• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #ifndef TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_
17 #define TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_
18 
19 #include "tensorflow/core/lib/io/inputstream_interface.h"
20 #include "tensorflow/core/platform/file_system.h"
21 
22 namespace tensorflow {
23 namespace io {
24 
25 // Provides a buffer on top of an InputStreamInterface. A single instance of
26 // BufferedInputStream is NOT safe for concurrent use by multiple threads.
27 class BufferedInputStream : public InputStreamInterface {
28  public:
29   // Does not take ownership of input_stream unless owns_input_stream is set
30   // to true. input_stream must outlive *this then.
31   // TODO(rohanj): Remove owns_input_stream once the constructor below is
32   // removed.
33   BufferedInputStream(InputStreamInterface* input_stream, size_t buffer_bytes,
34                       bool owns_input_stream = false);
35 
36   // For backwards compatibility, expose an interface that is similar to what
37   // InputBuffer exposes. Does not take ownership of file. file must outlive
38   // *this. This will be removed once we migrate all uses of this class to the
39   // constructor above.
40   BufferedInputStream(RandomAccessFile* file, size_t buffer_bytes);
41 
42   ~BufferedInputStream() override;
43 
44   Status ReadNBytes(int64 bytes_to_read, string* result) override;
45 
46   Status SkipNBytes(int64 bytes_to_skip) override;
47 
48   int64 Tell() const override;
49 
50   // Seek to this offset within the file.
51   //
52   // If we seek to somewhere within our pre-buffered data, we will re-use what
53   // data we can.  Otherwise, Seek() throws out the current buffer and the next
54   // read will trigger an underlying read.
55   //
56   // Note: When seeking backwards in a stream, this implementation uses
57   // Reset() + SkipNBytes(), so its performance will be dependent
58   // largely on the performance of SkipNBytes().
59   Status Seek(int64 position);
60 
61   // Read one text line of data into "*result" until end-of-file or a
62   // \n is read.  (The \n is not included in the result.)  Overwrites
63   // any existing data in *result.
64   //
65   // If successful, returns OK.  If we are already at the end of the
66   // file, we return an OUT_OF_RANGE error.  Otherwise, we return
67   // some other non-OK status.
68   Status ReadLine(string* result);
69 
70   // Returns one text line of data until end-of-file or a '\n' is read. The '\n'
71   // is included in the result.
72   // This method is a substitute for ReadLine() when called from Python which is
73   // the expectation in the python File::readline() API.
74   // Also, '\0's are treated like any other character within the line and given
75   // no special treatment.
76   string ReadLineAsString();
77 
78   // Reads the entire contents of the file into *result.
79   //
80   // Note: the amount of memory used by this function call is unbounded, so only
81   // use in ops that expect that behavior.
82   Status ReadAll(string* result);
83 
84   Status Reset() override;
85 
86  private:
87   Status FillBuffer();
88   Status ReadLineHelper(string* result, bool include_eol);
89 
90   InputStreamInterface* input_stream_;  // not owned.
91   size_t size_;                         // buffer size.
92   string buf_;                          // the buffer itself.
93   // buf_[pos_, limit_) holds the valid "read ahead" data in the file.
94   size_t pos_ = 0;    // current position in buf_.
95   size_t limit_ = 0;  // just past the end of valid data in buf_.
96   bool owns_input_stream_ = false;
97   // When EoF is reached, file_status_ contains the status to skip unnecessary
98   // buffer allocations.
99   Status file_status_ = Status::OK();
100 
101   TF_DISALLOW_COPY_AND_ASSIGN(BufferedInputStream);
102 };
103 
104 }  // namespace io
105 }  // namespace tensorflow
106 
107 #endif  // TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_
108