1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_ 17 #define TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_ 18 19 #include "tensorflow/core/lib/io/inputstream_interface.h" 20 #include "tensorflow/core/platform/file_system.h" 21 22 namespace tensorflow { 23 namespace io { 24 25 // Provides a buffer on top of an InputStreamInterface. A single instance of 26 // BufferedInputStream is NOT safe for concurrent use by multiple threads. 27 class BufferedInputStream : public InputStreamInterface { 28 public: 29 // Does not take ownership of input_stream unless owns_input_stream is set 30 // to true. input_stream must outlive *this then. 31 // TODO(rohanj): Remove owns_input_stream once the constructor below is 32 // removed. 33 BufferedInputStream(InputStreamInterface* input_stream, size_t buffer_bytes, 34 bool owns_input_stream = false); 35 36 // For backwards compatibility, expose an interface that is similar to what 37 // InputBuffer exposes. Does not take ownership of file. file must outlive 38 // *this. This will be removed once we migrate all uses of this class to the 39 // constructor above. 40 BufferedInputStream(RandomAccessFile* file, size_t buffer_bytes); 41 42 ~BufferedInputStream() override; 43 44 tensorflow::Status ReadNBytes(int64 bytes_to_read, tstring* result) override; 45 46 tensorflow::Status SkipNBytes(int64 bytes_to_skip) override; 47 48 int64 Tell() const override; 49 50 // Seek to this offset within the file. 51 // 52 // If we seek to somewhere within our pre-buffered data, we will re-use what 53 // data we can. Otherwise, Seek() throws out the current buffer and the next 54 // read will trigger an underlying read. 55 // 56 // Note: When seeking backwards in a stream, this implementation uses 57 // Reset() + SkipNBytes(), so its performance will be dependent 58 // largely on the performance of SkipNBytes(). 59 tensorflow::Status Seek(int64 position); 60 61 // Read one text line of data into "*result" until end-of-file or a 62 // \n is read. (The \n is not included in the result.) Overwrites 63 // any existing data in *result. 64 // 65 // If successful, returns OK. If we are already at the end of the 66 // file, we return an OUT_OF_RANGE error. Otherwise, we return 67 // some other non-OK status. 68 tensorflow::Status ReadLine(string* result); 69 tensorflow::Status ReadLine(tstring* result); 70 71 // Returns one text line of data until end-of-file or a '\n' is read. The '\n' 72 // is included in the result. 73 // This method is a substitute for ReadLine() when called from Python which is 74 // the expectation in the python File::readline() API. 75 // Also, '\0's are treated like any other character within the line and given 76 // no special treatment. 77 string ReadLineAsString(); 78 79 // Reads the entire contents of the file into *result. 80 // 81 // Note: the amount of memory used by this function call is unbounded, so only 82 // use in ops that expect that behavior. 83 template <typename T> 84 tensorflow::Status ReadAll(T* result); 85 86 tensorflow::Status Reset() override; 87 88 private: 89 tensorflow::Status FillBuffer(); 90 template <typename StringType> 91 tensorflow::Status ReadLineHelper(StringType* result, bool include_eol); 92 93 InputStreamInterface* input_stream_; // not owned. 94 size_t size_; // buffer size. 95 tstring buf_; // the buffer itself. 96 // buf_[pos_, limit_) holds the valid "read ahead" data in the file. 97 size_t pos_ = 0; // current position in buf_. 98 size_t limit_ = 0; // just past the end of valid data in buf_. 99 bool owns_input_stream_ = false; 100 // When EoF is reached, file_status_ contains the status to skip unnecessary 101 // buffer allocations. 102 tensorflow::Status file_status_ = Status::OK(); 103 104 TF_DISALLOW_COPY_AND_ASSIGN(BufferedInputStream); 105 }; 106 107 // Explicit instantiations defined in buffered_inputstream.cc. 108 #ifndef SWIG 109 extern template tensorflow::Status BufferedInputStream::ReadAll<string>( 110 string* result); 111 extern template tensorflow::Status BufferedInputStream::ReadAll<tstring>( 112 tstring* result); 113 #endif // SWIG 114 115 } // namespace io 116 } // namespace tensorflow 117 118 #endif // TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_ 119