1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #ifndef TENSORFLOW_LIB_IO_INPUTBUFFER_H_
17 #define TENSORFLOW_LIB_IO_INPUTBUFFER_H_
18
19 #include <string>
20 #include "tensorflow/core/lib/core/coding.h"
21 #include "tensorflow/core/lib/core/status.h"
22 #include "tensorflow/core/platform/env.h"
23 #include "tensorflow/core/platform/macros.h"
24 #include "tensorflow/core/platform/types.h"
25
26 namespace tensorflow {
27 namespace io {
28
29 // An InputBuffer provides a buffer on top of a RandomAccessFile.
30 // A given instance of an InputBuffer is NOT safe for concurrent use
31 // by multiple threads
32 class InputBuffer {
33 public:
34 // Create an InputBuffer for "file" with a buffer size of
35 // "buffer_bytes" bytes. 'file' must outlive *this.
36 InputBuffer(RandomAccessFile* file, size_t buffer_bytes);
37 ~InputBuffer();
38
39 // Read one text line of data into "*result" until end-of-file or a
40 // \n is read. (The \n is not included in the result.) Overwrites
41 // any existing data in *result.
42 //
43 // If successful, returns OK. If we are already at the end of the
44 // file, we return an OUT_OF_RANGE error. Otherwise, we return
45 // some other non-OK status.
46 Status ReadLine(string* result);
47
48 // Reads bytes_to_read bytes into *result, overwriting *result.
49 //
50 // If successful, returns OK. If we there are not enough bytes to
51 // read before the end of the file, we return an OUT_OF_RANGE error.
52 // Otherwise, we return some other non-OK status.
53 Status ReadNBytes(int64 bytes_to_read, string* result);
54
55 // An overload that writes to char*. Caller must ensure result[0,
56 // bytes_to_read) is valid to be overwritten. Returns OK iff "*bytes_read ==
57 // bytes_to_read".
58 Status ReadNBytes(int64 bytes_to_read, char* result, size_t* bytes_read);
59
60 // Reads a single varint32.
61 Status ReadVarint32(uint32* result);
62
63 // Reads a single varint64.
64 Status ReadVarint64(uint64* result);
65
66 // Like ReadNBytes() without returning the bytes read.
67 Status SkipNBytes(int64 bytes_to_skip);
68
69 // Seek to this offset within the file.
70 //
71 // If we seek to somewhere within our pre-buffered data, we will re-use what
72 // data we can. Otherwise, Seek() throws out the current buffer and the next
73 // read will trigger a File::Read().
74 Status Seek(int64 position);
75
76 // Returns the position in the file.
Tell()77 int64 Tell() const { return file_pos_ - (limit_ - pos_); }
78
79 // Returns the underlying RandomAccessFile.
file()80 RandomAccessFile* file() const { return file_; }
81
82 private:
83 Status FillBuffer();
84
85 // Internal slow-path routine used by ReadVarint32().
86 Status ReadVarint32Fallback(uint32* result);
87
88 // Internal slow-path routine used by ReadVarint64().
89 Status ReadVarint64Fallback(uint64* result);
90
91 // Helper method for reading a varint which can span at max `max_bytes`.
92 // If the varint is longer, a DataLoss error status is returned.
93 // If end of file is reached while reading, OutOfRange error is returned.
94 template <typename T>
95 Status ReadVarintFallback(T* result, int max_bytes);
96
97 RandomAccessFile* file_; // Not owned
98 int64 file_pos_; // Next position to read from in "file_"
99 size_t size_; // Size of "buf_"
100 char* buf_; // The buffer itself
101 // [pos_,limit_) hold the "limit_ - pos_" bytes just before "file_pos_"
102 char* pos_; // Current position in "buf"
103 char* limit_; // Just past end of valid data in "buf"
104
105 TF_DISALLOW_COPY_AND_ASSIGN(InputBuffer);
106 };
107
108 // Implementation details.
109
110 // Inlined for performance.
ReadVarint32(uint32 * result)111 inline Status InputBuffer::ReadVarint32(uint32* result) {
112 if (pos_ + core::kMaxVarint32Bytes <= limit_) {
113 // Fast path: directly parse from buffered data.
114 // Reads strictly from the range [pos_, limit_).
115 const char* offset = core::GetVarint32Ptr(pos_, limit_, result);
116 if (offset == nullptr) return errors::OutOfRange("Parsed past limit.");
117 pos_ = const_cast<char*>(offset);
118 return Status::OK();
119 } else {
120 return ReadVarint32Fallback(result);
121 }
122 }
123
124 // Inlined for performance.
ReadVarint64(uint64 * result)125 inline Status InputBuffer::ReadVarint64(uint64* result) {
126 if (pos_ + core::kMaxVarint64Bytes <= limit_) {
127 // Fast path: directly parse from buffered data.
128 // Reads strictly from the range [pos_, limit_).
129 const char* offset = core::GetVarint64Ptr(pos_, limit_, result);
130 if (offset == nullptr) return errors::OutOfRange("Parsed past limit.");
131 pos_ = const_cast<char*>(offset);
132 return Status::OK();
133 } else {
134 return ReadVarint64Fallback(result);
135 }
136 }
137
138 } // namespace io
139 } // namespace tensorflow
140
141 #endif // TENSORFLOW_LIB_IO_INPUTBUFFER_H_
142