1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include <dirent.h>
17 #include <errno.h>
18 #include <fcntl.h>
19 #include <stdio.h>
20 #include <sys/mman.h>
21 #if defined(__linux__)
22 #include <sys/sendfile.h>
23 #endif
24 #include <sys/stat.h>
25 #include <sys/time.h>
26 #include <sys/types.h>
27 #include <time.h>
28 #include <unistd.h>
29
30 #include "tensorflow/core/lib/core/error_codes.pb.h"
31 #include "tensorflow/core/lib/core/status.h"
32 #include "tensorflow/core/lib/strings/strcat.h"
33 #include "tensorflow/core/platform/env.h"
34 #include "tensorflow/core/platform/file_system_helper.h"
35 #include "tensorflow/core/platform/logging.h"
36 #include "tensorflow/core/platform/posix/error.h"
37 #include "tensorflow/core/platform/posix/posix_file_system.h"
38
39 namespace tensorflow {
40
41 // 128KB of copy buffer
42 constexpr size_t kPosixCopyFileBufferSize = 128 * 1024;
43
44 // pread() based random-access
45 class PosixRandomAccessFile : public RandomAccessFile {
46 private:
47 string filename_;
48 int fd_;
49
50 public:
PosixRandomAccessFile(const string & fname,int fd)51 PosixRandomAccessFile(const string& fname, int fd)
52 : filename_(fname), fd_(fd) {}
~PosixRandomAccessFile()53 ~PosixRandomAccessFile() override { close(fd_); }
54
Name(StringPiece * result) const55 Status Name(StringPiece* result) const override {
56 *result = filename_;
57 return Status::OK();
58 }
59
Read(uint64 offset,size_t n,StringPiece * result,char * scratch) const60 Status Read(uint64 offset, size_t n, StringPiece* result,
61 char* scratch) const override {
62 Status s;
63 char* dst = scratch;
64 while (n > 0 && s.ok()) {
65 ssize_t r = pread(fd_, dst, n, static_cast<off_t>(offset));
66 if (r > 0) {
67 dst += r;
68 n -= r;
69 offset += r;
70 } else if (r == 0) {
71 s = Status(error::OUT_OF_RANGE, "Read less bytes than requested");
72 } else if (errno == EINTR || errno == EAGAIN) {
73 // Retry
74 } else {
75 s = IOError(filename_, errno);
76 }
77 }
78 *result = StringPiece(scratch, dst - scratch);
79 return s;
80 }
81 };
82
83 class PosixWritableFile : public WritableFile {
84 private:
85 string filename_;
86 FILE* file_;
87
88 public:
PosixWritableFile(const string & fname,FILE * f)89 PosixWritableFile(const string& fname, FILE* f)
90 : filename_(fname), file_(f) {}
91
~PosixWritableFile()92 ~PosixWritableFile() override {
93 if (file_ != nullptr) {
94 // Ignoring any potential errors
95 fclose(file_);
96 }
97 }
98
Append(StringPiece data)99 Status Append(StringPiece data) override {
100 size_t r = fwrite(data.data(), 1, data.size(), file_);
101 if (r != data.size()) {
102 return IOError(filename_, errno);
103 }
104 return Status::OK();
105 }
106
Close()107 Status Close() override {
108 Status result;
109 if (fclose(file_) != 0) {
110 result = IOError(filename_, errno);
111 }
112 file_ = nullptr;
113 return result;
114 }
115
Flush()116 Status Flush() override {
117 if (fflush(file_) != 0) {
118 return IOError(filename_, errno);
119 }
120 return Status::OK();
121 }
122
Name(StringPiece * result) const123 Status Name(StringPiece* result) const override {
124 *result = filename_;
125 return Status::OK();
126 }
127
Sync()128 Status Sync() override {
129 Status s;
130 if (fflush(file_) != 0) {
131 s = IOError(filename_, errno);
132 }
133 return s;
134 }
135
Tell(int64 * position)136 Status Tell(int64* position) override {
137 Status s;
138 *position = ftell(file_);
139
140 if (*position == -1) {
141 s = IOError(filename_, errno);
142 }
143
144 return s;
145 }
146 };
147
148 class PosixReadOnlyMemoryRegion : public ReadOnlyMemoryRegion {
149 public:
PosixReadOnlyMemoryRegion(const void * address,uint64 length)150 PosixReadOnlyMemoryRegion(const void* address, uint64 length)
151 : address_(address), length_(length) {}
~PosixReadOnlyMemoryRegion()152 ~PosixReadOnlyMemoryRegion() override {
153 munmap(const_cast<void*>(address_), length_);
154 }
data()155 const void* data() override { return address_; }
length()156 uint64 length() override { return length_; }
157
158 private:
159 const void* const address_;
160 const uint64 length_;
161 };
162
NewRandomAccessFile(const string & fname,std::unique_ptr<RandomAccessFile> * result)163 Status PosixFileSystem::NewRandomAccessFile(
164 const string& fname, std::unique_ptr<RandomAccessFile>* result) {
165 string translated_fname = TranslateName(fname);
166 Status s;
167 int fd = open(translated_fname.c_str(), O_RDONLY);
168 if (fd < 0) {
169 s = IOError(fname, errno);
170 } else {
171 result->reset(new PosixRandomAccessFile(translated_fname, fd));
172 }
173 return s;
174 }
175
NewWritableFile(const string & fname,std::unique_ptr<WritableFile> * result)176 Status PosixFileSystem::NewWritableFile(const string& fname,
177 std::unique_ptr<WritableFile>* result) {
178 string translated_fname = TranslateName(fname);
179 Status s;
180 FILE* f = fopen(translated_fname.c_str(), "w");
181 if (f == nullptr) {
182 s = IOError(fname, errno);
183 } else {
184 result->reset(new PosixWritableFile(translated_fname, f));
185 }
186 return s;
187 }
188
NewAppendableFile(const string & fname,std::unique_ptr<WritableFile> * result)189 Status PosixFileSystem::NewAppendableFile(
190 const string& fname, std::unique_ptr<WritableFile>* result) {
191 string translated_fname = TranslateName(fname);
192 Status s;
193 FILE* f = fopen(translated_fname.c_str(), "a");
194 if (f == nullptr) {
195 s = IOError(fname, errno);
196 } else {
197 result->reset(new PosixWritableFile(translated_fname, f));
198 }
199 return s;
200 }
201
NewReadOnlyMemoryRegionFromFile(const string & fname,std::unique_ptr<ReadOnlyMemoryRegion> * result)202 Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile(
203 const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result) {
204 string translated_fname = TranslateName(fname);
205 Status s = Status::OK();
206 int fd = open(translated_fname.c_str(), O_RDONLY);
207 if (fd < 0) {
208 s = IOError(fname, errno);
209 } else {
210 struct stat st;
211 ::fstat(fd, &st);
212 const void* address =
213 mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
214 if (address == MAP_FAILED) {
215 s = IOError(fname, errno);
216 } else {
217 result->reset(new PosixReadOnlyMemoryRegion(address, st.st_size));
218 }
219 close(fd);
220 }
221 return s;
222 }
223
FileExists(const string & fname)224 Status PosixFileSystem::FileExists(const string& fname) {
225 if (access(TranslateName(fname).c_str(), F_OK) == 0) {
226 return Status::OK();
227 }
228 return errors::NotFound(fname, " not found");
229 }
230
GetChildren(const string & dir,std::vector<string> * result)231 Status PosixFileSystem::GetChildren(const string& dir,
232 std::vector<string>* result) {
233 string translated_dir = TranslateName(dir);
234 result->clear();
235 DIR* d = opendir(translated_dir.c_str());
236 if (d == nullptr) {
237 return IOError(dir, errno);
238 }
239 struct dirent* entry;
240 while ((entry = readdir(d)) != nullptr) {
241 StringPiece basename = entry->d_name;
242 if ((basename != ".") && (basename != "..")) {
243 result->push_back(entry->d_name);
244 }
245 }
246 closedir(d);
247 return Status::OK();
248 }
249
GetMatchingPaths(const string & pattern,std::vector<string> * results)250 Status PosixFileSystem::GetMatchingPaths(const string& pattern,
251 std::vector<string>* results) {
252 return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
253 }
254
DeleteFile(const string & fname)255 Status PosixFileSystem::DeleteFile(const string& fname) {
256 Status result;
257 if (unlink(TranslateName(fname).c_str()) != 0) {
258 result = IOError(fname, errno);
259 }
260 return result;
261 }
262
CreateDir(const string & name)263 Status PosixFileSystem::CreateDir(const string& name) {
264 string translated = TranslateName(name);
265 if (translated.empty()) {
266 return errors::AlreadyExists(name);
267 }
268 if (mkdir(translated.c_str(), 0755) != 0) {
269 return IOError(name, errno);
270 }
271 return Status::OK();
272 }
273
DeleteDir(const string & name)274 Status PosixFileSystem::DeleteDir(const string& name) {
275 Status result;
276 if (rmdir(TranslateName(name).c_str()) != 0) {
277 result = IOError(name, errno);
278 }
279 return result;
280 }
281
GetFileSize(const string & fname,uint64 * size)282 Status PosixFileSystem::GetFileSize(const string& fname, uint64* size) {
283 Status s;
284 struct stat sbuf;
285 if (stat(TranslateName(fname).c_str(), &sbuf) != 0) {
286 *size = 0;
287 s = IOError(fname, errno);
288 } else {
289 *size = sbuf.st_size;
290 }
291 return s;
292 }
293
Stat(const string & fname,FileStatistics * stats)294 Status PosixFileSystem::Stat(const string& fname, FileStatistics* stats) {
295 Status s;
296 struct stat sbuf;
297 if (stat(TranslateName(fname).c_str(), &sbuf) != 0) {
298 s = IOError(fname, errno);
299 } else {
300 stats->length = sbuf.st_size;
301 stats->mtime_nsec = sbuf.st_mtime * 1e9;
302 stats->is_directory = S_ISDIR(sbuf.st_mode);
303 }
304 return s;
305 }
306
RenameFile(const string & src,const string & target)307 Status PosixFileSystem::RenameFile(const string& src, const string& target) {
308 Status result;
309 if (rename(TranslateName(src).c_str(), TranslateName(target).c_str()) != 0) {
310 result = IOError(src, errno);
311 }
312 return result;
313 }
314
CopyFile(const string & src,const string & target)315 Status PosixFileSystem::CopyFile(const string& src, const string& target) {
316 string translated_src = TranslateName(src);
317 struct stat sbuf;
318 if (stat(translated_src.c_str(), &sbuf) != 0) {
319 return IOError(src, errno);
320 }
321 int src_fd = open(translated_src.c_str(), O_RDONLY);
322 if (src_fd < 0) {
323 return IOError(src, errno);
324 }
325 string translated_target = TranslateName(target);
326 // O_WRONLY | O_CREAT:
327 // Open file for write and if file does not exist, create the file.
328 // S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH:
329 // Create the file with permission of 0644
330 int target_fd = open(translated_target.c_str(), O_WRONLY | O_CREAT,
331 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
332 if (target_fd < 0) {
333 close(src_fd);
334 return IOError(target, errno);
335 }
336 int rc = 0;
337 off_t offset = 0;
338 std::unique_ptr<char[]> buffer(new char[kPosixCopyFileBufferSize]);
339 while (offset < sbuf.st_size) {
340 // Use uint64 for safe compare SSIZE_MAX
341 uint64 chunk = sbuf.st_size - offset;
342 if (chunk > SSIZE_MAX) {
343 chunk = SSIZE_MAX;
344 }
345 #if defined(__linux__) && !defined(__ANDROID__)
346 rc = sendfile(target_fd, src_fd, &offset, static_cast<size_t>(chunk));
347 #else
348 if (chunk > kPosixCopyFileBufferSize) {
349 chunk = kPosixCopyFileBufferSize;
350 }
351 rc = read(src_fd, buffer.get(), static_cast<size_t>(chunk));
352 if (rc <= 0) {
353 break;
354 }
355 rc = write(target_fd, buffer.get(), static_cast<size_t>(chunk));
356 offset += chunk;
357 #endif
358 if (rc <= 0) {
359 break;
360 }
361 }
362
363 Status result = Status::OK();
364 if (rc < 0) {
365 result = IOError(target, errno);
366 }
367
368 // Keep the error code
369 rc = close(target_fd);
370 if (rc < 0 && result == Status::OK()) {
371 result = IOError(target, errno);
372 }
373 rc = close(src_fd);
374 if (rc < 0 && result == Status::OK()) {
375 result = IOError(target, errno);
376 }
377
378 return result;
379 }
380
381 } // namespace tensorflow
382