1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #ifndef TENSORFLOW_CORE_PLATFORM_ENV_H_
17 #define TENSORFLOW_CORE_PLATFORM_ENV_H_
18
19 #include <stdint.h>
20
21 #include <memory>
22 #include <string>
23 #include <unordered_map>
24 #include <vector>
25
26 #include "tensorflow/core/platform/env_time.h"
27 #include "tensorflow/core/platform/errors.h"
28 #include "tensorflow/core/platform/file_system.h"
29 #include "tensorflow/core/platform/macros.h"
30 #include "tensorflow/core/platform/mutex.h"
31 #include "tensorflow/core/platform/numa.h"
32 #include "tensorflow/core/platform/platform.h"
33 #include "tensorflow/core/platform/protobuf.h"
34 #include "tensorflow/core/platform/status.h"
35 #include "tensorflow/core/platform/stringpiece.h"
36 #include "tensorflow/core/platform/types.h"
37
38 // Delete the definition of CopyFile as the linker gets confused.
39 #ifdef PLATFORM_WINDOWS
40 #undef CopyFile
41 #endif
42
43 namespace tensorflow {
44
45 class Thread;
46 struct ThreadOptions;
47
48 /// \brief An interface used by the tensorflow implementation to
49 /// access operating system functionality like the filesystem etc.
50 ///
51 /// Callers may wish to provide a custom Env object to get fine grain
52 /// control.
53 ///
54 /// All Env implementations are safe for concurrent access from
55 /// multiple threads without any external synchronization.
56 class Env {
57 public:
58 Env();
59 virtual ~Env() = default;
60
61 /// \brief Returns a default environment suitable for the current operating
62 /// system.
63 ///
64 /// Sophisticated users may wish to provide their own Env
65 /// implementation instead of relying on this default environment.
66 ///
67 /// The result of Default() belongs to this library and must never be deleted.
68 static Env* Default();
69
70 /// \brief Returns the FileSystem object to handle operations on the file
71 /// specified by 'fname'. The FileSystem object is used as the implementation
72 /// for the file system related (non-virtual) functions that follow.
73 /// Returned FileSystem object is still owned by the Env object and will
74 // (might) be destroyed when the environment is destroyed.
75 virtual Status GetFileSystemForFile(const std::string& fname,
76 FileSystem** result);
77
78 /// \brief Returns the file system schemes registered for this Env.
79 virtual Status GetRegisteredFileSystemSchemes(
80 std::vector<std::string>* schemes);
81
82 /// \brief Register a file system for a scheme.
83 virtual Status RegisterFileSystem(const std::string& scheme,
84 FileSystemRegistry::Factory factory);
85
86 /// \brief Register a modular file system for a scheme.
87 ///
88 /// Same as `RegisterFileSystem` but for filesystems provided by plugins.
89 ///
90 /// TODO(mihaimaruseac): After all filesystems are converted, make this be the
91 /// canonical registration function.
92 virtual Status RegisterFileSystem(const std::string& scheme,
93 std::unique_ptr<FileSystem> filesystem);
94
95 /// \brief Flush filesystem caches for all registered filesystems.
96 Status FlushFileSystemCaches();
97
98 /// \brief Creates a brand new random access read-only file with the
99 /// specified name.
100
101 /// On success, stores a pointer to the new file in
102 /// *result and returns OK. On failure stores NULL in *result and
103 /// returns non-OK. If the file does not exist, returns a non-OK
104 /// status.
105 ///
106 /// The returned file may be concurrently accessed by multiple threads.
107 ///
108 /// The ownership of the returned RandomAccessFile is passed to the caller
109 /// and the object should be deleted when is not used. The file object
110 /// shouldn't live longer than the Env object.
111 Status NewRandomAccessFile(const std::string& fname,
112 std::unique_ptr<RandomAccessFile>* result);
113
NewRandomAccessFile(const std::string & fname,TransactionToken * token,std::unique_ptr<RandomAccessFile> * result)114 Status NewRandomAccessFile(const std::string& fname, TransactionToken* token,
115 std::unique_ptr<RandomAccessFile>* result) {
116 // We duplicate these methods due to Google internal coding style prevents
117 // virtual functions with default arguments. See PR #41615.
118 return Status::OK();
119 }
120
121 /// \brief Creates an object that writes to a new file with the specified
122 /// name.
123 ///
124 /// Deletes any existing file with the same name and creates a
125 /// new file. On success, stores a pointer to the new file in
126 /// *result and returns OK. On failure stores NULL in *result and
127 /// returns non-OK.
128 ///
129 /// The returned file will only be accessed by one thread at a time.
130 ///
131 /// The ownership of the returned WritableFile is passed to the caller
132 /// and the object should be deleted when is not used. The file object
133 /// shouldn't live longer than the Env object.
134 Status NewWritableFile(const std::string& fname,
135 std::unique_ptr<WritableFile>* result);
136
NewWritableFile(const std::string & fname,TransactionToken * token,std::unique_ptr<WritableFile> * result)137 Status NewWritableFile(const std::string& fname, TransactionToken* token,
138 std::unique_ptr<WritableFile>* result) {
139 return Status::OK();
140 }
141
142 /// \brief Creates an object that either appends to an existing file, or
143 /// writes to a new file (if the file does not exist to begin with).
144 ///
145 /// On success, stores a pointer to the new file in *result and
146 /// returns OK. On failure stores NULL in *result and returns
147 /// non-OK.
148 ///
149 /// The returned file will only be accessed by one thread at a time.
150 ///
151 /// The ownership of the returned WritableFile is passed to the caller
152 /// and the object should be deleted when is not used. The file object
153 /// shouldn't live longer than the Env object.
154 Status NewAppendableFile(const std::string& fname,
155 std::unique_ptr<WritableFile>* result);
156
NewAppendableFile(const std::string & fname,TransactionToken * token,std::unique_ptr<WritableFile> * result)157 Status NewAppendableFile(const std::string& fname, TransactionToken* token,
158 std::unique_ptr<WritableFile>* result) {
159 return Status::OK();
160 }
161 /// \brief Creates a readonly region of memory with the file context.
162 ///
163 /// On success, it returns a pointer to read-only memory region
164 /// from the content of file fname. The ownership of the region is passed to
165 /// the caller. On failure stores nullptr in *result and returns non-OK.
166 ///
167 /// The returned memory region can be accessed from many threads in parallel.
168 ///
169 /// The ownership of the returned ReadOnlyMemoryRegion is passed to the caller
170 /// and the object should be deleted when is not used. The memory region
171 /// object shouldn't live longer than the Env object.
172 Status NewReadOnlyMemoryRegionFromFile(
173 const std::string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result);
174
NewReadOnlyMemoryRegionFromFile(const std::string & fname,TransactionToken * token,std::unique_ptr<ReadOnlyMemoryRegion> * result)175 Status NewReadOnlyMemoryRegionFromFile(
176 const std::string& fname, TransactionToken* token,
177 std::unique_ptr<ReadOnlyMemoryRegion>* result) {
178 return Status::OK();
179 }
180
181 /// Returns OK if the named path exists and NOT_FOUND otherwise.
182 Status FileExists(const std::string& fname);
183
FileExists(const std::string & fname,TransactionToken * token)184 Status FileExists(const std::string& fname, TransactionToken* token) {
185 return Status::OK();
186 }
187
188 /// Returns true if all the listed files exist, false otherwise.
189 /// if status is not null, populate the vector with a detailed status
190 /// for each file.
191 bool FilesExist(const std::vector<string>& files,
192 std::vector<Status>* status);
193
FilesExist(const std::vector<string> & files,TransactionToken * token,std::vector<Status> * status)194 bool FilesExist(const std::vector<string>& files, TransactionToken* token,
195 std::vector<Status>* status) {
196 return true;
197 }
198
199 /// \brief Stores in *result the names of the children of the specified
200 /// directory. The names are relative to "dir".
201 ///
202 /// Original contents of *results are dropped.
203 Status GetChildren(const std::string& dir, std::vector<string>* result);
204
GetChildren(const std::string & dir,TransactionToken * token,std::vector<string> * result)205 Status GetChildren(const std::string& dir, TransactionToken* token,
206 std::vector<string>* result) {
207 return Status::OK();
208 }
209
210 /// \brief Returns true if the path matches the given pattern. The wildcards
211 /// allowed in pattern are described in FileSystem::GetMatchingPaths.
212 virtual bool MatchPath(const std::string& path,
213 const std::string& pattern) = 0;
214
215 /// \brief Given a pattern, stores in *results the set of paths that matches
216 /// that pattern. *results is cleared.
217 ///
218 /// More details about `pattern` in FileSystem::GetMatchingPaths.
219 virtual Status GetMatchingPaths(const std::string& pattern,
220 std::vector<string>* results);
221
GetMatchingPaths(const std::string & pattern,TransactionToken * token,std::vector<string> * results)222 Status GetMatchingPaths(const std::string& pattern, TransactionToken* token,
223 std::vector<string>* results) {
224 return Status::OK();
225 }
226
227 /// Deletes the named file.
228 Status DeleteFile(const std::string& fname);
229
DeleteFile(const std::string & fname,TransactionToken * token)230 Status DeleteFile(const std::string& fname, TransactionToken* token) {
231 return Status::OK();
232 }
233
234 /// \brief Deletes the specified directory and all subdirectories and files
235 /// underneath it. This is accomplished by traversing the directory tree
236 /// rooted at dirname and deleting entries as they are encountered.
237 ///
238 /// If dirname itself is not readable or does not exist, *undeleted_dir_count
239 /// is set to 1, *undeleted_file_count is set to 0 and an appropriate status
240 /// (e.g. NOT_FOUND) is returned.
241 ///
242 /// If dirname and all its descendants were successfully deleted, TF_OK is
243 /// returned and both error counters are set to zero.
244 ///
245 /// Otherwise, while traversing the tree, undeleted_file_count and
246 /// undeleted_dir_count are updated if an entry of the corresponding type
247 /// could not be deleted. The returned error status represents the reason that
248 /// any one of these entries could not be deleted.
249 ///
250 /// REQUIRES: undeleted_files, undeleted_dirs to be not null.
251 ///
252 /// Typical return codes:
253 /// * OK - dirname exists and we were able to delete everything underneath.
254 /// * NOT_FOUND - dirname doesn't exist
255 /// * PERMISSION_DENIED - dirname or some descendant is not writable
256 /// * UNIMPLEMENTED - Some underlying functions (like Delete) are not
257 /// implemented
258 Status DeleteRecursively(const std::string& dirname, int64* undeleted_files,
259 int64* undeleted_dirs);
260
DeleteRecursively(const std::string & dirname,TransactionToken * token,int64 * undeleted_files,int64 * undeleted_dirs)261 Status DeleteRecursively(const std::string& dirname, TransactionToken* token,
262 int64* undeleted_files, int64* undeleted_dirs) {
263 return Status::OK();
264 }
265
266 /// \brief Creates the specified directory and all the necessary
267 /// subdirectories. Typical return codes.
268 /// * OK - successfully created the directory and sub directories, even if
269 /// they were already created.
270 /// * PERMISSION_DENIED - dirname or some subdirectory is not writable.
271 Status RecursivelyCreateDir(const std::string& dirname);
272
RecursivelyCreateDir(const std::string & dirname,TransactionToken * token)273 Status RecursivelyCreateDir(const std::string& dirname,
274 TransactionToken* token) {
275 return Status::OK();
276 }
277 /// \brief Creates the specified directory. Typical return codes
278 /// * OK - successfully created the directory.
279 /// * ALREADY_EXISTS - directory already exists.
280 /// * PERMISSION_DENIED - dirname is not writable.
281 Status CreateDir(const std::string& dirname);
282
CreateDir(const std::string & dirname,TransactionToken * token)283 Status CreateDir(const std::string& dirname, TransactionToken* token) {
284 return Status::OK();
285 }
286
287 /// Deletes the specified directory.
288 Status DeleteDir(const std::string& dirname);
289
DeleteDir(const std::string & dirname,TransactionToken * token)290 Status DeleteDir(const std::string& dirname, TransactionToken* token) {
291 return Status::OK();
292 }
293
294 /// Obtains statistics for the given path.
295 Status Stat(const std::string& fname, FileStatistics* stat);
296
Stat(const std::string & fname,TransactionToken * token,FileStatistics * stat)297 Status Stat(const std::string& fname, TransactionToken* token,
298 FileStatistics* stat) {
299 return Status::OK();
300 }
301
302 /// \brief Returns whether the given path is a directory or not.
303 /// Typical return codes (not guaranteed exhaustive):
304 /// * OK - The path exists and is a directory.
305 /// * FAILED_PRECONDITION - The path exists and is not a directory.
306 /// * NOT_FOUND - The path entry does not exist.
307 /// * PERMISSION_DENIED - Insufficient permissions.
308 /// * UNIMPLEMENTED - The file factory doesn't support directories.
309 Status IsDirectory(const std::string& fname);
310
311 /// \brief Returns whether the given path is on a file system
312 /// that has atomic move capabilities. This can be used
313 /// to determine if there needs to be a temp location to safely write objects.
314 /// The second boolean argument has_atomic_move contains this information.
315 ///
316 /// Returns one of the following status codes (not guaranteed exhaustive):
317 /// * OK - The path is on a recognized file system,
318 /// so has_atomic_move holds the above information.
319 /// * UNIMPLEMENTED - The file system of the path hasn't been implemented in
320 /// TF
321 Status HasAtomicMove(const std::string& path, bool* has_atomic_move);
322
323 /// Stores the size of `fname` in `*file_size`.
324 Status GetFileSize(const std::string& fname, uint64* file_size);
325
GetFileSize(const std::string & fname,TransactionToken * token,uint64 * file_size)326 Status GetFileSize(const std::string& fname, TransactionToken* token,
327 uint64* file_size) {
328 return Status::OK();
329 }
330
331 /// \brief Renames file src to target. If target already exists, it will be
332 /// replaced.
333 Status RenameFile(const std::string& src, const std::string& target);
334
RenameFile(const std::string & src,const std::string & target,TransactionToken * token)335 Status RenameFile(const std::string& src, const std::string& target,
336 TransactionToken* token) {
337 return Status::OK();
338 }
339
340 /// \brief Copy the src to target.
341 Status CopyFile(const std::string& src, const std::string& target);
342
CopyFile(const std::string & src,const std::string & target,TransactionToken * token)343 Status CopyFile(const std::string& src, const std::string& target,
344 TransactionToken* token) {
345 return Status::OK();
346 }
347
348 /// \brief starts a new transaction on the filesystem that handles filename
StartTransaction(const std::string & filename,TransactionToken ** token)349 Status StartTransaction(const std::string& filename,
350 TransactionToken** token) {
351 token = nullptr;
352 return Status::OK();
353 }
354
355 /// \brief Adds `path` to transaction in `token` if token belongs to
356 /// filesystem that handles the path.
AddToTransaction(const std::string & path,TransactionToken * token)357 Status AddToTransaction(const std::string& path, TransactionToken* token) {
358 return Status::OK();
359 }
360
361 /// \brief Get token for `path` or start a new transaction and add `path` to
362 /// it.
GetTokenOrStartTransaction(const std::string & path,TransactionToken ** token)363 Status GetTokenOrStartTransaction(const std::string& path,
364 TransactionToken** token) {
365 *token = nullptr;
366 return Status::OK();
367 }
368
369 /// \brief Returns the transaction for `path` or nullptr in `token`
GetTransactionForPath(const std::string & path,TransactionToken ** token)370 Status GetTransactionForPath(const std::string& path,
371 TransactionToken** token) {
372 token = nullptr;
373 return Status::OK();
374 }
375
376 /// \brief Finalizes the transaction
EndTransaction(TransactionToken * token)377 Status EndTransaction(TransactionToken* token) { return Status::OK(); }
378
379 /// \brief Returns the absolute path of the current executable. It resolves
380 /// symlinks if there is any.
381 std::string GetExecutablePath();
382
383 /// Creates a local unique temporary file name. Returns true if success.
384 bool LocalTempFilename(std::string* filename);
385
386 /// Creates a local unique file name that starts with |prefix| and ends with
387 /// |suffix|. Returns true if success.
388 bool CreateUniqueFileName(std::string* prefix, const std::string& suffix);
389
390 /// \brief Return the runfiles directory if running under bazel. Returns
391 /// the directory the executable is located in if not running under bazel.
392 virtual std::string GetRunfilesDir() = 0;
393
394 // TODO(jeff,sanjay): Add back thread/thread-pool support if needed.
395 // TODO(jeff,sanjay): if needed, tighten spec so relative to epoch, or
396 // provide a routine to get the absolute time.
397
398 /// \brief Returns the number of nano-seconds since the Unix epoch.
NowNanos()399 virtual uint64 NowNanos() const { return EnvTime::NowNanos(); }
400
401 /// \brief Returns the number of micro-seconds since the Unix epoch.
NowMicros()402 virtual uint64 NowMicros() const { return EnvTime::NowMicros(); }
403
404 /// \brief Returns the number of seconds since the Unix epoch.
NowSeconds()405 virtual uint64 NowSeconds() const { return EnvTime::NowSeconds(); }
406
407 /// Sleeps/delays the thread for the prescribed number of micro-seconds.
408 virtual void SleepForMicroseconds(int64 micros) = 0;
409
410 /// \brief Returns a new thread that is running fn() and is identified
411 /// (for debugging/performance-analysis) by "name".
412 ///
413 /// Caller takes ownership of the result and must delete it eventually
414 /// (the deletion will block until fn() stops running).
415 virtual Thread* StartThread(const ThreadOptions& thread_options,
416 const std::string& name,
417 std::function<void()> fn) TF_MUST_USE_RESULT = 0;
418
419 // Returns the thread id of calling thread.
420 // Posix: Returns pthread id which is only guaranteed to be unique within a
421 // process.
422 // Windows: Returns thread id which is unique.
423 virtual int32 GetCurrentThreadId() = 0;
424
425 // Copies current thread name to "name". Returns true if success.
426 virtual bool GetCurrentThreadName(std::string* name) = 0;
427
428 // \brief Schedules the given closure on a thread-pool.
429 //
430 // NOTE(mrry): This closure may block.
431 virtual void SchedClosure(std::function<void()> closure) = 0;
432
433 // \brief Schedules the given closure on a thread-pool after the given number
434 // of microseconds.
435 //
436 // NOTE(mrry): This closure must not block.
437 virtual void SchedClosureAfter(int64 micros,
438 std::function<void()> closure) = 0;
439
440 // \brief Load a dynamic library.
441 //
442 // Pass "library_filename" to a platform-specific mechanism for dynamically
443 // loading a library. The rules for determining the exact location of the
444 // library are platform-specific and are not documented here.
445 //
446 // On success, returns a handle to the library in "*handle" and returns
447 // OK from the function.
448 // Otherwise returns nullptr in "*handle" and an error status from the
449 // function.
450 virtual Status LoadDynamicLibrary(const char* library_filename,
451 void** handle) = 0;
452
453 // \brief Get a pointer to a symbol from a dynamic library.
454 //
455 // "handle" should be a pointer returned from a previous call to LoadLibrary.
456 // On success, store a pointer to the located symbol in "*symbol" and return
457 // OK from the function. Otherwise, returns nullptr in "*symbol" and an error
458 // status from the function.
459 virtual Status GetSymbolFromLibrary(void* handle, const char* symbol_name,
460 void** symbol) = 0;
461
462 // \brief build the name of dynamic library.
463 //
464 // "name" should be name of the library.
465 // "version" should be the version of the library or NULL
466 // returns the name that LoadLibrary() can use
467 virtual std::string FormatLibraryFileName(const std::string& name,
468 const std::string& version) = 0;
469
470 // Returns a possible list of local temporary directories.
471 virtual void GetLocalTempDirectories(std::vector<string>* list) = 0;
472
473 private:
474 std::unique_ptr<FileSystemRegistry> file_system_registry_;
475 TF_DISALLOW_COPY_AND_ASSIGN(Env);
476 };
477
478 /// \brief An implementation of Env that forwards all calls to another Env.
479 ///
480 /// May be useful to clients who wish to override just part of the
481 /// functionality of another Env.
482 class EnvWrapper : public Env {
483 public:
484 /// Initializes an EnvWrapper that delegates all calls to *t
EnvWrapper(Env * t)485 explicit EnvWrapper(Env* t) : target_(t) {}
486 ~EnvWrapper() override;
487
488 /// Returns the target to which this Env forwards all calls
target()489 Env* target() const { return target_; }
490
GetFileSystemForFile(const std::string & fname,FileSystem ** result)491 Status GetFileSystemForFile(const std::string& fname,
492 FileSystem** result) override {
493 return target_->GetFileSystemForFile(fname, result);
494 }
495
GetRegisteredFileSystemSchemes(std::vector<string> * schemes)496 Status GetRegisteredFileSystemSchemes(std::vector<string>* schemes) override {
497 return target_->GetRegisteredFileSystemSchemes(schemes);
498 }
499
RegisterFileSystem(const std::string & scheme,FileSystemRegistry::Factory factory)500 Status RegisterFileSystem(const std::string& scheme,
501 FileSystemRegistry::Factory factory) override {
502 return target_->RegisterFileSystem(scheme, factory);
503 }
504
MatchPath(const std::string & path,const std::string & pattern)505 bool MatchPath(const std::string& path, const std::string& pattern) override {
506 return target_->MatchPath(path, pattern);
507 }
508
NowMicros()509 uint64 NowMicros() const override { return target_->NowMicros(); }
SleepForMicroseconds(int64 micros)510 void SleepForMicroseconds(int64 micros) override {
511 target_->SleepForMicroseconds(micros);
512 }
StartThread(const ThreadOptions & thread_options,const std::string & name,std::function<void ()> fn)513 Thread* StartThread(const ThreadOptions& thread_options,
514 const std::string& name,
515 std::function<void()> fn) override {
516 return target_->StartThread(thread_options, name, fn);
517 }
GetCurrentThreadId()518 int32 GetCurrentThreadId() override { return target_->GetCurrentThreadId(); }
GetCurrentThreadName(std::string * name)519 bool GetCurrentThreadName(std::string* name) override {
520 return target_->GetCurrentThreadName(name);
521 }
SchedClosure(std::function<void ()> closure)522 void SchedClosure(std::function<void()> closure) override {
523 target_->SchedClosure(closure);
524 }
SchedClosureAfter(int64 micros,std::function<void ()> closure)525 void SchedClosureAfter(int64 micros, std::function<void()> closure) override {
526 target_->SchedClosureAfter(micros, closure);
527 }
LoadDynamicLibrary(const char * library_filename,void ** handle)528 Status LoadDynamicLibrary(const char* library_filename,
529 void** handle) override {
530 return target_->LoadDynamicLibrary(library_filename, handle);
531 }
GetSymbolFromLibrary(void * handle,const char * symbol_name,void ** symbol)532 Status GetSymbolFromLibrary(void* handle, const char* symbol_name,
533 void** symbol) override {
534 return target_->GetSymbolFromLibrary(handle, symbol_name, symbol);
535 }
FormatLibraryFileName(const std::string & name,const std::string & version)536 std::string FormatLibraryFileName(const std::string& name,
537 const std::string& version) override {
538 return target_->FormatLibraryFileName(name, version);
539 }
540
GetRunfilesDir()541 std::string GetRunfilesDir() override { return target_->GetRunfilesDir(); }
542
543 private:
GetLocalTempDirectories(std::vector<string> * list)544 void GetLocalTempDirectories(std::vector<string>* list) override {
545 target_->GetLocalTempDirectories(list);
546 }
547
548 Env* target_;
549 };
550
551 /// Represents a thread used to run a TensorFlow function.
552 class Thread {
553 public:
Thread()554 Thread() {}
555
556 /// Blocks until the thread of control stops running.
557 virtual ~Thread();
558
559 private:
560 TF_DISALLOW_COPY_AND_ASSIGN(Thread);
561 };
562
563 /// \brief Cross-platform setenv.
564 ///
565 /// Since setenv() is not available on windows, we provide an
566 /// alternative with platform specific implementations here.
567 int setenv(const char* name, const char* value, int overwrite);
568
569 /// Cross-platform unsetenv.
570 int unsetenv(const char* name);
571
572 /// \brief Options to configure a Thread.
573 ///
574 /// Note that the options are all hints, and the
575 /// underlying implementation may choose to ignore it.
576 struct ThreadOptions {
577 /// Thread stack size to use (in bytes).
578 size_t stack_size = 0; // 0: use system default value
579 /// Guard area size to use near thread stacks to use (in bytes)
580 size_t guard_size = 0; // 0: use system default value
581 int numa_node = port::kNUMANoAffinity;
582 };
583
584 /// A utility routine: copy contents of `src` in file system `src_fs`
585 /// to `target` in file system `target_fs`.
586 Status FileSystemCopyFile(FileSystem* src_fs, const std::string& src,
587 FileSystem* target_fs, const std::string& target);
588
589 /// A utility routine: reads contents of named file into `*data`
590 Status ReadFileToString(Env* env, const std::string& fname, std::string* data);
591
592 /// A utility routine: write contents of `data` to file named `fname`
593 /// (overwriting existing contents, if any).
594 Status WriteStringToFile(Env* env, const std::string& fname,
595 const StringPiece& data);
596
597 /// Write binary representation of "proto" to the named file.
598 Status WriteBinaryProto(Env* env, const std::string& fname,
599 const protobuf::MessageLite& proto);
600
601 /// Reads contents of named file and parse as binary encoded proto data
602 /// and store into `*proto`.
603 Status ReadBinaryProto(Env* env, const std::string& fname,
604 protobuf::MessageLite* proto);
605
606 /// Write the text representation of "proto" to the named file.
607 Status WriteTextProto(Env* env, const std::string& fname,
608 const protobuf::Message& proto);
609
610 /// Read contents of named file and parse as text encoded proto data
611 /// and store into `*proto`.
ReadTextProto(Env *,const std::string &,protobuf::MessageLite *)612 inline Status ReadTextProto(Env* /* env */, const std::string& /* fname */,
613 protobuf::MessageLite* /* proto */) {
614 return errors::Unimplemented("Can't parse text protos with protolite.");
615 }
616 Status ReadTextProto(Env* env, const std::string& fname,
617 protobuf::Message* proto);
618
619 /// Read contents of named file and parse as either text or binary encoded proto
620 /// data and store into `*proto`.
621 Status ReadTextOrBinaryProto(Env* env, const std::string& fname,
622 protobuf::Message* proto);
623 Status ReadTextOrBinaryProto(Env* env, const std::string& fname,
624 protobuf::MessageLite* proto);
625
626 // START_SKIP_DOXYGEN
627
628 // The following approach to register filesystems is deprecated and will be
629 // replaced with modular filesystem plugins registration.
630 // TODO(mihaimaruseac): After all filesystems are converted, remove this.
631 namespace register_file_system {
632
633 template <typename Factory>
634 struct Register {
RegisterRegister635 Register(Env* env, const std::string& scheme, bool legacy) {
636 // TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs
637 // after TF 2.6+.
638 if (legacy) {
639 const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM");
640 string enable_legacy =
641 enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : "";
642 if (!(enable_legacy == "true" || enable_legacy == "1")) {
643 return;
644 }
645 LOG(WARNING) << "Legacy file system for '" << scheme << "' is deprecated"
646 << " and will be removed in tensorflow 2.6 or higher."
647 << " Please switch to tensorflow-io"
648 << " (https://github.com/tensorflow/io) for file system"
649 << " support of '" << scheme << "'.";
650 }
651 // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
652 env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; })
653 .IgnoreError();
654 }
655 };
656
657 } // namespace register_file_system
658
659 // END_SKIP_DOXYGEN
660
661 } // namespace tensorflow
662
663 // Register a FileSystem implementation for a scheme. Files with names that have
664 // "scheme://" prefixes are routed to use this implementation.
665 #define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, legacy) \
666 REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, legacy)
667 #define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, legacy) \
668 REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy)
669 #define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy) \
670 static ::tensorflow::register_file_system::Register<factory> \
671 register_ff##ctr TF_ATTRIBUTE_UNUSED = \
672 ::tensorflow::register_file_system::Register<factory>(env, scheme, \
673 legacy)
674
675 #define REGISTER_FILE_SYSTEM(scheme, factory) \
676 REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, \
677 false);
678
679 #define REGISTER_LEGACY_FILE_SYSTEM(scheme, factory) \
680 REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, true);
681
682 #endif // TENSORFLOW_CORE_PLATFORM_ENV_H_
683