• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #ifndef TENSORFLOW_CORE_PLATFORM_ENV_H_
17 #define TENSORFLOW_CORE_PLATFORM_ENV_H_
18 
19 #include <stdint.h>
20 
21 #include <memory>
22 #include <string>
23 #include <unordered_map>
24 #include <vector>
25 
26 #include "tensorflow/core/platform/env_time.h"
27 #include "tensorflow/core/platform/errors.h"
28 #include "tensorflow/core/platform/file_system.h"
29 #include "tensorflow/core/platform/macros.h"
30 #include "tensorflow/core/platform/mutex.h"
31 #include "tensorflow/core/platform/numa.h"
32 #include "tensorflow/core/platform/platform.h"
33 #include "tensorflow/core/platform/protobuf.h"
34 #include "tensorflow/core/platform/status.h"
35 #include "tensorflow/core/platform/stringpiece.h"
36 #include "tensorflow/core/platform/types.h"
37 
38 // Delete the definition of CopyFile as the linker gets confused.
39 #ifdef PLATFORM_WINDOWS
40 #undef CopyFile
41 #endif
42 
43 namespace tensorflow {
44 
45 class Thread;
46 struct ThreadOptions;
47 
48 /// \brief An interface used by the tensorflow implementation to
49 /// access operating system functionality like the filesystem etc.
50 ///
51 /// Callers may wish to provide a custom Env object to get fine grain
52 /// control.
53 ///
54 /// All Env implementations are safe for concurrent access from
55 /// multiple threads without any external synchronization.
56 class Env {
57  public:
58   Env();
59   virtual ~Env() = default;
60 
61   /// \brief Returns a default environment suitable for the current operating
62   /// system.
63   ///
64   /// Sophisticated users may wish to provide their own Env
65   /// implementation instead of relying on this default environment.
66   ///
67   /// The result of Default() belongs to this library and must never be deleted.
68   static Env* Default();
69 
70   /// \brief Returns the FileSystem object to handle operations on the file
71   /// specified by 'fname'. The FileSystem object is used as the implementation
72   /// for the file system related (non-virtual) functions that follow.
73   /// Returned FileSystem object is still owned by the Env object and will
74   // (might) be destroyed when the environment is destroyed.
75   virtual Status GetFileSystemForFile(const std::string& fname,
76                                       FileSystem** result);
77 
78   /// \brief Returns the file system schemes registered for this Env.
79   virtual Status GetRegisteredFileSystemSchemes(
80       std::vector<std::string>* schemes);
81 
82   /// \brief Register a file system for a scheme.
83   virtual Status RegisterFileSystem(const std::string& scheme,
84                                     FileSystemRegistry::Factory factory);
85 
86   /// \brief Register a modular file system for a scheme.
87   ///
88   /// Same as `RegisterFileSystem` but for filesystems provided by plugins.
89   ///
90   /// TODO(mihaimaruseac): After all filesystems are converted, make this be the
91   /// canonical registration function.
92   virtual Status RegisterFileSystem(const std::string& scheme,
93                                     std::unique_ptr<FileSystem> filesystem);
94 
95   /// \brief Flush filesystem caches for all registered filesystems.
96   Status FlushFileSystemCaches();
97 
98   /// \brief Creates a brand new random access read-only file with the
99   /// specified name.
100 
101   /// On success, stores a pointer to the new file in
102   /// *result and returns OK.  On failure stores NULL in *result and
103   /// returns non-OK.  If the file does not exist, returns a non-OK
104   /// status.
105   ///
106   /// The returned file may be concurrently accessed by multiple threads.
107   ///
108   /// The ownership of the returned RandomAccessFile is passed to the caller
109   /// and the object should be deleted when is not used. The file object
110   /// shouldn't live longer than the Env object.
111   Status NewRandomAccessFile(const std::string& fname,
112                              std::unique_ptr<RandomAccessFile>* result);
113 
NewRandomAccessFile(const std::string & fname,TransactionToken * token,std::unique_ptr<RandomAccessFile> * result)114   Status NewRandomAccessFile(const std::string& fname, TransactionToken* token,
115                              std::unique_ptr<RandomAccessFile>* result) {
116     // We duplicate these methods due to Google internal coding style prevents
117     // virtual functions with default arguments. See PR #41615.
118     return Status::OK();
119   }
120 
121   /// \brief Creates an object that writes to a new file with the specified
122   /// name.
123   ///
124   /// Deletes any existing file with the same name and creates a
125   /// new file.  On success, stores a pointer to the new file in
126   /// *result and returns OK.  On failure stores NULL in *result and
127   /// returns non-OK.
128   ///
129   /// The returned file will only be accessed by one thread at a time.
130   ///
131   /// The ownership of the returned WritableFile is passed to the caller
132   /// and the object should be deleted when is not used. The file object
133   /// shouldn't live longer than the Env object.
134   Status NewWritableFile(const std::string& fname,
135                          std::unique_ptr<WritableFile>* result);
136 
NewWritableFile(const std::string & fname,TransactionToken * token,std::unique_ptr<WritableFile> * result)137   Status NewWritableFile(const std::string& fname, TransactionToken* token,
138                          std::unique_ptr<WritableFile>* result) {
139     return Status::OK();
140   }
141 
142   /// \brief Creates an object that either appends to an existing file, or
143   /// writes to a new file (if the file does not exist to begin with).
144   ///
145   /// On success, stores a pointer to the new file in *result and
146   /// returns OK.  On failure stores NULL in *result and returns
147   /// non-OK.
148   ///
149   /// The returned file will only be accessed by one thread at a time.
150   ///
151   /// The ownership of the returned WritableFile is passed to the caller
152   /// and the object should be deleted when is not used. The file object
153   /// shouldn't live longer than the Env object.
154   Status NewAppendableFile(const std::string& fname,
155                            std::unique_ptr<WritableFile>* result);
156 
NewAppendableFile(const std::string & fname,TransactionToken * token,std::unique_ptr<WritableFile> * result)157   Status NewAppendableFile(const std::string& fname, TransactionToken* token,
158                            std::unique_ptr<WritableFile>* result) {
159     return Status::OK();
160   }
161   /// \brief Creates a readonly region of memory with the file context.
162   ///
163   /// On success, it returns a pointer to read-only memory region
164   /// from the content of file fname. The ownership of the region is passed to
165   /// the caller. On failure stores nullptr in *result and returns non-OK.
166   ///
167   /// The returned memory region can be accessed from many threads in parallel.
168   ///
169   /// The ownership of the returned ReadOnlyMemoryRegion is passed to the caller
170   /// and the object should be deleted when is not used. The memory region
171   /// object shouldn't live longer than the Env object.
172   Status NewReadOnlyMemoryRegionFromFile(
173       const std::string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result);
174 
NewReadOnlyMemoryRegionFromFile(const std::string & fname,TransactionToken * token,std::unique_ptr<ReadOnlyMemoryRegion> * result)175   Status NewReadOnlyMemoryRegionFromFile(
176       const std::string& fname, TransactionToken* token,
177       std::unique_ptr<ReadOnlyMemoryRegion>* result) {
178     return Status::OK();
179   }
180 
181   /// Returns OK if the named path exists and NOT_FOUND otherwise.
182   Status FileExists(const std::string& fname);
183 
FileExists(const std::string & fname,TransactionToken * token)184   Status FileExists(const std::string& fname, TransactionToken* token) {
185     return Status::OK();
186   }
187 
188   /// Returns true if all the listed files exist, false otherwise.
189   /// if status is not null, populate the vector with a detailed status
190   /// for each file.
191   bool FilesExist(const std::vector<string>& files,
192                   std::vector<Status>* status);
193 
FilesExist(const std::vector<string> & files,TransactionToken * token,std::vector<Status> * status)194   bool FilesExist(const std::vector<string>& files, TransactionToken* token,
195                   std::vector<Status>* status) {
196     return true;
197   }
198 
199   /// \brief Stores in *result the names of the children of the specified
200   /// directory. The names are relative to "dir".
201   ///
202   /// Original contents of *results are dropped.
203   Status GetChildren(const std::string& dir, std::vector<string>* result);
204 
GetChildren(const std::string & dir,TransactionToken * token,std::vector<string> * result)205   Status GetChildren(const std::string& dir, TransactionToken* token,
206                      std::vector<string>* result) {
207     return Status::OK();
208   }
209 
210   /// \brief Returns true if the path matches the given pattern. The wildcards
211   /// allowed in pattern are described in FileSystem::GetMatchingPaths.
212   virtual bool MatchPath(const std::string& path,
213                          const std::string& pattern) = 0;
214 
215   /// \brief Given a pattern, stores in *results the set of paths that matches
216   /// that pattern. *results is cleared.
217   ///
218   /// More details about `pattern` in FileSystem::GetMatchingPaths.
219   virtual Status GetMatchingPaths(const std::string& pattern,
220                                   std::vector<string>* results);
221 
GetMatchingPaths(const std::string & pattern,TransactionToken * token,std::vector<string> * results)222   Status GetMatchingPaths(const std::string& pattern, TransactionToken* token,
223                           std::vector<string>* results) {
224     return Status::OK();
225   }
226 
227   /// Deletes the named file.
228   Status DeleteFile(const std::string& fname);
229 
DeleteFile(const std::string & fname,TransactionToken * token)230   Status DeleteFile(const std::string& fname, TransactionToken* token) {
231     return Status::OK();
232   }
233 
234   /// \brief Deletes the specified directory and all subdirectories and files
235   /// underneath it. This is accomplished by traversing the directory tree
236   /// rooted at dirname and deleting entries as they are encountered.
237   ///
238   /// If dirname itself is not readable or does not exist, *undeleted_dir_count
239   /// is set to 1, *undeleted_file_count is set to 0 and an appropriate status
240   /// (e.g. NOT_FOUND) is returned.
241   ///
242   /// If dirname and all its descendants were successfully deleted, TF_OK is
243   /// returned and both error counters are set to zero.
244   ///
245   /// Otherwise, while traversing the tree, undeleted_file_count and
246   /// undeleted_dir_count are updated if an entry of the corresponding type
247   /// could not be deleted. The returned error status represents the reason that
248   /// any one of these entries could not be deleted.
249   ///
250   /// REQUIRES: undeleted_files, undeleted_dirs to be not null.
251   ///
252   /// Typical return codes:
253   ///  * OK - dirname exists and we were able to delete everything underneath.
254   ///  * NOT_FOUND - dirname doesn't exist
255   ///  * PERMISSION_DENIED - dirname or some descendant is not writable
256   ///  * UNIMPLEMENTED - Some underlying functions (like Delete) are not
257   ///                    implemented
258   Status DeleteRecursively(const std::string& dirname, int64* undeleted_files,
259                            int64* undeleted_dirs);
260 
DeleteRecursively(const std::string & dirname,TransactionToken * token,int64 * undeleted_files,int64 * undeleted_dirs)261   Status DeleteRecursively(const std::string& dirname, TransactionToken* token,
262                            int64* undeleted_files, int64* undeleted_dirs) {
263     return Status::OK();
264   }
265 
266   /// \brief Creates the specified directory and all the necessary
267   /// subdirectories. Typical return codes.
268   ///  * OK - successfully created the directory and sub directories, even if
269   ///         they were already created.
270   ///  * PERMISSION_DENIED - dirname or some subdirectory is not writable.
271   Status RecursivelyCreateDir(const std::string& dirname);
272 
RecursivelyCreateDir(const std::string & dirname,TransactionToken * token)273   Status RecursivelyCreateDir(const std::string& dirname,
274                               TransactionToken* token) {
275     return Status::OK();
276   }
277   /// \brief Creates the specified directory. Typical return codes
278   ///  * OK - successfully created the directory.
279   ///  * ALREADY_EXISTS - directory already exists.
280   ///  * PERMISSION_DENIED - dirname is not writable.
281   Status CreateDir(const std::string& dirname);
282 
CreateDir(const std::string & dirname,TransactionToken * token)283   Status CreateDir(const std::string& dirname, TransactionToken* token) {
284     return Status::OK();
285   }
286 
287   /// Deletes the specified directory.
288   Status DeleteDir(const std::string& dirname);
289 
DeleteDir(const std::string & dirname,TransactionToken * token)290   Status DeleteDir(const std::string& dirname, TransactionToken* token) {
291     return Status::OK();
292   }
293 
294   /// Obtains statistics for the given path.
295   Status Stat(const std::string& fname, FileStatistics* stat);
296 
Stat(const std::string & fname,TransactionToken * token,FileStatistics * stat)297   Status Stat(const std::string& fname, TransactionToken* token,
298               FileStatistics* stat) {
299     return Status::OK();
300   }
301 
302   /// \brief Returns whether the given path is a directory or not.
303   /// Typical return codes (not guaranteed exhaustive):
304   ///  * OK - The path exists and is a directory.
305   ///  * FAILED_PRECONDITION - The path exists and is not a directory.
306   ///  * NOT_FOUND - The path entry does not exist.
307   ///  * PERMISSION_DENIED - Insufficient permissions.
308   ///  * UNIMPLEMENTED - The file factory doesn't support directories.
309   Status IsDirectory(const std::string& fname);
310 
311   /// \brief Returns whether the given path is on a file system
312   /// that has atomic move capabilities. This can be used
313   /// to determine if there needs to be a temp location to safely write objects.
314   /// The second boolean argument has_atomic_move contains this information.
315   ///
316   /// Returns one of the following status codes (not guaranteed exhaustive):
317   ///  * OK - The path is on a recognized file system,
318   ///         so has_atomic_move holds the above information.
319   ///  * UNIMPLEMENTED - The file system of the path hasn't been implemented in
320   ///  TF
321   Status HasAtomicMove(const std::string& path, bool* has_atomic_move);
322 
323   /// Stores the size of `fname` in `*file_size`.
324   Status GetFileSize(const std::string& fname, uint64* file_size);
325 
GetFileSize(const std::string & fname,TransactionToken * token,uint64 * file_size)326   Status GetFileSize(const std::string& fname, TransactionToken* token,
327                      uint64* file_size) {
328     return Status::OK();
329   }
330 
331   /// \brief Renames file src to target. If target already exists, it will be
332   /// replaced.
333   Status RenameFile(const std::string& src, const std::string& target);
334 
RenameFile(const std::string & src,const std::string & target,TransactionToken * token)335   Status RenameFile(const std::string& src, const std::string& target,
336                     TransactionToken* token) {
337     return Status::OK();
338   }
339 
340   /// \brief Copy the src to target.
341   Status CopyFile(const std::string& src, const std::string& target);
342 
CopyFile(const std::string & src,const std::string & target,TransactionToken * token)343   Status CopyFile(const std::string& src, const std::string& target,
344                   TransactionToken* token) {
345     return Status::OK();
346   }
347 
348   /// \brief starts a new transaction on the filesystem that handles filename
StartTransaction(const std::string & filename,TransactionToken ** token)349   Status StartTransaction(const std::string& filename,
350                           TransactionToken** token) {
351     token = nullptr;
352     return Status::OK();
353   }
354 
355   /// \brief Adds `path` to transaction in `token` if token belongs to
356   /// filesystem that handles the path.
AddToTransaction(const std::string & path,TransactionToken * token)357   Status AddToTransaction(const std::string& path, TransactionToken* token) {
358     return Status::OK();
359   }
360 
361   /// \brief Get token for `path` or start a new transaction and add `path` to
362   /// it.
GetTokenOrStartTransaction(const std::string & path,TransactionToken ** token)363   Status GetTokenOrStartTransaction(const std::string& path,
364                                     TransactionToken** token) {
365     *token = nullptr;
366     return Status::OK();
367   }
368 
369   /// \brief Returns the transaction for `path` or nullptr in `token`
GetTransactionForPath(const std::string & path,TransactionToken ** token)370   Status GetTransactionForPath(const std::string& path,
371                                TransactionToken** token) {
372     token = nullptr;
373     return Status::OK();
374   }
375 
376   /// \brief Finalizes the transaction
EndTransaction(TransactionToken * token)377   Status EndTransaction(TransactionToken* token) { return Status::OK(); }
378 
379   /// \brief Returns the absolute path of the current executable. It resolves
380   /// symlinks if there is any.
381   std::string GetExecutablePath();
382 
383   /// Creates a local unique temporary file name. Returns true if success.
384   bool LocalTempFilename(std::string* filename);
385 
386   /// Creates a local unique file name that starts with |prefix| and ends with
387   /// |suffix|. Returns true if success.
388   bool CreateUniqueFileName(std::string* prefix, const std::string& suffix);
389 
390   /// \brief Return the runfiles directory if running under bazel. Returns
391   /// the directory the executable is located in if not running under bazel.
392   virtual std::string GetRunfilesDir() = 0;
393 
394   // TODO(jeff,sanjay): Add back thread/thread-pool support if needed.
395   // TODO(jeff,sanjay): if needed, tighten spec so relative to epoch, or
396   // provide a routine to get the absolute time.
397 
398   /// \brief Returns the number of nano-seconds since the Unix epoch.
NowNanos()399   virtual uint64 NowNanos() const { return EnvTime::NowNanos(); }
400 
401   /// \brief Returns the number of micro-seconds since the Unix epoch.
NowMicros()402   virtual uint64 NowMicros() const { return EnvTime::NowMicros(); }
403 
404   /// \brief Returns the number of seconds since the Unix epoch.
NowSeconds()405   virtual uint64 NowSeconds() const { return EnvTime::NowSeconds(); }
406 
407   /// Sleeps/delays the thread for the prescribed number of micro-seconds.
408   virtual void SleepForMicroseconds(int64 micros) = 0;
409 
410   /// \brief Returns a new thread that is running fn() and is identified
411   /// (for debugging/performance-analysis) by "name".
412   ///
413   /// Caller takes ownership of the result and must delete it eventually
414   /// (the deletion will block until fn() stops running).
415   virtual Thread* StartThread(const ThreadOptions& thread_options,
416                               const std::string& name,
417                               std::function<void()> fn) TF_MUST_USE_RESULT = 0;
418 
419   // Returns the thread id of calling thread.
420   // Posix: Returns pthread id which is only guaranteed to be unique within a
421   //        process.
422   // Windows: Returns thread id which is unique.
423   virtual int32 GetCurrentThreadId() = 0;
424 
425   // Copies current thread name to "name". Returns true if success.
426   virtual bool GetCurrentThreadName(std::string* name) = 0;
427 
428   // \brief Schedules the given closure on a thread-pool.
429   //
430   // NOTE(mrry): This closure may block.
431   virtual void SchedClosure(std::function<void()> closure) = 0;
432 
433   // \brief Schedules the given closure on a thread-pool after the given number
434   // of microseconds.
435   //
436   // NOTE(mrry): This closure must not block.
437   virtual void SchedClosureAfter(int64 micros,
438                                  std::function<void()> closure) = 0;
439 
440   // \brief Load a dynamic library.
441   //
442   // Pass "library_filename" to a platform-specific mechanism for dynamically
443   // loading a library.  The rules for determining the exact location of the
444   // library are platform-specific and are not documented here.
445   //
446   // On success, returns a handle to the library in "*handle" and returns
447   // OK from the function.
448   // Otherwise returns nullptr in "*handle" and an error status from the
449   // function.
450   virtual Status LoadDynamicLibrary(const char* library_filename,
451                                     void** handle) = 0;
452 
453   // \brief Get a pointer to a symbol from a dynamic library.
454   //
455   // "handle" should be a pointer returned from a previous call to LoadLibrary.
456   // On success, store a pointer to the located symbol in "*symbol" and return
457   // OK from the function. Otherwise, returns nullptr in "*symbol" and an error
458   // status from the function.
459   virtual Status GetSymbolFromLibrary(void* handle, const char* symbol_name,
460                                       void** symbol) = 0;
461 
462   // \brief build the name of dynamic library.
463   //
464   // "name" should be name of the library.
465   // "version" should be the version of the library or NULL
466   // returns the name that LoadLibrary() can use
467   virtual std::string FormatLibraryFileName(const std::string& name,
468                                             const std::string& version) = 0;
469 
470   // Returns a possible list of local temporary directories.
471   virtual void GetLocalTempDirectories(std::vector<string>* list) = 0;
472 
473  private:
474   std::unique_ptr<FileSystemRegistry> file_system_registry_;
475   TF_DISALLOW_COPY_AND_ASSIGN(Env);
476 };
477 
478 /// \brief An implementation of Env that forwards all calls to another Env.
479 ///
480 /// May be useful to clients who wish to override just part of the
481 /// functionality of another Env.
482 class EnvWrapper : public Env {
483  public:
484   /// Initializes an EnvWrapper that delegates all calls to *t
EnvWrapper(Env * t)485   explicit EnvWrapper(Env* t) : target_(t) {}
486   ~EnvWrapper() override;
487 
488   /// Returns the target to which this Env forwards all calls
target()489   Env* target() const { return target_; }
490 
GetFileSystemForFile(const std::string & fname,FileSystem ** result)491   Status GetFileSystemForFile(const std::string& fname,
492                               FileSystem** result) override {
493     return target_->GetFileSystemForFile(fname, result);
494   }
495 
GetRegisteredFileSystemSchemes(std::vector<string> * schemes)496   Status GetRegisteredFileSystemSchemes(std::vector<string>* schemes) override {
497     return target_->GetRegisteredFileSystemSchemes(schemes);
498   }
499 
RegisterFileSystem(const std::string & scheme,FileSystemRegistry::Factory factory)500   Status RegisterFileSystem(const std::string& scheme,
501                             FileSystemRegistry::Factory factory) override {
502     return target_->RegisterFileSystem(scheme, factory);
503   }
504 
MatchPath(const std::string & path,const std::string & pattern)505   bool MatchPath(const std::string& path, const std::string& pattern) override {
506     return target_->MatchPath(path, pattern);
507   }
508 
NowMicros()509   uint64 NowMicros() const override { return target_->NowMicros(); }
SleepForMicroseconds(int64 micros)510   void SleepForMicroseconds(int64 micros) override {
511     target_->SleepForMicroseconds(micros);
512   }
StartThread(const ThreadOptions & thread_options,const std::string & name,std::function<void ()> fn)513   Thread* StartThread(const ThreadOptions& thread_options,
514                       const std::string& name,
515                       std::function<void()> fn) override {
516     return target_->StartThread(thread_options, name, fn);
517   }
GetCurrentThreadId()518   int32 GetCurrentThreadId() override { return target_->GetCurrentThreadId(); }
GetCurrentThreadName(std::string * name)519   bool GetCurrentThreadName(std::string* name) override {
520     return target_->GetCurrentThreadName(name);
521   }
SchedClosure(std::function<void ()> closure)522   void SchedClosure(std::function<void()> closure) override {
523     target_->SchedClosure(closure);
524   }
SchedClosureAfter(int64 micros,std::function<void ()> closure)525   void SchedClosureAfter(int64 micros, std::function<void()> closure) override {
526     target_->SchedClosureAfter(micros, closure);
527   }
LoadDynamicLibrary(const char * library_filename,void ** handle)528   Status LoadDynamicLibrary(const char* library_filename,
529                             void** handle) override {
530     return target_->LoadDynamicLibrary(library_filename, handle);
531   }
GetSymbolFromLibrary(void * handle,const char * symbol_name,void ** symbol)532   Status GetSymbolFromLibrary(void* handle, const char* symbol_name,
533                               void** symbol) override {
534     return target_->GetSymbolFromLibrary(handle, symbol_name, symbol);
535   }
FormatLibraryFileName(const std::string & name,const std::string & version)536   std::string FormatLibraryFileName(const std::string& name,
537                                     const std::string& version) override {
538     return target_->FormatLibraryFileName(name, version);
539   }
540 
GetRunfilesDir()541   std::string GetRunfilesDir() override { return target_->GetRunfilesDir(); }
542 
543  private:
GetLocalTempDirectories(std::vector<string> * list)544   void GetLocalTempDirectories(std::vector<string>* list) override {
545     target_->GetLocalTempDirectories(list);
546   }
547 
548   Env* target_;
549 };
550 
551 /// Represents a thread used to run a TensorFlow function.
552 class Thread {
553  public:
Thread()554   Thread() {}
555 
556   /// Blocks until the thread of control stops running.
557   virtual ~Thread();
558 
559  private:
560   TF_DISALLOW_COPY_AND_ASSIGN(Thread);
561 };
562 
563 /// \brief Cross-platform setenv.
564 ///
565 /// Since setenv() is not available on windows, we provide an
566 /// alternative with platform specific implementations here.
567 int setenv(const char* name, const char* value, int overwrite);
568 
569 /// Cross-platform unsetenv.
570 int unsetenv(const char* name);
571 
572 /// \brief Options to configure a Thread.
573 ///
574 /// Note that the options are all hints, and the
575 /// underlying implementation may choose to ignore it.
576 struct ThreadOptions {
577   /// Thread stack size to use (in bytes).
578   size_t stack_size = 0;  // 0: use system default value
579   /// Guard area size to use near thread stacks to use (in bytes)
580   size_t guard_size = 0;  // 0: use system default value
581   int numa_node = port::kNUMANoAffinity;
582 };
583 
584 /// A utility routine: copy contents of `src` in file system `src_fs`
585 /// to `target` in file system `target_fs`.
586 Status FileSystemCopyFile(FileSystem* src_fs, const std::string& src,
587                           FileSystem* target_fs, const std::string& target);
588 
589 /// A utility routine: reads contents of named file into `*data`
590 Status ReadFileToString(Env* env, const std::string& fname, std::string* data);
591 
592 /// A utility routine: write contents of `data` to file named `fname`
593 /// (overwriting existing contents, if any).
594 Status WriteStringToFile(Env* env, const std::string& fname,
595                          const StringPiece& data);
596 
597 /// Write binary representation of "proto" to the named file.
598 Status WriteBinaryProto(Env* env, const std::string& fname,
599                         const protobuf::MessageLite& proto);
600 
601 /// Reads contents of named file and parse as binary encoded proto data
602 /// and store into `*proto`.
603 Status ReadBinaryProto(Env* env, const std::string& fname,
604                        protobuf::MessageLite* proto);
605 
606 /// Write the text representation of "proto" to the named file.
607 Status WriteTextProto(Env* env, const std::string& fname,
608                       const protobuf::Message& proto);
609 
610 /// Read contents of named file and parse as text encoded proto data
611 /// and store into `*proto`.
ReadTextProto(Env *,const std::string &,protobuf::MessageLite *)612 inline Status ReadTextProto(Env* /* env */, const std::string& /* fname */,
613                             protobuf::MessageLite* /* proto */) {
614   return errors::Unimplemented("Can't parse text protos with protolite.");
615 }
616 Status ReadTextProto(Env* env, const std::string& fname,
617                      protobuf::Message* proto);
618 
619 /// Read contents of named file and parse as either text or binary encoded proto
620 /// data and store into `*proto`.
621 Status ReadTextOrBinaryProto(Env* env, const std::string& fname,
622                              protobuf::Message* proto);
623 Status ReadTextOrBinaryProto(Env* env, const std::string& fname,
624                              protobuf::MessageLite* proto);
625 
626 // START_SKIP_DOXYGEN
627 
628 // The following approach to register filesystems is deprecated and will be
629 // replaced with modular filesystem plugins registration.
630 // TODO(mihaimaruseac): After all filesystems are converted, remove this.
631 namespace register_file_system {
632 
633 template <typename Factory>
634 struct Register {
RegisterRegister635   Register(Env* env, const std::string& scheme, bool legacy) {
636     // TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs
637     // after TF 2.6+.
638     if (legacy) {
639       const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM");
640       string enable_legacy =
641           enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : "";
642       if (!(enable_legacy == "true" || enable_legacy == "1")) {
643         return;
644       }
645       LOG(WARNING) << "Legacy file system for '" << scheme << "' is deprecated"
646                    << " and will be removed in tensorflow 2.6 or higher."
647                    << " Please switch to tensorflow-io"
648                    << " (https://github.com/tensorflow/io) for file system"
649                    << " support of '" << scheme << "'.";
650     }
651     // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
652     env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; })
653         .IgnoreError();
654   }
655 };
656 
657 }  // namespace register_file_system
658 
659 // END_SKIP_DOXYGEN
660 
661 }  // namespace tensorflow
662 
663 // Register a FileSystem implementation for a scheme. Files with names that have
664 // "scheme://" prefixes are routed to use this implementation.
665 #define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, legacy) \
666   REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, legacy)
667 #define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, legacy) \
668   REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy)
669 #define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy)         \
670   static ::tensorflow::register_file_system::Register<factory>               \
671       register_ff##ctr TF_ATTRIBUTE_UNUSED =                                 \
672           ::tensorflow::register_file_system::Register<factory>(env, scheme, \
673                                                                 legacy)
674 
675 #define REGISTER_FILE_SYSTEM(scheme, factory)                             \
676   REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, \
677                            false);
678 
679 #define REGISTER_LEGACY_FILE_SYSTEM(scheme, factory) \
680   REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, true);
681 
682 #endif  // TENSORFLOW_CORE_PLATFORM_ENV_H_
683