1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 #ifndef TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ 16 #define TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ 17 18 #include <memory> 19 20 #include "tensorflow/c/experimental/filesystem/filesystem_interface.h" 21 #include "tensorflow/core/platform/file_system.h" 22 23 /// This file builds classes needed to hold a filesystem implementation in the 24 /// modular world. Once all TensorFlow filesystems are converted to use the 25 /// plugin based approach, this file will replace the one in core/platform and 26 /// the names will lose the `Modular` part. Until that point, the `Modular*` 27 /// classes here are experimental and subject to breaking changes. 28 /// For documentation on these methods, consult `core/platform/filesystem.h`. 29 30 namespace tensorflow { 31 32 // TODO(b/143949615): After all filesystems are converted, this file will be 33 // moved to core/platform, and this class can become a singleton and replace the 34 // need for `Env::Default()`. At that time, we might decide to remove the need 35 // for `Env::Default()` altogether, but that's a different project, not in 36 // scope for now. I'm just mentioning this here as that transition will mean 37 // removal of the registration part from `Env` and adding it here instead: we 38 // will need tables to hold for each scheme the function tables that implement 39 // the needed functionality instead of the current `FileSystemRegistry` code in 40 // `core/platform/env.cc`. 41 class ModularFileSystem final : public FileSystem { 42 public: ModularFileSystem(std::unique_ptr<TF_Filesystem> filesystem,std::unique_ptr<const TF_FilesystemOps> filesystem_ops,std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops,std::unique_ptr<const TF_WritableFileOps> writable_file_ops,std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> read_only_memory_region_ops,std::function<void * (size_t)> plugin_memory_allocate,std::function<void (void *)> plugin_memory_free)43 ModularFileSystem( 44 std::unique_ptr<TF_Filesystem> filesystem, 45 std::unique_ptr<const TF_FilesystemOps> filesystem_ops, 46 std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops, 47 std::unique_ptr<const TF_WritableFileOps> writable_file_ops, 48 std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> 49 read_only_memory_region_ops, 50 std::function<void*(size_t)> plugin_memory_allocate, 51 std::function<void(void*)> plugin_memory_free) 52 : filesystem_(std::move(filesystem)), 53 ops_(std::move(filesystem_ops)), 54 random_access_file_ops_(std::move(random_access_file_ops)), 55 writable_file_ops_(std::move(writable_file_ops)), 56 read_only_memory_region_ops_(std::move(read_only_memory_region_ops)), 57 plugin_memory_allocate_(std::move(plugin_memory_allocate)), 58 plugin_memory_free_(std::move(plugin_memory_free)) {} 59 ~ModularFileSystem()60 ~ModularFileSystem() override { ops_->cleanup(filesystem_.get()); } 61 62 TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; 63 64 Status NewRandomAccessFile( 65 const std::string& fname, TransactionToken* token, 66 std::unique_ptr<RandomAccessFile>* result) override; 67 Status NewWritableFile(const std::string& fname, TransactionToken* token, 68 std::unique_ptr<WritableFile>* result) override; 69 Status NewAppendableFile(const std::string& fname, TransactionToken* token, 70 std::unique_ptr<WritableFile>* result) override; 71 Status NewReadOnlyMemoryRegionFromFile( 72 const std::string& fname, TransactionToken* token, 73 std::unique_ptr<ReadOnlyMemoryRegion>* result) override; 74 Status FileExists(const std::string& fname, TransactionToken* token) override; 75 bool FilesExist(const std::vector<std::string>& files, 76 TransactionToken* token, 77 std::vector<Status>* status) override; 78 Status GetChildren(const std::string& dir, TransactionToken* token, 79 std::vector<std::string>* result) override; 80 Status GetMatchingPaths(const std::string& pattern, TransactionToken* token, 81 std::vector<std::string>* results) override; 82 Status DeleteFile(const std::string& fname, TransactionToken* token) override; 83 Status DeleteRecursively(const std::string& dirname, TransactionToken* token, 84 int64* undeleted_files, 85 int64* undeleted_dirs) override; 86 Status DeleteDir(const std::string& dirname, 87 TransactionToken* token) override; 88 Status RecursivelyCreateDir(const std::string& dirname, 89 TransactionToken* token) override; 90 Status CreateDir(const std::string& dirname, 91 TransactionToken* token) override; 92 Status Stat(const std::string& fname, TransactionToken* token, 93 FileStatistics* stat) override; 94 Status IsDirectory(const std::string& fname, 95 TransactionToken* token) override; 96 Status GetFileSize(const std::string& fname, TransactionToken* token, 97 uint64* file_size) override; 98 Status RenameFile(const std::string& src, const std::string& target, 99 TransactionToken* token) override; 100 Status CopyFile(const std::string& src, const std::string& target, 101 TransactionToken* token) override; 102 std::string TranslateName(const std::string& name) const override; 103 void FlushCaches(TransactionToken* token) override; 104 105 private: 106 std::unique_ptr<TF_Filesystem> filesystem_; 107 std::unique_ptr<const TF_FilesystemOps> ops_; 108 std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops_; 109 std::unique_ptr<const TF_WritableFileOps> writable_file_ops_; 110 std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> 111 read_only_memory_region_ops_; 112 std::function<void*(size_t)> plugin_memory_allocate_; 113 std::function<void(void*)> plugin_memory_free_; 114 TF_DISALLOW_COPY_AND_ASSIGN(ModularFileSystem); 115 }; 116 117 class ModularRandomAccessFile final : public RandomAccessFile { 118 public: ModularRandomAccessFile(const std::string & filename,std::unique_ptr<TF_RandomAccessFile> file,const TF_RandomAccessFileOps * ops)119 ModularRandomAccessFile(const std::string& filename, 120 std::unique_ptr<TF_RandomAccessFile> file, 121 const TF_RandomAccessFileOps* ops) 122 : filename_(filename), file_(std::move(file)), ops_(ops) {} 123 ~ModularRandomAccessFile()124 ~ModularRandomAccessFile() override { ops_->cleanup(file_.get()); } 125 126 Status Read(uint64 offset, size_t n, StringPiece* result, 127 char* scratch) const override; 128 Status Name(StringPiece* result) const override; 129 130 private: 131 std::string filename_; 132 std::unique_ptr<TF_RandomAccessFile> file_; 133 const TF_RandomAccessFileOps* ops_; // not owned 134 TF_DISALLOW_COPY_AND_ASSIGN(ModularRandomAccessFile); 135 }; 136 137 class ModularWritableFile final : public WritableFile { 138 public: ModularWritableFile(const std::string & filename,std::unique_ptr<TF_WritableFile> file,const TF_WritableFileOps * ops)139 ModularWritableFile(const std::string& filename, 140 std::unique_ptr<TF_WritableFile> file, 141 const TF_WritableFileOps* ops) 142 : filename_(filename), file_(std::move(file)), ops_(ops) {} 143 ~ModularWritableFile()144 ~ModularWritableFile() override { ops_->cleanup(file_.get()); } 145 146 Status Append(StringPiece data) override; 147 Status Close() override; 148 Status Flush() override; 149 Status Sync() override; 150 Status Name(StringPiece* result) const override; 151 Status Tell(int64* position) override; 152 153 private: 154 std::string filename_; 155 std::unique_ptr<TF_WritableFile> file_; 156 const TF_WritableFileOps* ops_; // not owned 157 TF_DISALLOW_COPY_AND_ASSIGN(ModularWritableFile); 158 }; 159 160 class ModularReadOnlyMemoryRegion final : public ReadOnlyMemoryRegion { 161 public: ModularReadOnlyMemoryRegion(std::unique_ptr<TF_ReadOnlyMemoryRegion> region,const TF_ReadOnlyMemoryRegionOps * ops)162 ModularReadOnlyMemoryRegion(std::unique_ptr<TF_ReadOnlyMemoryRegion> region, 163 const TF_ReadOnlyMemoryRegionOps* ops) 164 : region_(std::move(region)), ops_(ops) {} 165 ~ModularReadOnlyMemoryRegion()166 ~ModularReadOnlyMemoryRegion() override { ops_->cleanup(region_.get()); }; 167 data()168 const void* data() override { return ops_->data(region_.get()); } length()169 uint64 length() override { return ops_->length(region_.get()); } 170 171 private: 172 std::unique_ptr<TF_ReadOnlyMemoryRegion> region_; 173 const TF_ReadOnlyMemoryRegionOps* ops_; // not owned 174 TF_DISALLOW_COPY_AND_ASSIGN(ModularReadOnlyMemoryRegion); 175 }; 176 177 // Registers a filesystem plugin so that core TensorFlow can use it. 178 Status RegisterFilesystemPlugin(const std::string& dso_path); 179 180 } // namespace tensorflow 181 182 #endif // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ 183