1 /** 2 * Copyright 2019-2023 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MINDSPORE_CORE_UTILS_SYSTEM_FILE_SYSTEM_H_ 18 #define MINDSPORE_CORE_UTILS_SYSTEM_FILE_SYSTEM_H_ 19 20 #include <cerrno> 21 #include <cstdint> 22 #include <cstdlib> 23 #include <cstdio> 24 #include <functional> 25 #include <string> 26 #include <memory> 27 #include <vector> 28 #include "utils/system/base.h" 29 #include "utils/log_adapter.h" 30 #include "utils/os.h" 31 #include "utils/convert_utils_base.h" 32 #include "include/common/debug/common.h" 33 34 namespace mindspore { 35 namespace system { 36 class WriteFile; 37 class PosixWriteFile; 38 using WriteFilePtr = std::shared_ptr<WriteFile>; 39 using PosixWriteFilePtr = std::shared_ptr<PosixWriteFile>; 40 constexpr size_t kMaxFileRWLength = static_cast<size_t>(2047) * 1024 * 1024; 41 // File system of create or delete directory 42 class FileSystem { 43 public: 44 FileSystem() = default; 45 46 virtual ~FileSystem() = default; 47 48 // Create a new read/write file with mode 49 virtual WriteFilePtr CreateWriteFile(const string &file_name, const char *mode = "w+") = 0; 50 51 // Check the file is exist? 52 virtual bool FileExist(const string &file_name) = 0; 53 54 // Delete the file 55 virtual bool DeleteFile(const string &file_name) = 0; 56 57 // Create a directory 58 virtual bool CreateDir(const string &dir_name) = 0; 59 60 // Delete the specified directory 61 virtual bool DeleteDir(const string &dir_name) = 0; 62 }; 63 64 // A file that can be read and write 65 class WriteFile { 66 public: WriteFile(const string & file_name)67 explicit WriteFile(const string &file_name) : file_name_(file_name) {} 68 69 virtual ~WriteFile() = default; 70 71 // Open the file using a special mode 72 virtual bool Open(const char *mode = "w+") = 0; 73 74 // append the content to file Write(const std::string & data)75 virtual bool Write(const std::string &data) { 76 MS_LOG(WARNING) << "Attention: Maybe not call the function."; 77 return true; 78 } 79 80 // Write to a file at a given offset like linux function pwrite 81 virtual bool PWrite(const void *buf, size_t nbytes, size_t offset) = 0; 82 83 // Read from a file at a given offset like linux function pwrite 84 virtual bool PRead(void *buf, size_t nbytes, size_t offset) = 0; 85 86 // Trunc file size to length 87 virtual bool Trunc(size_t length) = 0; 88 89 // Get size of this file 90 virtual size_t Size() = 0; 91 92 // name: return the file name get_file_name()93 string get_file_name() { return file_name_; } 94 95 // flush: flush local buffer data to filesystem. 96 virtual bool Flush() = 0; 97 98 // sync: sync the content to disk 99 virtual bool Sync() = 0; 100 101 // close the file 102 virtual bool Close() = 0; 103 104 protected: 105 string file_name_; 106 107 // The size of this file. 108 size_t size_{0}; 109 }; 110 111 #if defined(SYSTEM_ENV_POSIX) 112 // File system of create or delete directory for posix system 113 class MS_CORE_API PosixFileSystem : public FileSystem { 114 public: 115 PosixFileSystem() = default; 116 117 ~PosixFileSystem() override = default; 118 119 // create a new write file using a special mode 120 WriteFilePtr CreateWriteFile(const string &file_name, const char *mode) override; 121 122 // check the file is exist? 123 bool FileExist(const string &file_name) override; 124 125 // delete the file 126 bool DeleteFile(const string &file_name) override; 127 128 // Create a Directory 129 bool CreateDir(const string &dir_name) override; 130 131 // Delete the specified directory. 132 bool DeleteDir(const string &dir_name) override; 133 }; 134 135 // A file that can be read and write for posix 136 class PosixWriteFile : public WriteFile { 137 public: PosixWriteFile(const string & file_name)138 explicit PosixWriteFile(const string &file_name) : WriteFile(file_name), file_(nullptr) {} 139 PosixWriteFile(const PosixWriteFile &); 140 PosixWriteFile &operator=(const PosixWriteFile &); 141 ~PosixWriteFile()142 ~PosixWriteFile() override { 143 try { 144 if (file_ != nullptr) { 145 (void)fclose(file_); 146 file_ = nullptr; 147 } 148 } catch (const std::exception &e) { 149 MS_LOG(ERROR) << "Exception when closing file."; 150 } catch (...) { 151 MS_LOG(ERROR) << "Non standard exception when closing file."; 152 } 153 } 154 Open(const char * mode)155 bool Open(const char *mode) override { 156 if (file_ != nullptr) { 157 MS_LOG(WARNING) << "The File(" << file_name_ << ") already open."; 158 return true; 159 } 160 // check the path 161 if (file_name_.c_str() == nullptr) { 162 MS_LOG(EXCEPTION) << "The file path is null."; 163 } 164 if (file_name_.size() >= PATH_MAX) { 165 MS_LOG(EXCEPTION) << "The file name is too long, file name is " << file_name_ << "."; 166 } 167 168 // open the file 169 file_ = fopen(file_name_.c_str(), mode); 170 if (file_ == nullptr) { 171 MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR. " << ErrnoToString(errno); 172 return false; 173 } 174 return true; 175 } 176 Write(const std::string & data)177 bool Write(const std::string &data) override { 178 MS_LOG(DEBUG) << "Write data(" << data.size() << ") to file(" << this->file_name_ << ")."; 179 size_t r = fwrite(data.data(), 1, data.size(), file_); 180 if (r != data.size()) { 181 MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR. " << ErrnoToString(errno); 182 return false; 183 } 184 return true; 185 } 186 PWrite(const void * buf,size_t nbytes,size_t offset)187 bool PWrite(const void *buf, size_t nbytes, size_t offset) override { 188 MS_LOG(DEBUG) << "Write data(" << nbytes << ") at offset(" << offset << ")to file(" << file_name_ << ")."; 189 return POperate(buf, nullptr, nbytes, offset, false); 190 } 191 PRead(void * buf,size_t nbytes,size_t offset)192 bool PRead(void *buf, size_t nbytes, size_t offset) override { 193 MS_LOG(DEBUG) << "Read data(" << nbytes << ") at offset(" << offset << ")to file(" << file_name_ << ")."; 194 return POperate(nullptr, buf, nbytes, offset, true); 195 } 196 Trunc(size_t length)197 bool Trunc(size_t length) override { 198 MS_LOG(DEBUG) << "Trunc file(" << file_name_ << ") to size(" << length << ")"; 199 if (length == size_) { 200 return true; 201 } 202 if (ftruncate(fileno(file_), length) != 0) { 203 MS_LOG(ERROR) << "File(" << file_name_ << ") Trunc ERROR. " << ErrnoToString(errno); 204 return false; 205 } 206 size_ = length; 207 return true; 208 } 209 Size()210 size_t Size() override { return size_; } 211 Close()212 bool Close() override { 213 if (file_ == nullptr) { 214 MS_LOG(INFO) << "File(" << file_name_ << ") already close."; 215 return true; 216 } 217 bool result = true; 218 if (fclose(file_) != 0) { 219 MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR. " << ErrnoToString(errno); 220 result = false; 221 } 222 file_ = nullptr; 223 return result; 224 } 225 Flush()226 bool Flush() override { 227 if (fflush(file_) != 0) { 228 MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR. " << ErrnoToString(errno); 229 return false; 230 } 231 return true; 232 } 233 Sync()234 bool Sync() override { return Flush(); } 235 236 private: POperate(const void * write_buf,void * read_buf,size_t nbytes,size_t offset,bool read)237 bool POperate(const void *write_buf, void *read_buf, size_t nbytes, size_t offset, bool read) { 238 size_t left = nbytes; 239 size_t buff_offset = 0; 240 auto fd = fileno(file_); 241 while (left > 0) { 242 size_t length = 0; 243 if (left > kMaxFileRWLength) { 244 length = kMaxFileRWLength; 245 } else { 246 length = left; 247 } 248 left -= length; 249 ssize_t r = 0; 250 if (read && read_buf != nullptr) { 251 auto buff_p = static_cast<uint8_t *>(read_buf) + buff_offset; 252 r = pread(fd, buff_p, length, SizeToLong(offset + buff_offset)); 253 } else if (write_buf != nullptr) { 254 auto buff_p = static_cast<const uint8_t *>(write_buf) + buff_offset; 255 r = pwrite(fd, buff_p, length, SizeToLong(offset + buff_offset)); 256 } 257 if (r >= 0 && LongToSize(r) != length) { 258 MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR. " << ErrnoToString(errno); 259 return false; 260 } 261 buff_offset += length; 262 } 263 return true; 264 } 265 266 FILE *file_; 267 }; 268 #endif 269 270 #if defined(SYSTEM_ENV_WINDOWS) 271 // File system of create or delete directory for windows system 272 class MS_CORE_API WinFileSystem : public FileSystem { 273 public: 274 WinFileSystem() = default; 275 276 ~WinFileSystem() override = default; 277 278 // create a new write file with mode 279 WriteFilePtr CreateWriteFile(const string &file_name, const char *mode) override; 280 281 // check the file is exist? 282 bool FileExist(const string &file_name) override; 283 284 // delete the file 285 bool DeleteFile(const string &file_name) override; 286 287 // Create a Directory 288 bool CreateDir(const string &dir_name) override; 289 290 // Delete the specified directory. 291 bool DeleteDir(const string &dir_name) override; 292 }; 293 294 // A file that can be read and write for windows 295 class WinWriteFile : public WriteFile { 296 public: WinWriteFile(const string & file_name)297 explicit WinWriteFile(const string &file_name) : WriteFile(file_name), file_(nullptr) {} 298 299 ~WinWriteFile() override; 300 301 bool Open(const char *mode) override; 302 303 bool Write(const std::string &data) override; 304 305 bool PWrite(const void *buf, size_t nbytes, size_t offset) override; 306 307 bool PRead(void *buf, size_t nbytes, size_t offset) override; 308 309 bool Trunc(size_t length) override; 310 311 size_t Size() override; 312 313 bool Close() override; 314 315 bool Flush() override; 316 317 bool Sync() override; 318 319 private: 320 FILE *file_; 321 }; 322 #endif 323 } // namespace system 324 } // namespace mindspore 325 326 #endif // MINDSPORE_CORE_UTILS_SYSTEM_FILE_SYSTEM_H_ 327