1 /**
2 * Copyright 2019 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "minddata/dataset/util/path.h"
17
18 #include <sys/stat.h>
19 #include <fcntl.h>
20 #include <new>
21 #include <sstream>
22
23 #ifdef _MSC_VER
24 #include <direct.h> // for _mkdir
25 #define stat _stat64 // for file size exceeds (1<<31)-1 bytes
26 #endif
27
28 #include "securec.h"
29 #ifndef BUILD_LITE
30 #include "mindspore/core/utils/file_utils.h"
31 #else
32 #include "mindspore/lite/src/common/file_utils.h"
33 #endif
34 #include "utils/ms_utils.h"
35 #include "minddata/dataset/util/log_adapter.h"
36
37 namespace mindspore {
38 namespace dataset {
39 #if defined(_WIN32) || defined(_WIN64)
40 char Path::separator_ = '\\';
41 #else
42 char Path::separator_ = '/';
43 #endif
44
Path(const std::string & s)45 Path::Path(const std::string &s) {
46 #if defined(_WIN32) || defined(_WIN64)
47 path_ = FileUtils::UTF_8ToGB2312(s.data());
48 #else
49 path_ = s;
50 #endif
51 }
52
Path(const char * p)53 Path::Path(const char *p) {
54 #if defined(_WIN32) || defined(_WIN64)
55 path_ = FileUtils::UTF_8ToGB2312(p);
56 #else
57 path_ = p;
58 #endif
59 }
60
61 Path::Path(const Path &p) = default;
62
operator =(const Path & p)63 Path &Path::operator=(const Path &p) {
64 if (&p != this) {
65 this->path_ = p.path_;
66 }
67 return *this;
68 }
69
operator =(Path && p)70 Path &Path::operator=(Path &&p) noexcept {
71 if (&p != this) {
72 this->path_ = std::move(p.path_);
73 }
74 return *this;
75 }
76
Path(Path && p)77 Path::Path(Path &&p) noexcept { this->path_ = std::move(p.path_); }
78
operator +(const Path & p)79 Path Path::operator+(const Path &p) {
80 std::string q = path_ + p.ToString();
81 return Path(q);
82 }
83
operator +(const std::string & p)84 Path Path::operator+(const std::string &p) {
85 std::string q = path_ + p;
86 return Path(q);
87 }
88
operator +(const char * p)89 Path Path::operator+(const char *p) {
90 std::string q = path_ + p;
91 return Path(q);
92 }
93
operator +=(const Path & rhs)94 Path &Path::operator+=(const Path &rhs) {
95 path_ += rhs.ToString();
96 return *this;
97 }
98
operator +=(const std::string & p)99 Path &Path::operator+=(const std::string &p) {
100 #if defined(_WIN32) || defined(_WIN64)
101 path_ += FileUtils::UTF_8ToGB2312(p.data());
102 #else
103 path_ += p;
104 #endif
105 return *this;
106 }
107
operator +=(const char * p)108 Path &Path::operator+=(const char *p) {
109 #if defined(_WIN32) || defined(_WIN64)
110 path_ += FileUtils::UTF_8ToGB2312(p);
111 #else
112 path_ += p;
113 #endif
114 return *this;
115 }
116
operator /(const Path & p)117 Path Path::operator/(const Path &p) {
118 std::string q = path_ + separator_ + p.ToString();
119 return Path(q);
120 }
121
operator /(const std::string & p)122 Path Path::operator/(const std::string &p) {
123 std::string q = path_ + separator_ + p;
124 return Path(q);
125 }
126
operator /(const char * p)127 Path Path::operator/(const char *p) {
128 std::string q = path_ + separator_ + p;
129 return Path(q);
130 }
131
Extension() const132 std::string Path::Extension() const {
133 std::size_t found = path_.find_last_of('.');
134 if (found != std::string::npos) {
135 return path_.substr(found);
136 } else {
137 return std::string("");
138 }
139 }
140
Exists()141 bool Path::Exists() {
142 struct stat sb {};
143 int rc = stat(common::SafeCStr(path_), &sb);
144 if (rc == -1 && errno != ENOENT) {
145 MS_LOG(WARNING) << "Unable to query the status of " << path_ << ". Errno = " << errno << ".";
146 }
147 return (rc == 0);
148 }
149
IsDirectory()150 bool Path::IsDirectory() {
151 struct stat sb {};
152 int rc = stat(common::SafeCStr(path_), &sb);
153 if (rc == 0) {
154 return S_ISDIR(sb.st_mode);
155 } else {
156 return false;
157 }
158 }
159
IsFile()160 bool Path::IsFile() {
161 struct stat sb {};
162 int rc = stat(common::SafeCStr(path_), &sb);
163 if (rc == 0) {
164 return S_ISREG(sb.st_mode);
165 } else {
166 return false;
167 }
168 }
169
CreateDirectory(bool is_common_dir)170 Status Path::CreateDirectory(bool is_common_dir) {
171 if (!Exists()) {
172 #if defined(_WIN32) || defined(_WIN64)
173 #ifndef _MSC_VER
174 int rc = mkdir(common::SafeCStr(path_));
175 #else
176 int rc = _mkdir(common::SafeCStr(path_));
177 #endif
178 #else
179 int rc = mkdir(common::SafeCStr(path_), S_IRUSR | S_IWUSR | S_IXUSR);
180 #endif
181 if (rc) {
182 std::ostringstream oss;
183 oss << "Unable to create directory " << path_ << ". Errno = " << errno;
184 RETURN_STATUS_UNEXPECTED(oss.str());
185 }
186 return Status::OK();
187 } else {
188 if (IsDirectory()) {
189 return Status::OK();
190 } else {
191 std::ostringstream oss;
192 oss << "Unable to create directory " << path_ << ". It exists but is not a directory";
193 RETURN_STATUS_UNEXPECTED(oss.str());
194 }
195 }
196 }
197
ParentPath()198 std::string Path::ParentPath() {
199 std::string r;
200 std::size_t found = path_.find_last_of(separator_);
201 if (found != std::string::npos) {
202 if (found == 0) {
203 r += separator_;
204 } else {
205 r = std::string(path_.substr(0, found));
206 }
207 }
208 return r;
209 }
210
CreateDirectories(bool is_common_dir)211 Status Path::CreateDirectories(bool is_common_dir) {
212 if (IsDirectory()) {
213 MS_LOG(DEBUG) << "Directory " << ToString() << " already exists.";
214 return Status::OK();
215 } else {
216 MS_LOG(DEBUG) << "Creating directory " << ToString() << ".";
217 std::string parent = ParentPath();
218 if (!parent.empty()) {
219 if (Path(parent).CreateDirectories(is_common_dir)) {
220 return CreateDirectory(is_common_dir);
221 }
222 } else {
223 return CreateDirectory(is_common_dir);
224 }
225 }
226 return Status::OK();
227 }
228
CreateCommonDirectories()229 Status Path::CreateCommonDirectories() { return CreateDirectories(true); }
230
Remove()231 Status Path::Remove() {
232 if (Exists()) {
233 if (IsDirectory()) {
234 #ifndef _MSC_VER
235 errno_t err = rmdir(common::SafeCStr(path_));
236 #else
237 errno_t err = _rmdir(common::SafeCStr(path_));
238 #endif
239 if (err == -1) {
240 std::ostringstream oss;
241 oss << "Unable to delete directory " << path_ << ". Errno = " << errno;
242 RETURN_STATUS_UNEXPECTED(oss.str());
243 }
244 } else {
245 errno_t err = unlink(common::SafeCStr(path_));
246 if (err == -1) {
247 std::ostringstream oss;
248 oss << "Unable to delete file " << path_ << ". Errno = " << errno;
249 RETURN_STATUS_UNEXPECTED(oss.str());
250 }
251 }
252 }
253 return Status::OK();
254 }
255
CreateFile(int * file_descriptor)256 Status Path::CreateFile(int *file_descriptor) { return OpenFile(file_descriptor, true); }
257
OpenFile(int * file_descriptor,bool create)258 Status Path::OpenFile(int *file_descriptor, bool create) {
259 int fd;
260 if (file_descriptor == nullptr) {
261 RETURN_STATUS_UNEXPECTED("null pointer");
262 }
263 if (IsDirectory()) {
264 std::ostringstream oss;
265 oss << "Unable to create file " << path_ << " which is a directory.";
266 RETURN_STATUS_UNEXPECTED(oss.str());
267 }
268 // Convert to canonical form.
269 if (strlen(common::SafeCStr(path_)) >= PATH_MAX) {
270 RETURN_STATUS_UNEXPECTED(strerror(errno));
271 }
272 char canonical_path[PATH_MAX] = {0x00};
273 #if defined(_WIN32) || defined(_WIN64)
274 auto err = _fullpath(canonical_path, common::SafeCStr(path_), PATH_MAX);
275 #else
276 auto err = realpath(common::SafeCStr(path_), canonical_path);
277 #endif
278 if (err == nullptr) {
279 if (errno == ENOENT && create) {
280 // File doesn't exist and we are to create it. Let's break it down.
281 auto file_part = Basename();
282 auto parent_part = ParentPath();
283 #if defined(_WIN32) || defined(_WIN64)
284 auto parent_err = _fullpath(canonical_path, common::SafeCStr(parent_part), PATH_MAX);
285 #else
286 auto parent_err = realpath(common::SafeCStr(parent_part), canonical_path);
287 #endif
288 if (parent_err == nullptr) {
289 RETURN_STATUS_UNEXPECTED(strerror(errno));
290 }
291 auto cur_inx = strlen(canonical_path);
292 if (cur_inx + file_part.length() >= PATH_MAX) {
293 RETURN_STATUS_UNEXPECTED(strerror(errno));
294 }
295 canonical_path[cur_inx++] = separator_;
296 if (strncpy_s(canonical_path + cur_inx, PATH_MAX - cur_inx, common::SafeCStr(file_part), file_part.length()) !=
297 EOK) {
298 RETURN_STATUS_UNEXPECTED(strerror(errno));
299 }
300 } else {
301 RETURN_STATUS_UNEXPECTED(strerror(errno));
302 }
303 }
304 if (create) {
305 fd = open(canonical_path, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR);
306 } else {
307 fd = open(canonical_path, O_RDWR);
308 }
309 if (fd == -1) {
310 RETURN_STATUS_UNEXPECTED(strerror(errno));
311 }
312 *file_descriptor = fd;
313 return Status::OK();
314 }
315
CloseFile(int fd) const316 Status Path::CloseFile(int fd) const {
317 if (close(fd) < 0) {
318 RETURN_STATUS_UNEXPECTED(strerror(errno));
319 }
320 return Status::OK();
321 }
322
TruncateFile(int fd) const323 Status Path::TruncateFile(int fd) const {
324 #ifdef _MSC_VER
325 int rc = _chsize(fd, 0);
326 #else
327 int rc = ftruncate(fd, 0);
328 #endif
329 if (rc == 0) {
330 return Status::OK();
331 } else {
332 RETURN_STATUS_UNEXPECTED(strerror(errno));
333 }
334 }
335
Basename()336 std::string Path::Basename() {
337 std::size_t found = path_.find_last_of(separator_);
338 if (found != std::string::npos) {
339 return path_.substr(found + 1);
340 } else {
341 return path_;
342 }
343 }
344
OpenDirectory(Path * f)345 std::shared_ptr<Path::DirIterator> Path::DirIterator::OpenDirectory(Path *f) {
346 auto it = new (std::nothrow) DirIterator(f);
347
348 if (it == nullptr) {
349 return nullptr;
350 }
351
352 if (it->dp_) {
353 return std::shared_ptr<DirIterator>(it);
354 } else {
355 delete it;
356 return nullptr;
357 }
358 }
359
~DirIterator()360 Path::DirIterator::~DirIterator() {
361 if (dp_) {
362 (void)closedir(dp_);
363 }
364 dp_ = nullptr;
365 dir_ = nullptr;
366 entry_ = nullptr;
367 }
368
DirIterator(Path * f)369 Path::DirIterator::DirIterator(Path *f) : dir_(f), dp_(nullptr), entry_(nullptr) {
370 MS_LOG(DEBUG) << "Open directory " << f->ToString() << ".";
371 dp_ = opendir(f->ToString().c_str());
372 }
373
HasNext()374 bool Path::DirIterator::HasNext() {
375 do {
376 entry_ = readdir(dp_);
377 if (entry_) {
378 if (strcmp(entry_->d_name, ".") == 0 || strcmp(entry_->d_name, "..") == 0) {
379 continue;
380 }
381 }
382 break;
383 } while (true);
384 return (entry_ != nullptr);
385 }
386
Next()387 Path Path::DirIterator::Next() { return (*(this->dir_) / Path(entry_->d_name)); }
388
RealPath(const std::string & path,std::string & realpath_str)389 Status Path::RealPath(const std::string &path, std::string &realpath_str) {
390 char real_path[PATH_MAX] = {0};
391 // input_path is only file_name
392 #if defined(_WIN32) || defined(_WIN64)
393 CHECK_FAIL_RETURN_UNEXPECTED(path.length() < PATH_MAX,
394 "The length of path: " + path + " exceeds limit: " + std::to_string(PATH_MAX));
395 auto ret = _fullpath(real_path, common::SafeCStr(path), PATH_MAX);
396 CHECK_FAIL_RETURN_UNEXPECTED(ret != nullptr, "The file " + path + " does not exist.");
397 #else
398 CHECK_FAIL_RETURN_UNEXPECTED(path.length() < NAME_MAX,
399 "The length of path: " + path + " exceeds limit: " + std::to_string(NAME_MAX));
400 auto ret = realpath(common::SafeCStr(path), real_path);
401 CHECK_FAIL_RETURN_UNEXPECTED(ret != nullptr, "The file " + path + " does not exist.");
402 #endif
403 realpath_str = std::string(real_path);
404 return Status::OK();
405 }
406
operator <<(std::ostream & os,const Path & s)407 std::ostream &operator<<(std::ostream &os, const Path &s) {
408 os << s.path_;
409 return os;
410 }
411 } // namespace dataset
412 } // namespace mindspore
413