1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/platform/file_system.h"
17
18 #include <sys/stat.h>
19
20 #include "tensorflow/core/lib/core/status_test_util.h"
21 #include "tensorflow/core/platform/null_file_system.h"
22 #include "tensorflow/core/platform/path.h"
23 #include "tensorflow/core/platform/str_util.h"
24 #include "tensorflow/core/platform/strcat.h"
25 #include "tensorflow/core/platform/test.h"
26
27 namespace tensorflow {
28
29 static const char* const kPrefix = "ipfs://solarsystem";
30
31 // A file system that has Planets, Satellites and Sub Satellites. Sub satellites
32 // cannot have children further.
33 class InterPlanetaryFileSystem : public NullFileSystem {
34 public:
35 TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
36
FileExists(const string & fname,TransactionToken * token)37 Status FileExists(const string& fname, TransactionToken* token) override {
38 string parsed_path;
39 ParsePath(fname, &parsed_path);
40 if (BodyExists(parsed_path)) {
41 return Status::OK();
42 }
43 return Status(tensorflow::error::NOT_FOUND, "File does not exist");
44 }
45
46 // Adds the dir to the parent's children list and creates an entry for itself.
CreateDir(const string & dirname,TransactionToken * token)47 Status CreateDir(const string& dirname, TransactionToken* token) override {
48 string parsed_path;
49 ParsePath(dirname, &parsed_path);
50 // If the directory already exists, throw an error.
51 if (celestial_bodies_.find(parsed_path) != celestial_bodies_.end()) {
52 return Status(tensorflow::error::ALREADY_EXISTS,
53 "dirname already exists.");
54 }
55 std::vector<string> split_path = str_util::Split(parsed_path, '/');
56 // If the path is too long then we don't support it.
57 if (split_path.size() > 3) {
58 return Status(tensorflow::error::INVALID_ARGUMENT, "Bad dirname");
59 }
60 if (split_path.empty()) {
61 return Status::OK();
62 }
63 if (split_path.size() == 1) {
64 celestial_bodies_[""].insert(parsed_path);
65 celestial_bodies_.insert(
66 std::pair<string, std::set<string>>(parsed_path, {}));
67 return Status::OK();
68 }
69 if (split_path.size() == 2) {
70 if (!BodyExists(split_path[0])) {
71 return Status(tensorflow::error::FAILED_PRECONDITION,
72 "Base dir not created");
73 }
74 celestial_bodies_[split_path[0]].insert(split_path[1]);
75 celestial_bodies_.insert(
76 std::pair<string, std::set<string>>(parsed_path, {}));
77 return Status::OK();
78 }
79 if (split_path.size() == 3) {
80 const string& parent_path = this->JoinPath(split_path[0], split_path[1]);
81 if (!BodyExists(parent_path)) {
82 return Status(tensorflow::error::FAILED_PRECONDITION,
83 "Base dir not created");
84 }
85 celestial_bodies_[parent_path].insert(split_path[2]);
86 celestial_bodies_.insert(
87 std::pair<string, std::set<string>>(parsed_path, {}));
88 return Status::OK();
89 }
90 return Status(tensorflow::error::FAILED_PRECONDITION, "Failed to create");
91 }
92
IsDirectory(const string & dirname,TransactionToken * token)93 Status IsDirectory(const string& dirname, TransactionToken* token) override {
94 string parsed_path;
95 ParsePath(dirname, &parsed_path);
96 // Simulate evil_directory has bad permissions by throwing a LOG(FATAL)
97 if (parsed_path == "evil_directory") {
98 LOG(FATAL) << "evil_directory cannot be accessed";
99 }
100 std::vector<string> split_path = str_util::Split(parsed_path, '/');
101 if (split_path.size() > 2) {
102 return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir");
103 }
104 if (celestial_bodies_.find(parsed_path) != celestial_bodies_.end()) {
105 return Status::OK();
106 }
107 return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir");
108 }
109
GetChildren(const string & dir,TransactionToken * token,std::vector<string> * result)110 Status GetChildren(const string& dir, TransactionToken* token,
111 std::vector<string>* result) override {
112 TF_RETURN_IF_ERROR(IsDirectory(dir, nullptr));
113 string parsed_path;
114 ParsePath(dir, &parsed_path);
115 result->insert(result->begin(), celestial_bodies_[parsed_path].begin(),
116 celestial_bodies_[parsed_path].end());
117 return Status::OK();
118 }
119
120 private:
BodyExists(const string & name)121 bool BodyExists(const string& name) {
122 return celestial_bodies_.find(name) != celestial_bodies_.end();
123 }
124
ParsePath(const string & name,string * parsed_path)125 void ParsePath(const string& name, string* parsed_path) {
126 StringPiece scheme, host, path;
127 this->ParseURI(name, &scheme, &host, &path);
128 ASSERT_EQ(scheme, "ipfs");
129 ASSERT_EQ(host, "solarsystem");
130 absl::ConsumePrefix(&path, "/");
131 *parsed_path = string(path);
132 }
133
134 std::map<string, std::set<string>> celestial_bodies_ = {
135 std::pair<string, std::set<string>>(
136 "", {"Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn",
137 "Uranus", "Neptune"}),
138 std::pair<string, std::set<string>>("Mercury", {}),
139 std::pair<string, std::set<string>>("Venus", {}),
140 std::pair<string, std::set<string>>("Earth", {"Moon"}),
141 std::pair<string, std::set<string>>("Mars", {}),
142 std::pair<string, std::set<string>>("Jupiter",
143 {"Europa", "Io", "Ganymede"}),
144 std::pair<string, std::set<string>>("Saturn", {}),
145 std::pair<string, std::set<string>>("Uranus", {}),
146 std::pair<string, std::set<string>>("Neptune", {}),
147 std::pair<string, std::set<string>>("Earth/Moon", {}),
148 std::pair<string, std::set<string>>("Jupiter/Europa", {}),
149 std::pair<string, std::set<string>>("Jupiter/Io", {}),
150 std::pair<string, std::set<string>>("Jupiter/Ganymede", {})};
151 };
152
153 // Returns all the matched entries as a comma separated string removing the
154 // common prefix of BaseDir().
Match(InterPlanetaryFileSystem * ipfs,const string & suffix_pattern)155 string Match(InterPlanetaryFileSystem* ipfs, const string& suffix_pattern) {
156 std::vector<string> results;
157 Status s = ipfs->GetMatchingPaths(ipfs->JoinPath(kPrefix, suffix_pattern),
158 nullptr, &results);
159 if (!s.ok()) {
160 return s.ToString();
161 } else {
162 std::vector<StringPiece> trimmed_results;
163 std::sort(results.begin(), results.end());
164 for (const string& result : results) {
165 StringPiece trimmed_result(result);
166 EXPECT_TRUE(
167 absl::ConsumePrefix(&trimmed_result, strings::StrCat(kPrefix, "/")));
168 trimmed_results.push_back(trimmed_result);
169 }
170 return absl::StrJoin(trimmed_results, ",");
171 }
172 }
173
TEST(InterPlanetaryFileSystemTest,IPFSMatch)174 TEST(InterPlanetaryFileSystemTest, IPFSMatch) {
175 InterPlanetaryFileSystem ipfs;
176 EXPECT_EQ(Match(&ipfs, "thereisnosuchfile"), "");
177 EXPECT_EQ(Match(&ipfs, "*"),
178 "Earth,Jupiter,Mars,Mercury,Neptune,Saturn,Uranus,Venus");
179 // Returns Jupiter's moons.
180 EXPECT_EQ(Match(&ipfs, "Jupiter/*"),
181 "Jupiter/Europa,Jupiter/Ganymede,Jupiter/Io");
182 // Returns Jupiter's and Earth's moons.
183 EXPECT_EQ(Match(&ipfs, "*/*"),
184 "Earth/Moon,Jupiter/Europa,Jupiter/Ganymede,Jupiter/Io");
185 TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "Planet0"), nullptr));
186 TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "Planet1"), nullptr));
187 EXPECT_EQ(Match(&ipfs, "Planet[0-1]"), "Planet0,Planet1");
188 EXPECT_EQ(Match(&ipfs, "Planet?"), "Planet0,Planet1");
189 }
190
TEST(InterPlanetaryFileSystemTest,MatchSimple)191 TEST(InterPlanetaryFileSystemTest, MatchSimple) {
192 InterPlanetaryFileSystem ipfs;
193 TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-00"), nullptr));
194 TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-0a"), nullptr));
195 TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-01"), nullptr));
196 TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-aaa"), nullptr));
197
198 EXPECT_EQ(Match(&ipfs, "match-*"), "match-00,match-01,match-0a,match-aaa");
199 EXPECT_EQ(Match(&ipfs, "match-0[0-9]"), "match-00,match-01");
200 EXPECT_EQ(Match(&ipfs, "match-?[0-9]"), "match-00,match-01");
201 EXPECT_EQ(Match(&ipfs, "match-?a*"), "match-0a,match-aaa");
202 EXPECT_EQ(Match(&ipfs, "match-??"), "match-00,match-01,match-0a");
203 }
204
205 // Create 2 directories abcd and evil_directory. Look for abcd and make sure
206 // that evil_directory isn't accessed.
TEST(InterPlanetaryFileSystemTest,MatchOnlyNeeded)207 TEST(InterPlanetaryFileSystemTest, MatchOnlyNeeded) {
208 InterPlanetaryFileSystem ipfs;
209 TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "abcd"), nullptr));
210 TF_EXPECT_OK(
211 ipfs.CreateDir(ipfs.JoinPath(kPrefix, "evil_directory"), nullptr));
212
213 EXPECT_EQ(Match(&ipfs, "abcd"), "abcd");
214 }
215
TEST(InterPlanetaryFileSystemTest,MatchDirectory)216 TEST(InterPlanetaryFileSystemTest, MatchDirectory) {
217 InterPlanetaryFileSystem ipfs;
218 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
219 ipfs.JoinPath(kPrefix, "match-00/abc/x"), nullptr));
220 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
221 ipfs.JoinPath(kPrefix, "match-0a/abc/x"), nullptr));
222 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
223 ipfs.JoinPath(kPrefix, "match-01/abc/x"), nullptr));
224 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
225 ipfs.JoinPath(kPrefix, "match-aaa/abc/x"), nullptr));
226
227 EXPECT_EQ(Match(&ipfs, "match-*/abc/x"),
228 "match-00/abc/x,match-01/abc/x,match-0a/abc/x,match-aaa/abc/x");
229 EXPECT_EQ(Match(&ipfs, "match-0[0-9]/abc/x"),
230 "match-00/abc/x,match-01/abc/x");
231 EXPECT_EQ(Match(&ipfs, "match-?[0-9]/abc/x"),
232 "match-00/abc/x,match-01/abc/x");
233 EXPECT_EQ(Match(&ipfs, "match-?a*/abc/x"), "match-0a/abc/x,match-aaa/abc/x");
234 EXPECT_EQ(Match(&ipfs, "match-?[^a]/abc/x"), "match-00/abc/x,match-01/abc/x");
235 }
236
TEST(InterPlanetaryFileSystemTest,MatchMultipleWildcards)237 TEST(InterPlanetaryFileSystemTest, MatchMultipleWildcards) {
238 InterPlanetaryFileSystem ipfs;
239 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
240 ipfs.JoinPath(kPrefix, "match-00/abc/00"), nullptr));
241 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
242 ipfs.JoinPath(kPrefix, "match-00/abc/01"), nullptr));
243 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
244 ipfs.JoinPath(kPrefix, "match-00/abc/09"), nullptr));
245 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
246 ipfs.JoinPath(kPrefix, "match-01/abc/00"), nullptr));
247 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
248 ipfs.JoinPath(kPrefix, "match-01/abc/04"), nullptr));
249 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
250 ipfs.JoinPath(kPrefix, "match-01/abc/10"), nullptr));
251 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(
252 ipfs.JoinPath(kPrefix, "match-02/abc/00"), nullptr));
253
254 EXPECT_EQ(Match(&ipfs, "match-0[0-1]/abc/0[0-8]"),
255 "match-00/abc/00,match-00/abc/01,match-01/abc/00,match-01/abc/04");
256 }
257
TEST(InterPlanetaryFileSystemTest,RecursivelyCreateAlreadyExistingDir)258 TEST(InterPlanetaryFileSystemTest, RecursivelyCreateAlreadyExistingDir) {
259 InterPlanetaryFileSystem ipfs;
260 const string dirname = ipfs.JoinPath(kPrefix, "match-00/abc/00");
261 TF_EXPECT_OK(ipfs.RecursivelyCreateDir(dirname));
262 // We no longer check for recursively creating the directory again because
263 // `ipfs.IsDirectory` is badly implemented, fixing it will break other tests
264 // in this suite and we already test creating the directory again in
265 // env_test.cc as well as in the modular filesystem tests.
266 }
267
TEST(InterPlanetaryFileSystemTest,HasAtomicMove)268 TEST(InterPlanetaryFileSystemTest, HasAtomicMove) {
269 InterPlanetaryFileSystem ipfs;
270 const string dirname = io::JoinPath(kPrefix, "match-00/abc/00");
271 bool has_atomic_move;
272 TF_EXPECT_OK(ipfs.HasAtomicMove(dirname, &has_atomic_move));
273 EXPECT_EQ(has_atomic_move, true);
274 }
275
276 // A simple file system with a root directory and a single file underneath it.
277 class TestFileSystem : public NullFileSystem {
278 public:
279 // Only allow for a single root directory.
IsDirectory(const string & dirname,TransactionToken * token)280 Status IsDirectory(const string& dirname, TransactionToken* token) override {
281 if (dirname == "." || dirname.empty()) {
282 return Status::OK();
283 }
284 return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir");
285 }
286
287 // Simulating a FS with a root dir and a single file underneath it.
GetChildren(const string & dir,TransactionToken * token,std::vector<string> * result)288 Status GetChildren(const string& dir, TransactionToken* token,
289 std::vector<string>* result) override {
290 if (dir == "." || dir.empty()) {
291 result->push_back("test");
292 }
293 return Status::OK();
294 }
295 };
296
297 // Making sure that ./<pattern> and <pattern> have the same result.
TEST(TestFileSystemTest,RootDirectory)298 TEST(TestFileSystemTest, RootDirectory) {
299 TestFileSystem fs;
300 std::vector<string> results;
301 auto ret = fs.GetMatchingPaths("./te*", nullptr, &results);
302 EXPECT_EQ(1, results.size());
303 EXPECT_EQ("./test", results[0]);
304 ret = fs.GetMatchingPaths("te*", nullptr, &results);
305 EXPECT_EQ(1, results.size());
306 EXPECT_EQ("./test", results[0]);
307 }
308
309 } // namespace tensorflow
310