1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/platform/s3/s3_file_system.h"
17
18 #include "tensorflow/core/lib/core/status_test_util.h"
19 #include "tensorflow/core/lib/gtl/stl_util.h"
20 #include "tensorflow/core/lib/io/path.h"
21 #include "tensorflow/core/platform/file_system.h"
22 #include "tensorflow/core/platform/test.h"
23
24 namespace tensorflow {
25
26 namespace {
27
28 class S3FileSystemTest : public ::testing::Test {
29 protected:
S3FileSystemTest()30 S3FileSystemTest() {}
31
TmpDir(const string & path)32 string TmpDir(const string& path) {
33 char* test_dir = getenv("S3_TEST_TMPDIR");
34 if (test_dir != nullptr) {
35 return io::JoinPath(string(test_dir), path);
36 } else {
37 return "s3://" + io::JoinPath(testing::TmpDir(), path);
38 }
39 }
40
WriteString(const string & fname,const string & content)41 Status WriteString(const string& fname, const string& content) {
42 std::unique_ptr<WritableFile> writer;
43 TF_RETURN_IF_ERROR(s3fs.NewWritableFile(fname, &writer));
44 TF_RETURN_IF_ERROR(writer->Append(content));
45 TF_RETURN_IF_ERROR(writer->Close());
46 return Status::OK();
47 }
48
ReadAll(const string & fname,string * content)49 Status ReadAll(const string& fname, string* content) {
50 std::unique_ptr<RandomAccessFile> reader;
51 TF_RETURN_IF_ERROR(s3fs.NewRandomAccessFile(fname, &reader));
52
53 uint64 file_size = 0;
54 TF_RETURN_IF_ERROR(s3fs.GetFileSize(fname, &file_size));
55
56 content->resize(file_size);
57 StringPiece result;
58 TF_RETURN_IF_ERROR(
59 reader->Read(0, file_size, &result, gtl::string_as_array(content)));
60 if (file_size != result.size()) {
61 return errors::DataLoss("expected ", file_size, " got ", result.size(),
62 " bytes");
63 }
64 return Status::OK();
65 }
66
67 S3FileSystem s3fs;
68 };
69
TEST_F(S3FileSystemTest,NewRandomAccessFile)70 TEST_F(S3FileSystemTest, NewRandomAccessFile) {
71 const string fname = TmpDir("RandomAccessFile");
72 const string content = "abcdefghijklmn";
73
74 TF_ASSERT_OK(WriteString(fname, content));
75
76 std::unique_ptr<RandomAccessFile> reader;
77 TF_EXPECT_OK(s3fs.NewRandomAccessFile(fname, &reader));
78
79 string got;
80 got.resize(content.size());
81 StringPiece result;
82 TF_EXPECT_OK(
83 reader->Read(0, content.size(), &result, gtl::string_as_array(&got)));
84 EXPECT_EQ(content.size(), result.size());
85 EXPECT_EQ(content, result);
86
87 got.clear();
88 got.resize(4);
89 TF_EXPECT_OK(reader->Read(2, 4, &result, gtl::string_as_array(&got)));
90 EXPECT_EQ(4, result.size());
91 EXPECT_EQ(content.substr(2, 4), result);
92 }
93
TEST_F(S3FileSystemTest,NewWritableFile)94 TEST_F(S3FileSystemTest, NewWritableFile) {
95 std::unique_ptr<WritableFile> writer;
96 const string fname = TmpDir("WritableFile");
97 TF_EXPECT_OK(s3fs.NewWritableFile(fname, &writer));
98 TF_EXPECT_OK(writer->Append("content1,"));
99 TF_EXPECT_OK(writer->Append("content2"));
100 TF_EXPECT_OK(writer->Flush());
101 TF_EXPECT_OK(writer->Sync());
102 TF_EXPECT_OK(writer->Close());
103
104 string content;
105 TF_EXPECT_OK(ReadAll(fname, &content));
106 EXPECT_EQ("content1,content2", content);
107 }
108
TEST_F(S3FileSystemTest,NewAppendableFile)109 TEST_F(S3FileSystemTest, NewAppendableFile) {
110 std::unique_ptr<WritableFile> writer;
111
112 const string fname = TmpDir("AppendableFile");
113 TF_ASSERT_OK(WriteString(fname, "test"));
114
115 TF_EXPECT_OK(s3fs.NewAppendableFile(fname, &writer));
116 TF_EXPECT_OK(writer->Append("content"));
117 TF_EXPECT_OK(writer->Close());
118 }
119
TEST_F(S3FileSystemTest,NewReadOnlyMemoryRegionFromFile)120 TEST_F(S3FileSystemTest, NewReadOnlyMemoryRegionFromFile) {
121 const string fname = TmpDir("MemoryFile");
122 const string content = "content";
123 TF_ASSERT_OK(WriteString(fname, content));
124 std::unique_ptr<ReadOnlyMemoryRegion> region;
125 TF_EXPECT_OK(s3fs.NewReadOnlyMemoryRegionFromFile(fname, ®ion));
126
127 EXPECT_EQ(content, StringPiece(reinterpret_cast<const char*>(region->data()),
128 region->length()));
129 }
130
TEST_F(S3FileSystemTest,FileExists)131 TEST_F(S3FileSystemTest, FileExists) {
132 const string fname = TmpDir("FileExists");
133 // Ensure the file doesn't yet exist.
134 TF_ASSERT_OK(s3fs.DeleteFile(fname));
135 EXPECT_EQ(error::Code::NOT_FOUND, s3fs.FileExists(fname).code());
136 TF_ASSERT_OK(WriteString(fname, "test"));
137 TF_EXPECT_OK(s3fs.FileExists(fname));
138 }
139
TEST_F(S3FileSystemTest,GetChildren)140 TEST_F(S3FileSystemTest, GetChildren) {
141 const string base = TmpDir("GetChildren");
142 TF_EXPECT_OK(s3fs.CreateDir(base));
143
144 const string file = io::JoinPath(base, "TestFile.csv");
145 TF_EXPECT_OK(WriteString(file, "test"));
146
147 const string subdir = io::JoinPath(base, "SubDir");
148 TF_EXPECT_OK(s3fs.CreateDir(subdir));
149 // s3 object storage doesn't support empty directory, we create file in the
150 // directory
151 const string subfile = io::JoinPath(subdir, "TestSubFile.csv");
152 TF_EXPECT_OK(WriteString(subfile, "test"));
153
154 std::vector<string> children;
155 TF_EXPECT_OK(s3fs.GetChildren(base, &children));
156 std::sort(children.begin(), children.end());
157 EXPECT_EQ(std::vector<string>({"SubDir", "TestFile.csv"}), children);
158 }
159
TEST_F(S3FileSystemTest,DeleteFile)160 TEST_F(S3FileSystemTest, DeleteFile) {
161 const string fname = TmpDir("DeleteFile");
162 TF_ASSERT_OK(WriteString(fname, "test"));
163 TF_EXPECT_OK(s3fs.DeleteFile(fname));
164 }
165
TEST_F(S3FileSystemTest,GetFileSize)166 TEST_F(S3FileSystemTest, GetFileSize) {
167 const string fname = TmpDir("GetFileSize");
168 TF_ASSERT_OK(WriteString(fname, "test"));
169 uint64 file_size = 0;
170 TF_EXPECT_OK(s3fs.GetFileSize(fname, &file_size));
171 EXPECT_EQ(4, file_size);
172 }
173
TEST_F(S3FileSystemTest,CreateDir)174 TEST_F(S3FileSystemTest, CreateDir) {
175 // s3 object storage doesn't support empty directory, we create file in the
176 // directory
177 const string dir = TmpDir("CreateDir");
178 TF_EXPECT_OK(s3fs.CreateDir(dir));
179
180 const string file = io::JoinPath(dir, "CreateDirFile.csv");
181 TF_EXPECT_OK(WriteString(file, "test"));
182 FileStatistics stat;
183 TF_EXPECT_OK(s3fs.Stat(dir, &stat));
184 EXPECT_TRUE(stat.is_directory);
185 }
186
TEST_F(S3FileSystemTest,DeleteDir)187 TEST_F(S3FileSystemTest, DeleteDir) {
188 // s3 object storage doesn't support empty directory, we create file in the
189 // directory
190 const string dir = TmpDir("DeleteDir");
191 const string file = io::JoinPath(dir, "DeleteDirFile.csv");
192 TF_EXPECT_OK(WriteString(file, "test"));
193 EXPECT_FALSE(s3fs.DeleteDir(dir).ok());
194
195 TF_EXPECT_OK(s3fs.DeleteFile(file));
196 TF_EXPECT_OK(s3fs.DeleteDir(dir));
197 FileStatistics stat;
198 EXPECT_FALSE(s3fs.Stat(dir, &stat).ok());
199 }
200
TEST_F(S3FileSystemTest,RenameFile)201 TEST_F(S3FileSystemTest, RenameFile) {
202 const string fname1 = TmpDir("RenameFile1");
203 const string fname2 = TmpDir("RenameFile2");
204 TF_ASSERT_OK(WriteString(fname1, "test"));
205 TF_EXPECT_OK(s3fs.RenameFile(fname1, fname2));
206 string content;
207 TF_EXPECT_OK(ReadAll(fname2, &content));
208 EXPECT_EQ("test", content);
209 }
210
TEST_F(S3FileSystemTest,RenameFile_Overwrite)211 TEST_F(S3FileSystemTest, RenameFile_Overwrite) {
212 const string fname1 = TmpDir("RenameFile1");
213 const string fname2 = TmpDir("RenameFile2");
214
215 TF_ASSERT_OK(WriteString(fname2, "test"));
216 TF_EXPECT_OK(s3fs.FileExists(fname2));
217
218 TF_ASSERT_OK(WriteString(fname1, "test"));
219 TF_EXPECT_OK(s3fs.RenameFile(fname1, fname2));
220 string content;
221 TF_EXPECT_OK(ReadAll(fname2, &content));
222 EXPECT_EQ("test", content);
223 }
224
TEST_F(S3FileSystemTest,StatFile)225 TEST_F(S3FileSystemTest, StatFile) {
226 const string fname = TmpDir("StatFile");
227 TF_ASSERT_OK(WriteString(fname, "test"));
228 FileStatistics stat;
229 TF_EXPECT_OK(s3fs.Stat(fname, &stat));
230 EXPECT_EQ(4, stat.length);
231 EXPECT_FALSE(stat.is_directory);
232 }
233
234 } // namespace
235 } // namespace tensorflow
236