1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/platform/cloud/gcs_file_system.h"
17 #include <fstream>
18 #include "tensorflow/core/lib/core/errors.h"
19 #include "tensorflow/core/lib/core/status_test_util.h"
20 #include "tensorflow/core/lib/strings/str_util.h"
21 #include "tensorflow/core/platform/cloud/http_request_fake.h"
22 #include "tensorflow/core/platform/test.h"
23
24 namespace tensorflow {
25 namespace {
26
27 static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30);
28 static RetryConfig kTestRetryConfig(0 /* init_delay_time_us */);
29
30 // Default (empty) constraint config
31 static std::unordered_set<string>* kAllowedLocationsDefault =
32 new std::unordered_set<string>();
33 // Constraint config if bucket location constraint is turned on, with no
34 // custom list
35 static std::unordered_set<string>* kAllowedLocationsAuto =
36 new std::unordered_set<string>({"auto"});
37
38 class FakeAuthProvider : public AuthProvider {
39 public:
GetToken(string * token)40 Status GetToken(string* token) override {
41 *token = "fake_token";
42 return Status::OK();
43 }
44 };
45
46 class FakeZoneProvider : public ZoneProvider {
47 public:
GetZone(string * zone)48 Status GetZone(string* zone) override {
49 *zone = "us-east1-b";
50 return Status::OK();
51 }
52 };
53
TEST(GcsFileSystemTest,NewRandomAccessFile_NoBlockCache)54 TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
55 std::vector<HttpRequest*> requests(
56 {new FakeHttpRequest(
57 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
58 "Auth Token: fake_token\n"
59 "Range: 0-5\n"
60 "Timeouts: 5 1 20\n",
61 "012345"),
62 new FakeHttpRequest(
63 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
64 "Auth Token: fake_token\n"
65 "Range: 6-11\n"
66 "Timeouts: 5 1 20\n",
67 "6789")});
68 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
69 std::unique_ptr<HttpRequest::Factory>(
70 new FakeHttpRequestFactory(&requests)),
71 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
72 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
73 0 /* stat cache max age */, 0 /* stat cache max entries */,
74 0 /* matching paths cache max age */,
75 0 /* matching paths cache max entries */, kTestRetryConfig,
76 kTestTimeoutConfig, *kAllowedLocationsDefault,
77 nullptr /* gcs additional header */);
78
79 std::unique_ptr<RandomAccessFile> file;
80 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
81
82 StringPiece filename;
83 TF_EXPECT_OK(file->Name(&filename));
84 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
85
86 char scratch[6];
87 StringPiece result;
88
89 // Read the first chunk.
90 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
91 EXPECT_EQ("012345", result);
92
93 // Read the second chunk.
94 EXPECT_EQ(
95 errors::Code::OUT_OF_RANGE,
96 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch).code());
97 EXPECT_EQ("6789", result);
98 }
99
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintInSameLocation)100 TEST(GcsFileSystemTest,
101 NewRandomAccessFile_WithLocationConstraintInSameLocation) {
102 std::vector<HttpRequest*> requests({new FakeHttpRequest(
103 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
104 "Auth Token: fake_token\n"
105 "Timeouts: 5 1 10\n",
106 R"(
107 {
108 "location":"US-EAST1"
109 })")});
110
111 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
112 std::unique_ptr<HttpRequest::Factory>(
113 new FakeHttpRequestFactory(&requests)),
114 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
115 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
116 0 /* stat cache max age */, 0 /* stat cache max entries */,
117 0 /* matching paths cache max age */,
118 0 /* matching paths cache max entries */, kTestRetryConfig,
119 kTestTimeoutConfig, *kAllowedLocationsAuto,
120 nullptr /* gcs additional header */);
121
122 std::unique_ptr<RandomAccessFile> file;
123 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
124 }
125
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintCaching)126 TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) {
127 std::vector<HttpRequest*> requests(
128 {new FakeHttpRequest(
129 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
130 "Auth Token: fake_token\n"
131 "Timeouts: 5 1 10\n",
132 R"(
133 {
134 "location":"US-EAST1"
135 })"),
136 new FakeHttpRequest(
137 "Uri: https://www.googleapis.com/storage/v1/b/anotherbucket\n"
138 "Auth Token: fake_token\n"
139 "Timeouts: 5 1 10\n",
140 R"(
141 {
142 "location":"US-EAST1"
143 })"),
144 new FakeHttpRequest(
145 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
146 "Auth Token: fake_token\n"
147 "Timeouts: 5 1 10\n",
148 R"(
149 {
150 "location":"US-EAST1"
151 })")});
152
153 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
154 std::unique_ptr<HttpRequest::Factory>(
155 new FakeHttpRequestFactory(&requests)),
156 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
157 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
158 0 /* stat cache max age */, 0 /* stat cache max entries */,
159 0 /* matching paths cache max age */,
160 0 /* matching paths cache max entries */, kTestRetryConfig,
161 kTestTimeoutConfig, *kAllowedLocationsAuto,
162 nullptr /* gcs additional header */);
163
164 std::unique_ptr<RandomAccessFile> file;
165
166 string bucket = "gs://bucket/random_access.txt";
167 string another_bucket = "gs://anotherbucket/random_access.txt";
168 // Multiple calls should only cause one request to the location api.
169 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file));
170 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file));
171
172 // A new bucket should have one cache miss
173 TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, &file));
174 // And then future calls to both should be cached
175 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file));
176 TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, &file));
177
178 // Trigger a flush, should then require one more call
179 fs.FlushCaches();
180 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file));
181 }
182
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintInDifferentLocation)183 TEST(GcsFileSystemTest,
184 NewRandomAccessFile_WithLocationConstraintInDifferentLocation) {
185 std::vector<HttpRequest*> requests({new FakeHttpRequest(
186 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
187 "Auth Token: fake_token\n"
188 "Timeouts: 5 1 10\n",
189 R"(
190 {
191 "location":"BARFOO"
192 })")});
193
194 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
195 std::unique_ptr<HttpRequest::Factory>(
196 new FakeHttpRequestFactory(&requests)),
197 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
198 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
199 0 /* stat cache max age */, 0 /* stat cache max entries */,
200 0 /* matching paths cache max age */,
201 0 /* matching paths cache max entries */, kTestRetryConfig,
202 kTestTimeoutConfig, *kAllowedLocationsAuto,
203 nullptr /* gcs additional header */);
204
205 std::unique_ptr<RandomAccessFile> file;
206 EXPECT_EQ(tensorflow::errors::FailedPrecondition(
207 "Bucket 'bucket' is in 'barfoo' location, allowed locations "
208 "are: (us-east1)."),
209 fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
210 }
211
TEST(GcsFileSystemTest,NewRandomAccessFile_NoBlockCache_DifferentN)212 TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) {
213 std::vector<HttpRequest*> requests(
214 {new FakeHttpRequest(
215 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
216 "Auth Token: fake_token\n"
217 "Range: 0-2\n"
218 "Timeouts: 5 1 20\n",
219 "012"),
220 new FakeHttpRequest(
221 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
222 "Auth Token: fake_token\n"
223 "Range: 3-12\n"
224 "Timeouts: 5 1 20\n",
225 "3456789")});
226 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
227 std::unique_ptr<HttpRequest::Factory>(
228 new FakeHttpRequestFactory(&requests)),
229 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
230 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
231 0 /* stat cache max age */, 0 /* stat cache max entries */,
232 0 /* matching paths cache max age */,
233 0 /* matching paths cache max entries */, kTestRetryConfig,
234 kTestTimeoutConfig, *kAllowedLocationsDefault,
235 nullptr /* gcs additional header */);
236
237 std::unique_ptr<RandomAccessFile> file;
238 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
239
240 char small_scratch[3];
241 StringPiece result;
242
243 // Read the first chunk.
244 TF_EXPECT_OK(file->Read(0, sizeof(small_scratch), &result, small_scratch));
245 EXPECT_EQ("012", result);
246
247 // Read the second chunk that is larger. Requires allocation of new buffer.
248 char large_scratch[10];
249
250 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
251 file->Read(sizeof(small_scratch), sizeof(large_scratch), &result,
252 large_scratch)
253 .code());
254 EXPECT_EQ("3456789", result);
255 }
256
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache)257 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) {
258 // Our underlying file in this test is a 15 byte file with contents
259 // "0123456789abcde".
260 std::vector<HttpRequest*> requests(
261 {new FakeHttpRequest(
262 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
263 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
264 "Auth Token: fake_token\n"
265 "Timeouts: 5 1 10\n",
266 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
267 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
268 new FakeHttpRequest(
269 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
270 "Auth Token: fake_token\n"
271 "Range: 0-8\n"
272 "Timeouts: 5 1 20\n",
273 "012345678"),
274 new FakeHttpRequest(
275 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
276 "Auth Token: fake_token\n"
277 "Range: 9-17\n"
278 "Timeouts: 5 1 20\n",
279 "9abcde"),
280 new FakeHttpRequest(
281 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
282 "Auth Token: fake_token\n"
283 "Range: 18-26\n"
284 "Timeouts: 5 1 20\n",
285 "")});
286 GcsFileSystem fs(
287 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
288 std::unique_ptr<HttpRequest::Factory>(
289 new FakeHttpRequestFactory(&requests)),
290 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
291 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
292 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
293 0 /* matching paths cache max entries */, kTestRetryConfig,
294 kTestTimeoutConfig, *kAllowedLocationsDefault,
295 nullptr /* gcs additional header */);
296
297 char scratch[100];
298 StringPiece result;
299 {
300 // We are instantiating this in an enclosed scope to make sure after the
301 // unique ptr goes out of scope, we can still access result.
302 std::unique_ptr<RandomAccessFile> file;
303 TF_EXPECT_OK(
304 fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
305
306 // Read the first chunk. The cache will be populated with the first block of
307 // 9 bytes.
308 scratch[5] = 'x';
309 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
310 EXPECT_EQ("0123", result);
311 EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes.
312
313 // The second chunk will be fully loaded from the cache, no requests are
314 // made.
315 TF_EXPECT_OK(file->Read(4, 4, &result, scratch));
316 EXPECT_EQ("4567", result);
317
318 // The chunk is only partially cached -- the request will be made to fetch
319 // the next block. 9 bytes will be requested, starting at offset 9.
320 TF_EXPECT_OK(file->Read(6, 5, &result, scratch));
321 EXPECT_EQ("6789a", result);
322
323 // The range can only be partially satisfied, as the second block contains
324 // only 6 bytes for a total of 9 + 6 = 15 bytes in the file.
325 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
326 file->Read(6, 10, &result, scratch).code());
327 EXPECT_EQ("6789abcde", result);
328
329 // The range cannot be satisfied, and the requested offset is past the end
330 // of the cache. A new request will be made to read 9 bytes starting at
331 // offset 18. This request will return an empty response, and there will not
332 // be another request.
333 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
334 file->Read(20, 10, &result, scratch).code());
335 EXPECT_TRUE(result.empty());
336
337 // The beginning of the file should still be in the LRU cache. There should
338 // not be another request. The buffer size is still 15.
339 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
340 }
341
342 EXPECT_EQ("0123", result);
343 }
344
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_Flush)345 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) {
346 // Our underlying file in this test is a 15 byte file with contents
347 // "0123456789abcde".
348 std::vector<HttpRequest*> requests(
349 {new FakeHttpRequest(
350 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
351 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
352 "Auth Token: fake_token\n"
353 "Timeouts: 5 1 10\n",
354 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
355 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
356 new FakeHttpRequest(
357 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
358 "Auth Token: fake_token\n"
359 "Range: 0-8\n"
360 "Timeouts: 5 1 20\n",
361 "012345678"),
362 new FakeHttpRequest(
363 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
364 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
365 "Auth Token: fake_token\n"
366 "Timeouts: 5 1 10\n",
367 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
368 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
369 new FakeHttpRequest(
370 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
371 "Auth Token: fake_token\n"
372 "Range: 0-8\n"
373 "Timeouts: 5 1 20\n",
374 "012345678")});
375 GcsFileSystem fs(
376 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
377 std::unique_ptr<HttpRequest::Factory>(
378 new FakeHttpRequestFactory(&requests)),
379 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
380 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
381 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
382 0 /* matching paths cache max entries */, kTestRetryConfig,
383 kTestTimeoutConfig, *kAllowedLocationsDefault,
384 nullptr /* gcs additional header */);
385
386 char scratch[100];
387 StringPiece result;
388 std::unique_ptr<RandomAccessFile> file;
389 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
390 // Read the first chunk. The cache will be populated with the first block of
391 // 9 bytes.
392 scratch[5] = 'x';
393 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
394 EXPECT_EQ("0123", result);
395 EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes.
396 // Flush caches and read the second chunk. This will be a cache miss, and
397 // the same block will be fetched again.
398 fs.FlushCaches();
399 TF_EXPECT_OK(file->Read(4, 4, &result, scratch));
400 EXPECT_EQ("4567", result);
401 }
402
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_MaxStaleness)403 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) {
404 // Our underlying file in this test is a 16 byte file with contents
405 // "0123456789abcdef".
406 std::vector<HttpRequest*> requests(
407 {new FakeHttpRequest(
408 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
409 "object?fields=size%2Cgeneration%2Cupdated\n"
410 "Auth Token: fake_token\n"
411 "Timeouts: 5 1 10\n",
412 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
413 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
414 new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
415 "Auth Token: fake_token\n"
416 "Range: 0-7\n"
417 "Timeouts: 5 1 20\n",
418 "01234567"),
419 new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
420 "Auth Token: fake_token\n"
421 "Range: 8-15\n"
422 "Timeouts: 5 1 20\n",
423 "89abcdef")});
424 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
425 std::unique_ptr<HttpRequest::Factory>(
426 new FakeHttpRequestFactory(&requests)),
427 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
428 8 /* block size */, 16 /* max bytes */,
429 3600 /* max staleness */, 3600 /* stat cache max age */,
430 0 /* stat cache max entries */,
431 0 /* matching paths cache max age */,
432 0 /* matching paths cache max entries */, kTestRetryConfig,
433 kTestTimeoutConfig, *kAllowedLocationsDefault,
434 nullptr /* gcs additional header */);
435 char scratch[100];
436 StringPiece result;
437 // There should only be two HTTP requests issued to GCS even though we iterate
438 // this loop 10 times. This shows that the underlying FileBlockCache persists
439 // across file close/open boundaries.
440 for (int i = 0; i < 10; i++) {
441 // Create two files. Since these files have the same name name and the max
442 // staleness of the filesystem is > 0, they will share the same blocks.
443 std::unique_ptr<RandomAccessFile> file1;
444 std::unique_ptr<RandomAccessFile> file2;
445 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &file1));
446 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &file2));
447 // Reading the first block from file1 should load it once.
448 TF_EXPECT_OK(file1->Read(0, 8, &result, scratch));
449 EXPECT_EQ("01234567", result);
450 // Reading the first block from file2 should not trigger a request to load
451 // the first block again, because the FileBlockCache shared by file1 and
452 // file2 already has the first block.
453 TF_EXPECT_OK(file2->Read(0, 8, &result, scratch));
454 EXPECT_EQ("01234567", result);
455 // Reading the second block from file2 should load it once.
456 TF_EXPECT_OK(file2->Read(8, 8, &result, scratch));
457 EXPECT_EQ("89abcdef", result);
458 // Reading the second block from file1 should not trigger a request to load
459 // the second block again, because the FileBlockCache shared by file1 and
460 // file2 already has the second block.
461 TF_EXPECT_OK(file1->Read(8, 8, &result, scratch));
462 EXPECT_EQ("89abcdef", result);
463 }
464 }
465
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_FileSignatureChanges)466 TEST(GcsFileSystemTest,
467 NewRandomAccessFile_WithBlockCache_FileSignatureChanges) {
468 std::vector<HttpRequest*> requests(
469 {new FakeHttpRequest(
470 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
471 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
472 "Auth Token: fake_token\n"
473 "Timeouts: 5 1 10\n",
474 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
475 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
476 new FakeHttpRequest(
477 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
478 "Auth Token: fake_token\n"
479 "Range: 0-8\n"
480 "Timeouts: 5 1 20\n",
481 "01234"),
482 new FakeHttpRequest(
483 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
484 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
485 "Auth Token: fake_token\n"
486 "Timeouts: 5 1 10\n",
487 strings::StrCat("{\"size\": \"5\",\"generation\": \"2\","
488 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
489 new FakeHttpRequest(
490 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
491 "Auth Token: fake_token\n"
492 "Range: 0-8\n"
493 "Timeouts: 5 1 20\n",
494 "43210")});
495 GcsFileSystem fs(
496 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
497 std::unique_ptr<HttpRequest::Factory>(
498 new FakeHttpRequestFactory(&requests)),
499 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
500 18 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
501 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
502 0 /* matching paths cache max entries */, kTestRetryConfig,
503 kTestTimeoutConfig, *kAllowedLocationsDefault,
504 nullptr /* gcs additional header */);
505
506 std::unique_ptr<RandomAccessFile> file;
507 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
508
509 char scratch[5];
510 StringPiece result;
511
512 // First read.
513 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
514 EXPECT_EQ("01234", result);
515
516 // Second read. File signatures are different.
517 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
518 EXPECT_EQ("43210", result);
519 }
520
TEST(GcsFileSystemTest,NewRandomAccessFile_NoObjectName)521 TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) {
522 std::vector<HttpRequest*> requests;
523 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
524 std::unique_ptr<HttpRequest::Factory>(
525 new FakeHttpRequestFactory(&requests)),
526 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
527 0 /* read ahead bytes */, 0 /* max bytes */,
528 0 /* max staleness */, 0 /* stat cache max age */,
529 0 /* stat cache max entries */,
530 0 /* matching paths cache max age */,
531 0 /* matching paths cache max entries */, kTestRetryConfig,
532 kTestTimeoutConfig, *kAllowedLocationsDefault,
533 nullptr /* gcs additional header */);
534
535 std::unique_ptr<RandomAccessFile> file;
536 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
537 fs.NewRandomAccessFile("gs://bucket/", &file).code());
538 }
539
TEST(GcsFileSystemTest,NewRandomAccessFile_InconsistentRead)540 TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) {
541 std::vector<HttpRequest*> requests(
542 {new FakeHttpRequest(
543 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
544 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
545 "Auth Token: fake_token\n"
546 "Timeouts: 5 1 10\n",
547 strings::StrCat("{\"size\": \"6\",\"generation\": \"1\","
548 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
549 new FakeHttpRequest(
550 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
551 "Auth Token: fake_token\n"
552 "Range: 0-5\n"
553 "Timeouts: 5 1 20\n",
554 "012")});
555
556 // Set stat_cache_max_age to 1000s so that StatCache could work.
557 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
558 std::unique_ptr<HttpRequest::Factory>(
559 new FakeHttpRequestFactory(&requests)),
560 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
561 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
562 1e3 /* stat cache max age */, 0 /* stat cache max entries */,
563 0 /* matching paths cache max age */,
564 0 /* matching paths cache max entries */, kTestRetryConfig,
565 kTestTimeoutConfig, *kAllowedLocationsDefault,
566 nullptr /* gcs additional header */);
567
568 // Stat the file first so that the file stats are cached.
569 FileStatistics stat;
570 TF_ASSERT_OK(fs.Stat("gs://bucket/random_access.txt", &stat));
571
572 std::unique_ptr<RandomAccessFile> file;
573 TF_ASSERT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
574
575 char scratch[6];
576 StringPiece result;
577
578 EXPECT_EQ(errors::Code::INTERNAL,
579 file->Read(0, sizeof(scratch), &result, scratch).code());
580 }
581
TEST(GcsFileSystemTest,NewWritableFile)582 TEST(GcsFileSystemTest, NewWritableFile) {
583 std::vector<HttpRequest*> requests(
584 {new FakeHttpRequest(
585 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
586 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
587 "Auth Token: fake_token\n"
588 "Timeouts: 5 1 10\n",
589 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
590 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
591 new FakeHttpRequest(
592 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
593 "Auth Token: fake_token\n"
594 "Range: 0-7\n"
595 "Timeouts: 5 1 20\n",
596 "01234567"),
597 new FakeHttpRequest(
598 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
599 "uploadType=resumable&name=path%2Fwriteable\n"
600 "Auth Token: fake_token\n"
601 "Header X-Upload-Content-Length: 17\n"
602 "Post: yes\n"
603 "Timeouts: 5 1 10\n",
604 "", {{"Location", "https://custom/upload/location"}}),
605 new FakeHttpRequest("Uri: https://custom/upload/location\n"
606 "Auth Token: fake_token\n"
607 "Header Content-Range: bytes 0-16/17\n"
608 "Timeouts: 5 1 30\n"
609 "Put body: content1,content2\n",
610 ""),
611 new FakeHttpRequest(
612 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
613 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
614 "Auth Token: fake_token\n"
615 "Timeouts: 5 1 10\n",
616 strings::StrCat("{\"size\": \"33\",\"generation\": \"2\","
617 "\"updated\": \"2016-04-29T23:15:34.896Z\"}")),
618 new FakeHttpRequest(
619 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
620 "Auth Token: fake_token\n"
621 "Range: 0-7\n"
622 "Timeouts: 5 1 20\n",
623 "01234567")});
624 GcsFileSystem fs(
625 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
626 std::unique_ptr<HttpRequest::Factory>(
627 new FakeHttpRequestFactory(&requests)),
628 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
629 8 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
630 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
631 0 /* matching paths cache max entries */, kTestRetryConfig,
632 kTestTimeoutConfig, *kAllowedLocationsDefault,
633 nullptr /* gcs additional header */);
634
635 // Read from the file first, to fill the block cache.
636 std::unique_ptr<RandomAccessFile> rfile;
637 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/writeable", &rfile));
638 char scratch[100];
639 StringPiece result;
640 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
641 EXPECT_EQ("0123", result);
642 // Open the writable file.
643 std::unique_ptr<WritableFile> wfile;
644 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable", &wfile));
645 TF_EXPECT_OK(wfile->Append("content1,"));
646 int64 pos;
647 TF_EXPECT_OK(wfile->Tell(&pos));
648 EXPECT_EQ(9, pos);
649 TF_EXPECT_OK(wfile->Append("content2"));
650 TF_EXPECT_OK(wfile->Flush());
651 // Re-reading the file should trigger another HTTP request to GCS.
652 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
653 EXPECT_EQ("0123", result);
654 // The calls to flush, sync, and close below should not cause uploads because
655 // the file is not dirty.
656 TF_EXPECT_OK(wfile->Flush());
657 TF_EXPECT_OK(wfile->Sync());
658 TF_EXPECT_OK(wfile->Close());
659 }
660
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadSucceeds)661 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
662 std::vector<HttpRequest*> requests(
663 {new FakeHttpRequest(
664 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
665 "uploadType=resumable&name=path%2Fwriteable.txt\n"
666 "Auth Token: fake_token\n"
667 "Header X-Upload-Content-Length: 17\n"
668 "Post: yes\n"
669 "Timeouts: 5 1 10\n",
670 "", {{"Location", "https://custom/upload/location"}}),
671 new FakeHttpRequest("Uri: https://custom/upload/location\n"
672 "Auth Token: fake_token\n"
673 "Header Content-Range: bytes 0-16/17\n"
674 "Timeouts: 5 1 30\n"
675 "Put body: content1,content2\n",
676 "", errors::Unavailable("503"), 503),
677 new FakeHttpRequest("Uri: https://custom/upload/location\n"
678 "Auth Token: fake_token\n"
679 "Timeouts: 5 1 10\n"
680 "Header Content-Range: bytes */17\n"
681 "Put: yes\n",
682 "", errors::Unavailable("308"), nullptr,
683 {{"Range", "0-10"}}, 308),
684 new FakeHttpRequest("Uri: https://custom/upload/location\n"
685 "Auth Token: fake_token\n"
686 "Header Content-Range: bytes 11-16/17\n"
687 "Timeouts: 5 1 30\n"
688 "Put body: ntent2\n",
689 "", errors::Unavailable("503"), 503),
690 new FakeHttpRequest("Uri: https://custom/upload/location\n"
691 "Auth Token: fake_token\n"
692 "Timeouts: 5 1 10\n"
693 "Header Content-Range: bytes */17\n"
694 "Put: yes\n",
695 "", errors::Unavailable("308"), nullptr,
696 {{"Range", "bytes=0-12"}}, 308),
697 new FakeHttpRequest("Uri: https://custom/upload/location\n"
698 "Auth Token: fake_token\n"
699 "Header Content-Range: bytes 13-16/17\n"
700 "Timeouts: 5 1 30\n"
701 "Put body: ent2\n",
702 "", errors::Unavailable("308"), 308),
703 new FakeHttpRequest("Uri: https://custom/upload/location\n"
704 "Auth Token: fake_token\n"
705 "Timeouts: 5 1 10\n"
706 "Header Content-Range: bytes */17\n"
707 "Put: yes\n",
708 "", errors::Unavailable("308"), nullptr,
709 {{"Range", "bytes=0-14"}}, 308),
710 new FakeHttpRequest("Uri: https://custom/upload/location\n"
711 "Auth Token: fake_token\n"
712 "Header Content-Range: bytes 15-16/17\n"
713 "Timeouts: 5 1 30\n"
714 "Put body: t2\n",
715 "")});
716 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
717 std::unique_ptr<HttpRequest::Factory>(
718 new FakeHttpRequestFactory(&requests)),
719 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
720 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
721 0 /* stat cache max age */, 0 /* stat cache max entries */,
722 0 /* matching paths cache max age */,
723 0 /* matching paths cache max entries */, kTestRetryConfig,
724 kTestTimeoutConfig, *kAllowedLocationsDefault,
725 nullptr /* gcs additional header */);
726
727 std::unique_ptr<WritableFile> file;
728 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
729
730 TF_EXPECT_OK(file->Append("content1,"));
731 TF_EXPECT_OK(file->Append("content2"));
732 TF_EXPECT_OK(file->Close());
733 }
734
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadSucceedsOnGetStatus)735 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) {
736 // This test also verifies that a file's blocks are purged from the cache when
737 // the file is written, even when the write takes the "succeeds on get status"
738 // path.
739 std::vector<HttpRequest*> requests(
740 {new FakeHttpRequest(
741 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
742 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
743 "Auth Token: fake_token\n"
744 "Timeouts: 5 1 10\n",
745 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
746 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
747 new FakeHttpRequest(
748 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
749 "Auth Token: fake_token\n"
750 "Range: 0-7\n"
751 "Timeouts: 5 1 20\n",
752 "01234567"),
753 new FakeHttpRequest(
754 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
755 "uploadType=resumable&name=path%2Fwriteable\n"
756 "Auth Token: fake_token\n"
757 "Header X-Upload-Content-Length: 17\n"
758 "Post: yes\n"
759 "Timeouts: 5 1 10\n",
760 "", {{"Location", "https://custom/upload/location"}}),
761 new FakeHttpRequest("Uri: https://custom/upload/location\n"
762 "Auth Token: fake_token\n"
763 "Header Content-Range: bytes 0-16/17\n"
764 "Timeouts: 5 1 30\n"
765 "Put body: content1,content2\n",
766 "", errors::Unavailable("503"), 503),
767 new FakeHttpRequest("Uri: https://custom/upload/location\n"
768 "Auth Token: fake_token\n"
769 "Timeouts: 5 1 10\n"
770 "Header Content-Range: bytes */17\n"
771 "Put: yes\n",
772 "", Status::OK(), nullptr, {}, 201),
773 new FakeHttpRequest(
774 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
775 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
776 "Auth Token: fake_token\n"
777 "Timeouts: 5 1 10\n",
778 strings::StrCat("{\"size\": \"33\",\"generation\": \"2\","
779 "\"updated\": \"2016-04-29T23:19:24.896Z\"}")),
780 new FakeHttpRequest(
781 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
782 "Auth Token: fake_token\n"
783 "Range: 0-7\n"
784 "Timeouts: 5 1 20\n",
785 "01234567")});
786 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
787 std::unique_ptr<HttpRequest::Factory>(
788 new FakeHttpRequestFactory(&requests)),
789 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
790 8 /* block size */, 8 /* max bytes */,
791 3600 /* max staleness */, 3600 /* stat cache max age */,
792 0 /* stat cache max entries */,
793 0 /* matching paths cache max age */,
794 0 /* matching paths cache max entries */, kTestRetryConfig,
795 kTestTimeoutConfig, *kAllowedLocationsDefault,
796 nullptr /* gcs additional header */);
797 // Pull the file's first block into the cache. This will trigger the first
798 // HTTP request to GCS.
799 std::unique_ptr<RandomAccessFile> rfile;
800 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/writeable", &rfile));
801 char scratch[100];
802 StringPiece result;
803 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
804 EXPECT_EQ("0123", result);
805 // Now write to the same file. Once the write succeeds, the cached block will
806 // be flushed.
807 std::unique_ptr<WritableFile> wfile;
808 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable", &wfile));
809 TF_EXPECT_OK(wfile->Append("content1,"));
810 TF_EXPECT_OK(wfile->Append("content2"));
811 // Appending doesn't invalidate the read cache - only flushing does. This read
812 // will not trigger an HTTP request to GCS.
813 TF_EXPECT_OK(rfile->Read(4, 4, &result, scratch));
814 EXPECT_EQ("4567", result);
815 // Closing the file triggers HTTP requests to GCS and invalidates the read
816 // cache for the file.
817 TF_EXPECT_OK(wfile->Close());
818 // Reading the first block of the file goes to GCS again.
819 TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch));
820 EXPECT_EQ("01234567", result);
821 }
822
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadAllAttemptsFail)823 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
824 std::vector<HttpRequest*> requests(
825 {new FakeHttpRequest(
826 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
827 "uploadType=resumable&name=path%2Fwriteable.txt\n"
828 "Auth Token: fake_token\n"
829 "Header X-Upload-Content-Length: 17\n"
830 "Post: yes\n"
831 "Timeouts: 5 1 10\n",
832 "", {{"Location", "https://custom/upload/location"}}),
833 new FakeHttpRequest("Uri: https://custom/upload/location\n"
834 "Auth Token: fake_token\n"
835 "Header Content-Range: bytes 0-16/17\n"
836 "Timeouts: 5 1 30\n"
837 "Put body: content1,content2\n",
838 "", errors::Unavailable("503"), 503)});
839 for (int i = 0; i < 10; i++) {
840 requests.emplace_back(
841 new FakeHttpRequest("Uri: https://custom/upload/location\n"
842 "Auth Token: fake_token\n"
843 "Timeouts: 5 1 10\n"
844 "Header Content-Range: bytes */17\n"
845 "Put: yes\n",
846 "", errors::Unavailable("important HTTP error 308"),
847 nullptr, {{"Range", "0-10"}}, 308));
848 requests.emplace_back(new FakeHttpRequest(
849 "Uri: https://custom/upload/location\n"
850 "Auth Token: fake_token\n"
851 "Header Content-Range: bytes 11-16/17\n"
852 "Timeouts: 5 1 30\n"
853 "Put body: ntent2\n",
854 "", errors::Unavailable("important HTTP error 503"), 503));
855 }
856 // These calls will be made in the Close() attempt from the destructor.
857 // Letting the destructor succeed.
858 requests.emplace_back(new FakeHttpRequest(
859 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
860 "uploadType=resumable&name=path%2Fwriteable.txt\n"
861 "Auth Token: fake_token\n"
862 "Header X-Upload-Content-Length: 17\n"
863 "Post: yes\n"
864 "Timeouts: 5 1 10\n",
865 "", {{"Location", "https://custom/upload/location"}}));
866 requests.emplace_back(
867 new FakeHttpRequest("Uri: https://custom/upload/location\n"
868 "Auth Token: fake_token\n"
869 "Header Content-Range: bytes 0-16/17\n"
870 "Timeouts: 5 1 30\n"
871 "Put body: content1,content2\n",
872 ""));
873 GcsFileSystem fs(
874 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
875 std::unique_ptr<HttpRequest::Factory>(
876 new FakeHttpRequestFactory(&requests)),
877 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
878 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
879 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
880 0 /* matching paths cache max entries */,
881 RetryConfig(2 /* .init_delay_time_us */), kTestTimeoutConfig,
882 *kAllowedLocationsDefault, nullptr /* gcs additional header */);
883
884 std::unique_ptr<WritableFile> file;
885 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
886
887 TF_EXPECT_OK(file->Append("content1,"));
888 TF_EXPECT_OK(file->Append("content2"));
889 const auto& status = file->Close();
890 EXPECT_EQ(errors::Code::ABORTED, status.code());
891 EXPECT_TRUE(
892 str_util::StrContains(status.error_message(),
893 "All 10 retry attempts failed. The last failure: "
894 "Unavailable: important HTTP error 503"))
895 << status;
896 }
897
TEST(GcsFileSystemTest,NewWritableFile_UploadReturns410)898 TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
899 std::vector<HttpRequest*> requests(
900 {new FakeHttpRequest(
901 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
902 "uploadType=resumable&name=path%2Fwriteable.txt\n"
903 "Auth Token: fake_token\n"
904 "Header X-Upload-Content-Length: 17\n"
905 "Post: yes\n"
906 "Timeouts: 5 1 10\n",
907 "", {{"Location", "https://custom/upload/location"}}),
908 new FakeHttpRequest("Uri: https://custom/upload/location\n"
909 "Auth Token: fake_token\n"
910 "Header Content-Range: bytes 0-16/17\n"
911 "Timeouts: 5 1 30\n"
912 "Put body: content1,content2\n",
913 "", errors::NotFound("important HTTP error 410"),
914 410),
915 // These calls will be made in the Close() attempt from the destructor.
916 // Letting the destructor succeed.
917 new FakeHttpRequest(
918 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
919 "uploadType=resumable&name=path%2Fwriteable.txt\n"
920 "Auth Token: fake_token\n"
921 "Header X-Upload-Content-Length: 17\n"
922 "Post: yes\n"
923 "Timeouts: 5 1 10\n",
924 "", {{"Location", "https://custom/upload/location"}}),
925 new FakeHttpRequest("Uri: https://custom/upload/location\n"
926 "Auth Token: fake_token\n"
927 "Header Content-Range: bytes 0-16/17\n"
928 "Timeouts: 5 1 30\n"
929 "Put body: content1,content2\n",
930 "")});
931 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
932 std::unique_ptr<HttpRequest::Factory>(
933 new FakeHttpRequestFactory(&requests)),
934 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
935 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
936 0 /* stat cache max age */, 0 /* stat cache max entries */,
937 0 /* matching paths cache max age */,
938 0 /* matching paths cache max entries */, kTestRetryConfig,
939 kTestTimeoutConfig, *kAllowedLocationsDefault,
940 nullptr /* gcs additional header */);
941
942 std::unique_ptr<WritableFile> file;
943 TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
944
945 TF_EXPECT_OK(file->Append("content1,"));
946 TF_EXPECT_OK(file->Append("content2"));
947 const auto& status = file->Close();
948 EXPECT_EQ(errors::Code::UNAVAILABLE, status.code());
949 EXPECT_TRUE(
950 str_util::StrContains(status.error_message(),
951 "Upload to gs://bucket/path/writeable.txt failed, "
952 "caused by: Not found: important HTTP error 410"))
953 << status;
954 EXPECT_TRUE(str_util::StrContains(
955 status.error_message(), "when uploading gs://bucket/path/writeable.txt"))
956 << status;
957 }
958
TEST(GcsFileSystemTest,NewWritableFile_NoObjectName)959 TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) {
960 std::vector<HttpRequest*> requests;
961 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
962 std::unique_ptr<HttpRequest::Factory>(
963 new FakeHttpRequestFactory(&requests)),
964 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
965 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
966 0 /* stat cache max age */, 0 /* stat cache max entries */,
967 0 /* matching paths cache max age */,
968 0 /* matching paths cache max entries */, kTestRetryConfig,
969 kTestTimeoutConfig, *kAllowedLocationsDefault,
970 nullptr /* gcs additional header */);
971
972 std::unique_ptr<WritableFile> file;
973 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
974 fs.NewWritableFile("gs://bucket/", &file).code());
975 }
976
TEST(GcsFileSystemTest,NewAppendableFile)977 TEST(GcsFileSystemTest, NewAppendableFile) {
978 std::vector<HttpRequest*> requests(
979 {new FakeHttpRequest(
980 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
981 "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
982 "Auth Token: fake_token\n"
983 "Timeouts: 5 1 10\n",
984 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
985 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
986 new FakeHttpRequest(
987 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
988 "Auth Token: fake_token\n"
989 "Range: 0-31\n"
990 "Timeouts: 5 1 20\n",
991 "content1,"),
992 new FakeHttpRequest(
993 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
994 "uploadType=resumable&name=path%2Fappendable\n"
995 "Auth Token: fake_token\n"
996 "Header X-Upload-Content-Length: 17\n"
997 "Post: yes\n"
998 "Timeouts: 5 1 10\n",
999 "", {{"Location", "https://custom/upload/location"}}),
1000 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1001 "Auth Token: fake_token\n"
1002 "Header Content-Range: bytes 0-16/17\n"
1003 "Timeouts: 5 1 30\n"
1004 "Put body: content1,content2\n",
1005 ""),
1006 new FakeHttpRequest(
1007 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1008 "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
1009 "Auth Token: fake_token\n"
1010 "Timeouts: 5 1 10\n",
1011 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
1012 "\"updated\": \"2016-04-29T23:25:24.896Z\"}")),
1013 new FakeHttpRequest(
1014 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
1015 "Auth Token: fake_token\n"
1016 "Range: 0-31\n"
1017 "Timeouts: 5 1 20\n",
1018 "01234567")});
1019 GcsFileSystem fs(
1020 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1021 std::unique_ptr<HttpRequest::Factory>(
1022 new FakeHttpRequestFactory(&requests)),
1023 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 32 /* block size */,
1024 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1025 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1026 0 /* matching paths cache max entries */, kTestRetryConfig,
1027 kTestTimeoutConfig, *kAllowedLocationsDefault,
1028 nullptr /* gcs additional header */);
1029
1030 // Create an appendable file. This should read the file from GCS, and pull its
1031 // contents into the block cache.
1032 std::unique_ptr<WritableFile> wfile;
1033 TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/path/appendable", &wfile));
1034 TF_EXPECT_OK(wfile->Append("content2"));
1035 // Verify that the file contents are in the block cache. This read should not
1036 // trigger an HTTP request to GCS.
1037 std::unique_ptr<RandomAccessFile> rfile;
1038 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/appendable", &rfile));
1039 char scratch[100];
1040 StringPiece result;
1041 TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch));
1042 EXPECT_EQ("content1", result);
1043 // Closing the appendable file will flush its contents to GCS, triggering HTTP
1044 // requests.
1045 TF_EXPECT_OK(wfile->Close());
1046 // Redo the read. The block should be reloaded from GCS, causing one more HTTP
1047 // request to load it.
1048 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
1049 EXPECT_EQ("0123", result);
1050 }
1051
TEST(GcsFileSystemTest,NewAppendableFile_NoObjectName)1052 TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) {
1053 std::vector<HttpRequest*> requests;
1054 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1055 std::unique_ptr<HttpRequest::Factory>(
1056 new FakeHttpRequestFactory(&requests)),
1057 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1058 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1059 0 /* stat cache max age */, 0 /* stat cache max entries */,
1060 0 /* matching paths cache max age */,
1061 0 /* matching paths cache max entries */, kTestRetryConfig,
1062 kTestTimeoutConfig, *kAllowedLocationsDefault,
1063 nullptr /* gcs additional header */);
1064
1065 std::unique_ptr<WritableFile> file;
1066 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1067 fs.NewAppendableFile("gs://bucket/", &file).code());
1068 }
1069
TEST(GcsFileSystemTest,NewReadOnlyMemoryRegionFromFile)1070 TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
1071 const string content = "file content";
1072 std::vector<HttpRequest*> requests(
1073 {new FakeHttpRequest(
1074 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1075 "path%2Frandom_access.txt?fields=size%2Cgeneration%2Cupdated\n"
1076 "Auth Token: fake_token\n"
1077 "Timeouts: 5 1 10\n",
1078 strings::StrCat("{\"size\": \"", content.size(), "\"",
1079 ", \"generation\": \"1\"",
1080 ", \"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1081 new FakeHttpRequest(
1082 strings::StrCat("Uri: https://storage.googleapis.com/bucket/"
1083 "path%2Frandom_access.txt\n"
1084 "Auth Token: fake_token\n"
1085 "Range: 0-",
1086 content.size() - 1, "\n", "Timeouts: 5 1 20\n"),
1087 content)});
1088 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1089 std::unique_ptr<HttpRequest::Factory>(
1090 new FakeHttpRequestFactory(&requests)),
1091 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1092 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1093 0 /* stat cache max age */, 0 /* stat cache max entries */,
1094 0 /* matching paths cache max age */,
1095 0 /* matching paths cache max entries */, kTestRetryConfig,
1096 kTestTimeoutConfig, *kAllowedLocationsDefault,
1097 nullptr /* gcs additional header */);
1098
1099 std::unique_ptr<ReadOnlyMemoryRegion> region;
1100 TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile(
1101 "gs://bucket/path/random_access.txt", ®ion));
1102
1103 EXPECT_EQ(content, StringPiece(reinterpret_cast<const char*>(region->data()),
1104 region->length()));
1105 }
1106
TEST(GcsFileSystemTest,NewReadOnlyMemoryRegionFromFile_NoObjectName)1107 TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) {
1108 std::vector<HttpRequest*> requests;
1109 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1110 std::unique_ptr<HttpRequest::Factory>(
1111 new FakeHttpRequestFactory(&requests)),
1112 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1113 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1114 0 /* stat cache max age */, 0 /* stat cache max entries */,
1115 0 /* matching paths cache max age */,
1116 0 /* matching paths cache max entries */, kTestRetryConfig,
1117 kTestTimeoutConfig, *kAllowedLocationsDefault,
1118 nullptr /* gcs additional header */);
1119
1120 std::unique_ptr<ReadOnlyMemoryRegion> region;
1121 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1122 fs.NewReadOnlyMemoryRegionFromFile("gs://bucket/", ®ion).code());
1123 }
1124
TEST(GcsFileSystemTest,FileExists_YesAsObject)1125 TEST(GcsFileSystemTest, FileExists_YesAsObject) {
1126 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1127 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1128 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1129 "Auth Token: fake_token\n"
1130 "Timeouts: 5 1 10\n",
1131 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
1132 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
1133 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1134 std::unique_ptr<HttpRequest::Factory>(
1135 new FakeHttpRequestFactory(&requests)),
1136 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1137 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1138 0 /* stat cache max age */, 0 /* stat cache max entries */,
1139 0 /* matching paths cache max age */,
1140 0 /* matching paths cache max entries */, kTestRetryConfig,
1141 kTestTimeoutConfig, *kAllowedLocationsDefault,
1142 nullptr /* gcs additional header */);
1143
1144 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt"));
1145 }
1146
TEST(GcsFileSystemTest,FileExists_YesAsFolder)1147 TEST(GcsFileSystemTest, FileExists_YesAsFolder) {
1148 std::vector<HttpRequest*> requests(
1149 {new FakeHttpRequest(
1150 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1151 "path%2Fsubfolder?fields=size%2Cgeneration%2Cupdated\n"
1152 "Auth Token: fake_token\n"
1153 "Timeouts: 5 1 10\n",
1154 "", errors::NotFound("404"), 404),
1155 new FakeHttpRequest(
1156 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1157 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F"
1158 "&maxResults=1\n"
1159 "Auth Token: fake_token\n"
1160 "Timeouts: 5 1 10\n",
1161 "{\"items\": [ "
1162 " { \"name\": \"path/subfolder/\" }]}")});
1163 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1164 std::unique_ptr<HttpRequest::Factory>(
1165 new FakeHttpRequestFactory(&requests)),
1166 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1167 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1168 0 /* stat cache max age */, 0 /* stat cache max entries */,
1169 0 /* matching paths cache max age */,
1170 0 /* matching paths cache max entries */, kTestRetryConfig,
1171 kTestTimeoutConfig, *kAllowedLocationsDefault,
1172 nullptr /* gcs additional header */);
1173
1174 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder"));
1175 }
1176
TEST(GcsFileSystemTest,FileExists_YesAsBucket)1177 TEST(GcsFileSystemTest, FileExists_YesAsBucket) {
1178 std::vector<HttpRequest*> requests(
1179 {new FakeHttpRequest(
1180 "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n"
1181 "Auth Token: fake_token\n"
1182 "Timeouts: 5 1 10\n",
1183 "{\"size\": \"100\"}"),
1184 new FakeHttpRequest(
1185 "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n"
1186 "Auth Token: fake_token\n"
1187 "Timeouts: 5 1 10\n",
1188 "{\"size\": \"100\"}")});
1189 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1190 std::unique_ptr<HttpRequest::Factory>(
1191 new FakeHttpRequestFactory(&requests)),
1192 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1193 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1194 0 /* stat cache max age */, 0 /* stat cache max entries */,
1195 0 /* matching paths cache max age */,
1196 0 /* matching paths cache max entries */, kTestRetryConfig,
1197 kTestTimeoutConfig, *kAllowedLocationsDefault,
1198 nullptr /* gcs additional header */);
1199
1200 TF_EXPECT_OK(fs.FileExists("gs://bucket1"));
1201 TF_EXPECT_OK(fs.FileExists("gs://bucket1/"));
1202 }
1203
TEST(GcsFileSystemTest,FileExists_NotAsObjectOrFolder)1204 TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) {
1205 std::vector<HttpRequest*> requests(
1206 {new FakeHttpRequest(
1207 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1208 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1209 "Auth Token: fake_token\n"
1210 "Timeouts: 5 1 10\n",
1211 "", errors::NotFound("404"), 404),
1212 new FakeHttpRequest(
1213 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1214 "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile1.txt%2F"
1215 "&maxResults=1\n"
1216 "Auth Token: fake_token\n"
1217 "Timeouts: 5 1 10\n",
1218 "{\"items\": []}")});
1219 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1220 std::unique_ptr<HttpRequest::Factory>(
1221 new FakeHttpRequestFactory(&requests)),
1222 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1223 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1224 0 /* stat cache max age */, 0 /* stat cache max entries */,
1225 0 /* matching paths cache max age */,
1226 0 /* matching paths cache max entries */, kTestRetryConfig,
1227 kTestTimeoutConfig, *kAllowedLocationsDefault,
1228 nullptr /* gcs additional header */);
1229
1230 EXPECT_EQ(errors::Code::NOT_FOUND,
1231 fs.FileExists("gs://bucket/path/file1.txt").code());
1232 }
1233
TEST(GcsFileSystemTest,FileExists_NotAsBucket)1234 TEST(GcsFileSystemTest, FileExists_NotAsBucket) {
1235 std::vector<HttpRequest*> requests(
1236 {new FakeHttpRequest(
1237 "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n"
1238 "Auth Token: fake_token\n"
1239 "Timeouts: 5 1 10\n",
1240 "", errors::NotFound("404"), 404),
1241 new FakeHttpRequest(
1242 "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n"
1243 "Auth Token: fake_token\n"
1244 "Timeouts: 5 1 10\n",
1245 "", errors::NotFound("404"), 404)});
1246 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1247 std::unique_ptr<HttpRequest::Factory>(
1248 new FakeHttpRequestFactory(&requests)),
1249 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1250 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1251 0 /* stat cache max age */, 0 /* stat cache max entries */,
1252 0 /* matching paths cache max age */,
1253 0 /* matching paths cache max entries */, kTestRetryConfig,
1254 kTestTimeoutConfig, *kAllowedLocationsDefault,
1255 nullptr /* gcs additional header */);
1256 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1257 fs.FileExists("gs://bucket2/").code());
1258 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1259 fs.FileExists("gs://bucket2").code());
1260 }
1261
TEST(GcsFileSystemTest,FileExists_StatCache)1262 TEST(GcsFileSystemTest, FileExists_StatCache) {
1263 std::vector<HttpRequest*> requests(
1264 {new FakeHttpRequest(
1265 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1266 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1267 "Auth Token: fake_token\n"
1268 "Timeouts: 5 1 10\n",
1269 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
1270 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1271 new FakeHttpRequest(
1272 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1273 "path%2Fsubfolder%2F?fields=size%2Cgeneration%2Cupdated\n"
1274 "Auth Token: fake_token\n"
1275 "Timeouts: 5 1 10\n",
1276 "", errors::NotFound("404"), 404),
1277 new FakeHttpRequest(
1278 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1279 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F"
1280 "&maxResults=1\n"
1281 "Auth Token: fake_token\n"
1282 "Timeouts: 5 1 10\n",
1283 "{\"items\": [ "
1284 " { \"name\": \"path/subfolder/\" }]}")});
1285 GcsFileSystem fs(
1286 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1287 std::unique_ptr<HttpRequest::Factory>(
1288 new FakeHttpRequestFactory(&requests)),
1289 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1290 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1291 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1292 0 /* matching paths cache max entries */, kTestRetryConfig,
1293 kTestTimeoutConfig, *kAllowedLocationsDefault,
1294 nullptr /* gcs additional header */);
1295
1296 // The stat cache will ensure that repeated lookups don't trigger additional
1297 // HTTP requests.
1298 for (int i = 0; i < 10; i++) {
1299 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt"));
1300 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder/"));
1301 }
1302 }
1303
TEST(GcsFileSystemTest,FileExists_DirectoryMark)1304 TEST(GcsFileSystemTest, FileExists_DirectoryMark) {
1305 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1306 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1307 "dir%2F?fields=size%2Cgeneration%2Cupdated\n"
1308 "Auth Token: fake_token\n"
1309 "Timeouts: 5 1 10\n",
1310 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
1311 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
1312 GcsFileSystem fs(
1313 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1314 std::unique_ptr<HttpRequest::Factory>(
1315 new FakeHttpRequestFactory(&requests)),
1316 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1317 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1318 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1319 0 /* matching paths cache max entries */, kTestRetryConfig,
1320 kTestTimeoutConfig, *kAllowedLocationsDefault,
1321 nullptr /* gcs additional header */);
1322
1323 TF_EXPECT_OK(fs.FileExists("gs://bucket/dir/"));
1324 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/dir/"));
1325 }
1326
TEST(GcsFileSystemTest,GetChildren_NoItems)1327 TEST(GcsFileSystemTest, GetChildren_NoItems) {
1328 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1329 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1330 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1331 "path%2F\n"
1332 "Auth Token: fake_token\n"
1333 "Timeouts: 5 1 10\n",
1334 "{\"prefixes\": [\"path/subpath/\"]}")});
1335 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1336 std::unique_ptr<HttpRequest::Factory>(
1337 new FakeHttpRequestFactory(&requests)),
1338 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1339 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1340 0 /* stat cache max age */, 0 /* stat cache max entries */,
1341 0 /* matching paths cache max age */,
1342 0 /* matching paths cache max entries */, kTestRetryConfig,
1343 kTestTimeoutConfig, *kAllowedLocationsDefault,
1344 nullptr /* gcs additional header */);
1345
1346 std::vector<string> children;
1347 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
1348
1349 EXPECT_EQ(std::vector<string>({"subpath/"}), children);
1350 }
1351
TEST(GcsFileSystemTest,GetChildren_ThreeFiles)1352 TEST(GcsFileSystemTest, GetChildren_ThreeFiles) {
1353 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1354 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1355 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1356 "path%2F\n"
1357 "Auth Token: fake_token\n"
1358 "Timeouts: 5 1 10\n",
1359 "{\"items\": [ "
1360 " { \"name\": \"path/file1.txt\" },"
1361 " { \"name\": \"path/file3.txt\" }],"
1362 "\"prefixes\": [\"path/subpath/\"]}")});
1363 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1364 std::unique_ptr<HttpRequest::Factory>(
1365 new FakeHttpRequestFactory(&requests)),
1366 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1367 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1368 0 /* stat cache max age */, 0 /* stat cache max entries */,
1369 0 /* matching paths cache max age */,
1370 0 /* matching paths cache max entries */, kTestRetryConfig,
1371 kTestTimeoutConfig, *kAllowedLocationsDefault,
1372 nullptr /* gcs additional header */);
1373
1374 std::vector<string> children;
1375 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
1376
1377 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/"}),
1378 children);
1379 }
1380
TEST(GcsFileSystemTest,GetChildren_SelfDirectoryMarker)1381 TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) {
1382 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1383 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1384 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1385 "path%2F\n"
1386 "Auth Token: fake_token\n"
1387 "Timeouts: 5 1 10\n",
1388 "{\"items\": [ "
1389 " { \"name\": \"path/\" },"
1390 " { \"name\": \"path/file3.txt\" }],"
1391 "\"prefixes\": [\"path/subpath/\"]}")});
1392 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1393 std::unique_ptr<HttpRequest::Factory>(
1394 new FakeHttpRequestFactory(&requests)),
1395 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1396 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1397 0 /* stat cache max age */, 0 /* stat cache max entries */,
1398 0 /* matching paths cache max age */,
1399 0 /* matching paths cache max entries */, kTestRetryConfig,
1400 kTestTimeoutConfig, *kAllowedLocationsDefault,
1401 nullptr /* gcs additional header */);
1402
1403 std::vector<string> children;
1404 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
1405
1406 EXPECT_EQ(std::vector<string>({"file3.txt", "subpath/"}), children);
1407 }
1408
TEST(GcsFileSystemTest,GetChildren_ThreeFiles_NoSlash)1409 TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) {
1410 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1411 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1412 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1413 "path%2F\n"
1414 "Auth Token: fake_token\n"
1415 "Timeouts: 5 1 10\n",
1416 "{\"items\": [ "
1417 " { \"name\": \"path/file1.txt\" },"
1418 " { \"name\": \"path/file3.txt\" }],"
1419 "\"prefixes\": [\"path/subpath/\"]}")});
1420 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1421 std::unique_ptr<HttpRequest::Factory>(
1422 new FakeHttpRequestFactory(&requests)),
1423 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1424 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1425 0 /* stat cache max age */, 0 /* stat cache max entries */,
1426 0 /* matching paths cache max age */,
1427 0 /* matching paths cache max entries */, kTestRetryConfig,
1428 kTestTimeoutConfig, *kAllowedLocationsDefault,
1429 nullptr /* gcs additional header */);
1430
1431 std::vector<string> children;
1432 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children));
1433
1434 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/"}),
1435 children);
1436 }
1437
TEST(GcsFileSystemTest,GetChildren_Root)1438 TEST(GcsFileSystemTest, GetChildren_Root) {
1439 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1440 "Uri: https://www.googleapis.com/storage/v1/b/bucket-a-b-c/o?"
1441 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F\n"
1442 "Auth Token: fake_token\n"
1443 "Timeouts: 5 1 10\n",
1444 "{}")});
1445 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1446 std::unique_ptr<HttpRequest::Factory>(
1447 new FakeHttpRequestFactory(&requests)),
1448 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1449 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1450 0 /* stat cache max age */, 0 /* stat cache max entries */,
1451 0 /* matching paths cache max age */,
1452 0 /* matching paths cache max entries */, kTestRetryConfig,
1453 kTestTimeoutConfig, *kAllowedLocationsDefault,
1454 nullptr /* gcs additional header */);
1455
1456 std::vector<string> children;
1457 TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children));
1458
1459 EXPECT_EQ(0, children.size());
1460 }
1461
TEST(GcsFileSystemTest,GetChildren_Empty)1462 TEST(GcsFileSystemTest, GetChildren_Empty) {
1463 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1464 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1465 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1466 "path%2F\n"
1467 "Auth Token: fake_token\n"
1468 "Timeouts: 5 1 10\n",
1469 "{}")});
1470 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1471 std::unique_ptr<HttpRequest::Factory>(
1472 new FakeHttpRequestFactory(&requests)),
1473 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1474 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1475 0 /* stat cache max age */, 0 /* stat cache max entries */,
1476 0 /* matching paths cache max age */,
1477 0 /* matching paths cache max entries */, kTestRetryConfig,
1478 kTestTimeoutConfig, *kAllowedLocationsDefault,
1479 nullptr /* gcs additional header */);
1480
1481 std::vector<string> children;
1482 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
1483
1484 EXPECT_EQ(0, children.size());
1485 }
1486
TEST(GcsFileSystemTest,GetChildren_Pagination)1487 TEST(GcsFileSystemTest, GetChildren_Pagination) {
1488 std::vector<HttpRequest*> requests(
1489 {new FakeHttpRequest(
1490 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1491 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&"
1492 "prefix=path%2F\n"
1493 "Auth Token: fake_token\n"
1494 "Timeouts: 5 1 10\n",
1495 "{\"nextPageToken\": \"ABCD==\", "
1496 "\"items\": [ "
1497 " { \"name\": \"path/file1.txt\" },"
1498 " { \"name\": \"path/file3.txt\" }],"
1499 "\"prefixes\": [\"path/subpath/\"]}"),
1500 new FakeHttpRequest(
1501 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1502 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&"
1503 "prefix=path%2F"
1504 "&pageToken=ABCD==\n"
1505 "Auth Token: fake_token\n"
1506 "Timeouts: 5 1 10\n",
1507 "{\"items\": [ "
1508 " { \"name\": \"path/file4.txt\" },"
1509 " { \"name\": \"path/file5.txt\" }]}")});
1510
1511 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1512 std::unique_ptr<HttpRequest::Factory>(
1513 new FakeHttpRequestFactory(&requests)),
1514 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1515 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1516 0 /* stat cache max age */, 0 /* stat cache max entries */,
1517 0 /* matching paths cache max age */,
1518 0 /* matching paths cache max entries */, kTestRetryConfig,
1519 kTestTimeoutConfig, *kAllowedLocationsDefault,
1520 nullptr /* gcs additional header */);
1521
1522 std::vector<string> children;
1523 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children));
1524
1525 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/",
1526 "file4.txt", "file5.txt"}),
1527 children);
1528 }
1529
TEST(GcsFileSystemTest,GetMatchingPaths_NoWildcard)1530 TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) {
1531 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1532 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1533 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
1534 "Auth Token: fake_token\n"
1535 "Timeouts: 5 1 10\n",
1536 "{\"items\": [ "
1537 " { \"name\": \"path/subpath/file2.txt\" }]}")});
1538 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1539 std::unique_ptr<HttpRequest::Factory>(
1540 new FakeHttpRequestFactory(&requests)),
1541 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1542 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1543 0 /* stat cache max age */, 0 /* stat cache max entries */,
1544 0 /* matching paths cache max age */,
1545 0 /* matching paths cache max entries */, kTestRetryConfig,
1546 kTestTimeoutConfig, *kAllowedLocationsDefault,
1547 nullptr /* gcs additional header */);
1548
1549 std::vector<string> result;
1550 TF_EXPECT_OK(
1551 fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result));
1552 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
1553 result);
1554 }
1555
TEST(GcsFileSystemTest,GetMatchingPaths_BucketAndWildcard)1556 TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) {
1557 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1558 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1559 "fields=items%2Fname%2CnextPageToken\n"
1560 "Auth Token: fake_token\n"
1561 "Timeouts: 5 1 10\n",
1562 "{\"items\": [ "
1563 " { \"name\": \"path/file1.txt\" },"
1564 " { \"name\": \"path/subpath/file2.txt\" },"
1565 " { \"name\": \"path/file3.txt\" }]}")});
1566 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1567 std::unique_ptr<HttpRequest::Factory>(
1568 new FakeHttpRequestFactory(&requests)),
1569 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1570 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1571 0 /* stat cache max age */, 0 /* stat cache max entries */,
1572 0 /* matching paths cache max age */,
1573 0 /* matching paths cache max entries */, kTestRetryConfig,
1574 kTestTimeoutConfig, *kAllowedLocationsDefault,
1575 nullptr /* gcs additional header */);
1576
1577 std::vector<string> result;
1578 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result));
1579 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file1.txt",
1580 "gs://bucket/path/file3.txt",
1581 "gs://bucket/path/subpath"}),
1582 result);
1583 }
1584
TEST(GcsFileSystemTest,GetMatchingPaths_FolderAndWildcard_Matches)1585 TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) {
1586 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1587 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1588 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
1589 "Auth Token: fake_token\n"
1590 "Timeouts: 5 1 10\n",
1591 "{\"items\": [ "
1592 " { \"name\": \"path/file1.txt\" },"
1593 " { \"name\": \"path/subpath/file2.txt\" },"
1594 " { \"name\": \"path/file3.txt\" }]}")});
1595 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1596 std::unique_ptr<HttpRequest::Factory>(
1597 new FakeHttpRequestFactory(&requests)),
1598 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1599 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1600 0 /* stat cache max age */, 0 /* stat cache max entries */,
1601 0 /* matching paths cache max age */,
1602 0 /* matching paths cache max entries */, kTestRetryConfig,
1603 kTestTimeoutConfig, *kAllowedLocationsDefault,
1604 nullptr /* gcs additional header */);
1605
1606 std::vector<string> result;
1607 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result));
1608 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
1609 result);
1610 }
1611
TEST(GcsFileSystemTest,GetMatchingPaths_SelfDirectoryMarker)1612 TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) {
1613 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1614 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1615 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
1616 "Auth Token: fake_token\n"
1617 "Timeouts: 5 1 10\n",
1618 "{\"items\": [ "
1619 " { \"name\": \"path/\" },"
1620 " { \"name\": \"path/file3.txt\" }]}")});
1621 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1622 std::unique_ptr<HttpRequest::Factory>(
1623 new FakeHttpRequestFactory(&requests)),
1624 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1625 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1626 0 /* stat cache max age */, 0 /* stat cache max entries */,
1627 0 /* matching paths cache max age */,
1628 0 /* matching paths cache max entries */, kTestRetryConfig,
1629 kTestTimeoutConfig, *kAllowedLocationsDefault,
1630 nullptr /* gcs additional header */);
1631
1632 std::vector<string> result;
1633 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result));
1634 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file3.txt"}), result);
1635 }
1636
TEST(GcsFileSystemTest,GetMatchingPaths_FolderAndWildcard_NoMatches)1637 TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) {
1638 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1639 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1640 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
1641 "Auth Token: fake_token\n"
1642 "Timeouts: 5 1 10\n",
1643 "{\"items\": [ "
1644 " { \"name\": \"path/file1.txt\" },"
1645 " { \"name\": \"path/subpath/file2.txt\" },"
1646 " { \"name\": \"path/file3.txt\" }]}")});
1647 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1648 std::unique_ptr<HttpRequest::Factory>(
1649 new FakeHttpRequestFactory(&requests)),
1650 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1651 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1652 0 /* stat cache max age */, 0 /* stat cache max entries */,
1653 0 /* matching paths cache max age */,
1654 0 /* matching paths cache max entries */, kTestRetryConfig,
1655 kTestTimeoutConfig, *kAllowedLocationsDefault,
1656 nullptr /* gcs additional header */);
1657
1658 std::vector<string> result;
1659 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result));
1660 EXPECT_EQ(std::vector<string>(), result);
1661 }
1662
TEST(GcsFileSystemTest,GetMatchingPaths_OnlyWildcard)1663 TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) {
1664 std::vector<HttpRequest*> requests;
1665 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1666 std::unique_ptr<HttpRequest::Factory>(
1667 new FakeHttpRequestFactory(&requests)),
1668 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1669 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1670 0 /* stat cache max age */, 0 /* stat cache max entries */,
1671 0 /* matching paths cache max age */,
1672 0 /* matching paths cache max entries */, kTestRetryConfig,
1673 kTestTimeoutConfig, *kAllowedLocationsDefault,
1674 nullptr /* gcs additional header */);
1675
1676 std::vector<string> result;
1677 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1678 fs.GetMatchingPaths("gs://*", &result).code());
1679 }
1680
TEST(GcsFileSystemTest,GetMatchingPaths_Cache)1681 TEST(GcsFileSystemTest, GetMatchingPaths_Cache) {
1682 std::vector<HttpRequest*> requests(
1683 {new FakeHttpRequest(
1684 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1685 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
1686 "Auth Token: fake_token\n"
1687 "Timeouts: 5 1 10\n",
1688 "{\"items\": [ "
1689 " { \"name\": \"path/subpath/file2.txt\" }]}"),
1690 new FakeHttpRequest(
1691 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1692 "fields=items%2Fname%2CnextPageToken\n"
1693 "Auth Token: fake_token\n"
1694 "Timeouts: 5 1 10\n",
1695 "{\"items\": [ "
1696 " { \"name\": \"path/file1.txt\" },"
1697 " { \"name\": \"path/subpath/file2.txt\" },"
1698 " { \"name\": \"path/file3.txt\" }]}")});
1699 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1700 std::unique_ptr<HttpRequest::Factory>(
1701 new FakeHttpRequestFactory(&requests)),
1702 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1703 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1704 0 /* stat cache max age */, 0 /* stat cache max entries */,
1705 3600 /* matching paths cache max age */,
1706 0 /* matching paths cache max entries */, kTestRetryConfig,
1707 kTestTimeoutConfig, *kAllowedLocationsDefault,
1708 nullptr /* gcs additional header */);
1709
1710 // Repeated calls to fs.GetMatchingPaths on these patterns should not lead to
1711 // any additional HTTP requests to GCS.
1712 for (int i = 0; i < 10; i++) {
1713 std::vector<string> result;
1714 TF_EXPECT_OK(
1715 fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result));
1716 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
1717 result);
1718 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result));
1719 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file1.txt",
1720 "gs://bucket/path/file3.txt",
1721 "gs://bucket/path/subpath"}),
1722 result);
1723 }
1724 }
1725
TEST(GcsFileSystemTest,GetMatchingPaths_Cache_Flush)1726 TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) {
1727 std::vector<HttpRequest*> requests(
1728 {new FakeHttpRequest(
1729 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1730 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
1731 "Auth Token: fake_token\n"
1732 "Timeouts: 5 1 10\n",
1733 "{\"items\": [ "
1734 " { \"name\": \"path/subpath/file2.txt\" }]}"),
1735 new FakeHttpRequest(
1736 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1737 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
1738 "Auth Token: fake_token\n"
1739 "Timeouts: 5 1 10\n",
1740 "{\"items\": [ "
1741 " { \"name\": \"path/subpath/file2.txt\" }]}")});
1742 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1743 std::unique_ptr<HttpRequest::Factory>(
1744 new FakeHttpRequestFactory(&requests)),
1745 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1746 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1747 0 /* stat cache max age */, 0 /* stat cache max entries */,
1748 3600 /* matching paths cache max age */,
1749 0 /* matching paths cache max entries */, kTestRetryConfig,
1750 kTestTimeoutConfig, *kAllowedLocationsDefault,
1751 nullptr /* gcs additional header */);
1752
1753 // This loop should trigger the first HTTP request to GCS.
1754 for (int i = 0; i < 10; i++) {
1755 std::vector<string> result;
1756 TF_EXPECT_OK(
1757 fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result));
1758 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
1759 result);
1760 }
1761 // After flushing caches, there should be another (identical) request to GCS.
1762 fs.FlushCaches();
1763 for (int i = 0; i < 10; i++) {
1764 std::vector<string> result;
1765 TF_EXPECT_OK(
1766 fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result));
1767 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
1768 result);
1769 }
1770 }
1771
TEST(GcsFileSystemTest,DeleteFile)1772 TEST(GcsFileSystemTest, DeleteFile) {
1773 std::vector<HttpRequest*> requests(
1774 {new FakeHttpRequest(
1775 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1776 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1777 "Auth Token: fake_token\n"
1778 "Timeouts: 5 1 10\n",
1779 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
1780 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1781 new FakeHttpRequest(
1782 "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
1783 "Auth Token: fake_token\n"
1784 "Range: 0-15\n"
1785 "Timeouts: 5 1 20\n",
1786 "01234567"),
1787 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
1788 "/bucket/o/path%2Ffile1.txt\n"
1789 "Auth Token: fake_token\n"
1790 "Timeouts: 5 1 10\n"
1791 "Delete: yes\n",
1792 ""),
1793 new FakeHttpRequest(
1794 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1795 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1796 "Auth Token: fake_token\n"
1797 "Timeouts: 5 1 10\n",
1798 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
1799 "\"updated\": \"2016-04-29T23:19:24.896Z\"}")),
1800 new FakeHttpRequest(
1801 "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
1802 "Auth Token: fake_token\n"
1803 "Range: 0-15\n"
1804 "Timeouts: 5 1 20\n",
1805 "76543210")});
1806 GcsFileSystem fs(
1807 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1808 std::unique_ptr<HttpRequest::Factory>(
1809 new FakeHttpRequestFactory(&requests)),
1810 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
1811 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1812 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1813 0 /* matching paths cache max entries */, kTestRetryConfig,
1814 kTestTimeoutConfig, *kAllowedLocationsDefault,
1815 nullptr /* gcs additional header */);
1816
1817 // Do an initial read of the file to load its contents into the block cache.
1818 char scratch[100];
1819 StringPiece result;
1820 std::unique_ptr<RandomAccessFile> file;
1821 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/file1.txt", &file));
1822 TF_EXPECT_OK(file->Read(0, 8, &result, scratch));
1823 EXPECT_EQ("01234567", result);
1824 // Deleting the file triggers the next HTTP request to GCS.
1825 TF_EXPECT_OK(fs.DeleteFile("gs://bucket/path/file1.txt"));
1826 // Re-reading the file causes its contents to be reloaded from GCS and not
1827 // from the block cache.
1828 TF_EXPECT_OK(file->Read(0, 8, &result, scratch));
1829 EXPECT_EQ("76543210", result);
1830 }
1831
TEST(GcsFileSystemTest,DeleteFile_NoObjectName)1832 TEST(GcsFileSystemTest, DeleteFile_NoObjectName) {
1833 std::vector<HttpRequest*> requests;
1834 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1835 std::unique_ptr<HttpRequest::Factory>(
1836 new FakeHttpRequestFactory(&requests)),
1837 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1838 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1839 0 /* stat cache max age */, 0 /* stat cache max entries */,
1840 0 /* matching paths cache max age */,
1841 0 /* matching paths cache max entries */, kTestRetryConfig,
1842 kTestTimeoutConfig, *kAllowedLocationsDefault,
1843 nullptr /* gcs additional header */);
1844
1845 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1846 fs.DeleteFile("gs://bucket/").code());
1847 }
1848
TEST(GcsFileSystemTest,DeleteFile_StatCacheRemoved)1849 TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) {
1850 std::vector<HttpRequest*> requests(
1851 {new FakeHttpRequest(
1852 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1853 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
1854 "Auth Token: fake_token\n"
1855 "Timeouts: 5 1 10\n",
1856 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
1857 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1858 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
1859 "/bucket/o/file.txt\n"
1860 "Auth Token: fake_token\n"
1861 "Timeouts: 5 1 10\n"
1862 "Delete: yes\n",
1863 ""),
1864 new FakeHttpRequest(
1865 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1866 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
1867 "Auth Token: fake_token\n"
1868 "Timeouts: 5 1 10\n",
1869 "", errors::NotFound("404"), 404),
1870 new FakeHttpRequest(
1871 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1872 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
1873 "&maxResults=1\n"
1874 "Auth Token: fake_token\n"
1875 "Timeouts: 5 1 10\n",
1876 "{}")});
1877 GcsFileSystem fs(
1878 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1879 std::unique_ptr<HttpRequest::Factory>(
1880 new FakeHttpRequestFactory(&requests)),
1881 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
1882 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1883 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1884 0 /* matching paths cache max entries */, kTestRetryConfig,
1885 kTestTimeoutConfig, *kAllowedLocationsDefault,
1886 nullptr /* gcs additional header */);
1887
1888 // Stats the file first so the stat is cached.
1889 FileStatistics stat_before_deletion;
1890 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat_before_deletion));
1891 EXPECT_EQ(1010, stat_before_deletion.length);
1892
1893 TF_EXPECT_OK(fs.DeleteFile("gs://bucket/file.txt"));
1894
1895 FileStatistics stat_after_deletion;
1896 EXPECT_EQ(error::Code::NOT_FOUND,
1897 fs.Stat("gs://bucket/file.txt", &stat_after_deletion).code());
1898 }
1899
TEST(GcsFileSystemTest,DeleteDir_Empty)1900 TEST(GcsFileSystemTest, DeleteDir_Empty) {
1901 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1902 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1903 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
1904 "Auth Token: fake_token\n"
1905 "Timeouts: 5 1 10\n",
1906 "{}")});
1907 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1908 std::unique_ptr<HttpRequest::Factory>(
1909 new FakeHttpRequestFactory(&requests)),
1910 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1911 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1912 0 /* stat cache max age */, 0 /* stat cache max entries */,
1913 0 /* matching paths cache max age */,
1914 0 /* matching paths cache max entries */, kTestRetryConfig,
1915 kTestTimeoutConfig, *kAllowedLocationsDefault,
1916 nullptr /* gcs additional header */);
1917
1918 TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/"));
1919 }
1920
TEST(GcsFileSystemTest,DeleteDir_OnlyDirMarkerLeft)1921 TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) {
1922 std::vector<HttpRequest*> requests(
1923 {new FakeHttpRequest(
1924 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1925 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
1926 "Auth Token: fake_token\n"
1927 "Timeouts: 5 1 10\n",
1928 "{\"items\": [ "
1929 " { \"name\": \"path/\" }]}"),
1930 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
1931 "/bucket/o/path%2F\n"
1932 "Auth Token: fake_token\n"
1933 "Timeouts: 5 1 10\n"
1934 "Delete: yes\n",
1935 "")});
1936 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1937 std::unique_ptr<HttpRequest::Factory>(
1938 new FakeHttpRequestFactory(&requests)),
1939 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1940 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1941 0 /* stat cache max age */, 0 /* stat cache max entries */,
1942 0 /* matching paths cache max age */,
1943 0 /* matching paths cache max entries */, kTestRetryConfig,
1944 kTestTimeoutConfig, *kAllowedLocationsDefault,
1945 nullptr /* gcs additional header */);
1946
1947 TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/"));
1948 }
1949
TEST(GcsFileSystemTest,DeleteDir_BucketOnly)1950 TEST(GcsFileSystemTest, DeleteDir_BucketOnly) {
1951 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1952 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?fields=items%2F"
1953 "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n"
1954 "Timeouts: 5 1 10\n",
1955 "{}")});
1956 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1957 std::unique_ptr<HttpRequest::Factory>(
1958 new FakeHttpRequestFactory(&requests)),
1959 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1960 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1961 0 /* stat cache max age */, 0 /* stat cache max entries */,
1962 0 /* matching paths cache max age */,
1963 0 /* matching paths cache max entries */, kTestRetryConfig,
1964 kTestTimeoutConfig, *kAllowedLocationsDefault,
1965 nullptr /* gcs additional header */);
1966
1967 TF_EXPECT_OK(fs.DeleteDir("gs://bucket"));
1968 }
1969
TEST(GcsFileSystemTest,DeleteDir_NonEmpty)1970 TEST(GcsFileSystemTest, DeleteDir_NonEmpty) {
1971 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1972 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1973 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
1974 "Auth Token: fake_token\n"
1975 "Timeouts: 5 1 10\n",
1976 "{\"items\": [ "
1977 " { \"name\": \"path/file1.txt\" }]}")});
1978 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1979 std::unique_ptr<HttpRequest::Factory>(
1980 new FakeHttpRequestFactory(&requests)),
1981 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
1982 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
1983 0 /* stat cache max age */, 0 /* stat cache max entries */,
1984 0 /* matching paths cache max age */,
1985 0 /* matching paths cache max entries */, kTestRetryConfig,
1986 kTestTimeoutConfig, *kAllowedLocationsDefault,
1987 nullptr /* gcs additional header */);
1988
1989 EXPECT_EQ(error::Code::FAILED_PRECONDITION,
1990 fs.DeleteDir("gs://bucket/path/").code());
1991 }
1992
TEST(GcsFileSystemTest,GetFileSize)1993 TEST(GcsFileSystemTest, GetFileSize) {
1994 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1995 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1996 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
1997 "Auth Token: fake_token\n"
1998 "Timeouts: 5 1 10\n",
1999 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2000 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2001 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2002 std::unique_ptr<HttpRequest::Factory>(
2003 new FakeHttpRequestFactory(&requests)),
2004 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2005 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2006 0 /* stat cache max age */, 0 /* stat cache max entries */,
2007 0 /* matching paths cache max age */,
2008 0 /* matching paths cache max entries */, kTestRetryConfig,
2009 kTestTimeoutConfig, *kAllowedLocationsDefault,
2010 nullptr /* gcs additional header */);
2011
2012 uint64 size;
2013 TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size));
2014 EXPECT_EQ(1010, size);
2015 }
2016
TEST(GcsFileSystemTest,GetFileSize_NoObjectName)2017 TEST(GcsFileSystemTest, GetFileSize_NoObjectName) {
2018 std::vector<HttpRequest*> requests;
2019 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2020 std::unique_ptr<HttpRequest::Factory>(
2021 new FakeHttpRequestFactory(&requests)),
2022 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2023 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2024 0 /* stat cache max age */, 0 /* stat cache max entries */,
2025 0 /* matching paths cache max age */,
2026 0 /* matching paths cache max entries */, kTestRetryConfig,
2027 kTestTimeoutConfig, *kAllowedLocationsDefault,
2028 nullptr /* gcs additional header */);
2029
2030 uint64 size;
2031 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
2032 fs.GetFileSize("gs://bucket/", &size).code());
2033 }
2034
TEST(GcsFileSystemTest,RenameFile_Folder)2035 TEST(GcsFileSystemTest, RenameFile_Folder) {
2036 std::vector<HttpRequest*> requests(
2037 {// Check if this is a folder or an object.
2038 new FakeHttpRequest(
2039 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2040 "fields=items%2Fname%2CnextPageToken&prefix=path1%2F"
2041 "&maxResults=1\n"
2042 "Auth Token: fake_token\n"
2043 "Timeouts: 5 1 10\n",
2044 "{\"items\": [ "
2045 " { \"name\": \"path1/subfolder/file1.txt\" }]}"),
2046 // Requesting the full list of files in the folder.
2047 new FakeHttpRequest(
2048 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2049 "fields=items%2Fname%2CnextPageToken&prefix=path1%2F\n"
2050 "Auth Token: fake_token\n"
2051 "Timeouts: 5 1 10\n",
2052 "{\"items\": [ "
2053 " { \"name\": \"path1/\" }," // A directory marker.
2054 " { \"name\": \"path1/subfolder/file1.txt\" },"
2055 " { \"name\": \"path1/file2.txt\" }]}"),
2056 // Copying the directory marker.
2057 new FakeHttpRequest(
2058 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2059 "path1%2F/rewriteTo/b/bucket/o/path2%2F\n"
2060 "Auth Token: fake_token\n"
2061 "Post: yes\n"
2062 "Timeouts: 5 1 10\n",
2063 "{\"done\": true}"),
2064 // Deleting the original directory marker.
2065 new FakeHttpRequest(
2066 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2067 "path1%2F\n"
2068 "Auth Token: fake_token\n"
2069 "Timeouts: 5 1 10\n"
2070 "Delete: yes\n",
2071 ""),
2072 // Copying the first file.
2073 new FakeHttpRequest(
2074 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2075 "path1%2Fsubfolder%2Ffile1.txt/rewriteTo/b/bucket/o/"
2076 "path2%2Fsubfolder%2Ffile1.txt\n"
2077 "Auth Token: fake_token\n"
2078 "Post: yes\n"
2079 "Timeouts: 5 1 10\n",
2080 "{\"done\": true}"),
2081 // Deleting the first original file.
2082 new FakeHttpRequest(
2083 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2084 "path1%2Fsubfolder%2Ffile1.txt\n"
2085 "Auth Token: fake_token\n"
2086 "Timeouts: 5 1 10\n"
2087 "Delete: yes\n",
2088 ""),
2089 // Copying the second file.
2090 new FakeHttpRequest(
2091 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2092 "path1%2Ffile2.txt/rewriteTo/b/bucket/o/path2%2Ffile2.txt\n"
2093 "Auth Token: fake_token\n"
2094 "Post: yes\n"
2095 "Timeouts: 5 1 10\n",
2096 "{\"done\": true}"),
2097 // Deleting the second original file.
2098 new FakeHttpRequest(
2099 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2100 "path1%2Ffile2.txt\n"
2101 "Auth Token: fake_token\n"
2102 "Timeouts: 5 1 10\n"
2103 "Delete: yes\n",
2104 "")});
2105 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2106 std::unique_ptr<HttpRequest::Factory>(
2107 new FakeHttpRequestFactory(&requests)),
2108 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2109 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2110 0 /* stat cache max age */, 0 /* stat cache max entries */,
2111 0 /* matching paths cache max age */,
2112 0 /* matching paths cache max entries */, kTestRetryConfig,
2113 kTestTimeoutConfig, *kAllowedLocationsDefault,
2114 nullptr /* gcs additional header */);
2115
2116 TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/"));
2117 }
2118
TEST(GcsFileSystemTest,RenameFile_Object)2119 TEST(GcsFileSystemTest, RenameFile_Object) {
2120 std::vector<HttpRequest*> requests(
2121 {new FakeHttpRequest(
2122 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2123 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2124 "Auth Token: fake_token\n"
2125 "Timeouts: 5 1 10\n",
2126 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
2127 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2128 new FakeHttpRequest(
2129 "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
2130 "Auth Token: fake_token\n"
2131 "Range: 0-15\n"
2132 "Timeouts: 5 1 20\n",
2133 "01234567"),
2134 new FakeHttpRequest(
2135 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2136 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2137 "Auth Token: fake_token\n"
2138 "Timeouts: 5 1 10\n",
2139 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
2140 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2141 new FakeHttpRequest(
2142 "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
2143 "Auth Token: fake_token\n"
2144 "Range: 0-15\n"
2145 "Timeouts: 5 1 20\n",
2146 "76543210"),
2147 // IsDirectory is checking whether there are children objects.
2148 new FakeHttpRequest(
2149 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2150 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2151 "&maxResults=1\n"
2152 "Auth Token: fake_token\n"
2153 "Timeouts: 5 1 10\n",
2154 "{}"),
2155 // Copying to the new location.
2156 new FakeHttpRequest(
2157 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2158 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2159 "Auth Token: fake_token\n"
2160 "Post: yes\n"
2161 "Timeouts: 5 1 10\n",
2162 "{\"done\": true}"),
2163 // Deleting the original file.
2164 new FakeHttpRequest(
2165 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2166 "path%2Fsrc.txt\n"
2167 "Auth Token: fake_token\n"
2168 "Timeouts: 5 1 10\n"
2169 "Delete: yes\n",
2170 ""),
2171 new FakeHttpRequest(
2172 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2173 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2174 "Auth Token: fake_token\n"
2175 "Timeouts: 5 1 10\n",
2176 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
2177 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2178 new FakeHttpRequest(
2179 "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
2180 "Auth Token: fake_token\n"
2181 "Range: 0-15\n"
2182 "Timeouts: 5 1 20\n",
2183 "89abcdef"),
2184 new FakeHttpRequest(
2185 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2186 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2187 "Auth Token: fake_token\n"
2188 "Timeouts: 5 1 10\n",
2189 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
2190 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2191 new FakeHttpRequest(
2192 "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
2193 "Auth Token: fake_token\n"
2194 "Range: 0-15\n"
2195 "Timeouts: 5 1 20\n",
2196 "fedcba98")});
2197 GcsFileSystem fs(
2198 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2199 std::unique_ptr<HttpRequest::Factory>(
2200 new FakeHttpRequestFactory(&requests)),
2201 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
2202 64 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2203 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2204 0 /* matching paths cache max entries */, kTestRetryConfig,
2205 kTestTimeoutConfig, *kAllowedLocationsDefault,
2206 nullptr /* gcs additional header */);
2207 // Do an initial read of the source and destination files to load their
2208 // contents into the block cache.
2209 char scratch[100];
2210 StringPiece result;
2211 std::unique_ptr<RandomAccessFile> src;
2212 std::unique_ptr<RandomAccessFile> dst;
2213 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/src.txt", &src));
2214 TF_EXPECT_OK(src->Read(0, 8, &result, scratch));
2215 EXPECT_EQ("01234567", result);
2216 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/dst.txt", &dst));
2217 TF_EXPECT_OK(dst->Read(0, 8, &result, scratch));
2218 EXPECT_EQ("76543210", result);
2219 // Now rename src to dst. This should flush the block cache for both files.
2220 TF_EXPECT_OK(
2221 fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt"));
2222 // Re-read both files. This should reload their contents from GCS.
2223 TF_EXPECT_OK(src->Read(0, 8, &result, scratch));
2224 EXPECT_EQ("89abcdef", result);
2225 TF_EXPECT_OK(dst->Read(0, 8, &result, scratch));
2226 EXPECT_EQ("fedcba98", result);
2227 }
2228
TEST(GcsFileSystemTest,RenameFile_Object_FlushTargetStatCache)2229 TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) {
2230 std::vector<HttpRequest*> requests(
2231 {// Stat the target file.
2232 new FakeHttpRequest(
2233 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2234 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2235 "Auth Token: fake_token\n"
2236 "Timeouts: 5 1 10\n",
2237 strings::StrCat("{\"size\": \"1000\",\"generation\": \"1\","
2238 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2239 // IsDirectory is checking whether there are children objects.
2240 new FakeHttpRequest(
2241 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2242 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2243 "&maxResults=1\n"
2244 "Auth Token: fake_token\n"
2245 "Timeouts: 5 1 10\n",
2246 "{}"),
2247 // IsDirectory is checking if the path exists as an object.
2248 new FakeHttpRequest(
2249 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2250 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2251 "Auth Token: fake_token\n"
2252 "Timeouts: 5 1 10\n",
2253 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2254 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2255 // Copying to the new location.
2256 new FakeHttpRequest(
2257 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2258 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2259 "Auth Token: fake_token\n"
2260 "Post: yes\n"
2261 "Timeouts: 5 1 10\n",
2262 "{\"done\": true}"),
2263 // Deleting the original file.
2264 new FakeHttpRequest(
2265 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2266 "path%2Fsrc.txt\n"
2267 "Auth Token: fake_token\n"
2268 "Timeouts: 5 1 10\n"
2269 "Delete: yes\n",
2270 ""),
2271 new FakeHttpRequest(
2272 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2273 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2274 "Auth Token: fake_token\n"
2275 "Timeouts: 5 1 10\n",
2276 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2277 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2278 GcsFileSystem fs(
2279 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2280 std::unique_ptr<HttpRequest::Factory>(
2281 new FakeHttpRequestFactory(&requests)),
2282 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2283 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2284 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2285 0 /* matching paths cache max entries */, kTestRetryConfig,
2286 kTestTimeoutConfig, *kAllowedLocationsDefault,
2287 nullptr /* gcs additional header */);
2288 // Do an initial stat of the destination file to load their contents into the
2289 // stat cache.
2290 FileStatistics stat_before_renaming;
2291 TF_EXPECT_OK(fs.Stat("gs://bucket/path/dst.txt", &stat_before_renaming));
2292 EXPECT_EQ(1000, stat_before_renaming.length);
2293
2294 TF_EXPECT_OK(
2295 fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt"));
2296
2297 FileStatistics stat_after_renaming;
2298 TF_EXPECT_OK(fs.Stat("gs://bucket/path/dst.txt", &stat_after_renaming));
2299 EXPECT_EQ(1010, stat_after_renaming.length);
2300 }
2301
2302 /// Tests the scenario when deletion returns a failure, but actually succeeds.
TEST(GcsFileSystemTest,RenameFile_Object_DeletionRetried)2303 TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
2304 std::vector<HttpRequest*> requests(
2305 {// IsDirectory is checking whether there are children objects.
2306 new FakeHttpRequest(
2307 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2308 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2309 "&maxResults=1\n"
2310 "Auth Token: fake_token\n"
2311 "Timeouts: 5 1 10\n",
2312 "{}"),
2313 // IsDirectory is checking if the path exists as an object.
2314 new FakeHttpRequest(
2315 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2316 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2317 "Auth Token: fake_token\n"
2318 "Timeouts: 5 1 10\n",
2319 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2320 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2321 // Copying to the new location.
2322 new FakeHttpRequest(
2323 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2324 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2325 "Auth Token: fake_token\n"
2326 "Post: yes\n"
2327 "Timeouts: 5 1 10\n",
2328 "{\"done\": true}"),
2329 // Deleting the original file - the deletion returns a failure.
2330 new FakeHttpRequest(
2331 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2332 "path%2Fsrc.txt\n"
2333 "Auth Token: fake_token\n"
2334 "Timeouts: 5 1 10\n"
2335 "Delete: yes\n",
2336 "", errors::Unavailable("503"), 503),
2337 // Deleting the original file again - the deletion returns NOT_FOUND.
2338 new FakeHttpRequest(
2339 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2340 "path%2Fsrc.txt\n"
2341 "Auth Token: fake_token\n"
2342 "Timeouts: 5 1 10\n"
2343 "Delete: yes\n",
2344 "", errors::NotFound("404"), 404)});
2345 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2346 std::unique_ptr<HttpRequest::Factory>(
2347 new FakeHttpRequestFactory(&requests)),
2348 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2349 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2350 0 /* stat cache max age */, 0 /* stat cache max entries */,
2351 0 /* matching paths cache max age */,
2352 0 /* matching paths cache max entries */, kTestRetryConfig,
2353 kTestTimeoutConfig, *kAllowedLocationsDefault,
2354 nullptr /* gcs additional header */);
2355
2356 TF_EXPECT_OK(
2357 fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt"));
2358 }
2359
2360 /// Tests the case when rewrite couldn't complete in one RPC.
TEST(GcsFileSystemTest,RenameFile_Object_Incomplete)2361 TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) {
2362 std::vector<HttpRequest*> requests(
2363 {// IsDirectory is checking whether there are children objects.
2364 new FakeHttpRequest(
2365 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2366 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2367 "&maxResults=1\n"
2368 "Auth Token: fake_token\n"
2369 "Timeouts: 5 1 10\n",
2370 "{}"),
2371 // IsDirectory is checking if the path exists as an object.
2372 new FakeHttpRequest(
2373 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2374 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2375 "Auth Token: fake_token\n"
2376 "Timeouts: 5 1 10\n",
2377 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2378 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2379 // Copying to the new location.
2380 new FakeHttpRequest(
2381 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2382 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2383 "Auth Token: fake_token\n"
2384 "Post: yes\n"
2385 "Timeouts: 5 1 10\n",
2386 "{\"done\": false}")});
2387 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2388 std::unique_ptr<HttpRequest::Factory>(
2389 new FakeHttpRequestFactory(&requests)),
2390 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2391 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2392 0 /* stat cache max age */, 0 /* stat cache max entries */,
2393 0 /* matching paths cache max age */,
2394 0 /* matching paths cache max entries */, kTestRetryConfig,
2395 kTestTimeoutConfig, *kAllowedLocationsDefault,
2396 nullptr /* gcs additional header */);
2397
2398 EXPECT_EQ(
2399 errors::Code::UNIMPLEMENTED,
2400 fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")
2401 .code());
2402 }
2403
TEST(GcsFileSystemTest,Stat_Object)2404 TEST(GcsFileSystemTest, Stat_Object) {
2405 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2406 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2407 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2408 "Auth Token: fake_token\n"
2409 "Timeouts: 5 1 10\n",
2410 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2411 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2412 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2413 std::unique_ptr<HttpRequest::Factory>(
2414 new FakeHttpRequestFactory(&requests)),
2415 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2416 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2417 0 /* stat cache max age */, 0 /* stat cache max entries */,
2418 0 /* matching paths cache max age */,
2419 0 /* matching paths cache max entries */, kTestRetryConfig,
2420 kTestTimeoutConfig, *kAllowedLocationsDefault,
2421 nullptr /* gcs additional header */);
2422
2423 FileStatistics stat;
2424 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
2425 EXPECT_EQ(1010, stat.length);
2426 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
2427 EXPECT_FALSE(stat.is_directory);
2428 }
2429
TEST(GcsFileSystemTest,Stat_Folder)2430 TEST(GcsFileSystemTest, Stat_Folder) {
2431 std::vector<HttpRequest*> requests(
2432 {new FakeHttpRequest(
2433 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2434 "subfolder?fields=size%2Cgeneration%2Cupdated\n"
2435 "Auth Token: fake_token\n"
2436 "Timeouts: 5 1 10\n",
2437 "", errors::NotFound("404"), 404),
2438 new FakeHttpRequest(
2439 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2440 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
2441 "&maxResults=1\n"
2442 "Auth Token: fake_token\n"
2443 "Timeouts: 5 1 10\n",
2444 "{\"items\": [ "
2445 " { \"name\": \"subfolder/\" }]}")});
2446 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2447 std::unique_ptr<HttpRequest::Factory>(
2448 new FakeHttpRequestFactory(&requests)),
2449 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2450 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2451 0 /* stat cache max age */, 0 /* stat cache max entries */,
2452 0 /* matching paths cache max age */,
2453 0 /* matching paths cache max entries */, kTestRetryConfig,
2454 kTestTimeoutConfig, *kAllowedLocationsDefault,
2455 nullptr /* gcs additional header */);
2456
2457 FileStatistics stat;
2458 TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat));
2459 EXPECT_EQ(0, stat.length);
2460 EXPECT_EQ(0, stat.mtime_nsec);
2461 EXPECT_TRUE(stat.is_directory);
2462 }
2463
TEST(GcsFileSystemTest,Stat_ObjectOrFolderNotFound)2464 TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) {
2465 std::vector<HttpRequest*> requests(
2466 {new FakeHttpRequest(
2467 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2468 "path?fields=size%2Cgeneration%2Cupdated\n"
2469 "Auth Token: fake_token\n"
2470 "Timeouts: 5 1 10\n",
2471 "", errors::NotFound("404"), 404),
2472 new FakeHttpRequest(
2473 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2474 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
2475 "&maxResults=1\n"
2476 "Auth Token: fake_token\n"
2477 "Timeouts: 5 1 10\n",
2478 "{}")});
2479 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2480 std::unique_ptr<HttpRequest::Factory>(
2481 new FakeHttpRequestFactory(&requests)),
2482 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2483 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2484 0 /* stat cache max age */, 0 /* stat cache max entries */,
2485 0 /* matching paths cache max age */,
2486 0 /* matching paths cache max entries */, kTestRetryConfig,
2487 kTestTimeoutConfig, *kAllowedLocationsDefault,
2488 nullptr /* gcs additional header */);
2489
2490 FileStatistics stat;
2491 EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code());
2492 }
2493
TEST(GcsFileSystemTest,Stat_Bucket)2494 TEST(GcsFileSystemTest, Stat_Bucket) {
2495 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2496 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2497 "Auth Token: fake_token\n"
2498 "Timeouts: 5 1 10\n",
2499 "{}")});
2500 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2501 std::unique_ptr<HttpRequest::Factory>(
2502 new FakeHttpRequestFactory(&requests)),
2503 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2504 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2505 0 /* stat cache max age */, 0 /* stat cache max entries */,
2506 0 /* matching paths cache max age */,
2507 0 /* matching paths cache max entries */, kTestRetryConfig,
2508 kTestTimeoutConfig, *kAllowedLocationsDefault,
2509 nullptr /* gcs additional header */);
2510
2511 FileStatistics stat;
2512 TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat));
2513 EXPECT_EQ(0, stat.length);
2514 EXPECT_EQ(0, stat.mtime_nsec);
2515 EXPECT_TRUE(stat.is_directory);
2516 }
2517
TEST(GcsFileSystemTest,Stat_BucketNotFound)2518 TEST(GcsFileSystemTest, Stat_BucketNotFound) {
2519 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2520 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2521 "Auth Token: fake_token\n"
2522 "Timeouts: 5 1 10\n",
2523 "", errors::NotFound("404"), 404)});
2524 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2525 std::unique_ptr<HttpRequest::Factory>(
2526 new FakeHttpRequestFactory(&requests)),
2527 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2528 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2529 0 /* stat cache max age */, 0 /* stat cache max entries */,
2530 0 /* matching paths cache max age */,
2531 0 /* matching paths cache max entries */, kTestRetryConfig,
2532 kTestTimeoutConfig, *kAllowedLocationsDefault,
2533 nullptr /* gcs additional header */);
2534
2535 FileStatistics stat;
2536 EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code());
2537 }
2538
TEST(GcsFileSystemTest,Stat_Cache)2539 TEST(GcsFileSystemTest, Stat_Cache) {
2540 std::vector<HttpRequest*> requests(
2541 {new FakeHttpRequest(
2542 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2543 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2544 "Auth Token: fake_token\n"
2545 "Timeouts: 5 1 10\n",
2546 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2547 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2548 new FakeHttpRequest(
2549 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2550 "subfolder%2F?fields=size%2Cgeneration%2Cupdated\n"
2551 "Auth Token: fake_token\n"
2552 "Timeouts: 5 1 10\n",
2553 "", errors::NotFound("404"), 404),
2554 new FakeHttpRequest(
2555 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2556 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
2557 "&maxResults=1\n"
2558 "Auth Token: fake_token\n"
2559 "Timeouts: 5 1 10\n",
2560 "{\"items\": [ "
2561 " { \"name\": \"subfolder/\" }]}")});
2562 GcsFileSystem fs(
2563 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2564 std::unique_ptr<HttpRequest::Factory>(
2565 new FakeHttpRequestFactory(&requests)),
2566 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2567 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2568 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2569 0 /* matching paths cache max entries */, kTestRetryConfig,
2570 kTestTimeoutConfig, *kAllowedLocationsDefault,
2571 nullptr /* gcs additional header */);
2572
2573 // Repeated calls to fs.Stat on these paths should not lead to any additional
2574 // HTTP requests to GCS.
2575 for (int i = 0; i < 10; i++) {
2576 FileStatistics stat;
2577 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
2578 EXPECT_EQ(1010, stat.length);
2579 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
2580 EXPECT_FALSE(stat.is_directory);
2581 TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder/", &stat));
2582 EXPECT_EQ(0, stat.length);
2583 EXPECT_EQ(0, stat.mtime_nsec);
2584 EXPECT_TRUE(stat.is_directory);
2585 }
2586 }
2587
TEST(GcsFileSystemTest,Stat_Cache_Flush)2588 TEST(GcsFileSystemTest, Stat_Cache_Flush) {
2589 std::vector<HttpRequest*> requests(
2590 {new FakeHttpRequest(
2591 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2592 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2593 "Auth Token: fake_token\n"
2594 "Timeouts: 5 1 10\n",
2595 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2596 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2597 new FakeHttpRequest(
2598 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2599 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2600 "Auth Token: fake_token\n"
2601 "Timeouts: 5 1 10\n",
2602 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2603 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2604 GcsFileSystem fs(
2605 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2606 std::unique_ptr<HttpRequest::Factory>(
2607 new FakeHttpRequestFactory(&requests)),
2608 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2609 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2610 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2611 0 /* matching paths cache max entries */, kTestRetryConfig,
2612 kTestTimeoutConfig, *kAllowedLocationsDefault,
2613 nullptr /* gcs additional header */);
2614 // There should be a single HTTP request to GCS for fs.Stat in this loop.
2615 for (int i = 0; i < 10; i++) {
2616 FileStatistics stat;
2617 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
2618 EXPECT_EQ(1010, stat.length);
2619 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
2620 EXPECT_FALSE(stat.is_directory);
2621 }
2622 // After flushing caches, there should be a second request to GCS for fs.Stat.
2623 fs.FlushCaches();
2624 for (int i = 0; i < 10; i++) {
2625 FileStatistics stat;
2626 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
2627 EXPECT_EQ(1010, stat.length);
2628 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
2629 EXPECT_FALSE(stat.is_directory);
2630 }
2631 }
2632
TEST(GcsFileSystemTest,Stat_FilenameEndingWithSlash)2633 TEST(GcsFileSystemTest, Stat_FilenameEndingWithSlash) {
2634 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2635 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2636 "dir%2F?fields=size%2Cgeneration%2Cupdated\n"
2637 "Auth Token: fake_token\n"
2638 "Timeouts: 5 1 10\n",
2639 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
2640 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2641 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2642 std::unique_ptr<HttpRequest::Factory>(
2643 new FakeHttpRequestFactory(&requests)),
2644 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2645 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2646 0 /* stat cache max age */, 0 /* stat cache max entries */,
2647 0 /* matching paths cache max age */,
2648 0 /* matching paths cache max entries */, kTestRetryConfig,
2649 kTestTimeoutConfig, *kAllowedLocationsDefault,
2650 nullptr /* gcs additional header */);
2651
2652 FileStatistics stat;
2653 TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", &stat));
2654 EXPECT_EQ(5, stat.length);
2655 EXPECT_TRUE(stat.is_directory);
2656 }
2657
TEST(GcsFileSystemTest,IsDirectory_NotFound)2658 TEST(GcsFileSystemTest, IsDirectory_NotFound) {
2659 std::vector<HttpRequest*> requests(
2660 {new FakeHttpRequest(
2661 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2662 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
2663 "&maxResults=1\n"
2664 "Auth Token: fake_token\n"
2665 "Timeouts: 5 1 10\n",
2666 "{}"),
2667 new FakeHttpRequest(
2668 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2669 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2670 "Auth Token: fake_token\n"
2671 "Timeouts: 5 1 10\n",
2672 "", errors::NotFound("404"), 404)});
2673 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2674 std::unique_ptr<HttpRequest::Factory>(
2675 new FakeHttpRequestFactory(&requests)),
2676 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2677 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2678 0 /* stat cache max age */, 0 /* stat cache max entries */,
2679 0 /* matching paths cache max age */,
2680 0 /* matching paths cache max entries */, kTestRetryConfig,
2681 kTestTimeoutConfig, *kAllowedLocationsDefault,
2682 nullptr /* gcs additional header */);
2683
2684 EXPECT_EQ(error::Code::NOT_FOUND,
2685 fs.IsDirectory("gs://bucket/file.txt").code());
2686 }
2687
TEST(GcsFileSystemTest,IsDirectory_NotDirectoryButObject)2688 TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) {
2689 std::vector<HttpRequest*> requests(
2690 {new FakeHttpRequest(
2691 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2692 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
2693 "&maxResults=1\n"
2694 "Auth Token: fake_token\n"
2695 "Timeouts: 5 1 10\n",
2696 "{}"),
2697 new FakeHttpRequest(
2698 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2699 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2700 "Auth Token: fake_token\n"
2701 "Timeouts: 5 1 10\n",
2702 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2703 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2704 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2705 std::unique_ptr<HttpRequest::Factory>(
2706 new FakeHttpRequestFactory(&requests)),
2707 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2708 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2709 0 /* stat cache max age */, 0 /* stat cache max entries */,
2710 0 /* matching paths cache max age */,
2711 0 /* matching paths cache max entries */, kTestRetryConfig,
2712 kTestTimeoutConfig, *kAllowedLocationsDefault,
2713 nullptr /* gcs additional header */);
2714
2715 EXPECT_EQ(error::Code::FAILED_PRECONDITION,
2716 fs.IsDirectory("gs://bucket/file.txt").code());
2717 }
2718
TEST(GcsFileSystemTest,IsDirectory_Yes)2719 TEST(GcsFileSystemTest, IsDirectory_Yes) {
2720 std::vector<HttpRequest*> requests(
2721 {new FakeHttpRequest(
2722 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2723 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
2724 "&maxResults=1\n"
2725 "Auth Token: fake_token\n"
2726 "Timeouts: 5 1 10\n",
2727 "{\"items\": [{\"name\": \"subfolder/\"}]}"),
2728 new FakeHttpRequest(
2729 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2730 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
2731 "&maxResults=1\n"
2732 "Auth Token: fake_token\n"
2733 "Timeouts: 5 1 10\n",
2734 "{\"items\": [{\"name\": \"subfolder/\"}]}")});
2735 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2736 std::unique_ptr<HttpRequest::Factory>(
2737 new FakeHttpRequestFactory(&requests)),
2738 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2739 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2740 0 /* stat cache max age */, 0 /* stat cache max entries */,
2741 0 /* matching paths cache max age */,
2742 0 /* matching paths cache max entries */, kTestRetryConfig,
2743 kTestTimeoutConfig, *kAllowedLocationsDefault,
2744 nullptr /* gcs additional header */);
2745
2746 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder"));
2747 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/"));
2748 }
2749
TEST(GcsFileSystemTest,IsDirectory_Bucket)2750 TEST(GcsFileSystemTest, IsDirectory_Bucket) {
2751 std::vector<HttpRequest*> requests(
2752 {new FakeHttpRequest(
2753 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2754 "Auth Token: fake_token\n"
2755 "Timeouts: 5 1 10\n",
2756 "{}"),
2757 new FakeHttpRequest(
2758 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2759 "Auth Token: fake_token\n"
2760 "Timeouts: 5 1 10\n",
2761 "{}")});
2762 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2763 std::unique_ptr<HttpRequest::Factory>(
2764 new FakeHttpRequestFactory(&requests)),
2765 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2766 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2767 0 /* stat cache max age */, 0 /* stat cache max entries */,
2768 0 /* matching paths cache max age */,
2769 0 /* matching paths cache max entries */, kTestRetryConfig,
2770 kTestTimeoutConfig, *kAllowedLocationsDefault,
2771 nullptr /* gcs additional header */);
2772
2773 TF_EXPECT_OK(fs.IsDirectory("gs://bucket"));
2774 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/"));
2775 }
2776
TEST(GcsFileSystemTest,IsDirectory_BucketNotFound)2777 TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) {
2778 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2779 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2780 "Auth Token: fake_token\n"
2781 "Timeouts: 5 1 10\n",
2782 "", errors::NotFound("404"), 404)});
2783 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2784 std::unique_ptr<HttpRequest::Factory>(
2785 new FakeHttpRequestFactory(&requests)),
2786 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2787 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2788 0 /* stat cache max age */, 0 /* stat cache max entries */,
2789 0 /* matching paths cache max age */,
2790 0 /* matching paths cache max entries */, kTestRetryConfig,
2791 kTestTimeoutConfig, *kAllowedLocationsDefault,
2792 nullptr /* gcs additional header */);
2793
2794 EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code());
2795 }
2796
TEST(GcsFileSystemTest,CreateDir_Folder)2797 TEST(GcsFileSystemTest, CreateDir_Folder) {
2798 std::vector<HttpRequest*> requests(
2799 {new FakeHttpRequest(
2800 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2801 "subpath%2F?fields=size%2Cgeneration%2Cupdated\n"
2802 "Auth Token: fake_token\n"
2803 "Timeouts: 5 1 10\n",
2804 "{}"),
2805 new FakeHttpRequest(
2806 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
2807 "uploadType=resumable&name=subpath%2F\n"
2808 "Auth Token: fake_token\n"
2809 "Header X-Upload-Content-Length: 0\n"
2810 "Post: yes\n"
2811 "Timeouts: 5 1 10\n",
2812 "", {{"Location", "https://custom/upload/location"}}),
2813 new FakeHttpRequest("Uri: https://custom/upload/location\n"
2814 "Auth Token: fake_token\n"
2815 "Timeouts: 5 1 30\n"
2816 "Put body: \n",
2817 ""),
2818 new FakeHttpRequest(
2819 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2820 "subpath%2F?fields=size%2Cgeneration%2Cupdated\n"
2821 "Auth Token: fake_token\n"
2822 "Timeouts: 5 1 10\n",
2823 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2824 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2825 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2826 std::unique_ptr<HttpRequest::Factory>(
2827 new FakeHttpRequestFactory(&requests)),
2828 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2829 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2830 0 /* stat cache max age */, 0 /* stat cache max entries */,
2831 0 /* matching paths cache max age */,
2832 0 /* matching paths cache max entries */, kTestRetryConfig,
2833 kTestTimeoutConfig, *kAllowedLocationsDefault,
2834 nullptr /* gcs additional header */);
2835
2836 TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath"));
2837 EXPECT_EQ(errors::AlreadyExists("gs://bucket/subpath/"),
2838 fs.CreateDir("gs://bucket/subpath/"));
2839 }
2840
TEST(GcsFileSystemTest,CreateDir_Bucket)2841 TEST(GcsFileSystemTest, CreateDir_Bucket) {
2842 std::vector<HttpRequest*> requests(
2843 {new FakeHttpRequest(
2844 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2845 "Auth Token: fake_token\n"
2846 "Timeouts: 5 1 10\n",
2847 ""),
2848 new FakeHttpRequest(
2849 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2850 "Auth Token: fake_token\n"
2851 "Timeouts: 5 1 10\n",
2852 "")});
2853 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2854 std::unique_ptr<HttpRequest::Factory>(
2855 new FakeHttpRequestFactory(&requests)),
2856 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2857 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2858 0 /* stat cache max age */, 0 /* stat cache max entries */,
2859 0 /* matching paths cache max age */,
2860 0 /* matching paths cache max entries */, kTestRetryConfig,
2861 kTestTimeoutConfig, *kAllowedLocationsDefault,
2862 nullptr /* gcs additional header */);
2863
2864 TF_EXPECT_OK(fs.CreateDir("gs://bucket/"));
2865 TF_EXPECT_OK(fs.CreateDir("gs://bucket"));
2866 }
2867
TEST(GcsFileSystemTest,DeleteRecursively_Ok)2868 TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
2869 std::vector<HttpRequest*> requests(
2870 {// IsDirectory is checking whether there are children objects.
2871 new FakeHttpRequest(
2872 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2873 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
2874 "&maxResults=1\n"
2875 "Auth Token: fake_token\n"
2876 "Timeouts: 5 1 10\n",
2877 "{\"items\": [ "
2878 " { \"name\": \"path/file1.txt\" }]}"),
2879 // GetChildren recursively.
2880 new FakeHttpRequest(
2881 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2882 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2883 "Auth Token: fake_token\n"
2884 "Timeouts: 5 1 10\n",
2885 "{\"items\": [ "
2886 " { \"name\": \"path/\" }," // The current directory's marker.
2887 " { \"name\": \"path/file1.txt\" },"
2888 " { \"name\": \"path/subpath/file2.txt\" },"
2889 " { \"name\": \"path/file3.txt\" }]}"),
2890 // Delete the current directory's marker.
2891 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2892 "/bucket/o/path%2F\n"
2893 "Auth Token: fake_token\n"
2894 "Timeouts: 5 1 10\n"
2895 "Delete: yes\n",
2896 ""),
2897 // Delete the object - fails and will be retried.
2898 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2899 "/bucket/o/path%2Ffile1.txt\n"
2900 "Auth Token: fake_token\n"
2901 "Timeouts: 5 1 10\n"
2902 "Delete: yes\n",
2903 "", errors::Unavailable("500"), 500),
2904 // Delete the object again.
2905 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2906 "/bucket/o/path%2Ffile1.txt\n"
2907 "Auth Token: fake_token\n"
2908 "Timeouts: 5 1 10\n"
2909 "Delete: yes\n",
2910 ""),
2911 // Delete the object.
2912 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2913 "/bucket/o/path%2Fsubpath%2Ffile2.txt\n"
2914 "Auth Token: fake_token\n"
2915 "Timeouts: 5 1 10\n"
2916 "Delete: yes\n",
2917 ""),
2918 // Delete the object.
2919 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2920 "/bucket/o/path%2Ffile3.txt\n"
2921 "Auth Token: fake_token\n"
2922 "Timeouts: 5 1 10\n"
2923 "Delete: yes\n",
2924 "")});
2925 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2926 std::unique_ptr<HttpRequest::Factory>(
2927 new FakeHttpRequestFactory(&requests)),
2928 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
2929 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
2930 0 /* stat cache max age */, 0 /* stat cache max entries */,
2931 0 /* matching paths cache max age */,
2932 0 /* matching paths cache max entries */, kTestRetryConfig,
2933 kTestTimeoutConfig, *kAllowedLocationsDefault,
2934 nullptr /* gcs additional header */);
2935
2936 int64 undeleted_files, undeleted_dirs;
2937 TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
2938 &undeleted_dirs));
2939 EXPECT_EQ(0, undeleted_files);
2940 EXPECT_EQ(0, undeleted_dirs);
2941 }
2942
TEST(GcsFileSystemTest,DeleteRecursively_DeletionErrors)2943 TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
2944 std::vector<HttpRequest*> requests(
2945 {// IsDirectory is checking whether there are children objects.
2946 new FakeHttpRequest(
2947 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2948 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
2949 "&maxResults=1\n"
2950 "Auth Token: fake_token\n"
2951 "Timeouts: 5 1 10\n",
2952 "{\"items\": [ "
2953 " { \"name\": \"path/file1.txt\" }]}"),
2954 // Calling GetChildren recursively.
2955 new FakeHttpRequest(
2956 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2957 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2958 "Auth Token: fake_token\n"
2959 "Timeouts: 5 1 10\n",
2960 "{\"items\": [ "
2961 " { \"name\": \"path/file1.txt\" },"
2962 " { \"name\": \"path/subpath/\" },"
2963 " { \"name\": \"path/subpath/file2.txt\" },"
2964 " { \"name\": \"path/file3.txt\" }]}"),
2965 // Deleting the object.
2966 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2967 "/bucket/o/path%2Ffile1.txt\n"
2968 "Auth Token: fake_token\n"
2969 "Timeouts: 5 1 10\n"
2970 "Delete: yes\n",
2971 ""),
2972 // Deleting the directory marker gs://bucket/path/ - fails with 404.
2973 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2974 "/bucket/o/path%2Fsubpath%2F\n"
2975 "Auth Token: fake_token\n"
2976 "Timeouts: 5 1 10\n"
2977 "Delete: yes\n",
2978 "", errors::NotFound("404"), 404),
2979 // Checking if gs://bucket/path/subpath/ is a folder - it is.
2980 new FakeHttpRequest(
2981 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2982 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F"
2983 "&maxResults=1\n"
2984 "Auth Token: fake_token\n"
2985 "Timeouts: 5 1 10\n",
2986 strings::StrCat("{\"items\": [ "
2987 " { \"name\": \"path/subpath/\" }]}")),
2988 // Deleting the object gs://bucket/path/subpath/file2.txt
2989 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2990 "/bucket/o/path%2Fsubpath%2Ffile2.txt\n"
2991 "Auth Token: fake_token\n"
2992 "Timeouts: 5 1 10\n"
2993 "Delete: yes\n",
2994 ""),
2995 // Deleting the object s://bucket/path/file3.txt - fails with 404.
2996 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2997 "/bucket/o/path%2Ffile3.txt\n"
2998 "Auth Token: fake_token\n"
2999 "Timeouts: 5 1 10\n"
3000 "Delete: yes\n",
3001 "", errors::NotFound("404"), 404),
3002 // Checking if gs://bucket/path/file3.txt/ is a folder - it's not.
3003 new FakeHttpRequest(
3004 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3005 "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile3.txt%2F"
3006 "&maxResults=1\n"
3007 "Auth Token: fake_token\n"
3008 "Timeouts: 5 1 10\n",
3009 "{}"),
3010 // Checking if gs://bucket/path/file3.txt is an object - fails with 404.
3011 new FakeHttpRequest(
3012 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3013 "path%2Ffile3.txt?fields=size%2Cgeneration%2Cupdated\n"
3014 "Auth Token: fake_token\n"
3015 "Timeouts: 5 1 10\n",
3016 "", errors::NotFound("404"), 404)});
3017
3018 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3019 std::unique_ptr<HttpRequest::Factory>(
3020 new FakeHttpRequestFactory(&requests)),
3021 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
3022 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
3023 0 /* stat cache max age */, 0 /* stat cache max entries */,
3024 0 /* matching paths cache max age */,
3025 0 /* matching paths cache max entries */, kTestRetryConfig,
3026 kTestTimeoutConfig, *kAllowedLocationsDefault,
3027 nullptr /* gcs additional header */);
3028
3029 int64 undeleted_files, undeleted_dirs;
3030 TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
3031 &undeleted_dirs));
3032 EXPECT_EQ(1, undeleted_files);
3033 EXPECT_EQ(1, undeleted_dirs);
3034 }
3035
TEST(GcsFileSystemTest,DeleteRecursively_NotAFolder)3036 TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) {
3037 std::vector<HttpRequest*> requests(
3038 {// IsDirectory is checking whether there are children objects.
3039 new FakeHttpRequest(
3040 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3041 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
3042 "&maxResults=1\n"
3043 "Auth Token: fake_token\n"
3044 "Timeouts: 5 1 10\n",
3045 "{}"),
3046 // IsDirectory is checking if the path exists as an object.
3047 new FakeHttpRequest(
3048 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3049 "path?fields=size%2Cgeneration%2Cupdated\n"
3050 "Auth Token: fake_token\n"
3051 "Timeouts: 5 1 10\n",
3052 "", errors::NotFound("404"), 404)});
3053 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3054 std::unique_ptr<HttpRequest::Factory>(
3055 new FakeHttpRequestFactory(&requests)),
3056 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
3057 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
3058 0 /* stat cache max age */, 0 /* stat cache max entries */,
3059 0 /* matching paths cache max age */,
3060 0 /* matching paths cache max entries */, kTestRetryConfig,
3061 kTestTimeoutConfig, *kAllowedLocationsDefault,
3062 nullptr /* gcs additional header */);
3063
3064 int64 undeleted_files, undeleted_dirs;
3065 EXPECT_EQ(error::Code::NOT_FOUND,
3066 fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
3067 &undeleted_dirs)
3068 .code());
3069 EXPECT_EQ(0, undeleted_files);
3070 EXPECT_EQ(1, undeleted_dirs);
3071 }
3072
TEST(GcsFileSystemTest,NoConstraintsEnvironmentVariableTest)3073 TEST(GcsFileSystemTest, NoConstraintsEnvironmentVariableTest) {
3074 unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS");
3075 // No constraints
3076 GcsFileSystem fs1;
3077 EXPECT_EQ(*kAllowedLocationsDefault, fs1.allowed_locations());
3078
3079 // Cover cache initialization code, any uninitialized cache will cause this to
3080 // fail
3081 fs1.FlushCaches();
3082 }
3083
TEST(GcsFileSystemTest,BucketLocationConstraintEnvironmentVariableTest)3084 TEST(GcsFileSystemTest, BucketLocationConstraintEnvironmentVariableTest) {
3085 unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS");
3086 setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "auto", 1);
3087 GcsFileSystem fs1;
3088 EXPECT_EQ(*kAllowedLocationsAuto, fs1.allowed_locations());
3089
3090 setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "CUSTOM,list", 1);
3091 GcsFileSystem fs2;
3092 EXPECT_EQ(std::unordered_set<string>({"custom", "list"}),
3093 fs2.allowed_locations());
3094 }
3095
TEST(GcsFileSystemTest,AdditionalRequestHeaderTest)3096 TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) {
3097 GcsFileSystem fs1;
3098 EXPECT_EQ("", fs1.additional_header_name());
3099 EXPECT_EQ("", fs1.additional_header_value());
3100
3101 setenv("GCS_ADDITIONAL_REQUEST_HEADER",
3102 "X-Add-Header:My Additional Header Value", 1);
3103 GcsFileSystem fs2;
3104 EXPECT_EQ("X-Add-Header", fs2.additional_header_name());
3105 EXPECT_EQ("My Additional Header Value", fs2.additional_header_value());
3106
3107 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "Someinvalidheadervalue", 1);
3108 GcsFileSystem fs3;
3109 EXPECT_EQ("", fs3.additional_header_name());
3110 EXPECT_EQ("", fs3.additional_header_value());
3111
3112 setenv("GCS_ADDITIONAL_REQUEST_HEADER", ":thisisinvalid", 1);
3113 GcsFileSystem fs4;
3114 EXPECT_EQ("", fs4.additional_header_name());
3115 EXPECT_EQ("", fs4.additional_header_value());
3116
3117 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "soisthis:", 1);
3118 GcsFileSystem fs5;
3119 EXPECT_EQ("", fs5.additional_header_name());
3120 EXPECT_EQ("", fs5.additional_header_value());
3121
3122 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "a:b", 1);
3123 GcsFileSystem fs6;
3124 EXPECT_EQ("a", fs6.additional_header_name());
3125 EXPECT_EQ("b", fs6.additional_header_value());
3126
3127 auto* add_header = new std::pair<const string, const string>(
3128 "mynewheader", "newheadercontents");
3129
3130 std::vector<HttpRequest*> requests(
3131 {// IsDirectory is checking whether there are children objects.
3132 new FakeHttpRequest("Uri: https://www.googleapis.com/fake\n"
3133 "Auth Token: fake_token\n"
3134 "Header mynewheader: newheadercontents\n"
3135 "Header Hello: world\n",
3136 "{}")});
3137 GcsFileSystem fs7(
3138 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3139 std::unique_ptr<HttpRequest::Factory>(
3140 new FakeHttpRequestFactory(&requests)),
3141 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3142 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3143 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3144 0 /* matching paths cache max entries */, kTestRetryConfig,
3145 kTestTimeoutConfig, *kAllowedLocationsDefault,
3146 add_header /* gcs additional header */);
3147
3148 std::unique_ptr<HttpRequest> request;
3149 TF_EXPECT_OK(fs7.CreateHttpRequest(&request));
3150 request->SetUri("https://www.googleapis.com/fake");
3151 request->AddHeader("Hello", "world");
3152 TF_EXPECT_OK(request->Send());
3153 }
3154
TEST(GcsFileSystemTest,OverrideCacheParameters)3155 TEST(GcsFileSystemTest, OverrideCacheParameters) {
3156 // Verify defaults are propagated correctly.
3157 GcsFileSystem fs1;
3158 EXPECT_EQ(16 * 1024 * 1024, fs1.block_size());
3159 EXPECT_EQ(fs1.block_size(), fs1.max_bytes());
3160 EXPECT_EQ(0, fs1.max_staleness());
3161 EXPECT_EQ(120, fs1.timeouts().connect);
3162 EXPECT_EQ(60, fs1.timeouts().idle);
3163 EXPECT_EQ(3600, fs1.timeouts().metadata);
3164 EXPECT_EQ(3600, fs1.timeouts().read);
3165 EXPECT_EQ(3600, fs1.timeouts().write);
3166
3167 // Verify legacy readahead buffer override sets block size.
3168 setenv("GCS_READAHEAD_BUFFER_SIZE_BYTES", "123456789", 1);
3169 GcsFileSystem fs2;
3170 EXPECT_EQ(123456789L, fs2.block_size());
3171
3172 // Verify block size, max size, and max staleness overrides.
3173 setenv("GCS_READ_CACHE_BLOCK_SIZE_MB", "1", 1);
3174 setenv("GCS_READ_CACHE_MAX_SIZE_MB", "16", 1);
3175 setenv("GCS_READ_CACHE_MAX_STALENESS", "60", 1);
3176 GcsFileSystem fs3;
3177 EXPECT_EQ(1048576L, fs3.block_size());
3178 EXPECT_EQ(16 * 1024 * 1024, fs3.max_bytes());
3179 EXPECT_EQ(60, fs3.max_staleness());
3180
3181 // Verify StatCache and MatchingPathsCache overrides.
3182 setenv("GCS_STAT_CACHE_MAX_AGE", "60", 1);
3183 setenv("GCS_STAT_CACHE_MAX_ENTRIES", "32", 1);
3184 setenv("GCS_MATCHING_PATHS_CACHE_MAX_AGE", "30", 1);
3185 setenv("GCS_MATCHING_PATHS_CACHE_MAX_ENTRIES", "64", 1);
3186 GcsFileSystem fs4;
3187 EXPECT_EQ(60, fs4.stat_cache_max_age());
3188 EXPECT_EQ(32, fs4.stat_cache_max_entries());
3189 EXPECT_EQ(30, fs4.matching_paths_cache_max_age());
3190 EXPECT_EQ(64, fs4.matching_paths_cache_max_entries());
3191
3192 // Verify timeout overrides.
3193 setenv("GCS_REQUEST_CONNECTION_TIMEOUT_SECS", "10", 1);
3194 setenv("GCS_REQUEST_IDLE_TIMEOUT_SECS", "5", 1);
3195 setenv("GCS_METADATA_REQUEST_TIMEOUT_SECS", "20", 1);
3196 setenv("GCS_READ_REQUEST_TIMEOUT_SECS", "30", 1);
3197 setenv("GCS_WRITE_REQUEST_TIMEOUT_SECS", "40", 1);
3198 GcsFileSystem fs5;
3199 EXPECT_EQ(10, fs5.timeouts().connect);
3200 EXPECT_EQ(5, fs5.timeouts().idle);
3201 EXPECT_EQ(20, fs5.timeouts().metadata);
3202 EXPECT_EQ(30, fs5.timeouts().read);
3203 EXPECT_EQ(40, fs5.timeouts().write);
3204 }
3205
TEST(GcsFileSystemTest,CreateHttpRequest)3206 TEST(GcsFileSystemTest, CreateHttpRequest) {
3207 std::vector<HttpRequest*> requests(
3208 {// IsDirectory is checking whether there are children objects.
3209 new FakeHttpRequest("Uri: https://www.googleapis.com/fake\n"
3210 "Auth Token: fake_token\n"
3211 "Header Hello: world\n",
3212 "{}")});
3213 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3214 std::unique_ptr<HttpRequest::Factory>(
3215 new FakeHttpRequestFactory(&requests)),
3216 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
3217 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
3218 0 /* stat cache max age */, 0 /* stat cache max entries */,
3219 0 /* matching paths cache max age */,
3220 0 /* matching paths cache max entries */, kTestRetryConfig,
3221 kTestTimeoutConfig, *kAllowedLocationsDefault,
3222 nullptr /* gcs additional header */);
3223
3224 std::unique_ptr<HttpRequest> request;
3225 TF_EXPECT_OK(fs.CreateHttpRequest(&request));
3226 request->SetUri("https://www.googleapis.com/fake");
3227 request->AddHeader("Hello", "world");
3228 TF_EXPECT_OK(request->Send());
3229 }
3230
3231 class TestGcsStats : public GcsStatsInterface {
3232 public:
Configure(GcsFileSystem * fs,GcsThrottle * throttle,const FileBlockCache * block_cache)3233 void Configure(GcsFileSystem* fs, GcsThrottle* throttle,
3234 const FileBlockCache* block_cache) override {
3235 CHECK(fs_ == nullptr);
3236 CHECK(throttle_ == nullptr);
3237 CHECK(block_cache_ == nullptr);
3238
3239 fs_ = fs;
3240 throttle_ = throttle;
3241 block_cache_ = block_cache;
3242 }
3243
RecordBlockLoadRequest(const string & file,size_t offset)3244 void RecordBlockLoadRequest(const string& file, size_t offset) override {
3245 block_load_request_file_ = file;
3246 }
3247
RecordBlockRetrieved(const string & file,size_t offset,size_t bytes_transferred)3248 void RecordBlockRetrieved(const string& file, size_t offset,
3249 size_t bytes_transferred) override {
3250 block_retrieved_file_ = file;
3251 block_retrieved_bytes_transferred_ = bytes_transferred;
3252 }
3253
RecordStatObjectRequest()3254 void RecordStatObjectRequest() override { stat_object_request_count_++; }
3255
HttpStats()3256 HttpRequest::RequestStats* HttpStats() override { return nullptr; }
3257
3258 GcsFileSystem* fs_ = nullptr;
3259 GcsThrottle* throttle_ = nullptr;
3260 const FileBlockCache* block_cache_ = nullptr;
3261
3262 string block_load_request_file_;
3263 string block_retrieved_file_;
3264 size_t block_retrieved_bytes_transferred_ = 0;
3265 int stat_object_request_count_ = 0;
3266 };
3267
TEST(GcsFileSystemTest,Stat_StatsRecording)3268 TEST(GcsFileSystemTest, Stat_StatsRecording) {
3269 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3270 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3271 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3272 "Auth Token: fake_token\n"
3273 "Timeouts: 5 1 10\n",
3274 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3275 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3276 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3277 std::unique_ptr<HttpRequest::Factory>(
3278 new FakeHttpRequestFactory(&requests)),
3279 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
3280 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
3281 0 /* stat cache max age */, 0 /* stat cache max entries */,
3282 0 /* matching paths cache max age */,
3283 0 /* matching paths cache max entries */, kTestRetryConfig,
3284 kTestTimeoutConfig, *kAllowedLocationsDefault,
3285 nullptr /* gcs additional header */);
3286
3287 TestGcsStats stats;
3288 fs.SetStats(&stats);
3289 EXPECT_EQ(stats.fs_, &fs);
3290
3291 FileStatistics stat;
3292 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
3293 EXPECT_EQ(1, stats.stat_object_request_count_);
3294 }
3295
TEST(GcsFileSystemTest,NewRandomAccessFile_StatsRecording)3296 TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) {
3297 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3298 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
3299 "Auth Token: fake_token\n"
3300 "Range: 0-5\n"
3301 "Timeouts: 5 1 20\n",
3302 "012345")});
3303 GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3304 std::unique_ptr<HttpRequest::Factory>(
3305 new FakeHttpRequestFactory(&requests)),
3306 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
3307 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
3308 0 /* stat cache max age */, 0 /* stat cache max entries */,
3309 0 /* matching paths cache max age */,
3310 0 /* matching paths cache max entries */, kTestRetryConfig,
3311 kTestTimeoutConfig, *kAllowedLocationsDefault,
3312 nullptr /* gcs additional header */);
3313
3314 TestGcsStats stats;
3315 fs.SetStats(&stats);
3316 EXPECT_EQ(stats.fs_, &fs);
3317
3318 std::unique_ptr<RandomAccessFile> file;
3319 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
3320
3321 char scratch[6];
3322 StringPiece result;
3323
3324 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
3325 EXPECT_EQ("012345", result);
3326
3327 EXPECT_EQ("gs://bucket/random_access.txt", stats.block_load_request_file_);
3328 EXPECT_EQ("gs://bucket/random_access.txt", stats.block_retrieved_file_);
3329 EXPECT_EQ(6, stats.block_retrieved_bytes_transferred_);
3330 }
3331
3332 } // namespace
3333 } // namespace tensorflow
3334