1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/platform/cloud/gcs_file_system.h"
17
18 #include <fstream>
19
20 #include "tensorflow/core/lib/core/errors.h"
21 #include "tensorflow/core/lib/core/status_test_util.h"
22 #include "tensorflow/core/platform/cloud/http_request_fake.h"
23 #include "tensorflow/core/platform/errors.h"
24 #include "tensorflow/core/platform/str_util.h"
25 #include "tensorflow/core/platform/strcat.h"
26 #include "tensorflow/core/platform/test.h"
27
28 // Undef DeleteFile macro defined in wndows.h.
29 #ifdef PLATFORM_WINDOWS
30 #undef DeleteFile
31 #endif
32
33 namespace tensorflow {
34 namespace {
35
36 static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30);
37 static RetryConfig kTestRetryConfig(0 /* init_delay_time_us */);
38
39 // Default (empty) constraint config
40 static std::unordered_set<string>* kAllowedLocationsDefault =
41 new std::unordered_set<string>();
42 // Constraint config if bucket location constraint is turned on, with no
43 // custom list
44 static std::unordered_set<string>* kAllowedLocationsAuto =
45 new std::unordered_set<string>({"auto"});
46
47 class FakeAuthProvider : public AuthProvider {
48 public:
GetToken(string * token)49 Status GetToken(string* token) override {
50 *token = "fake_token";
51 return Status::OK();
52 }
53 };
54
55 class FakeZoneProvider : public ZoneProvider {
56 public:
GetZone(string * zone)57 Status GetZone(string* zone) override {
58 *zone = "us-east1-b";
59 return Status::OK();
60 }
61 };
62
TEST(GcsFileSystemTest,NewRandomAccessFile_NoBlockCache)63 TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
64 std::vector<HttpRequest*> requests(
65 {new FakeHttpRequest(
66 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
67 "Auth Token: fake_token\n"
68 "Range: 0-5\n"
69 "Timeouts: 5 1 20\n",
70 "012345"),
71 new FakeHttpRequest(
72 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
73 "Auth Token: fake_token\n"
74 "Range: 6-11\n"
75 "Timeouts: 5 1 20\n",
76 "6789")});
77 GcsFileSystem fs(
78 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
79 std::unique_ptr<HttpRequest::Factory>(
80 new FakeHttpRequestFactory(&requests)),
81 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
82 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
83 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
84 0 /* matching paths cache max entries */, kTestRetryConfig,
85 kTestTimeoutConfig, *kAllowedLocationsDefault,
86 nullptr /* gcs additional header */, false /* compose append */);
87
88 std::unique_ptr<RandomAccessFile> file;
89 TF_EXPECT_OK(
90 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
91
92 StringPiece filename;
93 TF_EXPECT_OK(file->Name(&filename));
94 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
95
96 char scratch[6];
97 StringPiece result;
98
99 // Read the first chunk.
100 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
101 EXPECT_EQ("012345", result);
102
103 // Read the second chunk.
104 EXPECT_EQ(
105 errors::Code::OUT_OF_RANGE,
106 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch).code());
107 EXPECT_EQ("6789", result);
108 }
109
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered)110 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered) {
111 std::vector<HttpRequest*> requests({
112 new FakeHttpRequest(
113 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
114 "Auth Token: fake_token\n"
115 "Range: 0-9\n"
116 "Timeouts: 5 1 20\n",
117 "0123456789"),
118 new FakeHttpRequest(
119 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
120 "Auth Token: fake_token\n"
121 "Range: 10-19\n"
122 "Timeouts: 5 1 20\n",
123 ""),
124 });
125 GcsFileSystem fs(
126 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
127 std::unique_ptr<HttpRequest::Factory>(
128 new FakeHttpRequestFactory(&requests)),
129 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
130 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
131 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
132 0 /* matching paths cache max entries */, kTestRetryConfig,
133 kTestTimeoutConfig, *kAllowedLocationsDefault,
134 nullptr /* gcs additional header */, false /* compose append */);
135
136 std::unique_ptr<RandomAccessFile> file;
137 TF_EXPECT_OK(
138 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
139
140 StringPiece filename;
141 TF_EXPECT_OK(file->Name(&filename));
142 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
143
144 char scratch[6];
145 StringPiece result;
146
147 // Read the first chunk.
148 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
149 EXPECT_EQ("012345", result);
150
151 // Read the second chunk.
152 EXPECT_EQ(
153 errors::Code::OUT_OF_RANGE,
154 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch).code());
155 EXPECT_EQ("6789", result);
156 }
157
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_Errors)158 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_Errors) {
159 std::vector<HttpRequest*> requests({
160 new FakeHttpRequest(
161 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
162 "Auth Token: fake_token\n"
163 "Range: 0-9\n"
164 "Timeouts: 5 1 20\n",
165 "Server Not", errors::Unavailable("important HTTP error 308"),
166 nullptr, {}, 308),
167 new FakeHttpRequest(
168 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
169 "Auth Token: fake_token\n"
170 "Range: 6-15\n"
171 "Timeouts: 5 1 20\n",
172 "123"),
173 });
174 GcsFileSystem fs(
175 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
176 std::unique_ptr<HttpRequest::Factory>(
177 new FakeHttpRequestFactory(&requests)),
178 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
179 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
180 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
181 0 /* matching paths cache max entries */, kTestRetryConfig,
182 kTestTimeoutConfig, *kAllowedLocationsDefault,
183 nullptr /* gcs additional header */, false /* compose append */);
184
185 std::unique_ptr<RandomAccessFile> file;
186 TF_EXPECT_OK(
187 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
188
189 StringPiece filename;
190 TF_EXPECT_OK(file->Name(&filename));
191 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
192
193 char scratch[6];
194 StringPiece result;
195
196 // Read the first chunk.
197 EXPECT_EQ(errors::Code::UNAVAILABLE,
198 file->Read(0, sizeof(scratch), &result, scratch).code());
199 EXPECT_EQ("", result);
200
201 // Read the second chunk.
202 EXPECT_EQ(
203 errors::Code::OUT_OF_RANGE,
204 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch).code());
205 EXPECT_EQ("123", result);
206 }
207
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_ReadAtEOF)208 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_ReadAtEOF) {
209 std::vector<HttpRequest*> requests(
210 {new FakeHttpRequest(
211 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
212 "Auth Token: fake_token\n"
213 "Range: 0-9\n"
214 "Timeouts: 5 1 20\n",
215 "0123456789"),
216 new FakeHttpRequest(
217 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
218 "Auth Token: fake_token\n"
219 "Range: 10-19\n"
220 "Timeouts: 5 1 20\n",
221 "")});
222 GcsFileSystem fs(
223 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
224 std::unique_ptr<HttpRequest::Factory>(
225 new FakeHttpRequestFactory(&requests)),
226 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
227 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
228 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
229 0 /* matching paths cache max entries */, kTestRetryConfig,
230 kTestTimeoutConfig, *kAllowedLocationsDefault,
231 nullptr /* gcs additional header */, false /* compose append */);
232
233 std::unique_ptr<RandomAccessFile> file;
234 TF_EXPECT_OK(
235 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
236
237 StringPiece filename;
238 TF_EXPECT_OK(file->Name(&filename));
239 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
240
241 char scratch[10];
242 StringPiece result;
243
244 // Read the first chunk.
245 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
246 EXPECT_EQ("0123456789", result);
247
248 // Read the second chunk.
249 EXPECT_EQ(
250 errors::Code::OUT_OF_RANGE,
251 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch).code());
252 EXPECT_EQ("", result);
253 }
254
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_CachedOutOfRange)255 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_CachedOutOfRange) {
256 // In this test, there is only one backend request since we cache the file
257 // size.
258 std::vector<HttpRequest*> requests({new FakeHttpRequest(
259 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
260 "Auth Token: fake_token\n"
261 "Range: 0-9\n"
262 "Timeouts: 5 1 20\n",
263 "012345678")});
264 GcsFileSystem fs(
265 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
266 std::unique_ptr<HttpRequest::Factory>(
267 new FakeHttpRequestFactory(&requests)),
268 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
269 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
270 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
271 0 /* matching paths cache max entries */, kTestRetryConfig,
272 kTestTimeoutConfig, *kAllowedLocationsDefault,
273 nullptr /* gcs additional header */, false /* compose append */);
274
275 std::unique_ptr<RandomAccessFile> file;
276 TF_EXPECT_OK(
277 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
278
279 StringPiece filename;
280 TF_EXPECT_OK(file->Name(&filename));
281 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
282
283 char scratch[5];
284 StringPiece result;
285
286 // Read the first chunk. Even though the backend response is out-of-range,
287 // we should get a OK status since we're just reading the first 5 bytes.
288 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
289 EXPECT_EQ("01234", result);
290
291 TF_EXPECT_OK(file->Read(4, sizeof(scratch), &result, scratch));
292 EXPECT_EQ("45678", result);
293
294 // Return the cached error once the user starts reading out of range.
295 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
296 file->Read(5, sizeof(scratch), &result, scratch).code());
297 EXPECT_EQ("5678", result);
298 }
299
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_CachedNotSequential)300 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_CachedNotSequential) {
301 // In this test, the second read is seeking backwards, so it should trigger
302 // a backend request.
303 std::vector<HttpRequest*> requests(
304 {new FakeHttpRequest(
305 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
306 "Auth Token: fake_token\n"
307 "Range: 1-10\n"
308 "Timeouts: 5 1 20\n",
309 "12345678"),
310 new FakeHttpRequest(
311 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
312 "Auth Token: fake_token\n"
313 "Range: 0-9\n"
314 "Timeouts: 5 1 20\n",
315 "012345678")});
316 GcsFileSystem fs(
317 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
318 std::unique_ptr<HttpRequest::Factory>(
319 new FakeHttpRequestFactory(&requests)),
320 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
321 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
322 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
323 0 /* matching paths cache max entries */, kTestRetryConfig,
324 kTestTimeoutConfig, *kAllowedLocationsDefault,
325 nullptr /* gcs additional header */, false /* compose append */);
326
327 std::unique_ptr<RandomAccessFile> file;
328 TF_EXPECT_OK(
329 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
330
331 StringPiece filename;
332 TF_EXPECT_OK(file->Name(&filename));
333 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
334
335 char scratch[5];
336 StringPiece result;
337
338 TF_EXPECT_OK(file->Read(1, sizeof(scratch), &result, scratch));
339 EXPECT_EQ("12345", result);
340 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
341 EXPECT_EQ("01234", result);
342 }
343
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_Growing)344 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_Growing) {
345 std::vector<HttpRequest*> requests(
346 {new FakeHttpRequest(
347 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
348 "Auth Token: fake_token\n"
349 "Range: 0-9\n"
350 "Timeouts: 5 1 20\n",
351 "012345678"),
352 new FakeHttpRequest(
353 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
354 "Auth Token: fake_token\n"
355 "Range: 9-18\n"
356 "Timeouts: 5 1 20\n",
357 "9")});
358 GcsFileSystem fs(
359 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
360 std::unique_ptr<HttpRequest::Factory>(
361 new FakeHttpRequestFactory(&requests)),
362 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
363 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
364 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
365 0 /* matching paths cache max entries */, kTestRetryConfig,
366 kTestTimeoutConfig, *kAllowedLocationsDefault,
367 nullptr /* gcs additional header */, false /* compose append */);
368
369 std::unique_ptr<RandomAccessFile> file;
370 TF_EXPECT_OK(
371 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
372
373 StringPiece filename;
374 TF_EXPECT_OK(file->Name(&filename));
375 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
376
377 char scratch[10];
378 StringPiece result;
379
380 // Read the first chunk. Since the first read is out-of-range,
381 // we don't cache the out-of-range flag and each subsequent read triggers a
382 // backend call.
383 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
384 file->Read(0, sizeof(scratch), &result, scratch).code());
385 EXPECT_EQ("012345678", result);
386
387 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
388 EXPECT_EQ("0123456789", result);
389 }
390
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_ReadBackwards)391 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_ReadBackwards) {
392 // Go backwards in the file. It should trigger a new read.
393 std::vector<HttpRequest*> requests(
394 {new FakeHttpRequest(
395 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
396 "Auth Token: fake_token\n"
397 "Range: 5-14\n"
398 "Timeouts: 5 1 20\n",
399 "56789"),
400 new FakeHttpRequest(
401 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
402 "Auth Token: fake_token\n"
403 "Range: 0-9\n"
404 "Timeouts: 5 1 20\n",
405 "0123456789")});
406 GcsFileSystem fs(
407 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
408 std::unique_ptr<HttpRequest::Factory>(
409 new FakeHttpRequestFactory(&requests)),
410 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
411 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
412 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
413 0 /* matching paths cache max entries */, kTestRetryConfig,
414 kTestTimeoutConfig, *kAllowedLocationsDefault,
415 nullptr /* gcs additional header */, false /* compose append */);
416
417 std::unique_ptr<RandomAccessFile> file;
418 TF_EXPECT_OK(
419 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
420
421 StringPiece filename;
422 TF_EXPECT_OK(file->Name(&filename));
423 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
424
425 char scratch[10];
426 StringPiece result;
427
428 // Read the first chunk.
429 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
430 file->Read(5, sizeof(scratch), &result, scratch).code());
431 EXPECT_EQ("56789", result);
432
433 // Go back and read from the beginning of the file.
434 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
435 EXPECT_EQ("0123456789", result);
436 }
437
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintInSameLocation)438 TEST(GcsFileSystemTest,
439 NewRandomAccessFile_WithLocationConstraintInSameLocation) {
440 std::vector<HttpRequest*> requests({new FakeHttpRequest(
441 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
442 "Auth Token: fake_token\n"
443 "Timeouts: 5 1 10\n",
444 R"(
445 {
446 "location":"US-EAST1"
447 })")});
448
449 GcsFileSystem fs(
450 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
451 std::unique_ptr<HttpRequest::Factory>(
452 new FakeHttpRequestFactory(&requests)),
453 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
454 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
455 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
456 0 /* matching paths cache max entries */, kTestRetryConfig,
457 kTestTimeoutConfig, *kAllowedLocationsAuto,
458 nullptr /* gcs additional header */, false /* compose append */);
459
460 std::unique_ptr<RandomAccessFile> file;
461 TF_EXPECT_OK(
462 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
463 }
464
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintCaching)465 TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) {
466 std::vector<HttpRequest*> requests(
467 {new FakeHttpRequest(
468 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
469 "Auth Token: fake_token\n"
470 "Timeouts: 5 1 10\n",
471 R"(
472 {
473 "location":"US-EAST1"
474 })"),
475 new FakeHttpRequest(
476 "Uri: https://www.googleapis.com/storage/v1/b/anotherbucket\n"
477 "Auth Token: fake_token\n"
478 "Timeouts: 5 1 10\n",
479 R"(
480 {
481 "location":"US-EAST1"
482 })"),
483 new FakeHttpRequest(
484 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
485 "Auth Token: fake_token\n"
486 "Timeouts: 5 1 10\n",
487 R"(
488 {
489 "location":"US-EAST1"
490 })")});
491
492 GcsFileSystem fs(
493 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
494 std::unique_ptr<HttpRequest::Factory>(
495 new FakeHttpRequestFactory(&requests)),
496 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
497 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
498 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
499 0 /* matching paths cache max entries */, kTestRetryConfig,
500 kTestTimeoutConfig, *kAllowedLocationsAuto,
501 nullptr /* gcs additional header */, false /* compose append */);
502
503 std::unique_ptr<RandomAccessFile> file;
504
505 string bucket = "gs://bucket/random_access.txt";
506 string another_bucket = "gs://anotherbucket/random_access.txt";
507 // Multiple calls should only cause one request to the location API.
508 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file));
509 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file));
510
511 // A new bucket should have one cache miss
512 TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, nullptr, &file));
513 // And then future calls to both should be cached
514 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file));
515 TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, nullptr, &file));
516
517 // Trigger a flush, should then require one more call
518 fs.FlushCaches(nullptr);
519 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file));
520 }
521
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintInDifferentLocation)522 TEST(GcsFileSystemTest,
523 NewRandomAccessFile_WithLocationConstraintInDifferentLocation) {
524 std::vector<HttpRequest*> requests({new FakeHttpRequest(
525 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
526 "Auth Token: fake_token\n"
527 "Timeouts: 5 1 10\n",
528 R"(
529 {
530 "location":"BARFOO"
531 })")});
532
533 GcsFileSystem fs(
534 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
535 std::unique_ptr<HttpRequest::Factory>(
536 new FakeHttpRequestFactory(&requests)),
537 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
538 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
539 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
540 0 /* matching paths cache max entries */, kTestRetryConfig,
541 kTestTimeoutConfig, *kAllowedLocationsAuto,
542 nullptr /* gcs additional header */, false /* compose append */);
543
544 std::unique_ptr<RandomAccessFile> file;
545 EXPECT_EQ(
546 tensorflow::errors::FailedPrecondition(
547 "Bucket 'bucket' is in 'barfoo' location, allowed locations "
548 "are: (us-east1)."),
549 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
550 }
551
TEST(GcsFileSystemTest,NewRandomAccessFile_NoBlockCache_DifferentN)552 TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) {
553 std::vector<HttpRequest*> requests(
554 {new FakeHttpRequest(
555 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
556 "Auth Token: fake_token\n"
557 "Range: 0-2\n"
558 "Timeouts: 5 1 20\n",
559 "012"),
560 new FakeHttpRequest(
561 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
562 "Auth Token: fake_token\n"
563 "Range: 3-12\n"
564 "Timeouts: 5 1 20\n",
565 "3456789")});
566 GcsFileSystem fs(
567 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
568 std::unique_ptr<HttpRequest::Factory>(
569 new FakeHttpRequestFactory(&requests)),
570 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
571 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
572 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
573 0 /* matching paths cache max entries */, kTestRetryConfig,
574 kTestTimeoutConfig, *kAllowedLocationsDefault,
575 nullptr /* gcs additional header */, false /* compose append */);
576
577 std::unique_ptr<RandomAccessFile> file;
578 TF_EXPECT_OK(
579 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
580
581 char small_scratch[3];
582 StringPiece result;
583
584 // Read the first chunk.
585 TF_EXPECT_OK(file->Read(0, sizeof(small_scratch), &result, small_scratch));
586 EXPECT_EQ("012", result);
587
588 // Read the second chunk that is larger. Requires allocation of new buffer.
589 char large_scratch[10];
590
591 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
592 file->Read(sizeof(small_scratch), sizeof(large_scratch), &result,
593 large_scratch)
594 .code());
595 EXPECT_EQ("3456789", result);
596 }
597
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache)598 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) {
599 // Our underlying file in this test is a 15 byte file with contents
600 // "0123456789abcde".
601 std::vector<HttpRequest*> requests(
602 {new FakeHttpRequest(
603 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
604 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
605 "Auth Token: fake_token\n"
606 "Timeouts: 5 1 10\n",
607 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
608 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
609 new FakeHttpRequest(
610 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
611 "Auth Token: fake_token\n"
612 "Range: 0-8\n"
613 "Timeouts: 5 1 20\n",
614 "012345678"),
615 new FakeHttpRequest(
616 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
617 "Auth Token: fake_token\n"
618 "Range: 9-17\n"
619 "Timeouts: 5 1 20\n",
620 "9abcde"),
621 new FakeHttpRequest(
622 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
623 "Auth Token: fake_token\n"
624 "Range: 18-26\n"
625 "Timeouts: 5 1 20\n",
626 "")});
627 GcsFileSystem fs(
628 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
629 std::unique_ptr<HttpRequest::Factory>(
630 new FakeHttpRequestFactory(&requests)),
631 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
632 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
633 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
634 0 /* matching paths cache max entries */, kTestRetryConfig,
635 kTestTimeoutConfig, *kAllowedLocationsDefault,
636 nullptr /* gcs additional header */, false /* compose append */);
637
638 char scratch[100];
639 StringPiece result;
640 {
641 // We are instantiating this in an enclosed scope to make sure after the
642 // unique ptr goes out of scope, we can still access result.
643 std::unique_ptr<RandomAccessFile> file;
644 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt",
645 nullptr, &file));
646
647 // Read the first chunk. The cache will be populated with the first block of
648 // 9 bytes.
649 scratch[5] = 'x';
650 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
651 EXPECT_EQ("0123", result);
652 EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes.
653
654 // The second chunk will be fully loaded from the cache, no requests are
655 // made.
656 TF_EXPECT_OK(file->Read(4, 4, &result, scratch));
657 EXPECT_EQ("4567", result);
658
659 // The chunk is only partially cached -- the request will be made to fetch
660 // the next block. 9 bytes will be requested, starting at offset 9.
661 TF_EXPECT_OK(file->Read(6, 5, &result, scratch));
662 EXPECT_EQ("6789a", result);
663
664 // The range can only be partially satisfied, as the second block contains
665 // only 6 bytes for a total of 9 + 6 = 15 bytes in the file.
666 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
667 file->Read(6, 10, &result, scratch).code());
668 EXPECT_EQ("6789abcde", result);
669
670 // The range cannot be satisfied, and the requested offset is past the end
671 // of the cache. A new request will be made to read 9 bytes starting at
672 // offset 18. This request will return an empty response, and there will not
673 // be another request.
674 EXPECT_EQ(errors::Code::OUT_OF_RANGE,
675 file->Read(20, 10, &result, scratch).code());
676 EXPECT_TRUE(result.empty());
677
678 // The beginning of the file should still be in the LRU cache. There should
679 // not be another request. The buffer size is still 15.
680 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
681 }
682
683 EXPECT_EQ("0123", result);
684 }
685
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_Flush)686 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) {
687 // Our underlying file in this test is a 15 byte file with contents
688 // "0123456789abcde".
689 std::vector<HttpRequest*> requests(
690 {new FakeHttpRequest(
691 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
692 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
693 "Auth Token: fake_token\n"
694 "Timeouts: 5 1 10\n",
695 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
696 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
697 new FakeHttpRequest(
698 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
699 "Auth Token: fake_token\n"
700 "Range: 0-8\n"
701 "Timeouts: 5 1 20\n",
702 "012345678"),
703 new FakeHttpRequest(
704 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
705 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
706 "Auth Token: fake_token\n"
707 "Timeouts: 5 1 10\n",
708 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
709 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
710 new FakeHttpRequest(
711 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
712 "Auth Token: fake_token\n"
713 "Range: 0-8\n"
714 "Timeouts: 5 1 20\n",
715 "012345678")});
716 GcsFileSystem fs(
717 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
718 std::unique_ptr<HttpRequest::Factory>(
719 new FakeHttpRequestFactory(&requests)),
720 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
721 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
722 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
723 0 /* matching paths cache max entries */, kTestRetryConfig,
724 kTestTimeoutConfig, *kAllowedLocationsDefault,
725 nullptr /* gcs additional header */, false /* compose append */);
726
727 char scratch[100];
728 StringPiece result;
729 std::unique_ptr<RandomAccessFile> file;
730 TF_EXPECT_OK(
731 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
732 // Read the first chunk. The cache will be populated with the first block of
733 // 9 bytes.
734 scratch[5] = 'x';
735 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
736 EXPECT_EQ("0123", result);
737 EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes.
738 // Flush caches and read the second chunk. This will be a cache miss, and
739 // the same block will be fetched again.
740 fs.FlushCaches(nullptr);
741 TF_EXPECT_OK(file->Read(4, 4, &result, scratch));
742 EXPECT_EQ("4567", result);
743 }
744
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_MaxStaleness)745 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) {
746 // Our underlying file in this test is a 16 byte file with contents
747 // "0123456789abcdef".
748 std::vector<HttpRequest*> requests(
749 {new FakeHttpRequest(
750 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
751 "object?fields=size%2Cgeneration%2Cupdated\n"
752 "Auth Token: fake_token\n"
753 "Timeouts: 5 1 10\n",
754 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
755 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
756 new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
757 "Auth Token: fake_token\n"
758 "Range: 0-7\n"
759 "Timeouts: 5 1 20\n",
760 "01234567"),
761 new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
762 "Auth Token: fake_token\n"
763 "Range: 8-15\n"
764 "Timeouts: 5 1 20\n",
765 "89abcdef")});
766 GcsFileSystem fs(
767 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
768 std::unique_ptr<HttpRequest::Factory>(
769 new FakeHttpRequestFactory(&requests)),
770 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
771 16 /* max bytes */, 3600 /* max staleness */,
772 3600 /* stat cache max age */, 0 /* stat cache max entries */,
773 0 /* matching paths cache max age */,
774 0 /* matching paths cache max entries */, kTestRetryConfig,
775 kTestTimeoutConfig, *kAllowedLocationsDefault,
776 nullptr /* gcs additional header */, false /* compose append */);
777 char scratch[100];
778 StringPiece result;
779 // There should only be two HTTP requests issued to GCS even though we iterate
780 // this loop 10 times. This shows that the underlying FileBlockCache persists
781 // across file close/open boundaries.
782 for (int i = 0; i < 10; i++) {
783 // Create two files. Since these files have the same name and the max
784 // staleness of the filesystem is > 0, they will share the same blocks.
785 std::unique_ptr<RandomAccessFile> file1;
786 std::unique_ptr<RandomAccessFile> file2;
787 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", nullptr, &file1));
788 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", nullptr, &file2));
789 // Reading the first block from file1 should load it once.
790 TF_EXPECT_OK(file1->Read(0, 8, &result, scratch));
791 EXPECT_EQ("01234567", result);
792 // Reading the first block from file2 should not trigger a request to load
793 // the first block again, because the FileBlockCache shared by file1 and
794 // file2 already has the first block.
795 TF_EXPECT_OK(file2->Read(0, 8, &result, scratch));
796 EXPECT_EQ("01234567", result);
797 // Reading the second block from file2 should load it once.
798 TF_EXPECT_OK(file2->Read(8, 8, &result, scratch));
799 EXPECT_EQ("89abcdef", result);
800 // Reading the second block from file1 should not trigger a request to load
801 // the second block again, because the FileBlockCache shared by file1 and
802 // file2 already has the second block.
803 TF_EXPECT_OK(file1->Read(8, 8, &result, scratch));
804 EXPECT_EQ("89abcdef", result);
805 }
806 }
807
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_FileSignatureChanges)808 TEST(GcsFileSystemTest,
809 NewRandomAccessFile_WithBlockCache_FileSignatureChanges) {
810 std::vector<HttpRequest*> requests(
811 {new FakeHttpRequest(
812 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
813 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
814 "Auth Token: fake_token\n"
815 "Timeouts: 5 1 10\n",
816 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
817 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
818 new FakeHttpRequest(
819 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
820 "Auth Token: fake_token\n"
821 "Range: 0-8\n"
822 "Timeouts: 5 1 20\n",
823 "01234"),
824 new FakeHttpRequest(
825 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
826 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
827 "Auth Token: fake_token\n"
828 "Timeouts: 5 1 10\n",
829 strings::StrCat("{\"size\": \"5\",\"generation\": \"2\","
830 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
831 new FakeHttpRequest(
832 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
833 "Auth Token: fake_token\n"
834 "Range: 0-8\n"
835 "Timeouts: 5 1 20\n",
836 "43210")});
837 GcsFileSystem fs(
838 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
839 std::unique_ptr<HttpRequest::Factory>(
840 new FakeHttpRequestFactory(&requests)),
841 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
842 18 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
843 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
844 0 /* matching paths cache max entries */, kTestRetryConfig,
845 kTestTimeoutConfig, *kAllowedLocationsDefault,
846 nullptr /* gcs additional header */, false /* compose append */);
847
848 std::unique_ptr<RandomAccessFile> file;
849 TF_EXPECT_OK(
850 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
851
852 char scratch[5];
853 StringPiece result;
854
855 // First read.
856 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
857 EXPECT_EQ("01234", result);
858
859 // Second read. File signatures are different.
860 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
861 EXPECT_EQ("43210", result);
862 }
863
TEST(GcsFileSystemTest,NewRandomAccessFile_NoObjectName)864 TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) {
865 std::vector<HttpRequest*> requests;
866 GcsFileSystem fs(
867 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
868 std::unique_ptr<HttpRequest::Factory>(
869 new FakeHttpRequestFactory(&requests)),
870 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
871 0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */,
872 0 /* stat cache max age */, 0 /* stat cache max entries */,
873 0 /* matching paths cache max age */,
874 0 /* matching paths cache max entries */, kTestRetryConfig,
875 kTestTimeoutConfig, *kAllowedLocationsDefault,
876 nullptr /* gcs additional header */, false /* compose append */);
877
878 std::unique_ptr<RandomAccessFile> file;
879 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
880 fs.NewRandomAccessFile("gs://bucket/", nullptr, &file).code());
881 }
882
TEST(GcsFileSystemTest,NewRandomAccessFile_InconsistentRead)883 TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) {
884 std::vector<HttpRequest*> requests(
885 {new FakeHttpRequest(
886 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
887 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
888 "Auth Token: fake_token\n"
889 "Timeouts: 5 1 10\n",
890 strings::StrCat("{\"size\": \"6\",\"generation\": \"1\","
891 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
892 new FakeHttpRequest(
893 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
894 "Auth Token: fake_token\n"
895 "Range: 0-5\n"
896 "Timeouts: 5 1 20\n",
897 "012")});
898
899 // Set stat_cache_max_age to 1000s so that StatCache could work.
900 GcsFileSystem fs(
901 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
902 std::unique_ptr<HttpRequest::Factory>(
903 new FakeHttpRequestFactory(&requests)),
904 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
905 0 /* max bytes */, 0 /* max staleness */, 1e3 /* stat cache max age */,
906 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
907 0 /* matching paths cache max entries */, kTestRetryConfig,
908 kTestTimeoutConfig, *kAllowedLocationsDefault,
909 nullptr /* gcs additional header */, false /* compose append */);
910
911 // Stat the file first so that the file stats are cached.
912 FileStatistics stat;
913 TF_ASSERT_OK(fs.Stat("gs://bucket/random_access.txt", nullptr, &stat));
914
915 std::unique_ptr<RandomAccessFile> file;
916 TF_ASSERT_OK(
917 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
918
919 char scratch[6];
920 StringPiece result;
921
922 EXPECT_EQ(errors::Code::INTERNAL,
923 file->Read(0, sizeof(scratch), &result, scratch).code());
924 }
925
TEST(GcsFileSystemTest,NewWritableFile)926 TEST(GcsFileSystemTest, NewWritableFile) {
927 std::vector<HttpRequest*> requests(
928 {new FakeHttpRequest(
929 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
930 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
931 "Auth Token: fake_token\n"
932 "Timeouts: 5 1 10\n",
933 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
934 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
935 new FakeHttpRequest(
936 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
937 "Auth Token: fake_token\n"
938 "Range: 0-7\n"
939 "Timeouts: 5 1 20\n",
940 "01234567"),
941 new FakeHttpRequest(
942 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
943 "uploadType=resumable&name=path%2Fwriteable\n"
944 "Auth Token: fake_token\n"
945 "Header X-Upload-Content-Length: 17\n"
946 "Post: yes\n"
947 "Timeouts: 5 1 10\n",
948 "", {{"Location", "https://custom/upload/location"}}),
949 new FakeHttpRequest("Uri: https://custom/upload/location\n"
950 "Auth Token: fake_token\n"
951 "Header Content-Range: bytes 0-16/17\n"
952 "Timeouts: 5 1 30\n"
953 "Put body: content1,content2\n",
954 ""),
955 new FakeHttpRequest(
956 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
957 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
958 "Auth Token: fake_token\n"
959 "Timeouts: 5 1 10\n",
960 strings::StrCat("{\"size\": \"33\",\"generation\": \"2\","
961 "\"updated\": \"2016-04-29T23:15:34.896Z\"}")),
962 new FakeHttpRequest(
963 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
964 "Auth Token: fake_token\n"
965 "Range: 0-7\n"
966 "Timeouts: 5 1 20\n",
967 "01234567")});
968 GcsFileSystem fs(
969 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
970 std::unique_ptr<HttpRequest::Factory>(
971 new FakeHttpRequestFactory(&requests)),
972 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
973 8 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
974 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
975 0 /* matching paths cache max entries */, kTestRetryConfig,
976 kTestTimeoutConfig, *kAllowedLocationsDefault,
977 nullptr /* gcs additional header */, false /* compose append */);
978
979 // Read from the file first, to fill the block cache.
980 std::unique_ptr<RandomAccessFile> rfile;
981 TF_EXPECT_OK(
982 fs.NewRandomAccessFile("gs://bucket/path/writeable", nullptr, &rfile));
983 char scratch[100];
984 StringPiece result;
985 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
986 EXPECT_EQ("0123", result);
987 // Open the writable file.
988 std::unique_ptr<WritableFile> wfile;
989 TF_EXPECT_OK(
990 fs.NewWritableFile("gs://bucket/path/writeable", nullptr, &wfile));
991 TF_EXPECT_OK(wfile->Append("content1,"));
992 int64 pos;
993 TF_EXPECT_OK(wfile->Tell(&pos));
994 EXPECT_EQ(9, pos);
995 TF_EXPECT_OK(wfile->Append("content2"));
996 TF_EXPECT_OK(wfile->Flush());
997 // Re-reading the file should trigger another HTTP request to GCS.
998 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
999 EXPECT_EQ("0123", result);
1000 // The calls to flush, sync, and close below should not cause uploads because
1001 // the file is not dirty.
1002 TF_EXPECT_OK(wfile->Flush());
1003 TF_EXPECT_OK(wfile->Sync());
1004 TF_EXPECT_OK(wfile->Close());
1005 }
1006
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadSucceeds)1007 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
1008 std::vector<HttpRequest*> requests(
1009 {new FakeHttpRequest(
1010 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1011 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1012 "Auth Token: fake_token\n"
1013 "Header X-Upload-Content-Length: 17\n"
1014 "Post: yes\n"
1015 "Timeouts: 5 1 10\n",
1016 "", {{"Location", "https://custom/upload/location"}}),
1017 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1018 "Auth Token: fake_token\n"
1019 "Header Content-Range: bytes 0-16/17\n"
1020 "Timeouts: 5 1 30\n"
1021 "Put body: content1,content2\n",
1022 "", errors::Unavailable("503"), 503),
1023 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1024 "Auth Token: fake_token\n"
1025 "Timeouts: 5 1 10\n"
1026 "Header Content-Range: bytes */17\n"
1027 "Put: yes\n",
1028 "", errors::Unavailable("308"), nullptr,
1029 {{"Range", "0-10"}}, 308),
1030 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1031 "Auth Token: fake_token\n"
1032 "Header Content-Range: bytes 11-16/17\n"
1033 "Timeouts: 5 1 30\n"
1034 "Put body: ntent2\n",
1035 "", errors::Unavailable("503"), 503),
1036 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1037 "Auth Token: fake_token\n"
1038 "Timeouts: 5 1 10\n"
1039 "Header Content-Range: bytes */17\n"
1040 "Put: yes\n",
1041 "", errors::Unavailable("308"), nullptr,
1042 {{"Range", "bytes=0-12"}}, 308),
1043 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1044 "Auth Token: fake_token\n"
1045 "Header Content-Range: bytes 13-16/17\n"
1046 "Timeouts: 5 1 30\n"
1047 "Put body: ent2\n",
1048 "", errors::Unavailable("308"), 308),
1049 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1050 "Auth Token: fake_token\n"
1051 "Timeouts: 5 1 10\n"
1052 "Header Content-Range: bytes */17\n"
1053 "Put: yes\n",
1054 "", errors::Unavailable("308"), nullptr,
1055 {{"Range", "bytes=0-14"}}, 308),
1056 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1057 "Auth Token: fake_token\n"
1058 "Header Content-Range: bytes 15-16/17\n"
1059 "Timeouts: 5 1 30\n"
1060 "Put body: t2\n",
1061 "")});
1062 GcsFileSystem fs(
1063 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1064 std::unique_ptr<HttpRequest::Factory>(
1065 new FakeHttpRequestFactory(&requests)),
1066 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1067 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1068 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1069 0 /* matching paths cache max entries */, kTestRetryConfig,
1070 kTestTimeoutConfig, *kAllowedLocationsDefault,
1071 nullptr /* gcs additional header */, false /* compose append */);
1072
1073 std::unique_ptr<WritableFile> file;
1074 TF_EXPECT_OK(
1075 fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file));
1076
1077 TF_EXPECT_OK(file->Append("content1,"));
1078 TF_EXPECT_OK(file->Append("content2"));
1079 TF_EXPECT_OK(file->Close());
1080 }
1081
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadSucceedsOnGetStatus)1082 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) {
1083 // This test also verifies that a file's blocks are purged from the cache when
1084 // the file is written, even when the write takes the "succeeds on get status"
1085 // path.
1086 std::vector<HttpRequest*> requests(
1087 {new FakeHttpRequest(
1088 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1089 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
1090 "Auth Token: fake_token\n"
1091 "Timeouts: 5 1 10\n",
1092 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
1093 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1094 new FakeHttpRequest(
1095 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
1096 "Auth Token: fake_token\n"
1097 "Range: 0-7\n"
1098 "Timeouts: 5 1 20\n",
1099 "01234567"),
1100 new FakeHttpRequest(
1101 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1102 "uploadType=resumable&name=path%2Fwriteable\n"
1103 "Auth Token: fake_token\n"
1104 "Header X-Upload-Content-Length: 17\n"
1105 "Post: yes\n"
1106 "Timeouts: 5 1 10\n",
1107 "", {{"Location", "https://custom/upload/location"}}),
1108 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1109 "Auth Token: fake_token\n"
1110 "Header Content-Range: bytes 0-16/17\n"
1111 "Timeouts: 5 1 30\n"
1112 "Put body: content1,content2\n",
1113 "", errors::Unavailable("503"), 503),
1114 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1115 "Auth Token: fake_token\n"
1116 "Timeouts: 5 1 10\n"
1117 "Header Content-Range: bytes */17\n"
1118 "Put: yes\n",
1119 "", Status::OK(), nullptr, {}, 201),
1120 new FakeHttpRequest(
1121 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1122 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
1123 "Auth Token: fake_token\n"
1124 "Timeouts: 5 1 10\n",
1125 strings::StrCat("{\"size\": \"33\",\"generation\": \"2\","
1126 "\"updated\": \"2016-04-29T23:19:24.896Z\"}")),
1127 new FakeHttpRequest(
1128 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
1129 "Auth Token: fake_token\n"
1130 "Range: 0-7\n"
1131 "Timeouts: 5 1 20\n",
1132 "01234567")});
1133 GcsFileSystem fs(
1134 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1135 std::unique_ptr<HttpRequest::Factory>(
1136 new FakeHttpRequestFactory(&requests)),
1137 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
1138 8 /* max bytes */, 3600 /* max staleness */,
1139 3600 /* stat cache max age */, 0 /* stat cache max entries */,
1140 0 /* matching paths cache max age */,
1141 0 /* matching paths cache max entries */, kTestRetryConfig,
1142 kTestTimeoutConfig, *kAllowedLocationsDefault,
1143 nullptr /* gcs additional header */, false /* compose append */);
1144 // Pull the file's first block into the cache. This will trigger the first
1145 // HTTP request to GCS.
1146 std::unique_ptr<RandomAccessFile> rfile;
1147 TF_EXPECT_OK(
1148 fs.NewRandomAccessFile("gs://bucket/path/writeable", nullptr, &rfile));
1149 char scratch[100];
1150 StringPiece result;
1151 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
1152 EXPECT_EQ("0123", result);
1153 // Now write to the same file. Once the write succeeds, the cached block will
1154 // be flushed.
1155 std::unique_ptr<WritableFile> wfile;
1156 TF_EXPECT_OK(
1157 fs.NewWritableFile("gs://bucket/path/writeable", nullptr, &wfile));
1158 TF_EXPECT_OK(wfile->Append("content1,"));
1159 TF_EXPECT_OK(wfile->Append("content2"));
1160 // Appending doesn't invalidate the read cache - only flushing does. This read
1161 // will not trigger an HTTP request to GCS.
1162 TF_EXPECT_OK(rfile->Read(4, 4, &result, scratch));
1163 EXPECT_EQ("4567", result);
1164 // Closing the file triggers HTTP requests to GCS and invalidates the read
1165 // cache for the file.
1166 TF_EXPECT_OK(wfile->Close());
1167 // Reading the first block of the file goes to GCS again.
1168 TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch));
1169 EXPECT_EQ("01234567", result);
1170 }
1171
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadAllAttemptsFail)1172 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
1173 std::vector<HttpRequest*> requests(
1174 {new FakeHttpRequest(
1175 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1176 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1177 "Auth Token: fake_token\n"
1178 "Header X-Upload-Content-Length: 17\n"
1179 "Post: yes\n"
1180 "Timeouts: 5 1 10\n",
1181 "", {{"Location", "https://custom/upload/location"}}),
1182 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1183 "Auth Token: fake_token\n"
1184 "Header Content-Range: bytes 0-16/17\n"
1185 "Timeouts: 5 1 30\n"
1186 "Put body: content1,content2\n",
1187 "", errors::Unavailable("503"), 503)});
1188 for (int i = 0; i < 10; i++) {
1189 requests.emplace_back(
1190 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1191 "Auth Token: fake_token\n"
1192 "Timeouts: 5 1 10\n"
1193 "Header Content-Range: bytes */17\n"
1194 "Put: yes\n",
1195 "", errors::Unavailable("important HTTP error 308"),
1196 nullptr, {{"Range", "0-10"}}, 308));
1197 requests.emplace_back(new FakeHttpRequest(
1198 "Uri: https://custom/upload/location\n"
1199 "Auth Token: fake_token\n"
1200 "Header Content-Range: bytes 11-16/17\n"
1201 "Timeouts: 5 1 30\n"
1202 "Put body: ntent2\n",
1203 "", errors::Unavailable("important HTTP error 503"), 503));
1204 }
1205 // These calls will be made in the Close() attempt from the destructor.
1206 // Letting the destructor succeed.
1207 requests.emplace_back(new FakeHttpRequest(
1208 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1209 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1210 "Auth Token: fake_token\n"
1211 "Header X-Upload-Content-Length: 17\n"
1212 "Post: yes\n"
1213 "Timeouts: 5 1 10\n",
1214 "", {{"Location", "https://custom/upload/location"}}));
1215 requests.emplace_back(
1216 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1217 "Auth Token: fake_token\n"
1218 "Header Content-Range: bytes 0-16/17\n"
1219 "Timeouts: 5 1 30\n"
1220 "Put body: content1,content2\n",
1221 ""));
1222 GcsFileSystem fs(
1223 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1224 std::unique_ptr<HttpRequest::Factory>(
1225 new FakeHttpRequestFactory(&requests)),
1226 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1227 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1228 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1229 0 /* matching paths cache max entries */,
1230 RetryConfig(2 /* .init_delay_time_us */), kTestTimeoutConfig,
1231 *kAllowedLocationsDefault, nullptr /* gcs additional header */,
1232 false /* compose append */);
1233
1234 std::unique_ptr<WritableFile> file;
1235 TF_EXPECT_OK(
1236 fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file));
1237
1238 TF_EXPECT_OK(file->Append("content1,"));
1239 TF_EXPECT_OK(file->Append("content2"));
1240 const auto& status = file->Close();
1241 EXPECT_EQ(errors::Code::ABORTED, status.code());
1242 EXPECT_TRUE(
1243 absl::StrContains(status.error_message(),
1244 "All 10 retry attempts failed. The last failure: "
1245 "Unavailable: important HTTP error 503"))
1246 << status;
1247 }
1248
TEST(GcsFileSystemTest,NewWritableFile_UploadReturns410)1249 TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
1250 std::vector<string> results;
1251 TF_EXPECT_OK(
1252 Env::Default()->GetMatchingPaths("/tmp/tmp_file_tensorflow*", &results));
1253 const int64 tmp_files_before = results.size();
1254
1255 std::vector<HttpRequest*> requests(
1256 {new FakeHttpRequest(
1257 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1258 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1259 "Auth Token: fake_token\n"
1260 "Header X-Upload-Content-Length: 17\n"
1261 "Post: yes\n"
1262 "Timeouts: 5 1 10\n",
1263 "", {{"Location", "https://custom/upload/location"}}),
1264 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1265 "Auth Token: fake_token\n"
1266 "Header Content-Range: bytes 0-16/17\n"
1267 "Timeouts: 5 1 30\n"
1268 "Put body: content1,content2\n",
1269 "", errors::NotFound("important HTTP error 410"),
1270 410),
1271 // These calls will be made in the Close() attempt from the destructor.
1272 // Letting the destructor succeed.
1273 new FakeHttpRequest(
1274 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1275 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1276 "Auth Token: fake_token\n"
1277 "Header X-Upload-Content-Length: 17\n"
1278 "Post: yes\n"
1279 "Timeouts: 5 1 10\n",
1280 "", {{"Location", "https://custom/upload/location"}}),
1281 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1282 "Auth Token: fake_token\n"
1283 "Header Content-Range: bytes 0-16/17\n"
1284 "Timeouts: 5 1 30\n"
1285 "Put body: content1,content2\n",
1286 "")});
1287 GcsFileSystem fs(
1288 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1289 std::unique_ptr<HttpRequest::Factory>(
1290 new FakeHttpRequestFactory(&requests)),
1291 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1292 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1293 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1294 0 /* matching paths cache max entries */, kTestRetryConfig,
1295 kTestTimeoutConfig, *kAllowedLocationsDefault,
1296 nullptr /* gcs additional header */, false /* compose append */);
1297
1298 {
1299 std::unique_ptr<WritableFile> file;
1300 TF_EXPECT_OK(
1301 fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file));
1302
1303 TF_EXPECT_OK(file->Append("content1,"));
1304 TF_EXPECT_OK(file->Append("content2"));
1305 const auto& status = file->Close();
1306 EXPECT_EQ(errors::Code::UNAVAILABLE, status.code());
1307 EXPECT_TRUE(
1308 absl::StrContains(status.error_message(),
1309 "Upload to gs://bucket/path/writeable.txt failed, "
1310 "caused by: Not found: important HTTP error 410"))
1311 << status;
1312 EXPECT_TRUE(
1313 absl::StrContains(status.error_message(),
1314 "when uploading gs://bucket/path/writeable.txt"))
1315 << status;
1316 }
1317
1318 // Check that no new tempfiles were left over after failure and destruction
1319 // of the file.
1320 results.clear();
1321 TF_EXPECT_OK(
1322 Env::Default()->GetMatchingPaths("/tmp/tmp_file_tensorflow*", &results));
1323 EXPECT_EQ(tmp_files_before, results.size());
1324 }
1325
TEST(GcsFileSystemTest,NewWritableFile_NoObjectName)1326 TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) {
1327 std::vector<HttpRequest*> requests;
1328 GcsFileSystem fs(
1329 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1330 std::unique_ptr<HttpRequest::Factory>(
1331 new FakeHttpRequestFactory(&requests)),
1332 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1333 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1334 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1335 0 /* matching paths cache max entries */, kTestRetryConfig,
1336 kTestTimeoutConfig, *kAllowedLocationsDefault,
1337 nullptr /* gcs additional header */, false /* compose append */);
1338
1339 std::unique_ptr<WritableFile> file;
1340 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1341 fs.NewWritableFile("gs://bucket/", nullptr, &file).code());
1342 }
1343
TEST(GcsFileSystemTest,NewAppendableFile)1344 TEST(GcsFileSystemTest, NewAppendableFile) {
1345 std::vector<HttpRequest*> requests(
1346 {new FakeHttpRequest(
1347 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1348 "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
1349 "Auth Token: fake_token\n"
1350 "Timeouts: 5 1 10\n",
1351 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
1352 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1353 new FakeHttpRequest(
1354 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
1355 "Auth Token: fake_token\n"
1356 "Range: 0-1048575\n"
1357 "Timeouts: 5 1 20\n",
1358 "content1,"),
1359 new FakeHttpRequest(
1360 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
1361 "Auth Token: fake_token\n"
1362 "Range: 0-31\n"
1363 "Timeouts: 5 1 20\n",
1364 "content1,"),
1365 new FakeHttpRequest(
1366 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1367 "uploadType=resumable&name=path%2Fappendable\n"
1368 "Auth Token: fake_token\n"
1369 "Header X-Upload-Content-Length: 17\n"
1370 "Post: yes\n"
1371 "Timeouts: 5 1 10\n",
1372 "", {{"Location", "https://custom/upload/location"}}),
1373 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1374 "Auth Token: fake_token\n"
1375 "Header Content-Range: bytes 0-16/17\n"
1376 "Timeouts: 5 1 30\n"
1377 "Put body: content1,content2\n",
1378 ""),
1379 new FakeHttpRequest(
1380 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1381 "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
1382 "Auth Token: fake_token\n"
1383 "Timeouts: 5 1 10\n",
1384 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
1385 "\"updated\": \"2016-04-29T23:25:24.896Z\"}")),
1386 new FakeHttpRequest(
1387 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
1388 "Auth Token: fake_token\n"
1389 "Range: 0-31\n"
1390 "Timeouts: 5 1 20\n",
1391 "01234567")});
1392 GcsFileSystem fs(
1393 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1394 std::unique_ptr<HttpRequest::Factory>(
1395 new FakeHttpRequestFactory(&requests)),
1396 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 32 /* block size */,
1397 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1398 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1399 0 /* matching paths cache max entries */, kTestRetryConfig,
1400 kTestTimeoutConfig, *kAllowedLocationsDefault,
1401 nullptr /* gcs additional header */, false /* compose append */);
1402
1403 // Create an appendable file. This should read the file from GCS, and pull its
1404 // contents into the block cache.
1405 std::unique_ptr<WritableFile> wfile;
1406 TF_EXPECT_OK(
1407 fs.NewAppendableFile("gs://bucket/path/appendable", nullptr, &wfile));
1408 TF_EXPECT_OK(wfile->Append("content2"));
1409 // Verify that the file contents are in the block cache. This read should not
1410 // trigger an HTTP request to GCS.
1411 std::unique_ptr<RandomAccessFile> rfile;
1412 TF_EXPECT_OK(
1413 fs.NewRandomAccessFile("gs://bucket/path/appendable", nullptr, &rfile));
1414 char scratch[100];
1415 StringPiece result;
1416 TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch));
1417 EXPECT_EQ("content1", result);
1418 // Closing the appendable file will flush its contents to GCS, triggering HTTP
1419 // requests.
1420 TF_EXPECT_OK(wfile->Close());
1421 // Redo the read. The block should be reloaded from GCS, causing one more HTTP
1422 // request to load it.
1423 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
1424 EXPECT_EQ("0123", result);
1425 }
1426
TEST(GcsFileSystemTest,NewAppendableFile_NoObjectName)1427 TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) {
1428 std::vector<HttpRequest*> requests;
1429 GcsFileSystem fs(
1430 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1431 std::unique_ptr<HttpRequest::Factory>(
1432 new FakeHttpRequestFactory(&requests)),
1433 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1434 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1435 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1436 0 /* matching paths cache max entries */, kTestRetryConfig,
1437 kTestTimeoutConfig, *kAllowedLocationsDefault,
1438 nullptr /* gcs additional header */, false /* compose append */);
1439
1440 std::unique_ptr<WritableFile> file;
1441 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1442 fs.NewAppendableFile("gs://bucket/", nullptr, &file).code());
1443 }
1444
TEST(GcsFileSystemTest,NewAppendableFile_ObjectDoesNotExist)1445 TEST(GcsFileSystemTest, NewAppendableFile_ObjectDoesNotExist) {
1446 std::vector<HttpRequest*> requests(
1447 {new FakeHttpRequest(
1448 "Uri: https://storage.googleapis.com/bucket/filename\n"
1449 "Auth Token: fake_token\n"
1450 "Range: 0-1048575\n"
1451 "Timeouts: 5 1 20\n",
1452 "", errors::NotFound("404"), 404),
1453 new FakeHttpRequest(
1454 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o"
1455 "?uploadType=resumable&name=filename\n"
1456 "Auth Token: fake_token\n"
1457 "Header X-Upload-Content-Length: 0\n"
1458 "Post: yes\n"
1459 "Timeouts: 5 1 10\n",
1460 "")});
1461 GcsFileSystem fs(
1462 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1463 std::unique_ptr<HttpRequest::Factory>(
1464 new FakeHttpRequestFactory(&requests)),
1465 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1466 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1467 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1468 0 /* matching paths cache max entries */, kTestRetryConfig,
1469 kTestTimeoutConfig, *kAllowedLocationsDefault,
1470 nullptr /* gcs additional header */, false /* compose append */);
1471
1472 std::unique_ptr<WritableFile> file;
1473 TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/filename", nullptr, &file));
1474 }
1475
TEST(GcsFileSystemTest,NewReadOnlyMemoryRegionFromFile)1476 TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
1477 const string content = "file content";
1478 std::vector<HttpRequest*> requests(
1479 {new FakeHttpRequest(
1480 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1481 "path%2Frandom_access.txt?fields=size%2Cgeneration%2Cupdated\n"
1482 "Auth Token: fake_token\n"
1483 "Timeouts: 5 1 10\n",
1484 strings::StrCat("{\"size\": \"", content.size(), "\"",
1485 ", \"generation\": \"1\"",
1486 ", \"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1487 new FakeHttpRequest(
1488 strings::StrCat("Uri: https://storage.googleapis.com/bucket/"
1489 "path%2Frandom_access.txt\n"
1490 "Auth Token: fake_token\n"
1491 "Range: 0-",
1492 content.size() - 1, "\n", "Timeouts: 5 1 20\n"),
1493 content)});
1494 GcsFileSystem fs(
1495 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1496 std::unique_ptr<HttpRequest::Factory>(
1497 new FakeHttpRequestFactory(&requests)),
1498 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1499 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1500 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1501 0 /* matching paths cache max entries */, kTestRetryConfig,
1502 kTestTimeoutConfig, *kAllowedLocationsDefault,
1503 nullptr /* gcs additional header */, false /* compose append */);
1504
1505 std::unique_ptr<ReadOnlyMemoryRegion> region;
1506 TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile(
1507 "gs://bucket/path/random_access.txt", nullptr, ®ion));
1508
1509 EXPECT_EQ(content, StringPiece(reinterpret_cast<const char*>(region->data()),
1510 region->length()));
1511 }
1512
TEST(GcsFileSystemTest,NewReadOnlyMemoryRegionFromFile_NoObjectName)1513 TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) {
1514 std::vector<HttpRequest*> requests;
1515 GcsFileSystem fs(
1516 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1517 std::unique_ptr<HttpRequest::Factory>(
1518 new FakeHttpRequestFactory(&requests)),
1519 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1520 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1521 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1522 0 /* matching paths cache max entries */, kTestRetryConfig,
1523 kTestTimeoutConfig, *kAllowedLocationsDefault,
1524 nullptr /* gcs additional header */, false /* compose append */);
1525
1526 std::unique_ptr<ReadOnlyMemoryRegion> region;
1527 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1528 fs.NewReadOnlyMemoryRegionFromFile("gs://bucket/", nullptr, ®ion)
1529 .code());
1530 }
1531
TEST(GcsFileSystemTest,FileExists_YesAsObject)1532 TEST(GcsFileSystemTest, FileExists_YesAsObject) {
1533 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1534 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1535 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1536 "Auth Token: fake_token\n"
1537 "Timeouts: 5 1 10\n",
1538 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
1539 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
1540 GcsFileSystem fs(
1541 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1542 std::unique_ptr<HttpRequest::Factory>(
1543 new FakeHttpRequestFactory(&requests)),
1544 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1545 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1546 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1547 0 /* matching paths cache max entries */, kTestRetryConfig,
1548 kTestTimeoutConfig, *kAllowedLocationsDefault,
1549 nullptr /* gcs additional header */, false /* compose append */);
1550
1551 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt", nullptr));
1552 }
1553
TEST(GcsFileSystemTest,FileExists_YesAsFolder)1554 TEST(GcsFileSystemTest, FileExists_YesAsFolder) {
1555 std::vector<HttpRequest*> requests(
1556 {new FakeHttpRequest(
1557 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1558 "path%2Fsubfolder?fields=size%2Cgeneration%2Cupdated\n"
1559 "Auth Token: fake_token\n"
1560 "Timeouts: 5 1 10\n",
1561 "", errors::NotFound("404"), 404),
1562 new FakeHttpRequest(
1563 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1564 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F"
1565 "&maxResults=1\n"
1566 "Auth Token: fake_token\n"
1567 "Timeouts: 5 1 10\n",
1568 "{\"items\": [ "
1569 " { \"name\": \"path/subfolder/\" }]}")});
1570 GcsFileSystem fs(
1571 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1572 std::unique_ptr<HttpRequest::Factory>(
1573 new FakeHttpRequestFactory(&requests)),
1574 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1575 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1576 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1577 0 /* matching paths cache max entries */, kTestRetryConfig,
1578 kTestTimeoutConfig, *kAllowedLocationsDefault,
1579 nullptr /* gcs additional header */, false /* compose append */);
1580
1581 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder", nullptr));
1582 }
1583
TEST(GcsFileSystemTest,FileExists_YesAsBucket)1584 TEST(GcsFileSystemTest, FileExists_YesAsBucket) {
1585 std::vector<HttpRequest*> requests(
1586 {new FakeHttpRequest(
1587 "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n"
1588 "Auth Token: fake_token\n"
1589 "Timeouts: 5 1 10\n",
1590 "{\"size\": \"100\"}"),
1591 new FakeHttpRequest(
1592 "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n"
1593 "Auth Token: fake_token\n"
1594 "Timeouts: 5 1 10\n",
1595 "{\"size\": \"100\"}")});
1596 GcsFileSystem fs(
1597 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1598 std::unique_ptr<HttpRequest::Factory>(
1599 new FakeHttpRequestFactory(&requests)),
1600 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1601 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1602 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1603 0 /* matching paths cache max entries */, kTestRetryConfig,
1604 kTestTimeoutConfig, *kAllowedLocationsDefault,
1605 nullptr /* gcs additional header */, false /* compose append */);
1606
1607 TF_EXPECT_OK(fs.FileExists("gs://bucket1", nullptr));
1608 TF_EXPECT_OK(fs.FileExists("gs://bucket1/", nullptr));
1609 }
1610
TEST(GcsFileSystemTest,FileExists_NotAsObjectOrFolder)1611 TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) {
1612 std::vector<HttpRequest*> requests(
1613 {new FakeHttpRequest(
1614 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1615 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1616 "Auth Token: fake_token\n"
1617 "Timeouts: 5 1 10\n",
1618 "", errors::NotFound("404"), 404),
1619 new FakeHttpRequest(
1620 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1621 "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile1.txt%2F"
1622 "&maxResults=1\n"
1623 "Auth Token: fake_token\n"
1624 "Timeouts: 5 1 10\n",
1625 "{\"items\": []}")});
1626 GcsFileSystem fs(
1627 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1628 std::unique_ptr<HttpRequest::Factory>(
1629 new FakeHttpRequestFactory(&requests)),
1630 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1631 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1632 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1633 0 /* matching paths cache max entries */, kTestRetryConfig,
1634 kTestTimeoutConfig, *kAllowedLocationsDefault,
1635 nullptr /* gcs additional header */, false /* compose append */);
1636
1637 EXPECT_EQ(errors::Code::NOT_FOUND,
1638 fs.FileExists("gs://bucket/path/file1.txt", nullptr).code());
1639 }
1640
TEST(GcsFileSystemTest,FileExists_NotAsBucket)1641 TEST(GcsFileSystemTest, FileExists_NotAsBucket) {
1642 std::vector<HttpRequest*> requests(
1643 {new FakeHttpRequest(
1644 "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n"
1645 "Auth Token: fake_token\n"
1646 "Timeouts: 5 1 10\n",
1647 "", errors::NotFound("404"), 404),
1648 new FakeHttpRequest(
1649 "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n"
1650 "Auth Token: fake_token\n"
1651 "Timeouts: 5 1 10\n",
1652 "", errors::NotFound("404"), 404)});
1653 GcsFileSystem fs(
1654 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1655 std::unique_ptr<HttpRequest::Factory>(
1656 new FakeHttpRequestFactory(&requests)),
1657 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1658 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1659 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1660 0 /* matching paths cache max entries */, kTestRetryConfig,
1661 kTestTimeoutConfig, *kAllowedLocationsDefault,
1662 nullptr /* gcs additional header */, false /* compose append */);
1663 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1664 fs.FileExists("gs://bucket2/", nullptr).code());
1665 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
1666 fs.FileExists("gs://bucket2", nullptr).code());
1667 }
1668
TEST(GcsFileSystemTest,FileExists_StatCache)1669 TEST(GcsFileSystemTest, FileExists_StatCache) {
1670 std::vector<HttpRequest*> requests(
1671 {new FakeHttpRequest(
1672 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1673 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1674 "Auth Token: fake_token\n"
1675 "Timeouts: 5 1 10\n",
1676 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
1677 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1678 new FakeHttpRequest(
1679 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1680 "path%2Fsubfolder%2F?fields=size%2Cgeneration%2Cupdated\n"
1681 "Auth Token: fake_token\n"
1682 "Timeouts: 5 1 10\n",
1683 "", errors::NotFound("404"), 404),
1684 new FakeHttpRequest(
1685 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1686 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F"
1687 "&maxResults=1\n"
1688 "Auth Token: fake_token\n"
1689 "Timeouts: 5 1 10\n",
1690 "{\"items\": [ "
1691 " { \"name\": \"path/subfolder/\" }]}")});
1692 GcsFileSystem fs(
1693 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1694 std::unique_ptr<HttpRequest::Factory>(
1695 new FakeHttpRequestFactory(&requests)),
1696 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1697 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1698 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1699 0 /* matching paths cache max entries */, kTestRetryConfig,
1700 kTestTimeoutConfig, *kAllowedLocationsDefault,
1701 nullptr /* gcs additional header */, false /* compose append */);
1702
1703 // The stat cache will ensure that repeated lookups don't trigger additional
1704 // HTTP requests.
1705 for (int i = 0; i < 10; i++) {
1706 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt", nullptr));
1707 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder/", nullptr));
1708 }
1709 }
1710
TEST(GcsFileSystemTest,FileExists_DirectoryMark)1711 TEST(GcsFileSystemTest, FileExists_DirectoryMark) {
1712 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1713 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1714 "dir%2F?fields=size%2Cgeneration%2Cupdated\n"
1715 "Auth Token: fake_token\n"
1716 "Timeouts: 5 1 10\n",
1717 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
1718 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
1719 GcsFileSystem fs(
1720 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1721 std::unique_ptr<HttpRequest::Factory>(
1722 new FakeHttpRequestFactory(&requests)),
1723 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1724 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1725 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1726 0 /* matching paths cache max entries */, kTestRetryConfig,
1727 kTestTimeoutConfig, *kAllowedLocationsDefault,
1728 nullptr /* gcs additional header */, false /* compose append */);
1729
1730 TF_EXPECT_OK(fs.FileExists("gs://bucket/dir/", nullptr));
1731 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/dir/", nullptr));
1732 }
1733
TEST(GcsFileSystemTest,GetChildren_NoItems)1734 TEST(GcsFileSystemTest, GetChildren_NoItems) {
1735 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1736 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1737 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1738 "path%2F\n"
1739 "Auth Token: fake_token\n"
1740 "Timeouts: 5 1 10\n",
1741 "{\"prefixes\": [\"path/subpath/\"]}")});
1742 GcsFileSystem fs(
1743 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1744 std::unique_ptr<HttpRequest::Factory>(
1745 new FakeHttpRequestFactory(&requests)),
1746 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1747 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1748 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1749 0 /* matching paths cache max entries */, kTestRetryConfig,
1750 kTestTimeoutConfig, *kAllowedLocationsDefault,
1751 nullptr /* gcs additional header */, false /* compose append */);
1752
1753 std::vector<string> children;
1754 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children));
1755
1756 EXPECT_EQ(std::vector<string>({"subpath/"}), children);
1757 }
1758
TEST(GcsFileSystemTest,GetChildren_ThreeFiles)1759 TEST(GcsFileSystemTest, GetChildren_ThreeFiles) {
1760 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1761 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1762 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1763 "path%2F\n"
1764 "Auth Token: fake_token\n"
1765 "Timeouts: 5 1 10\n",
1766 "{\"items\": [ "
1767 " { \"name\": \"path/file1.txt\" },"
1768 " { \"name\": \"path/file3.txt\" }],"
1769 "\"prefixes\": [\"path/subpath/\"]}")});
1770 GcsFileSystem fs(
1771 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1772 std::unique_ptr<HttpRequest::Factory>(
1773 new FakeHttpRequestFactory(&requests)),
1774 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1775 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1776 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1777 0 /* matching paths cache max entries */, kTestRetryConfig,
1778 kTestTimeoutConfig, *kAllowedLocationsDefault,
1779 nullptr /* gcs additional header */, false /* compose append */);
1780
1781 std::vector<string> children;
1782 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children));
1783
1784 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/"}),
1785 children);
1786 }
1787
TEST(GcsFileSystemTest,GetChildren_SelfDirectoryMarker)1788 TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) {
1789 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1790 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1791 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1792 "path%2F\n"
1793 "Auth Token: fake_token\n"
1794 "Timeouts: 5 1 10\n",
1795 "{\"items\": [ "
1796 " { \"name\": \"path/\" },"
1797 " { \"name\": \"path/file3.txt\" }],"
1798 "\"prefixes\": [\"path/subpath/\"]}")});
1799 GcsFileSystem fs(
1800 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1801 std::unique_ptr<HttpRequest::Factory>(
1802 new FakeHttpRequestFactory(&requests)),
1803 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1804 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1805 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1806 0 /* matching paths cache max entries */, kTestRetryConfig,
1807 kTestTimeoutConfig, *kAllowedLocationsDefault,
1808 nullptr /* gcs additional header */, false /* compose append */);
1809
1810 std::vector<string> children;
1811 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children));
1812
1813 EXPECT_EQ(std::vector<string>({"file3.txt", "subpath/"}), children);
1814 }
1815
TEST(GcsFileSystemTest,GetChildren_ThreeFiles_NoSlash)1816 TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) {
1817 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1818 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1819 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1820 "path%2F\n"
1821 "Auth Token: fake_token\n"
1822 "Timeouts: 5 1 10\n",
1823 "{\"items\": [ "
1824 " { \"name\": \"path/file1.txt\" },"
1825 " { \"name\": \"path/file3.txt\" }],"
1826 "\"prefixes\": [\"path/subpath/\"]}")});
1827 GcsFileSystem fs(
1828 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1829 std::unique_ptr<HttpRequest::Factory>(
1830 new FakeHttpRequestFactory(&requests)),
1831 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1832 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1833 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1834 0 /* matching paths cache max entries */, kTestRetryConfig,
1835 kTestTimeoutConfig, *kAllowedLocationsDefault,
1836 nullptr /* gcs additional header */, false /* compose append */);
1837
1838 std::vector<string> children;
1839 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", nullptr, &children));
1840
1841 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/"}),
1842 children);
1843 }
1844
TEST(GcsFileSystemTest,GetChildren_Root)1845 TEST(GcsFileSystemTest, GetChildren_Root) {
1846 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1847 "Uri: https://www.googleapis.com/storage/v1/b/bucket-a-b-c/o?"
1848 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F\n"
1849 "Auth Token: fake_token\n"
1850 "Timeouts: 5 1 10\n",
1851 "{}")});
1852 GcsFileSystem fs(
1853 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1854 std::unique_ptr<HttpRequest::Factory>(
1855 new FakeHttpRequestFactory(&requests)),
1856 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1857 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1858 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1859 0 /* matching paths cache max entries */, kTestRetryConfig,
1860 kTestTimeoutConfig, *kAllowedLocationsDefault,
1861 nullptr /* gcs additional header */, false /* compose append */);
1862
1863 std::vector<string> children;
1864 TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", nullptr, &children));
1865
1866 EXPECT_EQ(0, children.size());
1867 }
1868
TEST(GcsFileSystemTest,GetChildren_Empty)1869 TEST(GcsFileSystemTest, GetChildren_Empty) {
1870 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1871 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1872 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1873 "path%2F\n"
1874 "Auth Token: fake_token\n"
1875 "Timeouts: 5 1 10\n",
1876 "{}")});
1877 GcsFileSystem fs(
1878 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1879 std::unique_ptr<HttpRequest::Factory>(
1880 new FakeHttpRequestFactory(&requests)),
1881 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1882 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1883 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1884 0 /* matching paths cache max entries */, kTestRetryConfig,
1885 kTestTimeoutConfig, *kAllowedLocationsDefault,
1886 nullptr /* gcs additional header */, false /* compose append */);
1887
1888 std::vector<string> children;
1889 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children));
1890
1891 EXPECT_EQ(0, children.size());
1892 }
1893
TEST(GcsFileSystemTest,GetChildren_Pagination)1894 TEST(GcsFileSystemTest, GetChildren_Pagination) {
1895 std::vector<HttpRequest*> requests(
1896 {new FakeHttpRequest(
1897 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1898 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&"
1899 "prefix=path%2F\n"
1900 "Auth Token: fake_token\n"
1901 "Timeouts: 5 1 10\n",
1902 "{\"nextPageToken\": \"ABCD==\", "
1903 "\"items\": [ "
1904 " { \"name\": \"path/file1.txt\" },"
1905 " { \"name\": \"path/file3.txt\" }],"
1906 "\"prefixes\": [\"path/subpath/\"]}"),
1907 new FakeHttpRequest(
1908 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1909 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&"
1910 "prefix=path%2F"
1911 "&pageToken=ABCD==\n"
1912 "Auth Token: fake_token\n"
1913 "Timeouts: 5 1 10\n",
1914 "{\"items\": [ "
1915 " { \"name\": \"path/file4.txt\" },"
1916 " { \"name\": \"path/file5.txt\" }]}")});
1917
1918 GcsFileSystem fs(
1919 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1920 std::unique_ptr<HttpRequest::Factory>(
1921 new FakeHttpRequestFactory(&requests)),
1922 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1923 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1924 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1925 0 /* matching paths cache max entries */, kTestRetryConfig,
1926 kTestTimeoutConfig, *kAllowedLocationsDefault,
1927 nullptr /* gcs additional header */, false /* compose append */);
1928
1929 std::vector<string> children;
1930 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", nullptr, &children));
1931
1932 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/",
1933 "file4.txt", "file5.txt"}),
1934 children);
1935 }
1936
TEST(GcsFileSystemTest,GetMatchingPaths_NoWildcard)1937 TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) {
1938 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1939 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1940 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
1941 "Auth Token: fake_token\n"
1942 "Timeouts: 5 1 10\n",
1943 "{\"items\": [ "
1944 " { \"name\": \"path/subpath/file2.txt\" }]}")});
1945 GcsFileSystem fs(
1946 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1947 std::unique_ptr<HttpRequest::Factory>(
1948 new FakeHttpRequestFactory(&requests)),
1949 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1950 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1951 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1952 0 /* matching paths cache max entries */, kTestRetryConfig,
1953 kTestTimeoutConfig, *kAllowedLocationsDefault,
1954 nullptr /* gcs additional header */, false /* compose append */);
1955
1956 std::vector<string> result;
1957 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt",
1958 nullptr, &result));
1959 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
1960 result);
1961 }
1962
TEST(GcsFileSystemTest,GetMatchingPaths_BucketAndWildcard)1963 TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) {
1964 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1965 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1966 "fields=items%2Fname%2CnextPageToken\n"
1967 "Auth Token: fake_token\n"
1968 "Timeouts: 5 1 10\n",
1969 "{\"items\": [ "
1970 " { \"name\": \"path/file1.txt\" },"
1971 " { \"name\": \"path/subpath/file2.txt\" },"
1972 " { \"name\": \"path/file3.txt\" }]}")});
1973 GcsFileSystem fs(
1974 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1975 std::unique_ptr<HttpRequest::Factory>(
1976 new FakeHttpRequestFactory(&requests)),
1977 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1978 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1979 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1980 0 /* matching paths cache max entries */, kTestRetryConfig,
1981 kTestTimeoutConfig, *kAllowedLocationsDefault,
1982 nullptr /* gcs additional header */, false /* compose append */);
1983
1984 std::vector<string> result;
1985 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", nullptr, &result));
1986 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file1.txt",
1987 "gs://bucket/path/file3.txt",
1988 "gs://bucket/path/subpath"}),
1989 result);
1990 }
1991
TEST(GcsFileSystemTest,GetMatchingPaths_FolderAndWildcard_Matches)1992 TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) {
1993 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1994 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1995 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
1996 "Auth Token: fake_token\n"
1997 "Timeouts: 5 1 10\n",
1998 "{\"items\": [ "
1999 " { \"name\": \"path/file1.txt\" },"
2000 " { \"name\": \"path/subpath/file2.txt\" },"
2001 " { \"name\": \"path/file3.txt\" }]}")});
2002 GcsFileSystem fs(
2003 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2004 std::unique_ptr<HttpRequest::Factory>(
2005 new FakeHttpRequestFactory(&requests)),
2006 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2007 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2008 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2009 0 /* matching paths cache max entries */, kTestRetryConfig,
2010 kTestTimeoutConfig, *kAllowedLocationsDefault,
2011 nullptr /* gcs additional header */, false /* compose append */);
2012
2013 std::vector<string> result;
2014 TF_EXPECT_OK(
2015 fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", nullptr, &result));
2016 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
2017 result);
2018 }
2019
TEST(GcsFileSystemTest,GetMatchingPaths_SelfDirectoryMarker)2020 TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) {
2021 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2022 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2023 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2024 "Auth Token: fake_token\n"
2025 "Timeouts: 5 1 10\n",
2026 "{\"items\": [ "
2027 " { \"name\": \"path/\" },"
2028 " { \"name\": \"path/file3.txt\" }]}")});
2029 GcsFileSystem fs(
2030 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2031 std::unique_ptr<HttpRequest::Factory>(
2032 new FakeHttpRequestFactory(&requests)),
2033 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2034 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2035 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2036 0 /* matching paths cache max entries */, kTestRetryConfig,
2037 kTestTimeoutConfig, *kAllowedLocationsDefault,
2038 nullptr /* gcs additional header */, false /* compose append */);
2039
2040 std::vector<string> result;
2041 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", nullptr, &result));
2042 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file3.txt"}), result);
2043 }
2044
TEST(GcsFileSystemTest,GetMatchingPaths_SlashInObjectName)2045 TEST(GcsFileSystemTest, GetMatchingPaths_SlashInObjectName) {
2046 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2047 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2048 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2049 "Auth Token: fake_token\n"
2050 "Timeouts: 5 1 10\n",
2051 "{\"items\": [ "
2052 " { \"name\": \"path/\" },"
2053 " { \"name\": \"path//foo.txt\" }]}")});
2054 GcsFileSystem fs(
2055 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2056 std::unique_ptr<HttpRequest::Factory>(
2057 new FakeHttpRequestFactory(&requests)),
2058 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2059 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2060 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2061 0 /* matching paths cache max entries */, kTestRetryConfig,
2062 kTestTimeoutConfig, *kAllowedLocationsDefault,
2063 nullptr /* gcs additional header */, false /* compose append */);
2064
2065 std::vector<string> result;
2066 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", nullptr, &result));
2067 EXPECT_EQ(std::vector<string>(), result);
2068 }
2069
TEST(GcsFileSystemTest,GetMatchingPaths_SlashInObjectNameEscaped)2070 TEST(GcsFileSystemTest, GetMatchingPaths_SlashInObjectNameEscaped) {
2071 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2072 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2073 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2074 "Auth Token: fake_token\n"
2075 "Timeouts: 5 1 10\n",
2076 "{\"items\": [ "
2077 " { \"name\": \"path/\" },"
2078 " { \"name\": \"path//foo.txt\" }]}")});
2079 GcsFileSystem fs(
2080 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2081 std::unique_ptr<HttpRequest::Factory>(
2082 new FakeHttpRequestFactory(&requests)),
2083 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2084 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2085 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2086 0 /* matching paths cache max entries */, kTestRetryConfig,
2087 kTestTimeoutConfig, *kAllowedLocationsDefault,
2088 nullptr /* gcs additional header */, false /* compose append */);
2089
2090 std::vector<string> result;
2091 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/\\/*", nullptr, &result));
2092 EXPECT_EQ(std::vector<string>({"gs://bucket/path//foo.txt"}), result);
2093 }
2094
TEST(GcsFileSystemTest,GetMatchingPaths_FolderAndWildcard_NoMatches)2095 TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) {
2096 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2097 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2098 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2099 "Auth Token: fake_token\n"
2100 "Timeouts: 5 1 10\n",
2101 "{\"items\": [ "
2102 " { \"name\": \"path/file1.txt\" },"
2103 " { \"name\": \"path/subpath/file2.txt\" },"
2104 " { \"name\": \"path/file3.txt\" }]}")});
2105 GcsFileSystem fs(
2106 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2107 std::unique_ptr<HttpRequest::Factory>(
2108 new FakeHttpRequestFactory(&requests)),
2109 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2110 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2111 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2112 0 /* matching paths cache max entries */, kTestRetryConfig,
2113 kTestTimeoutConfig, *kAllowedLocationsDefault,
2114 nullptr /* gcs additional header */, false /* compose append */);
2115
2116 std::vector<string> result;
2117 TF_EXPECT_OK(
2118 fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", nullptr, &result));
2119 EXPECT_EQ(std::vector<string>(), result);
2120 }
2121
TEST(GcsFileSystemTest,GetMatchingPaths_OnlyWildcard)2122 TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) {
2123 std::vector<HttpRequest*> requests;
2124 GcsFileSystem fs(
2125 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2126 std::unique_ptr<HttpRequest::Factory>(
2127 new FakeHttpRequestFactory(&requests)),
2128 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2129 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2130 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2131 0 /* matching paths cache max entries */, kTestRetryConfig,
2132 kTestTimeoutConfig, *kAllowedLocationsDefault,
2133 nullptr /* gcs additional header */, false /* compose append */);
2134
2135 std::vector<string> result;
2136 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
2137 fs.GetMatchingPaths("gs://*", nullptr, &result).code());
2138 }
2139
TEST(GcsFileSystemTest,GetMatchingPaths_Cache)2140 TEST(GcsFileSystemTest, GetMatchingPaths_Cache) {
2141 std::vector<HttpRequest*> requests(
2142 {new FakeHttpRequest(
2143 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2144 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
2145 "Auth Token: fake_token\n"
2146 "Timeouts: 5 1 10\n",
2147 "{\"items\": [ "
2148 " { \"name\": \"path/subpath/file2.txt\" }]}"),
2149 new FakeHttpRequest(
2150 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2151 "fields=items%2Fname%2CnextPageToken\n"
2152 "Auth Token: fake_token\n"
2153 "Timeouts: 5 1 10\n",
2154 "{\"items\": [ "
2155 " { \"name\": \"path/file1.txt\" },"
2156 " { \"name\": \"path/subpath/file2.txt\" },"
2157 " { \"name\": \"path/file3.txt\" }]}")});
2158 GcsFileSystem fs(
2159 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2160 std::unique_ptr<HttpRequest::Factory>(
2161 new FakeHttpRequestFactory(&requests)),
2162 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2163 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2164 0 /* stat cache max entries */, 3600 /* matching paths cache max age */,
2165 0 /* matching paths cache max entries */, kTestRetryConfig,
2166 kTestTimeoutConfig, *kAllowedLocationsDefault,
2167 nullptr /* gcs additional header */, false /* compose append */);
2168
2169 // Repeated calls to fs.GetMatchingPaths on these patterns should not lead to
2170 // any additional HTTP requests to GCS.
2171 for (int i = 0; i < 10; i++) {
2172 std::vector<string> result;
2173 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt",
2174 nullptr, &result));
2175 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
2176 result);
2177 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", nullptr, &result));
2178 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file1.txt",
2179 "gs://bucket/path/file3.txt",
2180 "gs://bucket/path/subpath"}),
2181 result);
2182 }
2183 }
2184
TEST(GcsFileSystemTest,GetMatchingPaths_Cache_Flush)2185 TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) {
2186 std::vector<HttpRequest*> requests(
2187 {new FakeHttpRequest(
2188 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2189 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
2190 "Auth Token: fake_token\n"
2191 "Timeouts: 5 1 10\n",
2192 "{\"items\": [ "
2193 " { \"name\": \"path/subpath/file2.txt\" }]}"),
2194 new FakeHttpRequest(
2195 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2196 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
2197 "Auth Token: fake_token\n"
2198 "Timeouts: 5 1 10\n",
2199 "{\"items\": [ "
2200 " { \"name\": \"path/subpath/file2.txt\" }]}")});
2201 GcsFileSystem fs(
2202 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2203 std::unique_ptr<HttpRequest::Factory>(
2204 new FakeHttpRequestFactory(&requests)),
2205 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2206 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2207 0 /* stat cache max entries */, 3600 /* matching paths cache max age */,
2208 0 /* matching paths cache max entries */, kTestRetryConfig,
2209 kTestTimeoutConfig, *kAllowedLocationsDefault,
2210 nullptr /* gcs additional header */, false /* compose append */);
2211
2212 // This loop should trigger the first HTTP request to GCS.
2213 for (int i = 0; i < 10; i++) {
2214 std::vector<string> result;
2215 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt",
2216 nullptr, &result));
2217 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
2218 result);
2219 }
2220 // After flushing caches, there should be another (identical) request to GCS.
2221 fs.FlushCaches(nullptr);
2222 for (int i = 0; i < 10; i++) {
2223 std::vector<string> result;
2224 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt",
2225 nullptr, &result));
2226 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
2227 result);
2228 }
2229 }
2230
TEST(GcsFileSystemTest,DeleteFile)2231 TEST(GcsFileSystemTest, DeleteFile) {
2232 std::vector<HttpRequest*> requests(
2233 {new FakeHttpRequest(
2234 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2235 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
2236 "Auth Token: fake_token\n"
2237 "Timeouts: 5 1 10\n",
2238 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
2239 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2240 new FakeHttpRequest(
2241 "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
2242 "Auth Token: fake_token\n"
2243 "Range: 0-15\n"
2244 "Timeouts: 5 1 20\n",
2245 "01234567"),
2246 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2247 "/bucket/o/path%2Ffile1.txt\n"
2248 "Auth Token: fake_token\n"
2249 "Timeouts: 5 1 10\n"
2250 "Delete: yes\n",
2251 ""),
2252 new FakeHttpRequest(
2253 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2254 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
2255 "Auth Token: fake_token\n"
2256 "Timeouts: 5 1 10\n",
2257 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
2258 "\"updated\": \"2016-04-29T23:19:24.896Z\"}")),
2259 new FakeHttpRequest(
2260 "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
2261 "Auth Token: fake_token\n"
2262 "Range: 0-15\n"
2263 "Timeouts: 5 1 20\n",
2264 "76543210")});
2265 GcsFileSystem fs(
2266 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2267 std::unique_ptr<HttpRequest::Factory>(
2268 new FakeHttpRequestFactory(&requests)),
2269 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
2270 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2271 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2272 0 /* matching paths cache max entries */, kTestRetryConfig,
2273 kTestTimeoutConfig, *kAllowedLocationsDefault,
2274 nullptr /* gcs additional header */, false /* compose append */);
2275
2276 // Do an initial read of the file to load its contents into the block cache.
2277 char scratch[100];
2278 StringPiece result;
2279 std::unique_ptr<RandomAccessFile> file;
2280 TF_EXPECT_OK(
2281 fs.NewRandomAccessFile("gs://bucket/path/file1.txt", nullptr, &file));
2282 TF_EXPECT_OK(file->Read(0, 8, &result, scratch));
2283 EXPECT_EQ("01234567", result);
2284 // Deleting the file triggers the next HTTP request to GCS.
2285 TF_EXPECT_OK(fs.DeleteFile("gs://bucket/path/file1.txt", nullptr));
2286 // Re-reading the file causes its contents to be reloaded from GCS and not
2287 // from the block cache.
2288 TF_EXPECT_OK(file->Read(0, 8, &result, scratch));
2289 EXPECT_EQ("76543210", result);
2290 }
2291
TEST(GcsFileSystemTest,DeleteFile_NoObjectName)2292 TEST(GcsFileSystemTest, DeleteFile_NoObjectName) {
2293 std::vector<HttpRequest*> requests;
2294 GcsFileSystem fs(
2295 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2296 std::unique_ptr<HttpRequest::Factory>(
2297 new FakeHttpRequestFactory(&requests)),
2298 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2299 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2300 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2301 0 /* matching paths cache max entries */, kTestRetryConfig,
2302 kTestTimeoutConfig, *kAllowedLocationsDefault,
2303 nullptr /* gcs additional header */, false /* compose append */);
2304
2305 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
2306 fs.DeleteFile("gs://bucket/", nullptr).code());
2307 }
2308
TEST(GcsFileSystemTest,DeleteFile_StatCacheRemoved)2309 TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) {
2310 std::vector<HttpRequest*> requests(
2311 {new FakeHttpRequest(
2312 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2313 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2314 "Auth Token: fake_token\n"
2315 "Timeouts: 5 1 10\n",
2316 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2317 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2318 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2319 "/bucket/o/file.txt\n"
2320 "Auth Token: fake_token\n"
2321 "Timeouts: 5 1 10\n"
2322 "Delete: yes\n",
2323 ""),
2324 new FakeHttpRequest(
2325 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2326 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2327 "Auth Token: fake_token\n"
2328 "Timeouts: 5 1 10\n",
2329 "", errors::NotFound("404"), 404),
2330 new FakeHttpRequest(
2331 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2332 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
2333 "&maxResults=1\n"
2334 "Auth Token: fake_token\n"
2335 "Timeouts: 5 1 10\n",
2336 "{}")});
2337 GcsFileSystem fs(
2338 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2339 std::unique_ptr<HttpRequest::Factory>(
2340 new FakeHttpRequestFactory(&requests)),
2341 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
2342 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2343 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2344 0 /* matching paths cache max entries */, kTestRetryConfig,
2345 kTestTimeoutConfig, *kAllowedLocationsDefault,
2346 nullptr /* gcs additional header */, false /* compose append */);
2347
2348 // Stats the file first so the stat is cached.
2349 FileStatistics stat_before_deletion;
2350 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat_before_deletion));
2351 EXPECT_EQ(1010, stat_before_deletion.length);
2352
2353 TF_EXPECT_OK(fs.DeleteFile("gs://bucket/file.txt", nullptr));
2354
2355 FileStatistics stat_after_deletion;
2356 EXPECT_EQ(
2357 error::Code::NOT_FOUND,
2358 fs.Stat("gs://bucket/file.txt", nullptr, &stat_after_deletion).code());
2359 }
2360
TEST(GcsFileSystemTest,DeleteDir_Empty)2361 TEST(GcsFileSystemTest, DeleteDir_Empty) {
2362 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2363 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2364 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
2365 "Auth Token: fake_token\n"
2366 "Timeouts: 5 1 10\n",
2367 "{}")});
2368 GcsFileSystem fs(
2369 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2370 std::unique_ptr<HttpRequest::Factory>(
2371 new FakeHttpRequestFactory(&requests)),
2372 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2373 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2374 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2375 0 /* matching paths cache max entries */, kTestRetryConfig,
2376 kTestTimeoutConfig, *kAllowedLocationsDefault,
2377 nullptr /* gcs additional header */, false /* compose append */);
2378
2379 TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/", nullptr));
2380 }
2381
TEST(GcsFileSystemTest,DeleteDir_OnlyDirMarkerLeft)2382 TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) {
2383 std::vector<HttpRequest*> requests(
2384 {new FakeHttpRequest(
2385 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2386 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
2387 "Auth Token: fake_token\n"
2388 "Timeouts: 5 1 10\n",
2389 "{\"items\": [ "
2390 " { \"name\": \"path/\" }]}"),
2391 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2392 "/bucket/o/path%2F\n"
2393 "Auth Token: fake_token\n"
2394 "Timeouts: 5 1 10\n"
2395 "Delete: yes\n",
2396 "")});
2397 GcsFileSystem fs(
2398 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2399 std::unique_ptr<HttpRequest::Factory>(
2400 new FakeHttpRequestFactory(&requests)),
2401 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2402 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2403 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2404 0 /* matching paths cache max entries */, kTestRetryConfig,
2405 kTestTimeoutConfig, *kAllowedLocationsDefault,
2406 nullptr /* gcs additional header */, false /* compose append */);
2407
2408 TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/", nullptr));
2409 }
2410
TEST(GcsFileSystemTest,DeleteDir_BucketOnly)2411 TEST(GcsFileSystemTest, DeleteDir_BucketOnly) {
2412 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2413 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?fields=items%2F"
2414 "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n"
2415 "Timeouts: 5 1 10\n",
2416 "{}")});
2417 GcsFileSystem fs(
2418 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2419 std::unique_ptr<HttpRequest::Factory>(
2420 new FakeHttpRequestFactory(&requests)),
2421 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2422 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2423 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2424 0 /* matching paths cache max entries */, kTestRetryConfig,
2425 kTestTimeoutConfig, *kAllowedLocationsDefault,
2426 nullptr /* gcs additional header */, false /* compose append */);
2427
2428 TF_EXPECT_OK(fs.DeleteDir("gs://bucket", nullptr));
2429 }
2430
TEST(GcsFileSystemTest,DeleteDir_NonEmpty)2431 TEST(GcsFileSystemTest, DeleteDir_NonEmpty) {
2432 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2433 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2434 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
2435 "Auth Token: fake_token\n"
2436 "Timeouts: 5 1 10\n",
2437 "{\"items\": [ "
2438 " { \"name\": \"path/file1.txt\" }]}")});
2439 GcsFileSystem fs(
2440 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2441 std::unique_ptr<HttpRequest::Factory>(
2442 new FakeHttpRequestFactory(&requests)),
2443 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2444 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2445 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2446 0 /* matching paths cache max entries */, kTestRetryConfig,
2447 kTestTimeoutConfig, *kAllowedLocationsDefault,
2448 nullptr /* gcs additional header */, false /* compose append */);
2449
2450 EXPECT_EQ(error::Code::FAILED_PRECONDITION,
2451 fs.DeleteDir("gs://bucket/path/", nullptr).code());
2452 }
2453
TEST(GcsFileSystemTest,GetFileSize)2454 TEST(GcsFileSystemTest, GetFileSize) {
2455 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2456 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2457 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2458 "Auth Token: fake_token\n"
2459 "Timeouts: 5 1 10\n",
2460 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2461 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2462 GcsFileSystem fs(
2463 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2464 std::unique_ptr<HttpRequest::Factory>(
2465 new FakeHttpRequestFactory(&requests)),
2466 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2467 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2468 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2469 0 /* matching paths cache max entries */, kTestRetryConfig,
2470 kTestTimeoutConfig, *kAllowedLocationsDefault,
2471 nullptr /* gcs additional header */, false /* compose append */);
2472
2473 uint64 size;
2474 TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", nullptr, &size));
2475 EXPECT_EQ(1010, size);
2476 }
2477
TEST(GcsFileSystemTest,GetFileSize_NoObjectName)2478 TEST(GcsFileSystemTest, GetFileSize_NoObjectName) {
2479 std::vector<HttpRequest*> requests;
2480 GcsFileSystem fs(
2481 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2482 std::unique_ptr<HttpRequest::Factory>(
2483 new FakeHttpRequestFactory(&requests)),
2484 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2485 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2486 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2487 0 /* matching paths cache max entries */, kTestRetryConfig,
2488 kTestTimeoutConfig, *kAllowedLocationsDefault,
2489 nullptr /* gcs additional header */, false /* compose append */);
2490
2491 uint64 size;
2492 EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
2493 fs.GetFileSize("gs://bucket/", nullptr, &size).code());
2494 }
2495
TEST(GcsFileSystemTest,RenameFile_Folder)2496 TEST(GcsFileSystemTest, RenameFile_Folder) {
2497 std::vector<HttpRequest*> requests(
2498 {// Check if this is a folder or an object.
2499 new FakeHttpRequest(
2500 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2501 "fields=items%2Fname%2CnextPageToken&prefix=path1%2F"
2502 "&maxResults=1\n"
2503 "Auth Token: fake_token\n"
2504 "Timeouts: 5 1 10\n",
2505 "{\"items\": [ "
2506 " { \"name\": \"path1/subfolder/file1.txt\" }]}"),
2507 // Requesting the full list of files in the folder.
2508 new FakeHttpRequest(
2509 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2510 "fields=items%2Fname%2CnextPageToken&prefix=path1%2F\n"
2511 "Auth Token: fake_token\n"
2512 "Timeouts: 5 1 10\n",
2513 "{\"items\": [ "
2514 " { \"name\": \"path1/\" }," // A directory marker.
2515 " { \"name\": \"path1/subfolder/file1.txt\" },"
2516 " { \"name\": \"path1/file2.txt\" }]}"),
2517 // Copying the directory marker.
2518 new FakeHttpRequest(
2519 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2520 "path1%2F/rewriteTo/b/bucket/o/path2%2F\n"
2521 "Auth Token: fake_token\n"
2522 "Post: yes\n"
2523 "Timeouts: 5 1 10\n",
2524 "{\"done\": true}"),
2525 // Deleting the original directory marker.
2526 new FakeHttpRequest(
2527 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2528 "path1%2F\n"
2529 "Auth Token: fake_token\n"
2530 "Timeouts: 5 1 10\n"
2531 "Delete: yes\n",
2532 ""),
2533 // Copying the first file.
2534 new FakeHttpRequest(
2535 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2536 "path1%2Fsubfolder%2Ffile1.txt/rewriteTo/b/bucket/o/"
2537 "path2%2Fsubfolder%2Ffile1.txt\n"
2538 "Auth Token: fake_token\n"
2539 "Post: yes\n"
2540 "Timeouts: 5 1 10\n",
2541 "{\"done\": true}"),
2542 // Deleting the first original file.
2543 new FakeHttpRequest(
2544 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2545 "path1%2Fsubfolder%2Ffile1.txt\n"
2546 "Auth Token: fake_token\n"
2547 "Timeouts: 5 1 10\n"
2548 "Delete: yes\n",
2549 ""),
2550 // Copying the second file.
2551 new FakeHttpRequest(
2552 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2553 "path1%2Ffile2.txt/rewriteTo/b/bucket/o/path2%2Ffile2.txt\n"
2554 "Auth Token: fake_token\n"
2555 "Post: yes\n"
2556 "Timeouts: 5 1 10\n",
2557 "{\"done\": true}"),
2558 // Deleting the second original file.
2559 new FakeHttpRequest(
2560 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2561 "path1%2Ffile2.txt\n"
2562 "Auth Token: fake_token\n"
2563 "Timeouts: 5 1 10\n"
2564 "Delete: yes\n",
2565 "")});
2566 GcsFileSystem fs(
2567 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2568 std::unique_ptr<HttpRequest::Factory>(
2569 new FakeHttpRequestFactory(&requests)),
2570 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2571 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2572 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2573 0 /* matching paths cache max entries */, kTestRetryConfig,
2574 kTestTimeoutConfig, *kAllowedLocationsDefault,
2575 nullptr /* gcs additional header */, false /* compose append */);
2576
2577 TF_EXPECT_OK(
2578 fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/", nullptr));
2579 }
2580
TEST(GcsFileSystemTest,RenameFile_Object)2581 TEST(GcsFileSystemTest, RenameFile_Object) {
2582 std::vector<HttpRequest*> requests(
2583 {new FakeHttpRequest(
2584 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2585 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2586 "Auth Token: fake_token\n"
2587 "Timeouts: 5 1 10\n",
2588 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
2589 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2590 new FakeHttpRequest(
2591 "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
2592 "Auth Token: fake_token\n"
2593 "Range: 0-15\n"
2594 "Timeouts: 5 1 20\n",
2595 "01234567"),
2596 new FakeHttpRequest(
2597 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2598 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2599 "Auth Token: fake_token\n"
2600 "Timeouts: 5 1 10\n",
2601 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
2602 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2603 new FakeHttpRequest(
2604 "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
2605 "Auth Token: fake_token\n"
2606 "Range: 0-15\n"
2607 "Timeouts: 5 1 20\n",
2608 "76543210"),
2609 // IsDirectory is checking whether there are children objects.
2610 new FakeHttpRequest(
2611 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2612 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2613 "&maxResults=1\n"
2614 "Auth Token: fake_token\n"
2615 "Timeouts: 5 1 10\n",
2616 "{}"),
2617 // Copying to the new location.
2618 new FakeHttpRequest(
2619 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2620 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2621 "Auth Token: fake_token\n"
2622 "Post: yes\n"
2623 "Timeouts: 5 1 10\n",
2624 "{\"done\": true}"),
2625 // Deleting the original file.
2626 new FakeHttpRequest(
2627 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2628 "path%2Fsrc.txt\n"
2629 "Auth Token: fake_token\n"
2630 "Timeouts: 5 1 10\n"
2631 "Delete: yes\n",
2632 ""),
2633 new FakeHttpRequest(
2634 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2635 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2636 "Auth Token: fake_token\n"
2637 "Timeouts: 5 1 10\n",
2638 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
2639 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2640 new FakeHttpRequest(
2641 "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
2642 "Auth Token: fake_token\n"
2643 "Range: 0-15\n"
2644 "Timeouts: 5 1 20\n",
2645 "89abcdef"),
2646 new FakeHttpRequest(
2647 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2648 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2649 "Auth Token: fake_token\n"
2650 "Timeouts: 5 1 10\n",
2651 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
2652 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2653 new FakeHttpRequest(
2654 "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
2655 "Auth Token: fake_token\n"
2656 "Range: 0-15\n"
2657 "Timeouts: 5 1 20\n",
2658 "fedcba98")});
2659 GcsFileSystem fs(
2660 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2661 std::unique_ptr<HttpRequest::Factory>(
2662 new FakeHttpRequestFactory(&requests)),
2663 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
2664 64 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2665 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2666 0 /* matching paths cache max entries */, kTestRetryConfig,
2667 kTestTimeoutConfig, *kAllowedLocationsDefault,
2668 nullptr /* gcs additional header */, false /* compose append */);
2669 // Do an initial read of the source and destination files to load their
2670 // contents into the block cache.
2671 char scratch[100];
2672 StringPiece result;
2673 std::unique_ptr<RandomAccessFile> src;
2674 std::unique_ptr<RandomAccessFile> dst;
2675 TF_EXPECT_OK(
2676 fs.NewRandomAccessFile("gs://bucket/path/src.txt", nullptr, &src));
2677 TF_EXPECT_OK(src->Read(0, 8, &result, scratch));
2678 EXPECT_EQ("01234567", result);
2679 TF_EXPECT_OK(
2680 fs.NewRandomAccessFile("gs://bucket/path/dst.txt", nullptr, &dst));
2681 TF_EXPECT_OK(dst->Read(0, 8, &result, scratch));
2682 EXPECT_EQ("76543210", result);
2683 // Now rename src to dst. This should flush the block cache for both files.
2684 TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt",
2685 "gs://bucket/path/dst.txt", nullptr));
2686 // Re-read both files. This should reload their contents from GCS.
2687 TF_EXPECT_OK(src->Read(0, 8, &result, scratch));
2688 EXPECT_EQ("89abcdef", result);
2689 TF_EXPECT_OK(dst->Read(0, 8, &result, scratch));
2690 EXPECT_EQ("fedcba98", result);
2691 }
2692
TEST(GcsFileSystemTest,RenameFile_Object_FlushTargetStatCache)2693 TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) {
2694 std::vector<HttpRequest*> requests(
2695 {// Stat the target file.
2696 new FakeHttpRequest(
2697 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2698 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2699 "Auth Token: fake_token\n"
2700 "Timeouts: 5 1 10\n",
2701 strings::StrCat("{\"size\": \"1000\",\"generation\": \"1\","
2702 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2703 // IsDirectory is checking whether there are children objects.
2704 new FakeHttpRequest(
2705 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2706 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2707 "&maxResults=1\n"
2708 "Auth Token: fake_token\n"
2709 "Timeouts: 5 1 10\n",
2710 "{}"),
2711 // IsDirectory is checking if the path exists as an object.
2712 new FakeHttpRequest(
2713 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2714 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2715 "Auth Token: fake_token\n"
2716 "Timeouts: 5 1 10\n",
2717 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2718 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2719 // Copying to the new location.
2720 new FakeHttpRequest(
2721 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2722 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2723 "Auth Token: fake_token\n"
2724 "Post: yes\n"
2725 "Timeouts: 5 1 10\n",
2726 "{\"done\": true}"),
2727 // Deleting the original file.
2728 new FakeHttpRequest(
2729 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2730 "path%2Fsrc.txt\n"
2731 "Auth Token: fake_token\n"
2732 "Timeouts: 5 1 10\n"
2733 "Delete: yes\n",
2734 ""),
2735 new FakeHttpRequest(
2736 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2737 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2738 "Auth Token: fake_token\n"
2739 "Timeouts: 5 1 10\n",
2740 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2741 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2742 GcsFileSystem fs(
2743 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2744 std::unique_ptr<HttpRequest::Factory>(
2745 new FakeHttpRequestFactory(&requests)),
2746 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2747 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2748 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2749 0 /* matching paths cache max entries */, kTestRetryConfig,
2750 kTestTimeoutConfig, *kAllowedLocationsDefault,
2751 nullptr /* gcs additional header */, false /* compose append */);
2752 // Do an initial stat of the destination file to load their contents into the
2753 // stat cache.
2754 FileStatistics stat_before_renaming;
2755 TF_EXPECT_OK(
2756 fs.Stat("gs://bucket/path/dst.txt", nullptr, &stat_before_renaming));
2757 EXPECT_EQ(1000, stat_before_renaming.length);
2758
2759 TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt",
2760 "gs://bucket/path/dst.txt", nullptr));
2761
2762 FileStatistics stat_after_renaming;
2763 TF_EXPECT_OK(
2764 fs.Stat("gs://bucket/path/dst.txt", nullptr, &stat_after_renaming));
2765 EXPECT_EQ(1010, stat_after_renaming.length);
2766 }
2767
2768 /// Tests the scenario when deletion returns a failure, but actually succeeds.
TEST(GcsFileSystemTest,RenameFile_Object_DeletionRetried)2769 TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
2770 std::vector<HttpRequest*> requests(
2771 {// IsDirectory is checking whether there are children objects.
2772 new FakeHttpRequest(
2773 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2774 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2775 "&maxResults=1\n"
2776 "Auth Token: fake_token\n"
2777 "Timeouts: 5 1 10\n",
2778 "{}"),
2779 // IsDirectory is checking if the path exists as an object.
2780 new FakeHttpRequest(
2781 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2782 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2783 "Auth Token: fake_token\n"
2784 "Timeouts: 5 1 10\n",
2785 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2786 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2787 // Copying to the new location.
2788 new FakeHttpRequest(
2789 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2790 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2791 "Auth Token: fake_token\n"
2792 "Post: yes\n"
2793 "Timeouts: 5 1 10\n",
2794 "{\"done\": true}"),
2795 // Deleting the original file - the deletion returns a failure.
2796 new FakeHttpRequest(
2797 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2798 "path%2Fsrc.txt\n"
2799 "Auth Token: fake_token\n"
2800 "Timeouts: 5 1 10\n"
2801 "Delete: yes\n",
2802 "", errors::Unavailable("503"), 503),
2803 // Deleting the original file again - the deletion returns NOT_FOUND.
2804 new FakeHttpRequest(
2805 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2806 "path%2Fsrc.txt\n"
2807 "Auth Token: fake_token\n"
2808 "Timeouts: 5 1 10\n"
2809 "Delete: yes\n",
2810 "", errors::NotFound("404"), 404)});
2811 GcsFileSystem fs(
2812 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2813 std::unique_ptr<HttpRequest::Factory>(
2814 new FakeHttpRequestFactory(&requests)),
2815 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2816 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2817 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2818 0 /* matching paths cache max entries */, kTestRetryConfig,
2819 kTestTimeoutConfig, *kAllowedLocationsDefault,
2820 nullptr /* gcs additional header */, false /* compose append */);
2821
2822 TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt",
2823 "gs://bucket/path/dst.txt", nullptr));
2824 }
2825
2826 /// Tests the case when rewrite couldn't complete in one RPC.
TEST(GcsFileSystemTest,RenameFile_Object_Incomplete)2827 TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) {
2828 std::vector<HttpRequest*> requests(
2829 {// IsDirectory is checking whether there are children objects.
2830 new FakeHttpRequest(
2831 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2832 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2833 "&maxResults=1\n"
2834 "Auth Token: fake_token\n"
2835 "Timeouts: 5 1 10\n",
2836 "{}"),
2837 // IsDirectory is checking if the path exists as an object.
2838 new FakeHttpRequest(
2839 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2840 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2841 "Auth Token: fake_token\n"
2842 "Timeouts: 5 1 10\n",
2843 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2844 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2845 // Copying to the new location.
2846 new FakeHttpRequest(
2847 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2848 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2849 "Auth Token: fake_token\n"
2850 "Post: yes\n"
2851 "Timeouts: 5 1 10\n",
2852 "{\"done\": false}")});
2853 GcsFileSystem fs(
2854 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2855 std::unique_ptr<HttpRequest::Factory>(
2856 new FakeHttpRequestFactory(&requests)),
2857 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2858 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2859 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2860 0 /* matching paths cache max entries */, kTestRetryConfig,
2861 kTestTimeoutConfig, *kAllowedLocationsDefault,
2862 nullptr /* gcs additional header */, false /* compose append */);
2863
2864 EXPECT_EQ(errors::Code::UNIMPLEMENTED,
2865 fs.RenameFile("gs://bucket/path/src.txt",
2866 "gs://bucket/path/dst.txt", nullptr)
2867 .code());
2868 }
2869
TEST(GcsFileSystemTest,Stat_Object)2870 TEST(GcsFileSystemTest, Stat_Object) {
2871 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2872 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2873 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2874 "Auth Token: fake_token\n"
2875 "Timeouts: 5 1 10\n",
2876 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2877 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2878 GcsFileSystem fs(
2879 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2880 std::unique_ptr<HttpRequest::Factory>(
2881 new FakeHttpRequestFactory(&requests)),
2882 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2883 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2884 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2885 0 /* matching paths cache max entries */, kTestRetryConfig,
2886 kTestTimeoutConfig, *kAllowedLocationsDefault,
2887 nullptr /* gcs additional header */, false /* compose append */);
2888
2889 FileStatistics stat;
2890 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
2891 EXPECT_EQ(1010, stat.length);
2892 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
2893 EXPECT_FALSE(stat.is_directory);
2894 }
2895
TEST(GcsFileSystemTest,Stat_Folder)2896 TEST(GcsFileSystemTest, Stat_Folder) {
2897 std::vector<HttpRequest*> requests(
2898 {new FakeHttpRequest(
2899 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2900 "subfolder?fields=size%2Cgeneration%2Cupdated\n"
2901 "Auth Token: fake_token\n"
2902 "Timeouts: 5 1 10\n",
2903 "", errors::NotFound("404"), 404),
2904 new FakeHttpRequest(
2905 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2906 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
2907 "&maxResults=1\n"
2908 "Auth Token: fake_token\n"
2909 "Timeouts: 5 1 10\n",
2910 "{\"items\": [ "
2911 " { \"name\": \"subfolder/\" }]}")});
2912 GcsFileSystem fs(
2913 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2914 std::unique_ptr<HttpRequest::Factory>(
2915 new FakeHttpRequestFactory(&requests)),
2916 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2917 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2918 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2919 0 /* matching paths cache max entries */, kTestRetryConfig,
2920 kTestTimeoutConfig, *kAllowedLocationsDefault,
2921 nullptr /* gcs additional header */, false /* compose append */);
2922
2923 FileStatistics stat;
2924 TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", nullptr, &stat));
2925 EXPECT_EQ(0, stat.length);
2926 EXPECT_EQ(0, stat.mtime_nsec);
2927 EXPECT_TRUE(stat.is_directory);
2928 }
2929
TEST(GcsFileSystemTest,Stat_ObjectOrFolderNotFound)2930 TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) {
2931 std::vector<HttpRequest*> requests(
2932 {new FakeHttpRequest(
2933 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2934 "path?fields=size%2Cgeneration%2Cupdated\n"
2935 "Auth Token: fake_token\n"
2936 "Timeouts: 5 1 10\n",
2937 "", errors::NotFound("404"), 404),
2938 new FakeHttpRequest(
2939 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2940 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
2941 "&maxResults=1\n"
2942 "Auth Token: fake_token\n"
2943 "Timeouts: 5 1 10\n",
2944 "{}")});
2945 GcsFileSystem fs(
2946 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2947 std::unique_ptr<HttpRequest::Factory>(
2948 new FakeHttpRequestFactory(&requests)),
2949 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2950 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2951 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2952 0 /* matching paths cache max entries */, kTestRetryConfig,
2953 kTestTimeoutConfig, *kAllowedLocationsDefault,
2954 nullptr /* gcs additional header */, false /* compose append */);
2955
2956 FileStatistics stat;
2957 EXPECT_EQ(error::Code::NOT_FOUND,
2958 fs.Stat("gs://bucket/path", nullptr, &stat).code());
2959 }
2960
TEST(GcsFileSystemTest,Stat_Bucket)2961 TEST(GcsFileSystemTest, Stat_Bucket) {
2962 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2963 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2964 "Auth Token: fake_token\n"
2965 "Timeouts: 5 1 10\n",
2966 "{}")});
2967 GcsFileSystem fs(
2968 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2969 std::unique_ptr<HttpRequest::Factory>(
2970 new FakeHttpRequestFactory(&requests)),
2971 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2972 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2973 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2974 0 /* matching paths cache max entries */, kTestRetryConfig,
2975 kTestTimeoutConfig, *kAllowedLocationsDefault,
2976 nullptr /* gcs additional header */, false /* compose append */);
2977
2978 FileStatistics stat;
2979 TF_EXPECT_OK(fs.Stat("gs://bucket/", nullptr, &stat));
2980 EXPECT_EQ(0, stat.length);
2981 EXPECT_EQ(0, stat.mtime_nsec);
2982 EXPECT_TRUE(stat.is_directory);
2983 }
2984
TEST(GcsFileSystemTest,Stat_BucketNotFound)2985 TEST(GcsFileSystemTest, Stat_BucketNotFound) {
2986 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2987 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2988 "Auth Token: fake_token\n"
2989 "Timeouts: 5 1 10\n",
2990 "", errors::NotFound("404"), 404)});
2991 GcsFileSystem fs(
2992 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2993 std::unique_ptr<HttpRequest::Factory>(
2994 new FakeHttpRequestFactory(&requests)),
2995 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2996 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2997 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2998 0 /* matching paths cache max entries */, kTestRetryConfig,
2999 kTestTimeoutConfig, *kAllowedLocationsDefault,
3000 nullptr /* gcs additional header */, false /* compose append */);
3001
3002 FileStatistics stat;
3003 EXPECT_EQ(error::Code::NOT_FOUND,
3004 fs.Stat("gs://bucket/", nullptr, &stat).code());
3005 }
3006
TEST(GcsFileSystemTest,Stat_Cache)3007 TEST(GcsFileSystemTest, Stat_Cache) {
3008 std::vector<HttpRequest*> requests(
3009 {new FakeHttpRequest(
3010 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3011 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3012 "Auth Token: fake_token\n"
3013 "Timeouts: 5 1 10\n",
3014 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3015 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3016 new FakeHttpRequest(
3017 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3018 "subfolder%2F?fields=size%2Cgeneration%2Cupdated\n"
3019 "Auth Token: fake_token\n"
3020 "Timeouts: 5 1 10\n",
3021 "", errors::NotFound("404"), 404),
3022 new FakeHttpRequest(
3023 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3024 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
3025 "&maxResults=1\n"
3026 "Auth Token: fake_token\n"
3027 "Timeouts: 5 1 10\n",
3028 "{\"items\": [ "
3029 " { \"name\": \"subfolder/\" }]}")});
3030 GcsFileSystem fs(
3031 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3032 std::unique_ptr<HttpRequest::Factory>(
3033 new FakeHttpRequestFactory(&requests)),
3034 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3035 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
3036 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3037 0 /* matching paths cache max entries */, kTestRetryConfig,
3038 kTestTimeoutConfig, *kAllowedLocationsDefault,
3039 nullptr /* gcs additional header */, false /* compose append */);
3040
3041 // Repeated calls to fs.Stat on these paths should not lead to any additional
3042 // HTTP requests to GCS.
3043 for (int i = 0; i < 10; i++) {
3044 FileStatistics stat;
3045 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
3046 EXPECT_EQ(1010, stat.length);
3047 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
3048 EXPECT_FALSE(stat.is_directory);
3049 TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder/", nullptr, &stat));
3050 EXPECT_EQ(0, stat.length);
3051 EXPECT_EQ(0, stat.mtime_nsec);
3052 EXPECT_TRUE(stat.is_directory);
3053 }
3054 }
3055
TEST(GcsFileSystemTest,Stat_Cache_Flush)3056 TEST(GcsFileSystemTest, Stat_Cache_Flush) {
3057 std::vector<HttpRequest*> requests(
3058 {new FakeHttpRequest(
3059 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3060 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3061 "Auth Token: fake_token\n"
3062 "Timeouts: 5 1 10\n",
3063 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3064 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3065 new FakeHttpRequest(
3066 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3067 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3068 "Auth Token: fake_token\n"
3069 "Timeouts: 5 1 10\n",
3070 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3071 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3072 GcsFileSystem fs(
3073 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3074 std::unique_ptr<HttpRequest::Factory>(
3075 new FakeHttpRequestFactory(&requests)),
3076 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3077 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
3078 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3079 0 /* matching paths cache max entries */, kTestRetryConfig,
3080 kTestTimeoutConfig, *kAllowedLocationsDefault,
3081 nullptr /* gcs additional header */, false /* compose append */);
3082 // There should be a single HTTP request to GCS for fs.Stat in this loop.
3083 for (int i = 0; i < 10; i++) {
3084 FileStatistics stat;
3085 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
3086 EXPECT_EQ(1010, stat.length);
3087 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
3088 EXPECT_FALSE(stat.is_directory);
3089 }
3090 // After flushing caches, there should be a second request to GCS for fs.Stat.
3091 fs.FlushCaches(nullptr);
3092 for (int i = 0; i < 10; i++) {
3093 FileStatistics stat;
3094 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
3095 EXPECT_EQ(1010, stat.length);
3096 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
3097 EXPECT_FALSE(stat.is_directory);
3098 }
3099 }
3100
TEST(GcsFileSystemTest,Stat_FilenameEndingWithSlash)3101 TEST(GcsFileSystemTest, Stat_FilenameEndingWithSlash) {
3102 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3103 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3104 "dir%2F?fields=size%2Cgeneration%2Cupdated\n"
3105 "Auth Token: fake_token\n"
3106 "Timeouts: 5 1 10\n",
3107 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
3108 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3109 GcsFileSystem fs(
3110 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3111 std::unique_ptr<HttpRequest::Factory>(
3112 new FakeHttpRequestFactory(&requests)),
3113 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3114 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3115 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3116 0 /* matching paths cache max entries */, kTestRetryConfig,
3117 kTestTimeoutConfig, *kAllowedLocationsDefault,
3118 nullptr /* gcs additional header */, false /* compose append */);
3119
3120 FileStatistics stat;
3121 TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", nullptr, &stat));
3122 EXPECT_EQ(5, stat.length);
3123 EXPECT_TRUE(stat.is_directory);
3124 }
3125
TEST(GcsFileSystemTest,IsDirectory_NotFound)3126 TEST(GcsFileSystemTest, IsDirectory_NotFound) {
3127 std::vector<HttpRequest*> requests(
3128 {new FakeHttpRequest(
3129 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3130 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
3131 "&maxResults=1\n"
3132 "Auth Token: fake_token\n"
3133 "Timeouts: 5 1 10\n",
3134 "{}"),
3135 new FakeHttpRequest(
3136 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3137 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3138 "Auth Token: fake_token\n"
3139 "Timeouts: 5 1 10\n",
3140 "", errors::NotFound("404"), 404)});
3141 GcsFileSystem fs(
3142 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3143 std::unique_ptr<HttpRequest::Factory>(
3144 new FakeHttpRequestFactory(&requests)),
3145 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3146 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3147 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3148 0 /* matching paths cache max entries */, kTestRetryConfig,
3149 kTestTimeoutConfig, *kAllowedLocationsDefault,
3150 nullptr /* gcs additional header */, false /* compose append */);
3151
3152 EXPECT_EQ(error::Code::NOT_FOUND,
3153 fs.IsDirectory("gs://bucket/file.txt", nullptr).code());
3154 }
3155
TEST(GcsFileSystemTest,IsDirectory_NotDirectoryButObject)3156 TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) {
3157 std::vector<HttpRequest*> requests(
3158 {new FakeHttpRequest(
3159 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3160 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
3161 "&maxResults=1\n"
3162 "Auth Token: fake_token\n"
3163 "Timeouts: 5 1 10\n",
3164 "{}"),
3165 new FakeHttpRequest(
3166 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3167 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3168 "Auth Token: fake_token\n"
3169 "Timeouts: 5 1 10\n",
3170 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3171 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3172 GcsFileSystem fs(
3173 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3174 std::unique_ptr<HttpRequest::Factory>(
3175 new FakeHttpRequestFactory(&requests)),
3176 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3177 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3178 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3179 0 /* matching paths cache max entries */, kTestRetryConfig,
3180 kTestTimeoutConfig, *kAllowedLocationsDefault,
3181 nullptr /* gcs additional header */, false /* compose append */);
3182
3183 EXPECT_EQ(error::Code::FAILED_PRECONDITION,
3184 fs.IsDirectory("gs://bucket/file.txt", nullptr).code());
3185 }
3186
TEST(GcsFileSystemTest,IsDirectory_Yes)3187 TEST(GcsFileSystemTest, IsDirectory_Yes) {
3188 std::vector<HttpRequest*> requests(
3189 {new FakeHttpRequest(
3190 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3191 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
3192 "&maxResults=1\n"
3193 "Auth Token: fake_token\n"
3194 "Timeouts: 5 1 10\n",
3195 "{\"items\": [{\"name\": \"subfolder/\"}]}"),
3196 new FakeHttpRequest(
3197 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3198 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
3199 "&maxResults=1\n"
3200 "Auth Token: fake_token\n"
3201 "Timeouts: 5 1 10\n",
3202 "{\"items\": [{\"name\": \"subfolder/\"}]}")});
3203 GcsFileSystem fs(
3204 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3205 std::unique_ptr<HttpRequest::Factory>(
3206 new FakeHttpRequestFactory(&requests)),
3207 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3208 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3209 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3210 0 /* matching paths cache max entries */, kTestRetryConfig,
3211 kTestTimeoutConfig, *kAllowedLocationsDefault,
3212 nullptr /* gcs additional header */, false /* compose append */);
3213
3214 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder", nullptr));
3215 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/", nullptr));
3216 }
3217
TEST(GcsFileSystemTest,IsDirectory_Bucket)3218 TEST(GcsFileSystemTest, IsDirectory_Bucket) {
3219 std::vector<HttpRequest*> requests(
3220 {new FakeHttpRequest(
3221 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3222 "Auth Token: fake_token\n"
3223 "Timeouts: 5 1 10\n",
3224 "{}"),
3225 new FakeHttpRequest(
3226 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3227 "Auth Token: fake_token\n"
3228 "Timeouts: 5 1 10\n",
3229 "{}")});
3230 GcsFileSystem fs(
3231 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3232 std::unique_ptr<HttpRequest::Factory>(
3233 new FakeHttpRequestFactory(&requests)),
3234 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3235 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3236 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3237 0 /* matching paths cache max entries */, kTestRetryConfig,
3238 kTestTimeoutConfig, *kAllowedLocationsDefault,
3239 nullptr /* gcs additional header */, false /* compose append */);
3240
3241 TF_EXPECT_OK(fs.IsDirectory("gs://bucket", nullptr));
3242 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/", nullptr));
3243 }
3244
TEST(GcsFileSystemTest,IsDirectory_BucketNotFound)3245 TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) {
3246 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3247 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3248 "Auth Token: fake_token\n"
3249 "Timeouts: 5 1 10\n",
3250 "", errors::NotFound("404"), 404)});
3251 GcsFileSystem fs(
3252 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3253 std::unique_ptr<HttpRequest::Factory>(
3254 new FakeHttpRequestFactory(&requests)),
3255 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3256 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3257 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3258 0 /* matching paths cache max entries */, kTestRetryConfig,
3259 kTestTimeoutConfig, *kAllowedLocationsDefault,
3260 nullptr /* gcs additional header */, false /* compose append */);
3261
3262 EXPECT_EQ(error::Code::NOT_FOUND,
3263 fs.IsDirectory("gs://bucket/", nullptr).code());
3264 }
3265
TEST(GcsFileSystemTest,CreateDir_Folder)3266 TEST(GcsFileSystemTest, CreateDir_Folder) {
3267 std::vector<HttpRequest*> requests(
3268
3269 {
3270 // File doesn't exist.
3271 new FakeHttpRequest(
3272 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3273 "subpath%2F?fields=size%2Cgeneration%2Cupdated\n"
3274 "Auth Token: fake_token\n"
3275 "Timeouts: 5 1 10\n",
3276 "{}"),
3277 // Simple upload.
3278 new FakeHttpRequest(
3279 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3280 "uploadType=media&name=subpath%2F&ifGenerationMatch=0\n"
3281 "Auth Token: fake_token\n"
3282 "Post: yes\n"
3283 "Timeouts: 5 1 10\n",
3284 ""),
3285 // File exists.
3286 new FakeHttpRequest(
3287 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3288 "subpath%2F?fields=size%2Cgeneration%2Cupdated\n"
3289 "Auth Token: fake_token\n"
3290 "Timeouts: 5 1 10\n",
3291 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3292 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3293 // File doesn't exist again.
3294 new FakeHttpRequest(
3295 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3296 "subpath%2F?fields=size%2Cgeneration%2Cupdated\n"
3297 "Auth Token: fake_token\n"
3298 "Timeouts: 5 1 10\n",
3299 "{}"),
3300 // Simulate object uploaded in between.
3301 new FakeHttpRequest(
3302 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3303 "uploadType=media&name=subpath%2F&ifGenerationMatch=0\n"
3304 "Auth Token: fake_token\n"
3305 "Post: yes\n"
3306 "Timeouts: 5 1 10\n",
3307 "", errors::FailedPrecondition("412"), 412),
3308 });
3309 GcsFileSystem fs(
3310 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3311 std::unique_ptr<HttpRequest::Factory>(
3312 new FakeHttpRequestFactory(&requests)),
3313 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3314 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3315 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3316 0 /* matching paths cache max entries */, kTestRetryConfig,
3317 kTestTimeoutConfig, *kAllowedLocationsDefault,
3318 nullptr /* gcs additional header */, false /* compose append */);
3319
3320 TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath", nullptr));
3321 // Check that when GCS returns the object already exists return that the
3322 // directory already exists.
3323 EXPECT_EQ(errors::AlreadyExists("gs://bucket/subpath"),
3324 fs.CreateDir("gs://bucket/subpath", nullptr));
3325 // Check that when GCS returns the object already has a version (failed
3326 // precondition) return directory already exists.
3327 EXPECT_EQ(errors::AlreadyExists("gs://bucket/subpath"),
3328 fs.CreateDir("gs://bucket/subpath", nullptr));
3329 }
3330
TEST(GcsFileSystemTest,CreateDir_Bucket)3331 TEST(GcsFileSystemTest, CreateDir_Bucket) {
3332 std::vector<HttpRequest*> requests(
3333 {new FakeHttpRequest(
3334 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3335 "Auth Token: fake_token\n"
3336 "Timeouts: 5 1 10\n",
3337 ""),
3338 new FakeHttpRequest(
3339 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3340 "Auth Token: fake_token\n"
3341 "Timeouts: 5 1 10\n",
3342 "")});
3343 GcsFileSystem fs(
3344 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3345 std::unique_ptr<HttpRequest::Factory>(
3346 new FakeHttpRequestFactory(&requests)),
3347 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3348 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3349 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3350 0 /* matching paths cache max entries */, kTestRetryConfig,
3351 kTestTimeoutConfig, *kAllowedLocationsDefault,
3352 nullptr /* gcs additional header */, false /* compose append */);
3353
3354 TF_EXPECT_OK(fs.CreateDir("gs://bucket/", nullptr));
3355 TF_EXPECT_OK(fs.CreateDir("gs://bucket", nullptr));
3356 }
3357
TEST(GcsFileSystemTest,DeleteRecursively_Ok)3358 TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
3359 std::vector<HttpRequest*> requests(
3360 {// IsDirectory is checking whether there are children objects.
3361 new FakeHttpRequest(
3362 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3363 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
3364 "&maxResults=1\n"
3365 "Auth Token: fake_token\n"
3366 "Timeouts: 5 1 10\n",
3367 "{\"items\": [ "
3368 " { \"name\": \"path/file1.txt\" }]}"),
3369 // GetChildren recursively.
3370 new FakeHttpRequest(
3371 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3372 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
3373 "Auth Token: fake_token\n"
3374 "Timeouts: 5 1 10\n",
3375 "{\"items\": [ "
3376 " { \"name\": \"path/\" }," // The current directory's marker.
3377 " { \"name\": \"path/file1.txt\" },"
3378 " { \"name\": \"path/subpath/file2.txt\" },"
3379 " { \"name\": \"path/file3.txt\" }]}"),
3380 // Delete the current directory's marker.
3381 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3382 "/bucket/o/path%2F\n"
3383 "Auth Token: fake_token\n"
3384 "Timeouts: 5 1 10\n"
3385 "Delete: yes\n",
3386 ""),
3387 // Delete the object - fails and will be retried.
3388 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3389 "/bucket/o/path%2Ffile1.txt\n"
3390 "Auth Token: fake_token\n"
3391 "Timeouts: 5 1 10\n"
3392 "Delete: yes\n",
3393 "", errors::Unavailable("500"), 500),
3394 // Delete the object again.
3395 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3396 "/bucket/o/path%2Ffile1.txt\n"
3397 "Auth Token: fake_token\n"
3398 "Timeouts: 5 1 10\n"
3399 "Delete: yes\n",
3400 ""),
3401 // Delete the object.
3402 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3403 "/bucket/o/path%2Fsubpath%2Ffile2.txt\n"
3404 "Auth Token: fake_token\n"
3405 "Timeouts: 5 1 10\n"
3406 "Delete: yes\n",
3407 ""),
3408 // Delete the object.
3409 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3410 "/bucket/o/path%2Ffile3.txt\n"
3411 "Auth Token: fake_token\n"
3412 "Timeouts: 5 1 10\n"
3413 "Delete: yes\n",
3414 "")});
3415 GcsFileSystem fs(
3416 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3417 std::unique_ptr<HttpRequest::Factory>(
3418 new FakeHttpRequestFactory(&requests)),
3419 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3420 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3421 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3422 0 /* matching paths cache max entries */, kTestRetryConfig,
3423 kTestTimeoutConfig, *kAllowedLocationsDefault,
3424 nullptr /* gcs additional header */, false /* compose append */);
3425
3426 int64 undeleted_files, undeleted_dirs;
3427 TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", nullptr,
3428 &undeleted_files, &undeleted_dirs));
3429 EXPECT_EQ(0, undeleted_files);
3430 EXPECT_EQ(0, undeleted_dirs);
3431 }
3432
TEST(GcsFileSystemTest,DeleteRecursively_DeletionErrors)3433 TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
3434 std::vector<HttpRequest*> requests(
3435 {// IsDirectory is checking whether there are children objects.
3436 new FakeHttpRequest(
3437 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3438 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
3439 "&maxResults=1\n"
3440 "Auth Token: fake_token\n"
3441 "Timeouts: 5 1 10\n",
3442 "{\"items\": [ "
3443 " { \"name\": \"path/file1.txt\" }]}"),
3444 // Calling GetChildren recursively.
3445 new FakeHttpRequest(
3446 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3447 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
3448 "Auth Token: fake_token\n"
3449 "Timeouts: 5 1 10\n",
3450 "{\"items\": [ "
3451 " { \"name\": \"path/file1.txt\" },"
3452 " { \"name\": \"path/subpath/\" },"
3453 " { \"name\": \"path/subpath/file2.txt\" },"
3454 " { \"name\": \"path/file3.txt\" }]}"),
3455 // Deleting the object.
3456 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3457 "/bucket/o/path%2Ffile1.txt\n"
3458 "Auth Token: fake_token\n"
3459 "Timeouts: 5 1 10\n"
3460 "Delete: yes\n",
3461 ""),
3462 // Deleting the directory marker gs://bucket/path/ - fails with 404.
3463 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3464 "/bucket/o/path%2Fsubpath%2F\n"
3465 "Auth Token: fake_token\n"
3466 "Timeouts: 5 1 10\n"
3467 "Delete: yes\n",
3468 "", errors::NotFound("404"), 404),
3469 // Checking if gs://bucket/path/subpath/ is a folder - it is.
3470 new FakeHttpRequest(
3471 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3472 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F"
3473 "&maxResults=1\n"
3474 "Auth Token: fake_token\n"
3475 "Timeouts: 5 1 10\n",
3476 strings::StrCat("{\"items\": [ "
3477 " { \"name\": \"path/subpath/\" }]}")),
3478 // Deleting the object gs://bucket/path/subpath/file2.txt
3479 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3480 "/bucket/o/path%2Fsubpath%2Ffile2.txt\n"
3481 "Auth Token: fake_token\n"
3482 "Timeouts: 5 1 10\n"
3483 "Delete: yes\n",
3484 ""),
3485 // Deleting the object s://bucket/path/file3.txt - fails with 404.
3486 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3487 "/bucket/o/path%2Ffile3.txt\n"
3488 "Auth Token: fake_token\n"
3489 "Timeouts: 5 1 10\n"
3490 "Delete: yes\n",
3491 "", errors::NotFound("404"), 404),
3492 // Checking if gs://bucket/path/file3.txt/ is a folder - it's not.
3493 new FakeHttpRequest(
3494 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3495 "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile3.txt%2F"
3496 "&maxResults=1\n"
3497 "Auth Token: fake_token\n"
3498 "Timeouts: 5 1 10\n",
3499 "{}"),
3500 // Checking if gs://bucket/path/file3.txt is an object - fails with 404.
3501 new FakeHttpRequest(
3502 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3503 "path%2Ffile3.txt?fields=size%2Cgeneration%2Cupdated\n"
3504 "Auth Token: fake_token\n"
3505 "Timeouts: 5 1 10\n",
3506 "", errors::NotFound("404"), 404)});
3507
3508 GcsFileSystem fs(
3509 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3510 std::unique_ptr<HttpRequest::Factory>(
3511 new FakeHttpRequestFactory(&requests)),
3512 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3513 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3514 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3515 0 /* matching paths cache max entries */, kTestRetryConfig,
3516 kTestTimeoutConfig, *kAllowedLocationsDefault,
3517 nullptr /* gcs additional header */, false /* compose append */);
3518
3519 int64 undeleted_files, undeleted_dirs;
3520 TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", nullptr,
3521 &undeleted_files, &undeleted_dirs));
3522 EXPECT_EQ(1, undeleted_files);
3523 EXPECT_EQ(1, undeleted_dirs);
3524 }
3525
TEST(GcsFileSystemTest,DeleteRecursively_NotAFolder)3526 TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) {
3527 std::vector<HttpRequest*> requests(
3528 {// IsDirectory is checking whether there are children objects.
3529 new FakeHttpRequest(
3530 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3531 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
3532 "&maxResults=1\n"
3533 "Auth Token: fake_token\n"
3534 "Timeouts: 5 1 10\n",
3535 "{}"),
3536 // IsDirectory is checking if the path exists as an object.
3537 new FakeHttpRequest(
3538 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3539 "path?fields=size%2Cgeneration%2Cupdated\n"
3540 "Auth Token: fake_token\n"
3541 "Timeouts: 5 1 10\n",
3542 "", errors::NotFound("404"), 404)});
3543 GcsFileSystem fs(
3544 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3545 std::unique_ptr<HttpRequest::Factory>(
3546 new FakeHttpRequestFactory(&requests)),
3547 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3548 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3549 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3550 0 /* matching paths cache max entries */, kTestRetryConfig,
3551 kTestTimeoutConfig, *kAllowedLocationsDefault,
3552 nullptr /* gcs additional header */, false /* compose append */);
3553
3554 int64 undeleted_files, undeleted_dirs;
3555 EXPECT_EQ(error::Code::NOT_FOUND,
3556 fs.DeleteRecursively("gs://bucket/path", nullptr, &undeleted_files,
3557 &undeleted_dirs)
3558 .code());
3559 EXPECT_EQ(0, undeleted_files);
3560 EXPECT_EQ(1, undeleted_dirs);
3561 }
3562
TEST(GcsFileSystemTest,NoConstraintsEnvironmentVariableTest)3563 TEST(GcsFileSystemTest, NoConstraintsEnvironmentVariableTest) {
3564 unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS");
3565 // No constraints
3566 GcsFileSystem fs1;
3567 EXPECT_EQ(*kAllowedLocationsDefault, fs1.allowed_locations());
3568
3569 // Cover cache initialization code, any uninitialized cache will cause this to
3570 // fail
3571 fs1.FlushCaches(nullptr);
3572 }
3573
TEST(GcsFileSystemTest,BucketLocationConstraintEnvironmentVariableTest)3574 TEST(GcsFileSystemTest, BucketLocationConstraintEnvironmentVariableTest) {
3575 unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS");
3576 setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "auto", 1);
3577 GcsFileSystem fs1;
3578 EXPECT_EQ(*kAllowedLocationsAuto, fs1.allowed_locations());
3579
3580 setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "CUSTOM,list", 1);
3581 GcsFileSystem fs2;
3582 EXPECT_EQ(std::unordered_set<string>({"custom", "list"}),
3583 fs2.allowed_locations());
3584 }
3585
TEST(GcsFileSystemTest,AdditionalRequestHeaderTest)3586 TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) {
3587 GcsFileSystem fs1;
3588 EXPECT_EQ("", fs1.additional_header_name());
3589 EXPECT_EQ("", fs1.additional_header_value());
3590
3591 setenv("GCS_ADDITIONAL_REQUEST_HEADER",
3592 "X-Add-Header:My Additional Header Value", 1);
3593 GcsFileSystem fs2;
3594 EXPECT_EQ("X-Add-Header", fs2.additional_header_name());
3595 EXPECT_EQ("My Additional Header Value", fs2.additional_header_value());
3596
3597 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "Someinvalidheadervalue", 1);
3598 GcsFileSystem fs3;
3599 EXPECT_EQ("", fs3.additional_header_name());
3600 EXPECT_EQ("", fs3.additional_header_value());
3601
3602 setenv("GCS_ADDITIONAL_REQUEST_HEADER", ":thisisinvalid", 1);
3603 GcsFileSystem fs4;
3604 EXPECT_EQ("", fs4.additional_header_name());
3605 EXPECT_EQ("", fs4.additional_header_value());
3606
3607 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "soisthis:", 1);
3608 GcsFileSystem fs5;
3609 EXPECT_EQ("", fs5.additional_header_name());
3610 EXPECT_EQ("", fs5.additional_header_value());
3611
3612 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "a:b", 1);
3613 GcsFileSystem fs6;
3614 EXPECT_EQ("a", fs6.additional_header_name());
3615 EXPECT_EQ("b", fs6.additional_header_value());
3616
3617 auto* add_header = new std::pair<const string, const string>(
3618 "mynewheader", "newheadercontents");
3619
3620 std::vector<HttpRequest*> requests(
3621 {// IsDirectory is checking whether there are children objects.
3622 new FakeHttpRequest("Uri: https://www.googleapis.com/fake\n"
3623 "Auth Token: fake_token\n"
3624 "Header mynewheader: newheadercontents\n"
3625 "Header Hello: world\n",
3626 "{}")});
3627 GcsFileSystem fs7(
3628 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3629 std::unique_ptr<HttpRequest::Factory>(
3630 new FakeHttpRequestFactory(&requests)),
3631 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3632 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3633 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3634 0 /* matching paths cache max entries */, kTestRetryConfig,
3635 kTestTimeoutConfig, *kAllowedLocationsDefault,
3636 add_header /* gcs additional header */, false /* compose append */);
3637
3638 std::unique_ptr<HttpRequest> request;
3639 TF_EXPECT_OK(fs7.CreateHttpRequest(&request));
3640 request->SetUri("https://www.googleapis.com/fake");
3641 request->AddHeader("Hello", "world");
3642 TF_EXPECT_OK(request->Send());
3643 }
3644
TEST(GcsFileSystemTest,OverrideCacheParameters)3645 TEST(GcsFileSystemTest, OverrideCacheParameters) {
3646 // Verify defaults are propagated correctly.
3647 setenv("GCS_READ_CACHE_BLOCK_SIZE_MB", "16", 1);
3648 setenv("GCS_READ_CACHE_MAX_SIZE_MB", "128", 1);
3649 GcsFileSystem fs1;
3650 EXPECT_EQ(16 * 1024 * 1024, fs1.block_size());
3651 EXPECT_EQ(128 * 1024 * 1024, fs1.max_bytes());
3652 EXPECT_EQ(0, fs1.max_staleness());
3653 EXPECT_EQ(120, fs1.timeouts().connect);
3654 EXPECT_EQ(60, fs1.timeouts().idle);
3655 EXPECT_EQ(3600, fs1.timeouts().metadata);
3656 EXPECT_EQ(3600, fs1.timeouts().read);
3657 EXPECT_EQ(3600, fs1.timeouts().write);
3658
3659 // Verify legacy readahead buffer override sets block size.
3660 unsetenv("GCS_READ_CACHE_BLOCK_SIZE_MB");
3661 setenv("GCS_READAHEAD_BUFFER_SIZE_BYTES", "123456789", 1);
3662 GcsFileSystem fs2;
3663 EXPECT_EQ(123456789L, fs2.block_size());
3664
3665 // Verify block size, max size, and max staleness overrides.
3666 setenv("GCS_READ_CACHE_BLOCK_SIZE_MB", "1", 1);
3667 setenv("GCS_READ_CACHE_MAX_SIZE_MB", "16", 1);
3668 setenv("GCS_READ_CACHE_MAX_STALENESS", "60", 1);
3669 GcsFileSystem fs3;
3670 EXPECT_EQ(1048576L, fs3.block_size());
3671 EXPECT_EQ(16 * 1024 * 1024, fs3.max_bytes());
3672 EXPECT_EQ(60, fs3.max_staleness());
3673
3674 // Verify StatCache and MatchingPathsCache overrides.
3675 setenv("GCS_STAT_CACHE_MAX_AGE", "60", 1);
3676 setenv("GCS_STAT_CACHE_MAX_ENTRIES", "32", 1);
3677 setenv("GCS_MATCHING_PATHS_CACHE_MAX_AGE", "30", 1);
3678 setenv("GCS_MATCHING_PATHS_CACHE_MAX_ENTRIES", "64", 1);
3679 GcsFileSystem fs4;
3680 EXPECT_EQ(60, fs4.stat_cache_max_age());
3681 EXPECT_EQ(32, fs4.stat_cache_max_entries());
3682 EXPECT_EQ(30, fs4.matching_paths_cache_max_age());
3683 EXPECT_EQ(64, fs4.matching_paths_cache_max_entries());
3684
3685 // Verify timeout overrides.
3686 setenv("GCS_REQUEST_CONNECTION_TIMEOUT_SECS", "10", 1);
3687 setenv("GCS_REQUEST_IDLE_TIMEOUT_SECS", "5", 1);
3688 setenv("GCS_METADATA_REQUEST_TIMEOUT_SECS", "20", 1);
3689 setenv("GCS_READ_REQUEST_TIMEOUT_SECS", "30", 1);
3690 setenv("GCS_WRITE_REQUEST_TIMEOUT_SECS", "40", 1);
3691 GcsFileSystem fs5;
3692 EXPECT_EQ(10, fs5.timeouts().connect);
3693 EXPECT_EQ(5, fs5.timeouts().idle);
3694 EXPECT_EQ(20, fs5.timeouts().metadata);
3695 EXPECT_EQ(30, fs5.timeouts().read);
3696 EXPECT_EQ(40, fs5.timeouts().write);
3697 }
3698
TEST(GcsFileSystemTest,CreateHttpRequest)3699 TEST(GcsFileSystemTest, CreateHttpRequest) {
3700 std::vector<HttpRequest*> requests(
3701 {// IsDirectory is checking whether there are children objects.
3702 new FakeHttpRequest("Uri: https://www.googleapis.com/fake\n"
3703 "Auth Token: fake_token\n"
3704 "Header Hello: world\n",
3705 "{}")});
3706 GcsFileSystem fs(
3707 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3708 std::unique_ptr<HttpRequest::Factory>(
3709 new FakeHttpRequestFactory(&requests)),
3710 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3711 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3712 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3713 0 /* matching paths cache max entries */, kTestRetryConfig,
3714 kTestTimeoutConfig, *kAllowedLocationsDefault,
3715 nullptr /* gcs additional header */, false /* compose append */);
3716
3717 std::unique_ptr<HttpRequest> request;
3718 TF_EXPECT_OK(fs.CreateHttpRequest(&request));
3719 request->SetUri("https://www.googleapis.com/fake");
3720 request->AddHeader("Hello", "world");
3721 TF_EXPECT_OK(request->Send());
3722 }
3723
3724 class TestGcsStats : public GcsStatsInterface {
3725 public:
Configure(GcsFileSystem * fs,GcsThrottle * throttle,const FileBlockCache * block_cache)3726 void Configure(GcsFileSystem* fs, GcsThrottle* throttle,
3727 const FileBlockCache* block_cache) override {
3728 CHECK(fs_ == nullptr);
3729 CHECK(throttle_ == nullptr);
3730 CHECK(block_cache_ == nullptr);
3731
3732 fs_ = fs;
3733 throttle_ = throttle;
3734 block_cache_ = block_cache;
3735 }
3736
RecordBlockLoadRequest(const string & file,size_t offset)3737 void RecordBlockLoadRequest(const string& file, size_t offset) override {
3738 block_load_request_file_ = file;
3739 }
3740
RecordBlockRetrieved(const string & file,size_t offset,size_t bytes_transferred)3741 void RecordBlockRetrieved(const string& file, size_t offset,
3742 size_t bytes_transferred) override {
3743 block_retrieved_file_ = file;
3744 block_retrieved_bytes_transferred_ = bytes_transferred;
3745 }
3746
RecordStatObjectRequest()3747 void RecordStatObjectRequest() override { stat_object_request_count_++; }
3748
HttpStats()3749 HttpRequest::RequestStats* HttpStats() override { return nullptr; }
3750
3751 GcsFileSystem* fs_ = nullptr;
3752 GcsThrottle* throttle_ = nullptr;
3753 const FileBlockCache* block_cache_ = nullptr;
3754
3755 string block_load_request_file_;
3756 string block_retrieved_file_;
3757 size_t block_retrieved_bytes_transferred_ = 0;
3758 int stat_object_request_count_ = 0;
3759 };
3760
TEST(GcsFileSystemTest,Stat_StatsRecording)3761 TEST(GcsFileSystemTest, Stat_StatsRecording) {
3762 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3763 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3764 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3765 "Auth Token: fake_token\n"
3766 "Timeouts: 5 1 10\n",
3767 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3768 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3769 GcsFileSystem fs(
3770 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3771 std::unique_ptr<HttpRequest::Factory>(
3772 new FakeHttpRequestFactory(&requests)),
3773 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3774 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3775 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3776 0 /* matching paths cache max entries */, kTestRetryConfig,
3777 kTestTimeoutConfig, *kAllowedLocationsDefault,
3778 nullptr /* gcs additional header */, false /* compose append */);
3779
3780 TestGcsStats stats;
3781 fs.SetStats(&stats);
3782 EXPECT_EQ(stats.fs_, &fs);
3783
3784 FileStatistics stat;
3785 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
3786 EXPECT_EQ(1, stats.stat_object_request_count_);
3787 }
3788
TEST(GcsFileSystemTest,NewRandomAccessFile_StatsRecording)3789 TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) {
3790 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3791 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
3792 "Auth Token: fake_token\n"
3793 "Range: 0-5\n"
3794 "Timeouts: 5 1 20\n",
3795 "012345")});
3796 GcsFileSystem fs(
3797 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3798 std::unique_ptr<HttpRequest::Factory>(
3799 new FakeHttpRequestFactory(&requests)),
3800 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3801 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3802 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3803 0 /* matching paths cache max entries */, kTestRetryConfig,
3804 kTestTimeoutConfig, *kAllowedLocationsDefault,
3805 nullptr /* gcs additional header */, false /* compose append */);
3806
3807 TestGcsStats stats;
3808 fs.SetStats(&stats);
3809 EXPECT_EQ(stats.fs_, &fs);
3810
3811 std::unique_ptr<RandomAccessFile> file;
3812 TF_EXPECT_OK(
3813 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
3814
3815 char scratch[6];
3816 StringPiece result;
3817
3818 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
3819 EXPECT_EQ("012345", result);
3820
3821 EXPECT_EQ("gs://bucket/random_access.txt", stats.block_load_request_file_);
3822 EXPECT_EQ("gs://bucket/random_access.txt", stats.block_retrieved_file_);
3823 EXPECT_EQ(6, stats.block_retrieved_bytes_transferred_);
3824 }
3825
TEST(GcsFileSystemTest,NewAppendableFile_MultipleFlushesWithCompose)3826 TEST(GcsFileSystemTest, NewAppendableFile_MultipleFlushesWithCompose) {
3827 std::vector<string> contents(
3828 {"content0,", "content1,", "content2,", "content3,"});
3829 std::vector<HttpRequest*> requests({
3830 // Fetch the file (stats and then content)
3831 new FakeHttpRequest(
3832 "Uri: "
3833 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3834 "some%2Fpath%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
3835 "Auth Token: fake_token\n"
3836 "Timeouts: 5 1 10\n",
3837 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
3838 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3839 new FakeHttpRequest(
3840 "Uri: "
3841 "https://storage.googleapis.com/bucket/some%2Fpath%2Fappendable\n"
3842 "Auth Token: fake_token\n"
3843 "Range: 0-1048575\n"
3844 "Timeouts: 5 1 20\n",
3845 contents[0]),
3846 // Upload entire file
3847 new FakeHttpRequest(
3848 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3849 "uploadType=resumable&name=some%2Fpath%2Fappendable\n"
3850 "Auth Token: fake_token\n"
3851 "Header X-Upload-Content-Length: 18\n"
3852 "Post: yes\n"
3853 "Timeouts: 5 1 10\n",
3854 "", {{"Location", "https://custom/upload/location"}}),
3855 new FakeHttpRequest(
3856 strings::StrCat("Uri: https://custom/upload/location\n"
3857 "Auth Token: fake_token\n"
3858 "Header Content-Range: bytes 0-17/18\n"
3859 "Timeouts: 5 1 30\n"
3860 "Put body: ",
3861 contents[0], contents[1], "\n"),
3862 ""),
3863 // Upload new part to a temporary object
3864 new FakeHttpRequest(
3865 "Uri: "
3866 "https://www.googleapis.com/upload/storage/v1/b/bucket/"
3867 "o?uploadType=resumable&name=some%2Fpath%2F.tmpcompose%2Fappendable."
3868 "18\n"
3869 "Auth Token: fake_token\n"
3870 "Header X-Upload-Content-Length: 9\n"
3871 "Post: yes\n"
3872 "Timeouts: 5 1 10\n",
3873 "",
3874 {{"Location",
3875 "https://custom/upload/"
3876 "location"}}),
3877 new FakeHttpRequest(
3878 strings::StrCat("Uri: https://custom/upload/location\n"
3879 "Auth Token: fake_token\n"
3880 "Header Content-Range: bytes 0-8/9\n"
3881 "Timeouts: 5 1 30\n"
3882 "Put body: ",
3883 contents[2], "\n"),
3884 ""),
3885 // Fetch generation
3886 new FakeHttpRequest(
3887 "Uri: "
3888 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3889 "some%2Fpath%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
3890 "Auth Token: fake_token\n"
3891 "Timeouts: 5 1 10\n",
3892 strings::StrCat("{\"size\": \"8\",\"generation\": \"1234\","
3893 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3894 // Compose the new part at the end of the original object.
3895 new FakeHttpRequest("Uri: "
3896 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3897 "some%2Fpath%2Fappendable/compose\n"
3898 "Auth Token: fake_token\n"
3899 "Timeouts: 5 1 10\n"
3900 "Header content-type: application/json\n"
3901 "Post body: {'sourceObjects': [{'name': "
3902 "'some/path/"
3903 "appendable','objectPrecondition':{'"
3904 "ifGenerationMatch':1234}},{'name': "
3905 "'some/path/.tmpcompose/appendable.18'}]}\n",
3906 ""),
3907 // Delete the temporary object.
3908 new FakeHttpRequest("Uri: "
3909 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3910 "some%2Fpath%2F.tmpcompose%2Fappendable.18\n"
3911 "Auth Token: fake_token\n"
3912 "Timeouts: 5 1 10\n"
3913 "Delete: yes\n",
3914 ""),
3915 new FakeHttpRequest(
3916 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3917 "uploadType=resumable&name=some%2Fpath%2F.tmpcompose%2Fappendable."
3918 "27\n"
3919 "Auth Token: fake_token\n"
3920 "Header X-Upload-Content-Length: 9\n"
3921 "Post: yes\n"
3922 "Timeouts: 5 1 10\n",
3923 "", {{"Location", "https://custom/upload/location"}}),
3924 new FakeHttpRequest(
3925 strings::StrCat("Uri: https://custom/upload/location\n"
3926 "Auth Token: fake_token\n"
3927 "Header Content-Range: bytes 0-8/9\n"
3928 "Timeouts: 5 1 30\n"
3929 "Put body: ",
3930 contents[3], "\n"),
3931 ""),
3932 // Fetch generation
3933 new FakeHttpRequest(
3934 "Uri: "
3935 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3936 "some%2Fpath%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
3937 "Auth Token: fake_token\n"
3938 "Timeouts: 5 1 10\n",
3939 strings::StrCat("{\"size\": \"8\",\"generation\": \"4567\","
3940 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3941 new FakeHttpRequest("Uri: "
3942 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3943 "some%2Fpath%2Fappendable/compose\n"
3944 "Auth Token: fake_token\n"
3945 "Timeouts: 5 1 10\n"
3946 "Header content-type: application/json\n"
3947 "Post body: {'sourceObjects': [{'name': "
3948 "'some/path/"
3949 "appendable','objectPrecondition':{'"
3950 "ifGenerationMatch':4567}},{'name': "
3951 "'some/path/.tmpcompose/appendable.27'}]}\n",
3952 ""),
3953 new FakeHttpRequest("Uri: "
3954 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3955 "some%2Fpath%2F.tmpcompose%2Fappendable."
3956 "27\n"
3957 "Auth Token: fake_token\n"
3958 "Timeouts: 5 1 10\n"
3959 "Delete: yes\n",
3960 ""),
3961 });
3962 GcsFileSystem fs(
3963 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3964 std::unique_ptr<HttpRequest::Factory>(
3965 new FakeHttpRequestFactory(&requests)),
3966 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 32 /* block size */,
3967 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
3968 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3969 0 /* matching paths cache max entries */, kTestRetryConfig,
3970 kTestTimeoutConfig, *kAllowedLocationsDefault,
3971 nullptr /* gcs additional header */, true /* compose append */);
3972
3973 // Create an appendable file. This should read the file from GCS, and pull its
3974 // contents into the block cache.
3975 std::unique_ptr<WritableFile> wfile;
3976 TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/some/path/appendable", nullptr,
3977 &wfile));
3978 TF_EXPECT_OK(wfile->Append(contents[1]));
3979 TF_EXPECT_OK(wfile->Flush());
3980 TF_EXPECT_OK(wfile->Append(contents[2]));
3981 TF_EXPECT_OK(wfile->Flush());
3982 TF_EXPECT_OK(wfile->Append(contents[3]));
3983 TF_EXPECT_OK(wfile->Close());
3984 }
3985
TEST(GcsFileSystemTest,NewAppendableFile_MultipleFlushesWithoutCompose)3986 TEST(GcsFileSystemTest, NewAppendableFile_MultipleFlushesWithoutCompose) {
3987 std::vector<string> contents(
3988 {"content0,", "content1,", "content2,", "content3,"});
3989 std::vector<HttpRequest*> requests({
3990 new FakeHttpRequest(
3991 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3992 "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
3993 "Auth Token: fake_token\n"
3994 "Timeouts: 5 1 10\n",
3995 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
3996 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3997 new FakeHttpRequest(
3998 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
3999 "Auth Token: fake_token\n"
4000 "Range: 0-1048575\n"
4001 "Timeouts: 5 1 20\n",
4002 contents[0]),
4003 new FakeHttpRequest(
4004 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
4005 "uploadType=resumable&name=path%2Fappendable\n"
4006 "Auth Token: fake_token\n"
4007 "Header X-Upload-Content-Length: 18\n"
4008 "Post: yes\n"
4009 "Timeouts: 5 1 10\n",
4010 "", {{"Location", "https://custom/upload/location"}}),
4011 // Uploads entire file.
4012 new FakeHttpRequest(
4013 strings::StrCat("Uri: https://custom/upload/location\n"
4014 "Auth Token: fake_token\n"
4015 "Header Content-Range: bytes 0-17/18\n"
4016 "Timeouts: 5 1 30\n"
4017 "Put body: ",
4018 contents[0], contents[1], "\n"),
4019 ""),
4020 new FakeHttpRequest("Uri: "
4021 "https://www.googleapis.com/upload/storage/v1/b/"
4022 "bucket/o?"
4023 "uploadType=resumable&name=path%2Fappendable\n"
4024 "Auth Token: fake_token\n"
4025 "Header X-Upload-Content-Length: 27\n"
4026 "Post: yes\n"
4027 "Timeouts: 5 1 10\n",
4028 "",
4029 {{"Location",
4030 "https://custom/upload/"
4031 "location"}}),
4032 // Uploads entire file again.
4033 new FakeHttpRequest(
4034 strings::StrCat("Uri: https://custom/upload/location\n"
4035 "Auth Token: fake_token\n"
4036 "Header Content-Range: bytes 0-26/27\n"
4037 "Timeouts: 5 1 30\n"
4038 "Put body: ",
4039 contents[0], contents[1], contents[2], "\n"),
4040 ""),
4041 new FakeHttpRequest(
4042 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
4043 "uploadType=resumable&name=path%2Fappendable\n"
4044 "Auth Token: fake_token\n"
4045 "Header X-Upload-Content-Length: 36\n"
4046 "Post: yes\n"
4047 "Timeouts: 5 1 10\n",
4048 "", {{"Location", "https://custom/upload/location"}}),
4049 // Uploads entire file again.
4050 new FakeHttpRequest(
4051 strings::StrCat("Uri: https://custom/upload/location\n"
4052 "Auth Token: fake_token\n"
4053 "Header Content-Range: bytes 0-35/36\n"
4054 "Timeouts: 5 1 30\n"
4055 "Put body: ",
4056 contents[0], contents[1], contents[2], contents[3],
4057 "\n"),
4058 ""),
4059 });
4060 GcsFileSystem fs(
4061 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
4062 std::unique_ptr<HttpRequest::Factory>(
4063 new FakeHttpRequestFactory(&requests)),
4064 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 32 /* block size */,
4065 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
4066 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
4067 0 /* matching paths cache max entries */, kTestRetryConfig,
4068 kTestTimeoutConfig, *kAllowedLocationsDefault,
4069 nullptr /* gcs additional header */, false /* compose append */);
4070
4071 // Create an appendable file. This should read the file from GCS, and pull its
4072 // contents into the block cache.
4073 std::unique_ptr<WritableFile> wfile;
4074 TF_EXPECT_OK(
4075 fs.NewAppendableFile("gs://bucket/path/appendable", nullptr, &wfile));
4076 TF_EXPECT_OK(wfile->Append(contents[1]));
4077 TF_EXPECT_OK(wfile->Flush());
4078 TF_EXPECT_OK(wfile->Append(contents[2]));
4079 TF_EXPECT_OK(wfile->Flush());
4080 TF_EXPECT_OK(wfile->Append(contents[3]));
4081 TF_EXPECT_OK(wfile->Close());
4082 }
4083
TEST(GcsFileSystemTest,AppendModeCompose)4084 TEST(GcsFileSystemTest, AppendModeCompose) {
4085 unsetenv("GCS_APPEND_MODE");
4086 setenv("GCS_APPEND_MODE", "compose", 1);
4087 GcsFileSystem fs1;
4088 EXPECT_EQ(true, fs1.compose_append());
4089 }
4090
TEST(GcsFileSystemTest,AppendModeDefault)4091 TEST(GcsFileSystemTest, AppendModeDefault) {
4092 unsetenv("GCS_APPEND_MODE");
4093 GcsFileSystem fs1;
4094 EXPECT_EQ(false, fs1.compose_append());
4095 }
4096
4097 } // namespace
4098 } // namespace tensorflow
4099