1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/platform/cloud/curl_http_request.h"
17 #include <fstream>
18 #include "tensorflow/core/lib/core/status_test_util.h"
19 #include "tensorflow/core/lib/io/path.h"
20 #include "tensorflow/core/platform/mem.h"
21 #include "tensorflow/core/platform/test.h"
22
23 namespace tensorflow {
24 namespace {
25
26 const string kTestContent = "random original scratch content";
27
28 class FakeEnv : public EnvWrapper {
29 public:
FakeEnv()30 FakeEnv() : EnvWrapper(Env::Default()) {}
31
NowSeconds()32 uint64 NowSeconds() override { return now_; }
33 uint64 now_ = 10000;
34 };
35
36 // A fake proxy that pretends to be libcurl.
37 class FakeLibCurl : public LibCurl {
38 public:
FakeLibCurl(const string & response_content,uint64 response_code)39 FakeLibCurl(const string& response_content, uint64 response_code)
40 : response_content_(response_content), response_code_(response_code) {}
FakeLibCurl(const string & response_content,uint64 response_code,std::vector<std::tuple<uint64,curl_off_t>> progress_ticks,FakeEnv * env)41 FakeLibCurl(const string& response_content, uint64 response_code,
42 std::vector<std::tuple<uint64, curl_off_t>> progress_ticks,
43 FakeEnv* env)
44 : response_content_(response_content),
45 response_code_(response_code),
46 progress_ticks_(std::move(progress_ticks)),
47 env_(env) {}
FakeLibCurl(const string & response_content,uint64 response_code,const std::vector<string> & response_headers)48 FakeLibCurl(const string& response_content, uint64 response_code,
49 const std::vector<string>& response_headers)
50 : response_content_(response_content),
51 response_code_(response_code),
52 response_headers_(response_headers) {}
curl_easy_init()53 CURL* curl_easy_init() override {
54 is_initialized_ = true;
55 // The reuslt just needs to be non-null.
56 return reinterpret_cast<CURL*>(this);
57 }
curl_easy_setopt(CURL * curl,CURLoption option,uint64 param)58 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
59 uint64 param) override {
60 switch (option) {
61 case CURLOPT_POST:
62 is_post_ = param;
63 break;
64 case CURLOPT_PUT:
65 is_put_ = param;
66 break;
67 default:
68 break;
69 }
70 return CURLE_OK;
71 }
curl_easy_setopt(CURL * curl,CURLoption option,const char * param)72 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
73 const char* param) override {
74 return curl_easy_setopt(curl, option,
75 reinterpret_cast<void*>(const_cast<char*>(param)));
76 }
curl_easy_setopt(CURL * curl,CURLoption option,void * param)77 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
78 void* param) override {
79 switch (option) {
80 case CURLOPT_URL:
81 url_ = reinterpret_cast<char*>(param);
82 break;
83 case CURLOPT_RANGE:
84 range_ = reinterpret_cast<char*>(param);
85 break;
86 case CURLOPT_CUSTOMREQUEST:
87 custom_request_ = reinterpret_cast<char*>(param);
88 break;
89 case CURLOPT_HTTPHEADER:
90 headers_ = reinterpret_cast<std::vector<string>*>(param);
91 break;
92 case CURLOPT_ERRORBUFFER:
93 error_buffer_ = reinterpret_cast<char*>(param);
94 break;
95 case CURLOPT_WRITEDATA:
96 write_data_ = reinterpret_cast<FILE*>(param);
97 break;
98 case CURLOPT_HEADERDATA:
99 header_data_ = reinterpret_cast<FILE*>(param);
100 break;
101 case CURLOPT_READDATA:
102 read_data_ = reinterpret_cast<FILE*>(param);
103 break;
104 case CURLOPT_XFERINFODATA:
105 progress_data_ = param;
106 break;
107 default:
108 break;
109 }
110 return CURLE_OK;
111 }
curl_easy_setopt(CURL * curl,CURLoption option,size_t (* param)(void *,size_t,size_t,FILE *))112 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
113 size_t (*param)(void*, size_t, size_t,
114 FILE*)) override {
115 read_callback_ = param;
116 return CURLE_OK;
117 }
curl_easy_setopt(CURL * curl,CURLoption option,size_t (* param)(const void *,size_t,size_t,void *))118 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
119 size_t (*param)(const void*, size_t, size_t,
120 void*)) override {
121 switch (option) {
122 case CURLOPT_WRITEFUNCTION:
123 write_callback_ = param;
124 break;
125 case CURLOPT_HEADERFUNCTION:
126 header_callback_ = param;
127 break;
128 default:
129 break;
130 }
131 return CURLE_OK;
132 }
curl_easy_setopt(CURL * curl,CURLoption option,int (* param)(void * clientp,curl_off_t dltotal,curl_off_t dlnow,curl_off_t ultotal,curl_off_t ulnow))133 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
134 int (*param)(void* clientp, curl_off_t dltotal,
135 curl_off_t dlnow, curl_off_t ultotal,
136 curl_off_t ulnow)) override {
137 progress_callback_ = param;
138 return CURLE_OK;
139 }
curl_easy_perform(CURL * curl)140 CURLcode curl_easy_perform(CURL* curl) override {
141 if (is_post_ || is_put_) {
142 char buffer[3];
143 int bytes_read;
144 posted_content_ = "";
145 do {
146 bytes_read = read_callback_(buffer, 1, sizeof(buffer), read_data_);
147 posted_content_ =
148 strings::StrCat(posted_content_, StringPiece(buffer, bytes_read));
149 } while (bytes_read > 0);
150 }
151 if (write_data_ || write_callback_) {
152 size_t bytes_handled = write_callback_(
153 response_content_.c_str(), 1, response_content_.size(), write_data_);
154 // Mimic real libcurl behavior by checking write callback return value.
155 if (bytes_handled != response_content_.size()) {
156 curl_easy_perform_result_ = CURLE_WRITE_ERROR;
157 }
158 }
159 for (const auto& header : response_headers_) {
160 header_callback_(header.c_str(), 1, header.size(), header_data_);
161 }
162 if (error_buffer_) {
163 strncpy(error_buffer_, curl_easy_perform_error_message_.c_str(),
164 curl_easy_perform_error_message_.size() + 1);
165 }
166 for (const auto& tick : progress_ticks_) {
167 env_->now_ = std::get<0>(tick);
168 if (progress_callback_(progress_data_, 0, std::get<1>(tick), 0, 0)) {
169 return CURLE_ABORTED_BY_CALLBACK;
170 }
171 }
172 return curl_easy_perform_result_;
173 }
curl_easy_getinfo(CURL * curl,CURLINFO info,uint64 * value)174 CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info,
175 uint64* value) override {
176 switch (info) {
177 case CURLINFO_RESPONSE_CODE:
178 *value = response_code_;
179 break;
180 default:
181 break;
182 }
183 return CURLE_OK;
184 }
curl_easy_getinfo(CURL * curl,CURLINFO info,double * value)185 CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info,
186 double* value) override {
187 switch (info) {
188 case CURLINFO_SIZE_DOWNLOAD:
189 *value = response_content_.size();
190 break;
191 default:
192 break;
193 }
194 return CURLE_OK;
195 }
curl_easy_cleanup(CURL * curl)196 void curl_easy_cleanup(CURL* curl) override { is_cleaned_up_ = true; }
curl_slist_append(curl_slist * list,const char * str)197 curl_slist* curl_slist_append(curl_slist* list, const char* str) override {
198 std::vector<string>* v = list ? reinterpret_cast<std::vector<string>*>(list)
199 : new std::vector<string>();
200 v->push_back(str);
201 return reinterpret_cast<curl_slist*>(v);
202 }
curl_easy_escape(CURL * curl,const char * str,int length)203 char* curl_easy_escape(CURL* curl, const char* str, int length) override {
204 // This function just does a simple replacing of "/" with "%2F" instead of
205 // full url encoding.
206 const string victim = "/";
207 const string encoded = "%2F";
208
209 string temp_str = str;
210 std::string::size_type n = 0;
211 while ((n = temp_str.find(victim, n)) != std::string::npos) {
212 temp_str.replace(n, victim.size(), encoded);
213 n += encoded.size();
214 }
215 char* out_char_str = reinterpret_cast<char*>(
216 port::Malloc(sizeof(char) * temp_str.size() + 1));
217 std::copy(temp_str.begin(), temp_str.end(), out_char_str);
218 out_char_str[temp_str.size()] = '\0';
219 return out_char_str;
220 }
curl_slist_free_all(curl_slist * list)221 void curl_slist_free_all(curl_slist* list) override {
222 delete reinterpret_cast<std::vector<string>*>(list);
223 }
curl_free(void * p)224 void curl_free(void* p) override { port::Free(p); }
225
226 // Variables defining the behavior of this fake.
227 string response_content_;
228 uint64 response_code_;
229 std::vector<string> response_headers_;
230
231 // Internal variables to store the libcurl state.
232 string url_;
233 string range_;
234 string custom_request_;
235 char* error_buffer_ = nullptr;
236 bool is_initialized_ = false;
237 bool is_cleaned_up_ = false;
238 std::vector<string>* headers_ = nullptr;
239 bool is_post_ = false;
240 bool is_put_ = false;
241 void* write_data_ = nullptr;
242 size_t (*write_callback_)(const void* ptr, size_t size, size_t nmemb,
243 void* userdata) = nullptr;
244 void* header_data_ = nullptr;
245 size_t (*header_callback_)(const void* ptr, size_t size, size_t nmemb,
246 void* userdata) = nullptr;
247 FILE* read_data_ = nullptr;
248 size_t (*read_callback_)(void* ptr, size_t size, size_t nmemb,
249 FILE* userdata) = &fread;
250 int (*progress_callback_)(void* clientp, curl_off_t dltotal, curl_off_t dlnow,
251 curl_off_t ultotal, curl_off_t ulnow) = nullptr;
252 void* progress_data_ = nullptr;
253 // Outcome of performing the request.
254 string posted_content_;
255 CURLcode curl_easy_perform_result_ = CURLE_OK;
256 string curl_easy_perform_error_message_;
257 // A vector of <timestamp, progress in bytes> pairs that represent the
258 // progress of a transmission.
259 std::vector<std::tuple<uint64, curl_off_t>> progress_ticks_;
260 FakeEnv* env_ = nullptr;
261 };
262
TEST(CurlHttpRequestTest,GetRequest)263 TEST(CurlHttpRequestTest, GetRequest) {
264 FakeLibCurl libcurl("get response", 200);
265 CurlHttpRequest http_request(&libcurl);
266
267 std::vector<char> scratch;
268 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
269 scratch.reserve(100);
270
271 http_request.SetUri("http://www.testuri.com");
272 http_request.AddAuthBearerHeader("fake-bearer");
273 http_request.SetRange(100, 199);
274 http_request.SetResultBuffer(&scratch);
275 TF_EXPECT_OK(http_request.Send());
276
277 EXPECT_EQ("get response", string(scratch.begin(), scratch.end()));
278
279 // Check interactions with libcurl.
280 EXPECT_TRUE(libcurl.is_initialized_);
281 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
282 EXPECT_EQ("100-199", libcurl.range_);
283 EXPECT_EQ("", libcurl.custom_request_);
284 EXPECT_EQ(1, libcurl.headers_->size());
285 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
286 EXPECT_FALSE(libcurl.is_post_);
287 EXPECT_EQ(200, http_request.GetResponseCode());
288 }
289
TEST(CurlHttpRequestTest,GetRequest_Direct)290 TEST(CurlHttpRequestTest, GetRequest_Direct) {
291 FakeLibCurl libcurl("get response", 200);
292 CurlHttpRequest http_request(&libcurl);
293
294 std::vector<char> scratch(100, 0);
295
296 http_request.SetUri("http://www.testuri.com");
297 http_request.AddAuthBearerHeader("fake-bearer");
298 http_request.SetRange(100, 199);
299 http_request.SetResultBufferDirect(scratch.data(), scratch.capacity());
300 TF_EXPECT_OK(http_request.Send());
301
302 string expected_response = "get response";
303 size_t response_bytes_transferred =
304 http_request.GetResultBufferDirectBytesTransferred();
305 EXPECT_EQ(expected_response.size(), response_bytes_transferred);
306 EXPECT_EQ(
307 "get response",
308 string(scratch.begin(), scratch.begin() + response_bytes_transferred));
309
310 // Check interactions with libcurl.
311 EXPECT_TRUE(libcurl.is_initialized_);
312 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
313 EXPECT_EQ("100-199", libcurl.range_);
314 EXPECT_EQ("", libcurl.custom_request_);
315 EXPECT_EQ(1, libcurl.headers_->size());
316 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
317 EXPECT_FALSE(libcurl.is_post_);
318 EXPECT_EQ(200, http_request.GetResponseCode());
319 }
320
TEST(CurlHttpRequestTest,GetRequest_Direct_ResponseTooLarge)321 TEST(CurlHttpRequestTest, GetRequest_Direct_ResponseTooLarge) {
322 FakeLibCurl libcurl("get response", 200);
323 CurlHttpRequest http_request(&libcurl);
324
325 std::vector<char> scratch(5, 0);
326
327 http_request.SetUri("http://www.testuri.com");
328 http_request.SetResultBufferDirect(scratch.data(), scratch.size());
329 const Status& status = http_request.Send();
330 EXPECT_EQ(error::FAILED_PRECONDITION, status.code());
331 EXPECT_EQ(
332 "Error executing an HTTP request: libcurl code 23 meaning "
333 "'Failed writing received data to disk/application', error details: "
334 "Received 12 response bytes for a 5-byte buffer",
335 status.error_message());
336
337 // As long as the request clearly fails, ok to leave truncated response here.
338 EXPECT_EQ(5, http_request.GetResultBufferDirectBytesTransferred());
339 EXPECT_EQ("get r", string(scratch.begin(), scratch.begin() + 5));
340 }
341
TEST(CurlHttpRequestTest,GetRequest_Direct_RangeOutOfBound)342 TEST(CurlHttpRequestTest, GetRequest_Direct_RangeOutOfBound) {
343 FakeLibCurl libcurl("get response", 416);
344 CurlHttpRequest http_request(&libcurl);
345
346 const string initialScratch = "abcde";
347 std::vector<char> scratch;
348 scratch.insert(scratch.end(), initialScratch.begin(), initialScratch.end());
349
350 http_request.SetUri("http://www.testuri.com");
351 http_request.SetRange(0, 4);
352 http_request.SetResultBufferDirect(scratch.data(), scratch.size());
353 TF_EXPECT_OK(http_request.Send());
354 EXPECT_EQ(416, http_request.GetResponseCode());
355
356 // Some servers (in particular, GCS) return an error message payload with a
357 // 416 Range Not Satisfiable response. We should pretend it's not there when
358 // reporting bytes transferred, but it's ok if it writes to scratch.
359 EXPECT_EQ(0, http_request.GetResultBufferDirectBytesTransferred());
360 EXPECT_EQ("get r", string(scratch.begin(), scratch.end()));
361 }
362
TEST(CurlHttpRequestTest,GetRequest_Empty)363 TEST(CurlHttpRequestTest, GetRequest_Empty) {
364 FakeLibCurl libcurl("", 200);
365 CurlHttpRequest http_request(&libcurl);
366
367 std::vector<char> scratch;
368 scratch.resize(0);
369
370 http_request.SetUri("http://www.testuri.com");
371 http_request.AddAuthBearerHeader("fake-bearer");
372 http_request.SetRange(100, 199);
373 http_request.SetResultBuffer(&scratch);
374 TF_EXPECT_OK(http_request.Send());
375
376 EXPECT_TRUE(scratch.empty());
377
378 // Check interactions with libcurl.
379 EXPECT_TRUE(libcurl.is_initialized_);
380 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
381 EXPECT_EQ("100-199", libcurl.range_);
382 EXPECT_EQ("", libcurl.custom_request_);
383 EXPECT_EQ(1, libcurl.headers_->size());
384 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
385 EXPECT_FALSE(libcurl.is_post_);
386 EXPECT_EQ(200, http_request.GetResponseCode());
387 }
388
TEST(CurlHttpRequestTest,GetRequest_RangeOutOfBound)389 TEST(CurlHttpRequestTest, GetRequest_RangeOutOfBound) {
390 FakeLibCurl libcurl("get response", 416);
391 CurlHttpRequest http_request(&libcurl);
392
393 std::vector<char> scratch;
394 scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end());
395
396 http_request.SetUri("http://www.testuri.com");
397 http_request.AddAuthBearerHeader("fake-bearer");
398 http_request.SetRange(100, 199);
399 http_request.SetResultBuffer(&scratch);
400 TF_EXPECT_OK(http_request.Send());
401
402 // Some servers (in particular, GCS) return an error message payload with a
403 // 416 Range Not Satisfiable response. We should pretend it's not there.
404 EXPECT_TRUE(scratch.empty());
405 EXPECT_EQ(416, http_request.GetResponseCode());
406 }
407
TEST(CurlHttpRequestTest,GetRequest_503)408 TEST(CurlHttpRequestTest, GetRequest_503) {
409 FakeLibCurl libcurl("get response", 503);
410 CurlHttpRequest http_request(&libcurl);
411
412 std::vector<char> scratch;
413 scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end());
414
415 http_request.SetUri("http://www.testuri.com");
416 http_request.SetResultBuffer(&scratch);
417 const auto& status = http_request.Send();
418 EXPECT_EQ(error::UNAVAILABLE, status.code());
419 EXPECT_EQ(
420 "Error executing an HTTP request: HTTP response code 503 with body "
421 "'get response'",
422 status.error_message());
423 }
424
TEST(CurlHttpRequestTest,GetRequest_HttpCode0)425 TEST(CurlHttpRequestTest, GetRequest_HttpCode0) {
426 FakeLibCurl libcurl("get response", 0);
427 libcurl.curl_easy_perform_result_ = CURLE_OPERATION_TIMEDOUT;
428 libcurl.curl_easy_perform_error_message_ = "Operation timed out";
429 CurlHttpRequest http_request(&libcurl);
430
431 std::vector<char> scratch;
432 scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end());
433
434 http_request.SetUri("http://www.testuri.com");
435 const auto& status = http_request.Send();
436 EXPECT_EQ(error::UNAVAILABLE, status.code());
437 EXPECT_EQ(
438 "Error executing an HTTP request: libcurl code 28 meaning "
439 "'Timeout was reached', error details: Operation timed out",
440 status.error_message());
441 EXPECT_EQ(0, http_request.GetResponseCode());
442 }
443
TEST(CurlHttpRequestTest,ResponseHeaders)444 TEST(CurlHttpRequestTest, ResponseHeaders) {
445 FakeLibCurl libcurl(
446 "get response", 200,
447 {"Location: abcd", "Content-Type: text", "unparsable header"});
448 CurlHttpRequest http_request(&libcurl);
449
450 http_request.SetUri("http://www.testuri.com");
451 TF_EXPECT_OK(http_request.Send());
452
453 EXPECT_EQ("abcd", http_request.GetResponseHeader("Location"));
454 EXPECT_EQ("text", http_request.GetResponseHeader("Content-Type"));
455 EXPECT_EQ("", http_request.GetResponseHeader("Not-Seen-Header"));
456 }
457
TEST(CurlHttpRequestTest,PutRequest_WithBody_FromFile)458 TEST(CurlHttpRequestTest, PutRequest_WithBody_FromFile) {
459 FakeLibCurl libcurl("", 200);
460 CurlHttpRequest http_request(&libcurl);
461
462 auto content_filename = io::JoinPath(testing::TmpDir(), "content");
463 std::ofstream content(content_filename, std::ofstream::binary);
464 content << "post body content";
465 content.close();
466
467 http_request.SetUri("http://www.testuri.com");
468 http_request.AddAuthBearerHeader("fake-bearer");
469 TF_EXPECT_OK(http_request.SetPutFromFile(content_filename, 0));
470 TF_EXPECT_OK(http_request.Send());
471
472 // Check interactions with libcurl.
473 EXPECT_TRUE(libcurl.is_initialized_);
474 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
475 EXPECT_EQ("", libcurl.custom_request_);
476 EXPECT_EQ(2, libcurl.headers_->size());
477 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
478 EXPECT_EQ("Content-Length: 17", (*libcurl.headers_)[1]);
479 EXPECT_TRUE(libcurl.is_put_);
480 EXPECT_EQ("post body content", libcurl.posted_content_);
481
482 std::remove(content_filename.c_str());
483 }
484
TEST(CurlHttpRequestTest,PutRequest_WithBody_FromFile_NonZeroOffset)485 TEST(CurlHttpRequestTest, PutRequest_WithBody_FromFile_NonZeroOffset) {
486 FakeLibCurl libcurl("", 200);
487 CurlHttpRequest http_request(&libcurl);
488
489 auto content_filename = io::JoinPath(testing::TmpDir(), "content");
490 std::ofstream content(content_filename, std::ofstream::binary);
491 content << "post body content";
492 content.close();
493
494 http_request.SetUri("http://www.testuri.com");
495 http_request.AddAuthBearerHeader("fake-bearer");
496 TF_EXPECT_OK(http_request.SetPutFromFile(content_filename, 7));
497 TF_EXPECT_OK(http_request.Send());
498
499 // Check interactions with libcurl.
500 EXPECT_EQ("dy content", libcurl.posted_content_);
501
502 std::remove(content_filename.c_str());
503 }
504
TEST(CurlHttpRequestTest,PutRequest_WithoutBody)505 TEST(CurlHttpRequestTest, PutRequest_WithoutBody) {
506 FakeLibCurl libcurl("", 200);
507 CurlHttpRequest http_request(&libcurl);
508
509 http_request.SetUri("http://www.testuri.com");
510 http_request.AddAuthBearerHeader("fake-bearer");
511 http_request.SetPutEmptyBody();
512 TF_EXPECT_OK(http_request.Send());
513
514 // Check interactions with libcurl.
515 EXPECT_TRUE(libcurl.is_initialized_);
516 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
517 EXPECT_EQ("", libcurl.custom_request_);
518 EXPECT_EQ(3, libcurl.headers_->size());
519 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
520 EXPECT_EQ("Content-Length: 0", (*libcurl.headers_)[1]);
521 EXPECT_EQ("Transfer-Encoding: identity", (*libcurl.headers_)[2]);
522 EXPECT_TRUE(libcurl.is_put_);
523 EXPECT_EQ("", libcurl.posted_content_);
524 }
525
TEST(CurlHttpRequestTest,PostRequest_WithBody_FromMemory)526 TEST(CurlHttpRequestTest, PostRequest_WithBody_FromMemory) {
527 FakeLibCurl libcurl("", 200);
528 CurlHttpRequest http_request(&libcurl);
529
530 string content = "post body content";
531
532 http_request.SetUri("http://www.testuri.com");
533 http_request.AddAuthBearerHeader("fake-bearer");
534 http_request.SetPostFromBuffer(content.c_str(), content.size());
535 TF_EXPECT_OK(http_request.Send());
536
537 // Check interactions with libcurl.
538 EXPECT_TRUE(libcurl.is_initialized_);
539 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
540 EXPECT_EQ("", libcurl.custom_request_);
541 EXPECT_EQ(2, libcurl.headers_->size());
542 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
543 EXPECT_EQ("Content-Length: 17", (*libcurl.headers_)[1]);
544 EXPECT_TRUE(libcurl.is_post_);
545 EXPECT_EQ("post body content", libcurl.posted_content_);
546 }
547
TEST(CurlHttpRequestTest,PostRequest_WithoutBody)548 TEST(CurlHttpRequestTest, PostRequest_WithoutBody) {
549 FakeLibCurl libcurl("", 200);
550 CurlHttpRequest http_request(&libcurl);
551 http_request.SetUri("http://www.testuri.com");
552 http_request.AddAuthBearerHeader("fake-bearer");
553 http_request.SetPostEmptyBody();
554 TF_EXPECT_OK(http_request.Send());
555
556 // Check interactions with libcurl.
557 EXPECT_TRUE(libcurl.is_initialized_);
558 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
559 EXPECT_EQ("", libcurl.custom_request_);
560 EXPECT_EQ(3, libcurl.headers_->size());
561 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
562 EXPECT_EQ("Content-Length: 0", (*libcurl.headers_)[1]);
563 EXPECT_EQ("Transfer-Encoding: identity", (*libcurl.headers_)[2]);
564 EXPECT_TRUE(libcurl.is_post_);
565 EXPECT_EQ("", libcurl.posted_content_);
566 }
567
TEST(CurlHttpRequestTest,DeleteRequest)568 TEST(CurlHttpRequestTest, DeleteRequest) {
569 FakeLibCurl libcurl("", 200);
570 CurlHttpRequest http_request(&libcurl);
571 http_request.SetUri("http://www.testuri.com");
572 http_request.AddAuthBearerHeader("fake-bearer");
573 http_request.SetDeleteRequest();
574 TF_EXPECT_OK(http_request.Send());
575
576 // Check interactions with libcurl.
577 EXPECT_TRUE(libcurl.is_initialized_);
578 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
579 EXPECT_EQ("DELETE", libcurl.custom_request_);
580 EXPECT_EQ(1, libcurl.headers_->size());
581 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
582 EXPECT_FALSE(libcurl.is_post_);
583 }
584
TEST(CurlHttpRequestTest,WrongSequenceOfCalls_NoUri)585 TEST(CurlHttpRequestTest, WrongSequenceOfCalls_NoUri) {
586 FakeLibCurl libcurl("", 200);
587 CurlHttpRequest http_request(&libcurl);
588 ASSERT_DEATH((void)http_request.Send(), "URI has not been set");
589 }
590
TEST(CurlHttpRequestTest,WrongSequenceOfCalls_TwoSends)591 TEST(CurlHttpRequestTest, WrongSequenceOfCalls_TwoSends) {
592 FakeLibCurl libcurl("", 200);
593 CurlHttpRequest http_request(&libcurl);
594 http_request.SetUri("http://www.google.com");
595 TF_EXPECT_OK(http_request.Send());
596 ASSERT_DEATH((void)http_request.Send(), "The request has already been sent");
597 }
598
TEST(CurlHttpRequestTest,WrongSequenceOfCalls_ReusingAfterSend)599 TEST(CurlHttpRequestTest, WrongSequenceOfCalls_ReusingAfterSend) {
600 FakeLibCurl libcurl("", 200);
601 CurlHttpRequest http_request(&libcurl);
602 http_request.SetUri("http://www.google.com");
603 TF_EXPECT_OK(http_request.Send());
604 ASSERT_DEATH(http_request.SetUri("http://mail.google.com"),
605 "The request has already been sent");
606 }
607
TEST(CurlHttpRequestTest,WrongSequenceOfCalls_SettingMethodTwice)608 TEST(CurlHttpRequestTest, WrongSequenceOfCalls_SettingMethodTwice) {
609 FakeLibCurl libcurl("", 200);
610 CurlHttpRequest http_request(&libcurl);
611 http_request.SetDeleteRequest();
612 ASSERT_DEATH(http_request.SetPostEmptyBody(),
613 "HTTP method has been already set");
614 }
615
TEST(CurlHttpRequestTest,EscapeString)616 TEST(CurlHttpRequestTest, EscapeString) {
617 FakeLibCurl libcurl("get response", 200);
618 CurlHttpRequest http_request(&libcurl);
619 const string test_string = "a/b/c";
620 EXPECT_EQ("a%2Fb%2Fc", http_request.EscapeString(test_string));
621 }
622
TEST(CurlHttpRequestTest,ErrorReturnsNoResponse)623 TEST(CurlHttpRequestTest, ErrorReturnsNoResponse) {
624 FakeLibCurl libcurl("get response", 500);
625 CurlHttpRequest http_request(&libcurl);
626
627 std::vector<char> scratch;
628 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
629 scratch.reserve(100);
630
631 http_request.SetUri("http://www.testuri.com");
632 http_request.AddAuthBearerHeader("fake-bearer");
633 http_request.SetRange(100, 199);
634 http_request.SetResultBuffer(&scratch);
635 EXPECT_EQ(error::UNAVAILABLE, http_request.Send().code());
636
637 EXPECT_EQ("", string(scratch.begin(), scratch.end()));
638 }
639
TEST(CurlHttpRequestTest,ProgressIsOk)640 TEST(CurlHttpRequestTest, ProgressIsOk) {
641 // Imitate a steady progress.
642 FakeEnv env;
643 FakeLibCurl libcurl(
644 "test", 200,
645 {
646 std::make_tuple(100, 0) /* timestamp 100, 0 bytes */,
647 std::make_tuple(110, 0) /* timestamp 110, 0 bytes */,
648 std::make_tuple(200, 100) /* timestamp 200, 100 bytes */
649 },
650 &env);
651 CurlHttpRequest http_request(&libcurl, &env);
652 http_request.SetUri("http://www.testuri.com");
653 TF_EXPECT_OK(http_request.Send());
654 }
655
TEST(CurlHttpRequestTest,ProgressIsStuck)656 TEST(CurlHttpRequestTest, ProgressIsStuck) {
657 // Imitate a transmission that got stuck for more than a minute.
658 FakeEnv env;
659 FakeLibCurl libcurl(
660 "test", 200,
661 {
662 std::make_tuple(100, 10) /* timestamp 100, 10 bytes */,
663 std::make_tuple(130, 10) /* timestamp 130, 10 bytes */,
664 std::make_tuple(170, 10) /* timestamp 170, 10 bytes */
665 },
666 &env);
667 CurlHttpRequest http_request(&libcurl, &env);
668 http_request.SetUri("http://www.testuri.com");
669 auto status = http_request.Send();
670 EXPECT_EQ(error::UNAVAILABLE, status.code());
671 EXPECT_EQ(
672 "Error executing an HTTP request: libcurl code 42 meaning 'Operation "
673 "was aborted by an application callback', error details: (none)",
674 status.error_message());
675 }
676
677 class TestStats : public HttpRequest::RequestStats {
678 public:
679 ~TestStats() override = default;
680
RecordRequest(const HttpRequest * request,const string & uri,HttpRequest::RequestMethod method)681 void RecordRequest(const HttpRequest* request, const string& uri,
682 HttpRequest::RequestMethod method) override {
683 has_recorded_request_ = true;
684 record_request_request_ = request;
685 record_request_uri_ = uri;
686 record_request_method_ = method;
687 }
688
RecordResponse(const HttpRequest * request,const string & uri,HttpRequest::RequestMethod method,const Status & result)689 void RecordResponse(const HttpRequest* request, const string& uri,
690 HttpRequest::RequestMethod method,
691 const Status& result) override {
692 has_recorded_response_ = true;
693 record_response_request_ = request;
694 record_response_uri_ = uri;
695 record_response_method_ = method;
696 record_response_result_ = result;
697 }
698
699 const HttpRequest* record_request_request_ = nullptr;
700 string record_request_uri_ = "http://www.testuri.com";
701 HttpRequest::RequestMethod record_request_method_ =
702 HttpRequest::RequestMethod::kGet;
703
704 const HttpRequest* record_response_request_ = nullptr;
705 string record_response_uri_ = "http://www.testuri.com";
706 HttpRequest::RequestMethod record_response_method_ =
707 HttpRequest::RequestMethod::kGet;
708 Status record_response_result_;
709
710 bool has_recorded_request_ = false;
711 bool has_recorded_response_ = false;
712 };
713
714 class StatsTestFakeLibCurl : public FakeLibCurl {
715 public:
StatsTestFakeLibCurl(TestStats * stats,const string & response_content,uint64 response_code)716 StatsTestFakeLibCurl(TestStats* stats, const string& response_content,
717 uint64 response_code)
718 : FakeLibCurl(response_content, response_code), stats_(stats) {}
curl_easy_perform(CURL * curl)719 CURLcode curl_easy_perform(CURL* curl) override {
720 CHECK(!performed_request_);
721 performed_request_ = true;
722 stats_had_recorded_request_ = stats_->has_recorded_request_;
723 stats_had_recorded_response_ = stats_->has_recorded_response_;
724 return FakeLibCurl::curl_easy_perform(curl);
725 };
726
727 TestStats* stats_;
728 bool performed_request_ = false;
729 bool stats_had_recorded_request_;
730 bool stats_had_recorded_response_;
731 };
732
TEST(CurlHttpRequestTest,StatsGetSuccessful)733 TEST(CurlHttpRequestTest, StatsGetSuccessful) {
734 TestStats stats;
735 StatsTestFakeLibCurl libcurl(&stats, "get response", 200);
736 CurlHttpRequest http_request(&libcurl);
737
738 std::vector<char> scratch;
739 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
740 scratch.reserve(100);
741
742 http_request.SetRequestStats(&stats);
743
744 http_request.SetUri("http://www.testuri.com");
745 http_request.AddAuthBearerHeader("fake-bearer");
746 http_request.SetRange(100, 199);
747 http_request.SetResultBuffer(&scratch);
748 TF_EXPECT_OK(http_request.Send());
749
750 EXPECT_EQ("get response", string(scratch.begin(), scratch.end()));
751
752 // Check interaction with stats.
753 ASSERT_TRUE(stats.has_recorded_request_);
754 EXPECT_EQ(&http_request, stats.record_request_request_);
755 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
756 EXPECT_EQ(HttpRequest::RequestMethod::kGet, stats.record_request_method_);
757
758 ASSERT_TRUE(stats.has_recorded_response_);
759 EXPECT_EQ(&http_request, stats.record_response_request_);
760 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
761 EXPECT_EQ(HttpRequest::RequestMethod::kGet, stats.record_response_method_);
762 TF_EXPECT_OK(stats.record_response_result_);
763
764 // Check interaction with libcurl.
765 EXPECT_TRUE(libcurl.performed_request_);
766 EXPECT_TRUE(libcurl.stats_had_recorded_request_);
767 EXPECT_FALSE(libcurl.stats_had_recorded_response_);
768 }
769
TEST(CurlHttpRequestTest,StatsGetNotFound)770 TEST(CurlHttpRequestTest, StatsGetNotFound) {
771 TestStats stats;
772 StatsTestFakeLibCurl libcurl(&stats, "get other response", 404);
773 CurlHttpRequest http_request(&libcurl);
774
775 std::vector<char> scratch;
776 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
777 scratch.reserve(100);
778
779 http_request.SetRequestStats(&stats);
780
781 http_request.SetUri("http://www.testuri.com");
782 http_request.AddAuthBearerHeader("fake-bearer");
783 http_request.SetRange(100, 199);
784 http_request.SetResultBuffer(&scratch);
785 Status s = http_request.Send();
786
787 // Check interaction with stats.
788 ASSERT_TRUE(stats.has_recorded_request_);
789 EXPECT_EQ(&http_request, stats.record_request_request_);
790 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
791 EXPECT_EQ(HttpRequest::RequestMethod::kGet, stats.record_request_method_);
792
793 ASSERT_TRUE(stats.has_recorded_response_);
794 EXPECT_EQ(&http_request, stats.record_response_request_);
795 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
796 EXPECT_EQ(HttpRequest::RequestMethod::kGet, stats.record_response_method_);
797 EXPECT_TRUE(errors::IsNotFound(stats.record_response_result_));
798 EXPECT_EQ(s, stats.record_response_result_);
799
800 // Check interaction with libcurl.
801 EXPECT_TRUE(libcurl.performed_request_);
802 EXPECT_TRUE(libcurl.stats_had_recorded_request_);
803 EXPECT_FALSE(libcurl.stats_had_recorded_response_);
804 }
805
TEST(CurlHttpRequestTest,StatsPost)806 TEST(CurlHttpRequestTest, StatsPost) {
807 TestStats stats;
808
809 FakeLibCurl libcurl("", 200);
810 CurlHttpRequest http_request(&libcurl);
811
812 http_request.SetRequestStats(&stats);
813
814 string content = "post body content";
815
816 http_request.SetUri("http://www.testuri.com");
817 http_request.SetPostFromBuffer(content.c_str(), content.size());
818 TF_EXPECT_OK(http_request.Send());
819
820 // Check interaction with stats.
821 ASSERT_TRUE(stats.has_recorded_request_);
822 EXPECT_EQ(&http_request, stats.record_request_request_);
823 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
824 EXPECT_EQ(HttpRequest::RequestMethod::kPost, stats.record_request_method_);
825
826 ASSERT_TRUE(stats.has_recorded_response_);
827 EXPECT_EQ(&http_request, stats.record_response_request_);
828 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
829 EXPECT_EQ(HttpRequest::RequestMethod::kPost, stats.record_response_method_);
830 TF_EXPECT_OK(stats.record_response_result_);
831 }
832
TEST(CurlHttpRequestTest,StatsDelete)833 TEST(CurlHttpRequestTest, StatsDelete) {
834 TestStats stats;
835
836 FakeLibCurl libcurl("", 200);
837 CurlHttpRequest http_request(&libcurl);
838 http_request.SetRequestStats(&stats);
839 http_request.SetUri("http://www.testuri.com");
840 http_request.SetDeleteRequest();
841 TF_EXPECT_OK(http_request.Send());
842
843 // Check interaction with stats.
844 ASSERT_TRUE(stats.has_recorded_request_);
845 EXPECT_EQ(&http_request, stats.record_request_request_);
846 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
847 EXPECT_EQ(HttpRequest::RequestMethod::kDelete, stats.record_request_method_);
848
849 ASSERT_TRUE(stats.has_recorded_response_);
850 EXPECT_EQ(&http_request, stats.record_response_request_);
851 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
852 EXPECT_EQ(HttpRequest::RequestMethod::kDelete, stats.record_response_method_);
853 TF_EXPECT_OK(stats.record_response_result_);
854 }
855
TEST(CurlHttpRequestTest,StatsPut)856 TEST(CurlHttpRequestTest, StatsPut) {
857 TestStats stats;
858
859 FakeLibCurl libcurl("", 200);
860 CurlHttpRequest http_request(&libcurl);
861 http_request.SetRequestStats(&stats);
862 http_request.SetUri("http://www.testuri.com");
863 http_request.AddAuthBearerHeader("fake-bearer");
864 http_request.SetPutEmptyBody();
865 TF_EXPECT_OK(http_request.Send());
866
867 // Check interaction with stats.
868 ASSERT_TRUE(stats.has_recorded_request_);
869 EXPECT_EQ(&http_request, stats.record_request_request_);
870 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
871 EXPECT_EQ(HttpRequest::RequestMethod::kPut, stats.record_request_method_);
872
873 ASSERT_TRUE(stats.has_recorded_response_);
874 EXPECT_EQ(&http_request, stats.record_response_request_);
875 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
876 EXPECT_EQ(HttpRequest::RequestMethod::kPut, stats.record_response_method_);
877 TF_EXPECT_OK(stats.record_response_result_);
878 }
879
880 } // namespace
881 } // namespace tensorflow
882