• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //
2 // Copyright (C) 2012 The Android Open Source Project
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 //      http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16 
17 #include <netinet/in.h>
18 #include <netinet/ip.h>
19 #include <sys/socket.h>
20 #include <unistd.h>
21 
22 #include <algorithm>
23 #include <memory>
24 #include <string>
25 #include <utility>
26 #include <vector>
27 
28 #include <base/bind.h>
29 #include <base/location.h>
30 #include <base/logging.h>
31 #if BASE_VER < 780000  // Android
32 #include <base/message_loop/message_loop.h>
33 #endif  // BASE_VER < 780000
34 #include <base/stl_util.h>
35 #include <base/strings/string_number_conversions.h>
36 #include <base/strings/string_util.h>
37 #include <base/strings/stringprintf.h>
38 #if BASE_VER >= 780000  // CrOS
39 #include <base/task/single_thread_task_executor.h>
40 #endif  // BASE_VER >= 780000
41 #include <base/time/time.h>
42 #include <brillo/message_loops/base_message_loop.h>
43 #include <brillo/message_loops/message_loop.h>
44 #include <brillo/message_loops/message_loop_utils.h>
45 #ifdef __CHROMEOS__
46 #include <brillo/process/process.h>
47 #else
48 #include <brillo/process.h>
49 #endif  // __CHROMEOS__
50 #include <brillo/streams/file_stream.h>
51 #include <brillo/streams/stream.h>
52 #include <gtest/gtest.h>
53 
54 #include "update_engine/common/fake_hardware.h"
55 #include "update_engine/common/file_fetcher.h"
56 #include "update_engine/common/http_common.h"
57 #include "update_engine/common/mock_http_fetcher.h"
58 #include "update_engine/common/mock_proxy_resolver.h"
59 #include "update_engine/common/multi_range_http_fetcher.h"
60 #include "update_engine/common/proxy_resolver.h"
61 #include "update_engine/common/test_utils.h"
62 #include "update_engine/common/utils.h"
63 #include "update_engine/libcurl_http_fetcher.h"
64 
65 using brillo::MessageLoop;
66 using std::make_pair;
67 using std::pair;
68 using std::string;
69 using std::unique_ptr;
70 using std::vector;
71 using testing::_;
72 using testing::DoAll;
73 using testing::Return;
74 using testing::SaveArg;
75 
76 namespace {
77 
78 const int kBigLength = 100000;
79 const int kMediumLength = 1000;
80 const int kFlakyTruncateLength = 29000;
81 const int kFlakySleepEvery = 3;
82 const int kFlakySleepSecs = 10;
83 
84 }  // namespace
85 
86 namespace chromeos_update_engine {
87 
88 static const char* kUnusedUrl = "unused://unused";
89 
LocalServerUrlForPath(in_port_t port,const string & path)90 static inline string LocalServerUrlForPath(in_port_t port, const string& path) {
91   string port_str = (port ? base::StringPrintf(":%hu", port) : "");
92   return base::StringPrintf(
93       "http://127.0.0.1%s%s", port_str.c_str(), path.c_str());
94 }
95 
96 //
97 // Class hierarchy for HTTP server implementations.
98 //
99 
100 namespace {
101 
102 class HttpServer {
103  public:
104   // This makes it an abstract class (dirty but works).
105   virtual ~HttpServer() = 0;
106 
GetPort() const107   virtual in_port_t GetPort() const { return 0; }
108 
109   bool started_;
110 };
111 
~HttpServer()112 HttpServer::~HttpServer() {}
113 
114 class NullHttpServer : public HttpServer {
115  public:
NullHttpServer()116   NullHttpServer() { started_ = true; }
117 };
118 
119 class PythonHttpServer : public HttpServer {
120  public:
PythonHttpServer()121   PythonHttpServer() : port_(0) {
122     started_ = false;
123 
124     // Spawn the server process.
125     unique_ptr<brillo::Process> http_server(new brillo::ProcessImpl());
126     http_server->AddArg(test_utils::GetBuildArtifactsPath("test_http_server"));
127     http_server->RedirectUsingPipe(STDOUT_FILENO, false);
128 
129     if (!http_server->Start()) {
130       ADD_FAILURE() << "failed to spawn http server process";
131       return;
132     }
133     LOG(INFO) << "started http server with pid " << http_server->pid();
134 
135     // Wait for server to begin accepting connections, obtain its port.
136     brillo::StreamPtr stdout = brillo::FileStream::FromFileDescriptor(
137         http_server->GetPipe(STDOUT_FILENO), false /* own */, nullptr);
138     if (!stdout)
139       return;
140 
141     vector<char> buf(128);
142     string line;
143     while (line.find('\n') == string::npos) {
144       size_t read;
145       if (!stdout->ReadBlocking(buf.data(), buf.size(), &read, nullptr)) {
146         ADD_FAILURE() << "error reading http server stdout";
147         return;
148       }
149       line.append(buf.data(), read);
150       if (read == 0)
151         break;
152     }
153     // Parse the port from the output line.
154     const size_t listening_msg_prefix_len = strlen(kServerListeningMsgPrefix);
155     if (line.size() < listening_msg_prefix_len) {
156       ADD_FAILURE() << "server output too short";
157       return;
158     }
159 
160     EXPECT_EQ(kServerListeningMsgPrefix,
161               line.substr(0, listening_msg_prefix_len));
162     string port_str = line.substr(listening_msg_prefix_len);
163     port_str.resize(port_str.find('\n'));
164     EXPECT_TRUE(base::StringToUint(port_str, &port_));
165 
166     started_ = true;
167     LOG(INFO) << "server running, listening on port " << port_;
168 
169     // Any failure before this point will SIGKILL the test server if started
170     // when the |http_server| goes out of scope.
171     http_server_ = std::move(http_server);
172   }
173 
~PythonHttpServer()174   ~PythonHttpServer() {
175     // If there's no process, do nothing.
176     if (!http_server_)
177       return;
178     // Wait up to 10 seconds for the process to finish. Destroying the process
179     // will kill it with a SIGKILL otherwise.
180     http_server_->Kill(SIGTERM, 10);
181   }
182 
GetPort() const183   in_port_t GetPort() const override { return port_; }
184 
185  private:
186   static const char* kServerListeningMsgPrefix;
187 
188   unique_ptr<brillo::Process> http_server_;
189   unsigned int port_;
190 };
191 
192 const char* PythonHttpServer::kServerListeningMsgPrefix = "listening on port ";
193 
194 //
195 // Class hierarchy for HTTP fetcher test wrappers.
196 //
197 
198 class AnyHttpFetcherFactory {
199  public:
AnyHttpFetcherFactory()200   AnyHttpFetcherFactory() {}
~AnyHttpFetcherFactory()201   virtual ~AnyHttpFetcherFactory() {}
202 
203   virtual HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) = 0;
NewLargeFetcher(size_t num_proxies)204   HttpFetcher* NewLargeFetcher(size_t num_proxies) {
205     proxy_resolver_.set_num_proxies(num_proxies);
206     return NewLargeFetcher(&proxy_resolver_);
207   }
NewLargeFetcher()208   HttpFetcher* NewLargeFetcher() { return NewLargeFetcher(1); }
209 
210   virtual HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) = 0;
NewSmallFetcher()211   HttpFetcher* NewSmallFetcher() {
212     proxy_resolver_.set_num_proxies(1);
213     return NewSmallFetcher(&proxy_resolver_);
214   }
215 
BigUrl(in_port_t port) const216   virtual string BigUrl(in_port_t port) const { return kUnusedUrl; }
SmallUrl(in_port_t port) const217   virtual string SmallUrl(in_port_t port) const { return kUnusedUrl; }
ErrorUrl(in_port_t port) const218   virtual string ErrorUrl(in_port_t port) const { return kUnusedUrl; }
219 
220   virtual bool IsMock() const = 0;
221   virtual bool IsMulti() const = 0;
222   virtual bool IsHttpSupported() const = 0;
223   virtual bool IsFileFetcher() const = 0;
224 
IgnoreServerAborting(HttpServer * server) const225   virtual void IgnoreServerAborting(HttpServer* server) const {}
226 
227   virtual HttpServer* CreateServer() = 0;
228 
fake_hardware()229   FakeHardware* fake_hardware() { return &fake_hardware_; }
230 
231  protected:
232   DirectProxyResolver proxy_resolver_;
233   FakeHardware fake_hardware_;
234 };
235 
236 class MockHttpFetcherFactory : public AnyHttpFetcherFactory {
237  public:
238   // Necessary to unhide the definition in the base class.
239   using AnyHttpFetcherFactory::NewLargeFetcher;
NewLargeFetcher(ProxyResolver * proxy_resolver)240   HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
241     brillo::Blob big_data(1000000);
242     return new MockHttpFetcher(
243         big_data.data(), big_data.size(), proxy_resolver);
244   }
245 
246   // Necessary to unhide the definition in the base class.
247   using AnyHttpFetcherFactory::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)248   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
249     return new MockHttpFetcher("x", 1, proxy_resolver);
250   }
251 
IsMock() const252   bool IsMock() const override { return true; }
IsMulti() const253   bool IsMulti() const override { return false; }
IsHttpSupported() const254   bool IsHttpSupported() const override { return true; }
IsFileFetcher() const255   bool IsFileFetcher() const override { return false; }
256 
CreateServer()257   HttpServer* CreateServer() override { return new NullHttpServer; }
258 };
259 
260 class LibcurlHttpFetcherFactory : public AnyHttpFetcherFactory {
261  public:
262   // Necessary to unhide the definition in the base class.
263   using AnyHttpFetcherFactory::NewLargeFetcher;
NewLargeFetcher(ProxyResolver * proxy_resolver)264   HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
265     LibcurlHttpFetcher* ret =
266         new LibcurlHttpFetcher(proxy_resolver, &fake_hardware_);
267     // Speed up test execution.
268     ret->set_idle_seconds(1);
269     ret->set_retry_seconds(1);
270     fake_hardware_.SetIsOfficialBuild(false);
271     return ret;
272   }
273 
274   // Necessary to unhide the definition in the base class.
275   using AnyHttpFetcherFactory::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)276   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
277     return NewLargeFetcher(proxy_resolver);
278   }
279 
BigUrl(in_port_t port) const280   string BigUrl(in_port_t port) const override {
281     return LocalServerUrlForPath(
282         port, base::StringPrintf("/download/%d", kBigLength));
283   }
SmallUrl(in_port_t port) const284   string SmallUrl(in_port_t port) const override {
285     return LocalServerUrlForPath(port, "/foo");
286   }
ErrorUrl(in_port_t port) const287   string ErrorUrl(in_port_t port) const override {
288     return LocalServerUrlForPath(port, "/error");
289   }
290 
IsMock() const291   bool IsMock() const override { return false; }
IsMulti() const292   bool IsMulti() const override { return false; }
IsHttpSupported() const293   bool IsHttpSupported() const override { return true; }
IsFileFetcher() const294   bool IsFileFetcher() const override { return false; }
295 
IgnoreServerAborting(HttpServer * server) const296   void IgnoreServerAborting(HttpServer* server) const override {
297     // Nothing to do.
298   }
299 
CreateServer()300   HttpServer* CreateServer() override { return new PythonHttpServer; }
301 };
302 
303 class MultiRangeHttpFetcherFactory : public LibcurlHttpFetcherFactory {
304  public:
305   // Necessary to unhide the definition in the base class.
306   using AnyHttpFetcherFactory::NewLargeFetcher;
NewLargeFetcher(ProxyResolver * proxy_resolver)307   HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
308     MultiRangeHttpFetcher* ret = new MultiRangeHttpFetcher(
309         new LibcurlHttpFetcher(proxy_resolver, &fake_hardware_));
310     ret->ClearRanges();
311     ret->AddRange(0);
312     // Speed up test execution.
313     ret->set_idle_seconds(1);
314     ret->set_retry_seconds(1);
315     fake_hardware_.SetIsOfficialBuild(false);
316     return ret;
317   }
318 
319   // Necessary to unhide the definition in the base class.
320   using AnyHttpFetcherFactory::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)321   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
322     return NewLargeFetcher(proxy_resolver);
323   }
324 
IsMulti() const325   bool IsMulti() const override { return true; }
326 };
327 
328 class FileFetcherFactory : public AnyHttpFetcherFactory {
329  public:
330   // Necessary to unhide the definition in the base class.
331   using AnyHttpFetcherFactory::NewLargeFetcher;
NewLargeFetcher(ProxyResolver *)332   HttpFetcher* NewLargeFetcher(ProxyResolver* /* proxy_resolver */) override {
333     return new FileFetcher();
334   }
335 
336   // Necessary to unhide the definition in the base class.
337   using AnyHttpFetcherFactory::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)338   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
339     return NewLargeFetcher(proxy_resolver);
340   }
341 
BigUrl(in_port_t port) const342   string BigUrl(in_port_t port) const override {
343     static string big_contents = []() {
344       string buf;
345       buf.reserve(kBigLength);
346       constexpr const char* kBigUrlContent = "abcdefghij";
347       for (size_t i = 0; i < kBigLength; i += strlen(kBigUrlContent)) {
348         buf.append(kBigUrlContent,
349                    std::min(kBigLength - i, strlen(kBigUrlContent)));
350       }
351       return buf;
352     }();
353     test_utils::WriteFileString(temp_file_.path(), big_contents);
354     return "file://" + temp_file_.path();
355   }
SmallUrl(in_port_t port) const356   string SmallUrl(in_port_t port) const override {
357     test_utils::WriteFileString(temp_file_.path(), "small contents");
358     return "file://" + temp_file_.path();
359   }
ErrorUrl(in_port_t port) const360   string ErrorUrl(in_port_t port) const override {
361     return "file:///path/to/non-existing-file";
362   }
363 
IsMock() const364   bool IsMock() const override { return false; }
IsMulti() const365   bool IsMulti() const override { return false; }
IsHttpSupported() const366   bool IsHttpSupported() const override { return false; }
IsFileFetcher() const367   bool IsFileFetcher() const override { return true; }
368 
IgnoreServerAborting(HttpServer * server) const369   void IgnoreServerAborting(HttpServer* server) const override {}
370 
CreateServer()371   HttpServer* CreateServer() override { return new NullHttpServer; }
372 
373  private:
374   ScopedTempFile temp_file_{"ue_file_fetcher.XXXXXX"};
375 };
376 
377 class MultiRangeHttpFetcherOverFileFetcherFactory : public FileFetcherFactory {
378  public:
379   // Necessary to unhide the definition in the base class.
380   using AnyHttpFetcherFactory::NewLargeFetcher;
NewLargeFetcher(ProxyResolver *)381   HttpFetcher* NewLargeFetcher(ProxyResolver* /* proxy_resolver */) override {
382     MultiRangeHttpFetcher* ret = new MultiRangeHttpFetcher(new FileFetcher());
383     ret->ClearRanges();
384     // FileFetcher doesn't support range with unspecified length.
385     ret->AddRange(0, 1);
386     // Speed up test execution.
387     ret->set_idle_seconds(1);
388     ret->set_retry_seconds(1);
389     fake_hardware_.SetIsOfficialBuild(false);
390     return ret;
391   }
392 
393   // Necessary to unhide the definition in the base class.
394   using AnyHttpFetcherFactory::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)395   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
396     return NewLargeFetcher(proxy_resolver);
397   }
398 
IsMulti() const399   bool IsMulti() const override { return true; }
400 };
401 
402 //
403 // Infrastructure for type tests of HTTP fetcher.
404 // See: http://code.google.com/p/googletest/wiki/AdvancedGuide#Typed_Tests
405 //
406 
407 // Fixture class template. We use an explicit constraint to guarantee that it
408 // can only be instantiated with an AnyHttpFetcherTest type, see:
409 // http://www2.research.att.com/~bs/bs_faq2.html#constraints
410 template <typename T>
411 class HttpFetcherTest : public ::testing::Test {
412  public:
413 #if BASE_VER < 780000  // Android
414   base::MessageLoopForIO base_loop_;
415   brillo::BaseMessageLoop loop_{&base_loop_};
416 #else   // Chrome OS
417   base::SingleThreadTaskExecutor base_loop_{base::MessagePumpType::IO};
418   brillo::BaseMessageLoop loop_{base_loop_.task_runner()};
419 #endif  // BASE_VER < 780000
420 
421   T test_;
422 
423  protected:
HttpFetcherTest()424   HttpFetcherTest() { loop_.SetAsCurrent(); }
425 
TearDown()426   void TearDown() override {
427     EXPECT_EQ(0, brillo::MessageLoopRunMaxIterations(&loop_, 1));
428   }
429 
430  private:
TypeConstraint(T * a)431   static void TypeConstraint(T* a) {
432     AnyHttpFetcherFactory* b = a;
433     if (b == 0)  // Silence compiler warning of unused variable.
434       *b = a;
435   }
436 };
437 
438 // Test case types list.
439 typedef ::testing::Types<LibcurlHttpFetcherFactory,
440                          MockHttpFetcherFactory,
441                          MultiRangeHttpFetcherFactory,
442                          FileFetcherFactory,
443                          MultiRangeHttpFetcherOverFileFetcherFactory>
444     HttpFetcherTestTypes;
445 TYPED_TEST_CASE(HttpFetcherTest, HttpFetcherTestTypes);
446 
447 class HttpFetcherTestDelegate : public HttpFetcherDelegate {
448  public:
449   HttpFetcherTestDelegate() = default;
450 
ReceivedBytes(HttpFetcher *,const void * bytes,size_t length)451   bool ReceivedBytes(HttpFetcher* /* fetcher */,
452                      const void* bytes,
453                      size_t length) override {
454     data.append(reinterpret_cast<const char*>(bytes), length);
455     // Update counters
456     times_received_bytes_called_++;
457     return true;
458   }
459 
TransferComplete(HttpFetcher * fetcher,bool successful)460   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
461     if (is_expect_error_)
462       EXPECT_EQ(kHttpResponseNotFound, fetcher->http_response_code());
463     else
464       EXPECT_EQ(kHttpResponseOk, fetcher->http_response_code());
465     MessageLoop::current()->BreakLoop();
466 
467     // Update counter
468     times_transfer_complete_called_++;
469   }
470 
TransferTerminated(HttpFetcher * fetcher)471   void TransferTerminated(HttpFetcher* fetcher) override {
472     times_transfer_terminated_called_++;
473     MessageLoop::current()->BreakLoop();
474   }
475 
476   // Are we expecting an error response? (default: no)
477   bool is_expect_error_{false};
478 
479   // Counters for callback invocations.
480   int times_transfer_complete_called_{0};
481   int times_transfer_terminated_called_{0};
482   int times_received_bytes_called_{0};
483 
484   // The received data bytes.
485   string data;
486 };
487 
StartTransfer(HttpFetcher * http_fetcher,const string & url)488 void StartTransfer(HttpFetcher* http_fetcher, const string& url) {
489   http_fetcher->BeginTransfer(url);
490 }
491 
TYPED_TEST(HttpFetcherTest,SimpleTest)492 TYPED_TEST(HttpFetcherTest, SimpleTest) {
493   HttpFetcherTestDelegate delegate;
494   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
495   fetcher->set_delegate(&delegate);
496 
497   unique_ptr<HttpServer> server(this->test_.CreateServer());
498   ASSERT_TRUE(server->started_);
499 
500   this->loop_.PostTask(FROM_HERE,
501                        base::Bind(StartTransfer,
502                                   fetcher.get(),
503                                   this->test_.SmallUrl(server->GetPort())));
504   this->loop_.Run();
505   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
506 }
507 
TYPED_TEST(HttpFetcherTest,SimpleBigTest)508 TYPED_TEST(HttpFetcherTest, SimpleBigTest) {
509   HttpFetcherTestDelegate delegate;
510   unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher());
511   fetcher->set_delegate(&delegate);
512 
513   unique_ptr<HttpServer> server(this->test_.CreateServer());
514   ASSERT_TRUE(server->started_);
515 
516   this->loop_.PostTask(
517       FROM_HERE,
518       base::Bind(
519           StartTransfer, fetcher.get(), this->test_.BigUrl(server->GetPort())));
520   this->loop_.Run();
521   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
522 }
523 
524 // Issue #9648: when server returns an error HTTP response, the fetcher needs to
525 // terminate transfer prematurely, rather than try to process the error payload.
TYPED_TEST(HttpFetcherTest,ErrorTest)526 TYPED_TEST(HttpFetcherTest, ErrorTest) {
527   if (this->test_.IsMock() || this->test_.IsMulti())
528     return;
529   HttpFetcherTestDelegate delegate;
530 
531   // Delegate should expect an error response.
532   delegate.is_expect_error_ = true;
533 
534   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
535   fetcher->set_delegate(&delegate);
536 
537   unique_ptr<HttpServer> server(this->test_.CreateServer());
538   ASSERT_TRUE(server->started_);
539 
540   this->loop_.PostTask(FROM_HERE,
541                        base::Bind(StartTransfer,
542                                   fetcher.get(),
543                                   this->test_.ErrorUrl(server->GetPort())));
544   this->loop_.Run();
545 
546   // Make sure that no bytes were received.
547   EXPECT_EQ(0, delegate.times_received_bytes_called_);
548   EXPECT_EQ(0U, fetcher->GetBytesDownloaded());
549 
550   // Make sure that transfer completion was signaled once, and no termination
551   // was signaled.
552   EXPECT_EQ(1, delegate.times_transfer_complete_called_);
553   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
554 }
555 
TYPED_TEST(HttpFetcherTest,ExtraHeadersInRequestTest)556 TYPED_TEST(HttpFetcherTest, ExtraHeadersInRequestTest) {
557   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
558     return;
559 
560   HttpFetcherTestDelegate delegate;
561   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
562   fetcher->set_delegate(&delegate);
563   fetcher->SetHeader("User-Agent", "MyTest");
564   fetcher->SetHeader("user-agent", "Override that header");
565   fetcher->SetHeader("Authorization", "Basic user:passwd");
566 
567   // Invalid headers.
568   fetcher->SetHeader("X-Foo", "Invalid\nHeader\nIgnored");
569   fetcher->SetHeader("X-Bar: ", "I do not know how to parse");
570 
571   // Hide Accept header normally added by default.
572   fetcher->SetHeader("Accept", "");
573 
574   PythonHttpServer server;
575   int port = server.GetPort();
576   ASSERT_TRUE(server.started_);
577 
578   this->loop_.PostTask(
579       FROM_HERE,
580       base::Bind(StartTransfer,
581                  fetcher.get(),
582                  LocalServerUrlForPath(port, "/echo-headers")));
583   this->loop_.Run();
584 
585   EXPECT_NE(string::npos,
586             delegate.data.find("user-agent: Override that header\r\n"));
587   EXPECT_NE(string::npos,
588             delegate.data.find("Authorization: Basic user:passwd\r\n"));
589 
590   EXPECT_EQ(string::npos, delegate.data.find("\nAccept:"));
591   EXPECT_EQ(string::npos, delegate.data.find("X-Foo: Invalid"));
592   EXPECT_EQ(string::npos, delegate.data.find("X-Bar: I do not"));
593 }
594 
595 class PausingHttpFetcherTestDelegate : public HttpFetcherDelegate {
596  public:
ReceivedBytes(HttpFetcher * fetcher,const void *,size_t)597   bool ReceivedBytes(HttpFetcher* fetcher,
598                      const void* /* bytes */,
599                      size_t /* length */) override {
600     CHECK(!paused_);
601     paused_ = true;
602     fetcher->Pause();
603     return true;
604   }
TransferComplete(HttpFetcher * fetcher,bool successful)605   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
606     MessageLoop::current()->BreakLoop();
607   }
TransferTerminated(HttpFetcher * fetcher)608   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
Unpause()609   void Unpause() {
610     CHECK(paused_);
611     paused_ = false;
612     fetcher_->Unpause();
613   }
614   bool paused_;
615   HttpFetcher* fetcher_;
616 };
617 
UnpausingTimeoutCallback(PausingHttpFetcherTestDelegate * delegate,MessageLoop::TaskId * my_id)618 void UnpausingTimeoutCallback(PausingHttpFetcherTestDelegate* delegate,
619                               MessageLoop::TaskId* my_id) {
620   if (delegate->paused_)
621     delegate->Unpause();
622   // Update the task id with the new scheduled callback.
623   *my_id = MessageLoop::current()->PostDelayedTask(
624       FROM_HERE,
625       base::Bind(&UnpausingTimeoutCallback, delegate, my_id),
626       base::TimeDelta::FromMilliseconds(200));
627 }
628 
TYPED_TEST(HttpFetcherTest,PauseTest)629 TYPED_TEST(HttpFetcherTest, PauseTest) {
630   PausingHttpFetcherTestDelegate delegate;
631   unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher());
632   delegate.paused_ = false;
633   delegate.fetcher_ = fetcher.get();
634   fetcher->set_delegate(&delegate);
635 
636   unique_ptr<HttpServer> server(this->test_.CreateServer());
637   ASSERT_TRUE(server->started_);
638 
639   MessageLoop::TaskId callback_id;
640   callback_id = this->loop_.PostDelayedTask(
641       FROM_HERE,
642       base::Bind(&UnpausingTimeoutCallback, &delegate, &callback_id),
643       base::TimeDelta::FromMilliseconds(200));
644   fetcher->BeginTransfer(this->test_.BigUrl(server->GetPort()));
645 
646   this->loop_.Run();
647   EXPECT_TRUE(this->loop_.CancelTask(callback_id));
648 }
649 
650 // This test will pause the fetcher while the download is not yet started
651 // because it is waiting for the proxy to be resolved.
TYPED_TEST(HttpFetcherTest,PauseWhileResolvingProxyTest)652 TYPED_TEST(HttpFetcherTest, PauseWhileResolvingProxyTest) {
653   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
654     return;
655   MockProxyResolver mock_resolver;
656   unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher(&mock_resolver));
657 
658   // Saved arguments from the proxy call.
659   ProxiesResolvedFn proxy_callback;
660   EXPECT_CALL(mock_resolver, GetProxiesForUrl("http://fake_url", _))
661       .WillOnce(DoAll(SaveArg<1>(&proxy_callback), Return(true)));
662   fetcher->BeginTransfer("http://fake_url");
663   testing::Mock::VerifyAndClearExpectations(&mock_resolver);
664 
665   // Pausing and unpausing while resolving the proxy should not affect anything.
666   fetcher->Pause();
667   fetcher->Unpause();
668   fetcher->Pause();
669   // Proxy resolver comes back after we paused the fetcher.
670   ASSERT_FALSE(proxy_callback.is_null());
671   proxy_callback.Run({1, kNoProxy});
672 }
673 
674 class AbortingHttpFetcherTestDelegate : public HttpFetcherDelegate {
675  public:
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)676   bool ReceivedBytes(HttpFetcher* fetcher,
677                      const void* bytes,
678                      size_t length) override {
679     return true;
680   }
TransferComplete(HttpFetcher * fetcher,bool successful)681   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
682     ADD_FAILURE();  // We should never get here
683     MessageLoop::current()->BreakLoop();
684   }
TransferTerminated(HttpFetcher * fetcher)685   void TransferTerminated(HttpFetcher* fetcher) override {
686     EXPECT_EQ(fetcher, fetcher_.get());
687     EXPECT_FALSE(once_);
688     EXPECT_TRUE(callback_once_);
689     callback_once_ = false;
690     // The fetcher could have a callback scheduled on the ProxyResolver that
691     // can fire after this callback. We wait until the end of the test to
692     // delete the fetcher.
693   }
TerminateTransfer()694   void TerminateTransfer() {
695     CHECK(once_);
696     once_ = false;
697     fetcher_->TerminateTransfer();
698   }
EndLoop()699   void EndLoop() { MessageLoop::current()->BreakLoop(); }
700   bool once_;
701   bool callback_once_;
702   unique_ptr<HttpFetcher> fetcher_;
703 };
704 
AbortingTimeoutCallback(AbortingHttpFetcherTestDelegate * delegate,MessageLoop::TaskId * my_id)705 void AbortingTimeoutCallback(AbortingHttpFetcherTestDelegate* delegate,
706                              MessageLoop::TaskId* my_id) {
707   if (delegate->once_) {
708     delegate->TerminateTransfer();
709     *my_id = MessageLoop::current()->PostTask(
710         FROM_HERE, base::Bind(AbortingTimeoutCallback, delegate, my_id));
711   } else {
712     delegate->EndLoop();
713     *my_id = MessageLoop::kTaskIdNull;
714   }
715 }
716 
TYPED_TEST(HttpFetcherTest,AbortTest)717 TYPED_TEST(HttpFetcherTest, AbortTest) {
718   AbortingHttpFetcherTestDelegate delegate;
719   delegate.fetcher_.reset(this->test_.NewLargeFetcher());
720   delegate.once_ = true;
721   delegate.callback_once_ = true;
722   delegate.fetcher_->set_delegate(&delegate);
723 
724   unique_ptr<HttpServer> server(this->test_.CreateServer());
725   this->test_.IgnoreServerAborting(server.get());
726   ASSERT_TRUE(server->started_);
727 
728   MessageLoop::TaskId task_id = MessageLoop::kTaskIdNull;
729 
730   task_id = this->loop_.PostTask(
731       FROM_HERE, base::Bind(AbortingTimeoutCallback, &delegate, &task_id));
732   delegate.fetcher_->BeginTransfer(this->test_.BigUrl(server->GetPort()));
733 
734   this->loop_.Run();
735   CHECK(!delegate.once_);
736   CHECK(!delegate.callback_once_);
737   this->loop_.CancelTask(task_id);
738 }
739 
TYPED_TEST(HttpFetcherTest,TerminateTransferWhileResolvingProxyTest)740 TYPED_TEST(HttpFetcherTest, TerminateTransferWhileResolvingProxyTest) {
741   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
742     return;
743   MockProxyResolver mock_resolver;
744   unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher(&mock_resolver));
745 
746   HttpFetcherTestDelegate delegate;
747   fetcher->set_delegate(&delegate);
748 
749   EXPECT_CALL(mock_resolver, GetProxiesForUrl(_, _)).WillOnce(Return(123));
750   fetcher->BeginTransfer("http://fake_url");
751   // Run the message loop until idle. This must call the MockProxyResolver with
752   // the request.
753   while (this->loop_.RunOnce(false)) {
754   }
755   testing::Mock::VerifyAndClearExpectations(&mock_resolver);
756 
757   EXPECT_CALL(mock_resolver, CancelProxyRequest(123)).WillOnce(Return(true));
758 
759   // Terminate the transfer right before the proxy resolution response.
760   fetcher->TerminateTransfer();
761   EXPECT_EQ(0, delegate.times_received_bytes_called_);
762   EXPECT_EQ(0, delegate.times_transfer_complete_called_);
763   EXPECT_EQ(1, delegate.times_transfer_terminated_called_);
764 }
765 
766 class FlakyHttpFetcherTestDelegate : public HttpFetcherDelegate {
767  public:
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)768   bool ReceivedBytes(HttpFetcher* fetcher,
769                      const void* bytes,
770                      size_t length) override {
771     data.append(reinterpret_cast<const char*>(bytes), length);
772     return true;
773   }
TransferComplete(HttpFetcher * fetcher,bool successful)774   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
775     EXPECT_TRUE(successful);
776     EXPECT_EQ(kHttpResponsePartialContent, fetcher->http_response_code());
777     MessageLoop::current()->BreakLoop();
778   }
TransferTerminated(HttpFetcher * fetcher)779   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
780   string data;
781 };
782 
TYPED_TEST(HttpFetcherTest,FlakyTest)783 TYPED_TEST(HttpFetcherTest, FlakyTest) {
784   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
785     return;
786   {
787     FlakyHttpFetcherTestDelegate delegate;
788     unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
789     fetcher->set_delegate(&delegate);
790 
791     unique_ptr<HttpServer> server(this->test_.CreateServer());
792     ASSERT_TRUE(server->started_);
793 
794     this->loop_.PostTask(FROM_HERE,
795                          base::Bind(&StartTransfer,
796                                     fetcher.get(),
797                                     LocalServerUrlForPath(
798                                         server->GetPort(),
799                                         base::StringPrintf("/flaky/%d/%d/%d/%d",
800                                                            kBigLength,
801                                                            kFlakyTruncateLength,
802                                                            kFlakySleepEvery,
803                                                            kFlakySleepSecs))));
804     this->loop_.Run();
805 
806     // verify the data we get back
807     ASSERT_EQ(kBigLength, static_cast<int>(delegate.data.size()));
808     for (int i = 0; i < kBigLength; i += 10) {
809       // Assert so that we don't flood the screen w/ EXPECT errors on failure.
810       ASSERT_EQ(delegate.data.substr(i, 10), "abcdefghij");
811     }
812   }
813 }
814 
815 // This delegate kills the server attached to it after receiving any bytes.
816 // This can be used for testing what happens when you try to fetch data and
817 // the server dies.
818 class FailureHttpFetcherTestDelegate : public HttpFetcherDelegate {
819  public:
FailureHttpFetcherTestDelegate(PythonHttpServer * server)820   explicit FailureHttpFetcherTestDelegate(PythonHttpServer* server)
821       : server_(server) {}
822 
~FailureHttpFetcherTestDelegate()823   ~FailureHttpFetcherTestDelegate() override {
824     if (server_) {
825       LOG(INFO) << "Stopping server in destructor";
826       server_.reset();
827       LOG(INFO) << "server stopped";
828     }
829   }
830 
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)831   bool ReceivedBytes(HttpFetcher* fetcher,
832                      const void* bytes,
833                      size_t length) override {
834     if (server_) {
835       LOG(INFO) << "Stopping server in ReceivedBytes";
836       server_.reset();
837       LOG(INFO) << "server stopped";
838     }
839     return true;
840   }
TransferComplete(HttpFetcher * fetcher,bool successful)841   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
842     EXPECT_FALSE(successful);
843     EXPECT_EQ(0, fetcher->http_response_code());
844     times_transfer_complete_called_++;
845     MessageLoop::current()->BreakLoop();
846   }
TransferTerminated(HttpFetcher * fetcher)847   void TransferTerminated(HttpFetcher* fetcher) override {
848     times_transfer_terminated_called_++;
849     MessageLoop::current()->BreakLoop();
850   }
851   unique_ptr<PythonHttpServer> server_;
852   int times_transfer_terminated_called_{0};
853   int times_transfer_complete_called_{0};
854 };
855 
TYPED_TEST(HttpFetcherTest,FailureTest)856 TYPED_TEST(HttpFetcherTest, FailureTest) {
857   // This test ensures that a fetcher responds correctly when a server isn't
858   // available at all.
859   if (this->test_.IsMock())
860     return;
861   FailureHttpFetcherTestDelegate delegate(nullptr);
862   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
863   fetcher->set_delegate(&delegate);
864 
865   this->loop_.PostTask(
866       FROM_HERE,
867       base::Bind(
868           StartTransfer, fetcher.get(), "http://host_doesnt_exist99999999"));
869   this->loop_.Run();
870   EXPECT_EQ(1, delegate.times_transfer_complete_called_);
871   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
872 
873   // Exiting and testing happens in the delegate
874 }
875 
TYPED_TEST(HttpFetcherTest,NoResponseTest)876 TYPED_TEST(HttpFetcherTest, NoResponseTest) {
877   // This test starts a new http server but the server doesn't respond and just
878   // closes the connection.
879   if (this->test_.IsMock())
880     return;
881 
882   PythonHttpServer* server = new PythonHttpServer();
883   int port = server->GetPort();
884   ASSERT_TRUE(server->started_);
885 
886   // Handles destruction and claims ownership.
887   FailureHttpFetcherTestDelegate delegate(server);
888   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
889   fetcher->set_delegate(&delegate);
890   // The server will not reply at all, so we can limit the execution time of the
891   // test by reducing the low-speed timeout to something small. The test will
892   // finish once the TimeoutCallback() triggers (every second) and the timeout
893   // expired.
894   fetcher->set_low_speed_limit(kDownloadLowSpeedLimitBps, 1);
895 
896   this->loop_.PostTask(
897       FROM_HERE,
898       base::Bind(
899           StartTransfer, fetcher.get(), LocalServerUrlForPath(port, "/hang")));
900   this->loop_.Run();
901   EXPECT_EQ(1, delegate.times_transfer_complete_called_);
902   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
903 
904   // Check that no other callback runs in the next two seconds. That would
905   // indicate a leaked callback.
906   bool timeout = false;
907   auto callback = base::Bind([](bool* timeout) { *timeout = true; },
908                              base::Unretained(&timeout));
909   this->loop_.PostDelayedTask(
910       FROM_HERE, callback, base::TimeDelta::FromSeconds(2));
911   EXPECT_TRUE(this->loop_.RunOnce(true));
912   EXPECT_TRUE(timeout);
913 }
914 
TYPED_TEST(HttpFetcherTest,ServerDiesTest)915 TYPED_TEST(HttpFetcherTest, ServerDiesTest) {
916   // This test starts a new http server and kills it after receiving its first
917   // set of bytes. It test whether or not our fetcher eventually gives up on
918   // retries and aborts correctly.
919   if (this->test_.IsMock())
920     return;
921   PythonHttpServer* server = new PythonHttpServer();
922   int port = server->GetPort();
923   ASSERT_TRUE(server->started_);
924 
925   // Handles destruction and claims ownership.
926   FailureHttpFetcherTestDelegate delegate(server);
927   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
928   fetcher->set_delegate(&delegate);
929 
930   this->loop_.PostTask(
931       FROM_HERE,
932       base::Bind(StartTransfer,
933                  fetcher.get(),
934                  LocalServerUrlForPath(port,
935                                        base::StringPrintf("/flaky/%d/%d/%d/%d",
936                                                           kBigLength,
937                                                           kFlakyTruncateLength,
938                                                           kFlakySleepEvery,
939                                                           kFlakySleepSecs))));
940   this->loop_.Run();
941   EXPECT_EQ(1, delegate.times_transfer_complete_called_);
942   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
943 
944   // Exiting and testing happens in the delegate
945 }
946 
947 // Test that we can cancel a transfer while it is still trying to connect to the
948 // server. This test kills the server after a few bytes are received.
TYPED_TEST(HttpFetcherTest,TerminateTransferWhenServerDiedTest)949 TYPED_TEST(HttpFetcherTest, TerminateTransferWhenServerDiedTest) {
950   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
951     return;
952 
953   PythonHttpServer* server = new PythonHttpServer();
954   int port = server->GetPort();
955   ASSERT_TRUE(server->started_);
956 
957   // Handles destruction and claims ownership.
958   FailureHttpFetcherTestDelegate delegate(server);
959   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
960   fetcher->set_delegate(&delegate);
961 
962   this->loop_.PostTask(
963       FROM_HERE,
964       base::Bind(StartTransfer,
965                  fetcher.get(),
966                  LocalServerUrlForPath(port,
967                                        base::StringPrintf("/flaky/%d/%d/%d/%d",
968                                                           kBigLength,
969                                                           kFlakyTruncateLength,
970                                                           kFlakySleepEvery,
971                                                           kFlakySleepSecs))));
972   // Terminating the transfer after 3 seconds gives it a chance to contact the
973   // server and enter the retry loop.
974   this->loop_.PostDelayedTask(FROM_HERE,
975                               base::Bind(&HttpFetcher::TerminateTransfer,
976                                          base::Unretained(fetcher.get())),
977                               base::TimeDelta::FromSeconds(3));
978 
979   // Exiting and testing happens in the delegate.
980   this->loop_.Run();
981   EXPECT_EQ(0, delegate.times_transfer_complete_called_);
982   EXPECT_EQ(1, delegate.times_transfer_terminated_called_);
983 
984   // Check that no other callback runs in the next two seconds. That would
985   // indicate a leaked callback.
986   bool timeout = false;
987   auto callback = base::Bind([](bool* timeout) { *timeout = true; },
988                              base::Unretained(&timeout));
989   this->loop_.PostDelayedTask(
990       FROM_HERE, callback, base::TimeDelta::FromSeconds(2));
991   EXPECT_TRUE(this->loop_.RunOnce(true));
992   EXPECT_TRUE(timeout);
993 }
994 
995 const HttpResponseCode kRedirectCodes[] = {kHttpResponseMovedPermanently,
996                                            kHttpResponseFound,
997                                            kHttpResponseSeeOther,
998                                            kHttpResponseTempRedirect};
999 
1000 class RedirectHttpFetcherTestDelegate : public HttpFetcherDelegate {
1001  public:
RedirectHttpFetcherTestDelegate(bool expected_successful)1002   explicit RedirectHttpFetcherTestDelegate(bool expected_successful)
1003       : expected_successful_(expected_successful) {}
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)1004   bool ReceivedBytes(HttpFetcher* fetcher,
1005                      const void* bytes,
1006                      size_t length) override {
1007     data.append(reinterpret_cast<const char*>(bytes), length);
1008     return true;
1009   }
TransferComplete(HttpFetcher * fetcher,bool successful)1010   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1011     EXPECT_EQ(expected_successful_, successful);
1012     if (expected_successful_) {
1013       EXPECT_EQ(kHttpResponseOk, fetcher->http_response_code());
1014     } else {
1015       EXPECT_GE(fetcher->http_response_code(), kHttpResponseMovedPermanently);
1016       EXPECT_LE(fetcher->http_response_code(), kHttpResponseTempRedirect);
1017     }
1018     MessageLoop::current()->BreakLoop();
1019   }
TransferTerminated(HttpFetcher * fetcher)1020   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
1021   bool expected_successful_;
1022   string data;
1023 };
1024 
1025 // RedirectTest takes ownership of |http_fetcher|.
RedirectTest(const HttpServer * server,bool expected_successful,const string & url,HttpFetcher * http_fetcher)1026 void RedirectTest(const HttpServer* server,
1027                   bool expected_successful,
1028                   const string& url,
1029                   HttpFetcher* http_fetcher) {
1030   RedirectHttpFetcherTestDelegate delegate(expected_successful);
1031   unique_ptr<HttpFetcher> fetcher(http_fetcher);
1032   fetcher->set_delegate(&delegate);
1033 
1034   MessageLoop::current()->PostTask(
1035       FROM_HERE,
1036       base::Bind(StartTransfer,
1037                  fetcher.get(),
1038                  LocalServerUrlForPath(server->GetPort(), url)));
1039   MessageLoop::current()->Run();
1040   if (expected_successful) {
1041     // verify the data we get back
1042     ASSERT_EQ(static_cast<size_t>(kMediumLength), delegate.data.size());
1043     for (int i = 0; i < kMediumLength; i += 10) {
1044       // Assert so that we don't flood the screen w/ EXPECT errors on failure.
1045       ASSERT_EQ(delegate.data.substr(i, 10), "abcdefghij");
1046     }
1047   }
1048 }
1049 
TYPED_TEST(HttpFetcherTest,SimpleRedirectTest)1050 TYPED_TEST(HttpFetcherTest, SimpleRedirectTest) {
1051   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
1052     return;
1053 
1054   unique_ptr<HttpServer> server(this->test_.CreateServer());
1055   ASSERT_TRUE(server->started_);
1056 
1057   for (size_t c = 0; c < base::size(kRedirectCodes); ++c) {
1058     const string url = base::StringPrintf(
1059         "/redirect/%d/download/%d", kRedirectCodes[c], kMediumLength);
1060     RedirectTest(server.get(), true, url, this->test_.NewLargeFetcher());
1061   }
1062 }
1063 
TYPED_TEST(HttpFetcherTest,MaxRedirectTest)1064 TYPED_TEST(HttpFetcherTest, MaxRedirectTest) {
1065   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
1066     return;
1067 
1068   unique_ptr<HttpServer> server(this->test_.CreateServer());
1069   ASSERT_TRUE(server->started_);
1070 
1071   string url;
1072   for (int r = 0; r < kDownloadMaxRedirects; r++) {
1073     url += base::StringPrintf("/redirect/%d",
1074                               kRedirectCodes[r % base::size(kRedirectCodes)]);
1075   }
1076   url += base::StringPrintf("/download/%d", kMediumLength);
1077   RedirectTest(server.get(), true, url, this->test_.NewLargeFetcher());
1078 }
1079 
TYPED_TEST(HttpFetcherTest,BeyondMaxRedirectTest)1080 TYPED_TEST(HttpFetcherTest, BeyondMaxRedirectTest) {
1081   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
1082     return;
1083 
1084   unique_ptr<HttpServer> server(this->test_.CreateServer());
1085   ASSERT_TRUE(server->started_);
1086 
1087   string url;
1088   for (int r = 0; r < kDownloadMaxRedirects + 1; r++) {
1089     url += base::StringPrintf("/redirect/%d",
1090                               kRedirectCodes[r % base::size(kRedirectCodes)]);
1091   }
1092   url += base::StringPrintf("/download/%d", kMediumLength);
1093   RedirectTest(server.get(), false, url, this->test_.NewLargeFetcher());
1094 }
1095 
1096 class MultiHttpFetcherTestDelegate : public HttpFetcherDelegate {
1097  public:
MultiHttpFetcherTestDelegate(int expected_response_code)1098   explicit MultiHttpFetcherTestDelegate(int expected_response_code)
1099       : expected_response_code_(expected_response_code) {}
1100 
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)1101   bool ReceivedBytes(HttpFetcher* fetcher,
1102                      const void* bytes,
1103                      size_t length) override {
1104     EXPECT_EQ(fetcher, fetcher_.get());
1105     data.append(reinterpret_cast<const char*>(bytes), length);
1106     return true;
1107   }
1108 
TransferComplete(HttpFetcher * fetcher,bool successful)1109   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1110     EXPECT_EQ(fetcher, fetcher_.get());
1111     EXPECT_EQ(expected_response_code_ != kHttpResponseUndefined, successful);
1112     if (expected_response_code_ != 0)
1113       EXPECT_EQ(expected_response_code_, fetcher->http_response_code());
1114     // Destroy the fetcher (because we're allowed to).
1115     fetcher_.reset(nullptr);
1116     MessageLoop::current()->BreakLoop();
1117   }
1118 
TransferTerminated(HttpFetcher * fetcher)1119   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
1120 
1121   unique_ptr<HttpFetcher> fetcher_;
1122   int expected_response_code_;
1123   string data;
1124 };
1125 
MultiTest(HttpFetcher * fetcher_in,FakeHardware * fake_hardware,const string & url,const vector<pair<off_t,off_t>> & ranges,const string & expected_prefix,size_t expected_size,HttpResponseCode expected_response_code)1126 void MultiTest(HttpFetcher* fetcher_in,
1127                FakeHardware* fake_hardware,
1128                const string& url,
1129                const vector<pair<off_t, off_t>>& ranges,
1130                const string& expected_prefix,
1131                size_t expected_size,
1132                HttpResponseCode expected_response_code) {
1133   MultiHttpFetcherTestDelegate delegate(expected_response_code);
1134   delegate.fetcher_.reset(fetcher_in);
1135 
1136   MultiRangeHttpFetcher* multi_fetcher =
1137       static_cast<MultiRangeHttpFetcher*>(fetcher_in);
1138   ASSERT_TRUE(multi_fetcher);
1139   multi_fetcher->ClearRanges();
1140   for (vector<pair<off_t, off_t>>::const_iterator it = ranges.begin(),
1141                                                   e = ranges.end();
1142        it != e;
1143        ++it) {
1144     string tmp_str = base::StringPrintf("%jd+", it->first);
1145     if (it->second > 0) {
1146       base::StringAppendF(&tmp_str, "%jd", it->second);
1147       multi_fetcher->AddRange(it->first, it->second);
1148     } else {
1149       base::StringAppendF(&tmp_str, "?");
1150       multi_fetcher->AddRange(it->first);
1151     }
1152     LOG(INFO) << "added range: " << tmp_str;
1153   }
1154   fake_hardware->SetIsOfficialBuild(false);
1155   multi_fetcher->set_delegate(&delegate);
1156 
1157   MessageLoop::current()->PostTask(
1158       FROM_HERE, base::Bind(StartTransfer, multi_fetcher, url));
1159   MessageLoop::current()->Run();
1160 
1161   EXPECT_EQ(expected_size, delegate.data.size());
1162   EXPECT_EQ(expected_prefix,
1163             string(delegate.data.data(), expected_prefix.size()));
1164 }
1165 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherSimpleTest)1166 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherSimpleTest) {
1167   if (!this->test_.IsMulti())
1168     return;
1169 
1170   unique_ptr<HttpServer> server(this->test_.CreateServer());
1171   ASSERT_TRUE(server->started_);
1172 
1173   vector<pair<off_t, off_t>> ranges;
1174   ranges.push_back(make_pair(0, 25));
1175   ranges.push_back(make_pair(99, 17));
1176   MultiTest(this->test_.NewLargeFetcher(),
1177             this->test_.fake_hardware(),
1178             this->test_.BigUrl(server->GetPort()),
1179             ranges,
1180             "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1181             25 + 17,
1182             this->test_.IsFileFetcher() ? kHttpResponseOk
1183                                         : kHttpResponsePartialContent);
1184 }
1185 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherUnspecifiedEndTest)1186 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherUnspecifiedEndTest) {
1187   if (!this->test_.IsMulti() || this->test_.IsFileFetcher())
1188     return;
1189 
1190   unique_ptr<HttpServer> server(this->test_.CreateServer());
1191   ASSERT_TRUE(server->started_);
1192 
1193   vector<pair<off_t, off_t>> ranges;
1194   ranges.push_back(make_pair(0, 25));
1195   ranges.push_back(make_pair(99, 0));
1196   MultiTest(this->test_.NewLargeFetcher(),
1197             this->test_.fake_hardware(),
1198             this->test_.BigUrl(server->GetPort()),
1199             ranges,
1200             "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1201             kBigLength - (99 - 25),
1202             kHttpResponsePartialContent);
1203 }
1204 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherLengthLimitTest)1205 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherLengthLimitTest) {
1206   if (!this->test_.IsMulti())
1207     return;
1208 
1209   unique_ptr<HttpServer> server(this->test_.CreateServer());
1210   ASSERT_TRUE(server->started_);
1211 
1212   vector<pair<off_t, off_t>> ranges;
1213   ranges.push_back(make_pair(0, 24));
1214   MultiTest(this->test_.NewLargeFetcher(),
1215             this->test_.fake_hardware(),
1216             this->test_.BigUrl(server->GetPort()),
1217             ranges,
1218             "abcdefghijabcdefghijabcd",
1219             24,
1220             this->test_.IsFileFetcher() ? kHttpResponseOk
1221                                         : kHttpResponsePartialContent);
1222 }
1223 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherMultiEndTest)1224 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherMultiEndTest) {
1225   if (!this->test_.IsMulti() || this->test_.IsFileFetcher())
1226     return;
1227 
1228   unique_ptr<HttpServer> server(this->test_.CreateServer());
1229   ASSERT_TRUE(server->started_);
1230 
1231   vector<pair<off_t, off_t>> ranges;
1232   ranges.push_back(make_pair(kBigLength - 2, 0));
1233   ranges.push_back(make_pair(kBigLength - 3, 0));
1234   MultiTest(this->test_.NewLargeFetcher(),
1235             this->test_.fake_hardware(),
1236             this->test_.BigUrl(server->GetPort()),
1237             ranges,
1238             "ijhij",
1239             5,
1240             kHttpResponsePartialContent);
1241 }
1242 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherInsufficientTest)1243 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherInsufficientTest) {
1244   if (!this->test_.IsMulti())
1245     return;
1246 
1247   unique_ptr<HttpServer> server(this->test_.CreateServer());
1248   ASSERT_TRUE(server->started_);
1249 
1250   vector<pair<off_t, off_t>> ranges;
1251   ranges.push_back(make_pair(kBigLength - 2, 4));
1252   for (int i = 0; i < 2; ++i) {
1253     LOG(INFO) << "i = " << i;
1254     MultiTest(this->test_.NewLargeFetcher(),
1255               this->test_.fake_hardware(),
1256               this->test_.BigUrl(server->GetPort()),
1257               ranges,
1258               "ij",
1259               2,
1260               kHttpResponseUndefined);
1261     ranges.push_back(make_pair(0, 5));
1262   }
1263 }
1264 
1265 // Issue #18143: when a fetch of a secondary chunk out of a chain, then it
1266 // should retry with other proxies listed before giving up.
1267 //
1268 // (1) successful recovery: The offset fetch will fail twice but succeed with
1269 // the third proxy.
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherErrorIfOffsetRecoverableTest)1270 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherErrorIfOffsetRecoverableTest) {
1271   if (!this->test_.IsMulti() || this->test_.IsFileFetcher())
1272     return;
1273 
1274   unique_ptr<HttpServer> server(this->test_.CreateServer());
1275   ASSERT_TRUE(server->started_);
1276 
1277   vector<pair<off_t, off_t>> ranges;
1278   ranges.push_back(make_pair(0, 25));
1279   ranges.push_back(make_pair(99, 0));
1280   MultiTest(this->test_.NewLargeFetcher(3),
1281             this->test_.fake_hardware(),
1282             LocalServerUrlForPath(
1283                 server->GetPort(),
1284                 base::StringPrintf("/error-if-offset/%d/2", kBigLength)),
1285             ranges,
1286             "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1287             kBigLength - (99 - 25),
1288             kHttpResponsePartialContent);
1289 }
1290 
1291 // (2) unsuccessful recovery: The offset fetch will fail repeatedly.  The
1292 // fetcher will signal a (failed) completed transfer to the delegate.
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherErrorIfOffsetUnrecoverableTest)1293 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherErrorIfOffsetUnrecoverableTest) {
1294   if (!this->test_.IsMulti() || this->test_.IsFileFetcher())
1295     return;
1296 
1297   unique_ptr<HttpServer> server(this->test_.CreateServer());
1298   ASSERT_TRUE(server->started_);
1299 
1300   vector<pair<off_t, off_t>> ranges;
1301   ranges.push_back(make_pair(0, 25));
1302   ranges.push_back(make_pair(99, 0));
1303   MultiTest(this->test_.NewLargeFetcher(2),
1304             this->test_.fake_hardware(),
1305             LocalServerUrlForPath(
1306                 server->GetPort(),
1307                 base::StringPrintf("/error-if-offset/%d/3", kBigLength)),
1308             ranges,
1309             "abcdefghijabcdefghijabcde",  // only received the first chunk
1310             25,
1311             kHttpResponseUndefined);
1312 }
1313 
1314 // This HttpFetcherDelegate calls TerminateTransfer at a configurable point.
1315 class MultiHttpFetcherTerminateTestDelegate : public HttpFetcherDelegate {
1316  public:
MultiHttpFetcherTerminateTestDelegate(size_t terminate_trigger_bytes)1317   explicit MultiHttpFetcherTerminateTestDelegate(size_t terminate_trigger_bytes)
1318       : terminate_trigger_bytes_(terminate_trigger_bytes) {}
1319 
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)1320   bool ReceivedBytes(HttpFetcher* fetcher,
1321                      const void* bytes,
1322                      size_t length) override {
1323     LOG(INFO) << "ReceivedBytes, " << length << " bytes.";
1324     EXPECT_EQ(fetcher, fetcher_.get());
1325     bool should_terminate = false;
1326     if (bytes_downloaded_ < terminate_trigger_bytes_ &&
1327         bytes_downloaded_ + length >= terminate_trigger_bytes_) {
1328       MessageLoop::current()->PostTask(
1329           FROM_HERE,
1330           base::Bind(&HttpFetcher::TerminateTransfer,
1331                      base::Unretained(fetcher_.get())));
1332       should_terminate = true;
1333     }
1334     bytes_downloaded_ += length;
1335     return !should_terminate;
1336   }
1337 
TransferComplete(HttpFetcher * fetcher,bool successful)1338   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1339     ADD_FAILURE() << "TransferComplete called but expected a failure";
1340     // Destroy the fetcher (because we're allowed to).
1341     fetcher_.reset(nullptr);
1342     MessageLoop::current()->BreakLoop();
1343   }
1344 
TransferTerminated(HttpFetcher * fetcher)1345   void TransferTerminated(HttpFetcher* fetcher) override {
1346     // Destroy the fetcher (because we're allowed to).
1347     fetcher_.reset(nullptr);
1348     MessageLoop::current()->BreakLoop();
1349   }
1350 
1351   unique_ptr<HttpFetcher> fetcher_;
1352   size_t bytes_downloaded_{0};
1353   size_t terminate_trigger_bytes_;
1354 };
1355 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherTerminateBetweenRangesTest)1356 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherTerminateBetweenRangesTest) {
1357   if (!this->test_.IsMulti())
1358     return;
1359   const size_t kRangeTrigger = 1000;
1360   MultiHttpFetcherTerminateTestDelegate delegate(kRangeTrigger);
1361 
1362   unique_ptr<HttpServer> server(this->test_.CreateServer());
1363   ASSERT_TRUE(server->started_);
1364 
1365   MultiRangeHttpFetcher* multi_fetcher =
1366       static_cast<MultiRangeHttpFetcher*>(this->test_.NewLargeFetcher());
1367   ASSERT_TRUE(multi_fetcher);
1368   // Transfer ownership of the fetcher to the delegate.
1369   delegate.fetcher_.reset(multi_fetcher);
1370   multi_fetcher->set_delegate(&delegate);
1371 
1372   multi_fetcher->ClearRanges();
1373   multi_fetcher->AddRange(45, kRangeTrigger);
1374   multi_fetcher->AddRange(2000, 100);
1375 
1376   this->test_.fake_hardware()->SetIsOfficialBuild(false);
1377 
1378   StartTransfer(multi_fetcher, this->test_.BigUrl(server->GetPort()));
1379   MessageLoop::current()->Run();
1380 
1381   // Check that the delegate made it to the trigger point.
1382   EXPECT_EQ(kRangeTrigger, delegate.bytes_downloaded_);
1383 }
1384 
1385 class BlockedTransferTestDelegate : public HttpFetcherDelegate {
1386  public:
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)1387   bool ReceivedBytes(HttpFetcher* fetcher,
1388                      const void* bytes,
1389                      size_t length) override {
1390     ADD_FAILURE();
1391     return true;
1392   }
TransferComplete(HttpFetcher * fetcher,bool successful)1393   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1394     EXPECT_FALSE(successful);
1395     MessageLoop::current()->BreakLoop();
1396   }
TransferTerminated(HttpFetcher * fetcher)1397   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
1398 };
1399 
BlockedTransferTestHelper(AnyHttpFetcherFactory * fetcher_test,bool is_official_build)1400 void BlockedTransferTestHelper(AnyHttpFetcherFactory* fetcher_test,
1401                                bool is_official_build) {
1402   if (fetcher_test->IsMock() || fetcher_test->IsMulti())
1403     return;
1404 
1405   unique_ptr<HttpServer> server(fetcher_test->CreateServer());
1406   ASSERT_TRUE(server->started_);
1407 
1408   BlockedTransferTestDelegate delegate;
1409   unique_ptr<HttpFetcher> fetcher(fetcher_test->NewLargeFetcher());
1410   LOG(INFO) << "is_official_build: " << is_official_build;
1411   // NewLargeFetcher creates the HttpFetcher* with a FakeSystemState.
1412   fetcher_test->fake_hardware()->SetIsOfficialBuild(is_official_build);
1413   fetcher->set_delegate(&delegate);
1414 
1415   MessageLoop::current()->PostTask(
1416       FROM_HERE,
1417       base::Bind(
1418           StartTransfer,
1419           fetcher.get(),
1420           LocalServerUrlForPath(server->GetPort(),
1421                                 fetcher_test->SmallUrl(server->GetPort()))));
1422   MessageLoop::current()->Run();
1423 }
1424 
TYPED_TEST(HttpFetcherTest,BlockedTransferTest)1425 TYPED_TEST(HttpFetcherTest, BlockedTransferTest) {
1426   BlockedTransferTestHelper(&this->test_, false);
1427 }
1428 
TYPED_TEST(HttpFetcherTest,BlockedTransferOfficialBuildTest)1429 TYPED_TEST(HttpFetcherTest, BlockedTransferOfficialBuildTest) {
1430   BlockedTransferTestHelper(&this->test_, true);
1431 }
1432 
1433 }  // namespace
1434 
1435 }  // namespace chromeos_update_engine
1436