1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "net/url_request/url_fetcher_core.h"
6
7 #include "base/bind.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/sequenced_task_runner.h"
11 #include "base/single_thread_task_runner.h"
12 #include "base/stl_util.h"
13 #include "base/thread_task_runner_handle.h"
14 #include "base/tracked_objects.h"
15 #include "net/base/io_buffer.h"
16 #include "net/base/load_flags.h"
17 #include "net/base/net_errors.h"
18 #include "net/base/request_priority.h"
19 #include "net/base/upload_bytes_element_reader.h"
20 #include "net/base/upload_data_stream.h"
21 #include "net/base/upload_file_element_reader.h"
22 #include "net/http/http_response_headers.h"
23 #include "net/url_request/url_fetcher_delegate.h"
24 #include "net/url_request/url_fetcher_response_writer.h"
25 #include "net/url_request/url_request_context.h"
26 #include "net/url_request/url_request_context_getter.h"
27 #include "net/url_request/url_request_throttler_manager.h"
28
29 namespace {
30
31 const int kBufferSize = 4096;
32 const int kUploadProgressTimerInterval = 100;
33 bool g_interception_enabled = false;
34 bool g_ignore_certificate_requests = false;
35
EmptyCompletionCallback(int result)36 void EmptyCompletionCallback(int result) {}
37
38 } // namespace
39
40 namespace net {
41
42 // URLFetcherCore::Registry ---------------------------------------------------
43
Registry()44 URLFetcherCore::Registry::Registry() {}
~Registry()45 URLFetcherCore::Registry::~Registry() {}
46
AddURLFetcherCore(URLFetcherCore * core)47 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
48 DCHECK(!ContainsKey(fetchers_, core));
49 fetchers_.insert(core);
50 }
51
RemoveURLFetcherCore(URLFetcherCore * core)52 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
53 DCHECK(ContainsKey(fetchers_, core));
54 fetchers_.erase(core);
55 }
56
CancelAll()57 void URLFetcherCore::Registry::CancelAll() {
58 while (!fetchers_.empty())
59 (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
60 }
61
62 // URLFetcherCore -------------------------------------------------------------
63
64 // static
65 base::LazyInstance<URLFetcherCore::Registry>
66 URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
67
URLFetcherCore(URLFetcher * fetcher,const GURL & original_url,URLFetcher::RequestType request_type,URLFetcherDelegate * d)68 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
69 const GURL& original_url,
70 URLFetcher::RequestType request_type,
71 URLFetcherDelegate* d)
72 : fetcher_(fetcher),
73 original_url_(original_url),
74 request_type_(request_type),
75 delegate_(d),
76 delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
77 load_flags_(LOAD_NORMAL),
78 response_code_(URLFetcher::RESPONSE_CODE_INVALID),
79 buffer_(new IOBuffer(kBufferSize)),
80 url_request_data_key_(NULL),
81 was_fetched_via_proxy_(false),
82 upload_content_set_(false),
83 upload_range_offset_(0),
84 upload_range_length_(0),
85 referrer_policy_(
86 URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
87 is_chunked_upload_(false),
88 was_cancelled_(false),
89 stop_on_redirect_(false),
90 stopped_on_redirect_(false),
91 automatically_retry_on_5xx_(true),
92 num_retries_on_5xx_(0),
93 max_retries_on_5xx_(0),
94 num_retries_on_network_changes_(0),
95 max_retries_on_network_changes_(0),
96 current_upload_bytes_(-1),
97 current_response_bytes_(0),
98 total_response_bytes_(-1) {
99 CHECK(original_url_.is_valid());
100 }
101
Start()102 void URLFetcherCore::Start() {
103 DCHECK(delegate_task_runner_.get());
104 DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
105 if (network_task_runner_.get()) {
106 DCHECK_EQ(network_task_runner_,
107 request_context_getter_->GetNetworkTaskRunner());
108 } else {
109 network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
110 }
111 DCHECK(network_task_runner_.get()) << "We need an IO task runner";
112
113 network_task_runner_->PostTask(
114 FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
115 }
116
Stop()117 void URLFetcherCore::Stop() {
118 if (delegate_task_runner_.get()) // May be NULL in tests.
119 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
120
121 delegate_ = NULL;
122 fetcher_ = NULL;
123 if (!network_task_runner_.get())
124 return;
125 if (network_task_runner_->RunsTasksOnCurrentThread()) {
126 CancelURLRequest(ERR_ABORTED);
127 } else {
128 network_task_runner_->PostTask(
129 FROM_HERE,
130 base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
131 }
132 }
133
SetUploadData(const std::string & upload_content_type,const std::string & upload_content)134 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
135 const std::string& upload_content) {
136 DCHECK(!is_chunked_upload_);
137 DCHECK(!upload_content_set_);
138 DCHECK(upload_content_.empty());
139 DCHECK(upload_file_path_.empty());
140 DCHECK(upload_content_type_.empty());
141
142 // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
143 DCHECK(upload_content.empty() || !upload_content_type.empty());
144
145 upload_content_type_ = upload_content_type;
146 upload_content_ = upload_content;
147 upload_content_set_ = true;
148 }
149
SetUploadFilePath(const std::string & upload_content_type,const base::FilePath & file_path,uint64 range_offset,uint64 range_length,scoped_refptr<base::TaskRunner> file_task_runner)150 void URLFetcherCore::SetUploadFilePath(
151 const std::string& upload_content_type,
152 const base::FilePath& file_path,
153 uint64 range_offset,
154 uint64 range_length,
155 scoped_refptr<base::TaskRunner> file_task_runner) {
156 DCHECK(!is_chunked_upload_);
157 DCHECK(!upload_content_set_);
158 DCHECK(upload_content_.empty());
159 DCHECK(upload_file_path_.empty());
160 DCHECK_EQ(upload_range_offset_, 0ULL);
161 DCHECK_EQ(upload_range_length_, 0ULL);
162 DCHECK(upload_content_type_.empty());
163 DCHECK(!upload_content_type.empty());
164
165 upload_content_type_ = upload_content_type;
166 upload_file_path_ = file_path;
167 upload_range_offset_ = range_offset;
168 upload_range_length_ = range_length;
169 upload_file_task_runner_ = file_task_runner;
170 upload_content_set_ = true;
171 }
172
SetChunkedUpload(const std::string & content_type)173 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
174 DCHECK(is_chunked_upload_ ||
175 (upload_content_type_.empty() &&
176 upload_content_.empty()));
177
178 // Empty |content_type| is not allowed here, because it is impossible
179 // to ensure non-empty upload content as it is not yet supplied.
180 DCHECK(!content_type.empty());
181
182 upload_content_type_ = content_type;
183 upload_content_.clear();
184 is_chunked_upload_ = true;
185 }
186
AppendChunkToUpload(const std::string & content,bool is_last_chunk)187 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
188 bool is_last_chunk) {
189 DCHECK(delegate_task_runner_.get());
190 DCHECK(network_task_runner_.get());
191 network_task_runner_->PostTask(
192 FROM_HERE,
193 base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
194 is_last_chunk));
195 }
196
SetLoadFlags(int load_flags)197 void URLFetcherCore::SetLoadFlags(int load_flags) {
198 load_flags_ = load_flags;
199 }
200
GetLoadFlags() const201 int URLFetcherCore::GetLoadFlags() const {
202 return load_flags_;
203 }
204
SetReferrer(const std::string & referrer)205 void URLFetcherCore::SetReferrer(const std::string& referrer) {
206 referrer_ = referrer;
207 }
208
SetReferrerPolicy(URLRequest::ReferrerPolicy referrer_policy)209 void URLFetcherCore::SetReferrerPolicy(
210 URLRequest::ReferrerPolicy referrer_policy) {
211 referrer_policy_ = referrer_policy;
212 }
213
SetExtraRequestHeaders(const std::string & extra_request_headers)214 void URLFetcherCore::SetExtraRequestHeaders(
215 const std::string& extra_request_headers) {
216 extra_request_headers_.Clear();
217 extra_request_headers_.AddHeadersFromString(extra_request_headers);
218 }
219
AddExtraRequestHeader(const std::string & header_line)220 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
221 extra_request_headers_.AddHeaderFromString(header_line);
222 }
223
SetRequestContext(URLRequestContextGetter * request_context_getter)224 void URLFetcherCore::SetRequestContext(
225 URLRequestContextGetter* request_context_getter) {
226 DCHECK(!request_context_getter_.get());
227 DCHECK(request_context_getter);
228 request_context_getter_ = request_context_getter;
229 }
230
SetFirstPartyForCookies(const GURL & first_party_for_cookies)231 void URLFetcherCore::SetFirstPartyForCookies(
232 const GURL& first_party_for_cookies) {
233 DCHECK(first_party_for_cookies_.is_empty());
234 first_party_for_cookies_ = first_party_for_cookies;
235 }
236
SetURLRequestUserData(const void * key,const URLFetcher::CreateDataCallback & create_data_callback)237 void URLFetcherCore::SetURLRequestUserData(
238 const void* key,
239 const URLFetcher::CreateDataCallback& create_data_callback) {
240 DCHECK(key);
241 DCHECK(!create_data_callback.is_null());
242 url_request_data_key_ = key;
243 url_request_create_data_callback_ = create_data_callback;
244 }
245
SetStopOnRedirect(bool stop_on_redirect)246 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
247 stop_on_redirect_ = stop_on_redirect;
248 }
249
SetAutomaticallyRetryOn5xx(bool retry)250 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
251 automatically_retry_on_5xx_ = retry;
252 }
253
SetMaxRetriesOn5xx(int max_retries)254 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
255 max_retries_on_5xx_ = max_retries;
256 }
257
GetMaxRetriesOn5xx() const258 int URLFetcherCore::GetMaxRetriesOn5xx() const {
259 return max_retries_on_5xx_;
260 }
261
GetBackoffDelay() const262 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
263 return backoff_delay_;
264 }
265
SetAutomaticallyRetryOnNetworkChanges(int max_retries)266 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
267 max_retries_on_network_changes_ = max_retries;
268 }
269
SaveResponseToFileAtPath(const base::FilePath & file_path,scoped_refptr<base::SequencedTaskRunner> file_task_runner)270 void URLFetcherCore::SaveResponseToFileAtPath(
271 const base::FilePath& file_path,
272 scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
273 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
274 SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
275 new URLFetcherFileWriter(file_task_runner, file_path)));
276 }
277
SaveResponseToTemporaryFile(scoped_refptr<base::SequencedTaskRunner> file_task_runner)278 void URLFetcherCore::SaveResponseToTemporaryFile(
279 scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
280 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
281 SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
282 new URLFetcherFileWriter(file_task_runner, base::FilePath())));
283 }
284
SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter> response_writer)285 void URLFetcherCore::SaveResponseWithWriter(
286 scoped_ptr<URLFetcherResponseWriter> response_writer) {
287 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
288 response_writer_ = response_writer.Pass();
289 }
290
GetResponseHeaders() const291 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
292 return response_headers_.get();
293 }
294
295 // TODO(panayiotis): socket_address_ is written in the IO thread,
296 // if this is accessed in the UI thread, this could result in a race.
297 // Same for response_headers_ above and was_fetched_via_proxy_ below.
GetSocketAddress() const298 HostPortPair URLFetcherCore::GetSocketAddress() const {
299 return socket_address_;
300 }
301
WasFetchedViaProxy() const302 bool URLFetcherCore::WasFetchedViaProxy() const {
303 return was_fetched_via_proxy_;
304 }
305
GetOriginalURL() const306 const GURL& URLFetcherCore::GetOriginalURL() const {
307 return original_url_;
308 }
309
GetURL() const310 const GURL& URLFetcherCore::GetURL() const {
311 return url_;
312 }
313
GetStatus() const314 const URLRequestStatus& URLFetcherCore::GetStatus() const {
315 return status_;
316 }
317
GetResponseCode() const318 int URLFetcherCore::GetResponseCode() const {
319 return response_code_;
320 }
321
GetCookies() const322 const ResponseCookies& URLFetcherCore::GetCookies() const {
323 return cookies_;
324 }
325
ReceivedContentWasMalformed()326 void URLFetcherCore::ReceivedContentWasMalformed() {
327 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
328 if (network_task_runner_.get()) {
329 network_task_runner_->PostTask(
330 FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
331 }
332 }
333
GetResponseAsString(std::string * out_response_string) const334 bool URLFetcherCore::GetResponseAsString(
335 std::string* out_response_string) const {
336 URLFetcherStringWriter* string_writer =
337 response_writer_ ? response_writer_->AsStringWriter() : NULL;
338 if (!string_writer)
339 return false;
340
341 *out_response_string = string_writer->data();
342 UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
343 (string_writer->data().length() / 1024));
344 return true;
345 }
346
GetResponseAsFilePath(bool take_ownership,base::FilePath * out_response_path)347 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
348 base::FilePath* out_response_path) {
349 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
350
351 URLFetcherFileWriter* file_writer =
352 response_writer_ ? response_writer_->AsFileWriter() : NULL;
353 if (!file_writer)
354 return false;
355
356 *out_response_path = file_writer->file_path();
357
358 if (take_ownership) {
359 // Intentionally calling a file_writer_ method directly without posting
360 // the task to network_task_runner_.
361 //
362 // This is for correctly handling the case when file_writer_->DisownFile()
363 // is soon followed by URLFetcherCore::Stop(). We have to make sure that
364 // DisownFile takes effect before Stop deletes file_writer_.
365 //
366 // This direct call should be thread-safe, since DisownFile itself does no
367 // file operation. It just flips the state to be referred in destruction.
368 file_writer->DisownFile();
369 }
370 return true;
371 }
372
OnReceivedRedirect(URLRequest * request,const GURL & new_url,bool * defer_redirect)373 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
374 const GURL& new_url,
375 bool* defer_redirect) {
376 DCHECK_EQ(request, request_.get());
377 DCHECK(network_task_runner_->BelongsToCurrentThread());
378 if (stop_on_redirect_) {
379 stopped_on_redirect_ = true;
380 url_ = new_url;
381 response_code_ = request_->GetResponseCode();
382 was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
383 request->Cancel();
384 OnReadCompleted(request, 0);
385 }
386 }
387
OnResponseStarted(URLRequest * request)388 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
389 DCHECK_EQ(request, request_.get());
390 DCHECK(network_task_runner_->BelongsToCurrentThread());
391 if (request_->status().is_success()) {
392 response_code_ = request_->GetResponseCode();
393 response_headers_ = request_->response_headers();
394 socket_address_ = request_->GetSocketAddress();
395 was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
396 total_response_bytes_ = request_->GetExpectedContentSize();
397 }
398
399 ReadResponse();
400 }
401
OnCertificateRequested(URLRequest * request,SSLCertRequestInfo * cert_request_info)402 void URLFetcherCore::OnCertificateRequested(
403 URLRequest* request,
404 SSLCertRequestInfo* cert_request_info) {
405 DCHECK_EQ(request, request_.get());
406 DCHECK(network_task_runner_->BelongsToCurrentThread());
407
408 if (g_ignore_certificate_requests) {
409 request->ContinueWithCertificate(NULL);
410 } else {
411 request->Cancel();
412 }
413 }
414
OnReadCompleted(URLRequest * request,int bytes_read)415 void URLFetcherCore::OnReadCompleted(URLRequest* request,
416 int bytes_read) {
417 DCHECK(request == request_);
418 DCHECK(network_task_runner_->BelongsToCurrentThread());
419
420 if (!stopped_on_redirect_)
421 url_ = request->url();
422 URLRequestThrottlerManager* throttler_manager =
423 request->context()->throttler_manager();
424 if (throttler_manager) {
425 url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
426 }
427
428 do {
429 if (!request_->status().is_success() || bytes_read <= 0)
430 break;
431
432 current_response_bytes_ += bytes_read;
433 InformDelegateDownloadProgress();
434
435 const int result =
436 WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
437 if (result < 0) {
438 // Write failed or waiting for write completion.
439 return;
440 }
441 } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
442
443 const URLRequestStatus status = request_->status();
444
445 if (status.is_success())
446 request_->GetResponseCookies(&cookies_);
447
448 // See comments re: HEAD requests in ReadResponse().
449 if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
450 status_ = status;
451 ReleaseRequest();
452
453 // No more data to write.
454 const int result = response_writer_->Finish(
455 base::Bind(&URLFetcherCore::DidFinishWriting, this));
456 if (result != ERR_IO_PENDING)
457 DidFinishWriting(result);
458 }
459 }
460
CancelAll()461 void URLFetcherCore::CancelAll() {
462 g_registry.Get().CancelAll();
463 }
464
GetNumFetcherCores()465 int URLFetcherCore::GetNumFetcherCores() {
466 return g_registry.Get().size();
467 }
468
SetEnableInterceptionForTests(bool enabled)469 void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
470 g_interception_enabled = enabled;
471 }
472
SetIgnoreCertificateRequests(bool ignored)473 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
474 g_ignore_certificate_requests = ignored;
475 }
476
~URLFetcherCore()477 URLFetcherCore::~URLFetcherCore() {
478 // |request_| should be NULL. If not, it's unsafe to delete it here since we
479 // may not be on the IO thread.
480 DCHECK(!request_.get());
481 }
482
StartOnIOThread()483 void URLFetcherCore::StartOnIOThread() {
484 DCHECK(network_task_runner_->BelongsToCurrentThread());
485
486 if (!response_writer_)
487 response_writer_.reset(new URLFetcherStringWriter);
488
489 const int result = response_writer_->Initialize(
490 base::Bind(&URLFetcherCore::DidInitializeWriter, this));
491 if (result != ERR_IO_PENDING)
492 DidInitializeWriter(result);
493 }
494
StartURLRequest()495 void URLFetcherCore::StartURLRequest() {
496 DCHECK(network_task_runner_->BelongsToCurrentThread());
497
498 if (was_cancelled_) {
499 // Since StartURLRequest() is posted as a *delayed* task, it may
500 // run after the URLFetcher was already stopped.
501 return;
502 }
503
504 DCHECK(request_context_getter_.get());
505 DCHECK(!request_.get());
506
507 g_registry.Get().AddURLFetcherCore(this);
508 current_response_bytes_ = 0;
509 request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
510 original_url_, DEFAULT_PRIORITY, this, NULL);
511 request_->set_stack_trace(stack_trace_);
512 int flags = request_->load_flags() | load_flags_;
513 if (!g_interception_enabled)
514 flags = flags | LOAD_DISABLE_INTERCEPT;
515
516 if (is_chunked_upload_)
517 request_->EnableChunkedUpload();
518 request_->SetLoadFlags(flags);
519 request_->SetReferrer(referrer_);
520 request_->set_referrer_policy(referrer_policy_);
521 request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
522 original_url_ : first_party_for_cookies_);
523 if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
524 request_->SetUserData(url_request_data_key_,
525 url_request_create_data_callback_.Run());
526 }
527
528 switch (request_type_) {
529 case URLFetcher::GET:
530 break;
531
532 case URLFetcher::POST:
533 case URLFetcher::PUT:
534 case URLFetcher::PATCH:
535 // Upload content must be set.
536 DCHECK(is_chunked_upload_ || upload_content_set_);
537
538 request_->set_method(
539 request_type_ == URLFetcher::POST ? "POST" :
540 request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
541 if (!upload_content_type_.empty()) {
542 extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
543 upload_content_type_);
544 }
545 if (!upload_content_.empty()) {
546 scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
547 upload_content_.data(), upload_content_.size()));
548 request_->set_upload(make_scoped_ptr(
549 UploadDataStream::CreateWithReader(reader.Pass(), 0)));
550 } else if (!upload_file_path_.empty()) {
551 scoped_ptr<UploadElementReader> reader(
552 new UploadFileElementReader(upload_file_task_runner_.get(),
553 upload_file_path_,
554 upload_range_offset_,
555 upload_range_length_,
556 base::Time()));
557 request_->set_upload(make_scoped_ptr(
558 UploadDataStream::CreateWithReader(reader.Pass(), 0)));
559 }
560
561 current_upload_bytes_ = -1;
562 // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
563 // layer and avoid using timer here.
564 upload_progress_checker_timer_.reset(
565 new base::RepeatingTimer<URLFetcherCore>());
566 upload_progress_checker_timer_->Start(
567 FROM_HERE,
568 base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
569 this,
570 &URLFetcherCore::InformDelegateUploadProgress);
571 break;
572
573 case URLFetcher::HEAD:
574 request_->set_method("HEAD");
575 break;
576
577 case URLFetcher::DELETE_REQUEST:
578 request_->set_method("DELETE");
579 break;
580
581 default:
582 NOTREACHED();
583 }
584
585 if (!extra_request_headers_.IsEmpty())
586 request_->SetExtraRequestHeaders(extra_request_headers_);
587
588 request_->Start();
589 }
590
DidInitializeWriter(int result)591 void URLFetcherCore::DidInitializeWriter(int result) {
592 if (result != OK) {
593 CancelURLRequest(result);
594 delegate_task_runner_->PostTask(
595 FROM_HERE,
596 base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
597 return;
598 }
599 StartURLRequestWhenAppropriate();
600 }
601
StartURLRequestWhenAppropriate()602 void URLFetcherCore::StartURLRequestWhenAppropriate() {
603 DCHECK(network_task_runner_->BelongsToCurrentThread());
604
605 if (was_cancelled_)
606 return;
607
608 DCHECK(request_context_getter_.get());
609
610 int64 delay = 0LL;
611 if (original_url_throttler_entry_.get() == NULL) {
612 URLRequestThrottlerManager* manager =
613 request_context_getter_->GetURLRequestContext()->throttler_manager();
614 if (manager) {
615 original_url_throttler_entry_ =
616 manager->RegisterRequestUrl(original_url_);
617 }
618 }
619 if (original_url_throttler_entry_.get() != NULL) {
620 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
621 GetBackoffReleaseTime());
622 }
623
624 if (delay == 0) {
625 StartURLRequest();
626 } else {
627 base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
628 FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
629 base::TimeDelta::FromMilliseconds(delay));
630 }
631 }
632
CancelURLRequest(int error)633 void URLFetcherCore::CancelURLRequest(int error) {
634 DCHECK(network_task_runner_->BelongsToCurrentThread());
635
636 if (request_.get()) {
637 request_->CancelWithError(error);
638 ReleaseRequest();
639 }
640
641 // Set the error manually.
642 // Normally, calling URLRequest::CancelWithError() results in calling
643 // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
644 // URLRequestJob::NotifyDone(). But, because the request was released
645 // immediately after being canceled, the request could not call
646 // OnReadCompleted() which overwrites |status_| with the error status.
647 status_.set_status(URLRequestStatus::CANCELED);
648 status_.set_error(error);
649
650 // Release the reference to the request context. There could be multiple
651 // references to URLFetcher::Core at this point so it may take a while to
652 // delete the object, but we cannot delay the destruction of the request
653 // context.
654 request_context_getter_ = NULL;
655 first_party_for_cookies_ = GURL();
656 url_request_data_key_ = NULL;
657 url_request_create_data_callback_.Reset();
658 was_cancelled_ = true;
659 }
660
OnCompletedURLRequest(base::TimeDelta backoff_delay)661 void URLFetcherCore::OnCompletedURLRequest(
662 base::TimeDelta backoff_delay) {
663 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
664
665 // Save the status and backoff_delay so that delegates can read it.
666 if (delegate_) {
667 backoff_delay_ = backoff_delay;
668 InformDelegateFetchIsComplete();
669 }
670 }
671
InformDelegateFetchIsComplete()672 void URLFetcherCore::InformDelegateFetchIsComplete() {
673 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
674 if (delegate_)
675 delegate_->OnURLFetchComplete(fetcher_);
676 }
677
NotifyMalformedContent()678 void URLFetcherCore::NotifyMalformedContent() {
679 DCHECK(network_task_runner_->BelongsToCurrentThread());
680 if (url_throttler_entry_.get() != NULL) {
681 int status_code = response_code_;
682 if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
683 // The status code will generally be known by the time clients
684 // call the |ReceivedContentWasMalformed()| function (which ends up
685 // calling the current function) but if it's not, we need to assume
686 // the response was successful so that the total failure count
687 // used to calculate exponential back-off goes up.
688 status_code = 200;
689 }
690 url_throttler_entry_->ReceivedContentWasMalformed(status_code);
691 }
692 }
693
DidFinishWriting(int result)694 void URLFetcherCore::DidFinishWriting(int result) {
695 if (result != OK) {
696 CancelURLRequest(result);
697 delegate_task_runner_->PostTask(
698 FROM_HERE,
699 base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
700 return;
701 }
702 // If the file was successfully closed, then the URL request is complete.
703 RetryOrCompleteUrlFetch();
704 }
705
RetryOrCompleteUrlFetch()706 void URLFetcherCore::RetryOrCompleteUrlFetch() {
707 DCHECK(network_task_runner_->BelongsToCurrentThread());
708 base::TimeDelta backoff_delay;
709
710 // Checks the response from server.
711 if (response_code_ >= 500 ||
712 status_.error() == ERR_TEMPORARILY_THROTTLED) {
713 // When encountering a server error, we will send the request again
714 // after backoff time.
715 ++num_retries_on_5xx_;
716
717 // Note that backoff_delay may be 0 because (a) the
718 // URLRequestThrottlerManager and related code does not
719 // necessarily back off on the first error, (b) it only backs off
720 // on some of the 5xx status codes, (c) not all URLRequestContexts
721 // have a throttler manager.
722 base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
723 backoff_delay = backoff_release_time - base::TimeTicks::Now();
724 if (backoff_delay < base::TimeDelta())
725 backoff_delay = base::TimeDelta();
726
727 if (automatically_retry_on_5xx_ &&
728 num_retries_on_5xx_ <= max_retries_on_5xx_) {
729 StartOnIOThread();
730 return;
731 }
732 } else {
733 backoff_delay = base::TimeDelta();
734 }
735
736 // Retry if the request failed due to network changes.
737 if (status_.error() == ERR_NETWORK_CHANGED &&
738 num_retries_on_network_changes_ < max_retries_on_network_changes_) {
739 ++num_retries_on_network_changes_;
740
741 // Retry soon, after flushing all the current tasks which may include
742 // further network change observers.
743 network_task_runner_->PostTask(
744 FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
745 return;
746 }
747
748 request_context_getter_ = NULL;
749 first_party_for_cookies_ = GURL();
750 url_request_data_key_ = NULL;
751 url_request_create_data_callback_.Reset();
752 bool posted = delegate_task_runner_->PostTask(
753 FROM_HERE,
754 base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
755
756 // If the delegate message loop does not exist any more, then the delegate
757 // should be gone too.
758 DCHECK(posted || !delegate_);
759 }
760
ReleaseRequest()761 void URLFetcherCore::ReleaseRequest() {
762 upload_progress_checker_timer_.reset();
763 request_.reset();
764 g_registry.Get().RemoveURLFetcherCore(this);
765 }
766
GetBackoffReleaseTime()767 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
768 DCHECK(network_task_runner_->BelongsToCurrentThread());
769
770 if (original_url_throttler_entry_.get()) {
771 base::TimeTicks original_url_backoff =
772 original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
773 base::TimeTicks destination_url_backoff;
774 if (url_throttler_entry_.get() != NULL &&
775 original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
776 destination_url_backoff =
777 url_throttler_entry_->GetExponentialBackoffReleaseTime();
778 }
779
780 return original_url_backoff > destination_url_backoff ?
781 original_url_backoff : destination_url_backoff;
782 } else {
783 return base::TimeTicks();
784 }
785 }
786
CompleteAddingUploadDataChunk(const std::string & content,bool is_last_chunk)787 void URLFetcherCore::CompleteAddingUploadDataChunk(
788 const std::string& content, bool is_last_chunk) {
789 if (was_cancelled_) {
790 // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
791 // may run after the URLFetcher was already stopped.
792 return;
793 }
794 DCHECK(is_chunked_upload_);
795 DCHECK(request_.get());
796 DCHECK(!content.empty());
797 request_->AppendChunkToUpload(content.data(),
798 static_cast<int>(content.length()),
799 is_last_chunk);
800 }
801
WriteBuffer(scoped_refptr<DrainableIOBuffer> data)802 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
803 while (data->BytesRemaining() > 0) {
804 const int result = response_writer_->Write(
805 data.get(),
806 data->BytesRemaining(),
807 base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
808 if (result < 0) {
809 if (result != ERR_IO_PENDING)
810 DidWriteBuffer(data, result);
811 return result;
812 }
813 data->DidConsume(result);
814 }
815 return OK;
816 }
817
DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,int result)818 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
819 int result) {
820 if (result < 0) { // Handle errors.
821 CancelURLRequest(result);
822 response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
823 delegate_task_runner_->PostTask(
824 FROM_HERE,
825 base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
826 return;
827 }
828
829 // Continue writing.
830 data->DidConsume(result);
831 if (WriteBuffer(data) < 0)
832 return;
833
834 // Finished writing buffer_. Read some more, unless the request has been
835 // cancelled and deleted.
836 DCHECK_EQ(0, data->BytesRemaining());
837 if (request_.get())
838 ReadResponse();
839 }
840
ReadResponse()841 void URLFetcherCore::ReadResponse() {
842 // Some servers may treat HEAD requests as GET requests. To free up the
843 // network connection as soon as possible, signal that the request has
844 // completed immediately, without trying to read any data back (all we care
845 // about is the response code and headers, which we already have).
846 int bytes_read = 0;
847 if (request_->status().is_success() &&
848 (request_type_ != URLFetcher::HEAD))
849 request_->Read(buffer_.get(), kBufferSize, &bytes_read);
850 OnReadCompleted(request_.get(), bytes_read);
851 }
852
InformDelegateUploadProgress()853 void URLFetcherCore::InformDelegateUploadProgress() {
854 DCHECK(network_task_runner_->BelongsToCurrentThread());
855 if (request_.get()) {
856 int64 current = request_->GetUploadProgress().position();
857 if (current_upload_bytes_ != current) {
858 current_upload_bytes_ = current;
859 int64 total = -1;
860 if (!is_chunked_upload_) {
861 total = static_cast<int64>(request_->GetUploadProgress().size());
862 // Total may be zero if the UploadDataStream::Init has not been called
863 // yet. Don't send the upload progress until the size is initialized.
864 if (!total)
865 return;
866 }
867 delegate_task_runner_->PostTask(
868 FROM_HERE,
869 base::Bind(
870 &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
871 this, current, total));
872 }
873 }
874 }
875
InformDelegateUploadProgressInDelegateThread(int64 current,int64 total)876 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
877 int64 current, int64 total) {
878 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
879 if (delegate_)
880 delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
881 }
882
InformDelegateDownloadProgress()883 void URLFetcherCore::InformDelegateDownloadProgress() {
884 DCHECK(network_task_runner_->BelongsToCurrentThread());
885 delegate_task_runner_->PostTask(
886 FROM_HERE,
887 base::Bind(
888 &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
889 this, current_response_bytes_, total_response_bytes_));
890 }
891
InformDelegateDownloadProgressInDelegateThread(int64 current,int64 total)892 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
893 int64 current, int64 total) {
894 DCHECK(delegate_task_runner_->BelongsToCurrentThread());
895 if (delegate_)
896 delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
897 }
898
899 } // namespace net
900