1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "net/url_request/url_request_http_job.h"
6
7 #include "base/base_switches.h"
8 #include "base/bind.h"
9 #include "base/bind_helpers.h"
10 #include "base/command_line.h"
11 #include "base/compiler_specific.h"
12 #include "base/file_version_info.h"
13 #include "base/message_loop/message_loop.h"
14 #include "base/metrics/field_trial.h"
15 #include "base/metrics/histogram.h"
16 #include "base/rand_util.h"
17 #include "base/strings/string_util.h"
18 #include "base/time/time.h"
19 #include "net/base/filter.h"
20 #include "net/base/host_port_pair.h"
21 #include "net/base/load_flags.h"
22 #include "net/base/mime_util.h"
23 #include "net/base/net_errors.h"
24 #include "net/base/net_util.h"
25 #include "net/base/network_delegate.h"
26 #include "net/base/sdch_manager.h"
27 #include "net/cert/cert_status_flags.h"
28 #include "net/cookies/cookie_monster.h"
29 #include "net/http/http_network_session.h"
30 #include "net/http/http_request_headers.h"
31 #include "net/http/http_response_headers.h"
32 #include "net/http/http_response_info.h"
33 #include "net/http/http_status_code.h"
34 #include "net/http/http_transaction.h"
35 #include "net/http/http_transaction_delegate.h"
36 #include "net/http/http_transaction_factory.h"
37 #include "net/http/http_util.h"
38 #include "net/ssl/ssl_cert_request_info.h"
39 #include "net/ssl/ssl_config_service.h"
40 #include "net/url_request/fraudulent_certificate_reporter.h"
41 #include "net/url_request/http_user_agent_settings.h"
42 #include "net/url_request/url_request.h"
43 #include "net/url_request/url_request_context.h"
44 #include "net/url_request/url_request_error_job.h"
45 #include "net/url_request/url_request_job_factory.h"
46 #include "net/url_request/url_request_redirect_job.h"
47 #include "net/url_request/url_request_throttler_header_adapter.h"
48 #include "net/url_request/url_request_throttler_manager.h"
49 #include "net/websockets/websocket_handshake_stream_base.h"
50
51 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
52
53 namespace net {
54
55 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
56 public:
57 explicit HttpFilterContext(URLRequestHttpJob* job);
58 virtual ~HttpFilterContext();
59
60 // FilterContext implementation.
61 virtual bool GetMimeType(std::string* mime_type) const OVERRIDE;
62 virtual bool GetURL(GURL* gurl) const OVERRIDE;
63 virtual base::Time GetRequestTime() const OVERRIDE;
64 virtual bool IsCachedContent() const OVERRIDE;
65 virtual bool IsDownload() const OVERRIDE;
66 virtual bool IsSdchResponse() const OVERRIDE;
67 virtual int64 GetByteReadCount() const OVERRIDE;
68 virtual int GetResponseCode() const OVERRIDE;
69 virtual void RecordPacketStats(StatisticSelector statistic) const OVERRIDE;
70
71 // Method to allow us to reset filter context for a response that should have
72 // been SDCH encoded when there is an update due to an explicit HTTP header.
73 void ResetSdchResponseToFalse();
74
75 private:
76 URLRequestHttpJob* job_;
77
78 DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
79 };
80
81 class URLRequestHttpJob::HttpTransactionDelegateImpl
82 : public HttpTransactionDelegate {
83 public:
HttpTransactionDelegateImpl(URLRequest * request,NetworkDelegate * network_delegate)84 HttpTransactionDelegateImpl(URLRequest* request,
85 NetworkDelegate* network_delegate)
86 : request_(request),
87 network_delegate_(network_delegate),
88 state_(NONE_ACTIVE) {}
~HttpTransactionDelegateImpl()89 virtual ~HttpTransactionDelegateImpl() { OnDetachRequest(); }
OnDetachRequest()90 void OnDetachRequest() {
91 if (!IsRequestAndDelegateActive())
92 return;
93 NotifyStateChange(NetworkDelegate::REQUEST_WAIT_STATE_RESET);
94 state_ = NONE_ACTIVE;
95 request_ = NULL;
96 }
OnCacheActionStart()97 virtual void OnCacheActionStart() OVERRIDE {
98 HandleStateChange(NONE_ACTIVE,
99 CACHE_ACTIVE,
100 NetworkDelegate::REQUEST_WAIT_STATE_CACHE_START);
101 }
OnCacheActionFinish()102 virtual void OnCacheActionFinish() OVERRIDE {
103 HandleStateChange(CACHE_ACTIVE,
104 NONE_ACTIVE,
105 NetworkDelegate::REQUEST_WAIT_STATE_CACHE_FINISH);
106 }
OnNetworkActionStart()107 virtual void OnNetworkActionStart() OVERRIDE {
108 HandleStateChange(NONE_ACTIVE,
109 NETWORK_ACTIVE,
110 NetworkDelegate::REQUEST_WAIT_STATE_NETWORK_START);
111 }
OnNetworkActionFinish()112 virtual void OnNetworkActionFinish() OVERRIDE {
113 HandleStateChange(NETWORK_ACTIVE,
114 NONE_ACTIVE,
115 NetworkDelegate::REQUEST_WAIT_STATE_NETWORK_FINISH);
116 }
117
118 private:
119 enum State {
120 NONE_ACTIVE,
121 CACHE_ACTIVE,
122 NETWORK_ACTIVE
123 };
124
125 // Returns true if this object still has an active request and network
126 // delegate.
IsRequestAndDelegateActive() const127 bool IsRequestAndDelegateActive() const {
128 return request_ && network_delegate_;
129 }
130
131 // Notifies the |network_delegate_| object of a change in the state of the
132 // |request_| to the state given by the |request_wait_state| argument.
NotifyStateChange(NetworkDelegate::RequestWaitState request_wait_state)133 void NotifyStateChange(NetworkDelegate::RequestWaitState request_wait_state) {
134 network_delegate_->NotifyRequestWaitStateChange(*request_,
135 request_wait_state);
136 }
137
138 // Checks the request and delegate are still active, changes |state_| from
139 // |expected_state| to |next_state|, and then notifies the network delegate of
140 // the change to |request_wait_state|.
HandleStateChange(State expected_state,State next_state,NetworkDelegate::RequestWaitState request_wait_state)141 void HandleStateChange(State expected_state,
142 State next_state,
143 NetworkDelegate::RequestWaitState request_wait_state) {
144 if (!IsRequestAndDelegateActive())
145 return;
146 DCHECK_EQ(expected_state, state_);
147 state_ = next_state;
148 NotifyStateChange(request_wait_state);
149 }
150
151 URLRequest* request_;
152 NetworkDelegate* network_delegate_;
153 // Internal state tracking, for sanity checking.
154 State state_;
155
156 DISALLOW_COPY_AND_ASSIGN(HttpTransactionDelegateImpl);
157 };
158
HttpFilterContext(URLRequestHttpJob * job)159 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
160 : job_(job) {
161 DCHECK(job_);
162 }
163
~HttpFilterContext()164 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
165 }
166
GetMimeType(std::string * mime_type) const167 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
168 std::string* mime_type) const {
169 return job_->GetMimeType(mime_type);
170 }
171
GetURL(GURL * gurl) const172 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
173 if (!job_->request())
174 return false;
175 *gurl = job_->request()->url();
176 return true;
177 }
178
GetRequestTime() const179 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
180 return job_->request() ? job_->request()->request_time() : base::Time();
181 }
182
IsCachedContent() const183 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
184 return job_->is_cached_content_;
185 }
186
IsDownload() const187 bool URLRequestHttpJob::HttpFilterContext::IsDownload() const {
188 return (job_->request_info_.load_flags & LOAD_IS_DOWNLOAD) != 0;
189 }
190
ResetSdchResponseToFalse()191 void URLRequestHttpJob::HttpFilterContext::ResetSdchResponseToFalse() {
192 DCHECK(job_->sdch_dictionary_advertised_);
193 job_->sdch_dictionary_advertised_ = false;
194 }
195
IsSdchResponse() const196 bool URLRequestHttpJob::HttpFilterContext::IsSdchResponse() const {
197 return job_->sdch_dictionary_advertised_;
198 }
199
GetByteReadCount() const200 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
201 return job_->filter_input_byte_count();
202 }
203
GetResponseCode() const204 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
205 return job_->GetResponseCode();
206 }
207
RecordPacketStats(StatisticSelector statistic) const208 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
209 StatisticSelector statistic) const {
210 job_->RecordPacketStats(statistic);
211 }
212
213 // TODO(darin): make sure the port blocking code is not lost
214 // static
Factory(URLRequest * request,NetworkDelegate * network_delegate,const std::string & scheme)215 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
216 NetworkDelegate* network_delegate,
217 const std::string& scheme) {
218 DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
219 scheme == "wss");
220
221 if (!request->context()->http_transaction_factory()) {
222 NOTREACHED() << "requires a valid context";
223 return new URLRequestErrorJob(
224 request, network_delegate, ERR_INVALID_ARGUMENT);
225 }
226
227 GURL redirect_url;
228 if (request->GetHSTSRedirect(&redirect_url)) {
229 return new URLRequestRedirectJob(
230 request, network_delegate, redirect_url,
231 // Use status code 307 to preserve the method, so POST requests work.
232 URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT);
233 }
234 return new URLRequestHttpJob(request,
235 network_delegate,
236 request->context()->http_user_agent_settings());
237 }
238
URLRequestHttpJob(URLRequest * request,NetworkDelegate * network_delegate,const HttpUserAgentSettings * http_user_agent_settings)239 URLRequestHttpJob::URLRequestHttpJob(
240 URLRequest* request,
241 NetworkDelegate* network_delegate,
242 const HttpUserAgentSettings* http_user_agent_settings)
243 : URLRequestJob(request, network_delegate),
244 priority_(DEFAULT_PRIORITY),
245 response_info_(NULL),
246 response_cookies_save_index_(0),
247 proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
248 server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
249 start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
250 base::Unretained(this))),
251 notify_before_headers_sent_callback_(
252 base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
253 base::Unretained(this))),
254 read_in_progress_(false),
255 throttling_entry_(NULL),
256 sdch_dictionary_advertised_(false),
257 sdch_test_activated_(false),
258 sdch_test_control_(false),
259 is_cached_content_(false),
260 request_creation_time_(),
261 packet_timing_enabled_(false),
262 done_(false),
263 bytes_observed_in_packets_(0),
264 request_time_snapshot_(),
265 final_packet_time_(),
266 filter_context_(new HttpFilterContext(this)),
267 weak_factory_(this),
268 on_headers_received_callback_(
269 base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
270 base::Unretained(this))),
271 awaiting_callback_(false),
272 http_transaction_delegate_(
273 new HttpTransactionDelegateImpl(request, network_delegate)),
274 http_user_agent_settings_(http_user_agent_settings) {
275 URLRequestThrottlerManager* manager = request->context()->throttler_manager();
276 if (manager)
277 throttling_entry_ = manager->RegisterRequestUrl(request->url());
278
279 ResetTimer();
280 }
281
~URLRequestHttpJob()282 URLRequestHttpJob::~URLRequestHttpJob() {
283 CHECK(!awaiting_callback_);
284
285 DCHECK(!sdch_test_control_ || !sdch_test_activated_);
286 if (!is_cached_content_) {
287 if (sdch_test_control_)
288 RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
289 if (sdch_test_activated_)
290 RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
291 }
292 // Make sure SDCH filters are told to emit histogram data while
293 // filter_context_ is still alive.
294 DestroyFilters();
295
296 if (sdch_dictionary_url_.is_valid()) {
297 // Prior to reaching the destructor, request_ has been set to a NULL
298 // pointer, so request_->url() is no longer valid in the destructor, and we
299 // use an alternate copy |request_info_.url|.
300 SdchManager* manager = SdchManager::Global();
301 // To be extra safe, since this is a "different time" from when we decided
302 // to get the dictionary, we'll validate that an SdchManager is available.
303 // At shutdown time, care is taken to be sure that we don't delete this
304 // globally useful instance "too soon," so this check is just defensive
305 // coding to assure that IF the system is shutting down, we don't have any
306 // problem if the manager was deleted ahead of time.
307 if (manager) // Defensive programming.
308 manager->FetchDictionary(request_info_.url, sdch_dictionary_url_);
309 }
310 DoneWithRequest(ABORTED);
311 }
312
SetPriority(RequestPriority priority)313 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
314 priority_ = priority;
315 if (transaction_)
316 transaction_->SetPriority(priority_);
317 }
318
Start()319 void URLRequestHttpJob::Start() {
320 DCHECK(!transaction_.get());
321
322 // URLRequest::SetReferrer ensures that we do not send username and password
323 // fields in the referrer.
324 GURL referrer(request_->referrer());
325
326 request_info_.url = request_->url();
327 request_info_.method = request_->method();
328 request_info_.load_flags = request_->load_flags();
329 // Enable privacy mode if cookie settings or flags tell us not send or
330 // save cookies.
331 bool enable_privacy_mode =
332 (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
333 (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
334 CanEnablePrivacyMode();
335 // Privacy mode could still be disabled in OnCookiesLoaded if we are going
336 // to send previously saved cookies.
337 request_info_.privacy_mode = enable_privacy_mode ?
338 kPrivacyModeEnabled : kPrivacyModeDisabled;
339
340 // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
341 // from overriding headers that are controlled using other means. Otherwise a
342 // plugin could set a referrer although sending the referrer is inhibited.
343 request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
344
345 // Our consumer should have made sure that this is a safe referrer. See for
346 // instance WebCore::FrameLoader::HideReferrer.
347 if (referrer.is_valid()) {
348 request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
349 referrer.spec());
350 }
351
352 request_info_.extra_headers.SetHeaderIfMissing(
353 HttpRequestHeaders::kUserAgent,
354 http_user_agent_settings_ ?
355 http_user_agent_settings_->GetUserAgent(request_->url()) :
356 std::string());
357
358 AddExtraHeaders();
359 AddCookieHeaderAndStart();
360 }
361
Kill()362 void URLRequestHttpJob::Kill() {
363 http_transaction_delegate_->OnDetachRequest();
364
365 if (!transaction_.get())
366 return;
367
368 weak_factory_.InvalidateWeakPtrs();
369 DestroyTransaction();
370 URLRequestJob::Kill();
371 }
372
NotifyHeadersComplete()373 void URLRequestHttpJob::NotifyHeadersComplete() {
374 DCHECK(!response_info_);
375
376 response_info_ = transaction_->GetResponseInfo();
377
378 // Save boolean, as we'll need this info at destruction time, and filters may
379 // also need this info.
380 is_cached_content_ = response_info_->was_cached;
381
382 if (!is_cached_content_ && throttling_entry_.get()) {
383 URLRequestThrottlerHeaderAdapter response_adapter(GetResponseHeaders());
384 throttling_entry_->UpdateWithResponse(request_info_.url.host(),
385 &response_adapter);
386 }
387
388 // The ordering of these calls is not important.
389 ProcessStrictTransportSecurityHeader();
390 ProcessPublicKeyPinsHeader();
391
392 if (SdchManager::Global() &&
393 SdchManager::Global()->IsInSupportedDomain(request_->url())) {
394 const std::string name = "Get-Dictionary";
395 std::string url_text;
396 void* iter = NULL;
397 // TODO(jar): We need to not fetch dictionaries the first time they are
398 // seen, but rather wait until we can justify their usefulness.
399 // For now, we will only fetch the first dictionary, which will at least
400 // require multiple suggestions before we get additional ones for this site.
401 // Eventually we should wait until a dictionary is requested several times
402 // before we even download it (so that we don't waste memory or bandwidth).
403 if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
404 // request_->url() won't be valid in the destructor, so we use an
405 // alternate copy.
406 DCHECK_EQ(request_->url(), request_info_.url);
407 // Resolve suggested URL relative to request url.
408 sdch_dictionary_url_ = request_info_.url.Resolve(url_text);
409 }
410 }
411
412 // The HTTP transaction may be restarted several times for the purposes
413 // of sending authorization information. Each time it restarts, we get
414 // notified of the headers completion so that we can update the cookie store.
415 if (transaction_->IsReadyToRestartForAuth()) {
416 DCHECK(!response_info_->auth_challenge.get());
417 // TODO(battre): This breaks the webrequest API for
418 // URLRequestTestHTTP.BasicAuthWithCookies
419 // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
420 // occurs.
421 RestartTransactionWithAuth(AuthCredentials());
422 return;
423 }
424
425 URLRequestJob::NotifyHeadersComplete();
426 }
427
NotifyDone(const URLRequestStatus & status)428 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
429 DoneWithRequest(FINISHED);
430 URLRequestJob::NotifyDone(status);
431 }
432
DestroyTransaction()433 void URLRequestHttpJob::DestroyTransaction() {
434 DCHECK(transaction_.get());
435
436 DoneWithRequest(ABORTED);
437 transaction_.reset();
438 response_info_ = NULL;
439 receive_headers_end_ = base::TimeTicks();
440 }
441
StartTransaction()442 void URLRequestHttpJob::StartTransaction() {
443 if (network_delegate()) {
444 OnCallToDelegate();
445 int rv = network_delegate()->NotifyBeforeSendHeaders(
446 request_, notify_before_headers_sent_callback_,
447 &request_info_.extra_headers);
448 // If an extension blocks the request, we rely on the callback to
449 // MaybeStartTransactionInternal().
450 if (rv == ERR_IO_PENDING)
451 return;
452 MaybeStartTransactionInternal(rv);
453 return;
454 }
455 StartTransactionInternal();
456 }
457
NotifyBeforeSendHeadersCallback(int result)458 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
459 // Check that there are no callbacks to already canceled requests.
460 DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
461
462 MaybeStartTransactionInternal(result);
463 }
464
MaybeStartTransactionInternal(int result)465 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
466 OnCallToDelegateComplete();
467 if (result == OK) {
468 StartTransactionInternal();
469 } else {
470 std::string source("delegate");
471 request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
472 NetLog::StringCallback("source", &source));
473 NotifyCanceled();
474 NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
475 }
476 }
477
StartTransactionInternal()478 void URLRequestHttpJob::StartTransactionInternal() {
479 // NOTE: This method assumes that request_info_ is already setup properly.
480
481 // If we already have a transaction, then we should restart the transaction
482 // with auth provided by auth_credentials_.
483
484 int rv;
485
486 if (network_delegate()) {
487 network_delegate()->NotifySendHeaders(
488 request_, request_info_.extra_headers);
489 }
490
491 if (transaction_.get()) {
492 rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
493 auth_credentials_ = AuthCredentials();
494 } else {
495 DCHECK(request_->context()->http_transaction_factory());
496
497 rv = request_->context()->http_transaction_factory()->CreateTransaction(
498 priority_, &transaction_, http_transaction_delegate_.get());
499
500 if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
501 // TODO(ricea): Implement WebSocket throttling semantics as defined in
502 // RFC6455 Section 4.1.
503 base::SupportsUserData::Data* data = request_->GetUserData(
504 WebSocketHandshakeStreamBase::CreateHelper::DataKey());
505 if (data) {
506 transaction_->SetWebSocketHandshakeStreamCreateHelper(
507 static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
508 } else {
509 rv = ERR_DISALLOWED_URL_SCHEME;
510 }
511 }
512
513 if (rv == OK) {
514 if (!throttling_entry_.get() ||
515 !throttling_entry_->ShouldRejectRequest(*request_)) {
516 rv = transaction_->Start(
517 &request_info_, start_callback_, request_->net_log());
518 start_time_ = base::TimeTicks::Now();
519 } else {
520 // Special error code for the exponential back-off module.
521 rv = ERR_TEMPORARILY_THROTTLED;
522 }
523 }
524 }
525
526 if (rv == ERR_IO_PENDING)
527 return;
528
529 // The transaction started synchronously, but we need to notify the
530 // URLRequest delegate via the message loop.
531 base::MessageLoop::current()->PostTask(
532 FROM_HERE,
533 base::Bind(&URLRequestHttpJob::OnStartCompleted,
534 weak_factory_.GetWeakPtr(), rv));
535 }
536
AddExtraHeaders()537 void URLRequestHttpJob::AddExtraHeaders() {
538 // Supply Accept-Encoding field only if it is not already provided.
539 // It should be provided IF the content is known to have restrictions on
540 // potential encoding, such as streaming multi-media.
541 // For details see bug 47381.
542 // TODO(jar, enal): jpeg files etc. should set up a request header if
543 // possible. Right now it is done only by buffered_resource_loader and
544 // simple_data_source.
545 if (!request_info_.extra_headers.HasHeader(
546 HttpRequestHeaders::kAcceptEncoding)) {
547 bool advertise_sdch = SdchManager::Global() &&
548 SdchManager::Global()->IsInSupportedDomain(request_->url());
549 std::string avail_dictionaries;
550 if (advertise_sdch) {
551 SdchManager::Global()->GetAvailDictionaryList(request_->url(),
552 &avail_dictionaries);
553
554 // The AllowLatencyExperiment() is only true if we've successfully done a
555 // full SDCH compression recently in this browser session for this host.
556 // Note that for this path, there might be no applicable dictionaries,
557 // and hence we can't participate in the experiment.
558 if (!avail_dictionaries.empty() &&
559 SdchManager::Global()->AllowLatencyExperiment(request_->url())) {
560 // We are participating in the test (or control), and hence we'll
561 // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
562 // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
563 packet_timing_enabled_ = true;
564 if (base::RandDouble() < .01) {
565 sdch_test_control_ = true; // 1% probability.
566 advertise_sdch = false;
567 } else {
568 sdch_test_activated_ = true;
569 }
570 }
571 }
572
573 // Supply Accept-Encoding headers first so that it is more likely that they
574 // will be in the first transmitted packet. This can sometimes make it
575 // easier to filter and analyze the streams to assure that a proxy has not
576 // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
577 // headers.
578 if (!advertise_sdch) {
579 // Tell the server what compression formats we support (other than SDCH).
580 request_info_.extra_headers.SetHeader(
581 HttpRequestHeaders::kAcceptEncoding, "gzip,deflate");
582 } else {
583 // Include SDCH in acceptable list.
584 request_info_.extra_headers.SetHeader(
585 HttpRequestHeaders::kAcceptEncoding, "gzip,deflate,sdch");
586 if (!avail_dictionaries.empty()) {
587 request_info_.extra_headers.SetHeader(
588 kAvailDictionaryHeader,
589 avail_dictionaries);
590 sdch_dictionary_advertised_ = true;
591 // Since we're tagging this transaction as advertising a dictionary,
592 // we'll definitely employ an SDCH filter (or tentative sdch filter)
593 // when we get a response. When done, we'll record histograms via
594 // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
595 // arrival times.
596 packet_timing_enabled_ = true;
597 }
598 }
599 }
600
601 if (http_user_agent_settings_) {
602 // Only add default Accept-Language if the request didn't have it
603 // specified.
604 std::string accept_language =
605 http_user_agent_settings_->GetAcceptLanguage();
606 if (!accept_language.empty()) {
607 request_info_.extra_headers.SetHeaderIfMissing(
608 HttpRequestHeaders::kAcceptLanguage,
609 accept_language);
610 }
611 }
612 }
613
AddCookieHeaderAndStart()614 void URLRequestHttpJob::AddCookieHeaderAndStart() {
615 // No matter what, we want to report our status as IO pending since we will
616 // be notifying our consumer asynchronously via OnStartCompleted.
617 SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
618
619 // If the request was destroyed, then there is no more work to do.
620 if (!request_)
621 return;
622
623 CookieStore* cookie_store = request_->context()->cookie_store();
624 if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
625 net::CookieMonster* cookie_monster = cookie_store->GetCookieMonster();
626 if (cookie_monster) {
627 cookie_monster->GetAllCookiesForURLAsync(
628 request_->url(),
629 base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
630 weak_factory_.GetWeakPtr()));
631 } else {
632 CheckCookiePolicyAndLoad(CookieList());
633 }
634 } else {
635 DoStartTransaction();
636 }
637 }
638
DoLoadCookies()639 void URLRequestHttpJob::DoLoadCookies() {
640 CookieOptions options;
641 options.set_include_httponly();
642 request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
643 request_->url(), options,
644 base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
645 weak_factory_.GetWeakPtr()));
646 }
647
CheckCookiePolicyAndLoad(const CookieList & cookie_list)648 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
649 const CookieList& cookie_list) {
650 if (CanGetCookies(cookie_list))
651 DoLoadCookies();
652 else
653 DoStartTransaction();
654 }
655
OnCookiesLoaded(const std::string & cookie_line)656 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
657 if (!cookie_line.empty()) {
658 request_info_.extra_headers.SetHeader(
659 HttpRequestHeaders::kCookie, cookie_line);
660 // Disable privacy mode as we are sending cookies anyway.
661 request_info_.privacy_mode = kPrivacyModeDisabled;
662 }
663 DoStartTransaction();
664 }
665
DoStartTransaction()666 void URLRequestHttpJob::DoStartTransaction() {
667 // We may have been canceled while retrieving cookies.
668 if (GetStatus().is_success()) {
669 StartTransaction();
670 } else {
671 NotifyCanceled();
672 }
673 }
674
SaveCookiesAndNotifyHeadersComplete(int result)675 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
676 // End of the call started in OnStartCompleted.
677 OnCallToDelegateComplete();
678
679 if (result != net::OK) {
680 std::string source("delegate");
681 request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
682 NetLog::StringCallback("source", &source));
683 NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
684 return;
685 }
686
687 DCHECK(transaction_.get());
688
689 const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
690 DCHECK(response_info);
691
692 response_cookies_.clear();
693 response_cookies_save_index_ = 0;
694
695 FetchResponseCookies(&response_cookies_);
696
697 if (!GetResponseHeaders()->GetDateValue(&response_date_))
698 response_date_ = base::Time();
699
700 // Now, loop over the response cookies, and attempt to persist each.
701 SaveNextCookie();
702 }
703
704 // If the save occurs synchronously, SaveNextCookie will loop and save the next
705 // cookie. If the save is deferred, the callback is responsible for continuing
706 // to iterate through the cookies.
707 // TODO(erikwright): Modify the CookieStore API to indicate via return value
708 // whether it completed synchronously or asynchronously.
709 // See http://crbug.com/131066.
SaveNextCookie()710 void URLRequestHttpJob::SaveNextCookie() {
711 // No matter what, we want to report our status as IO pending since we will
712 // be notifying our consumer asynchronously via OnStartCompleted.
713 SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
714
715 // Used to communicate with the callback. See the implementation of
716 // OnCookieSaved.
717 scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
718 scoped_refptr<SharedBoolean> save_next_cookie_running =
719 new SharedBoolean(true);
720
721 if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
722 request_->context()->cookie_store() &&
723 response_cookies_.size() > 0) {
724 CookieOptions options;
725 options.set_include_httponly();
726 options.set_server_time(response_date_);
727
728 net::CookieStore::SetCookiesCallback callback(
729 base::Bind(&URLRequestHttpJob::OnCookieSaved,
730 weak_factory_.GetWeakPtr(),
731 save_next_cookie_running,
732 callback_pending));
733
734 // Loop through the cookies as long as SetCookieWithOptionsAsync completes
735 // synchronously.
736 while (!callback_pending->data &&
737 response_cookies_save_index_ < response_cookies_.size()) {
738 if (CanSetCookie(
739 response_cookies_[response_cookies_save_index_], &options)) {
740 callback_pending->data = true;
741 request_->context()->cookie_store()->SetCookieWithOptionsAsync(
742 request_->url(), response_cookies_[response_cookies_save_index_],
743 options, callback);
744 }
745 ++response_cookies_save_index_;
746 }
747 }
748
749 save_next_cookie_running->data = false;
750
751 if (!callback_pending->data) {
752 response_cookies_.clear();
753 response_cookies_save_index_ = 0;
754 SetStatus(URLRequestStatus()); // Clear the IO_PENDING status
755 NotifyHeadersComplete();
756 return;
757 }
758 }
759
760 // |save_next_cookie_running| is true when the callback is bound and set to
761 // false when SaveNextCookie exits, allowing the callback to determine if the
762 // save occurred synchronously or asynchronously.
763 // |callback_pending| is false when the callback is invoked and will be set to
764 // true by the callback, allowing SaveNextCookie to detect whether the save
765 // occurred synchronously.
766 // See SaveNextCookie() for more information.
OnCookieSaved(scoped_refptr<SharedBoolean> save_next_cookie_running,scoped_refptr<SharedBoolean> callback_pending,bool cookie_status)767 void URLRequestHttpJob::OnCookieSaved(
768 scoped_refptr<SharedBoolean> save_next_cookie_running,
769 scoped_refptr<SharedBoolean> callback_pending,
770 bool cookie_status) {
771 callback_pending->data = false;
772
773 // If we were called synchronously, return.
774 if (save_next_cookie_running->data) {
775 return;
776 }
777
778 // We were called asynchronously, so trigger the next save.
779 // We may have been canceled within OnSetCookie.
780 if (GetStatus().is_success()) {
781 SaveNextCookie();
782 } else {
783 NotifyCanceled();
784 }
785 }
786
FetchResponseCookies(std::vector<std::string> * cookies)787 void URLRequestHttpJob::FetchResponseCookies(
788 std::vector<std::string>* cookies) {
789 const std::string name = "Set-Cookie";
790 std::string value;
791
792 void* iter = NULL;
793 HttpResponseHeaders* headers = GetResponseHeaders();
794 while (headers->EnumerateHeader(&iter, name, &value)) {
795 if (!value.empty())
796 cookies->push_back(value);
797 }
798 }
799
800 // NOTE: |ProcessStrictTransportSecurityHeader| and
801 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
ProcessStrictTransportSecurityHeader()802 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
803 DCHECK(response_info_);
804 TransportSecurityState* security_state =
805 request_->context()->transport_security_state();
806 const SSLInfo& ssl_info = response_info_->ssl_info;
807
808 // Only accept HSTS headers on HTTPS connections that have no
809 // certificate errors.
810 if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
811 !security_state)
812 return;
813
814 // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
815 //
816 // If a UA receives more than one STS header field in a HTTP response
817 // message over secure transport, then the UA MUST process only the
818 // first such header field.
819 HttpResponseHeaders* headers = GetResponseHeaders();
820 std::string value;
821 if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
822 security_state->AddHSTSHeader(request_info_.url.host(), value);
823 }
824
ProcessPublicKeyPinsHeader()825 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
826 DCHECK(response_info_);
827 TransportSecurityState* security_state =
828 request_->context()->transport_security_state();
829 const SSLInfo& ssl_info = response_info_->ssl_info;
830
831 // Only accept HPKP headers on HTTPS connections that have no
832 // certificate errors.
833 if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
834 !security_state)
835 return;
836
837 // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
838 //
839 // If a UA receives more than one PKP header field in an HTTP
840 // response message over secure transport, then the UA MUST process
841 // only the first such header field.
842 HttpResponseHeaders* headers = GetResponseHeaders();
843 std::string value;
844 if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
845 security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
846 }
847
OnStartCompleted(int result)848 void URLRequestHttpJob::OnStartCompleted(int result) {
849 RecordTimer();
850
851 // If the request was destroyed, then there is no more work to do.
852 if (!request_)
853 return;
854
855 // If the transaction was destroyed, then the job was cancelled, and
856 // we can just ignore this notification.
857 if (!transaction_.get())
858 return;
859
860 receive_headers_end_ = base::TimeTicks::Now();
861
862 // Clear the IO_PENDING status
863 SetStatus(URLRequestStatus());
864
865 const URLRequestContext* context = request_->context();
866
867 if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
868 transaction_->GetResponseInfo() != NULL) {
869 FraudulentCertificateReporter* reporter =
870 context->fraudulent_certificate_reporter();
871 if (reporter != NULL) {
872 const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
873 bool sni_available = SSLConfigService::IsSNIAvailable(
874 context->ssl_config_service());
875 const std::string& host = request_->url().host();
876
877 reporter->SendReport(host, ssl_info, sni_available);
878 }
879 }
880
881 if (result == OK) {
882 scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
883 if (network_delegate()) {
884 // Note that |this| may not be deleted until
885 // |on_headers_received_callback_| or
886 // |NetworkDelegate::URLRequestDestroyed()| has been called.
887 OnCallToDelegate();
888 int error = network_delegate()->NotifyHeadersReceived(
889 request_,
890 on_headers_received_callback_,
891 headers.get(),
892 &override_response_headers_);
893 if (error != net::OK) {
894 if (error == net::ERR_IO_PENDING) {
895 awaiting_callback_ = true;
896 } else {
897 std::string source("delegate");
898 request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
899 NetLog::StringCallback("source",
900 &source));
901 OnCallToDelegateComplete();
902 NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
903 }
904 return;
905 }
906 }
907
908 SaveCookiesAndNotifyHeadersComplete(net::OK);
909 } else if (IsCertificateError(result)) {
910 // We encountered an SSL certificate error.
911 if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
912 result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
913 // These are hard failures. They're handled separately and don't have
914 // the correct cert status, so set it here.
915 SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
916 info.cert_status = MapNetErrorToCertStatus(result);
917 NotifySSLCertificateError(info, true);
918 } else {
919 // Maybe overridable, maybe not. Ask the delegate to decide.
920 TransportSecurityState::DomainState domain_state;
921 const URLRequestContext* context = request_->context();
922 const bool fatal = context->transport_security_state() &&
923 context->transport_security_state()->GetDomainState(
924 request_info_.url.host(),
925 SSLConfigService::IsSNIAvailable(context->ssl_config_service()),
926 &domain_state) &&
927 domain_state.ShouldSSLErrorsBeFatal();
928 NotifySSLCertificateError(
929 transaction_->GetResponseInfo()->ssl_info, fatal);
930 }
931 } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
932 NotifyCertificateRequested(
933 transaction_->GetResponseInfo()->cert_request_info.get());
934 } else {
935 NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
936 }
937 }
938
OnHeadersReceivedCallback(int result)939 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
940 awaiting_callback_ = false;
941
942 // Check that there are no callbacks to already canceled requests.
943 DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
944
945 SaveCookiesAndNotifyHeadersComplete(result);
946 }
947
OnReadCompleted(int result)948 void URLRequestHttpJob::OnReadCompleted(int result) {
949 read_in_progress_ = false;
950
951 if (ShouldFixMismatchedContentLength(result))
952 result = OK;
953
954 if (result == OK) {
955 NotifyDone(URLRequestStatus());
956 } else if (result < 0) {
957 NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
958 } else {
959 // Clear the IO_PENDING status
960 SetStatus(URLRequestStatus());
961 }
962
963 NotifyReadComplete(result);
964 }
965
RestartTransactionWithAuth(const AuthCredentials & credentials)966 void URLRequestHttpJob::RestartTransactionWithAuth(
967 const AuthCredentials& credentials) {
968 auth_credentials_ = credentials;
969
970 // These will be reset in OnStartCompleted.
971 response_info_ = NULL;
972 receive_headers_end_ = base::TimeTicks();
973 response_cookies_.clear();
974
975 ResetTimer();
976
977 // Update the cookies, since the cookie store may have been updated from the
978 // headers in the 401/407. Since cookies were already appended to
979 // extra_headers, we need to strip them out before adding them again.
980 request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
981
982 AddCookieHeaderAndStart();
983 }
984
SetUpload(UploadDataStream * upload)985 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
986 DCHECK(!transaction_.get()) << "cannot change once started";
987 request_info_.upload_data_stream = upload;
988 }
989
SetExtraRequestHeaders(const HttpRequestHeaders & headers)990 void URLRequestHttpJob::SetExtraRequestHeaders(
991 const HttpRequestHeaders& headers) {
992 DCHECK(!transaction_.get()) << "cannot change once started";
993 request_info_.extra_headers.CopyFrom(headers);
994 }
995
GetLoadState() const996 LoadState URLRequestHttpJob::GetLoadState() const {
997 return transaction_.get() ?
998 transaction_->GetLoadState() : LOAD_STATE_IDLE;
999 }
1000
GetUploadProgress() const1001 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1002 return transaction_.get() ?
1003 transaction_->GetUploadProgress() : UploadProgress();
1004 }
1005
GetMimeType(std::string * mime_type) const1006 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1007 DCHECK(transaction_.get());
1008
1009 if (!response_info_)
1010 return false;
1011
1012 return GetResponseHeaders()->GetMimeType(mime_type);
1013 }
1014
GetCharset(std::string * charset)1015 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1016 DCHECK(transaction_.get());
1017
1018 if (!response_info_)
1019 return false;
1020
1021 return GetResponseHeaders()->GetCharset(charset);
1022 }
1023
GetResponseInfo(HttpResponseInfo * info)1024 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1025 DCHECK(request_);
1026 DCHECK(transaction_.get());
1027
1028 if (response_info_) {
1029 *info = *response_info_;
1030 if (override_response_headers_.get())
1031 info->headers = override_response_headers_;
1032 }
1033 }
1034
GetLoadTimingInfo(LoadTimingInfo * load_timing_info) const1035 void URLRequestHttpJob::GetLoadTimingInfo(
1036 LoadTimingInfo* load_timing_info) const {
1037 // If haven't made it far enough to receive any headers, don't return
1038 // anything. This makes for more consistent behavior in the case of errors.
1039 if (!transaction_ || receive_headers_end_.is_null())
1040 return;
1041 if (transaction_->GetLoadTimingInfo(load_timing_info))
1042 load_timing_info->receive_headers_end = receive_headers_end_;
1043 }
1044
GetResponseCookies(std::vector<std::string> * cookies)1045 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1046 DCHECK(transaction_.get());
1047
1048 if (!response_info_)
1049 return false;
1050
1051 // TODO(darin): Why are we extracting response cookies again? Perhaps we
1052 // should just leverage response_cookies_.
1053
1054 cookies->clear();
1055 FetchResponseCookies(cookies);
1056 return true;
1057 }
1058
GetResponseCode() const1059 int URLRequestHttpJob::GetResponseCode() const {
1060 DCHECK(transaction_.get());
1061
1062 if (!response_info_)
1063 return -1;
1064
1065 return GetResponseHeaders()->response_code();
1066 }
1067
SetupFilter() const1068 Filter* URLRequestHttpJob::SetupFilter() const {
1069 DCHECK(transaction_.get());
1070 if (!response_info_)
1071 return NULL;
1072
1073 std::vector<Filter::FilterType> encoding_types;
1074 std::string encoding_type;
1075 HttpResponseHeaders* headers = GetResponseHeaders();
1076 void* iter = NULL;
1077 while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1078 encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1079 }
1080
1081 if (filter_context_->IsSdchResponse()) {
1082 // We are wary of proxies that discard or damage SDCH encoding. If a server
1083 // explicitly states that this is not SDCH content, then we can correct our
1084 // assumption that this is an SDCH response, and avoid the need to recover
1085 // as though the content is corrupted (when we discover it is not SDCH
1086 // encoded).
1087 std::string sdch_response_status;
1088 iter = NULL;
1089 while (headers->EnumerateHeader(&iter, "X-Sdch-Encode",
1090 &sdch_response_status)) {
1091 if (sdch_response_status == "0") {
1092 filter_context_->ResetSdchResponseToFalse();
1093 break;
1094 }
1095 }
1096 }
1097
1098 // Even if encoding types are empty, there is a chance that we need to add
1099 // some decoding, as some proxies strip encoding completely. In such cases,
1100 // we may need to add (for example) SDCH filtering (when the context suggests
1101 // it is appropriate).
1102 Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1103
1104 return !encoding_types.empty()
1105 ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1106 }
1107
IsSafeRedirect(const GURL & location)1108 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1109 // HTTP is always safe.
1110 // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1111 if (location.is_valid() &&
1112 (location.scheme() == "http" || location.scheme() == "https")) {
1113 return true;
1114 }
1115 // Query URLRequestJobFactory as to whether |location| would be safe to
1116 // redirect to.
1117 return request_->context()->job_factory() &&
1118 request_->context()->job_factory()->IsSafeRedirectTarget(location);
1119 }
1120
NeedsAuth()1121 bool URLRequestHttpJob::NeedsAuth() {
1122 int code = GetResponseCode();
1123 if (code == -1)
1124 return false;
1125
1126 // Check if we need either Proxy or WWW Authentication. This could happen
1127 // because we either provided no auth info, or provided incorrect info.
1128 switch (code) {
1129 case 407:
1130 if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1131 return false;
1132 proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1133 return true;
1134 case 401:
1135 if (server_auth_state_ == AUTH_STATE_CANCELED)
1136 return false;
1137 server_auth_state_ = AUTH_STATE_NEED_AUTH;
1138 return true;
1139 }
1140 return false;
1141 }
1142
GetAuthChallengeInfo(scoped_refptr<AuthChallengeInfo> * result)1143 void URLRequestHttpJob::GetAuthChallengeInfo(
1144 scoped_refptr<AuthChallengeInfo>* result) {
1145 DCHECK(transaction_.get());
1146 DCHECK(response_info_);
1147
1148 // sanity checks:
1149 DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1150 server_auth_state_ == AUTH_STATE_NEED_AUTH);
1151 DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1152 (GetResponseHeaders()->response_code() ==
1153 HTTP_PROXY_AUTHENTICATION_REQUIRED));
1154
1155 *result = response_info_->auth_challenge;
1156 }
1157
SetAuth(const AuthCredentials & credentials)1158 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1159 DCHECK(transaction_.get());
1160
1161 // Proxy gets set first, then WWW.
1162 if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1163 proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1164 } else {
1165 DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1166 server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1167 }
1168
1169 RestartTransactionWithAuth(credentials);
1170 }
1171
CancelAuth()1172 void URLRequestHttpJob::CancelAuth() {
1173 // Proxy gets set first, then WWW.
1174 if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1175 proxy_auth_state_ = AUTH_STATE_CANCELED;
1176 } else {
1177 DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1178 server_auth_state_ = AUTH_STATE_CANCELED;
1179 }
1180
1181 // These will be reset in OnStartCompleted.
1182 response_info_ = NULL;
1183 receive_headers_end_ = base::TimeTicks::Now();
1184 response_cookies_.clear();
1185
1186 ResetTimer();
1187
1188 // OK, let the consumer read the error page...
1189 //
1190 // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1191 // which will cause the consumer to receive OnResponseStarted instead of
1192 // OnAuthRequired.
1193 //
1194 // We have to do this via InvokeLater to avoid "recursing" the consumer.
1195 //
1196 base::MessageLoop::current()->PostTask(
1197 FROM_HERE,
1198 base::Bind(&URLRequestHttpJob::OnStartCompleted,
1199 weak_factory_.GetWeakPtr(), OK));
1200 }
1201
ContinueWithCertificate(X509Certificate * client_cert)1202 void URLRequestHttpJob::ContinueWithCertificate(
1203 X509Certificate* client_cert) {
1204 DCHECK(transaction_.get());
1205
1206 DCHECK(!response_info_) << "should not have a response yet";
1207 receive_headers_end_ = base::TimeTicks();
1208
1209 ResetTimer();
1210
1211 // No matter what, we want to report our status as IO pending since we will
1212 // be notifying our consumer asynchronously via OnStartCompleted.
1213 SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1214
1215 int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1216 if (rv == ERR_IO_PENDING)
1217 return;
1218
1219 // The transaction started synchronously, but we need to notify the
1220 // URLRequest delegate via the message loop.
1221 base::MessageLoop::current()->PostTask(
1222 FROM_HERE,
1223 base::Bind(&URLRequestHttpJob::OnStartCompleted,
1224 weak_factory_.GetWeakPtr(), rv));
1225 }
1226
ContinueDespiteLastError()1227 void URLRequestHttpJob::ContinueDespiteLastError() {
1228 // If the transaction was destroyed, then the job was cancelled.
1229 if (!transaction_.get())
1230 return;
1231
1232 DCHECK(!response_info_) << "should not have a response yet";
1233 receive_headers_end_ = base::TimeTicks();
1234
1235 ResetTimer();
1236
1237 // No matter what, we want to report our status as IO pending since we will
1238 // be notifying our consumer asynchronously via OnStartCompleted.
1239 SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1240
1241 int rv = transaction_->RestartIgnoringLastError(start_callback_);
1242 if (rv == ERR_IO_PENDING)
1243 return;
1244
1245 // The transaction started synchronously, but we need to notify the
1246 // URLRequest delegate via the message loop.
1247 base::MessageLoop::current()->PostTask(
1248 FROM_HERE,
1249 base::Bind(&URLRequestHttpJob::OnStartCompleted,
1250 weak_factory_.GetWeakPtr(), rv));
1251 }
1252
ShouldFixMismatchedContentLength(int rv) const1253 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1254 // Some servers send the body compressed, but specify the content length as
1255 // the uncompressed size. Although this violates the HTTP spec we want to
1256 // support it (as IE and FireFox do), but *only* for an exact match.
1257 // See http://crbug.com/79694.
1258 if (rv == net::ERR_CONTENT_LENGTH_MISMATCH ||
1259 rv == net::ERR_INCOMPLETE_CHUNKED_ENCODING) {
1260 if (request_ && request_->response_headers()) {
1261 int64 expected_length = request_->response_headers()->GetContentLength();
1262 VLOG(1) << __FUNCTION__ << "() "
1263 << "\"" << request_->url().spec() << "\""
1264 << " content-length = " << expected_length
1265 << " pre total = " << prefilter_bytes_read()
1266 << " post total = " << postfilter_bytes_read();
1267 if (postfilter_bytes_read() == expected_length) {
1268 // Clear the error.
1269 return true;
1270 }
1271 }
1272 }
1273 return false;
1274 }
1275
ReadRawData(IOBuffer * buf,int buf_size,int * bytes_read)1276 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1277 int* bytes_read) {
1278 DCHECK_NE(buf_size, 0);
1279 DCHECK(bytes_read);
1280 DCHECK(!read_in_progress_);
1281
1282 int rv = transaction_->Read(
1283 buf, buf_size,
1284 base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1285
1286 if (ShouldFixMismatchedContentLength(rv))
1287 rv = 0;
1288
1289 if (rv >= 0) {
1290 *bytes_read = rv;
1291 if (!rv)
1292 DoneWithRequest(FINISHED);
1293 return true;
1294 }
1295
1296 if (rv == ERR_IO_PENDING) {
1297 read_in_progress_ = true;
1298 SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1299 } else {
1300 NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1301 }
1302
1303 return false;
1304 }
1305
StopCaching()1306 void URLRequestHttpJob::StopCaching() {
1307 if (transaction_.get())
1308 transaction_->StopCaching();
1309 }
1310
GetFullRequestHeaders(HttpRequestHeaders * headers) const1311 bool URLRequestHttpJob::GetFullRequestHeaders(
1312 HttpRequestHeaders* headers) const {
1313 if (!transaction_)
1314 return false;
1315
1316 return transaction_->GetFullRequestHeaders(headers);
1317 }
1318
DoneReading()1319 void URLRequestHttpJob::DoneReading() {
1320 if (transaction_.get())
1321 transaction_->DoneReading();
1322 DoneWithRequest(FINISHED);
1323 }
1324
GetSocketAddress() const1325 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1326 return response_info_ ? response_info_->socket_address : HostPortPair();
1327 }
1328
RecordTimer()1329 void URLRequestHttpJob::RecordTimer() {
1330 if (request_creation_time_.is_null()) {
1331 NOTREACHED()
1332 << "The same transaction shouldn't start twice without new timing.";
1333 return;
1334 }
1335
1336 base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1337 request_creation_time_ = base::Time();
1338
1339 UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1340 }
1341
ResetTimer()1342 void URLRequestHttpJob::ResetTimer() {
1343 if (!request_creation_time_.is_null()) {
1344 NOTREACHED()
1345 << "The timer was reset before it was recorded.";
1346 return;
1347 }
1348 request_creation_time_ = base::Time::Now();
1349 }
1350
UpdatePacketReadTimes()1351 void URLRequestHttpJob::UpdatePacketReadTimes() {
1352 if (!packet_timing_enabled_)
1353 return;
1354
1355 if (filter_input_byte_count() <= bytes_observed_in_packets_) {
1356 DCHECK_EQ(filter_input_byte_count(), bytes_observed_in_packets_);
1357 return; // No new bytes have arrived.
1358 }
1359
1360 final_packet_time_ = base::Time::Now();
1361 if (!bytes_observed_in_packets_)
1362 request_time_snapshot_ = request_ ? request_->request_time() : base::Time();
1363
1364 bytes_observed_in_packets_ = filter_input_byte_count();
1365 }
1366
RecordPacketStats(FilterContext::StatisticSelector statistic) const1367 void URLRequestHttpJob::RecordPacketStats(
1368 FilterContext::StatisticSelector statistic) const {
1369 if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1370 return;
1371
1372 base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1373 switch (statistic) {
1374 case FilterContext::SDCH_DECODE: {
1375 UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1376 static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1377 return;
1378 }
1379 case FilterContext::SDCH_PASSTHROUGH: {
1380 // Despite advertising a dictionary, we handled non-sdch compressed
1381 // content.
1382 return;
1383 }
1384
1385 case FilterContext::SDCH_EXPERIMENT_DECODE: {
1386 UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment2_Decode",
1387 duration,
1388 base::TimeDelta::FromMilliseconds(20),
1389 base::TimeDelta::FromMinutes(10), 100);
1390 return;
1391 }
1392 case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1393 UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment2_Holdback",
1394 duration,
1395 base::TimeDelta::FromMilliseconds(20),
1396 base::TimeDelta::FromMinutes(10), 100);
1397 return;
1398 }
1399 default:
1400 NOTREACHED();
1401 return;
1402 }
1403 }
1404
1405 // The common type of histogram we use for all compression-tracking histograms.
1406 #define COMPRESSION_HISTOGRAM(name, sample) \
1407 do { \
1408 UMA_HISTOGRAM_CUSTOM_COUNTS("Net.Compress." name, sample, \
1409 500, 1000000, 100); \
1410 } while (0)
1411
RecordCompressionHistograms()1412 void URLRequestHttpJob::RecordCompressionHistograms() {
1413 DCHECK(request_);
1414 if (!request_)
1415 return;
1416
1417 if (is_cached_content_ || // Don't record cached content
1418 !GetStatus().is_success() || // Don't record failed content
1419 !IsCompressibleContent() || // Only record compressible content
1420 !prefilter_bytes_read()) // Zero-byte responses aren't useful.
1421 return;
1422
1423 // Miniature requests aren't really compressible. Don't count them.
1424 const int kMinSize = 16;
1425 if (prefilter_bytes_read() < kMinSize)
1426 return;
1427
1428 // Only record for http or https urls.
1429 bool is_http = request_->url().SchemeIs("http");
1430 bool is_https = request_->url().SchemeIs("https");
1431 if (!is_http && !is_https)
1432 return;
1433
1434 int compressed_B = prefilter_bytes_read();
1435 int decompressed_B = postfilter_bytes_read();
1436 bool was_filtered = HasFilter();
1437
1438 // We want to record how often downloaded resources are compressed.
1439 // But, we recognize that different protocols may have different
1440 // properties. So, for each request, we'll put it into one of 3
1441 // groups:
1442 // a) SSL resources
1443 // Proxies cannot tamper with compression headers with SSL.
1444 // b) Non-SSL, loaded-via-proxy resources
1445 // In this case, we know a proxy might have interfered.
1446 // c) Non-SSL, loaded-without-proxy resources
1447 // In this case, we know there was no explicit proxy. However,
1448 // it is possible that a transparent proxy was still interfering.
1449 //
1450 // For each group, we record the same 3 histograms.
1451
1452 if (is_https) {
1453 if (was_filtered) {
1454 COMPRESSION_HISTOGRAM("SSL.BytesBeforeCompression", compressed_B);
1455 COMPRESSION_HISTOGRAM("SSL.BytesAfterCompression", decompressed_B);
1456 } else {
1457 COMPRESSION_HISTOGRAM("SSL.ShouldHaveBeenCompressed", decompressed_B);
1458 }
1459 return;
1460 }
1461
1462 if (request_->was_fetched_via_proxy()) {
1463 if (was_filtered) {
1464 COMPRESSION_HISTOGRAM("Proxy.BytesBeforeCompression", compressed_B);
1465 COMPRESSION_HISTOGRAM("Proxy.BytesAfterCompression", decompressed_B);
1466 } else {
1467 COMPRESSION_HISTOGRAM("Proxy.ShouldHaveBeenCompressed", decompressed_B);
1468 }
1469 return;
1470 }
1471
1472 if (was_filtered) {
1473 COMPRESSION_HISTOGRAM("NoProxy.BytesBeforeCompression", compressed_B);
1474 COMPRESSION_HISTOGRAM("NoProxy.BytesAfterCompression", decompressed_B);
1475 } else {
1476 COMPRESSION_HISTOGRAM("NoProxy.ShouldHaveBeenCompressed", decompressed_B);
1477 }
1478 }
1479
IsCompressibleContent() const1480 bool URLRequestHttpJob::IsCompressibleContent() const {
1481 std::string mime_type;
1482 return GetMimeType(&mime_type) &&
1483 (IsSupportedJavascriptMimeType(mime_type.c_str()) ||
1484 IsSupportedNonImageMimeType(mime_type.c_str()));
1485 }
1486
RecordPerfHistograms(CompletionCause reason)1487 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1488 if (start_time_.is_null())
1489 return;
1490
1491 base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1492 UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1493
1494 if (reason == FINISHED) {
1495 UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1496 } else {
1497 UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1498 }
1499
1500 if (response_info_) {
1501 if (response_info_->was_cached) {
1502 UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1503 } else {
1504 UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1505 }
1506 }
1507
1508 if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1509 UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1510 prefilter_bytes_read());
1511
1512 start_time_ = base::TimeTicks();
1513 }
1514
DoneWithRequest(CompletionCause reason)1515 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1516 if (done_)
1517 return;
1518 done_ = true;
1519 RecordPerfHistograms(reason);
1520 if (reason == FINISHED) {
1521 request_->set_received_response_content_length(prefilter_bytes_read());
1522 RecordCompressionHistograms();
1523 }
1524 }
1525
GetResponseHeaders() const1526 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1527 DCHECK(transaction_.get());
1528 DCHECK(transaction_->GetResponseInfo());
1529 return override_response_headers_.get() ?
1530 override_response_headers_.get() :
1531 transaction_->GetResponseInfo()->headers.get();
1532 }
1533
NotifyURLRequestDestroyed()1534 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1535 awaiting_callback_ = false;
1536 }
1537
OnDetachRequest()1538 void URLRequestHttpJob::OnDetachRequest() {
1539 http_transaction_delegate_->OnDetachRequest();
1540 }
1541
1542 } // namespace net
1543