• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "net/url_request/url_request_http_job.h"
6 
7 #include "base/base_switches.h"
8 #include "base/bind.h"
9 #include "base/bind_helpers.h"
10 #include "base/command_line.h"
11 #include "base/compiler_specific.h"
12 #include "base/file_version_info.h"
13 #include "base/message_loop/message_loop.h"
14 #include "base/metrics/field_trial.h"
15 #include "base/metrics/histogram.h"
16 #include "base/rand_util.h"
17 #include "base/strings/string_util.h"
18 #include "base/time/time.h"
19 #include "net/base/host_port_pair.h"
20 #include "net/base/load_flags.h"
21 #include "net/base/mime_util.h"
22 #include "net/base/net_errors.h"
23 #include "net/base/net_util.h"
24 #include "net/base/network_delegate.h"
25 #include "net/base/sdch_manager.h"
26 #include "net/cert/cert_status_flags.h"
27 #include "net/cookies/cookie_store.h"
28 #include "net/http/http_content_disposition.h"
29 #include "net/http/http_network_session.h"
30 #include "net/http/http_request_headers.h"
31 #include "net/http/http_response_headers.h"
32 #include "net/http/http_response_info.h"
33 #include "net/http/http_status_code.h"
34 #include "net/http/http_transaction.h"
35 #include "net/http/http_transaction_factory.h"
36 #include "net/http/http_util.h"
37 #include "net/proxy/proxy_info.h"
38 #include "net/ssl/ssl_cert_request_info.h"
39 #include "net/ssl/ssl_config_service.h"
40 #include "net/url_request/fraudulent_certificate_reporter.h"
41 #include "net/url_request/http_user_agent_settings.h"
42 #include "net/url_request/url_request.h"
43 #include "net/url_request/url_request_context.h"
44 #include "net/url_request/url_request_error_job.h"
45 #include "net/url_request/url_request_job_factory.h"
46 #include "net/url_request/url_request_redirect_job.h"
47 #include "net/url_request/url_request_throttler_header_adapter.h"
48 #include "net/url_request/url_request_throttler_manager.h"
49 #include "net/websockets/websocket_handshake_stream_base.h"
50 
51 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
52 
53 namespace net {
54 
55 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
56  public:
57   explicit HttpFilterContext(URLRequestHttpJob* job);
58   virtual ~HttpFilterContext();
59 
60   // FilterContext implementation.
61   virtual bool GetMimeType(std::string* mime_type) const OVERRIDE;
62   virtual bool GetURL(GURL* gurl) const OVERRIDE;
63   virtual bool GetContentDisposition(std::string* disposition) const OVERRIDE;
64   virtual base::Time GetRequestTime() const OVERRIDE;
65   virtual bool IsCachedContent() const OVERRIDE;
66   virtual bool IsDownload() const OVERRIDE;
67   virtual bool SdchResponseExpected() const OVERRIDE;
68   virtual int64 GetByteReadCount() const OVERRIDE;
69   virtual int GetResponseCode() const OVERRIDE;
70   virtual const URLRequestContext* GetURLRequestContext() const OVERRIDE;
71   virtual void RecordPacketStats(StatisticSelector statistic) const OVERRIDE;
72 
73   // Method to allow us to reset filter context for a response that should have
74   // been SDCH encoded when there is an update due to an explicit HTTP header.
75   void ResetSdchResponseToFalse();
76 
77  private:
78   URLRequestHttpJob* job_;
79 
80   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
81 };
82 
HttpFilterContext(URLRequestHttpJob * job)83 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
84     : job_(job) {
85   DCHECK(job_);
86 }
87 
~HttpFilterContext()88 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
89 }
90 
GetMimeType(std::string * mime_type) const91 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
92     std::string* mime_type) const {
93   return job_->GetMimeType(mime_type);
94 }
95 
GetURL(GURL * gurl) const96 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
97   if (!job_->request())
98     return false;
99   *gurl = job_->request()->url();
100   return true;
101 }
102 
GetContentDisposition(std::string * disposition) const103 bool URLRequestHttpJob::HttpFilterContext::GetContentDisposition(
104     std::string* disposition) const {
105   HttpResponseHeaders* headers = job_->GetResponseHeaders();
106   void *iter = NULL;
107   return headers->EnumerateHeader(&iter, "Content-Disposition", disposition);
108 }
109 
GetRequestTime() const110 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
111   return job_->request() ? job_->request()->request_time() : base::Time();
112 }
113 
IsCachedContent() const114 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
115   return job_->is_cached_content_;
116 }
117 
IsDownload() const118 bool URLRequestHttpJob::HttpFilterContext::IsDownload() const {
119   return (job_->request_info_.load_flags & LOAD_IS_DOWNLOAD) != 0;
120 }
121 
ResetSdchResponseToFalse()122 void URLRequestHttpJob::HttpFilterContext::ResetSdchResponseToFalse() {
123   DCHECK(job_->sdch_dictionary_advertised_);
124   job_->sdch_dictionary_advertised_ = false;
125 }
126 
SdchResponseExpected() const127 bool URLRequestHttpJob::HttpFilterContext::SdchResponseExpected() const {
128   return job_->sdch_dictionary_advertised_;
129 }
130 
GetByteReadCount() const131 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
132   return job_->filter_input_byte_count();
133 }
134 
GetResponseCode() const135 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
136   return job_->GetResponseCode();
137 }
138 
139 const URLRequestContext*
GetURLRequestContext() const140 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
141   return job_->request() ? job_->request()->context() : NULL;
142 }
143 
RecordPacketStats(StatisticSelector statistic) const144 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
145     StatisticSelector statistic) const {
146   job_->RecordPacketStats(statistic);
147 }
148 
149 // TODO(darin): make sure the port blocking code is not lost
150 // static
Factory(URLRequest * request,NetworkDelegate * network_delegate,const std::string & scheme)151 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
152                                           NetworkDelegate* network_delegate,
153                                           const std::string& scheme) {
154   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
155          scheme == "wss");
156 
157   if (!request->context()->http_transaction_factory()) {
158     NOTREACHED() << "requires a valid context";
159     return new URLRequestErrorJob(
160         request, network_delegate, ERR_INVALID_ARGUMENT);
161   }
162 
163   GURL redirect_url;
164   if (request->GetHSTSRedirect(&redirect_url)) {
165     return new URLRequestRedirectJob(
166         request, network_delegate, redirect_url,
167         // Use status code 307 to preserve the method, so POST requests work.
168         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
169   }
170   return new URLRequestHttpJob(request,
171                                network_delegate,
172                                request->context()->http_user_agent_settings());
173 }
174 
URLRequestHttpJob(URLRequest * request,NetworkDelegate * network_delegate,const HttpUserAgentSettings * http_user_agent_settings)175 URLRequestHttpJob::URLRequestHttpJob(
176     URLRequest* request,
177     NetworkDelegate* network_delegate,
178     const HttpUserAgentSettings* http_user_agent_settings)
179     : URLRequestJob(request, network_delegate),
180       priority_(DEFAULT_PRIORITY),
181       response_info_(NULL),
182       response_cookies_save_index_(0),
183       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
184       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
185       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
186                                  base::Unretained(this))),
187       notify_before_headers_sent_callback_(
188           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
189                      base::Unretained(this))),
190       read_in_progress_(false),
191       throttling_entry_(NULL),
192       sdch_dictionary_advertised_(false),
193       sdch_test_activated_(false),
194       sdch_test_control_(false),
195       is_cached_content_(false),
196       request_creation_time_(),
197       packet_timing_enabled_(false),
198       done_(false),
199       bytes_observed_in_packets_(0),
200       request_time_snapshot_(),
201       final_packet_time_(),
202       filter_context_(new HttpFilterContext(this)),
203       on_headers_received_callback_(
204           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
205                      base::Unretained(this))),
206       awaiting_callback_(false),
207       http_user_agent_settings_(http_user_agent_settings),
208       weak_factory_(this) {
209   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
210   if (manager)
211     throttling_entry_ = manager->RegisterRequestUrl(request->url());
212 
213   ResetTimer();
214 }
215 
~URLRequestHttpJob()216 URLRequestHttpJob::~URLRequestHttpJob() {
217   CHECK(!awaiting_callback_);
218 
219   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
220   if (!is_cached_content_) {
221     if (sdch_test_control_)
222       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
223     if (sdch_test_activated_)
224       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
225   }
226   // Make sure SDCH filters are told to emit histogram data while
227   // filter_context_ is still alive.
228   DestroyFilters();
229 
230   DoneWithRequest(ABORTED);
231 }
232 
SetPriority(RequestPriority priority)233 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
234   priority_ = priority;
235   if (transaction_)
236     transaction_->SetPriority(priority_);
237 }
238 
Start()239 void URLRequestHttpJob::Start() {
240   DCHECK(!transaction_.get());
241 
242   // URLRequest::SetReferrer ensures that we do not send username and password
243   // fields in the referrer.
244   GURL referrer(request_->referrer());
245 
246   request_info_.url = request_->url();
247   request_info_.method = request_->method();
248   request_info_.load_flags = request_->load_flags();
249   // Enable privacy mode if cookie settings or flags tell us not send or
250   // save cookies.
251   bool enable_privacy_mode =
252       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
253       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
254       CanEnablePrivacyMode();
255   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
256   // to send previously saved cookies.
257   request_info_.privacy_mode = enable_privacy_mode ?
258       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
259 
260   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
261   // from overriding headers that are controlled using other means. Otherwise a
262   // plugin could set a referrer although sending the referrer is inhibited.
263   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
264 
265   // Our consumer should have made sure that this is a safe referrer.  See for
266   // instance WebCore::FrameLoader::HideReferrer.
267   if (referrer.is_valid()) {
268     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
269                                           referrer.spec());
270   }
271 
272   request_info_.extra_headers.SetHeaderIfMissing(
273       HttpRequestHeaders::kUserAgent,
274       http_user_agent_settings_ ?
275           http_user_agent_settings_->GetUserAgent() : std::string());
276 
277   AddExtraHeaders();
278   AddCookieHeaderAndStart();
279 }
280 
Kill()281 void URLRequestHttpJob::Kill() {
282   if (!transaction_.get())
283     return;
284 
285   weak_factory_.InvalidateWeakPtrs();
286   DestroyTransaction();
287   URLRequestJob::Kill();
288 }
289 
NotifyBeforeSendProxyHeadersCallback(const ProxyInfo & proxy_info,HttpRequestHeaders * request_headers)290 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
291     const ProxyInfo& proxy_info,
292     HttpRequestHeaders* request_headers) {
293   DCHECK(request_headers);
294   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
295   if (network_delegate()) {
296     network_delegate()->NotifyBeforeSendProxyHeaders(
297         request_,
298         proxy_info,
299         request_headers);
300   }
301 }
302 
NotifyHeadersComplete()303 void URLRequestHttpJob::NotifyHeadersComplete() {
304   DCHECK(!response_info_);
305 
306   response_info_ = transaction_->GetResponseInfo();
307 
308   // Save boolean, as we'll need this info at destruction time, and filters may
309   // also need this info.
310   is_cached_content_ = response_info_->was_cached;
311 
312   if (!is_cached_content_ && throttling_entry_.get()) {
313     URLRequestThrottlerHeaderAdapter response_adapter(GetResponseHeaders());
314     throttling_entry_->UpdateWithResponse(request_info_.url.host(),
315                                           &response_adapter);
316   }
317 
318   // The ordering of these calls is not important.
319   ProcessStrictTransportSecurityHeader();
320   ProcessPublicKeyPinsHeader();
321 
322   SdchManager* sdch_manager(request()->context()->sdch_manager());
323   if (sdch_manager && sdch_manager->IsInSupportedDomain(request_->url())) {
324     const std::string name = "Get-Dictionary";
325     std::string url_text;
326     void* iter = NULL;
327     // TODO(jar): We need to not fetch dictionaries the first time they are
328     // seen, but rather wait until we can justify their usefulness.
329     // For now, we will only fetch the first dictionary, which will at least
330     // require multiple suggestions before we get additional ones for this site.
331     // Eventually we should wait until a dictionary is requested several times
332     // before we even download it (so that we don't waste memory or bandwidth).
333     if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
334       // Resolve suggested URL relative to request url.
335       GURL sdch_dictionary_url = request_->url().Resolve(url_text);
336       if (sdch_dictionary_url.is_valid()) {
337         sdch_manager->FetchDictionary(request_->url(), sdch_dictionary_url);
338       }
339     }
340   }
341 
342   // The HTTP transaction may be restarted several times for the purposes
343   // of sending authorization information. Each time it restarts, we get
344   // notified of the headers completion so that we can update the cookie store.
345   if (transaction_->IsReadyToRestartForAuth()) {
346     DCHECK(!response_info_->auth_challenge.get());
347     // TODO(battre): This breaks the webrequest API for
348     // URLRequestTestHTTP.BasicAuthWithCookies
349     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
350     // occurs.
351     RestartTransactionWithAuth(AuthCredentials());
352     return;
353   }
354 
355   URLRequestJob::NotifyHeadersComplete();
356 }
357 
NotifyDone(const URLRequestStatus & status)358 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
359   DoneWithRequest(FINISHED);
360   URLRequestJob::NotifyDone(status);
361 }
362 
DestroyTransaction()363 void URLRequestHttpJob::DestroyTransaction() {
364   DCHECK(transaction_.get());
365 
366   DoneWithRequest(ABORTED);
367   transaction_.reset();
368   response_info_ = NULL;
369   receive_headers_end_ = base::TimeTicks();
370 }
371 
StartTransaction()372 void URLRequestHttpJob::StartTransaction() {
373   if (network_delegate()) {
374     OnCallToDelegate();
375     int rv = network_delegate()->NotifyBeforeSendHeaders(
376         request_, notify_before_headers_sent_callback_,
377         &request_info_.extra_headers);
378     // If an extension blocks the request, we rely on the callback to
379     // MaybeStartTransactionInternal().
380     if (rv == ERR_IO_PENDING)
381       return;
382     MaybeStartTransactionInternal(rv);
383     return;
384   }
385   StartTransactionInternal();
386 }
387 
NotifyBeforeSendHeadersCallback(int result)388 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
389   // Check that there are no callbacks to already canceled requests.
390   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
391 
392   MaybeStartTransactionInternal(result);
393 }
394 
MaybeStartTransactionInternal(int result)395 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
396   OnCallToDelegateComplete();
397   if (result == OK) {
398     StartTransactionInternal();
399   } else {
400     std::string source("delegate");
401     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
402                                  NetLog::StringCallback("source", &source));
403     NotifyCanceled();
404     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
405   }
406 }
407 
StartTransactionInternal()408 void URLRequestHttpJob::StartTransactionInternal() {
409   // NOTE: This method assumes that request_info_ is already setup properly.
410 
411   // If we already have a transaction, then we should restart the transaction
412   // with auth provided by auth_credentials_.
413 
414   int rv;
415 
416   if (network_delegate()) {
417     network_delegate()->NotifySendHeaders(
418         request_, request_info_.extra_headers);
419   }
420 
421   if (transaction_.get()) {
422     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
423     auth_credentials_ = AuthCredentials();
424   } else {
425     DCHECK(request_->context()->http_transaction_factory());
426 
427     rv = request_->context()->http_transaction_factory()->CreateTransaction(
428         priority_, &transaction_);
429 
430     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
431       base::SupportsUserData::Data* data = request_->GetUserData(
432           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
433       if (data) {
434         transaction_->SetWebSocketHandshakeStreamCreateHelper(
435             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
436       } else {
437         rv = ERR_DISALLOWED_URL_SCHEME;
438       }
439     }
440 
441     if (rv == OK) {
442       transaction_->SetBeforeNetworkStartCallback(
443           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
444                      base::Unretained(this)));
445       transaction_->SetBeforeProxyHeadersSentCallback(
446           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
447                      base::Unretained(this)));
448 
449       if (!throttling_entry_.get() ||
450           !throttling_entry_->ShouldRejectRequest(*request_,
451                                                   network_delegate())) {
452         rv = transaction_->Start(
453             &request_info_, start_callback_, request_->net_log());
454         start_time_ = base::TimeTicks::Now();
455       } else {
456         // Special error code for the exponential back-off module.
457         rv = ERR_TEMPORARILY_THROTTLED;
458       }
459     }
460   }
461 
462   if (rv == ERR_IO_PENDING)
463     return;
464 
465   // The transaction started synchronously, but we need to notify the
466   // URLRequest delegate via the message loop.
467   base::MessageLoop::current()->PostTask(
468       FROM_HERE,
469       base::Bind(&URLRequestHttpJob::OnStartCompleted,
470                  weak_factory_.GetWeakPtr(), rv));
471 }
472 
AddExtraHeaders()473 void URLRequestHttpJob::AddExtraHeaders() {
474   SdchManager* sdch_manager = request()->context()->sdch_manager();
475 
476   // Supply Accept-Encoding field only if it is not already provided.
477   // It should be provided IF the content is known to have restrictions on
478   // potential encoding, such as streaming multi-media.
479   // For details see bug 47381.
480   // TODO(jar, enal): jpeg files etc. should set up a request header if
481   // possible. Right now it is done only by buffered_resource_loader and
482   // simple_data_source.
483   if (!request_info_.extra_headers.HasHeader(
484       HttpRequestHeaders::kAcceptEncoding)) {
485     bool advertise_sdch = sdch_manager &&
486         // We don't support SDCH responses to POST as there is a possibility
487         // of having SDCH encoded responses returned (e.g. by the cache)
488         // which we cannot decode, and in those situations, we will need
489         // to retransmit the request without SDCH, which is illegal for a POST.
490         request()->method() != "POST" &&
491         sdch_manager->IsInSupportedDomain(request_->url());
492     std::string avail_dictionaries;
493     if (advertise_sdch) {
494       sdch_manager->GetAvailDictionaryList(request_->url(),
495                                            &avail_dictionaries);
496 
497       // The AllowLatencyExperiment() is only true if we've successfully done a
498       // full SDCH compression recently in this browser session for this host.
499       // Note that for this path, there might be no applicable dictionaries,
500       // and hence we can't participate in the experiment.
501       if (!avail_dictionaries.empty() &&
502           sdch_manager->AllowLatencyExperiment(request_->url())) {
503         // We are participating in the test (or control), and hence we'll
504         // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
505         // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
506         packet_timing_enabled_ = true;
507         if (base::RandDouble() < .01) {
508           sdch_test_control_ = true;  // 1% probability.
509           advertise_sdch = false;
510         } else {
511           sdch_test_activated_ = true;
512         }
513       }
514     }
515 
516     // Supply Accept-Encoding headers first so that it is more likely that they
517     // will be in the first transmitted packet.  This can sometimes make it
518     // easier to filter and analyze the streams to assure that a proxy has not
519     // damaged these headers.  Some proxies deliberately corrupt Accept-Encoding
520     // headers.
521     if (!advertise_sdch) {
522       // Tell the server what compression formats we support (other than SDCH).
523       request_info_.extra_headers.SetHeader(
524           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
525     } else {
526       // Include SDCH in acceptable list.
527       request_info_.extra_headers.SetHeader(
528           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
529       if (!avail_dictionaries.empty()) {
530         request_info_.extra_headers.SetHeader(
531             kAvailDictionaryHeader,
532             avail_dictionaries);
533         sdch_dictionary_advertised_ = true;
534         // Since we're tagging this transaction as advertising a dictionary,
535         // we'll definitely employ an SDCH filter (or tentative sdch filter)
536         // when we get a response.  When done, we'll record histograms via
537         // SDCH_DECODE or SDCH_PASSTHROUGH.  Hence we need to record packet
538         // arrival times.
539         packet_timing_enabled_ = true;
540       }
541     }
542   }
543 
544   if (http_user_agent_settings_) {
545     // Only add default Accept-Language if the request didn't have it
546     // specified.
547     std::string accept_language =
548         http_user_agent_settings_->GetAcceptLanguage();
549     if (!accept_language.empty()) {
550       request_info_.extra_headers.SetHeaderIfMissing(
551           HttpRequestHeaders::kAcceptLanguage,
552           accept_language);
553     }
554   }
555 }
556 
AddCookieHeaderAndStart()557 void URLRequestHttpJob::AddCookieHeaderAndStart() {
558   // No matter what, we want to report our status as IO pending since we will
559   // be notifying our consumer asynchronously via OnStartCompleted.
560   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
561 
562   // If the request was destroyed, then there is no more work to do.
563   if (!request_)
564     return;
565 
566   CookieStore* cookie_store = GetCookieStore();
567   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
568     cookie_store->GetAllCookiesForURLAsync(
569         request_->url(),
570         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
571                    weak_factory_.GetWeakPtr()));
572   } else {
573     DoStartTransaction();
574   }
575 }
576 
DoLoadCookies()577 void URLRequestHttpJob::DoLoadCookies() {
578   CookieOptions options;
579   options.set_include_httponly();
580   GetCookieStore()->GetCookiesWithOptionsAsync(
581       request_->url(), options,
582       base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
583                  weak_factory_.GetWeakPtr()));
584 }
585 
CheckCookiePolicyAndLoad(const CookieList & cookie_list)586 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
587     const CookieList& cookie_list) {
588   if (CanGetCookies(cookie_list))
589     DoLoadCookies();
590   else
591     DoStartTransaction();
592 }
593 
OnCookiesLoaded(const std::string & cookie_line)594 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
595   if (!cookie_line.empty()) {
596     request_info_.extra_headers.SetHeader(
597         HttpRequestHeaders::kCookie, cookie_line);
598     // Disable privacy mode as we are sending cookies anyway.
599     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
600   }
601   DoStartTransaction();
602 }
603 
DoStartTransaction()604 void URLRequestHttpJob::DoStartTransaction() {
605   // We may have been canceled while retrieving cookies.
606   if (GetStatus().is_success()) {
607     StartTransaction();
608   } else {
609     NotifyCanceled();
610   }
611 }
612 
SaveCookiesAndNotifyHeadersComplete(int result)613 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
614   // End of the call started in OnStartCompleted.
615   OnCallToDelegateComplete();
616 
617   if (result != net::OK) {
618     std::string source("delegate");
619     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
620                                  NetLog::StringCallback("source", &source));
621     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
622     return;
623   }
624 
625   DCHECK(transaction_.get());
626 
627   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
628   DCHECK(response_info);
629 
630   response_cookies_.clear();
631   response_cookies_save_index_ = 0;
632 
633   FetchResponseCookies(&response_cookies_);
634 
635   if (!GetResponseHeaders()->GetDateValue(&response_date_))
636     response_date_ = base::Time();
637 
638   // Now, loop over the response cookies, and attempt to persist each.
639   SaveNextCookie();
640 }
641 
642 // If the save occurs synchronously, SaveNextCookie will loop and save the next
643 // cookie. If the save is deferred, the callback is responsible for continuing
644 // to iterate through the cookies.
645 // TODO(erikwright): Modify the CookieStore API to indicate via return value
646 // whether it completed synchronously or asynchronously.
647 // See http://crbug.com/131066.
SaveNextCookie()648 void URLRequestHttpJob::SaveNextCookie() {
649   // No matter what, we want to report our status as IO pending since we will
650   // be notifying our consumer asynchronously via OnStartCompleted.
651   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
652 
653   // Used to communicate with the callback. See the implementation of
654   // OnCookieSaved.
655   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
656   scoped_refptr<SharedBoolean> save_next_cookie_running =
657       new SharedBoolean(true);
658 
659   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
660       GetCookieStore() && response_cookies_.size() > 0) {
661     CookieOptions options;
662     options.set_include_httponly();
663     options.set_server_time(response_date_);
664 
665     net::CookieStore::SetCookiesCallback callback(
666         base::Bind(&URLRequestHttpJob::OnCookieSaved,
667                    weak_factory_.GetWeakPtr(),
668                    save_next_cookie_running,
669                    callback_pending));
670 
671     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
672     // synchronously.
673     while (!callback_pending->data &&
674            response_cookies_save_index_ < response_cookies_.size()) {
675       if (CanSetCookie(
676           response_cookies_[response_cookies_save_index_], &options)) {
677         callback_pending->data = true;
678         GetCookieStore()->SetCookieWithOptionsAsync(
679             request_->url(), response_cookies_[response_cookies_save_index_],
680             options, callback);
681       }
682       ++response_cookies_save_index_;
683     }
684   }
685 
686   save_next_cookie_running->data = false;
687 
688   if (!callback_pending->data) {
689     response_cookies_.clear();
690     response_cookies_save_index_ = 0;
691     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
692     NotifyHeadersComplete();
693     return;
694   }
695 }
696 
697 // |save_next_cookie_running| is true when the callback is bound and set to
698 // false when SaveNextCookie exits, allowing the callback to determine if the
699 // save occurred synchronously or asynchronously.
700 // |callback_pending| is false when the callback is invoked and will be set to
701 // true by the callback, allowing SaveNextCookie to detect whether the save
702 // occurred synchronously.
703 // See SaveNextCookie() for more information.
OnCookieSaved(scoped_refptr<SharedBoolean> save_next_cookie_running,scoped_refptr<SharedBoolean> callback_pending,bool cookie_status)704 void URLRequestHttpJob::OnCookieSaved(
705     scoped_refptr<SharedBoolean> save_next_cookie_running,
706     scoped_refptr<SharedBoolean> callback_pending,
707     bool cookie_status) {
708   callback_pending->data = false;
709 
710   // If we were called synchronously, return.
711   if (save_next_cookie_running->data) {
712     return;
713   }
714 
715   // We were called asynchronously, so trigger the next save.
716   // We may have been canceled within OnSetCookie.
717   if (GetStatus().is_success()) {
718     SaveNextCookie();
719   } else {
720     NotifyCanceled();
721   }
722 }
723 
FetchResponseCookies(std::vector<std::string> * cookies)724 void URLRequestHttpJob::FetchResponseCookies(
725     std::vector<std::string>* cookies) {
726   const std::string name = "Set-Cookie";
727   std::string value;
728 
729   void* iter = NULL;
730   HttpResponseHeaders* headers = GetResponseHeaders();
731   while (headers->EnumerateHeader(&iter, name, &value)) {
732     if (!value.empty())
733       cookies->push_back(value);
734   }
735 }
736 
737 // NOTE: |ProcessStrictTransportSecurityHeader| and
738 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
ProcessStrictTransportSecurityHeader()739 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
740   DCHECK(response_info_);
741   TransportSecurityState* security_state =
742       request_->context()->transport_security_state();
743   const SSLInfo& ssl_info = response_info_->ssl_info;
744 
745   // Only accept HSTS headers on HTTPS connections that have no
746   // certificate errors.
747   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
748       !security_state)
749     return;
750 
751   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
752   //
753   //   If a UA receives more than one STS header field in a HTTP response
754   //   message over secure transport, then the UA MUST process only the
755   //   first such header field.
756   HttpResponseHeaders* headers = GetResponseHeaders();
757   std::string value;
758   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
759     security_state->AddHSTSHeader(request_info_.url.host(), value);
760 }
761 
ProcessPublicKeyPinsHeader()762 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
763   DCHECK(response_info_);
764   TransportSecurityState* security_state =
765       request_->context()->transport_security_state();
766   const SSLInfo& ssl_info = response_info_->ssl_info;
767 
768   // Only accept HPKP headers on HTTPS connections that have no
769   // certificate errors.
770   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
771       !security_state)
772     return;
773 
774   // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
775   //
776   //   If a UA receives more than one PKP header field in an HTTP
777   //   response message over secure transport, then the UA MUST process
778   //   only the first such header field.
779   HttpResponseHeaders* headers = GetResponseHeaders();
780   std::string value;
781   if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
782     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
783 }
784 
OnStartCompleted(int result)785 void URLRequestHttpJob::OnStartCompleted(int result) {
786   RecordTimer();
787 
788   // If the request was destroyed, then there is no more work to do.
789   if (!request_)
790     return;
791 
792   // If the job is done (due to cancellation), can just ignore this
793   // notification.
794   if (done_)
795     return;
796 
797   receive_headers_end_ = base::TimeTicks::Now();
798 
799   // Clear the IO_PENDING status
800   SetStatus(URLRequestStatus());
801 
802   const URLRequestContext* context = request_->context();
803 
804   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
805       transaction_->GetResponseInfo() != NULL) {
806     FraudulentCertificateReporter* reporter =
807       context->fraudulent_certificate_reporter();
808     if (reporter != NULL) {
809       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
810       const std::string& host = request_->url().host();
811 
812       reporter->SendReport(host, ssl_info);
813     }
814   }
815 
816   if (result == OK) {
817     if (transaction_ && transaction_->GetResponseInfo()) {
818       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
819     }
820     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
821     if (network_delegate()) {
822       // Note that |this| may not be deleted until
823       // |on_headers_received_callback_| or
824       // |NetworkDelegate::URLRequestDestroyed()| has been called.
825       OnCallToDelegate();
826       allowed_unsafe_redirect_url_ = GURL();
827       int error = network_delegate()->NotifyHeadersReceived(
828           request_,
829           on_headers_received_callback_,
830           headers.get(),
831           &override_response_headers_,
832           &allowed_unsafe_redirect_url_);
833       if (error != net::OK) {
834         if (error == net::ERR_IO_PENDING) {
835           awaiting_callback_ = true;
836         } else {
837           std::string source("delegate");
838           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
839                                        NetLog::StringCallback("source",
840                                                               &source));
841           OnCallToDelegateComplete();
842           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
843         }
844         return;
845       }
846     }
847 
848     SaveCookiesAndNotifyHeadersComplete(net::OK);
849   } else if (IsCertificateError(result)) {
850     // We encountered an SSL certificate error.
851     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
852         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
853       // These are hard failures. They're handled separately and don't have
854       // the correct cert status, so set it here.
855       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
856       info.cert_status = MapNetErrorToCertStatus(result);
857       NotifySSLCertificateError(info, true);
858     } else {
859       // Maybe overridable, maybe not. Ask the delegate to decide.
860       const URLRequestContext* context = request_->context();
861       TransportSecurityState* state = context->transport_security_state();
862       const bool fatal =
863           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
864       NotifySSLCertificateError(
865           transaction_->GetResponseInfo()->ssl_info, fatal);
866     }
867   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
868     NotifyCertificateRequested(
869         transaction_->GetResponseInfo()->cert_request_info.get());
870   } else {
871     // Even on an error, there may be useful information in the response
872     // info (e.g. whether there's a cached copy).
873     if (transaction_.get())
874       response_info_ = transaction_->GetResponseInfo();
875     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
876   }
877 }
878 
OnHeadersReceivedCallback(int result)879 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
880   awaiting_callback_ = false;
881 
882   // Check that there are no callbacks to already canceled requests.
883   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
884 
885   SaveCookiesAndNotifyHeadersComplete(result);
886 }
887 
OnReadCompleted(int result)888 void URLRequestHttpJob::OnReadCompleted(int result) {
889   read_in_progress_ = false;
890 
891   if (ShouldFixMismatchedContentLength(result))
892     result = OK;
893 
894   if (result == OK) {
895     NotifyDone(URLRequestStatus());
896   } else if (result < 0) {
897     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
898   } else {
899     // Clear the IO_PENDING status
900     SetStatus(URLRequestStatus());
901   }
902 
903   NotifyReadComplete(result);
904 }
905 
RestartTransactionWithAuth(const AuthCredentials & credentials)906 void URLRequestHttpJob::RestartTransactionWithAuth(
907     const AuthCredentials& credentials) {
908   auth_credentials_ = credentials;
909 
910   // These will be reset in OnStartCompleted.
911   response_info_ = NULL;
912   receive_headers_end_ = base::TimeTicks();
913   response_cookies_.clear();
914 
915   ResetTimer();
916 
917   // Update the cookies, since the cookie store may have been updated from the
918   // headers in the 401/407. Since cookies were already appended to
919   // extra_headers, we need to strip them out before adding them again.
920   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
921 
922   AddCookieHeaderAndStart();
923 }
924 
SetUpload(UploadDataStream * upload)925 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
926   DCHECK(!transaction_.get()) << "cannot change once started";
927   request_info_.upload_data_stream = upload;
928 }
929 
SetExtraRequestHeaders(const HttpRequestHeaders & headers)930 void URLRequestHttpJob::SetExtraRequestHeaders(
931     const HttpRequestHeaders& headers) {
932   DCHECK(!transaction_.get()) << "cannot change once started";
933   request_info_.extra_headers.CopyFrom(headers);
934 }
935 
GetLoadState() const936 LoadState URLRequestHttpJob::GetLoadState() const {
937   return transaction_.get() ?
938       transaction_->GetLoadState() : LOAD_STATE_IDLE;
939 }
940 
GetUploadProgress() const941 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
942   return transaction_.get() ?
943       transaction_->GetUploadProgress() : UploadProgress();
944 }
945 
GetMimeType(std::string * mime_type) const946 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
947   DCHECK(transaction_.get());
948 
949   if (!response_info_)
950     return false;
951 
952   return GetResponseHeaders()->GetMimeType(mime_type);
953 }
954 
GetCharset(std::string * charset)955 bool URLRequestHttpJob::GetCharset(std::string* charset) {
956   DCHECK(transaction_.get());
957 
958   if (!response_info_)
959     return false;
960 
961   return GetResponseHeaders()->GetCharset(charset);
962 }
963 
GetResponseInfo(HttpResponseInfo * info)964 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
965   DCHECK(request_);
966 
967   if (response_info_) {
968     DCHECK(transaction_.get());
969 
970     *info = *response_info_;
971     if (override_response_headers_.get())
972       info->headers = override_response_headers_;
973   }
974 }
975 
GetLoadTimingInfo(LoadTimingInfo * load_timing_info) const976 void URLRequestHttpJob::GetLoadTimingInfo(
977     LoadTimingInfo* load_timing_info) const {
978   // If haven't made it far enough to receive any headers, don't return
979   // anything.  This makes for more consistent behavior in the case of errors.
980   if (!transaction_ || receive_headers_end_.is_null())
981     return;
982   if (transaction_->GetLoadTimingInfo(load_timing_info))
983     load_timing_info->receive_headers_end = receive_headers_end_;
984 }
985 
GetResponseCookies(std::vector<std::string> * cookies)986 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
987   DCHECK(transaction_.get());
988 
989   if (!response_info_)
990     return false;
991 
992   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
993   // should just leverage response_cookies_.
994 
995   cookies->clear();
996   FetchResponseCookies(cookies);
997   return true;
998 }
999 
GetResponseCode() const1000 int URLRequestHttpJob::GetResponseCode() const {
1001   DCHECK(transaction_.get());
1002 
1003   if (!response_info_)
1004     return -1;
1005 
1006   return GetResponseHeaders()->response_code();
1007 }
1008 
SetupFilter() const1009 Filter* URLRequestHttpJob::SetupFilter() const {
1010   DCHECK(transaction_.get());
1011   if (!response_info_)
1012     return NULL;
1013 
1014   std::vector<Filter::FilterType> encoding_types;
1015   std::string encoding_type;
1016   HttpResponseHeaders* headers = GetResponseHeaders();
1017   void* iter = NULL;
1018   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1019     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1020   }
1021 
1022   if (filter_context_->SdchResponseExpected()) {
1023     // We are wary of proxies that discard or damage SDCH encoding.  If a server
1024     // explicitly states that this is not SDCH content, then we can correct our
1025     // assumption that this is an SDCH response, and avoid the need to recover
1026     // as though the content is corrupted (when we discover it is not SDCH
1027     // encoded).
1028     std::string sdch_response_status;
1029     iter = NULL;
1030     while (headers->EnumerateHeader(&iter, "X-Sdch-Encode",
1031                                     &sdch_response_status)) {
1032       if (sdch_response_status == "0") {
1033         filter_context_->ResetSdchResponseToFalse();
1034         break;
1035       }
1036     }
1037   }
1038 
1039   // Even if encoding types are empty, there is a chance that we need to add
1040   // some decoding, as some proxies strip encoding completely. In such cases,
1041   // we may need to add (for example) SDCH filtering (when the context suggests
1042   // it is appropriate).
1043   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1044 
1045   return !encoding_types.empty()
1046       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1047 }
1048 
CopyFragmentOnRedirect(const GURL & location) const1049 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1050   // Allow modification of reference fragments by default, unless
1051   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1052   // When this is the case, we assume that the network delegate has set the
1053   // desired redirect URL (with or without fragment), so it must not be changed
1054   // any more.
1055   return !allowed_unsafe_redirect_url_.is_valid() ||
1056        allowed_unsafe_redirect_url_ != location;
1057 }
1058 
IsSafeRedirect(const GURL & location)1059 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1060   // HTTP is always safe.
1061   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1062   if (location.is_valid() &&
1063       (location.scheme() == "http" || location.scheme() == "https")) {
1064     return true;
1065   }
1066   // Delegates may mark a URL as safe for redirection.
1067   if (allowed_unsafe_redirect_url_.is_valid() &&
1068       allowed_unsafe_redirect_url_ == location) {
1069     return true;
1070   }
1071   // Query URLRequestJobFactory as to whether |location| would be safe to
1072   // redirect to.
1073   return request_->context()->job_factory() &&
1074       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1075 }
1076 
NeedsAuth()1077 bool URLRequestHttpJob::NeedsAuth() {
1078   int code = GetResponseCode();
1079   if (code == -1)
1080     return false;
1081 
1082   // Check if we need either Proxy or WWW Authentication.  This could happen
1083   // because we either provided no auth info, or provided incorrect info.
1084   switch (code) {
1085     case 407:
1086       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1087         return false;
1088       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1089       return true;
1090     case 401:
1091       if (server_auth_state_ == AUTH_STATE_CANCELED)
1092         return false;
1093       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1094       return true;
1095   }
1096   return false;
1097 }
1098 
GetAuthChallengeInfo(scoped_refptr<AuthChallengeInfo> * result)1099 void URLRequestHttpJob::GetAuthChallengeInfo(
1100     scoped_refptr<AuthChallengeInfo>* result) {
1101   DCHECK(transaction_.get());
1102   DCHECK(response_info_);
1103 
1104   // sanity checks:
1105   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1106          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1107   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1108          (GetResponseHeaders()->response_code() ==
1109           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1110 
1111   *result = response_info_->auth_challenge;
1112 }
1113 
SetAuth(const AuthCredentials & credentials)1114 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1115   DCHECK(transaction_.get());
1116 
1117   // Proxy gets set first, then WWW.
1118   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1119     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1120   } else {
1121     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1122     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1123   }
1124 
1125   RestartTransactionWithAuth(credentials);
1126 }
1127 
CancelAuth()1128 void URLRequestHttpJob::CancelAuth() {
1129   // Proxy gets set first, then WWW.
1130   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1131     proxy_auth_state_ = AUTH_STATE_CANCELED;
1132   } else {
1133     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1134     server_auth_state_ = AUTH_STATE_CANCELED;
1135   }
1136 
1137   // These will be reset in OnStartCompleted.
1138   response_info_ = NULL;
1139   receive_headers_end_ = base::TimeTicks::Now();
1140   response_cookies_.clear();
1141 
1142   ResetTimer();
1143 
1144   // OK, let the consumer read the error page...
1145   //
1146   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1147   // which will cause the consumer to receive OnResponseStarted instead of
1148   // OnAuthRequired.
1149   //
1150   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1151   //
1152   base::MessageLoop::current()->PostTask(
1153       FROM_HERE,
1154       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1155                  weak_factory_.GetWeakPtr(), OK));
1156 }
1157 
ContinueWithCertificate(X509Certificate * client_cert)1158 void URLRequestHttpJob::ContinueWithCertificate(
1159     X509Certificate* client_cert) {
1160   DCHECK(transaction_.get());
1161 
1162   DCHECK(!response_info_) << "should not have a response yet";
1163   receive_headers_end_ = base::TimeTicks();
1164 
1165   ResetTimer();
1166 
1167   // No matter what, we want to report our status as IO pending since we will
1168   // be notifying our consumer asynchronously via OnStartCompleted.
1169   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1170 
1171   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1172   if (rv == ERR_IO_PENDING)
1173     return;
1174 
1175   // The transaction started synchronously, but we need to notify the
1176   // URLRequest delegate via the message loop.
1177   base::MessageLoop::current()->PostTask(
1178       FROM_HERE,
1179       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1180                  weak_factory_.GetWeakPtr(), rv));
1181 }
1182 
ContinueDespiteLastError()1183 void URLRequestHttpJob::ContinueDespiteLastError() {
1184   // If the transaction was destroyed, then the job was cancelled.
1185   if (!transaction_.get())
1186     return;
1187 
1188   DCHECK(!response_info_) << "should not have a response yet";
1189   receive_headers_end_ = base::TimeTicks();
1190 
1191   ResetTimer();
1192 
1193   // No matter what, we want to report our status as IO pending since we will
1194   // be notifying our consumer asynchronously via OnStartCompleted.
1195   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1196 
1197   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1198   if (rv == ERR_IO_PENDING)
1199     return;
1200 
1201   // The transaction started synchronously, but we need to notify the
1202   // URLRequest delegate via the message loop.
1203   base::MessageLoop::current()->PostTask(
1204       FROM_HERE,
1205       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1206                  weak_factory_.GetWeakPtr(), rv));
1207 }
1208 
ResumeNetworkStart()1209 void URLRequestHttpJob::ResumeNetworkStart() {
1210   DCHECK(transaction_.get());
1211   transaction_->ResumeNetworkStart();
1212 }
1213 
ShouldFixMismatchedContentLength(int rv) const1214 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1215   // Some servers send the body compressed, but specify the content length as
1216   // the uncompressed size.  Although this violates the HTTP spec we want to
1217   // support it (as IE and FireFox do), but *only* for an exact match.
1218   // See http://crbug.com/79694.
1219   if (rv == net::ERR_CONTENT_LENGTH_MISMATCH ||
1220       rv == net::ERR_INCOMPLETE_CHUNKED_ENCODING) {
1221     if (request_ && request_->response_headers()) {
1222       int64 expected_length = request_->response_headers()->GetContentLength();
1223       VLOG(1) << __FUNCTION__ << "() "
1224               << "\"" << request_->url().spec() << "\""
1225               << " content-length = " << expected_length
1226               << " pre total = " << prefilter_bytes_read()
1227               << " post total = " << postfilter_bytes_read();
1228       if (postfilter_bytes_read() == expected_length) {
1229         // Clear the error.
1230         return true;
1231       }
1232     }
1233   }
1234   return false;
1235 }
1236 
ReadRawData(IOBuffer * buf,int buf_size,int * bytes_read)1237 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1238                                     int* bytes_read) {
1239   DCHECK_NE(buf_size, 0);
1240   DCHECK(bytes_read);
1241   DCHECK(!read_in_progress_);
1242 
1243   int rv = transaction_->Read(
1244       buf, buf_size,
1245       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1246 
1247   if (ShouldFixMismatchedContentLength(rv))
1248     rv = 0;
1249 
1250   if (rv >= 0) {
1251     *bytes_read = rv;
1252     if (!rv)
1253       DoneWithRequest(FINISHED);
1254     return true;
1255   }
1256 
1257   if (rv == ERR_IO_PENDING) {
1258     read_in_progress_ = true;
1259     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1260   } else {
1261     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1262   }
1263 
1264   return false;
1265 }
1266 
StopCaching()1267 void URLRequestHttpJob::StopCaching() {
1268   if (transaction_.get())
1269     transaction_->StopCaching();
1270 }
1271 
GetFullRequestHeaders(HttpRequestHeaders * headers) const1272 bool URLRequestHttpJob::GetFullRequestHeaders(
1273     HttpRequestHeaders* headers) const {
1274   if (!transaction_)
1275     return false;
1276 
1277   return transaction_->GetFullRequestHeaders(headers);
1278 }
1279 
GetTotalReceivedBytes() const1280 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1281   if (!transaction_)
1282     return 0;
1283 
1284   return transaction_->GetTotalReceivedBytes();
1285 }
1286 
DoneReading()1287 void URLRequestHttpJob::DoneReading() {
1288   if (transaction_) {
1289     transaction_->DoneReading();
1290   }
1291   DoneWithRequest(FINISHED);
1292 }
1293 
DoneReadingRedirectResponse()1294 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1295   if (transaction_) {
1296     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1297       // If the original headers indicate a redirect, go ahead and cache the
1298       // response, even if the |override_response_headers_| are a redirect to
1299       // another location.
1300       transaction_->DoneReading();
1301     } else {
1302       // Otherwise, |override_response_headers_| must be non-NULL and contain
1303       // bogus headers indicating a redirect.
1304       DCHECK(override_response_headers_.get());
1305       DCHECK(override_response_headers_->IsRedirect(NULL));
1306       transaction_->StopCaching();
1307     }
1308   }
1309   DoneWithRequest(FINISHED);
1310 }
1311 
GetSocketAddress() const1312 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1313   return response_info_ ? response_info_->socket_address : HostPortPair();
1314 }
1315 
RecordTimer()1316 void URLRequestHttpJob::RecordTimer() {
1317   if (request_creation_time_.is_null()) {
1318     NOTREACHED()
1319         << "The same transaction shouldn't start twice without new timing.";
1320     return;
1321   }
1322 
1323   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1324   request_creation_time_ = base::Time();
1325 
1326   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1327 }
1328 
ResetTimer()1329 void URLRequestHttpJob::ResetTimer() {
1330   if (!request_creation_time_.is_null()) {
1331     NOTREACHED()
1332         << "The timer was reset before it was recorded.";
1333     return;
1334   }
1335   request_creation_time_ = base::Time::Now();
1336 }
1337 
UpdatePacketReadTimes()1338 void URLRequestHttpJob::UpdatePacketReadTimes() {
1339   if (!packet_timing_enabled_)
1340     return;
1341 
1342   if (filter_input_byte_count() <= bytes_observed_in_packets_) {
1343     DCHECK_EQ(filter_input_byte_count(), bytes_observed_in_packets_);
1344     return;  // No new bytes have arrived.
1345   }
1346 
1347   base::Time now(base::Time::Now());
1348   if (!bytes_observed_in_packets_)
1349     request_time_snapshot_ = now;
1350   final_packet_time_ = now;
1351 
1352   bytes_observed_in_packets_ = filter_input_byte_count();
1353 }
1354 
RecordPacketStats(FilterContext::StatisticSelector statistic) const1355 void URLRequestHttpJob::RecordPacketStats(
1356     FilterContext::StatisticSelector statistic) const {
1357   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1358     return;
1359 
1360   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1361   switch (statistic) {
1362     case FilterContext::SDCH_DECODE: {
1363       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1364           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1365       return;
1366     }
1367     case FilterContext::SDCH_PASSTHROUGH: {
1368       // Despite advertising a dictionary, we handled non-sdch compressed
1369       // content.
1370       return;
1371     }
1372 
1373     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1374       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1375                                   duration,
1376                                   base::TimeDelta::FromMilliseconds(20),
1377                                   base::TimeDelta::FromMinutes(10), 100);
1378       return;
1379     }
1380     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1381       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1382                                   duration,
1383                                   base::TimeDelta::FromMilliseconds(20),
1384                                   base::TimeDelta::FromMinutes(10), 100);
1385       return;
1386     }
1387     default:
1388       NOTREACHED();
1389       return;
1390   }
1391 }
1392 
1393 // The common type of histogram we use for all compression-tracking histograms.
1394 #define COMPRESSION_HISTOGRAM(name, sample) \
1395     do { \
1396       UMA_HISTOGRAM_CUSTOM_COUNTS("Net.Compress." name, sample, \
1397                                   500, 1000000, 100); \
1398     } while (0)
1399 
RecordCompressionHistograms()1400 void URLRequestHttpJob::RecordCompressionHistograms() {
1401   DCHECK(request_);
1402   if (!request_)
1403     return;
1404 
1405   if (is_cached_content_ ||                // Don't record cached content
1406       !GetStatus().is_success() ||         // Don't record failed content
1407       !IsCompressibleContent() ||          // Only record compressible content
1408       !prefilter_bytes_read())       // Zero-byte responses aren't useful.
1409     return;
1410 
1411   // Miniature requests aren't really compressible.  Don't count them.
1412   const int kMinSize = 16;
1413   if (prefilter_bytes_read() < kMinSize)
1414     return;
1415 
1416   // Only record for http or https urls.
1417   bool is_http = request_->url().SchemeIs("http");
1418   bool is_https = request_->url().SchemeIs("https");
1419   if (!is_http && !is_https)
1420     return;
1421 
1422   int compressed_B = prefilter_bytes_read();
1423   int decompressed_B = postfilter_bytes_read();
1424   bool was_filtered = HasFilter();
1425 
1426   // We want to record how often downloaded resources are compressed.
1427   // But, we recognize that different protocols may have different
1428   // properties.  So, for each request, we'll put it into one of 3
1429   // groups:
1430   //      a) SSL resources
1431   //         Proxies cannot tamper with compression headers with SSL.
1432   //      b) Non-SSL, loaded-via-proxy resources
1433   //         In this case, we know a proxy might have interfered.
1434   //      c) Non-SSL, loaded-without-proxy resources
1435   //         In this case, we know there was no explicit proxy.  However,
1436   //         it is possible that a transparent proxy was still interfering.
1437   //
1438   // For each group, we record the same 3 histograms.
1439 
1440   if (is_https) {
1441     if (was_filtered) {
1442       COMPRESSION_HISTOGRAM("SSL.BytesBeforeCompression", compressed_B);
1443       COMPRESSION_HISTOGRAM("SSL.BytesAfterCompression", decompressed_B);
1444     } else {
1445       COMPRESSION_HISTOGRAM("SSL.ShouldHaveBeenCompressed", decompressed_B);
1446     }
1447     return;
1448   }
1449 
1450   if (request_->was_fetched_via_proxy()) {
1451     if (was_filtered) {
1452       COMPRESSION_HISTOGRAM("Proxy.BytesBeforeCompression", compressed_B);
1453       COMPRESSION_HISTOGRAM("Proxy.BytesAfterCompression", decompressed_B);
1454     } else {
1455       COMPRESSION_HISTOGRAM("Proxy.ShouldHaveBeenCompressed", decompressed_B);
1456     }
1457     return;
1458   }
1459 
1460   if (was_filtered) {
1461     COMPRESSION_HISTOGRAM("NoProxy.BytesBeforeCompression", compressed_B);
1462     COMPRESSION_HISTOGRAM("NoProxy.BytesAfterCompression", decompressed_B);
1463   } else {
1464     COMPRESSION_HISTOGRAM("NoProxy.ShouldHaveBeenCompressed", decompressed_B);
1465   }
1466 }
1467 
IsCompressibleContent() const1468 bool URLRequestHttpJob::IsCompressibleContent() const {
1469   std::string mime_type;
1470   return GetMimeType(&mime_type) &&
1471       (IsSupportedJavascriptMimeType(mime_type.c_str()) ||
1472        IsSupportedNonImageMimeType(mime_type.c_str()));
1473 }
1474 
RecordPerfHistograms(CompletionCause reason)1475 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1476   if (start_time_.is_null())
1477     return;
1478 
1479   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1480   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1481 
1482   if (reason == FINISHED) {
1483     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1484   } else {
1485     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1486   }
1487 
1488   if (response_info_) {
1489     if (response_info_->was_cached) {
1490       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1491     } else  {
1492       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1493     }
1494   }
1495 
1496   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1497     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1498                          prefilter_bytes_read());
1499 
1500   start_time_ = base::TimeTicks();
1501 }
1502 
DoneWithRequest(CompletionCause reason)1503 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1504   if (done_)
1505     return;
1506   done_ = true;
1507   RecordPerfHistograms(reason);
1508   if (reason == FINISHED) {
1509     request_->set_received_response_content_length(prefilter_bytes_read());
1510     RecordCompressionHistograms();
1511   }
1512 }
1513 
GetResponseHeaders() const1514 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1515   DCHECK(transaction_.get());
1516   DCHECK(transaction_->GetResponseInfo());
1517   return override_response_headers_.get() ?
1518              override_response_headers_.get() :
1519              transaction_->GetResponseInfo()->headers.get();
1520 }
1521 
NotifyURLRequestDestroyed()1522 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1523   awaiting_callback_ = false;
1524 }
1525 
1526 }  // namespace net
1527