1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef NET_URL_REQUEST_URL_FETCHER_CORE_H_ 6 #define NET_URL_REQUEST_URL_FETCHER_CORE_H_ 7 8 #include <set> 9 #include <string> 10 11 #include "base/basictypes.h" 12 #include "base/compiler_specific.h" 13 #include "base/debug/stack_trace.h" 14 #include "base/files/file_path.h" 15 #include "base/lazy_instance.h" 16 #include "base/memory/ref_counted.h" 17 #include "base/memory/scoped_ptr.h" 18 #include "base/timer/timer.h" 19 #include "net/base/host_port_pair.h" 20 #include "net/http/http_request_headers.h" 21 #include "net/url_request/url_fetcher.h" 22 #include "net/url_request/url_request.h" 23 #include "net/url_request/url_request_status.h" 24 #include "url/gurl.h" 25 26 namespace base { 27 class SingleThreadTaskRunner; 28 } // namespace base 29 30 namespace net { 31 class DrainableIOBuffer; 32 class HttpResponseHeaders; 33 class IOBuffer; 34 class URLFetcherDelegate; 35 class URLFetcherResponseWriter; 36 class URLRequestContextGetter; 37 class URLRequestThrottlerEntryInterface; 38 39 class URLFetcherCore 40 : public base::RefCountedThreadSafe<URLFetcherCore>, 41 public URLRequest::Delegate { 42 public: 43 URLFetcherCore(URLFetcher* fetcher, 44 const GURL& original_url, 45 URLFetcher::RequestType request_type, 46 URLFetcherDelegate* d); 47 48 // Starts the load. It's important that this not happen in the constructor 49 // because it causes the IO thread to begin AddRef()ing and Release()ing 50 // us. If our caller hasn't had time to fully construct us and take a 51 // reference, the IO thread could interrupt things, run a task, Release() 52 // us, and destroy us, leaving the caller with an already-destroyed object 53 // when construction finishes. 54 void Start(); 55 56 // Stops any in-progress load and ensures no callback will happen. It is 57 // safe to call this multiple times. 58 void Stop(); 59 60 // URLFetcher-like functions. 61 62 // For POST requests, set |content_type| to the MIME type of the 63 // content and set |content| to the data to upload. 64 void SetUploadData(const std::string& upload_content_type, 65 const std::string& upload_content); 66 void SetUploadFilePath(const std::string& upload_content_type, 67 const base::FilePath& file_path, 68 uint64 range_offset, 69 uint64 range_length, 70 scoped_refptr<base::TaskRunner> file_task_runner); 71 void SetChunkedUpload(const std::string& upload_content_type); 72 // Adds a block of data to be uploaded in a POST body. This can only be 73 // called after Start(). 74 void AppendChunkToUpload(const std::string& data, bool is_last_chunk); 75 // |flags| are flags to apply to the load operation--these should be 76 // one or more of the LOAD_* flags defined in net/base/load_flags.h. 77 void SetLoadFlags(int load_flags); 78 int GetLoadFlags() const; 79 void SetReferrer(const std::string& referrer); 80 void SetReferrerPolicy(URLRequest::ReferrerPolicy referrer_policy); 81 void SetExtraRequestHeaders(const std::string& extra_request_headers); 82 void AddExtraRequestHeader(const std::string& header_line); 83 void SetRequestContext(URLRequestContextGetter* request_context_getter); 84 // Set the URL that should be consulted for the third-party cookie 85 // blocking policy. 86 void SetFirstPartyForCookies(const GURL& first_party_for_cookies); 87 // Set the key and data callback that is used when setting the user 88 // data on any URLRequest objects this object creates. 89 void SetURLRequestUserData( 90 const void* key, 91 const URLFetcher::CreateDataCallback& create_data_callback); 92 void SetStopOnRedirect(bool stop_on_redirect); 93 void SetAutomaticallyRetryOn5xx(bool retry); 94 void SetMaxRetriesOn5xx(int max_retries); 95 int GetMaxRetriesOn5xx() const; 96 base::TimeDelta GetBackoffDelay() const; 97 void SetAutomaticallyRetryOnNetworkChanges(int max_retries); 98 void SaveResponseToFileAtPath( 99 const base::FilePath& file_path, 100 scoped_refptr<base::SequencedTaskRunner> file_task_runner); 101 void SaveResponseToTemporaryFile( 102 scoped_refptr<base::SequencedTaskRunner> file_task_runner); 103 void SaveResponseWithWriter( 104 scoped_ptr<URLFetcherResponseWriter> response_writer); 105 HttpResponseHeaders* GetResponseHeaders() const; 106 HostPortPair GetSocketAddress() const; 107 bool WasFetchedViaProxy() const; 108 const GURL& GetOriginalURL() const; 109 const GURL& GetURL() const; 110 const URLRequestStatus& GetStatus() const; 111 int GetResponseCode() const; 112 const ResponseCookies& GetCookies() const; 113 // Reports that the received content was malformed (i.e. failed parsing 114 // or validation). This makes the throttling logic that does exponential 115 // back-off when servers are having problems treat the current request as 116 // a failure. Your call to this method will be ignored if your request is 117 // already considered a failure based on the HTTP response code or response 118 // headers. 119 void ReceivedContentWasMalformed(); 120 bool GetResponseAsString(std::string* out_response_string) const; 121 bool GetResponseAsFilePath(bool take_ownership, 122 base::FilePath* out_response_path); 123 124 // Overridden from URLRequest::Delegate: 125 virtual void OnReceivedRedirect(URLRequest* request, 126 const GURL& new_url, 127 bool* defer_redirect) OVERRIDE; 128 virtual void OnResponseStarted(URLRequest* request) OVERRIDE; 129 virtual void OnReadCompleted(URLRequest* request, 130 int bytes_read) OVERRIDE; 131 virtual void OnCertificateRequested( 132 URLRequest* request, 133 SSLCertRequestInfo* cert_request_info) OVERRIDE; 134 delegate()135 URLFetcherDelegate* delegate() const { return delegate_; } 136 static void CancelAll(); 137 static int GetNumFetcherCores(); 138 static void SetEnableInterceptionForTests(bool enabled); 139 static void SetIgnoreCertificateRequests(bool ignored); 140 141 private: 142 friend class base::RefCountedThreadSafe<URLFetcherCore>; 143 144 class Registry { 145 public: 146 Registry(); 147 ~Registry(); 148 149 void AddURLFetcherCore(URLFetcherCore* core); 150 void RemoveURLFetcherCore(URLFetcherCore* core); 151 152 void CancelAll(); 153 size()154 int size() const { 155 return fetchers_.size(); 156 } 157 158 private: 159 std::set<URLFetcherCore*> fetchers_; 160 161 DISALLOW_COPY_AND_ASSIGN(Registry); 162 }; 163 164 virtual ~URLFetcherCore(); 165 166 // Wrapper functions that allow us to ensure actions happen on the right 167 // thread. 168 void StartOnIOThread(); 169 void StartURLRequest(); 170 void DidInitializeWriter(int result); 171 void StartURLRequestWhenAppropriate(); 172 void CancelURLRequest(int error); 173 void OnCompletedURLRequest(base::TimeDelta backoff_delay); 174 void InformDelegateFetchIsComplete(); 175 void NotifyMalformedContent(); 176 void DidFinishWriting(int result); 177 void RetryOrCompleteUrlFetch(); 178 179 // Deletes the request, removes it from the registry, and removes the 180 // destruction observer. 181 void ReleaseRequest(); 182 183 // Returns the max value of exponential back-off release time for 184 // |original_url_| and |url_|. 185 base::TimeTicks GetBackoffReleaseTime(); 186 187 void CompleteAddingUploadDataChunk(const std::string& data, 188 bool is_last_chunk); 189 190 // Writes all bytes stored in |data| with |response_writer_|. 191 // Returns OK if all bytes in |data| get written synchronously. Otherwise, 192 // returns ERR_IO_PENDING or a network error code. 193 int WriteBuffer(scoped_refptr<DrainableIOBuffer> data); 194 195 // Used to implement WriteBuffer(). 196 void DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data, int result); 197 198 // Read response bytes from the request. 199 void ReadResponse(); 200 201 // Notify Delegate about the progress of upload/download. 202 void InformDelegateUploadProgress(); 203 void InformDelegateUploadProgressInDelegateThread(int64 current, int64 total); 204 void InformDelegateDownloadProgress(); 205 void InformDelegateDownloadProgressInDelegateThread(int64 current, 206 int64 total); 207 208 URLFetcher* fetcher_; // Corresponding fetcher object 209 GURL original_url_; // The URL we were asked to fetch 210 GURL url_; // The URL we eventually wound up at 211 URLFetcher::RequestType request_type_; // What type of request is this? 212 URLRequestStatus status_; // Status of the request 213 URLFetcherDelegate* delegate_; // Object to notify on completion 214 // Task runner for the creating thread. Used to interact with the delegate. 215 scoped_refptr<base::SingleThreadTaskRunner> delegate_task_runner_; 216 // Task runner for network operations. 217 scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_; 218 // Task runner for upload file access. 219 scoped_refptr<base::TaskRunner> upload_file_task_runner_; 220 scoped_ptr<URLRequest> request_; // The actual request this wraps 221 int load_flags_; // Flags for the load operation 222 int response_code_; // HTTP status code for the request 223 scoped_refptr<IOBuffer> buffer_; 224 // Read buffer 225 scoped_refptr<URLRequestContextGetter> request_context_getter_; 226 // Cookie/cache info for the request 227 GURL first_party_for_cookies_; // The first party URL for the request 228 // The user data to add to each newly-created URLRequest. 229 const void* url_request_data_key_; 230 URLFetcher::CreateDataCallback url_request_create_data_callback_; 231 ResponseCookies cookies_; // Response cookies 232 HttpRequestHeaders extra_request_headers_; 233 scoped_refptr<HttpResponseHeaders> response_headers_; 234 bool was_fetched_via_proxy_; 235 HostPortPair socket_address_; 236 237 bool upload_content_set_; // SetUploadData has been called 238 std::string upload_content_; // HTTP POST payload 239 base::FilePath upload_file_path_; // Path to file containing POST payload 240 uint64 upload_range_offset_; // Offset from the beginning of the file 241 // to be uploaded. 242 uint64 upload_range_length_; // The length of the part of file to be 243 // uploaded. 244 std::string upload_content_type_; // MIME type of POST payload 245 std::string referrer_; // HTTP Referer header value and policy 246 URLRequest::ReferrerPolicy referrer_policy_; 247 bool is_chunked_upload_; // True if using chunked transfer encoding 248 249 // Used to determine how long to wait before making a request or doing a 250 // retry. 251 // 252 // Both of them can only be accessed on the IO thread. 253 // 254 // We need not only the throttler entry for |original_URL|, but also 255 // the one for |url|. For example, consider the case that URL A 256 // redirects to URL B, for which the server returns a 500 257 // response. In this case, the exponential back-off release time of 258 // URL A won't increase. If we retry without considering the 259 // back-off constraint of URL B, we may send out too many requests 260 // for URL A in a short period of time. 261 // 262 // Both of these will be NULL if 263 // URLRequestContext::throttler_manager() is NULL. 264 scoped_refptr<URLRequestThrottlerEntryInterface> 265 original_url_throttler_entry_; 266 scoped_refptr<URLRequestThrottlerEntryInterface> url_throttler_entry_; 267 268 // True if the URLFetcher has been cancelled. 269 bool was_cancelled_; 270 271 // Writer object to write response to the destination like file and string. 272 scoped_ptr<URLFetcherResponseWriter> response_writer_; 273 274 // By default any server-initiated redirects are automatically followed. If 275 // this flag is set to true, however, a redirect will halt the fetch and call 276 // back to to the delegate immediately. 277 bool stop_on_redirect_; 278 // True when we're actually stopped due to a redirect halted by the above. We 279 // use this to ensure that |url_| is set to the redirect destination rather 280 // than the originally-fetched URL. 281 bool stopped_on_redirect_; 282 283 // If |automatically_retry_on_5xx_| is false, 5xx responses will be 284 // propagated to the observer, if it is true URLFetcher will automatically 285 // re-execute the request, after the back-off delay has expired. 286 // true by default. 287 bool automatically_retry_on_5xx_; 288 // |num_retries_on_5xx_| indicates how many times we've failed to successfully 289 // fetch this URL due to 5xx responses. Once this value exceeds the maximum 290 // number of retries specified by the owner URLFetcher instance, 291 // we'll give up. 292 int num_retries_on_5xx_; 293 // Maximum retries allowed when 5xx responses are received. 294 int max_retries_on_5xx_; 295 // Back-off time delay. 0 by default. 296 base::TimeDelta backoff_delay_; 297 298 // The number of retries that have been attempted due to ERR_NETWORK_CHANGED. 299 int num_retries_on_network_changes_; 300 // Maximum retries allowed when the request fails with ERR_NETWORK_CHANGED. 301 // 0 by default. 302 int max_retries_on_network_changes_; 303 304 // Timer to poll the progress of uploading for POST and PUT requests. 305 // When crbug.com/119629 is fixed, scoped_ptr is not necessary here. 306 scoped_ptr<base::RepeatingTimer<URLFetcherCore> > 307 upload_progress_checker_timer_; 308 // Number of bytes sent so far. 309 int64 current_upload_bytes_; 310 // Number of bytes received so far. 311 int64 current_response_bytes_; 312 // Total expected bytes to receive (-1 if it cannot be determined). 313 int64 total_response_bytes_; 314 315 // TODO(willchan): Get rid of this after debugging crbug.com/90971. 316 base::debug::StackTrace stack_trace_; 317 318 static base::LazyInstance<Registry> g_registry; 319 320 DISALLOW_COPY_AND_ASSIGN(URLFetcherCore); 321 }; 322 323 } // namespace net 324 325 #endif // NET_URL_REQUEST_URL_FETCHER_CORE_H_ 326