• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "components/precache/core/precache_fetcher.h"
6 
7 #include <string>
8 
9 #include "base/bind.h"
10 #include "base/callback.h"
11 #include "base/command_line.h"
12 #include "base/compiler_specific.h"
13 #include "base/containers/hash_tables.h"
14 #include "components/precache/core/precache_switches.h"
15 #include "components/precache/core/proto/precache.pb.h"
16 #include "net/base/escape.h"
17 #include "net/base/load_flags.h"
18 #include "net/url_request/url_fetcher.h"
19 #include "net/url_request/url_fetcher_delegate.h"
20 #include "net/url_request/url_request_context_getter.h"
21 #include "net/url_request/url_request_status.h"
22 
23 using net::URLFetcher;
24 
25 namespace precache {
26 
27 namespace {
28 
GetConfigURL()29 GURL GetConfigURL() {
30   const CommandLine& command_line = *CommandLine::ForCurrentProcess();
31   if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
32     return GURL(
33         command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
34   }
35 
36 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
37   return GURL(PRECACHE_CONFIG_SETTINGS_URL);
38 #else
39   // The precache config settings URL could not be determined, so return an
40   // empty, invalid GURL.
41   return GURL();
42 #endif
43 }
44 
GetManifestURLPrefix()45 std::string GetManifestURLPrefix() {
46   const CommandLine& command_line = *CommandLine::ForCurrentProcess();
47   if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
48     return command_line.GetSwitchValueASCII(
49         switches::kPrecacheManifestURLPrefix);
50   }
51 
52 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
53   return PRECACHE_MANIFEST_URL_PREFIX;
54 #else
55   // The precache manifest URL prefix could not be determined, so return an
56   // empty string.
57   return std::string();
58 #endif
59 }
60 
61 // Construct the URL of the precache manifest for the given starting URL.
62 // The server is expecting a request for a URL consisting of the manifest URL
63 // prefix followed by the doubly escaped starting URL.
ConstructManifestURL(const GURL & starting_url)64 GURL ConstructManifestURL(const GURL& starting_url) {
65   return GURL(
66       GetManifestURLPrefix() +
67       net::EscapeQueryParamValue(
68           net::EscapeQueryParamValue(starting_url.spec(), false), false));
69 }
70 
71 // Attempts to parse a protobuf message from the response string of a
72 // URLFetcher. If parsing is successful, the message parameter will contain the
73 // parsed protobuf and this function will return true. Otherwise, returns false.
ParseProtoFromFetchResponse(const URLFetcher & source,::google::protobuf::MessageLite * message)74 bool ParseProtoFromFetchResponse(const URLFetcher& source,
75                                  ::google::protobuf::MessageLite* message) {
76   std::string response_string;
77 
78   if (!source.GetStatus().is_success()) {
79     DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
80     return false;
81   }
82   if (!source.GetResponseAsString(&response_string)) {
83     DLOG(WARNING) << "No response string present: "
84                   << source.GetOriginalURL().spec();
85     return false;
86   }
87   if (!message->ParseFromString(response_string)) {
88     DLOG(WARNING) << "Unable to parse proto served from "
89                   << source.GetOriginalURL().spec();
90     return false;
91   }
92   return true;
93 }
94 
95 }  // namespace
96 
97 // Class that fetches a URL, and runs the specified callback when the fetch is
98 // complete. This class exists so that a different method can be run in
99 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
100 // configuration settings are fetched, OnManifestFetchComplete when a manifest
101 // is fetched, etc.
102 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
103  public:
104   // Construct a new Fetcher. This will create and start a new URLFetcher for
105   // the specified URL using the specified request context.
106   Fetcher(net::URLRequestContextGetter* request_context, const GURL& url,
107           const base::Callback<void(const URLFetcher&)>& callback);
~Fetcher()108   virtual ~Fetcher() {}
109   virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
110 
111  private:
112   const base::Callback<void(const URLFetcher&)> callback_;
113   scoped_ptr<URLFetcher> url_fetcher_;
114 
115   DISALLOW_COPY_AND_ASSIGN(Fetcher);
116 };
117 
Fetcher(net::URLRequestContextGetter * request_context,const GURL & url,const base::Callback<void (const URLFetcher &)> & callback)118 PrecacheFetcher::Fetcher::Fetcher(
119     net::URLRequestContextGetter* request_context, const GURL& url,
120     const base::Callback<void(const URLFetcher&)>& callback)
121     : callback_(callback) {
122   url_fetcher_.reset(URLFetcher::Create(url, URLFetcher::GET, this));
123   url_fetcher_->SetRequestContext(request_context);
124   url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_PROMPT_FOR_LOGIN);
125   url_fetcher_->Start();
126 }
127 
OnURLFetchComplete(const URLFetcher * source)128 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
129   callback_.Run(*source);
130 }
131 
PrecacheFetcher(const std::list<GURL> & starting_urls,net::URLRequestContextGetter * request_context,PrecacheFetcher::PrecacheDelegate * precache_delegate)132 PrecacheFetcher::PrecacheFetcher(
133     const std::list<GURL>& starting_urls,
134     net::URLRequestContextGetter* request_context,
135     PrecacheFetcher::PrecacheDelegate* precache_delegate)
136     : starting_urls_(starting_urls),
137       request_context_(request_context),
138       precache_delegate_(precache_delegate) {
139   DCHECK(request_context_.get());  // Request context must be non-NULL.
140   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
141 
142   DCHECK_NE(GURL(), GetConfigURL())
143       << "Could not determine the precache config settings URL.";
144   DCHECK_NE(std::string(), GetManifestURLPrefix())
145       << "Could not determine the precache manifest URL prefix.";
146 }
147 
~PrecacheFetcher()148 PrecacheFetcher::~PrecacheFetcher() {
149 }
150 
Start()151 void PrecacheFetcher::Start() {
152   DCHECK(!fetcher_);  // Start shouldn't be called repeatedly.
153 
154   GURL config_url = GetConfigURL();
155   DCHECK(config_url.is_valid());
156 
157   // Fetch the precache configuration settings from the server.
158   fetcher_.reset(new Fetcher(request_context_.get(),
159                              config_url,
160                              base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
161                                         base::Unretained(this))));
162 }
163 
StartNextFetch()164 void PrecacheFetcher::StartNextFetch() {
165   if (!resource_urls_to_fetch_.empty()) {
166     // Fetch the next resource URL.
167     fetcher_.reset(
168         new Fetcher(request_context_.get(),
169                     resource_urls_to_fetch_.front(),
170                     base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
171                                base::Unretained(this))));
172 
173     resource_urls_to_fetch_.pop_front();
174     return;
175   }
176 
177   if (!manifest_urls_to_fetch_.empty()) {
178     // Fetch the next manifest URL.
179     fetcher_.reset(
180         new Fetcher(request_context_.get(),
181                     manifest_urls_to_fetch_.front(),
182                     base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
183                                base::Unretained(this))));
184 
185     manifest_urls_to_fetch_.pop_front();
186     return;
187   }
188 
189   // There are no more URLs to fetch, so end the precache cycle.
190   precache_delegate_->OnDone();
191   // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
192   // is called.
193 }
194 
OnConfigFetchComplete(const URLFetcher & source)195 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
196   PrecacheConfigurationSettings config;
197 
198   if (ParseProtoFromFetchResponse(source, &config)) {
199     // Keep track of starting URLs that manifests are being fetched for, in
200     // order to remove duplicates. This is a hash set on strings, and not GURLs,
201     // because there is no hash function defined for GURL.
202     base::hash_set<std::string> unique_starting_urls;
203 
204     // Attempt to fetch manifests for starting URLs up to the maximum top sites
205     // count. If a manifest does not exist for a particular starting URL, then
206     // the fetch will fail, and that starting URL will be ignored.
207     int64 rank = 0;
208     for (std::list<GURL>::const_iterator it = starting_urls_.begin();
209          it != starting_urls_.end() && rank < config.top_sites_count();
210          ++it, ++rank) {
211       if (unique_starting_urls.find(it->spec()) == unique_starting_urls.end()) {
212         // Only add a fetch for the manifest URL if this manifest isn't already
213         // going to be fetched.
214         manifest_urls_to_fetch_.push_back(ConstructManifestURL(*it));
215         unique_starting_urls.insert(it->spec());
216       }
217     }
218 
219     for (int i = 0; i < config.forced_starting_url_size(); ++i) {
220       // Convert the string URL into a GURL and take the spec() of it so that
221       // the URL string gets canonicalized.
222       GURL url(config.forced_starting_url(i));
223       if (unique_starting_urls.find(url.spec()) == unique_starting_urls.end()) {
224         // Only add a fetch for the manifest URL if this manifest isn't already
225         // going to be fetched.
226         manifest_urls_to_fetch_.push_back(ConstructManifestURL(url));
227         unique_starting_urls.insert(url.spec());
228       }
229     }
230   }
231 
232   StartNextFetch();
233 }
234 
OnManifestFetchComplete(const URLFetcher & source)235 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
236   PrecacheManifest manifest;
237 
238   if (ParseProtoFromFetchResponse(source, &manifest)) {
239     for (int i = 0; i < manifest.resource_size(); ++i) {
240       if (manifest.resource(i).has_url()) {
241         resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
242       }
243     }
244   }
245 
246   StartNextFetch();
247 }
248 
OnResourceFetchComplete(const URLFetcher & source)249 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
250   // The resource has already been put in the cache during the fetch process, so
251   // nothing more needs to be done for the resource.
252   StartNextFetch();
253 }
254 
255 }  // namespace precache
256