1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "components/precache/core/precache_fetcher.h"
10 #include "base/callback.h"
11 #include "base/command_line.h"
12 #include "base/compiler_specific.h"
13 #include "base/containers/hash_tables.h"
14 #include "components/precache/core/precache_switches.h"
15 #include "components/precache/core/proto/precache.pb.h"
16 #include "net/base/escape.h"
17 #include "net/base/load_flags.h"
18 #include "net/url_request/url_fetcher.h"
19 #include "net/url_request/url_fetcher_delegate.h"
20 #include "net/url_request/url_request_context_getter.h"
21 #include "net/url_request/url_request_status.h"
23 using net::URLFetcher;
30 const CommandLine& command_line = *CommandLine::ForCurrentProcess();
31 if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
33 command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
36 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
37 return GURL(PRECACHE_CONFIG_SETTINGS_URL);
39 // The precache config settings URL could not be determined, so return an
40 // empty, invalid GURL.
45 std::string GetManifestURLPrefix() {
46 const CommandLine& command_line = *CommandLine::ForCurrentProcess();
47 if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
48 return command_line.GetSwitchValueASCII(
49 switches::kPrecacheManifestURLPrefix);
52 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
53 return PRECACHE_MANIFEST_URL_PREFIX;
55 // The precache manifest URL prefix could not be determined, so return an
61 // Construct the URL of the precache manifest for the given starting URL.
62 // The server is expecting a request for a URL consisting of the manifest URL
63 // prefix followed by the doubly escaped starting URL.
64 GURL ConstructManifestURL(const GURL& starting_url) {
66 GetManifestURLPrefix() +
67 net::EscapeQueryParamValue(
68 net::EscapeQueryParamValue(starting_url.spec(), false), false));
71 // Attempts to parse a protobuf message from the response string of a
72 // URLFetcher. If parsing is successful, the message parameter will contain the
73 // parsed protobuf and this function will return true. Otherwise, returns false.
74 bool ParseProtoFromFetchResponse(const URLFetcher& source,
75 ::google::protobuf::MessageLite* message) {
76 std::string response_string;
78 if (!source.GetStatus().is_success()) {
79 DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
82 if (!source.GetResponseAsString(&response_string)) {
83 DLOG(WARNING) << "No response string present: "
84 << source.GetOriginalURL().spec();
87 if (!message->ParseFromString(response_string)) {
88 DLOG(WARNING) << "Unable to parse proto served from "
89 << source.GetOriginalURL().spec();
97 // Class that fetches a URL, and runs the specified callback when the fetch is
98 // complete. This class exists so that a different method can be run in
99 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
100 // configuration settings are fetched, OnManifestFetchComplete when a manifest
102 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
104 // Construct a new Fetcher. This will create and start a new URLFetcher for
105 // the specified URL using the specified request context.
106 Fetcher(net::URLRequestContextGetter* request_context, const GURL& url,
107 const base::Callback<void(const URLFetcher&)>& callback);
108 virtual ~Fetcher() {}
109 virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
112 const base::Callback<void(const URLFetcher&)> callback_;
113 scoped_ptr<URLFetcher> url_fetcher_;
115 DISALLOW_COPY_AND_ASSIGN(Fetcher);
118 PrecacheFetcher::Fetcher::Fetcher(
119 net::URLRequestContextGetter* request_context, const GURL& url,
120 const base::Callback<void(const URLFetcher&)>& callback)
121 : callback_(callback) {
122 url_fetcher_.reset(URLFetcher::Create(url, URLFetcher::GET, this));
123 url_fetcher_->SetRequestContext(request_context);
124 url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_PROMPT_FOR_LOGIN);
125 url_fetcher_->Start();
128 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
129 callback_.Run(*source);
132 PrecacheFetcher::PrecacheFetcher(
133 const std::list<GURL>& starting_urls,
134 net::URLRequestContextGetter* request_context,
135 PrecacheFetcher::PrecacheDelegate* precache_delegate)
136 : starting_urls_(starting_urls),
137 request_context_(request_context),
138 precache_delegate_(precache_delegate) {
139 DCHECK(request_context_); // Request context must be non-NULL.
140 DCHECK(precache_delegate_); // Precache delegate must be non-NULL.
142 DCHECK_NE(GURL(), GetConfigURL())
143 << "Could not determine the precache config settings URL.";
144 DCHECK_NE(std::string(), GetManifestURLPrefix())
145 << "Could not determine the precache manifest URL prefix.";
148 PrecacheFetcher::~PrecacheFetcher() {
151 void PrecacheFetcher::Start() {
152 DCHECK(!fetcher_); // Start shouldn't be called repeatedly.
154 GURL config_url = GetConfigURL();
155 DCHECK(config_url.is_valid());
157 // Fetch the precache configuration settings from the server.
158 fetcher_.reset(new Fetcher(request_context_, config_url,
159 base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
160 base::Unretained(this))));
163 void PrecacheFetcher::StartNextFetch() {
164 if (!resource_urls_to_fetch_.empty()) {
165 // Fetch the next resource URL.
167 new Fetcher(request_context_, resource_urls_to_fetch_.front(),
168 base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
169 base::Unretained(this))));
171 resource_urls_to_fetch_.pop_front();
175 if (!manifest_urls_to_fetch_.empty()) {
176 // Fetch the next manifest URL.
178 new Fetcher(request_context_, manifest_urls_to_fetch_.front(),
179 base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
180 base::Unretained(this))));
182 manifest_urls_to_fetch_.pop_front();
186 // There are no more URLs to fetch, so end the precache cycle.
187 precache_delegate_->OnDone();
188 // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
192 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
193 PrecacheConfigurationSettings config;
195 if (ParseProtoFromFetchResponse(source, &config)) {
196 // A hash set of strings is used instead of GURLs because there is no
197 // standard hash function defined for GURLs.
198 base::hash_set<std::string> whitelisted_urls;
199 for (int i = 0; i < config.whitelisted_starting_url_size(); ++i) {
200 // Instead of using the raw URL string, construct a GURL and take the spec
201 // of it so that the URL string is canonicalized.
202 whitelisted_urls.insert(GURL(config.whitelisted_starting_url(i)).spec());
205 // Only fetch manifests for starting URLs up to the maximum rank that are in
208 for (std::list<GURL>::const_iterator it = starting_urls_.begin();
209 it != starting_urls_.end() &&
210 rank < config.maximum_rank_starting_url();
212 if (whitelisted_urls.find(it->spec()) != whitelisted_urls.end()) {
213 manifest_urls_to_fetch_.push_back(ConstructManifestURL(*it));
221 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
222 PrecacheManifest manifest;
224 if (ParseProtoFromFetchResponse(source, &manifest)) {
225 for (int i = 0; i < manifest.resource_size(); ++i) {
226 if (manifest.resource(i).has_url()) {
227 resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
235 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
236 // The resource has already been put in the cache during the fetch process, so
237 // nothing more needs to be done for the resource.
241 } // namespace precache