1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef CHROME_BROWSER_SAFE_BROWSING_PROTOCOL_MANAGER_H_
6 #define CHROME_BROWSER_SAFE_BROWSING_PROTOCOL_MANAGER_H_
8 // A class that implements Chrome's interface with the SafeBrowsing protocol.
9 // See https://developers.google.com/safe-browsing/developers_guide_v2 for
12 // The SafeBrowsingProtocolManager handles formatting and making requests of,
13 // and handling responses from, Google's SafeBrowsing servers. This class uses
14 // The SafeBrowsingProtocolParser class to do the actual parsing.
21 #include "base/containers/hash_tables.h"
22 #include "base/gtest_prod_util.h"
23 #include "base/memory/scoped_ptr.h"
24 #include "base/threading/non_thread_safe.h"
25 #include "base/time/time.h"
26 #include "base/timer/timer.h"
27 #include "chrome/browser/safe_browsing/chunk_range.h"
28 #include "chrome/browser/safe_browsing/protocol_manager_helper.h"
29 #include "chrome/browser/safe_browsing/protocol_parser.h"
30 #include "chrome/browser/safe_browsing/safe_browsing_util.h"
31 #include "net/url_request/url_fetcher_delegate.h"
36 class URLRequestContextGetter;
39 #if defined(COMPILER_GCC)
40 // Allows us to use URLFetchers in a hash_map with gcc (MSVC is okay without
42 namespace BASE_HASH_NAMESPACE {
44 struct hash<const net::URLFetcher*> {
45 size_t operator()(const net::URLFetcher* fetcher) const {
46 return reinterpret_cast<size_t>(fetcher);
52 class SBProtocolManagerFactory;
53 class SafeBrowsingProtocolManagerDelegate;
55 class SafeBrowsingProtocolManager : public net::URLFetcherDelegate,
56 public base::NonThreadSafe {
58 // FullHashCallback is invoked when GetFullHash completes.
60 // - The vector of full hash results. If empty, indicates that there
61 // were no matches, and that the resource is safe.
62 // - Whether the result can be cached. This may not be the case when
63 // the result did not come from the SB server, for example.
64 typedef base::Callback<void(const std::vector<SBFullHashResult>&,
65 bool)> FullHashCallback;
67 virtual ~SafeBrowsingProtocolManager();
69 // Makes the passed |factory| the factory used to instantiate
70 // a SafeBrowsingService. Useful for tests.
71 static void RegisterFactory(SBProtocolManagerFactory* factory) {
75 // Create an instance of the safe browsing protocol manager.
76 static SafeBrowsingProtocolManager* Create(
77 SafeBrowsingProtocolManagerDelegate* delegate,
78 net::URLRequestContextGetter* request_context_getter,
79 const SafeBrowsingProtocolConfig& config);
81 // Sets up the update schedule and internal state for making periodic requests
82 // of the Safebrowsing servers.
83 virtual void Initialize();
85 // net::URLFetcherDelegate interface.
86 virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
88 // Retrieve the full hash for a set of prefixes, and invoke the callback
89 // argument when the results are retrieved. The callback may be invoked
91 virtual void GetFullHash(const std::vector<SBPrefix>& prefixes,
92 FullHashCallback callback,
95 // Forces the start of next update after |interval| time.
96 void ForceScheduleNextUpdate(base::TimeDelta interval);
98 // Scheduled update callback.
101 // Called by the SafeBrowsingService when our request for a list of all chunks
102 // for each list is done. If database_error is true, that means the protocol
103 // manager shouldn't fetch updates since they can't be written to disk. It
104 // should try again later to open the database.
105 void OnGetChunksComplete(const std::vector<SBListChunkRanges>& list,
106 bool database_error);
108 // The last time we received an update.
109 base::Time last_update() const { return last_update_; }
111 // Setter for additional_query_. To make sure the additional_query_ won't
112 // be changed in the middle of an update, caller (e.g.: SafeBrowsingService)
113 // should call this after callbacks triggered in UpdateFinished() or before
114 // IssueUpdateRequest().
115 void set_additional_query(const std::string& query) {
116 additional_query_ = query;
118 const std::string& additional_query() const {
119 return additional_query_;
122 // Enumerate failures for histogramming purposes. DO NOT CHANGE THE
123 // ORDERING OF THESE VALUES.
125 // 200 response code means that the server recognized the hash
126 // prefix, while 204 is an empty response indicating that the
127 // server did not recognize it.
131 // Subset of successful responses which returned no full hashes.
132 // This includes the STATUS_204 case, and the *_ERROR cases.
133 GET_HASH_FULL_HASH_EMPTY,
135 // Subset of successful responses for which one or more of the
136 // full hashes matched (should lead to an interstitial).
137 GET_HASH_FULL_HASH_HIT,
139 // Subset of successful responses which weren't empty and have no
140 // matches. It means that there was a prefix collision which was
141 // cleared up by the full hashes.
142 GET_HASH_FULL_HASH_MISS,
144 // Subset of successful responses where the response body wasn't parsable.
145 GET_HASH_PARSE_ERROR,
147 // Gethash request failed (network error).
148 GET_HASH_NETWORK_ERROR,
150 // Gethash request returned HTTP result code other than 200 or 204.
153 // Gethash attempted during error backoff, no request sent.
154 GET_HASH_BACKOFF_ERROR,
156 // Memory space for histograms is determined by the max. ALWAYS
157 // ADD NEW VALUES BEFORE THIS ONE.
161 // Record a GetHash result. |is_download| indicates if the get
162 // hash is triggered by download related lookup.
163 static void RecordGetHashResult(bool is_download,
164 ResultType result_type);
166 // Returns whether another update is currently scheduled.
167 bool IsUpdateScheduled() const;
170 // Constructs a SafeBrowsingProtocolManager for |delegate| that issues
171 // network requests using |request_context_getter|.
172 SafeBrowsingProtocolManager(
173 SafeBrowsingProtocolManagerDelegate* delegate,
174 net::URLRequestContextGetter* request_context_getter,
175 const SafeBrowsingProtocolConfig& config);
178 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest, TestBackOffTimes);
179 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest, TestChunkStrings);
180 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest, TestGetHashUrl);
181 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest,
182 TestGetHashBackOffTimes);
183 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest, TestNextChunkUrl);
184 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest, TestUpdateUrl);
185 friend class SafeBrowsingServerTest;
186 friend class SBProtocolManagerFactoryImpl;
188 // Internal API for fetching information from the SafeBrowsing servers. The
189 // GetHash requests are higher priority since they can block user requests
190 // so are handled separately.
191 enum SafeBrowsingRequestType {
192 NO_REQUEST = 0, // No requests in progress
193 UPDATE_REQUEST, // Request for redirect URLs
194 BACKUP_UPDATE_REQUEST, // Request for redirect URLs to a backup URL.
195 CHUNK_REQUEST, // Request for a specific chunk
198 // Which type of backup update request is being used.
199 enum BackupUpdateReason {
200 BACKUP_UPDATE_REASON_CONNECT,
201 BACKUP_UPDATE_REASON_HTTP,
202 BACKUP_UPDATE_REASON_NETWORK,
203 BACKUP_UPDATE_REASON_MAX,
206 // Generates Update URL for querying about the latest set of chunk updates.
207 GURL UpdateUrl() const;
209 // Generates backup Update URL for querying about the latest set of chunk
210 // updates. |url_prefix| is the base prefix to use.
211 GURL BackupUpdateUrl(BackupUpdateReason reason) const;
213 // Generates GetHash request URL for retrieving full hashes.
214 GURL GetHashUrl() const;
215 // Generates URL for reporting safe browsing hits for UMA users.
217 // Composes a ChunkUrl based on input string.
218 GURL NextChunkUrl(const std::string& input) const;
220 // Returns the time for the next update request. If |back_off| is true,
221 // the time returned will increment an error count and return the appriate
222 // next time (see ScheduleNextUpdate below).
223 base::TimeDelta GetNextUpdateInterval(bool back_off);
225 // Worker function for calculating GetHash and Update backoff times (in
226 // seconds). |multiplier| is doubled for each consecutive error between the
227 // 2nd and 5th, and |error_count| is incremented with each call.
228 base::TimeDelta GetNextBackOffInterval(int* error_count,
229 int* multiplier) const;
231 // Manages our update with the next allowable update time. If 'back_off_' is
232 // true, we must decrease the frequency of requests of the SafeBrowsing
233 // service according to section 5 of the protocol specification.
234 // When disable_auto_update_ is set, ScheduleNextUpdate will do nothing.
235 // ForceScheduleNextUpdate has to be called to trigger the update.
236 void ScheduleNextUpdate(bool back_off);
238 // Sends a request for a list of chunks we should download to the SafeBrowsing
239 // servers. In order to format this request, we need to send all the chunk
240 // numbers for each list that we have to the server. Getting the chunk numbers
241 // requires a database query (run on the database thread), and the request
242 // is sent upon completion of that query in OnGetChunksComplete.
243 void IssueUpdateRequest();
245 // Sends a backup request for a list of chunks to download, when the primary
246 // update request failed. |reason| specifies why the backup is needed. Unlike
247 // the primary IssueUpdateRequest, this does not need to hit the local
248 // SafeBrowsing database since the existing chunk numbers are remembered from
249 // the primary update request. Returns whether the backup request was issued -
250 // this may be false in cases where there is not a prefix specified.
251 bool IssueBackupUpdateRequest(BackupUpdateReason reason);
253 // Sends a request for a chunk to the SafeBrowsing servers.
254 void IssueChunkRequest();
256 // Formats a string returned from the database into:
257 // "list_name;a:<add_chunk_ranges>:s:<sub_chunk_ranges>\n"
258 static std::string FormatList(const SBListChunkRanges& list);
260 // Runs the protocol parser on received data and update the
261 // SafeBrowsingService with the new content. Returns 'true' on successful
262 // parse, 'false' on error.
263 bool HandleServiceResponse(const GURL& url, const char* data, int length);
265 // Updates internal state for each GetHash response error, assuming that the
266 // current time is |now|.
267 void HandleGetHashError(const base::Time& now);
269 // Helper function for update completion.
270 void UpdateFinished(bool success);
271 void UpdateFinished(bool success, bool back_off);
273 // A callback that runs if we timeout waiting for a response to an update
274 // request. We use this to properly set our update state.
275 void UpdateResponseTimeout();
277 // Called after the chunks are added to the database.
278 void OnAddChunksComplete();
281 // Map of GetHash requests to parameters which created it.
282 struct FullHashDetails {
284 FullHashDetails(FullHashCallback callback, bool is_download);
287 FullHashCallback callback;
290 typedef base::hash_map<const net::URLFetcher*, FullHashDetails> HashRequests;
292 // The factory that controls the creation of SafeBrowsingProtocolManager.
293 // This is used by tests.
294 static SBProtocolManagerFactory* factory_;
297 SafeBrowsingProtocolManagerDelegate* delegate_;
299 // Current active request (in case we need to cancel) for updates or chunks
300 // from the SafeBrowsing service. We can only have one of these outstanding
301 // at any given time unlike GetHash requests, which are tracked separately.
302 scoped_ptr<net::URLFetcher> request_;
304 // The kind of request that is currently in progress.
305 SafeBrowsingRequestType request_type_;
307 // The number of HTTP response errors, used for request backoff timing.
308 int update_error_count_;
309 int gethash_error_count_;
311 // Multipliers which double (max == 8) for each error after the second.
312 int update_back_off_mult_;
313 int gethash_back_off_mult_;
315 // Multiplier between 0 and 1 to spread clients over an interval.
316 float back_off_fuzz_;
318 // The list for which we are make a request.
319 std::string list_name_;
321 // For managing the next earliest time to query the SafeBrowsing servers for
323 base::TimeDelta next_update_interval_;
324 base::OneShotTimer<SafeBrowsingProtocolManager> update_timer_;
326 // timeout_timer_ is used to interrupt update requests which are taking
328 base::OneShotTimer<SafeBrowsingProtocolManager> timeout_timer_;
330 // All chunk requests that need to be made.
331 std::deque<ChunkUrl> chunk_request_urls_;
333 HashRequests hash_requests_;
335 // The next scheduled update has special behavior for the first 2 requests.
336 enum UpdateRequestState {
341 UpdateRequestState update_state_;
343 // True if the service has been given an add/sub chunk but it hasn't been
344 // added to the database yet.
345 bool chunk_pending_to_write_;
347 // The last time we successfully received an update.
348 base::Time last_update_;
350 // While in GetHash backoff, we can't make another GetHash until this time.
351 base::Time next_gethash_time_;
353 // Current product version sent in each request.
354 std::string version_;
356 // Used for measuring chunk request latency.
357 base::Time chunk_request_start_;
359 // Tracks the size of each update (in bytes).
362 // The safe browsing client name sent in each request.
363 std::string client_name_;
365 // A string that is appended to the end of URLs for download, gethash,
366 // safebrowsing hits and chunk update requests.
367 std::string additional_query_;
369 // The context we use to issue network requests.
370 scoped_refptr<net::URLRequestContextGetter> request_context_getter_;
372 // URL prefix where browser fetches safebrowsing chunk updates, and hashes.
373 std::string url_prefix_;
375 // Backup URL prefixes for updates.
376 std::string backup_url_prefixes_[BACKUP_UPDATE_REASON_MAX];
378 // The current reason why the backup update request is happening.
379 BackupUpdateReason backup_update_reason_;
381 // Data to POST when doing an update.
382 std::string update_list_data_;
384 // When true, protocol manager will not start an update unless
385 // ForceScheduleNextUpdate() is called. This is set for testing purpose.
386 bool disable_auto_update_;
388 // ID for URLFetchers for testing.
391 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingProtocolManager);
394 // Interface of a factory to create ProtocolManager. Useful for tests.
395 class SBProtocolManagerFactory {
397 SBProtocolManagerFactory() {}
398 virtual ~SBProtocolManagerFactory() {}
399 virtual SafeBrowsingProtocolManager* CreateProtocolManager(
400 SafeBrowsingProtocolManagerDelegate* delegate,
401 net::URLRequestContextGetter* request_context_getter,
402 const SafeBrowsingProtocolConfig& config) = 0;
404 DISALLOW_COPY_AND_ASSIGN(SBProtocolManagerFactory);
407 // Delegate interface for the SafeBrowsingProtocolManager.
408 class SafeBrowsingProtocolManagerDelegate {
410 typedef base::Callback<void(const std::vector<SBListChunkRanges>&, bool)>
412 typedef base::Callback<void(void)> AddChunksCallback;
414 virtual ~SafeBrowsingProtocolManagerDelegate();
416 // |UpdateStarted()| is called just before the SafeBrowsing update protocol
418 virtual void UpdateStarted() = 0;
420 // |UpdateFinished()| is called just after the SafeBrowsing update protocol
422 virtual void UpdateFinished(bool success) = 0;
424 // Wipe out the local database. The SafeBrowsing server can request this.
425 virtual void ResetDatabase() = 0;
427 // Retrieve all the local database chunks, and invoke |callback| with the
428 // results. The SafeBrowsingProtocolManagerDelegate must only invoke the
429 // callback if the SafeBrowsingProtocolManager is still alive. Only one call
430 // may be made to GetChunks at a time.
431 virtual void GetChunks(GetChunksCallback callback) = 0;
433 // Add new chunks to the database. Invokes |callback| when complete, but must
434 // call at a later time.
435 virtual void AddChunks(const std::string& list, SBChunkList* chunks,
436 AddChunksCallback callback) = 0;
438 // Delete chunks from the database.
439 virtual void DeleteChunks(
440 std::vector<SBChunkDelete>* delete_chunks) = 0;
443 #endif // CHROME_BROWSER_SAFE_BROWSING_PROTOCOL_MANAGER_H_