1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // The Safe Browsing service is responsible for downloading anti-phishing and
6 // anti-malware tables and checking urls against them.
8 #ifndef CHROME_BROWSER_SAFE_BROWSING_DATABASE_MANAGER_H_
9 #define CHROME_BROWSER_SAFE_BROWSING_DATABASE_MANAGER_H_
17 #include "base/callback.h"
18 #include "base/containers/hash_tables.h"
19 #include "base/memory/ref_counted.h"
20 #include "base/memory/scoped_ptr.h"
21 #include "base/synchronization/lock.h"
22 #include "base/time/time.h"
23 #include "chrome/browser/safe_browsing/protocol_manager.h"
24 #include "chrome/browser/safe_browsing/safe_browsing_util.h"
27 class SafeBrowsingService;
28 class SafeBrowsingDatabase;
35 class URLRequestContext;
36 class URLRequestContextGetter;
39 namespace safe_browsing {
40 class ClientSideDetectionService;
41 class DownloadProtectionService;
44 // Construction needs to happen on the main thread.
45 class SafeBrowsingDatabaseManager
46 : public base::RefCountedThreadSafe<SafeBrowsingDatabaseManager>,
47 public SafeBrowsingProtocolManagerDelegate {
51 // Bundle of SafeBrowsing state while performing a URL or hash prefix check.
52 struct SafeBrowsingCheck {
53 // |check_type| should correspond to the type of item that is being
54 // checked, either a URL or a binary hash/URL. We store this for two
55 // purposes: to know which of Client's methods to call when a result is
56 // known, and for logging purposes. It *isn't* used to predict the response
57 // list type, that is information that the server gives us.
58 SafeBrowsingCheck(const std::vector<GURL>& urls,
59 const std::vector<SBFullHash>& full_hashes,
61 safe_browsing_util::ListType check_type,
62 const std::vector<SBThreatType>& expected_threats);
65 // Either |urls| or |full_hashes| is used to lookup database. |*_results|
66 // are parallel vectors containing the results. They are initialized to
67 // contain SB_THREAT_TYPE_SAFE.
68 std::vector<GURL> urls;
69 std::vector<SBThreatType> url_results;
70 std::vector<SBFullHash> full_hashes;
71 std::vector<SBThreatType> full_hash_results;
75 base::TimeTicks start; // When check was sent to SB service.
76 safe_browsing_util::ListType check_type; // See comment in constructor.
77 std::vector<SBThreatType> expected_threats;
78 std::vector<SBPrefix> prefix_hits;
79 std::vector<SBFullHashResult> cache_hits;
81 // Vends weak pointers for TimeoutCallback(). If the response is
82 // received before the timeout fires, factory is destructed and
83 // the timeout won't be fired.
84 // TODO(lzheng): We should consider to use this time out check
85 // for browsing too (instead of implementin in
86 // safe_browsing_resource_handler.cc).
87 scoped_ptr<base::WeakPtrFactory<
88 SafeBrowsingDatabaseManager> > timeout_factory_;
91 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingCheck);
96 void OnSafeBrowsingResult(const SafeBrowsingCheck& check);
101 // Called when the result of checking a browse URL is known.
102 virtual void OnCheckBrowseUrlResult(const GURL& url,
103 SBThreatType threat_type) {}
105 // Called when the result of checking a download URL is known.
106 virtual void OnCheckDownloadUrlResult(const std::vector<GURL>& url_chain,
107 SBThreatType threat_type) {}
109 // Called when the result of checking a set of extensions is known.
110 virtual void OnCheckExtensionsResult(
111 const std::set<std::string>& threats) {}
114 // Creates the safe browsing service. Need to initialize before using.
115 explicit SafeBrowsingDatabaseManager(
116 const scoped_refptr<SafeBrowsingService>& service);
118 // Returns true if the url's scheme can be checked.
119 bool CanCheckUrl(const GURL& url) const;
121 // Returns whether download protection is enabled.
122 bool download_protection_enabled() const {
123 return enable_download_protection_;
126 // Called on the IO thread to check if the given url is safe or not. If we
127 // can synchronously determine that the url is safe, CheckUrl returns true.
128 // Otherwise it returns false, and "client" is called asynchronously with the
129 // result when it is ready.
130 virtual bool CheckBrowseUrl(const GURL& url, Client* client);
132 // Check if the prefix for |url| is in safebrowsing download add lists.
133 // Result will be passed to callback in |client|.
134 virtual bool CheckDownloadUrl(const std::vector<GURL>& url_chain,
137 // Check which prefixes in |extension_ids| are in the safebrowsing blacklist.
138 // Returns true if not, false if further checks need to be made in which case
139 // the result will be passed to |client|.
140 virtual bool CheckExtensionIDs(const std::set<std::string>& extension_ids,
143 // Check if the given url is on the side-effect free whitelist.
144 // Can be called on any thread. Returns false if the check cannot be performed
145 // (e.g. because we are disabled or because of an invalid scheme in the URL).
146 // Otherwise, returns true if the URL is on the whitelist based on matching
147 // the hash prefix only (so there may be false positives).
148 virtual bool CheckSideEffectFreeWhitelistUrl(const GURL& url);
150 // Check if the |url| matches any of the full-length hashes from the
151 // client-side phishing detection whitelist. Returns true if there was a
152 // match and false otherwise. To make sure we are conservative we will return
153 // true if an error occurs. This method is expected to be called on the IO
155 virtual bool MatchCsdWhitelistUrl(const GURL& url);
157 // Check if the given IP address (either IPv4 or IPv6) matches the malware
159 virtual bool MatchMalwareIP(const std::string& ip_address);
161 // Check if the |url| matches any of the full-length hashes from the
162 // download whitelist. Returns true if there was a match and false otherwise.
163 // To make sure we are conservative we will return true if an error occurs.
164 // This method is expected to be called on the IO thread.
165 virtual bool MatchDownloadWhitelistUrl(const GURL& url);
167 // Check if |str| matches any of the full-length hashes from the download
168 // whitelist. Returns true if there was a match and false otherwise.
169 // To make sure we are conservative we will return true if an error occurs.
170 // This method is expected to be called on the IO thread.
171 virtual bool MatchDownloadWhitelistString(const std::string& str);
173 // Check if the CSD malware IP matching kill switch is turned on.
174 virtual bool IsMalwareKillSwitchOn();
176 // Check if the CSD whitelist kill switch is turned on.
177 virtual bool IsCsdWhitelistKillSwitchOn();
179 // Called on the IO thread to cancel a pending check if the result is no
181 void CancelCheck(Client* client);
183 // Called on the IO thread when the SafeBrowsingProtocolManager has received
184 // the full hash results for prefix hits detected in the database.
185 void HandleGetHashResults(SafeBrowsingCheck* check,
186 const std::vector<SBFullHashResult>& full_hashes,
187 const base::TimeDelta& cache_lifetime);
189 // Log the user perceived delay caused by SafeBrowsing. This delay is the time
190 // delta starting from when we would have started reading data from the
191 // network, and ending when the SafeBrowsing check completes indicating that
192 // the current page is 'safe'.
193 void LogPauseDelay(base::TimeDelta time);
195 // Called to initialize objects that are used on the io_thread. This may be
196 // called multiple times during the life of the DatabaseManager. Should be
197 // called on IO thread.
198 void StartOnIOThread();
200 // Called to stop or shutdown operations on the io_thread. This may be called
201 // multiple times during the life of the DatabaseManager. Should be called
202 // on IO thread. If shutdown is true, the manager is disabled permanently.
203 void StopOnIOThread(bool shutdown);
206 virtual ~SafeBrowsingDatabaseManager();
208 // protected for tests.
209 void NotifyDatabaseUpdateFinished(bool update_succeeded);
212 friend class base::RefCountedThreadSafe<SafeBrowsingDatabaseManager>;
213 friend class SafeBrowsingServerTest;
214 friend class SafeBrowsingServiceTest;
215 friend class SafeBrowsingServiceTestHelper;
216 friend class SafeBrowsingDatabaseManagerTest;
217 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingDatabaseManagerTest, GetUrlThreatType);
219 typedef std::set<SafeBrowsingCheck*> CurrentChecks;
220 typedef std::vector<SafeBrowsingCheck*> GetHashRequestors;
221 typedef base::hash_map<SBPrefix, GetHashRequestors> GetHashRequests;
223 // Clients that we've queued up for checking later once the database is ready.
225 QueuedCheck(const safe_browsing_util::ListType check_type,
228 const std::vector<SBThreatType>& expected_threats,
229 const base::TimeTicks& start);
231 safe_browsing_util::ListType check_type;
234 std::vector<SBThreatType> expected_threats;
235 base::TimeTicks start; // When check was queued.
238 // Return the threat type from the first result in |full_hashes| which matches
239 // |hash|, or SAFE if none match.
240 static SBThreatType GetHashThreatType(
241 const SBFullHash& hash,
242 const std::vector<SBFullHashResult>& full_hashes);
244 // Given a URL, compare all the possible host + path full hashes to the set of
245 // provided full hashes. Returns the threat type of the matching result from
246 // |full_hashes|, or SAFE if none match.
247 static SBThreatType GetUrlThreatType(
249 const std::vector<SBFullHashResult>& full_hashes);
251 // Called to stop operations on the io_thread. This may be called multiple
252 // times during the life of the DatabaseManager. Should be called on IO
254 void DoStopOnIOThread();
256 // Returns whether |database_| exists and is accessible.
257 bool DatabaseAvailable() const;
259 // Called on the IO thread. If the database does not exist, queues up a call
260 // on the db thread to create it. Returns whether the database is available.
262 // Note that this is only needed outside the db thread, since functions on the
263 // db thread can call GetDatabase() directly.
264 bool MakeDatabaseAvailable();
266 // Should only be called on db thread as SafeBrowsingDatabase is not
268 SafeBrowsingDatabase* GetDatabase();
270 // Called on the IO thread with the check result.
271 void OnCheckDone(SafeBrowsingCheck* info);
273 // Called on the database thread to retrieve chunks.
274 void GetAllChunksFromDatabase(GetChunksCallback callback);
276 // Called on the IO thread with the results of all chunks.
277 void OnGetAllChunksFromDatabase(const std::vector<SBListChunkRanges>& lists,
279 GetChunksCallback callback);
281 // Called on the IO thread after the database reports that it added a chunk.
282 void OnAddChunksComplete(AddChunksCallback callback);
284 // Notification that the database is done loading its bloom filter. We may
285 // have had to queue checks until the database is ready, and if so, this
287 void DatabaseLoadComplete();
289 // Called on the database thread to add/remove chunks and host keys.
290 void AddDatabaseChunks(const std::string& list,
291 scoped_ptr<ScopedVector<SBChunkData> > chunks,
292 AddChunksCallback callback);
294 void DeleteDatabaseChunks(
295 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes);
297 void NotifyClientBlockingComplete(Client* client, bool proceed);
299 void DatabaseUpdateFinished(bool update_succeeded);
301 // Called on the db thread to close the database. See CloseDatabase().
302 void OnCloseDatabase();
304 // Runs on the db thread to reset the database. We assume that resetting the
305 // database is a synchronous operation.
306 void OnResetDatabase();
308 // Internal worker function for processing full hashes.
309 void OnHandleGetHashResults(SafeBrowsingCheck* check,
310 const std::vector<SBFullHashResult>& full_hashes);
312 // Run one check against |full_hashes|. Returns |true| if the check
313 // finds a match in |full_hashes|.
314 bool HandleOneCheck(SafeBrowsingCheck* check,
315 const std::vector<SBFullHashResult>& full_hashes);
317 // Invoked by CheckDownloadUrl. It checks the download URL on
318 // safe_browsing_thread_.
319 void CheckDownloadUrlOnSBThread(SafeBrowsingCheck* check);
321 // The callback function when a safebrowsing check is timed out. Client will
322 // be notified that the safebrowsing check is SAFE when this happens.
323 void TimeoutCallback(SafeBrowsingCheck* check);
325 // Calls the Client's callback on IO thread after CheckDownloadUrl finishes.
326 void CheckDownloadUrlDone(SafeBrowsingCheck* check);
328 // Checks all extension ID hashes on safe_browsing_thread_.
329 void CheckExtensionIDsOnSBThread(SafeBrowsingCheck* check);
331 // Helper function that calls safe browsing client and cleans up |checks_|.
332 void SafeBrowsingCheckDone(SafeBrowsingCheck* check);
334 // Helper function to set |check| with default values and start a safe
335 // browsing check with timeout of |timeout|. |task| will be called on
336 // success, otherwise TimeoutCallback will be called.
337 void StartSafeBrowsingCheck(SafeBrowsingCheck* check,
338 const base::Closure& task);
340 // SafeBrowsingProtocolManageDelegate override
341 virtual void ResetDatabase() OVERRIDE;
342 virtual void UpdateStarted() OVERRIDE;
343 virtual void UpdateFinished(bool success) OVERRIDE;
344 virtual void GetChunks(GetChunksCallback callback) OVERRIDE;
345 virtual void AddChunks(const std::string& list,
346 scoped_ptr<ScopedVector<SBChunkData> > chunks,
347 AddChunksCallback callback) OVERRIDE;
348 virtual void DeleteChunks(
349 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) OVERRIDE;
351 scoped_refptr<SafeBrowsingService> sb_service_;
353 CurrentChecks checks_;
355 // Used for issuing only one GetHash request for a given prefix.
356 GetHashRequests gethash_requests_;
358 // The persistent database. We don't use a scoped_ptr because it
359 // needs to be destroyed on a different thread than this object.
360 SafeBrowsingDatabase* database_;
362 // Lock used to prevent possible data races due to compiler optimizations.
363 mutable base::Lock database_lock_;
365 // Whether the service is running. 'enabled_' is used by the
366 // SafeBrowsingDatabaseManager on the IO thread during normal operations.
369 // Indicate if download_protection is enabled by command switch
370 // so we allow this feature to be exersized.
371 bool enable_download_protection_;
373 // Indicate if client-side phishing detection whitelist should be enabled
375 bool enable_csd_whitelist_;
377 // Indicate if the download whitelist should be enabled or not.
378 bool enable_download_whitelist_;
380 // Indicate if the extension blacklist should be enabled.
381 bool enable_extension_blacklist_;
383 // Indicate if the side effect free whitelist should be enabled.
384 bool enable_side_effect_free_whitelist_;
386 // Indicate if the csd malware IP blacklist should be enabled.
387 bool enable_ip_blacklist_;
389 // The SafeBrowsing thread that runs database operations.
391 // Note: Functions that run on this thread should run synchronously and return
392 // to the IO thread, not post additional tasks back to this thread, lest we
393 // cause a race condition at shutdown time that leads to a database leak.
394 scoped_ptr<base::Thread> safe_browsing_thread_;
396 // Indicates if we're currently in an update cycle.
397 bool update_in_progress_;
399 // When true, newly fetched chunks may not in the database yet since the
400 // database is still updating.
401 bool database_update_in_progress_;
403 // Indicates if we're in the midst of trying to close the database. If this
404 // is true, nothing on the IO thread should access the database.
405 bool closing_database_;
407 std::deque<QueuedCheck> queued_checks_;
409 // Timeout to use for safe browsing checks.
410 base::TimeDelta check_timeout_;
412 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseManager);
415 #endif // CHROME_BROWSER_SAFE_BROWSING_DATABASE_MANAGER_H_