1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // The Safe Browsing service is responsible for downloading anti-phishing and
6 // anti-malware tables and checking urls against them.
8 #ifndef CHROME_BROWSER_SAFE_BROWSING_DATABASE_MANAGER_H_
9 #define CHROME_BROWSER_SAFE_BROWSING_DATABASE_MANAGER_H_
17 #include "base/callback.h"
18 #include "base/containers/hash_tables.h"
19 #include "base/memory/ref_counted.h"
20 #include "base/memory/scoped_ptr.h"
21 #include "base/synchronization/lock.h"
22 #include "base/time/time.h"
23 #include "chrome/browser/safe_browsing/protocol_manager.h"
24 #include "chrome/browser/safe_browsing/safe_browsing_util.h"
27 class SafeBrowsingService;
28 class SafeBrowsingDatabase;
35 class URLRequestContext;
36 class URLRequestContextGetter;
39 namespace safe_browsing {
40 class ClientSideDetectionService;
41 class DownloadProtectionService;
44 // Construction needs to happen on the main thread.
45 class SafeBrowsingDatabaseManager
46 : public base::RefCountedThreadSafe<SafeBrowsingDatabaseManager>,
47 public SafeBrowsingProtocolManagerDelegate {
51 // Bundle of SafeBrowsing state while performing a URL or hash prefix check.
52 struct SafeBrowsingCheck {
53 // |check_type| should correspond to the type of item that is being
54 // checked, either a URL or a binary hash/URL. We store this for two
55 // purposes: to know which of Client's methods to call when a result is
56 // known, and for logging purposes. It *isn't* used to predict the response
57 // list type, that is information that the server gives us.
58 SafeBrowsingCheck(const std::vector<GURL>& urls,
59 const std::vector<SBFullHash>& full_hashes,
61 safe_browsing_util::ListType check_type,
62 const std::vector<SBThreatType>& expected_threats);
65 // Either |urls| or |full_hashes| is used to lookup database. |*_results|
66 // are parallel vectors containing the results. They are initialized to
67 // contain SB_THREAT_TYPE_SAFE.
68 std::vector<GURL> urls;
69 std::vector<SBThreatType> url_results;
70 std::vector<std::string> url_metadata;
71 std::vector<SBFullHash> full_hashes;
72 std::vector<SBThreatType> full_hash_results;
76 base::TimeTicks start; // When check was sent to SB service.
77 safe_browsing_util::ListType check_type; // See comment in constructor.
78 std::vector<SBThreatType> expected_threats;
79 std::vector<SBPrefix> prefix_hits;
80 std::vector<SBFullHashResult> cache_hits;
82 // Vends weak pointers for TimeoutCallback(). If the response is
83 // received before the timeout fires, factory is destructed and
84 // the timeout won't be fired.
85 // TODO(lzheng): We should consider to use this time out check
86 // for browsing too (instead of implementin in
87 // safe_browsing_resource_handler.cc).
88 scoped_ptr<base::WeakPtrFactory<
89 SafeBrowsingDatabaseManager> > timeout_factory_;
92 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingCheck);
97 void OnSafeBrowsingResult(const SafeBrowsingCheck& check);
102 // Called when the result of checking a browse URL is known.
103 virtual void OnCheckBrowseUrlResult(const GURL& url,
104 SBThreatType threat_type,
105 const std::string& metadata) {}
107 // Called when the result of checking a download URL is known.
108 virtual void OnCheckDownloadUrlResult(const std::vector<GURL>& url_chain,
109 SBThreatType threat_type) {}
111 // Called when the result of checking a set of extensions is known.
112 virtual void OnCheckExtensionsResult(
113 const std::set<std::string>& threats) {}
116 // Creates the safe browsing service. Need to initialize before using.
117 explicit SafeBrowsingDatabaseManager(
118 const scoped_refptr<SafeBrowsingService>& service);
120 // Returns true if the url's scheme can be checked.
121 bool CanCheckUrl(const GURL& url) const;
123 // Returns whether download protection is enabled.
124 bool download_protection_enabled() const {
125 return enable_download_protection_;
128 // Called on the IO thread to check if the given url is safe or not. If we
129 // can synchronously determine that the url is safe, CheckUrl returns true.
130 // Otherwise it returns false, and "client" is called asynchronously with the
131 // result when it is ready.
132 virtual bool CheckBrowseUrl(const GURL& url, Client* client);
134 // Check if the prefix for |url| is in safebrowsing download add lists.
135 // Result will be passed to callback in |client|.
136 virtual bool CheckDownloadUrl(const std::vector<GURL>& url_chain,
139 // Check which prefixes in |extension_ids| are in the safebrowsing blacklist.
140 // Returns true if not, false if further checks need to be made in which case
141 // the result will be passed to |client|.
142 virtual bool CheckExtensionIDs(const std::set<std::string>& extension_ids,
145 // Check if the given url is on the side-effect free whitelist.
146 // Can be called on any thread. Returns false if the check cannot be performed
147 // (e.g. because we are disabled or because of an invalid scheme in the URL).
148 // Otherwise, returns true if the URL is on the whitelist based on matching
149 // the hash prefix only (so there may be false positives).
150 virtual bool CheckSideEffectFreeWhitelistUrl(const GURL& url);
152 // Check if the |url| matches any of the full-length hashes from the
153 // client-side phishing detection whitelist. Returns true if there was a
154 // match and false otherwise. To make sure we are conservative we will return
155 // true if an error occurs. This method is expected to be called on the IO
157 virtual bool MatchCsdWhitelistUrl(const GURL& url);
159 // Check if the given IP address (either IPv4 or IPv6) matches the malware
161 virtual bool MatchMalwareIP(const std::string& ip_address);
163 // Check if the |url| matches any of the full-length hashes from the
164 // download whitelist. Returns true if there was a match and false otherwise.
165 // To make sure we are conservative we will return true if an error occurs.
166 // This method is expected to be called on the IO thread.
167 virtual bool MatchDownloadWhitelistUrl(const GURL& url);
169 // Check if |str| matches any of the full-length hashes from the download
170 // whitelist. Returns true if there was a match and false otherwise.
171 // To make sure we are conservative we will return true if an error occurs.
172 // This method is expected to be called on the IO thread.
173 virtual bool MatchDownloadWhitelistString(const std::string& str);
175 // Check if the CSD malware IP matching kill switch is turned on.
176 virtual bool IsMalwareKillSwitchOn();
178 // Check if the CSD whitelist kill switch is turned on.
179 virtual bool IsCsdWhitelistKillSwitchOn();
181 // Called on the IO thread to cancel a pending check if the result is no
183 void CancelCheck(Client* client);
185 // Called on the IO thread when the SafeBrowsingProtocolManager has received
186 // the full hash results for prefix hits detected in the database.
187 void HandleGetHashResults(SafeBrowsingCheck* check,
188 const std::vector<SBFullHashResult>& full_hashes,
189 const base::TimeDelta& cache_lifetime);
191 // Log the user perceived delay caused by SafeBrowsing. This delay is the time
192 // delta starting from when we would have started reading data from the
193 // network, and ending when the SafeBrowsing check completes indicating that
194 // the current page is 'safe'.
195 void LogPauseDelay(base::TimeDelta time);
197 // Called to initialize objects that are used on the io_thread. This may be
198 // called multiple times during the life of the DatabaseManager. Should be
199 // called on IO thread.
200 void StartOnIOThread();
202 // Called to stop or shutdown operations on the io_thread. This may be called
203 // multiple times during the life of the DatabaseManager. Should be called
204 // on IO thread. If shutdown is true, the manager is disabled permanently.
205 void StopOnIOThread(bool shutdown);
208 ~SafeBrowsingDatabaseManager() override;
210 // protected for tests.
211 void NotifyDatabaseUpdateFinished(bool update_succeeded);
214 friend class base::RefCountedThreadSafe<SafeBrowsingDatabaseManager>;
215 friend class SafeBrowsingServerTest;
216 friend class SafeBrowsingServiceTest;
217 friend class SafeBrowsingServiceTestHelper;
218 friend class SafeBrowsingDatabaseManagerTest;
219 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingDatabaseManagerTest,
220 GetUrlSeverestThreatType);
222 typedef std::set<SafeBrowsingCheck*> CurrentChecks;
223 typedef std::vector<SafeBrowsingCheck*> GetHashRequestors;
224 typedef base::hash_map<SBPrefix, GetHashRequestors> GetHashRequests;
226 // Clients that we've queued up for checking later once the database is ready.
228 QueuedCheck(const safe_browsing_util::ListType check_type,
231 const std::vector<SBThreatType>& expected_threats,
232 const base::TimeTicks& start);
234 safe_browsing_util::ListType check_type;
237 std::vector<SBThreatType> expected_threats;
238 base::TimeTicks start; // When check was queued.
241 // Return the threat type of the severest entry in |full_hashes| which matches
242 // |hash|, or SAFE if none match.
243 static SBThreatType GetHashSeverestThreatType(
244 const SBFullHash& hash,
245 const std::vector<SBFullHashResult>& full_hashes);
247 // Given a URL, compare all the possible host + path full hashes to the set of
248 // provided full hashes. Returns the threat type of the severest matching
249 // result from |full_hashes|, or SAFE if none match.
250 static SBThreatType GetUrlSeverestThreatType(
252 const std::vector<SBFullHashResult>& full_hashes,
255 // Called to stop operations on the io_thread. This may be called multiple
256 // times during the life of the DatabaseManager. Should be called on IO
258 void DoStopOnIOThread();
260 // Returns whether |database_| exists and is accessible.
261 bool DatabaseAvailable() const;
263 // Called on the IO thread. If the database does not exist, queues up a call
264 // on the db thread to create it. Returns whether the database is available.
266 // Note that this is only needed outside the db thread, since functions on the
267 // db thread can call GetDatabase() directly.
268 bool MakeDatabaseAvailable();
270 // Should only be called on db thread as SafeBrowsingDatabase is not
272 SafeBrowsingDatabase* GetDatabase();
274 // Called on the IO thread with the check result.
275 void OnCheckDone(SafeBrowsingCheck* info);
277 // Called on the database thread to retrieve chunks.
278 void GetAllChunksFromDatabase(GetChunksCallback callback);
280 // Called on the IO thread with the results of all chunks.
281 void OnGetAllChunksFromDatabase(const std::vector<SBListChunkRanges>& lists,
283 GetChunksCallback callback);
285 // Called on the IO thread after the database reports that it added a chunk.
286 void OnAddChunksComplete(AddChunksCallback callback);
288 // Notification that the database is done loading its bloom filter. We may
289 // have had to queue checks until the database is ready, and if so, this
291 void DatabaseLoadComplete();
293 // Called on the database thread to add/remove chunks and host keys.
294 void AddDatabaseChunks(const std::string& list,
295 scoped_ptr<ScopedVector<SBChunkData> > chunks,
296 AddChunksCallback callback);
298 void DeleteDatabaseChunks(
299 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes);
301 void NotifyClientBlockingComplete(Client* client, bool proceed);
303 void DatabaseUpdateFinished(bool update_succeeded);
305 // Called on the db thread to close the database. See CloseDatabase().
306 void OnCloseDatabase();
308 // Runs on the db thread to reset the database. We assume that resetting the
309 // database is a synchronous operation.
310 void OnResetDatabase();
312 // Internal worker function for processing full hashes.
313 void OnHandleGetHashResults(SafeBrowsingCheck* check,
314 const std::vector<SBFullHashResult>& full_hashes);
316 // Run one check against |full_hashes|. Returns |true| if the check
317 // finds a match in |full_hashes|.
318 bool HandleOneCheck(SafeBrowsingCheck* check,
319 const std::vector<SBFullHashResult>& full_hashes);
321 // Invoked by CheckDownloadUrl. It checks the download URL on
322 // safe_browsing_thread_.
323 void CheckDownloadUrlOnSBThread(SafeBrowsingCheck* check);
325 // The callback function when a safebrowsing check is timed out. Client will
326 // be notified that the safebrowsing check is SAFE when this happens.
327 void TimeoutCallback(SafeBrowsingCheck* check);
329 // Calls the Client's callback on IO thread after CheckDownloadUrl finishes.
330 void CheckDownloadUrlDone(SafeBrowsingCheck* check);
332 // Checks all extension ID hashes on safe_browsing_thread_.
333 void CheckExtensionIDsOnSBThread(SafeBrowsingCheck* check);
335 // Helper function that calls safe browsing client and cleans up |checks_|.
336 void SafeBrowsingCheckDone(SafeBrowsingCheck* check);
338 // Helper function to set |check| with default values and start a safe
339 // browsing check with timeout of |timeout|. |task| will be called on
340 // success, otherwise TimeoutCallback will be called.
341 void StartSafeBrowsingCheck(SafeBrowsingCheck* check,
342 const base::Closure& task);
344 // SafeBrowsingProtocolManageDelegate override
345 void ResetDatabase() override;
346 void UpdateStarted() override;
347 void UpdateFinished(bool success) override;
348 void GetChunks(GetChunksCallback callback) override;
349 void AddChunks(const std::string& list,
350 scoped_ptr<ScopedVector<SBChunkData>> chunks,
351 AddChunksCallback callback) override;
353 scoped_ptr<std::vector<SBChunkDelete>> chunk_deletes) override;
355 scoped_refptr<SafeBrowsingService> sb_service_;
357 CurrentChecks checks_;
359 // Used for issuing only one GetHash request for a given prefix.
360 GetHashRequests gethash_requests_;
362 // The persistent database. We don't use a scoped_ptr because it
363 // needs to be destroyed on a different thread than this object.
364 SafeBrowsingDatabase* database_;
366 // Lock used to prevent possible data races due to compiler optimizations.
367 mutable base::Lock database_lock_;
369 // Whether the service is running. 'enabled_' is used by the
370 // SafeBrowsingDatabaseManager on the IO thread during normal operations.
373 // Indicate if download_protection is enabled by command switch
374 // so we allow this feature to be exersized.
375 bool enable_download_protection_;
377 // Indicate if client-side phishing detection whitelist should be enabled
379 bool enable_csd_whitelist_;
381 // Indicate if the download whitelist should be enabled or not.
382 bool enable_download_whitelist_;
384 // Indicate if the extension blacklist should be enabled.
385 bool enable_extension_blacklist_;
387 // Indicate if the side effect free whitelist should be enabled.
388 bool enable_side_effect_free_whitelist_;
390 // Indicate if the csd malware IP blacklist should be enabled.
391 bool enable_ip_blacklist_;
393 // Indicate if the unwanted software blacklist should be enabled.
394 bool enable_unwanted_software_blacklist_;
396 // The SafeBrowsing thread that runs database operations.
398 // Note: Functions that run on this thread should run synchronously and return
399 // to the IO thread, not post additional tasks back to this thread, lest we
400 // cause a race condition at shutdown time that leads to a database leak.
401 scoped_ptr<base::Thread> safe_browsing_thread_;
403 // Indicates if we're currently in an update cycle.
404 bool update_in_progress_;
406 // When true, newly fetched chunks may not in the database yet since the
407 // database is still updating.
408 bool database_update_in_progress_;
410 // Indicates if we're in the midst of trying to close the database. If this
411 // is true, nothing on the IO thread should access the database.
412 bool closing_database_;
414 std::deque<QueuedCheck> queued_checks_;
416 // Timeout to use for safe browsing checks.
417 base::TimeDelta check_timeout_;
419 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseManager);
422 #endif // CHROME_BROWSER_SAFE_BROWSING_DATABASE_MANAGER_H_