1 // Copyright 2017 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef COMPONENTS_UKM_UKM_RECORDER_IMPL_H_
6 #define COMPONENTS_UKM_UKM_RECORDER_IMPL_H_
13 #include <unordered_set>
16 #include "base/component_export.h"
17 #include "base/containers/flat_map.h"
18 #include "base/containers/flat_set.h"
19 #include "base/functional/callback_forward.h"
20 #include "base/gtest_prod_util.h"
21 #include "base/observer_list_threadsafe.h"
22 #include "base/sequence_checker.h"
23 #include "base/strings/string_piece.h"
24 #include "base/synchronization/lock.h"
25 #include "components/ukm/ukm_consent_state.h"
26 #include "components/ukm/ukm_entry_filter.h"
27 #include "services/metrics/public/cpp/ukm_decode.h"
28 #include "services/metrics/public/cpp/ukm_recorder.h"
29 #include "services/metrics/public/cpp/ukm_source_id.h"
30 #include "services/metrics/public/mojom/ukm_interface.mojom-forward.h"
31 #include "ukm_consent_state.h"
34 class UkmBrowserTestBase;
40 class UkmRecorderImplTest;
41 class UkmRecorderObserver;
44 class UkmUtilsForTest;
46 COMPONENT_EXPORT(UKM_RECORDER) BASE_DECLARE_FEATURE(kUkmSamplingRateFeature);
49 class UkmDebugDataExtractor;
52 class COMPONENT_EXPORT(UKM_RECORDER) UkmRecorderImpl : public UkmRecorder {
53 using IsWebstoreExtensionCallback =
54 base::RepeatingCallback<bool(base::StringPiece id)>;
58 ~UkmRecorderImpl() override;
60 // Enables/disables recording control if data is allowed to be collected.
61 // |state| defines what is allowed to be collected.
62 // See components/ukm/ukm_consent_state.h for details.
63 void UpdateRecording(const ukm::UkmConsentState state);
64 // Enables recording if MSBB is consented.
65 void EnableRecording();
66 // Disables recording without updating the consent state.
67 void DisableRecording();
69 // Controls sampling for testing purposes. Sampling is 1-in-N (N==rate).
70 void SetSamplingForTesting(int rate) override;
72 // True if sampling has been configured.
73 bool IsSamplingConfigured() const;
75 // Deletes all stored recordings.
78 // Deletes stored Sources containing URLs of the given scheme and events
79 // attributed with these Sources.
80 void PurgeRecordingsWithUrlScheme(const std::string& url_scheme);
82 // Deletes stored Sources with the given Source id type and events
83 // attributed with these Sources.
84 void PurgeRecordingsWithSourceIdType(ukm::SourceIdType source_id_type);
86 // Deletes stored Sources with any Source Id related to MSBB. This included
87 // all SourceIds that are not of type APP_ID.
88 void PurgeRecordingsWithMsbbSources();
90 // Marks a source as no longer needed to be kept alive in memory. The source
91 // with given id will be removed from in-memory recordings at the next
93 void MarkSourceForDeletion(ukm::SourceId source_id) override;
95 // Sets a callback for determining if an extension URL can be recorded.
96 void SetIsWebstoreExtensionCallback(
97 const IsWebstoreExtensionCallback& callback);
99 // Sets the UkmEntryFilter that will be applied to all subsequent entries
100 // reported via AddEntry(). Does not apply the filter to any entries that are
103 // Currently only accommodates one entry filter.
104 void SetEntryFilter(std::unique_ptr<UkmEntryFilter> entry_filter);
106 // Register an observer to be notified when a new UKM entry that comes with
107 // one of the |event_hashes| is added. This method can be called on any
109 void AddUkmRecorderObserver(const base::flat_set<uint64_t>& event_hashes,
110 UkmRecorderObserver* observer);
112 // Clears the given |observer| from |observers_|. This method can be called
113 // on any thread. If an observer is registered for multiple event sets, it
114 // will be removed from all the sets. If an event set no longer has any
115 // observers as a result of this call, it will be removed from |observers_|
117 void RemoveUkmRecorderObserver(UkmRecorderObserver* observer);
119 // Called when UKM consent state changed.
120 void OnUkmAllowedStateChanged(UkmConsentState state);
122 // Sets the sampling seed for testing purposes.
123 void SetSamplingSeedForTesting(uint32_t seed) {
124 // Normally the seed is set during object construction and remains
125 // constant in order to provide consistent results when doing an "is
126 // sampled in" query for a given source and event. A "const cast" is
127 // necessary to override that.
128 *const_cast<uint32_t*>(&sampling_seed_) = seed;
131 bool recording_enabled() const { return recording_enabled_; }
133 bool recording_enabled(ukm::UkmConsentType type) const {
134 return recording_state_.Has(type);
137 bool ShouldDropEntryForTesting(mojom::UkmEntry* entry);
140 // Calculates sampled in/out for a specific source/event based on internal
141 // configuration. This function is guaranteed to always return the same
142 // result over the life of this object for the same config & input parameters.
143 bool IsSampledIn(int64_t source_id, uint64_t event_id);
145 // Like above but uses a passed |sampling_rate| instead of internal config.
146 bool IsSampledIn(int64_t source_id, uint64_t event_id, int sampling_rate);
148 void InitDecodeMap();
150 // Writes recordings into a report proto, and clears recordings.
151 void StoreRecordingsInReport(Report* report);
153 // Prunes data after storing records in the report. Returns the time elapsed
154 // in seconds from the moment the newest truncated source was created to the
155 // moment it was discarded from memory, if pruning happened due to number
156 // of sources exceeding the max threshold.
157 int PruneData(std::set<SourceId>& source_ids_seen);
159 // Deletes Sources and Events with these source_ids.
160 void PurgeSourcesAndEventsBySourceIds(
161 const std::unordered_set<SourceId>& source_ids);
163 const std::map<SourceId, std::unique_ptr<UkmSource>>& sources() const {
164 return recordings_.sources;
167 const std::vector<mojom::UkmEntryPtr>& entries() const {
168 return recordings_.entries;
171 // Keep only newest |max_kept_sources| sources when the number of sources
172 // in recordings_ exceeds this threshold. We only consider the set of ids
173 // contained in |pruning_set|. Returns the age of newest truncated
174 // source in seconds.
175 int PruneOldSources(size_t max_kept_sources,
176 const std::set<SourceId>& pruning_set);
179 void AddEntry(mojom::UkmEntryPtr entry) override;
180 void UpdateSourceURL(SourceId source_id, const GURL& url) override;
181 void UpdateAppURL(SourceId source_id,
183 const AppType app_type) override;
184 void RecordNavigation(
186 const UkmSource::NavigationData& navigation_data) override;
187 using UkmRecorder::RecordOtherURL;
189 // Get the UkmConsentType associated for a given SourceIdType.
190 static UkmConsentType GetConsentType(SourceIdType type);
193 // Get the set of hashes of event types that are observed by any of the
194 // |observers_|. These observers_ need to be notified of a new UKM event with
195 // event_hash in set of observed event_hashes even when UKM recording is
197 std::set<uint64_t> GetObservedEventHashes();
198 // Update the MojoUkmRecorder clients about any update in parameters. This
199 // method can be called on any thread.
200 virtual void OnRecorderParametersChanged() {}
203 friend ::metrics::UkmBrowserTestBase;
204 friend ::ukm::debug::UkmDebugDataExtractor;
205 friend ::ukm::UkmRecorderImplTest;
206 friend ::ukm::UkmTestHelper;
207 friend ::ukm::UkmUtilsForTest;
208 FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, IsSampledIn);
209 FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, PurgeExtensionRecordings);
210 FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, WebApkSourceUrl);
211 FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, PaymentAppScopeUrl);
212 FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, WebIdentityScopeUrl);
213 FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, ObserverNotifiedOnNewEntry);
214 FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, AddRemoveObserver);
216 struct MetricAggregate {
217 uint64_t total_count = 0;
218 double value_sum = 0;
219 double value_square_sum = 0.0;
220 uint64_t dropped_due_to_limits = 0;
221 uint64_t dropped_due_to_sampling = 0;
222 uint64_t dropped_due_to_filter = 0;
223 uint64_t dropped_due_to_unconfigured = 0;
226 struct EventAggregate {
230 // Fills the proto message from the struct.
231 void FillProto(Aggregate* proto_aggregate) const;
233 base::flat_map<uint64_t, MetricAggregate> metrics;
234 uint64_t total_count = 0;
235 uint64_t dropped_due_to_limits = 0;
236 uint64_t dropped_due_to_sampling = 0;
237 uint64_t dropped_due_to_filter = 0;
238 uint64_t dropped_due_to_unconfigured = 0;
241 // Result for ShouldRecordUrl() method.
242 enum class ShouldRecordUrlResult {
243 kOk = 0, // URL will be recorded and observers will be notified.
244 kObserverOnly, // The client has opted out from uploading UKM metrics.
245 // As a result, observers will be notified but URL will not
247 kDropped, // The URL is not allowed to be recorded and will be
248 // dropped. Observers are not nofitied either.
251 using MetricAggregateMap = std::map<uint64_t, MetricAggregate>;
253 // Marks for deletion if the |source_id| is of a certain type.
254 void MaybeMarkForDeletion(SourceId source_id);
256 // Checks if the given |sanitized_extension_url| should be dropped because of
257 // invalid scheme, extension URL recording consent, or whether it's a webstore
258 // extension, and records the dropped reason if so.
259 bool ShouldDropExtensionUrl(const GURL& sanitized_extension_url,
260 bool has_recorded_reason) const;
262 // Returns the result whether |sanitized_url| should be recorded.
263 ShouldRecordUrlResult ShouldRecordUrl(SourceId source_id,
264 const GURL& sanitized_url) const;
266 void RecordSource(std::unique_ptr<UkmSource> source);
268 // Determines if an UkmEntry should be dropped and records reason if so.
269 bool ShouldDropEntry(mojom::UkmEntry* entry);
271 // Applies UkmEntryFilter if there is one registered.
272 bool ApplyEntryFilter(mojom::UkmEntry* entry);
274 // Loads sampling configurations from field-trial information.
275 void LoadExperimentSamplingInfo();
277 // Loads sampling configuration from the key/value "params" of a field-trial.
278 // This is separated from the above to ease testing.
279 void LoadExperimentSamplingParams(
280 const std::map<std::string, std::string>& params);
282 // Called to notify interested observers about a newly added UKM entry.
283 void NotifyObserversWithNewEntry(const mojom::UkmEntry& entry);
285 // Helper method to notify all observers on UKM events.
286 template <typename Method, typename... Params>
287 void NotifyAllObservers(Method m, Params&&... params);
289 // Whether recording new data is currently allowed.
290 bool recording_enabled_ = false;
292 // Whether recording new data is enabled and what type is allowed.
293 ukm::UkmConsentState recording_state_;
295 // Indicates whether recording continuity has been broken since last report.
296 bool recording_is_continuous_ = true;
298 // Indicates if sampling has been forced for testing.
299 bool sampling_forced_for_testing_ = false;
301 // A pseudo-random number used as the base for sampling choices. This
302 // allows consistent "is sampled in" results for a given source and event
303 // type throughout the life of this object.
304 const uint32_t sampling_seed_;
306 // Callback for checking extension IDs.
307 IsWebstoreExtensionCallback is_webstore_extension_callback_;
309 // Filter applied to AddEntry().
310 std::unique_ptr<UkmEntryFilter> entry_filter_;
312 // Map from hashes to entry and metric names.
313 ukm::builders::DecodeMap decode_map_;
315 // Sampling configurations, loaded from a field-trial.
316 int default_sampling_rate_ = -1; // -1 == not yet loaded
317 base::flat_map<uint64_t, int> event_sampling_rates_;
319 // If an event's sampling is "slaved" to another, the hashes of the slave
320 // and the master are recorded here.
321 base::flat_map<uint64_t, uint64_t> event_sampling_master_;
323 // Contains data from various recordings which periodically get serialized
324 // and cleared by StoreRecordingsInReport() and may be Purged().
327 Recordings& operator=(Recordings&&);
330 // Data captured by UpdateSourceUrl().
331 std::map<SourceId, std::unique_ptr<UkmSource>> sources;
333 // Data captured by AddEntry().
334 std::vector<mojom::UkmEntryPtr> entries;
336 // Source ids that have been marked as no longer needed, to denote the
337 // subset of |sources| that can be purged after next report.
338 std::unordered_set<ukm::SourceId> obsolete_source_ids;
340 // URLs of sources that matched a allowlist url, but were not included in
341 // the report generated by the last log rotation because we haven't seen any
342 // events for that source yet.
343 std::unordered_set<std::string> carryover_urls_allowlist;
345 // Aggregate information for collected event metrics.
346 std::map<uint64_t, EventAggregate> event_aggregations;
348 // Aggregated counters about Sources recorded in the current log.
349 struct SourceCounts {
350 // Count of URLs recorded for all sources.
352 // Count of URLs recorded for all SourceIdType::NAVIGATION_ID Sources.
353 size_t navigation_sources = 0;
354 // Sources carried over (not recorded) from a previous logging rotation.
355 size_t carryover_sources = 0;
357 // Resets all of the data.
360 SourceCounts source_counts;
362 // Resets all of the data.
365 Recordings recordings_;
367 // The maximum number of Sources we'll keep in memory before discarding any
368 // new ones being added.
369 size_t max_sources_ = 500;
371 // The maximum number of Sources we can keep in memory at the end of the
372 // current reporting cycle that will stay accessible in the next reporting
374 size_t max_kept_sources_ = 100;
376 // The maximum number of Entries we'll keep in memory before discarding any
377 // new ones being added.
378 size_t max_entries_ = 5000;
380 using UkmRecorderObserverList =
381 base::ObserverListThreadSafe<UkmRecorderObserver>;
382 // Map from event hashes to observers. The key is a set of event hashes that
383 // their corresponding value pair will be norified when one of those events
384 // is added. The value is a non-empty observer list whose members are
385 // observing those events.
386 using UkmRecorderObserverMap =
387 base::flat_map<base::flat_set<uint64_t> /*event_hashes*/,
388 scoped_refptr<UkmRecorderObserverList>>;
389 // Lock used to ensure mutual exclusive access to |observers_|.
390 mutable base::Lock lock_;
392 // Observers that will be notified on UKM events.
393 UkmRecorderObserverMap observers_ GUARDED_BY(lock_);
395 SEQUENCE_CHECKER(sequence_checker_);
400 #endif // COMPONENTS_UKM_UKM_RECORDER_IMPL_H_