1 // Copyright 2017 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "components/ukm/ukm_service.h"
12 #include "base/bind.h"
13 #include "base/hash.h"
14 #include "base/metrics/metrics_hashes.h"
15 #include "base/strings/string_number_conversions.h"
16 #include "base/strings/string_piece.h"
17 #include "base/strings/string_util.h"
18 #include "base/test/scoped_feature_list.h"
19 #include "base/test/test_simple_task_runner.h"
20 #include "base/threading/platform_thread.h"
21 #include "base/threading/thread_task_runner_handle.h"
22 #include "base/time/time.h"
23 #include "components/metrics/test_metrics_provider.h"
24 #include "components/metrics/test_metrics_service_client.h"
25 #include "components/prefs/testing_pref_service.h"
26 #include "components/ukm/persisted_logs_metrics_impl.h"
27 #include "components/ukm/ukm_pref_names.h"
28 #include "components/variations/variations_associated_data.h"
29 #include "services/metrics/public/cpp/ukm_builders.h"
30 #include "services/metrics/public/cpp/ukm_entry_builder.h"
31 #include "services/metrics/public/cpp/ukm_source.h"
32 #include "testing/gtest/include/gtest/gtest.h"
33 #include "third_party/metrics_proto/ukm/report.pb.h"
34 #include "third_party/metrics_proto/ukm/source.pb.h"
35 #include "third_party/zlib/google/compression_utils.h"
39 // Some arbitrary events used in tests.
40 using TestEvent1 = ukm::builders::PageLoad;
41 const char* kTestEvent1Metric1 =
42 TestEvent1::kPaintTiming_NavigationToFirstContentfulPaintName;
43 const char* kTestEvent1Metric2 = TestEvent1::kNet_CacheBytesName;
44 using TestEvent2 = ukm::builders::Memory_Experimental;
45 const char* kTestEvent2Metric1 = TestEvent2::kArrayBufferName;
46 const char* kTestEvent2Metric2 = TestEvent2::kBlinkGCName;
47 using TestEvent3 = ukm::builders::Previews;
49 std::string Entry1And2Whitelist() {
50 return std::string(TestEvent1::kEntryName) + ',' + TestEvent2::kEntryName;
53 // A small shim exposing UkmRecorder methods to tests.
54 class TestRecordingHelper {
56 explicit TestRecordingHelper(UkmRecorder* recorder) : recorder_(recorder) {
57 recorder_->DisableSamplingForTesting();
60 void UpdateSourceURL(SourceId source_id, const GURL& url) {
61 recorder_->UpdateSourceURL(source_id, url);
64 void RecordNavigation(SourceId source_id,
65 const UkmSource::NavigationData& navigation_data) {
66 recorder_->RecordNavigation(source_id, navigation_data);
70 UkmRecorder* recorder_;
72 DISALLOW_COPY_AND_ASSIGN(TestRecordingHelper);
77 bool TestIsWebstoreExtension(base::StringPiece id) {
78 return (id == "bhcnanendmgjjeghamaccjnochlnhcgj");
81 // TODO(rkaplow): consider making this a generic testing class in
82 // components/variations.
83 class ScopedUkmFeatureParams {
85 ScopedUkmFeatureParams(
86 base::FeatureList::OverrideState feature_state,
87 const std::map<std::string, std::string>& variation_params) {
88 static const char kTestFieldTrialName[] = "TestTrial";
89 static const char kTestExperimentGroupName[] = "TestGroup";
91 variations::testing::ClearAllVariationParams();
93 EXPECT_TRUE(variations::AssociateVariationParams(
94 kTestFieldTrialName, kTestExperimentGroupName, variation_params));
96 base::FieldTrial* field_trial = base::FieldTrialList::CreateFieldTrial(
97 kTestFieldTrialName, kTestExperimentGroupName);
99 std::unique_ptr<base::FeatureList> feature_list(new base::FeatureList);
100 feature_list->RegisterFieldTrialOverride(kUkmFeature.name, feature_state,
103 // Since we are adding a scoped feature list after browser start, copy over
104 // the existing feature list to prevent inconsistency.
105 base::FeatureList* existing_feature_list = base::FeatureList::GetInstance();
106 if (existing_feature_list) {
107 std::string enabled_features;
108 std::string disabled_features;
109 base::FeatureList::GetInstance()->GetFeatureOverrides(&enabled_features,
111 feature_list->InitializeFromCommandLine(enabled_features,
115 scoped_feature_list_.InitWithFeatureList(std::move(feature_list));
118 ~ScopedUkmFeatureParams() { variations::testing::ClearAllVariationParams(); }
121 base::test::ScopedFeatureList scoped_feature_list_;
123 DISALLOW_COPY_AND_ASSIGN(ScopedUkmFeatureParams);
126 class UkmServiceTest : public testing::Test {
129 : task_runner_(new base::TestSimpleTaskRunner),
130 task_runner_handle_(task_runner_) {
131 UkmService::RegisterPrefs(prefs_.registry());
136 prefs_.ClearPref(prefs::kUkmClientId);
137 prefs_.ClearPref(prefs::kUkmSessionId);
138 prefs_.ClearPref(prefs::kUkmPersistedLogs);
141 int GetPersistedLogCount() {
142 const base::ListValue* list_value =
143 prefs_.GetList(prefs::kUkmPersistedLogs);
144 return list_value->GetSize();
147 Report GetPersistedReport() {
148 EXPECT_GE(GetPersistedLogCount(), 1);
149 metrics::PersistedLogs result_persisted_logs(
150 std::make_unique<ukm::PersistedLogsMetricsImpl>(), &prefs_,
151 prefs::kUkmPersistedLogs,
152 3, // log count limit
156 result_persisted_logs.LoadPersistedUnsentLogs();
157 result_persisted_logs.StageNextLog();
159 std::string uncompressed_log_data;
160 EXPECT_TRUE(compression::GzipUncompress(result_persisted_logs.staged_log(),
161 &uncompressed_log_data));
164 EXPECT_TRUE(report.ParseFromString(uncompressed_log_data));
168 static SourceId GetWhitelistedSourceId(int64_t id) {
169 return ConvertToSourceId(id, SourceIdType::NAVIGATION_ID);
172 static SourceId GetNonWhitelistedSourceId(int64_t id) {
173 return ConvertToSourceId(id, SourceIdType::UKM);
177 TestingPrefServiceSimple prefs_;
178 metrics::TestMetricsServiceClient client_;
180 scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
181 base::ThreadTaskRunnerHandle task_runner_handle_;
184 DISALLOW_COPY_AND_ASSIGN(UkmServiceTest);
189 TEST_F(UkmServiceTest, EnableDisableSchedule) {
190 UkmService service(&prefs_, &client_,
191 true /* restrict_to_whitelisted_entries */);
192 EXPECT_FALSE(task_runner_->HasPendingTask());
193 service.Initialize();
194 EXPECT_FALSE(task_runner_->HasPendingTask());
195 service.EnableRecording(/*extensions=*/false);
196 service.EnableReporting();
197 EXPECT_TRUE(task_runner_->HasPendingTask());
198 service.DisableReporting();
199 task_runner_->RunPendingTasks();
200 EXPECT_FALSE(task_runner_->HasPendingTask());
203 TEST_F(UkmServiceTest, PersistAndPurge) {
204 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
205 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE,
206 {{"WhitelistEntries", Entry1And2Whitelist()}});
208 UkmService service(&prefs_, &client_,
209 true /* restrict_to_whitelisted_entries */);
210 TestRecordingHelper recorder(&service);
211 EXPECT_EQ(GetPersistedLogCount(), 0);
212 service.Initialize();
213 task_runner_->RunUntilIdle();
214 service.EnableRecording(/*extensions=*/false);
215 service.EnableReporting();
217 SourceId id = GetWhitelistedSourceId(0);
218 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
219 // Should init, generate a log, and start an upload for source.
220 task_runner_->RunPendingTasks();
221 EXPECT_TRUE(client_.uploader()->is_uploading());
222 // Flushes the generated log to disk and generates a new entry.
223 TestEvent1(id).Record(&service);
225 EXPECT_EQ(GetPersistedLogCount(), 2);
227 EXPECT_EQ(GetPersistedLogCount(), 0);
230 TEST_F(UkmServiceTest, Purge) {
231 UkmService service(&prefs_, &client_,
232 true /* restrict_to_whitelisted_entries */);
233 TestRecordingHelper recorder(&service);
234 EXPECT_EQ(GetPersistedLogCount(), 0);
235 service.Initialize();
236 task_runner_->RunUntilIdle();
237 service.EnableRecording(/*extensions=*/false);
238 service.EnableReporting();
241 auto id = GetWhitelistedSourceId(0);
242 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar1"));
243 TestEvent1(id).Record(&service);
245 // Purge should delete data, so there shouldn't be anything left to upload.
248 EXPECT_EQ(0, GetPersistedLogCount());
251 TEST_F(UkmServiceTest, SourceSerialization) {
252 UkmService service(&prefs_, &client_,
253 true /* restrict_to_whitelisted_entries */);
254 TestRecordingHelper recorder(&service);
255 EXPECT_EQ(GetPersistedLogCount(), 0);
256 service.Initialize();
257 task_runner_->RunUntilIdle();
258 service.EnableRecording(/*extensions=*/false);
259 service.EnableReporting();
261 UkmSource::NavigationData navigation_data;
262 navigation_data.urls = {GURL("https://google.com/initial"),
263 GURL("https://google.com/final")};
265 ukm::SourceId id = GetWhitelistedSourceId(0);
266 recorder.RecordNavigation(id, navigation_data);
269 EXPECT_EQ(GetPersistedLogCount(), 1);
271 Report proto_report = GetPersistedReport();
272 EXPECT_EQ(1, proto_report.sources_size());
273 EXPECT_TRUE(proto_report.has_session_id());
274 const Source& proto_source = proto_report.sources(0);
276 EXPECT_EQ(id, proto_source.id());
277 EXPECT_EQ(GURL("https://google.com/final").spec(), proto_source.url());
278 EXPECT_FALSE(proto_source.has_initial_url());
281 TEST_F(UkmServiceTest, AddEntryWithEmptyMetrics) {
282 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
283 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE,
284 {{"WhitelistEntries", Entry1And2Whitelist()}});
286 UkmService service(&prefs_, &client_,
287 true /* restrict_to_whitelisted_entries */);
288 TestRecordingHelper recorder(&service);
289 ASSERT_EQ(0, GetPersistedLogCount());
290 service.Initialize();
291 task_runner_->RunUntilIdle();
292 service.EnableRecording(/*extensions=*/false);
293 service.EnableReporting();
295 ukm::SourceId id = GetWhitelistedSourceId(0);
296 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
298 TestEvent1(id).Record(&service);
300 ASSERT_EQ(1, GetPersistedLogCount());
301 Report proto_report = GetPersistedReport();
302 EXPECT_EQ(1, proto_report.entries_size());
305 TEST_F(UkmServiceTest, MetricsProviderTest) {
306 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
307 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE,
308 {{"WhitelistEntries", Entry1And2Whitelist()}});
310 UkmService service(&prefs_, &client_,
311 true /* restrict_to_whitelisted_entries */);
312 TestRecordingHelper recorder(&service);
314 metrics::TestMetricsProvider* provider = new metrics::TestMetricsProvider();
315 service.RegisterMetricsProvider(
316 std::unique_ptr<metrics::MetricsProvider>(provider));
318 service.Initialize();
320 // Providers have not supplied system profile information yet.
321 EXPECT_FALSE(provider->provide_system_profile_metrics_called());
323 task_runner_->RunUntilIdle();
324 service.EnableRecording(/*extensions=*/false);
325 service.EnableReporting();
327 ukm::SourceId id = GetWhitelistedSourceId(0);
328 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
329 TestEvent1(id).Record(&service);
331 EXPECT_EQ(GetPersistedLogCount(), 1);
333 Report proto_report = GetPersistedReport();
334 EXPECT_EQ(1, proto_report.sources_size());
335 EXPECT_EQ(1, proto_report.entries_size());
337 // Providers have now supplied system profile information.
338 EXPECT_TRUE(provider->provide_system_profile_metrics_called());
341 TEST_F(UkmServiceTest, LogsRotation) {
342 UkmService service(&prefs_, &client_,
343 true /* restrict_to_whitelisted_entries */);
344 TestRecordingHelper recorder(&service);
345 EXPECT_EQ(GetPersistedLogCount(), 0);
346 service.Initialize();
347 task_runner_->RunUntilIdle();
348 service.EnableRecording(/*extensions=*/false);
349 service.EnableReporting();
351 EXPECT_EQ(0, service.report_count());
353 // Log rotation should generate a log.
354 const ukm::SourceId id = GetWhitelistedSourceId(0);
355 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
356 task_runner_->RunPendingTasks();
357 EXPECT_EQ(1, service.report_count());
358 EXPECT_TRUE(client_.uploader()->is_uploading());
360 // Rotation shouldn't generate a log due to one being pending.
361 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
362 task_runner_->RunPendingTasks();
363 EXPECT_EQ(1, service.report_count());
364 EXPECT_TRUE(client_.uploader()->is_uploading());
366 // Completing the upload should clear pending log, then log rotation should
367 // generate another log.
368 client_.uploader()->CompleteUpload(200);
369 task_runner_->RunPendingTasks();
370 EXPECT_EQ(2, service.report_count());
372 // Check that rotations keep working.
373 for (int i = 3; i < 6; i++) {
374 task_runner_->RunPendingTasks();
375 client_.uploader()->CompleteUpload(200);
376 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
377 task_runner_->RunPendingTasks();
378 EXPECT_EQ(i, service.report_count());
382 TEST_F(UkmServiceTest, LogsUploadedOnlyWhenHavingSourcesOrEntries) {
383 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
384 // Testing two whitelisted Entries.
385 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE,
386 {{"WhitelistEntries", Entry1And2Whitelist()}});
388 UkmService service(&prefs_, &client_,
389 true /* restrict_to_whitelisted_entries */);
390 TestRecordingHelper recorder(&service);
391 EXPECT_EQ(GetPersistedLogCount(), 0);
392 service.Initialize();
393 task_runner_->RunUntilIdle();
394 service.EnableRecording(/*extensions=*/false);
395 service.EnableReporting();
397 EXPECT_TRUE(task_runner_->HasPendingTask());
398 // Neither rotation or Flush should generate logs
399 task_runner_->RunPendingTasks();
401 EXPECT_EQ(GetPersistedLogCount(), 0);
403 ukm::SourceId id = GetWhitelistedSourceId(0);
404 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
405 // Includes a Source, so will persist.
407 EXPECT_EQ(GetPersistedLogCount(), 1);
409 TestEvent1(id).Record(&service);
410 // Includes an Entry, so will persist.
412 EXPECT_EQ(GetPersistedLogCount(), 2);
414 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
415 TestEvent1(id).Record(&service);
416 // Includes a Source and an Entry, so will persist.
418 EXPECT_EQ(GetPersistedLogCount(), 3);
420 // Current log has no Sources.
422 EXPECT_EQ(GetPersistedLogCount(), 3);
425 TEST_F(UkmServiceTest, GetNewSourceID) {
426 ukm::SourceId id1 = UkmRecorder::GetNewSourceID();
427 ukm::SourceId id2 = UkmRecorder::GetNewSourceID();
428 ukm::SourceId id3 = UkmRecorder::GetNewSourceID();
434 TEST_F(UkmServiceTest, RecordInitialUrl) {
435 for (bool should_record_initial_url : {true, false}) {
436 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
437 ScopedUkmFeatureParams params(
438 base::FeatureList::OVERRIDE_ENABLE_FEATURE,
439 {{"RecordInitialUrl", should_record_initial_url ? "true" : "false"}});
442 UkmService service(&prefs_, &client_,
443 true /* restrict_to_whitelisted_entries */);
444 TestRecordingHelper recorder(&service);
445 EXPECT_EQ(GetPersistedLogCount(), 0);
446 service.Initialize();
447 task_runner_->RunUntilIdle();
448 service.EnableRecording(/*extensions=*/false);
449 service.EnableReporting();
451 ukm::SourceId id = GetWhitelistedSourceId(0);
452 UkmSource::NavigationData navigation_data;
453 navigation_data.urls = {GURL("https://google.com/initial"),
454 GURL("https://google.com/final")};
455 recorder.RecordNavigation(id, navigation_data);
458 EXPECT_EQ(GetPersistedLogCount(), 1);
460 Report proto_report = GetPersistedReport();
461 EXPECT_EQ(1, proto_report.sources_size());
462 const Source& proto_source = proto_report.sources(0);
464 EXPECT_EQ(id, proto_source.id());
465 EXPECT_EQ(GURL("https://google.com/final").spec(), proto_source.url());
466 EXPECT_EQ(should_record_initial_url, proto_source.has_initial_url());
467 if (should_record_initial_url) {
468 EXPECT_EQ(GURL("https://google.com/initial").spec(),
469 proto_source.initial_url());
474 TEST_F(UkmServiceTest, RestrictToWhitelistedSourceIds) {
475 const GURL kURL = GURL("https://example.com/");
476 for (bool restrict_to_whitelisted_source_ids : {true, false}) {
477 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
478 ScopedUkmFeatureParams params(
479 base::FeatureList::OVERRIDE_ENABLE_FEATURE,
480 {{"RestrictToWhitelistedSourceIds",
481 restrict_to_whitelisted_source_ids ? "true" : "false"},
482 {"WhitelistEntries", Entry1And2Whitelist()}});
485 UkmService service(&prefs_, &client_,
486 true /* restrict_to_whitelisted_entries */);
487 TestRecordingHelper recorder(&service);
488 EXPECT_EQ(GetPersistedLogCount(), 0);
489 service.Initialize();
490 task_runner_->RunUntilIdle();
491 service.EnableRecording(/*extensions=*/false);
492 service.EnableReporting();
494 ukm::SourceId id1 = GetWhitelistedSourceId(0);
495 recorder.UpdateSourceURL(id1, kURL);
496 TestEvent1(id1).Record(&service);
498 // Create a non-navigation-based sourceid, which should not be whitelisted.
499 ukm::SourceId id2 = GetNonWhitelistedSourceId(1);
500 recorder.UpdateSourceURL(id2, kURL);
501 TestEvent1(id2).Record(&service);
504 ASSERT_EQ(GetPersistedLogCount(), 1);
505 Report proto_report = GetPersistedReport();
506 ASSERT_GE(proto_report.sources_size(), 1);
508 // The whitelisted source should always be recorded.
509 const Source& proto_source1 = proto_report.sources(0);
510 EXPECT_EQ(id1, proto_source1.id());
511 EXPECT_EQ(kURL.spec(), proto_source1.url());
513 // The non-whitelisted source should only be recorded if we aren't
514 // restricted to whitelisted source ids.
515 if (restrict_to_whitelisted_source_ids) {
516 ASSERT_EQ(1, proto_report.sources_size());
518 ASSERT_EQ(2, proto_report.sources_size());
519 const Source& proto_source2 = proto_report.sources(1);
520 EXPECT_EQ(id2, proto_source2.id());
521 EXPECT_EQ(kURL.spec(), proto_source2.url());
526 TEST_F(UkmServiceTest, RecordSessionId) {
528 UkmService service(&prefs_, &client_,
529 true /* restrict_to_whitelisted_entries */);
530 TestRecordingHelper recorder(&service);
531 EXPECT_EQ(0, GetPersistedLogCount());
532 service.Initialize();
533 task_runner_->RunUntilIdle();
534 service.EnableRecording(/*extensions=*/false);
535 service.EnableReporting();
537 auto id = GetWhitelistedSourceId(0);
538 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
541 EXPECT_EQ(1, GetPersistedLogCount());
543 auto proto_report = GetPersistedReport();
544 EXPECT_TRUE(proto_report.has_session_id());
545 EXPECT_EQ(1, proto_report.report_id());
548 TEST_F(UkmServiceTest, SourceSize) {
549 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
550 // Set a threshold of number of Sources via Feature Params.
551 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE,
552 {{"MaxSources", "2"}});
555 UkmService service(&prefs_, &client_,
556 true /* restrict_to_whitelisted_entries */);
557 TestRecordingHelper recorder(&service);
558 EXPECT_EQ(0, GetPersistedLogCount());
559 service.Initialize();
560 task_runner_->RunUntilIdle();
561 service.EnableRecording(/*extensions=*/false);
562 service.EnableReporting();
564 auto id = GetWhitelistedSourceId(0);
565 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar1"));
566 id = GetWhitelistedSourceId(1);
567 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar2"));
568 id = GetWhitelistedSourceId(2);
569 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar3"));
572 EXPECT_EQ(1, GetPersistedLogCount());
574 auto proto_report = GetPersistedReport();
575 // Note, 2 instead of 3 sources, since we overrode the max number of sources
576 // via Feature params.
577 EXPECT_EQ(2, proto_report.sources_size());
580 TEST_F(UkmServiceTest, PurgeMidUpload) {
581 UkmService service(&prefs_, &client_,
582 true /* restrict_to_whitelisted_entries */);
583 TestRecordingHelper recorder(&service);
584 EXPECT_EQ(GetPersistedLogCount(), 0);
585 service.Initialize();
586 task_runner_->RunUntilIdle();
587 service.EnableRecording(/*extensions=*/false);
588 service.EnableReporting();
590 auto id = GetWhitelistedSourceId(0);
591 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar1"));
592 // Should init, generate a log, and start an upload.
593 task_runner_->RunPendingTasks();
594 EXPECT_TRUE(client_.uploader()->is_uploading());
595 // Purge should delete all logs, including the one being sent.
597 // Upload succeeds after logs was deleted.
598 client_.uploader()->CompleteUpload(200);
599 EXPECT_EQ(GetPersistedLogCount(), 0);
600 EXPECT_FALSE(client_.uploader()->is_uploading());
603 TEST_F(UkmServiceTest, WhitelistEntryTest) {
604 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
605 // Testing two whitelisted Entries.
606 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE,
607 {{"WhitelistEntries", Entry1And2Whitelist()}});
610 UkmService service(&prefs_, &client_,
611 true /* restrict_to_whitelisted_entries */);
612 TestRecordingHelper recorder(&service);
613 EXPECT_EQ(0, GetPersistedLogCount());
614 service.Initialize();
615 task_runner_->RunUntilIdle();
616 service.EnableRecording(/*extensions=*/false);
617 service.EnableReporting();
619 auto id = GetWhitelistedSourceId(0);
620 recorder.UpdateSourceURL(id, GURL("https://google.com/foobar1"));
622 TestEvent1(id).Record(&service);
623 TestEvent2(id).Record(&service);
624 // Note that this third entry is not in the whitelist.
625 TestEvent3(id).Record(&service);
628 EXPECT_EQ(1, GetPersistedLogCount());
629 Report proto_report = GetPersistedReport();
631 // Verify we've added one source and 2 entries.
632 EXPECT_EQ(1, proto_report.sources_size());
633 ASSERT_EQ(2, proto_report.entries_size());
635 const Entry& proto_entry_a = proto_report.entries(0);
636 EXPECT_EQ(id, proto_entry_a.source_id());
637 EXPECT_EQ(base::HashMetricName(TestEvent1::kEntryName),
638 proto_entry_a.event_hash());
640 const Entry& proto_entry_b = proto_report.entries(1);
641 EXPECT_EQ(id, proto_entry_b.source_id());
642 EXPECT_EQ(base::HashMetricName(TestEvent2::kEntryName),
643 proto_entry_b.event_hash());
646 TEST_F(UkmServiceTest, SourceURLLength) {
647 UkmService service(&prefs_, &client_,
648 true /* restrict_to_whitelisted_entries */);
649 TestRecordingHelper recorder(&service);
650 EXPECT_EQ(0, GetPersistedLogCount());
651 service.Initialize();
652 task_runner_->RunUntilIdle();
653 service.EnableRecording(/*extensions=*/false);
654 service.EnableReporting();
656 auto id = GetWhitelistedSourceId(0);
658 // This URL is too long to be recorded fully.
659 const std::string long_string =
660 "https://example.com/" + std::string(10000, 'a');
661 recorder.UpdateSourceURL(id, GURL(long_string));
664 EXPECT_EQ(1, GetPersistedLogCount());
666 auto proto_report = GetPersistedReport();
667 ASSERT_EQ(1, proto_report.sources_size());
668 const Source& proto_source = proto_report.sources(0);
669 EXPECT_EQ("URLTooLong", proto_source.url());
672 TEST_F(UkmServiceTest, UnreferencedNonWhitelistedSources) {
673 const GURL kURL("https://google.com/foobar");
674 for (bool restrict_to_whitelisted_source_ids : {true, false}) {
675 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
676 // Set a threshold of number of Sources via Feature Params.
677 ScopedUkmFeatureParams params(
678 base::FeatureList::OVERRIDE_ENABLE_FEATURE,
679 {{"MaxKeptSources", "3"},
680 {"WhitelistEntries", Entry1And2Whitelist()},
681 {"RestrictToWhitelistedSourceIds",
682 restrict_to_whitelisted_source_ids ? "true" : "false"}});
685 UkmService service(&prefs_, &client_,
686 true /* restrict_to_whitelisted_entries */);
687 TestRecordingHelper recorder(&service);
688 EXPECT_EQ(0, GetPersistedLogCount());
689 service.Initialize();
690 task_runner_->RunUntilIdle();
691 service.EnableRecording(/*extensions=*/false);
692 service.EnableReporting();
694 // Record with whitelisted ID to whitelist the URL.
695 // Use a larger ID to make it last in the proto.
696 ukm::SourceId whitelisted_id = GetWhitelistedSourceId(100);
697 recorder.UpdateSourceURL(whitelisted_id, kURL);
699 std::vector<SourceId> ids;
700 base::TimeTicks last_time = base::TimeTicks::Now();
701 for (int i = 0; i < 6; ++i) {
702 // Wait until base::TimeTicks::Now() no longer equals |last_time|. This
703 // ensures each source has a unique timestamp to avoid flakes. Should take
704 // between 1-15ms per documented resolution of base::TimeTicks.
705 while (base::TimeTicks::Now() == last_time) {
706 base::PlatformThread::Sleep(base::TimeDelta::FromMilliseconds(1));
709 ids.push_back(GetNonWhitelistedSourceId(i));
710 recorder.UpdateSourceURL(ids.back(), kURL);
711 last_time = base::TimeTicks::Now();
714 // Add whitelisted entries for 0, 2 and non-whitelisted entries for 2, 3.
715 TestEvent1(ids[0]).Record(&service);
716 TestEvent2(ids[2]).Record(&service);
717 TestEvent3(ids[2]).Record(&service);
718 TestEvent3(ids[3]).Record(&service);
721 EXPECT_EQ(1, GetPersistedLogCount());
722 auto proto_report = GetPersistedReport();
724 if (restrict_to_whitelisted_source_ids) {
725 EXPECT_EQ(1, proto_report.source_counts().observed());
726 EXPECT_EQ(1, proto_report.source_counts().navigation_sources());
727 EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
728 EXPECT_EQ(0, proto_report.source_counts().deferred_sources());
729 EXPECT_EQ(0, proto_report.source_counts().carryover_sources());
731 ASSERT_EQ(1, proto_report.sources_size());
733 EXPECT_EQ(7, proto_report.source_counts().observed());
734 EXPECT_EQ(1, proto_report.source_counts().navigation_sources());
735 EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
736 EXPECT_EQ(4, proto_report.source_counts().deferred_sources());
737 EXPECT_EQ(0, proto_report.source_counts().carryover_sources());
739 ASSERT_EQ(3, proto_report.sources_size());
740 EXPECT_EQ(ids[0], proto_report.sources(0).id());
741 EXPECT_EQ(kURL.spec(), proto_report.sources(0).url());
742 EXPECT_EQ(ids[2], proto_report.sources(1).id());
743 EXPECT_EQ(kURL.spec(), proto_report.sources(1).url());
746 // Since MaxKeptSources is 3, only Sources 5, 4, 3 should be retained.
747 // Log entries under 0, 1, 3 and 4. Log them in reverse order - which
748 // shouldn't affect source ordering in the output.
749 // - Source 0 should not be re-transmitted since it was sent before.
750 // - Source 1 should not be transmitted due to MaxKeptSources param.
751 // - Sources 3 and 4 should be transmitted since they were not sent before.
752 TestEvent1(ids[4]).Record(&service);
753 TestEvent1(ids[3]).Record(&service);
754 TestEvent1(ids[1]).Record(&service);
755 TestEvent1(ids[0]).Record(&service);
758 EXPECT_EQ(2, GetPersistedLogCount());
759 proto_report = GetPersistedReport();
761 if (restrict_to_whitelisted_source_ids) {
762 EXPECT_EQ(0, proto_report.source_counts().observed());
763 EXPECT_EQ(0, proto_report.source_counts().navigation_sources());
764 EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
765 EXPECT_EQ(0, proto_report.source_counts().deferred_sources());
766 EXPECT_EQ(0, proto_report.source_counts().carryover_sources());
768 ASSERT_EQ(0, proto_report.sources_size());
770 EXPECT_EQ(0, proto_report.source_counts().observed());
771 EXPECT_EQ(0, proto_report.source_counts().navigation_sources());
772 EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
773 EXPECT_EQ(1, proto_report.source_counts().deferred_sources());
774 EXPECT_EQ(3, proto_report.source_counts().carryover_sources());
776 ASSERT_EQ(2, proto_report.sources_size());
777 EXPECT_EQ(ids[3], proto_report.sources(0).id());
778 EXPECT_EQ(kURL.spec(), proto_report.sources(0).url());
779 EXPECT_EQ(ids[4], proto_report.sources(1).id());
780 EXPECT_EQ(kURL.spec(), proto_report.sources(1).url());
785 TEST_F(UkmServiceTest, NonWhitelistedUrls) {
786 const GURL kURL("https://google.com/foobar");
791 {GURL("https://google.com/foobar"), true},
792 // For origin-only URLs, only the origin needs to be matched.
793 {GURL("https://google.com"), true},
794 {GURL("https://google.com/foobar2"), false},
795 {GURL("https://other.com"), false},
798 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
799 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE,
800 {{"WhitelistEntries", Entry1And2Whitelist()}});
802 for (const auto& test : test_cases) {
804 UkmService service(&prefs_, &client_,
805 true /* restrict_to_whitelisted_entries */);
806 TestRecordingHelper recorder(&service);
808 ASSERT_EQ(GetPersistedLogCount(), 0);
809 service.Initialize();
810 task_runner_->RunUntilIdle();
811 service.EnableRecording(/*extensions=*/false);
812 service.EnableReporting();
814 // Record with whitelisted ID to whitelist the URL.
815 ukm::SourceId whitelist_id = GetWhitelistedSourceId(1);
816 recorder.UpdateSourceURL(whitelist_id, kURL);
818 // Record non whitelisted ID with a entry.
819 ukm::SourceId nonwhitelist_id = GetNonWhitelistedSourceId(100);
820 recorder.UpdateSourceURL(nonwhitelist_id, test.url);
821 TestEvent1(nonwhitelist_id).Record(&service);
824 ASSERT_EQ(1, GetPersistedLogCount());
825 auto proto_report = GetPersistedReport();
827 EXPECT_EQ(2, proto_report.source_counts().observed());
828 EXPECT_EQ(1, proto_report.source_counts().navigation_sources());
829 if (test.expected_kept) {
830 EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
831 ASSERT_EQ(2, proto_report.sources_size());
832 EXPECT_EQ(whitelist_id, proto_report.sources(0).id());
833 EXPECT_EQ(kURL, proto_report.sources(0).url());
834 EXPECT_EQ(nonwhitelist_id, proto_report.sources(1).id());
835 EXPECT_EQ(test.url, proto_report.sources(1).url());
837 EXPECT_EQ(1, proto_report.source_counts().unmatched_sources());
838 ASSERT_EQ(1, proto_report.sources_size());
839 EXPECT_EQ(whitelist_id, proto_report.sources(0).id());
840 EXPECT_EQ(kURL, proto_report.sources(0).url());
845 TEST_F(UkmServiceTest, NonWhitelistedCarryoverUrls) {
846 const GURL kURL("https://google.com/foobar");
849 // Source1 is recorded during the first rotation with no entry.
850 // An entry for it is recorded in the second rotation.
852 // Should Source1 be seen in second rotation's log.
854 // Source2 is recorded during the second rotation with an entry.
856 // Should Source2 be seen in second rotation's log.
859 // Recording the URL captures in the whitelist, which will also allow
860 // exact matches of the same URL.
861 {GURL("https://google.com/foobar"), true,
862 GURL("https://google.com/foobar"), true},
863 // Capturing a full URL shouldn't allow origin matches.
864 {GURL("https://google.com/foobar"), true, GURL("https://google.com"),
866 // Uncaptured URLs won't get matched.
867 {GURL("https://google.com/foobar"), true, GURL("https://other.com"),
869 // Origin should be capturable, and will remember the same origin.
870 {GURL("https://google.com"), true, GURL("https://google.com"), true},
871 // If the origin is captured, only the origin is remembered.
872 {GURL("https://google.com"), true, GURL("https://google.com/foobar"),
874 // Uncaptured URLs won't get matched.
875 {GURL("https://google.com"), true, GURL("https://other.com"), false},
876 // If the URL isn't captured in the first round, it won't capture later.
877 {GURL("https://other.com"), false, GURL("https://google.com/foobar"),
879 {GURL("https://other.com"), false, GURL("https://google.com"), false},
880 // Entries shouldn't whitelist themselves.
881 {GURL("https://other.com"), false, GURL("https://other.com"), false},
884 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
885 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE,
886 {{"WhitelistEntries", Entry1And2Whitelist()}});
888 for (const auto& test : test_cases) {
890 UkmService service(&prefs_, &client_,
891 true /* restrict_to_whitelisted_entries */);
892 TestRecordingHelper recorder(&service);
894 EXPECT_EQ(GetPersistedLogCount(), 0);
895 service.Initialize();
896 task_runner_->RunUntilIdle();
897 service.EnableRecording(/*extensions=*/false);
898 service.EnableReporting();
900 // Record with whitelisted ID to whitelist the URL.
901 ukm::SourceId whitelist_id = GetWhitelistedSourceId(1);
902 recorder.UpdateSourceURL(whitelist_id, kURL);
904 // Record test Source1 without an event.
905 ukm::SourceId nonwhitelist_id1 = GetNonWhitelistedSourceId(100);
906 recorder.UpdateSourceURL(nonwhitelist_id1, test.source1_url);
909 ASSERT_EQ(1, GetPersistedLogCount());
910 auto proto_report = GetPersistedReport();
912 EXPECT_EQ(2, proto_report.source_counts().observed());
913 EXPECT_EQ(1, proto_report.source_counts().navigation_sources());
914 EXPECT_EQ(0, proto_report.source_counts().carryover_sources());
915 if (test.expect_source1) {
916 EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
917 EXPECT_EQ(1, proto_report.source_counts().deferred_sources());
919 EXPECT_EQ(1, proto_report.source_counts().unmatched_sources());
920 EXPECT_EQ(0, proto_report.source_counts().deferred_sources());
922 ASSERT_EQ(1, proto_report.sources_size());
923 EXPECT_EQ(whitelist_id, proto_report.sources(0).id());
924 EXPECT_EQ(kURL, proto_report.sources(0).url());
926 // Record the Source2 and events for Source1 and Source2.
927 ukm::SourceId nonwhitelist_id2 = GetNonWhitelistedSourceId(101);
928 recorder.UpdateSourceURL(nonwhitelist_id2, test.source2_url);
929 TestEvent1(nonwhitelist_id1).Record(&service);
930 TestEvent1(nonwhitelist_id2).Record(&service);
933 ASSERT_EQ(2, GetPersistedLogCount());
934 proto_report = GetPersistedReport();
936 EXPECT_EQ(1, proto_report.source_counts().observed());
937 EXPECT_EQ(0, proto_report.source_counts().navigation_sources());
938 EXPECT_EQ(0, proto_report.source_counts().deferred_sources());
939 if (!test.expect_source1) {
940 EXPECT_FALSE(test.expect_source2);
941 EXPECT_EQ(1, proto_report.source_counts().unmatched_sources());
942 EXPECT_EQ(0, proto_report.source_counts().carryover_sources());
943 ASSERT_EQ(0, proto_report.sources_size());
944 } else if (!test.expect_source2) {
945 EXPECT_EQ(1, proto_report.source_counts().unmatched_sources());
946 EXPECT_EQ(1, proto_report.source_counts().carryover_sources());
947 ASSERT_EQ(1, proto_report.sources_size());
948 EXPECT_EQ(nonwhitelist_id1, proto_report.sources(0).id());
949 EXPECT_EQ(test.source1_url, proto_report.sources(0).url());
951 EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
952 EXPECT_EQ(1, proto_report.source_counts().carryover_sources());
953 ASSERT_EQ(2, proto_report.sources_size());
954 EXPECT_EQ(nonwhitelist_id1, proto_report.sources(0).id());
955 EXPECT_EQ(test.source1_url, proto_report.sources(0).url());
956 EXPECT_EQ(nonwhitelist_id2, proto_report.sources(1).id());
957 EXPECT_EQ(test.source2_url, proto_report.sources(1).url());
962 TEST_F(UkmServiceTest, SupportedSchemes) {
967 {"http://google.ca/", true},
968 {"https://google.ca/", true},
969 {"ftp://google.ca/", true},
970 {"about:blank", true},
971 {"chrome://version/", true},
972 {"app://play/abcdefghijklmnopqrstuvwxyzabcdef/", true},
973 // chrome-extension are controlled by TestIsWebstoreExtension, above.
974 {"chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj/", true},
975 {"chrome-extension://abcdefghijklmnopqrstuvwxyzabcdef/", false},
976 {"file:///tmp/", false},
977 {"abc://google.ca/", false},
978 {"www.google.ca/", false},
981 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
982 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, {});
983 UkmService service(&prefs_, &client_,
984 true /* restrict_to_whitelisted_entries */);
985 TestRecordingHelper recorder(&service);
986 service.SetIsWebstoreExtensionCallback(
987 base::BindRepeating(&TestIsWebstoreExtension));
989 EXPECT_EQ(GetPersistedLogCount(), 0);
990 service.Initialize();
991 task_runner_->RunUntilIdle();
992 service.EnableRecording(/*extensions=*/true);
993 service.EnableReporting();
995 int64_t id_counter = 1;
996 int expected_kept_count = 0;
997 for (const auto& test : test_cases) {
998 auto source_id = GetWhitelistedSourceId(id_counter++);
999 recorder.UpdateSourceURL(source_id, GURL(test.url));
1000 TestEvent1(source_id).Record(&service);
1001 if (test.expected_kept)
1002 ++expected_kept_count;
1006 EXPECT_EQ(GetPersistedLogCount(), 1);
1007 Report proto_report = GetPersistedReport();
1009 EXPECT_EQ(expected_kept_count, proto_report.sources_size());
1010 for (const auto& test : test_cases) {
1012 for (int i = 0; i < proto_report.sources_size(); ++i) {
1013 if (proto_report.sources(i).url() == test.url) {
1018 EXPECT_EQ(test.expected_kept, found) << test.url;
1022 TEST_F(UkmServiceTest, SupportedSchemesNoExtensions) {
1027 {"http://google.ca/", true},
1028 {"https://google.ca/", true},
1029 {"ftp://google.ca/", true},
1030 {"about:blank", true},
1031 {"chrome://version/", true},
1032 {"app://play/abcdefghijklmnopqrstuvwxyzabcdef/", true},
1033 {"chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj/", false},
1034 {"chrome-extension://abcdefghijklmnopqrstuvwxyzabcdef/", false},
1035 {"file:///tmp/", false},
1036 {"abc://google.ca/", false},
1037 {"www.google.ca/", false},
1040 base::FieldTrialList field_trial_list(nullptr /* entropy_provider */);
1041 ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, {});
1042 UkmService service(&prefs_, &client_,
1043 true /* restrict_to_whitelisted_entries */);
1044 TestRecordingHelper recorder(&service);
1046 EXPECT_EQ(GetPersistedLogCount(), 0);
1047 service.Initialize();
1048 task_runner_->RunUntilIdle();
1049 service.EnableRecording(/*extensions=*/false);
1050 service.EnableReporting();
1052 int64_t id_counter = 1;
1053 int expected_kept_count = 0;
1054 for (const auto& test : test_cases) {
1055 auto source_id = GetWhitelistedSourceId(id_counter++);
1056 recorder.UpdateSourceURL(source_id, GURL(test.url));
1057 TestEvent1(source_id).Record(&service);
1058 if (test.expected_kept)
1059 ++expected_kept_count;
1063 EXPECT_EQ(GetPersistedLogCount(), 1);
1064 Report proto_report = GetPersistedReport();
1066 EXPECT_EQ(expected_kept_count, proto_report.sources_size());
1067 for (const auto& test : test_cases) {
1069 for (int i = 0; i < proto_report.sources_size(); ++i) {
1070 if (proto_report.sources(i).url() == test.url) {
1075 EXPECT_EQ(test.expected_kept, found) << test.url;
1079 TEST_F(UkmServiceTest, SanitizeUrlAuthParams) {
1080 UkmService service(&prefs_, &client_,
1081 true /* restrict_to_whitelisted_entries */);
1082 TestRecordingHelper recorder(&service);
1083 EXPECT_EQ(0, GetPersistedLogCount());
1084 service.Initialize();
1085 task_runner_->RunUntilIdle();
1086 service.EnableRecording(/*extensions=*/false);
1087 service.EnableReporting();
1089 auto id = GetWhitelistedSourceId(0);
1090 recorder.UpdateSourceURL(id, GURL("https://username:password@example.com/"));
1093 EXPECT_EQ(1, GetPersistedLogCount());
1095 auto proto_report = GetPersistedReport();
1096 ASSERT_EQ(1, proto_report.sources_size());
1097 const Source& proto_source = proto_report.sources(0);
1098 EXPECT_EQ("https://example.com/", proto_source.url());
1101 TEST_F(UkmServiceTest, SanitizeChromeUrlParams) {
1104 const char* expected_url;
1106 {"chrome://version/?foo=bar", "chrome://version/"},
1107 {"about:blank?foo=bar", "about:blank"},
1108 {"chrome://histograms/Variations", "chrome://histograms/Variations"},
1109 {"http://google.ca/?foo=bar", "http://google.ca/?foo=bar"},
1110 {"https://google.ca/?foo=bar", "https://google.ca/?foo=bar"},
1111 {"ftp://google.ca/?foo=bar", "ftp://google.ca/?foo=bar"},
1112 {"chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj/foo.html?a=b",
1113 "chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj/"},
1116 for (const auto& test : test_cases) {
1119 UkmService service(&prefs_, &client_,
1120 true /* restrict_to_whitelisted_entries */);
1121 TestRecordingHelper recorder(&service);
1122 service.SetIsWebstoreExtensionCallback(
1123 base::BindRepeating(&TestIsWebstoreExtension));
1125 EXPECT_EQ(0, GetPersistedLogCount());
1126 service.Initialize();
1127 task_runner_->RunUntilIdle();
1128 service.EnableRecording(/*extensions=*/true);
1129 service.EnableReporting();
1131 auto id = GetWhitelistedSourceId(0);
1132 recorder.UpdateSourceURL(id, GURL(test.url));
1135 EXPECT_EQ(1, GetPersistedLogCount());
1137 auto proto_report = GetPersistedReport();
1138 ASSERT_EQ(1, proto_report.sources_size());
1139 const Source& proto_source = proto_report.sources(0);
1140 EXPECT_EQ(test.expected_url, proto_source.url());