components/ukm/ukm_service_unittest.cc

   1 // Copyright 2017 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/ukm/ukm_service.h"
   6
   7 #include <map>
   8 #include <memory>
   9 #include <string>
  10 #include <utility>
  11 #include <vector>
  12
  13 #include "base/containers/flat_map.h"
  14 #include "base/containers/flat_set.h"
  15 #include "base/functional/bind.h"
  16 #include "base/hash/hash.h"
  17 #include "base/memory/raw_ptr.h"
  18 #include "base/metrics/metrics_hashes.h"
  19 #include "base/ranges/algorithm.h"
  20 #include "base/run_loop.h"
  21 #include "base/strings/string_number_conversions.h"
  22 #include "base/strings/string_piece.h"
  23 #include "base/strings/string_util.h"
  24 #include "base/task/single_thread_task_runner.h"
  25 #include "base/test/scoped_feature_list.h"
  26 #include "base/test/task_environment.h"
  27 #include "base/test/test_simple_task_runner.h"
  28 #include "base/threading/platform_thread.h"
  29 #include "base/time/time.h"
  30 #include "components/metrics/cloned_install_detector.h"
  31 #include "components/metrics/log_decoder.h"
  32 #include "components/metrics/metrics_features.h"
  33 #include "components/metrics/metrics_log_uploader.h"
  34 #include "components/metrics/metrics_pref_names.h"
  35 #include "components/metrics/test/test_metrics_provider.h"
  36 #include "components/metrics/test/test_metrics_service_client.h"
  37 #include "components/metrics/ukm_demographic_metrics_provider.h"
  38 #include "components/metrics/unsent_log_store.h"
  39 #include "components/prefs/testing_pref_service.h"
  40 #include "components/ukm/observers/ukm_consent_state_observer.h"
  41 #include "components/ukm/test_ukm_recorder.h"
  42 #include "components/ukm/ukm_entry_filter.h"
  43 #include "components/ukm/ukm_pref_names.h"
  44 #include "components/ukm/ukm_recorder_impl.h"
  45 #include "components/ukm/ukm_recorder_observer.h"
  46 #include "components/ukm/unsent_log_store_metrics_impl.h"
  47 #include "services/metrics/public/cpp/mojo_ukm_recorder.h"
  48 #include "services/metrics/public/cpp/ukm_builders.h"
  49 #include "services/metrics/public/cpp/ukm_entry_builder.h"
  50 #include "services/metrics/public/cpp/ukm_recorder.h"
  51 #include "services/metrics/public/cpp/ukm_source.h"
  52 #include "services/metrics/public/cpp/ukm_source_id.h"
  53 #include "services/metrics/ukm_recorder_factory_impl.h"
  54 #include "testing/gmock/include/gmock/gmock.h"
  55 #include "testing/gtest/include/gtest/gtest.h"
  56 #include "third_party/metrics_proto/ukm/report.pb.h"
  57 #include "third_party/metrics_proto/ukm/source.pb.h"
  58 #include "third_party/metrics_proto/user_demographics.pb.h"
  59
  60 namespace ukm {
  61
  62 // Some arbitrary events used in tests.
  63 using TestEvent1 = builders::PageLoad;
  64 const char* kTestEvent1Metric1 =
  65     TestEvent1::kPaintTiming_NavigationToFirstContentfulPaintName;
  66 const char* kTestEvent1Metric2 = TestEvent1::kNet_CacheBytes2Name;
  67 using TestEvent2 = builders::Memory_Experimental;
  68 const char* kTestEvent2Metric1 = TestEvent2::kArrayBufferName;
  69 const char* kTestEvent2Metric2 = TestEvent2::kBlinkGCName;
  70 using TestEvent3 = builders::PageWithPassword;
  71 using TestProviderEvent = builders::ScreenBrightness;
  72
  73 SourceId ConvertSourceIdToAllowlistedType(SourceId id, SourceIdType type) {
  74   return ukm::SourceIdObj::FromOtherId(id, type).ToInt64();
  75 }
  76
  77 // A small shim exposing UkmRecorder methods to tests.
  78 class TestRecordingHelper {
  79  public:
  80   explicit TestRecordingHelper(UkmRecorder* recorder) : recorder_(recorder) {
  81     recorder_->SetSamplingForTesting(1);
  82   }
  83
  84   TestRecordingHelper(const TestRecordingHelper&) = delete;
  85   TestRecordingHelper& operator=(const TestRecordingHelper&) = delete;
  86
  87   void UpdateSourceURL(SourceId source_id, const GURL& url) {
  88     recorder_->UpdateSourceURL(source_id, url);
  89   }
  90
  91   void RecordNavigation(SourceId source_id,
  92                         const UkmSource::NavigationData& navigation_data) {
  93     recorder_->RecordNavigation(source_id, navigation_data);
  94   }
  95
  96   void MarkSourceForDeletion(SourceId source_id) {
  97     recorder_->MarkSourceForDeletion(source_id);
  98   }
  99
 100  private:
 101   raw_ptr<UkmRecorder> recorder_;
 102 };
 103
 104 class TestMetricsServiceClientWithClonedInstallDetector
 105     : public metrics::TestMetricsServiceClient {
 106  public:
 107   TestMetricsServiceClientWithClonedInstallDetector() = default;
 108
 109   TestMetricsServiceClientWithClonedInstallDetector(
 110       const TestMetricsServiceClientWithClonedInstallDetector&) = delete;
 111   TestMetricsServiceClientWithClonedInstallDetector& operator=(
 112       const TestMetricsServiceClientWithClonedInstallDetector&) = delete;
 113
 114   ~TestMetricsServiceClientWithClonedInstallDetector() override = default;
 115
 116   // metrics::MetricsServiceClient:
 117   base::CallbackListSubscription AddOnClonedInstallDetectedCallback(
 118       base::OnceClosure callback) override {
 119     return cloned_install_detector_.AddOnClonedInstallDetectedCallback(
 120         std::move(callback));
 121   }
 122
 123   metrics::ClonedInstallDetector* cloned_install_detector() {
 124     return &cloned_install_detector_;
 125   }
 126
 127  private:
 128   metrics::ClonedInstallDetector cloned_install_detector_;
 129 };
 130
 131 namespace {
 132
 133 bool TestIsWebstoreExtension(base::StringPiece id) {
 134   return (id == "bhcnanendmgjjeghamaccjnochlnhcgj");
 135 }
 136
 137 int GetPersistedLogCount(TestingPrefServiceSimple& prefs) {
 138   return prefs.GetList(prefs::kUkmUnsentLogStore).size();
 139 }
 140
 141 Report GetPersistedReport(TestingPrefServiceSimple& prefs) {
 142   EXPECT_GE(GetPersistedLogCount(prefs), 1);
 143   metrics::UnsentLogStore result_unsent_log_store(
 144       std::make_unique<UnsentLogStoreMetricsImpl>(), &prefs,
 145       prefs::kUkmUnsentLogStore, /*metadata_pref_name=*/nullptr,
 146       /*min_log_count=*/3, /*min_log_bytes=*/1000,
 147       /*max_log_size=*/0,
 148       /*signing_key=*/std::string(),
 149       /*logs_event_manager=*/nullptr);
 150
 151   result_unsent_log_store.LoadPersistedUnsentLogs();
 152   result_unsent_log_store.StageNextLog();
 153
 154   Report report;
 155   EXPECT_TRUE(metrics::DecodeLogDataToProto(
 156       result_unsent_log_store.staged_log(), &report));
 157   return report;
 158 }
 159
 160 class ScopedUkmFeatureParams {
 161  public:
 162   explicit ScopedUkmFeatureParams(const base::FieldTrialParams& params) {
 163     scoped_feature_list_.InitAndEnableFeatureWithParameters(kUkmFeature,
 164                                                             params);
 165   }
 166
 167   ScopedUkmFeatureParams(const ScopedUkmFeatureParams&) = delete;
 168   ScopedUkmFeatureParams& operator=(const ScopedUkmFeatureParams&) = delete;
 169
 170   ~ScopedUkmFeatureParams() {}
 171
 172  private:
 173   base::test::ScopedFeatureList scoped_feature_list_;
 174 };
 175
 176 class MockDemographicMetricsProvider
 177     : public metrics::UkmDemographicMetricsProvider {
 178  public:
 179   ~MockDemographicMetricsProvider() override {}
 180
 181   // DemographicMetricsProvider:
 182   MOCK_METHOD1(ProvideSyncedUserNoisedBirthYearAndGenderToReport,
 183                void(Report* report));
 184 };
 185
 186 // A simple Provider that emits a 'TestProviderEvent' on session close (i.e. a
 187 // Report being emitted).
 188 class UkmTestMetricsProvider : public metrics::TestMetricsProvider {
 189  public:
 190   explicit UkmTestMetricsProvider(UkmRecorder* test_recording_helper)
 191       : test_recording_helper_(test_recording_helper) {}
 192
 193   void ProvideCurrentSessionUKMData() override {
 194     // An Event emitted during a Provider will frequently not not associated
 195     // with a URL.
 196     SourceId id = ukm::NoURLSourceId();
 197     TestProviderEvent(id).Record(test_recording_helper_);
 198   }
 199
 200  private:
 201   raw_ptr<UkmRecorder> test_recording_helper_;
 202 };
 203
 204 class UkmServiceTest : public testing::Test,
 205                        public testing::WithParamInterface<std::tuple<bool>> {
 206  public:
 207   UkmServiceTest()
 208       : task_runner_(new base::TestSimpleTaskRunner),
 209         task_runner_current_default_handle_(task_runner_) {
 210     UkmService::RegisterPrefs(prefs_.registry());
 211     ClearPrefs();
 212   }
 213
 214   UkmServiceTest(const UkmServiceTest&) = delete;
 215   UkmServiceTest& operator=(const UkmServiceTest&) = delete;
 216
 217   bool ShouldClearLogsOnClonedInstall() { return std::get<0>(GetParam()); }
 218
 219   void SetUp() override {
 220     testing::Test::SetUp();
 221     std::vector<base::test::FeatureRefAndParams> enabled_features;
 222     std::vector<base::test::FeatureRef> disabled_features;
 223
 224     if (ShouldClearLogsOnClonedInstall()) {
 225       enabled_features.emplace_back(
 226           metrics::features::kMetricsClearLogsOnClonedInstall,
 227           /*params=*/std::map<std::string, std::string>());
 228     } else {
 229       disabled_features.emplace_back(
 230           metrics::features::kMetricsClearLogsOnClonedInstall);
 231     }
 232
 233     feature_list_.InitWithFeaturesAndParameters(enabled_features,
 234                                                 disabled_features);
 235   }
 236
 237   void ClearPrefs() {
 238     prefs_.ClearPref(prefs::kUkmClientId);
 239     prefs_.ClearPref(prefs::kUkmSessionId);
 240     prefs_.ClearPref(prefs::kUkmUnsentLogStore);
 241   }
 242
 243   int GetPersistedLogCount() { return ukm::GetPersistedLogCount(prefs_); }
 244
 245   Report GetPersistedReport() { return ukm::GetPersistedReport(prefs_); }
 246
 247   static SourceId GetAllowlistedSourceId(int64_t id) {
 248     return ConvertToSourceId(id, SourceIdType::NAVIGATION_ID);
 249   }
 250
 251   static SourceId GetAppIDSourceId(int64_t id) {
 252     return ConvertToSourceId(id, SourceIdType::APP_ID);
 253   }
 254
 255   static SourceId GetNonAllowlistedSourceId(int64_t id) {
 256     return ConvertToSourceId(id, SourceIdType::DEFAULT);
 257   }
 258
 259  protected:
 260   base::test::ScopedFeatureList feature_list_;
 261   TestingPrefServiceSimple prefs_;
 262   metrics::TestMetricsServiceClient client_;
 263
 264   scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
 265   base::SingleThreadTaskRunner::CurrentDefaultHandle
 266       task_runner_current_default_handle_;
 267 };
 268
 269 class UkmReduceAddEntryIpcTest : public testing::Test {
 270  public:
 271   UkmReduceAddEntryIpcTest() {
 272     UkmService::RegisterPrefs(prefs_.registry());
 273     ClearPrefs();
 274     scoped_feature_list_.InitAndEnableFeature(ukm::kUkmReduceAddEntryIPC);
 275   }
 276
 277   UkmReduceAddEntryIpcTest(const UkmReduceAddEntryIpcTest&) = delete;
 278   UkmReduceAddEntryIpcTest& operator=(const UkmReduceAddEntryIpcTest&) = delete;
 279
 280   void ClearPrefs() {
 281     prefs_.ClearPref(prefs::kUkmClientId);
 282     prefs_.ClearPref(prefs::kUkmSessionId);
 283     prefs_.ClearPref(prefs::kUkmUnsentLogStore);
 284   }
 285
 286  protected:
 287   TestingPrefServiceSimple prefs_;
 288   metrics::TestMetricsServiceClient client_;
 289   base::test::ScopedFeatureList scoped_feature_list_;
 290
 291  private:
 292   base::test::TaskEnvironment task_environment;
 293 };
 294 }  // namespace
 295
 296 INSTANTIATE_TEST_SUITE_P(All,
 297                          UkmServiceTest,
 298                          testing::Combine(testing::Bool()));
 299
 300 TEST_P(UkmServiceTest, ClientIdMigration) {
 301   prefs_.SetInt64(prefs::kUkmClientId, -1);
 302   UkmService service(&prefs_, &client_,
 303                      std::make_unique<MockDemographicMetricsProvider>());
 304   service.Initialize();
 305   uint64_t migrated_id = prefs_.GetUint64(prefs::kUkmClientId);
 306   // -1 migrates to the max UInt 64 value.
 307   EXPECT_EQ(migrated_id, 18446744073709551615ULL);
 308 }
 309
 310 TEST_P(UkmServiceTest, ClientIdClonedInstall) {
 311   prefs_.SetInt64(prefs::kUkmClientId, 123);
 312   UkmService service(&prefs_, &client_,
 313                      std::make_unique<MockDemographicMetricsProvider>());
 314
 315   EXPECT_FALSE(client_.ShouldResetClientIdsOnClonedInstall());
 316   client_.set_should_reset_client_ids_on_cloned_install(true);
 317   EXPECT_TRUE(client_.ShouldResetClientIdsOnClonedInstall());
 318
 319   uint64_t original_id = prefs_.GetUint64(prefs::kUkmClientId);
 320   service.Initialize();
 321   uint64_t new_id = prefs_.GetUint64(prefs::kUkmClientId);
 322   EXPECT_NE(original_id, new_id);
 323 }
 324
 325 TEST_P(UkmServiceTest, EnableDisableSchedule) {
 326   UkmService service(&prefs_, &client_,
 327                      std::make_unique<MockDemographicMetricsProvider>());
 328   EXPECT_FALSE(task_runner_->HasPendingTask());
 329   service.Initialize();
 330   EXPECT_FALSE(task_runner_->HasPendingTask());
 331   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 332   service.EnableReporting();
 333   EXPECT_TRUE(task_runner_->HasPendingTask());
 334   service.DisableReporting();
 335   task_runner_->RunPendingTasks();
 336   EXPECT_FALSE(task_runner_->HasPendingTask());
 337 }
 338
 339 TEST_P(UkmServiceTest, PersistAndPurge) {
 340   UkmService service(&prefs_, &client_,
 341                      std::make_unique<MockDemographicMetricsProvider>());
 342   TestRecordingHelper recorder(&service);
 343   EXPECT_EQ(GetPersistedLogCount(), 0);
 344   service.Initialize();
 345   task_runner_->RunUntilIdle();
 346   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 347   service.EnableReporting();
 348
 349   SourceId id = GetAllowlistedSourceId(0);
 350   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 351   // Should init, generate a log, and start an upload for source.
 352   task_runner_->RunPendingTasks();
 353   EXPECT_TRUE(client_.uploader()->is_uploading());
 354   // Flushes the generated log to disk and generates a new entry.
 355   TestEvent1(id).Record(&service);
 356   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 357   EXPECT_EQ(GetPersistedLogCount(), 2);
 358   service.Purge();
 359   EXPECT_EQ(GetPersistedLogCount(), 0);
 360 }
 361
 362 TEST_P(UkmServiceTest, Purge) {
 363   UkmService service(&prefs_, &client_,
 364                      std::make_unique<MockDemographicMetricsProvider>());
 365   TestRecordingHelper recorder(&service);
 366   EXPECT_EQ(GetPersistedLogCount(), 0);
 367   service.Initialize();
 368   task_runner_->RunUntilIdle();
 369   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 370   service.EnableReporting();
 371
 372   // Record some data
 373   auto id = GetAllowlistedSourceId(0);
 374   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar1"));
 375   TestEvent1(id).Record(&service);
 376
 377   // Purge should delete data, so there shouldn't be anything left to upload.
 378   service.Purge();
 379   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 380   EXPECT_EQ(0, GetPersistedLogCount());
 381 }
 382
 383 TEST_P(UkmServiceTest, PurgeExtensionDataFromUnsentLogStore) {
 384   UkmService service(&prefs_, &client_,
 385                      std::make_unique<MockDemographicMetricsProvider>());
 386   auto* unsent_log_store = service.reporting_service_.ukm_log_store();
 387
 388   // Initialize a Report to be saved to the log store.
 389   Report report;
 390   report.set_client_id(1);
 391   report.set_session_id(1);
 392   report.set_report_id(1);
 393
 394   std::string non_extension_url = "https://www.google.ca";
 395   std::string extension_url =
 396       "chrome-extension://bmnlcjabgnpnenekpadlanbbkooimhnj/manifest.json";
 397
 398   // Add both extension- and non-extension-related sources to the Report.
 399   Source* proto_source_1 = report.add_sources();
 400   SourceId source_id_1 = ConvertToSourceId(1, SourceIdType::NAVIGATION_ID);
 401   proto_source_1->set_id(source_id_1);
 402   proto_source_1->add_urls()->set_url(non_extension_url);
 403   Source* proto_source_2 = report.add_sources();
 404   SourceId source_id_2 = ConvertToSourceId(2, SourceIdType::NAVIGATION_ID);
 405   proto_source_2->set_id(source_id_2);
 406   proto_source_2->add_urls()->set_url(extension_url);
 407
 408   // Add some entries for both sources.
 409   Entry* entry_1 = report.add_entries();
 410   entry_1->set_source_id(source_id_2);
 411   Entry* entry_2 = report.add_entries();
 412   entry_2->set_source_id(source_id_1);
 413   Entry* entry_3 = report.add_entries();
 414   entry_3->set_source_id(source_id_2);
 415
 416   // Save the Report to the store.
 417   std::string serialized_log;
 418   report.SerializeToString(&serialized_log);
 419   // Makes sure that the serialized ukm report can be parsed.
 420   ASSERT_TRUE(UkmService::LogCanBeParsed(serialized_log));
 421   metrics::LogMetadata log_metadata;
 422   unsent_log_store->StoreLog(
 423       serialized_log, log_metadata,
 424       metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 425
 426   // Do extension purging.
 427   service.PurgeExtensionsData();
 428
 429   // Get the Report in the log store and verify extension-related data have been
 430   // filtered.
 431   unsent_log_store->StageNextLog();
 432   const std::string& compressed_log_data = unsent_log_store->staged_log();
 433
 434   Report filtered_report;
 435   ASSERT_TRUE(
 436       metrics::DecodeLogDataToProto(compressed_log_data, &filtered_report));
 437
 438   // Only proto_source_1 with non-extension URL is kept.
 439   EXPECT_EQ(1, filtered_report.sources_size());
 440   EXPECT_EQ(source_id_1, filtered_report.sources(0).id());
 441   EXPECT_EQ(non_extension_url, filtered_report.sources(0).urls(0).url());
 442
 443   // Only entry_2 from the non-extension source is kept.
 444   EXPECT_EQ(1, filtered_report.entries_size());
 445   EXPECT_EQ(source_id_1, filtered_report.entries(0).source_id());
 446 }
 447
 448 TEST_P(UkmServiceTest, PurgeAppDataFromUnsentLogStore) {
 449   UkmService service(&prefs_, &client_,
 450                      std::make_unique<MockDemographicMetricsProvider>());
 451   auto* unsent_log_store = service.reporting_service_.ukm_log_store();
 452
 453   // Initialize a Report to be saved to the log store.
 454   Report report;
 455   report.set_client_id(1);
 456   report.set_session_id(1);
 457   report.set_report_id(1);
 458
 459   // A URL from browser navigation.
 460   std::string non_app_url = "https://www.google.ca";
 461   // A URL with app:// scheme.
 462   std::string app_url = "app://mgndgikekgjfcpckkfioiadnlibdjbkf";
 463   // OS Settings is an app on ChromeOS without the app:// scheme.
 464   std::string os_settings_url = "chrome://os-settings";
 465
 466   // Add sources to the Report.
 467   // Non-app source.
 468   Source* proto_source_1 = report.add_sources();
 469   SourceId source_id_1 = ConvertToSourceId(1, SourceIdType::NAVIGATION_ID);
 470   proto_source_1->set_id(source_id_1);
 471   proto_source_1->add_urls()->set_url(non_app_url);
 472   // App source with app:// URL.
 473   Source* proto_source_2 = report.add_sources();
 474   SourceId source_id_2 = ConvertToSourceId(2, SourceIdType::APP_ID);
 475   proto_source_2->set_id(source_id_2);
 476   proto_source_2->add_urls()->set_url(app_url);
 477   // App source with non-app:// URL.
 478   Source* proto_source_3 = report.add_sources();
 479   SourceId source_id_3 = ConvertToSourceId(3, SourceIdType::APP_ID);
 480   proto_source_3->set_id(source_id_3);
 481   proto_source_3->add_urls()->set_url(os_settings_url);
 482   // Non-app source with app:// URL. This shouldn't happen in practice, but
 483   // if it does, this source should be purged when app data are purged.
 484   Source* proto_source_4 = report.add_sources();
 485   SourceId source_id_4 = ConvertToSourceId(4, SourceIdType::NAVIGATION_ID);
 486   proto_source_4->set_id(source_id_4);
 487   proto_source_4->add_urls()->set_url(app_url);
 488
 489   // Add entries to each of the sources.
 490   Entry* entry_1 = report.add_entries();
 491   entry_1->set_source_id(source_id_2);
 492   Entry* entry_2 = report.add_entries();
 493   entry_2->set_source_id(source_id_1);
 494   Entry* entry_3 = report.add_entries();
 495   entry_3->set_source_id(source_id_2);
 496   Entry* entry_4 = report.add_entries();
 497   entry_4->set_source_id(source_id_3);
 498   Entry* entry_5 = report.add_entries();
 499   entry_5->set_source_id(source_id_4);
 500
 501   // Save the Report to the store.
 502   std::string serialized_log;
 503   report.SerializeToString(&serialized_log);
 504   // Make sure that the serialized ukm report can be parsed.
 505   ASSERT_TRUE(UkmService::LogCanBeParsed(serialized_log));
 506   metrics::LogMetadata log_metadata;
 507   unsent_log_store->StoreLog(
 508       serialized_log, log_metadata,
 509       metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 510
 511   // Do app data purging.
 512   service.PurgeAppsData();
 513
 514   // Get the Report in the log store and verify app-related data have been
 515   // filtered.
 516   unsent_log_store->StageNextLog();
 517   const std::string& compressed_log_data = unsent_log_store->staged_log();
 518
 519   Report filtered_report;
 520   ASSERT_TRUE(
 521       metrics::DecodeLogDataToProto(compressed_log_data, &filtered_report));
 522
 523   // Only proto_source_1 with non-app URL is kept.
 524   EXPECT_EQ(1, filtered_report.sources_size());
 525   EXPECT_EQ(source_id_1, filtered_report.sources(0).id());
 526   EXPECT_EQ(non_app_url, filtered_report.sources(0).urls(0).url());
 527
 528   // Only entry_2 from the non-app source is kept.
 529   EXPECT_EQ(1, filtered_report.entries_size());
 530   EXPECT_EQ(source_id_1, filtered_report.entries(0).source_id());
 531 }
 532
 533 TEST_P(UkmServiceTest, PurgeMsbbDataFromUnsentLogStore) {
 534   UkmService service(&prefs_, &client_,
 535                      std::make_unique<MockDemographicMetricsProvider>());
 536   auto* unsent_log_store = service.reporting_service_.ukm_log_store();
 537
 538   // Initialize a Report to be saved to the log store.
 539   Report report;
 540   report.set_client_id(1);
 541   report.set_session_id(1);
 542   report.set_report_id(1);
 543
 544   // A URL from browser navigation.
 545   std::string non_app_url = "https://www.google.ca";
 546   std::string non_app_url2 = "https://www.google.com";
 547   // A URL with app:// scheme.
 548   std::string app_url = "app://mgndgikekgjfcpckkfioiadnlibdjbkf";
 549   // OS Settings is an app on ChromeOS without the app:// scheme.
 550   std::string os_settings_url = "chrome://os-settings";
 551
 552   // Add sources to the Report.
 553   // Non-app source.
 554   Source* proto_source_1 = report.add_sources();
 555   SourceId source_id_1 = ConvertToSourceId(1, SourceIdType::NAVIGATION_ID);
 556   proto_source_1->set_id(source_id_1);
 557   proto_source_1->add_urls()->set_url(non_app_url);
 558   // Non-app source 2.
 559   Source* proto_source_2 = report.add_sources();
 560   SourceId source_id_2 = ConvertToSourceId(2, SourceIdType::NAVIGATION_ID);
 561   proto_source_2->set_id(source_id_2);
 562   proto_source_2->add_urls()->set_url(non_app_url);
 563   // App source with app:// URL.
 564   Source* proto_source_3 = report.add_sources();
 565   SourceId source_id_3 = ConvertToSourceId(3, SourceIdType::APP_ID);
 566   proto_source_3->set_id(source_id_3);
 567   proto_source_3->add_urls()->set_url(app_url);
 568   // App source with non-app:// URL.
 569   Source* proto_source_4 = report.add_sources();
 570   SourceId source_id_4 = ConvertToSourceId(4, SourceIdType::APP_ID);
 571   proto_source_4->set_id(source_id_4);
 572   proto_source_4->add_urls()->set_url(os_settings_url);
 573   // Non-app source with app:// URL. This shouldn't happen in practice, but
 574   // if it does, this source should be purged when app data are purged.
 575   Source* proto_source_5 = report.add_sources();
 576   SourceId source_id_5 = ConvertToSourceId(5, SourceIdType::NAVIGATION_ID);
 577   proto_source_5->set_id(source_id_5);
 578   proto_source_5->add_urls()->set_url(app_url);
 579
 580   // Add entries to each of the sources.
 581   Entry* entry_1 = report.add_entries();
 582   entry_1->set_source_id(source_id_2);
 583   Entry* entry_2 = report.add_entries();
 584   entry_2->set_source_id(source_id_1);
 585   Entry* entry_3 = report.add_entries();
 586   entry_3->set_source_id(source_id_2);
 587   Entry* entry_4 = report.add_entries();
 588   entry_4->set_source_id(source_id_3);
 589   Entry* entry_5 = report.add_entries();
 590   entry_5->set_source_id(source_id_4);
 591   Entry* entry_6 = report.add_entries();
 592   entry_6->set_source_id(source_id_5);
 593
 594   // Save the Report to the store.
 595   std::string serialized_log;
 596   report.SerializeToString(&serialized_log);
 597   // Make sure that the serialized ukm report can be parsed.
 598   ASSERT_TRUE(UkmService::LogCanBeParsed(serialized_log));
 599   metrics::LogMetadata log_metadata;
 600   unsent_log_store->StoreLog(
 601       serialized_log, log_metadata,
 602       metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 603
 604   // Purge MSBB data.
 605   service.PurgeMsbbData();
 606
 607   // Get the Report in the log store and verify non-app-related data have been
 608   // filtered.
 609   unsent_log_store->StageNextLog();
 610   const std::string& compressed_log_data = unsent_log_store->staged_log();
 611
 612   Report filtered_report;
 613   ASSERT_TRUE(
 614       metrics::DecodeLogDataToProto(compressed_log_data, &filtered_report));
 615
 616   // Source proto_source_1 with app URL is kept.
 617   EXPECT_EQ(2, filtered_report.sources_size());
 618   EXPECT_EQ(source_id_4, filtered_report.sources(0).id());
 619   EXPECT_EQ(os_settings_url, filtered_report.sources(0).urls(0).url());
 620
 621   EXPECT_EQ(source_id_3, filtered_report.sources(1).id());
 622   EXPECT_EQ(app_url, filtered_report.sources(1).urls(0).url());
 623
 624   // Entry entry_2 from the app source is kept.
 625   EXPECT_EQ(2, filtered_report.entries_size());
 626   EXPECT_EQ(source_id_4, filtered_report.entries(0).source_id());
 627   EXPECT_EQ(source_id_3, filtered_report.entries(1).source_id());
 628 }
 629
 630 TEST_P(UkmServiceTest, SourceSerialization) {
 631   UkmService service(&prefs_, &client_,
 632                      std::make_unique<MockDemographicMetricsProvider>());
 633   TestRecordingHelper recorder(&service);
 634   EXPECT_EQ(GetPersistedLogCount(), 0);
 635   service.Initialize();
 636   task_runner_->RunUntilIdle();
 637   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 638   service.EnableReporting();
 639
 640   UkmSource::NavigationData navigation_data;
 641   navigation_data.urls = {GURL("https://google.com/initial"),
 642                           GURL("https://google.com/final")};
 643
 644   SourceId id = GetAllowlistedSourceId(0);
 645   recorder.RecordNavigation(id, navigation_data);
 646
 647   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 648   EXPECT_EQ(GetPersistedLogCount(), 1);
 649
 650   Report proto_report = GetPersistedReport();
 651   EXPECT_EQ(1, proto_report.sources_size());
 652   EXPECT_TRUE(proto_report.has_session_id());
 653   const Source& proto_source = proto_report.sources(0);
 654
 655   EXPECT_EQ(id, proto_source.id());
 656   EXPECT_EQ(GURL("https://google.com/final").spec(),
 657             proto_source.urls(1).url());
 658 }
 659
 660 TEST_P(UkmServiceTest, AddEntryWithEmptyMetrics) {
 661   UkmService service(&prefs_, &client_,
 662                      std::make_unique<MockDemographicMetricsProvider>());
 663   TestRecordingHelper recorder(&service);
 664   ASSERT_EQ(0, GetPersistedLogCount());
 665   service.Initialize();
 666   task_runner_->RunUntilIdle();
 667   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 668   service.EnableReporting();
 669
 670   SourceId id = GetAllowlistedSourceId(0);
 671   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 672
 673   TestEvent1(id).Record(&service);
 674   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 675   ASSERT_EQ(1, GetPersistedLogCount());
 676   Report proto_report = GetPersistedReport();
 677   EXPECT_EQ(1, proto_report.entries_size());
 678 }
 679
 680 TEST_P(UkmServiceTest, MetricsProviderTest) {
 681   UkmService service(&prefs_, &client_,
 682                      std::make_unique<MockDemographicMetricsProvider>());
 683   TestRecordingHelper recorder(&service);
 684
 685   auto* provider = new UkmTestMetricsProvider(&service);
 686   service.RegisterMetricsProvider(
 687       std::unique_ptr<metrics::MetricsProvider>(provider));
 688
 689   service.Initialize();
 690
 691   // Providers have not supplied system profile information yet.
 692   EXPECT_FALSE(provider->provide_system_profile_metrics_called());
 693
 694   task_runner_->RunUntilIdle();
 695   service.UpdateRecording(UkmConsentState(MSBB));
 696   service.EnableReporting();
 697
 698   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 699   EXPECT_EQ(GetPersistedLogCount(), 1);
 700
 701   Report proto_report = GetPersistedReport();
 702   // We should have an Event from a Provider provided metric, however, it is not
 703   // attached to a Source (which should be typical for a Provider metric).
 704   EXPECT_EQ(proto_report.sources_size(), 0);
 705
 706   // Providers have now supplied system profile.
 707   EXPECT_TRUE(provider->provide_system_profile_metrics_called());
 708   // Providers has also supplied a UKM Event.
 709   const Entry& entry = proto_report.entries(0);
 710   EXPECT_EQ(base::HashMetricName(TestProviderEvent::kEntryName),
 711             entry.event_hash());
 712 }
 713
 714 // Currently just testing brand is set, would be good to test other core
 715 // system profile fields.
 716 TEST_P(UkmServiceTest, SystemProfileTest) {
 717   UkmService service(&prefs_, &client_,
 718                      std::make_unique<MockDemographicMetricsProvider>());
 719   TestRecordingHelper recorder(&service);
 720
 721   service.Initialize();
 722
 723   task_runner_->RunUntilIdle();
 724   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 725   service.EnableReporting();
 726
 727   SourceId id = GetAllowlistedSourceId(0);
 728   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 729   TestEvent1(id).Record(&service);
 730   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 731   EXPECT_EQ(GetPersistedLogCount(), 1);
 732
 733   Report proto_report = GetPersistedReport();
 734   EXPECT_EQ(metrics::TestMetricsServiceClient::kBrandForTesting,
 735             proto_report.system_profile().brand_code());
 736 }
 737
 738 TEST_P(UkmServiceTest, AddUserDemograhicsWhenAvailableAndFeatureEnabled) {
 739   int number_of_invocations = 0;
 740   int test_birth_year = 1983;
 741   metrics::UserDemographicsProto::Gender test_gender =
 742       metrics::UserDemographicsProto::GENDER_FEMALE;
 743
 744   auto provider = std::make_unique<MockDemographicMetricsProvider>();
 745   // Expect the synced user's noised birth year and gender to be added 2 times
 746   // to the UKM report: on the event trigger and when tearing down the UKM
 747   // service.
 748   EXPECT_CALL(*provider,
 749               ProvideSyncedUserNoisedBirthYearAndGenderToReport(testing::_))
 750       .Times(2)
 751       .WillRepeatedly([&number_of_invocations, test_gender,
 752                        test_birth_year](Report* report) {
 753         report->mutable_user_demographics()->set_birth_year(test_birth_year);
 754         report->mutable_user_demographics()->set_gender(test_gender);
 755         ++number_of_invocations;
 756       });
 757
 758   UkmService service(&prefs_, &client_, std::move(provider));
 759   TestRecordingHelper recorder(&service);
 760
 761   service.Initialize();
 762
 763   // Verify that the provider hasn't provided the synced user's noised birth
 764   // year and gender yet.
 765   EXPECT_EQ(0, number_of_invocations);
 766
 767   task_runner_->RunUntilIdle();
 768   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 769   service.EnableReporting();
 770
 771   SourceId id = GetAllowlistedSourceId(0);
 772   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 773   TestEvent1(id).Record(&service);
 774   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 775   EXPECT_EQ(1, GetPersistedLogCount());
 776
 777   // Verify that the synced user's noised birth year and gender were added to
 778   // the UKM report.
 779   Report proto_report = GetPersistedReport();
 780   EXPECT_EQ(test_birth_year, proto_report.user_demographics().birth_year());
 781   EXPECT_EQ(test_gender, proto_report.user_demographics().gender());
 782
 783   // Verify that the provider's method was only called once before the
 784   // destruction of the service.
 785   EXPECT_EQ(1, number_of_invocations);
 786 }
 787
 788 TEST_P(UkmServiceTest,
 789        DontAddUserDemograhicsWhenNotAvailableAndFeatureEnabled) {
 790   auto provider = std::make_unique<MockDemographicMetricsProvider>();
 791   EXPECT_CALL(*provider,
 792               ProvideSyncedUserNoisedBirthYearAndGenderToReport(testing::_))
 793       .Times(2)
 794       .WillRepeatedly([](Report* report) {});
 795
 796   UkmService service(&prefs_, &client_, std::move(provider));
 797   TestRecordingHelper recorder(&service);
 798   service.Initialize();
 799
 800   task_runner_->RunUntilIdle();
 801   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 802   service.EnableReporting();
 803
 804   SourceId id = GetAllowlistedSourceId(0);
 805   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 806   TestEvent1(id).Record(&service);
 807   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 808   EXPECT_EQ(1, GetPersistedLogCount());
 809
 810   // Verify that the synced user's noised birth year and gender are not in the
 811   // report when they are not available.
 812   Report proto_report = GetPersistedReport();
 813   EXPECT_FALSE(proto_report.has_user_demographics());
 814 }
 815
 816 TEST_P(UkmServiceTest, DontAddUserDemograhicsWhenFeatureDisabled) {
 817   base::test::ScopedFeatureList local_feature;
 818   local_feature.InitAndDisableFeature(kReportUserNoisedUserBirthYearAndGender);
 819
 820   // The demographics provider should not be called.
 821   auto provider = std::make_unique<MockDemographicMetricsProvider>();
 822   EXPECT_CALL(*provider,
 823               ProvideSyncedUserNoisedBirthYearAndGenderToReport(testing::_))
 824       .Times(0);
 825
 826   UkmService service(&prefs_, &client_, std::move(provider));
 827   TestRecordingHelper recorder(&service);
 828
 829   service.Initialize();
 830
 831   task_runner_->RunUntilIdle();
 832   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 833   service.EnableReporting();
 834
 835   SourceId id = GetAllowlistedSourceId(0);
 836   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 837   TestEvent1(id).Record(&service);
 838   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 839   EXPECT_EQ(1, GetPersistedLogCount());
 840
 841   // Verify that the synced user's noised birth year and gender are not in the
 842   // report when they are not available.
 843   Report proto_report = GetPersistedReport();
 844   EXPECT_FALSE(proto_report.has_user_demographics());
 845 }
 846
 847 TEST_P(UkmServiceTest, LogsRotation) {
 848   UkmService service(&prefs_, &client_,
 849                      std::make_unique<MockDemographicMetricsProvider>());
 850   TestRecordingHelper recorder(&service);
 851   EXPECT_EQ(GetPersistedLogCount(), 0);
 852   service.Initialize();
 853   task_runner_->RunUntilIdle();
 854   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 855   service.EnableReporting();
 856
 857   EXPECT_EQ(0, service.report_count());
 858
 859   // Log rotation should generate a log.
 860   const SourceId id = GetAllowlistedSourceId(0);
 861   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 862   task_runner_->RunPendingTasks();
 863   EXPECT_EQ(1, service.report_count());
 864   EXPECT_TRUE(client_.uploader()->is_uploading());
 865
 866   // Rotation shouldn't generate a log due to one being pending.
 867   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 868   task_runner_->RunPendingTasks();
 869   EXPECT_EQ(1, service.report_count());
 870   EXPECT_TRUE(client_.uploader()->is_uploading());
 871
 872   // Completing the upload should clear pending log, then log rotation should
 873   // generate another log.
 874   client_.uploader()->CompleteUpload(200);
 875   task_runner_->RunPendingTasks();
 876   EXPECT_EQ(2, service.report_count());
 877
 878   // Check that rotations keep working.
 879   for (int i = 3; i < 6; i++) {
 880     task_runner_->RunPendingTasks();
 881     client_.uploader()->CompleteUpload(200);
 882     recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 883     task_runner_->RunPendingTasks();
 884     EXPECT_EQ(i, service.report_count());
 885   }
 886 }
 887
 888 TEST_P(UkmServiceTest, LogsUploadedOnlyWhenHavingSourcesOrEntries) {
 889   UkmService service(&prefs_, &client_,
 890                      std::make_unique<MockDemographicMetricsProvider>());
 891   TestRecordingHelper recorder(&service);
 892   EXPECT_EQ(GetPersistedLogCount(), 0);
 893   service.Initialize();
 894   task_runner_->RunUntilIdle();
 895   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 896   service.EnableReporting();
 897
 898   EXPECT_TRUE(task_runner_->HasPendingTask());
 899   // Neither rotation or Flush should generate logs
 900   task_runner_->RunPendingTasks();
 901   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 902   EXPECT_EQ(GetPersistedLogCount(), 0);
 903
 904   SourceId id = GetAllowlistedSourceId(0);
 905   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 906   // Includes a Source, so will persist.
 907   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 908   EXPECT_EQ(GetPersistedLogCount(), 1);
 909
 910   TestEvent1(id).Record(&service);
 911   // Includes an Entry, so will persist.
 912   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 913   EXPECT_EQ(GetPersistedLogCount(), 2);
 914
 915   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 916   TestEvent1(id).Record(&service);
 917   // Do not keep the source in the recorder after the current log.
 918   recorder.MarkSourceForDeletion(id);
 919   // Includes a Source and an Entry, so will persist.
 920   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 921   EXPECT_EQ(GetPersistedLogCount(), 3);
 922
 923   // The recorder contains no Sources or Entries thus will not create a new log.
 924   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 925   EXPECT_EQ(GetPersistedLogCount(), 3);
 926 }
 927
 928 TEST_P(UkmServiceTest, GetNewSourceID) {
 929   SourceId id1 = UkmRecorder::GetNewSourceID();
 930   SourceId id2 = UkmRecorder::GetNewSourceID();
 931   SourceId id3 = UkmRecorder::GetNewSourceID();
 932   EXPECT_NE(id1, id2);
 933   EXPECT_NE(id1, id3);
 934   EXPECT_NE(id2, id3);
 935 }
 936
 937 TEST_P(UkmServiceTest, RecordRedirectedUrl) {
 938   ClearPrefs();
 939   UkmService service(&prefs_, &client_,
 940                      std::make_unique<MockDemographicMetricsProvider>());
 941   TestRecordingHelper recorder(&service);
 942   EXPECT_EQ(GetPersistedLogCount(), 0);
 943   service.Initialize();
 944   task_runner_->RunUntilIdle();
 945   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 946   service.EnableReporting();
 947
 948   SourceId id = GetAllowlistedSourceId(0);
 949   UkmSource::NavigationData navigation_data;
 950   navigation_data.urls = {GURL("https://google.com/initial"),
 951                           GURL("https://google.com/final")};
 952   recorder.RecordNavigation(id, navigation_data);
 953
 954   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 955   EXPECT_EQ(GetPersistedLogCount(), 1);
 956
 957   Report proto_report = GetPersistedReport();
 958   EXPECT_EQ(1, proto_report.sources_size());
 959   const Source& proto_source = proto_report.sources(0);
 960
 961   EXPECT_EQ(id, proto_source.id());
 962   EXPECT_EQ(GURL("https://google.com/initial").spec(),
 963             proto_source.urls(0).url());
 964   EXPECT_EQ(GURL("https://google.com/final").spec(),
 965             proto_source.urls(1).url());
 966 }
 967
 968 TEST_P(UkmServiceTest, RecordSessionId) {
 969   ClearPrefs();
 970   UkmService service(&prefs_, &client_,
 971                      std::make_unique<MockDemographicMetricsProvider>());
 972   TestRecordingHelper recorder(&service);
 973   EXPECT_EQ(0, GetPersistedLogCount());
 974   service.Initialize();
 975   task_runner_->RunUntilIdle();
 976   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 977   service.EnableReporting();
 978
 979   auto id = GetAllowlistedSourceId(0);
 980   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
 981
 982   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
 983   EXPECT_EQ(1, GetPersistedLogCount());
 984
 985   auto proto_report = GetPersistedReport();
 986   EXPECT_TRUE(proto_report.has_session_id());
 987   EXPECT_EQ(1, proto_report.report_id());
 988 }
 989
 990 TEST_P(UkmServiceTest, SourceSize) {
 991   ClearPrefs();
 992   UkmService service(&prefs_, &client_,
 993                      std::make_unique<MockDemographicMetricsProvider>());
 994   TestRecordingHelper recorder(&service);
 995   EXPECT_EQ(0, GetPersistedLogCount());
 996   service.Initialize();
 997   task_runner_->RunUntilIdle();
 998   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
 999   service.EnableReporting();
1000
1001   // Add a large number of sources, more than the hardcoded max.
1002   for (int i = 0; i < 1000; ++i) {
1003     auto id = GetAllowlistedSourceId(i);
1004     recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
1005   }
1006
1007   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1008   EXPECT_EQ(1, GetPersistedLogCount());
1009
1010   auto proto_report = GetPersistedReport();
1011   // Note, 500 instead of 1000 sources, since 500 is the maximum.
1012   EXPECT_EQ(500, proto_report.sources_size());
1013 }
1014
1015 TEST_P(UkmServiceTest, PurgeMidUpload) {
1016   UkmService service(&prefs_, &client_,
1017                      std::make_unique<MockDemographicMetricsProvider>());
1018   TestRecordingHelper recorder(&service);
1019   EXPECT_EQ(GetPersistedLogCount(), 0);
1020   service.Initialize();
1021   task_runner_->RunUntilIdle();
1022   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1023   service.EnableReporting();
1024
1025   auto id = GetAllowlistedSourceId(0);
1026   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar1"));
1027   // Should init, generate a log, and start an upload.
1028   task_runner_->RunPendingTasks();
1029   EXPECT_TRUE(client_.uploader()->is_uploading());
1030   // Purge should delete all logs, including the one being sent.
1031   service.Purge();
1032   // Upload succeeds after logs was deleted.
1033   client_.uploader()->CompleteUpload(200);
1034   EXPECT_EQ(GetPersistedLogCount(), 0);
1035   EXPECT_FALSE(client_.uploader()->is_uploading());
1036 }
1037
1038 TEST_P(UkmServiceTest, SourceURLLength) {
1039   UkmService service(&prefs_, &client_,
1040                      std::make_unique<MockDemographicMetricsProvider>());
1041   TestRecordingHelper recorder(&service);
1042   EXPECT_EQ(0, GetPersistedLogCount());
1043   service.Initialize();
1044   task_runner_->RunUntilIdle();
1045   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1046   service.EnableReporting();
1047
1048   auto id = GetAllowlistedSourceId(0);
1049
1050   // This URL is too long to be recorded fully.
1051   const std::string long_string =
1052       "https://example.com/" + std::string(10000, 'a');
1053   recorder.UpdateSourceURL(id, GURL(long_string));
1054
1055   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1056   EXPECT_EQ(1, GetPersistedLogCount());
1057
1058   auto proto_report = GetPersistedReport();
1059   ASSERT_EQ(1, proto_report.sources_size());
1060   const Source& proto_source = proto_report.sources(0);
1061   EXPECT_EQ("URLTooLong", proto_source.urls(0).url());
1062 }
1063
1064 TEST_P(UkmServiceTest, UnreferencedNonAllowlistedSources) {
1065   const GURL kURL("https://google.com/foobar");
1066
1067   ClearPrefs();
1068   UkmService service(&prefs_, &client_,
1069                      std::make_unique<MockDemographicMetricsProvider>());
1070   TestRecordingHelper recorder(&service);
1071   EXPECT_EQ(0, GetPersistedLogCount());
1072   service.Initialize();
1073   task_runner_->RunUntilIdle();
1074   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1075   service.EnableReporting();
1076
1077   // Record with Allowlisted ID to allowlist the URL.
1078   // Use a larger ID to make it last in the proto.
1079   SourceId allowlisted_id = GetAllowlistedSourceId(100);
1080   recorder.UpdateSourceURL(allowlisted_id, kURL);
1081
1082   std::vector<SourceId> ids;
1083   base::TimeTicks last_time = base::TimeTicks::Now();
1084   for (int i = 1; i < 7; ++i) {
1085     // Wait until base::TimeTicks::Now() no longer equals |last_time|. This
1086     // ensures each source has a unique timestamp to avoid flakes. Should take
1087     // between 1-15ms per documented resolution of base::TimeTicks.
1088     while (base::TimeTicks::Now() == last_time) {
1089       base::PlatformThread::Sleep(base::Milliseconds(1));
1090     }
1091
1092     ids.push_back(GetNonAllowlistedSourceId(i));
1093     recorder.UpdateSourceURL(ids.back(), kURL);
1094     last_time = base::TimeTicks::Now();
1095   }
1096
1097   TestEvent1(ids[0]).Record(&service);
1098   TestEvent2(ids[2]).Record(&service);
1099   TestEvent3(ids[2]).Record(&service);
1100   TestEvent3(ids[3]).Record(&service);
1101
1102   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1103   EXPECT_EQ(1, GetPersistedLogCount());
1104   auto proto_report = GetPersistedReport();
1105
1106   // 1 allowlisted source and 6 non-allowlisted source.
1107   EXPECT_EQ(7, proto_report.source_counts().observed());
1108   // The one allowlisted source is of navigation type.
1109   EXPECT_EQ(1, proto_report.source_counts().navigation_sources());
1110   EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
1111
1112   EXPECT_EQ(4, proto_report.source_counts().deferred_sources());
1113   EXPECT_EQ(0, proto_report.source_counts().carryover_sources());
1114
1115   ASSERT_EQ(4, proto_report.sources_size());
1116   EXPECT_EQ(ids[0], proto_report.sources(0).id());
1117   EXPECT_EQ(kURL.spec(), proto_report.sources(0).urls(0).url());
1118   EXPECT_EQ(ids[2], proto_report.sources(1).id());
1119   EXPECT_EQ(kURL.spec(), proto_report.sources(1).urls(0).url());
1120   // Since MaxKeptSources is 3, only Sources 5, 4, 3 should be retained.
1121   // Log entries under 0, 1, 3 and 4. Log them in reverse order - which
1122   // shouldn't affect source ordering in the output.
1123   //  - Source 0 should not be re-transmitted since it was sent before.
1124   //  - Source 1 should not be transmitted due to MaxKeptSources param.
1125   //  - Sources 3 and 4 should be transmitted since they were not sent before.
1126   TestEvent1(ids[4]).Record(&service);
1127   TestEvent1(ids[3]).Record(&service);
1128   TestEvent1(ids[1]).Record(&service);
1129   TestEvent1(ids[0]).Record(&service);
1130
1131   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1132   EXPECT_EQ(2, GetPersistedLogCount());
1133   proto_report = GetPersistedReport();
1134
1135   EXPECT_EQ(0, proto_report.source_counts().observed());
1136   EXPECT_EQ(0, proto_report.source_counts().navigation_sources());
1137   EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
1138
1139   EXPECT_EQ(2, proto_report.source_counts().deferred_sources());
1140
1141   EXPECT_EQ(4, proto_report.source_counts().carryover_sources());
1142   ASSERT_EQ(3, proto_report.sources_size());
1143 }
1144
1145 TEST_P(UkmServiceTest, NonAllowlistedUrls) {
1146   // URL to be manually allowlisted using allowlisted source type.
1147   const GURL kURL("https://google.com/foobar");
1148   struct {
1149     GURL url;
1150     bool expect_in_report;
1151   } test_cases[] = {
1152       {GURL("https://google.com/foobar"), true},
1153       // For origin-only URLs, only the origin needs to be matched.
1154       {GURL("https://google.com"), true},
1155       {GURL("https://google.com/foobar2"), false},
1156       {GURL("https://other.com"), false},
1157   };
1158
1159   for (const auto& test : test_cases) {
1160     ClearPrefs();
1161     UkmService service(&prefs_, &client_,
1162                        std::make_unique<MockDemographicMetricsProvider>());
1163     TestRecordingHelper recorder(&service);
1164
1165     ASSERT_EQ(GetPersistedLogCount(), 0);
1166     service.Initialize();
1167     task_runner_->RunUntilIdle();
1168     service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1169     service.EnableReporting();
1170
1171     // Record with allowlisted ID to allowlist the URL.
1172     SourceId allowlist_id = GetAllowlistedSourceId(1);
1173     recorder.UpdateSourceURL(allowlist_id, kURL);
1174
1175     // Record non allowlisted ID with an entry.
1176     SourceId nonallowlist_id = GetNonAllowlistedSourceId(100);
1177     recorder.UpdateSourceURL(nonallowlist_id, test.url);
1178     TestEvent1(nonallowlist_id).Record(&service);
1179
1180     service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1181     ASSERT_EQ(1, GetPersistedLogCount());
1182     auto proto_report = GetPersistedReport();
1183
1184     EXPECT_EQ(2, proto_report.source_counts().observed());
1185     EXPECT_EQ(1, proto_report.source_counts().navigation_sources());
1186
1187     // If the source id is not allowlisted, don't send it unless it has
1188     // associated entries and the URL matches that of the allowlisted source.
1189     if (test.expect_in_report) {
1190       EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
1191       ASSERT_EQ(2, proto_report.sources_size());
1192       EXPECT_EQ(allowlist_id, proto_report.sources(0).id());
1193       EXPECT_EQ(kURL, proto_report.sources(0).urls(0).url());
1194       EXPECT_EQ(nonallowlist_id, proto_report.sources(1).id());
1195       EXPECT_EQ(test.url, proto_report.sources(1).urls(0).url());
1196     } else {
1197       EXPECT_EQ(1, proto_report.source_counts().unmatched_sources());
1198       ASSERT_EQ(1, proto_report.sources_size());
1199       EXPECT_EQ(allowlist_id, proto_report.sources(0).id());
1200       EXPECT_EQ(kURL, proto_report.sources(0).urls(0).url());
1201     }
1202
1203     // Do a log rotation again, with the same test URL associated to a new
1204     // source id. Since the previous source id of the test case is of
1205     // non-allowlisted type, the carryover URLs list is expected to remain
1206     // be unchanged, thus the the report should still contain the same numbers
1207     // of sources as before, that is, non-allowlisted URLs should not have
1208     // allowlisted themselves during the previous log rotation.
1209     SourceId nonallowlist_id2 = GetNonAllowlistedSourceId(101);
1210     recorder.UpdateSourceURL(nonallowlist_id2, test.url);
1211     TestEvent1(nonallowlist_id2).Record(&service);
1212     service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1213     ASSERT_EQ(2, GetPersistedLogCount());
1214     proto_report = GetPersistedReport();
1215
1216     if (test.expect_in_report) {
1217       EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
1218       ASSERT_EQ(2, proto_report.sources_size());
1219       EXPECT_EQ(allowlist_id, proto_report.sources(0).id());
1220       EXPECT_EQ(kURL, proto_report.sources(0).urls(0).url());
1221       EXPECT_EQ(nonallowlist_id2, proto_report.sources(1).id());
1222       EXPECT_EQ(test.url, proto_report.sources(1).urls(0).url());
1223     } else {
1224       EXPECT_EQ(1, proto_report.source_counts().unmatched_sources());
1225       ASSERT_EQ(1, proto_report.sources_size());
1226       EXPECT_EQ(allowlist_id, proto_report.sources(0).id());
1227       EXPECT_EQ(kURL, proto_report.sources(0).urls(0).url());
1228     }
1229   }
1230 }
1231
1232 TEST_P(UkmServiceTest, AllowlistIdType) {
1233   std::map<SourceIdType, bool> source_id_type_allowlisted = {
1234       {SourceIdType::DEFAULT, false},  {SourceIdType::NAVIGATION_ID, true},
1235       {SourceIdType::APP_ID, true},    {SourceIdType::HISTORY_ID, true},
1236       {SourceIdType::WEBAPK_ID, true},
1237   };
1238
1239   for (std::pair<SourceIdType, bool> type : source_id_type_allowlisted) {
1240     ClearPrefs();
1241     UkmService service(&prefs_, &client_,
1242                        std::make_unique<MockDemographicMetricsProvider>());
1243     TestRecordingHelper recorder(&service);
1244     EXPECT_EQ(0, GetPersistedLogCount());
1245     service.Initialize();
1246     task_runner_->RunUntilIdle();
1247     service.UpdateRecording(
1248         UkmConsentState(UkmConsentType::MSBB, UkmConsentType::APPS));
1249     service.EnableReporting();
1250
1251     SourceId id = ConvertSourceIdToAllowlistedType(
1252         1, static_cast<SourceIdType>(type.first));
1253     ASSERT_EQ(GetSourceIdType(id), type.first);
1254
1255     recorder.UpdateSourceURL(id, GURL("https://google.com/foobar1"));
1256
1257     TestEvent1(id).Record(&service);
1258
1259     service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1260     EXPECT_EQ(1, GetPersistedLogCount());
1261     Report proto_report = GetPersistedReport();
1262
1263     if (type.second) {
1264       // Verify we've added one source.
1265       EXPECT_EQ(1, proto_report.sources_size());
1266       EXPECT_EQ(GURL("https://google.com/foobar1").spec(),
1267                 proto_report.sources(0).urls(0).url());
1268     } else {
1269       // No source added when id is not allowlisted type.
1270       EXPECT_EQ(0, proto_report.sources_size());
1271     }
1272
1273     // We've added the entry whether source is added or not.
1274     ASSERT_EQ(1, proto_report.entries_size());
1275     const Entry& proto_entry_a = proto_report.entries(0);
1276     EXPECT_EQ(id, proto_entry_a.source_id());
1277     EXPECT_EQ(base::HashMetricName(TestEvent1::kEntryName),
1278               proto_entry_a.event_hash());
1279   }
1280 }
1281
1282 TEST_P(UkmServiceTest, SupportedSchemes) {
1283   struct {
1284     const char* url;
1285     bool expected_kept;
1286   } test_cases[] = {
1287       {"http://google.ca/", true},
1288       {"https://google.ca/", true},
1289       {"about:blank", true},
1290       {"chrome://version/", true},
1291       {"app://play/abcdefghijklmnopqrstuvwxyzabcdef/", true},
1292       // chrome-extension are controlled by TestIsWebstoreExtension, above.
1293       {"chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj/", true},
1294       {"chrome-extension://abcdefghijklmnopqrstuvwxyzabcdef/", false},
1295       {"ftp://google.ca/", false},
1296       {"file:///tmp/", false},
1297       {"abc://google.ca/", false},
1298       {"www.google.ca/", false},
1299   };
1300
1301   ScopedUkmFeatureParams params({});
1302   UkmService service(&prefs_, &client_,
1303                      std::make_unique<MockDemographicMetricsProvider>());
1304   TestRecordingHelper recorder(&service);
1305   service.SetIsWebstoreExtensionCallback(
1306       base::BindRepeating(&TestIsWebstoreExtension));
1307
1308   EXPECT_EQ(GetPersistedLogCount(), 0);
1309   service.Initialize();
1310   task_runner_->RunUntilIdle();
1311   service.UpdateRecording(
1312       UkmConsentState(UkmConsentType::MSBB, UkmConsentType::EXTENSIONS));
1313   service.EnableReporting();
1314
1315   int64_t id_counter = 1;
1316   int expected_kept_count = 0;
1317   for (const auto& test : test_cases) {
1318     auto source_id = GetAllowlistedSourceId(id_counter++);
1319     recorder.UpdateSourceURL(source_id, GURL(test.url));
1320     TestEvent1(source_id).Record(&service);
1321     if (test.expected_kept)
1322       ++expected_kept_count;
1323   }
1324
1325   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1326   EXPECT_EQ(GetPersistedLogCount(), 1);
1327   Report proto_report = GetPersistedReport();
1328
1329   EXPECT_EQ(expected_kept_count, proto_report.sources_size());
1330   for (const auto& test : test_cases) {
1331     bool found = false;
1332     for (int i = 0; i < proto_report.sources_size(); ++i) {
1333       if (proto_report.sources(i).urls(0).url() == test.url) {
1334         found = true;
1335         break;
1336       }
1337     }
1338     EXPECT_EQ(test.expected_kept, found) << test.url;
1339   }
1340 }
1341
1342 TEST_P(UkmServiceTest, SupportedSchemesNoExtensions) {
1343   struct {
1344     const char* url;
1345     bool expected_kept;
1346   } test_cases[] = {
1347       {"http://google.ca/", true},
1348       {"https://google.ca/", true},
1349       {"about:blank", true},
1350       {"chrome://version/", true},
1351       {"app://play/abcdefghijklmnopqrstuvwxyzabcdef/", true},
1352       {"chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj/", false},
1353       {"chrome-extension://abcdefghijklmnopqrstuvwxyzabcdef/", false},
1354       {"ftp://google.ca/", false},
1355       {"file:///tmp/", false},
1356       {"abc://google.ca/", false},
1357       {"www.google.ca/", false},
1358   };
1359
1360   ScopedUkmFeatureParams params({});
1361   UkmService service(&prefs_, &client_,
1362                      std::make_unique<MockDemographicMetricsProvider>());
1363   TestRecordingHelper recorder(&service);
1364
1365   EXPECT_EQ(GetPersistedLogCount(), 0);
1366   service.Initialize();
1367   task_runner_->RunUntilIdle();
1368   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1369   service.EnableReporting();
1370
1371   int64_t id_counter = 1;
1372   int expected_kept_count = 0;
1373   for (const auto& test : test_cases) {
1374     auto source_id = GetAllowlistedSourceId(id_counter++);
1375     recorder.UpdateSourceURL(source_id, GURL(test.url));
1376     TestEvent1(source_id).Record(&service);
1377     if (test.expected_kept)
1378       ++expected_kept_count;
1379   }
1380
1381   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1382   EXPECT_EQ(GetPersistedLogCount(), 1);
1383   Report proto_report = GetPersistedReport();
1384
1385   EXPECT_EQ(expected_kept_count, proto_report.sources_size());
1386   for (const auto& test : test_cases) {
1387     bool found = false;
1388     for (int i = 0; i < proto_report.sources_size(); ++i) {
1389       if (proto_report.sources(i).urls(0).url() == test.url) {
1390         found = true;
1391         break;
1392       }
1393     }
1394     EXPECT_EQ(test.expected_kept, found) << test.url;
1395   }
1396 }
1397
1398 TEST_P(UkmServiceTest, SanitizeUrlAuthParams) {
1399   UkmService service(&prefs_, &client_,
1400                      std::make_unique<MockDemographicMetricsProvider>());
1401   TestRecordingHelper recorder(&service);
1402   EXPECT_EQ(0, GetPersistedLogCount());
1403   service.Initialize();
1404   task_runner_->RunUntilIdle();
1405   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1406   service.EnableReporting();
1407
1408   auto id = GetAllowlistedSourceId(0);
1409   recorder.UpdateSourceURL(id, GURL("https://username:password@example.com/"));
1410
1411   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1412   EXPECT_EQ(1, GetPersistedLogCount());
1413
1414   auto proto_report = GetPersistedReport();
1415   ASSERT_EQ(1, proto_report.sources_size());
1416   const Source& proto_source = proto_report.sources(0);
1417   EXPECT_EQ("https://example.com/", proto_source.urls(0).url());
1418 }
1419
1420 TEST_P(UkmServiceTest, SanitizeChromeUrlParams) {
1421   struct {
1422     const char* url;
1423     const char* expected_url;
1424   } test_cases[] = {
1425       {"chrome://version/?foo=bar", "chrome://version/"},
1426       {"about:blank?foo=bar", "about:blank"},
1427       {"chrome://histograms/Variations", "chrome://histograms/Variations"},
1428       {"http://google.ca/?foo=bar", "http://google.ca/?foo=bar"},
1429       {"https://google.ca/?foo=bar", "https://google.ca/?foo=bar"},
1430       {"chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj/foo.html?a=b",
1431        "chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj/"},
1432   };
1433
1434   for (const auto& test : test_cases) {
1435     ClearPrefs();
1436
1437     UkmService service(&prefs_, &client_,
1438                        std::make_unique<MockDemographicMetricsProvider>());
1439     TestRecordingHelper recorder(&service);
1440     service.SetIsWebstoreExtensionCallback(
1441         base::BindRepeating(&TestIsWebstoreExtension));
1442
1443     EXPECT_EQ(0, GetPersistedLogCount());
1444     service.Initialize();
1445     task_runner_->RunUntilIdle();
1446     service.UpdateRecording(
1447         UkmConsentState(UkmConsentType::MSBB, UkmConsentType::EXTENSIONS));
1448     service.EnableReporting();
1449
1450     auto id = GetAllowlistedSourceId(0);
1451     recorder.UpdateSourceURL(id, GURL(test.url));
1452
1453     service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1454     EXPECT_EQ(1, GetPersistedLogCount());
1455
1456     auto proto_report = GetPersistedReport();
1457     ASSERT_EQ(1, proto_report.sources_size());
1458     const Source& proto_source = proto_report.sources(0);
1459     EXPECT_EQ(test.expected_url, proto_source.urls(0).url());
1460   }
1461 }
1462
1463 TEST_P(UkmServiceTest, MarkSourceForDeletion) {
1464   UkmService service(&prefs_, &client_,
1465                      std::make_unique<MockDemographicMetricsProvider>());
1466   TestRecordingHelper recorder(&service);
1467   EXPECT_EQ(0, GetPersistedLogCount());
1468   service.Initialize();
1469   task_runner_->RunUntilIdle();
1470   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1471   service.EnableReporting();
1472
1473   // Seed some dummy sources.
1474   SourceId id0 = GetAllowlistedSourceId(0);
1475   recorder.UpdateSourceURL(id0, GURL("https://www.example0.com/"));
1476   SourceId id1 = GetAllowlistedSourceId(1);
1477   recorder.UpdateSourceURL(id1, GURL("https://www.example1.com/"));
1478   SourceId id2 = GetAllowlistedSourceId(2);
1479   recorder.UpdateSourceURL(id2, GURL("https://www.example2.com/"));
1480
1481   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1482   int logs_count = 0;
1483   EXPECT_EQ(++logs_count, GetPersistedLogCount());
1484
1485   // All sources are present in the report.
1486   Report proto_report = GetPersistedReport();
1487   ASSERT_EQ(3, proto_report.sources_size());
1488   EXPECT_EQ(id0, proto_report.sources(0).id());
1489   EXPECT_EQ(id1, proto_report.sources(1).id());
1490   EXPECT_EQ(id2, proto_report.sources(2).id());
1491
1492   // Mark source 1 for deletion. Next report will still contain source 1 because
1493   // we might have associated entries. It will no longer be in further report at
1494   // the following cycle.
1495   service.MarkSourceForDeletion(id1);
1496   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1497   EXPECT_EQ(++logs_count, GetPersistedLogCount());
1498
1499   proto_report = GetPersistedReport();
1500   ASSERT_EQ(3, proto_report.sources_size());
1501
1502   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1503   EXPECT_EQ(++logs_count, GetPersistedLogCount());
1504
1505   proto_report = GetPersistedReport();
1506   ASSERT_EQ(2, proto_report.sources_size());
1507   EXPECT_EQ(id0, proto_report.sources(0).id());
1508   EXPECT_EQ(id2, proto_report.sources(1).id());
1509 }
1510
1511 // Verifies that sources of some types are deleted at the end of reporting
1512 // cycle.
1513 TEST_P(UkmServiceTest, PurgeNonCarriedOverSources) {
1514   UkmService service(&prefs_, &client_,
1515                      std::make_unique<MockDemographicMetricsProvider>());
1516   TestRecordingHelper recorder(&service);
1517   EXPECT_EQ(0, GetPersistedLogCount());
1518   service.Initialize();
1519   task_runner_->RunUntilIdle();
1520   service.UpdateRecording(UkmConsentState(
1521       UkmConsentType::MSBB, UkmConsentType::APPS, UkmConsentType::EXTENSIONS));
1522   service.EnableReporting();
1523   service.SetIsWebstoreExtensionCallback(
1524       base::BindRepeating(&TestIsWebstoreExtension));
1525
1526   // Seed some fake sources.
1527   SourceId ukm_id = ConvertToSourceId(0, SourceIdType::DEFAULT);
1528   recorder.UpdateSourceURL(ukm_id, GURL("https://www.example0.com/"));
1529   SourceId navigation_id =
1530       ConvertSourceIdToAllowlistedType(1, SourceIdType::NAVIGATION_ID);
1531   recorder.UpdateSourceURL(navigation_id, GURL("https://www.example1.com/"));
1532   SourceId app_id = ConvertSourceIdToAllowlistedType(2, SourceIdType::APP_ID);
1533   recorder.UpdateSourceURL(app_id, GURL("https://www.example2.com/"));
1534   SourceId history_id =
1535       ConvertSourceIdToAllowlistedType(3, SourceIdType::HISTORY_ID);
1536   recorder.UpdateSourceURL(history_id, GURL("https://www.example3.com/"));
1537   SourceId webapk_id =
1538       ConvertSourceIdToAllowlistedType(4, SourceIdType::WEBAPK_ID);
1539   recorder.UpdateSourceURL(webapk_id, GURL("https://www.example4.com/"));
1540   SourceId payment_app_id =
1541       ConvertSourceIdToAllowlistedType(5, SourceIdType::PAYMENT_APP_ID);
1542   recorder.UpdateSourceURL(payment_app_id, GURL("https://www.example5.com/"));
1543   SourceId web_identity_id =
1544       ConvertSourceIdToAllowlistedType(6, SourceIdType::WEB_IDENTITY_ID);
1545   recorder.UpdateSourceURL(web_identity_id, GURL("https://www.example6.com/"));
1546   SourceId extension_id =
1547       ConvertSourceIdToAllowlistedType(7, SourceIdType::EXTENSION_ID);
1548   recorder.UpdateSourceURL(
1549       extension_id,
1550       GURL("chrome-extension://bhcnanendmgjjeghamaccjnochlnhcgj"));
1551
1552   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1553   int logs_count = 0;
1554   EXPECT_EQ(++logs_count, GetPersistedLogCount());
1555
1556   // All sources are present except ukm_id of non-allowlisted UKM type.
1557   Report proto_report = GetPersistedReport();
1558   ASSERT_EQ(7, proto_report.sources_size());
1559   EXPECT_EQ(navigation_id, proto_report.sources(0).id());
1560   EXPECT_EQ(app_id, proto_report.sources(1).id());
1561   EXPECT_EQ(history_id, proto_report.sources(2).id());
1562   EXPECT_EQ(webapk_id, proto_report.sources(3).id());
1563   EXPECT_EQ(payment_app_id, proto_report.sources(4).id());
1564   EXPECT_EQ(web_identity_id, proto_report.sources(5).id());
1565   EXPECT_EQ(extension_id, proto_report.sources(6).id());
1566
1567   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1568   EXPECT_EQ(++logs_count, GetPersistedLogCount());
1569
1570   // Sources of HISTORY_ID, WEBAPK_ID, PAYMENT_APP_ID, WEB_IDENTITY_ID,and
1571   // EXTENSION_ID types are not kept between reporting cycles, thus only 2
1572   // sources remain.
1573   proto_report = GetPersistedReport();
1574   ASSERT_EQ(2, proto_report.sources_size());
1575   EXPECT_EQ(navigation_id, proto_report.sources(0).id());
1576   EXPECT_EQ(app_id, proto_report.sources(1).id());
1577 }
1578
1579 TEST_P(UkmServiceTest, IdentifiabilityMetricsDontExplode) {
1580   UkmService service(&prefs_, &client_,
1581                      std::make_unique<MockDemographicMetricsProvider>());
1582   TestRecordingHelper recorder(&service);
1583   ASSERT_EQ(0, GetPersistedLogCount());
1584   service.Initialize();
1585   task_runner_->RunUntilIdle();
1586   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1587   service.EnableReporting();
1588
1589   SourceId id = GetAllowlistedSourceId(0);
1590   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
1591
1592   builders::Identifiability(id).SetStudyGeneration_626(0).Record(&service);
1593   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1594   ASSERT_EQ(1, GetPersistedLogCount());
1595   Report proto_report = GetPersistedReport();
1596   EXPECT_EQ(1, proto_report.entries_size());
1597 }
1598
1599 TEST_P(UkmServiceTest, FilterCanRemoveMetrics) {
1600   class TestEntryFilter : public UkmEntryFilter {
1601    public:
1602     // This implementation removes the last metric in an event and returns it in
1603     // |filtered_metric_hashes|.
1604     bool FilterEntry(
1605         mojom::UkmEntry* entry,
1606         base::flat_set<uint64_t>* filtered_metric_hashes) override {
1607       EXPECT_FALSE(entry->metrics.empty());
1608       auto last_iter = --entry->metrics.end();
1609       filtered_metric_hashes->insert(last_iter->first);
1610       entry->metrics.erase(last_iter);
1611       return !entry->metrics.empty();
1612     }
1613   };
1614
1615   UkmService service(&prefs_, &client_,
1616                      std::make_unique<MockDemographicMetricsProvider>());
1617   service.RegisterEventFilter(std::make_unique<TestEntryFilter>());
1618   TestRecordingHelper recorder(&service);
1619   ASSERT_EQ(0, GetPersistedLogCount());
1620   service.Initialize();
1621   task_runner_->RunUntilIdle();
1622   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1623   service.EnableReporting();
1624
1625   SourceId id = GetAllowlistedSourceId(0);
1626   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
1627
1628   // This event sticks around albeit with a single metric instead of two.
1629   TestEvent1(id).SetCpuTime(1).SetNet_CacheBytes2(0).Record(&service);
1630
1631   // This event is discarded because its only metric gets stripped out.
1632   TestEvent1(id).SetNet_CacheBytes2(0).Record(&service);
1633
1634   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1635   ASSERT_EQ(1, GetPersistedLogCount());
1636   Report proto_report = GetPersistedReport();
1637   ASSERT_EQ(1, proto_report.entries_size());
1638   EXPECT_EQ(1, proto_report.entries(0).metrics_size());
1639   ASSERT_EQ(1, proto_report.aggregates().size());
1640   EXPECT_EQ(1u, proto_report.aggregates(0).dropped_due_to_filter());
1641   EXPECT_EQ(2, proto_report.aggregates(0).metrics_size());
1642   EXPECT_EQ(0u, proto_report.aggregates(0).metrics(0).dropped_due_to_filter());
1643   EXPECT_EQ(2u, proto_report.aggregates(0).metrics(1).dropped_due_to_filter());
1644 }
1645
1646 TEST_P(UkmServiceTest, FilterRejectsEvent) {
1647   static const auto kTestEvent1EntryNameHash =
1648       base::HashMetricName(TestEvent1::kEntryName);
1649
1650   class TestEntryFilter : public UkmEntryFilter {
1651    public:
1652     // This filter rejects all events that are not TestEvent1.
1653     bool FilterEntry(
1654         mojom::UkmEntry* entry,
1655         base::flat_set<uint64_t>* filtered_metric_hashes) override {
1656       if (entry->event_hash == kTestEvent1EntryNameHash)
1657         return true;
1658
1659       std::vector<uint64_t> filtered_metrics;
1660       filtered_metrics.resize(entry->metrics.size());
1661       base::ranges::transform(entry->metrics, filtered_metrics.begin(),
1662                               &decltype(entry->metrics)::value_type::first);
1663       filtered_metric_hashes->replace(std::move(filtered_metrics));
1664       // Note that the event still contains metrics.
1665       return false;
1666     }
1667   };
1668
1669   UkmService service(&prefs_, &client_,
1670                      std::make_unique<MockDemographicMetricsProvider>());
1671   service.RegisterEventFilter(std::make_unique<TestEntryFilter>());
1672   TestRecordingHelper recorder(&service);
1673   ASSERT_EQ(0, GetPersistedLogCount());
1674   service.Initialize();
1675   task_runner_->RunUntilIdle();
1676   service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1677   service.EnableReporting();
1678
1679   SourceId id = GetAllowlistedSourceId(0);
1680   recorder.UpdateSourceURL(id, GURL("https://google.com/foobar"));
1681
1682   TestEvent1(id).SetCpuTime(0).Record(&service);
1683   TestEvent2(id).SetDownloadService(3).Record(&service);
1684
1685   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1686   ASSERT_EQ(1, GetPersistedLogCount());
1687   Report proto_report = GetPersistedReport();
1688   EXPECT_EQ(1, proto_report.entries_size());
1689   EXPECT_EQ(kTestEvent1EntryNameHash, proto_report.entries(0).event_hash());
1690   ASSERT_EQ(2, proto_report.aggregates_size());
1691   EXPECT_EQ(1u, proto_report.aggregates(0).dropped_due_to_filter());
1692   ASSERT_EQ(1, proto_report.aggregates(0).metrics_size());
1693
1694   // No dropped_due_to_filter due to the value being equal to the entry's
1695   // droppeddropped_due_to_filter.
1696   EXPECT_FALSE(
1697       proto_report.aggregates(0).metrics(0).has_dropped_due_to_filter());
1698 }
1699
1700 TEST_P(UkmServiceTest, PruneUnseenFirst) {
1701   // We will be testing with the prune unseen feature both off and on.
1702   for (bool prune_unseen_sources_first : {true, false}) {
1703     const GURL kURL("https://google.com/foobar");
1704
1705     // Set the 'MaxKeptSources' value to 3 so it is easier to test.
1706     ScopedUkmFeatureParams params(
1707         {{"MaxKeptSources", "3"},
1708          {"PruneUnseenSourcesFirst",
1709           prune_unseen_sources_first ? "true" : "false"}});
1710
1711     ClearPrefs();
1712     UkmService service(&prefs_, &client_,
1713                        std::make_unique<MockDemographicMetricsProvider>());
1714     TestRecordingHelper recorder(&service);
1715     EXPECT_EQ(0, GetPersistedLogCount());
1716     service.Initialize();
1717     task_runner_->RunUntilIdle();
1718     service.UpdateRecording(UkmConsentState(UkmConsentType::MSBB));
1719     service.EnableReporting();
1720
1721     // Create 5 allowlisted ids. Allowlisted ids (like APP_ID) will not be
1722     // automatically removed when they emit events. They're only removed via the
1723     // pruning mechanism. Note that the are added in order, so 4 is the
1724     // youngest/newest.
1725     std::vector<SourceId> ids;
1726     base::TimeTicks last_time = base::TimeTicks::Now();
1727     for (int i = 0; i < 5; ++i) {
1728       // Wait until base::TimeTicks::Now() no longer equals |last_time|. This
1729       // ensures each source has a unique timestamp to avoid flakes. Should take
1730       // between 1-15ms per documented resolution of base::TimeTicks.
1731       while (base::TimeTicks::Now() == last_time) {
1732         base::PlatformThread::Sleep(base::Milliseconds(1));
1733       }
1734       ids.push_back(GetAllowlistedSourceId(i));
1735       recorder.UpdateSourceURL(ids.back(), kURL);
1736       last_time = base::TimeTicks::Now();
1737     }
1738
1739     // Events on 0 and 4. This will be important to this test, as we are testing
1740     // how pruning will vary based on this. So keep this in mind.
1741     TestEvent1(ids[0]).Record(&service);
1742     TestEvent1(ids[4]).Record(&service);
1743
1744     service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1745     EXPECT_EQ(1, GetPersistedLogCount());
1746     auto proto_report = GetPersistedReport();
1747
1748     EXPECT_EQ(5, proto_report.source_counts().observed());
1749     // All are navigation sources.
1750     EXPECT_EQ(5, proto_report.source_counts().navigation_sources());
1751     EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
1752
1753     // In all cases, 3 will be deferred since that is our max allowed.
1754     EXPECT_EQ(3, proto_report.source_counts().deferred_sources());
1755     // This is from last time, so none there.
1756     EXPECT_EQ(0, proto_report.source_counts().carryover_sources());
1757
1758     // All 5 sources will be included in this first report.
1759     ASSERT_EQ(5, proto_report.sources_size());
1760     EXPECT_EQ(ids[0], proto_report.sources(0).id());
1761     EXPECT_EQ(ids[1], proto_report.sources(1).id());
1762     EXPECT_EQ(ids[2], proto_report.sources(2).id());
1763     EXPECT_EQ(ids[3], proto_report.sources(3).id());
1764     EXPECT_EQ(ids[4], proto_report.sources(4).id());
1765
1766     // Depending on the PruneUnseenSourcesFirst setting, different ones will be
1767     // removed.
1768     // We have MaxKeptSources=3.
1769     // If PruneUnseenSourcesFirst was set, then the ones kept should be the two
1770     // that were used, which are 0 and 4. The one remaining one will be picked
1771     // via age which will be 3, so 0, 3, 4 are kept.
1772     // Otherwise, it will be entirely based on age, which is 2,3,4.
1773
1774     // New events on 0,2,4. This actually doesn't matter with respect to what
1775     // sources are emitted here, as some sources are already pruned.
1776     TestEvent1(ids[0]).Record(&service);
1777     TestEvent1(ids[2]).Record(&service);
1778     TestEvent1(ids[4]).Record(&service);
1779
1780     service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1781     EXPECT_EQ(2, GetPersistedLogCount());
1782     proto_report = GetPersistedReport();
1783
1784     // No new sources observed.
1785     EXPECT_EQ(0, proto_report.source_counts().observed());
1786     // 0 again, as this is for newly observed ones.
1787     EXPECT_EQ(0, proto_report.source_counts().navigation_sources());
1788     EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
1789
1790     // Since no new sources added, we still are keeping the same 3. So all 3 are
1791     // kept and retained, in both cases.
1792     EXPECT_EQ(3, proto_report.source_counts().deferred_sources());
1793     EXPECT_EQ(3, proto_report.source_counts().carryover_sources());
1794     ASSERT_EQ(3, proto_report.sources_size());
1795
1796     if (prune_unseen_sources_first) {
1797       // 0, 3, 4 as 0 and 4 were used last time, and 3 is the newest of the
1798       // remaining.
1799       EXPECT_EQ(ids[0], proto_report.sources(0).id());
1800       EXPECT_EQ(ids[3], proto_report.sources(1).id());
1801       EXPECT_EQ(ids[4], proto_report.sources(2).id());
1802     } else {
1803       // 2, 3, 4 as these are the 3 newest, which is the only criteria we are
1804       // using for this test.
1805       EXPECT_EQ(ids[2], proto_report.sources(0).id());
1806       EXPECT_EQ(ids[3], proto_report.sources(1).id());
1807       EXPECT_EQ(ids[4], proto_report.sources(2).id());
1808     }
1809   }
1810 }
1811
1812 TEST_P(UkmServiceTest, PruneAppIDLast) {
1813   // We will be testing with the PruneAppIdLast feature both off and on.
1814   for (bool prune_app_id_last : {true, false}) {
1815     const GURL kURL("https://google.com/foobar");
1816
1817     // Set the 'MaxKeptSources' value to 3 so it is easier to test.
1818     ScopedUkmFeatureParams params(
1819         {{"MaxKeptSources", "3"},
1820          {"PruneAppIdLast", prune_app_id_last ? "true" : "false"}});
1821
1822     ClearPrefs();
1823     UkmService service(&prefs_, &client_,
1824                        std::make_unique<MockDemographicMetricsProvider>());
1825     TestRecordingHelper recorder(&service);
1826     EXPECT_EQ(0, GetPersistedLogCount());
1827     service.Initialize();
1828     task_runner_->RunUntilIdle();
1829     service.UpdateRecording(
1830         UkmConsentState(UkmConsentType::MSBB, UkmConsentType::APPS));
1831     service.EnableReporting();
1832
1833     // Create 5 sources. We set source 0 and 4 to be APP_ID Sources, where
1834     // 1,2,3 are allowlisted/navigation sources.
1835     std::vector<SourceId> ids;
1836     base::TimeTicks last_time = base::TimeTicks::Now();
1837     for (int i = 0; i < 5; ++i) {
1838       // Wait until base::TimeTicks::Now() no longer equals |last_time|. This
1839       // ensures each source has a unique timestamp to avoid flakes. Should take
1840       // between 1-15ms per documented resolution of base::TimeTicks.
1841       while (base::TimeTicks::Now() == last_time) {
1842         base::PlatformThread::Sleep(base::Milliseconds(1));
1843       }
1844       // Note, this is where we are setting the source types. Important for the
1845       // testing.
1846       if (i == 0 || i == 4) {
1847         ids.push_back(GetAppIDSourceId(i));
1848       } else {
1849         ids.push_back(GetAllowlistedSourceId(i));
1850       }
1851       recorder.UpdateSourceURL(ids.back(), kURL);
1852       last_time = base::TimeTicks::Now();
1853     }
1854
1855     service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1856     EXPECT_EQ(1, GetPersistedLogCount());
1857     auto proto_report = GetPersistedReport();
1858
1859     EXPECT_EQ(5, proto_report.source_counts().observed());
1860
1861     // In all cases, 3 will be deferred since that is our max allowed.
1862     EXPECT_EQ(3, proto_report.source_counts().deferred_sources());
1863     // This is from last time, so none there.
1864     EXPECT_EQ(0, proto_report.source_counts().carryover_sources());
1865
1866     // All 5 sources will be included in this first report.
1867     ASSERT_EQ(5, proto_report.sources_size());
1868     EXPECT_EQ(ids[0], proto_report.sources(0).id());
1869     EXPECT_EQ(ids[1], proto_report.sources(1).id());
1870     EXPECT_EQ(ids[2], proto_report.sources(2).id());
1871     EXPECT_EQ(ids[3], proto_report.sources(3).id());
1872     EXPECT_EQ(ids[4], proto_report.sources(4).id());
1873
1874     // We have MaxKeptSources=3.
1875     // If PruneAppIdLast was set, then the ones kept should be the two that were
1876     // set as APP_ID, which are 0 and 4. The one remaining one will be picked
1877     // via age which will be 3, so 0, 3, 4 are kept.
1878     // Otherwise, it will be entirely based on age, which is 2,3,4.
1879
1880     service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1881     EXPECT_EQ(2, GetPersistedLogCount());
1882     proto_report = GetPersistedReport();
1883
1884     // No new sources observed.
1885     EXPECT_EQ(0, proto_report.source_counts().observed());
1886     // 0 again, as this is for newly observed ones.
1887     EXPECT_EQ(0, proto_report.source_counts().unmatched_sources());
1888
1889     // Since no new sources added, we still are keeping the same 3. So all 3 are
1890     // kept and retained, in both cases.
1891     EXPECT_EQ(3, proto_report.source_counts().deferred_sources());
1892     EXPECT_EQ(3, proto_report.source_counts().carryover_sources());
1893     ASSERT_EQ(3, proto_report.sources_size());
1894
1895     if (prune_app_id_last) {
1896       // 0, 3, 4 as 0 and 4 are APP_ID, and 3 is the newest of the remaining.
1897       EXPECT_EQ(ids[0], proto_report.sources(0).id());
1898       EXPECT_EQ(ids[3], proto_report.sources(1).id());
1899       EXPECT_EQ(ids[4], proto_report.sources(2).id());
1900     } else {
1901       // 2, 3, 4 as these are the 3 newest, which is the only criteria we are
1902       // using for this test.
1903       EXPECT_EQ(ids[2], proto_report.sources(0).id());
1904       EXPECT_EQ(ids[3], proto_report.sources(1).id());
1905       EXPECT_EQ(ids[4], proto_report.sources(2).id());
1906     }
1907   }
1908 }
1909
1910 TEST_P(UkmServiceTest, UseExternalClientID) {
1911   prefs_.SetUint64(prefs::kUkmClientId, 1234);
1912   uint64_t external_client_id = 5678;
1913   UkmService service(&prefs_, &client_,
1914                      std::make_unique<MockDemographicMetricsProvider>(),
1915                      external_client_id);
1916   service.Initialize();
1917   EXPECT_EQ(external_client_id, service.client_id());
1918   EXPECT_EQ(external_client_id, prefs_.GetUint64(prefs::kUkmClientId));
1919 }
1920
1921 // Verifies that when a cloned install is detected, logs are purged.
1922 TEST_P(UkmServiceTest, PurgeLogsOnClonedInstallDetected) {
1923   TestMetricsServiceClientWithClonedInstallDetector client;
1924   UkmService service(&prefs_, &client,
1925                      std::make_unique<MockDemographicMetricsProvider>());
1926   service.Initialize();
1927
1928   // Store various logs.
1929   metrics::UnsentLogStore* test_log_store =
1930       service.reporting_service_for_testing().ukm_log_store();
1931   test_log_store->StoreLog(
1932       "dummy log data", metrics::LogMetadata(),
1933       metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1934   test_log_store->StageNextLog();
1935   test_log_store->StoreLog(
1936       "more dummy log data", metrics::LogMetadata(),
1937       metrics::MetricsLogsEventManager::CreateReason::kUnknown);
1938   EXPECT_TRUE(test_log_store->has_staged_log());
1939   EXPECT_TRUE(test_log_store->has_unsent_logs());
1940
1941   metrics::ClonedInstallDetector* cloned_install_detector =
1942       client.cloned_install_detector();
1943   cloned_install_detector->RegisterPrefs(prefs_.registry());
1944
1945   static constexpr char kTestRawId[] = "test";
1946   // Hashed machine id for |kTestRawId|.
1947   static constexpr int kTestHashedId = 2216819;
1948
1949   // Save a machine id that will not cause a clone to be detected.
1950   prefs_.SetInteger(metrics::prefs::kMetricsMachineId, kTestHashedId);
1951   cloned_install_detector->SaveMachineIdForTesting(&prefs_, kTestRawId);
1952   // Verify that the logs are still present.
1953   EXPECT_TRUE(test_log_store->has_staged_log());
1954   EXPECT_TRUE(test_log_store->has_unsent_logs());
1955
1956   // Save a machine id that will cause a clone to be detected.
1957   prefs_.SetInteger(metrics::prefs::kMetricsMachineId, kTestHashedId + 1);
1958   cloned_install_detector->SaveMachineIdForTesting(&prefs_, kTestRawId);
1959   // Verify that the logs were purged if the |kMetricsClearLogsOnClonedInstall|
1960   // feature is enabled.
1961   if (ShouldClearLogsOnClonedInstall()) {
1962     EXPECT_FALSE(test_log_store->has_staged_log());
1963     EXPECT_FALSE(test_log_store->has_unsent_logs());
1964   } else {
1965     EXPECT_TRUE(test_log_store->has_staged_log());
1966     EXPECT_TRUE(test_log_store->has_unsent_logs());
1967   }
1968 }
1969
1970 #if BUILDFLAG(IS_CHROMEOS_ASH)
1971 namespace {
1972
1973 class UkmServiceTestWithIndependentAppKM
1974     : public testing::TestWithParam<UkmConsentType> {
1975  public:
1976   UkmServiceTestWithIndependentAppKM()
1977       : task_runner_(new base::TestSimpleTaskRunner),
1978         task_runner_current_default_handle_(task_runner_) {
1979     UkmService::RegisterPrefs(prefs_.registry());
1980
1981     prefs_.ClearPref(prefs::kUkmClientId);
1982     prefs_.ClearPref(prefs::kUkmSessionId);
1983     prefs_.ClearPref(prefs::kUkmUnsentLogStore);
1984
1985     scoped_feature_list_.InitAndEnableFeature({kAppMetricsOnlyRelyOnAppSync});
1986   }
1987
1988   int GetPersistedLogCount() { return ukm::GetPersistedLogCount(prefs_); }
1989
1990   Report GetPersistedReport() { return ukm::GetPersistedReport(prefs_); }
1991
1992  protected:
1993   TestingPrefServiceSimple prefs_;
1994   metrics::TestMetricsServiceClient client_;
1995   scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
1996   base::SingleThreadTaskRunner::CurrentDefaultHandle
1997       task_runner_current_default_handle_;
1998   base::test::ScopedFeatureList scoped_feature_list_;
1999 };
2000
2001 }  // namespace
2002
2003 TEST_P(UkmServiceTestWithIndependentAppKM, RejectWhenNotConsented) {
2004   const GURL kURL("https://google.com/foobar");
2005   const GURL kAppURL("app://google.com/foobar");
2006
2007   // Setup test constants from param.
2008   const auto consent = GetParam();
2009   const std::vector<int> app_indices = {1, 4};
2010   const std::vector<int> url_indices = {0, 2, 3};
2011   const std::vector<int>& expected_source_indices =
2012       (consent == UkmConsentType::APPS ? app_indices : url_indices);
2013
2014   const int expected_result = expected_source_indices.size();
2015
2016   UkmService service(&prefs_, &client_,
2017                      std::make_unique<MockDemographicMetricsProvider>());
2018   TestRecordingHelper recorder(&service);
2019   EXPECT_EQ(0, GetPersistedLogCount());
2020   service.Initialize();
2021   task_runner_->RunUntilIdle();
2022   service.UpdateRecording(UkmConsentState(consent));
2023   service.EnableReporting();
2024
2025   std::vector<SourceId> source_ids;
2026   for (int i = 0; i < 5; ++i) {
2027     if (std::find(app_indices.begin(), app_indices.end(), i) !=
2028         app_indices.end()) {
2029       source_ids.push_back(UkmServiceTest::GetAppIDSourceId(i));
2030       recorder.UpdateSourceURL(source_ids.back(), kAppURL);
2031     } else {
2032       source_ids.push_back(UkmServiceTest::GetAllowlistedSourceId(i));
2033       recorder.UpdateSourceURL(source_ids.back(), kURL);
2034     }
2035
2036     TestEvent1(source_ids.back()).Record(&service);
2037   }
2038
2039   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
2040   EXPECT_EQ(1, GetPersistedLogCount());
2041
2042   // Has the sources and entries associated with AppIDs.
2043   Report report = GetPersistedReport();
2044   EXPECT_EQ(expected_result, report.sources_size());
2045   EXPECT_EQ(expected_result, report.entries_size());
2046
2047   for (int i = 0; i < expected_result; ++i) {
2048     EXPECT_EQ(source_ids[expected_source_indices[i]], report.sources(i).id());
2049     EXPECT_EQ(source_ids[expected_source_indices[i]],
2050               report.entries(i).source_id());
2051   }
2052 }
2053
2054 INSTANTIATE_TEST_SUITE_P(
2055     UkmServiceTestWithIndependentAppKMGroup,
2056     UkmServiceTestWithIndependentAppKM,
2057     testing::Values(UkmConsentType::APPS, UkmConsentType::MSBB),
2058     [](const testing::TestParamInfo<
2059         UkmServiceTestWithIndependentAppKM::ParamType>& info) {
2060       if (info.param == UkmConsentType::APPS)
2061         return "TestApps";
2062       else
2063         return "TestMSBB";
2064     });
2065
2066 namespace {
2067
2068 class UkmServiceTestWithIndependentAppKMFullConsent
2069     : public testing::TestWithParam<bool> {
2070  public:
2071   UkmServiceTestWithIndependentAppKMFullConsent()
2072       : task_runner_(new base::TestSimpleTaskRunner),
2073         task_runner_current_default_handle_(task_runner_) {
2074     UkmService::RegisterPrefs(prefs_.registry());
2075
2076     prefs_.ClearPref(prefs::kUkmClientId);
2077     prefs_.ClearPref(prefs::kUkmSessionId);
2078     prefs_.ClearPref(prefs::kUkmUnsentLogStore);
2079
2080     scoped_feature_list_.InitAndEnableFeature({kAppMetricsOnlyRelyOnAppSync});
2081   }
2082
2083   int GetPersistedLogCount() { return ukm::GetPersistedLogCount(prefs_); }
2084
2085   Report GetPersistedReport() { return ukm::GetPersistedReport(prefs_); }
2086
2087  protected:
2088   TestingPrefServiceSimple prefs_;
2089   metrics::TestMetricsServiceClient client_;
2090   scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
2091   base::SingleThreadTaskRunner::CurrentDefaultHandle
2092       task_runner_current_default_handle_;
2093   base::test::ScopedFeatureList scoped_feature_list_;
2094 };
2095
2096 }  // namespace
2097
2098 TEST_P(UkmServiceTestWithIndependentAppKMFullConsent, VerifyAllAndNoneConsent) {
2099   const GURL kURL("https://google.com/foobar");
2100   const GURL kAppURL("app://google.com/foobar");
2101   const int kExpectedResultWithConsent = 5;
2102
2103   // Setup test constants from param.
2104   const auto has_consent = GetParam();
2105   const auto consent_state =
2106       (has_consent ? UkmConsentState::All() : UkmConsentState());
2107
2108   UkmService service(&prefs_, &client_,
2109                      std::make_unique<MockDemographicMetricsProvider>());
2110   TestRecordingHelper recorder(&service);
2111   EXPECT_EQ(0, GetPersistedLogCount());
2112   service.Initialize();
2113   task_runner_->RunUntilIdle();
2114   service.UpdateRecording(consent_state);
2115   service.EnableReporting();
2116
2117   const std::vector<int> app_indices = {1, 4};
2118   const std::vector<int> url_indices = {0, 2, 3};
2119
2120   std::vector<SourceId> source_ids;
2121   for (int i = 0; i < 5; ++i) {
2122     if (std::find(app_indices.begin(), app_indices.end(), i) !=
2123         app_indices.end()) {
2124       source_ids.push_back(UkmServiceTest::GetAppIDSourceId(i));
2125       recorder.UpdateSourceURL(source_ids.back(), kAppURL);
2126     } else {
2127       source_ids.push_back(UkmServiceTest::GetAllowlistedSourceId(i));
2128       recorder.UpdateSourceURL(source_ids.back(), kURL);
2129     }
2130
2131     TestEvent1(source_ids.back()).Record(&service);
2132   }
2133
2134   service.Flush(metrics::MetricsLogsEventManager::CreateReason::kUnknown);
2135
2136   EXPECT_EQ(GetPersistedLogCount(), static_cast<int>(has_consent));
2137
2138   if (has_consent) {
2139     const auto report = GetPersistedReport();
2140
2141     EXPECT_EQ(report.sources_size(), kExpectedResultWithConsent);
2142     EXPECT_EQ(report.entries_size(), kExpectedResultWithConsent);
2143
2144     for (int i = 0; i < kExpectedResultWithConsent; ++i) {
2145       EXPECT_EQ(source_ids[i], report.sources(i).id());
2146       EXPECT_EQ(source_ids[i], report.entries(i).source_id());
2147     }
2148   }
2149 }
2150
2151 INSTANTIATE_TEST_SUITE_P(
2152     UkmServiceTestWithIndependentAppKMFullConsentGroup,
2153     UkmServiceTestWithIndependentAppKMFullConsent,
2154     testing::Values(true, false),
2155     [](const testing::TestParamInfo<
2156         UkmServiceTestWithIndependentAppKMFullConsent::ParamType>& info) {
2157       if (info.param)
2158         return "TestAllConsent";
2159       else
2160         return "TestNoConsent";
2161     });
2162
2163 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
2164
2165 class MockUkmRecorder : public ukm::UkmRecorder {
2166  public:
2167   MockUkmRecorder() = default;
2168   ~MockUkmRecorder() override = default;
2169
2170   MOCK_METHOD(void, AddEntry, (mojom::UkmEntryPtr entry), (override));
2171   MOCK_METHOD(void,
2172               UpdateSourceURL,
2173               (SourceId source_id, const GURL& url),
2174               (override));
2175
2176   MOCK_METHOD(void,
2177               UpdateAppURL,
2178               (SourceId source_id, const GURL& url, const AppType app_type),
2179               (override));
2180
2181   MOCK_METHOD(void,
2182               RecordNavigation,
2183               (SourceId source_id,
2184                const UkmSource::NavigationData& navigation_data),
2185               (override));
2186
2187   MOCK_METHOD(void,
2188               MarkSourceForDeletion,
2189               (ukm::SourceId source_id),
2190               (override));
2191 };
2192
2193 TEST_F(UkmReduceAddEntryIpcTest, RecordingEnabled) {
2194   ASSERT_TRUE(base::FeatureList::IsEnabled(ukm::kUkmReduceAddEntryIPC));
2195
2196   base::RunLoop run_loop;
2197   UkmService service(&prefs_, &client_,
2198                      std::make_unique<MockDemographicMetricsProvider>());
2199   // Initialize UkmService.
2200   service.Initialize();
2201
2202   ukm::UkmEntryBuilder builder(ukm::NoURLSourceId(),
2203                                "Event.ScrollUpdate.Touch");
2204   builder.SetMetric("TimeToScrollUpdateSwapBegin", 17);
2205
2206   // Custom UkmRecorder to intercept messages to and from UkmService to clients.
2207   MockUkmRecorder mock_recorder;
2208   mojo::Remote<ukm::mojom::UkmRecorderFactory> factory;
2209   metrics::UkmRecorderFactoryImpl::Create(&mock_recorder,
2210                                           factory.BindNewPipeAndPassReceiver());
2211
2212   // MojoUkmRecorder (client).
2213   auto mojo_recorder = MojoUkmRecorder::Create(*factory);
2214
2215   service.EnableRecording();
2216   run_loop.RunUntilIdle();
2217
2218   // Since UkmObservers list is empty, the final decision regarding sending the
2219   // AddEntry IPC from clients to UkmService depends on recording being
2220   // enabled/disabled.
2221   EXPECT_CALL(mock_recorder, AddEntry).Times(1);
2222
2223   builder.Record(mojo_recorder.get());
2224   run_loop.RunUntilIdle();
2225 }
2226
2227 TEST_F(UkmReduceAddEntryIpcTest, RecordingDisabled) {
2228   ASSERT_TRUE(base::FeatureList::IsEnabled(ukm::kUkmReduceAddEntryIPC));
2229
2230   base::RunLoop run_loop;
2231   ukm::UkmEntryBuilder builder(ukm::NoURLSourceId(),
2232                                "Event.ScrollUpdate.Touch");
2233   builder.SetMetric("TimeToScrollUpdateSwapBegin", 17);
2234
2235   UkmService service(&prefs_, &client_,
2236                      std::make_unique<MockDemographicMetricsProvider>());
2237   // Initialize UkmService.
2238   service.Initialize();
2239
2240   // Custom UkmRecorder to intercept messages to and from UkmService to clients.
2241   MockUkmRecorder mock_recorder;
2242   mojo::Remote<ukm::mojom::UkmRecorderFactory> factory;
2243   metrics::UkmRecorderFactoryImpl::Create(&mock_recorder,
2244                                           factory.BindNewPipeAndPassReceiver());
2245
2246   // MojoUkmRecorder (client).
2247   auto mojo_recorder = MojoUkmRecorder::Create(*factory);
2248
2249   service.DisableRecording();
2250   run_loop.RunUntilIdle();
2251
2252   // Since UkmObservers list is empty, the final decision regarding sending the
2253   // AddEntry IPC from clients to UkmService depends on recording being
2254   // enabled/disabled.
2255   EXPECT_CALL(mock_recorder, AddEntry).Times(0);
2256
2257   builder.Record(mojo_recorder.get());
2258   run_loop.RunUntilIdle();
2259 }
2260
2261 TEST_F(UkmReduceAddEntryIpcTest, AddRemoveUkmObserver) {
2262   ASSERT_TRUE(base::FeatureList::IsEnabled(ukm::kUkmReduceAddEntryIPC));
2263
2264   base::RunLoop run_loop;
2265   UkmService service(&prefs_, &client_,
2266                      std::make_unique<MockDemographicMetricsProvider>());
2267   // Initialize UkmService.
2268   service.Initialize();
2269
2270   // Custom UkmRecorder to intercept messages to and from UkmService to clients.
2271   MockUkmRecorder mock_recorder;
2272   mojo::Remote<ukm::mojom::UkmRecorderFactory> factory;
2273   metrics::UkmRecorderFactoryImpl::Create(&mock_recorder,
2274                                           factory.BindNewPipeAndPassReceiver());
2275
2276   // MojoUkmRecorder (client).
2277   auto mojo_recorder = MojoUkmRecorder::Create(*factory);
2278
2279   // Recording Disabled.
2280   service.DisableRecording();
2281   run_loop.RunUntilIdle();
2282
2283   mojom::UkmEntryPtr observed_ukm_entry;
2284
2285   // UkmRecorderObservers with different event hashes. If an entry is seen at
2286   // client where event_hash matches with one from observers, we need to send
2287   // AddEntry IPC even if recording is disabled.
2288   UkmRecorderObserver obs1, obs2;
2289   base::flat_set<uint64_t> events1 = {
2290       base::HashMetricName("Event.ScrollUpdate.Touch")};
2291   service.AddUkmRecorderObserver(events1, &obs1);
2292   base::flat_set<uint64_t> events2 = {
2293       base::HashMetricName("Event.ScrollBegin.Wheel")};
2294   service.AddUkmRecorderObserver(events2, &obs2);
2295   run_loop.RunUntilIdle();
2296
2297   {
2298     // This event is being observed by UkmRecorderObserver obs1. Hence, even
2299     // when UKM recording has been disabled, the event needs to be sent to
2300     // browser process.
2301     ukm::UkmEntryBuilder builder(ukm::NoURLSourceId(),
2302                                  "Event.ScrollUpdate.Touch");
2303     builder.SetMetric("TimeToScrollUpdateSwapBegin", 17);
2304     builder.SetMetric("IsMainThread", 21);
2305     auto expected_ukm_entry = builder.GetEntryForTesting();
2306     // We expect this event will not be filtered at client,i.e., MojoUkmRecorder
2307     // and the mock method AddEntry would be called once.
2308     EXPECT_CALL(mock_recorder, AddEntry)
2309         .Times(1)
2310         .WillOnce(testing::Invoke([&](mojom::UkmEntryPtr entry) {
2311           observed_ukm_entry = std::move(entry);
2312         }));
2313
2314     builder.Record(mojo_recorder.get());
2315     run_loop.RunUntilIdle();
2316     // Expects the UkmEntry seen at both sent(MojoUkmRecorder) and
2317     // receive(MockUkmRecorder) to be same.
2318     EXPECT_EQ(expected_ukm_entry, observed_ukm_entry);
2319   }
2320   {
2321     // This event is not being observed by any of UkmRecorderObservers and since
2322     // UKM recording has been disabled, the event will not be sent to browser
2323     // process.
2324     ukm::UkmEntryBuilder builder2(ukm::NoURLSourceId(), "Download.Interrupted");
2325     builder2.SetMetric("BytesWasted", 10);
2326     // Expect 0 calls to MockUkmRecorer::AddEntry since the UKM event will be
2327     // filtered out at the client.
2328     EXPECT_CALL(mock_recorder, AddEntry).Times(0);
2329     builder2.Record(mojo_recorder.get());
2330     run_loop.RunUntilIdle();
2331   }
2332   {
2333     // This event is being observed by UkmRecorderObserver obs2. Hence, even
2334     // when UKM recording has been disabled, the event needs to be sent to
2335     // browser process.
2336     ukm::UkmEntryBuilder builder3(ukm::NoURLSourceId(),
2337                                   "Event.ScrollBegin.Wheel");
2338     builder3.SetMetric("TimeToScrollUpdateSwapBegin", 25);
2339     auto expected_ukm_entry = builder3.GetEntryForTesting();
2340
2341     EXPECT_CALL(mock_recorder, AddEntry)
2342         .Times(1)
2343         .WillOnce(testing::Invoke([&](mojom::UkmEntryPtr entry) {
2344           observed_ukm_entry = std::move(entry);
2345         }));
2346     builder3.Record(mojo_recorder.get());
2347     run_loop.RunUntilIdle();
2348     EXPECT_EQ(expected_ukm_entry, observed_ukm_entry);
2349   }
2350   // Remove UkmRecorderObserver obs1.
2351   service.RemoveUkmRecorderObserver(&obs1);
2352   run_loop.RunUntilIdle();
2353   {
2354     // This event is not being observed by any of the UkmRecorderObservers now,
2355     // and since UKM recording is disabled, it will be filtered out.
2356     ukm::UkmEntryBuilder builder4(ukm::NoURLSourceId(),
2357                                   "Event.ScrollUpdate.Touch");
2358     builder4.SetMetric("TimeToScrollUpdateSwapBegin", 17);
2359     // Expect 0 calls to MockUkmRecorer::AddEntry since the UKM event will be
2360     // filtered out at the client, because of UKM recording being disabled.
2361     EXPECT_CALL(mock_recorder, AddEntry).Times(0);
2362     builder4.Record(mojo_recorder.get());
2363     run_loop.RunUntilIdle();
2364   }
2365 }
2366
2367 TEST_F(UkmReduceAddEntryIpcTest, MultipleDelegates) {
2368   ASSERT_TRUE(base::FeatureList::IsEnabled(ukm::kUkmReduceAddEntryIPC));
2369
2370   base::RunLoop run_loop;
2371   ukm::UkmEntryBuilder builder(ukm::NoURLSourceId(),
2372                                "Event.ScrollUpdate.Touch");
2373   builder.SetMetric("TimeToScrollUpdateSwapBegin", 17);
2374
2375   UkmService service(&prefs_, &client_,
2376                      std::make_unique<MockDemographicMetricsProvider>());
2377   // Initialize UkmService.
2378   service.Initialize();
2379   MockUkmRecorder mock_recorder;
2380   mojo::Remote<ukm::mojom::UkmRecorderFactory> factory;
2381   metrics::UkmRecorderFactoryImpl::Create(&mock_recorder,
2382                                           factory.BindNewPipeAndPassReceiver());
2383
2384   // MojoUkmRecorder (client).
2385   auto mojo_recorder = MojoUkmRecorder::Create(*factory);
2386
2387   // Disabled recording but having multiple delegates will default clients
2388   // sending all AddEntry IPCs to the browser.
2389   service.DisableRecording();
2390   run_loop.RunUntilIdle();
2391
2392   ukm::TestAutoSetUkmRecorder test_recorder;
2393   run_loop.RunUntilIdle();
2394
2395   EXPECT_CALL(mock_recorder, AddEntry).Times(1);
2396
2397   builder.Record(mojo_recorder.get());
2398   run_loop.RunUntilIdle();
2399 }
2400 }  // namespace ukm