src/chrome/browser/history/most_visited_tiles_experiment_unittest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/history/most_visited_tiles_experiment.h"
   6
   7 #include <algorithm>
   8 #include <sstream>
   9
  10 #include "base/metrics/field_trial.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/metrics/statistics_recorder.h"
  13 #include "base/strings/stringprintf.h"
  14 #include "base/values.h"
  15 #include "chrome/browser/history/history_types.h"
  16 #include "chrome/common/instant_types.h"
  17 #include "components/variations/entropy_provider.h"
  18 #include "testing/gtest/include/gtest/gtest.h"
  19 #include "url/gurl.h"
  20
  21 namespace history {
  22
  23 namespace {
  24
  25 // Constants for the most visited tile placement field trial.
  26 // See field trial config (MostVisitedTilePlacement.json) for details.
  27 const char kMostVisitedFieldTrialName[] = "MostVisitedTilePlacement";
  28 const char kOneEightAGroupName[] = "OneEight_A_Flipped";
  29 const char kOneFourAGroupName[] = "OneFour_A_Flipped";
  30 const char kDontShowOpenURLsGroupName[] = "DontShowOpenTabs";
  31 const char kGmailURL[] = "http://www.gmail.com/";
  32 // Name of histogram tracking types of actions carried out by the field trial.
  33 const char kMostVisitedExperimentHistogramName[] =
  34     "NewTabPage.MostVisitedTilePlacementExperiment";
  35 // Minimum number of Most Visited suggestions required in order for the Most
  36 // Visited Field Trial to remove a URL already open in the browser.
  37 const size_t kMinUrlSuggestions = 8;
  38
  39 // The indexes of the tiles that are affected in the experiment.
  40 enum FlippedIndexes {
  41   TILE_ONE = 0,
  42   TILE_FOUR = 3,
  43   TILE_EIGHT = 7
  44 };
  45
  46 // Creates a DictionaryValue using |url| and appends to |list|.
  47 void AppendURLToListValue(const std::string& url_string,
  48                           base::ListValue* list) {
  49   DictionaryValue* page_value = new DictionaryValue();
  50   page_value->SetString("url", url_string);
  51   list->Append(page_value);
  52 }
  53
  54 // Creates an InstantMostVisitedItem using |url| and appends to |list|.
  55 void AppendInstantURLToVector(const std::string& url_string,
  56                               std::vector<InstantMostVisitedItem>* list) {
  57   InstantMostVisitedItem item;
  58   item.url = GURL(url_string);
  59   list->push_back(item);
  60 }
  61
  62 // Creates an MostVisitedURL using |url| and appends to |list|.
  63 void AppendMostVisitedURLToVector(const std::string& url_string,
  64                                   std::vector<history::MostVisitedURL>* list) {
  65   history::MostVisitedURL most_visited;
  66   most_visited.url = GURL(url_string);
  67   list->push_back(most_visited);
  68 }
  69
  70 void SetUpMaybeShuffle(const int& max_urls,
  71                        MostVisitedURLList* most_visited_urls,
  72                        MostVisitedURLList* test_urls) {
  73   // |most_visited_urls| must have > 8 MostVisitedURLs for any URLs to be
  74   // flipped by experiment.
  75   for (int i = 0; i < max_urls; ++i) {
  76     std::string url;
  77     base::SStringPrintf(&url, "http://www.test%d.com", i);
  78     AppendMostVisitedURLToVector(url, most_visited_urls);
  79     AppendMostVisitedURLToVector(url, test_urls);
  80   }
  81 }
  82
  83 }  // namespace
  84
  85 class MostVisitedTilesExperimentTest : public testing::Test {
  86  public:
  87   MostVisitedTilesExperimentTest()
  88       : histogram_(NULL),
  89         field_trial_list_(new metrics::SHA1EntropyProvider("foo")) {}
  90
  91   virtual ~MostVisitedTilesExperimentTest() {}
  92
  93  protected:
  94   virtual void SetUp() OVERRIDE {
  95     base::StatisticsRecorder::Initialize();
  96     previous_metrics_count_.resize(NUM_NTP_TILE_EXPERIMENT_ACTIONS, 0);
  97     base::HistogramBase* histogram = GetHistogram();
  98     if (histogram) {
  99       scoped_ptr<base::HistogramSamples> samples(histogram->SnapshotSamples());
 100       if (samples.get()) {
 101         for (int state = NTP_TILE_EXPERIMENT_ACTION_REMOVED_URL;
 102              state < NUM_NTP_TILE_EXPERIMENT_ACTIONS;
 103              ++state) {
 104           previous_metrics_count_[state] = samples->GetCount(state);
 105         }
 106       }
 107     }
 108   }
 109
 110   void ValidateMetrics(const base::HistogramBase::Sample& value) {
 111     base::HistogramBase* histogram = GetHistogram();
 112     ASSERT_TRUE(histogram != NULL);
 113     scoped_ptr<base::HistogramSamples> samples(histogram->SnapshotSamples());
 114     if (samples.get()) {
 115       for (int state = NTP_TILE_EXPERIMENT_ACTION_REMOVED_URL;
 116            state < NUM_NTP_TILE_EXPERIMENT_ACTIONS;
 117            ++state) {
 118         if (state == value) {
 119           EXPECT_EQ(previous_metrics_count_[state] + 1,
 120                     samples->GetCount(state));
 121         } else {
 122           EXPECT_EQ(previous_metrics_count_[state], samples->GetCount(state));
 123         }
 124       }
 125     }
 126   }
 127
 128  private:
 129   base::HistogramBase* GetHistogram() {
 130     if (!histogram_) {
 131       histogram_ = base::StatisticsRecorder::FindHistogram(
 132           kMostVisitedExperimentHistogramName);
 133     }
 134     return histogram_;
 135   }
 136
 137   // Owned by base::StatisticsRecorder
 138   base::HistogramBase* histogram_;
 139   base::FieldTrialList field_trial_list_;
 140   std::vector<int> previous_metrics_count_;
 141
 142   DISALLOW_COPY_AND_ASSIGN(MostVisitedTilesExperimentTest);
 143 };
 144
 145 // For pre-instant extended clients.
 146 TEST_F(MostVisitedTilesExperimentTest,
 147        RemovePageValuesMatchingOpenTabsTooFewURLs) {
 148   base::FieldTrialList::CreateFieldTrial(kMostVisitedFieldTrialName,
 149                                          kDontShowOpenURLsGroupName);
 150
 151   // Ensure the field trial is created with the correct group.
 152   EXPECT_TRUE(MostVisitedTilesExperiment::IsDontShowOpenURLsEnabled());
 153
 154   std::set<std::string> open_urls;
 155   open_urls.insert(kGmailURL);
 156
 157   base::ListValue pages_value;
 158   AppendURLToListValue(kGmailURL, &pages_value);
 159
 160   // Test the method when there are not enough URLs to force removal.
 161   MostVisitedTilesExperiment::RemovePageValuesMatchingOpenTabs(
 162       open_urls, &pages_value);
 163   DictionaryValue gmail_value;
 164   gmail_value.SetString("url", kGmailURL);
 165   // Ensure the open url has not been removed from |pages_value|.
 166   EXPECT_NE(pages_value.end(), pages_value.Find(gmail_value));
 167
 168   // Ensure counts have been incremented correctly.
 169   EXPECT_NO_FATAL_FAILURE(
 170       ValidateMetrics(NTP_TILE_EXPERIMENT_ACTION_DID_NOT_REMOVE_URL));
 171 }
 172
 173 // For pre-instant extended clients.
 174 TEST_F(MostVisitedTilesExperimentTest, RemovePageValuesMatchingOpenTabs) {
 175   base::FieldTrialList::CreateFieldTrial(kMostVisitedFieldTrialName,
 176                                          kDontShowOpenURLsGroupName);
 177
 178   // Ensure the field trial is created with the correct group.
 179   EXPECT_TRUE(MostVisitedTilesExperiment::IsDontShowOpenURLsEnabled());
 180
 181   std::set<std::string> open_urls;
 182   open_urls.insert(kGmailURL);
 183
 184   base::ListValue pages_value;
 185   AppendURLToListValue(kGmailURL, &pages_value);
 186
 187   // |pages_value| must have > 8 page values for any URLs to be removed by
 188   // experiment.
 189   for (size_t i = 0; i < kMinUrlSuggestions; ++i) {
 190     std::string url;
 191     base::SStringPrintf(&url, "http://www.test%d.com", static_cast<int>(i));
 192     AppendURLToListValue(url, &pages_value);
 193   }
 194
 195   // Call method with enough URLs to force removal.
 196   MostVisitedTilesExperiment::RemovePageValuesMatchingOpenTabs(
 197       open_urls, &pages_value);
 198   // Ensure the open url has been removed from |pages_value|.
 199   DictionaryValue gmail_value;
 200   gmail_value.SetString("url", kGmailURL);
 201   EXPECT_EQ(pages_value.end(), pages_value.Find(gmail_value));
 202
 203   // Ensure counts have been incremented correctly.
 204   EXPECT_NO_FATAL_FAILURE(
 205       ValidateMetrics(NTP_TILE_EXPERIMENT_ACTION_REMOVED_URL));
 206 }
 207
 208 // For instant extended clients.
 209 TEST_F(MostVisitedTilesExperimentTest, RemoveItemsMatchingOpenTabsTooFewURLs) {
 210   base::FieldTrialList::CreateFieldTrial(kMostVisitedFieldTrialName,
 211                                          kDontShowOpenURLsGroupName);
 212
 213   // Ensure the field trial is created with the correct group.
 214   EXPECT_TRUE(MostVisitedTilesExperiment::IsDontShowOpenURLsEnabled());
 215
 216   std::set<std::string> open_urls;
 217   open_urls.insert(kGmailURL);
 218   std::vector<InstantMostVisitedItem> items;
 219   AppendInstantURLToVector(kGmailURL, &items);
 220
 221   // Call the method when there are not enough URLs to force removal.
 222   MostVisitedTilesExperiment::RemoveItemsMatchingOpenTabs(open_urls, &items);
 223
 224   // Ensure the open url has not been removed from |items|.
 225   for (size_t i = 0; i < items.size(); i++) {
 226     const std::string& item_url = items[i].url.spec();
 227     EXPECT_NE(0u, open_urls.count(item_url));
 228   }
 229
 230   // Ensure counts have been incremented correctly.
 231   EXPECT_NO_FATAL_FAILURE(
 232       ValidateMetrics(NTP_TILE_EXPERIMENT_ACTION_DID_NOT_REMOVE_URL));
 233 }
 234
 235 // For instant extended clients.
 236 TEST_F(MostVisitedTilesExperimentTest, RemoveItemsMatchingOpenTabs) {
 237   base::FieldTrialList::CreateFieldTrial(
 238       kMostVisitedFieldTrialName,
 239       kDontShowOpenURLsGroupName);
 240
 241   // Ensure the field trial is created with the correct group.
 242   EXPECT_TRUE(MostVisitedTilesExperiment::IsDontShowOpenURLsEnabled());
 243
 244   std::set<std::string> open_urls;
 245   open_urls.insert(kGmailURL);
 246   std::vector<InstantMostVisitedItem> items;
 247   AppendInstantURLToVector(kGmailURL, &items);
 248
 249   // |items| must have > 8 InstantMostVisitedItems for any URLs to be removed by
 250   // experiment.
 251   for (size_t i = 0; i < kMinUrlSuggestions; ++i) {
 252     std::string url;
 253     base::SStringPrintf(&url, "http://www.test%d.com", static_cast<int>(i));
 254     AppendInstantURLToVector(url, &items);
 255   }
 256
 257   // Call method with enough URLs to force removal.
 258   MostVisitedTilesExperiment::RemoveItemsMatchingOpenTabs(open_urls, &items);
 259
 260   // Ensure the open URL has been removed from |items|.
 261   for (size_t i = 0; i < items.size(); i++) {
 262     const std::string& item_url = items[i].url.spec();
 263     EXPECT_EQ(0u, open_urls.count(item_url));
 264   }
 265
 266   // Ensure counts have been incremented correctly.
 267   EXPECT_NO_FATAL_FAILURE(
 268       ValidateMetrics(NTP_TILE_EXPERIMENT_ACTION_REMOVED_URL));
 269 }
 270
 271 TEST_F(MostVisitedTilesExperimentTest, MaybeShuffleOneEight) {
 272   base::FieldTrialList::CreateFieldTrial(kMostVisitedFieldTrialName,
 273                                          kOneEightAGroupName);
 274
 275   // Ensure the field trial is created with the correct group.
 276   EXPECT_EQ(kOneEightAGroupName,
 277             base::FieldTrialList::FindFullName(kMostVisitedFieldTrialName));
 278
 279   MostVisitedURLList most_visited_urls;
 280   MostVisitedURLList test_urls;
 281   SetUpMaybeShuffle(kMinUrlSuggestions, &most_visited_urls, &test_urls);
 282
 283   history::MostVisitedTilesExperiment::MaybeShuffle(&most_visited_urls);
 284   // Ensure the 1st and 8th URLs have been switched.
 285   EXPECT_EQ(most_visited_urls[TILE_ONE].url.spec(),
 286             test_urls[TILE_EIGHT].url.spec());
 287 }
 288
 289 TEST_F(MostVisitedTilesExperimentTest, MaybeShuffleOneEightTooFewURLs) {
 290   base::FieldTrialList::CreateFieldTrial(kMostVisitedFieldTrialName,
 291                                          kOneEightAGroupName);
 292
 293   // Ensure the field trial is created with the correct group.
 294   EXPECT_EQ(kOneEightAGroupName,
 295             base::FieldTrialList::FindFullName(kMostVisitedFieldTrialName));
 296
 297   MostVisitedURLList most_visited_urls;
 298   MostVisitedURLList test_urls;
 299   // If |most_visited_urls| has < 8 URLs, experiment will not flip any tiles.
 300   SetUpMaybeShuffle(kMinUrlSuggestions - 1, &most_visited_urls, &test_urls);
 301
 302   history::MostVisitedTilesExperiment::MaybeShuffle(&most_visited_urls);
 303   // Ensure no URLs have been switched.
 304   EXPECT_EQ(most_visited_urls[TILE_ONE].url.spec(),
 305             test_urls[TILE_ONE].url.spec());
 306   EXPECT_EQ(most_visited_urls[TILE_EIGHT - 1].url.spec(),
 307             test_urls[TILE_EIGHT - 1].url.spec());
 308
 309   // Ensure counts are correct.
 310   EXPECT_NO_FATAL_FAILURE(
 311       ValidateMetrics(NTP_TILE_EXPERIMENT_ACTION_TOO_FEW_URLS_TILES_1_8));
 312 }
 313
 314 TEST_F(MostVisitedTilesExperimentTest, MaybeShuffleOneFour) {
 315   base::FieldTrialList::CreateFieldTrial(kMostVisitedFieldTrialName,
 316                                          kOneFourAGroupName);
 317
 318   // Ensure the field trial is created with the correct group.
 319   EXPECT_EQ(kOneFourAGroupName,
 320             base::FieldTrialList::FindFullName(kMostVisitedFieldTrialName));
 321
 322   MostVisitedURLList most_visited_urls;
 323   MostVisitedURLList test_urls;
 324   SetUpMaybeShuffle(kMinUrlSuggestions, &most_visited_urls, &test_urls);
 325
 326   history::MostVisitedTilesExperiment::MaybeShuffle(&most_visited_urls);
 327   // Ensure the 1st and 4th URLs have been switched.
 328   EXPECT_EQ(most_visited_urls[TILE_ONE].url.spec(),
 329             test_urls[TILE_FOUR].url.spec());
 330 }
 331
 332 TEST_F(MostVisitedTilesExperimentTest, MaybeShuffleOneFourTooFewURLs) {
 333   base::FieldTrialList::CreateFieldTrial(
 334       kMostVisitedFieldTrialName,
 335       kOneFourAGroupName);
 336
 337   // Ensure the field trial is created with the correct group.
 338   EXPECT_EQ(kOneFourAGroupName,
 339             base::FieldTrialList::FindFullName(kMostVisitedFieldTrialName));
 340
 341   MostVisitedURLList most_visited_urls;
 342   MostVisitedURLList test_urls;
 343   // If |most_visited_urls| has < 4 URLs, experiment will not flip any tiles.
 344   SetUpMaybeShuffle(kMinUrlSuggestions - 5, &most_visited_urls, &test_urls);
 345
 346   history::MostVisitedTilesExperiment::MaybeShuffle(&most_visited_urls);
 347   // Ensure no URLs have been switched.
 348   EXPECT_EQ(most_visited_urls[TILE_ONE].url.spec(),
 349            test_urls[TILE_ONE].url.spec());
 350   EXPECT_EQ(most_visited_urls[TILE_FOUR-1].url.spec(),
 351             test_urls[TILE_FOUR-1].url.spec());
 352
 353   // Ensure counts are correct.
 354   EXPECT_NO_FATAL_FAILURE(
 355       ValidateMetrics(NTP_TILE_EXPERIMENT_ACTION_TOO_FEW_URLS_TILES_1_4));
 356 }
 357
 358 }  // namespace history