src/content/public/test/test_launcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/public/test/test_launcher.h"
   6
   7 #include <map>
   8 #include <string>
   9 #include <vector>
  10
  11 #include "base/command_line.h"
  12 #include "base/containers/hash_tables.h"
  13 #include "base/environment.h"
  14 #include "base/file_util.h"
  15 #include "base/files/scoped_temp_dir.h"
  16 #include "base/logging.h"
  17 #include "base/memory/linked_ptr.h"
  18 #include "base/memory/scoped_ptr.h"
  19 #include "base/message_loop/message_loop.h"
  20 #include "base/stl_util.h"
  21 #include "base/strings/string_number_conversions.h"
  22 #include "base/strings/string_util.h"
  23 #include "base/strings/utf_string_conversions.h"
  24 #include "base/test/launcher/test_launcher.h"
  25 #include "base/test/test_suite.h"
  26 #include "base/test/test_switches.h"
  27 #include "base/test/test_timeouts.h"
  28 #include "base/time/time.h"
  29 #include "content/public/app/content_main.h"
  30 #include "content/public/app/content_main_delegate.h"
  31 #include "content/public/app/startup_helper_win.h"
  32 #include "content/public/common/content_switches.h"
  33 #include "content/public/common/sandbox_init.h"
  34 #include "content/public/test/browser_test.h"
  35 #include "net/base/escape.h"
  36 #include "testing/gtest/include/gtest/gtest.h"
  37
  38 #if defined(OS_WIN)
  39 #include "base/base_switches.h"
  40 #include "content/common/sandbox_win.h"
  41 #include "sandbox/win/src/sandbox_factory.h"
  42 #include "sandbox/win/src/sandbox_types.h"
  43 #elif defined(OS_MACOSX)
  44 #include "base/mac/scoped_nsautorelease_pool.h"
  45 #endif
  46
  47 namespace content {
  48
  49 namespace {
  50
  51 // Tests with this prefix run before the same test without it, and use the same
  52 // profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
  53 // that span browser restarts.
  54 const char kPreTestPrefix[] = "PRE_";
  55
  56 // Manual tests only run when --run-manual is specified. This allows writing
  57 // tests that don't run automatically but are still in the same test binary.
  58 // This is useful so that a team that wants to run a few tests doesn't have to
  59 // add a new binary that must be compiled on all builds.
  60 const char kManualTestPrefix[] = "MANUAL_";
  61
  62 TestLauncherDelegate* g_launcher_delegate;
  63
  64 std::string RemoveAnyPrePrefixes(const std::string& test_name) {
  65   std::string result(test_name);
  66   ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string());
  67   return result;
  68 }
  69
  70 void PrintUsage() {
  71   fprintf(stdout,
  72       "Runs tests using the gtest framework, each test being run in its own\n"
  73       "process.  Any gtest flags can be specified.\n"
  74       "  --single_process\n"
  75       "    Runs the tests and the launcher in the same process. Useful for \n"
  76       "    debugging a specific test in a debugger.\n"
  77       "  --single-process\n"
  78       "    Same as above, and also runs Chrome in single-process mode.\n"
  79       "  --help\n"
  80       "    Shows this message.\n"
  81       "  --gtest_help\n"
  82       "    Shows the gtest help message.\n");
  83 }
  84
  85 // Implementation of base::TestLauncherDelegate. This is also a test launcher,
  86 // wrapping a lower-level test launcher with content-specific code.
  87 class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
  88  public:
  89   explicit WrapperTestLauncherDelegate(
  90       content::TestLauncherDelegate* launcher_delegate)
  91       : launcher_delegate_(launcher_delegate) {
  92     CHECK(temp_dir_.CreateUniqueTempDir());
  93   }
  94
  95   // base::TestLauncherDelegate:
  96   virtual void OnTestIterationStarting() OVERRIDE;
  97   virtual std::string GetTestNameForFiltering(
  98       const testing::TestCase* test_case,
  99       const testing::TestInfo* test_info) OVERRIDE;
 100   virtual bool ShouldRunTest(const testing::TestCase* test_case,
 101                              const testing::TestInfo* test_info) OVERRIDE;
 102   virtual size_t RunTests(base::TestLauncher* test_launcher,
 103                           const std::vector<std::string>& test_names) OVERRIDE;
 104   virtual size_t RetryTests(
 105       base::TestLauncher* test_launcher,
 106       const std::vector<std::string>& test_names) OVERRIDE;
 107
 108  private:
 109   void DoRunTest(base::TestLauncher* test_launcher,
 110                  const std::string& test_name);
 111
 112   // Launches test named |test_name| using parallel launcher,
 113   // given result of PRE_ test |pre_test_result|.
 114   void RunDependentTest(base::TestLauncher* test_launcher,
 115                         const std::string test_name,
 116                         const base::TestResult& pre_test_result);
 117
 118   // Callback to receive result of a test.
 119   void GTestCallback(
 120       base::TestLauncher* test_launcher,
 121       const std::string& test_name,
 122       int exit_code,
 123       const base::TimeDelta& elapsed_time,
 124       bool was_timeout,
 125       const std::string& output);
 126
 127   content::TestLauncherDelegate* launcher_delegate_;
 128
 129   // Store dependent test name (map is indexed by full test name).
 130   typedef std::map<std::string, std::string> DependentTestMap;
 131   DependentTestMap dependent_test_map_;
 132   DependentTestMap reverse_dependent_test_map_;
 133
 134   // Store unique data directory prefix for test names (without PRE_ prefixes).
 135   // PRE_ tests and tests that depend on them must share the same
 136   // data directory. Using test name as directory name leads to too long
 137   // names (exceeding UNIX_PATH_MAX, which creates a problem with
 138   // process_singleton_linux). Create a randomly-named temporary directory
 139   // and keep track of the names so that PRE_ tests can still re-use them.
 140   typedef std::map<std::string, base::FilePath> UserDataDirMap;
 141   UserDataDirMap user_data_dir_map_;
 142
 143   // Store names of all seen tests to properly handle PRE_ tests.
 144   std::set<std::string> all_test_names_;
 145
 146   // Temporary directory for user data directories.
 147   base::ScopedTempDir temp_dir_;
 148
 149   DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
 150 };
 151
 152 void WrapperTestLauncherDelegate::OnTestIterationStarting() {
 153   dependent_test_map_.clear();
 154   user_data_dir_map_.clear();
 155 }
 156
 157 std::string WrapperTestLauncherDelegate::GetTestNameForFiltering(
 158     const testing::TestCase* test_case,
 159     const testing::TestInfo* test_info) {
 160   return RemoveAnyPrePrefixes(
 161       std::string(test_case->name()) + "." + test_info->name());
 162 }
 163
 164 bool WrapperTestLauncherDelegate::ShouldRunTest(
 165     const testing::TestCase* test_case,
 166     const testing::TestInfo* test_info) {
 167   all_test_names_.insert(
 168       std::string(test_case->name()) + "." + test_info->name());
 169
 170   if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
 171       !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
 172     return false;
 173   }
 174
 175   if (StartsWithASCII(test_info->name(), kPreTestPrefix, true)) {
 176     // We will actually run PRE_ tests, but to ensure they run on the same shard
 177     // as dependent tests, handle all these details internally.
 178     return false;
 179   }
 180
 181   return true;
 182 }
 183
 184 std::string GetPreTestName(const std::string& full_name) {
 185   size_t dot_pos = full_name.find('.');
 186   CHECK_NE(dot_pos, std::string::npos);
 187   std::string test_case_name = full_name.substr(0, dot_pos);
 188   std::string test_name = full_name.substr(dot_pos + 1);
 189   return test_case_name + "." + kPreTestPrefix + test_name;
 190 }
 191
 192 size_t WrapperTestLauncherDelegate::RunTests(
 193     base::TestLauncher* test_launcher,
 194     const std::vector<std::string>& test_names) {
 195   // Number of additional tests to run because of dependencies.
 196   size_t additional_tests_to_run_count = 0;
 197
 198   // Compute dependencies of tests to be run.
 199   for (size_t i = 0; i < test_names.size(); i++) {
 200     std::string full_name(test_names[i]);
 201     std::string pre_test_name(GetPreTestName(full_name));
 202
 203     while (ContainsKey(all_test_names_, pre_test_name)) {
 204       additional_tests_to_run_count++;
 205
 206       DCHECK(!ContainsKey(dependent_test_map_, pre_test_name));
 207       dependent_test_map_[pre_test_name] = full_name;
 208
 209       DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name));
 210       reverse_dependent_test_map_[full_name] = pre_test_name;
 211
 212       full_name = pre_test_name;
 213       pre_test_name = GetPreTestName(pre_test_name);
 214     }
 215   }
 216
 217   for (size_t i = 0; i < test_names.size(); i++) {
 218     std::string full_name(test_names[i]);
 219
 220     // Make sure no PRE_ tests were requested explicitly.
 221     DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name));
 222
 223     if (!ContainsKey(user_data_dir_map_, full_name)) {
 224       base::FilePath temp_dir;
 225       CHECK(file_util::CreateTemporaryDirInDir(
 226                 temp_dir_.path(), FILE_PATH_LITERAL("d"), &temp_dir));
 227       user_data_dir_map_[full_name] = temp_dir;
 228     }
 229
 230     // If the test has any dependencies, get to the root and start with that.
 231     while (ContainsKey(reverse_dependent_test_map_, full_name))
 232       full_name = GetPreTestName(full_name);
 233
 234     DoRunTest(test_launcher, full_name);
 235   }
 236
 237   return test_names.size() + additional_tests_to_run_count;
 238 }
 239
 240 size_t WrapperTestLauncherDelegate::RetryTests(
 241     base::TestLauncher* test_launcher,
 242     const std::vector<std::string>& test_names) {
 243   // List of tests we can kick off right now, depending on no other tests.
 244   std::vector<std::string> tests_to_run_now;
 245
 246   // We retry at least the tests requested to retry.
 247   std::set<std::string> test_names_set(test_names.begin(), test_names.end());
 248
 249   // In the face of PRE_ tests, we need to retry the entire chain of tests,
 250   // from the very first one.
 251   for (size_t i = 0; i < test_names.size(); i++) {
 252     std::string test_name(test_names[i]);
 253     while (ContainsKey(reverse_dependent_test_map_, test_name)) {
 254       test_name = reverse_dependent_test_map_[test_name];
 255       test_names_set.insert(test_name);
 256     }
 257   }
 258
 259   // Discard user data directories from any previous runs. Start with
 260   // fresh state.
 261   for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin();
 262        i != user_data_dir_map_.end();
 263        ++i) {
 264     // Delete temporary directories now to avoid using too much space in /tmp.
 265     if (!base::DeleteFile(i->second, true)) {
 266       LOG(WARNING) << "Failed to delete " << i->second.value();
 267     }
 268   }
 269   user_data_dir_map_.clear();
 270
 271   for (std::set<std::string>::const_iterator i = test_names_set.begin();
 272        i != test_names_set.end();
 273        ++i) {
 274     std::string full_name(*i);
 275
 276     // Make sure PRE_ tests and tests that depend on them share the same
 277     // data directory - based it on the test name without prefixes.
 278     std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name));
 279     if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) {
 280       base::FilePath temp_dir;
 281       CHECK(file_util::CreateTemporaryDirInDir(
 282                 temp_dir_.path(), FILE_PATH_LITERAL("d"), &temp_dir));
 283       user_data_dir_map_[test_name_no_pre] = temp_dir;
 284     }
 285
 286     size_t dot_pos = full_name.find('.');
 287     CHECK_NE(dot_pos, std::string::npos);
 288     std::string test_case_name = full_name.substr(0, dot_pos);
 289     std::string test_name = full_name.substr(dot_pos + 1);
 290     std::string pre_test_name(
 291         test_case_name + "." + kPreTestPrefix + test_name);
 292     if (!ContainsKey(test_names_set, pre_test_name))
 293       tests_to_run_now.push_back(full_name);
 294   }
 295
 296   for (size_t i = 0; i < tests_to_run_now.size(); i++)
 297     DoRunTest(test_launcher, tests_to_run_now[i]);
 298
 299   return test_names_set.size();
 300 }
 301
 302 void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher,
 303                                             const std::string& test_name) {
 304   std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
 305
 306   CommandLine cmd_line(*CommandLine::ForCurrentProcess());
 307   CHECK(launcher_delegate_->AdjustChildProcessCommandLine(
 308             &cmd_line, user_data_dir_map_[test_name_no_pre]));
 309
 310   CommandLine new_cmd_line(cmd_line.GetProgram());
 311   CommandLine::SwitchMap switches = cmd_line.GetSwitches();
 312
 313   // Strip out gtest_output flag because otherwise we would overwrite results
 314   // of the other tests.
 315   switches.erase(base::kGTestOutputFlag);
 316
 317   for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
 318        iter != switches.end(); ++iter) {
 319     new_cmd_line.AppendSwitchNative(iter->first, iter->second);
 320   }
 321
 322   // Always enable disabled tests.  This method is not called with disabled
 323   // tests unless this flag was specified to the browser test executable.
 324   new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
 325   new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
 326   new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
 327
 328   char* browser_wrapper = getenv("BROWSER_WRAPPER");
 329
 330   test_launcher->LaunchChildGTestProcess(
 331       new_cmd_line,
 332       browser_wrapper ? browser_wrapper : std::string(),
 333       TestTimeouts::action_max_timeout(),
 334       base::Bind(&WrapperTestLauncherDelegate::GTestCallback,
 335                  base::Unretained(this),
 336                  test_launcher,
 337                  test_name));
 338 }
 339
 340 void WrapperTestLauncherDelegate::RunDependentTest(
 341     base::TestLauncher* test_launcher,
 342     const std::string test_name,
 343     const base::TestResult& pre_test_result) {
 344   if (pre_test_result.status == base::TestResult::TEST_SUCCESS) {
 345     // Only run the dependent test if PRE_ test succeeded.
 346     DoRunTest(test_launcher, test_name);
 347   } else {
 348     // Otherwise skip the test.
 349     base::TestResult test_result;
 350     test_result.full_name = test_name;
 351     test_result.status = base::TestResult::TEST_SKIPPED;
 352     test_launcher->OnTestFinished(test_result);
 353
 354     if (ContainsKey(dependent_test_map_, test_name)) {
 355       RunDependentTest(test_launcher,
 356                        dependent_test_map_[test_name],
 357                        test_result);
 358     }
 359   }
 360 }
 361
 362 void WrapperTestLauncherDelegate::GTestCallback(
 363     base::TestLauncher* test_launcher,
 364     const std::string& test_name,
 365     int exit_code,
 366     const base::TimeDelta& elapsed_time,
 367     bool was_timeout,
 368     const std::string& output) {
 369   base::TestResult result;
 370   result.full_name = test_name;
 371
 372   // TODO(phajdan.jr): Recognize crashes.
 373   if (exit_code == 0)
 374     result.status = base::TestResult::TEST_SUCCESS;
 375   else if (was_timeout)
 376     result.status = base::TestResult::TEST_TIMEOUT;
 377   else
 378     result.status = base::TestResult::TEST_FAILURE;
 379
 380   result.elapsed_time = elapsed_time;
 381
 382   result.output_snippet = GetTestOutputSnippet(result, output);
 383
 384   if (ContainsKey(dependent_test_map_, test_name)) {
 385     RunDependentTest(test_launcher, dependent_test_map_[test_name], result);
 386   } else {
 387     // No other tests depend on this, we can delete the temporary directory now.
 388     // Do so to avoid too many temporary files using lots of disk space.
 389     std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
 390     if (ContainsKey(user_data_dir_map_, test_name_no_pre)) {
 391       if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) {
 392         LOG(WARNING) << "Failed to delete "
 393                      << user_data_dir_map_[test_name_no_pre].value();
 394       }
 395       user_data_dir_map_.erase(test_name_no_pre);
 396     }
 397   }
 398
 399   test_launcher->OnTestFinished(result);
 400 }
 401
 402 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
 403   if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
 404     return true;
 405
 406   std::string switch_value =
 407       CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
 408   if (!base::StringToInt(switch_value, result) || *result < 1) {
 409     LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
 410     return false;
 411   }
 412
 413   return true;
 414 }
 415
 416 }  // namespace
 417
 418 const char kHelpFlag[]   = "help";
 419
 420 const char kLaunchAsBrowser[] = "as-browser";
 421
 422 // See kManualTestPrefix above.
 423 const char kRunManualTestsFlag[] = "run-manual";
 424
 425 const char kSingleProcessTestsFlag[]   = "single_process";
 426
 427
 428 TestLauncherDelegate::~TestLauncherDelegate() {
 429 }
 430
 431 bool ShouldRunContentMain() {
 432 #if defined(OS_WIN) || defined(OS_LINUX)
 433   CommandLine* command_line = CommandLine::ForCurrentProcess();
 434   return command_line->HasSwitch(switches::kProcessType) ||
 435          command_line->HasSwitch(kLaunchAsBrowser);
 436 #else
 437   return false;
 438 #endif  // defined(OS_WIN) || defined(OS_LINUX)
 439 }
 440
 441 int RunContentMain(int argc, char** argv,
 442                    TestLauncherDelegate* launcher_delegate) {
 443 #if defined(OS_WIN)
 444   sandbox::SandboxInterfaceInfo sandbox_info = {0};
 445   InitializeSandboxInfo(&sandbox_info);
 446   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
 447       launcher_delegate->CreateContentMainDelegate());
 448   return ContentMain(GetModuleHandle(NULL),
 449                      &sandbox_info,
 450                      chrome_main_delegate.get());
 451 #elif defined(OS_LINUX)
 452   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
 453       launcher_delegate->CreateContentMainDelegate());
 454   return ContentMain(argc, const_cast<const char**>(argv),
 455                      chrome_main_delegate.get());
 456 #endif  // defined(OS_WIN)
 457   NOTREACHED();
 458   return 0;
 459 }
 460
 461 int LaunchTests(TestLauncherDelegate* launcher_delegate,
 462                 int default_jobs,
 463                 int argc,
 464                 char** argv) {
 465   DCHECK(!g_launcher_delegate);
 466   g_launcher_delegate = launcher_delegate;
 467
 468   CommandLine::Init(argc, argv);
 469   const CommandLine* command_line = CommandLine::ForCurrentProcess();
 470
 471   if (command_line->HasSwitch(kHelpFlag)) {
 472     PrintUsage();
 473     return 0;
 474   }
 475
 476   if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
 477       (command_line->HasSwitch(switches::kSingleProcess) &&
 478        command_line->HasSwitch(base::kGTestFilterFlag)) ||
 479       command_line->HasSwitch(base::kGTestListTestsFlag) ||
 480       command_line->HasSwitch(base::kGTestHelpFlag)) {
 481 #if defined(OS_WIN)
 482     if (command_line->HasSwitch(kSingleProcessTestsFlag)) {
 483       sandbox::SandboxInterfaceInfo sandbox_info;
 484       InitializeSandboxInfo(&sandbox_info);
 485       InitializeSandbox(&sandbox_info);
 486     }
 487 #endif
 488     return launcher_delegate->RunTestSuite(argc, argv);
 489   }
 490
 491   if (ShouldRunContentMain())
 492     return RunContentMain(argc, argv, launcher_delegate);
 493
 494   base::AtExitManager at_exit;
 495   testing::InitGoogleTest(&argc, argv);
 496   TestTimeouts::Initialize();
 497
 498   int jobs = default_jobs;
 499   if (!GetSwitchValueAsInt(switches::kTestLauncherJobs, &jobs))
 500     return 1;
 501
 502   fprintf(stdout,
 503       "Starting tests (using %d parallel jobs)...\n"
 504       "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
 505       "For debugging a test inside a debugger, use the\n"
 506       "--gtest_filter=<your_test_name> flag along with either\n"
 507       "--single_process (to run the test in one launcher/browser process) or\n"
 508       "--single-process (to do the above, and also run Chrome in single-"
 509           "process mode).\n", jobs);
 510
 511   base::MessageLoopForIO message_loop;
 512
 513   WrapperTestLauncherDelegate delegate(launcher_delegate);
 514   base::TestLauncher launcher(&delegate, jobs);
 515   bool success = launcher.Run(argc, argv);
 516   return (success ? 0 : 1);
 517 }
 518
 519 TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
 520   return g_launcher_delegate;
 521 }
 522
 523 }  // namespace content