src/base/test/launcher/unit_test_launcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/launcher/unit_test_launcher.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback_helpers.h"
   9 #include "base/command_line.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/file_util.h"
  12 #include "base/files/scoped_temp_dir.h"
  13 #include "base/format_macros.h"
  14 #include "base/message_loop/message_loop.h"
  15 #include "base/stl_util.h"
  16 #include "base/strings/string_number_conversions.h"
  17 #include "base/strings/string_util.h"
  18 #include "base/sys_info.h"
  19 #include "base/test/gtest_xml_util.h"
  20 #include "base/test/launcher/test_launcher.h"
  21 #include "base/test/test_switches.h"
  22 #include "base/test/test_timeouts.h"
  23 #include "base/threading/thread_checker.h"
  24 #include "testing/gtest/include/gtest/gtest.h"
  25
  26 namespace base {
  27
  28 namespace {
  29
  30 // This constant controls how many tests are run in a single batch by default.
  31 const size_t kDefaultTestBatchLimit = 10;
  32
  33 const char kHelpFlag[] = "help";
  34
  35 // Flag to enable the new launcher logic.
  36 // TODO(phajdan.jr): Remove it, http://crbug.com/236893 .
  37 const char kBraveNewTestLauncherFlag[] = "brave-new-test-launcher";
  38
  39 // Flag to run all tests in a single process.
  40 const char kSingleProcessTestsFlag[] = "single-process-tests";
  41
  42 void PrintUsage() {
  43   fprintf(stdout,
  44           "Runs tests using the gtest framework, each batch of tests being\n"
  45           "run in their own process. Supported command-line flags:\n"
  46           "\n"
  47           "  --single-process-tests\n"
  48           "    Runs the tests and the launcher in the same process. Useful\n"
  49           "    for debugging a specific test in a debugger.\n"
  50           "  --test-launcher-jobs=N\n"
  51           "    Sets the number of parallel test jobs to N.\n"
  52           "  --test-launcher-batch-limit=N\n"
  53           "    Sets the limit of test batch to run in a single process to N.\n"
  54           "  --gtest_filter=...\n"
  55           "    Runs a subset of tests (see --gtest_help for more info).\n"
  56           "  --help\n"
  57           "    Shows this message.\n"
  58           "  --gtest_help\n"
  59           "    Shows the gtest help message.\n");
  60   fflush(stdout);
  61 }
  62
  63 // Returns command line for child GTest process based on the command line
  64 // of current process. |test_names| is a vector of test full names
  65 // (e.g. "A.B"), |output_file| is path to the GTest XML output file.
  66 CommandLine GetCommandLineForChildGTestProcess(
  67     const std::vector<std::string>& test_names,
  68     const base::FilePath& output_file) {
  69   CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
  70
  71   new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
  72   new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":"));
  73   new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
  74   new_cmd_line.AppendSwitch(kBraveNewTestLauncherFlag);
  75
  76   return new_cmd_line;
  77 }
  78
  79 class UnitTestLauncherDelegate : public TestLauncherDelegate {
  80  public:
  81   explicit UnitTestLauncherDelegate(size_t batch_limit)
  82       : batch_limit_(batch_limit) {
  83   }
  84
  85   virtual ~UnitTestLauncherDelegate() {
  86     DCHECK(thread_checker_.CalledOnValidThread());
  87   }
  88
  89  private:
  90   struct GTestCallbackState {
  91     TestLauncher* test_launcher;
  92     std::vector<std::string> test_names;
  93     FilePath output_file;
  94   };
  95
  96   virtual void OnTestIterationStarting() OVERRIDE {
  97     // Nothing to do.
  98   }
  99
 100   virtual std::string GetTestNameForFiltering(
 101       const testing::TestCase* test_case,
 102       const testing::TestInfo* test_info) OVERRIDE {
 103     DCHECK(thread_checker_.CalledOnValidThread());
 104
 105     return std::string(test_case->name()) + "." + test_info->name();
 106   }
 107
 108   virtual bool ShouldRunTest(const testing::TestCase* test_case,
 109                              const testing::TestInfo* test_info) OVERRIDE {
 110     DCHECK(thread_checker_.CalledOnValidThread());
 111
 112     // There is no additional logic to disable specific tests.
 113     return true;
 114   }
 115
 116   virtual size_t RunTests(TestLauncher* test_launcher,
 117                           const std::vector<std::string>& test_names) OVERRIDE {
 118     DCHECK(thread_checker_.CalledOnValidThread());
 119
 120     std::vector<std::string> batch;
 121     for (size_t i = 0; i < test_names.size(); i++) {
 122       batch.push_back(test_names[i]);
 123
 124       if (batch.size() >= batch_limit_) {
 125         RunBatch(test_launcher, batch);
 126         batch.clear();
 127       }
 128     }
 129
 130     RunBatch(test_launcher, batch);
 131
 132     return test_names.size();
 133   }
 134
 135   virtual size_t RetryTests(
 136       TestLauncher* test_launcher,
 137       const std::vector<std::string>& test_names) OVERRIDE {
 138     MessageLoop::current()->PostTask(
 139         FROM_HERE,
 140         Bind(&UnitTestLauncherDelegate::RunSerially,
 141              Unretained(this),
 142              test_launcher,
 143              test_names));
 144     return test_names.size();
 145   }
 146
 147   void RunSerially(TestLauncher* test_launcher,
 148                    const std::vector<std::string>& test_names) {
 149     if (test_names.empty())
 150       return;
 151
 152     std::vector<std::string> new_test_names(test_names);
 153     std::string test_name(new_test_names.back());
 154     new_test_names.pop_back();
 155
 156     // Create a dedicated temporary directory to store the xml result data
 157     // per run to ensure clean state and make it possible to launch multiple
 158     // processes in parallel.
 159     base::FilePath output_file;
 160     CHECK(file_util::CreateNewTempDirectory(FilePath::StringType(),
 161                                             &output_file));
 162     output_file = output_file.AppendASCII("test_results.xml");
 163
 164     std::vector<std::string> current_test_names;
 165     current_test_names.push_back(test_name);
 166     CommandLine cmd_line(
 167         GetCommandLineForChildGTestProcess(current_test_names, output_file));
 168
 169     GTestCallbackState callback_state;
 170     callback_state.test_launcher = test_launcher;
 171     callback_state.test_names = current_test_names;
 172     callback_state.output_file = output_file;
 173
 174     test_launcher->LaunchChildGTestProcess(
 175         cmd_line,
 176         std::string(),
 177         TestTimeouts::test_launcher_timeout(),
 178         Bind(&UnitTestLauncherDelegate::SerialGTestCallback,
 179              Unretained(this),
 180              callback_state,
 181              new_test_names));
 182   }
 183
 184   void RunBatch(TestLauncher* test_launcher,
 185                 const std::vector<std::string>& test_names) {
 186     DCHECK(thread_checker_.CalledOnValidThread());
 187
 188     if (test_names.empty())
 189       return;
 190
 191     // Create a dedicated temporary directory to store the xml result data
 192     // per run to ensure clean state and make it possible to launch multiple
 193     // processes in parallel.
 194     base::FilePath output_file;
 195     CHECK(file_util::CreateNewTempDirectory(FilePath::StringType(),
 196                                             &output_file));
 197     output_file = output_file.AppendASCII("test_results.xml");
 198
 199     CommandLine cmd_line(
 200         GetCommandLineForChildGTestProcess(test_names, output_file));
 201
 202     // Adjust the timeout depending on how many tests we're running
 203     // (note that e.g. the last batch of tests will be smaller).
 204     // TODO(phajdan.jr): Consider an adaptive timeout, which can change
 205     // depending on how many tests ran and how many remain.
 206     // Note: do NOT parse child's stdout to do that, it's known to be
 207     // unreliable (e.g. buffering issues can mix up the output).
 208     base::TimeDelta timeout =
 209         test_names.size() * TestTimeouts::test_launcher_timeout();
 210
 211     GTestCallbackState callback_state;
 212     callback_state.test_launcher = test_launcher;
 213     callback_state.test_names = test_names;
 214     callback_state.output_file = output_file;
 215
 216     test_launcher->LaunchChildGTestProcess(
 217         cmd_line,
 218         std::string(),
 219         timeout,
 220         Bind(&UnitTestLauncherDelegate::GTestCallback,
 221              Unretained(this),
 222              callback_state));
 223   }
 224
 225   void GTestCallback(const GTestCallbackState& callback_state,
 226                      int exit_code,
 227                      const TimeDelta& elapsed_time,
 228                      bool was_timeout,
 229                      const std::string& output) {
 230     DCHECK(thread_checker_.CalledOnValidThread());
 231     std::vector<std::string> tests_to_relaunch_after_interruption;
 232     ProcessTestResults(callback_state.test_launcher,
 233                        callback_state.test_names,
 234                        callback_state.output_file,
 235                        output,
 236                        exit_code,
 237                        was_timeout,
 238                        &tests_to_relaunch_after_interruption);
 239
 240     RunBatch(callback_state.test_launcher,
 241              tests_to_relaunch_after_interruption);
 242
 243     // The temporary file's directory is also temporary.
 244     DeleteFile(callback_state.output_file.DirName(), true);
 245   }
 246
 247   void SerialGTestCallback(const GTestCallbackState& callback_state,
 248                            const std::vector<std::string>& test_names,
 249                            int exit_code,
 250                            const TimeDelta& elapsed_time,
 251                            bool was_timeout,
 252                            const std::string& output) {
 253     DCHECK(thread_checker_.CalledOnValidThread());
 254     std::vector<std::string> tests_to_relaunch_after_interruption;
 255     bool called_any_callbacks =
 256         ProcessTestResults(callback_state.test_launcher,
 257                            callback_state.test_names,
 258                            callback_state.output_file,
 259                            output,
 260                            exit_code,
 261                            was_timeout,
 262                            &tests_to_relaunch_after_interruption);
 263
 264     // There is only one test, there cannot be other tests to relaunch
 265     // due to a crash.
 266     DCHECK(tests_to_relaunch_after_interruption.empty());
 267
 268     // There is only one test, we should have called back with its result.
 269     DCHECK(called_any_callbacks);
 270
 271     // The temporary file's directory is also temporary.
 272     DeleteFile(callback_state.output_file.DirName(), true);
 273
 274     MessageLoop::current()->PostTask(
 275         FROM_HERE,
 276         Bind(&UnitTestLauncherDelegate::RunSerially,
 277              Unretained(this),
 278              callback_state.test_launcher,
 279              test_names));
 280   }
 281
 282   static bool ProcessTestResults(
 283       TestLauncher* test_launcher,
 284       const std::vector<std::string>& test_names,
 285       const base::FilePath& output_file,
 286       const std::string& output,
 287       int exit_code,
 288       bool was_timeout,
 289       std::vector<std::string>* tests_to_relaunch_after_interruption) {
 290     std::vector<TestResult> test_results;
 291     bool crashed = false;
 292     bool have_test_results =
 293         ProcessGTestOutput(output_file, &test_results, &crashed);
 294
 295     bool called_any_callback = false;
 296
 297     if (have_test_results) {
 298       // TODO(phajdan.jr): Check for duplicates and mismatches between
 299       // the results we got from XML file and tests we intended to run.
 300       std::map<std::string, TestResult> results_map;
 301       for (size_t i = 0; i < test_results.size(); i++)
 302         results_map[test_results[i].full_name] = test_results[i];
 303
 304       bool had_interrupted_test = false;
 305
 306       for (size_t i = 0; i < test_names.size(); i++) {
 307         if (ContainsKey(results_map, test_names[i])) {
 308           TestResult test_result = results_map[test_names[i]];
 309           if (test_result.status == TestResult::TEST_CRASH) {
 310             had_interrupted_test = true;
 311
 312             if (was_timeout) {
 313               // Fix up the test status: we forcibly kill the child process
 314               // after the timeout, so from XML results it looks just like
 315               // a crash.
 316               test_result.status = TestResult::TEST_TIMEOUT;
 317             }
 318           } else if (test_result.status == TestResult::TEST_SUCCESS ||
 319                      test_result.status == TestResult::TEST_FAILURE) {
 320             // We run multiple tests in a batch with a timeout applied
 321             // to the entire batch. It is possible that with other tests
 322             // running quickly some tests take longer than the per-test timeout.
 323             // For consistent handling of tests independent of order and other
 324             // factors, mark them as timing out.
 325             if (test_result.elapsed_time >
 326                 TestTimeouts::test_launcher_timeout()) {
 327               test_result.status = TestResult::TEST_TIMEOUT;
 328             }
 329           }
 330           test_result.output_snippet =
 331               GetTestOutputSnippet(test_result, output);
 332           test_launcher->OnTestFinished(test_result);
 333           called_any_callback = true;
 334         } else if (had_interrupted_test) {
 335           tests_to_relaunch_after_interruption->push_back(test_names[i]);
 336         } else {
 337           // TODO(phajdan.jr): Explicitly pass the info that the test didn't
 338           // run for a mysterious reason.
 339           LOG(ERROR) << "no test result for " << test_names[i];
 340           TestResult test_result;
 341           test_result.full_name = test_names[i];
 342           test_result.status = TestResult::TEST_UNKNOWN;
 343           test_result.output_snippet =
 344               GetTestOutputSnippet(test_result, output);
 345           test_launcher->OnTestFinished(test_result);
 346           called_any_callback = true;
 347         }
 348       }
 349
 350       // TODO(phajdan.jr): Handle the case where processing XML output
 351       // indicates a crash but none of the test results is marked as crashing.
 352
 353       // TODO(phajdan.jr): Handle the case where the exit code is non-zero
 354       // but results file indicates that all tests passed (e.g. crash during
 355       // shutdown).
 356     } else {
 357       fprintf(stdout,
 358               "Failed to get out-of-band test success data, "
 359               "dumping full stdio below:\n%s\n",
 360               output.c_str());
 361       fflush(stdout);
 362
 363       // We do not have reliable details about test results (parsing test
 364       // stdout is known to be unreliable), apply the executable exit code
 365       // to all tests.
 366       // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
 367       // individually.
 368       for (size_t i = 0; i < test_names.size(); i++) {
 369         TestResult test_result;
 370         test_result.full_name = test_names[i];
 371         test_result.status = TestResult::TEST_UNKNOWN;
 372         test_launcher->OnTestFinished(test_result);
 373         called_any_callback = true;
 374       }
 375     }
 376
 377     return called_any_callback;
 378   }
 379
 380   ThreadChecker thread_checker_;
 381
 382   // Maximum number of tests to run in a single batch.
 383   size_t batch_limit_;
 384 };
 385
 386 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
 387   if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
 388     return true;
 389
 390   std::string switch_value =
 391       CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
 392   if (!StringToInt(switch_value, result) || *result < 1) {
 393     LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
 394     return false;
 395   }
 396
 397   return true;
 398 }
 399
 400 }  // namespace
 401
 402 int LaunchUnitTests(int argc,
 403                     char** argv,
 404                     const RunTestSuiteCallback& run_test_suite) {
 405   CommandLine::Init(argc, argv);
 406   if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
 407       CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) ||
 408       !CommandLine::ForCurrentProcess()->HasSwitch(kBraveNewTestLauncherFlag)) {
 409     return run_test_suite.Run();
 410   }
 411
 412   if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
 413     PrintUsage();
 414     return 0;
 415   }
 416
 417   base::TimeTicks start_time(base::TimeTicks::Now());
 418
 419   testing::InitGoogleTest(&argc, argv);
 420   TestTimeouts::Initialize();
 421
 422   int jobs = SysInfo::NumberOfProcessors();
 423   if (!GetSwitchValueAsInt(switches::kTestLauncherJobs, &jobs))
 424     return 1;
 425
 426   int batch_limit = kDefaultTestBatchLimit;
 427   if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
 428     return 1;
 429
 430   fprintf(stdout,
 431           "Starting tests (using %d parallel jobs)...\n"
 432           "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
 433           "own process. For debugging a test inside a debugger, use the\n"
 434           "--gtest_filter=<your_test_name> flag along with\n"
 435           "--single-process-tests.\n", jobs);
 436   fflush(stdout);
 437
 438   MessageLoopForIO message_loop;
 439
 440   base::UnitTestLauncherDelegate delegate(batch_limit);
 441   base::TestLauncher launcher(&delegate, jobs);
 442   bool success = launcher.Run(argc, argv);
 443
 444   fprintf(stdout,
 445           "Tests took %" PRId64 " seconds.\n",
 446           (base::TimeTicks::Now() - start_time).InSeconds());
 447   fflush(stdout);
 448
 449   return (success ? 0 : 1);
 450 }
 451
 452 }  // namespace base