[test] Return variant and random seed on failures.

BUG=chromium:511215
LOG=n

Review URL: https://codereview.chromium.org/1276853002

Cr-Commit-Position: refs/heads/master@{#30057}

diff --git a/tools/run-deopt-fuzzer.py b/tools/run-deopt-fuzzer.py
index 4e361ae..7fbf402 100755 (executable)
--- a/tools/run-deopt-fuzzer.py
+++ b/tools/run-deopt-fuzzer.py
@@ -417,7 +417,7 @@ def Execute(arch, mode, args, options, suites, workspace):
     test_backup[s] = s.tests
     analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT,
                       "--print-deopt-stress"]
-    s.tests = [ t.CopyAddingFlags(analysis_flags) for t in s.tests ]
+    s.tests = [ t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests ]
     num_tests += len(s.tests)
     for t in s.tests:
       t.id = test_id
@@ -464,7 +464,7 @@ def Execute(arch, mode, args, options, suites, workspace):
         print "%s %s" % (t.path, distribution)
       for i in distribution:
         fuzzing_flags = ["--deopt-every-n-times", "%d" % i]
-        s.tests.append(t.CopyAddingFlags(fuzzing_flags))
+        s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags))
     num_tests += len(s.tests)
     for t in s.tests:
       t.id = test_id

diff --git a/tools/run-tests.py b/tools/run-tests.py
index 934d9d7..a9a81f8 100755 (executable)
--- a/tools/run-tests.py
+++ b/tools/run-tests.py
@@ -622,7 +622,7 @@ def Execute(arch, mode, args, options, suites, workspace):
       verbose.PrintTestSource(s.tests)
       continue
     variant_gen = s.CreateVariantGenerator(VARIANTS)
-    variant_tests = [ t.CopyAddingFlags(flags)
+    variant_tests = [ t.CopyAddingFlags(v, flags)
                       for t in s.tests
                       for v in variant_gen.FilterVariantsByTest(t)
                       for flags in variant_gen.GetFlagSets(t, v) ]
@@ -638,7 +638,7 @@ def Execute(arch, mode, args, options, suites, workspace):
           else:
             yield ["--random-seed=%d" % RandomSeed()]
       s.tests = [
-        t.CopyAddingFlags(flags)
+        t.CopyAddingFlags(t.variant, flags)
         for t in variant_tests
         for flags in iter_seed_flags()
       ]
@@ -663,7 +663,8 @@ def Execute(arch, mode, args, options, suites, workspace):
         options.junitout, options.junittestsuite))
   if options.json_test_results:
     progress_indicator.Register(progress.JsonTestProgressIndicator(
-        options.json_test_results, arch, MODES[mode]["execution_mode"]))
+        options.json_test_results, arch, MODES[mode]["execution_mode"],
+        ctx.random_seed))
 
   run_networked = not options.no_network
   if not run_networked:

diff --git a/tools/testrunner/local/progress.py b/tools/testrunner/local/progress.py
index 469d64b..60ec635 100644 (file)
--- a/tools/testrunner/local/progress.py
+++ b/tools/testrunner/local/progress.py
@@ -313,10 +313,11 @@ class JUnitTestProgressIndicator(ProgressIndicator):
 
 class JsonTestProgressIndicator(ProgressIndicator):
 
-  def __init__(self, json_test_results, arch, mode):
+  def __init__(self, json_test_results, arch, mode, random_seed):
     self.json_test_results = json_test_results
     self.arch = arch
     self.mode = mode
+    self.random_seed = random_seed
     self.results = []
     self.tests = []
 
@@ -370,6 +371,11 @@ class JsonTestProgressIndicator(ProgressIndicator):
       "result": test.suite.GetOutcome(test),
       "expected": list(test.outcomes or ["PASS"]),
       "duration": test.duration,
+
+      # TODO(machenbach): This stores only the global random seed from the
+      # context and not possible overrides when using random-seed stress.
+      "random_seed": self.random_seed,
+      "variant": test.variant,
     })
 
 

diff --git a/tools/testrunner/objects/testcase.py b/tools/testrunner/objects/testcase.py
index c7b445e..0ab0636 100644 (file)
--- a/tools/testrunner/objects/testcase.py
+++ b/tools/testrunner/objects/testcase.py
@@ -29,10 +29,12 @@
 from . import output
 
 class TestCase(object):
-  def __init__(self, suite, path, flags=None, dependency=None):
+  def __init__(self, suite, path, variant='default', flags=None,
+               dependency=None):
     self.suite = suite        # TestSuite object
     self.path = path          # string, e.g. 'div-mod', 'test-api/foo'
     self.flags = flags or []  # list of strings, flags specific to this test
+    self.variant = variant    # name of the used testing variant
     self.dependency = dependency  # |path| for testcase that must be run first
     self.outcomes = set([])
     self.output = None
@@ -40,8 +42,9 @@ class TestCase(object):
     self.duration = None  # assigned during execution
     self.run = 1  # The nth time this test is executed.
 
-  def CopyAddingFlags(self, flags):
-    copy = TestCase(self.suite, self.path, self.flags + flags, self.dependency)
+  def CopyAddingFlags(self, variant, flags):
+    copy = TestCase(self.suite, self.path, variant, self.flags + flags,
+                    self.dependency)
     copy.outcomes = self.outcomes
     return copy
 
@@ -51,16 +54,16 @@ class TestCase(object):
     and returns them as a JSON serializable object.
     """
     assert self.id is not None
-    return [self.suitename(), self.path, self.flags,
+    return [self.suitename(), self.path, self.variant, self.flags,
             self.dependency, list(self.outcomes or []), self.id]
 
   @staticmethod
   def UnpackTask(task):
     """Creates a new TestCase object based on packed task data."""
     # For the order of the fields, refer to PackTask() above.
-    test = TestCase(str(task[0]), task[1], task[2], task[3])
-    test.outcomes = set(task[4])
-    test.id = task[5]
+    test = TestCase(str(task[0]), task[1], task[2], task[3], task[4])
+    test.outcomes = set(task[5])
+    test.id = task[6]
     test.run = 1
     return test