Teach benchmark runner to understand generic traces.

BUG=406405
LOG=n
TEST=python -m unittest run_benchmarks_test
R=bmeurer@chromium.org

Review URL: https://codereview.chromium.org/502473002

git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@23329 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

diff --git a/tools/run_benchmarks.py b/tools/run_benchmarks.py
index d6e9145dacea3bdf73062d5fa88954d5b807bdd8..b74f3588f360e2ed869ab76cbf2fcb543df69559 100755 (executable)
--- a/tools/run_benchmarks.py
+++ b/tools/run_benchmarks.py
@@ -113,6 +113,8 @@ SUPPORTED_ARCHS = ["android_arm",
                    "x64",
                    "arm64"]
 
+GENERIC_RESULTS_RE = re.compile(
+    r"^Trace\(([^\)]+)\), Result\(([^\)]+)\), StdDev\(([^\)]+)\)$")
 
 class Results(object):
   """Place holder for result traces."""
@@ -249,7 +251,7 @@ class Runnable(Graph):
   """
   @property
   def main(self):
-    return self._suite["main"]
+    return self._suite.get("main", "")
 
   def ChangeCWD(self, suite_path):
     """Changes the cwd to to path defined in the current graph.
@@ -289,6 +291,33 @@ class RunnableTrace(Trace, Runnable):
     return self.GetResults()
 
 
+class RunnableGeneric(Runnable):
+  """Represents a runnable benchmark suite definition with generic traces."""
+  def __init__(self, suite, parent, arch):
+    super(RunnableGeneric, self).__init__(suite, parent, arch)
+
+  def Run(self, runner):
+    """Iterates over several runs and handles the output."""
+    traces = {}
+    for stdout in runner():
+      for line in stdout.strip().splitlines():
+        match = GENERIC_RESULTS_RE.match(line)
+        if match:
+          trace = match.group(1)
+          result = match.group(2)
+          stddev = match.group(3)
+          trace_result = traces.setdefault(trace, Results([{
+            "graphs": self.graphs + [trace],
+            "units": self.units,
+            "results": [],
+            "stddev": "",
+          }], []))
+          trace_result.traces[0]["results"].append(result)
+          trace_result.traces[0]["stddev"] = stddev
+
+    return reduce(lambda r, t: r + t, traces.itervalues(), Results())
+
+
 def MakeGraph(suite, arch, parent):
   """Factory method for making graph objects."""
   if isinstance(parent, Runnable):
@@ -302,6 +331,10 @@ def MakeGraph(suite, arch, parent):
     else:
       # This graph has no subbenchmarks, it's a leaf.
       return RunnableTrace(suite, parent, arch)
+  elif suite.get("generic"):
+    # This is a generic suite definition. It is either a runnable executable
+    # or has a main js file.
+    return RunnableGeneric(suite, parent, arch)
   elif suite.get("benchmarks"):
     # This is neither a leaf nor a runnable.
     return Graph(suite, parent, arch)

diff --git a/tools/unittests/run_benchmarks_test.py b/tools/unittests/run_benchmarks_test.py
index 37a816e760d2a59d5107f86a54efa2eeddfe2828..054522091ed1738143ac30cb686b808eed0af2aa 100644 (file)
--- a/tools/unittests/run_benchmarks_test.py
+++ b/tools/unittests/run_benchmarks_test.py
@@ -68,6 +68,15 @@ V8_NESTED_SUITES_JSON = {
   ]
 }
 
+V8_GENERIC_JSON = {
+  "path": ["."],
+  "binary": "cc",
+  "flags": ["--flag"],
+  "generic": True,
+  "run_count": 1,
+  "units": "ms",
+}
+
 Output = namedtuple("Output", "stdout, stderr")
 
 class BenchmarksTest(unittest.TestCase):
@@ -295,3 +304,17 @@ class BenchmarksTest(unittest.TestCase):
     self._VerifyErrors(
         ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards."])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
+
+  def testOneRunGeneric(self):
+    test_input = dict(V8_GENERIC_JSON)
+    self._WriteTestInput(test_input)
+    self._MockCommand(["."], [
+      "Trace(Test1), Result(1.234), StdDev(0.23)\n"
+      "Trace(Test2), Result(10657567), StdDev(106)\n"])
+    self.assertEquals(0, self._CallMain())
+    self._VerifyResults("test", "ms", [
+      {"name": "Test1", "results": ["1.234"], "stddev": "0.23"},
+      {"name": "Test2", "results": ["10657567"], "stddev": "106"},
+    ])
+    self._VerifyErrors([])
+    self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")