Add test flags feature to perf runner.

TBR=svenpanne@chromium.org
NOTRY=true

Review URL: https://codereview.chromium.org/722023006

Cr-Commit-Position: refs/heads/master@{#25559}

diff --git a/tools/run_perf.py b/tools/run_perf.py
index da139d7a6b871a43f71714e5785acf190755965e..3d388f6b4d899e767bdf22e7dde852c7afb98e04 100755 (executable)
--- a/tools/run_perf.py
+++ b/tools/run_perf.py
@@ -15,6 +15,7 @@ The suite json format is expected to be:
   "archs": [<architecture name for which this suite is run>, ...],
   "binary": <name of binary to run, default "d8">,
   "flags": [<flag to d8>, ...],
+  "test_flags": [<flag to the test file>, ...],
   "run_count": <how often will this suite run (optional)>,
   "run_count_XXX": <how often will this suite run for arch XXX (optional)>,
   "resources": [<js file to be loaded before main>, ...]
@@ -54,6 +55,7 @@ Full example (suite with one runner):
 {
   "path": ["."],
   "flags": ["--expose-gc"],
+  "test_flags": ["5"],
   "archs": ["ia32", "x64"],
   "run_count": 5,
   "run_count_ia32": 3,
@@ -89,6 +91,8 @@ Full example (suite with several runners):
 }
 
 Path pieces are concatenated. D8 is always run with the suite's path as cwd.
+
+The test flags are passed to the js test file after '--'.
 """
 
 from collections import OrderedDict
@@ -171,6 +175,7 @@ class DefaultSentinel(Node):
     self.path = []
     self.graphs = []
     self.flags = []
+    self.test_flags = []
     self.resources = []
     self.results_regexp = None
     self.stddev_regexp = None
@@ -190,12 +195,14 @@ class Graph(Node):
     assert isinstance(suite.get("path", []), list)
     assert isinstance(suite["name"], basestring)
     assert isinstance(suite.get("flags", []), list)
+    assert isinstance(suite.get("test_flags", []), list)
     assert isinstance(suite.get("resources", []), list)
 
     # Accumulated values.
     self.path = parent.path[:] + suite.get("path", [])
     self.graphs = parent.graphs[:] + [suite["name"]]
     self.flags = parent.flags[:] + suite.get("flags", [])
+    self.test_flags = parent.test_flags[:] + suite.get("test_flags", [])
     self.resources = parent.resources[:] + suite.get("resources", [])
 
     # Descrete values (with parent defaults).
@@ -282,11 +289,13 @@ class Runnable(Graph):
 
   def GetCommand(self, shell_dir):
     # TODO(machenbach): This requires +.exe if run on windows.
+    suffix = ["--"] + self.test_flags if self.test_flags else []
     return (
       [os.path.join(shell_dir, self.binary)] +
       self.flags +
       self.resources +
-      [self.main]
+      [self.main] +
+      suffix
     )
 
   def Run(self, runner):

diff --git a/tools/unittests/run_perf_test.py b/tools/unittests/run_perf_test.py
index 0c3d6f60777defb7cfe91000503995f2efc3c670..0f84a73aecc25a95c2dd35bc154fa8fc573f5819 100644 (file)
--- a/tools/unittests/run_perf_test.py
+++ b/tools/unittests/run_perf_test.py
@@ -179,6 +179,20 @@ class PerfTest(unittest.TestCase):
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 
+  def testOneRunWithTestFlags(self):
+    test_input = dict(V8_JSON)
+    test_input["test_flags"] = ["2", "test_name"]
+    self._WriteTestInput(test_input)
+    self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567"])
+    self.assertEquals(0, self._CallMain())
+    self._VerifyResults("test", "score", [
+      {"name": "Richards", "results": ["1.234"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+    ])
+    self._VerifyErrors([])
+    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js",
+                     "--", "2", "test_name")
+
   def testTwoRuns_Units_SuiteName(self):
     test_input = dict(V8_JSON)
     test_input["run_count"] = 2