Make perf test runner more robust.

Convert all results to floats and log errors on failing
conversions. Otherwise these results bubble up into
buildbot.

TBR=tandrii@chromium.org
BUG=chromium:374740
LOG=n
NOTRY=true

Review URL: https://codereview.chromium.org/811483008

Cr-Commit-Position: refs/heads/master@{#25913}

diff --git a/tools/run_perf.py b/tools/run_perf.py
index a4fc8b3..14d67c5 100755 (executable)
--- a/tools/run_perf.py
+++ b/tools/run_perf.py
@@ -265,8 +265,11 @@ class Trace(Graph):
 
   def ConsumeOutput(self, stdout):
     try:
-      self.results.append(
-          re.search(self.results_regexp, stdout, re.M).group(1))
+      result = re.search(self.results_regexp, stdout, re.M).group(1)
+      self.results.append(str(float(result)))
+    except ValueError:
+      self.errors.append("Regexp \"%s\" returned a non-numeric for test %s."
+                         % (self.results_regexp, self.graphs[-1]))
     except:
       self.errors.append("Regexp \"%s\" didn't match for test %s."
                          % (self.results_regexp, self.graphs[-1]))
@@ -374,6 +377,7 @@ class RunnableGeneric(Runnable):
           units = match.group(4)
           match_stddev = RESULT_STDDEV_RE.match(body)
           match_list = RESULT_LIST_RE.match(body)
+          errors = []
           if match_stddev:
             result, stddev = map(str.strip, match_stddev.group(1).split(","))
             results = [result]
@@ -382,12 +386,19 @@ class RunnableGeneric(Runnable):
           else:
             results = [body.strip()]
 
+          try:
+            results = map(lambda r: str(float(r)), results)
+          except ValueError:
+            results = []
+            errors = ["Found non-numeric in %s" %
+                      "/".join(self.graphs + [graph, trace])]
+
           trace_result = traces.setdefault(trace, Results([{
             "graphs": self.graphs + [graph, trace],
             "units": (units or self.units).strip(),
             "results": [],
             "stddev": "",
-          }], []))
+          }], errors))
           trace_result.traces[0]["results"].extend(results)
           trace_result.traces[0]["stddev"] = stddev
 

diff --git a/tools/unittests/run_perf_test.py b/tools/unittests/run_perf_test.py
index 5ac0789..f9ea0c0 100644 (file)
--- a/tools/unittests/run_perf_test.py
+++ b/tools/unittests/run_perf_test.py
@@ -174,7 +174,7 @@ class PerfTest(unittest.TestCase):
     self.assertEquals(0, self._CallMain())
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["1.234"], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
     ])
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
@@ -187,7 +187,7 @@ class PerfTest(unittest.TestCase):
     self.assertEquals(0, self._CallMain())
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["1.234"], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
     ])
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js",
@@ -204,8 +204,8 @@ class PerfTest(unittest.TestCase):
                        "Richards: 50\nDeltaBlue: 300\n"])
     self.assertEquals(0, self._CallMain())
     self._VerifyResults("v8", "ms", [
-      {"name": "Richards", "results": ["50", "100"], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
+      {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""},
     ])
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
@@ -222,8 +222,8 @@ class PerfTest(unittest.TestCase):
                        "Richards: 50\nDeltaBlue: 300\n"])
     self.assertEquals(0, self._CallMain())
     self._VerifyResults("test", "score", [
-      {"name": "Richards", "results": ["50", "100"], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
+      {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""},
     ])
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
@@ -241,15 +241,15 @@ class PerfTest(unittest.TestCase):
     self.assertEquals([
       {"units": "score",
        "graphs": ["test", "Richards"],
-       "results": ["50", "100"],
+       "results": ["50.0", "100.0"],
        "stddev": ""},
       {"units": "ms",
        "graphs": ["test", "Sub", "Leaf"],
-       "results": ["3", "2", "1"],
+       "results": ["3.0", "2.0", "1.0"],
        "stddev": ""},
       {"units": "score",
        "graphs": ["test", "DeltaBlue"],
-       "results": ["200"],
+       "results": ["200.0"],
        "stddev": ""},
       ], self._LoadResults()["traces"])
     self._VerifyErrors([])
@@ -270,7 +270,7 @@ class PerfTest(unittest.TestCase):
     self.assertEquals(0, self._CallMain())
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["1.234"], "stddev": "0.23"},
-      {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"},
+      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": "106"},
     ])
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
@@ -286,8 +286,8 @@ class PerfTest(unittest.TestCase):
                               "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"])
     self.assertEquals(1, self._CallMain())
     self._VerifyResults("test", "score", [
-      {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"},
-      {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"},
+      {"name": "Richards", "results": ["2.0", "3.0"], "stddev": "0.7"},
+      {"name": "DeltaBlue", "results": ["5.0", "6.0"], "stddev": "0.8"},
     ])
     self._VerifyErrors(
         ["Test Richards should only run once since a stddev is provided "
@@ -304,7 +304,7 @@ class PerfTest(unittest.TestCase):
     self.assertEquals(0, self._CallMain("--buildbot"))
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["1.234"], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
     ])
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
@@ -317,7 +317,7 @@ class PerfTest(unittest.TestCase):
     self.assertEquals(0, self._CallMain("--buildbot"))
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["1.234"], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
       {"name": "Total", "results": ["3626.49109719"], "stddev": ""},
     ])
     self._VerifyErrors([])
@@ -327,14 +327,15 @@ class PerfTest(unittest.TestCase):
     test_input = dict(V8_JSON)
     test_input["total"] = True
     self._WriteTestInput(test_input)
-    self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
+    self._MockCommand(["."], ["x\nRichards: bla\nDeltaBlue: 10657567\ny\n"])
     self.assertEquals(1, self._CallMain("--buildbot"))
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": [], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
     ])
     self._VerifyErrors(
-        ["Regexp \"^Richards: (.+)$\" didn't match for test Richards.",
+        ["Regexp \"^Richards: (.+)$\" "
+         "returned a non-numeric for test Richards.",
          "Not all traces have the same number of results."])
     self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
 
@@ -344,7 +345,7 @@ class PerfTest(unittest.TestCase):
     self.assertEquals(1, self._CallMain())
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": [], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
     ])
     self._VerifyErrors(
         ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."])
@@ -356,23 +357,28 @@ class PerfTest(unittest.TestCase):
     self._MockCommand(["."], [
       "RESULT Infra: Constant1= 11 count\n"
       "RESULT Infra: Constant2= [10,5,10,15] count\n"
-      "RESULT Infra: Constant3= {12,1.2} count\n"])
-    self.assertEquals(0, self._CallMain())
+      "RESULT Infra: Constant3= {12,1.2} count\n"
+      "RESULT Infra: Constant4= [10,5,error,15] count\n"])
+    self.assertEquals(1, self._CallMain())
     self.assertEquals([
       {"units": "count",
        "graphs": ["test", "Infra", "Constant1"],
-       "results": ["11"],
+       "results": ["11.0"],
        "stddev": ""},
       {"units": "count",
        "graphs": ["test", "Infra", "Constant2"],
-       "results": ["10", "5", "10", "15"],
+       "results": ["10.0", "5.0", "10.0", "15.0"],
        "stddev": ""},
       {"units": "count",
        "graphs": ["test", "Infra", "Constant3"],
-       "results": ["12"],
+       "results": ["12.0"],
        "stddev": "1.2"},
+      {"units": "count",
+       "graphs": ["test", "Infra", "Constant4"],
+       "results": [],
+       "stddev": ""},
       ], self._LoadResults()["traces"])
-    self._VerifyErrors([])
+    self._VerifyErrors(["Found non-numeric in test/Infra/Constant4"])
     self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")
 
   def testOneRunTimingOut(self):
@@ -408,5 +414,5 @@ class PerfTest(unittest.TestCase):
                           "--arch", "android_arm"))
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["1.234"], "stddev": ""},
-      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
     ])