src/v8/tools/unittests/run_perf_test.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 the V8 project authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 from collections import namedtuple
   7 import coverage
   8 import json
   9 from mock import DEFAULT
  10 from mock import MagicMock
  11 import os
  12 from os import path, sys
  13 import shutil
  14 import tempfile
  15 import unittest
  16
  17 # Requires python-coverage and python-mock. Native python coverage
  18 # version >= 3.7.1 should be installed to get the best speed.
  19
  20 TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf")
  21
  22 V8_JSON = {
  23   "path": ["."],
  24   "binary": "d7",
  25   "flags": ["--flag"],
  26   "main": "run.js",
  27   "run_count": 1,
  28   "results_regexp": "^%s: (.+)$",
  29   "tests": [
  30     {"name": "Richards"},
  31     {"name": "DeltaBlue"},
  32   ]
  33 }
  34
  35 V8_NESTED_SUITES_JSON = {
  36   "path": ["."],
  37   "flags": ["--flag"],
  38   "run_count": 1,
  39   "units": "score",
  40   "tests": [
  41     {"name": "Richards",
  42      "path": ["richards"],
  43      "binary": "d7",
  44      "main": "run.js",
  45      "resources": ["file1.js", "file2.js"],
  46      "run_count": 2,
  47      "results_regexp": "^Richards: (.+)$"},
  48     {"name": "Sub",
  49      "path": ["sub"],
  50      "tests": [
  51        {"name": "Leaf",
  52         "path": ["leaf"],
  53         "run_count_x64": 3,
  54         "units": "ms",
  55         "main": "run.js",
  56         "results_regexp": "^Simple: (.+) ms.$"},
  57      ]
  58     },
  59     {"name": "DeltaBlue",
  60      "path": ["delta_blue"],
  61      "main": "run.js",
  62      "flags": ["--flag2"],
  63      "results_regexp": "^DeltaBlue: (.+)$"},
  64     {"name": "ShouldntRun",
  65      "path": ["."],
  66      "archs": ["arm"],
  67      "main": "run.js"},
  68   ]
  69 }
  70
  71 V8_GENERIC_JSON = {
  72   "path": ["."],
  73   "binary": "cc",
  74   "flags": ["--flag"],
  75   "generic": True,
  76   "run_count": 1,
  77   "units": "ms",
  78 }
  79
  80 Output = namedtuple("Output", "stdout, stderr, timed_out")
  81
  82 class PerfTest(unittest.TestCase):
  83   @classmethod
  84   def setUpClass(cls):
  85     cls.base = path.dirname(path.dirname(path.abspath(__file__)))
  86     sys.path.append(cls.base)
  87     cls._cov = coverage.coverage(
  88         include=([os.path.join(cls.base, "run_perf.py")]))
  89     cls._cov.start()
  90     import run_perf
  91     from testrunner.local import commands
  92     global commands
  93     global run_perf
  94
  95   @classmethod
  96   def tearDownClass(cls):
  97     cls._cov.stop()
  98     print ""
  99     print cls._cov.report()
 100
 101   def setUp(self):
 102     self.maxDiff = None
 103     if path.exists(TEST_WORKSPACE):
 104       shutil.rmtree(TEST_WORKSPACE)
 105     os.makedirs(TEST_WORKSPACE)
 106
 107   def tearDown(self):
 108     if path.exists(TEST_WORKSPACE):
 109       shutil.rmtree(TEST_WORKSPACE)
 110
 111   def _WriteTestInput(self, json_content):
 112     self._test_input = path.join(TEST_WORKSPACE, "test.json")
 113     with open(self._test_input, "w") as f:
 114       f.write(json.dumps(json_content))
 115
 116   def _MockCommand(self, *args, **kwargs):
 117     # Fake output for each test run.
 118     test_outputs = [Output(stdout=arg,
 119                            stderr=None,
 120                            timed_out=kwargs.get("timed_out", False))
 121                     for arg in args[1]]
 122     def execute(*args, **kwargs):
 123       return test_outputs.pop()
 124     commands.Execute = MagicMock(side_effect=execute)
 125
 126     # Check that d8 is called from the correct cwd for each test run.
 127     dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]]
 128     def chdir(*args, **kwargs):
 129       self.assertEquals(dirs.pop(), args[0])
 130     os.chdir = MagicMock(side_effect=chdir)
 131
 132   def _CallMain(self, *args):
 133     self._test_output = path.join(TEST_WORKSPACE, "results.json")
 134     all_args=[
 135       "--json-test-results",
 136       self._test_output,
 137       self._test_input,
 138     ]
 139     all_args += args
 140     return run_perf.Main(all_args)
 141
 142   def _LoadResults(self):
 143     with open(self._test_output) as f:
 144       return json.load(f)
 145
 146   def _VerifyResults(self, suite, units, traces):
 147     self.assertEquals([
 148       {"units": units,
 149        "graphs": [suite, trace["name"]],
 150        "results": trace["results"],
 151        "stddev": trace["stddev"]} for trace in traces],
 152       self._LoadResults()["traces"])
 153
 154   def _VerifyErrors(self, errors):
 155     self.assertEquals(errors, self._LoadResults()["errors"])
 156
 157   def _VerifyMock(self, binary, *args, **kwargs):
 158     arg = [path.join(path.dirname(self.base), binary)]
 159     arg += args
 160     commands.Execute.assert_called_with(
 161         arg, timeout=kwargs.get("timeout", 60))
 162
 163   def _VerifyMockMultiple(self, *args, **kwargs):
 164     expected = []
 165     for arg in args:
 166       a = [path.join(path.dirname(self.base), arg[0])]
 167       a += arg[1:]
 168       expected.append(((a,), {"timeout": kwargs.get("timeout", 60)}))
 169     self.assertEquals(expected, commands.Execute.call_args_list)
 170
 171   def testOneRun(self):
 172     self._WriteTestInput(V8_JSON)
 173     self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"])
 174     self.assertEquals(0, self._CallMain())
 175     self._VerifyResults("test", "score", [
 176       {"name": "Richards", "results": ["1.234"], "stddev": ""},
 177       {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
 178     ])
 179     self._VerifyErrors([])
 180     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 181
 182   def testTwoRuns_Units_SuiteName(self):
 183     test_input = dict(V8_JSON)
 184     test_input["run_count"] = 2
 185     test_input["name"] = "v8"
 186     test_input["units"] = "ms"
 187     self._WriteTestInput(test_input)
 188     self._MockCommand([".", "."],
 189                       ["Richards: 100\nDeltaBlue: 200\n",
 190                        "Richards: 50\nDeltaBlue: 300\n"])
 191     self.assertEquals(0, self._CallMain())
 192     self._VerifyResults("v8", "ms", [
 193       {"name": "Richards", "results": ["50", "100"], "stddev": ""},
 194       {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
 195     ])
 196     self._VerifyErrors([])
 197     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 198
 199   def testTwoRuns_SubRegexp(self):
 200     test_input = dict(V8_JSON)
 201     test_input["run_count"] = 2
 202     del test_input["results_regexp"]
 203     test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$"
 204     test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$"
 205     self._WriteTestInput(test_input)
 206     self._MockCommand([".", "."],
 207                       ["Richards: 100\nDeltaBlue: 200\n",
 208                        "Richards: 50\nDeltaBlue: 300\n"])
 209     self.assertEquals(0, self._CallMain())
 210     self._VerifyResults("test", "score", [
 211       {"name": "Richards", "results": ["50", "100"], "stddev": ""},
 212       {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
 213     ])
 214     self._VerifyErrors([])
 215     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 216
 217   def testNestedSuite(self):
 218     self._WriteTestInput(V8_NESTED_SUITES_JSON)
 219     self._MockCommand(["delta_blue", "sub/leaf", "richards"],
 220                       ["DeltaBlue: 200\n",
 221                        "Simple: 1 ms.\n",
 222                        "Simple: 2 ms.\n",
 223                        "Simple: 3 ms.\n",
 224                        "Richards: 100\n",
 225                        "Richards: 50\n"])
 226     self.assertEquals(0, self._CallMain())
 227     self.assertEquals([
 228       {"units": "score",
 229        "graphs": ["test", "Richards"],
 230        "results": ["50", "100"],
 231        "stddev": ""},
 232       {"units": "ms",
 233        "graphs": ["test", "Sub", "Leaf"],
 234        "results": ["3", "2", "1"],
 235        "stddev": ""},
 236       {"units": "score",
 237        "graphs": ["test", "DeltaBlue"],
 238        "results": ["200"],
 239        "stddev": ""},
 240       ], self._LoadResults()["traces"])
 241     self._VerifyErrors([])
 242     self._VerifyMockMultiple(
 243         (path.join("out", "x64.release", "d7"), "--flag", "file1.js",
 244          "file2.js", "run.js"),
 245         (path.join("out", "x64.release", "d7"), "--flag", "file1.js",
 246          "file2.js", "run.js"),
 247         (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
 248         (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
 249         (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
 250         (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js"))
 251
 252   def testOneRunStdDevRegExp(self):
 253     test_input = dict(V8_JSON)
 254     test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
 255     self._WriteTestInput(test_input)
 256     self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n"
 257                               "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"])
 258     self.assertEquals(0, self._CallMain())
 259     self._VerifyResults("test", "score", [
 260       {"name": "Richards", "results": ["1.234"], "stddev": "0.23"},
 261       {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"},
 262     ])
 263     self._VerifyErrors([])
 264     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 265
 266   def testTwoRunsStdDevRegExp(self):
 267     test_input = dict(V8_JSON)
 268     test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
 269     test_input["run_count"] = 2
 270     self._WriteTestInput(test_input)
 271     self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n"
 272                               "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n",
 273                               "Richards: 2\nRichards-stddev: 0.5\n"
 274                               "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"])
 275     self.assertEquals(1, self._CallMain())
 276     self._VerifyResults("test", "score", [
 277       {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"},
 278       {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"},
 279     ])
 280     self._VerifyErrors(
 281         ["Test Richards should only run once since a stddev is provided "
 282          "by the test.",
 283          "Test DeltaBlue should only run once since a stddev is provided "
 284          "by the test.",
 285          "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test "
 286          "DeltaBlue."])
 287     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 288
 289   def testBuildbot(self):
 290     self._WriteTestInput(V8_JSON)
 291     self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
 292     self.assertEquals(0, self._CallMain("--buildbot"))
 293     self._VerifyResults("test", "score", [
 294       {"name": "Richards", "results": ["1.234"], "stddev": ""},
 295       {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
 296     ])
 297     self._VerifyErrors([])
 298     self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
 299
 300   def testBuildbotWithTotal(self):
 301     test_input = dict(V8_JSON)
 302     test_input["total"] = True
 303     self._WriteTestInput(test_input)
 304     self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
 305     self.assertEquals(0, self._CallMain("--buildbot"))
 306     self._VerifyResults("test", "score", [
 307       {"name": "Richards", "results": ["1.234"], "stddev": ""},
 308       {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
 309       {"name": "Total", "results": ["3626.49109719"], "stddev": ""},
 310     ])
 311     self._VerifyErrors([])
 312     self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
 313
 314   def testBuildbotWithTotalAndErrors(self):
 315     test_input = dict(V8_JSON)
 316     test_input["total"] = True
 317     self._WriteTestInput(test_input)
 318     self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
 319     self.assertEquals(1, self._CallMain("--buildbot"))
 320     self._VerifyResults("test", "score", [
 321       {"name": "Richards", "results": [], "stddev": ""},
 322       {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
 323     ])
 324     self._VerifyErrors(
 325         ["Regexp \"^Richards: (.+)$\" didn't match for test Richards.",
 326          "Not all traces have the same number of results."])
 327     self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
 328
 329   def testRegexpNoMatch(self):
 330     self._WriteTestInput(V8_JSON)
 331     self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
 332     self.assertEquals(1, self._CallMain())
 333     self._VerifyResults("test", "score", [
 334       {"name": "Richards", "results": [], "stddev": ""},
 335       {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
 336     ])
 337     self._VerifyErrors(
 338         ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."])
 339     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 340
 341   def testOneRunGeneric(self):
 342     test_input = dict(V8_GENERIC_JSON)
 343     self._WriteTestInput(test_input)
 344     self._MockCommand(["."], [
 345       "RESULT Infra: Constant1= 11 count\n"
 346       "RESULT Infra: Constant2= [10,5,10,15] count\n"
 347       "RESULT Infra: Constant3= {12,1.2} count\n"])
 348     self.assertEquals(0, self._CallMain())
 349     self.assertEquals([
 350       {"units": "count",
 351        "graphs": ["test", "Infra", "Constant1"],
 352        "results": ["11"],
 353        "stddev": ""},
 354       {"units": "count",
 355        "graphs": ["test", "Infra", "Constant2"],
 356        "results": ["10", "5", "10", "15"],
 357        "stddev": ""},
 358       {"units": "count",
 359        "graphs": ["test", "Infra", "Constant3"],
 360        "results": ["12"],
 361        "stddev": "1.2"},
 362       ], self._LoadResults()["traces"])
 363     self._VerifyErrors([])
 364     self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")
 365
 366   def testOneRunTimingOut(self):
 367     test_input = dict(V8_JSON)
 368     test_input["timeout"] = 70
 369     self._WriteTestInput(test_input)
 370     self._MockCommand(["."], [""], timed_out=True)
 371     self.assertEquals(1, self._CallMain())
 372     self._VerifyResults("test", "score", [
 373       {"name": "Richards", "results": [], "stddev": ""},
 374       {"name": "DeltaBlue", "results": [], "stddev": ""},
 375     ])
 376     self._VerifyErrors([
 377       "Regexp \"^Richards: (.+)$\" didn't match for test Richards.",
 378       "Regexp \"^DeltaBlue: (.+)$\" didn't match for test DeltaBlue.",
 379     ])
 380     self._VerifyMock(
 381         path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70)