/build/Release
/obj/
/out/
+/test/cctest/cctest.status2
/test/es5conform/data
+/test/messages/messages.status2
+/test/mjsunit/mjsunit.status2
+/test/mozilla/CHECKED_OUT_VERSION
/test/mozilla/data
+/test/mozilla/downloaded_*
+/test/mozilla/mozilla.status2
+/test/preparser/preparser.status2
/test/sputnik/sputniktests
/test/test262/data
+/test/test262/test262-*
+/test/test262/test262.status2
/third_party
/tools/jsfunfuzz
/tools/jsfunfuzz.zip
import os
from os.path import join, split
+def GetSuite(name, root):
+ # Not implemented.
+ return None
+
+
def IsNumber(string):
try:
float(string)
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import test
import os
-from os.path import join, dirname, exists
-import platform
-import utils
+import shutil
+
+from testrunner.local import commands
+from testrunner.local import testsuite
+from testrunner.local import utils
+from testrunner.objects import testcase
+
+
+class CcTestSuite(testsuite.TestSuite):
+
+ def __init__(self, name, root):
+ super(CcTestSuite, self).__init__(name, root)
+ self.serdes_dir = normpath(join(root, "..", "..", "out", ".serdes"))
+ if exists(self.serdes_dir):
+ shutil.rmtree(self.serdes_dir, True)
+ os.makedirs(self.serdes_dir)
+
+ def ListTests(self, context):
+ shell = join(context.shell_dir, self.shell())
+ if utils.IsWindows():
+ shell += '.exe'
+ output = commands.Execute([shell, '--list'])
+ if output.exit_code != 0:
+ print output.stdout
+ print output.stderr
+ return []
+ tests = []
+ for test_desc in output.stdout.strip().split():
+ raw_test, dependency = test_desc.split('<')
+ if dependency != '':
+ dependency = raw_test.split('/')[0] + '/' + dependency
+ else:
+ dependency = None
+ test = testcase.TestCase(self, raw_test, dependency=dependency)
+ tests.append(test)
+ tests.sort()
+ return tests
+
+ def GetFlagsForTestCase(self, testcase, context):
+ testname = testcase.path.split(os.path.sep)[-1]
+ serialization_file = join(self.serdes_dir, "serdes_" + testname)
+ serialization_file += ''.join(testcase.flags).replace('-', '_')
+ return (testcase.flags + [testcase.path] + context.mode_flags +
+ ["--testing_serialization_file=" + serialization_file])
+
+ def shell(self):
+ return "cctest"
+
+
+def GetSuite(name, root):
+ return CcTestSuite(name, root)
+
+
+# Deprecated definitions below.
+# TODO(jkummerow): Remove when SCons is no longer supported.
+
+
+from os.path import exists, join, normpath
+import test
class CcTestCase(test.TestCase):
from os.path import join, exists
+def GetSuite(name, root):
+ # Not implemented.
+ return None
+
+
HARNESS_FILES = ['sth.js']
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import test
+import itertools
import os
-from os.path import join, dirname, exists, basename, isdir
import re
+from testrunner.local import testsuite
+from testrunner.local import utils
+from testrunner.objects import testcase
+
+
FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
+
+class MessageTestSuite(testsuite.TestSuite):
+ def __init__(self, name, root):
+ super(MessageTestSuite, self).__init__(name, root)
+
+ def ListTests(self, context):
+ tests = []
+ for dirname, dirs, files in os.walk(self.root):
+ for dotted in [x for x in dirs if x.startswith('.')]:
+ dirs.remove(dotted)
+ dirs.sort()
+ files.sort()
+ for filename in files:
+ if filename.endswith(".js"):
+ testname = join(dirname[len(self.root) + 1:], filename[:-3])
+ test = testcase.TestCase(self, testname)
+ tests.append(test)
+ return tests
+
+ def GetFlagsForTestCase(self, testcase, context):
+ source = self.GetSourceForTest(testcase)
+ result = []
+ flags_match = re.findall(FLAGS_PATTERN, source)
+ for match in flags_match:
+ result += match.strip().split()
+ result += context.mode_flags
+ result.append(os.path.join(self.root, testcase.path + ".js"))
+ return testcase.flags + result
+
+ def GetSourceForTest(self, testcase):
+ filename = os.path.join(self.root, testcase.path + self.suffix())
+ with open(filename) as f:
+ return f.read()
+
+ def _IgnoreLine(self, string):
+ """Ignore empty lines, valgrind output and Android output."""
+ if not string: return True
+ return (string.startswith("==") or string.startswith("**") or
+ string.startswith("ANDROID"))
+
+ def IsFailureOutput(self, output, testpath):
+ expected_path = os.path.join(self.root, testpath + ".out")
+ expected_lines = []
+ # Can't use utils.ReadLinesFrom() here because it strips whitespace.
+ with open(expected_path) as f:
+ for line in f:
+ if line.startswith("#") or not line.strip(): continue
+ expected_lines.append(line)
+ raw_lines = output.stdout.splitlines()
+ actual_lines = [ s for s in raw_lines if not self._IgnoreLine(s) ]
+ env = { "basename": os.path.basename(testpath + ".js") }
+ if len(expected_lines) != len(actual_lines):
+ return True
+ for (expected, actual) in itertools.izip(expected_lines, actual_lines):
+ pattern = re.escape(expected.rstrip() % env)
+ pattern = pattern.replace("\\*", ".*")
+ pattern = "^%s$" % pattern
+ if not re.match(pattern, actual):
+ return True
+ return False
+
+ def StripOutputForTransmit(self, testcase):
+ pass
+
+
+def GetSuite(name, root):
+ return MessageTestSuite(name, root)
+
+
+# Deprecated definitions below.
+# TODO(jkummerow): Remove when SCons is no longer supported.
+
+
+import test
+from os.path import join, exists, basename, isdir
+
class MessageTestCase(test.TestCase):
def __init__(self, path, file, expected, mode, context, config):
-// Copyright 2008 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# Copyright 2008 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import test
import os
-from os.path import join, dirname, exists
import re
-import tempfile
+
+from testrunner.local import testsuite
+from testrunner.objects import testcase
FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
+class MjsunitTestSuite(testsuite.TestSuite):
+
+ def __init__(self, name, root):
+ super(MjsunitTestSuite, self).__init__(name, root)
+
+ def ListTests(self, context):
+ tests = []
+ for dirname, dirs, files in os.walk(self.root):
+ for dotted in [x for x in dirs if x.startswith('.')]:
+ dirs.remove(dotted)
+ dirs.sort()
+ files.sort()
+ for filename in files:
+ if filename.endswith(".js") and filename != "mjsunit.js":
+ testname = join(dirname[len(self.root) + 1:], filename[:-3])
+ test = testcase.TestCase(self, testname)
+ tests.append(test)
+ return tests
+
+ def GetFlagsForTestCase(self, testcase, context):
+ source = self.GetSourceForTest(testcase)
+ flags = []
+ flags_match = re.findall(FLAGS_PATTERN, source)
+ for match in flags_match:
+ flags += match.strip().split()
+ flags += context.mode_flags
+
+ files_list = [] # List of file names to append to command arguments.
+ files_match = FILES_PATTERN.search(source);
+ # Accept several lines of 'Files:'.
+ while True:
+ if files_match:
+ files_list += files_match.group(1).strip().split()
+ files_match = FILES_PATTERN.search(source, files_match.end())
+ else:
+ break
+ files = [ os.path.normpath(os.path.join(self.root, '..', '..', f))
+ for f in files_list ]
+ testfilename = os.path.join(self.root, testcase.path + self.suffix())
+ if SELF_SCRIPT_PATTERN.search(source):
+ env = ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename]
+ files = env + files
+ files.append(os.path.join(self.root, "mjsunit.js"))
+ files.append(testfilename)
+
+ flags += files
+ if context.isolates:
+ flags.append("--isolate")
+ flags += files
+
+ return testcase.flags + flags
+
+ def GetSourceForTest(self, testcase):
+ filename = os.path.join(self.root, testcase.path + self.suffix())
+ with open(filename) as f:
+ return f.read()
+
+
+def GetSuite(name, root):
+ return MjsunitTestSuite(name, root)
+
+
+# Deprecated definitions below.
+# TODO(jkummerow): Remove when SCons is no longer supported.
+
+
+from os.path import dirname, exists, join, normpath
+import tempfile
+import test
+
+
class MjsunitTestCase(test.TestCase):
def __init__(self, path, file, mode, context, config, isolates):
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import test
import os
-from os.path import join, exists
+import shutil
+import subprocess
+import tarfile
+
+from testrunner.local import testsuite
+from testrunner.objects import testcase
+
+MOZILLA_VERSION = "2010-06-29"
-EXCLUDED = ['CVS']
+
+EXCLUDED = ["CVS"]
FRAMEWORK = """
""".split()
+class MozillaTestSuite(testsuite.TestSuite):
+
+ def __init__(self, name, root):
+ super(MozillaTestSuite, self).__init__(name, root)
+ self.testroot = os.path.join(root, "data")
+
+ def ListTests(self, context):
+ tests = []
+ for testdir in TEST_DIRS:
+ current_root = os.path.join(self.testroot, testdir)
+ for dirname, dirs, files in os.walk(current_root):
+ for dotted in [x for x in dirs if x.startswith(".")]:
+ dirs.remove(dotted)
+ for excluded in EXCLUDED:
+ if excluded in dirs:
+ dirs.remove(excluded)
+ dirs.sort()
+ files.sort()
+ for filename in files:
+ if filename.endswith(".js") and not filename in FRAMEWORK:
+ testname = os.path.join(dirname[len(self.testroot) + 1:],
+ filename[:-3])
+ case = testcase.TestCase(self, testname)
+ tests.append(case)
+ return tests
+
+ def GetFlagsForTestCase(self, testcase, context):
+ result = []
+ result += context.mode_flags
+ result += ["--expose-gc"]
+ result += [os.path.join(self.root, "mozilla-shell-emulation.js")]
+ testfilename = testcase.path + ".js"
+ testfilepath = testfilename.split(os.path.sep)
+ for i in xrange(len(testfilepath)):
+ script = os.path.join(self.testroot,
+ reduce(os.path.join, testfilepath[:i], ""),
+ "shell.js")
+ if os.path.exists(script):
+ result.append(script)
+ result.append(os.path.join(self.testroot, testfilename))
+ return testcase.flags + result
+
+ def GetSourceForTest(self, testcase):
+ filename = join(self.testroot, testcase.path + ".js")
+ with open(filename) as f:
+ return f.read()
+
+ def IsNegativeTest(self, testcase):
+ return testcase.path.endswith("-n")
+
+ def IsFailureOutput(self, output, testpath):
+ if output.exit_code != 0:
+ return True
+ return "FAILED!" in output.stdout
+
+ def DownloadData(self):
+ old_cwd = os.getcwd()
+ os.chdir(os.path.abspath(self.root))
+
+ # Maybe we're still up to date?
+ versionfile = "CHECKED_OUT_VERSION"
+ checked_out_version = None
+ if os.path.exists(versionfile):
+ with open(versionfile) as f:
+ checked_out_version = f.read()
+ if checked_out_version == MOZILLA_VERSION:
+ os.chdir(old_cwd)
+ return
+
+ # If we have a local archive file with the test data, extract it.
+ directory_name = "data"
+ if os.path.exists(directory_name):
+ os.rename(directory_name, "data.old")
+ archive_file = "downloaded_%s.tar.gz" % MOZILLA_VERSION
+ if os.path.exists(archive_file):
+ with tarfile.open(archive_file, "r:gz") as tar:
+ tar.extractall()
+ with open(versionfile, "w") as f:
+ f.write(MOZILLA_VERSION)
+ os.chdir(old_cwd)
+ return
+
+ # No cached copy. Check out via CVS, and pack as .tar.gz for later use.
+ command = ("cvs -d :pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot"
+ " co -D %s mozilla/js/tests" % MOZILLA_VERSION)
+ code = subprocess.call(command, shell=True)
+ if code != 0:
+ os.chdir(old_cwd)
+ raise Exception("Error checking out Mozilla test suite!")
+ os.rename(join("mozilla", "js", "tests"), directory_name)
+ shutil.rmtree("mozilla")
+ with tarfile.open(archive_file, "w:gz") as tar:
+ tar.add("data")
+ with open(versionfile, "w") as f:
+ f.write(MOZILLA_VERSION)
+ os.chdir(old_cwd)
+
+
+def GetSuite(name, root):
+ return MozillaTestSuite(name, root)
+
+
+# Deprecated definitions below.
+# TODO(jkummerow): Remove when SCons is no longer supported.
+
+
+from os.path import exists
+from os.path import join
+import test
+
+
class MozillaTestCase(test.TestCase):
def __init__(self, filename, path, context, root, mode, framework):
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import test
+
import os
-from os.path import join, dirname, exists, isfile
-import platform
-import utils
import re
+from testrunner.local import testsuite
+from testrunner.local import utils
+from testrunner.objects import testcase
+
+
+class PreparserTestSuite(testsuite.TestSuite):
+ def __init__(self, name, root):
+ super(PreparserTestSuite, self).__init__(name, root)
+
+ def shell(self):
+ return "preparser"
+
+ def _GetExpectations(self):
+ expects_file = join(self.root, "preparser.expectation")
+ expectations_map = {}
+ if not os.path.exists(expects_file): return expectations_map
+ rule_regex = re.compile("^([\w\-]+)(?::([\w\-]+))?(?::(\d+),(\d+))?$")
+ for line in utils.ReadLinesFrom(expects_file):
+ rule_match = rule_regex.match(line)
+ if not rule_match: continue
+ expects = []
+ if (rule_match.group(2)):
+ expects += [rule_match.group(2)]
+ if (rule_match.group(3)):
+ expects += [rule_match.group(3), rule_match.group(4)]
+ expectations_map[rule_match.group(1)] = " ".join(expects)
+ return expectations_map
+
+ def _ParsePythonTestTemplates(self, result, filename):
+ pathname = join(self.root, filename + ".pyt")
+ def Test(name, source, expectation):
+ source = source.replace("\n", " ")
+ testname = os.path.join(filename, name)
+ flags = ["-e", source]
+ if expectation:
+ flags += ["throws", expectation]
+ test = testcase.TestCase(self, testname, flags=flags)
+ result.append(test)
+ def Template(name, source):
+ def MkTest(replacement, expectation):
+ testname = name
+ testsource = source
+ for key in replacement.keys():
+ testname = testname.replace("$" + key, replacement[key]);
+ testsource = testsource.replace("$" + key, replacement[key]);
+ Test(testname, testsource, expectation)
+ return MkTest
+ execfile(pathname, {"Test": Test, "Template": Template})
+
+ def ListTests(self, context):
+ expectations = self._GetExpectations()
+ result = []
+
+ # Find all .js files in this directory.
+ filenames = [f[:-3] for f in os.listdir(self.root) if f.endswith(".js")]
+ filenames.sort()
+ for f in filenames:
+ throws = expectations.get(f, None)
+ flags = [f + ".js"]
+ if throws:
+ flags += ["throws", throws]
+ test = testcase.TestCase(self, f, flags=flags)
+ result.append(test)
+
+ # Find all .pyt files in this directory.
+ filenames = [f[:-4] for f in os.listdir(self.root) if f.endswith(".pyt")]
+ filenames.sort()
+ for f in filenames:
+ self._ParsePythonTestTemplates(result, f)
+ return result
+
+ def GetFlagsForTestCase(self, testcase, context):
+ first = testcase.flags[0]
+ if first != "-e":
+ testcase.flags[0] = os.path.join(self.root, first)
+ return testcase.flags
+
+ def GetSourceForTest(self, testcase):
+ if testcase.flags[0] == "-e":
+ return testcase.flags[1]
+ with open(testcase.flags[0]) as f:
+ return f.read()
+
+ def VariantFlags(self):
+ return [[]];
+
+
+def GetSuite(name, root):
+ return PreparserTestSuite(name, root)
+
+
+# Deprecated definitions below.
+# TODO(jkummerow): Remove when SCons is no longer supported.
+
+
+from os.path import join, exists, isfile
+import test
+
+
class PreparserTestCase(test.TestCase):
def __init__(self, root, path, executable, mode, throws, context, source):
def HasSource(self):
return self.source is not None
- def GetSource():
+ def GetSource(self):
return self.source
def BuildCommand(self, path):
import time
+def GetSuite(name, root):
+ # Not implemented.
+ return None
+
+
class SputnikTestCase(test.TestCase):
def __init__(self, case, path, context, mode):
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import test
-import os
-from os.path import join, exists
-import urllib
import hashlib
+import os
import sys
import tarfile
+import urllib
+
+from testrunner.local import testsuite
+from testrunner.objects import testcase
+
+
+TEST_262_ARCHIVE_REVISION = "fb327c439e20" # This is the r334 revision.
+TEST_262_ARCHIVE_MD5 = "307acd166ec34629592f240dc12d57ed"
+TEST_262_URL = "http://hg.ecmascript.org/tests/test262/archive/%s.tar.bz2"
+TEST_262_HARNESS = ["sta.js"]
+
+
+class Test262TestSuite(testsuite.TestSuite):
+
+ def __init__(self, name, root):
+ super(Test262TestSuite, self).__init__(name, root)
+ self.testroot = os.path.join(root, "data", "test", "suite")
+ self.harness = [os.path.join(self.root, "data", "test", "harness", f)
+ for f in TEST_262_HARNESS]
+ self.harness += [os.path.join(self.root, "harness-adapt.js")]
+
+ def CommonTestName(self, testcase):
+ return testcase.path.split(os.path.sep)[-1]
+ def ListTests(self, context):
+ tests = []
+ for dirname, dirs, files in os.walk(self.testroot):
+ for dotted in [x for x in dirs if x.startswith(".")]:
+ dirs.remove(dotted)
+ dirs.sort()
+ files.sort()
+ for filename in files:
+ if filename.endswith(".js"):
+ testname = os.path.join(dirname[len(self.testroot) + 1:],
+ filename[:-3])
+ case = testcase.TestCase(self, testname)
+ tests.append(case)
+ return tests
+
+ def GetFlagsForTestCase(self, testcase, context):
+ return (testcase.flags + context.mode_flags + self.harness +
+ [os.path.join(self.testroot, testcase.path + ".js")])
+
+ def GetSourceForTest(self, testcase):
+ filename = os.path.join(self.testroot, testcase.path + ".js")
+ with open(filename) as f:
+ return f.read()
+
+ def IsNegativeTest(self, testcase):
+ return "@negative" in self.GetSourceForTest(testcase)
+
+ def IsFailureOutput(self, output, testpath):
+ if output.exit_code != 0:
+ return True
+ return "FAILED!" in output.stdout
-TEST_262_ARCHIVE_REVISION = 'fb327c439e20' # This is the r334 revision.
-TEST_262_ARCHIVE_MD5 = '307acd166ec34629592f240dc12d57ed'
-TEST_262_URL = 'http://hg.ecmascript.org/tests/test262/archive/%s.tar.bz2'
-TEST_262_HARNESS = ['sta.js']
+ def DownloadData(self):
+ revision = TEST_262_ARCHIVE_REVISION
+ archive_url = TEST_262_URL % revision
+ archive_name = os.path.join(self.root, "test262-%s.tar.bz2" % revision)
+ directory_name = os.path.join(self.root, "data")
+ directory_old_name = os.path.join(self.root, "data.old")
+ if not os.path.exists(archive_name):
+ print "Downloading test data from %s ..." % archive_url
+ urllib.urlretrieve(archive_url, archive_name)
+ if os.path.exists(directory_name):
+ os.rename(directory_name, directory_old_name)
+ if not os.path.exists(directory_name):
+ print "Extracting test262-%s.tar.bz2 ..." % revision
+ md5 = hashlib.md5()
+ with open(archive_name, "rb") as f:
+ for chunk in iter(lambda: f.read(8192), ""):
+ md5.update(chunk)
+ if md5.hexdigest() != TEST_262_ARCHIVE_MD5:
+ os.remove(archive_name)
+ raise Exception("Hash mismatch of test data file")
+ archive = tarfile.open(archive_name, "r:bz2")
+ if sys.platform in ("win32", "cygwin"):
+ # Magic incantation to allow longer path names on Windows.
+ archive.extractall(u"\\\\?\\%s" % self.root)
+ else:
+ archive.extractall(self.root)
+ os.rename(os.path.join(self.root, "test262-%s" % revision),
+ directory_name)
+
+
+def GetSuite(name, root):
+ return Test262TestSuite(name, root)
+
+
+# Deprecated definitions below.
+# TODO(jkummerow): Remove when SCons is no longer supported.
+
+
+from os.path import exists
+from os.path import join
+import test
class Test262TestCase(test.TestCase):
or (name == 'DerivedSources'))
IGNORE_COPYRIGHTS = ['cpplint.py',
+ 'daemon.py',
'earley-boyer.js',
'raytrace.js',
'crypto.js',
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import multiprocessing
+import optparse
+import os
+from os.path import join
+import subprocess
+import sys
+import time
+
+from testrunner.local import execution
+from testrunner.local import progress
+from testrunner.local import testsuite
+from testrunner.local import utils
+from testrunner.local import verbose
+from testrunner.network import network_execution
+from testrunner.objects import context
+
+
+ARCH_GUESS = utils.DefaultArch()
+DEFAULT_TESTS = ["mjsunit", "cctest", "message", "preparser"]
+TIMEOUT_DEFAULT = 60
+TIMEOUT_SCALEFACTOR = {"debug" : 4,
+ "release" : 1 }
+
+# Use this to run several variants of the tests.
+VARIANT_FLAGS = [[],
+ ["--stress-opt", "--always-opt"],
+ ["--nocrankshaft"]]
+MODE_FLAGS = {
+ "debug" : ["--nobreak-on-abort", "--enable-slow-asserts",
+ "--debug-code", "--verify-heap"],
+ "release" : ["--nobreak-on-abort"]}
+
+
+def BuildOptions():
+ result = optparse.OptionParser()
+ result.add_option("--arch",
+ help=("The architecture to run tests for, "
+ "'auto' or 'native' for auto-detect"),
+ default="ia32,x64,arm")
+ result.add_option("--arch-and-mode",
+ help="Architecture and mode in the format 'arch.mode'",
+ default=None)
+ result.add_option("--buildbot",
+ help="Adapt to path structure used on buildbots",
+ default=False, action="store_true")
+ result.add_option("--cat", help="Print the source of the tests",
+ default=False, action="store_true")
+ result.add_option("--command-prefix",
+ help="Prepended to each shell command used to run a test",
+ default="")
+ result.add_option("--download-data", help="Download missing test suite data",
+ default=False, action="store_true")
+ result.add_option("--extra-flags",
+ help="Additional flags to pass to each test command",
+ default="")
+ result.add_option("--isolates", help="Whether to test isolates",
+ default=False, action="store_true")
+ result.add_option("-j", help="The number of parallel tasks to run",
+ default=0, type="int")
+ result.add_option("-m", "--mode",
+ help="The test modes in which to run (comma-separated)",
+ default="release,debug")
+ result.add_option("--no-network", "--nonetwork",
+ help="Don't distribute tests on the network",
+ default=(utils.GuessOS() != "linux"),
+ dest="no_network", action="store_true")
+ result.add_option("--no-presubmit", "--nopresubmit",
+ help='Skip presubmit checks',
+ default=False, dest="no_presubmit", action="store_true")
+ result.add_option("--no-stress", "--nostress",
+ help="Don't run crankshaft --always-opt --stress-op test",
+ default=False, dest="no_stress", action="store_true")
+ result.add_option("--outdir", help="Base directory with compile output",
+ default="out")
+ result.add_option("-p", "--progress",
+ help=("The style of progress indicator"
+ " (verbose, dots, color, mono)"),
+ choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
+ result.add_option("--report", help="Print a summary of the tests to be run",
+ default=False, action="store_true")
+ result.add_option("--shard-count",
+ help="Split testsuites into this number of shards",
+ default=1, type="int")
+ result.add_option("--shard-run",
+ help="Run this shard from the split up tests.",
+ default=1, type="int")
+ result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
+ result.add_option("--shell-dir", help="Directory containing executables",
+ default="")
+ result.add_option("--stress-only",
+ help="Only run tests with --always-opt --stress-opt",
+ default=False, action="store_true")
+ result.add_option("--time", help="Print timing information after running",
+ default=False, action="store_true")
+ result.add_option("-t", "--timeout", help="Timeout in seconds",
+ default= -1, type="int")
+ result.add_option("-v", "--verbose", help="Verbose output",
+ default=False, action="store_true")
+ result.add_option("--valgrind", help="Run tests through valgrind",
+ default=False, action="store_true")
+ result.add_option("--warn-unused", help="Report unused rules",
+ default=False, action="store_true")
+ return result
+
+
+def ProcessOptions(options):
+ global VARIANT_FLAGS
+
+ # Architecture and mode related stuff.
+ if options.arch_and_mode:
+ tokens = options.arch_and_mode.split(".")
+ options.arch = tokens[0]
+ options.mode = tokens[1]
+ options.mode = options.mode.split(",")
+ for mode in options.mode:
+ if not mode in ["debug", "release"]:
+ print "Unknown mode %s" % mode
+ return False
+ if options.arch in ["auto", "native"]:
+ options.arch = ARCH_GUESS
+ options.arch = options.arch.split(",")
+ for arch in options.arch:
+ if not arch in ['ia32', 'x64', 'arm', 'mipsel']:
+ print "Unknown architecture %s" % arch
+ return False
+
+ # Special processing of other options, sorted alphabetically.
+
+ if options.buildbot:
+ # Buildbots run presubmit tests as a separate step.
+ options.no_presubmit = True
+ options.no_network = True
+ if options.command_prefix:
+ print("Specifying --command-prefix disables network distribution, "
+ "running tests locally.")
+ options.no_network = True
+ if options.j == 0:
+ options.j = multiprocessing.cpu_count()
+ if options.no_stress:
+ VARIANT_FLAGS = [[], ["--nocrankshaft"]]
+ if not options.shell_dir:
+ if options.shell:
+ print "Warning: --shell is deprecated, use --shell-dir instead."
+ options.shell_dir = os.path.dirname(options.shell)
+ if options.stress_only:
+ VARIANT_FLAGS = [["--stress-opt", "--always-opt"]]
+ # Simulators are slow, therefore allow a longer default timeout.
+ if options.timeout == -1:
+ if options.arch == "arm" or options.arch == "mipsel":
+ options.timeout = 2 * TIMEOUT_DEFAULT;
+ else:
+ options.timeout = TIMEOUT_DEFAULT;
+ if options.valgrind:
+ run_valgrind = os.path.join("tools", "run-valgrind.py")
+ # This is OK for distributed running, so we don't need to set no_network.
+ options.command_prefix = ("python -u " + run_valgrind +
+ options.command_prefix)
+ return True
+
+
+def ShardTests(tests, shard_count, shard_run):
+ if shard_count < 2:
+ return tests
+ if shard_run < 1 or shard_run > shard_count:
+ print "shard-run not a valid number, should be in [1:shard-count]"
+ print "defaulting back to running all tests"
+ return tests
+ count = 0
+ shard = []
+ for test in tests:
+ if count % shard_count == shard_run - 1:
+ shard.append(test)
+ count += 1
+ return shard
+
+
+def Main():
+ parser = BuildOptions()
+ (options, args) = parser.parse_args()
+ if not ProcessOptions(options):
+ parser.print_help()
+ return 1
+
+ exit_code = 0
+ workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
+ if not options.no_presubmit:
+ print ">>> running presubmit tests"
+ code = subprocess.call(
+ [sys.executable, join(workspace, "tools", "presubmit.py")])
+ exit_code = code
+
+ suite_paths = utils.GetSuitePaths(join(workspace, "test"))
+ print "all suite_paths:", suite_paths
+
+ if len(args) == 0:
+ suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
+ else:
+ args_suites = set()
+ for arg in args:
+ suite = arg.split(os.path.sep)[0]
+ if not suite in args_suites:
+ args_suites.add(suite)
+ suite_paths = [ s for s in suite_paths if s in args_suites ]
+
+ suites = []
+ for root in suite_paths:
+ suite = testsuite.TestSuite.LoadTestSuite(
+ os.path.join(workspace, "test", root))
+ if suite:
+ suites.append(suite)
+
+ if options.download_data:
+ for s in suites:
+ s.DownloadData()
+
+ for mode in options.mode:
+ for arch in options.arch:
+ code = Execute(arch, mode, args, options, suites, workspace)
+ exit_code = exit_code or code
+ return exit_code
+
+
+def Execute(arch, mode, args, options, suites, workspace):
+ print(">>> Running tests for %s.%s" % (arch, mode))
+
+ shell_dir = options.shell_dir
+ if not shell_dir:
+ if options.buildbot:
+ shell_dir = os.path.join(workspace, options.outdir, mode)
+ mode = mode.lower()
+ else:
+ shell_dir = os.path.join(workspace, options.outdir,
+ "%s.%s" % (arch, mode))
+ shell_dir = os.path.relpath(shell_dir)
+
+ # Populate context object.
+ mode_flags = MODE_FLAGS[mode]
+ options.timeout *= TIMEOUT_SCALEFACTOR[mode]
+ ctx = context.Context(arch, mode, shell_dir,
+ mode_flags, options.verbose,
+ options.timeout, options.isolates,
+ options.command_prefix,
+ options.extra_flags)
+
+ # Find available test suites and read test cases from them.
+ variables = {
+ "mode": mode,
+ "arch": arch,
+ "system": utils.GuessOS(),
+ "isolates": options.isolates
+ }
+ all_tests = []
+ num_tests = 0
+ test_id = 0
+ for s in suites:
+ s.ReadStatusFile(variables)
+ s.ReadTestCases(ctx)
+ all_tests += s.tests
+ if len(args) > 0:
+ s.FilterTestCasesByArgs(args)
+ s.FilterTestCasesByStatus(options.warn_unused)
+ if options.cat:
+ verbose.PrintTestSource(s.tests)
+ continue
+ variant_flags = s.VariantFlags() or VARIANT_FLAGS
+ s.tests = [ t.CopyAddingFlags(v) for t in s.tests for v in variant_flags ]
+ s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
+ num_tests += len(s.tests)
+ for t in s.tests:
+ t.id = test_id
+ test_id += 1
+
+ if options.cat:
+ return 0 # We're done here.
+
+ if options.report:
+ verbose.PrintReport(all_tests)
+
+ if num_tests == 0:
+ print "No tests to run."
+ return 0
+
+ # Run the tests, either locally or distributed on the network.
+ try:
+ start_time = time.time()
+ progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
+
+ run_networked = not options.no_network
+ if not run_networked:
+ print("Network distribution disabled, running tests locally.")
+ elif utils.GuessOS() != "linux":
+ print("Network distribution is only supported on Linux, sorry!")
+ run_networked = False
+ peers = []
+ if run_networked:
+ peers = network_execution.GetPeers()
+ if not peers:
+ print("No connection to distribution server; running tests locally.")
+ run_networked = False
+ elif len(peers) == 1:
+ print("No other peers on the network; running tests locally.")
+ run_networked = False
+ elif num_tests <= 100:
+ print("Less than 100 tests, running them locally.")
+ run_networked = False
+
+ if run_networked:
+ runner = network_execution.NetworkedRunner(suites, progress_indicator,
+ ctx, peers, workspace)
+ else:
+ runner = execution.Runner(suites, progress_indicator, ctx)
+
+ exit_code = runner.Run(options.j)
+ if runner.terminate:
+ return exit_code
+ overall_duration = time.time() - start_time
+ except KeyboardInterrupt:
+ return 1
+
+ if options.time:
+ verbose.PrintTestDurations(suites, overall_duration)
+ return exit_code
+
+
+if __name__ == "__main__":
+ sys.exit(Main())
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import sys
+from testrunner.local import old_statusfile
+
+if len(sys.argv) != 2:
+ print "Usage: %s foo.status" % sys.argv[0]
+ print "Will read foo.status and print the converted version to stdout."
+ sys.exit(1)
+
+print old_statusfile.ConvertNotation(sys.argv[1]).GetOutput()
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import subprocess
+import sys
+
+
+PIDFILE = "/tmp/v8-distributed-testing-server.pid"
+ROOT = os.path.abspath(os.path.dirname(sys.argv[0]))
+
+
+def _PrintUsage():
+ print("""Usage: python %s COMMAND
+
+Where COMMAND can be any of:
+ start Starts the server. Forks to the background.
+ stop Stops the server.
+ restart Stops, then restarts the server.
+ setup Creates or updates the environment for the server to run.
+ update Alias for "setup".
+ trust <keyfile> Adds the given public key to the list of trusted keys.
+ help Displays this help text.
+ """ % sys.argv[0])
+
+
+def _IsDaemonRunning():
+ return os.path.exists(PIDFILE)
+
+
+def _Cmd(cmd):
+ code = subprocess.call(cmd, shell=True)
+ if code != 0:
+ print("Command '%s' returned error code %d" % (cmd, code))
+ sys.exit(code)
+
+
+def Update():
+ # Create directory for private data storage.
+ data_dir = os.path.join(ROOT, "data")
+ if not os.path.exists(data_dir):
+ os.makedirs(data_dir)
+
+ # Create directory for trusted public keys of peers (and self).
+ trusted_dir = os.path.join(ROOT, "trusted")
+ if not os.path.exists(trusted_dir):
+ os.makedirs(trusted_dir)
+
+ # Install UltraJSON. It is much faster than Python's builtin json.
+ try:
+ import ujson #@UnusedImport
+ except ImportError:
+ # Install pip if it doesn't exist.
+ code = subprocess.call("which pip", shell=True)
+ if code != 0:
+ apt_get_code = subprocess.call("which apt-get", shell=True)
+ if apt_get_code == 0:
+ print("Installing pip...")
+ _Cmd("sudo apt-get install python-pip")
+ print("Updating pip using itself...")
+ _Cmd("sudo pip install --upgrade pip")
+ else:
+ print("Please install pip on your machine. You can get it at: "
+ "http://www.pip-installer.org/en/latest/installing.html "
+ "or via your distro's package manager.")
+ sys.exit(1)
+ print("Using pip to install UltraJSON...")
+ _Cmd("sudo pip install ujson")
+
+ # Make sure we have a key pair for signing binaries.
+ privkeyfile = os.path.expanduser("~/.ssh/v8_dtest")
+ if not os.path.exists(privkeyfile):
+ _Cmd("ssh-keygen -t rsa -f %s -N '' -q" % privkeyfile)
+ fingerprint = subprocess.check_output("ssh-keygen -lf %s" % privkeyfile,
+ shell=True)
+ fingerprint = fingerprint.split(" ")[1].replace(":", "")[:16]
+ pubkeyfile = os.path.join(trusted_dir, "%s.pem" % fingerprint)
+ if (not os.path.exists(pubkeyfile) or
+ os.path.getmtime(pubkeyfile) < os.path.getmtime(privkeyfile)):
+ _Cmd("openssl rsa -in %s -out %s -pubout" % (privkeyfile, pubkeyfile))
+ with open(pubkeyfile, "a") as f:
+ f.write(fingerprint + "\n")
+ datafile = os.path.join(data_dir, "mypubkey")
+ with open(datafile, "w") as f:
+ f.write(fingerprint + "\n")
+
+ # Check out or update the server implementation in the current directory.
+ testrunner_dir = os.path.join(ROOT, "testrunner")
+ if os.path.exists(os.path.join(testrunner_dir, "server/daemon.py")):
+ _Cmd("cd %s; svn up" % testrunner_dir)
+ else:
+ path = ("http://v8.googlecode.com/svn/branches/bleeding_edge/"
+ "tools/testrunner")
+ _Cmd("svn checkout --force %s %s" % (path, testrunner_dir))
+
+ # Update this very script.
+ path = ("http://v8.googlecode.com/svn/branches/bleeding_edge/"
+ "tools/server.py")
+ scriptname = os.path.abspath(sys.argv[0])
+ _Cmd("svn cat %s > %s" % (path, scriptname))
+
+ # Check out or update V8.
+ v8_dir = os.path.join(ROOT, "v8")
+ if os.path.exists(v8_dir):
+ _Cmd("cd %s; git fetch" % v8_dir)
+ else:
+ _Cmd("git clone git://github.com/v8/v8.git %s" % v8_dir)
+
+ print("Finished.")
+
+
+# Handle "setup" here, because when executing that we can't import anything
+# else yet.
+if __name__ == "__main__" and len(sys.argv) == 2:
+ if sys.argv[1] in ("setup", "update"):
+ if _IsDaemonRunning():
+ print("Please stop the server before updating. Exiting.")
+ sys.exit(1)
+ Update()
+ sys.exit(0)
+ # Other parameters are handled below.
+
+
+#==========================================================
+# At this point we can assume that the implementation is available,
+# so we can import it.
+try:
+ from testrunner.server import constants
+ from testrunner.server import local_handler
+ from testrunner.server import main
+except Exception, e:
+ print(e)
+ print("Failed to import implementation. Have you run 'setup'?")
+ sys.exit(1)
+
+
+def _StartDaemon(daemon):
+ if not os.path.isdir(os.path.join(ROOT, "v8")):
+ print("No 'v8' working directory found. Have you run 'setup'?")
+ sys.exit(1)
+ daemon.start()
+
+
+if __name__ == "__main__":
+ if len(sys.argv) == 2:
+ arg = sys.argv[1]
+ if arg == "start":
+ daemon = main.Server(PIDFILE, ROOT)
+ _StartDaemon(daemon)
+ elif arg == "stop":
+ daemon = main.Server(PIDFILE, ROOT)
+ daemon.stop()
+ elif arg == "restart":
+ daemon = main.Server(PIDFILE, ROOT)
+ daemon.stop()
+ _StartDaemon(daemon)
+ elif arg in ("help", "-h", "--help"):
+ _PrintUsage()
+ elif arg == "status":
+ if not _IsDaemonRunning():
+ print("Server not running.")
+ else:
+ print(local_handler.LocalQuery([constants.REQUEST_STATUS]))
+ else:
+ print("Unknown command")
+ _PrintUsage()
+ sys.exit(2)
+ elif len(sys.argv) == 3:
+ arg = sys.argv[1]
+ if arg == "approve":
+ filename = sys.argv[2]
+ if not os.path.exists(filename):
+ print("%s does not exist.")
+ sys.exit(1)
+ filename = os.path.abspath(filename)
+ if _IsDaemonRunning():
+ response = local_handler.LocalQuery([constants.ADD_TRUSTED, filename])
+ else:
+ daemon = main.Server(PIDFILE, ROOT)
+ response = daemon.CopyToTrusted(filename)
+ print("Added certificate %s to trusted certificates." % response)
+ else:
+ print("Unknown command")
+ _PrintUsage()
+ sys.exit(2)
+ else:
+ print("Unknown command")
+ _PrintUsage()
+ sys.exit(2)
+ sys.exit(0)
--- /dev/null
+Test suite runner for V8, including support for distributed running.
+====================================================================
+
+
+Local usage instructions:
+=========================
+
+Run the main script with --help to get detailed usage instructions:
+
+$ tools/run-tests.py --help
+
+The interface is mostly the same as it was for the old test runner.
+You'll likely want something like this:
+
+$ tools/run-tests.py --nonetwork --arch ia32 --mode release
+
+--nonetwork is the default on Mac and Windows. If you don't specify --arch
+and/or --mode, all available values will be used and run in turn (e.g.,
+omitting --mode from the above example will run ia32 in both Release and Debug
+modes).
+
+
+Networked usage instructions:
+=============================
+
+Networked running is only supported on Linux currently. Make sure that all
+machines participating in the cluster are binary-compatible (e.g. mixing
+Ubuntu Lucid and Precise doesn't work).
+
+Setup:
+------
+
+1.) Copy tools/test-server.py to a new empty directory anywhere on your hard
+ drive (preferably not inside your V8 checkout just to keep things clean).
+ Please do create a copy, not just a symlink.
+
+2.) Navigate to the new directory and let the server setup itself:
+
+$ ./test-server.py setup
+
+ This will install PIP and UltraJSON, create a V8 working directory, and
+ generate a keypair.
+
+3.) Swap public keys with someone who's already part of the networked cluster.
+
+$ cp trusted/`cat data/mypubkey`.pem /where/peers/can/see/it/myname.pem
+$ ./test-server.py approve /wherever/they/put/it/yourname.pem
+
+
+Usage:
+------
+
+1.) Start your server:
+
+$ ./test-server.py start
+
+2.) (Optionally) inspect the server's status:
+
+$ ./test-server.py status
+
+3.) From your regular V8 working directory, run tests:
+
+$ tool/run-tests.py --arch ia32 --mode debug
+
+4.) (Optionally) enjoy the speeeeeeeeeeeeeeeed
+
+
+Architecture overview:
+======================
+
+Code organization:
+------------------
+
+This section is written from the point of view of the tools/ directory.
+
+./run-tests.py:
+ Main script. Parses command-line options and drives the test execution
+ procedure from a high level. Imports the actual implementation of all
+ steps from the testrunner/ directory.
+
+./test-server.py:
+ Interface to interact with the server. Contains code to setup the server's
+ working environment and can start and stop server daemon processes.
+ Imports some stuff from the testrunner/server/ directory.
+
+./testrunner/local/*:
+ Implementation needed to run tests locally. Used by run-tests.py. Inspired by
+ (and partly copied verbatim from) the original test.py script.
+
+./testrunner/local/old_statusfile.py:
+ Provides functionality to read an old-style <testsuite>.status file and
+ convert it to new-style syntax. This can be removed once the new-style
+ syntax becomes authoritative (and old-style syntax is no longer supported).
+ ./status-file-converter.py provides a stand-alone interface to this.
+
+./testrunner/objects/*:
+ A bunch of data container classes, used by the scripts in the various other
+ directories; serializable for transmission over the network.
+
+./testrunner/network/*:
+ Equivalents and extensions of some of the functionality in ./testrunner/local/
+ as required when dispatching tests to peers on the network.
+
+./testrunner/network/network_execution.py:
+ Drop-in replacement for ./testrunner/local/execution that distributes
+ test jobs to network peers instead of running them locally.
+
+./testrunner/network/endpoint.py:
+ Receiving end of a network distributed job, uses the implementation
+ in ./testrunner/local/execution.py for actually running the tests.
+
+./testrunner/server/*:
+ Implementation of the daemon that accepts and runs test execution jobs from
+ peers on the network. Should ideally have no dependencies on any of the other
+ directories, but that turned out to be impractical, so there are a few
+ exceptions.
+
+./testrunner/server/compression.py:
+ Defines a wrapper around Python TCP sockets that provides JSON based
+ serialization, gzip based compression, and ensures message completeness.
+
+
+Networking architecture:
+------------------------
+
+The distribution stuff is designed to be a layer between deciding which tests
+to run on the one side, and actually running them on the other. The frontend
+that the user interacts with is the same for local and networked execution,
+and the actual test execution and result gathering code is the same too.
+
+The server daemon starts four separate servers, each listening on another port:
+- "Local": Communication with a run-tests.py script running on the same host.
+ The test driving script e.g. needs to ask for available peers. It then talks
+ to those peers directly (one of them will be the locally running server).
+- "Work": Listens for test job requests from run-tests.py scripts on the network
+ (including localhost). Accepts an arbitrary number of connections at the
+ same time, but only works on them in a serialized fashion.
+- "Status": Used for communication with other servers on the network, e.g. for
+ exchanging trusted public keys to create the transitive trust closure.
+- "Discovery": Used to detect presence of other peers on the network.
+ In contrast to the other three, this uses UDP (as opposed to TCP).
+
+
+Give us a diagram! We love diagrams!
+------------------------------------
+ .
+ Machine A . Machine B
+ .
++------------------------------+ .
+| run-tests.py | .
+| with flag: | .
+|--nonetwork --network | .
+| | / | | .
+| | / | | .
+| v / v | .
+|BACKEND / distribution | .
++--------- / --------| \ ------+ .
+ / | \_____________________
+ / | . \
+ / | . \
++----- v ----------- v --------+ . +---- v -----------------------+
+| LocalHandler | WorkHandler | . | WorkHandler | LocalHandler |
+| | | | . | | | |
+| | v | . | v | |
+| | BACKEND | . | BACKEND | |
+|------------- +---------------| . |---------------+--------------|
+| Discovery | StatusHandler <----------> StatusHandler | Discovery |
++---- ^ -----------------------+ . +-------------------- ^ -------+
+ | . |
+ +---------------------------------------------------------+
+
+Note that the three occurrences of "BACKEND" are the same code
+(testrunner/local/execution.py and its imports), but running from three
+distinct directories (and on two different machines).
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+
+from ..local import utils
+from ..objects import output
+
+
+def KillProcessWithID(pid):
+ if utils.IsWindows():
+ os.popen('taskkill /T /F /PID %d' % pid)
+ else:
+ os.kill(pid, signal.SIGTERM)
+
+
+MAX_SLEEP_TIME = 0.1
+INITIAL_SLEEP_TIME = 0.0001
+SLEEP_TIME_FACTOR = 1.25
+
+SEM_INVALID_VALUE = -1
+SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h
+
+
+def Win32SetErrorMode(mode):
+ prev_error_mode = SEM_INVALID_VALUE
+ try:
+ import ctypes
+ prev_error_mode = \
+ ctypes.windll.kernel32.SetErrorMode(mode) #@UndefinedVariable
+ except ImportError:
+ pass
+ return prev_error_mode
+
+
+def RunProcess(verbose, timeout, args, **rest):
+ if verbose: print "#", " ".join(args)
+ popen_args = args
+ prev_error_mode = SEM_INVALID_VALUE
+ if utils.IsWindows():
+ popen_args = subprocess.list2cmdline(args)
+ # Try to change the error mode to avoid dialogs on fatal errors. Don't
+ # touch any existing error mode flags by merging the existing error mode.
+ # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
+ error_mode = SEM_NOGPFAULTERRORBOX
+ prev_error_mode = Win32SetErrorMode(error_mode)
+ Win32SetErrorMode(error_mode | prev_error_mode)
+ process = subprocess.Popen(
+ shell=utils.IsWindows(),
+ args=popen_args,
+ **rest
+ )
+ if (utils.IsWindows() and prev_error_mode != SEM_INVALID_VALUE):
+ Win32SetErrorMode(prev_error_mode)
+ # Compute the end time - if the process crosses this limit we
+ # consider it timed out.
+ if timeout is None: end_time = None
+ else: end_time = time.time() + timeout
+ timed_out = False
+ # Repeatedly check the exit code from the process in a
+ # loop and keep track of whether or not it times out.
+ exit_code = None
+ sleep_time = INITIAL_SLEEP_TIME
+ try:
+ while exit_code is None:
+ if (not end_time is None) and (time.time() >= end_time):
+ # Kill the process and wait for it to exit.
+ KillProcessWithID(process.pid)
+ exit_code = process.wait()
+ timed_out = True
+ else:
+ exit_code = process.poll()
+ time.sleep(sleep_time)
+ sleep_time = sleep_time * SLEEP_TIME_FACTOR
+ if sleep_time > MAX_SLEEP_TIME:
+ sleep_time = MAX_SLEEP_TIME
+ return (exit_code, timed_out)
+ except KeyboardInterrupt:
+ raise
+
+
+def PrintError(string):
+ sys.stderr.write(string)
+ sys.stderr.write("\n")
+
+
+def CheckedUnlink(name):
+ # On Windows, when run with -jN in parallel processes,
+ # OS often fails to unlink the temp file. Not sure why.
+ # Need to retry.
+ # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch
+ retry_count = 0
+ while retry_count < 30:
+ try:
+ os.unlink(name)
+ return
+ except OSError, e:
+ retry_count += 1
+ time.sleep(retry_count * 0.1)
+ PrintError("os.unlink() " + str(e))
+
+
+def Execute(args, verbose=False, timeout=None):
+ (fd_out, outname) = tempfile.mkstemp()
+ (fd_err, errname) = tempfile.mkstemp()
+ try:
+ (exit_code, timed_out) = RunProcess(
+ verbose,
+ timeout,
+ args=args,
+ stdout=fd_out,
+ stderr=fd_err
+ )
+ except:
+ raise
+ os.close(fd_out)
+ os.close(fd_err)
+ out = file(outname).read()
+ errors = file(errname).read()
+ CheckedUnlink(outname)
+ CheckedUnlink(errname)
+ return output.Output(exit_code, timed_out, out, errors)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import multiprocessing
+import os
+import threading
+import time
+
+from . import commands
+from . import utils
+
+
+class Job(object):
+ def __init__(self, command, dep_command, test_id, timeout, verbose):
+ self.command = command
+ self.dep_command = dep_command
+ self.id = test_id
+ self.timeout = timeout
+ self.verbose = verbose
+
+
+def RunTest(job):
+ try:
+ start_time = time.time()
+ if job.dep_command is not None:
+ dep_output = commands.Execute(job.dep_command, job.verbose, job.timeout)
+ # TODO(jkummerow): We approximate the test suite specific function
+ # IsFailureOutput() by just checking the exit code here. Currently
+ # only cctests define dependencies, for which this simplification is
+ # correct.
+ if dep_output.exit_code != 0:
+ return (job.id, dep_output, time.time() - start_time)
+ output = commands.Execute(job.command, job.verbose, job.timeout)
+ return (job.id, output, time.time() - start_time)
+ except Exception, e:
+ print(">>> EXCEPTION: %s" % e)
+ return (-1, -1, 0)
+
+
+class Runner(object):
+
+ def __init__(self, suites, progress_indicator, context):
+ self.tests = [ t for s in suites for t in s.tests ]
+ self._CommonInit(len(self.tests), progress_indicator, context)
+
+ def _CommonInit(self, num_tests, progress_indicator, context):
+ self.indicator = progress_indicator
+ progress_indicator.runner = self
+ self.context = context
+ self.succeeded = 0
+ self.total = num_tests
+ self.remaining = num_tests
+ self.failed = []
+ self.crashed = 0
+ self.terminate = False
+ self.lock = threading.Lock()
+
+ def Run(self, jobs):
+ self.indicator.Starting()
+ self._RunInternal(jobs)
+ self.indicator.Done()
+ return not self.failed
+
+ def _RunInternal(self, jobs):
+ pool = multiprocessing.Pool(processes=jobs)
+ test_map = {}
+ queue = []
+ for test in self.tests:
+ assert test.id >= 0
+ test_map[test.id] = test
+ command = self.GetCommand(test)
+ timeout = self.context.timeout
+ if ("--stress-opt" in test.flags or
+ "--stress-opt" in self.context.mode_flags or
+ "--stress-opt" in self.context.extra_flags):
+ timeout *= 4
+ if test.dependency is not None:
+ dep_command = [ c.replace(test.path, test.dependency) for c in command ]
+ else:
+ dep_command = None
+ job = Job(command, dep_command, test.id, timeout, self.context.verbose)
+ queue.append(job)
+ try:
+ kChunkSize = 1
+ it = pool.imap_unordered(RunTest, queue, kChunkSize)
+ for result in it:
+ test_id = result[0]
+ if test_id < 0:
+ raise BreakNowException("User pressed Ctrl+C or IO went wrong")
+ test = test_map[test_id]
+ self.indicator.AboutToRun(test)
+ test.output = result[1]
+ test.duration = result[2]
+ if test.suite.HasUnexpectedOutput(test):
+ self.failed.append(test)
+ if test.output.HasCrashed():
+ self.crashed += 1
+ else:
+ self.succeeded += 1
+ self.remaining -= 1
+ self.indicator.HasRun(test)
+ except:
+ pool.terminate()
+ pool.join()
+ raise
+ return
+
+
+ def GetCommand(self, test):
+ d8testflag = []
+ shell = test.suite.shell()
+ if shell == "d8":
+ d8testflag = ["--test"]
+ if utils.IsWindows():
+ shell += ".exe"
+ cmd = ([self.context.command_prefix] +
+ [os.path.join(self.context.shell_dir, shell)] +
+ d8testflag +
+ test.suite.GetFlagsForTestCase(test, self.context) +
+ [self.context.extra_flags])
+ cmd = [ c for c in cmd if c != "" ]
+ return cmd
+
+
+class BreakNowException(Exception):
+ def __init__(self, value):
+ self.value = value
+ def __str__(self):
+ return repr(self.value)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import cStringIO
+import re
+
+# These outcomes can occur in a TestCase's outcomes list:
+SKIP = 'SKIP'
+FAIL = 'FAIL'
+PASS = 'PASS'
+OKAY = 'OKAY'
+TIMEOUT = 'TIMEOUT'
+CRASH = 'CRASH'
+SLOW = 'SLOW'
+# These are just for the status files and are mapped below in DEFS:
+FAIL_OK = 'FAIL_OK'
+PASS_OR_FAIL = 'PASS_OR_FAIL'
+
+KEYWORDS = {SKIP: SKIP,
+ FAIL: FAIL,
+ PASS: PASS,
+ OKAY: OKAY,
+ TIMEOUT: TIMEOUT,
+ CRASH: CRASH,
+ SLOW: SLOW,
+ FAIL_OK: FAIL_OK,
+ PASS_OR_FAIL: PASS_OR_FAIL}
+
+class Expression(object):
+ pass
+
+
+class Constant(Expression):
+
+ def __init__(self, value):
+ self.value = value
+
+ def Evaluate(self, env, defs):
+ return self.value
+
+
+class Variable(Expression):
+
+ def __init__(self, name):
+ self.name = name
+
+ def GetOutcomes(self, env, defs):
+ if self.name in env: return set([env[self.name]])
+ else: return set([])
+
+ def Evaluate(self, env, defs):
+ return env[self.name]
+
+ def __str__(self):
+ return self.name
+
+ def string(self, logical):
+ return self.__str__()
+
+
+class Outcome(Expression):
+
+ def __init__(self, name):
+ self.name = name
+
+ def GetOutcomes(self, env, defs):
+ if self.name in defs:
+ return defs[self.name].GetOutcomes(env, defs)
+ else:
+ return set([self.name])
+
+ def __str__(self):
+ if self.name in KEYWORDS:
+ return "%s" % KEYWORDS[self.name]
+ return "'%s'" % self.name
+
+ def string(self, logical):
+ if logical:
+ return "%s" % self.name
+ return self.__str__()
+
+
+class Operation(Expression):
+
+ def __init__(self, left, op, right):
+ self.left = left
+ self.op = op
+ self.right = right
+
+ def Evaluate(self, env, defs):
+ if self.op == '||' or self.op == ',':
+ return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs)
+ elif self.op == 'if':
+ return False
+ elif self.op == '==':
+ return not self.left.GetOutcomes(env, defs).isdisjoint(self.right.GetOutcomes(env, defs))
+ elif self.op == '!=':
+ return self.left.GetOutcomes(env, defs).isdisjoint(self.right.GetOutcomes(env, defs))
+ else:
+ assert self.op == '&&'
+ return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs)
+
+ def GetOutcomes(self, env, defs):
+ if self.op == '||' or self.op == ',':
+ return self.left.GetOutcomes(env, defs) | self.right.GetOutcomes(env, defs)
+ elif self.op == 'if':
+ if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs)
+ else: return set([])
+ else:
+ assert self.op == '&&'
+ return self.left.GetOutcomes(env, defs) & self.right.GetOutcomes(env, defs)
+
+ def __str__(self):
+ return self.string(False)
+
+ def string(self, logical=False):
+ if self.op == 'if':
+ return "['%s', %s]" % (self.right.string(True), self.left.string(logical))
+ elif self.op == "||" or self.op == ",":
+ if logical:
+ return "%s or %s" % (self.left.string(True), self.right.string(True))
+ else:
+ return "%s, %s" % (self.left, self.right)
+ elif self.op == "&&":
+ return "%s and %s" % (self.left.string(True), self.right.string(True))
+ return "%s %s %s" % (self.left.string(logical), self.op,
+ self.right.string(logical))
+
+
+def IsAlpha(string):
+ for char in string:
+ if not (char.isalpha() or char.isdigit() or char == '_'):
+ return False
+ return True
+
+
+class Tokenizer(object):
+ """A simple string tokenizer that chops expressions into variables,
+ parens and operators"""
+
+ def __init__(self, expr):
+ self.index = 0
+ self.expr = expr
+ self.length = len(expr)
+ self.tokens = None
+
+ def Current(self, length=1):
+ if not self.HasMore(length): return ""
+ return self.expr[self.index:self.index + length]
+
+ def HasMore(self, length=1):
+ return self.index < self.length + (length - 1)
+
+ def Advance(self, count=1):
+ self.index = self.index + count
+
+ def AddToken(self, token):
+ self.tokens.append(token)
+
+ def SkipSpaces(self):
+ while self.HasMore() and self.Current().isspace():
+ self.Advance()
+
+ def Tokenize(self):
+ self.tokens = [ ]
+ while self.HasMore():
+ self.SkipSpaces()
+ if not self.HasMore():
+ return None
+ if self.Current() == '(':
+ self.AddToken('(')
+ self.Advance()
+ elif self.Current() == ')':
+ self.AddToken(')')
+ self.Advance()
+ elif self.Current() == '$':
+ self.AddToken('$')
+ self.Advance()
+ elif self.Current() == ',':
+ self.AddToken(',')
+ self.Advance()
+ elif IsAlpha(self.Current()):
+ buf = ""
+ while self.HasMore() and IsAlpha(self.Current()):
+ buf += self.Current()
+ self.Advance()
+ self.AddToken(buf)
+ elif self.Current(2) == '&&':
+ self.AddToken('&&')
+ self.Advance(2)
+ elif self.Current(2) == '||':
+ self.AddToken('||')
+ self.Advance(2)
+ elif self.Current(2) == '==':
+ self.AddToken('==')
+ self.Advance(2)
+ elif self.Current(2) == '!=':
+ self.AddToken('!=')
+ self.Advance(2)
+ else:
+ return None
+ return self.tokens
+
+
+class Scanner(object):
+ """A simple scanner that can serve out tokens from a given list"""
+
+ def __init__(self, tokens):
+ self.tokens = tokens
+ self.length = len(tokens)
+ self.index = 0
+
+ def HasMore(self):
+ return self.index < self.length
+
+ def Current(self):
+ return self.tokens[self.index]
+
+ def Advance(self):
+ self.index = self.index + 1
+
+
+def ParseAtomicExpression(scan):
+ if scan.Current() == "true":
+ scan.Advance()
+ return Constant(True)
+ elif scan.Current() == "false":
+ scan.Advance()
+ return Constant(False)
+ elif IsAlpha(scan.Current()):
+ name = scan.Current()
+ scan.Advance()
+ return Outcome(name)
+ elif scan.Current() == '$':
+ scan.Advance()
+ if not IsAlpha(scan.Current()):
+ return None
+ name = scan.Current()
+ scan.Advance()
+ return Variable(name.lower())
+ elif scan.Current() == '(':
+ scan.Advance()
+ result = ParseLogicalExpression(scan)
+ if (not result) or (scan.Current() != ')'):
+ return None
+ scan.Advance()
+ return result
+ else:
+ return None
+
+
+BINARIES = ['==', '!=']
+def ParseOperatorExpression(scan):
+ left = ParseAtomicExpression(scan)
+ if not left: return None
+ while scan.HasMore() and (scan.Current() in BINARIES):
+ op = scan.Current()
+ scan.Advance()
+ right = ParseOperatorExpression(scan)
+ if not right:
+ return None
+ left = Operation(left, op, right)
+ return left
+
+
+def ParseConditionalExpression(scan):
+ left = ParseOperatorExpression(scan)
+ if not left: return None
+ while scan.HasMore() and (scan.Current() == 'if'):
+ scan.Advance()
+ right = ParseOperatorExpression(scan)
+ if not right:
+ return None
+ left = Operation(left, 'if', right)
+ return left
+
+
+LOGICALS = ["&&", "||", ","]
+def ParseLogicalExpression(scan):
+ left = ParseConditionalExpression(scan)
+ if not left: return None
+ while scan.HasMore() and (scan.Current() in LOGICALS):
+ op = scan.Current()
+ scan.Advance()
+ right = ParseConditionalExpression(scan)
+ if not right:
+ return None
+ left = Operation(left, op, right)
+ return left
+
+
+def ParseCondition(expr):
+ """Parses a logical expression into an Expression object"""
+ tokens = Tokenizer(expr).Tokenize()
+ if not tokens:
+ print "Malformed expression: '%s'" % expr
+ return None
+ scan = Scanner(tokens)
+ ast = ParseLogicalExpression(scan)
+ if not ast:
+ print "Malformed expression: '%s'" % expr
+ return None
+ if scan.HasMore():
+ print "Malformed expression: '%s'" % expr
+ return None
+ return ast
+
+
+class Section(object):
+ """A section of the configuration file. Sections are enabled or
+ disabled prior to running the tests, based on their conditions"""
+
+ def __init__(self, condition):
+ self.condition = condition
+ self.rules = [ ]
+
+ def AddRule(self, rule):
+ self.rules.append(rule)
+
+
+class Rule(object):
+ """A single rule that specifies the expected outcome for a single
+ test."""
+
+ def __init__(self, raw_path, path, value):
+ self.raw_path = raw_path
+ self.path = path
+ self.value = value
+
+ def GetOutcomes(self, env, defs):
+ return self.value.GetOutcomes(env, defs)
+
+ def Contains(self, path):
+ if len(self.path) > len(path):
+ return False
+ for i in xrange(len(self.path)):
+ if not self.path[i].match(path[i]):
+ return False
+ return True
+
+
+HEADER_PATTERN = re.compile(r'\[([^]]+)\]')
+RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)')
+DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$')
+PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$')
+
+
+class ConvertNotation(object):
+ def __init__(self, path):
+ self.path = path
+ self.indent = ""
+ self.comment = []
+ self.init = False
+ self.section = False
+ self.out = cStringIO.StringIO()
+
+ def OpenGlobal(self):
+ if self.init: return
+ self.WriteComment()
+ print >> self.out, "["
+ self.init = True
+
+ def CloseGlobal(self):
+ if not self.init: return
+ print >> self.out, "]"
+ self.init = False
+
+ def OpenSection(self, condition="ALWAYS"):
+ if self.section: return
+ self.OpenGlobal()
+ if type(condition) != str:
+ condition = "'%s'" % condition.string(True)
+ print >> self.out, "%s[%s, {" % (self.indent, condition)
+ self.indent += " " * 2
+ self.section = condition
+
+ def CloseSection(self):
+ if not self.section: return
+ self.indent = self.indent[:-2]
+ print >> self.out, "%s}], # %s" % (self.indent, self.section)
+ self.section = False
+
+ def WriteComment(self):
+ if not self.comment: return
+ for c in self.comment:
+ if len(c.strip()) == 0:
+ print >> self.out, ""
+ else:
+ print >> self.out, "%s%s" % (self.indent, c),
+ self.comment = []
+
+ def GetOutput(self):
+ with open(self.path) as f:
+ for line in f:
+ if line[0] == '#':
+ self.comment += [line]
+ continue
+ if len(line.strip()) == 0:
+ self.comment += [line]
+ continue
+ header_match = HEADER_PATTERN.match(line)
+ if header_match:
+ condition = ParseCondition(header_match.group(1).strip())
+ self.CloseSection()
+ self.WriteComment()
+ self.OpenSection(condition)
+ continue
+ rule_match = RULE_PATTERN.match(line)
+ if rule_match:
+ self.OpenSection()
+ self.WriteComment()
+ path = rule_match.group(1).strip()
+ value_str = rule_match.group(2).strip()
+ comment = ""
+ if '#' in value_str:
+ pos = value_str.find('#')
+ comment = " %s" % value_str[pos:].strip()
+ value_str = value_str[:pos].strip()
+ value = ParseCondition(value_str)
+ print >> self.out, ("%s'%s': [%s],%s" %
+ (self.indent, path, value, comment))
+ continue
+ def_match = DEF_PATTERN.match(line)
+ if def_match:
+ # Custom definitions are deprecated.
+ continue
+ prefix_match = PREFIX_PATTERN.match(line)
+ if prefix_match:
+ continue
+ print "Malformed line: '%s'." % line
+ self.CloseSection()
+ self.CloseGlobal()
+ result = self.out.getvalue()
+ self.out.close()
+ return result
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import sys
+import time
+
+def EscapeCommand(command):
+ parts = []
+ for part in command:
+ if ' ' in part:
+ # Escape spaces. We may need to escape more characters for this
+ # to work properly.
+ parts.append('"%s"' % part)
+ else:
+ parts.append(part)
+ return " ".join(parts)
+
+
+class ProgressIndicator(object):
+
+ def __init__(self):
+ self.runner = None
+
+ def Starting(self):
+ pass
+
+ def Done(self):
+ pass
+
+ def AboutToRun(self, test):
+ pass
+
+ def HasRun(self, test):
+ pass
+
+ def PrintFailureHeader(self, test):
+ if test.suite.IsNegativeTest(test):
+ negative_marker = '[negative] '
+ else:
+ negative_marker = ''
+ print "=== %(label)s %(negative)s===" % {
+ 'label': test.GetLabel(),
+ 'negative': negative_marker
+ }
+
+
+class SimpleProgressIndicator(ProgressIndicator):
+ """Abstract base class for {Verbose,Dots}ProgressIndicator"""
+
+ def Starting(self):
+ print 'Running %i tests' % self.runner.total
+
+ def Done(self):
+ print
+ for failed in self.runner.failed:
+ self.PrintFailureHeader(failed)
+ if failed.output.stderr:
+ print "--- stderr ---"
+ print failed.output.stderr.strip()
+ if failed.output.stdout:
+ print "--- stdout ---"
+ print failed.output.stdout.strip()
+ print "Command: %s" % EscapeCommand(self.runner.GetCommand(failed))
+ if failed.output.HasCrashed():
+ print "--- CRASHED ---"
+ if failed.output.HasTimedOut():
+ print "--- TIMEOUT ---"
+ if len(self.runner.failed) == 0:
+ print "==="
+ print "=== All tests succeeded"
+ print "==="
+ else:
+ print
+ print "==="
+ print "=== %i tests failed" % len(self.runner.failed)
+ if self.runner.crashed > 0:
+ print "=== %i tests CRASHED" % self.runner.crashed
+ print "==="
+
+
+class VerboseProgressIndicator(SimpleProgressIndicator):
+
+ def AboutToRun(self, test):
+ print 'Starting %s...' % test.GetLabel()
+ sys.stdout.flush()
+
+ def HasRun(self, test):
+ if test.suite.HasUnexpectedOutput(test):
+ if test.output.HasCrashed():
+ outcome = 'CRASH'
+ else:
+ outcome = 'FAIL'
+ else:
+ outcome = 'pass'
+ print 'Done running %s: %s' % (test.GetLabel(), outcome)
+
+
+class DotsProgressIndicator(SimpleProgressIndicator):
+
+ def HasRun(self, test):
+ total = self.runner.succeeded + len(self.runner.failed)
+ if (total > 1) and (total % 50 == 1):
+ sys.stdout.write('\n')
+ if test.suite.HasUnexpectedOutput(test):
+ if test.output.HasCrashed():
+ sys.stdout.write('C')
+ sys.stdout.flush()
+ elif test.output.HasTimedOut():
+ sys.stdout.write('T')
+ sys.stdout.flush()
+ else:
+ sys.stdout.write('F')
+ sys.stdout.flush()
+ else:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+
+
+class CompactProgressIndicator(ProgressIndicator):
+ """Abstract base class for {Color,Monochrome}ProgressIndicator"""
+
+ def __init__(self, templates):
+ super(CompactProgressIndicator, self).__init__()
+ self.templates = templates
+ self.last_status_length = 0
+ self.start_time = time.time()
+
+ def Done(self):
+ self.PrintProgress('Done')
+
+ def AboutToRun(self, test):
+ self.PrintProgress(test.GetLabel())
+
+ def HasRun(self, test):
+ if test.suite.HasUnexpectedOutput(test):
+ self.ClearLine(self.last_status_length)
+ self.PrintFailureHeader(test)
+ stdout = test.output.stdout.strip()
+ if len(stdout):
+ print self.templates['stdout'] % stdout
+ stderr = test.output.stderr.strip()
+ if len(stderr):
+ print self.templates['stderr'] % stderr
+ print "Command: %s" % EscapeCommand(self.runner.GetCommand(test))
+ if test.output.HasCrashed():
+ print "exit code: %d" % test.output.exit_code
+ print "--- CRASHED ---"
+ if test.output.HasTimedOut():
+ print "--- TIMEOUT ---"
+
+ def Truncate(self, string, length):
+ if length and (len(string) > (length - 3)):
+ return string[:(length - 3)] + "..."
+ else:
+ return string
+
+ def PrintProgress(self, name):
+ self.ClearLine(self.last_status_length)
+ elapsed = time.time() - self.start_time
+ status = self.templates['status_line'] % {
+ 'passed': self.runner.succeeded,
+ 'remaining': (((self.runner.total - self.runner.remaining) * 100) //
+ self.runner.total),
+ 'failed': len(self.runner.failed),
+ 'test': name,
+ 'mins': int(elapsed) / 60,
+ 'secs': int(elapsed) % 60
+ }
+ status = self.Truncate(status, 78)
+ self.last_status_length = len(status)
+ print status,
+ sys.stdout.flush()
+
+
+class ColorProgressIndicator(CompactProgressIndicator):
+
+ def __init__(self):
+ templates = {
+ 'status_line': ("[%(mins)02i:%(secs)02i|"
+ "\033[34m%%%(remaining) 4d\033[0m|"
+ "\033[32m+%(passed) 4d\033[0m|"
+ "\033[31m-%(failed) 4d\033[0m]: %(test)s"),
+ 'stdout': "\033[1m%s\033[0m",
+ 'stderr': "\033[31m%s\033[0m",
+ }
+ super(ColorProgressIndicator, self).__init__(templates)
+
+ def ClearLine(self, last_line_length):
+ print "\033[1K\r",
+
+
+class MonochromeProgressIndicator(CompactProgressIndicator):
+
+ def __init__(self):
+ templates = {
+ 'status_line': ("[%(mins)02i:%(secs)02i|%%%(remaining) 4d|"
+ "+%(passed) 4d|-%(failed) 4d]: %(test)s"),
+ 'stdout': '%s',
+ 'stderr': '%s',
+ }
+ super(MonochromeProgressIndicator, self).__init__(templates)
+
+ def ClearLine(self, last_line_length):
+ print ("\r" + (" " * last_line_length) + "\r"),
+
+
+PROGRESS_INDICATORS = {
+ 'verbose': VerboseProgressIndicator,
+ 'dots': DotsProgressIndicator,
+ 'color': ColorProgressIndicator,
+ 'mono': MonochromeProgressIndicator
+}
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# These imports are required for the on-demand conversion from
+# old to new status file format.
+from os.path import exists
+from os.path import getmtime
+
+from . import old_statusfile
+
+
+# These outcomes can occur in a TestCase's outcomes list:
+SKIP = "SKIP"
+FAIL = "FAIL"
+PASS = "PASS"
+OKAY = "OKAY"
+TIMEOUT = "TIMEOUT"
+CRASH = "CRASH"
+SLOW = "SLOW"
+# These are just for the status files and are mapped below in DEFS:
+FAIL_OK = "FAIL_OK"
+PASS_OR_FAIL = "PASS_OR_FAIL"
+
+ALWAYS = "ALWAYS"
+
+KEYWORDS = {}
+for key in [SKIP, FAIL, PASS, OKAY, TIMEOUT, CRASH, SLOW, FAIL_OK,
+ PASS_OR_FAIL, ALWAYS]:
+ KEYWORDS[key] = key
+
+DEFS = {FAIL_OK: [FAIL, OKAY],
+ PASS_OR_FAIL: [PASS, FAIL]}
+
+# Support arches, modes to be written as keywords instead of strings.
+VARIABLES = {ALWAYS: True}
+for var in ["debug", "release", "android_arm", "android_ia32", "arm", "ia32",
+ "mipsel", "x64"]:
+ VARIABLES[var] = var
+
+
+def DoSkip(outcomes):
+ return SKIP in outcomes or SLOW in outcomes
+
+
+def IsFlaky(outcomes):
+ return ((PASS in outcomes) and (FAIL in outcomes) and
+ (not CRASH in outcomes) and (not OKAY in outcomes))
+
+
+def IsFailOk(outcomes):
+ return (FAIL in outcomes) and (OKAY in outcomes)
+
+
+def _AddOutcome(result, new):
+ global DEFS
+ if new in DEFS:
+ mapped = DEFS[new]
+ if type(mapped) == list:
+ for m in mapped:
+ _AddOutcome(result, m)
+ elif type(mapped) == str:
+ _AddOutcome(result, mapped)
+ else:
+ result.add(new)
+
+
+def _ParseOutcomeList(rule, outcomes, target_dict, variables):
+ result = set([])
+ if type(outcomes) == str:
+ outcomes = [outcomes]
+ for item in outcomes:
+ if type(item) == str:
+ _AddOutcome(result, item)
+ elif type(item) == list:
+ if not eval(item[0], variables): continue
+ for outcome in item[1:]:
+ assert type(outcome) == str
+ _AddOutcome(result, outcome)
+ else:
+ assert False
+ if len(result) == 0: return
+ if rule in target_dict:
+ target_dict[rule] |= result
+ else:
+ target_dict[rule] = result
+
+
+def ReadStatusFile(path, variables):
+ # As long as the old-format .status files are authoritative, just
+ # create the converted version on demand and cache it to speed up
+ # subsequent runs.
+ if path.endswith(".status"):
+ newpath = path + "2"
+ if not exists(newpath) or getmtime(newpath) < getmtime(path):
+ print "Converting status file."
+ converted = old_statusfile.ConvertNotation(path).GetOutput()
+ with open(newpath, 'w') as f:
+ f.write(converted)
+ path = newpath
+
+ with open(path) as f:
+ global KEYWORDS
+ contents = eval(f.read(), KEYWORDS)
+
+ rules = {}
+ wildcards = {}
+ variables.update(VARIABLES)
+ for section in contents:
+ assert type(section) == list
+ assert len(section) == 2
+ if not eval(section[0], variables): continue
+ section = section[1]
+ assert type(section) == dict
+ for rule in section:
+ assert type(rule) == str
+ if rule[-1] == '*':
+ _ParseOutcomeList(rule, section[rule], wildcards, variables)
+ else:
+ _ParseOutcomeList(rule, section[rule], rules, variables)
+ return rules, wildcards
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import imp
+import os
+
+from . import statusfile
+
+class TestSuite(object):
+
+ @staticmethod
+ def LoadTestSuite(root):
+ name = root.split(os.path.sep)[-1]
+ f = None
+ try:
+ (f, pathname, description) = imp.find_module("testcfg", [root])
+ module = imp.load_module("testcfg", f, pathname, description)
+ suite = module.GetSuite(name, root)
+ finally:
+ if f:
+ f.close()
+ return suite
+
+ def __init__(self, name, root):
+ self.name = name # string
+ self.root = root # string containing path
+ self.tests = None # list of TestCase objects
+ self.rules = None # dictionary mapping test path to list of outcomes
+ self.wildcards = None # dictionary mapping test paths to list of outcomes
+ self.total_duration = None # float, assigned on demand
+
+ def shell(self):
+ return "d8"
+
+ def suffix(self):
+ return ".js"
+
+ def status_file(self):
+ return "%s/%s.status" % (self.root, self.name)
+
+ # Used in the status file and for stdout printing.
+ def CommonTestName(self, testcase):
+ return testcase.path
+
+ def ListTests(self, context):
+ raise NotImplementedError
+
+ def VariantFlags(self):
+ return None
+
+ def DownloadData(self):
+ pass
+
+ def ReadStatusFile(self, variables):
+ (self.rules, self.wildcards) = \
+ statusfile.ReadStatusFile(self.status_file(), variables)
+
+ def ReadTestCases(self, context):
+ self.tests = self.ListTests(context)
+
+ def FilterTestCasesByStatus(self, warn_unused_rules):
+ filtered = []
+ used_rules = set()
+ for t in self.tests:
+ testname = self.CommonTestName(t)
+ if testname in self.rules:
+ used_rules.add(testname)
+ outcomes = self.rules[testname]
+ t.outcomes = outcomes # Even for skipped tests, as the TestCase
+ # object stays around and PrintReport() uses it.
+ if statusfile.DoSkip(outcomes):
+ continue # Don't add skipped tests to |filtered|.
+ if len(self.wildcards) != 0:
+ for rule in self.wildcards:
+ assert rule[-1] == '*'
+ if testname.startswith(rule[:-1]):
+ used_rules.add(rule)
+ outcomes = self.wildcards[rule]
+ if statusfile.DoSkip(outcomes):
+ continue
+ t.outcomes = outcomes
+ filtered.append(t)
+ self.tests = filtered
+
+ if not warn_unused_rules:
+ return
+
+ for rule in self.rules:
+ if rule not in used_rules:
+ print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
+ for rule in self.wildcards:
+ if rule not in used_rules:
+ print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
+
+ def FilterTestCasesByArgs(self, args):
+ filtered = []
+ filtered_args = []
+ for a in args:
+ argpath = a.split(os.path.sep)
+ if argpath[0] != self.name:
+ continue
+ if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
+ return # Don't filter, run all tests in this suite.
+ path = os.path.sep.join(argpath[1:])
+ if path[-1] == '*':
+ path = path[:-1]
+ filtered_args.append(path)
+ for t in self.tests:
+ for a in filtered_args:
+ if t.path.startswith(a):
+ filtered.append(t)
+ break
+ self.tests = filtered
+
+ def GetFlagsForTestCase(self, testcase, context):
+ raise NotImplementedError
+
+ def GetSourceForTest(self, testcase):
+ return "(no source available)"
+
+ def IsFailureOutput(self, output, testpath):
+ return output.exit_code != 0
+
+ def IsNegativeTest(self, testcase):
+ return False
+
+ def HasFailed(self, testcase):
+ execution_failed = self.IsFailureOutput(testcase.output, testcase.path)
+ if self.IsNegativeTest(testcase):
+ return not execution_failed
+ else:
+ return execution_failed
+
+ def HasUnexpectedOutput(self, testcase):
+ if testcase.output.HasCrashed():
+ outcome = statusfile.CRASH
+ elif testcase.output.HasTimedOut():
+ outcome = statusfile.TIMEOUT
+ elif self.HasFailed(testcase):
+ outcome = statusfile.FAIL
+ else:
+ outcome = statusfile.PASS
+ if not testcase.outcomes:
+ return outcome != statusfile.PASS
+ return not outcome in testcase.outcomes
+
+ def StripOutputForTransmit(self, testcase):
+ if not self.HasUnexpectedOutput(testcase):
+ testcase.output.stdout = ""
+ testcase.output.stderr = ""
+
+ def CalculateTotalDuration(self):
+ self.total_duration = 0.0
+ for t in self.tests:
+ self.total_duration += t.duration
+ return self.total_duration
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+from os.path import exists
+from os.path import isdir
+from os.path import join
+import platform
+import re
+
+
+def GetSuitePaths(test_root):
+ def IsSuite(path):
+ return isdir(path) and exists(join(path, 'testcfg.py'))
+ return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ]
+
+
+# Reads a file into an array of strings
+def ReadLinesFrom(name):
+ lines = []
+ with open(name) as f:
+ for line in f:
+ if line.startswith('#'): continue
+ if '#' in line:
+ line = line[:line.find('#')]
+ line = line.strip()
+ if not line: continue
+ lines.append(line)
+ return lines
+
+
+def GuessOS():
+ system = platform.system()
+ if system == 'Linux':
+ return 'linux'
+ elif system == 'Darwin':
+ return 'macos'
+ elif system.find('CYGWIN') >= 0:
+ return 'cygwin'
+ elif system == 'Windows' or system == 'Microsoft':
+ # On Windows Vista platform.system() can return 'Microsoft' with some
+ # versions of Python, see http://bugs.python.org/issue1082
+ return 'win32'
+ elif system == 'FreeBSD':
+ return 'freebsd'
+ elif system == 'OpenBSD':
+ return 'openbsd'
+ elif system == 'SunOS':
+ return 'solaris'
+ elif system == 'NetBSD':
+ return 'netbsd'
+ else:
+ return None
+
+
+# This will default to building the 32 bit VM even on machines that are
+# capable of running the 64 bit VM.
+def DefaultArch():
+ machine = platform.machine()
+ machine = machine.lower() # Windows 7 capitalizes 'AMD64'.
+ if machine.startswith('arm'):
+ return 'arm'
+ elif (not machine) or (not re.match('(x|i[3-6])86$', machine) is None):
+ return 'ia32'
+ elif machine == 'i86pc':
+ return 'ia32'
+ elif machine == 'x86_64':
+ return 'ia32'
+ elif machine == 'amd64':
+ return 'ia32'
+ else:
+ return None
+
+
+def GuessWordsize():
+ if '64' in platform.machine():
+ return '64'
+ else:
+ return '32'
+
+
+def IsWindows():
+ return GuessOS() == 'win32'
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import sys
+import time
+
+from . import statusfile
+
+
+REPORT_TEMPLATE = (
+"""Total: %(total)i tests
+ * %(skipped)4d tests will be skipped
+ * %(timeout)4d tests are expected to timeout sometimes
+ * %(nocrash)4d tests are expected to be flaky but not crash
+ * %(pass)4d tests are expected to pass
+ * %(fail_ok)4d tests are expected to fail that we won't fix
+ * %(fail)4d tests are expected to fail that we should fix""")
+
+
+def PrintReport(tests):
+ total = len(tests)
+ skipped = timeout = nocrash = passes = fail_ok = fail = 0
+ for t in tests:
+ if "outcomes" not in dir(t) or not t.outcomes:
+ passes += 1
+ continue
+ o = t.outcomes
+ if statusfile.DoSkip(o):
+ skipped += 1
+ continue
+ if statusfile.TIMEOUT in o: timeout += 1
+ if statusfile.IsFlaky(o): nocrash += 1
+ if list(o) == [statusfile.PASS]: passes += 1
+ if statusfile.IsFailOk(o): fail_ok += 1
+ if list(o) == [statusfile.FAIL]: fail += 1
+ print REPORT_TEMPLATE % {
+ "total": total,
+ "skipped": skipped,
+ "timeout": timeout,
+ "nocrash": nocrash,
+ "pass": passes,
+ "fail_ok": fail_ok,
+ "fail": fail
+ }
+
+
+def PrintTestSource(tests):
+ for test in tests:
+ suite = test.suite
+ source = suite.GetSourceForTest(test).strip()
+ if len(source) > 0:
+ print "--- begin source: %s/%s ---" % (suite.name, test.path)
+ print source
+ print "--- end source: %s/%s ---" % (suite.name, test.path)
+
+
+def FormatTime(d):
+ millis = round(d * 1000) % 1000
+ return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis)
+
+
+def PrintTestDurations(suites, overall_time):
+ # Write the times to stderr to make it easy to separate from the
+ # test output.
+ print
+ sys.stderr.write("--- Total time: %s ---\n" % FormatTime(overall_time))
+ timed_tests = [ t for s in suites for t in s.tests
+ if t.duration is not None ]
+ timed_tests.sort(lambda a, b: cmp(b.duration, a.duration))
+ index = 1
+ for entry in timed_tests[:20]:
+ t = FormatTime(entry.duration)
+ sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
+ index += 1
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+class Shell(object):
+ def __init__(self, shell):
+ self.shell = shell
+ self.tests = []
+ self.total_duration = 0.0
+
+ def AddSuite(self, suite):
+ self.tests += suite.tests
+ self.total_duration += suite.total_duration
+
+ def SortTests(self):
+ self.tests.sort(cmp=lambda x, y: cmp(x.duration, y.duration))
+
+
+def Assign(suites, peers):
+ total_work = 0.0
+ for s in suites:
+ total_work += s.CalculateTotalDuration()
+
+ total_power = 0.0
+ for p in peers:
+ p.assigned_work = 0.0
+ total_power += p.jobs * p.relative_performance
+ for p in peers:
+ p.needed_work = total_work * p.jobs * p.relative_performance / total_power
+
+ shells = {}
+ for s in suites:
+ shell = s.shell()
+ if not shell in shells:
+ shells[shell] = Shell(shell)
+ shells[shell].AddSuite(s)
+ # Convert |shells| to list and sort it, shortest total_duration first.
+ shells = [ shells[s] for s in shells ]
+ shells.sort(cmp=lambda x, y: cmp(x.total_duration, y.total_duration))
+ # Sort tests within each shell, longest duration last (so it's
+ # pop()'ed first).
+ for s in shells: s.SortTests()
+ # Sort peers, least needed_work first.
+ peers.sort(cmp=lambda x, y: cmp(x.needed_work, y.needed_work))
+ index = 0
+ for shell in shells:
+ while len(shell.tests) > 0:
+ while peers[index].needed_work <= 0:
+ index += 1
+ if index == len(peers):
+ print("BIG FAT WARNING: Assigning tests to peers failed. "
+ "Remaining tests: %d. Going to slow mode." % len(shell.tests))
+ # Pick the least-busy peer. Sorting the list for each test
+ # is terribly slow, but this is just an emergency fallback anyway.
+ peers.sort(cmp=lambda x, y: cmp(x.needed_work, y.needed_work))
+ peers[0].ForceAddOneTest(shell.tests.pop(), shell)
+ # If the peer already has a shell assigned and would need this one
+ # and then yet another, try to avoid it.
+ peer = peers[index]
+ if (shell.total_duration < peer.needed_work and
+ len(peer.shells) > 0 and
+ index < len(peers) - 1 and
+ shell.total_duration <= peers[index + 1].needed_work):
+ peers[index + 1].AddTests(shell)
+ else:
+ peer.AddTests(shell)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import multiprocessing
+import os
+import Queue
+import threading
+import time
+
+from ..local import execution
+from ..local import progress
+from ..local import testsuite
+from ..local import utils
+from ..server import compression
+
+
+class EndpointProgress(progress.ProgressIndicator):
+ def __init__(self, sock, server, ctx):
+ super(EndpointProgress, self).__init__()
+ self.sock = sock
+ self.server = server
+ self.context = ctx
+ self.results_queue = [] # Accessors must synchronize themselves.
+ self.sender_lock = threading.Lock()
+ self.senderthread = threading.Thread(target=self._SenderThread)
+ self.senderthread.start()
+
+ def HasRun(self, test):
+ # The runners that call this have a lock anyway, so this is safe.
+ self.results_queue.append(test)
+
+ def _SenderThread(self):
+ keep_running = True
+ tests = []
+ self.sender_lock.acquire()
+ while keep_running:
+ time.sleep(0.1)
+ t1 = time.time()
+ # This should be "atomic enough" without locking :-)
+ # (We don't care which list any new elements get appended to, as long
+ # as we don't lose any and the last one comes last.)
+ current = self.results_queue
+ self.results_queue = []
+ for c in current:
+ if c is None:
+ keep_running = False
+ else:
+ tests.append(c)
+ if keep_running and len(tests) < 1:
+ continue # Wait for more results.
+ if len(tests) < 1: break # We're done here.
+ result = []
+ for t in tests:
+ result.append(t.PackResult())
+ compression.Send(result, self.sock)
+ for t in tests:
+ self.server.CompareOwnPerf(t, self.context.arch, self.context.mode)
+ tests = []
+ self.sender_lock.release()
+
+
+def Execute(workspace, ctx, tests, sock, server):
+ suite_paths = utils.GetSuitePaths(os.path.join(workspace, "test"))
+ suites = []
+ for root in suite_paths:
+ suite = testsuite.TestSuite.LoadTestSuite(
+ os.path.join(workspace, "test", root))
+ if suite:
+ suites.append(suite)
+
+ suites_dict = {}
+ for s in suites:
+ suites_dict[s.name] = s
+ s.tests = []
+ for t in tests:
+ suite = suites_dict[t.suite]
+ t.suite = suite
+ suite.tests.append(t)
+
+ suites = [ s for s in suites if len(s.tests) > 0 ]
+ for s in suites:
+ s.DownloadData()
+
+ progress_indicator = EndpointProgress(sock, server, ctx)
+ runner = execution.Runner(suites, progress_indicator, ctx)
+ runner.Run(server.jobs)
+ progress_indicator.HasRun(None) # Sentinel to signal the end.
+ progress_indicator.sender_lock.acquire() # Released when sending is done.
+ progress_indicator.sender_lock.release()
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import socket
+import subprocess
+import threading
+import time
+
+from . import distro
+from . import perfdata
+from ..local import execution
+from ..objects import peer
+from ..objects import workpacket
+from ..server import compression
+from ..server import constants
+from ..server import local_handler
+from ..server import signatures
+
+
+def GetPeers():
+ data = local_handler.LocalQuery([constants.REQUEST_PEERS])
+ if not data: return []
+ return [ peer.Peer.Unpack(p) for p in data ]
+
+
+class NetworkedRunner(execution.Runner):
+ def __init__(self, suites, progress_indicator, context, peers, workspace):
+ self.suites = suites
+ num_tests = 0
+ datapath = os.path.join("out", "testrunner_data")
+ self.perf_data_manager = perfdata.PerfDataManager(datapath)
+ self.perfdata = self.perf_data_manager.GetStore(context.arch, context.mode)
+ for s in suites:
+ for t in s.tests:
+ t.duration = self.perfdata.FetchPerfData(t) or 1.0
+ num_tests += len(s.tests)
+ self._CommonInit(num_tests, progress_indicator, context)
+ self.tests = [] # Only used if we need to fall back to local execution.
+ self.tests_lock = threading.Lock()
+ self.peers = peers
+ self.pubkey_fingerprint = None # Fetched later.
+ self.base_rev = subprocess.check_output(
+ "cd %s; git log -1 --format=%%H --grep=git-svn-id" % workspace,
+ shell=True)
+ self.patch = subprocess.check_output(
+ "cd %s; git diff %s" % (workspace, self.base_rev), shell=True)
+ self.binaries = {}
+ self.initialization_lock = threading.Lock()
+ self.initialization_lock.acquire() # Released when init is done.
+ self._OpenLocalConnection()
+ self.local_receiver_thread = threading.Thread(
+ target=self._ListenLocalConnection)
+ self.local_receiver_thread.daemon = True
+ self.local_receiver_thread.start()
+ self.initialization_lock.acquire()
+ self.initialization_lock.release()
+
+ def _OpenLocalConnection(self):
+ self.local_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ code = self.local_socket.connect_ex(("localhost", constants.CLIENT_PORT))
+ if code != 0:
+ raise RuntimeError("Failed to connect to local server")
+ compression.Send([constants.REQUEST_PUBKEY_FINGERPRINT], self.local_socket)
+
+ def _ListenLocalConnection(self):
+ release_lock_countdown = 1 # Pubkey.
+ self.local_receiver = compression.Receiver(self.local_socket)
+ while not self.local_receiver.IsDone():
+ data = self.local_receiver.Current()
+ if data[0] == constants.REQUEST_PUBKEY_FINGERPRINT:
+ pubkey = data[1]
+ if not pubkey: raise RuntimeError("Received empty public key")
+ self.pubkey_fingerprint = pubkey
+ release_lock_countdown -= 1
+ if release_lock_countdown == 0:
+ self.initialization_lock.release()
+ release_lock_countdown -= 1 # Prevent repeated triggering.
+ self.local_receiver.Advance()
+
+ def Run(self, jobs):
+ self.indicator.Starting()
+ need_libv8 = False
+ for s in self.suites:
+ shell = s.shell()
+ if shell not in self.binaries:
+ path = os.path.join(self.context.shell_dir, shell)
+ # Check if this is a shared library build.
+ try:
+ ldd = subprocess.check_output("ldd %s | grep libv8\\.so" % (path),
+ shell=True)
+ ldd = ldd.strip().split(" ")
+ assert ldd[0] == "libv8.so"
+ assert ldd[1] == "=>"
+ need_libv8 = True
+ binary_needs_libv8 = True
+ libv8 = signatures.ReadFileAndSignature(ldd[2])
+ except:
+ binary_needs_libv8 = False
+ binary = signatures.ReadFileAndSignature(path)
+ if binary[0] is None:
+ print("Error: Failed to create signature.")
+ assert binary[1] != 0
+ return binary[1]
+ binary.append(binary_needs_libv8)
+ self.binaries[shell] = binary
+ if need_libv8:
+ self.binaries["libv8.so"] = libv8
+ distro.Assign(self.suites, self.peers)
+ # Spawn one thread for each peer.
+ threads = []
+ for p in self.peers:
+ thread = threading.Thread(target=self._TalkToPeer, args=[p])
+ threads.append(thread)
+ thread.start()
+ try:
+ for thread in threads:
+ # Use a timeout so that signals (Ctrl+C) will be processed.
+ thread.join(timeout=10000000)
+ self._AnalyzePeerRuntimes()
+ except KeyboardInterrupt:
+ self.terminate = True
+ raise
+ except Exception, _e:
+ # If there's an exception we schedule an interruption for any
+ # remaining threads...
+ self.terminate = True
+ # ...and then reraise the exception to bail out.
+ raise
+ compression.Send(constants.END_OF_STREAM, self.local_socket)
+ self.local_socket.close()
+ if self.tests:
+ self._RunInternal(jobs)
+ self.indicator.Done()
+ return not self.failed
+
+ def _TalkToPeer(self, peer):
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.settimeout(self.context.timeout + 10)
+ code = sock.connect_ex((peer.address, constants.PEER_PORT))
+ if code == 0:
+ try:
+ peer.runtime = None
+ start_time = time.time()
+ packet = workpacket.WorkPacket(peer=peer, context=self.context,
+ base_revision=self.base_rev,
+ patch=self.patch,
+ pubkey=self.pubkey_fingerprint)
+ data, test_map = packet.Pack(self.binaries)
+ compression.Send(data, sock)
+ compression.Send(constants.END_OF_STREAM, sock)
+ rec = compression.Receiver(sock)
+ while not rec.IsDone() and not self.terminate:
+ data_list = rec.Current()
+ for data in data_list:
+ test_id = data[0]
+ if test_id < 0:
+ # The peer is reporting an error.
+ print("Peer %s reports error: %s" % (peer.address, data[1]))
+ rec.Advance()
+ continue
+ test = test_map.pop(test_id)
+ test.MergeResult(data)
+ try:
+ self.perfdata.UpdatePerfData(test)
+ except Exception, e:
+ print("UpdatePerfData exception: %s" % e)
+ pass # Just keep working.
+ with self.lock:
+ perf_key = self.perfdata.GetKey(test)
+ compression.Send(
+ [constants.INFORM_DURATION, perf_key, test.duration,
+ self.context.arch, self.context.mode],
+ self.local_socket)
+ self.indicator.AboutToRun(test)
+ if test.suite.HasUnexpectedOutput(test):
+ self.failed.append(test)
+ if test.output.HasCrashed():
+ self.crashed += 1
+ else:
+ self.succeeded += 1
+ self.remaining -= 1
+ self.indicator.HasRun(test)
+ rec.Advance()
+ peer.runtime = time.time() - start_time
+ except Exception:
+ pass # Fall back to local execution.
+ else:
+ compression.Send([constants.UNRESPONSIVE_PEER, peer.address],
+ self.local_socket)
+ sock.close()
+ if len(test_map) > 0:
+ # Some tests have not received any results. Run them locally.
+ print("No results for %d tests, running them locally." % len(test_map))
+ self._EnqueueLocally(test_map)
+
+ def _EnqueueLocally(self, test_map):
+ with self.tests_lock:
+ for test in test_map:
+ self.tests.append(test_map[test])
+
+ def _AnalyzePeerRuntimes(self):
+ total_runtime = 0.0
+ total_work = 0.0
+ for p in self.peers:
+ if p.runtime is None:
+ return
+ total_runtime += p.runtime
+ total_work += p.assigned_work
+ for p in self.peers:
+ p.assigned_work /= total_work
+ p.runtime /= total_runtime
+ perf_correction = p.assigned_work / p.runtime
+ old_perf = p.relative_performance
+ p.relative_performance = (old_perf + perf_correction) / 2.0
+ compression.Send([constants.UPDATE_PERF, p.address,
+ p.relative_performance],
+ self.local_socket)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import shelve
+import threading
+
+
+class PerfDataEntry(object):
+ def __init__(self):
+ self.avg = 0.0
+ self.count = 0
+
+ def AddResult(self, result):
+ kLearnRateLimiter = 99 # Greater value means slower learning.
+ # We use an approximation of the average of the last 100 results here:
+ # The existing average is weighted with kLearnRateLimiter (or less
+ # if there are fewer data points).
+ effective_count = min(self.count, kLearnRateLimiter)
+ self.avg = self.avg * effective_count + result
+ self.count = effective_count + 1
+ self.avg /= self.count
+
+
+class PerfDataStore(object):
+ def __init__(self, datadir, arch, mode):
+ filename = os.path.join(datadir, "%s.%s.perfdata" % (arch, mode))
+ self.database = shelve.open(filename, protocol=2)
+ self.closed = False
+ self.lock = threading.Lock()
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ if self.closed: return
+ self.database.close()
+ self.closed = True
+
+ def GetKey(self, test):
+ """Computes the key used to access data for the given testcase."""
+ flags = "".join(test.flags)
+ return str("%s.%s.%s" % (test.suitename(), test.path, flags))
+
+ def FetchPerfData(self, test):
+ """Returns the observed duration for |test| as read from the store."""
+ key = self.GetKey(test)
+ if key in self.database:
+ return self.database[key].avg
+ return None
+
+ def UpdatePerfData(self, test):
+ """Updates the persisted value in the store with test.duration."""
+ testkey = self.GetKey(test)
+ self.RawUpdatePerfData(testkey, test.duration)
+
+ def RawUpdatePerfData(self, testkey, duration):
+ with self.lock:
+ if testkey in self.database:
+ entry = self.database[testkey]
+ else:
+ entry = PerfDataEntry()
+ entry.AddResult(duration)
+ self.database[testkey] = entry
+
+
+class PerfDataManager(object):
+ def __init__(self, datadir):
+ self.datadir = os.path.abspath(datadir)
+ if not os.path.exists(self.datadir):
+ os.makedirs(self.datadir)
+ self.stores = {} # Keyed by arch, then mode.
+ self.closed = False
+ self.lock = threading.Lock()
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ if self.closed: return
+ for arch in self.stores:
+ modes = self.stores[arch]
+ for mode in modes:
+ store = modes[mode]
+ store.close()
+ self.closed = True
+
+ def GetStore(self, arch, mode):
+ with self.lock:
+ if not arch in self.stores:
+ self.stores[arch] = {}
+ modes = self.stores[arch]
+ if not mode in modes:
+ modes[mode] = PerfDataStore(self.datadir, arch, mode)
+ return modes[mode]
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+class Context():
+ def __init__(self, arch, mode, shell_dir, mode_flags, verbose, timeout,
+ isolates, command_prefix, extra_flags):
+ self.arch = arch
+ self.mode = mode
+ self.shell_dir = shell_dir
+ self.mode_flags = mode_flags
+ self.verbose = verbose
+ self.timeout = timeout
+ self.isolates = isolates
+ self.command_prefix = command_prefix
+ self.extra_flags = extra_flags
+
+ def Pack(self):
+ return [self.arch, self.mode, self.mode_flags, self.timeout, self.isolates,
+ self.extra_flags]
+
+ @staticmethod
+ def Unpack(packed):
+ # For the order of the fields, refer to Pack() above.
+ return Context(packed[0], packed[1], None, packed[2], False,
+ packed[3], packed[4], "", packed[5])
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import signal
+
+from ..local import utils
+
+class Output(object):
+
+ def __init__(self, exit_code, timed_out, stdout, stderr):
+ self.exit_code = exit_code
+ self.timed_out = timed_out
+ self.stdout = stdout
+ self.stderr = stderr
+
+ def HasCrashed(self):
+ if utils.IsWindows():
+ return 0x80000000 & self.exit_code and not (0x3FFFFF00 & self.exit_code)
+ else:
+ # Timed out tests will have exit_code -signal.SIGTERM.
+ if self.timed_out:
+ return False
+ return (self.exit_code < 0 and
+ self.exit_code != -signal.SIGABRT)
+
+ def HasTimedOut(self):
+ return self.timed_out
+
+ def Pack(self):
+ return [self.exit_code, self.timed_out, self.stdout, self.stderr]
+
+ @staticmethod
+ def Unpack(packed):
+ # For the order of the fields, refer to Pack() above.
+ return Output(packed[0], packed[1], packed[2], packed[3])
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+class Peer(object):
+ def __init__(self, address, jobs, rel_perf, pubkey):
+ self.address = address # string: IP address
+ self.jobs = jobs # integer: number of CPUs
+ self.relative_performance = rel_perf
+ self.pubkey = pubkey # string: pubkey's fingerprint
+ self.shells = set() # set of strings
+ self.needed_work = 0
+ self.assigned_work = 0
+ self.tests = [] # list of TestCase objects
+ self.trusting_me = False # This peer trusts my public key.
+ self.trusted = False # I trust this peer's public key.
+
+ def __str__(self):
+ return ("Peer at %s, jobs: %d, performance: %.2f, trust I/O: %s/%s" %
+ (self.address, self.jobs, self.relative_performance,
+ self.trusting_me, self.trusted))
+
+ def AddTests(self, shell):
+ """Adds tests from |shell| to this peer.
+
+ Stops when self.needed_work reaches zero, or when all of shell's tests
+ are assigned."""
+ assert self.needed_work > 0
+ if shell.shell not in self.shells:
+ self.shells.add(shell.shell)
+ while len(shell.tests) > 0 and self.needed_work > 0:
+ t = shell.tests.pop()
+ self.needed_work -= t.duration
+ self.assigned_work += t.duration
+ shell.total_duration -= t.duration
+ self.tests.append(t)
+
+ def ForceAddOneTest(self, test, shell):
+ """Forcibly adds another test to this peer, disregarding needed_work."""
+ if shell.shell not in self.shells:
+ self.shells.add(shell.shell)
+ self.needed_work -= test.duration
+ self.assigned_work += test.duration
+ shell.total_duration -= test.duration
+ self.tests.append(test)
+
+
+ def Pack(self):
+ """Creates a JSON serializable representation of this Peer."""
+ return [self.address, self.jobs, self.relative_performance]
+
+ @staticmethod
+ def Unpack(packed):
+ """Creates a Peer object built from a packed representation."""
+ pubkey_dummy = "" # Callers of this don't care (only the server does).
+ return Peer(packed[0], packed[1], packed[2], pubkey_dummy)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+from . import output
+
+class TestCase(object):
+ def __init__(self, suite, path, flags=[], dependency=None):
+ self.suite = suite # TestSuite object
+ self.path = path # string, e.g. 'div-mod', 'test-api/foo'
+ self.flags = flags # list of strings, flags specific to this test case
+ self.dependency = dependency # |path| for testcase that must be run first
+ self.outcomes = None
+ self.output = None
+ self.id = None # int, used to map result back to TestCase instance
+ self.duration = None # assigned during execution
+
+ def CopyAddingFlags(self, flags):
+ copy = TestCase(self.suite, self.path, self.flags + flags, self.dependency)
+ copy.outcomes = self.outcomes
+ return copy
+
+ def PackTask(self):
+ """
+ Extracts those parts of this object that are required to run the test
+ and returns them as a JSON serializable object.
+ """
+ assert self.id is not None
+ return [self.suitename(), self.path, self.flags,
+ self.dependency, list(self.outcomes or []), self.id]
+
+ @staticmethod
+ def UnpackTask(task):
+ """Creates a new TestCase object based on packed task data."""
+ # For the order of the fields, refer to PackTask() above.
+ test = TestCase(str(task[0]), task[1], task[2], task[3])
+ test.outcomes = set(task[4])
+ test.id = task[5]
+ return test
+
+ def SetSuiteObject(self, suites):
+ self.suite = suites[self.suite]
+
+ def PackResult(self):
+ """Serializes the output of the TestCase after it has run."""
+ self.suite.StripOutputForTransmit(self)
+ return [self.id, self.output.Pack(), self.duration]
+
+ def MergeResult(self, result):
+ """Applies the contents of a Result to this object."""
+ assert result[0] == self.id
+ self.output = output.Output.Unpack(result[1])
+ self.duration = result[2]
+
+ def suitename(self):
+ return self.suite.name
+
+ def GetLabel(self):
+ return self.suitename() + "/" + self.suite.CommonTestName(self)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+from . import context
+from . import testcase
+
+class WorkPacket(object):
+ def __init__(self, peer=None, context=None, tests=None, binaries=None,
+ base_revision=None, patch=None, pubkey=None):
+ self.peer = peer
+ self.context = context
+ self.tests = tests
+ self.binaries = binaries
+ self.base_revision = base_revision
+ self.patch = patch
+ self.pubkey_fingerprint = pubkey
+
+ def Pack(self, binaries_dict):
+ """
+ Creates a JSON serializable object containing the data of this
+ work packet.
+ """
+ need_libv8 = False
+ binaries = []
+ for shell in self.peer.shells:
+ prefetched_binary = binaries_dict[shell]
+ binaries.append({"name": shell,
+ "blob": prefetched_binary[0],
+ "sign": prefetched_binary[1]})
+ if prefetched_binary[2]:
+ need_libv8 = True
+ if need_libv8:
+ libv8 = binaries_dict["libv8.so"]
+ binaries.append({"name": "libv8.so",
+ "blob": libv8[0],
+ "sign": libv8[1]})
+ tests = []
+ test_map = {}
+ for t in self.peer.tests:
+ test_map[t.id] = t
+ tests.append(t.PackTask())
+ result = {
+ "binaries": binaries,
+ "pubkey": self.pubkey_fingerprint,
+ "context": self.context.Pack(),
+ "base_revision": self.base_revision,
+ "patch": self.patch,
+ "tests": tests
+ }
+ return result, test_map
+
+ @staticmethod
+ def Unpack(packed):
+ """
+ Creates a WorkPacket object from the given packed representation.
+ """
+ binaries = packed["binaries"]
+ pubkey_fingerprint = packed["pubkey"]
+ ctx = context.Context.Unpack(packed["context"])
+ base_revision = packed["base_revision"]
+ patch = packed["patch"]
+ tests = [ testcase.TestCase.UnpackTask(t) for t in packed["tests"] ]
+ return WorkPacket(context=ctx, tests=tests, binaries=binaries,
+ base_revision=base_revision, patch=patch,
+ pubkey=pubkey_fingerprint)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import cStringIO as StringIO
+try:
+ import ujson as json
+except ImportError:
+ print("You should install UltraJSON, it is much faster!")
+ import json
+import os
+import struct
+import zlib
+
+from . import constants
+
+def Send(obj, sock):
+ """
+ Sends a JSON encodable object over the specified socket (zlib-compressed).
+ """
+ obj = json.dumps(obj)
+ compression_level = 2 # 1 = fastest, 9 = best compression
+ compressed = zlib.compress(obj, compression_level)
+ payload = struct.pack('>i', len(compressed)) + compressed
+ sock.sendall(payload)
+
+
+class Receiver(object):
+ def __init__(self, sock):
+ self.sock = sock
+ self.data = StringIO.StringIO()
+ self.datalength = 0
+ self._next = self._GetNext()
+
+ def IsDone(self):
+ return self._next == None
+
+ def Current(self):
+ return self._next
+
+ def Advance(self):
+ try:
+ self._next = self._GetNext()
+ except:
+ raise
+
+ def _GetNext(self):
+ try:
+ while self.datalength < constants.SIZE_T:
+ try:
+ chunk = self.sock.recv(8192)
+ except:
+ raise
+ if not chunk: return None
+ self._AppendData(chunk)
+ size = self._PopData(constants.SIZE_T)
+ size = struct.unpack(">i", size)[0]
+ while self.datalength < size:
+ try:
+ chunk = self.sock.recv(8192)
+ except:
+ raise
+ if not chunk: return None
+ self._AppendData(chunk)
+ result = self._PopData(size)
+ result = zlib.decompress(result)
+ result = json.loads(result)
+ if result == constants.END_OF_STREAM:
+ return None
+ return result
+ except:
+ raise
+
+ def _AppendData(self, new):
+ self.data.seek(0, os.SEEK_END)
+ self.data.write(new)
+ self.datalength += len(new)
+
+ def _PopData(self, length):
+ self.data.seek(0)
+ chunk = self.data.read(length)
+ remaining = self.data.read()
+ self.data.close()
+ self.data = StringIO.StringIO()
+ self.data.write(remaining)
+ assert self.datalength - length == len(remaining)
+ self.datalength = len(remaining)
+ return chunk
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+CLIENT_PORT = 9991 # Port for the local client to connect to.
+PEER_PORT = 9992 # Port for peers on the network to connect to.
+PRESENCE_PORT = 9993 # Port for presence daemon.
+STATUS_PORT = 9994 # Port for network requests not related to workpackets.
+
+END_OF_STREAM = "end of dtest stream" # Marker for end of network requests.
+SIZE_T = 4 # Number of bytes used for network request size header.
+
+# Messages understood by the local request handler.
+ADD_TRUSTED = "add trusted"
+INFORM_DURATION = "inform about duration"
+REQUEST_PEERS = "get peers"
+UNRESPONSIVE_PEER = "unresponsive peer"
+REQUEST_PUBKEY_FINGERPRINT = "get pubkey fingerprint"
+REQUEST_STATUS = "get status"
+UPDATE_PERF = "update performance"
+
+# Messages understood by the status request handler.
+LIST_TRUSTED_PUBKEYS = "list trusted pubkeys"
+GET_SIGNED_PUBKEY = "pass on signed pubkey"
+NOTIFY_NEW_TRUSTED = "new trusted peer"
+TRUST_YOU_NOW = "trust you now"
+DO_YOU_TRUST = "do you trust"
--- /dev/null
+#!/usr/bin/env python
+
+# This code has been written by Sander Marechal and published at:
+# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/
+# where the author has placed it in the public domain (see comment #6 at
+# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/#c6
+# ).
+# Some minor modifications have been made by the V8 authors. The work remains
+# in the public domain.
+
+import atexit
+import os
+from signal import SIGTERM
+from signal import SIGINT
+import sys
+import time
+
+
+class Daemon(object):
+ """
+ A generic daemon class.
+
+ Usage: subclass the Daemon class and override the run() method
+ """
+ def __init__(self, pidfile, stdin='/dev/null',
+ stdout='/dev/null', stderr='/dev/null'):
+ self.stdin = stdin
+ self.stdout = stdout
+ self.stderr = stderr
+ self.pidfile = pidfile
+
+ def daemonize(self):
+ """
+ do the UNIX double-fork magic, see Stevens' "Advanced
+ Programming in the UNIX Environment" for details (ISBN 0201563177)
+ http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
+ """
+ try:
+ pid = os.fork()
+ if pid > 0:
+ # exit first parent
+ sys.exit(0)
+ except OSError, e:
+ sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
+ sys.exit(1)
+
+ # decouple from parent environment
+ os.chdir("/")
+ os.setsid()
+ os.umask(0)
+
+ # do second fork
+ try:
+ pid = os.fork()
+ if pid > 0:
+ # exit from second parent
+ sys.exit(0)
+ except OSError, e:
+ sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
+ sys.exit(1)
+
+ # redirect standard file descriptors
+ sys.stdout.flush()
+ sys.stderr.flush()
+ si = file(self.stdin, 'r')
+ so = file(self.stdout, 'a+')
+ se = file(self.stderr, 'a+', 0)
+ # TODO: (debug) re-enable this!
+ #os.dup2(si.fileno(), sys.stdin.fileno())
+ #os.dup2(so.fileno(), sys.stdout.fileno())
+ #os.dup2(se.fileno(), sys.stderr.fileno())
+
+ # write pidfile
+ atexit.register(self.delpid)
+ pid = str(os.getpid())
+ file(self.pidfile, 'w+').write("%s\n" % pid)
+
+ def delpid(self):
+ os.remove(self.pidfile)
+
+ def start(self):
+ """
+ Start the daemon
+ """
+ # Check for a pidfile to see if the daemon already runs
+ try:
+ pf = file(self.pidfile, 'r')
+ pid = int(pf.read().strip())
+ pf.close()
+ except IOError:
+ pid = None
+
+ if pid:
+ message = "pidfile %s already exist. Daemon already running?\n"
+ sys.stderr.write(message % self.pidfile)
+ sys.exit(1)
+
+ # Start the daemon
+ self.daemonize()
+ self.run()
+
+ def stop(self):
+ """
+ Stop the daemon
+ """
+ # Get the pid from the pidfile
+ try:
+ pf = file(self.pidfile, 'r')
+ pid = int(pf.read().strip())
+ pf.close()
+ except IOError:
+ pid = None
+
+ if not pid:
+ message = "pidfile %s does not exist. Daemon not running?\n"
+ sys.stderr.write(message % self.pidfile)
+ return # not an error in a restart
+
+ # Try killing the daemon process
+ try:
+ # Give the process a one-second chance to exit gracefully.
+ os.kill(pid, SIGINT)
+ time.sleep(1)
+ while 1:
+ os.kill(pid, SIGTERM)
+ time.sleep(0.1)
+ except OSError, err:
+ err = str(err)
+ if err.find("No such process") > 0:
+ if os.path.exists(self.pidfile):
+ os.remove(self.pidfile)
+ else:
+ print str(err)
+ sys.exit(1)
+
+ def restart(self):
+ """
+ Restart the daemon
+ """
+ self.stop()
+ self.start()
+
+ def run(self):
+ """
+ You should override this method when you subclass Daemon. It will be
+ called after the process has been daemonized by start() or restart().
+ """
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import socket
+import SocketServer
+import StringIO
+
+from . import compression
+from . import constants
+
+
+def LocalQuery(query):
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ code = sock.connect_ex(("localhost", constants.CLIENT_PORT))
+ if code != 0: return None
+ compression.Send(query, sock)
+ compression.Send(constants.END_OF_STREAM, sock)
+ rec = compression.Receiver(sock)
+ data = None
+ while not rec.IsDone():
+ data = rec.Current()
+ assert data[0] == query[0]
+ data = data[1]
+ rec.Advance()
+ sock.close()
+ return data
+
+
+class LocalHandler(SocketServer.BaseRequestHandler):
+ def handle(self):
+ rec = compression.Receiver(self.request)
+ while not rec.IsDone():
+ data = rec.Current()
+ action = data[0]
+
+ if action == constants.REQUEST_PEERS:
+ with self.server.daemon.peer_list_lock:
+ response = [ p.Pack() for p in self.server.daemon.peers
+ if p.trusting_me ]
+ compression.Send([action, response], self.request)
+
+ elif action == constants.UNRESPONSIVE_PEER:
+ self.server.daemon.DeletePeer(data[1])
+
+ elif action == constants.REQUEST_PUBKEY_FINGERPRINT:
+ compression.Send([action, self.server.daemon.pubkey_fingerprint],
+ self.request)
+
+ elif action == constants.REQUEST_STATUS:
+ compression.Send([action, self._GetStatusMessage()], self.request)
+
+ elif action == constants.ADD_TRUSTED:
+ fingerprint = self.server.daemon.CopyToTrusted(data[1])
+ compression.Send([action, fingerprint], self.request)
+
+ elif action == constants.INFORM_DURATION:
+ test_key = data[1]
+ test_duration = data[2]
+ arch = data[3]
+ mode = data[4]
+ self.server.daemon.AddPerfData(test_key, test_duration, arch, mode)
+
+ elif action == constants.UPDATE_PERF:
+ address = data[1]
+ perf = data[2]
+ self.server.daemon.UpdatePeerPerformance(data[1], data[2])
+
+ rec.Advance()
+ compression.Send(constants.END_OF_STREAM, self.request)
+
+ def _GetStatusMessage(self):
+ sio = StringIO.StringIO()
+ sio.write("Peers:\n")
+ with self.server.daemon.peer_list_lock:
+ for p in self.server.daemon.peers:
+ sio.write("%s\n" % p)
+ sio.write("My own jobs: %d, relative performance: %.2f\n" %
+ (self.server.daemon.jobs, self.server.daemon.relative_perf))
+ # Low-priority TODO: Return more information. Ideas:
+ # - currently running anything,
+ # - time since last job,
+ # - time since last repository fetch
+ # - number of workpackets/testcases handled since startup
+ # - slowest test(s)
+ result = sio.getvalue()
+ sio.close()
+ return result
+
+
+class LocalSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+ def __init__(self, daemon):
+ SocketServer.TCPServer.__init__(self, ("localhost", constants.CLIENT_PORT),
+ LocalHandler)
+ self.daemon = daemon
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import multiprocessing
+import os
+import shutil
+import subprocess
+import threading
+import time
+
+from . import daemon
+from . import discovery
+from . import local_handler
+from . import signatures
+from . import status_handler
+from . import work_handler
+from ..network import perfdata
+
+
+class Server(daemon.Daemon):
+
+ def __init__(self, pidfile, root, stdin="/dev/null",
+ stdout="/dev/null", stderr="/dev/null"):
+ super(Server, self).__init__(pidfile, stdin, stdout, stderr)
+ self.root = root
+ self.local_handler = None
+ self.local_handler_thread = None
+ self.work_handler = None
+ self.work_handler_thread = None
+ self.status_handler = None
+ self.status_handler_thread = None
+ self.presence_daemon = None
+ self.presence_daemon_thread = None
+ self.peers = []
+ self.jobs = multiprocessing.cpu_count()
+ self.peer_list_lock = threading.Lock()
+ self.perf_data_lock = None
+ self.presence_daemon_lock = None
+ self.datadir = os.path.join(self.root, "data")
+ pubkey_fingerprint_filename = os.path.join(self.datadir, "mypubkey")
+ with open(pubkey_fingerprint_filename) as f:
+ self.pubkey_fingerprint = f.read().strip()
+ self.relative_perf_filename = os.path.join(self.datadir, "myperf")
+ if os.path.exists(self.relative_perf_filename):
+ with open(self.relative_perf_filename) as f:
+ try:
+ self.relative_perf = float(f.read())
+ except:
+ self.relative_perf = 1.0
+ else:
+ self.relative_perf = 1.0
+
+ def run(self):
+ os.nice(20)
+ self.ip = discovery.GetOwnIP()
+ self.perf_data_manager = perfdata.PerfDataManager(self.datadir)
+ self.perf_data_lock = threading.Lock()
+
+ self.local_handler = local_handler.LocalSocketServer(self)
+ self.local_handler_thread = threading.Thread(
+ target=self.local_handler.serve_forever)
+ self.local_handler_thread.start()
+
+ self.work_handler = work_handler.WorkSocketServer(self)
+ self.work_handler_thread = threading.Thread(
+ target=self.work_handler.serve_forever)
+ self.work_handler_thread.start()
+
+ self.status_handler = status_handler.StatusSocketServer(self)
+ self.status_handler_thread = threading.Thread(
+ target=self.status_handler.serve_forever)
+ self.status_handler_thread.start()
+
+ self.presence_daemon = discovery.PresenceDaemon(self)
+ self.presence_daemon_thread = threading.Thread(
+ target=self.presence_daemon.serve_forever)
+ self.presence_daemon_thread.start()
+
+ self.presence_daemon.FindPeers()
+ time.sleep(0.5) # Give those peers some time to reply.
+
+ with self.peer_list_lock:
+ for p in self.peers:
+ if p.address == self.ip: continue
+ status_handler.RequestTrustedPubkeys(p, self)
+
+ while True:
+ try:
+ self.PeriodicTasks()
+ time.sleep(60)
+ except Exception, e:
+ print("MAIN LOOP EXCEPTION: %s" % e)
+ self.Shutdown()
+ break
+ except KeyboardInterrupt:
+ self.Shutdown()
+ break
+
+ def Shutdown(self):
+ with open(self.relative_perf_filename, "w") as f:
+ f.write("%s" % self.relative_perf)
+ self.presence_daemon.shutdown()
+ self.presence_daemon.server_close()
+ self.local_handler.shutdown()
+ self.local_handler.server_close()
+ self.work_handler.shutdown()
+ self.work_handler.server_close()
+ self.status_handler.shutdown()
+ self.status_handler.server_close()
+
+ def PeriodicTasks(self):
+ # If we know peers we don't trust, see if someone else trusts them.
+ with self.peer_list_lock:
+ for p in self.peers:
+ if p.trusted: continue
+ if self.IsTrusted(p.pubkey):
+ p.trusted = True
+ status_handler.ITrustYouNow(p)
+ continue
+ for p2 in self.peers:
+ if not p2.trusted: continue
+ status_handler.TryTransitiveTrust(p2, p.pubkey, self)
+ # TODO: Ping for more peers waiting to be discovered.
+ # TODO: Update the checkout (if currently idle).
+
+ def AddPeer(self, peer):
+ with self.peer_list_lock:
+ for p in self.peers:
+ if p.address == peer.address:
+ return
+ self.peers.append(peer)
+ if peer.trusted:
+ status_handler.ITrustYouNow(peer)
+
+ def DeletePeer(self, peer_address):
+ with self.peer_list_lock:
+ for i in xrange(len(self.peers)):
+ if self.peers[i].address == peer_address:
+ del self.peers[i]
+ return
+
+ def MarkPeerAsTrusting(self, peer_address):
+ with self.peer_list_lock:
+ for p in self.peers:
+ if p.address == peer_address:
+ p.trusting_me = True
+ break
+
+ def UpdatePeerPerformance(self, peer_address, performance):
+ with self.peer_list_lock:
+ for p in self.peers:
+ if p.address == peer_address:
+ p.relative_performance = performance
+
+ def CopyToTrusted(self, pubkey_filename):
+ with open(pubkey_filename, "r") as f:
+ lines = f.readlines()
+ fingerprint = lines[-1].strip()
+ target_filename = self._PubkeyFilename(fingerprint)
+ shutil.copy(pubkey_filename, target_filename)
+ with self.peer_list_lock:
+ for peer in self.peers:
+ if peer.address == self.ip: continue
+ if peer.pubkey == fingerprint:
+ status_handler.ITrustYouNow(peer)
+ else:
+ result = self.SignTrusted(fingerprint)
+ status_handler.NotifyNewTrusted(peer, result)
+ return fingerprint
+
+ def _PubkeyFilename(self, pubkey_fingerprint):
+ return os.path.join(self.root, "trusted", "%s.pem" % pubkey_fingerprint)
+
+ def IsTrusted(self, pubkey_fingerprint):
+ return os.path.exists(self._PubkeyFilename(pubkey_fingerprint))
+
+ def ListTrusted(self):
+ path = os.path.join(self.root, "trusted")
+ if not os.path.exists(path): return []
+ return [ f[:-4] for f in os.listdir(path) if f.endswith(".pem") ]
+
+ def SignTrusted(self, pubkey_fingerprint):
+ if not self.IsTrusted(pubkey_fingerprint):
+ return []
+ filename = self._PubkeyFilename(pubkey_fingerprint)
+ result = signatures.ReadFileAndSignature(filename) # Format: [key, sig].
+ return [pubkey_fingerprint, result[0], result[1], self.pubkey_fingerprint]
+
+ def AcceptNewTrusted(self, data):
+ # The format of |data| matches the return value of |SignTrusted()|.
+ if not data: return
+ fingerprint = data[0]
+ pubkey = data[1]
+ signature = data[2]
+ signer = data[3]
+ if not self.IsTrusted(signer):
+ return
+ if self.IsTrusted(fingerprint):
+ return # Already trust this guy.
+ filename = self._PubkeyFilename(fingerprint)
+ signer_pubkeyfile = self._PubkeyFilename(signer)
+ if not signatures.VerifySignature(filename, pubkey, signature,
+ signer_pubkeyfile):
+ return
+ return # Nothing more to do.
+
+ def AddPerfData(self, test_key, duration, arch, mode):
+ data_store = self.perf_data_manager.GetStore(arch, mode)
+ data_store.RawUpdatePerfData(str(test_key), duration)
+
+ def CompareOwnPerf(self, test, arch, mode):
+ data_store = self.perf_data_manager.GetStore(arch, mode)
+ observed = data_store.FetchPerfData(test)
+ if not observed: return
+ own_perf_estimate = observed / test.duration
+ with self.perf_data_lock:
+ kLearnRateLimiter = 9999
+ self.relative_perf *= kLearnRateLimiter
+ self.relative_perf += own_perf_estimate
+ self.relative_perf /= (kLearnRateLimiter + 1)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import socket
+import SocketServer
+import threading
+try:
+ import ujson as json
+except:
+ import json
+
+from . import constants
+from ..objects import peer
+
+
+STARTUP_REQUEST = "V8 test peer starting up"
+STARTUP_RESPONSE = "Let's rock some tests!"
+EXIT_REQUEST = "V8 testing peer going down"
+
+
+def GetOwnIP():
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ s.connect(("8.8.8.8", 80))
+ ip = s.getsockname()[0]
+ s.close()
+ return ip
+
+
+class PresenceHandler(SocketServer.BaseRequestHandler):
+
+ def handle(self):
+ data = json.loads(self.request[0].strip())
+
+ if data[0] == STARTUP_REQUEST:
+ jobs = data[1]
+ relative_perf = data[2]
+ pubkey_fingerprint = data[3]
+ trusted = self.server.daemon.IsTrusted(pubkey_fingerprint)
+ response = [STARTUP_RESPONSE, self.server.daemon.jobs,
+ self.server.daemon.relative_perf,
+ self.server.daemon.pubkey_fingerprint, trusted]
+ response = json.dumps(response)
+ self.server.SendTo(self.client_address[0], response)
+ p = peer.Peer(self.client_address[0], jobs, relative_perf,
+ pubkey_fingerprint)
+ p.trusted = trusted
+ self.server.daemon.AddPeer(p)
+
+ elif data[0] == STARTUP_RESPONSE:
+ jobs = data[1]
+ perf = data[2]
+ pubkey_fingerprint = data[3]
+ p = peer.Peer(self.client_address[0], jobs, perf, pubkey_fingerprint)
+ p.trusted = self.server.daemon.IsTrusted(pubkey_fingerprint)
+ p.trusting_me = data[4]
+ self.server.daemon.AddPeer(p)
+
+ elif data[0] == EXIT_REQUEST:
+ self.server.daemon.DeletePeer(self.client_address[0])
+ if self.client_address[0] == self.server.daemon.ip:
+ self.server.shutdown_lock.release()
+
+
+class PresenceDaemon(SocketServer.ThreadingMixIn, SocketServer.UDPServer):
+ def __init__(self, daemon):
+ self.daemon = daemon
+ address = (daemon.ip, constants.PRESENCE_PORT)
+ SocketServer.UDPServer.__init__(self, address, PresenceHandler)
+ self.shutdown_lock = threading.Lock()
+
+ def shutdown(self):
+ self.shutdown_lock.acquire()
+ self.SendToAll(json.dumps([EXIT_REQUEST]))
+ self.shutdown_lock.acquire()
+ self.shutdown_lock.release()
+ SocketServer.UDPServer.shutdown(self)
+
+ def SendTo(self, target, message):
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ sock.sendto(message, (target, constants.PRESENCE_PORT))
+ sock.close()
+
+ def SendToAll(self, message):
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ ip = self.daemon.ip.split(".")
+ for i in range(1, 254):
+ ip[-1] = str(i)
+ sock.sendto(message, (".".join(ip), constants.PRESENCE_PORT))
+ sock.close()
+
+ def FindPeers(self):
+ request = [STARTUP_REQUEST, self.daemon.jobs, self.daemon.relative_perf,
+ self.daemon.pubkey_fingerprint]
+ request = json.dumps(request)
+ self.SendToAll(request)
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import base64
+import os
+import subprocess
+
+
+def ReadFileAndSignature(filename):
+ with open(filename, "rb") as f:
+ file_contents = base64.b64encode(f.read())
+ signature_file = filename + ".signature"
+ if (not os.path.exists(signature_file) or
+ os.path.getmtime(signature_file) < os.path.getmtime(filename)):
+ private_key = "~/.ssh/v8_dtest"
+ code = subprocess.call("openssl dgst -out %s -sign %s %s" %
+ (signature_file, private_key, filename),
+ shell=True)
+ if code != 0: return [None, code]
+ with open(signature_file) as f:
+ signature = base64.b64encode(f.read())
+ return [file_contents, signature]
+
+
+def VerifySignature(filename, file_contents, signature, pubkeyfile):
+ with open(filename, "wb") as f:
+ f.write(base64.b64decode(file_contents))
+ signature_file = filename + ".foreign_signature"
+ with open(signature_file, "wb") as f:
+ f.write(base64.b64decode(signature))
+ code = subprocess.call("openssl dgst -verify %s -signature %s %s" %
+ (pubkeyfile, signature_file, filename),
+ shell=True)
+ matched = (code == 0)
+ if not matched:
+ os.remove(signature_file)
+ os.remove(filename)
+ return matched
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import socket
+import SocketServer
+
+from . import compression
+from . import constants
+from . import discovery
+
+
+def _StatusQuery(peer, query):
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ code = sock.connect_ex((peer.address, constants.STATUS_PORT))
+ if code != 0:
+ # TODO(jkummerow): disconnect (after 3 failures?)
+ return
+ compression.Send(query, sock)
+ compression.Send(constants.END_OF_STREAM, sock)
+ rec = compression.Receiver(sock)
+ data = None
+ while not rec.IsDone():
+ data = rec.Current()
+ assert data[0] == query[0]
+ data = data[1]
+ rec.Advance()
+ sock.close()
+ return data
+
+
+def RequestTrustedPubkeys(peer, server):
+ pubkey_list = _StatusQuery(peer, [constants.LIST_TRUSTED_PUBKEYS])
+ for pubkey in pubkey_list:
+ if server.IsTrusted(pubkey): continue
+ result = _StatusQuery(peer, [constants.GET_SIGNED_PUBKEY, pubkey])
+ server.AcceptNewTrusted(result)
+
+
+def NotifyNewTrusted(peer, data):
+ _StatusQuery(peer, [constants.NOTIFY_NEW_TRUSTED] + data)
+
+
+def ITrustYouNow(peer):
+ _StatusQuery(peer, [constants.TRUST_YOU_NOW])
+
+
+def TryTransitiveTrust(peer, pubkey, server):
+ if _StatusQuery(peer, [constants.DO_YOU_TRUST, pubkey]):
+ result = _StatusQuery(peer, [constants.GET_SIGNED_PUBKEY, pubkey])
+ server.AcceptNewTrusted(result)
+
+
+class StatusHandler(SocketServer.BaseRequestHandler):
+ def handle(self):
+ rec = compression.Receiver(self.request)
+ while not rec.IsDone():
+ data = rec.Current()
+ action = data[0]
+
+ if action == constants.LIST_TRUSTED_PUBKEYS:
+ response = self.server.daemon.ListTrusted()
+ compression.Send([action, response], self.request)
+
+ elif action == constants.GET_SIGNED_PUBKEY:
+ response = self.server.daemon.SignTrusted(data[1])
+ compression.Send([action, response], self.request)
+
+ elif action == constants.NOTIFY_NEW_TRUSTED:
+ self.server.daemon.AcceptNewTrusted(data[1:])
+ pass # No response.
+
+ elif action == constants.TRUST_YOU_NOW:
+ self.server.daemon.MarkPeerAsTrusting(self.client_address[0])
+ pass # No response.
+
+ elif action == constants.DO_YOU_TRUST:
+ response = self.server.daemon.IsTrusted(data[1])
+ compression.Send([action, response], self.request)
+
+ rec.Advance()
+ compression.Send(constants.END_OF_STREAM, self.request)
+
+
+class StatusSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+ def __init__(self, daemon):
+ address = (daemon.ip, constants.STATUS_PORT)
+ SocketServer.TCPServer.__init__(self, address, StatusHandler)
+ self.daemon = daemon
--- /dev/null
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import SocketServer
+import stat
+import subprocess
+import threading
+
+from . import compression
+from . import constants
+from . import discovery
+from . import signatures
+from ..network import endpoint
+from ..objects import workpacket
+
+
+class WorkHandler(SocketServer.BaseRequestHandler):
+
+ def handle(self):
+ rec = compression.Receiver(self.request)
+ while not rec.IsDone():
+ data = rec.Current()
+ with self.server.job_lock:
+ self._WorkOnWorkPacket(data)
+ rec.Advance()
+
+ def _WorkOnWorkPacket(self, data):
+ server_root = self.server.daemon.root
+ v8_root = os.path.join(server_root, "v8")
+ os.chdir(v8_root)
+ packet = workpacket.WorkPacket.Unpack(data)
+ self.ctx = packet.context
+ self.ctx.shell_dir = os.path.join("out",
+ "%s.%s" % (self.ctx.arch, self.ctx.mode))
+ if not os.path.isdir(self.ctx.shell_dir):
+ os.makedirs(self.ctx.shell_dir)
+ for binary in packet.binaries:
+ if not self._UnpackBinary(binary, packet.pubkey_fingerprint):
+ return
+
+ if not self._CheckoutRevision(packet.base_revision):
+ return
+
+ if not self._ApplyPatch(packet.patch):
+ return
+
+ tests = packet.tests
+ endpoint.Execute(v8_root, self.ctx, tests, self.request, self.server.daemon)
+ self._SendResponse()
+
+ def _SendResponse(self, error_message=None):
+ try:
+ if error_message:
+ compression.Send([-1, error_message], self.request)
+ compression.Send(constants.END_OF_STREAM, self.request)
+ return
+ except Exception, e:
+ pass # Peer is gone. There's nothing we can do.
+ # Clean up.
+ self._Call("git checkout -f")
+ self._Call("git clean -f -d")
+ self._Call("rm -rf %s" % self.ctx.shell_dir)
+
+ def _UnpackBinary(self, binary, pubkey_fingerprint):
+ binary_name = binary["name"]
+ if binary_name == "libv8.so":
+ libdir = os.path.join(self.ctx.shell_dir, "lib.target")
+ if not os.path.exists(libdir): os.makedirs(libdir)
+ target = os.path.join(libdir, binary_name)
+ else:
+ target = os.path.join(self.ctx.shell_dir, binary_name)
+ pubkeyfile = "../trusted/%s.pem" % pubkey_fingerprint
+ if not signatures.VerifySignature(target, binary["blob"],
+ binary["sign"], pubkeyfile):
+ self._SendResponse("Signature verification failed")
+ return False
+ os.chmod(target, stat.S_IRWXU)
+ return True
+
+ def _CheckoutRevision(self, base_revision):
+ code = self._Call("git checkout -f %s" % base_revision)
+ if code != 0:
+ self._Call("git fetch")
+ code = self._Call("git checkout -f %s" % base_revision)
+ if code != 0:
+ self._SendResponse("Error trying to check out base revision.")
+ return False
+ code = self._Call("git clean -f -d")
+ if code != 0:
+ self._SendResponse("Failed to reset checkout")
+ return False
+ return True
+
+ def _ApplyPatch(self, patch):
+ patchfilename = "_dtest_incoming_patch.patch"
+ with open(patchfilename, "w") as f:
+ f.write(patch)
+ code = self._Call("git apply %s" % patchfilename)
+ if code != 0:
+ self._SendResponse("Error applying patch.")
+ return False
+ return True
+
+ def _Call(self, cmd):
+ return subprocess.call(cmd, shell=True)
+
+
+class WorkSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+ def __init__(self, daemon):
+ address = (daemon.ip, constants.PEER_PORT)
+ SocketServer.TCPServer.__init__(self, address, WorkHandler)
+ self.job_lock = threading.Lock()
+ self.daemon = daemon
projects / platform / upstream / v8.git / commitdiff