-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
Handles test configuration, but all the logic for
actually running the test is in Test and PageRunner."""
+import hashlib
import inspect
import json
import os
import sys
-from telemetry import test
+from telemetry import benchmark
+from telemetry import decorators
+from telemetry.core import browser_finder
from telemetry.core import browser_options
from telemetry.core import command_line
from telemetry.core import discover
+from telemetry.core import environment
from telemetry.core import util
+from telemetry.page import page_set
from telemetry.page import page_test
from telemetry.page import profile_creator
from telemetry.util import find_dependencies
class Help(command_line.OptparseCommand):
- """Display help information"""
+ """Display help information about a command"""
+
+ usage = '[command]'
def Run(self, args):
- print >> sys.stderr, ('usage: %s <command> [<options>]' % _ScriptName())
+ if len(args.positional_args) == 1:
+ commands = _MatchingCommands(args.positional_args[0])
+ if len(commands) == 1:
+ command = commands[0]
+ parser = command.CreateParser()
+ command.AddCommandLineArgs(parser)
+ parser.print_help()
+ return 0
+
+ print >> sys.stderr, ('usage: %s [command] [<options>]' % _ScriptName())
print >> sys.stderr, 'Available commands are:'
for command in _Commands():
print >> sys.stderr, ' %-10s %s' % (
command.Name(), command.Description())
+ print >> sys.stderr, ('"%s help <command>" to see usage information '
+ 'for a specific command.' % _ScriptName())
return 0
usage = '[test_name] [<options>]'
@classmethod
+ def CreateParser(cls):
+ options = browser_options.BrowserFinderOptions()
+ parser = options.CreateParser('%%prog %s %s' % (cls.Name(), cls.usage))
+ return parser
+
+ @classmethod
def AddCommandLineArgs(cls, parser):
- parser.add_option('-j', '--json', action='store_true')
+ parser.add_option('-j', '--json-output-file', type='string')
+ parser.add_option('-n', '--num-shards', type='int', default=1)
@classmethod
def ProcessCommandLineArgs(cls, parser, args):
parser.error('Must provide at most one test name.')
def Run(self, args):
- if args.json:
- test_list = []
- for test_class in sorted(args.tests, key=lambda t: t.Name()):
- test_list.append({
- 'name': test_class.Name(),
- 'description': test_class.Description(),
- 'options': test_class.options,
- })
- print json.dumps(test_list)
+ if args.json_output_file:
+ possible_browser = browser_finder.FindBrowser(args)
+ with open(args.json_output_file, 'w') as f:
+ f.write(_GetJsonTestList(possible_browser, args.tests, args.num_shards))
else:
_PrintTestList(args.tests)
return 0
class Run(command_line.OptparseCommand):
- """Run one or more tests"""
+ """Run one or more tests (default)"""
usage = 'test_name [page_set] [<options>]'
@classmethod
def AddCommandLineArgs(cls, parser):
- test.AddCommandLineArgs(parser)
+ benchmark.AddCommandLineArgs(parser)
# Allow tests to add their own command line options.
matching_tests = []
if not os.path.exists(page_set_path):
parser.error('Page set not found.')
if not (os.path.isfile(page_set_path) and
- os.path.splitext(page_set_path)[1] == '.json'):
- parser.error('Page set is not a JSON file.')
+ discover.IsPageSetFile(page_set_path)):
+ parser.error('Unsupported page set file format.')
- class TestWrapper(test.Test):
+ class TestWrapper(benchmark.Benchmark):
test = test_class
- page_set = page_set_path
+
+ @classmethod
+ def CreatePageSet(cls, options):
+ return page_set.PageSet.FromFile(page_set_path)
+
test_class = TestWrapper
else:
if len(args.positional_args) > 1:
parser.error('Too many arguments.')
- assert issubclass(test_class, test.Test), 'Trying to run a non-Test?!'
+ assert issubclass(test_class, benchmark.Benchmark), (
+ 'Trying to run a non-Benchmark?!')
+ benchmark.ProcessCommandLineArgs(parser, args)
test_class.ProcessCommandLineArgs(parser, args)
- test.ProcessCommandLineArgs(parser, args)
cls._test = test_class
continue
yield cls
+def _MatchingCommands(string):
+ return [command for command in _Commands()
+ if command.Name().startswith(string)]
+@decorators.Cache
def _Tests():
- base_dir = util.GetBaseDir()
- tests = discover.DiscoverClasses(base_dir, base_dir, test.Test,
- index_by_class_name=True).values()
- page_tests = discover.DiscoverClasses(base_dir, base_dir, page_test.PageTest,
- index_by_class_name=True).values()
- page_tests = [test_class for test_class in page_tests
- if not issubclass(test_class, profile_creator.ProfileCreator)]
- return tests + page_tests
+ tests = []
+ for base_dir in config.base_paths:
+ tests += discover.DiscoverClasses(base_dir, base_dir, benchmark.Benchmark,
+ index_by_class_name=True).values()
+ page_tests = discover.DiscoverClasses(base_dir, base_dir,
+ page_test.PageTest,
+ index_by_class_name=True).values()
+ tests += [test_class for test_class in page_tests
+ if not issubclass(test_class, profile_creator.ProfileCreator)]
+ return tests
def _MatchTestName(input_test_name, exact_matches=True):
# Exact matching.
if exact_matches:
# Don't add aliases to search dict, only allow exact matching for them.
- if input_test_name in test_aliases:
- exact_match = test_aliases[input_test_name]
+ if input_test_name in config.test_aliases:
+ exact_match = config.test_aliases[input_test_name]
else:
exact_match = input_test_name
if _Matches(input_test_name, test_class.Name())]
+def _GetJsonTestList(possible_browser, test_classes, num_shards):
+ """Returns a list of all enabled tests in a JSON format expected by buildbots.
+
+ JSON format (see build/android/pylib/perf/test_runner.py):
+ { "version": int,
+ "steps": {
+ "foo": {
+ "device_affinity": int,
+ "cmd": "script_to_execute foo"
+ },
+ "bar": {
+ "device_affinity": int,
+ "cmd": "script_to_execute bar"
+ }
+ }
+ }
+ """
+ output = {
+ 'version': 1,
+ 'steps': {
+ }
+ }
+ for test_class in test_classes:
+ if not issubclass(test_class, benchmark.Benchmark):
+ continue
+ if not decorators.IsEnabled(test_class, possible_browser):
+ continue
+ name = test_class.Name()
+ output['steps'][name] = {
+ 'cmd': ' '.join([sys.executable, os.path.realpath(sys.argv[0]),
+ '--browser=%s' % possible_browser.browser_type,
+ '-v', '--output-format=buildbot', name]),
+ # TODO(tonyg): Currently we set the device affinity to a stable hash of
+ # the test name. This somewhat evenly distributes benchmarks among the
+ # requested number of shards. However, it is far from optimal in terms of
+ # cycle time. We should add a test size decorator (e.g. small, medium,
+ # large) and let that inform sharding.
+ 'device_affinity': int(hashlib.sha1(name).hexdigest(), 16) % num_shards
+ }
+ return json.dumps(output, indent=2, sort_keys=True)
+
+
def _PrintTestList(tests):
if not tests:
print >> sys.stderr, 'No tests found!'
format_string = ' %%-%ds %%s' % max(len(t.Name()) for t in tests)
filtered_tests = [test_class for test_class in tests
- if issubclass(test_class, test.Test)]
+ if issubclass(test_class, benchmark.Benchmark)]
if filtered_tests:
print >> sys.stderr, 'Available tests are:'
for test_class in sorted(filtered_tests, key=lambda t: t.Name()):
print >> sys.stderr
-test_aliases = {}
+config = environment.Environment([util.GetBaseDir()])
-def Main():
+def main():
# Get the command name from the command line.
if len(sys.argv) > 1 and sys.argv[1] == '--help':
sys.argv[1] = 'help'
break
# Validate and interpret the command name.
- commands = [command for command in _Commands()
- if command.Name().startswith(command_name)]
+ commands = _MatchingCommands(command_name)
if len(commands) > 1:
print >> sys.stderr, ('"%s" is not a %s command. Did you mean one of these?'
% (command_name, _ScriptName()))
projects / platform / framework / web / crosswalk.git / blobdiff