2 # Copyright 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Runs the WebDriver Java acceptance tests.
8 This script is called from chrome/test/chromedriver/run_all_tests.py and reports
9 results using the buildbot annotation scheme.
11 For ChromeDriver documentation, refer to http://code.google.com/p/chromedriver.
18 import xml.dom.minidom as minidom
20 _THIS_DIR = os.path.abspath(os.path.dirname(__file__))
21 sys.path.insert(1, os.path.join(_THIS_DIR, os.pardir))
24 import test_environment
28 sys.path.insert(0, os.path.join(chrome_paths.GetSrc(), 'build', 'android'))
29 from pylib import constants
32 class TestResult(object):
33 """A result for an attempted single test case."""
35 def __init__(self, name, time, failure):
36 """Initializes a test result.
39 name: the full name of the test.
40 time: the amount of time the test ran, in seconds.
41 failure: the test error or failure message, or None if the test passed.
45 self._failure = failure
48 """Returns the test name."""
52 """Returns the time it took to run the test."""
56 """Returns whether the test passed."""
57 return self._failure is None
59 def GetFailureMessage(self):
60 """Returns the test failure message, or None if the test passed."""
64 def _Run(java_tests_src_dir, test_filter,
65 chromedriver_path, chrome_path, log_path, android_package_key,
67 """Run the WebDriver Java tests and return the test results.
70 java_tests_src_dir: the java test source code directory.
71 test_filter: the filter to use when choosing tests to run. Format is same
72 as Google C++ Test format.
73 chromedriver_path: path to ChromeDriver exe.
74 chrome_path: path to Chrome exe.
75 log_path: path to server log.
76 android_package_key: name of Chrome's Android package.
77 verbose: whether the output should be verbose.
78 debug: whether the tests should wait until attached by a debugger.
81 A list of |TestResult|s.
83 test_dir = util.MakeTempDir()
84 keystore_path = ('java', 'client', 'test', 'keystore')
85 required_dirs = [keystore_path[:-1],
87 ('third_party', 'closure', 'goog'),
88 ('third_party', 'js')]
89 for required_dir in required_dirs:
90 os.makedirs(os.path.join(test_dir, *required_dir))
92 test_jar = 'test-standalone.jar'
94 shutil.copyfile(os.path.join(java_tests_src_dir, 'keystore'),
95 os.path.join(test_dir, *keystore_path))
96 util.Unzip(os.path.join(java_tests_src_dir, 'common.zip'), test_dir)
97 shutil.copyfile(os.path.join(java_tests_src_dir, test_jar),
98 os.path.join(test_dir, test_jar))
100 sys_props = ['selenium.browser=chrome',
101 'webdriver.chrome.driver=' + os.path.abspath(chromedriver_path)]
103 sys_props += ['webdriver.chrome.binary=' + os.path.abspath(chrome_path)]
105 sys_props += ['webdriver.chrome.logfile=' + log_path]
106 if android_package_key:
107 android_package = constants.PACKAGE_INFO[android_package_key].package
108 sys_props += ['webdriver.chrome.android_package=' + android_package]
109 if android_package_key == 'chromedriver_webview_shell':
110 android_activity = constants.PACKAGE_INFO[android_package_key].activity
111 android_process = '%s:main' % android_package
112 sys_props += ['webdriver.chrome.android_activity=' + android_activity]
113 sys_props += ['webdriver.chrome.android_process=' + android_process]
115 # Test jar actually takes a regex. Convert from glob.
116 test_filter = test_filter.replace('*', '.*')
117 sys_props += ['filter=' + test_filter]
121 transport = 'dt_socket'
123 transport = 'dt_shmem'
124 jvm_args += ['-agentlib:jdwp=transport=%s,server=y,suspend=y,'
125 'address=33081' % transport]
126 # Unpack the sources into the test directory and add to the class path
127 # for ease of debugging, particularly with jdb.
128 util.Unzip(os.path.join(java_tests_src_dir, 'test-nodeps-srcs.jar'),
130 class_path += ':' + test_dir
133 test_dir, 'org.openqa.selenium.chrome.ChromeDriverTests',
134 class_path, sys_props, jvm_args, verbose)
137 def _RunAntTest(test_dir, test_class, class_path, sys_props, jvm_args, verbose):
138 """Runs a single Ant JUnit test suite and returns the |TestResult|s.
141 test_dir: the directory to run the tests in.
142 test_class: the name of the JUnit test suite class to run.
143 class_path: the Java class path used when running the tests, colon delimited
144 sys_props: Java system properties to set when running the tests.
145 jvm_args: Java VM command line args to use.
146 verbose: whether the output should be verbose.
149 A list of |TestResult|s.
151 def _CreateBuildConfig(test_name, results_file, class_path, junit_props,
152 sys_props, jvm_args):
153 def _SystemPropToXml(prop):
154 key, value = prop.split('=')
155 return '<sysproperty key="%s" value="%s"/>' % (key, value)
156 def _JvmArgToXml(arg):
157 return '<jvmarg value="%s"/>' % arg
160 ' <target name="test">',
161 ' <junit %s>' % ' '.join(junit_props),
162 ' <formatter type="xml"/>',
164 ' <pathelement path="%s"/>' % class_path,
166 ' ' + '\n '.join(map(_SystemPropToXml, sys_props)),
167 ' ' + '\n '.join(map(_JvmArgToXml, jvm_args)),
168 ' <test name="%s" outfile="%s"/>' % (test_name, results_file),
173 def _ProcessResults(results_path):
174 doc = minidom.parse(results_path)
176 for test in doc.getElementsByTagName('testcase'):
177 name = test.getAttribute('classname') + '.' + test.getAttribute('name')
178 time = test.getAttribute('time')
180 error_nodes = test.getElementsByTagName('error')
181 failure_nodes = test.getElementsByTagName('failure')
183 failure = error_nodes[0].childNodes[0].nodeValue
185 failure = failure_nodes[0].childNodes[0].nodeValue
186 tests += [TestResult(name, time, failure)]
189 junit_props = ['printsummary="yes"',
191 'haltonfailure="no"',
194 junit_props += ['showoutput="yes"']
196 ant_file = open(os.path.join(test_dir, 'build.xml'), 'w')
197 ant_file.write(_CreateBuildConfig(
198 test_class, 'results', class_path, junit_props, sys_props, jvm_args))
205 code = util.RunCommand([ant_name, 'test'], cwd=test_dir)
207 print 'FAILED to run java tests of %s through ant' % test_class
209 return _ProcessResults(os.path.join(test_dir, 'results.xml'))
212 def PrintTestResults(results):
213 """Prints the given results in a format recognized by the buildbot."""
216 for result in results:
217 if not result.IsPass():
219 failure_names += ['.'.join(result.GetName().split('.')[-2:])]
221 print 'Ran %s tests' % len(results)
222 print 'Failed %s:' % len(failures)
223 util.AddBuildStepText('failed %s/%s' % (len(failures), len(results)))
224 for result in failures:
226 print '=' * 10, result.GetName(), '(%ss)' % result.GetTime()
227 print result.GetFailureMessage()
228 if len(failures) < 10:
229 util.AddBuildStepText('.'.join(result.GetName().split('.')[-2:]))
230 print 'Rerun failing tests with filter:', ':'.join(failure_names)
235 parser = optparse.OptionParser()
237 '', '--verbose', action='store_true', default=False,
238 help='Whether output should be verbose')
240 '', '--debug', action='store_true', default=False,
241 help='Whether to wait to be attached by a debugger')
243 '', '--chromedriver', type='string', default=None,
244 help='Path to a build of the chromedriver library(REQUIRED!)')
246 '', '--chrome', type='string', default=None,
247 help='Path to a build of the chrome binary')
250 help='Output verbose server logs to this file')
252 '', '--chrome-version', default='HEAD',
253 help='Version of chrome. Default is \'HEAD\'')
255 '', '--android-package', help='Android package key')
257 '', '--filter', type='string', default=None,
258 help='Filter for specifying what tests to run, "*" will run all. E.g., '
259 '*testShouldReturnTitleOfPageIfSet')
261 '', '--also-run-disabled-tests', action='store_true', default=False,
262 help='Include disabled tests while running the tests')
264 '', '--isolate-tests', action='store_true', default=False,
265 help='Relaunch the jar test harness after each test')
266 options, _ = parser.parse_args()
268 options.chromedriver = util.GetAbsolutePathOfUserPath(options.chromedriver)
269 if options.chromedriver is None or not os.path.exists(options.chromedriver):
270 parser.error('chromedriver is required or the given path is invalid.' +
271 'Please run "%s --help" for help' % __file__)
273 if options.android_package:
274 if options.android_package not in constants.PACKAGE_INFO:
275 parser.error('Invalid --android-package')
276 if options.chrome_version != 'HEAD':
277 parser.error('Android does not support the --chrome-version argument.')
278 environment = test_environment.AndroidTestEnvironment(
279 options.android_package)
281 environment = test_environment.DesktopTestEnvironment(
282 options.chrome_version)
285 environment.GlobalSetUp()
286 # Run passed tests when filter is not provided.
287 if options.isolate_tests:
288 test_filters = environment.GetPassedJavaTests()
291 test_filter = options.filter
294 if not options.also_run_disabled_tests:
295 if '-' in test_filter:
299 test_filter += ':'.join(environment.GetDisabledJavaTestMatchers())
300 test_filters = [test_filter]
302 java_tests_src_dir = os.path.join(chrome_paths.GetSrc(), 'chrome', 'test',
303 'chromedriver', 'third_party',
305 if (not os.path.exists(java_tests_src_dir) or
306 not os.listdir(java_tests_src_dir)):
307 java_tests_url = ('http://src.chromium.org/svn/trunk/deps/third_party'
309 print ('"%s" is empty or it doesn\'t exist. ' % java_tests_src_dir +
310 'Need to map <chrome-svn>/trunk/deps/third_party/webdriver to '
311 'chrome/test/chromedriver/third_party/java_tests in .gclient.\n'
312 'Alternatively, do:\n'
313 ' $ cd chrome/test/chromedriver/third_party\n'
314 ' $ svn co %s java_tests' % java_tests_url)
318 for filter in test_filters:
320 java_tests_src_dir=java_tests_src_dir,
322 chromedriver_path=options.chromedriver,
323 chrome_path=util.GetAbsolutePathOfUserPath(options.chrome),
324 log_path=options.log_path,
325 android_package_key=options.android_package,
326 verbose=options.verbose,
328 return PrintTestResults(results)
330 environment.GlobalTearDown()
333 if __name__ == '__main__':