1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Generates test runner factory and tests for GTests."""
6 # pylint: disable=W0212
15 from pylib import cmd_helper
16 from pylib import constants
18 from pylib.base import base_test_result
19 from pylib.base import test_dispatcher
20 from pylib.gtest import test_package_apk
21 from pylib.gtest import test_package_exe
22 from pylib.gtest import test_runner
25 os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib',
27 import unittest_util # pylint: disable=F0401
30 _ISOLATE_FILE_PATHS = {
31 'base_unittests': 'base/base_unittests.isolate',
32 'blink_heap_unittests':
33 'third_party/WebKit/Source/platform/heap/BlinkHeapUnitTests.isolate',
34 'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
35 'cc_perftests': 'cc/cc_perftests.isolate',
36 'components_unittests': 'components/components_unittests.isolate',
37 'content_browsertests': 'content/content_browsertests.isolate',
38 'content_unittests': 'content/content_unittests.isolate',
39 'media_perftests': 'media/media_perftests.isolate',
40 'media_unittests': 'media/media_unittests.isolate',
41 'net_unittests': 'net/net_unittests.isolate',
42 'ui_unittests': 'ui/ui_unittests.isolate',
43 'unit_tests': 'chrome/unit_tests.isolate',
45 'third_party/WebKit/Source/web/WebKitUnitTests.isolate',
48 # Paths relative to third_party/webrtc/ (kept separate for readability).
49 _WEBRTC_ISOLATE_FILE_PATHS = {
50 'audio_decoder_unittests':
51 'modules/audio_coding/neteq4/audio_decoder_unittests.isolate',
52 'common_audio_unittests': 'common_audio/common_audio_unittests.isolate',
53 'common_video_unittests': 'common_video/common_video_unittests.isolate',
54 'modules_tests': 'modules/modules_tests.isolate',
55 'modules_unittests': 'modules/modules_unittests.isolate',
56 'system_wrappers_unittests':
57 'system_wrappers/source/system_wrappers_unittests.isolate',
58 'test_support_unittests': 'test/test_support_unittests.isolate',
59 'tools_unittests': 'tools/tools_unittests.isolate',
60 'video_engine_core_unittests':
61 'video_engine/video_engine_core_unittests.isolate',
62 'voice_engine_unittests': 'voice_engine/voice_engine_unittests.isolate',
65 # Append the WebRTC tests with the full path from Chromium's src/ root.
66 for webrtc_test, isolate_path in _WEBRTC_ISOLATE_FILE_PATHS.items():
67 _ISOLATE_FILE_PATHS[webrtc_test] = 'third_party/webrtc/%s' % isolate_path
69 # Used for filtering large data deps at a finer grain than what's allowed in
70 # isolate files since pushing deps to devices is expensive.
71 # Wildcards are allowed.
72 _DEPS_EXCLUSION_LIST = [
73 'chrome/test/data/extensions/api_test',
74 'chrome/test/data/extensions/secure_shell',
75 'chrome/test/data/firefox*',
76 'chrome/test/data/gpu',
77 'chrome/test/data/image_decoding',
78 'chrome/test/data/import',
79 'chrome/test/data/page_cycler',
80 'chrome/test/data/perf',
81 'chrome/test/data/pyauto_private',
82 'chrome/test/data/safari_import',
83 'chrome/test/data/scroll',
84 'chrome/test/data/third_party',
85 'third_party/hunspell_dictionaries/*.dic',
87 'webkit/data/bmp_decoder',
88 'webkit/data/ico_decoder',
91 _ISOLATE_SCRIPT = os.path.join(
92 constants.DIR_SOURCE_ROOT, 'tools', 'swarming_client', 'isolate.py')
95 def _GenerateDepsDirUsingIsolate(suite_name):
96 """Generate the dependency dir for the test suite using isolate.
99 suite_name: Name of the test suite (e.g. base_unittests).
101 if os.path.isdir(constants.ISOLATE_DEPS_DIR):
102 shutil.rmtree(constants.ISOLATE_DEPS_DIR)
104 isolate_rel_path = _ISOLATE_FILE_PATHS.get(suite_name)
105 if not isolate_rel_path:
106 logging.info('Did not find an isolate file for the test suite.')
109 isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
110 isolated_abs_path = os.path.join(
111 constants.GetOutDirectory(), '%s.isolated' % suite_name)
112 assert os.path.exists(isolate_abs_path)
113 # This needs to be kept in sync with the cmd line options for isolate.py
114 # in src/build/isolate.gypi.
116 'python', _ISOLATE_SCRIPT,
118 '--isolate', isolate_abs_path,
119 '--isolated', isolated_abs_path,
120 '--outdir', constants.ISOLATE_DEPS_DIR,
122 '--path-variable', 'PRODUCT_DIR', constants.GetOutDirectory(),
124 '--config-variable', 'OS', 'android',
125 '--config-variable', 'chromeos', '0',
126 '--config-variable', 'component', 'static_library',
127 '--config-variable', 'icu_use_data_file_flag', '1',
128 '--config-variable', 'use_openssl', '0',
130 assert not cmd_helper.RunCmd(isolate_cmd)
132 # We're relying on the fact that timestamps are preserved
133 # by the remap command (hardlinked). Otherwise, all the data
134 # will be pushed to the device once we move to using time diff
135 # instead of md5sum. Perform a sanity check here.
136 for root, _, filenames in os.walk(constants.ISOLATE_DEPS_DIR):
138 linked_file = os.path.join(root, filenames[0])
139 orig_file = os.path.join(
140 constants.DIR_SOURCE_ROOT,
141 os.path.relpath(linked_file, constants.ISOLATE_DEPS_DIR))
142 if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino:
145 raise Exception('isolate remap command did not use hardlinks.')
147 # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
148 old_cwd = os.getcwd()
150 os.chdir(constants.ISOLATE_DEPS_DIR)
151 excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)]
153 logging.info('Excluding the following from dependency list: %s',
155 for p in excluded_paths:
163 # On Android, all pak files need to be in the top-level 'paks' directory.
164 paks_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'paks')
167 deps_out_dir = os.path.join(
168 constants.ISOLATE_DEPS_DIR,
169 os.path.relpath(os.path.join(constants.GetOutDirectory(), os.pardir),
170 constants.DIR_SOURCE_ROOT))
171 for root, _, filenames in os.walk(deps_out_dir):
172 for filename in fnmatch.filter(filenames, '*.pak'):
173 shutil.move(os.path.join(root, filename), paks_dir)
175 # Move everything in PRODUCT_DIR to top level.
176 deps_product_dir = os.path.join(deps_out_dir, constants.GetBuildType())
177 if os.path.isdir(deps_product_dir):
178 for p in os.listdir(deps_product_dir):
179 shutil.move(os.path.join(deps_product_dir, p), constants.ISOLATE_DEPS_DIR)
180 os.rmdir(deps_product_dir)
181 os.rmdir(deps_out_dir)
184 def _GetDisabledTestsFilterFromFile(suite_name):
185 """Returns a gtest filter based on the *_disabled file.
188 suite_name: Name of the test suite (e.g. base_unittests).
191 A gtest filter which excludes disabled tests.
192 Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc'
194 filter_file_path = os.path.join(
195 os.path.abspath(os.path.dirname(__file__)),
196 'filter', '%s_disabled' % suite_name)
198 if not filter_file_path or not os.path.exists(filter_file_path):
199 logging.info('No filter file found at %s', filter_file_path)
202 filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()]
203 if x and x[0] != '#']
204 disabled_filter = '*-%s' % ':'.join(filters)
205 logging.info('Applying filter "%s" obtained from %s',
206 disabled_filter, filter_file_path)
207 return disabled_filter
210 def _GetTests(test_options, test_package, devices):
211 """Get a list of tests.
214 test_options: A GTestOptions object.
215 test_package: A TestPackageApk object.
216 devices: A list of attached devices.
219 A list of all the tests in the test suite.
221 def TestListerRunnerFactory(device, _shard_index):
222 class TestListerRunner(test_runner.TestRunner):
223 def RunTest(self, _test):
224 result = base_test_result.BaseTestResult(
225 'gtest_list_tests', base_test_result.ResultType.PASS)
226 self.test_package.Install(self.device)
227 result.test_list = self.test_package.GetAllTests(self.device)
228 results = base_test_result.TestRunResults()
229 results.AddResult(result)
231 return TestListerRunner(test_options, device, test_package)
233 results, _no_retry = test_dispatcher.RunTests(
234 ['gtest_list_tests'], TestListerRunnerFactory, devices)
236 for r in results.GetAll():
237 tests.extend(r.test_list)
241 def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False):
242 """Removes tests with disabled prefixes.
245 all_tests: List of tests to filter.
246 pre: If True, include tests with PRE_ prefix.
247 manual: If True, include tests with MANUAL_ prefix.
250 List of tests remaining.
253 filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_']
256 filter_prefixes.append('PRE_')
259 filter_prefixes.append('MANUAL_')
262 test_case, test = t.split('.', 1)
263 if not any([test_case.startswith(prefix) or test.startswith(prefix) for
264 prefix in filter_prefixes]):
265 filtered_tests.append(t)
266 return filtered_tests
269 def _FilterDisabledTests(tests, suite_name, has_gtest_filter):
270 """Removes disabled tests from |tests|.
272 Applies the following filters in order:
273 1. Remove tests with disabled prefixes.
274 2. Remove tests specified in the *_disabled files in the 'filter' dir
277 tests: List of tests.
278 suite_name: Name of the test suite (e.g. base_unittests).
279 has_gtest_filter: Whether a gtest_filter is provided.
282 List of tests remaining.
284 tests = _FilterTestsUsingPrefixes(
285 tests, has_gtest_filter, has_gtest_filter)
286 tests = unittest_util.FilterTestNames(
287 tests, _GetDisabledTestsFilterFromFile(suite_name))
292 def Setup(test_options, devices):
293 """Create the test runner factory and tests.
296 test_options: A GTestOptions object.
297 devices: A list of attached devices.
300 A tuple of (TestRunnerFactory, tests).
302 test_package = test_package_apk.TestPackageApk(test_options.suite_name)
303 if not os.path.exists(test_package.suite_path):
304 test_package = test_package_exe.TestPackageExecutable(
305 test_options.suite_name)
306 if not os.path.exists(test_package.suite_path):
308 'Did not find %s target. Ensure it has been built.'
309 % test_options.suite_name)
310 logging.warning('Found target %s', test_package.suite_path)
312 _GenerateDepsDirUsingIsolate(test_options.suite_name)
314 tests = _GetTests(test_options, test_package, devices)
316 # Constructs a new TestRunner with the current options.
317 def TestRunnerFactory(device, _shard_index):
318 return test_runner.TestRunner(
323 if test_options.run_disabled:
324 test_options = test_options._replace(
325 test_arguments=('%s --gtest_also_run_disabled_tests' %
326 test_options.test_arguments))
328 tests = _FilterDisabledTests(tests, test_options.suite_name,
329 bool(test_options.gtest_filter))
330 if test_options.gtest_filter:
331 tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter)
333 # Coalesce unit tests into a single test per device
334 if test_options.suite_name != 'content_browsertests':
335 num_devices = len(devices)
336 tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
337 tests = [t for t in tests if t]
339 return (TestRunnerFactory, tests)