1 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
13 # * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 """Abstract base class of Port-specific entry points for the layout tests
30 test infrastructure (the Port and Driver classes)."""
45 from collections import OrderedDict
47 # Needed for Python < 2.7
48 from webkitpy.thirdparty.ordered_dict import OrderedDict
51 from webkitpy.common import find_files
52 from webkitpy.common import read_checksum_from_png
53 from webkitpy.common.memoized import memoized
54 from webkitpy.common.system import path
55 from webkitpy.common.system.executive import ScriptError
56 from webkitpy.common.system.path import cygpath
57 from webkitpy.common.system.systemhost import SystemHost
58 from webkitpy.common.webkit_finder import WebKitFinder
59 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
60 from webkitpy.layout_tests.models import test_run_results
61 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
62 from webkitpy.layout_tests.port import config as port_config
63 from webkitpy.layout_tests.port import driver
64 from webkitpy.layout_tests.port import server_process
65 from webkitpy.layout_tests.port.factory import PortFactory
66 from webkitpy.layout_tests.servers import apache_http
67 from webkitpy.layout_tests.servers import pywebsocket
69 _log = logging.getLogger(__name__)
72 # FIXME: This class should merge with WebKitPort now that Chromium behaves mostly like other webkit ports.
74 """Abstract class for Port-specific hooks for the layout_test package."""
76 # Subclasses override this. This should indicate the basic implementation
77 # part of the port name, e.g., 'mac', 'win', 'gtk'; there is probably (?)
78 # one unique value per class.
80 # FIXME: We should probably rename this to something like 'implementation_name'.
83 # Test names resemble unix relative paths, and use '/' as a directory separator.
84 TEST_PATH_SEPARATOR = '/'
86 ALL_BUILD_TYPES = ('debug', 'release')
88 CONTENT_SHELL_NAME = 'content_shell'
90 # True if the port as aac and mp3 codecs built in.
91 PORT_HAS_AUDIO_CODECS_BUILT_IN = False
94 ('snowleopard', 'x86'),
97 # FIXME: We treat Retina (High-DPI) devices as if they are running
98 # a different operating system version. This isn't accurate, but will work until
99 # we need to test and support baselines across multiple O/S versions.
102 ('mountainlion', 'x86'),
103 ('mavericks', 'x86'),
108 # FIXME: Technically this should be 'arm', but adding a third architecture type breaks TestConfigurationConverter.
109 # If we need this to be 'arm' in the future, then we first have to fix TestConfigurationConverter.
110 ('icecreamsandwich', 'x86'),
113 ALL_BASELINE_VARIANTS = [
114 'mac-mavericks', 'mac-mountainlion', 'mac-retina', 'mac-lion', 'mac-snowleopard',
115 'win-win7', 'win-xp',
116 'linux-x86_64', 'linux-x86',
119 CONFIGURATION_SPECIFIER_MACROS = {
120 'mac': ['snowleopard', 'lion', 'retina', 'mountainlion', 'mavericks'],
121 'win': ['xp', 'win7'],
123 'android': ['icecreamsandwich'],
126 DEFAULT_BUILD_DIRECTORIES = ('out',)
128 # overridden in subclasses.
131 SUPPORTED_VERSIONS = []
133 # URL to the build requirements page.
134 BUILD_REQUIREMENTS_URL = ''
137 def latest_platform_fallback_path(cls):
138 return cls.FALLBACK_PATHS[cls.SUPPORTED_VERSIONS[-1]]
141 def _static_build_path(cls, filesystem, build_directory, chromium_base, configuration, comps):
143 return filesystem.join(build_directory, configuration, *comps)
146 for directory in cls.DEFAULT_BUILD_DIRECTORIES:
147 base_dir = filesystem.join(chromium_base, directory, configuration)
148 path = filesystem.join(base_dir, *comps)
149 if filesystem.exists(path):
150 hits.append((filesystem.mtime(path), path))
153 hits.sort(reverse=True)
154 return hits[0][1] # Return the newest file found.
156 # We have to default to something, so pick the last one.
157 return filesystem.join(base_dir, *comps)
160 def determine_full_port_name(cls, host, options, port_name):
161 """Return a fully-specified port name that can be used to construct objects."""
162 # Subclasses will usually override this.
163 assert port_name.startswith(cls.port_name)
166 def __init__(self, host, port_name, options=None, **kwargs):
168 # This value may be different from cls.port_name by having version modifiers
169 # and other fields appended to it (for example, 'qt-arm' or 'mac-wk2').
170 self._name = port_name
172 # These are default values that should be overridden in a subclasses.
174 self._architecture = 'x86'
176 # FIXME: Ideally we'd have a package-wide way to get a
177 # well-formed options object that had all of the necessary
178 # options defined on it.
179 self._options = options or optparse.Values()
182 self._executive = host.executive
183 self._filesystem = host.filesystem
184 self._webkit_finder = WebKitFinder(host.filesystem)
185 self._config = port_config.Config(self._executive, self._filesystem, self.port_name)
188 self._http_server = None
189 self._websocket_server = None
190 self._image_differ = None
191 self._server_process_constructor = server_process.ServerProcess # overridable for testing
192 self._http_lock = None # FIXME: Why does this live on the port object?
193 self._dump_reader = None
195 # Python's Popen has a bug that causes any pipes opened to a
196 # process that can't be executed to be leaked. Since this
197 # code is specifically designed to tolerate exec failures
198 # to gracefully handle cases where wdiff is not installed,
199 # the bug results in a massive file descriptor leak. As a
200 # workaround, if an exec failure is ever experienced for
201 # wdiff, assume it's not available. This will leak one
202 # file descriptor but that's better than leaking each time
203 # wdiff would be run.
205 # http://mail.python.org/pipermail/python-list/
206 # 2008-August/505753.html
207 # http://bugs.python.org/issue3210
208 self._wdiff_available = None
210 # FIXME: prettypatch.py knows this path, why is it copied here?
211 self._pretty_patch_path = self.path_from_webkit_base("Tools", "Scripts", "webkitruby", "PrettyPatch", "prettify.rb")
212 self._pretty_patch_available = None
214 if not hasattr(options, 'configuration') or not options.configuration:
215 self.set_option_default('configuration', self.default_configuration())
216 self._test_configuration = None
217 self._reftest_list = {}
218 self._results_directory = None
219 self._virtual_test_suites = None
221 def buildbot_archives_baselines(self):
224 def additional_drt_flag(self):
225 if self.driver_name() == self.CONTENT_SHELL_NAME:
226 return ['--dump-render-tree']
229 def supports_per_test_timeout(self):
232 def default_pixel_tests(self):
235 def default_smoke_test_only(self):
238 def default_timeout_ms(self):
239 timeout_ms = 6 * 1000
240 if self.get_option('configuration') == 'Debug':
241 # Debug is usually 2x-3x slower than Release.
242 return 3 * timeout_ms
245 def driver_stop_timeout(self):
246 """ Returns the amount of time in seconds to wait before killing the process in driver.stop()."""
247 # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as
248 # well (for things like ASAN, Valgrind, etc.)
249 return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms()
251 def wdiff_available(self):
252 if self._wdiff_available is None:
253 self._wdiff_available = self.check_wdiff(logging=False)
254 return self._wdiff_available
256 def pretty_patch_available(self):
257 if self._pretty_patch_available is None:
258 self._pretty_patch_available = self.check_pretty_patch(logging=False)
259 return self._pretty_patch_available
261 def default_child_processes(self):
262 """Return the number of drivers to use for this port."""
263 if self.get_option('enable_sanitizer'):
264 # ASAN/MSAN/TSAN are more cpu- and memory- intensive than regular
265 # content_shell, and so we need to run fewer of them in parallel.
266 return max(int(self._executive.cpu_count() * 0.75), 1)
267 return self._executive.cpu_count()
269 def default_max_locked_shards(self):
270 """Return the number of "locked" shards to run in parallel (like the http tests)."""
271 max_locked_shards = int(self.default_child_processes()) / 4
272 if not max_locked_shards:
274 return max_locked_shards
276 def baseline_path(self):
277 """Return the absolute path to the directory to store new baselines in for this port."""
278 # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir()
279 return self.baseline_version_dir()
281 def baseline_platform_dir(self):
282 """Return the absolute path to the default (version-independent) platform-specific results."""
283 return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name)
285 def baseline_version_dir(self):
286 """Return the absolute path to the platform-and-version-specific results."""
287 baseline_search_paths = self.baseline_search_path()
288 return baseline_search_paths[0]
290 def virtual_baseline_search_path(self, test_name):
291 suite = self.lookup_virtual_suite(test_name)
294 return [self._filesystem.join(path, suite.name) for path in self.default_baseline_search_path()]
296 def baseline_search_path(self):
297 return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path()
299 def default_baseline_search_path(self):
300 """Return a list of absolute paths to directories to search under for
301 baselines. The directories are searched in order."""
302 return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version()])
305 def _compare_baseline(self):
306 factory = PortFactory(self.host)
307 target_port = self.get_option('compare_port')
309 return factory.get(target_port).default_baseline_search_path()
312 def _check_file_exists(self, path_to_file, file_description,
313 override_step=None, logging=True):
314 """Verify the file is present where expected or log an error.
317 file_name: The (human friendly) name or description of the file
318 you're looking for (e.g., "HTTP Server"). Used for error logging.
319 override_step: An optional string to be logged if the check fails.
320 logging: Whether or not log the error messages."""
321 if not self._filesystem.exists(path_to_file):
323 _log.error('Unable to find %s' % file_description)
324 _log.error(' at %s' % path_to_file)
326 _log.error(' %s' % override_step)
331 def check_build(self, needs_http, printer):
334 dump_render_tree_binary_path = self._path_to_driver()
335 result = self._check_file_exists(dump_render_tree_binary_path,
336 'test driver') and result
337 if not result and self.get_option('build'):
338 result = self._check_driver_build_up_to_date(
339 self.get_option('configuration'))
343 helper_path = self._path_to_helper()
345 result = self._check_file_exists(helper_path,
346 'layout test helper') and result
348 if self.get_option('pixel_tests'):
349 result = self.check_image_diff(
350 'To override, invoke with --no-pixel-tests') and result
352 # It's okay if pretty patch and wdiff aren't available, but we will at least log messages.
353 self._pretty_patch_available = self.check_pretty_patch()
354 self._wdiff_available = self.check_wdiff()
356 if self._dump_reader:
357 result = self._dump_reader.check_is_functional() and result
360 result = self.check_httpd() and result
362 return test_run_results.OK_EXIT_STATUS if result else test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
364 def _check_driver(self):
365 driver_path = self._path_to_driver()
366 if not self._filesystem.exists(driver_path):
367 _log.error("%s was not found at %s" % (self.driver_name(), driver_path))
371 def _check_port_build(self):
372 # Ports can override this method to do additional checks.
375 def check_sys_deps(self, needs_http):
376 """If the port needs to do some runtime checks to ensure that the
377 tests can be run successfully, it should override this routine.
378 This step can be skipped with --nocheck-sys-deps.
380 Returns whether the system is properly configured."""
381 cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
383 local_error = ScriptError()
385 def error_handler(script_error):
386 local_error.exit_code = script_error.exit_code
388 output = self._executive.run_command(cmd, error_handler=error_handler)
389 if local_error.exit_code:
390 _log.error('System dependencies check failed.')
391 _log.error('To override, invoke with --nocheck-sys-deps')
394 if self.BUILD_REQUIREMENTS_URL is not '':
396 _log.error('For complete build requirements, please see:')
397 _log.error(self.BUILD_REQUIREMENTS_URL)
398 return test_run_results.SYS_DEPS_EXIT_STATUS
399 return test_run_results.OK_EXIT_STATUS
401 def check_image_diff(self, override_step=None, logging=True):
402 """This routine is used to check whether image_diff binary exists."""
403 image_diff_path = self._path_to_image_diff()
404 if not self._filesystem.exists(image_diff_path):
405 _log.error("image_diff was not found at %s" % image_diff_path)
409 def check_pretty_patch(self, logging=True):
410 """Checks whether we can use the PrettyPatch ruby script."""
412 _ = self._executive.run_command(['ruby', '--version'])
414 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
416 _log.warning("Ruby is not installed; can't generate pretty patches.")
420 if not self._filesystem.exists(self._pretty_patch_path):
422 _log.warning("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path)
428 def check_wdiff(self, logging=True):
429 if not self._path_to_wdiff():
430 # Don't need to log here since this is the port choosing not to use wdiff.
434 _ = self._executive.run_command([self._path_to_wdiff(), '--help'])
437 message = self._wdiff_missing_message()
439 for line in message.splitlines():
440 _log.warning(' ' + line)
446 def _wdiff_missing_message(self):
447 return 'wdiff is not installed; please install it to generate word-by-word diffs.'
449 def check_httpd(self):
450 httpd_path = self.path_to_apache()
452 server_name = self._filesystem.basename(httpd_path)
453 env = self.setup_environ_for_server(server_name)
454 if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0:
455 _log.error("httpd seems broken. Cannot run http tests.")
459 _log.error("No httpd found. Cannot run http tests.")
462 def do_text_results_differ(self, expected_text, actual_text):
463 return expected_text != actual_text
465 def do_audio_results_differ(self, expected_audio, actual_audio):
466 return expected_audio != actual_audio
468 def diff_image(self, expected_contents, actual_contents):
469 """Compare two images and return a tuple of an image diff, and an error string.
471 If an error occurs (like image_diff isn't found, or crashes, we log an error and return True (for a diff).
473 # If only one of them exists, return that one.
474 if not actual_contents and not expected_contents:
476 if not actual_contents:
477 return (expected_contents, None)
478 if not expected_contents:
479 return (actual_contents, None)
481 tempdir = self._filesystem.mkdtemp()
483 expected_filename = self._filesystem.join(str(tempdir), "expected.png")
484 self._filesystem.write_binary_file(expected_filename, expected_contents)
486 actual_filename = self._filesystem.join(str(tempdir), "actual.png")
487 self._filesystem.write_binary_file(actual_filename, actual_contents)
489 diff_filename = self._filesystem.join(str(tempdir), "diff.png")
491 # image_diff needs native win paths as arguments, so we need to convert them if running under cygwin.
492 native_expected_filename = self._convert_path(expected_filename)
493 native_actual_filename = self._convert_path(actual_filename)
494 native_diff_filename = self._convert_path(diff_filename)
496 executable = self._path_to_image_diff()
497 # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'.
498 comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename]
503 exit_code = self._executive.run_command(comand, return_exit_code=True)
505 # The images are the same.
508 result = self._filesystem.read_binary_file(native_diff_filename)
510 err_str = "Image diff returned an exit code of %s. See http://crbug.com/278596" % exit_code
512 err_str = 'error running image diff: %s' % str(e)
514 self._filesystem.rmtree(str(tempdir))
516 return (result, err_str or None)
518 def diff_text(self, expected_text, actual_text, expected_filename, actual_filename):
519 """Returns a string containing the diff of the two text strings
520 in 'unified diff' format."""
522 # The filenames show up in the diff output, make sure they're
523 # raw bytes and not unicode, so that they don't trigger join()
524 # trying to decode the input.
525 def to_raw_bytes(string_value):
526 if isinstance(string_value, unicode):
527 return string_value.encode('utf-8')
529 expected_filename = to_raw_bytes(expected_filename)
530 actual_filename = to_raw_bytes(actual_filename)
531 diff = difflib.unified_diff(expected_text.splitlines(True),
532 actual_text.splitlines(True),
536 # The diff generated by the difflib is incorrect if one of the files
537 # does not have a newline at the end of the file and it is present in
538 # the diff. Relevant Python issue: http://bugs.python.org/issue2142
539 def diff_fixup(diff):
542 if not line.endswith('\n'):
543 yield '\n\ No newline at end of file\n'
545 return ''.join(diff_fixup(diff))
547 def driver_name(self):
548 if self.get_option('driver_name'):
549 return self.get_option('driver_name')
550 return self.CONTENT_SHELL_NAME
552 def expected_baselines_by_extension(self, test_name):
553 """Returns a dict mapping baseline suffix to relative path for each baseline in
554 a test. For reftests, it returns ".==" or ".!=" instead of the suffix."""
555 # FIXME: The name similarity between this and expected_baselines() below, is unfortunate.
556 # We should probably rename them both.
558 reference_files = self.reference_files(test_name)
560 # FIXME: How should this handle more than one type of reftest?
561 baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1])
563 for extension in self.baseline_extensions():
564 path = self.expected_filename(test_name, extension, return_default=False)
565 baseline_dict[extension] = self.relative_test_filename(path) if path else path
569 def baseline_extensions(self):
570 """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'."""
571 return ('.wav', '.txt', '.png')
573 def expected_baselines(self, test_name, suffix, all_baselines=False):
574 """Given a test name, finds where the baseline results are located.
577 test_name: name of test file (usually a relative path under LayoutTests/)
578 suffix: file suffix of the expected results, including dot; e.g.
579 '.txt' or '.png'. This should not be None, but may be an empty
581 all_baselines: If True, return an ordered list of all baseline paths
582 for the given platform. If False, return only the first one.
584 a list of ( platform_dir, results_filename ), where
585 platform_dir - abs path to the top of the results tree (or test
587 results_filename - relative path from top of tree to the results
589 (port.join() of the two gives you the full path to the file,
590 unless None was returned.)
591 Return values will be in the format appropriate for the current
592 platform (e.g., "\\" for path separators on Windows). If the results
593 file is not found, then None will be returned for the directory,
594 but the expected relative pathname will still be returned.
596 This routine is generic but lives here since it is used in
597 conjunction with the other baseline and filename routines that are
600 baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix
601 baseline_search_path = self.baseline_search_path()
604 for platform_dir in baseline_search_path:
605 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
606 baselines.append((platform_dir, baseline_filename))
608 if not all_baselines and baselines:
611 # If it wasn't found in a platform directory, return the expected
612 # result in the test directory, even if no such file actually exists.
613 platform_dir = self.layout_tests_dir()
614 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
615 baselines.append((platform_dir, baseline_filename))
620 return [(None, baseline_filename)]
622 def expected_filename(self, test_name, suffix, return_default=True):
623 """Given a test name, returns an absolute path to its expected results.
625 If no expected results are found in any of the searched directories,
626 the directory in which the test itself is located will be returned.
627 The return value is in the format appropriate for the platform
628 (e.g., "\\" for path separators on windows).
631 test_name: name of test file (usually a relative path under LayoutTests/)
632 suffix: file suffix of the expected results, including dot; e.g. '.txt'
633 or '.png'. This should not be None, but may be an empty string.
634 platform: the most-specific directory name to use to build the
635 search list of directories, e.g., 'win', or
636 'chromium-cg-mac-leopard' (we follow the WebKit format)
637 return_default: if True, returns the path to the generic expectation if nothing
638 else is found; if False, returns None.
640 This routine is generic but is implemented here to live alongside
641 the other baseline and filename manipulation routines.
643 # FIXME: The [0] here is very mysterious, as is the destructured return.
644 platform_dir, baseline_filename = self.expected_baselines(test_name, suffix)[0]
646 return self._filesystem.join(platform_dir, baseline_filename)
648 actual_test_name = self.lookup_virtual_test_base(test_name)
650 return self.expected_filename(actual_test_name, suffix)
653 return self._filesystem.join(self.layout_tests_dir(), baseline_filename)
656 def expected_checksum(self, test_name):
657 """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
658 png_path = self.expected_filename(test_name, '.png')
660 if self._filesystem.exists(png_path):
661 with self._filesystem.open_binary_file_for_reading(png_path) as filehandle:
662 return read_checksum_from_png.read_checksum(filehandle)
666 def expected_image(self, test_name):
667 """Returns the image we expect the test to produce."""
668 baseline_path = self.expected_filename(test_name, '.png')
669 if not self._filesystem.exists(baseline_path):
671 return self._filesystem.read_binary_file(baseline_path)
673 def expected_audio(self, test_name):
674 baseline_path = self.expected_filename(test_name, '.wav')
675 if not self._filesystem.exists(baseline_path):
677 return self._filesystem.read_binary_file(baseline_path)
679 def expected_text(self, test_name):
680 """Returns the text output we expect the test to produce, or None
681 if we don't expect there to be any text output.
682 End-of-line characters are normalized to '\n'."""
683 # FIXME: DRT output is actually utf-8, but since we don't decode the
684 # output from DRT (instead treating it as a binary string), we read the
685 # baselines as a binary string, too.
686 baseline_path = self.expected_filename(test_name, '.txt')
687 if not self._filesystem.exists(baseline_path):
689 text = self._filesystem.read_binary_file(baseline_path)
690 return text.replace("\r\n", "\n")
692 def _get_reftest_list(self, test_name):
693 dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
694 if dirname not in self._reftest_list:
695 self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname)
696 return self._reftest_list[dirname]
699 def _parse_reftest_list(filesystem, test_dirpath):
700 reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
701 if not filesystem.isfile(reftest_list_path):
703 reftest_list_file = filesystem.read_text_file(reftest_list_path)
706 for line in reftest_list_file.split('\n'):
707 line = re.sub('#.+$', '', line)
708 split_line = line.split()
709 if len(split_line) == 4:
710 # FIXME: Probably one of mozilla's extensions in the reftest.list format. Do we need to support this?
711 _log.warning("unsupported reftest.list line '%s' in %s" % (line, reftest_list_path))
713 if len(split_line) < 3:
715 expectation_type, test_file, ref_file = split_line
716 parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file)))
719 def reference_files(self, test_name):
720 """Return a list of expectation (== or !=) and filename pairs"""
722 reftest_list = self._get_reftest_list(test_name)
725 for expectation, prefix in (('==', ''), ('!=', '-mismatch')):
726 for extention in Port._supported_file_extensions:
727 path = self.expected_filename(test_name, prefix + extention)
728 if self._filesystem.exists(path):
729 reftest_list.append((expectation, path))
732 return reftest_list.get(self._filesystem.join(self.layout_tests_dir(), test_name), []) # pylint: disable=E1103
734 def tests(self, paths):
735 """Return the list of tests found matching paths."""
736 tests = self._real_tests(paths)
738 suites = self.virtual_test_suites()
740 tests.extend(self._virtual_tests_matching_paths(paths, suites))
742 tests.extend(self._all_virtual_tests(suites))
745 def _real_tests(self, paths):
746 # When collecting test cases, skip these directories
747 skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'support', 'script-tests', 'reference', 'reftest'])
748 files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port.is_test_file, self.test_key)
749 return [self.relative_test_filename(f) for f in files]
751 # When collecting test cases, we include any file with these extensions.
752 _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl',
753 '.htm', '.php', '.svg', '.mht', '.pdf'])
756 # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well.
757 def is_reference_html_file(filesystem, dirname, filename):
758 if filename.startswith('ref-') or filename.startswith('notref-'):
760 filename_wihout_ext, unused = filesystem.splitext(filename)
761 for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
762 if filename_wihout_ext.endswith(suffix):
767 def _has_supported_extension(filesystem, filename):
768 """Return true if filename is one of the file extensions we want to run a test on."""
769 extension = filesystem.splitext(filename)[1]
770 return extension in Port._supported_file_extensions
773 def is_test_file(filesystem, dirname, filename):
774 return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename)
776 ALL_TEST_TYPES = ['audio', 'harness', 'pixel', 'ref', 'text', 'unknown']
778 def test_type(self, test_name):
779 fs = self._filesystem
780 if fs.exists(self.expected_filename(test_name, '.png')):
782 if fs.exists(self.expected_filename(test_name, '.wav')):
784 if self.reference_files(test_name):
786 txt = self.expected_text(test_name)
788 if 'layer at (0,0) size 800x600' in txt:
790 for line in txt.splitlines():
791 if line.startswith('FAIL') or line.startswith('TIMEOUT') or line.startswith('PASS'):
796 def test_key(self, test_name):
797 """Turns a test name into a list with two sublists, the natural key of the
798 dirname, and the natural key of the basename.
800 This can be used when sorting paths so that files in a directory.
801 directory are kept together rather than being mixed in with files in
803 dirname, basename = self.split_test(test_name)
804 return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename))
806 def _natural_sort_key(self, string_to_split):
807 """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"]
809 This can be used to implement "natural sort" order. See:
810 http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
811 http://nedbatchelder.com/blog/200712.html#e20071211T054956
819 return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
822 """Returns the list of top-level test directories."""
823 layout_tests_dir = self.layout_tests_dir()
824 return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
825 self._filesystem.listdir(layout_tests_dir))
828 def test_isfile(self, test_name):
829 """Return True if the test name refers to a directory of tests."""
830 # Used by test_expectations.py to apply rules to whole directories.
831 if self._filesystem.isfile(self.abspath_for_test(test_name)):
833 base = self.lookup_virtual_test_base(test_name)
834 return base and self._filesystem.isfile(self.abspath_for_test(base))
837 def test_isdir(self, test_name):
838 """Return True if the test name refers to a directory of tests."""
839 # Used by test_expectations.py to apply rules to whole directories.
840 if self._filesystem.isdir(self.abspath_for_test(test_name)):
842 base = self.lookup_virtual_test_base(test_name)
843 return base and self._filesystem.isdir(self.abspath_for_test(base))
846 def test_exists(self, test_name):
847 """Return True if the test name refers to an existing test or baseline."""
848 # Used by test_expectations.py to determine if an entry refers to a
849 # valid test and by printing.py to determine if baselines exist.
850 return self.test_isfile(test_name) or self.test_isdir(test_name)
852 def split_test(self, test_name):
853 """Splits a test name into the 'directory' part and the 'basename' part."""
854 index = test_name.rfind(self.TEST_PATH_SEPARATOR)
856 return ('', test_name)
857 return (test_name[0:index], test_name[index:])
859 def normalize_test_name(self, test_name):
860 """Returns a normalized version of the test name or test directory."""
861 if test_name.endswith('/'):
863 if self.test_isdir(test_name):
864 return test_name + '/'
867 def driver_cmd_line(self):
868 """Prints the DRT command line that will be used."""
869 driver = self.create_driver(0)
870 return driver.cmd_line(self.get_option('pixel_tests'), [])
872 def update_baseline(self, baseline_path, data):
873 """Updates the baseline for a test.
876 baseline_path: the actual path to use for baseline, not the path to
877 the test. This function is used to update either generic or
878 platform-specific baselines, but we can't infer which here.
879 data: contents of the baseline.
881 self._filesystem.write_binary_file(baseline_path, data)
883 # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected).
884 def webkit_base(self):
885 return self._webkit_finder.webkit_base()
887 def path_from_webkit_base(self, *comps):
888 return self._webkit_finder.path_from_webkit_base(*comps)
890 def path_from_chromium_base(self, *comps):
891 return self._webkit_finder.path_from_chromium_base(*comps)
893 def path_to_script(self, script_name):
894 return self._webkit_finder.path_to_script(script_name)
896 def layout_tests_dir(self):
897 return self._webkit_finder.layout_tests_dir()
899 def perf_tests_dir(self):
900 return self._webkit_finder.perf_tests_dir()
902 def skipped_layout_tests(self, test_list):
903 """Returns tests skipped outside of the TestExpectations files."""
904 tests = set(self._skipped_tests_for_unsupported_features(test_list))
906 # We explicitly skip any tests in LayoutTests/w3c if need be to avoid running any tests
907 # left over from the old DEPS-pulled repos.
908 # We also will warn at the end of the test run if these directories still exist.
910 # TODO(dpranke): Remove this check after 1/1/2015 and let people deal with the warnings.
911 # Remove the check in controllers/manager.py as well.
912 if self._filesystem.isdir(self._filesystem.join(self.layout_tests_dir(), 'w3c')):
917 def _tests_from_skipped_file_contents(self, skipped_file_contents):
919 for line in skipped_file_contents.split('\n'):
921 line = line.rstrip('/') # Best to normalize directory names to not include the trailing slash.
922 if line.startswith('#') or not len(line):
924 tests_to_skip.append(line)
927 def _expectations_from_skipped_files(self, skipped_file_paths):
929 for search_path in skipped_file_paths:
930 filename = self._filesystem.join(self._webkit_baseline_path(search_path), "Skipped")
931 if not self._filesystem.exists(filename):
932 _log.debug("Skipped does not exist: %s" % filename)
934 _log.debug("Using Skipped file: %s" % filename)
935 skipped_file_contents = self._filesystem.read_text_file(filename)
936 tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
940 def skipped_perf_tests(self):
941 return self._expectations_from_skipped_files([self.perf_tests_dir()])
943 def skips_perf_test(self, test_name):
944 for test_or_category in self.skipped_perf_tests():
945 if test_or_category == test_name:
947 category = self._filesystem.join(self.perf_tests_dir(), test_or_category)
948 if self._filesystem.isdir(category) and test_name.startswith(test_or_category):
952 def is_chromium(self):
956 """Returns a name that uniquely identifies this particular type of port
957 (e.g., "mac-snowleopard" or "linux-x86_x64" and can be passed
958 to factory.get() to instantiate the port."""
961 def operating_system(self):
962 # Subclasses should override this default implementation.
966 """Returns a string indicating the version of a given platform, e.g.
969 This is used to help identify the exact port when parsing test
970 expectations, determining search paths, and logging information."""
973 def architecture(self):
974 return self._architecture
976 def get_option(self, name, default_value=None):
977 return getattr(self._options, name, default_value)
979 def set_option_default(self, name, default_value):
980 return self._options.ensure_value(name, default_value)
983 def path_to_generic_test_expectations_file(self):
984 return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations')
986 def relative_test_filename(self, filename):
987 """Returns a test_name a relative unix-style path for a filename under the LayoutTests
988 directory. Ports may legitimately return abspaths here if no relpath makes sense."""
989 # Ports that run on windows need to override this method to deal with
990 # filenames with backslashes in them.
991 if filename.startswith(self.layout_tests_dir()):
992 return self.host.filesystem.relpath(filename, self.layout_tests_dir())
994 return self.host.filesystem.abspath(filename)
997 def abspath_for_test(self, test_name):
998 """Returns the full path to the file for a given test name. This is the
999 inverse of relative_test_filename()."""
1000 return self._filesystem.join(self.layout_tests_dir(), test_name)
1002 def results_directory(self):
1003 """Absolute path to the place to store the test results (uses --results-directory)."""
1004 if not self._results_directory:
1005 option_val = self.get_option('results_directory') or self.default_results_directory()
1006 self._results_directory = self._filesystem.abspath(option_val)
1007 return self._results_directory
1009 def perf_results_directory(self):
1010 return self._build_path()
1012 def default_results_directory(self):
1013 """Absolute path to the default place to store the test results."""
1015 return self.path_from_chromium_base('webkit', self.get_option('configuration'), 'layout-test-results')
1016 except AssertionError:
1017 return self._build_path('layout-test-results')
1019 def setup_test_run(self):
1020 """Perform port-specific work at the beginning of a test run."""
1021 # Delete the disk cache if any to ensure a clean test run.
1022 dump_render_tree_binary_path = self._path_to_driver()
1023 cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
1024 cachedir = self._filesystem.join(cachedir, "cache")
1025 if self._filesystem.exists(cachedir):
1026 self._filesystem.rmtree(cachedir)
1028 if self._dump_reader:
1029 self._filesystem.maybe_make_directory(self._dump_reader.crash_dumps_directory())
1031 def num_workers(self, requested_num_workers):
1032 """Returns the number of available workers (possibly less than the number requested)."""
1033 return requested_num_workers
1035 def clean_up_test_run(self):
1036 """Perform port-specific work at the end of a test run."""
1037 if self._image_differ:
1038 self._image_differ.stop()
1039 self._image_differ = None
1041 # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable.
1042 def _value_or_default_from_environ(self, name, default=None):
1043 if name in os.environ:
1044 return os.environ[name]
1047 def _copy_value_from_environ_if_set(self, clean_env, name):
1048 if name in os.environ:
1049 clean_env[name] = os.environ[name]
1051 def setup_environ_for_server(self, server_name=None):
1052 # We intentionally copy only a subset of os.environ when
1053 # launching subprocesses to ensure consistent test results.
1055 'LOCAL_RESOURCE_ROOT': self.layout_tests_dir(), # FIXME: Is this used?
1057 variables_to_copy = [
1058 'WEBKIT_TESTFONTS', # FIXME: Is this still used?
1059 'WEBKITOUTPUTDIR', # FIXME: Is this still used?
1060 'CHROME_DEVEL_SANDBOX',
1061 'CHROME_IPC_LOGGING',
1068 'VALGRIND_LIB_INNER',
1070 if self.host.platform.is_linux() or self.host.platform.is_freebsd():
1071 variables_to_copy += [
1076 'DBUS_SESSION_BUS_ADDRESS',
1079 clean_env['DISPLAY'] = self._value_or_default_from_environ('DISPLAY', ':1')
1080 if self.host.platform.is_mac():
1081 clean_env['DYLD_LIBRARY_PATH'] = self._build_path()
1082 clean_env['DYLD_FRAMEWORK_PATH'] = self._build_path()
1083 variables_to_copy += [
1086 if self.host.platform.is_win():
1087 variables_to_copy += [
1089 'GYP_DEFINES', # Required to locate win sdk.
1091 if self.host.platform.is_cygwin():
1092 variables_to_copy += [
1098 for variable in variables_to_copy:
1099 self._copy_value_from_environ_if_set(clean_env, variable)
1101 for string_variable in self.get_option('additional_env_var', []):
1102 [name, value] = string_variable.split('=', 1)
1103 clean_env[name] = value
1107 def show_results_html_file(self, results_filename):
1108 """This routine should display the HTML file pointed at by
1109 results_filename in a users' browser."""
1110 return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename))
1112 def create_driver(self, worker_number, no_timeout=False):
1113 """Return a newly created Driver subclass for starting/stopping the test driver."""
1114 return self._driver_class()(self, worker_number, pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout)
1116 def start_helper(self):
1117 """If a port needs to reconfigure graphics settings or do other
1118 things to ensure a known test configuration, it should override this
1120 helper_path = self._path_to_helper()
1122 _log.debug("Starting layout helper %s" % helper_path)
1123 # Note: Not thread safe: http://bugs.python.org/issue2320
1124 self._helper = self._executive.popen([helper_path],
1125 stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None)
1126 is_ready = self._helper.stdout.readline()
1127 if not is_ready.startswith('ready'):
1128 _log.error("layout_test_helper failed to be ready")
1130 def requires_http_server(self):
1131 """Does the port require an HTTP server for running tests? This could
1132 be the case when the tests aren't run on the host platform."""
1135 def start_http_server(self, additional_dirs, number_of_drivers):
1136 """Start a web server. Raise an error if it can't start or is already running.
1138 Ports can stub this out if they don't need a web server to be running."""
1139 assert not self._http_server, 'Already running an http server.'
1141 server = apache_http.ApacheHTTP(self, self.results_directory(),
1142 additional_dirs=additional_dirs,
1143 number_of_servers=(number_of_drivers * 4))
1145 self._http_server = server
1147 def start_websocket_server(self):
1148 """Start a web server. Raise an error if it can't start or is already running.
1150 Ports can stub this out if they don't need a websocket server to be running."""
1151 assert not self._websocket_server, 'Already running a websocket server.'
1153 server = pywebsocket.PyWebSocket(self, self.results_directory())
1155 self._websocket_server = server
1157 def http_server_supports_ipv6(self):
1158 # Apache < 2.4 on win32 does not support IPv6, nor does cygwin apache.
1159 if self.host.platform.is_cygwin() or self.host.platform.is_win():
1163 def stop_helper(self):
1164 """Shut down the test helper if it is running. Do nothing if
1165 it isn't, or it isn't available. If a port overrides start_helper()
1166 it must override this routine as well."""
1168 _log.debug("Stopping layout test helper")
1170 self._helper.stdin.write("x\n")
1171 self._helper.stdin.close()
1178 def stop_http_server(self):
1179 """Shut down the http server if it is running. Do nothing if it isn't."""
1180 if self._http_server:
1181 self._http_server.stop()
1182 self._http_server = None
1184 def stop_websocket_server(self):
1185 """Shut down the websocket server if it is running. Do nothing if it isn't."""
1186 if self._websocket_server:
1187 self._websocket_server.stop()
1188 self._websocket_server = None
1191 # TEST EXPECTATION-RELATED METHODS
1194 def test_configuration(self):
1195 """Returns the current TestConfiguration for the port."""
1196 if not self._test_configuration:
1197 self._test_configuration = TestConfiguration(self._version, self._architecture, self._options.configuration.lower())
1198 return self._test_configuration
1200 # FIXME: Belongs on a Platform object.
1202 def all_test_configurations(self):
1203 """Returns a list of TestConfiguration instances, representing all available
1204 test configurations for this port."""
1205 return self._generate_all_test_configurations()
1207 # FIXME: Belongs on a Platform object.
1208 def configuration_specifier_macros(self):
1209 """Ports may provide a way to abbreviate configuration specifiers to conveniently
1210 refer to them as one term or alias specific values to more generic ones. For example:
1212 (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake.
1213 (lucid) -> linux # Change specific name of the Linux distro to a more generic term.
1215 Returns a dictionary, each key representing a macro term ('win', for example),
1216 and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7'])."""
1217 return self.CONFIGURATION_SPECIFIER_MACROS
1219 def all_baseline_variants(self):
1220 """Returns a list of platform names sufficient to cover all the baselines.
1222 The list should be sorted so that a later platform will reuse
1223 an earlier platform's baselines if they are the same (e.g.,
1224 'snowleopard' should precede 'leopard')."""
1225 return self.ALL_BASELINE_VARIANTS
1227 def _generate_all_test_configurations(self):
1228 """Returns a sequence of the TestConfigurations the port supports."""
1229 # By default, we assume we want to test every graphics type in
1230 # every configuration on every system.
1231 test_configurations = []
1232 for version, architecture in self.ALL_SYSTEMS:
1233 for build_type in self.ALL_BUILD_TYPES:
1234 test_configurations.append(TestConfiguration(version, architecture, build_type))
1235 return test_configurations
1237 try_builder_names = frozenset([
1246 def warn_if_bug_missing_in_test_expectations(self):
1249 def _port_specific_expectations_files(self):
1251 paths.append(self.path_from_chromium_base('skia', 'skia_test_expectations.txt'))
1252 paths.append(self._filesystem.join(self.layout_tests_dir(), 'NeverFixTests'))
1253 paths.append(self._filesystem.join(self.layout_tests_dir(), 'StaleTestExpectations'))
1254 paths.append(self._filesystem.join(self.layout_tests_dir(), 'SlowTests'))
1255 paths.append(self._filesystem.join(self.layout_tests_dir(), 'FlakyTests'))
1259 def expectations_dict(self):
1260 """Returns an OrderedDict of name -> expectations strings.
1261 The names are expected to be (but not required to be) paths in the filesystem.
1262 If the name is a path, the file can be considered updatable for things like rebaselining,
1263 so don't use names that are paths if they're not paths.
1264 Generally speaking the ordering should be files in the filesystem in cascade order
1265 (TestExpectations followed by Skipped, if the port honors both formats),
1266 then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options."""
1267 # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict.
1268 expectations = OrderedDict()
1270 for path in self.expectations_files():
1271 if self._filesystem.exists(path):
1272 expectations[path] = self._filesystem.read_text_file(path)
1274 for path in self.get_option('additional_expectations', []):
1275 expanded_path = self._filesystem.expanduser(path)
1276 if self._filesystem.exists(expanded_path):
1277 _log.debug("reading additional_expectations from path '%s'" % path)
1278 expectations[path] = self._filesystem.read_text_file(expanded_path)
1280 _log.warning("additional_expectations path '%s' does not exist" % path)
1283 def bot_expectations(self):
1284 if not self.get_option('ignore_flaky_tests'):
1287 full_port_name = self.determine_full_port_name(self.host, self._options, self.port_name)
1288 builder_category = self.get_option('ignore_builder_category', 'layout')
1289 factory = BotTestExpectationsFactory()
1290 # FIXME: This only grabs release builder's flakiness data. If we're running debug,
1291 # when we should grab the debug builder's data.
1292 expectations = factory.expectations_for_port(full_port_name, builder_category)
1294 if not expectations:
1297 ignore_mode = self.get_option('ignore_flaky_tests')
1298 if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky':
1299 return expectations.flakes_by_path(ignore_mode == 'very-flaky')
1300 if ignore_mode == 'unexpected':
1301 return expectations.unexpected_results_by_path()
1302 _log.warning("Unexpected ignore mode: '%s'." % ignore_mode)
1305 def expectations_files(self):
1306 return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files()
1308 def repository_paths(self):
1309 """Returns a list of (repository_name, repository_path) tuples of its depending code base."""
1310 return [('blink', self.layout_tests_dir()),
1311 ('chromium', self.path_from_chromium_base('build'))]
1313 _WDIFF_DEL = '##WDIFF_DEL##'
1314 _WDIFF_ADD = '##WDIFF_ADD##'
1315 _WDIFF_END = '##WDIFF_END##'
1317 def _format_wdiff_output_as_html(self, wdiff):
1318 wdiff = cgi.escape(wdiff)
1319 wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
1320 wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
1321 wdiff = wdiff.replace(self._WDIFF_END, "</span>")
1322 html = "<head><style>.del { background: #faa; } "
1323 html += ".add { background: #afa; }</style></head>"
1324 html += "<pre>%s</pre>" % wdiff
1327 def _wdiff_command(self, actual_filename, expected_filename):
1328 executable = self._path_to_wdiff()
1330 "--start-delete=%s" % self._WDIFF_DEL,
1331 "--end-delete=%s" % self._WDIFF_END,
1332 "--start-insert=%s" % self._WDIFF_ADD,
1333 "--end-insert=%s" % self._WDIFF_END,
1338 def _handle_wdiff_error(script_error):
1339 # Exit 1 means the files differed, any other exit code is an error.
1340 if script_error.exit_code != 1:
1343 def _run_wdiff(self, actual_filename, expected_filename):
1344 """Runs wdiff and may throw exceptions.
1345 This is mostly a hook for unit testing."""
1346 # Diffs are treated as binary as they may include multiple files
1347 # with conflicting encodings. Thus we do not decode the output.
1348 command = self._wdiff_command(actual_filename, expected_filename)
1349 wdiff = self._executive.run_command(command, decode_output=False,
1350 error_handler=self._handle_wdiff_error)
1351 return self._format_wdiff_output_as_html(wdiff)
1353 _wdiff_error_html = "Failed to run wdiff, see error log."
1355 def wdiff_text(self, actual_filename, expected_filename):
1356 """Returns a string of HTML indicating the word-level diff of the
1357 contents of the two filenames. Returns an empty string if word-level
1358 diffing isn't available."""
1359 if not self.wdiff_available():
1362 # It's possible to raise a ScriptError we pass wdiff invalid paths.
1363 return self._run_wdiff(actual_filename, expected_filename)
1364 except OSError as e:
1365 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
1366 # Silently ignore cases where wdiff is missing.
1367 self._wdiff_available = False
1370 except ScriptError as e:
1371 _log.error("Failed to run wdiff: %s" % e)
1372 self._wdiff_available = False
1373 return self._wdiff_error_html
1375 # This is a class variable so we can test error output easily.
1376 _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."
1378 def pretty_patch_text(self, diff_path):
1379 if self._pretty_patch_available is None:
1380 self._pretty_patch_available = self.check_pretty_patch(logging=False)
1381 if not self._pretty_patch_available:
1382 return self._pretty_patch_error_html
1383 command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path),
1384 self._pretty_patch_path, diff_path)
1386 # Diffs are treated as binary (we pass decode_output=False) as they
1387 # may contain multiple files of conflicting encodings.
1388 return self._executive.run_command(command, decode_output=False)
1390 # If the system is missing ruby log the error and stop trying.
1391 self._pretty_patch_available = False
1392 _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
1393 return self._pretty_patch_error_html
1394 except ScriptError, e:
1395 # If ruby failed to run for some reason, log the command
1396 # output and stop trying.
1397 self._pretty_patch_available = False
1398 _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output()))
1399 return self._pretty_patch_error_html
1401 def default_configuration(self):
1402 return self._config.default_configuration()
1404 def clobber_old_port_specific_results(self):
1407 # FIXME: This does not belong on the port object.
1409 def path_to_apache(self):
1410 """Returns the full path to the apache binary.
1412 This is needed only by ports that use the apache_http_server module."""
1413 raise NotImplementedError('Port.path_to_apache')
1415 def path_to_apache_config_file(self):
1416 """Returns the full path to the apache configuration file.
1418 If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its
1419 contents will be used instead.
1421 This is needed only by ports that use the apache_http_server module."""
1422 config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH')
1423 if config_file_from_env:
1424 if not self._filesystem.exists(config_file_from_env):
1425 raise IOError('%s was not found on the system' % config_file_from_env)
1426 return config_file_from_env
1428 config_file_name = self._apache_config_file_name_for_platform(sys.platform)
1429 return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name)
1432 # PROTECTED ROUTINES
1434 # The routines below should only be called by routines in this class
1435 # or any of its subclasses.
1438 # FIXME: This belongs on some platform abstraction instead of Port.
1439 def _is_redhat_based(self):
1440 return self._filesystem.exists('/etc/redhat-release')
1442 def _is_debian_based(self):
1443 return self._filesystem.exists('/etc/debian_version')
1445 def _apache_version(self):
1446 config = self._executive.run_command([self.path_to_apache(), '-v'])
1447 return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config)
1449 # We pass sys_platform into this method to make it easy to unit test.
1450 def _apache_config_file_name_for_platform(self, sys_platform):
1451 if sys_platform == 'cygwin':
1452 return 'cygwin-httpd.conf' # CYGWIN is the only platform to still use Apache 1.3.
1453 if sys_platform.startswith('linux'):
1454 if self._is_redhat_based():
1455 return 'fedora-httpd-' + self._apache_version() + '.conf'
1456 if self._is_debian_based():
1457 return 'debian-httpd-' + self._apache_version() + '.conf'
1458 # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support).
1459 return "apache2-httpd.conf"
1461 def _path_to_driver(self, configuration=None):
1462 """Returns the full path to the test driver."""
1463 return self._build_path(self.driver_name())
1465 def _path_to_webcore_library(self):
1466 """Returns the full path to a built copy of WebCore."""
1469 def _path_to_helper(self):
1470 """Returns the full path to the layout_test_helper binary, which
1471 is used to help configure the system for the test run, or None
1472 if no helper is needed.
1474 This is likely only used by start/stop_helper()."""
1477 def _path_to_image_diff(self):
1478 """Returns the full path to the image_diff binary, or None if it is not available.
1480 This is likely used only by diff_image()"""
1481 return self._build_path('image_diff')
1484 def _path_to_wdiff(self):
1485 """Returns the full path to the wdiff binary, or None if it is not available.
1487 This is likely used only by wdiff_text()"""
1488 for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"):
1489 if self._filesystem.exists(path):
1493 def _webkit_baseline_path(self, platform):
1494 """Return the full path to the top of the baseline tree for a
1496 return self._filesystem.join(self.layout_tests_dir(), 'platform', platform)
1498 def _driver_class(self):
1499 """Returns the port's driver implementation."""
1500 return driver.Driver
1502 def _output_contains_sanitizer_messages(self, output):
1505 if 'AddressSanitizer' in output:
1506 return 'AddressSanitizer'
1507 if 'MemorySanitizer' in output:
1508 return 'MemorySanitizer'
1511 def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
1512 if self._output_contains_sanitizer_messages(stderr):
1513 # Running the symbolizer script can take a lot of memory, so we need to
1514 # serialize access to it across all the concurrently running drivers.
1516 llvm_symbolizer_path = self.path_from_chromium_base('third_party', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer')
1517 if self._filesystem.exists(llvm_symbolizer_path):
1518 env = os.environ.copy()
1519 env['LLVM_SYMBOLIZER_PATH'] = llvm_symbolizer_path
1522 sanitizer_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py')
1523 sanitizer_strip_path_prefix = 'Release/../../'
1524 if self._filesystem.exists(sanitizer_filter_path):
1525 stderr = self._executive.run_command(['flock', sys.executable, sanitizer_filter_path, sanitizer_strip_path_prefix], input=stderr, decode_output=False, env=env)
1527 name_str = name or '<unknown process name>'
1528 pid_str = str(pid or '<unknown>')
1529 stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines()
1530 stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines()
1531 return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str,
1532 '\n'.join(('STDOUT: ' + l) for l in stdout_lines),
1533 '\n'.join(('STDERR: ' + l) for l in stderr_lines)))
1535 def look_for_new_crash_logs(self, crashed_processes, start_time):
1538 def look_for_new_samples(self, unresponsive_processes, start_time):
1541 def sample_process(self, name, pid):
1544 def physical_test_suites(self):
1546 # For example, to turn on force-compositing-mode in the svg/ directory:
1547 # PhysicalTestSuite('svg',
1548 # ['--force-compositing-mode']),
1551 def virtual_test_suites(self):
1552 if self._virtual_test_suites is None:
1553 path_to_virtual_test_suites = self._filesystem.join(self.layout_tests_dir(), 'VirtualTestSuites')
1554 assert self._filesystem.exists(path_to_virtual_test_suites), 'LayoutTests/VirtualTestSuites not found'
1556 test_suite_json = json.loads(self._filesystem.read_text_file(path_to_virtual_test_suites))
1557 self._virtual_test_suites = [VirtualTestSuite(**d) for d in test_suite_json]
1558 except ValueError as e:
1559 raise ValueError("LayoutTests/VirtualTestSuites is not a valid JSON file: %s" % str(e))
1560 return self._virtual_test_suites
1562 def _all_virtual_tests(self, suites):
1564 for suite in suites:
1565 self._populate_virtual_suite(suite)
1566 tests.extend(suite.tests.keys())
1569 def _virtual_tests_matching_paths(self, paths, suites):
1571 for suite in suites:
1572 if any(p.startswith(suite.name) for p in paths):
1573 self._populate_virtual_suite(suite)
1574 for test in suite.tests:
1575 if any(test.startswith(p) for p in paths):
1579 def _populate_virtual_suite(self, suite):
1581 base_tests = self._real_tests([suite.base])
1583 for test in base_tests:
1584 suite.tests[test.replace(suite.base, suite.name, 1)] = test
1586 def is_virtual_test(self, test_name):
1587 return bool(self.lookup_virtual_suite(test_name))
1589 def lookup_virtual_suite(self, test_name):
1590 for suite in self.virtual_test_suites():
1591 if test_name.startswith(suite.name):
1595 def lookup_virtual_test_base(self, test_name):
1596 suite = self.lookup_virtual_suite(test_name)
1599 return test_name.replace(suite.name, suite.base, 1)
1601 def lookup_virtual_test_args(self, test_name):
1602 for suite in self.virtual_test_suites():
1603 if test_name.startswith(suite.name):
1607 def lookup_physical_test_args(self, test_name):
1608 for suite in self.physical_test_suites():
1609 if test_name.startswith(suite.name):
1613 def should_run_as_pixel_test(self, test_input):
1614 if not self._options.pixel_tests:
1616 if self._options.pixel_test_directories:
1617 return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories)
1620 def _modules_to_search_for_symbols(self):
1621 path = self._path_to_webcore_library()
1626 def _symbols_string(self):
1628 for path_to_module in self._modules_to_search_for_symbols():
1630 symbols += self._executive.run_command(['nm', path_to_module], error_handler=self._executive.ignore_error)
1632 _log.warn("Failed to run nm: %s. Can't determine supported features correctly." % e)
1635 # Ports which use compile-time feature detection should define this method and return
1636 # a dictionary mapping from symbol substrings to possibly disabled test directories.
1637 # When the symbol substrings are not matched, the directories will be skipped.
1638 # If ports don't ever enable certain features, then those directories can just be
1639 # in the Skipped list instead of compile-time-checked here.
1640 def _missing_symbol_to_skipped_tests(self):
1641 if self.PORT_HAS_AUDIO_CODECS_BUILT_IN:
1645 "ff_mp3_decoder": ["webaudio/codec-tests/mp3"],
1646 "ff_aac_decoder": ["webaudio/codec-tests/aac"],
1649 def _has_test_in_directories(self, directory_lists, test_list):
1653 directories = itertools.chain.from_iterable(directory_lists)
1654 for directory, test in itertools.product(directories, test_list):
1655 if test.startswith(directory):
1659 def _skipped_tests_for_unsupported_features(self, test_list):
1660 # Only check the symbols of there are tests in the test_list that might get skipped.
1661 # This is a performance optimization to avoid the calling nm.
1662 # Runtime feature detection not supported, fallback to static detection:
1663 # Disable any tests for symbols missing from the executable or libraries.
1664 if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list):
1665 symbols_string = self._symbols_string()
1666 if symbols_string is not None:
1667 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], [])
1670 def _convert_path(self, path):
1671 """Handles filename conversion for subprocess command line args."""
1672 # See note above in diff_image() for why we need this.
1673 if sys.platform == 'cygwin':
1674 return cygpath(path)
1677 def _build_path(self, *comps):
1678 return self._build_path_with_configuration(None, *comps)
1680 def _build_path_with_configuration(self, configuration, *comps):
1681 # Note that we don't do the option caching that the
1682 # base class does, because finding the right directory is relatively
1684 configuration = configuration or self.get_option('configuration')
1685 return self._static_build_path(self._filesystem, self.get_option('build_directory'),
1686 self.path_from_chromium_base(), configuration, comps)
1688 def _check_driver_build_up_to_date(self, configuration):
1689 if configuration in ('Debug', 'Release'):
1691 debug_path = self._path_to_driver('Debug')
1692 release_path = self._path_to_driver('Release')
1694 debug_mtime = self._filesystem.mtime(debug_path)
1695 release_mtime = self._filesystem.mtime(release_path)
1697 if (debug_mtime > release_mtime and configuration == 'Release' or
1698 release_mtime > debug_mtime and configuration == 'Debug'):
1699 most_recent_binary = 'Release' if configuration == 'Debug' else 'Debug'
1700 _log.warning('You are running the %s binary. However the %s binary appears to be more recent. '
1701 'Please pass --%s.', configuration, most_recent_binary, most_recent_binary.lower())
1703 # This will fail if we don't have both a debug and release binary.
1704 # That's fine because, in this case, we must already be running the
1705 # most up-to-date one.
1710 def _chromium_baseline_path(self, platform):
1711 if platform is None:
1712 platform = self.name()
1713 return self.path_from_webkit_base('LayoutTests', 'platform', platform)
1715 class VirtualTestSuite(object):
1716 def __init__(self, prefix=None, base=None, args=None):
1719 assert prefix.find('/') == -1, "Virtual test suites prefixes cannot contain /'s: %s" % prefix
1720 self.name = 'virtual/' + prefix + '/' + base
1726 return "VirtualTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)
1729 class PhysicalTestSuite(object):
1730 def __init__(self, base, args):
1737 return "PhysicalTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)