src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import logging
  30 import optparse
  31 import sys
  32 import tempfile
  33 import unittest
  34
  35 from webkitpy.common.system.executive import Executive, ScriptError
  36 from webkitpy.common.system import executive_mock
  37 from webkitpy.common.system.filesystem_mock import MockFileSystem
  38 from webkitpy.common.system.outputcapture import OutputCapture
  39 from webkitpy.common.system.path import abspath_to_uri
  40 from webkitpy.tool.mocktool import MockOptions
  41 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
  42 from webkitpy.common.system.systemhost import SystemHost
  43 from webkitpy.common.system.systemhost_mock import MockSystemHost
  44
  45 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
  46 from webkitpy.layout_tests.port.base import VirtualTestSuite
  47 from webkitpy.layout_tests.port.test import add_unit_tests_to_mock_filesystem, TestPort
  48
  49 class PortTest(unittest.TestCase):
  50     def make_port(self, executive=None, with_tests=False, port_name=None, **kwargs):
  51         host = MockSystemHost()
  52         if executive:
  53             host.executive = executive
  54         if with_tests:
  55             add_unit_tests_to_mock_filesystem(host.filesystem)
  56             return TestPort(host, **kwargs)
  57         return Port(host, port_name or 'baseport', **kwargs)
  58
  59     def test_default_child_processes(self):
  60         port = self.make_port()
  61         self.assertIsNotNone(port.default_child_processes())
  62
  63     def test_format_wdiff_output_as_html(self):
  64         output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
  65         html = self.make_port()._format_wdiff_output_as_html(output)
  66         expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
  67         self.assertEqual(html, expected_html)
  68
  69     def test_wdiff_command(self):
  70         port = self.make_port()
  71         port._path_to_wdiff = lambda: "/path/to/wdiff"
  72         command = port._wdiff_command("/actual/path", "/expected/path")
  73         expected_command = [
  74             "/path/to/wdiff",
  75             "--start-delete=##WDIFF_DEL##",
  76             "--end-delete=##WDIFF_END##",
  77             "--start-insert=##WDIFF_ADD##",
  78             "--end-insert=##WDIFF_END##",
  79             "/actual/path",
  80             "/expected/path",
  81         ]
  82         self.assertEqual(command, expected_command)
  83
  84     def _file_with_contents(self, contents, encoding="utf-8"):
  85         new_file = tempfile.NamedTemporaryFile()
  86         new_file.write(contents.encode(encoding))
  87         new_file.flush()
  88         return new_file
  89
  90     def test_pretty_patch_os_error(self):
  91         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
  92         oc = OutputCapture()
  93         oc.capture_output()
  94         self.assertEqual(port.pretty_patch_text("patch.txt"),
  95                          port._pretty_patch_error_html)
  96
  97         # This tests repeated calls to make sure we cache the result.
  98         self.assertEqual(port.pretty_patch_text("patch.txt"),
  99                          port._pretty_patch_error_html)
 100         oc.restore_output()
 101
 102     def test_pretty_patch_script_error(self):
 103         # FIXME: This is some ugly white-box test hacking ...
 104         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
 105         port._pretty_patch_available = True
 106         self.assertEqual(port.pretty_patch_text("patch.txt"),
 107                          port._pretty_patch_error_html)
 108
 109         # This tests repeated calls to make sure we cache the result.
 110         self.assertEqual(port.pretty_patch_text("patch.txt"),
 111                          port._pretty_patch_error_html)
 112
 113     def test_wdiff_text(self):
 114         port = self.make_port()
 115         port.wdiff_available = lambda: True
 116         port._run_wdiff = lambda a, b: 'PASS'
 117         self.assertEqual('PASS', port.wdiff_text(None, None))
 118
 119     def test_diff_text(self):
 120         port = self.make_port()
 121         # Make sure that we don't run into decoding exceptions when the
 122         # filenames are unicode, with regular or malformed input (expected or
 123         # actual input is always raw bytes, not unicode).
 124         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
 125         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
 126         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
 127
 128         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
 129         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
 130
 131         # Though expected and actual files should always be read in with no
 132         # encoding (and be stored as str objects), test unicode inputs just to
 133         # be safe.
 134         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
 135         port.diff_text(
 136             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
 137
 138         # And make sure we actually get diff output.
 139         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
 140         self.assertIn('foo', diff)
 141         self.assertIn('bar', diff)
 142         self.assertIn('exp.txt', diff)
 143         self.assertIn('act.txt', diff)
 144         self.assertNotIn('nosuchthing', diff)
 145
 146         # Test for missing newline at end of file diff output.
 147         content_a = "Hello\n\nWorld"
 148         content_b = "Hello\n\nWorld\n\n\n"
 149         expected = "--- exp.txt\n+++ act.txt\n@@ -1,3 +1,5 @@\n Hello\n \n-World\n\ No newline at end of file\n+World\n+\n+\n"
 150         self.assertEqual(expected, port.diff_text(content_a, content_b, 'exp.txt', 'act.txt'))
 151
 152     def test_setup_test_run(self):
 153         port = self.make_port()
 154         # This routine is a no-op. We just test it for coverage.
 155         port.setup_test_run()
 156
 157     def test_test_dirs(self):
 158         port = self.make_port()
 159         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/canvas/test', '')
 160         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/css2.1/test', '')
 161         dirs = port.test_dirs()
 162         self.assertIn('canvas', dirs)
 163         self.assertIn('css2.1', dirs)
 164
 165     def test_skipped_perf_tests(self):
 166         port = self.make_port()
 167
 168         def add_text_file(dirname, filename, content='some content'):
 169             dirname = port.host.filesystem.join(port.perf_tests_dir(), dirname)
 170             port.host.filesystem.maybe_make_directory(dirname)
 171             port.host.filesystem.write_text_file(port.host.filesystem.join(dirname, filename), content)
 172
 173         add_text_file('inspector', 'test1.html')
 174         add_text_file('inspector', 'unsupported_test1.html')
 175         add_text_file('inspector', 'test2.html')
 176         add_text_file('inspector/resources', 'resource_file.html')
 177         add_text_file('unsupported', 'unsupported_test2.html')
 178         add_text_file('', 'Skipped', '\n'.join(['Layout', '', 'SunSpider', 'Supported/some-test.html']))
 179         self.assertEqual(port.skipped_perf_tests(), ['Layout', 'SunSpider', 'Supported/some-test.html'])
 180
 181     def test_get_option__set(self):
 182         options, args = optparse.OptionParser().parse_args([])
 183         options.foo = 'bar'
 184         port = self.make_port(options=options)
 185         self.assertEqual(port.get_option('foo'), 'bar')
 186
 187     def test_get_option__unset(self):
 188         port = self.make_port()
 189         self.assertIsNone(port.get_option('foo'))
 190
 191     def test_get_option__default(self):
 192         port = self.make_port()
 193         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
 194
 195     def test_additional_platform_directory(self):
 196         port = self.make_port(port_name='foo')
 197         port.default_baseline_search_path = lambda: ['LayoutTests/platform/foo']
 198         layout_test_dir = port.layout_tests_dir()
 199         test_file = 'fast/test.html'
 200
 201         # No additional platform directory
 202         self.assertEqual(
 203             port.expected_baselines(test_file, '.txt'),
 204             [(None, 'fast/test-expected.txt')])
 205         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
 206
 207         # Simple additional platform directory
 208         port._options.additional_platform_directory = ['/tmp/local-baselines']
 209         port._filesystem.write_text_file('/tmp/local-baselines/fast/test-expected.txt', 'foo')
 210         self.assertEqual(
 211             port.expected_baselines(test_file, '.txt'),
 212             [('/tmp/local-baselines', 'fast/test-expected.txt')])
 213         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
 214
 215         # Multiple additional platform directories
 216         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
 217         self.assertEqual(
 218             port.expected_baselines(test_file, '.txt'),
 219             [('/tmp/local-baselines', 'fast/test-expected.txt')])
 220         self.assertEqual(port.baseline_path(), '/foo')
 221
 222     def test_nonexistant_expectations(self):
 223         port = self.make_port(port_name='foo')
 224         port.expectations_files = lambda: ['/mock-checkout/third_party/WebKit/LayoutTests/platform/exists/TestExpectations', '/mock-checkout/third_party/WebKit/LayoutTests/platform/nonexistant/TestExpectations']
 225         port._filesystem.write_text_file('/mock-checkout/third_party/WebKit/LayoutTests/platform/exists/TestExpectations', '')
 226         self.assertEqual('\n'.join(port.expectations_dict().keys()), '/mock-checkout/third_party/WebKit/LayoutTests/platform/exists/TestExpectations')
 227
 228     def test_additional_expectations(self):
 229         port = self.make_port(port_name='foo')
 230         port.port_name = 'foo'
 231         port._filesystem.write_text_file('/mock-checkout/third_party/WebKit/LayoutTests/platform/foo/TestExpectations', '')
 232         port._filesystem.write_text_file(
 233             '/tmp/additional-expectations-1.txt', 'content1\n')
 234         port._filesystem.write_text_file(
 235             '/tmp/additional-expectations-2.txt', 'content2\n')
 236
 237         self.assertEqual('\n'.join(port.expectations_dict().values()), '')
 238
 239         port._options.additional_expectations = [
 240             '/tmp/additional-expectations-1.txt']
 241         self.assertEqual('\n'.join(port.expectations_dict().values()), 'content1\n')
 242
 243         port._options.additional_expectations = [
 244             '/tmp/nonexistent-file', '/tmp/additional-expectations-1.txt']
 245         self.assertEqual('\n'.join(port.expectations_dict().values()), 'content1\n')
 246
 247         port._options.additional_expectations = [
 248             '/tmp/additional-expectations-1.txt', '/tmp/additional-expectations-2.txt']
 249         self.assertEqual('\n'.join(port.expectations_dict().values()), 'content1\n\ncontent2\n')
 250
 251     def test_additional_env_var(self):
 252         port = self.make_port(options=optparse.Values({'additional_env_var': ['FOO=BAR', 'BAR=FOO']}))
 253         self.assertEqual(port.get_option('additional_env_var'), ['FOO=BAR', 'BAR=FOO'])
 254         environment = port.setup_environ_for_server()
 255         self.assertTrue(('FOO' in environment) & ('BAR' in environment))
 256         self.assertEqual(environment['FOO'], 'BAR')
 257         self.assertEqual(environment['BAR'], 'FOO')
 258
 259     def test_find_no_paths_specified(self):
 260         port = self.make_port(with_tests=True)
 261         layout_tests_dir = port.layout_tests_dir()
 262         tests = port.tests([])
 263         self.assertNotEqual(len(tests), 0)
 264
 265     def test_find_one_test(self):
 266         port = self.make_port(with_tests=True)
 267         tests = port.tests(['failures/expected/image.html'])
 268         self.assertEqual(len(tests), 1)
 269
 270     def test_find_glob(self):
 271         port = self.make_port(with_tests=True)
 272         tests = port.tests(['failures/expected/im*'])
 273         self.assertEqual(len(tests), 2)
 274
 275     def test_find_with_skipped_directories(self):
 276         port = self.make_port(with_tests=True)
 277         tests = port.tests(['userscripts'])
 278         self.assertNotIn('userscripts/resources/iframe.html', tests)
 279
 280     def test_find_with_skipped_directories_2(self):
 281         port = self.make_port(with_tests=True)
 282         tests = port.tests(['userscripts/resources'])
 283         self.assertEqual(tests, [])
 284
 285     def test_is_test_file(self):
 286         filesystem = MockFileSystem()
 287         self.assertTrue(Port.is_test_file(filesystem, '', 'foo.html'))
 288         self.assertTrue(Port.is_test_file(filesystem, '', 'foo.svg'))
 289         self.assertTrue(Port.is_test_file(filesystem, '', 'test-ref-test.html'))
 290         self.assertFalse(Port.is_test_file(filesystem, '', 'foo.png'))
 291         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.html'))
 292         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.svg'))
 293         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.xht'))
 294         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
 295         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.svg'))
 296         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.xhtml'))
 297         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.html'))
 298         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-notref.html'))
 299         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-notref.xht'))
 300         self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.xhtml'))
 301         self.assertFalse(Port.is_test_file(filesystem, '', 'ref-foo.html'))
 302         self.assertFalse(Port.is_test_file(filesystem, '', 'notref-foo.xhr'))
 303
 304     def test_parse_reftest_list(self):
 305         port = self.make_port(with_tests=True)
 306         port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
 307         "",
 308         "# some comment",
 309         "!= test-2.html test-notref.html # more comments",
 310         "== test-3.html test-ref.html",
 311         "== test-3.html test-ref2.html",
 312         "!= test-3.html test-notref.html",
 313         "fuzzy(80,500) == test-3 test-ref.html"])
 314
 315         # Note that we don't support the syntax in the last line; the code should ignore it, rather than crashing.
 316
 317         reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar')
 318         self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')],
 319             'bar/test-2.html': [('!=', 'bar/test-notref.html')],
 320             'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]})
 321
 322     def test_reference_files(self):
 323         port = self.make_port(with_tests=True)
 324         self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')])
 325         self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.html')])
 326         self.assertEqual(port.reference_files('passes/phpreftest.php'), [('!=', port.layout_tests_dir() + '/passes/phpreftest-expected-mismatch.svg')])
 327
 328     def test_operating_system(self):
 329         self.assertEqual('mac', self.make_port().operating_system())
 330
 331     def test_http_server_supports_ipv6(self):
 332         port = self.make_port()
 333         self.assertTrue(port.http_server_supports_ipv6())
 334         port.host.platform.os_name = 'cygwin'
 335         self.assertFalse(port.http_server_supports_ipv6())
 336         port.host.platform.os_name = 'win'
 337         self.assertFalse(port.http_server_supports_ipv6())
 338
 339     def test_check_httpd_success(self):
 340         port = self.make_port(executive=MockExecutive2())
 341         port.path_to_apache = lambda: '/usr/sbin/httpd'
 342         capture = OutputCapture()
 343         capture.capture_output()
 344         self.assertTrue(port.check_httpd())
 345         _, _, logs = capture.restore_output()
 346         self.assertEqual('', logs)
 347
 348     def test_httpd_returns_error_code(self):
 349         port = self.make_port(executive=MockExecutive2(exit_code=1))
 350         port.path_to_apache = lambda: '/usr/sbin/httpd'
 351         capture = OutputCapture()
 352         capture.capture_output()
 353         self.assertFalse(port.check_httpd())
 354         _, _, logs = capture.restore_output()
 355         self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs)
 356
 357     def test_test_exists(self):
 358         port = self.make_port(with_tests=True)
 359         self.assertTrue(port.test_exists('passes'))
 360         self.assertTrue(port.test_exists('passes/text.html'))
 361         self.assertFalse(port.test_exists('passes/does_not_exist.html'))
 362
 363         self.assertTrue(port.test_exists('virtual'))
 364         self.assertFalse(port.test_exists('virtual/does_not_exist.html'))
 365         self.assertTrue(port.test_exists('virtual/virtual_passes/passes/text.html'))
 366
 367     def test_test_isfile(self):
 368         port = self.make_port(with_tests=True)
 369         self.assertFalse(port.test_isfile('passes'))
 370         self.assertTrue(port.test_isfile('passes/text.html'))
 371         self.assertFalse(port.test_isfile('passes/does_not_exist.html'))
 372
 373         self.assertFalse(port.test_isfile('virtual'))
 374         self.assertTrue(port.test_isfile('virtual/virtual_passes/passes/text.html'))
 375         self.assertFalse(port.test_isfile('virtual/does_not_exist.html'))
 376
 377     def test_test_isdir(self):
 378         port = self.make_port(with_tests=True)
 379         self.assertTrue(port.test_isdir('passes'))
 380         self.assertFalse(port.test_isdir('passes/text.html'))
 381         self.assertFalse(port.test_isdir('passes/does_not_exist.html'))
 382         self.assertFalse(port.test_isdir('passes/does_not_exist/'))
 383
 384         self.assertTrue(port.test_isdir('virtual'))
 385         self.assertFalse(port.test_isdir('virtual/does_not_exist.html'))
 386         self.assertFalse(port.test_isdir('virtual/does_not_exist/'))
 387         self.assertFalse(port.test_isdir('virtual/virtual_passes/passes/text.html'))
 388
 389     def test_tests(self):
 390         port = self.make_port(with_tests=True)
 391         tests = port.tests([])
 392         self.assertIn('passes/text.html', tests)
 393         self.assertIn('virtual/virtual_passes/passes/text.html', tests)
 394
 395         tests = port.tests(['passes'])
 396         self.assertIn('passes/text.html', tests)
 397         self.assertIn('passes/virtual_passes/test-virtual-passes.html', tests)
 398         self.assertNotIn('virtual/virtual_passes/passes/text.html', tests)
 399
 400         tests = port.tests(['virtual/virtual_passes/passes'])
 401         self.assertNotIn('passes/text.html', tests)
 402         self.assertIn('virtual/virtual_passes/passes/test-virtual-passes.html', tests)
 403         self.assertNotIn('passes/test-virtual-passes.html', tests)
 404         self.assertNotIn('virtual/virtual_passes/passes/test-virtual-virtual/passes.html', tests)
 405         self.assertNotIn('virtual/virtual_passes/passes/virtual_passes/passes/test-virtual-passes.html', tests)
 406
 407     def test_build_path(self):
 408         port = self.make_port(options=optparse.Values({'build_directory': '/my-build-directory/'}))
 409         self.assertEqual(port._build_path(), '/my-build-directory/Release')
 410
 411     def test_dont_require_http_server(self):
 412         port = self.make_port()
 413         self.assertEqual(port.requires_http_server(), False)
 414
 415     def test_can_load_actual_virtual_test_suite_file(self):
 416         port = Port(SystemHost(), 'baseport')
 417
 418         # If this call returns successfully, we found and loaded the LayoutTests/VirtualTestSuites.
 419         _ = port.virtual_test_suites()
 420
 421     def test_good_virtual_test_suite_file(self):
 422         port = self.make_port()
 423         fs = port._filesystem
 424         fs.write_text_file(fs.join(port.layout_tests_dir(), 'VirtualTestSuites'),
 425                            '[{"prefix": "bar", "base": "fast/bar", "args": ["--bar"]}]')
 426
 427         # If this call returns successfully, we found and loaded the LayoutTests/VirtualTestSuites.
 428         _ = port.virtual_test_suites()
 429
 430     def test_virtual_test_suite_file_is_not_json(self):
 431         port = self.make_port()
 432         fs = port._filesystem
 433         fs.write_text_file(fs.join(port.layout_tests_dir(), 'VirtualTestSuites'),
 434                            '{[{[')
 435         self.assertRaises(ValueError, port.virtual_test_suites)
 436
 437     def test_missing_virtual_test_suite_file(self):
 438         port = self.make_port()
 439         self.assertRaises(AssertionError, port.virtual_test_suites)
 440
 441
 442 class NaturalCompareTest(unittest.TestCase):
 443     def setUp(self):
 444         self._port = TestPort(MockSystemHost())
 445
 446     def assert_cmp(self, x, y, result):
 447         self.assertEqual(cmp(self._port._natural_sort_key(x), self._port._natural_sort_key(y)), result)
 448
 449     def test_natural_compare(self):
 450         self.assert_cmp('a', 'a', 0)
 451         self.assert_cmp('ab', 'a', 1)
 452         self.assert_cmp('a', 'ab', -1)
 453         self.assert_cmp('', '', 0)
 454         self.assert_cmp('', 'ab', -1)
 455         self.assert_cmp('1', '2', -1)
 456         self.assert_cmp('2', '1', 1)
 457         self.assert_cmp('1', '10', -1)
 458         self.assert_cmp('2', '10', -1)
 459         self.assert_cmp('foo_1.html', 'foo_2.html', -1)
 460         self.assert_cmp('foo_1.1.html', 'foo_2.html', -1)
 461         self.assert_cmp('foo_1.html', 'foo_10.html', -1)
 462         self.assert_cmp('foo_2.html', 'foo_10.html', -1)
 463         self.assert_cmp('foo_23.html', 'foo_10.html', 1)
 464         self.assert_cmp('foo_23.html', 'foo_100.html', -1)
 465
 466
 467 class KeyCompareTest(unittest.TestCase):
 468     def setUp(self):
 469         self._port = TestPort(MockSystemHost())
 470
 471     def assert_cmp(self, x, y, result):
 472         self.assertEqual(cmp(self._port.test_key(x), self._port.test_key(y)), result)
 473
 474     def test_test_key(self):
 475         self.assert_cmp('/a', '/a', 0)
 476         self.assert_cmp('/a', '/b', -1)
 477         self.assert_cmp('/a2', '/a10', -1)
 478         self.assert_cmp('/a2/foo', '/a10/foo', -1)
 479         self.assert_cmp('/a/foo11', '/a/foo2', 1)
 480         self.assert_cmp('/ab', '/a/a/b', -1)
 481         self.assert_cmp('/a/a/b', '/ab', 1)
 482         self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
 483
 484
 485 class VirtualTestSuiteTest(unittest.TestCase):
 486     def test_basic(self):
 487         suite = VirtualTestSuite(prefix='suite', base='base/foo', args=['--args'])
 488         self.assertEqual(suite.name, 'virtual/suite/base/foo')
 489         self.assertEqual(suite.base, 'base/foo')
 490         self.assertEqual(suite.args, ['--args'])
 491
 492     def test_no_slash(self):
 493         self.assertRaises(AssertionError, VirtualTestSuite, prefix='suite/bar', base='base/foo', args=['--args'])