nrwt: make --skip-pixel-test-if-no-baseline option

https://bugs.webkit.org/show_bug.cgi?id=70484

Patch by Fehér Zsolt <feherzs@inf.u-szeged.hu> on 2012-02-08
Reviewed by Dirk Pranke.

* Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
(SingleTestRunner.__init__):
(SingleTestRunner._should_fetch_expected_checksum):
* Scripts/webkitpy/layout_tests/controllers/worker.py:
(Worker.handle_test_list):
* Scripts/webkitpy/layout_tests/models/test_input.py:
(TestInput.__init__):
* Scripts/webkitpy/layout_tests/port/webkit.py:
(WebKitDriver.cmd_line):
* Scripts/webkitpy/layout_tests/run_webkit_tests.py:
(_set_up_derived_options):
(parse_args):
* Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
(MainTest.test_skip_pixel_test_if_no_baseline_option):
* WebKitTestRunner/TestController.cpp:
(WTR::TestController::TestController):
(WTR::TestController::initialize):
(WTR::TestController::runTest):
* WebKitTestRunner/TestController.h:
(TestController):
* WebKitTestRunner/TestInvocation.cpp:
(WTR::TestInvocation::TestInvocation):
(WTR::TestInvocation::setIsPixelTest):
* WebKitTestRunner/TestInvocation.h:
(WTR::TestInvocation::setSkipPixelTestOption):
(TestInvocation):

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@107113 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index 7453672..918e489 100644 (file)
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,37 @@
+2012-02-08  Fehér Zsolt  <feherzs@inf.u-szeged.hu>
+
+        nrwt: make --skip-pixel-test-if-no-baseline option
+        https://bugs.webkit.org/show_bug.cgi?id=70484
+
+        Reviewed by Dirk Pranke.
+
+        * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+        (SingleTestRunner.__init__):
+        (SingleTestRunner._should_fetch_expected_checksum):
+        * Scripts/webkitpy/layout_tests/controllers/worker.py:
+        (Worker.handle_test_list):
+        * Scripts/webkitpy/layout_tests/models/test_input.py:
+        (TestInput.__init__):
+        * Scripts/webkitpy/layout_tests/port/webkit.py:
+        (WebKitDriver.cmd_line):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        (_set_up_derived_options):
+        (parse_args):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+        (MainTest.test_skip_pixel_test_if_no_baseline_option):
+        * WebKitTestRunner/TestController.cpp:
+        (WTR::TestController::TestController):
+        (WTR::TestController::initialize):
+        (WTR::TestController::runTest):
+        * WebKitTestRunner/TestController.h:
+        (TestController):
+        * WebKitTestRunner/TestInvocation.cpp:
+        (WTR::TestInvocation::TestInvocation):
+        (WTR::TestInvocation::setIsPixelTest):
+        * WebKitTestRunner/TestInvocation.h:
+        (WTR::TestInvocation::setSkipPixelTestOption):
+        (TestInvocation):
+
 2012-02-08  Rob Buis  <rbuis@rim.com>
 
         [BlackBerry] Upstream DumpRenderTreeBlackBerry

diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py b/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
index 791fb62..987a779 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
@@ -55,6 +55,7 @@ class SingleTestRunner:
         self._timeout = test_input.timeout
         self._worker_name = worker_name
         self._test_name = test_input.test_name
+        self._should_run_pixel_test = test_input.should_run_pixel_test
 
         self._is_reftest = False
         self._reference_files = port.reference_files(self._test_name)
@@ -77,6 +78,8 @@ class SingleTestRunner:
                                  self._port.expected_audio(self._test_name))
 
     def _should_fetch_expected_checksum(self):
+        if not self._should_run_pixel_test:
+            return False
         return (self._options.pixel_tests and
                 not (self._options.new_baseline or self._options.reset_results))
 

diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py b/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py
index b66ce9b..fc7831d 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py
@@ -108,6 +108,9 @@ class Worker(manager_worker_broker.AbstractWorker):
         start_time = time.time()
         num_tests = 0
         for test_input in test_list:
+            #FIXME: When the DRT support also this function, that would be useful
+            if self._port.driver_name() == "WebKitTestRunner" and self._port.get_option('skip_pixel_test_if_no_baseline') and self._port.get_option('pixel_tests'):
+                test_input.should_run_pixel_test = (self._port.expected_image(test_input.test_name) != None)
             self._run_test(test_input)
             num_tests += 1
             self._worker_connection.yield_to_broker()

diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_input.py b/Tools/Scripts/webkitpy/layout_tests/models/test_input.py
index 6b5d148..5c8b30d 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/models/test_input.py
+++ b/Tools/Scripts/webkitpy/layout_tests/models/test_input.py
@@ -36,7 +36,7 @@ class TestInput:
     ref_file = None
     is_mismatch_reftest = None
 
-    def __init__(self, test_name, timeout):
+    def __init__(self, test_name, timeout, should_run_pixel_test=True):
         """Holds the input parameters for a test.
         Args:
           test: name of test (not an absolute path!)
@@ -46,6 +46,7 @@ class TestInput:
           """
         self.test_name = test_name
         self.timeout = timeout
+        self.should_run_pixel_test = should_run_pixel_test
 
     def __repr__(self):
         return "TestInput('%s', %d)" % (self.test_name, self.timeout)

diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py
index 1c78053..967e1f1 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py
@@ -467,6 +467,8 @@ class WebKitDriver(Driver):
     def cmd_line(self):
         cmd = self._command_wrapper(self._port.get_option('wrapper'))
         cmd.append(self._port._path_to_driver())
+        if self._port.get_option('skip_pixel_test_if_no_baseline'):
+            cmd.append('--skip-pixel-test-if-no-baseline')
         if self._pixel_tests:
             cmd.append('--pixel-tests')
         if self._port.get_option('gc_between_tests'):

diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
index ac09fc5..96508ca 100755 (executable)
--- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
@@ -175,6 +175,9 @@ def _set_up_derived_options(port, options):
         warnings.append("--no-http is ignored since --force is also provided")
         options.http = True
 
+    if options.skip_pixel_test_if_no_baseline and not options.pixel_tests:
+        warnings.append("--skip-pixel-test-if-no-baseline is only supported with -p (--pixel-tests)")
+
     if options.ignore_metrics and (options.new_baseline or options.reset_results):
         warnings.append("--ignore-metrics has no effect with --new-baselines or with --reset-results")
 
@@ -314,6 +317,9 @@ def parse_args(args=None):
         optparse.make_option("--no-new-test-results", action="store_false",
             dest="new_test_results", default=True,
             help="Don't create new baselines when no expected results exist"),
+        optparse.make_option("--skip-pixel-test-if-no-baseline", action="store_true",
+            dest="skip_pixel_test_if_no_baseline", help="Do not generate and check pixel result in the case when "
+                 "no image baseline is available for the test."),
         optparse.make_option("--skip-failing-tests", action="store_true",
             default=False, help="Skip tests that are expected to fail. "
                  "Note: When using this option, you might miss new crashes "

diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py
index 77f7359..e872d58 100755 (executable)
--- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py
@@ -385,6 +385,11 @@ class MainTest(unittest.TestCase):
         tests_run = get_tests_run(['--repeat-each', '2'] + tests_to_run, tests_included=True, flatten_batches=True)
         self.assertEquals(tests_run, ['passes/image.html', 'passes/image.html', 'passes/text.html', 'passes/text.html'])
 
+    def test_skip_pixel_test_if_no_baseline_option(self):
+        tests_to_run = ['passes/image.html', 'passes/text.html']
+        tests_run = get_tests_run(['--skip-pixel-test-if-no-baseline'] + tests_to_run, tests_included=True, flatten_batches=True)
+        self.assertEquals(tests_run, ['passes/image.html', 'passes/text.html'])
+
     def test_iterations(self):
         tests_to_run = ['passes/image.html', 'passes/text.html']
         tests_run = get_tests_run(['--iterations', '2'] + tests_to_run, tests_included=True, flatten_batches=True)

diff --git a/Tools/WebKitTestRunner/TestController.cpp b/Tools/WebKitTestRunner/TestController.cpp
index 0e5db19..db6e914 100644 (file)
--- a/Tools/WebKitTestRunner/TestController.cpp
+++ b/Tools/WebKitTestRunner/TestController.cpp
@@ -63,6 +63,7 @@ TestController& TestController::shared()
 
 TestController::TestController(int argc, const char* argv[])
     : m_dumpPixels(false)
+    , m_skipPixelTestOption(false)
     , m_verbose(false)
     , m_printSeparators(false)
     , m_usingServerMode(false)
@@ -246,6 +247,12 @@ void TestController::initialize(int argc, const char* argv[])
             m_shortTimeout = defaultShortTimeout * m_longTimeout / defaultLongTimeout;
             continue;
         }
+
+        if (argument == "--skip-pixel-test-if-no-baseline") {
+            m_skipPixelTestOption = true;
+            continue;
+        }
+
         if (argument == "--pixel-tests") {
             m_dumpPixels = true;
             continue;
@@ -489,6 +496,7 @@ bool TestController::runTest(const char* test)
     m_state = RunningTest;
 
     m_currentInvocation = adoptPtr(new TestInvocation(pathOrURL));
+    m_currentInvocation->setSkipPixelTestOption(m_skipPixelTestOption);
     if (m_dumpPixels)
         m_currentInvocation->setIsPixelTest(expectedPixelHash);    
 

diff --git a/Tools/WebKitTestRunner/TestController.h b/Tools/WebKitTestRunner/TestController.h
index d0d2151..32b4ade 100644 (file)
--- a/Tools/WebKitTestRunner/TestController.h
+++ b/Tools/WebKitTestRunner/TestController.h
@@ -104,6 +104,7 @@ private:
     OwnPtr<TestInvocation> m_currentInvocation;
 
     bool m_dumpPixels;
+    bool m_skipPixelTestOption;
     bool m_verbose;
     bool m_printSeparators;
     bool m_usingServerMode;

diff --git a/Tools/WebKitTestRunner/TestInvocation.cpp b/Tools/WebKitTestRunner/TestInvocation.cpp
index 8e3c229..42a8851 100644 (file)
--- a/Tools/WebKitTestRunner/TestInvocation.cpp
+++ b/Tools/WebKitTestRunner/TestInvocation.cpp
@@ -93,6 +93,7 @@ TestInvocation::TestInvocation(const std::string& pathOrURL)
     : m_url(AdoptWK, createWKURL(pathOrURL.c_str()))
     , m_pathOrURL(pathOrURL)
     , m_dumpPixels(false)
+    , m_skipPixelTestOption(false)
     , m_gotInitialResponse(false)
     , m_gotFinalMessage(false)
     , m_gotRepaint(false)
@@ -106,6 +107,8 @@ TestInvocation::~TestInvocation()
 
 void TestInvocation::setIsPixelTest(const std::string& expectedPixelHash)
 {
+    if (m_skipPixelTestOption && !expectedPixelHash.length())
+        return;
     m_dumpPixels = true;
     m_expectedPixelHash = expectedPixelHash;
 }

diff --git a/Tools/WebKitTestRunner/TestInvocation.h b/Tools/WebKitTestRunner/TestInvocation.h
index 5ff7ea0..af9f38f 100644 (file)
--- a/Tools/WebKitTestRunner/TestInvocation.h
+++ b/Tools/WebKitTestRunner/TestInvocation.h
@@ -39,7 +39,8 @@ public:
     ~TestInvocation();
 
     void setIsPixelTest(const std::string& expectedPixelHash);
-    
+    void setSkipPixelTestOption(bool option) { m_skipPixelTestOption = option; }
+
     void invoke();
     void didReceiveMessageFromInjectedBundle(WKStringRef messageName, WKTypeRef messageBody);
     WKRetainPtr<WKTypeRef> didReceiveSynchronousMessageFromInjectedBundle(WKStringRef messageName, WKTypeRef messageBody);
@@ -54,6 +55,7 @@ private:
     
     bool m_dumpPixels;
     std::string m_expectedPixelHash;
+    bool m_skipPixelTestOption;
 
     // Invocation state
     bool m_gotInitialResponse;