validate:tools: Blacklist some scenario/protocol combinations

And add the option for user to easilly blacklist tests

diff --git a/validate/tools/launcher/apps/ges-launch.py b/validate/tools/launcher/apps/ges-launch.py
index 7c0ffb6..ca3b8da 100644 (file)
--- a/validate/tools/launcher/apps/ges-launch.py
+++ b/validate/tools/launcher/apps/ges-launch.py
@@ -212,18 +212,18 @@ class GESTestsManager(TestsManager):
             # First playback casses
             for scenario in SCENARIOS:
                 classname = "ges.playback.%s.%s" % (scenario, os.path.basename(proj).replace(".xges", ""))
-                self.tests.append(GESPlaybackTest(classname,
-                                                  self.options,
-                                                  self.reporter,
-                                                  proj,
-                                                  scenario=scenario)
+                self.add_test(GESPlaybackTest(classname,
+                                              self.options,
+                                              self.reporter,
+                                              proj,
+                                              scenario=scenario)
                                   )
 
             # And now rendering casses
             for comb in COMBINATIONS:
                 classname = "ges.render.%s.%s" % (str(comb).replace(' ', '_'),
                                                   os.path.splitext(os.path.basename(proj))[0])
-                self.tests.append(GESRenderTest(classname, self.options,
-                                                self.reporter, proj,
-                                                combination=comb)
+                self.add_test(GESRenderTest(classname, self.options,
+                                            self.reporter, proj,
+                                            combination=comb)
                                   )

diff --git a/validate/tools/launcher/apps/gst-validate.py b/validate/tools/launcher/apps/gst-validate.py
index bd518ec..c63438d 100644 (file)
--- a/validate/tools/launcher/apps/gst-validate.py
+++ b/validate/tools/launcher/apps/gst-validate.py
@@ -149,11 +149,11 @@ class GstValidateManager(TestsManager, Loggable):
 
         for uri, mediainfo in self._list_uris():
             classname = "validate.media_check.%s" % (os.path.splitext(os.path.basename(uri))[0].replace(".", "_"))
-            self.tests.append(GstValidateMediaCheckTest(classname,
-                                                        self.options,
-                                                        self.reporter,
-                                                        mediainfo.path,
-                                                        uri))
+            self.add_test(GstValidateMediaCheckTest(classname,
+                                                    self.options,
+                                                    self.reporter,
+                                                    mediainfo.path,
+                                                    uri))
 
         for uri, mediainfo in self._list_uris():
             if mediainfo.config.getboolean("media-info", "is-image") is True:
@@ -162,11 +162,11 @@ class GstValidateManager(TestsManager, Loggable):
                 classname = "validate.%s.transcode.to_%s.%s" % (mediainfo.config.get("file-info", "protocol"),
                                                                 str(comb).replace(' ', '_'),
                                                                 os.path.splitext(os.path.basename(uri))[0].replace(".", "_"))
-                self.tests.append(GstValidateTranscodingTest(classname,
-                                                             self.options,
-                                                             self.reporter,
-                                                             comb, uri,
-                                                             mediainfo.config))
+                self.add_test(GstValidateTranscodingTest(classname,
+                                                         self.options,
+                                                         self.reporter,
+                                                         comb, uri,
+                                                         mediainfo.config))
 
     def _check_discovering_info(self, media_info, uri=None):
         self.debug("Checking %s", media_info)
@@ -254,6 +254,8 @@ class GstValidateManager(TestsManager, Loggable):
         if "__uri__" in pipe:
             for uri, minfo in self._list_uris():
                 npipe = pipe
+                protocol = minfo.config.get("file-info", "protocol")
+
                 if scenario != "none":
                     if minfo.config.getboolean("media-info", "seekable") is False:
                         self.debug("Do not run %s as %s does not support seeking",
@@ -266,23 +268,23 @@ class GstValidateManager(TestsManager, Loggable):
                         npipe = pipe.replace("fakesink", "'fakesink sync=true'")
 
                 fname = "%s.%s" % (self._get_fname(scenario,
-                                   minfo.config.get("file-info", "protocol")),
+                                   protocol),
                                    os.path.basename(uri).replace(".", "_"))
                 self.debug("Adding: %s", fname)
 
-                self.tests.append(GstValidateLaunchTest(fname,
-                                                  self.options,
-                                                  self.reporter,
-                                                  npipe.replace("__uri__", uri),
-                                                  scenario=scenario,
-                                                  file_infos=minfo.config)
+                self.add_test(GstValidateLaunchTest(fname,
+                                                    self.options,
+                                                    self.reporter,
+                                                    npipe.replace("__uri__", uri),
+                                                    scenario=scenario,
+                                                    file_infos=minfo.config)
                                  )
         else:
-            self.tests.append(GstValidateLaunchTest(self._get_fname(scenario, "testing"),
-                                              self.options,
-                                              self.reporter,
-                                              pipe,
-                                              scenario=scenario))
+            self.add_test(GstValidateLaunchTest(self._get_fname(scenario, "testing"),
+                                                self.options,
+                                                self.reporter,
+                                                pipe,
+                                                scenario=scenario))
 
     def needs_http_server(self):
         for uri, mediainfo in self._list_uris():

diff --git a/validate/tools/launcher/baseclasses.py b/validate/tools/launcher/baseclasses.py
index 01d3070..efa6eca 100644 (file)
--- a/validate/tools/launcher/baseclasses.py
+++ b/validate/tools/launcher/baseclasses.py
@@ -47,7 +47,7 @@ class Test(Loggable):
         self.process = None
 
         self.message = ""
-        self.error = ""
+        self.error_str = ""
         self.time_taken = 0.0
         self._starting_time = None
         self.result = Result.NOT_RUN
@@ -83,7 +83,7 @@ class Test(Loggable):
     def set_result(self, result, message="", error=""):
         self.result = result
         self.message = message
-        self.error = error
+        self.error_str = error
 
     def check_results(self):
         if self.result is Result.FAILED:
@@ -197,14 +197,14 @@ class GstValidateTest(Test):
         super(GstValidateTest, self).__init__(application_name, classname, options,
                                               reporter, timeout=DEFAULT_TIMEOUT)
 
-        if scenario is None or scenario.lower() == "none":
+        if scenario is None or scenario.name.lower() == "none":
             self.scenario = None
         else:
             self.scenario = scenario
 
     def build_arguments(self):
         if self.scenario is not None:
-            self.add_arguments("--set-scenario", self.scenario)
+            self.add_arguments("--set-scenario", self.scenario.name)
 
     def get_validate_criticals_errors(self):
         self.reporter.out.seek(0)
@@ -258,11 +258,13 @@ class TestsManager(Loggable):
 
         Loggable.__init__(self)
 
-        self.tests = []
+        self.tests = set([])
+        self.unwanted_tests = set([])
         self.options = None
         self.args = None
         self.reporter = None
         self.wanted_tests_patterns = []
+        self.blacklisted_tests_patterns = []
 
     def init(self):
         return False
@@ -270,6 +272,12 @@ class TestsManager(Loggable):
     def list_tests(self):
         pass
 
+    def add_test(self, test):
+        if self._is_test_wanted(test):
+            self.tests.add(test)
+        else:
+            self.unwanted_tests.add(test)
+
     def get_tests(self):
         return self.tests
 
@@ -284,11 +292,26 @@ class TestsManager(Loggable):
         self.reporter = reporter
 
         if options.wanted_tests:
-            for pattern in options.wanted_tests.split(','):
-                self.wanted_tests_patterns.append(re.compile(pattern))
+            for patterns in options.wanted_tests:
+                for pattern in patterns.split(","):
+                    self.wanted_tests_patterns.append(re.compile(pattern))
+
+        if options.blacklisted_tests:
+            for patterns in options.blacklisted_tests:
+                for pattern in patterns.split(","):
+                    self.blacklisted_tests_patterns.append(re.compile(pattern))
+
+    def _check_blacklisted(self, test):
+        for pattern in self.blacklisted_tests_patterns:
+            if pattern.findall(test.classname):
+                return True
 
+        return False
 
     def _is_test_wanted(self, test):
+        if self._check_blacklisted(test):
+            return False
+
         if not self.wanted_tests_patterns:
             return True
 
@@ -342,10 +365,12 @@ class _TestsLauncher(Loggable):
         for f in os.listdir(os.path.join(d, "apps")):
             if f.endswith(".py"):
                 execfile(os.path.join(d, "apps", f), env)
+                print f
 
         testers = [i() for i in get_subclasses(TestsManager, env)]
         for tester in testers:
             if tester.init() is True:
+                print tester
                 self.testers.append(tester)
             else:
                 self.warning("Can not init tester: %s -- PATH is %s"

diff --git a/validate/tools/launcher/main.py b/validate/tools/launcher/main.py
index e1265a9..bf878c3 100644 (file)
--- a/validate/tools/launcher/main.py
+++ b/validate/tools/launcher/main.py
@@ -17,6 +17,7 @@
 # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 # Boston, MA 02110-1301, USA.
 import os
+import sys
 import utils
 import urlparse
 import loggable
@@ -26,7 +27,13 @@ from httpserver import HTTPServer
 from baseclasses import _TestsLauncher
 from utils import printc, path2url, DEFAULT_GST_QA_ASSETS, launch_command
 
+
 DEFAULT_GST_QA_ASSETS_REPO = "git://people.freedesktop.org/~tsaunier/gst-qa-assets/"
+BLACKLISTED_TESTS = ["validate.hls.playback.simple_backward",  # bug 698155
+                     "validate.hls.playback.fast_forward",  # bug 698155
+                     "validate.*.simple_backward.*webm$",  # bug 679250
+                     ]
+
 
 def main():
     parser = OptionParser()
@@ -47,8 +54,14 @@ def main():
                       help=("Path to xml file to store the xunit report in. "
                       "Default is xunit.xml the logs-dir directory"))
     parser.add_option("-t", "--wanted-tests", dest="wanted_tests",
-                      default=None,
+                      default=[],
+                      action="append",
                       help="Define the tests to execute, it can be a regex")
+    parser.add_option("-b", "--blacklisted-tests", dest="blacklisted_tests",
+                      default=[],
+                      action="append",
+                      help="Define the tests not to execute, it can be a regex."
+                      " Currently blacklisted tests are: %s" % BLACKLISTED_TESTS)
     parser.add_option("-L", "--list-tests",
                       dest="list_tests",
                       action="store_true",
@@ -85,6 +98,10 @@ def main():
 
     tests_launcher = _TestsLauncher()
     tests_launcher.add_options(parser)
+
+    for p in BLACKLISTED_TESTS:
+        sys.argv.extend(["-b", p])
+
     (options, args) = parser.parse_args()
     if options.xunit_file is None:
         options.xunit_file = os.path.join(options.logsdir, "xunit.xml")