From d38307a0736184d8381cd6599c2b47840f03f3b0 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sat, 29 Nov 2014 00:03:04 +0100 Subject: [PATCH] validate: Add a cleaner API to setup tests in testsuite files With the testsuite format you will get a setup_tests(tests_manager, options) function called for each TestManager. The function will have the exact same role as with old config file but with a clean API and not magic global variables. This implies that we need default blacklist to be directly set on the TestManager and not on options.blacklisted_test --- validate/launcher/apps/gstvalidate.py | 10 ++-- validate/launcher/baseclasses.py | 94 +++++++++++++++++++++++++++++++---- validate/launcher/main.py | 65 ++++++++++++++---------- 3 files changed, 129 insertions(+), 40 deletions(-) diff --git a/validate/launcher/apps/gstvalidate.py b/validate/launcher/apps/gstvalidate.py index 899ce91..f79cdf6 100644 --- a/validate/launcher/apps/gstvalidate.py +++ b/validate/launcher/apps/gstvalidate.py @@ -458,6 +458,7 @@ class GstValidateTestManager(GstValidateBaseTestManager): self._uris = [] self._run_defaults = True self._is_populated = False + self._default_generators_registered = False def init(self): if which(GST_VALIDATE_COMMAND) and which(GST_VALIDATE_TRANSCODING_COMMAND): @@ -475,7 +476,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") if self._is_populated is True: return - if not self.options.config: + if not self.options.config and not self.options.testsuites: if self._run_defaults: self.register_defaults() else: @@ -687,9 +688,6 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") "https://bugzilla.gnome.org/show_bug.cgi?id=606382"), ("validate.hls.*seek_forward.*", "https://bugzilla.gnome.org/show_bug.cgi?id=606382"), - ("validate.hls.*", - "FIXME! The HLS tests are not stable enough " - "(at least on the server), try again later."), # Matroska/WEBM known issues: ("validate.*.reverse_playback.*webm$", @@ -747,6 +745,10 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") """ Registers default test generators """ + if self._default_generators_registered: + return + self.add_generators([GstValidatePlaybinTestsGenerator(self), GstValidateMediaCheckTestsGenerator(self), GstValidateTranscodingTestsGenerator(self)]) + self._default_generators_registered = True diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index 8df10c9..c078e04 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -21,6 +21,7 @@ import os import sys +import imp import re import time import utils @@ -658,13 +659,23 @@ class TestsManager(Loggable): else: self._generators.append(generators) + self._generators = list(set(self._generators)) + def get_generators(self): return self._generators + def _add_blacklist(self, blacklisted_tests): + if not isinstance(blacklisted_tests, list): + blacklisted_tests = [blacklisted_tests] + + for patterns in blacklisted_tests: + for pattern in patterns.split(","): + self.blacklisted_tests_patterns.append(re.compile(pattern)) + def set_default_blacklist(self, default_blacklist): msg = "\nCurrently 'hardcoded' %s blacklisted tests:\n\n" % self.name for name, bug in default_blacklist: - self.options.blacklisted_tests.append(name) + self._add_blacklist(name) msg += " + %s \n --> bug: %s\n" % (name, bug) printc(msg, Colors.FAIL, True) @@ -687,8 +698,7 @@ class TestsManager(Loggable): if options.blacklisted_tests: for patterns in options.blacklisted_tests: - for pattern in patterns.split(","): - self.blacklisted_tests_patterns.append(re.compile(pattern)) + self._add_blacklist(patterns) def _check_blacklisted(self, test): for pattern in self.blacklisted_tests_patterns: @@ -824,6 +834,65 @@ class _TestsLauncher(Loggable): for tester in self.testers: tester.add_options(parser) + def _load_testsuites(self): + testsuites = [] + for testsuite in self.options.testsuites: + if not os.path.isabs(testsuite): + testsuite = os.path.join(self.options.testsuites_dir, testsuite + ".py") + + try: + sys.path.insert(0, os.path.dirname(testsuite)) + module = __import__(os.path.basename(testsuite).replace(".py", "")) + except Exception as e: + printc("Could not load testsuite: %s, reason: %s" + % (testsuite, e), Colors.FAIL) + continue + finally: + sys.path.remove(os.path.dirname(testsuite)) + + testsuites.append(module) + if not hasattr(module, "TEST_MANAGER"): + module.TEST_MANAGER = [tester.name for tester in self.testers] + elif not isinstance(module.TEST_MANAGER, list): + module.TEST_MANAGER = [module.TEST_MANAGER] + + self.options.testsuites = testsuites + + def _setup_testsuites(self): + for testsuite in self.options.testsuites: + loaded = False + wanted_test_manager = None + if hasattr(testsuite, "TEST_MANAGER"): + wanted_test_manager = testsuite.TEST_MANAGER + if not isinstance(wanted_test_manager, list): + wanted_test_manager = [wanted_test_manager] + + for tester in self.testers: + if wanted_test_manager is not None and \ + tester.name not in wanted_test_manager: + continue + + if testsuite.setup_tests(tester, self.options): + loaded = True + + if not loaded: + printc("Could not load testsuite: %s" + " maybe because of missing TestManager" + % (testsuite), Colors.FAIL) + + def _load_config(self): + printc("Loading config files is DEPRECATED" + " you should use the new testsuite format now",) + + for tester in self.testers: + tester.options = options + globals()[tester.name] = tester + globals()["options"] = options + c__file__ = __file__ + globals()["__file__"] = self.options.config + execfile(self.options.config, globals()) + globals()["__file__"] = c__file__ + def set_settings(self, options, args): self.reporter = reporters.XunitReporter(options) if not options.logsdir in[sys.stderr, sys.stdout]: @@ -834,6 +903,7 @@ class _TestsLauncher(Loggable): for tester in self.testers: if tester.name in args: wanted_testers = tester.name + if wanted_testers: testers = self.testers self.testers = [] @@ -843,18 +913,16 @@ class _TestsLauncher(Loggable): args.remove(tester.name) if options.config: - for tester in self.testers: - tester.options = options - globals()[tester.name] = tester - globals()["options"] = options - c__file__ = __file__ - globals()["__file__"] = self.options.config - execfile(self.options.config, globals()) - globals()["__file__"] = c__file__ + self._load_config() + + self._load_testsuites() for tester in self.testers: tester.set_settings(options, args, self.reporter) + if not options.config and options.testsuites: + self._setup_testsuites() + def list_tests(self): for tester in self.testers: self.tests.extend(tester.list_tests()) @@ -1069,6 +1137,8 @@ class GstValidateBaseTestManager(TestsManager): else: self._scenarios.append(scenarios) + self._scenarios = list(set(self._scenarios)) + def get_scenarios(self): return self._scenarios @@ -1085,6 +1155,8 @@ class GstValidateBaseTestManager(TestsManager): else: self._encoding_formats.append(encoding_formats) + self._encoding_formats = list(set(self._encoding_formats)) + def get_encoding_formats(self): return self._encoding_formats diff --git a/validate/launcher/main.py b/validate/launcher/main.py index e9585a7..90285a7 100644 --- a/validate/launcher/main.py +++ b/validate/launcher/main.py @@ -133,10 +133,7 @@ MEDIAS_FOLDER = "medias" DEFAULT_GST_QA_ASSETS_REPO = "git://people.freedesktop.org/~tsaunier/gst-qa-assets/" DEFAULT_SYNC_ASSET_COMMAND = "git fetch origin && git checkout origin/master && git annex get medias/default/" DEFAULT_SYNC_ALL_ASSET_COMMAND = "git fetch origin && git checkout origin/master && git annex get ." -DEFAULT_VALIDATE_TESTSUITE = os.path.join(DEFAULT_MAIN_DIR, - QA_ASSETS, - "testsuites", - "gst-validate-default.testsuite") +DEFAULT_TESTSUITES_DIR = os.path.join(DEFAULT_MAIN_DIR, QA_ASSETS, "testsuites") def update_assets(options): @@ -194,6 +191,37 @@ def main(libsdir): parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, prog='gst-validate-launcher', description=HELP) + parser.add_argument('testsuites', metavar='N', nargs='*', + help="""Lets you specify a file where the testsuite to execute is defined. + +In the module if you want to work with a specific test manager(s) (for example, +'ges' or 'validate'), you should define the TEST_MANAGER variable in the +testsuite file (it can be a list of test manager names) + +In this file you should implement a setup_tests function. That function takes +a TestManager and the GstValidateLauncher option as parameters and return True +if it succeeded loading the tests, False otherwise. +You will be able to configure the TestManager with its various methods. This +function will be called with each TestManager usable, for example you will be +passed the 'validate' TestManager in case the GstValidateManager launcher is +avalaible. You should configure it using: + + * test_manager.add_scenarios: which allows you to register a list of scenario names to be run + * test_manager.set_default_blacklist: Lets you set a list of tuple of the form: + (@regex_defining_blacklister_test_names, @reason_for_the_blacklisting) + * test_manager.add_generators: which allows you to register a list of #GstValidateTestsGenerator + to be used to generate tests + * test_manager.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests + +You can also set default values with: + * test_manager.register_defaults: Sets default values for all parametters + * test_manager.register_default_test_generators: Sets default values for the TestsGenerators to be used + * test_manager.register_default_scenarios: Sets default values for the scenarios to be executed + * test_manager.register_default_encoding_formats: Sets default values for the encoding formats to be tested + +Note that all testsuite should be inside python modules, so the directory should contain a __init__.py file +""", + default=["validate", "ges"]) parser.add_argument("-d", "--debug", dest="debug", action="store_true", default=False, @@ -243,27 +271,9 @@ def main(libsdir): " note that 0 will enable all tests", type=int), parser.add_argument("-c", "--config", dest="config", - default=DEFAULT_VALIDATE_TESTSUITE, - help="""Lets you specify a file where the testsuite to execute is defined. -In this file you will have acces to the TestManager objects that you can configure with -its various methods. For example you can find the 'validate' variable, in case the GstValidateManager -launcher is avalaible. You should configure it using: - * validate.add_scenarios: which allows you to register a list of scenario names to be run - * validate.set_default_blacklist: Lets you set a list of tuple of the form: - (@regex_defining_blacklister_test_names, @reason_for_the_blacklisting) - * validate.add_generators: which allows you to register a list of #GstValidateTestsGenerator - to be used to generate tests - * validate.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests - -You can also set default values with: - * validate.register_defaults: Sets default values for all parameters - * validate.register_default_test_generators: Sets default values for the TestsGenerators to be used - * gst_validate_register_default_scenarios: Sets default values for the scenarios to be executed - * gst_validate_register_default_encoding_formats: Sets default values for the encoding formats to be tested - -Note: In the config file, you have access to the options variable resulting from the parsing of the command line -user argument, you can thus override command line options using that. -""") + default=None, + help="This is DEPRECATED, prefer using the testsuite format" + " to configure testsuites") dir_group = parser.add_argument_group( "Directories and files to be used by the launcher") parser.add_argument('--xunit-file', action='store', @@ -274,6 +284,11 @@ user argument, you can thus override command line options using that. dir_group.add_argument("-M", "--main-dir", dest="main_dir", default=DEFAULT_MAIN_DIR, help="Main directory where to put files. Default is %s" % DEFAULT_MAIN_DIR) + dir_group.add_argument("--testsuites-dir", dest="testsuites_dir", + default=DEFAULT_TESTSUITES_DIR, + help="Directory where to look for testsuites. Default is %s" + " Note that GstValidate expect testsuite file to have .testsuite" + " as an extension in this folder." % DEFAULT_TESTSUITES_DIR) dir_group.add_argument("-o", "--output-dir", dest="output_dir", default=None, help="Directory where to store logs and rendered files. Default is MAIN_DIR") -- 2.7.4