self._uris = []
self._run_defaults = True
self._is_populated = False
+ self._default_generators_registered = False
def init(self):
if which(GST_VALIDATE_COMMAND) and which(GST_VALIDATE_TRANSCODING_COMMAND):
if self._is_populated is True:
return
- if not self.options.config:
+ if not self.options.config and not self.options.testsuites:
if self._run_defaults:
self.register_defaults()
else:
"https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
("validate.hls.*seek_forward.*",
"https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
- ("validate.hls.*",
- "FIXME! The HLS tests are not stable enough "
- "(at least on the server), try again later."),
# Matroska/WEBM known issues:
("validate.*.reverse_playback.*webm$",
"""
Registers default test generators
"""
+ if self._default_generators_registered:
+ return
+
self.add_generators([GstValidatePlaybinTestsGenerator(self),
GstValidateMediaCheckTestsGenerator(self),
GstValidateTranscodingTestsGenerator(self)])
+ self._default_generators_registered = True
import os
import sys
+import imp
import re
import time
import utils
else:
self._generators.append(generators)
+ self._generators = list(set(self._generators))
+
def get_generators(self):
return self._generators
+ def _add_blacklist(self, blacklisted_tests):
+ if not isinstance(blacklisted_tests, list):
+ blacklisted_tests = [blacklisted_tests]
+
+ for patterns in blacklisted_tests:
+ for pattern in patterns.split(","):
+ self.blacklisted_tests_patterns.append(re.compile(pattern))
+
def set_default_blacklist(self, default_blacklist):
msg = "\nCurrently 'hardcoded' %s blacklisted tests:\n\n" % self.name
for name, bug in default_blacklist:
- self.options.blacklisted_tests.append(name)
+ self._add_blacklist(name)
msg += " + %s \n --> bug: %s\n" % (name, bug)
printc(msg, Colors.FAIL, True)
if options.blacklisted_tests:
for patterns in options.blacklisted_tests:
- for pattern in patterns.split(","):
- self.blacklisted_tests_patterns.append(re.compile(pattern))
+ self._add_blacklist(patterns)
def _check_blacklisted(self, test):
for pattern in self.blacklisted_tests_patterns:
for tester in self.testers:
tester.add_options(parser)
+ def _load_testsuites(self):
+ testsuites = []
+ for testsuite in self.options.testsuites:
+ if not os.path.isabs(testsuite):
+ testsuite = os.path.join(self.options.testsuites_dir, testsuite + ".py")
+
+ try:
+ sys.path.insert(0, os.path.dirname(testsuite))
+ module = __import__(os.path.basename(testsuite).replace(".py", ""))
+ except Exception as e:
+ printc("Could not load testsuite: %s, reason: %s"
+ % (testsuite, e), Colors.FAIL)
+ continue
+ finally:
+ sys.path.remove(os.path.dirname(testsuite))
+
+ testsuites.append(module)
+ if not hasattr(module, "TEST_MANAGER"):
+ module.TEST_MANAGER = [tester.name for tester in self.testers]
+ elif not isinstance(module.TEST_MANAGER, list):
+ module.TEST_MANAGER = [module.TEST_MANAGER]
+
+ self.options.testsuites = testsuites
+
+ def _setup_testsuites(self):
+ for testsuite in self.options.testsuites:
+ loaded = False
+ wanted_test_manager = None
+ if hasattr(testsuite, "TEST_MANAGER"):
+ wanted_test_manager = testsuite.TEST_MANAGER
+ if not isinstance(wanted_test_manager, list):
+ wanted_test_manager = [wanted_test_manager]
+
+ for tester in self.testers:
+ if wanted_test_manager is not None and \
+ tester.name not in wanted_test_manager:
+ continue
+
+ if testsuite.setup_tests(tester, self.options):
+ loaded = True
+
+ if not loaded:
+ printc("Could not load testsuite: %s"
+ " maybe because of missing TestManager"
+ % (testsuite), Colors.FAIL)
+
+ def _load_config(self):
+ printc("Loading config files is DEPRECATED"
+ " you should use the new testsuite format now",)
+
+ for tester in self.testers:
+ tester.options = options
+ globals()[tester.name] = tester
+ globals()["options"] = options
+ c__file__ = __file__
+ globals()["__file__"] = self.options.config
+ execfile(self.options.config, globals())
+ globals()["__file__"] = c__file__
+
def set_settings(self, options, args):
self.reporter = reporters.XunitReporter(options)
if not options.logsdir in[sys.stderr, sys.stdout]:
for tester in self.testers:
if tester.name in args:
wanted_testers = tester.name
+
if wanted_testers:
testers = self.testers
self.testers = []
args.remove(tester.name)
if options.config:
- for tester in self.testers:
- tester.options = options
- globals()[tester.name] = tester
- globals()["options"] = options
- c__file__ = __file__
- globals()["__file__"] = self.options.config
- execfile(self.options.config, globals())
- globals()["__file__"] = c__file__
+ self._load_config()
+
+ self._load_testsuites()
for tester in self.testers:
tester.set_settings(options, args, self.reporter)
+ if not options.config and options.testsuites:
+ self._setup_testsuites()
+
def list_tests(self):
for tester in self.testers:
self.tests.extend(tester.list_tests())
else:
self._scenarios.append(scenarios)
+ self._scenarios = list(set(self._scenarios))
+
def get_scenarios(self):
return self._scenarios
else:
self._encoding_formats.append(encoding_formats)
+ self._encoding_formats = list(set(self._encoding_formats))
+
def get_encoding_formats(self):
return self._encoding_formats
DEFAULT_GST_QA_ASSETS_REPO = "git://people.freedesktop.org/~tsaunier/gst-qa-assets/"
DEFAULT_SYNC_ASSET_COMMAND = "git fetch origin && git checkout origin/master && git annex get medias/default/"
DEFAULT_SYNC_ALL_ASSET_COMMAND = "git fetch origin && git checkout origin/master && git annex get ."
-DEFAULT_VALIDATE_TESTSUITE = os.path.join(DEFAULT_MAIN_DIR,
- QA_ASSETS,
- "testsuites",
- "gst-validate-default.testsuite")
+DEFAULT_TESTSUITES_DIR = os.path.join(DEFAULT_MAIN_DIR, QA_ASSETS, "testsuites")
def update_assets(options):
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
prog='gst-validate-launcher', description=HELP)
+ parser.add_argument('testsuites', metavar='N', nargs='*',
+ help="""Lets you specify a file where the testsuite to execute is defined.
+
+In the module if you want to work with a specific test manager(s) (for example,
+'ges' or 'validate'), you should define the TEST_MANAGER variable in the
+testsuite file (it can be a list of test manager names)
+
+In this file you should implement a setup_tests function. That function takes
+a TestManager and the GstValidateLauncher option as parameters and return True
+if it succeeded loading the tests, False otherwise.
+You will be able to configure the TestManager with its various methods. This
+function will be called with each TestManager usable, for example you will be
+passed the 'validate' TestManager in case the GstValidateManager launcher is
+avalaible. You should configure it using:
+
+ * test_manager.add_scenarios: which allows you to register a list of scenario names to be run
+ * test_manager.set_default_blacklist: Lets you set a list of tuple of the form:
+ (@regex_defining_blacklister_test_names, @reason_for_the_blacklisting)
+ * test_manager.add_generators: which allows you to register a list of #GstValidateTestsGenerator
+ to be used to generate tests
+ * test_manager.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests
+
+You can also set default values with:
+ * test_manager.register_defaults: Sets default values for all parametters
+ * test_manager.register_default_test_generators: Sets default values for the TestsGenerators to be used
+ * test_manager.register_default_scenarios: Sets default values for the scenarios to be executed
+ * test_manager.register_default_encoding_formats: Sets default values for the encoding formats to be tested
+
+Note that all testsuite should be inside python modules, so the directory should contain a __init__.py file
+""",
+ default=["validate", "ges"])
parser.add_argument("-d", "--debug", dest="debug",
action="store_true",
default=False,
" note that 0 will enable all tests",
type=int),
parser.add_argument("-c", "--config", dest="config",
- default=DEFAULT_VALIDATE_TESTSUITE,
- help="""Lets you specify a file where the testsuite to execute is defined.
-In this file you will have acces to the TestManager objects that you can configure with
-its various methods. For example you can find the 'validate' variable, in case the GstValidateManager
-launcher is avalaible. You should configure it using:
- * validate.add_scenarios: which allows you to register a list of scenario names to be run
- * validate.set_default_blacklist: Lets you set a list of tuple of the form:
- (@regex_defining_blacklister_test_names, @reason_for_the_blacklisting)
- * validate.add_generators: which allows you to register a list of #GstValidateTestsGenerator
- to be used to generate tests
- * validate.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests
-
-You can also set default values with:
- * validate.register_defaults: Sets default values for all parameters
- * validate.register_default_test_generators: Sets default values for the TestsGenerators to be used
- * gst_validate_register_default_scenarios: Sets default values for the scenarios to be executed
- * gst_validate_register_default_encoding_formats: Sets default values for the encoding formats to be tested
-
-Note: In the config file, you have access to the options variable resulting from the parsing of the command line
-user argument, you can thus override command line options using that.
-""")
+ default=None,
+ help="This is DEPRECATED, prefer using the testsuite format"
+ " to configure testsuites")
dir_group = parser.add_argument_group(
"Directories and files to be used by the launcher")
parser.add_argument('--xunit-file', action='store',
dir_group.add_argument("-M", "--main-dir", dest="main_dir",
default=DEFAULT_MAIN_DIR,
help="Main directory where to put files. Default is %s" % DEFAULT_MAIN_DIR)
+ dir_group.add_argument("--testsuites-dir", dest="testsuites_dir",
+ default=DEFAULT_TESTSUITES_DIR,
+ help="Directory where to look for testsuites. Default is %s"
+ " Note that GstValidate expect testsuite file to have .testsuite"
+ " as an extension in this folder." % DEFAULT_TESTSUITES_DIR)
dir_group.add_argument("-o", "--output-dir", dest="output_dir",
default=None,
help="Directory where to store logs and rendered files. Default is MAIN_DIR")
projects / platform / upstream / gstreamer.git / commitdiff