From: Stéphane Cerveau Date: Mon, 2 Dec 2019 13:46:59 +0000 (+0100) Subject: gst-validate-launcher: update documentation X-Git-Tag: 1.19.3~491^2~222 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=2581fef6843bfb53f3fc6f629577c1f013ef84e7;p=platform%2Fupstream%2Fgstreamer.git gst-validate-launcher: update documentation Use the new api to create your custom testsuite. Fix some broken links and enhance the logging system. --- diff --git a/docs/gst-validate-launcher.md b/docs/gst-validate-launcher.md index 15298de..c0e6c1f 100644 --- a/docs/gst-validate-launcher.md +++ b/docs/gst-validate-launcher.md @@ -71,59 +71,81 @@ Then you will need to write the `testsuite.py` file. You can for example implement the following testsuite: ``` python -import os - -# Make sure gst-validate-launcher uses our media files -options.paths = os.path.dirname(os.path.realpath(__file__)) - -# Make sure GstValidate is able to use our scenarios -# from the testsuite_folder/scenarios folder -os.environ["GST_VALIDATE_SCENARIOS_PATH"] = \ - os.path.join(os.path.dirname(os.path.realpath(__file__)), "scenarios") - -# You can activate the following if you only care about critical issues in -# the report: -# os.environ["GST_VALIDATE"] = "print_criticals" - -# Make gst-validate use our scenarios -validate.add_scenarios(["scenario", "scenario1"]) - +""" +The GstValidate custom testsuite +""" -# Now add "Theora and Vorbis in OGG container" as a wanted transcoding format. That means -# that conversion to this format will be tested on all the media files/streams. -validate.add_encoding_formats([MediaFormatCombination("ogg", "vorbis", "theora")]) - -# Use the GstValidatePlaybinTestsGenerator to generate tests that will use playbin -# and GstValidateTranscodingTestsGenerator to create media transcoding tests that -# will use all the media format added with validate.add_encoding_formats -validate.add_generators([validate.GstValidatePlaybinTestsGenerator(validate), - GstValidateTranscodingTestsGenerator(self)]) - -# Blacklist some tests that are known to fail because a feature is not supported -# or due to any other reason. -# The tuple defining those tests is of the form: -# ("regex defining the test name", "Reason why the test should be disabled") -validate.set_default_blacklist([ - ("validate.*.scenario1.*ogv$" - "oggdemux does not support some action executed in scenario1")] - ) +import os +from launcher.baseclasses import MediaFormatCombination +from launcher.apps.gstvalidate import * +TEST_MANAGER = "validate" + +KNOWN_ISSUES = {} + +def setup_tests(test_manager, options): + print("Setting up the custom testsuite") + assets_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".", "samples_files")) + options.add_paths(assets_dir) + + # This step will register default data for the test manager: + # - scenarios such as `play_15s`, `reverse_playback` etc. + # - encoding formats such as "mp4,h264,mp3" etc. + # - blacklist such as dash.media_check.* + # - test generators: + # - GstValidatePlaybinTestsGenerator + # - GstValidateMediaCheckTestsGenerator + # - GstValidateTranscodingTestsGenerator + # This 'defaults' can be found in 'gst-devtools/validate/launcher/apps/gstvalidate.py#register_defaults' + # test_manager.register_defaults() + + # Add scenarios + scenarios = [] + scenarios.append("play_5s") + scenarios.append("seek_backward") + test_manager.set_scenarios(scenarios) + + # Add encoding formats used by the transcoding generator + test_manager.add_encoding_formats([ + MediaFormatCombination("mp4", "mp3", "h264"),]) + + # Add generators + # GstValidatePlaybinTestsGenerator needs at least one media file + test_manager.add_generators([GstValidateMediaCheckTestsGenerator(test_manager)]) + # GstValidatePlaybinTestsGenerator needs at least one scenario + test_manager.add_generators([GstValidatePlaybinTestsGenerator(test_manager)]) + # GstValidateTranscodingTestsGenerator needs at least one MediaFormatCombination + test_manager.add_generators([GstValidateTranscodingTestsGenerator(test_manager)]) + + # list of combo to blacklist tests. Here it blacklists all tests with playback.seek_backward + test_manager.set_default_blacklist([ + ("custom_testsuite.file.playback.seek_backward.*", + "Not supported by this testsuite."),]) + + # you can even pass known issues to bypass an existing error in your custom testsuite + test_manager.add_expected_issues(KNOWN_ISSUES) + return True ``` Once this is done, you've got a testsuite that will: - Run playbin pipelines on `file.mp4`, `file1.mkv` and `file2.ogv`> - executing `scenario` and `scenario1` scenarios + executing `play_5s` and `seek_backward` scenarios -- Transcode `file.mp4,` `file1.mkv` and `file2.ogv` to Theora and - Vorbis in a OGG container +- Transcode `file.mp4,` `file1.mkv` and `file2.ogv` to h264 and + mp3 in a MP4 container The only thing to do to run the testsuite is: - gst-validate-launcher --config /path/to/testsuite_folder/testsuite.py + gst-validate-launcher --testsuites-dir=/path/to/testsuite_folder/ testsuite + # Invocation You can find detailed information about the launcher by launching it: gst-validate-launcher --help + +You can list all the tests with: + + gst-validate-launcher --testsuites-dir=/path/to/testsuite_folder/ testsuite -L diff --git a/validate/launcher/apps/gstvalidate.py b/validate/launcher/apps/gstvalidate.py index 9742a68..fe256ec 100644 --- a/validate/launcher/apps/gstvalidate.py +++ b/validate/launcher/apps/gstvalidate.py @@ -844,7 +844,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") group.add_argument("--validate-gdb-server", dest="validate_gdb_server", help="Run the server in GDB.") group.add_argument("--validate-disable-rtsp", dest="disable_rtsp", - help="Disable RTSP tests.") + help="Disable RTSP tests.", default=False, action='store_true') group.add_argument("--validate-enable-iqa-tests", dest="validate_enable_iqa_tests", help="Enable Image Quality Assessment validation tests.", default=False, action='store_true') diff --git a/validate/launcher/main.py b/validate/launcher/main.py index a39bd22..f77b1e0 100644 --- a/validate/launcher/main.py +++ b/validate/launcher/main.py @@ -59,7 +59,7 @@ been compiled against GstValidate. 2. Default test suite --------------------- -A default suite of tests is provided and is available at: http://cgit.freedesktop.org/gstreamer/gst-integration-testsuites/ +A default suite of tests is provided and is available at: http://gitlab.freedesktop.org/gstreamer/gst-integration-testsuites/ You can run it pretty simply doing: . $gst-validate-launcher --sync @@ -125,8 +125,11 @@ same way as if they were local files. ---------------------------------------- You can activate debug logs setting the environment variable GST_VALIDATE_LAUNCHER_DEBUG. + +. $GST_VALIDATE_LAUNCHER_DEBUG=6 gst-validate-launcher + It uses the same syntax as PITIVI_DEBUG (more information at: -http://wiki.pitivi.org/wiki/Bug_reporting#Debug_logs). +https://developer.pitivi.org/Bug_reporting.html#debug-logs). ''' % ("\n * ".join([reporter.name for reporter in utils.get_subclasses(reporters.Reporter, reporters.__dict__)] ), @@ -145,6 +148,8 @@ DEFAULT_GST_QA_ASSETS_REPO = "https://gitlab.freedesktop.org/gstreamer/gst-integ def download_assets(options): try: + printc("About to download assets from %s to %s" % options.remote_assets_url, + options.clone_dir) launch_command("%s %s %s" % (options.get_assets_command, options.remote_assets_url, options.clone_dir), @@ -311,7 +316,7 @@ class LauncherConfig(Loggable): if not self.sync and not os.path.exists(self.clone_dir) and \ self.clone_dir == os.path.join(self.clone_dir, MEDIAS_FOLDER): printc("Media path (%s) does not exists. Forgot to run --sync ?" - % self.clone_dir, Colors.FAIL, True) + % self.clone_dir, Colors.FAIL, True) return False if (self.main_dir != DEFAULT_MAIN_DIR or self.clone_dir != QA_ASSETS):