subprojects/gst-integration-testsuites/testsuites/validate.py

   1 # -*- Mode: Python -*- vi:si:et:sw=4:sts=4:ts=4:syntax=python
   2 #
   3 # Copyright (c) 2014,Thibault Saunier <thibault.saunier@collabora.com>
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Lesser General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2.1 of the License, or (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Lesser General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Lesser General Public
  16 # License along with this program; if not, write to the
  17 # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  18 # Boston, MA 02110-1301, USA.
  19
  20 """
  21 The GstValidate default testsuite
  22 """
  23
  24 import os
  25 import glob
  26 import pathlib
  27 import re
  28 import subprocess
  29
  30 from testsuiteutils import update_assets
  31 from launcher import utils
  32 from launcher.baseclasses import MediaFormatCombination
  33 from launcher.apps.gstvalidate import GstValidateSimpleTestsGenerator
  34 from validate_known_issues import KNOWN_ISSUES
  35
  36
  37 TEST_MANAGER = "validate"
  38
  39 BLACKLIST = [('validate.file.transcode.to_vorbis_and_vp8_in_webm.GH1_00094_1920x1280_MTS',
  40               'Got error: Internal data stream error. -- Debug message: mpegtsbase.c(1371):'
  41               'mpegts_base_loop (): ...: stream stopped, reason not-negotiated'),
  42              ('validate.testbin.transcode.*',
  43               "Encoding testsrc is not so interesting and the source is now unlimited"),
  44              ('validate.file.*.simple.fast_forward.synchronized',
  45               'https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/541'),
  46              ('validate.hls.playback.change_state_intensive.*',
  47               'https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/482'),
  48             ('validate.rtsp.*playback.switch.*',
  49              'https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/357'),
  50             ('validate.rtsp.*playback.*seek.*mxf$|validate.rtsp.*playback.*change_state_intensive.*mxf$',
  51              'Actions on MXF streams with rtsp-server fail in racy ways.'
  52              ' (Deactivating as it is not very important.)'),
  53             ('validate.rtsp.*pal-dv25_mxf$',
  54              'File has decoding issues with rtsp-server.'
  55              ' (Deactivating as it is not very important.)'),
  56              ("(?!.*.media_check.qtdemux-test-frag-basic_zero_dur_no_mehd_mp4).*.qtdemux-test-frag-basic_zero_dur_no_mehd_mp4",
  57               '`qtdemux-test-frag-basic_zero_dur_no_mehd_mp4` is there only for media_check tests.'),
  58              ('validate.rtsp.*playback.*mp3_h265_0_mp4$',
  59               'The version of libav shipped by Fedora 29 crashes in various ways with these tests.'),
  60              ('validate.rtsp.*playback.seek.*GH1_00094_1920x1280_MTS',
  61               'Do not preroll after pause.'),
  62              ('validate.file.playback.reverse_playback.sample_mpeg_program_stream_scr_mpg',
  63               'Do not decode any frame in reverse playback with SCR.'),
  64              ('validate.rtsp.*playback.seek.*sample_mpeg_program_stream_scr_mpg',
  65               'Racy with CI. No frames decoded before the end of the stream.'),
  66              ]
  67
  68 def add_accurate_seek_tests(test_manager, media_dir, extra_data):
  69     accurate_seeks_media_infos = []
  70     for f in [
  71             'mp4/timecoded_jpeg_23976fps.mp4.media_info.skipped',
  72             'mp4/timecoded_jpeg_2997fps.mp4.media_info.skipped',
  73             'mp4/timecoded_jpeg_30fps.mp4.media_info.skipped',
  74
  75             'mp4/timecoded_vp8_23976fps.mp4.media_info.skipped',
  76             'mp4/timecoded_vp8_2997fps.mp4.media_info.skipped',
  77             'mp4/timecoded_vp8_30fps.mp4.media_info.skipped',
  78
  79             'mp4/timecoded_h264_23976fps.mp4.media_info.skipped',
  80             'mp4/timecoded_h264_2997fps.mp4.media_info.skipped',
  81             'mp4/timecoded_h264_30fps.mp4.media_info.skipped',
  82         ]:
  83         dirname = os.path.join(media_dir, "defaults", os.path.dirname(f))
  84         filename = os.path.basename(f)
  85         media_info = os.path.join(dirname, filename)
  86         reference_frames_dir = os.path.join(dirname, re.sub(r"\.media_info.*", "_reference_frames", filename).replace('.', '_'))
  87         accurate_seeks_media_infos.append((media_info, reference_frames_dir))
  88
  89     test_manager.add_generators(
  90         test_manager.GstValidateCheckAccurateSeekingTestGenerator(
  91             'accurate_seeks',
  92             test_manager,
  93             [(os.path.join(media_dir, media_info), os.path.join(media_dir, reference_frames_dir)) for media_info, reference_frames_dir in accurate_seeks_media_infos],
  94             extra_data=extra_data)
  95     )
  96
  97
  98 def setup_tests(test_manager, options):
  99     testsuite_dir = os.path.realpath(os.path.join(os.path.dirname(__file__)))
 100     media_dir = os.path.realpath(os.path.join(testsuite_dir, os.path.pardir, "medias"))
 101
 102     assets_dir = os.path.realpath(os.path.join(testsuite_dir, os.path.pardir, "medias", "defaults"))
 103     if options.sync:
 104         if not utils.USING_SUBPROJECT:
 105             if not update_assets(options, assets_dir):
 106                 return False
 107         else:
 108             print("Syncing gst-integration-testsuites media files")
 109             subprocess.check_call(['git', 'submodule', 'update', '--init'],
 110                                 cwd=utils.DEFAULT_GST_QA_ASSETS)
 111             subprocess.check_call(['git', 'lfs', 'pull', '--exclude='],
 112                                 cwd=pathlib.Path(utils.DEFAULT_GST_QA_ASSETS) / 'medias')
 113
 114     options.add_paths(assets_dir)
 115     options.set_http_server_dir(media_dir)
 116     test_manager.set_default_blacklist(BLACKLIST)
 117
 118     extra_data = {
 119         "config_path": os.path.dirname(testsuite_dir),
 120         "medias": media_dir,
 121         "validate-flow-expectations-dir": os.path.join(testsuite_dir, "validate", "flow-expectations"),
 122         "validate-flow-actual-results-dir": test_manager.options.logsdir,
 123         "ssim-results-dir": os.path.join(test_manager.options.logsdir, "ssim-results"),
 124     }
 125     add_accurate_seek_tests(test_manager, media_dir, extra_data)
 126
 127     test_manager.add_generators(
 128         GstValidateSimpleTestsGenerator("simple", test_manager,
 129             os.path.join(testsuite_dir, "validate"))
 130     )
 131
 132     test_manager.add_expected_issues(KNOWN_ISSUES)
 133     test_manager.register_defaults()
 134
 135     test_manager.add_encoding_formats([MediaFormatCombination("quicktime", "rawaudio", "prores")])
 136
 137     valid_mixing_scenarios = ["play_15s",
 138                               "fast_forward",
 139                               "seek_forward",
 140                               "seek_backward",
 141                               "seek_with_stop",
 142                               "scrub_forward_seeking"]
 143
 144     for compositor in ["compositor", "glvideomixer"]:
 145         test_manager.add_generators(
 146             test_manager.GstValidateMixerTestsGenerator(compositor + ".simple", test_manager,
 147                                                         compositor,
 148                                                         "video",
 149                                                         converter="deinterlace ! videoconvert",
 150                                                         mixed_srcs={
 151                                                              "synchronized": {"mixer_props": "sink_1::alpha=0.5 sink_1::xpos=50 sink_1::ypos=50",  # noqa
 152                                                                               "sources":
 153                                                                               ("videotestsrc pattern=snow timestamp-offset=3000000000 ! 'video/x-raw,format=AYUV,width=640,height=480,framerate=(fraction)30/1' !  timeoverlay",  # noqa
 154                                                                                "videotestsrc pattern=smpte ! 'video/x-raw,format=AYUV,width=800,height=600,framerate=(fraction)10/1' ! timeoverlay")},  # noqa
 155                                                              "bgra": ("videotestsrc ! video/x-raw, framerate=\(fraction\)10/1, width=100, height=100",  # noqa
 156                                                                       "videotestsrc ! video/x-raw, framerate=\(fraction\)5/1, width=320, height=240")
 157                                                         },
 158                                                         valid_scenarios=valid_mixing_scenarios))
 159
 160     test_manager.add_generators(
 161         test_manager.GstValidateMixerTestsGenerator("audiomixer.simple", test_manager,
 162                                                     "audiomixer",
 163                                                     "audio",
 164                                                     converter="audioconvert ! audioresample",
 165                                                     mixed_srcs={"basic": {"mixer_props": "",
 166                                                                 "sources": ("audiotestsrc wave=triangle",
 167                                                                             "audiotestsrc wave=ticks")}},
 168                                                     valid_scenarios=valid_mixing_scenarios))
 169
 170     return True