ci: fedora-docker: Cache media files

As of now all integration tests endup fetching 600MB of media every run. This
will increase the docker side, but this is to the traceof that the docker
image will quickly be cached on all runners.

Fixes #841

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1413>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dc5791d..917cf6e 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,7 @@ variables:
   # If you are hacking on them or need a them to rebuild, its enough
   # to change any part of the string of the image you want.
   ###
-  FEDORA_TAG:  '2021-10-05.0'
+  FEDORA_TAG:  '2021-12-03.1'
   INDENT_TAG: '2021-10-04.0'
   WINDOWS_TAG: "2021-10-12.0"
 
@@ -136,7 +136,7 @@ trigger:
     FDO_DISTRIBUTION_VERSION: '31'
     FDO_REPO_SUFFIX: "$FEDORA_AMD64_SUFFIX"
     FDO_DISTRIBUTION_TAG: "$FEDORA_TAG-$GST_UPSTREAM_BRANCH"
-    FDO_DISTRIBUTION_EXEC: 'DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH bash ci/docker/fedora/prepare.sh'
+    FDO_DISTRIBUTION_EXEC: 'GIT_BRANCH=$CI_COMMIT_REF_NAME GIT_URL=$CI_REPOSITORY_URL bash ci/docker/fedora/prepare.sh'
 
 fedora amd64 docker:
   extends:
@@ -267,8 +267,10 @@ gst indent:
 # build setup templates
 #
 .build_template: &build
+  - ci/scripts/handle-subprojects-cache.py subprojects/
+  # Update subprojects to respect `.wrap` content
+  - meson subprojects update --reset
   - echo $MESON_ARGS
-
   - meson build/ $MESON_ARGS
   - ninja -C build/
   - ccache --show-stats
@@ -323,14 +325,6 @@ gst indent:
   timeout: '45min'
   variables:
     MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} ${MESON_GST_WERROR}"
-  before_script:
-    - export RUSTUP_HOME="/usr/local/rustup"
-    - export CARGO_HOME="/usr/local/cargo"
-    - export PATH="/usr/local/cargo/bin:$PATH"
-
-    - ci/scripts/handle-subprojects-cache.py subprojects/
-    # Update subprojects to respect `.wrap` content
-    - meson subprojects update --reset
   script:
     *build
   after_script:
@@ -399,9 +393,8 @@ build clang fedora x86_64:
       when: manual
     - changes:
         *modules_changes
-  before_script:
-    - ci/scripts/handle-subprojects-cache.py subprojects/
   script:
+    - ci/scripts/handle-subprojects-cache.py subprojects/
     # For some reason, options are separated by newline instead of space, so we
     # have to replace them first.
     - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ")
@@ -685,6 +678,8 @@ valgrind ges:
   - pip3 install --upgrade meson
   - *build
   - ./gst-env.py ninja -C build/ plugins_doc_caches
+  # Ignore modifications to wrap files made by meson
+  - git checkout subprojects/*.wrap
   - ./ci/scripts/check-documentation-diff.py
   - ./gst-env.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings
   - mv build/subprojects/gst-docs/GStreamer-doc/html documentation/

diff --git a/ci/docker/fedora/prepare.sh b/ci/docker/fedora/prepare.sh
index 9ab1d98..5e0563f 100644 (file)
--- a/ci/docker/fedora/prepare.sh
+++ b/ci/docker/fedora/prepare.sh
@@ -238,17 +238,20 @@ cargo --version
 rustc --version
 
 # get gstreamer and make all subprojects available
-# FIXME: we don't really need to clone it now as we have the repo checkout out already
-git clone -b  ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gstreamer.git /gstreamer
+git clone -b ${GIT_BRANCH} ${GIT_URL} /gstreamer
+git -C /gstreamer submodule update --init --depth=1
 meson subprojects download --sourcedir /gstreamer
 /gstreamer/ci/scripts/handle-subprojects-cache.py --build /gstreamer/subprojects/
 
 # Run git gc to prune unwanted refs and reduce the size of the image
-for i in $(find subprojects/ -mindepth 1 -maxdepth 1 -type d);
+for i in $(find /subprojects/ -mindepth 1 -maxdepth 1 -type d);
 do
     git -C $i gc --aggressive || true;
 done
 
+# Now remove the gstreamer clone
+rm -rf /gstreamer
+
 echo "Removing DNF cache"
 dnf clean all
 

diff --git a/ci/scripts/handle-subprojects-cache.py b/ci/scripts/handle-subprojects-cache.py
index 6281e09..9e37221 100755 (executable)
--- a/ci/scripts/handle-subprojects-cache.py
+++ b/ci/scripts/handle-subprojects-cache.py
@@ -24,9 +24,23 @@ def create_cache_in_image(options):
         if project_name != "packagecache" and not os.path.exists(os.path.join(project_path, '.git')):
             continue
 
+        if os.path.exists(os.path.join(DEST, project_name)):
+            continue
+
         print("Copying %s" % project_name)
         shutil.copytree(project_path, os.path.join(DEST, project_name))
 
+    media_path = os.path.join(options.subprojects_dir, '..', '.git',
+        'modules', 'subprojects', 'gst-integration-testsuites', 'medias')
+    if os.path.exists(os.path.join(DEST, 'medias.git')):
+        return
+
+    if os.path.exists(media_path):
+        print("Creating media cache")
+        shutil.copytree(media_path, os.path.join(DEST, 'medias.git'))
+    else:
+        print("Did not find medias in %s" % media_path)
+
 
 def copy_cache(options):
     # FIXME Remove when not needed anymore.
@@ -37,8 +51,14 @@ def copy_cache(options):
 
         for project_name in os.listdir(path):
             project_path = os.path.join(options.subprojects_dir, project_name)
-
             cache_dir = os.path.join(path, project_name)
+
+            if project_name == 'medias.git':
+                project_path = os.path.join(options.subprojects_dir, '..', '.git', 'modules',
+                    'subprojects', 'gst-integration-testsuites')
+                os.makedirs(project_path, exist_ok=True)
+                project_path = os.path.join(project_path, 'medias')
+
             if os.path.exists(project_path):
                 print("- Ignoring %s" % cache_dir)
                 continue
@@ -47,7 +67,7 @@ def copy_cache(options):
                 print("- Ignoring %s" % cache_dir)
                 continue
 
-            print("Copying from %s" % cache_dir)
+            print("Copying from %s -> %s" % (cache_dir, project_path))
             shutil.copytree(cache_dir, project_path)