cerbero ci: Refactor cerbero_before_script

We were doing a few things suboptimally:

1. Making a symlink from the current directory to
   ../../gstreamer/cerbero so that build-tools kept working, which
   should not be necessary since cerbero relocates all binaries inside
   the cerbero homedir after extracting the cache.

2. Hard-coding the cerbero builddir when building on Windows because
   `pwd` was outputting an MSYS-style path which Python can't read

3. rsync-ing all the contents of $CERBERO_HOST_DIR, which is
   bad because we should be building on top of the artifact cache.
   The only thing we want from $CERBERO_HOST_DIR is cerbero-sources
   (the network cache), and we can setup the toolchain during
   bootstrap with the new --system and --toolchains arguments which
   were split out of --system-only.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-ci/-/merge_requests/350>

diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora
index 2e8c0e3..dad3e99 100644 (file)
--- a/docker/cerbero/Dockerfile-fedora
+++ b/docker/cerbero/Dockerfile-fedora
@@ -20,18 +20,18 @@ RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerb
     cd cerbero && \
       echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \
       echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \
-      ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --system-only && \
+      ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \
       ./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap && \
       ./cerbero-uninstalled -t -c localconf.cbc fetch-package gstreamer-1.0 && \
       ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \
       ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \
-      ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --system-only && \
+      ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --build-tools=no --toolchains=no && \
       ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \
       ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \
-      ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \
+      ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --build-tools=no --toolchains=no && \
       ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \
       ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \
-      ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \
+      ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --build-tools=no --toolchains=no && \
       cd .. && \
     rm /cerbero/cerbero-sources/android-ndk-*.zip && \
     rm /cerbero/cerbero-sources/mingw-*.tar.xz && \

diff --git a/docker/windows/prepare_cerbero_env.sh b/docker/windows/prepare_cerbero_env.sh
index 446ed24..7cafe09 100644 (file)
--- a/docker/windows/prepare_cerbero_env.sh
+++ b/docker/windows/prepare_cerbero_env.sh
@@ -18,8 +18,8 @@ echo 'vs_install_version = "vs15"' >> localconf.cbc
 # Fetch all package requirements for a visualstudio gstreamer build
 ./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0
 
-# Extract the toolchain and fixup the MSYS installation
-./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --system-only --offline
+# Fixup the MSYS installation
+./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline
 
 # Delete mingw toolchain binary tarball
 rm /c/cerbero/cerbero-sources/mingw-*.tar.xz

diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh
index 217265e..38f8aea 100644 (file)
--- a/gitlab/cerbero_setup.sh
+++ b/gitlab/cerbero_setup.sh
@@ -8,16 +8,60 @@ show_ccache_sum() {
     fi
 }
 
-# Produces runtime and devel tarball packages for linux/android or .pkg for macos
-cerbero_package_and_check() {
-    # FIXME: mingw translates $(pwd) in a broken way
-    if [[ $CONFIG == win??.cbc ]]; then
-        PACKAGE_PATH="$CI_PROJECT_DIR"
+# XXX: This is copied and modified from the cerbero-uninstalled script
+# Use `mount` to get a list of MSYS mount points that the MSYS shell uses.
+# That's our reference point for translating from MSYS paths to Win32 paths.
+# We assume that the MSYS mount point directories are only in the filesystem
+# root. This will break if people add their own custom mount points beyond what
+# MSYS automatically creates, which is highly unlikely.
+#
+# /d -> d:/
+# /c -> c:/
+# /d/projects/cerbero -> d:/projects/cerbero/
+# /home/USERNAME/cerbero -> C:\\MinGW\\msys\\1.0/home/USERNAME/
+# /mingw -> C:\\MinGW/
+# /mingw/bin/foobar -> C:\\MinGW\\bin/foobar/
+# /tmp/baz -> C:\\Users\\USERNAME\\AppData\\Local\\Temp/baz/
+msys_dir_to_win32() {
+    set -e
+    local msys_path stripped_path mount_point path mounted_path
+    # If the path is already a native path, just return that
+    if [[ $1 == ?:/* ]] || [[ $1 == ?:\\* ]]; then
+      echo $1
+      return
+    fi
+    # Convert /c or /mingw etc to /c/ or /mingw/ etc; gives us a necessary
+    # anchor to split the path into components
+    msys_path="$1/"
+    # Strip leading slash
+    stripped_path="${msys_path#/}"
+    # Get the first path component, which may be a mount point
+    mount_point="/${stripped_path%%/*}"
+    # Get the path inside the mountp oint
+    path="/${stripped_path#*/}"
+    mounted_path="$(mount | sed -n "s|\(.*\) on $mount_point type.*|\1|p")"
+    # If it's not a mounted path (like /c or /tmp or /mingw), then it's in the
+    # general MSYS root mount
+    if [[ -z $mounted_path ]]; then
+        mounted_path="$(mount | sed -n "s|\(.*\) on / type.*|\1|p")"
+        path="$1"
+    fi
+    echo ${mounted_path}${path%/}
+}
+
+# Print the working directory in the native OS path format, but with forward
+# slashes
+pwd_native() {
+    if [[ -n "$MSYSTEM" ]]; then
+        msys_dir_to_win32 "$(pwd)"
     else
-        PACKAGE_PATH=$(pwd)
+        pwd
     fi
+}
 
-    $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$PACKAGE_PATH" gstreamer-1.0
+# Produces runtime and devel tarball packages for linux/android or .pkg for macos
+cerbero_package_and_check() {
+    $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0
 
     # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course.
     if [[ $CONFIG != *universal* ]]; then
@@ -29,52 +73,30 @@ cerbero_package_and_check() {
 }
 
 cerbero_before_script() {
-    # FIXME Wrong namespace
-    # Workaround build-tools having hardcoded internal path
     pwd
-    mkdir -p "../../gstreamer"
-    time ln -sf "$(pwd)" "../../gstreamer/cerbero"
-    # Don't try to symlink twice because on MSYS `ln` does a `cp` since it
-    # doesn't support the new NTFS symlink feature.
-    if [[ ${CI_PROJECT_NAMESPACE} != gstreamer ]]; then
-        mkdir -p "../../${CI_PROJECT_NAMESPACE}"
-        time ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero"
-    fi
+    ls -lh
 
-    # Make sure there isn't a pre-existing config hanging around
-    rm -v -f localconf.cbc
-    rm -v -f ${CERBERO_HOST_DIR}/localconf.cbc
+    # Copy cerbero git repo stored on the image
+    cp -a "${CERBERO_HOST_DIR}/.git" .
+    git checkout .
+    git status
 
-    if [[ $CONFIG == win??.cbc ]]; then
-        # For windows hardcode the path so it doesn't get
-        # mangled by msys path handling
-        # FIXME: make the sources point to pwd/$CI_PROJECT_DIR like the rest
-        echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc
-        echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc
+    # If there's no cerbero-sources directory in the runner cache, copy it from
+    # the image cache
+    if ! [[ -d ${CERBERO_SOURCES} ]]; then
+        time cp -a "${CERBERO_HOST_DIR}/${CERBERO_SOURCES}" .
+    fi
+    du -sch "${CERBERO_SOURCES}"
 
+    echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc
+    echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc
+    if [[ $CONFIG == win??.cbc ]]; then
         # Visual Studio 2017 build tools install path
         echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc
         echo 'vs_install_version = "vs15"' >> localconf.cbc
-    else
-        echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" > localconf.cbc
-        echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc
     fi
-
     cat localconf.cbc
 
-    time rsync -aH "${CERBERO_HOST_DIR}" .
-
-    cat localconf.cbc
-
-    # FIXME: if you comment out this line it fails like so, no clue why. Its not windows defender either.
-    # From https://gitlab.freedesktop.org/gstreamer/cerbero
-    #    b02080cb..d6923e42  master     -> origin/master
-    # Fetching origin
-    # error: unable to create file cerbero-uninstalled: Permission denied
-    # fatal: Could not reset index file to revision 'd6923e4216c8a17759527a3db070d15cf7ff10a0'.
-    # ERROR: Failed to proceed with self update Command Error: Running ['git', 'reset', '--hard', 'd6923e4216c8a17759527a3db070d15cf7ff10a0'] returned 128
-    git status
-
     time ./cerbero-uninstalled --self-update manifest.xml
 }
 
@@ -82,8 +104,10 @@ cerbero_script() {
     show_ccache_sum
 
     $CERBERO $CERBERO_ARGS show-config
-    $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
+    $CERBERO $CERBERO_ARGS fetch-bootstrap
     $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
+    du -sch "${CERBERO_SOURCES}"
+
     $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}"
 
     if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then
@@ -92,7 +116,7 @@ cerbero_script() {
         time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}"
     fi
 
-    $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only
+    $CERBERO $CERBERO_ARGS bootstrap --offline --system=no
     cerbero_package_and_check
 }
 
@@ -100,9 +124,9 @@ cerbero_deps_script() {
     show_ccache_sum
 
     $CERBERO $CERBERO_ARGS show-config
-    $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
+    $CERBERO $CERBERO_ARGS fetch-bootstrap
     $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
-    $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only
+    $CERBERO $CERBERO_ARGS bootstrap --offline --system=no
     $CERBERO $CERBERO_ARGS build-deps --offline \
         gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \
         gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \

diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml
index 6f1fd99..28bb936 100644 (file)
--- a/gitlab/ci_template.yml
+++ b/gitlab/ci_template.yml
@@ -683,7 +683,7 @@ valgrind ges:
     HAVE_CCACHE: "yes"
     # used by macos packages as we only ever install to a fixed directory
     CERBERO_OVERRIDDEN_DIST_DIR: ""
-    # location where cerbero is cached on the host
+    # location where the cerbero git repo is stored on the image
     CERBERO_HOST_DIR: "/cerbero/"
   before_script:
     - echo "Fetching ${CERBERO_SCRIPT_URL}"
@@ -1196,10 +1196,7 @@ build gst-omx tizonia fedora x86_64:
   variables:
     CONFIG: 'win64.cbc'
     ARCH: 'msvc_x86_64'
-    # rsync doesn't like windows paths like `C:/cerbero`
-    CERBERO_HOST_DIR: "/c/cerbero/"
-    CERBERO_HOME: "C:/cerbero-build"
-    CERBERO_SOURCES: "C:/cerbero-sources"
+    CERBERO_HOST_DIR: "C:/cerbero"
     CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v visualstudio -v nowerror"
     CERBERO_RUN_SUFFIX: ".exe"
     HAVE_CCACHE: ""