From 98e3c7d8b81a92f4fab9ea4bf39cb7fd472bb4e9 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 22 Sep 2020 00:04:08 +0530 Subject: [PATCH] cerbero ci: Refactor cerbero_before_script We were doing a few things suboptimally: 1. Making a symlink from the current directory to ../../gstreamer/cerbero so that build-tools kept working, which should not be necessary since cerbero relocates all binaries inside the cerbero homedir after extracting the cache. 2. Hard-coding the cerbero builddir when building on Windows because `pwd` was outputting an MSYS-style path which Python can't read 3. rsync-ing all the contents of $CERBERO_HOST_DIR, which is bad because we should be building on top of the artifact cache. The only thing we want from $CERBERO_HOST_DIR is cerbero-sources (the network cache), and we can setup the toolchain during bootstrap with the new --system and --toolchains arguments which were split out of --system-only. Part-of: --- docker/cerbero/Dockerfile-fedora | 8 +-- docker/windows/prepare_cerbero_env.sh | 4 +- gitlab/cerbero_setup.sh | 118 ++++++++++++++++++++-------------- gitlab/ci_template.yml | 7 +- 4 files changed, 79 insertions(+), 58 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 2e8c0e3..dad3e99 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -20,18 +20,18 @@ RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerb cd cerbero && \ echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \ echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \ - ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc fetch-package gstreamer-1.0 && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --build-tools=no --toolchains=no && \ cd .. && \ rm /cerbero/cerbero-sources/android-ndk-*.zip && \ rm /cerbero/cerbero-sources/mingw-*.tar.xz && \ diff --git a/docker/windows/prepare_cerbero_env.sh b/docker/windows/prepare_cerbero_env.sh index 446ed24..7cafe09 100644 --- a/docker/windows/prepare_cerbero_env.sh +++ b/docker/windows/prepare_cerbero_env.sh @@ -18,8 +18,8 @@ echo 'vs_install_version = "vs15"' >> localconf.cbc # Fetch all package requirements for a visualstudio gstreamer build ./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0 -# Extract the toolchain and fixup the MSYS installation -./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --system-only --offline +# Fixup the MSYS installation +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline # Delete mingw toolchain binary tarball rm /c/cerbero/cerbero-sources/mingw-*.tar.xz diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 217265e..38f8aea 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -8,16 +8,60 @@ show_ccache_sum() { fi } -# Produces runtime and devel tarball packages for linux/android or .pkg for macos -cerbero_package_and_check() { - # FIXME: mingw translates $(pwd) in a broken way - if [[ $CONFIG == win??.cbc ]]; then - PACKAGE_PATH="$CI_PROJECT_DIR" +# XXX: This is copied and modified from the cerbero-uninstalled script +# Use `mount` to get a list of MSYS mount points that the MSYS shell uses. +# That's our reference point for translating from MSYS paths to Win32 paths. +# We assume that the MSYS mount point directories are only in the filesystem +# root. This will break if people add their own custom mount points beyond what +# MSYS automatically creates, which is highly unlikely. +# +# /d -> d:/ +# /c -> c:/ +# /d/projects/cerbero -> d:/projects/cerbero/ +# /home/USERNAME/cerbero -> C:\\MinGW\\msys\\1.0/home/USERNAME/ +# /mingw -> C:\\MinGW/ +# /mingw/bin/foobar -> C:\\MinGW\\bin/foobar/ +# /tmp/baz -> C:\\Users\\USERNAME\\AppData\\Local\\Temp/baz/ +msys_dir_to_win32() { + set -e + local msys_path stripped_path mount_point path mounted_path + # If the path is already a native path, just return that + if [[ $1 == ?:/* ]] || [[ $1 == ?:\\* ]]; then + echo $1 + return + fi + # Convert /c or /mingw etc to /c/ or /mingw/ etc; gives us a necessary + # anchor to split the path into components + msys_path="$1/" + # Strip leading slash + stripped_path="${msys_path#/}" + # Get the first path component, which may be a mount point + mount_point="/${stripped_path%%/*}" + # Get the path inside the mountp oint + path="/${stripped_path#*/}" + mounted_path="$(mount | sed -n "s|\(.*\) on $mount_point type.*|\1|p")" + # If it's not a mounted path (like /c or /tmp or /mingw), then it's in the + # general MSYS root mount + if [[ -z $mounted_path ]]; then + mounted_path="$(mount | sed -n "s|\(.*\) on / type.*|\1|p")" + path="$1" + fi + echo ${mounted_path}${path%/} +} + +# Print the working directory in the native OS path format, but with forward +# slashes +pwd_native() { + if [[ -n "$MSYSTEM" ]]; then + msys_dir_to_win32 "$(pwd)" else - PACKAGE_PATH=$(pwd) + pwd fi +} - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$PACKAGE_PATH" gstreamer-1.0 +# Produces runtime and devel tarball packages for linux/android or .pkg for macos +cerbero_package_and_check() { + $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0 # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. if [[ $CONFIG != *universal* ]]; then @@ -29,52 +73,30 @@ cerbero_package_and_check() { } cerbero_before_script() { - # FIXME Wrong namespace - # Workaround build-tools having hardcoded internal path pwd - mkdir -p "../../gstreamer" - time ln -sf "$(pwd)" "../../gstreamer/cerbero" - # Don't try to symlink twice because on MSYS `ln` does a `cp` since it - # doesn't support the new NTFS symlink feature. - if [[ ${CI_PROJECT_NAMESPACE} != gstreamer ]]; then - mkdir -p "../../${CI_PROJECT_NAMESPACE}" - time ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" - fi + ls -lh - # Make sure there isn't a pre-existing config hanging around - rm -v -f localconf.cbc - rm -v -f ${CERBERO_HOST_DIR}/localconf.cbc + # Copy cerbero git repo stored on the image + cp -a "${CERBERO_HOST_DIR}/.git" . + git checkout . + git status - if [[ $CONFIG == win??.cbc ]]; then - # For windows hardcode the path so it doesn't get - # mangled by msys path handling - # FIXME: make the sources point to pwd/$CI_PROJECT_DIR like the rest - echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc - echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc + # If there's no cerbero-sources directory in the runner cache, copy it from + # the image cache + if ! [[ -d ${CERBERO_SOURCES} ]]; then + time cp -a "${CERBERO_HOST_DIR}/${CERBERO_SOURCES}" . + fi + du -sch "${CERBERO_SOURCES}" + echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc + echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc + if [[ $CONFIG == win??.cbc ]]; then # Visual Studio 2017 build tools install path echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc echo 'vs_install_version = "vs15"' >> localconf.cbc - else - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" > localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc fi - cat localconf.cbc - time rsync -aH "${CERBERO_HOST_DIR}" . - - cat localconf.cbc - - # FIXME: if you comment out this line it fails like so, no clue why. Its not windows defender either. - # From https://gitlab.freedesktop.org/gstreamer/cerbero - # b02080cb..d6923e42 master -> origin/master - # Fetching origin - # error: unable to create file cerbero-uninstalled: Permission denied - # fatal: Could not reset index file to revision 'd6923e4216c8a17759527a3db070d15cf7ff10a0'. - # ERROR: Failed to proceed with self update Command Error: Running ['git', 'reset', '--hard', 'd6923e4216c8a17759527a3db070d15cf7ff10a0'] returned 128 - git status - time ./cerbero-uninstalled --self-update manifest.xml } @@ -82,8 +104,10 @@ cerbero_script() { show_ccache_sum $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only + $CERBERO $CERBERO_ARGS fetch-bootstrap $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 + du -sch "${CERBERO_SOURCES}" + $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}" if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then @@ -92,7 +116,7 @@ cerbero_script() { time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}" fi - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only + $CERBERO $CERBERO_ARGS bootstrap --offline --system=no cerbero_package_and_check } @@ -100,9 +124,9 @@ cerbero_deps_script() { show_ccache_sum $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only + $CERBERO $CERBERO_ARGS fetch-bootstrap $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only + $CERBERO $CERBERO_ARGS bootstrap --offline --system=no $CERBERO $CERBERO_ARGS build-deps --offline \ gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \ gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6f1fd99..28bb936 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -683,7 +683,7 @@ valgrind ges: HAVE_CCACHE: "yes" # used by macos packages as we only ever install to a fixed directory CERBERO_OVERRIDDEN_DIST_DIR: "" - # location where cerbero is cached on the host + # location where the cerbero git repo is stored on the image CERBERO_HOST_DIR: "/cerbero/" before_script: - echo "Fetching ${CERBERO_SCRIPT_URL}" @@ -1196,10 +1196,7 @@ build gst-omx tizonia fedora x86_64: variables: CONFIG: 'win64.cbc' ARCH: 'msvc_x86_64' - # rsync doesn't like windows paths like `C:/cerbero` - CERBERO_HOST_DIR: "/c/cerbero/" - CERBERO_HOME: "C:/cerbero-build" - CERBERO_SOURCES: "C:/cerbero-sources" + CERBERO_HOST_DIR: "C:/cerbero" CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v visualstudio -v nowerror" CERBERO_RUN_SUFFIX: ".exe" HAVE_CCACHE: "" -- 2.7.4