From b857a5ada178563f53dd65bbdbf58b021b4ff3f2 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 7 Aug 2020 15:29:39 +0300 Subject: [PATCH] citemplate: add initial cerbero msvc builds This commit adds the last bits needed to have a working CI job of cerbero using mingw and msvc17 toolchain. It adds a new stage to the docker image that mirrors the setup of the cerbero fedora docker build, and tweaks the cerbero_setup script run by jobs with a couple windows specific fixes. Close #39 Part-of: --- docker/windows/Dockerfile | 3 +++ docker/windows/build_image.ps1 | 4 +-- docker/windows/prepare_cerbero_env.sh | 29 +++++++++++++++++++++ gitlab/cerbero_setup.sh | 45 ++++++++++++++++++++++++++++++--- gitlab/ci_template.yml | 47 +++++++++++++++++++++++++++++++++-- 5 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 docker/windows/prepare_cerbero_env.sh diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 882082a..e36e4fa 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -18,3 +18,6 @@ ARG DEFAULT_BRANCH="master" COPY prepare_gst_env.ps1 C:\ RUN C:\prepare_gst_env.ps1 + +COPY prepare_cerbero_env.sh C:\ +RUN C:\MinGW\msys\1.0\bin\bash.exe --login -c "C:/prepare_cerbero_env.sh" diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 490ecc6..c092f6d 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,6 +1,6 @@ $env:DEFAULT_BRANCH='master' -$env:VERSION='v11' +$env:VERSION='v12' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" echo "Building $env:tag" docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . -# docker push $tag +# docker push $env:tag diff --git a/docker/windows/prepare_cerbero_env.sh b/docker/windows/prepare_cerbero_env.sh new file mode 100644 index 0000000..446ed24 --- /dev/null +++ b/docker/windows/prepare_cerbero_env.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +set -eux + +cd C:/ +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git +cd cerbero + +echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc +echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc +echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc +echo 'vs_install_version = "vs15"' >> localconf.cbc + +# Fetch all bootstrap requirements +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-bootstrap +# Fetch all package requirements for a mingw gstreamer build +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0 +# Fetch all package requirements for a visualstudio gstreamer build +./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0 + +# Extract the toolchain and fixup the MSYS installation +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --system-only --offline + +# Delete mingw toolchain binary tarball +rm /c/cerbero/cerbero-sources/mingw-*.tar.xz +# Wipe visualstudio package dist, sources, logs, and the build tools recipes +./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc wipe --force --build-tools +# clean the localconf +rm /c/cerbero/localconf.cbc diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 364aba5..c4e5251 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -8,7 +8,14 @@ show_ccache_sum() { # Produces runtime and devel tarball packages for linux/android or .pkg for macos cerbero_package_and_check() { - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd)" gstreamer-1.0 + # FIXME: mingw translates $(pwd) in a broken way + if [[ $CONFIG == win??.cbc ]]; then + PACKAGE_PATH="$CI_PROJECT_DIR" + else + PACKAGE_PATH=$(pwd) + fi + + $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$PACKAGE_PATH" gstreamer-1.0 # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. if [[ $CONFIG != *universal* ]]; then @@ -27,9 +34,41 @@ cerbero_before_script() { ln -sf "$(pwd)" "../../gstreamer/cerbero" mkdir -p "../../${CI_PROJECT_NAMESPACE}" ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + + # Make sure there isn't a pre-existing config hanging around + rm -v -f localconf.cbc + rm -v -f ${CERBERO_HOST_DIR}/localconf.cbc + + if [[ $CONFIG == win??.cbc ]]; then + # For windows hardcode the path so it doesn't get + # mangled by msys path handling + # FIXME: make the sources point to pwd/$CI_PROJECT_DIR like the rest + echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc + echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc + + # Visual Studio 2017 build tools install path + echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc + echo 'vs_install_version = "vs15"' >> localconf.cbc + else + echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" > localconf.cbc + echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc + fi + + cat localconf.cbc + rsync -aH "${CERBERO_HOST_DIR}" . - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc + + cat localconf.cbc + + # FIXME: if you comment out this line it fails like so, no clue why. Its not windows defender either. + # From https://gitlab.freedesktop.org/gstreamer/cerbero + # b02080cb..d6923e42 master -> origin/master + # Fetching origin + # error: unable to create file cerbero-uninstalled: Permission denied + # fatal: Could not reset index file to revision 'd6923e4216c8a17759527a3db070d15cf7ff10a0'. + # ERROR: Failed to proceed with self update Command Error: Running ['git', 'reset', '--hard', 'd6923e4216c8a17759527a3db070d15cf7ff10a0'] returned 128 + git status + ./cerbero-uninstalled --self-update manifest.xml } diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index bcdb6fb..90a3178 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -45,11 +45,11 @@ variables: TEST_MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$TEST_MANIFEST_AMD64_SUFFIX:$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v11-master' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v12-master' # FIXME: after merging the script # CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" - CERBERO_SCRIPT_URL: 'https://gitlab.freedesktop.org/alatiera/gst-ci/-/raw/alatiera/cerbero-script-2/gitlab/cerbero_setup.sh' + CERBERO_SCRIPT_URL: 'https://gitlab.freedesktop.org/alatiera/gst-ci/-/raw/alatiera/cerbero-windows-attempt-6/gitlab/cerbero_setup.sh' GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both @@ -1178,3 +1178,46 @@ build gst-omx tizonia fedora x86_64: - ninja -C build - ninja -C build install - cd .. + +# +# Cerbero Native Windows builds +# +.cerbero windows native: + needs: ['manifest'] + image: $WINDOWS_IMAGE + tags: + - 'docker' + - 'windows' + - 'gstreamer-windows' + - '1809' + variables: + CONFIG: 'win64.cbc' + ARCH: 'msvc_x86_64' + # rsync doesn't like windows paths like `C:/cerbero` + CERBERO_HOST_DIR: "/c/cerbero/" + CERBERO_HOME: "C:/cerbero-build" + CERBERO_SOURCES: "C:/cerbero-sources" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v visualstudio -v nowerror" + CERBERO_RUN_SUFFIX: ".exe" + HAVE_CCACHE: "" + # FIXME: for some reason the runner is hanging when trying to upload artifacts + artifacts: + paths: [] + before_script: + - $env:CI_PROJECT_DIR = $env:CI_PROJECT_DIR.replace('\', '/') + - $env:CERBERO_SCRIPTS_PATH = "$env:CI_PROJECT_DIR/cerbero_setup.sh" + + - Invoke-WebRequest -Uri $env:CERBERO_SCRIPT_URL -OutFile $env:CERBERO_SCRIPTS_PATH + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_before_script" + +cerbero deps msvc x86_64: + extends: ['.cerbero deps', '.cerbero windows native'] + script: + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_deps_script" + +build cerbero msvc x86_64: + extends: ['.cerbero', '.cerbero windows native'] + script: + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_script" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' -- 2.7.4