From ab6536682bdad1cff785e6ac3e7df0ea67436b8b Mon Sep 17 00:00:00 2001 From: Benjamin Tissoires Date: Thu, 14 Mar 2019 08:59:53 +0100 Subject: [PATCH] CI: heavily rework the container creation Now everybody gets to rebuild their own containers if there is a change (too old or change in the packages). This should allow the MR touching the package list to succeed. Removal of the container_check stage, we can just have this in a before_script. Removal of the manual creation of the containers, not needed as we better just increment _TAG. Signed-off-by: Benjamin Tissoires --- .gitlab-ci.yml | 277 +++++++++++++++++++++------------------------------------ 1 file changed, 99 insertions(+), 178 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c394f5..03120ab 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,17 +47,27 @@ variables: FREEBSD_BUILD_PKGS: 'meson' FREEBSD_PKGS: 'libepoll-shim libudev-devd libevdev libwacom gtk3 libmtdev ' ############################ end of package lists ############################# + + # these tags should be incremented each time the list of packages is updated + # changing these will force rebuilding the associated image + # Note: these tags have no meaning and are not tied to a particular + # libinput version + FEDORA_TAG: libinput-2 + UBUNTU_TAG: libinput-2 + ARCH_TAG: libinput-2 + FREEBSD_TAG: libinput-2 + + UPSTREAM_REPO: libinput/libinput + BUILDAH_IMAGE: $CI_REGISTRY/wayland/ci-templates/buildah:latest + FEDORA_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/fedora/$FEDORA_VERSION:$FEDORA_TAG + UBUNTU_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/ubuntu/$UBUNTU_VERSION:$UBUNTU_TAG + ARCH_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/arch/rolling:$ARCH_TAG + FREEBSD_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/freebsd/11.2:$FREEBSD_TAG + MESON_BUILDDIR: "build dir" NINJA_ARGS: 'test' MESON_ARGS: '' - # We do not use CI_PROJECT_NAMESPACE or CI_REGISTRY_IMAGE because we want - # forks to use these particular images - PROJECT_NAMESPACE: libinput - BUILDAH_IMAGE: $CI_REGISTRY/wayland/ci-templates/buildah:latest - FEDORA_CONTAINER_IMAGE: $CI_REGISTRY/$PROJECT_NAMESPACE/$CI_PROJECT_NAME/fedora/$FEDORA_VERSION - UBUNTU_CONTAINER_IMAGE: $CI_REGISTRY/$PROJECT_NAMESPACE/$CI_PROJECT_NAME/ubuntu/$UBUNTU_VERSION - ARCH_CONTAINER_IMAGE: $CI_REGISTRY/$PROJECT_NAMESPACE/$CI_PROJECT_NAME/arch/rolling - FREEBSD_CONTAINER_IMAGE: $CI_REGISTRY/$PROJECT_NAMESPACE/$CI_PROJECT_NAME/freebsd/11.2 + # Until we have a VM with full access, we cannot run the test suite runner SKIP_LIBINPUT_TEST_SUITE_RUNNER: 1 # udev isn't available/working properly in the containers @@ -80,122 +90,55 @@ variables: - meson configure "$MESON_BUILDDIR" - ninja -C "$MESON_BUILDDIR" $NINJA_ARGS -# special rule to not expose the container creation runners to other users -# than those who have set up the CI to push on the registry. -# Users who have write access to libinput/libinput will have write -# access to the registry, so the libinput/libinput is a catch-all for -# our core developers. -# -# we can add as many users as we want by adding a new line like: -# - $GITLAB_USER_LOGIN == "someone" -.restrict_container_creation: &restrict_container_creation - only: - variables: - # Note: this is a set of logical OR, not AND - - $CI_PROJECT_PATH == "libinput/libinput" - ################################################################# # # -# container check stage # +# container prep stage # # # ################################################################# -.container-check: - stage: container_check - image: $BUILDAH_IMAGE - script: - # get the full container image name (CURRENT_CONTAINER_IMAGE still has indirections) - - CONTAINER_IMAGE=$(eval echo "$CURRENT_CONTAINER_IMAGE") - - # get the date of the current image - - IMG_DATE=$(skopeo inspect docker://$CONTAINER_IMAGE | jq -r '.Created' | cut -dT -f1) - - - TODAY_SECS=$(date -u +%s) - - IMG_SECS=$(date -u --date="$IMG_DATE" +%s) - - echo "today $TODAY_SECS, image $IMG_SECS" - - echo "image age $(($TODAY_SECS - $IMG_SECS))s" - - # check if image is less than a week old - - test $(($IMG_SECS + 604800)) -gt $TODAY_SECS - - # export an artefact telling the next stage that the image is valid - - touch .img_ready - artifacts: - name: image-$CURRENT_CONTAINER_IMAGE-check - expire_in: 6 hrs - paths: - - .img_ready - allow_failure: true - <<: *restrict_container_creation - - -# TODO: check that the RPMS/DEBS are all in the current images - -fedora:28@container-check: - extends: .container-check - variables: - GIT_STRATEGY: none - FEDORA_VERSION: 28 - CURRENT_CONTAINER_IMAGE: $FEDORA_CONTAINER_IMAGE:latest +# +# This stage will recreate the container images only if the image +# is too old or if it is missing some dependencies. +# -fedora:29@container-check: - extends: .container-check - variables: - GIT_STRATEGY: none - FEDORA_VERSION: 29 - CURRENT_CONTAINER_IMAGE: $FEDORA_CONTAINER_IMAGE:latest +.check_if_older_than_a_week: &check_if_older_than_a_week + before_script: + # log in to the registry + - podman login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY -ubuntu:18.10@container-check: - extends: .container-check - variables: - GIT_STRATEGY: none - UBUNTU_VERSION: "18.10" - CURRENT_CONTAINER_IMAGE: $UBUNTU_CONTAINER_IMAGE:latest + # get the full container image name (DISTRIB_VERSION still has indirections) + - IMAGE=$(eval echo "$DISTRIB_FLAVOR/$DISTRIB_VERSION:$TAG") -ubuntu:18.04@container-check: - extends: .container-check - variables: - GIT_STRATEGY: none - UBUNTU_VERSION: "18.04" - CURRENT_CONTAINER_IMAGE: $UBUNTU_CONTAINER_IMAGE:latest + # check if our image is already in the current registry and get its date + - LOCAL_IMG_DATE=$(skopeo inspect docker://$CI_REGISTRY_IMAGE/$IMAGE | jq -r '.Created' | cut -dT -f1 || + echo 1970-01-01) -arch:rolling@container-check: - extends: .container-check - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $ARCH_CONTAINER_IMAGE:latest + # get the date of the upstream image + - UPSTREAM_IMG_DATE=$(skopeo inspect docker://$CI_REGISTRY/$UPSTREAM_REPO/$IMAGE | jq -r '.Created' | cut -dT -f1 || + echo 1970-01-01) -freebsd:11.2@container-check: - extends: .container-check - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $FREEBSD_CONTAINER_IMAGE:latest + - TODAY_SECS=$(date -u +%s) + - LOCAL_IMG_SECS=$(date -u --date="$LOCAL_IMG_DATE" +%s) + - UPSTREAM_IMG_SECS=$(date -u --date="$UPSTREAM_IMG_DATE" +%s) + - echo "today $TODAY_SECS, local image $LOCAL_IMG_SECS, upstream image $UPSTREAM_IMG_SECS" + - echo "image ages $(($TODAY_SECS - $LOCAL_IMG_SECS))s and $(($TODAY_SECS - $UPSTREAM_IMG_SECS))s" + + # if the upstream image is more recent, use it + - if [[ $UPSTREAM_IMG_SECS -gt $LOCAL_IMG_SECS ]] ; + then + skopeo copy docker://$CI_REGISTRY/$UPSTREAM_REPO/$IMAGE + docker://$CI_REGISTRY_IMAGE/$IMAGE ; + LOCAL_IMG_SECS=UPSTREAM_IMG_SECS ; + fi -################################################################# -# # -# container prep stage # -# # -################################################################# + # check if image is less than a week old + - test $(($LOCAL_IMG_SECS + 604800)) -gt $TODAY_SECS && exit 0 -# -# This stage will recreate the container images only if the previous -# stage had a build failure, i.e. the image is too old or if it is -# missing some dependencies. -# -.container-prep: +.fedora@container-prep: stage: container_prep image: $BUILDAH_IMAGE - before_script: - # if the check was successful, we just skip recreating the container image - - test -e .img_ready && exit 0 - - # log in to the registry - - podman login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - <<: *restrict_container_creation - -.fedora@container-prep: - extends: .container-prep + <<: *check_if_older_than_a_week script: - buildcntr=$(buildah from --quiet fedora:$FEDORA_VERSION) - buildah run $buildcntr dnf upgrade -y @@ -203,34 +146,37 @@ freebsd:11.2@container-check: - buildah run $buildcntr dnf clean all - buildah config --workingdir /app $buildcntr # tag the current container - - buildah commit --quiet $buildcntr $FEDORA_CONTAINER_IMAGE:latest + - buildah commit --quiet $buildcntr $FEDORA_CONTAINER_IMAGE # clean up the working container - buildah rm $buildcntr # push the container image to the libinput registry - - podman push --quiet $FEDORA_CONTAINER_IMAGE:latest - - skopeo copy docker://$FEDORA_CONTAINER_IMAGE:latest docker://$FEDORA_CONTAINER_IMAGE:$CI_JOB_ID + - podman push --quiet $FEDORA_CONTAINER_IMAGE + - skopeo copy docker://$FEDORA_CONTAINER_IMAGE docker://$CI_REGISTRY_IMAGE/fedora/$FEDORA_VERSION:$CI_JOB_ID fedora:28@container-prep: extends: .fedora@container-prep variables: GIT_STRATEGY: none FEDORA_VERSION: 28 - dependencies: - # Note: we can not use $FEDORA_VERSION here - - fedora:28@container-check + DISTRIB_FLAVOR: fedora + DISTRIB_VERSION: $FEDORA_VERSION + TAG: $FEDORA_TAG fedora:29@container-prep: extends: .fedora@container-prep variables: GIT_STRATEGY: none FEDORA_VERSION: 29 - dependencies: - # Note: we can not use $FEDORA_VERSION here - - fedora:29@container-check + DISTRIB_FLAVOR: fedora + DISTRIB_VERSION: $FEDORA_VERSION + TAG: $FEDORA_TAG + .ubuntu@container-prep: - extends: .container-prep + stage: container_prep + image: $BUILDAH_IMAGE + <<: *check_if_older_than_a_week script: - buildcntr=$(buildah from --quiet ubuntu:$UBUNTU_VERSION) - buildah run $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get update @@ -241,34 +187,37 @@ fedora:29@container-prep: - buildah run $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get clean - buildah config --workingdir /app $buildcntr # tag the current container - - buildah commit --quiet $buildcntr $UBUNTU_CONTAINER_IMAGE:latest + - buildah commit --quiet $buildcntr $UBUNTU_CONTAINER_IMAGE # clean up the working container - buildah rm $buildcntr # push the container image to the libinput registry - - podman push --quiet $UBUNTU_CONTAINER_IMAGE:latest - - skopeo copy docker://$UBUNTU_CONTAINER_IMAGE:latest docker://$UBUNTU_CONTAINER_IMAGE:$CI_JOB_ID + - podman push --quiet $UBUNTU_CONTAINER_IMAGE + - skopeo copy docker://$UBUNTU_CONTAINER_IMAGE docker://$CI_REGISTRY_IMAGE/ubuntu/$UBUNTU_VERSION:$CI_JOB_ID ubuntu:18.10@container-prep: extends: .ubuntu@container-prep variables: GIT_STRATEGY: none UBUNTU_VERSION: "18.10" - dependencies: - # Note: we can not use $UBUNTU_VERSION here - - ubuntu:18.10@container-check + DISTRIB_FLAVOR: ubuntu + DISTRIB_VERSION: $UBUNTU_VERSION + TAG: $UBUNTU_TAG ubuntu:18.04@container-prep: extends: .ubuntu@container-prep variables: GIT_STRATEGY: none UBUNTU_VERSION: "18.04" - dependencies: - # Note: we can not use $UBUNTU_VERSION here - - ubuntu:18.04@container-check + DISTRIB_FLAVOR: ubuntu + DISTRIB_VERSION: $UBUNTU_VERSION + TAG: $UBUNTU_TAG + .arch@container-prep: - extends: .container-prep + stage: container_prep + image: $BUILDAH_IMAGE + <<: *check_if_older_than_a_week script: - buildcntr=$(buildah from --quiet archlinux/base) - buildah run $buildcntr pacman -S --refresh @@ -279,23 +228,27 @@ ubuntu:18.04@container-prep: - buildah run $buildcntr pacman -S --clean --noconfirm - buildah config --workingdir /app $buildcntr # tag the current container - - buildah commit --quiet $buildcntr $ARCH_CONTAINER_IMAGE:latest + - buildah commit --quiet $buildcntr $ARCH_CONTAINER_IMAGE # clean up the working container - buildah rm $buildcntr # push the container image to the libinput registry - - podman push --quiet $ARCH_CONTAINER_IMAGE:latest - - skopeo copy docker://$ARCH_CONTAINER_IMAGE:latest docker://$ARCH_CONTAINER_IMAGE:$CI_JOB_ID + - podman push --quiet $ARCH_CONTAINER_IMAGE + - skopeo copy docker://$ARCH_CONTAINER_IMAGE docker://$CI_REGISTRY_IMAGE/arch/rolling:$CI_JOB_ID arch:rolling@container-prep: extends: .arch@container-prep variables: GIT_STRATEGY: none - dependencies: - - arch:rolling@container-check + ARCH_VERSION: rolling + DISTRIB_FLAVOR: arch + DISTRIB_VERSION: $ARCH_VERSION + TAG: $ARCH_TAG .freebsd@container-prep: - extends: .container-prep + stage: container_prep + image: $BUILDAH_IMAGE + <<: *check_if_older_than_a_week script: - buildcntr=$(buildah from --quiet myfreeweb/freebsd-cross:latest) - buildah run $buildcntr apk add --no-cache $FREEBSD_BUILD_PKGS @@ -303,54 +256,23 @@ arch:rolling@container-prep: - buildah run $buildcntr pkg -r /freebsd install -y $FREEBSD_PKGS - buildah config --workingdir /app $buildcntr # tag the current container - - buildah commit --quiet $buildcntr $FREEBSD_CONTAINER_IMAGE:latest + - buildah commit --quiet $buildcntr $FREEBSD_CONTAINER_IMAGE # clean up the working container - buildah rm $buildcntr # push the container image to the libinput registry - - podman push --quiet $FREEBSD_CONTAINER_IMAGE:latest - - skopeo copy docker://$FREEBSD_CONTAINER_IMAGE:latest docker://$FREEBSD_CONTAINER_IMAGE:$CI_JOB_ID + - podman push --quiet $FREEBSD_CONTAINER_IMAGE + - skopeo copy docker://$FREEBSD_CONTAINER_IMAGE docker://$CI_REGISTRY_IMAGE/freebsd/$FREEBSD_VERSION:$CI_JOB_ID freebsd:11.2@container-prep: extends: .freebsd@container-prep variables: GIT_STRATEGY: none - dependencies: - # Note: we can not use $FREEBSD_VERSION here - - freebsd:11.2@container-check + FREEBSD_VERSION: "11.2" + DISTRIB_FLAVOR: freebsd + DISTRIB_VERSION: $FREEBSD_VERSION + TAG: $FREEBSD_TAG -# Add some manual runners to be able to recreate the cache on a day -# the list of the rpms changed - -fedora:28@force-container-prep: - extends: fedora:28@container-prep - when: manual - dependencies: [] - -fedora:29@force-container-prep: - extends: fedora:29@container-prep - when: manual - dependencies: [] - -ubuntu:18.10@force-container-prep: - extends: ubuntu:18.10@container-prep - when: manual - dependencies: [] - -ubuntu:18.04@force-container-prep: - extends: ubuntu:18.04@container-prep - when: manual - dependencies: [] - -arch:rolling@force-container-prep: - extends: arch:rolling@container-prep - when: manual - dependencies: [] - -freebsd:11.2@force-container-prep: - extends: freebsd:11.2@container-prep - when: manual - dependencies: [] ################################################################# # # @@ -413,7 +335,6 @@ freebsd:11.2@force-container-prep: ;done dependencies: [] allow_failure: true - <<: *restrict_container_creation fedora:28@container-clean: extends: .container-clean @@ -473,7 +394,7 @@ freebsd:11.2@container-clean: .fedora-build@template: extends: .build@template - image: $FEDORA_CONTAINER_IMAGE:latest + image: $FEDORA_CONTAINER_IMAGE fedora:28@default-build: extends: .fedora-build@template @@ -584,7 +505,7 @@ fedora:29@valgrind: .ubuntu@template: extends: .build@template - image: $UBUNTU_CONTAINER_IMAGE:latest + image: $UBUNTU_CONTAINER_IMAGE ubuntu:18.10@default-build: extends: .ubuntu@template @@ -601,7 +522,7 @@ ubuntu:18.04@default-build: # .arch@template: extends: .build@template - image: $ARCH_CONTAINER_IMAGE:latest + image: $ARCH_CONTAINER_IMAGE arch:rolling@default-build: extends: .arch@template @@ -611,7 +532,7 @@ arch:rolling@default-build: # .freebsd@template: extends: .build@template - image: $FREEBSD_CONTAINER_IMAGE:latest + image: $FREEBSD_CONTAINER_IMAGE variables: MESON_ARGS: '--cross-file freebsd -Ddocumentation=false -Dtests=false -Depoll-dir=/freebsd/usr/local/' # Can't run FreeBSD tests on Linux machine, so NINJA_ARGS shouldn't be "test" -- 2.7.4