From 2fe7f9f2aefc62bcd16e9bcbeaa0c9862d0b006a Mon Sep 17 00:00:00 2001 From: Tomeu Vizoso Date: Fri, 7 Jun 2019 10:20:28 +0200 Subject: [PATCH] panfrost: ci: Switch from direct Docker use to buildah MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Use the infrastructure in wayland/ci-templates to build the container images. This prevents from getting into some situations in which the images wouldn't be rebuilt, and allows us to share some infrastructure with other projects in freedesktop.org. Signed-off-by: Tomeu Vizoso Suggested-by: Michel Dänzer Acked-by: Alyssa Rosenzweig --- src/gallium/drivers/panfrost/ci/Dockerfile | 135 ---------------------- src/gallium/drivers/panfrost/ci/debian-install.sh | 125 ++++++++++++++++++++ src/gallium/drivers/panfrost/ci/gitlab-ci.yml | 66 +++++++---- 3 files changed, 166 insertions(+), 160 deletions(-) delete mode 100644 src/gallium/drivers/panfrost/ci/Dockerfile create mode 100644 src/gallium/drivers/panfrost/ci/debian-install.sh diff --git a/src/gallium/drivers/panfrost/ci/Dockerfile b/src/gallium/drivers/panfrost/ci/Dockerfile deleted file mode 100644 index fe0e67c..0000000 --- a/src/gallium/drivers/panfrost/ci/Dockerfile +++ /dev/null @@ -1,135 +0,0 @@ -FROM debian:testing - -ENV DEBIAN_FRONTEND=noninteractive - -RUN echo 'path-exclude=/usr/share/doc/*' > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft -RUN echo 'path-exclude=/usr/share/man/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft -RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d -RUN echo 'exit 101' >> /usr/sbin/policy-rc.d -RUN chmod +x /usr/sbin/policy-rc.d - -############### Install packages for building - -ARG DEBIAN_ARCH -RUN dpkg --add-architecture ${DEBIAN_ARCH} -RUN echo deb-src http://deb.debian.org/debian testing main >> /etc/apt/sources.list -RUN apt-get update && \ - apt-get -y install ca-certificates && \ - apt-get -y install --no-install-recommends \ - crossbuild-essential-${DEBIAN_ARCH} \ - meson \ - g++ \ - git \ - ccache \ - pkg-config \ - python3-mako \ - python-numpy \ - python-six \ - python-mako \ - python3-pip \ - python3-setuptools \ - python3-six \ - python3-wheel \ - python3-jinja2 \ - bison \ - flex \ - libwayland-dev \ - gettext \ - cmake \ - bc \ - libssl-dev \ - lavacli \ - csvkit \ - curl \ - unzip \ - wget \ - debootstrap \ - procps \ - qemu-user-static \ - cpio \ - \ - libdrm-dev:${DEBIAN_ARCH} \ - libx11-dev:${DEBIAN_ARCH} \ - libxxf86vm-dev:${DEBIAN_ARCH} \ - libexpat1-dev:${DEBIAN_ARCH} \ - libsensors-dev:${DEBIAN_ARCH} \ - libxfixes-dev:${DEBIAN_ARCH} \ - libxdamage-dev:${DEBIAN_ARCH} \ - libxext-dev:${DEBIAN_ARCH} \ - x11proto-dev:${DEBIAN_ARCH} \ - libx11-xcb-dev:${DEBIAN_ARCH} \ - libxcb-dri2-0-dev:${DEBIAN_ARCH} \ - libxcb-glx0-dev:${DEBIAN_ARCH} \ - libxcb-xfixes0-dev:${DEBIAN_ARCH} \ - libxcb-dri3-dev:${DEBIAN_ARCH} \ - libxcb-present-dev:${DEBIAN_ARCH} \ - libxcb-randr0-dev:${DEBIAN_ARCH} \ - libxcb-sync-dev:${DEBIAN_ARCH} \ - libxrandr-dev:${DEBIAN_ARCH} \ - libxshmfence-dev:${DEBIAN_ARCH} \ - libelf-dev:${DEBIAN_ARCH} \ - libwayland-dev:${DEBIAN_ARCH} \ - libwayland-egl-backend-dev:${DEBIAN_ARCH} \ - libclang-7-dev:${DEBIAN_ARCH} \ - zlib1g-dev:${DEBIAN_ARCH} \ - libglvnd-core-dev:${DEBIAN_ARCH} \ - wayland-protocols:${DEBIAN_ARCH} \ - libpng-dev:${DEBIAN_ARCH} && \ - rm -rf /var/lib/apt/lists - -############### Cross-build dEQP -ARG GCC_ARCH -RUN mkdir -p /artifacts/rootfs/deqp && \ - wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip && \ - unzip -q opengl-es-cts-3.2.5.0.zip -d / && \ - rm opengl-es-cts-3.2.5.0.zip && \ - cd /VK-GL-CTS-opengl-es-cts-3.2.5.0 && \ - python3 external/fetch_sources.py && \ - cd /artifacts/rootfs/deqp && \ - cmake -DDEQP_TARGET=wayland \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc \ - -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++ \ - /VK-GL-CTS-opengl-es-cts-3.2.5.0 && \ - make -j$(nproc) && \ - rm -rf /artifacts/rootfs/deqp/external && \ - rm -rf /artifacts/rootfs/deqp/modules/gles3 && \ - rm -rf /artifacts/rootfs/deqp/modules/gles31 && \ - rm -rf /artifacts/rootfs/deqp/modules/internal && \ - rm -rf /artifacts/rootfs/deqp/executor && \ - rm -rf /artifacts/rootfs/deqp/execserver && \ - rm -rf /artifacts/rootfs/deqp/modules/egl && \ - rm -rf /artifacts/rootfs/deqp/framework && \ - find . -name CMakeFiles | xargs rm -rf && \ - find . -name lib\*.a | xargs rm -rf && \ - du -sh * && \ - rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0 - - -############### Cross-build kernel - -ARG KERNEL_ARCH -ARG DEFCONFIG -ARG DEVICE_TREES -ARG KERNEL_IMAGE_NAME -ENV KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz" - -COPY ${KERNEL_ARCH}.config /panfrost-ci/ -RUN mkdir -p /kernel && \ - wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel && \ - cd /kernel && \ - ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" ./scripts/kconfig/merge_config.sh ${DEFCONFIG} /panfrost-ci/${KERNEL_ARCH}.config && \ - ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" make -j12 ${KERNEL_IMAGE_NAME} dtbs && \ - cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/. && \ - cp ${DEVICE_TREES} /artifacts/. && \ - rm -rf /kernel - - -############### Create rootfs - -COPY create-rootfs.sh /artifacts/rootfs/ -RUN debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian && \ - chroot /artifacts/rootfs sh /create-rootfs.sh && \ - rm /artifacts/rootfs/create-rootfs.sh - -ENTRYPOINT [""] diff --git a/src/gallium/drivers/panfrost/ci/debian-install.sh b/src/gallium/drivers/panfrost/ci/debian-install.sh new file mode 100644 index 0000000..8a8b4ec --- /dev/null +++ b/src/gallium/drivers/panfrost/ci/debian-install.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +set -e +set -o xtrace + +PANFROST_CI_DIR=/tmp/clone/src/gallium/drivers/panfrost/ci + +############### Install packages for building +dpkg --add-architecture ${DEBIAN_ARCH} +echo 'deb-src https://deb.debian.org/debian testing main' > /etc/apt/sources.list.d/deb-src.list +apt-get update +apt-get -y install ca-certificates +apt-get -y install --no-install-recommends \ + crossbuild-essential-${DEBIAN_ARCH} \ + meson \ + g++ \ + git \ + ccache \ + pkg-config \ + python3-mako \ + python-numpy \ + python-six \ + python-mako \ + python3-pip \ + python3-setuptools \ + python3-six \ + python3-wheel \ + python3-jinja2 \ + bison \ + flex \ + libwayland-dev \ + gettext \ + cmake \ + bc \ + libssl-dev \ + lavacli \ + csvkit \ + curl \ + unzip \ + wget \ + debootstrap \ + procps \ + qemu-user-static \ + cpio \ + \ + libdrm-dev:${DEBIAN_ARCH} \ + libx11-dev:${DEBIAN_ARCH} \ + libxxf86vm-dev:${DEBIAN_ARCH} \ + libexpat1-dev:${DEBIAN_ARCH} \ + libsensors-dev:${DEBIAN_ARCH} \ + libxfixes-dev:${DEBIAN_ARCH} \ + libxdamage-dev:${DEBIAN_ARCH} \ + libxext-dev:${DEBIAN_ARCH} \ + x11proto-dev:${DEBIAN_ARCH} \ + libx11-xcb-dev:${DEBIAN_ARCH} \ + libxcb-dri2-0-dev:${DEBIAN_ARCH} \ + libxcb-glx0-dev:${DEBIAN_ARCH} \ + libxcb-xfixes0-dev:${DEBIAN_ARCH} \ + libxcb-dri3-dev:${DEBIAN_ARCH} \ + libxcb-present-dev:${DEBIAN_ARCH} \ + libxcb-randr0-dev:${DEBIAN_ARCH} \ + libxcb-sync-dev:${DEBIAN_ARCH} \ + libxrandr-dev:${DEBIAN_ARCH} \ + libxshmfence-dev:${DEBIAN_ARCH} \ + libelf-dev:${DEBIAN_ARCH} \ + libwayland-dev:${DEBIAN_ARCH} \ + libwayland-egl-backend-dev:${DEBIAN_ARCH} \ + libclang-7-dev:${DEBIAN_ARCH} \ + zlib1g-dev:${DEBIAN_ARCH} \ + libglvnd-core-dev:${DEBIAN_ARCH} \ + wayland-protocols:${DEBIAN_ARCH} \ + libpng-dev:${DEBIAN_ARCH} + +############### Cross-build dEQP +mkdir -p /artifacts/rootfs/deqp + +wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip +unzip -q opengl-es-cts-3.2.5.0.zip -d / +rm opengl-es-cts-3.2.5.0.zip + +cd /VK-GL-CTS-opengl-es-cts-3.2.5.0 +python3 external/fetch_sources.py + +cd /artifacts/rootfs/deqp +cmake -DDEQP_TARGET=wayland \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc \ + -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++ \ + /VK-GL-CTS-opengl-es-cts-3.2.5.0 +make -j$(nproc) +rm -rf /artifacts/rootfs/deqp/external +rm -rf /artifacts/rootfs/deqp/modules/gles3 +rm -rf /artifacts/rootfs/deqp/modules/gles31 +rm -rf /artifacts/rootfs/deqp/modules/internal +rm -rf /artifacts/rootfs/deqp/executor +rm -rf /artifacts/rootfs/deqp/execserver +rm -rf /artifacts/rootfs/deqp/modules/egl +rm -rf /artifacts/rootfs/deqp/framework +find . -name CMakeFiles | xargs rm -rf +find . -name lib\*.a | xargs rm -rf +du -sh * +rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0 + + +############### Cross-build kernel +KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz" +export ARCH=${KERNEL_ARCH} +export CROSS_COMPILE="${GCC_ARCH}-" + +mkdir -p /kernel +wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel +cd /kernel +./scripts/kconfig/merge_config.sh ${DEFCONFIG} ${PANFROST_CI_DIR}/${KERNEL_ARCH}.config +make -j12 ${KERNEL_IMAGE_NAME} dtbs +cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/. +cp ${DEVICE_TREES} /artifacts/. +rm -rf /kernel + + +############### Create rootfs +cp ${PANFROST_CI_DIR}/create-rootfs.sh /artifacts/rootfs/. +debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian +chroot /artifacts/rootfs sh /create-rootfs.sh +rm /artifacts/rootfs/create-rootfs.sh + diff --git a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml index 3491b9f..d67c1ff 100644 --- a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml +++ b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml @@ -1,39 +1,55 @@ +# DEBIAN_TAG is the tag of the docker image used for the build jobs. If the +# image doesn't exist yet, the docker-image stage generates it. +# +# In order to generate a new image, one should generally change the tag. +# While removing the image from the registry would also work, that's not +# recommended except for ephemeral images during development: Replacing an +# image after a significant amount of time might pull in newer versions of +# gcc/clang or other packages, which might break the build with older commits +# using the same tag. +# +# After merging a change resulting in generating a new image to the main +# repository, it's recommended to remove the image from the source repository's +# container registry, so that the image from the main repository's registry +# will be used there as well. You can manage your images on your fork of: +# https://gitlab.freedesktop.org/xorg/xserver/container_registry variables: - DOCKER_DRIVER: overlay2 + UPSTREAM_REPO: mesa/mesa + DEBIAN_VERSION: testing-slim + DEBIAN_TAG: "2019-06-07-1" + +include: + - project: 'wayland/ci-templates' + ref: c73dae8b84697ef18e2dbbf4fed7386d9652b0cd + file: '/templates/debian.yml' stages: - containers - build - test +# Retry jobs after runner system failures +.retry: &retry + retry: + max: 2 + when: + - runner_system_failure + # Build Docker image with deqp, the rootfs and the build deps for Mesa .container: + extends: .debian@container-ifnot-exists stage: containers - image: docker:stable - only: - changes: - - src/gallium/drivers/panfrost/ci/gitlab-ci.yml - - src/gallium/drivers/panfrost/ci/Dockerfile - - src/gallium/drivers/panfrost/ci/create-rootfs.sh - - src/gallium/drivers/panfrost/ci/${KERNEL_ARCH}.config - services: - - docker:dind + <<: *retry variables: - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - time docker build - -t $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH} - --build-arg DEBIAN_ARCH=${DEBIAN_ARCH} - --build-arg GCC_ARCH=${GCC_ARCH} - --build-arg KERNEL_ARCH=${KERNEL_ARCH} - --build-arg DEFCONFIG=${DEFCONFIG} - --build-arg DEVICE_TREES=${DEVICE_TREES} - --build-arg KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME} - src/gallium/drivers/panfrost/ci - - time docker push $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH} - - docker history $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH} + GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image + IMAGE_LOCAL: "$CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}:latest" + DEBIAN_EXEC: 'DEBIAN_ARCH=${DEBIAN_ARCH} + GCC_ARCH=${GCC_ARCH} + KERNEL_ARCH=${KERNEL_ARCH} + DEFCONFIG=${DEFCONFIG} + DEVICE_TREES=${DEVICE_TREES} + KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME} + bash src/gallium/drivers/panfrost/ci/debian-install.sh' container:armhf: extends: .container -- 2.7.4