.gitlab-ci/container/lava_build.sh

   1 #!/bin/bash
   2
   3 set -e
   4 set -o xtrace
   5
   6 check_minio()
   7 {
   8     MINIO_PATH="${MINIO_HOST}/mesa-lava/$1/${DISTRIBUTION_TAG}/${DEBIAN_ARCH}"
   9     if wget -q --method=HEAD "https://${MINIO_PATH}/done"; then
  10         exit
  11     fi
  12 }
  13
  14 # If remote files are up-to-date, skip rebuilding them
  15 check_minio "${FDO_UPSTREAM_REPO}"
  16 check_minio "${CI_PROJECT_PATH}"
  17
  18 . .gitlab-ci/container/container_pre_build.sh
  19
  20 # Install rust, which we'll be using for deqp-runner.  It will be cleaned up at the end.
  21 . .gitlab-ci/build-rust.sh
  22
  23 if [[ "$DEBIAN_ARCH" = "arm64" ]]; then
  24     GCC_ARCH="aarch64-linux-gnu"
  25     KERNEL_ARCH="arm64"
  26     DEFCONFIG="arch/arm64/configs/defconfig"
  27     DEVICE_TREES="arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dtb arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dtb arch/arm64/boot/dts/allwinner/sun50i-h6-pine-h64.dtb arch/arm64/boot/dts/amlogic/meson-gxm-khadas-vim2.dtb arch/arm64/boot/dts/qcom/apq8016-sbc.dtb arch/arm64/boot/dts/amlogic/meson-g12b-a311d-khadas-vim3.dtb"
  28     KERNEL_IMAGE_NAME="Image"
  29 elif [[ "$DEBIAN_ARCH" = "armhf" ]]; then
  30     GCC_ARCH="arm-linux-gnueabihf"
  31     KERNEL_ARCH="arm"
  32     DEFCONFIG="arch/arm/configs/multi_v7_defconfig"
  33     DEVICE_TREES="arch/arm/boot/dts/rk3288-veyron-jaq.dtb arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dtb"
  34     KERNEL_IMAGE_NAME="zImage"
  35     . .gitlab-ci/create-cross-file.sh armhf
  36 else
  37     GCC_ARCH="x86_64-linux-gnu"
  38     KERNEL_ARCH="x86_64"
  39     DEFCONFIG="arch/x86/configs/x86_64_defconfig"
  40     DEVICE_TREES=""
  41     KERNEL_IMAGE_NAME="bzImage"
  42 fi
  43
  44 # Determine if we're in a cross build.
  45 if [[ -e /cross_file-$DEBIAN_ARCH.txt ]]; then
  46     EXTRA_MESON_ARGS="--cross-file /cross_file-$DEBIAN_ARCH.txt"
  47     EXTRA_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=/toolchain-$DEBIAN_ARCH.cmake"
  48
  49     if [ $DEBIAN_ARCH = arm64 ]; then
  50         RUST_TARGET="aarch64-unknown-linux-gnu"
  51     elif [ $DEBIAN_ARCH = armhf ]; then
  52         RUST_TARGET="armv7-unknown-linux-gnueabihf"
  53     fi
  54     rustup target add $RUST_TARGET
  55     export EXTRA_CARGO_ARGS="--target $RUST_TARGET"
  56
  57     export ARCH=${KERNEL_ARCH}
  58     export CROSS_COMPILE="${GCC_ARCH}-"
  59 fi
  60
  61 apt-get update
  62 apt-get install -y automake \
  63                    bc \
  64                    cmake \
  65                    debootstrap \
  66                    git \
  67                    libboost-dev \
  68                    libegl1-mesa-dev \
  69                    libgbm-dev \
  70                    libgles2-mesa-dev \
  71                    libpcre3-dev \
  72                    libpng-dev \
  73                    libpython3-dev \
  74                    libssl-dev \
  75                    libvulkan-dev \
  76                    libwaffle-dev \
  77                    libxcb-keysyms1-dev \
  78                    libxkbcommon-dev \
  79                    patch \
  80                    python3-dev \
  81                    python3-distutils \
  82                    python3-mako \
  83                    python3-numpy \
  84                    python3-serial \
  85                    qt5-default \
  86                    qt5-qmake \
  87                    qtbase5-dev \
  88                    wget
  89
  90
  91 if [[ "$DEBIAN_ARCH" = "armhf" ]]; then
  92     apt-get install -y libboost-dev:armhf \
  93                        libegl1-mesa-dev:armhf \
  94                        libelf-dev:armhf \
  95                        libgbm-dev:armhf \
  96                        libgles2-mesa-dev:armhf \
  97                        libpcre3-dev:armhf \
  98                        libpng-dev:armhf \
  99                        libpython3-dev:armhf \
 100                        libvulkan-dev:armhf \
 101                        libwaffle-dev:armhf \
 102                        libxcb-keysyms1-dev:armhf \
 103                        libxkbcommon-dev:armhf \
 104                        qtbase5-dev:armhf
 105 fi
 106
 107
 108 ############### Building
 109 STRIP_CMD="${GCC_ARCH}-strip"
 110 mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}
 111
 112
 113 ############### Build dEQP runner
 114 . .gitlab-ci/build-deqp-runner.sh
 115 mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/usr/bin
 116 mv /usr/local/bin/deqp-runner /lava-files/rootfs-${DEBIAN_ARCH}/usr/bin/.
 117
 118
 119 ############### Build dEQP
 120 DEQP_TARGET=surfaceless . .gitlab-ci/build-deqp.sh
 121
 122 mv /deqp /lava-files/rootfs-${DEBIAN_ARCH}/.
 123
 124
 125 ############### Build piglit
 126 if [ -n "$INCLUDE_PIGLIT" ]; then
 127     . .gitlab-ci/build-piglit.sh
 128     mv /piglit /lava-files/rootfs-${DEBIAN_ARCH}/.
 129 fi
 130
 131
 132 ############### Build apitrace
 133 . .gitlab-ci/build-apitrace.sh
 134 mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
 135 mv /apitrace/build /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
 136 rm -rf /apitrace
 137
 138 mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/waffle
 139 mv /waffle/build /lava-files/rootfs-${DEBIAN_ARCH}/waffle
 140 rm -rf /waffle
 141
 142
 143 ############### Build renderdoc
 144 EXTRA_CMAKE_ARGS+=" -DENABLE_XCB=false"
 145 . .gitlab-ci/build-renderdoc.sh
 146 mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/renderdoc
 147 mv /renderdoc/build /lava-files/rootfs-${DEBIAN_ARCH}/renderdoc
 148 rm -rf /renderdoc
 149
 150
 151 ############### Build libdrm
 152 EXTRA_MESON_ARGS+=" -D prefix=/libdrm"
 153 . .gitlab-ci/build-libdrm.sh
 154
 155
 156 ############### Cross-build kernel
 157 mkdir -p kernel
 158 wget -qO- ${KERNEL_URL} | tar -xz --strip-components=1 -C kernel
 159 pushd kernel
 160
 161 # The kernel doesn't like the gold linker (or the old lld in our debians).
 162 # Sneak in some override symlinks during kernel build until we can update
 163 # debian (they'll get blown away by the rm of the kernel dir at the end).
 164 mkdir -p ld-links
 165 for i in /usr/bin/*-ld /usr/bin/ld; do
 166     i=`basename $i`
 167     ln -sf /usr/bin/$i.bfd ld-links/$i
 168 done
 169 export PATH=`pwd`/ld-links:$PATH
 170
 171 if [ -n "$INSTALL_KERNEL_MODULES" ]; then
 172     # Disable all modules in defconfig, so we only build the ones we want
 173     sed -i 's/=m/=n/g' ${DEFCONFIG}
 174 fi
 175
 176 ./scripts/kconfig/merge_config.sh ${DEFCONFIG} ../.gitlab-ci/${KERNEL_ARCH}.config
 177 make ${KERNEL_IMAGE_NAME}
 178 for image in ${KERNEL_IMAGE_NAME}; do
 179     cp arch/${KERNEL_ARCH}/boot/${image} /lava-files/.
 180 done
 181
 182 if [[ -n ${DEVICE_TREES} ]]; then
 183     make dtbs
 184     cp ${DEVICE_TREES} /lava-files/.
 185 fi
 186
 187 if [ -n "$INSTALL_KERNEL_MODULES" ]; then
 188     make modules
 189     INSTALL_MOD_PATH=/lava-files/rootfs-${DEBIAN_ARCH}/ make modules_install
 190 fi
 191
 192 if [[ ${DEBIAN_ARCH} = "arm64" ]] && which mkimage > /dev/null; then
 193     make Image.lzma
 194     mkimage \
 195         -f auto \
 196         -A arm \
 197         -O linux \
 198         -d arch/arm64/boot/Image.lzma \
 199         -C lzma\
 200         -b arch/arm64/boot/dts/qcom/sdm845-cheza-r3.dtb \
 201         /lava-files/cheza-kernel
 202 fi
 203
 204 popd
 205 rm -rf kernel
 206
 207 ############### Delete rust, since the tests won't be compiling anything.
 208 rm -rf /root/.rustup /root/.cargo
 209
 210 ############### Create rootfs
 211 set +e
 212 debootstrap \
 213     --variant=minbase \
 214     --arch=${DEBIAN_ARCH} \
 215      --components main,contrib,non-free \
 216     buster \
 217     /lava-files/rootfs-${DEBIAN_ARCH}/ \
 218     http://deb.debian.org/debian
 219
 220 cat /lava-files/rootfs-${DEBIAN_ARCH}/debootstrap/debootstrap.log
 221 set -e
 222
 223 cp .gitlab-ci/create-rootfs.sh /lava-files/rootfs-${DEBIAN_ARCH}/.
 224 cp .gitlab-ci/container/llvm-snapshot.gpg.key /lava-files/rootfs-${DEBIAN_ARCH}/.
 225 chroot /lava-files/rootfs-${DEBIAN_ARCH} \
 226     sh -c "INCLUDE_PIGLIT=$INCLUDE_PIGLIT sh /create-rootfs.sh"
 227 rm /lava-files/rootfs-${DEBIAN_ARCH}/create-rootfs.sh
 228 rm /lava-files/rootfs-${DEBIAN_ARCH}/llvm-snapshot.gpg.key
 229
 230
 231 ############### Install the built libdrm
 232 # Dependencies pulled during the creation of the rootfs may overwrite
 233 # the built libdrm. Hence, we add it after the rootfs has been already
 234 # created.
 235 mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/usr/lib/$GCC_ARCH
 236 find /libdrm/ -name lib\*\.so\* | xargs cp -t /lava-files/rootfs-${DEBIAN_ARCH}/usr/lib/$GCC_ARCH/.
 237 rm -rf /libdrm
 238
 239
 240 du -ah /lava-files/rootfs-${DEBIAN_ARCH} | sort -h | tail -100
 241 pushd /lava-files/rootfs-${DEBIAN_ARCH}
 242   tar czf /lava-files/lava-rootfs.tgz .
 243 popd
 244
 245 if [ ${DEBIAN_ARCH} = arm64 ]; then
 246     # Pull down a specific build of qcomlt/release/qcomlt-5.4 8c79b3d12355
 247     # ("Merge tag 'v5.4.23' into release/qcomlt-5.4"), where I used the
 248     # .config from
 249     # http://snapshots.linaro.org/96boards/dragonboard820c/linaro/debian/457/config-5.4.0-qcomlt-arm64
 250     # with the following merged in:
 251     #
 252     # CONFIG_DRM=y
 253     # CONFIG_DRM_MSM=y
 254     # CONFIG_ATL1C=y
 255     #
 256     # Reason: 5.5 has a big stack of oopses and warns on db820c.  4.14-5.4
 257     # linaro kernel binaries (see above .config link) have these as modules
 258     # and distributed the modules only in the debian system, not the initrd,
 259     # so they're very hard to extract (involving simg2img and loopback
 260     # mounting).  4.11 is missing d72fea538fe6 ("drm/msm: Fix the check for
 261     # the command size") so it can't actually run fredreno.  qcomlt-4.14 is
 262     # unstable at boot (~10% instaboot rate).  The 5.4 qcomlt kernel with msm
 263     # built in seems like the easiest way to go.
 264     wget https://people.freedesktop.org/~anholt/qcomlt-5.4-msm-build/Image.gz -O Image.gz \
 265          -O /lava-files/db820c-kernel
 266     wget https://people.freedesktop.org/~anholt/qcomlt-5.4-msm-build/apq8096-db820c.dtb \
 267          -O /lava-files/db820c.dtb
 268
 269     # Make a gzipped copy of the Image for db410c.
 270     gzip -k /lava-files/Image
 271
 272     # Add missing a630 firmware, added to debian packge in apr 2020
 273     wget https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/qcom/a630_gmu.bin \
 274          -O /lava-files/rootfs-arm64/lib/firmware/qcom/a630_gmu.bin
 275     wget https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/qcom/a630_sqe.fw \
 276          -O /lava-files/rootfs-arm64/lib/firmware/qcom/a630_sqe.fw
 277 fi
 278
 279 . .gitlab-ci/container/container_post_build.sh
 280
 281 ############### Upload the files!
 282 if [ -n "$UPLOAD_FOR_LAVA" ]; then
 283     ci-fairy minio login $CI_JOB_JWT
 284     FILES_TO_UPLOAD="lava-rootfs.tgz \
 285                      $KERNEL_IMAGE_NAME"
 286
 287     if [[ -n $DEVICE_TREES ]]; then
 288         FILES_TO_UPLOAD="$FILES_TO_UPLOAD $(basename -a $DEVICE_TREES)"
 289     fi
 290
 291     for f in $FILES_TO_UPLOAD; do
 292         ci-fairy minio cp /lava-files/$f \
 293             minio://${MINIO_PATH}/$f
 294     done
 295
 296     touch /lava-files/done
 297     ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
 298 fi
 299