ci: Uprev deqp-runner and use "suite" support to merge softpipe runs.

The deqp-runner "suite" support lets us specify all the deqp variants at
once and shard them across gitlab jobs, so we reduce docker overhead and
get fewer jobs to review the output of.

Acked-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Reviewed-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12110>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b150c44..b59c3e7 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -407,7 +407,7 @@ debian/android_build:
 debian/x86_test-base:
   extends: debian/x86_build-base
   variables:
-    MESA_IMAGE_TAG: &debian-x86_test-base "2021-07-02-bump-libdrm"
+    MESA_IMAGE_TAG: &debian-x86_test-base "2021-07-30-deqp-runner"
 
 .use-debian/x86_test-base:
   extends:
@@ -480,7 +480,7 @@ fedora/x86_build:
   variables:
     GIT_STRATEGY: fetch
     KERNEL_URL: *kernel-rootfs-url
-    MESA_ROOTFS_TAG: &kernel-rootfs "2021-08-02-rust-bump"
+    MESA_ROOTFS_TAG: &kernel-rootfs "2021-08-03-deqp-runner"
     DISTRIBUTION_TAG: &distribution-tag-arm "${MESA_ROOTFS_TAG}--${MESA_ARTIFACTS_TAG}--${MESA_TEMPLATES_COMMIT}"
   script:
     - .gitlab-ci/container/lava_build.sh

diff --git a/.gitlab-ci/container/build-deqp-runner.sh b/.gitlab-ci/container/build-deqp-runner.sh
index 532e71c..3c12f43 100644 (file)
--- a/.gitlab-ci/container/build-deqp-runner.sh
+++ b/.gitlab-ci/container/build-deqp-runner.sh
@@ -4,6 +4,6 @@ set -ex
 
 cargo install --locked deqp-runner \
   -j ${FDO_CI_CONCURRENT:-4} \
-  --version 0.7.2 \
+  --version 0.8.2 \
   --root /usr/local \
   $EXTRA_CARGO_ARGS

diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh
index e9c02f4..cc4a539 100755 (executable)
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@@ -47,47 +47,51 @@ HANG_DETECTION_CMD=""
 
 # Generate test case list file.
 if [ "$DEQP_VER" = "vk" ]; then
-   cp /deqp/mustpass/vk-$DEQP_VARIANT.txt /tmp/case-list.txt
+   MUSTPASS=/deqp/mustpass/vk-$DEQP_VARIANT.txt
    DEQP=/deqp/external/vulkancts/modules/vulkan/deqp-vk
    HANG_DETECTION_CMD="/parallel-deqp-runner/build/bin/hang-detection"
 elif [ "$DEQP_VER" = "gles2" -o "$DEQP_VER" = "gles3" -o "$DEQP_VER" = "gles31" -o "$DEQP_VER" = "egl" ]; then
-   cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt
+   MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
    DEQP=/deqp/modules/$DEQP_VER/deqp-$DEQP_VER
    SUITE=dEQP
 elif [ "$DEQP_VER" = "gles2-khr" -o "$DEQP_VER" = "gles3-khr" -o "$DEQP_VER" = "gles31-khr" -o "$DEQP_VER" = "gles32-khr" ]; then
-   cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt
+   MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
    DEQP=/deqp/external/openglcts/modules/glcts
    SUITE=dEQP
 else
-   cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt
+   MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
    DEQP=/deqp/external/openglcts/modules/glcts
    SUITE=KHR
 fi
 
-# If the caselist is too long to run in a reasonable amount of time, let the job
-# specify what fraction (1/n) of the caselist we should run.  Note: N~M is a gnu
-# sed extension to match every nth line (first line is #1).
-if [ -n "$DEQP_FRACTION" ]; then
-   sed -ni 1~$DEQP_FRACTION"p" /tmp/case-list.txt
-fi
+if [ -z "$DEQP_SUITE" ]; then
+    cp $MUSTPASS /tmp/case-list.txt
 
-# If the job is parallel at the gitab job level, take the corresponding fraction
-# of the caselist.
-if [ -n "$CI_NODE_INDEX" ]; then
-   sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
-fi
+    # If the caselist is too long to run in a reasonable amount of time, let the job
+    # specify what fraction (1/n) of the caselist we should run.  Note: N~M is a gnu
+    # sed extension to match every nth line (first line is #1).
+    if [ -n "$DEQP_FRACTION" ]; then
+       sed -ni 1~$DEQP_FRACTION"p" /tmp/case-list.txt
+    fi
 
-if [ -n "$DEQP_CASELIST_FILTER" ]; then
-    sed -ni "/$DEQP_CASELIST_FILTER/p" /tmp/case-list.txt
-fi
+    # If the job is parallel at the gitab job level, take the corresponding fraction
+    # of the caselist.
+    if [ -n "$CI_NODE_INDEX" ]; then
+       sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
+    fi
 
-if [ -n "$DEQP_CASELIST_INV_FILTER" ]; then
-    sed -ni "/$DEQP_CASELIST_INV_FILTER/!p" /tmp/case-list.txt
-fi
+    if [ -n "$DEQP_CASELIST_FILTER" ]; then
+        sed -ni "/$DEQP_CASELIST_FILTER/p" /tmp/case-list.txt
+    fi
+
+    if [ -n "$DEQP_CASELIST_INV_FILTER" ]; then
+        sed -ni "/$DEQP_CASELIST_INV_FILTER/!p" /tmp/case-list.txt
+    fi
 
-if [ ! -s /tmp/case-list.txt ]; then
-    echo "Caselist generation failed"
-    exit 1
+    if [ ! -s /tmp/case-list.txt ]; then
+        echo "Caselist generation failed"
+        exit 1
+    fi
 fi
 
 if [ -e "$INSTALL/deqp-$GPU_VERSION-fails.txt" ]; then
@@ -116,25 +120,6 @@ if [ -n "$DEQP_NO_SAVE_RESULTS" ]; then
    SUMMARY_LIMIT="--summary-limit 0"
 fi
 
-run_cts() {
-    deqp=$1
-    caselist=$2
-    output=$3
-    deqp-runner \
-        run \
-        --deqp $deqp \
-        --output $RESULTS \
-        --caselist $caselist \
-        --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \
-        --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
-        --testlog-to-xml  /deqp/executor/testlog-to-xml \
-        $JOB \
-        $SUMMARY_LIMIT \
-       $DEQP_RUNNER_OPTIONS \
-        -- \
-        $DEQP_OPTIONS
-}
-
 parse_renderer() {
     RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
     VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
@@ -214,7 +199,35 @@ FAILURES_CSV=$RESULTS/failures.csv
 
 export LD_PRELOAD=$TEST_LD_PRELOAD
 
-run_cts $DEQP /tmp/case-list.txt $RESULTS_CSV
+if [ -z "$DEQP_SUITE" ]; then
+    deqp-runner \
+        run \
+        --deqp $DEQP \
+        --output $RESULTS \
+        --caselist /tmp/case-list.txt \
+        --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \
+        --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
+        --testlog-to-xml /deqp/executor/testlog-to-xml \
+        $JOB \
+        $SUMMARY_LIMIT \
+       $DEQP_RUNNER_OPTIONS \
+        -- \
+        $DEQP_OPTIONS
+else
+    deqp-runner \
+        suite \
+        --suite $INSTALL/deqp-$DEQP_SUITE.toml \
+        --output $RESULTS \
+        --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \
+        --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
+        --testlog-to-xml /deqp/executor/testlog-to-xml \
+        --fraction-start $CI_NODE_INDEX \
+        --fraction $CI_NODE_TOTAL \
+        $JOB \
+        $SUMMARY_LIMIT \
+       $DEQP_RUNNER_OPTIONS
+fi
+
 DEQP_EXITCODE=$?
 
 export LD_PRELOAD=

diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index a9a47da..393329e 100755 (executable)
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -38,6 +38,7 @@ cp -Rp .gitlab-ci/deqp-*.txt install/
 cp -Rp .gitlab-ci/report-flakes.py install/
 cp -Rp .gitlab-ci/vkd3d-proton install/
 find . -path \*/ci/\*.txt \
+    -o -path \*/ci/\*.toml \
     -o -path \*/ci/\*traces\*.yml \
     | xargs -I '{}' cp -p '{}' install/
 

diff --git a/src/gallium/drivers/softpipe/ci/deqp-softpipe.toml b/src/gallium/drivers/softpipe/ci/deqp-softpipe.toml
new file mode 100644 (file)
index 0000000..a5995e6
--- /dev/null
+++ b/src/gallium/drivers/softpipe/ci/deqp-softpipe.toml
@@ -0,0 +1,51 @@
+[[deqp]]
+deqp = "/deqp/modules/gles2/deqp-gles2"
+caselists = ["/deqp/mustpass/gles2-master.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+
+[[deqp]]
+deqp = "/deqp/modules/gles3/deqp-gles3"
+caselists = ["/deqp/mustpass/gles3-master.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+
+[[deqp]]
+deqp = "/deqp/modules/gles31/deqp-gles31"
+caselists = ["/deqp/mustpass/gles31-master.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+
+# Note that KHR-GL3* test sets include all tests from the previous
+# version, so we only need to run one test list (unlike dEQP-GLES,
+# where the test sets are separate).
+[[deqp]]
+deqp = "/deqp/external/openglcts/modules/glcts"
+caselists = [
+    "/deqp/mustpass/gles2-khr-master.txt",
+    "/deqp/mustpass/gles3-khr-master.txt",
+    "/deqp/mustpass/gles31-khr-master.txt",
+    "/deqp/mustpass/gl33-master.txt",
+]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]

diff --git a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml
index 7f21bde..fdb925e 100644 (file)
--- a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml
@@ -1,38 +1,21 @@
-softpipe-gles2:
+.softpipe-deqp-test:
   extends:
-    - llvmpipe-gles2
     - .softpipe-rules
+    - .test-gl
+    - .deqp-test
   variables:
     GPU_VERSION: softpipe
+    LIBGL_ALWAYS_SOFTWARE: "true"
     GALLIUM_DRIVER: "softpipe"
     DEQP_EXPECTED_RENDERER: softpipe
     FLAKES_CHANNEL: "#mesa-swrast-ci"
 
-softpipe-gles3:
+softpipe-deqp:
   variables:
-    DEQP_VER: gles3
-  extends: softpipe-gles2
-
-softpipe-gles31:
+    DEQP_VER: gles2 # for the renderer check
+    DEQP_SUITE: softpipe
   parallel: 2
-  variables:
-    DEQP_VER: gles31
-  extends: softpipe-gles2
-
-softpipe-khr-gles2:
-  variables:
-    DEQP_VER: gles2-khr
-  extends: softpipe-gles2
-
-softpipe-khr-gles3:
-  variables:
-    DEQP_VER: gles3-khr
-  extends: softpipe-gles2
-
-softpipe-khr-gles31:
-  variables:
-    DEQP_VER: gles31-khr
-  extends: softpipe-gles2
+  extends: .softpipe-deqp-test
 
 softpipe-asan-gles31:
   variables:
@@ -40,7 +23,7 @@ softpipe-asan-gles31:
     DEQP_FRACTION: 10
     DEQP_VER: gles31
     TEST_LD_PRELOAD: libasan.so.6
-  extends: softpipe-gles2
+  extends: .softpipe-deqp-test
   needs:
     - debian/x86_test-gl
     - debian-testing-asan
@@ -86,12 +69,3 @@ softpipe-piglit-quick:
       -x glsl-uniform-interstage-limits
     PIGLIT_PROFILES: quick_gl quick_shader
     PIGLIT_RESULTS: softpipe-quick
-
-# Note that KHR-GL3* test sets include all tests from the previous
-# version, so we only need to run one test list (unlike dEQP-GLES,
-# where the test sets are separate).
-softpipe-gl:
-  variables:
-    DEQP_VER: gl33
-  extends:
-    - softpipe-gles2