.gitlab-ci/deqp-runner.sh

   1 #!/bin/bash
   2
   3 echo -e "\e[0Ksection_start:$(date +%s):test_setup[collapsed=true]\r\e[0Kpreparing test setup"
   4
   5 set -ex
   6
   7 # Needed so configuration files can contain paths to files in /install
   8 ln -sf $CI_PROJECT_DIR/install /install
   9
  10 if [ -z "$GPU_VERSION" ]; then
  11    echo 'GPU_VERSION must be set to something like "llvmpipe" or "freedreno-a630" (the name used in .gitlab-ci/gpu-version-*.txt)'
  12    exit 1
  13 fi
  14
  15 INSTALL=`pwd`/install
  16
  17 # Set up the driver environment.
  18 export LD_LIBRARY_PATH=`pwd`/install/lib/
  19 export EGL_PLATFORM=surfaceless
  20 export VK_ICD_FILENAMES=`pwd`/install/share/vulkan/icd.d/"$VK_DRIVER"_icd.${VK_CPU:-`uname -m`}.json
  21
  22 RESULTS=`pwd`/${DEQP_RESULTS_DIR:-results}
  23 mkdir -p $RESULTS
  24
  25 # Ensure Mesa Shader Cache resides on tmpfs.
  26 SHADER_CACHE_HOME=${XDG_CACHE_HOME:-${HOME}/.cache}
  27 SHADER_CACHE_DIR=${MESA_SHADER_CACHE_DIR:-${SHADER_CACHE_HOME}/mesa_shader_cache}
  28
  29 findmnt -n tmpfs ${SHADER_CACHE_HOME} || findmnt -n tmpfs ${SHADER_CACHE_DIR} || {
  30     mkdir -p ${SHADER_CACHE_DIR}
  31     mount -t tmpfs -o nosuid,nodev,size=2G,mode=1755 tmpfs ${SHADER_CACHE_DIR}
  32 }
  33
  34 HANG_DETECTION_CMD=""
  35
  36 if [ -z "$DEQP_SUITE" ]; then
  37     if [ -z "$DEQP_VER" ]; then
  38         echo 'DEQP_SUITE must be set to the name of your deqp-gpu_version.toml, or DEQP_VER must be set to something like "gles2", "gles31-khr" or "vk" for the test run'
  39         exit 1
  40     fi
  41
  42     DEQP_WIDTH=${DEQP_WIDTH:-256}
  43     DEQP_HEIGHT=${DEQP_HEIGHT:-256}
  44     DEQP_CONFIG=${DEQP_CONFIG:-rgba8888d24s8ms0}
  45     DEQP_VARIANT=${DEQP_VARIANT:-master}
  46
  47     DEQP_OPTIONS="$DEQP_OPTIONS --deqp-surface-width=$DEQP_WIDTH --deqp-surface-height=$DEQP_HEIGHT"
  48     DEQP_OPTIONS="$DEQP_OPTIONS --deqp-surface-type=${DEQP_SURFACE_TYPE:-pbuffer}"
  49     DEQP_OPTIONS="$DEQP_OPTIONS --deqp-gl-config-name=$DEQP_CONFIG"
  50     DEQP_OPTIONS="$DEQP_OPTIONS --deqp-visibility=hidden"
  51
  52     if [ "$DEQP_VER" = "vk" -a -z "$VK_DRIVER" ]; then
  53         echo 'VK_DRIVER must be to something like "radeon" or "intel" for the test run'
  54         exit 1
  55     fi
  56
  57     # Generate test case list file.
  58     if [ "$DEQP_VER" = "vk" ]; then
  59        MUSTPASS=/deqp/mustpass/vk-$DEQP_VARIANT.txt
  60        DEQP=/deqp/external/vulkancts/modules/vulkan/deqp-vk
  61        HANG_DETECTION_CMD="/parallel-deqp-runner/build/bin/hang-detection"
  62     elif [ "$DEQP_VER" = "gles2" -o "$DEQP_VER" = "gles3" -o "$DEQP_VER" = "gles31" -o "$DEQP_VER" = "egl" ]; then
  63        MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
  64        DEQP=/deqp/modules/$DEQP_VER/deqp-$DEQP_VER
  65     elif [ "$DEQP_VER" = "gles2-khr" -o "$DEQP_VER" = "gles3-khr" -o "$DEQP_VER" = "gles31-khr" -o "$DEQP_VER" = "gles32-khr" ]; then
  66        MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
  67        DEQP=/deqp/external/openglcts/modules/glcts
  68     else
  69        MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
  70        DEQP=/deqp/external/openglcts/modules/glcts
  71     fi
  72
  73     cp $MUSTPASS /tmp/case-list.txt
  74
  75     # If the caselist is too long to run in a reasonable amount of time, let the job
  76     # specify what fraction (1/n) of the caselist we should run.  Note: N~M is a gnu
  77     # sed extension to match every nth line (first line is #1).
  78     if [ -n "$DEQP_FRACTION" ]; then
  79        sed -ni 1~$DEQP_FRACTION"p" /tmp/case-list.txt
  80     fi
  81
  82     # If the job is parallel at the gitab job level, take the corresponding fraction
  83     # of the caselist.
  84     if [ -n "$CI_NODE_INDEX" ]; then
  85        sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
  86     fi
  87
  88     if [ -n "$DEQP_CASELIST_FILTER" ]; then
  89         sed -ni "/$DEQP_CASELIST_FILTER/p" /tmp/case-list.txt
  90     fi
  91
  92     if [ -n "$DEQP_CASELIST_INV_FILTER" ]; then
  93         sed -ni "/$DEQP_CASELIST_INV_FILTER/!p" /tmp/case-list.txt
  94     fi
  95
  96     if [ ! -s /tmp/case-list.txt ]; then
  97         echo "Caselist generation failed"
  98         exit 1
  99     fi
 100 fi
 101
 102 if [ -e "$INSTALL/$GPU_VERSION-fails.txt" ]; then
 103     DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --baseline $INSTALL/$GPU_VERSION-fails.txt"
 104 fi
 105
 106 # Default to an empty known flakes file if it doesn't exist.
 107 touch $INSTALL/$GPU_VERSION-flakes.txt
 108
 109
 110 if [ -n "$VK_DRIVER" ] && [ -e "$INSTALL/$VK_DRIVER-skips.txt" ]; then
 111     DEQP_SKIPS="$DEQP_SKIPS $INSTALL/$VK_DRIVER-skips.txt"
 112 fi
 113
 114 if [ -n "$GALLIUM_DRIVER" ] && [ -e "$INSTALL/$GALLIUM_DRIVER-skips.txt" ]; then
 115     DEQP_SKIPS="$DEQP_SKIPS $INSTALL/$GALLIUM_DRIVER-skips.txt"
 116 fi
 117
 118 if [ -n "$DRIVER_NAME" ] && [ -e "$INSTALL/$DRIVER_NAME-skips.txt" ]; then
 119     DEQP_SKIPS="$DEQP_SKIPS $INSTALL/$DRIVER_NAME-skips.txt"
 120 fi
 121
 122 if [ -e "$INSTALL/$GPU_VERSION-skips.txt" ]; then
 123     DEQP_SKIPS="$DEQP_SKIPS $INSTALL/$GPU_VERSION-skips.txt"
 124 fi
 125
 126 report_load() {
 127     echo "System load: $(cut -d' ' -f1-3 < /proc/loadavg)"
 128     echo "# of CPU cores: $(cat /proc/cpuinfo | grep processor | wc -l)"
 129 }
 130
 131 # wrapper to supress +x to avoid spamming the log
 132 quiet() {
 133     set +x
 134     "$@"
 135     set -x
 136 }
 137
 138 if [ "$GALLIUM_DRIVER" = "virpipe" ]; then
 139     # deqp is to use virpipe, and virgl_test_server llvmpipe
 140     export GALLIUM_DRIVER="$GALLIUM_DRIVER"
 141
 142     VTEST_ARGS="--use-egl-surfaceless"
 143     if [ "$VIRGL_HOST_API" = "GLES" ]; then
 144         VTEST_ARGS="$VTEST_ARGS --use-gles"
 145     fi
 146
 147     GALLIUM_DRIVER=llvmpipe \
 148     virgl_test_server $VTEST_ARGS >$RESULTS/vtest-log.txt 2>&1 &
 149
 150     sleep 1
 151 fi
 152
 153 if [ -z "$DEQP_SUITE" ]; then
 154     if [ -n "$DEQP_EXPECTED_RENDERER" ]; then
 155         export DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --renderer-check "$DEQP_EXPECTED_RENDERER""
 156     fi
 157     if [ $DEQP_VER != vk -a $DEQP_VER != egl ]; then
 158         export DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --version-check `cat $INSTALL/VERSION | sed 's/[() ]/./g'`"
 159     fi
 160 fi
 161
 162 set +x
 163 echo -e "\e[0Ksection_end:$(date +%s):test_setup\r\e[0K"
 164
 165 echo -e "\e[0Ksection_start:$(date +%s):deqp[collapsed=false]\r\e[0Kdeqp-runner"
 166 set -x
 167
 168 set +e
 169 if [ -z "$DEQP_SUITE" ]; then
 170     deqp-runner \
 171         run \
 172         --deqp $DEQP \
 173         --output $RESULTS \
 174         --caselist /tmp/case-list.txt \
 175         --skips $INSTALL/all-skips.txt $DEQP_SKIPS \
 176         --flakes $INSTALL/$GPU_VERSION-flakes.txt \
 177         --testlog-to-xml /deqp/executor/testlog-to-xml \
 178         --jobs ${FDO_CI_CONCURRENT:-4} \
 179         $DEQP_RUNNER_OPTIONS \
 180         -- \
 181         $DEQP_OPTIONS
 182 else
 183     deqp-runner \
 184         suite \
 185         --suite $INSTALL/deqp-$DEQP_SUITE.toml \
 186         --output $RESULTS \
 187         --skips $INSTALL/all-skips.txt $DEQP_SKIPS \
 188         --flakes $INSTALL/$GPU_VERSION-flakes.txt \
 189         --testlog-to-xml /deqp/executor/testlog-to-xml \
 190         --fraction-start $CI_NODE_INDEX \
 191         --fraction `expr $CI_NODE_TOTAL \* ${DEQP_FRACTION:-1}` \
 192         --jobs ${FDO_CI_CONCURRENT:-4} \
 193         $DEQP_RUNNER_OPTIONS
 194 fi
 195
 196 DEQP_EXITCODE=$?
 197
 198 set +x
 199 echo -e "\e[0Ksection_end:$(date +%s):deqp\r\e[0K"
 200
 201 report_load
 202
 203 echo -e "\e[0Ksection_start:$(date +%s):test_post_process[collapsed=true]\r\e[0Kpost-processing test results"
 204 set -x
 205
 206 # Remove all but the first 50 individual XML files uploaded as artifacts, to
 207 # save fd.o space when you break everything.
 208 find $RESULTS -name \*.xml | \
 209     sort -n |
 210     sed -n '1,+49!p' | \
 211     xargs rm -f
 212
 213 # If any QPA XMLs are there, then include the XSL/CSS in our artifacts.
 214 find $RESULTS -name \*.xml \
 215     -exec cp /deqp/testlog.css /deqp/testlog.xsl "$RESULTS/" ";" \
 216     -quit
 217
 218 deqp-runner junit \
 219    --testsuite dEQP \
 220    --results $RESULTS/failures.csv \
 221    --output $RESULTS/junit.xml \
 222    --limit 50 \
 223    --template "See https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml"
 224
 225 # Report the flakes to the IRC channel for monitoring (if configured):
 226 if [ -n "$FLAKES_CHANNEL" ]; then
 227   python3 $INSTALL/report-flakes.py \
 228          --host irc.oftc.net \
 229          --port 6667 \
 230          --results $RESULTS/results.csv \
 231          --known-flakes $INSTALL/$GPU_VERSION-flakes.txt \
 232          --channel "$FLAKES_CHANNEL" \
 233          --runner "$CI_RUNNER_DESCRIPTION" \
 234          --job "$CI_JOB_ID" \
 235          --url "$CI_JOB_URL" \
 236          --branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-$CI_COMMIT_BRANCH}" \
 237          --branch-title "${CI_MERGE_REQUEST_TITLE:-$CI_COMMIT_TITLE}"
 238 fi
 239
 240 echo -e "\e[0Ksection_end:$(date +%s):test_post_process\r\e[0K"
 241
 242 exit $DEQP_EXITCODE