gitlab CI: split the qemu test runs up into multiple jobs

The tests are split by topic but have varying runtime. Specifically, the
longest test (touchpad) takes ~170s whereas many of the others can take less
than a second. Splitting them all up into separate VMs costs too much in
startup time so here's the middle ground of some custom grouping to make the
tests roughly run the same time.

This list will need to be manually maintained but given that groups are rarely
added anyway this shouldn't be too much of a maintenance burden. And bonus:
since the kvm tests often fail due to timing issues, re-running one is
significantly faster now.

This shaves about 8min of the CI run

Signed-off-by: Peter Hutterer <peter.hutterer@who-t.net>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c3a99e1bbede627681b1cf06569368f17214fc9a..0a23c89720394f18682161d04a6a251d122dbf71 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -47,6 +47,7 @@ stages:
   - container_prep   # rebuild the container images if there is a change
   - build            # for actually building and testing things in a container
   - VM               # for running the test suite in a VM
+  - valgrind         # for running the test suite under valgrind in a VM
   - distro           # distribs test
   - deploy           # trigger wayland's website generation
   - container_clean  # clean up unused container images
@@ -414,7 +415,11 @@ freebsd:11.2@container-clean:
     exit 1 ;
   fi
 
-fedora:30@test-suite-vm:
+# Run in a test suite. Special variables:
+# - SUITES: the meson test suites to run, or
+# - SUITE_NAMES: all elements will be expanded to libinput-test-suite-$value
+# Set one or the other, not both.
+.fedora:30@test-suite-vm:
   stage: VM
   image: $QEMU_CONTAINER_IMAGE
   tags:
@@ -422,6 +427,11 @@ fedora:30@test-suite-vm:
   variables:
     FEDORA_VERSION: 30
     MESON_BUILDDIR: build_dir
+    # remove the global --no-suite=hardware
+    MESON_TEST_ARGS: ''
+  before_script:
+    - if ! [[ -z $SUITE_NAMES ]]; then SUITES=$(echo $SUITE_NAMES | sed 's/\([^ ]*\)/libinput-test-suite-\1/g'); fi
+    - echo "Testing $SUITES"
   script:
     # start our vm, no args required
     - /app/start_vm.sh
@@ -433,7 +443,8 @@ fedora:30@test-suite-vm:
     - ssh localhost -p 5555 "cd $CI_PROJECT_NAME ; meson \"$MESON_BUILDDIR\" $MESON_ARGS"
     - ssh localhost -p 5555 "cd $CI_PROJECT_NAME ; meson configure \"$MESON_BUILDDIR\" "
     - ssh localhost -p 5555 "cd $CI_PROJECT_NAME ; ninja -C \"$MESON_BUILDDIR\" $NINJA_ARGS"
-    - ssh localhost -p 5555 "cd $CI_PROJECT_NAME ; meson test -C \"$MESON_BUILDDIR\" $MESON_TEST_ARGS --print-errorlogs" && touch .success || true
+    - echo "meson test -C \"$MESON_BUILDDIR\" $MESON_TEST_ARGS $SUITES --print-errorlogs"
+    - ssh localhost -p 5555 "cd $CI_PROJECT_NAME ; meson test -C \"$MESON_BUILDDIR\" $MESON_TEST_ARGS --print-errorlogs $SUITES" && touch .success || true
 
     # no matter the results of the tests, we want to fetch the logs
     - scp -P 5555 -r localhost:$CI_PROJECT_NAME/$MESON_BUILDDIR .
@@ -464,10 +475,74 @@ fedora:30@test-suite-vm:
   allow_failure: true
   needs: ['fedora:30@qemu-prep']
 
-fedora:30@test-suite-vm-valgrind:
-  extends: fedora:30@test-suite-vm
+
+# in reverse order of duration to get the slowest ones started first
+fedora:30@vm-touchpad:
+  extends: .fedora:30@test-suite-vm
+  variables:
+    SUITE_NAMES: 'touchpad'
+
+fedora:30@vm-tap:
+  extends: .fedora:30@test-suite-vm
+  variables:
+    SUITE_NAMES: 'tap'
+
+fedora:30@vm-tablet:
+  extends: .fedora:30@test-suite-vm
+  variables:
+    SUITE_NAMES: 'tablet'
+
+fedora:30@vm-gestures-device:
+  extends: .fedora:30@test-suite-vm
+  variables:
+    SUITE_NAMES: 'gestures device'
+
+fedora:30@vm-others:
+  extends: .fedora:30@test-suite-vm
+  variables:
+    SUITE_NAMES: 'context config misc events totem udev lid log timer tablet-mode quirks trackball pad path keyboard switch touch trackpoint'
+
+fedora:30@vm-pointer:
+  extends: .fedora:30@test-suite-vm
+  variables:
+    SUITE_NAMES: 'pointer'
+
+fedora:30@valgrind-touchpad:
+  stage: valgrind
+  extends: fedora:30@vm-touchpad
+  variables:
+    MESON_TEST_ARGS: '--setup=valgrind'
+
+fedora:30@valgrind-tap:
+  stage: valgrind
+  extends: fedora:30@vm-tap
+  variables:
+    MESON_TEST_ARGS: '--setup=valgrind'
+
+fedora:30@valgrind-tablet:
+  stage: valgrind
+  extends: fedora:30@vm-tablet
+  variables:
+    MESON_TEST_ARGS: '--setup=valgrind'
+
+fedora:30@valgrind-gestures-device:
+  stage: valgrind
+  extends: fedora:30@vm-gestures-device
+  variables:
+    MESON_TEST_ARGS: '--setup=valgrind'
+
+
+fedora:30@valgrind-others:
+  stage: valgrind
+  extends: fedora:30@vm-others
+  variables:
+    MESON_TEST_ARGS: '--setup=valgrind'
+
+fedora:30@valgrind-pointer:
+  stage: valgrind
+  extends: fedora:30@vm-pointer
   variables:
-    MESON_TEST_ARGS: '--suite=valgrind --setup=valgrind'
+    MESON_TEST_ARGS: '--setup=valgrind'
 
 .fedora-build@template:
   extends: .build@template