gitlab CI: hook up junit test reports to the meson results

The KVM tests use this for now, not the container builds where we run meson
directly.

The python script to convert meson test logs to junit results expects suite
names, so let's add all tests to suites so we don't need to carry local
modifications.

Signed-off-by: Peter Hutterer <peter.hutterer@who-t.net>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d3df62829dae29043d1dad878a02f24312021a8f..84e7b28213f6f4001ab0daf07a13a4d93c64e9a4 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -853,6 +853,8 @@ soname:
     paths:
       - $MESON_BUILDDIR/meson-logs
       - console.out
+    reports:
+      junit: $MESON_BUILDDIR/junit-*.xml
 
   allow_failure: true
   retry:

diff --git a/.gitlab-ci/gitlab-ci.tmpl b/.gitlab-ci/gitlab-ci.tmpl
index 7dd9f422fc48a4510538285ee0d253552a985e55..8fb165dc0c585976beef2c0b0ba8830354b6811c 100644 (file)
--- a/.gitlab-ci/gitlab-ci.tmpl
+++ b/.gitlab-ci/gitlab-ci.tmpl
@@ -509,6 +509,8 @@ soname:
     paths:
       - $MESON_BUILDDIR/meson-logs
       - console.out
+    reports:
+      junit: $MESON_BUILDDIR/junit-*.xml
 
   allow_failure: true
   retry:

diff --git a/.gitlab-ci/meson-build.sh b/.gitlab-ci/meson-build.sh
index 8fcbe3a9f4193fea9f18ce0cd1d1f98ee42c61c3..cf8acd5b81bcc69f32cb83ccbc31ea745d7df64c 100755 (executable)
--- a/.gitlab-ci/meson-build.sh
+++ b/.gitlab-ci/meson-build.sh
@@ -41,4 +41,12 @@ meson test -C "$MESON_BUILDDIR" $MESON_TEST_ARGS --print-errorlogs
 exit_code=$?
 set -e
 
+# We need the glob for the testlog so that it picks up those suffixed by a
+# suite (e.g. testlog-valgrind.json)
+./.gitlab-ci/meson-junit-report.py \
+       --project-name=libevdev \
+       --job-id="$CI_JOB_ID" \
+       --output="$MESON_BUILDDIR/junit-$CI_JOB_NAME-report.xml" \
+       "$MESON_BUILDDIR"/meson-logs/testlog*.json; \
+
 exit $exit_code

diff --git a/.gitlab-ci/meson-junit-report.py b/.gitlab-ci/meson-junit-report.py
new file mode 100755 (executable)
index 0000000..94a9a61
--- /dev/null
+++ b/.gitlab-ci/meson-junit-report.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+#
+# meson-junit-report.py: Turns a Meson test log into a JUnit report
+#
+# Copyright 2019  GNOME Foundation
+#
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+import argparse
+import datetime
+import json
+import sys
+import xml.etree.ElementTree as ET
+
+aparser = argparse.ArgumentParser(description='Turns a Meson test log into a JUnit report')
+aparser.add_argument('--project-name', metavar='NAME',
+                     help='The project name',
+                     default='unknown')
+aparser.add_argument('--job-id', metavar='ID',
+                     help='The job ID for the report',
+                     default='Unknown')
+aparser.add_argument('--branch', metavar='NAME',
+                     help='Branch of the project being tested',
+                     default='master')
+aparser.add_argument('--output', metavar='FILE',
+                     help='The output file, stdout by default',
+                     type=argparse.FileType('w', encoding='UTF-8'),
+                     default=sys.stdout)
+aparser.add_argument('infile', metavar='FILE',
+                     help='The input testlog.json, stdin by default',
+                     type=argparse.FileType('r', encoding='UTF-8'),
+                     default=sys.stdin)
+
+args = aparser.parse_args()
+
+outfile = args.output
+
+testsuites = ET.Element('testsuites')
+testsuites.set('id', '{}/{}'.format(args.job_id, args.branch))
+testsuites.set('package', args.project_name)
+testsuites.set('timestamp', datetime.datetime.utcnow().isoformat(timespec='minutes'))
+
+suites = {}
+for line in args.infile:
+    data = json.loads(line)
+    (full_suite, unit_name) = data['name'].split(' / ')
+    (project_name, suite_name) = full_suite.split(':')
+
+    duration = data['duration']
+    return_code = data['returncode']
+    log = data['stdout']
+
+    unit = {
+        'suite': suite_name,
+        'name': unit_name,
+        'duration': duration,
+        'returncode': return_code,
+        'stdout': log,
+    }
+
+    units = suites.setdefault(suite_name, [])
+    units.append(unit)
+
+for name, units in suites.items():
+    print('Processing suite {} (units: {})'.format(name, len(units)))
+
+    def if_failed(unit):
+        if not if_skipped(unit) and unit['returncode'] != 0:
+            return True
+        return False
+
+    def if_skipped(unit):
+        if unit['returncode'] == 77:
+            return True
+        return False
+
+    def if_succeded(unit):
+        if unit['returncode'] == 0:
+            return True
+        return False
+
+    successes = list(filter(if_succeded, units))
+    failures = list(filter(if_failed, units))
+    skips = list(filter(if_skipped, units))
+    print(' - {}: {} pass, {} fail, {} skipped'.format(name, len(successes), len(failures), len(skips)))
+
+    testsuite = ET.SubElement(testsuites, 'testsuite')
+    testsuite.set('name', '{}/{}'.format(args.project_name, name))
+    testsuite.set('tests', str(len(units)))
+    testsuite.set('errors', str(len(failures)))
+    testsuite.set('skipped', str(len(skips)))
+    testsuite.set('failures', str(len(failures)))
+
+    for unit in successes:
+        testcase = ET.SubElement(testsuite, 'testcase')
+        testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        testcase.set('name', unit['name'])
+        testcase.set('time', str(unit['duration']))
+
+    for unit in skips:
+        testcase = ET.SubElement(testsuite, 'testcase')
+        testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        testcase.set('name', unit['name'])
+        testcase.set('time', str(unit['duration']))
+
+        skip = ET.SubElement(testcase, 'skipped')
+
+    for unit in failures:
+        testcase = ET.SubElement(testsuite, 'testcase')
+        testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        testcase.set('name', unit['name'])
+        testcase.set('time', str(unit['duration']))
+
+        failure = ET.SubElement(testcase, 'failure')
+        failure.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        failure.set('name', unit['name'])
+        failure.set('type', 'error')
+        failure.text = unit['stdout']
+
+output = ET.tostring(testsuites, encoding='unicode')
+outfile.write(output)

diff --git a/meson.build b/meson.build
index e6c43db52eee86de9992e14284166cab0fa65384..18e4d2e8ecf2815dfaac57aeff060b5bf7af8779 100644 (file)
--- a/meson.build
+++ b/meson.build
@@ -147,7 +147,7 @@ if dep_check.found()
                                      ],
                                      dependencies: [dep_libevdev, dep_check],
                                      install: false)
-       test('test-event-codes', test_event_codes)
+       test('test-event-codes', test_event_codes, suite: 'library')
 
        test_internals = executable('test-internals',
                                    sources: src_common + [
@@ -155,7 +155,7 @@ if dep_check.found()
                                    ],
                                    dependencies: [dep_libevdev, dep_check],
                                    install: false)
-       test('test-internals', test_internals)
+       test('test-internals', test_internals, suite: 'library')
 
        test_uinput = executable('test-uinput',
                                 sources: src_common + [
@@ -163,7 +163,7 @@ if dep_check.found()
                                 ],
                                 dependencies: [dep_libevdev, dep_check],
                                 install: false)
-       test('test-uinput', test_uinput)
+       test('test-uinput', test_uinput, suite: 'library')
 
        test_libevdev = executable('test-libevdev',
                                   sources: src_common + [
@@ -173,7 +173,7 @@ if dep_check.found()
                                   ],
                                   dependencies: [dep_libevdev, dep_check],
                                   install: false)
-       test('test-libevdev', test_libevdev)
+       test('test-libevdev', test_libevdev, suite: 'library')
 
        test_kernel = executable('test-kernel',
                                 sources: src_common + [
@@ -181,7 +181,7 @@ if dep_check.found()
                                 ],
                                 dependencies: [dep_libevdev, dep_check],
                                 install: false)
-       test('test-kernel', test_kernel)
+       test('test-kernel', test_kernel, suite: 'kernel')
 
 
        valgrind = find_program('valgrind', required: false)
@@ -205,7 +205,8 @@ if dep_check.found()
 
        test_static_link = find_program('test/test-static-symbols-leak.sh')
        test('static-symbols-leak', test_static_link,
-               args: [meson.current_build_dir()])
+            args: [meson.current_build_dir()],
+            suite: 'static')
 endif
 
 doxygen = find_program('doxygen', required: get_option('documentation'))