gitlab CI: hook up junit test reports to the meson results

The KVM tests use this for now, not the container builds where we run meson
directly.

The python script to convert meson test logs to junit results expects suite
names, so let's add all tests to suites so we don't need to carry local
modifications.

Signed-off-by: Peter Hutterer <peter.hutterer@who-t.net>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d3df628..84e7b28 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -853,6 +853,8 @@ soname:
     paths:
       - $MESON_BUILDDIR/meson-logs
       - console.out
+    reports:
+      junit: $MESON_BUILDDIR/junit-*.xml
 
   allow_failure: true
   retry:

diff --git a/.gitlab-ci/gitlab-ci.tmpl b/.gitlab-ci/gitlab-ci.tmpl
index 7dd9f42..8fb165d 100644 (file)
--- a/.gitlab-ci/gitlab-ci.tmpl
+++ b/.gitlab-ci/gitlab-ci.tmpl
@@ -509,6 +509,8 @@ soname:
     paths:
       - $MESON_BUILDDIR/meson-logs
       - console.out
+    reports:
+      junit: $MESON_BUILDDIR/junit-*.xml
 
   allow_failure: true
   retry:

diff --git a/.gitlab-ci/meson-build.sh b/.gitlab-ci/meson-build.sh
index 8fcbe3a..cf8acd5 100755 (executable)
--- a/.gitlab-ci/meson-build.sh
+++ b/.gitlab-ci/meson-build.sh
@@ -41,4 +41,12 @@ meson test -C "$MESON_BUILDDIR" $MESON_TEST_ARGS --print-errorlogs
 exit_code=$?
 set -e
 
+# We need the glob for the testlog so that it picks up those suffixed by a
+# suite (e.g. testlog-valgrind.json)
+./.gitlab-ci/meson-junit-report.py \
+       --project-name=libevdev \
+       --job-id="$CI_JOB_ID" \
+       --output="$MESON_BUILDDIR/junit-$CI_JOB_NAME-report.xml" \
+       "$MESON_BUILDDIR"/meson-logs/testlog*.json; \
+
 exit $exit_code

diff --git a/.gitlab-ci/meson-junit-report.py b/.gitlab-ci/meson-junit-report.py
new file mode 100755 (executable)
index 0000000..94a9a61
--- /dev/null
+++ b/.gitlab-ci/meson-junit-report.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+#
+# meson-junit-report.py: Turns a Meson test log into a JUnit report
+#
+# Copyright 2019  GNOME Foundation
+#
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+import argparse
+import datetime
+import json
+import sys
+import xml.etree.ElementTree as ET
+
+aparser = argparse.ArgumentParser(description='Turns a Meson test log into a JUnit report')
+aparser.add_argument('--project-name', metavar='NAME',
+                     help='The project name',
+                     default='unknown')
+aparser.add_argument('--job-id', metavar='ID',
+                     help='The job ID for the report',
+                     default='Unknown')
+aparser.add_argument('--branch', metavar='NAME',
+                     help='Branch of the project being tested',
+                     default='master')
+aparser.add_argument('--output', metavar='FILE',
+                     help='The output file, stdout by default',
+                     type=argparse.FileType('w', encoding='UTF-8'),
+                     default=sys.stdout)
+aparser.add_argument('infile', metavar='FILE',
+                     help='The input testlog.json, stdin by default',
+                     type=argparse.FileType('r', encoding='UTF-8'),
+                     default=sys.stdin)
+
+args = aparser.parse_args()
+
+outfile = args.output
+
+testsuites = ET.Element('testsuites')
+testsuites.set('id', '{}/{}'.format(args.job_id, args.branch))
+testsuites.set('package', args.project_name)
+testsuites.set('timestamp', datetime.datetime.utcnow().isoformat(timespec='minutes'))
+
+suites = {}
+for line in args.infile:
+    data = json.loads(line)
+    (full_suite, unit_name) = data['name'].split(' / ')
+    (project_name, suite_name) = full_suite.split(':')
+
+    duration = data['duration']
+    return_code = data['returncode']
+    log = data['stdout']
+
+    unit = {
+        'suite': suite_name,
+        'name': unit_name,
+        'duration': duration,
+        'returncode': return_code,
+        'stdout': log,
+    }
+
+    units = suites.setdefault(suite_name, [])
+    units.append(unit)
+
+for name, units in suites.items():
+    print('Processing suite {} (units: {})'.format(name, len(units)))
+
+    def if_failed(unit):
+        if not if_skipped(unit) and unit['returncode'] != 0:
+            return True
+        return False
+
+    def if_skipped(unit):
+        if unit['returncode'] == 77:
+            return True
+        return False
+
+    def if_succeded(unit):
+        if unit['returncode'] == 0:
+            return True
+        return False
+
+    successes = list(filter(if_succeded, units))
+    failures = list(filter(if_failed, units))
+    skips = list(filter(if_skipped, units))
+    print(' - {}: {} pass, {} fail, {} skipped'.format(name, len(successes), len(failures), len(skips)))
+
+    testsuite = ET.SubElement(testsuites, 'testsuite')
+    testsuite.set('name', '{}/{}'.format(args.project_name, name))
+    testsuite.set('tests', str(len(units)))
+    testsuite.set('errors', str(len(failures)))
+    testsuite.set('skipped', str(len(skips)))
+    testsuite.set('failures', str(len(failures)))
+
+    for unit in successes:
+        testcase = ET.SubElement(testsuite, 'testcase')
+        testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        testcase.set('name', unit['name'])
+        testcase.set('time', str(unit['duration']))
+
+    for unit in skips:
+        testcase = ET.SubElement(testsuite, 'testcase')
+        testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        testcase.set('name', unit['name'])
+        testcase.set('time', str(unit['duration']))
+
+        skip = ET.SubElement(testcase, 'skipped')
+
+    for unit in failures:
+        testcase = ET.SubElement(testsuite, 'testcase')
+        testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        testcase.set('name', unit['name'])
+        testcase.set('time', str(unit['duration']))
+
+        failure = ET.SubElement(testcase, 'failure')
+        failure.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        failure.set('name', unit['name'])
+        failure.set('type', 'error')
+        failure.text = unit['stdout']
+
+output = ET.tostring(testsuites, encoding='unicode')
+outfile.write(output)

diff --git a/meson.build b/meson.build
index e6c43db..18e4d2e 100644 (file)
--- a/meson.build
+++ b/meson.build
@@ -147,7 +147,7 @@ if dep_check.found()
                                      ],
                                      dependencies: [dep_libevdev, dep_check],
                                      install: false)
-       test('test-event-codes', test_event_codes)
+       test('test-event-codes', test_event_codes, suite: 'library')
 
        test_internals = executable('test-internals',
                                    sources: src_common + [
@@ -155,7 +155,7 @@ if dep_check.found()
                                    ],
                                    dependencies: [dep_libevdev, dep_check],
                                    install: false)
-       test('test-internals', test_internals)
+       test('test-internals', test_internals, suite: 'library')
 
        test_uinput = executable('test-uinput',
                                 sources: src_common + [
@@ -163,7 +163,7 @@ if dep_check.found()
                                 ],
                                 dependencies: [dep_libevdev, dep_check],
                                 install: false)
-       test('test-uinput', test_uinput)
+       test('test-uinput', test_uinput, suite: 'library')
 
        test_libevdev = executable('test-libevdev',
                                   sources: src_common + [
@@ -173,7 +173,7 @@ if dep_check.found()
                                   ],
                                   dependencies: [dep_libevdev, dep_check],
                                   install: false)
-       test('test-libevdev', test_libevdev)
+       test('test-libevdev', test_libevdev, suite: 'library')
 
        test_kernel = executable('test-kernel',
                                 sources: src_common + [
@@ -181,7 +181,7 @@ if dep_check.found()
                                 ],
                                 dependencies: [dep_libevdev, dep_check],
                                 install: false)
-       test('test-kernel', test_kernel)
+       test('test-kernel', test_kernel, suite: 'kernel')
 
 
        valgrind = find_program('valgrind', required: false)
@@ -205,7 +205,8 @@ if dep_check.found()
 
        test_static_link = find_program('test/test-static-symbols-leak.sh')
        test('static-symbols-leak', test_static_link,
-               args: [meson.current_build_dir()])
+            args: [meson.current_build_dir()],
+            suite: 'static')
 endif
 
 doxygen = find_program('doxygen', required: get_option('documentation'))