From 5f0b84379024787bae24ad20e81d26ab2f082389 Mon Sep 17 00:00:00 2001 From: Florian Weimer Date: Wed, 28 Dec 2016 13:37:18 +0100 Subject: [PATCH] support: Add support for delayed test failure reporting The new functions support_record_failure records a test failure, but does not terminate the process. The macros TEST_VERIFY and TEST_VERIFY_EXIT check that a condition is true. --- ChangeLog | 20 +++++ support/Makefile | 18 +++- support/check.h | 38 +++++++- support/support_record_failure.c | 106 ++++++++++++++++++++++ support/support_test_main.c | 22 +++-- support/support_test_verify_impl.c | 33 +++++++ support/tst-support_record_failure-2.sh | 66 ++++++++++++++ support/tst-support_record_failure.c | 150 ++++++++++++++++++++++++++++++++ support/xfork.c | 34 ++++++++ support/xunistd.h | 35 ++++++++ support/xwaitpid.c | 35 ++++++++ 11 files changed, 550 insertions(+), 7 deletions(-) create mode 100644 support/support_record_failure.c create mode 100644 support/support_test_verify_impl.c create mode 100644 support/tst-support_record_failure-2.sh create mode 100644 support/tst-support_record_failure.c create mode 100644 support/xfork.c create mode 100644 support/xunistd.h create mode 100644 support/xwaitpid.c diff --git a/ChangeLog b/ChangeLog index ea8af3a..03700e2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,23 @@ +2016-12-28 Florian Weimer + + * support/Makefile (libsupport-routines): Add + support_test_verify_impl, support_record_failure, xfork, xwaitpid. + (tests): Add tst-support_record_failure. + (tests-special): tst-support_record_failure-2. + (tst-support_record_failure-2.out): Depend on + tst-support_record_failure-2.sh and tst-support_record_failure. + * support/check.h (TEST_VERIFY, TEST_VERIFY_EXIT): Define. + (support_test_verify_impl, support_record_failure) + (support_report_failure, support_report_failure_reset): Declare. + * support/support_test_main.c (adjust_exit_status): New function. + (support_test_main): Call it to incorporate record test failures. + * support/support_record_failure.c: New file. + * support/tst-support_record_failure.c: Likewise. + * support/tst-support_record_failure-2.sh: Likewise. + * support/xunistd.h: Likewise. + * support/xfork.c: Likewise. + * support/xwaitpid.c: Likewise. + 2016-12-27 Steve Ellcey * scripts/check-c++-types.sh: Add comments. diff --git a/support/Makefile b/support/Makefile index bd425af..1bde8bd 100644 --- a/support/Makefile +++ b/support/Makefile @@ -30,11 +30,14 @@ libsupport-routines = \ ignore_stderr \ oom_error \ set_fortify_handler \ + support_record_failure \ support_test_main \ + support_test_verify_impl \ temp_file \ write_message \ xasprintf \ xcalloc \ + xfork \ xmalloc \ xpthread_barrier_destroy \ xpthread_barrier_init \ @@ -51,6 +54,7 @@ libsupport-routines = \ xpthread_spin_lock \ xpthread_spin_unlock \ xrealloc \ + xwaitpid \ libsupport-static-only-routines := $(libsupport-routines) # Only build one variant of the library. @@ -59,6 +63,18 @@ ifeq ($(build-shared),yes) libsupport-inhibit-o += .o endif -tests = README-testing +tests = \ + README-testing \ + tst-support_record_failure \ + +tests-special = \ + $(objpfx)tst-support_record_failure-2.out + +$(objpfx)tst-support_record_failure-2.out: tst-support_record_failure-2.sh \ + $(objpfx)tst-support_record_failure + $(SHELL) $< $(common-objpfx) '$(test-program-prefix-before-env)' \ + '$(run-program-env)' '$(test-program-prefix-after-env)' \ + > $@; \ + $(evaluate-test) include ../Rules diff --git a/support/check.h b/support/check.h index ff2652c..fb2cd91 100644 --- a/support/check.h +++ b/support/check.h @@ -1,4 +1,4 @@ -/* Macros for reporting test results. +/* Functionality for reporting test results. Copyright (C) 2016 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -35,6 +35,25 @@ __BEGIN_DECLS #define FAIL_EXIT1(...) \ support_exit_failure_impl (1, __FILE__, __LINE__, __VA_ARGS__) +/* Record a test failure (but continue executing) if EXPR evaluates to + false. */ +#define TEST_VERIFY(expr) \ + ({ \ + if (expr) \ + ; \ + else \ + support_test_verify_impl (-1, __FILE__, __LINE__, #expr); \ + }) + +/* Record a test failure and exit if EXPR evaluates to false. */ +#define TEST_VERIFY_EXIT(expr) \ + ({ \ + if (expr) \ + ; \ + else \ + support_test_verify_impl (1, __FILE__, __LINE__, #expr); \ + }) + int support_print_failure_impl (const char *file, int line, const char *format, ...) __attribute__ ((nonnull (1), format (printf, 3, 4))); @@ -42,7 +61,24 @@ void support_exit_failure_impl (int exit_status, const char *file, int line, const char *format, ...) __attribute__ ((noreturn, nonnull (2), format (printf, 4, 5))); +void support_test_verify_impl (int status, const char *file, int line, + const char *expr); + +/* Record a test failure. This function returns and does not + terminate the process. The failure counter is stored in a shared + memory mapping, so that failures reported in child processes are + visible to the parent process and test driver. This function + depends on initialization by an ELF constructor, so it can only be + invoked after the test driver has run. Note that this function + does not support reporting failures from a DSO. */ +void support_record_failure (void); + +/* Internal function called by the test driver. */ +int support_report_failure (int status) + __attribute__ ((weak, warn_unused_result)); +/* Internal function used to test the failure recording framework. */ +void support_record_failure_reset (void); __END_DECLS diff --git a/support/support_record_failure.c b/support/support_record_failure.c new file mode 100644 index 0000000..24b2d6e --- /dev/null +++ b/support/support_record_failure.c @@ -0,0 +1,106 @@ +/* Global test failure counter. + Copyright (C) 2016 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +/* This structure keeps track of test failures. The counter is + incremented on each failure. The failed member is set to true if a + failure is detected, so that even if the counter wraps around to + zero, the failure of a test can be detected. + + The init constructor function below puts *state on a shared + annonymous mapping, so that failure reports from subprocesses + propagate to the parent process. */ +struct test_failures +{ + unsigned counter; + unsigned failed; +}; +static struct test_failures *state; + +static __attribute__ ((constructor)) void +init (void) +{ + void *ptr = mmap (NULL, sizeof (*state), PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_SHARED, -1, 0); + if (ptr == MAP_FAILED) + { + printf ("error: could not map %zu bytes: %m\n", sizeof (*state)); + exit (1); + } + /* Zero-initialization of the struct is sufficient. */ + state = ptr; +} + +void +support_record_failure (void) +{ + if (state == NULL) + { + write_message + ("error: support_record_failure called without initialization\n"); + _exit (1); + } + /* Relaxed MO is sufficient because we are only interested in the + values themselves, in isolation. */ + __atomic_store_n (&state->failed, 1, __ATOMIC_RELEASE); + __atomic_add_fetch (&state->counter, 1, __ATOMIC_RELEASE); +} + +int +support_report_failure (int status) +{ + if (state == NULL) + { + write_message + ("error: support_report_failure called without initialization\n"); + return 1; + } + + /* Relaxed MO is sufficient because acquire test result reporting + assumes that exiting from the main thread happens before the + error reporting via support_record_failure, which requires some + form of external synchronization. */ + bool failed = __atomic_load_n (&state->failed, __ATOMIC_RELAXED); + if (failed) + printf ("error: %u test failures\n", + __atomic_load_n (&state->counter, __ATOMIC_RELAXED)); + + if ((status == 0 || status == EXIT_UNSUPPORTED) && failed) + /* If we have a recorded failure, it overrides a non-failure + report from the test function. */ + status = 1; + return status; +} + +void +support_record_failure_reset (void) +{ + /* Only used for testing the test framework, with external + synchronization, but use release MO for consistency. */ + __atomic_store_n (&state->failed, 0, __ATOMIC_RELAXED); + __atomic_add_fetch (&state->counter, 0, __ATOMIC_RELAXED); +} diff --git a/support/support_test_main.c b/support/support_test_main.c index 0582230..8d31e2f 100644 --- a/support/support_test_main.c +++ b/support/support_test_main.c @@ -17,6 +17,7 @@ . */ #include +#include #include #include @@ -164,6 +165,17 @@ static bool test_main_called; const char *test_dir = NULL; + +/* If test failure reporting has been linked in, it may contribute + additional test failures. */ +static int +adjust_exit_status (int status) +{ + if (support_report_failure != NULL) + return support_report_failure (status); + return status; +} + int support_test_main (int argc, char **argv, const struct test_config *config) { @@ -300,7 +312,7 @@ support_test_main (int argc, char **argv, const struct test_config *config) /* If we are not expected to fork run the function immediately. */ if (direct) - return run_test_function (argc, argv, config); + return adjust_exit_status (run_test_function (argc, argv, config)); /* Set up the test environment: - prevent core dumps @@ -363,8 +375,8 @@ support_test_main (int argc, char **argv, const struct test_config *config) if (config->expected_status == 0) { if (config->expected_signal == 0) - /* Simply exit with the return value of the test. */ - return WEXITSTATUS (status); + /* Exit with the return value of the test. */ + return adjust_exit_status (WEXITSTATUS (status)); else { printf ("Expected signal '%s' from child, got none\n", @@ -382,7 +394,7 @@ support_test_main (int argc, char **argv, const struct test_config *config) exit (1); } } - return 0; + return adjust_exit_status (0); } /* Process was killed by timer or other signal. */ else @@ -401,6 +413,6 @@ support_test_main (int argc, char **argv, const struct test_config *config) exit (1); } - return 0; + return adjust_exit_status (0); } } diff --git a/support/support_test_verify_impl.c b/support/support_test_verify_impl.c new file mode 100644 index 0000000..28b1524 --- /dev/null +++ b/support/support_test_verify_impl.c @@ -0,0 +1,33 @@ +/* Implementation of the TEST_VERIFY and TEST_VERIFY_EXIT macros. + Copyright (C) 2016 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#include +#include + +void +support_test_verify_impl (int status, const char *file, int line, + const char *expr) +{ + support_record_failure (); + printf ("FAIL %s:%d: not true: %s\n", file, line, expr); + if (status >= 0) + exit (status); + +} diff --git a/support/tst-support_record_failure-2.sh b/support/tst-support_record_failure-2.sh new file mode 100644 index 0000000..71af382 --- /dev/null +++ b/support/tst-support_record_failure-2.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# Test failure recording (with and without --direct). +# Copyright (C) 2016 Free Software Foundation, Inc. +# This file is part of the GNU C Library. + +# The GNU C Library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# The GNU C Library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with the GNU C Library; if not, see +# . */ + +set -e + +common_objpfx=$1; shift +test_program_prefix_before_env=$1; shift +run_program_env=$1; shift +test_program_prefix_after_env=$1; shift + +run_test () { + expected_status="$1" + expected_output="$2" + shift 2 + args="${common_objpfx}support/tst-support_record_failure $*" + echo "running: $args" + set +e + output="$(${test_program_prefix_before_env} \ + ${run_program} ${test_program_prefix_after_env} $args)" + status=$? + set -e + echo " exit status: $status" + if test "$output" != "$expected_output" ; then + echo "error: unexpected ouput: $output" + exit 1 + fi + if test "$status" -ne "$expected_status" ; then + echo "error: exit status $expected_status expected" + exit 1 + fi +} + +different_status () { + direct="$1" + run_test 1 "error: 1 test failures" $direct --status=0 + run_test 1 "error: 1 test failures" $direct --status=1 + run_test 2 "error: 1 test failures" $direct --status=2 + run_test 1 "error: 1 test failures" $direct --status=77 + run_test 2 "FAIL tst-support_record_failure.c:108: not true: false +error: 1 test failures" $direct --test-verify +} + +different_status +different_status --direct + +run_test 1 "FAIL tst-support_record_failure.c:113: not true: false +error: 1 test failures" --test-verify-exit +# --direct does not print the summary error message if exit is called. +run_test 1 "FAIL tst-support_record_failure.c:113: not true: false" \ + --direct --test-verify-exit diff --git a/support/tst-support_record_failure.c b/support/tst-support_record_failure.c new file mode 100644 index 0000000..a999f70 --- /dev/null +++ b/support/tst-support_record_failure.c @@ -0,0 +1,150 @@ +/* Test support_record_failure state sharing. + Copyright (C) 2016 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +static int exit_status_with_failure = -1; +static bool test_verify; +static bool test_verify_exit; +enum + { + OPT_STATUS = 10001, + OPT_TEST_VERIFY, + OPT_TEST_VERIFY_EXIT, + }; +#define CMDLINE_OPTIONS \ + { "status", required_argument, NULL, OPT_STATUS }, \ + { "test-verify", no_argument, NULL, OPT_TEST_VERIFY }, \ + { "test-verify-exit", no_argument, NULL, OPT_TEST_VERIFY_EXIT }, +static void +cmdline_process (int c) +{ + switch (c) + { + case OPT_STATUS: + exit_status_with_failure = atoi (optarg); + break; + case OPT_TEST_VERIFY: + test_verify = true; + break; + case OPT_TEST_VERIFY_EXIT: + test_verify_exit = true; + break; + } +} +#define CMDLINE_PROCESS cmdline_process + +static void +check_failure_reporting (int phase, int zero, int unsupported) +{ + int status = support_report_failure (0); + if (status != zero) + { + printf ("real-error (phase %d): support_report_failure (0) == %d\n", + phase, status); + exit (1); + } + status = support_report_failure (1); + if (status != 1) + { + printf ("real-error (phase %d): support_report_failure (1) == %d\n", + phase, status); + exit (1); + } + status = support_report_failure (2); + if (status != 2) + { + printf ("real-error (phase %d): support_report_failure (2) == %d\n", + phase, status); + exit (1); + } + status = support_report_failure (EXIT_UNSUPPORTED); + if (status != unsupported) + { + printf ("real-error (phase %d): " + "support_report_failure (EXIT_UNSUPPORTED) == %d\n", + phase, status); + exit (1); + } +} + +static int +do_test (void) +{ + if (exit_status_with_failure >= 0) + { + /* External invocation with requested error status. Used by + tst-support_report_failure-2.sh. */ + support_record_failure (); + return exit_status_with_failure; + } + TEST_VERIFY (true); + TEST_VERIFY_EXIT (true); + if (test_verify) + { + TEST_VERIFY (false); + return 2; /* Expected exit status. */ + } + if (test_verify_exit) + { + TEST_VERIFY_EXIT (false); + return 3; /* Not reached. Expected exit status is 1. */ + } + + printf ("info: This test tests the test framework.\n" + "info: It reports some expected errors on stdout.\n"); + + /* Check that the status is passed through unchanged. */ + check_failure_reporting (1, 0, EXIT_UNSUPPORTED); + + /* Check state propagation from a subprocess. */ + pid_t pid = xfork (); + if (pid == 0) + { + support_record_failure (); + _exit (0); + } + int status; + xwaitpid (pid, &status, 0); + if (status != 0) + { + printf ("real-error: incorrect status from subprocess: %d\n", status); + return 1; + } + check_failure_reporting (2, 1, 1); + + /* Also test directly in the parent process. */ + support_record_failure_reset (); + check_failure_reporting (3, 0, EXIT_UNSUPPORTED); + support_record_failure (); + check_failure_reporting (4, 1, 1); + + /* We need to mask the failure above. */ + support_record_failure_reset (); + return 0; +} + +#include diff --git a/support/xfork.c b/support/xfork.c new file mode 100644 index 0000000..4b2ce91 --- /dev/null +++ b/support/xfork.c @@ -0,0 +1,34 @@ +/* fork with error checking. + Copyright (C) 2016 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#include +#include + +pid_t +xfork (void) +{ + pid_t result = fork (); + if (result < 0) + { + printf ("error: fork: %m\n"); + exit (1); + } + return result; +} diff --git a/support/xunistd.h b/support/xunistd.h new file mode 100644 index 0000000..f0c7419 --- /dev/null +++ b/support/xunistd.h @@ -0,0 +1,35 @@ +/* POSIX-specific extra functions. + Copyright (C) 2016 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* These wrapper functions use POSIX types and therefore cannot be + declared in . */ + +#ifndef SUPPORT_XUNISTD_H +#define SUPPORT_XUNISTD_H + +#include +#include + +__BEGIN_DECLS + +pid_t xfork (void); +pid_t xwaitpid (pid_t, int *status, int flags); + +__END_DECLS + +#endif /* SUPPORT_XUNISTD_H */ diff --git a/support/xwaitpid.c b/support/xwaitpid.c new file mode 100644 index 0000000..5a6e540 --- /dev/null +++ b/support/xwaitpid.c @@ -0,0 +1,35 @@ +/* waitpid with error checking. + Copyright (C) 2016 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#include +#include +#include + +int +xwaitpid (int pid, int *status, int flags) +{ + pid_t result = waitpid (pid, status, flags); + if (result < 0) + { + printf ("error: waitpid: %m\n"); + exit (1); + } + return result; +} -- 2.7.4