From: David Gow Date: Wed, 4 Aug 2021 05:08:08 +0000 (-0700) Subject: kunit: Print test statistics on failure X-Git-Tag: accepted/tizen/unified/20230118.172025~6492^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=acd8e8407b8fcc3229d6d8558cac338bea801aed;p=platform%2Fkernel%2Flinux-rpi.git kunit: Print test statistics on failure When a number of tests fail, it can be useful to get higher-level statistics of how many tests are failing (or how many parameters are failing in parameterised tests), and in what cases or suites. This is already done by some non-KUnit tests, so add support for automatically generating these for KUnit tests. This change adds a 'kunit.stats_enabled' switch which has three values: - 0: No stats are printed (current behaviour) - 1: Stats are printed only for tests/suites with more than one subtest (new default) - 2: Always print test statistics For parameterised tests, the summary line looks as follows: " # inode_test_xtimestamp_decoding: pass:16 fail:0 skip:0 total:16" For test suites, there are two lines looking like this: "# ext4_inode_test: pass:1 fail:0 skip:0 total:1" "# Totals: pass:16 fail:0 skip:0 total:16" The first line gives the number of direct subtests, the second "Totals" line is the accumulated sum of all tests and test parameters. This format is based on the one used by kselftest[1]. [1]: https://elixir.bootlin.com/linux/latest/source/tools/testing/selftests/kselftest.h#L109 Signed-off-by: David Gow Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- diff --git a/lib/kunit/test.c b/lib/kunit/test.c index d79ecb8..f246b84 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -52,6 +53,51 @@ EXPORT_SYMBOL_GPL(__kunit_fail_current_test); #endif /* + * KUnit statistic mode: + * 0 - disabled + * 1 - only when there is more than one subtest + * 2 - enabled + */ +static int kunit_stats_enabled = 1; +module_param_named(stats_enabled, kunit_stats_enabled, int, 0644); +MODULE_PARM_DESC(stats_enabled, + "Print test stats: never (0), only for multiple subtests (1), or always (2)"); + +struct kunit_result_stats { + unsigned long passed; + unsigned long skipped; + unsigned long failed; + unsigned long total; +}; + +static bool kunit_should_print_stats(struct kunit_result_stats stats) +{ + if (kunit_stats_enabled == 0) + return false; + + if (kunit_stats_enabled == 2) + return true; + + return (stats.total > 1); +} + +static void kunit_print_test_stats(struct kunit *test, + struct kunit_result_stats stats) +{ + if (!kunit_should_print_stats(stats)) + return; + + kunit_log(KERN_INFO, test, + KUNIT_SUBTEST_INDENT + "# %s: pass:%lu fail:%lu skip:%lu total:%lu", + test->name, + stats.passed, + stats.failed, + stats.skipped, + stats.total); +} + +/* * Append formatted message to log, size of which is limited to * KUNIT_LOG_SIZE bytes (including null terminating byte). */ @@ -393,15 +439,69 @@ static void kunit_run_case_catch_errors(struct kunit_suite *suite, test_case->status = KUNIT_SUCCESS; } +static void kunit_print_suite_stats(struct kunit_suite *suite, + struct kunit_result_stats suite_stats, + struct kunit_result_stats param_stats) +{ + if (kunit_should_print_stats(suite_stats)) { + kunit_log(KERN_INFO, suite, + "# %s: pass:%lu fail:%lu skip:%lu total:%lu", + suite->name, + suite_stats.passed, + suite_stats.failed, + suite_stats.skipped, + suite_stats.total); + } + + if (kunit_should_print_stats(param_stats)) { + kunit_log(KERN_INFO, suite, + "# Totals: pass:%lu fail:%lu skip:%lu total:%lu", + param_stats.passed, + param_stats.failed, + param_stats.skipped, + param_stats.total); + } +} + +static void kunit_update_stats(struct kunit_result_stats *stats, + enum kunit_status status) +{ + switch (status) { + case KUNIT_SUCCESS: + stats->passed++; + break; + case KUNIT_SKIPPED: + stats->skipped++; + break; + case KUNIT_FAILURE: + stats->failed++; + break; + } + + stats->total++; +} + +static void kunit_accumulate_stats(struct kunit_result_stats *total, + struct kunit_result_stats add) +{ + total->passed += add.passed; + total->skipped += add.skipped; + total->failed += add.failed; + total->total += add.total; +} + int kunit_run_tests(struct kunit_suite *suite) { char param_desc[KUNIT_PARAM_DESC_SIZE]; struct kunit_case *test_case; + struct kunit_result_stats suite_stats = { 0 }; + struct kunit_result_stats total_stats = { 0 }; kunit_print_subtest_start(suite); kunit_suite_for_each_test_case(suite, test_case) { struct kunit test = { .param_value = NULL, .param_index = 0 }; + struct kunit_result_stats param_stats = { 0 }; test_case->status = KUNIT_SKIPPED; if (test_case->generate_params) { @@ -431,14 +531,23 @@ int kunit_run_tests(struct kunit_suite *suite) test.param_value = test_case->generate_params(test.param_value, param_desc); test.param_index++; } + + kunit_update_stats(¶m_stats, test.status); + } while (test.param_value); + kunit_print_test_stats(&test, param_stats); + kunit_print_ok_not_ok(&test, true, test_case->status, kunit_test_case_num(suite, test_case), test_case->name, test.status_comment); + + kunit_update_stats(&suite_stats, test_case->status); + kunit_accumulate_stats(&total_stats, param_stats); } + kunit_print_suite_stats(suite, suite_stats, total_stats); kunit_print_subtest_end(suite); return 0; diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 84938fe..6310a64 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -133,7 +133,7 @@ def print_log(log) -> None: for m in log: print_with_timestamp(m) -TAP_ENTRIES = re.compile(r'^(TAP|[\s]*ok|[\s]*not ok|[\s]*[0-9]+\.\.[0-9]+|[\s]*#).*$') +TAP_ENTRIES = re.compile(r'^(TAP|[\s]*ok|[\s]*not ok|[\s]*[0-9]+\.\.[0-9]+|[\s]*# (Subtest:|.*: kunit test case crashed!)).*$') def consume_non_diagnostic(lines: LineStream) -> None: while lines and not TAP_ENTRIES.match(lines.peek()):