From: Heidi Fahim Date: Mon, 16 Mar 2020 20:21:24 +0000 (-0700) Subject: kunit: kunit_parser: make parser more robust X-Git-Tag: v5.15~4187^2~11 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=afc63da64f1e5e41875c98707020e85050f8a0c5;p=platform%2Fkernel%2Flinux-starfive.git kunit: kunit_parser: make parser more robust Previously, kunit_parser did not properly handle kunit TAP output that - had any prefixes (generated from different configs e.g. CONFIG_PRINTK_TIME) - had unrelated kernel output mixed in the middle of it, which has shown up when testing with allyesconfig To remove prefixes, the parser looks for the first line that includes TAP output, "TAP version 14". It then determines the length of the string before this sequence, and strips that number of characters off the beginning of the following lines until the last KUnit output line is reached. These fixes have been tested with additional tests in the KUnitParseTest and their associated logs have also been added. Signed-off-by: Heidi Fahim Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- diff --git a/tools/testing/kunit/.gitattributes b/tools/testing/kunit/.gitattributes new file mode 100644 index 0000000..5b7da1f --- /dev/null +++ b/tools/testing/kunit/.gitattributes @@ -0,0 +1 @@ +test_data/* binary diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 4ffbae0..adf8674 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -46,19 +46,21 @@ class TestStatus(Enum): TEST_CRASHED = auto() NO_TESTS = auto() -kunit_start_re = re.compile(r'^TAP version [0-9]+$') -kunit_end_re = re.compile('List of all partitions:') +kunit_start_re = re.compile(r'TAP version [0-9]+$') +kunit_end_re = re.compile('(List of all partitions:|' + 'Kernel panic - not syncing: VFS:|reboot: System halted)') def isolate_kunit_output(kernel_output): started = False for line in kernel_output: - if kunit_start_re.match(line): + if kunit_start_re.search(line): + prefix_len = len(line.split('TAP version')[0]) started = True - yield line - elif kunit_end_re.match(line): + yield line[prefix_len:] if prefix_len > 0 else line + elif kunit_end_re.search(line): break elif started: - yield line + yield line[prefix_len:] if prefix_len > 0 else line def raw_output(kernel_output): for line in kernel_output: @@ -108,18 +110,16 @@ OK_NOT_OK_SUBTEST = re.compile(r'^\t(ok|not ok) [0-9]+ - (.*)$') OK_NOT_OK_MODULE = re.compile(r'^(ok|not ok) [0-9]+ - (.*)$') -def parse_ok_not_ok_test_case(lines: List[str], - test_case: TestCase, - expecting_test_case: bool) -> bool: +def parse_ok_not_ok_test_case(lines: List[str], test_case: TestCase) -> bool: save_non_diagnositic(lines, test_case) if not lines: - if expecting_test_case: - test_case.status = TestStatus.TEST_CRASHED - return True - else: - return False + test_case.status = TestStatus.TEST_CRASHED + return True line = lines[0] match = OK_NOT_OK_SUBTEST.match(line) + while not match and lines: + line = lines.pop(0) + match = OK_NOT_OK_SUBTEST.match(line) if match: test_case.log.append(lines.pop(0)) test_case.name = match.group(2) @@ -150,12 +150,12 @@ def parse_diagnostic(lines: List[str], test_case: TestCase) -> bool: else: return False -def parse_test_case(lines: List[str], expecting_test_case: bool) -> TestCase: +def parse_test_case(lines: List[str]) -> TestCase: test_case = TestCase() save_non_diagnositic(lines, test_case) while parse_diagnostic(lines, test_case): pass - if parse_ok_not_ok_test_case(lines, test_case, expecting_test_case): + if parse_ok_not_ok_test_case(lines, test_case): return test_case else: return None @@ -234,11 +234,11 @@ def parse_test_suite(lines: List[str]) -> TestSuite: expected_test_case_num = parse_subtest_plan(lines) if not expected_test_case_num: return None - test_case = parse_test_case(lines, expected_test_case_num > 0) - expected_test_case_num -= 1 - while test_case: + while expected_test_case_num > 0: + test_case = parse_test_case(lines) + if not test_case: + break test_suite.cases.append(test_case) - test_case = parse_test_case(lines, expected_test_case_num > 0) expected_test_case_num -= 1 if parse_ok_not_ok_test_suite(lines, test_suite): test_suite.status = bubble_up_test_case_errors(test_suite) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index cba9775..0efae69 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -108,6 +108,36 @@ class KUnitParserTest(unittest.TestCase): self.assertContains('ok 1 - example', result) file.close() + def test_output_with_prefix_isolated_correctly(self): + log_path = get_absolute_path( + 'test_data/test_pound_sign.log') + with open(log_path) as file: + result = kunit_parser.isolate_kunit_output(file.readlines()) + self.assertContains('TAP version 14\n', result) + self.assertContains(' # Subtest: kunit-resource-test', result) + self.assertContains(' 1..5', result) + self.assertContains(' ok 1 - kunit_resource_test_init_resources', result) + self.assertContains(' ok 2 - kunit_resource_test_alloc_resource', result) + self.assertContains(' ok 3 - kunit_resource_test_destroy_resource', result) + self.assertContains(' foo bar #', result) + self.assertContains(' ok 4 - kunit_resource_test_cleanup_resources', result) + self.assertContains(' ok 5 - kunit_resource_test_proper_free_ordering', result) + self.assertContains('ok 1 - kunit-resource-test', result) + self.assertContains(' foo bar # non-kunit output', result) + self.assertContains(' # Subtest: kunit-try-catch-test', result) + self.assertContains(' 1..2', result) + self.assertContains(' ok 1 - kunit_test_try_catch_successful_try_no_catch', + result) + self.assertContains(' ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch', + result) + self.assertContains('ok 2 - kunit-try-catch-test', result) + self.assertContains(' # Subtest: string-stream-test', result) + self.assertContains(' 1..3', result) + self.assertContains(' ok 1 - string_stream_test_empty_on_creation', result) + self.assertContains(' ok 2 - string_stream_test_not_empty_after_add', result) + self.assertContains(' ok 3 - string_stream_test_get_string', result) + self.assertContains('ok 3 - string-stream-test', result) + def test_parse_successful_test_log(self): all_passed_log = get_absolute_path( 'test_data/test_is_test_passed-all_passed.log') @@ -150,6 +180,45 @@ class KUnitParserTest(unittest.TestCase): result.status) file.close() + def test_ignores_prefix_printk_time(self): + prefix_log = get_absolute_path( + 'test_data/test_config_printk_time.log') + with open(prefix_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_ignores_multiple_prefixes(self): + prefix_log = get_absolute_path( + 'test_data/test_multiple_prefixes.log') + with open(prefix_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_prefix_mixed_kernel_output(self): + mixed_prefix_log = get_absolute_path( + 'test_data/test_interrupted_tap_output.log') + with open(mixed_prefix_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_prefix_poundsign(self): + pound_log = get_absolute_path('test_data/test_pound_sign.log') + with open(pound_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_kernel_panic_end(self): + panic_log = get_absolute_path('test_data/test_kernel_panic_interrupt.log') + with open(panic_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_pound_no_prefix(self): + pound_log = get_absolute_path('test_data/test_pound_no_prefix.log') + with open(pound_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + class StrContains(str): def __eq__(self, other): return self in other diff --git a/tools/testing/kunit/test_data/test_config_printk_time.log b/tools/testing/kunit/test_data/test_config_printk_time.log new file mode 100644 index 0000000..c02ca77 --- /dev/null +++ b/tools/testing/kunit/test_data/test_config_printk_time.log @@ -0,0 +1,31 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000] ok 3 - string_stream_test_get_string +[ 0.060000] ok 3 - string-stream-test +[ 0.060000] List of all partitions: +[ 0.060000] No filesystem could mount root, tried: +[ 0.060000] +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010 \ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_interrupted_tap_output.log b/tools/testing/kunit/test_data/test_interrupted_tap_output.log new file mode 100644 index 0000000..5c73fb3 --- /dev/null +++ b/tools/testing/kunit/test_data/test_interrupted_tap_output.log @@ -0,0 +1,37 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] kAFS: Red Hat AFS client v0.1 registering. +[ 0.060000] FS-Cache: Netfs 'afs' registered for caching +[ 0.060000] *** VALIDATE kAFS *** +[ 0.060000] Btrfs loaded, crc32c=crc32c-generic, debug=on, assert=on, integrity-checker=on, ref-verify=on +[ 0.060000] BTRFS: selftest: sectorsize: 4096 nodesize: 4096 +[ 0.060000] BTRFS: selftest: running btrfs free space cache tests +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000] ok 3 - string_stream_test_get_string +[ 0.060000] ok 3 - string-stream-test +[ 0.060000] List of all partitions: +[ 0.060000] No filesystem could mount root, tried: +[ 0.060000] +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010 \ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_kernel_panic_interrupt.log b/tools/testing/kunit/test_data/test_kernel_panic_interrupt.log new file mode 100644 index 0000000..c045eee --- /dev/null +++ b/tools/testing/kunit/test_data/test_kernel_panic_interrupt.log @@ -0,0 +1,25 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010 \ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_multiple_prefixes.log b/tools/testing/kunit/test_data/test_multiple_prefixes.log new file mode 100644 index 0000000..bc48407 --- /dev/null +++ b/tools/testing/kunit/test_data/test_multiple_prefixes.log @@ -0,0 +1,31 @@ +[ 0.060000][ T1] printk: console [mc-1] enabled +[ 0.060000][ T1] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000][ T1] TAP version 14 +[ 0.060000][ T1] # Subtest: kunit-resource-test +[ 0.060000][ T1] 1..5 +[ 0.060000][ T1] ok 1 - kunit_resource_test_init_resources +[ 0.060000][ T1] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000][ T1] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000][ T1] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000][ T1] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000][ T1] ok 1 - kunit-resource-test +[ 0.060000][ T1] # Subtest: kunit-try-catch-test +[ 0.060000][ T1] 1..2 +[ 0.060000][ T1] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000][ T1] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000][ T1] ok 2 - kunit-try-catch-test +[ 0.060000][ T1] # Subtest: string-stream-test +[ 0.060000][ T1] 1..3 +[ 0.060000][ T1] ok 1 - string_stream_test_empty_on_creation +[ 0.060000][ T1] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000][ T1] ok 3 - string_stream_test_get_string +[ 0.060000][ T1] ok 3 - string-stream-test +[ 0.060000][ T1] List of all partitions: +[ 0.060000][ T1] No filesystem could mount root, tried: +[ 0.060000][ T1] +[ 0.060000][ T1] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000][ T1] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000][ T1] Stack: +[ 0.060000][ T1] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000][ T1] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000][ T1] 6005fcec 601bc260 705c0000 3000000010 \ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_output_with_prefix_isolated_correctly.log b/tools/testing/kunit/test_data/test_output_with_prefix_isolated_correctly.log new file mode 100644 index 0000000..0f87cda --- /dev/null +++ b/tools/testing/kunit/test_data/test_output_with_prefix_isolated_correctly.log @@ -0,0 +1,33 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] foo bar # +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] foo bar # non-kunit output +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000] ok 3 - string_stream_test_get_string +[ 0.060000] ok 3 - string-stream-test +[ 0.060000] List of all partitions: +[ 0.060000] No filesystem could mount root, tried: +[ 0.060000] +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010 \ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_pound_no_prefix.log b/tools/testing/kunit/test_data/test_pound_no_prefix.log new file mode 100644 index 0000000..2ceb360 --- /dev/null +++ b/tools/testing/kunit/test_data/test_pound_no_prefix.log @@ -0,0 +1,33 @@ + printk: console [mc-1] enabled + random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 + TAP version 14 + # Subtest: kunit-resource-test + 1..5 + ok 1 - kunit_resource_test_init_resources + ok 2 - kunit_resource_test_alloc_resource + ok 3 - kunit_resource_test_destroy_resource + foo bar # + ok 4 - kunit_resource_test_cleanup_resources + ok 5 - kunit_resource_test_proper_free_ordering + ok 1 - kunit-resource-test + foo bar # non-kunit output + # Subtest: kunit-try-catch-test + 1..2 + ok 1 - kunit_test_try_catch_successful_try_no_catch + ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch + ok 2 - kunit-try-catch-test + # Subtest: string-stream-test + 1..3 + ok 1 - string_stream_test_empty_on_creation + ok 2 - string_stream_test_not_empty_after_add + ok 3 - string_stream_test_get_string + ok 3 - string-stream-test + List of all partitions: + No filesystem could mount root, tried: + + Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) + CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 + Stack: + 602086f8 601bc260 705c0000 705c0000 + 602086f8 6005fcec 705c0000 6002c6ab + 6005fcec 601bc260 705c0000 3000000010 \ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_pound_sign.log b/tools/testing/kunit/test_data/test_pound_sign.log new file mode 100644 index 0000000..e69de29