From f9b63a1c91d4d2b4fcac5c786cdd7de1242c162b Mon Sep 17 00:00:00 2001 From: "Brenda J. Butler" Date: Fri, 23 Feb 2018 12:16:38 -0500 Subject: [PATCH] tools: tc-testing: better error reporting Do a better job with error handling - in pre- and post-suite, in pre- and post-case. Show a traceback for errors. Signed-off-by: Brenda J. Butler Signed-off-by: David S. Miller --- tools/testing/selftests/tc-testing/tdc.py | 84 +++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py index ab28373..6c220bc 100755 --- a/tools/testing/selftests/tc-testing/tdc.py +++ b/tools/testing/selftests/tc-testing/tdc.py @@ -15,6 +15,7 @@ import importlib import json import subprocess import time +import traceback from collections import OrderedDict from string import Template @@ -23,6 +24,13 @@ from tdc_helper import * import TdcPlugin + +class PluginMgrTestFail(Exception): + def __init__(self, stage, output, message): + self.stage = stage + self.output = output + self.message = message + class PluginMgr: def __init__(self, argparser): super().__init__() @@ -135,7 +143,7 @@ def exec_cmd(args, pm, stage, command): return proc, foutput -def prepare_env(args, pm, stage, prefix, cmdlist): +def prepare_env(args, pm, stage, prefix, cmdlist, output = None): """ Execute the setup/teardown commands for a test case. Optionally terminate test execution if the command fails. @@ -164,7 +172,9 @@ def prepare_env(args, pm, stage, prefix, cmdlist): print("\n{} *** Aborting test run.".format(prefix), file=sys.stderr) print("\n\n{} *** stdout ***".format(proc.stdout), file=sys.stderr) print("\n\n{} *** stderr ***".format(proc.stderr), file=sys.stderr) - raise Exception('"{}" did not complete successfully'.format(prefix)) + raise PluginMgrTestFail( + stage, output, + '"{}" did not complete successfully'.format(prefix)) def run_one_test(pm, args, index, tidx): result = True @@ -194,8 +204,11 @@ def run_one_test(pm, args, index, tidx): match_pattern = re.compile( str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE) (p, procout) = exec_cmd(args, pm, 'verify', tidx["verifyCmd"]) - match_index = re.findall(match_pattern, procout) - if len(match_index) != int(tidx["matchCount"]): + if procout: + match_index = re.findall(match_pattern, procout) + if len(match_index) != int(tidx["matchCount"]): + result = False + elif int(tidx["matchCount"]) != 0: result = False if not result: @@ -204,9 +217,12 @@ def run_one_test(pm, args, index, tidx): tap += tresult if result == False: - tap += procout + if procout: + tap += procout + else: + tap += 'No output!\n' - prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown']) + prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown'], procout) pm.call_post_case() index += 1 @@ -227,30 +243,70 @@ def test_runner(pm, args, filtered_tests): index = 1 tap = str(index) + ".." + str(tcount) + "\n" badtest = None + stage = None + emergency_exit = False + emergency_exit_message = '' - pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist]) - + try: + pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist]) + except Exception as ee: + ex_type, ex, ex_tb = sys.exc_info() + print('Exception {} {} (caught in pre_suite).'. + format(ex_type, ex)) + # when the extra print statements are uncommented, + # the traceback does not appear between them + # (it appears way earlier in the tdc.py output) + # so don't bother ... + # print('--------------------(') + # print('traceback') + traceback.print_tb(ex_tb) + # print('--------------------)') + emergency_exit_message = 'EMERGENCY EXIT, call_pre_suite failed with exception {} {}\n'.format(ex_type, ex) + emergency_exit = True + stage = 'pre-SUITE' + + if emergency_exit: + pm.call_post_suite(index) + return emergency_exit_message if args.verbose > 1: - print('Run tests here') + print('give test rig 2 seconds to stabilize') + time.sleep(2) for tidx in testlist: if "flower" in tidx["category"] and args.device == None: + if args.verbose > 1: + print('Not executing test {} {} because DEV2 not defined'. + format(tidx['id'], tidx['name'])) continue try: badtest = tidx # in case it goes bad tap += run_one_test(pm, args, index, tidx) - except Exception as ee: - print('Exception {} (caught in test_runner, running test {} {} {})'. - format(ee, index, tidx['id'], tidx['name'])) + except PluginMgrTestFail as pmtf: + ex_type, ex, ex_tb = sys.exc_info() + stage = pmtf.stage + message = pmtf.message + output = pmtf.output + print(message) + print('Exception {} {} (caught in test_runner, running test {} {} {} stage {})'. + format(ex_type, ex, index, tidx['id'], tidx['name'], stage)) + print('---------------') + print('traceback') + traceback.print_tb(ex_tb) + print('---------------') + if stage == 'teardown': + print('accumulated output for this test:') + if pmtf.output: + print(pmtf.output) + print('---------------') break index += 1 # if we failed in setup or teardown, - # fill in the remaining tests with not ok + # fill in the remaining tests with ok-skipped count = index tap += 'about to flush the tap output if tests need to be skipped\n' if tcount + 1 != index: for tidx in testlist[index - 1:]: - msg = 'skipped - previous setup or teardown failed' + msg = 'skipped - previous {} failed'.format(stage) tap += 'ok {} - {} # {} {} {}\n'.format( count, tidx['id'], msg, index, badtest.get('id', '--Unknown--')) count += 1 -- 2.7.4