From 39a0be8e0e57debceab9e2c56008468049d3d11f Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 23 Feb 2016 11:29:56 +0100 Subject: [PATCH] validate:launcher: Add a way to fail if test have been removed/added --- validate/launcher/baseclasses.py | 12 ++++++++++-- validate/launcher/main.py | 11 ++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index c74b361..3b031f2 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -1252,6 +1252,7 @@ class _TestsLauncher(Loggable): return tests_names = [test.classname for test in tests] + testlist_changed = False for testsuite in self.options.testsuites: if not self._check_tester_has_other_testsuite(testsuite, tester) \ and tester.check_testslist: @@ -1269,6 +1270,7 @@ class _TestsLauncher(Loggable): for test in know_tests: if test and test not in tests_names: + testlist_changed = True printc("Test %s Not in testsuite %s anymore" % (test, testsuite.__file__), Colors.FAIL) @@ -1277,14 +1279,20 @@ class _TestsLauncher(Loggable): if test and test not in know_tests: printc("Test %s is NEW in testsuite %s" % (test, testsuite.__file__), Colors.OKGREEN) + testlist_changed = True testlist_file.close() - return + break + + return testlist_changed def list_tests(self): for tester in self.testers: tests = tester.list_tests() - self._check_defined_tests(tester, tests) + if self._check_defined_tests(tester, tests) and \ + self.options.fail_on_testlist_change: + return -1 + self.tests.extend(tests) return sorted(list(self.tests)) diff --git a/validate/launcher/main.py b/validate/launcher/main.py index 5dc77c9..85344f0 100644 --- a/validate/launcher/main.py +++ b/validate/launcher/main.py @@ -375,6 +375,12 @@ Note that all testsuite should be inside python modules, so the directory should parser.add_argument("-F", "--fatal-error", dest="fatal_error", action="store_true", help="Stop on first fail") + parser.add_argument("--fail-on-testlist-change", + dest="fail_on_testlist_change", + action="store_true", + help="Fail the testsuite if a test has been added" + " or removed without being explicitely added/removed " + "from the testlist file.") parser.add_argument("-t", "--wanted-tests", dest="wanted_tests", action="append", help="Define the tests to execute, it can be a regex." @@ -510,7 +516,10 @@ Note that all testsuite should be inside python modules, so the directory should # Ensure that the scenario manager singleton is ready to be used ScenarioManager().config = options tests_launcher.set_settings(options, []) - tests_launcher.list_tests() + if tests_launcher.list_tests() == -1: + printc("\nFailling as tests have been removed/added " + " (--fail-on-testlist-change)", Colors.FAIL) + exit(1) if options.list_tests: l = tests_launcher.tests -- 2.7.4