validate:launcher: Add a way to fail if test have been removed/added

diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py
index c74b361..3b031f2 100644 (file)
--- a/validate/launcher/baseclasses.py
+++ b/validate/launcher/baseclasses.py
@@ -1252,6 +1252,7 @@ class _TestsLauncher(Loggable):
             return
 
         tests_names = [test.classname for test in tests]
+        testlist_changed = False
         for testsuite in self.options.testsuites:
             if not self._check_tester_has_other_testsuite(testsuite, tester) \
                     and tester.check_testslist:
@@ -1269,6 +1270,7 @@ class _TestsLauncher(Loggable):
 
                 for test in know_tests:
                     if test and test not in tests_names:
+                        testlist_changed = True
                         printc("Test %s Not in testsuite %s anymore"
                                % (test, testsuite.__file__), Colors.FAIL)
 
@@ -1277,14 +1279,20 @@ class _TestsLauncher(Loggable):
                     if test and test not in know_tests:
                         printc("Test %s is NEW in testsuite %s"
                                % (test, testsuite.__file__), Colors.OKGREEN)
+                        testlist_changed = True
 
                 testlist_file.close()
-                return
+                break
+
+        return testlist_changed
 
     def list_tests(self):
         for tester in self.testers:
             tests = tester.list_tests()
-            self._check_defined_tests(tester, tests)
+            if self._check_defined_tests(tester, tests) and \
+                    self.options.fail_on_testlist_change:
+                return -1
+
             self.tests.extend(tests)
         return sorted(list(self.tests))
 

diff --git a/validate/launcher/main.py b/validate/launcher/main.py
index 5dc77c9..85344f0 100644 (file)
--- a/validate/launcher/main.py
+++ b/validate/launcher/main.py
@@ -375,6 +375,12 @@ Note that all testsuite should be inside python modules, so the directory should
     parser.add_argument("-F", "--fatal-error", dest="fatal_error",
                         action="store_true",
                         help="Stop on first fail")
+    parser.add_argument("--fail-on-testlist-change",
+                        dest="fail_on_testlist_change",
+                        action="store_true",
+                        help="Fail the testsuite if a test has been added"
+                        " or removed without being explicitely added/removed "
+                        "from the testlist file.")
     parser.add_argument("-t", "--wanted-tests", dest="wanted_tests",
                         action="append",
                         help="Define the tests to execute, it can be a regex."
@@ -510,7 +516,10 @@ Note that all testsuite should be inside python modules, so the directory should
     # Ensure that the scenario manager singleton is ready to be used
     ScenarioManager().config = options
     tests_launcher.set_settings(options, [])
-    tests_launcher.list_tests()
+    if tests_launcher.list_tests() == -1:
+        printc("\nFailling as tests have been removed/added "
+               " (--fail-on-testlist-change)", Colors.FAIL)
+        exit(1)
 
     if options.list_tests:
         l = tests_launcher.tests