From f8ed1a0772784d3e6c6a15168040f8718100b444 Mon Sep 17 00:00:00 2001 From: Mark Lobodzinski Date: Mon, 8 Jan 2018 09:08:06 -0700 Subject: [PATCH] scripts: Default doc validator to non-verbose mode Change-Id: Ib62e92a3b84609df55179e4868e5030a010f712f --- scripts/vk_validation_stats.py | 31 +++++++++++++++++++------------ tests/_run_all_tests.ps1 | 2 +- tests/_vkvalidatelayerdoc.ps1 | 6 +----- tests/run_all_tests.sh | 2 +- tests/vkvalidatelayerdoc.sh | 2 +- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/scripts/vk_validation_stats.py b/scripts/vk_validation_stats.py index 58c5c0b..0ec5797 100755 --- a/scripts/vk_validation_stats.py +++ b/scripts/vk_validation_stats.py @@ -24,6 +24,13 @@ import sys import platform # vk_validation_stats.py overview +# +# usage: +# python vk_validation_stats.py [verbose] +# +# Arguments: +# verbose - enables verbose output, including VUID duplicates +# # This script is intended to generate statistics on the state of validation code # based on information parsed from the source files and the database file # Here's what it currently does: @@ -303,7 +310,7 @@ class bcolors: def main(argv): result = 0 # Non-zero result indicates an error case - terse_mode = 'terse_mode' in sys.argv + verbose_mode = 'verbose' in sys.argv # parse db val_db = ValidationDatabase() val_db.read() @@ -320,14 +327,14 @@ def main(argv): # Process stats - Just doing this inline in main, could make a fancy class to handle # all the processing of data and then get results from that txt_color = bcolors() - if terse_mode == False: + if verbose_mode: print("Validation Statistics") else: - print("Validation/Documentation Consistency Test)") + print("Validation/Documentation Consistency Test") # First give number of checks in db & header and report any discrepancies db_enums = len(val_db.db_dict.keys()) hdr_enums = len(val_header.enums) - if not terse_mode: + if verbose_mode: print(" Database file includes %d unique checks" % (db_enums)) print(" Header file declares %d unique checks" % (hdr_enums)) @@ -346,7 +353,7 @@ def main(argv): if not tmp_db_dict.pop(enum, False): db_missing.append(enum) if db_enums == hdr_enums and len(db_missing) == 0 and len(tmp_db_dict.keys()) == 0: - if not terse_mode: + if verbose_mode: print(txt_color.green() + " Database and Header match, GREAT!" + txt_color.endc()) else: print(txt_color.red() + " Uh oh, Database doesn't match Header :(" + txt_color.endc()) @@ -372,15 +379,15 @@ def main(argv): multiple_uses = True if src_enum not in val_db.db_implemented_enums: imp_not_claimed.append(src_enum) - if not terse_mode: + if verbose_mode: print(" Database file claims that %d checks (%s) are implemented in source." % (len(val_db.db_implemented_enums), "{0:.0f}%".format(float(len(val_db.db_implemented_enums))/db_enums * 100))) - if len(val_db.db_unimplemented_implicit) > 0 and not terse_mode: + if len(val_db.db_unimplemented_implicit) > 0 and verbose_mode: print(" Database file claims %d implicit checks (%s) that are not implemented." % (len(val_db.db_unimplemented_implicit), "{0:.0f}%".format(float(len(val_db.db_unimplemented_implicit))/db_enums * 100))) total_checks = len(val_db.db_implemented_enums) + len(val_db.db_unimplemented_implicit) print(" If all implicit checks are handled by parameter validation this is a total of %d (%s) checks covered." % (total_checks, "{0:.0f}%".format(float(total_checks)/db_enums * 100))) if len(imp_not_found) == 0 and len(imp_not_claimed) == 0: - if not terse_mode: + if verbose_mode: print(txt_color.green() + " All claimed Database implemented checks have been found in source, and no source checks aren't claimed in Database, GREAT!" + txt_color.endc()) else: result = 1 @@ -394,7 +401,7 @@ def main(argv): for imp_enum in imp_not_claimed: print(txt_color.red() + " %s" % (imp_enum) + txt_color.endc()) - if multiple_uses and not terse_mode: + if multiple_uses and verbose_mode: print(txt_color.yellow() + " Note that some checks are used multiple times. These may be good candidates for new valid usage spec language." + txt_color.endc()) print(txt_color.yellow() + " Here is a list of each check used multiple times with its number of uses:" + txt_color.endc()) for enum in val_source.enum_count_dict: @@ -421,7 +428,7 @@ def main(argv): if testname not in tests_missing_enum: tests_missing_enum[testname] = [] tests_missing_enum[testname].append(enum) - if tests_missing_enum and not terse_mode: + if tests_missing_enum and verbose_mode: print(txt_color.yellow() + " \nThe following tests do not use their reported enums to check for the validation error. You may want to update these to pass the expected enum to SetDesiredFailureMsg:" + txt_color.endc()) for testname in tests_missing_enum: print(txt_color.yellow() + " Testname %s does not explicitly check for these ids:" % (testname) + txt_color.endc()) @@ -429,10 +436,10 @@ def main(argv): print(txt_color.yellow() + " %s" % (enum) + txt_color.endc()) # TODO : Go through all enums found in the test file and make sure they're correctly documented in the database file - if not terse_mode: + if verbose_mode: print(" Database file claims that %d checks have tests written." % len(val_db.db_enum_to_tests)) if len(bad_testnames) == 0: - if not terse_mode: + if verbose_mode: print(txt_color.green() + " All claimed tests have valid names. That's good!" + txt_color.endc()) else: print(txt_color.red() + " The following testnames in Database appear to be invalid:") diff --git a/tests/_run_all_tests.ps1 b/tests/_run_all_tests.ps1 index 05c2141..9c63adf 100644 --- a/tests/_run_all_tests.ps1 +++ b/tests/_run_all_tests.ps1 @@ -34,6 +34,6 @@ if ($lastexitcode -ne 0) { exit 1 } -& .\vkvalidatelayerdoc.ps1 terse_mode +& .\vkvalidatelayerdoc.ps1 exit $lastexitcode diff --git a/tests/_vkvalidatelayerdoc.ps1 b/tests/_vkvalidatelayerdoc.ps1 index a3b5dc3..1ef84d6 100644 --- a/tests/_vkvalidatelayerdoc.ps1 +++ b/tests/_vkvalidatelayerdoc.ps1 @@ -18,10 +18,6 @@ if ($args[0] -eq "-Debug") { $dPath = "Release" } -if (($args[0] -eq "terse_mode") -Or ($args[1] -eq "terse_mode")) { - $output_mode = "terse_mode" -} - write-host -background black -foreground green "[ RUN ] " -nonewline write-host "vkvalidatelayerdoc.ps1: Validate layer documentation" @@ -29,7 +25,7 @@ write-host "vkvalidatelayerdoc.ps1: Validate layer documentation" push-location ..\..\scripts # Validate that layer documentation matches source contents -python vk_validation_stats.py $output_mode +python vk_validation_stats.py # Report result based on exit code if (!$LASTEXITCODE) { diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index b47f888..8cf691e 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -10,7 +10,7 @@ set -e ./run_loader_tests.sh # Verify that validation checks in source match documentation -./vkvalidatelayerdoc.sh terse_mode +./vkvalidatelayerdoc.sh # vk_layer_validation_tests check to see that validation layers will # catch the errors that they are supposed to by intentionally doing things diff --git a/tests/vkvalidatelayerdoc.sh b/tests/vkvalidatelayerdoc.sh index ef25bd0..2ce25ac 100755 --- a/tests/vkvalidatelayerdoc.sh +++ b/tests/vkvalidatelayerdoc.sh @@ -24,7 +24,7 @@ printf "$GREEN[ RUN ]$NC $0\n" pushd ../../scripts # Validate that layer database matches source contents -python3 vk_validation_stats.py $1 +python3 vk_validation_stats.py RES=$? -- 2.7.4