scripts/check_swiftshader_runtime.py

   1 # Copyright 2021 Google LLC.
   2 # Copyright 2021 The Khronos Group Inc.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     https://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # - GO needs to be installed to use regres. (apt install golang-go)
  17
  18 import os
  19 import json
  20 import tempfile
  21 import subprocess
  22
  23 from argparse import ArgumentParser
  24 from shutil import which, copyfile, move
  25 from pathlib import Path
  26 from datetime import datetime
  27
  28 AP = ArgumentParser()
  29 AP.add_argument(
  30     "-d",
  31     "--directory",
  32     metavar="DIRECTORY",
  33     type=str,
  34     help="Path to directory that will be used as root for cloning and file saving.",
  35     default=str(Path(tempfile.gettempdir()) / "deqp-swiftshader")
  36 )
  37 AP.add_argument(
  38     "-u",
  39     "--url",
  40     metavar="URL",
  41     type=str,
  42     help="URL of SwiftShader Git repository.",
  43     default="https://swiftshader.googlesource.com/SwiftShader",
  44 )
  45 AP.add_argument(
  46     "-l",
  47     "--vlayer_url",
  48     metavar="VURL",
  49     type=str,
  50     help="URL of Validation Layers Git repository.",
  51     default="https://github.com/KhronosGroup/Vulkan-ValidationLayers.git",
  52 )
  53 AP.add_argument(
  54     "-b",
  55     "--sws_build_type",
  56     metavar="SWS_BUILD_TYPE",
  57     type=str,
  58     help="SwiftShader build type.",
  59     choices=["debug", "release"],
  60     default="debug",
  61 )
  62 AP.add_argument(
  63     "-q",
  64     "--deqp_vk",
  65     metavar="DEQP_VK",
  66     type=str,
  67     help="Path to deqp-vk binary.",
  68 )
  69 AP.add_argument(
  70     "-v",
  71     "--vk_gl_cts",
  72     metavar="VK_GL_CTS",
  73     type=str,
  74     help="Path to vk-gl-cts source directory.",
  75 )
  76 AP.add_argument(
  77     "-w",
  78     "--vk_gl_cts_build",
  79     metavar="VK_GL_CTS_BUILD",
  80     type=str,
  81     help="Path to vk-gl-cts build directory.",
  82     default=str(Path(tempfile.gettempdir()) / "deqp-swiftshader" / "vk-gl-cts-build"),
  83 )
  84 AP.add_argument(
  85     "-t",
  86     "--vk_gl_cts_build_type",
  87     metavar="VK_GL_CTS_BUILD_TYPE",
  88     type=str,
  89     help="vk-gl-cts build type.",
  90     choices=["debug", "release"],
  91     default="debug",
  92 )
  93 AP.add_argument(
  94     "-r",
  95     "--recipe",
  96     metavar="RECIPE",
  97     type=str,
  98     help="Recipes to only run parts of script.",
  99     choices=["run-deqp", "check-comparison"],
 100     default="run-deqp",
 101 )
 102 AP.add_argument(
 103     "-f",
 104     "--files",
 105     nargs=2,
 106     metavar=("NEWER_FILE_PATH", "OLDER_FILE_PATH"),
 107     type=str,
 108     help="Compare two different run results.",
 109 )
 110 AP.add_argument(
 111     "-a",
 112     "--validation",
 113     metavar="VALIDATION",
 114     type=str,
 115     help="Enable vulkan validation layers.",
 116     choices=["true", "false"],
 117     default="false",
 118 )
 119 AP.add_argument(
 120     "-o",
 121     "--result_output",
 122     metavar="OUTPUT",
 123     type=str,
 124     help="Filename of the regres results.",
 125     default=str("result_" + str(datetime.now().strftime('%m_%d_%Y_%H_%M_%S')) + ".json"),
 126 )
 127
 128 ARGS = AP.parse_args()
 129
 130 # Check that we have everything needed to run the script when using recipe run-deqp.
 131 if ARGS.recipe == "run-deqp":
 132     if which("go") is None:
 133         raise RuntimeError("go not found.")
 134     if which("cmake") is None:
 135         raise RuntimeError("CMake not found.")
 136     if which("ninja") is None:
 137         raise RuntimeError("Ninja not found.")
 138     if which("git") is None:
 139         raise RuntimeError("Git not found.")
 140     if ARGS.vk_gl_cts is None:
 141         raise RuntimeError("vk-gl-cts source directory must be provided. Use --help for more info.")
 142
 143 PARENT_DIR = Path(ARGS.directory).resolve()
 144
 145 SWS_SRC_DIR = PARENT_DIR / "SwiftShader"
 146 SWS_BUILD_DIR = SWS_SRC_DIR / "build"
 147 SWIFTSHADER_URL = ARGS.url
 148
 149 LAYERS_PARENT_DIR = Path(ARGS.directory).resolve()
 150 LAYERS_SRC_DIR = LAYERS_PARENT_DIR / "Vulkan_Validation_Layers"
 151 LAYERS_URL = ARGS.vlayer_url
 152 LAYERS_BUILD_DIR = LAYERS_SRC_DIR / "build"
 153
 154 LINUX_SWS_ICD_DIR = SWS_BUILD_DIR / "Linux"
 155 REGRES_DIR = SWS_SRC_DIR / "tests" / "regres"
 156 RESULT_DIR = PARENT_DIR / "regres_results"
 157 COMP_RESULTS_DIR = PARENT_DIR / "comparison_results"
 158
 159 VK_GL_CTS_ROOT_DIR = Path(ARGS.vk_gl_cts)
 160 VK_GL_CTS_BUILD_DIR = Path(ARGS.vk_gl_cts_build)
 161 MUSTPASS_LIST = VK_GL_CTS_ROOT_DIR / "external" / "vulkancts" / "mustpass" / "master" / "vk-default.txt"
 162 if ARGS.deqp_vk is None:
 163     DEQP_VK_BINARY = VK_GL_CTS_BUILD_DIR / "external" / "vulkancts" / "modules" / "vulkan" / "deqp-vk"
 164 else:
 165     DEQP_VK_BINARY = str(ARGS.deqp_vk)
 166
 167 new_pass = []
 168 new_fail = []
 169 new_crash = []
 170 new_notsupported = []
 171 has_been_removed = []
 172 status_change = []
 173 compatibility_warning = []
 174 quality_warning = []
 175 internal_errors = []
 176 waivers = []
 177
 178 class Result:
 179     def __init__(self, filename):
 180         self.filename = filename
 181         self.f = open(filename)
 182         # Skip the first four lines and check that the file order has not been changed.
 183         tmp = ""
 184         for i in range(4):
 185             tmp = tmp + self.f.readline()
 186         if "Tests" not in tmp:
 187             raise RuntimeError("Skipped four lines, no starting line found. Has the file order changed?")
 188
 189     # Reads one test item from the file.
 190     def readResult(self):
 191         while True:
 192             tmp = ""
 193             while "}" not in tmp:
 194                 tmp = tmp + self.f.readline()
 195             if "Test" in tmp:
 196                 tmp = tmp[tmp.find("{") : tmp.find("}") + 1]
 197                 return json.loads(tmp)
 198             else:
 199                 return None
 200
 201     # Search for a test name. Returns the test data if found and otherwise False.
 202     def searchTest(self, test):
 203         line = self.f.readline()
 204         while line:
 205             if line.find(test) != -1:
 206                 # Found the test.
 207                 while "}" not in line:
 208                     line = line + self.f.readline()
 209
 210                 line = line[line.find("{") : line.find("}") + 1]
 211                 return json.loads(line)
 212             line = self.f.readline()
 213
 214 # Run deqp-vk with regres.
 215 def runDeqp(deqp_path, testlist_path):
 216     deqpVkParam = "--deqp-vk=" + deqp_path
 217     validationLayerParam = "--validation=" + ARGS.validation
 218     testListParam = "--test-list=" + testlist_path
 219     run(["./run_testlist.sh", deqpVkParam, validationLayerParam, testListParam], working_dir=REGRES_DIR)
 220
 221 # Run commands.
 222 def run(command: str, working_dir: str = Path.cwd()) -> None:
 223     """Run command using subprocess.run()"""
 224     subprocess.run(command, cwd=working_dir, check=True)
 225
 226 # Set VK_ICD_FILENAMES
 227 def setVkIcdFilenames():
 228     os.environ["VK_ICD_FILENAMES"] = str(LINUX_SWS_ICD_DIR / "vk_swiftshader_icd.json")
 229     print(f"VK_ICD_FILENAMES = {os.getenv('VK_ICD_FILENAMES')}")
 230
 231 # Choose the category/status to write results to.
 232 def writeToStatus(test):
 233     if test['Status'] == "PASS":
 234         new_pass.append(test['Test'])
 235     elif test['Status'] == "FAIL":
 236         new_fail.append(test['Test'])
 237     elif test['Status'] == "NOT_SUPPORTED" or test['Status'] == "UNSUPPORTED":
 238         new_notsupported.append(test['Test'])
 239     elif test['Status'] == "CRASH":
 240         new_crash.append(test['Test'])
 241     elif test['Status'] == "COMPATIBILITY_WARNING":
 242         compatibility_warning.append(test['Test'])
 243     elif test['Status'] == "QUALITY_WARNING":
 244         quality_warning.append(test['Test'])
 245     elif test['Status'] == "INTERNAL_ERROR":
 246         internal_errors.append(test['Test'])
 247     elif test['Status'] == "WAIVER":
 248         waivers.append(test['Test'])
 249     else:
 250         raise RuntimeError(f"Expected PASS, FAIL, NOT_SUPPORTED, UNSUPPORTED, CRASH, COMPATIBILITY_WARNING, " +
 251                            f"QUALITY_WARNING, INTERNAL_ERROR or WAIVER as status, " +
 252                            f"got {test['Status']}. Is there an unhandled status case?")
 253
 254 # Compare two result.json files for regression.
 255 def compareRuns(new_result, old_result):
 256     print(f"Comparing files: {old_result} and {new_result}")
 257
 258     r0 = Result(new_result)
 259     r1 = Result(old_result)
 260
 261     t0 = r0.readResult()
 262     t1 = r1.readResult()
 263
 264     done = False
 265
 266     while not done:
 267         # Old result file has ended, continue with new.
 268         if t1 == None and t0 != None:
 269             advance1 = False
 270             writeToStatus(t0)
 271         # New result file has ended, continue with old.
 272         elif t0 == None and t1 != None:
 273             advance0 = False
 274             has_been_removed.append(t1['Test'])
 275         # Both files have ended, stop iteration.
 276         elif t1 == None and t0 == None:
 277             done = True
 278         # By default advance both files.
 279         else:
 280             advance0 = True
 281             advance1 = True
 282
 283             if t0['Test'] == t1['Test']:
 284                 # The normal case where both files are in sync. Just check if the status matches.
 285                 if t0['Status'] != t1['Status']:
 286                     status_change.append(f"{t0['Test']}, new status: {t0['Status']}, old status: {t1['Status']}")
 287                     print(f"Status changed: {t0['Test']} {t0['Status']} vs {t1['Status']}")
 288             else:
 289                 # Create temporary objects for searching through the whole file.
 290                 tmp0 = Result(r0.filename)
 291                 tmp1 = Result(r1.filename)
 292
 293                 # Search the mismatching test cases from the opposite file.
 294                 s0 = tmp0.searchTest(t1['Test'])
 295                 s1 = tmp1.searchTest(t0['Test'])
 296
 297                 # Old test not in new results
 298                 if not s0:
 299                     print(f"Missing old test {t1['Test']} from new file: {r0.filename}\n")
 300                     has_been_removed.append(t1['Test'])
 301                     # Don't advance this file since we already read a test case other than the missing one.
 302                     advance0 = False
 303
 304                 # New test not in old results
 305                 if not s1:
 306                     print(f"Missing new test {t0['Test']} from old file: {r1.filename}\n")
 307                     writeToStatus(t0)
 308                     # Don't advance this file since we already read a test case other than the missing one.
 309                     advance1 = False
 310
 311                 if s0 and s1:
 312                     # This should never happen because the test cases are in alphabetical order.
 313                     # Print an error and bail out.
 314                     raise RuntimeError(f"Tests in different locations: {t0['Test']}\n")
 315
 316             if not advance0 and not advance1:
 317                 # An exotic case where both tests are missing from the other file.
 318                 # Need to skip both.
 319                 advance0 = True
 320                 advance1 = True
 321
 322         if advance0:
 323             t0 = r0.readResult()
 324         if advance1:
 325             t1 = r1.readResult()
 326
 327     result_file = str(COMP_RESULTS_DIR / "comparison_results_") + str(datetime.now().strftime('%m_%d_%Y_%H_%M_%S')) + ".txt"
 328     print(f"Writing to file {result_file}")
 329     COMP_RESULTS_DIR.mkdir(parents=True, exist_ok=True)
 330
 331     with open(result_file, "w") as log_file:
 332         log_file.write("New passes:\n")
 333         for line in new_pass:
 334             log_file.write(line + "\n")
 335         log_file.write("\n")
 336
 337         log_file.write("New fails:\n")
 338         for line in new_fail:
 339             log_file.write(line + "\n")
 340         log_file.write("\n")
 341
 342         log_file.write("New crashes:\n")
 343         for line in new_crash:
 344             log_file.write(line + "\n")
 345         log_file.write("\n")
 346
 347         log_file.write("New not_supported:\n")
 348         for line in new_notsupported:
 349             log_file.write(line + "\n")
 350         log_file.write("\n")
 351
 352         log_file.write("Tests removed:\n")
 353         for line in has_been_removed:
 354             log_file.write(line + "\n")
 355         log_file.write("\n")
 356
 357         log_file.write("Status changes:\n")
 358         for line in status_change:
 359             log_file.write(line + "\n")
 360         log_file.write("\n")
 361
 362         log_file.write("Compatibility warnings:\n")
 363         for line in compatibility_warning:
 364             log_file.write(line + "\n")
 365         log_file.write("\n")
 366
 367         log_file.write("Quality warnings:\n")
 368         for line in quality_warning:
 369             log_file.write(line + "\n")
 370         log_file.write("\n")
 371
 372         log_file.write("Internal errors:\n")
 373         for line in internal_errors:
 374             log_file.write(line + "\n")
 375         log_file.write("\n")
 376
 377         log_file.write("Waiver:\n")
 378         for line in waivers:
 379             log_file.write(line + "\n")
 380
 381     print(f"Comparison done. Results have been written to: {COMP_RESULTS_DIR}")
 382
 383 # Build VK-GL-CTS
 384 def buildCts():
 385     VK_GL_CTS_BUILD_DIR.mkdir(parents=True, exist_ok=True)
 386
 387     FETCH_SOURCES = str(VK_GL_CTS_ROOT_DIR / "external" / "fetch_sources.py")
 388     run([which("python3"), FETCH_SOURCES], working_dir=VK_GL_CTS_ROOT_DIR)
 389
 390     # Build VK-GL-CTS
 391     buildType = "-DCMAKE_BUILD_TYPE=" + ARGS.vk_gl_cts_build_type
 392     run([which("cmake"), "-GNinja", str(VK_GL_CTS_ROOT_DIR), buildType], working_dir=VK_GL_CTS_BUILD_DIR)
 393     run([which("ninja")], working_dir=VK_GL_CTS_BUILD_DIR)
 394     print(f"vk-gl-cts built to: {VK_GL_CTS_BUILD_DIR}")
 395
 396 # Clone and build SwiftShader and Vulkan validation layers.
 397 def cloneSwsAndLayers():
 398     # Clone SwiftShader or update if it already exists.
 399     if not SWS_SRC_DIR.exists():
 400         SWS_SRC_DIR.mkdir(parents=True, exist_ok=True)
 401         run([which("git"), "clone", SWIFTSHADER_URL, SWS_SRC_DIR])
 402     else:
 403         run([which("git"), "pull", "origin"], working_dir=SWS_SRC_DIR)
 404
 405     # Build SwiftShader.
 406     buildType = "-DCMAKE_BUILD_TYPE=" + ARGS.sws_build_type
 407     # Set env variables if clang build path is set.
 408     if os.getenv("CXX") is None:
 409         os.environ["CXX"] = "clang++"
 410     if os.getenv("CC") is None:
 411         os.environ["CC"] = "clang"
 412     run([which("cmake"), "-GNinja", str(SWS_SRC_DIR), buildType], working_dir=SWS_BUILD_DIR)
 413     run([which("cmake"), "--build", "."], working_dir=SWS_BUILD_DIR)
 414
 415     # Set Vulkan validation layers if flag is set.
 416     if ARGS.validation == "true":
 417         # Clone Vulkan validation layers or update if they already exist.
 418         if not LAYERS_SRC_DIR.exists():
 419             LAYERS_SRC_DIR.mkdir(parents=True, exist_ok=True)
 420             run([which("git"), "clone", LAYERS_URL, LAYERS_SRC_DIR])
 421         else:
 422             run([which("git"), "pull", "origin"], working_dir=LAYERS_SRC_DIR)
 423
 424         # Build and set Vulkan validation layers.
 425         LAYERS_BUILD_DIR.mkdir(parents=True, exist_ok=True)
 426         UPDATE_DEPS = str(LAYERS_SRC_DIR / "scripts" / "update_deps.py")
 427         run([which("python3"), UPDATE_DEPS], working_dir=LAYERS_BUILD_DIR)
 428         run([which("cmake"),
 429                 "-GNinja",
 430                 "-C",
 431                 "helper.cmake",
 432                 LAYERS_SRC_DIR],
 433                 working_dir=LAYERS_BUILD_DIR)
 434         run([which("cmake"), "--build", "."], working_dir=LAYERS_BUILD_DIR)
 435         LAYERS_PATH = str(LAYERS_BUILD_DIR / "layers")
 436         os.environ["VK_LAYER_PATH"] = LAYERS_PATH
 437     print(f"Tools cloned and built in: {PARENT_DIR}")
 438
 439 # Run cts with regres and move result files accordingly.
 440 def runCts():
 441     setVkIcdFilenames()
 442
 443     # Run cts and copy the resulting file to RESULT_DIR.
 444     print("Running cts...")
 445     runDeqp(str(DEQP_VK_BINARY), str(MUSTPASS_LIST))
 446     RESULT_DIR.mkdir(parents=True, exist_ok=True)
 447     copyfile(str(REGRES_DIR / "results.json"), str(RESULT_DIR / ARGS.result_output))
 448     print("Run completed.")
 449     print(f"Result file copied to: {RESULT_DIR}")
 450     exit(0)
 451
 452 # Recipe for running cts.
 453 if ARGS.recipe == "run-deqp":
 454     cloneSwsAndLayers()
 455     if ARGS.deqp_vk is None:
 456         buildCts()
 457     runCts()
 458
 459 # Recipe for only comparing the already existing result files.
 460 if ARGS.recipe == "check-comparison":
 461     if ARGS.files is None:
 462         raise RuntimeError("No comparable files provided. Please provide them with flag --files. Use --help for more info.")
 463     newFile, oldFile = ARGS.files
 464     compareRuns(str(newFile), str(oldFile))