scripts/check_swiftshader_runtime.py

   1 # Copyright 2021 Google LLC.
   2 # Copyright 2021 The Khronos Group Inc.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     https://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # Requirements to run the script:
  17 # - Python3 (apt-get install -y python3.x)
  18 # - GO      (apt-get install -y golang-go)
  19 # - cmake   (version 3.13 or later)
  20 # - ninja   (apt-get install -y ninja-build)
  21 # - git     (sudo apt-get install -y git)
  22
  23 # GO dependencies needed:
  24 # - crypto/openpgp (go get -u golang.org/x/crypto/openpgp...)
  25
  26 import os
  27 import json
  28 import tempfile
  29 import subprocess
  30 import sys
  31
  32 from argparse import ArgumentParser
  33 from shutil import which, copyfile
  34 from pathlib import Path
  35 from datetime import datetime
  36
  37 # Check for correct python version (python3) before doing anything.
  38 if sys.version_info.major < 3:
  39         raise RuntimeError("Python version needs to be 3 or greater.")
  40
  41 AP = ArgumentParser()
  42 AP.add_argument(
  43     "-d",
  44     "--directory",
  45     metavar="DIRECTORY",
  46     type=str,
  47     help="Path to directory that will be used as root for cloning and file saving.",
  48     default=str(Path(tempfile.gettempdir()) / "deqp-swiftshader")
  49 )
  50 AP.add_argument(
  51     "-u",
  52     "--url",
  53     metavar="URL",
  54     type=str,
  55     help="URL of SwiftShader Git repository.",
  56     default="https://swiftshader.googlesource.com/SwiftShader",
  57 )
  58 AP.add_argument(
  59     "-l",
  60     "--vlayer_url",
  61     metavar="VURL",
  62     type=str,
  63     help="URL of Validation Layers Git repository.",
  64     default="https://github.com/KhronosGroup/Vulkan-ValidationLayers.git",
  65 )
  66 AP.add_argument(
  67     "-b",
  68     "--sws_build_type",
  69     metavar="SWS_BUILD_TYPE",
  70     type=str,
  71     help="SwiftShader build type.",
  72     choices=["debug", "release"],
  73     default="debug",
  74 )
  75 AP.add_argument(
  76     "-q",
  77     "--deqp_vk",
  78     metavar="DEQP_VK",
  79     type=str,
  80     help="Path to deqp-vk binary.",
  81 )
  82 AP.add_argument(
  83     "-v",
  84     "--vk_gl_cts",
  85     metavar="VK_GL_CTS",
  86     type=str,
  87     help="Path to vk-gl-cts source directory.",
  88 )
  89 AP.add_argument(
  90     "-w",
  91     "--vk_gl_cts_build",
  92     metavar="VK_GL_CTS_BUILD",
  93     type=str,
  94     help="Path to vk-gl-cts build directory.",
  95     default=str(Path(tempfile.gettempdir()) / "deqp-swiftshader" / "vk-gl-cts-build"),
  96 )
  97 AP.add_argument(
  98     "-t",
  99     "--vk_gl_cts_build_type",
 100     metavar="VK_GL_CTS_BUILD_TYPE",
 101     type=str,
 102     help="vk-gl-cts build type.",
 103     choices=["debug", "release"],
 104     default="debug",
 105 )
 106 AP.add_argument(
 107     "-r",
 108     "--recipe",
 109     metavar="RECIPE",
 110     type=str,
 111     help="Recipes to only run parts of script.",
 112     choices=["run-deqp", "check-comparison"],
 113     default="run-deqp",
 114 )
 115 AP.add_argument(
 116     "-f",
 117     "--files",
 118     nargs=2,
 119     metavar=("NEWER_FILE_PATH", "OLDER_FILE_PATH"),
 120     type=str,
 121     help="Compare two different run results.",
 122 )
 123 AP.add_argument(
 124     "-a",
 125     "--validation",
 126     metavar="VALIDATION",
 127     type=str,
 128     help="Enable vulkan validation layers.",
 129     choices=["true", "false"],
 130     default="false",
 131 )
 132 AP.add_argument(
 133     "-o",
 134     "--result_output",
 135     metavar="OUTPUT",
 136     type=str,
 137     help="Filename of the regres results.",
 138     default=str("result_" + str(datetime.now().strftime('%m_%d_%Y_%H_%M_%S')) + ".json"),
 139 )
 140
 141 ARGS = AP.parse_args()
 142
 143 # Check that we have everything needed to run the script when using recipe run-deqp.
 144 if ARGS.recipe == "run-deqp":
 145     if which("go") is None:
 146         raise RuntimeError("go not found. (apt-get install -y golang-go)")
 147     if which("cmake") is None:
 148         raise RuntimeError("CMake not found. (version 3.13 or later needed)")
 149     if which("ninja") is None:
 150         raise RuntimeError("Ninja not found. (apt-get install -y ninja-build)")
 151     if which("git") is None:
 152         raise RuntimeError("Git not found. (apt-get install -y git)")
 153     if ARGS.vk_gl_cts is None:
 154         raise RuntimeError("vk-gl-cts source directory must be provided. Use --help for more info.")
 155
 156 PARENT_DIR = Path(ARGS.directory).resolve()
 157
 158 SWS_SRC_DIR = PARENT_DIR / "SwiftShader"
 159 SWS_BUILD_DIR = SWS_SRC_DIR / "build"
 160 SWIFTSHADER_URL = ARGS.url
 161
 162 LAYERS_PARENT_DIR = Path(ARGS.directory).resolve()
 163 LAYERS_SRC_DIR = LAYERS_PARENT_DIR / "Vulkan_Validation_Layers"
 164 LAYERS_URL = ARGS.vlayer_url
 165 LAYERS_BUILD_DIR = LAYERS_SRC_DIR / "build"
 166
 167 LINUX_SWS_ICD_DIR = SWS_BUILD_DIR / "Linux"
 168 REGRES_DIR = SWS_SRC_DIR / "tests" / "regres"
 169 RESULT_DIR = PARENT_DIR / "regres_results"
 170 COMP_RESULTS_DIR = PARENT_DIR / "comparison_results"
 171
 172 VK_GL_CTS_ROOT_DIR = Path(ARGS.vk_gl_cts)
 173 VK_GL_CTS_BUILD_DIR = Path(ARGS.vk_gl_cts_build)
 174 MUSTPASS_LIST = VK_GL_CTS_ROOT_DIR / "external" / "vulkancts" / "mustpass" / "master" / "vk-default.txt"
 175 if ARGS.deqp_vk is None:
 176     DEQP_VK_BINARY = VK_GL_CTS_BUILD_DIR / "external" / "vulkancts" / "modules" / "vulkan" / "deqp-vk"
 177 else:
 178     DEQP_VK_BINARY = str(ARGS.deqp_vk)
 179
 180 new_pass = []
 181 new_fail = []
 182 new_crash = []
 183 new_notsupported = []
 184 has_been_removed = []
 185 status_change = []
 186 compatibility_warning = []
 187 quality_warning = []
 188 internal_errors = []
 189 waivers = []
 190
 191 class Result:
 192     def __init__(self, filename):
 193         self.filename = filename
 194         self.f = open(filename)
 195         # Skip the first four lines and check that the file order has not been changed.
 196         tmp = ""
 197         for i in range(4):
 198             tmp = tmp + self.f.readline()
 199         if "Tests" not in tmp:
 200             raise RuntimeError("Skipped four lines, no starting line found. Has the file order changed?")
 201
 202     # Reads one test item from the file.
 203     def readResult(self):
 204         while True:
 205             tmp = ""
 206             while "}" not in tmp:
 207                 tmp = tmp + self.f.readline()
 208             if "Test" in tmp:
 209                 tmp = tmp[tmp.find("{") : tmp.find("}") + 1]
 210                 return json.loads(tmp)
 211             else:
 212                 return None
 213
 214     # Search for a test name. Returns the test data if found and otherwise False.
 215     def searchTest(self, test):
 216         line = self.f.readline()
 217         while line:
 218             if line.find(test) != -1:
 219                 # Found the test.
 220                 while "}" not in line:
 221                     line = line + self.f.readline()
 222
 223                 line = line[line.find("{") : line.find("}") + 1]
 224                 return json.loads(line)
 225             line = self.f.readline()
 226
 227 # Run deqp-vk with regres.
 228 def runDeqp(deqp_path, testlist_path):
 229     deqpVkParam = "--deqp-vk=" + deqp_path
 230     validationLayerParam = "--validation=" + ARGS.validation
 231     testListParam = "--test-list=" + testlist_path
 232     run(["./run_testlist.sh", deqpVkParam, validationLayerParam, testListParam], working_dir=REGRES_DIR)
 233
 234 # Run commands.
 235 def run(command: str, working_dir: str = Path.cwd()) -> None:
 236     """Run command using subprocess.run()"""
 237     subprocess.run(command, cwd=working_dir, check=True)
 238
 239 # Set VK_ICD_FILENAMES
 240 def setVkIcdFilenames():
 241     os.environ["VK_ICD_FILENAMES"] = str(LINUX_SWS_ICD_DIR / "vk_swiftshader_icd.json")
 242     print(f"VK_ICD_FILENAMES = {os.getenv('VK_ICD_FILENAMES')}")
 243
 244 # Choose the category/status to write results to.
 245 def writeToStatus(test):
 246     if test['Status'] == "PASS":
 247         new_pass.append(test['Test'])
 248     elif test['Status'] == "FAIL":
 249         new_fail.append(test['Test'])
 250     elif test['Status'] == "NOT_SUPPORTED" or test['Status'] == "UNSUPPORTED":
 251         new_notsupported.append(test['Test'])
 252     elif test['Status'] == "CRASH":
 253         new_crash.append(test['Test'])
 254     elif test['Status'] == "COMPATIBILITY_WARNING":
 255         compatibility_warning.append(test['Test'])
 256     elif test['Status'] == "QUALITY_WARNING":
 257         quality_warning.append(test['Test'])
 258     elif test['Status'] == "INTERNAL_ERROR":
 259         internal_errors.append(test['Test'])
 260     elif test['Status'] == "WAIVER":
 261         waivers.append(test['Test'])
 262     else:
 263         raise RuntimeError(f"Expected PASS, FAIL, NOT_SUPPORTED, UNSUPPORTED, CRASH, COMPATIBILITY_WARNING, " +
 264                            f"QUALITY_WARNING, INTERNAL_ERROR or WAIVER as status, " +
 265                            f"got {test['Status']}. Is there an unhandled status case?")
 266
 267 # Compare two result.json files for regression.
 268 def compareRuns(new_result, old_result):
 269     print(f"Comparing files: {old_result} and {new_result}")
 270
 271     r0 = Result(new_result)
 272     r1 = Result(old_result)
 273
 274     t0 = r0.readResult()
 275     t1 = r1.readResult()
 276
 277     done = False
 278
 279     while not done:
 280         # Old result file has ended, continue with new.
 281         if t1 == None and t0 != None:
 282             advance1 = False
 283             writeToStatus(t0)
 284         # New result file has ended, continue with old.
 285         elif t0 == None and t1 != None:
 286             advance0 = False
 287             has_been_removed.append(t1['Test'])
 288         # Both files have ended, stop iteration.
 289         elif t1 == None and t0 == None:
 290             done = True
 291         # By default advance both files.
 292         else:
 293             advance0 = True
 294             advance1 = True
 295
 296             if t0['Test'] == t1['Test']:
 297                 # The normal case where both files are in sync. Just check if the status matches.
 298                 if t0['Status'] != t1['Status']:
 299                     status_change.append(f"{t0['Test']}, new status: {t0['Status']}, old status: {t1['Status']}")
 300                     print(f"Status changed: {t0['Test']} {t0['Status']} vs {t1['Status']}")
 301             else:
 302                 # Create temporary objects for searching through the whole file.
 303                 tmp0 = Result(r0.filename)
 304                 tmp1 = Result(r1.filename)
 305
 306                 # Search the mismatching test cases from the opposite file.
 307                 s0 = tmp0.searchTest(t1['Test'])
 308                 s1 = tmp1.searchTest(t0['Test'])
 309
 310                 # Old test not in new results
 311                 if not s0:
 312                     print(f"Missing old test {t1['Test']} from new file: {r0.filename}\n")
 313                     has_been_removed.append(t1['Test'])
 314                     # Don't advance this file since we already read a test case other than the missing one.
 315                     advance0 = False
 316
 317                 # New test not in old results
 318                 if not s1:
 319                     print(f"Missing new test {t0['Test']} from old file: {r1.filename}\n")
 320                     writeToStatus(t0)
 321                     # Don't advance this file since we already read a test case other than the missing one.
 322                     advance1 = False
 323
 324                 if s0 and s1:
 325                     # This should never happen because the test cases are in alphabetical order.
 326                     # Print an error and bail out.
 327                     raise RuntimeError(f"Tests in different locations: {t0['Test']}\n")
 328
 329             if not advance0 and not advance1:
 330                 # An exotic case where both tests are missing from the other file.
 331                 # Need to skip both.
 332                 advance0 = True
 333                 advance1 = True
 334
 335         if advance0:
 336             t0 = r0.readResult()
 337         if advance1:
 338             t1 = r1.readResult()
 339
 340     result_file = str(COMP_RESULTS_DIR / "comparison_results_") + str(datetime.now().strftime('%m_%d_%Y_%H_%M_%S')) + ".txt"
 341     print(f"Writing to file {result_file}")
 342     COMP_RESULTS_DIR.mkdir(parents=True, exist_ok=True)
 343
 344     with open(result_file, "w") as log_file:
 345         log_file.write("New passes:\n")
 346         for line in new_pass:
 347             log_file.write(line + "\n")
 348         log_file.write("\n")
 349
 350         log_file.write("New fails:\n")
 351         for line in new_fail:
 352             log_file.write(line + "\n")
 353         log_file.write("\n")
 354
 355         log_file.write("New crashes:\n")
 356         for line in new_crash:
 357             log_file.write(line + "\n")
 358         log_file.write("\n")
 359
 360         log_file.write("New not_supported:\n")
 361         for line in new_notsupported:
 362             log_file.write(line + "\n")
 363         log_file.write("\n")
 364
 365         log_file.write("Tests removed:\n")
 366         for line in has_been_removed:
 367             log_file.write(line + "\n")
 368         log_file.write("\n")
 369
 370         log_file.write("Status changes:\n")
 371         for line in status_change:
 372             log_file.write(line + "\n")
 373         log_file.write("\n")
 374
 375         log_file.write("Compatibility warnings:\n")
 376         for line in compatibility_warning:
 377             log_file.write(line + "\n")
 378         log_file.write("\n")
 379
 380         log_file.write("Quality warnings:\n")
 381         for line in quality_warning:
 382             log_file.write(line + "\n")
 383         log_file.write("\n")
 384
 385         log_file.write("Internal errors:\n")
 386         for line in internal_errors:
 387             log_file.write(line + "\n")
 388         log_file.write("\n")
 389
 390         log_file.write("Waiver:\n")
 391         for line in waivers:
 392             log_file.write(line + "\n")
 393
 394     print(f"Comparison done. Results have been written to: {COMP_RESULTS_DIR}")
 395
 396 # Build VK-GL-CTS
 397 def buildCts():
 398     VK_GL_CTS_BUILD_DIR.mkdir(parents=True, exist_ok=True)
 399
 400     FETCH_SOURCES = str(VK_GL_CTS_ROOT_DIR / "external" / "fetch_sources.py")
 401     run([which("python3"), FETCH_SOURCES], working_dir=VK_GL_CTS_ROOT_DIR)
 402
 403     # Build VK-GL-CTS
 404     buildType = "-DCMAKE_BUILD_TYPE=" + ARGS.vk_gl_cts_build_type
 405     run([which("cmake"), "-GNinja", str(VK_GL_CTS_ROOT_DIR), buildType], working_dir=VK_GL_CTS_BUILD_DIR)
 406     run([which("ninja"), "deqp-vk"], working_dir=VK_GL_CTS_BUILD_DIR)
 407     print(f"vk-gl-cts built to: {VK_GL_CTS_BUILD_DIR}")
 408
 409 # Clone and build SwiftShader and Vulkan validation layers.
 410 def cloneSwsAndLayers():
 411     # Clone SwiftShader or update if it already exists.
 412     if not SWS_SRC_DIR.exists():
 413         SWS_SRC_DIR.mkdir(parents=True, exist_ok=True)
 414         run([which("git"), "clone", SWIFTSHADER_URL, SWS_SRC_DIR])
 415     else:
 416         run([which("git"), "pull", "origin"], working_dir=SWS_SRC_DIR)
 417
 418     # Build SwiftShader.
 419     run([which("cmake"),
 420             "-GNinja",
 421             str(SWS_SRC_DIR),
 422             "-DSWIFTSHADER_BUILD_EGL:BOOL=OFF",
 423             "-DSWIFTSHADER_BUILD_GLESv2:BOOL=OFF",
 424             "-DSWIFTSHADER_BUILD_TESTS:BOOL=OFF",
 425             "-DINSTALL_GTEST=OFF",
 426             "-DBUILD_TESTING:BOOL=OFF",
 427             "-DENABLE_CTEST:BOOL=OFF",
 428             "-DCMAKE_BUILD_TYPE=" + ARGS.sws_build_type],
 429             working_dir=SWS_BUILD_DIR)
 430     run([which("cmake"), "--build", ".", "--target", "vk_swiftshader"], working_dir=SWS_BUILD_DIR)
 431
 432     # Set Vulkan validation layers if flag is set.
 433     if ARGS.validation == "true":
 434         # Clone Vulkan validation layers or update if they already exist.
 435         if not LAYERS_SRC_DIR.exists():
 436             LAYERS_SRC_DIR.mkdir(parents=True, exist_ok=True)
 437             run([which("git"), "clone", LAYERS_URL, LAYERS_SRC_DIR])
 438         else:
 439             run([which("git"), "pull", "origin"], working_dir=LAYERS_SRC_DIR)
 440
 441         # Build and set Vulkan validation layers.
 442         LAYERS_BUILD_DIR.mkdir(parents=True, exist_ok=True)
 443         UPDATE_DEPS = str(LAYERS_SRC_DIR / "scripts" / "update_deps.py")
 444         run([which("python3"), UPDATE_DEPS], working_dir=LAYERS_BUILD_DIR)
 445         run([which("cmake"),
 446                 "-GNinja",
 447                 "-C",
 448                 "helper.cmake",
 449                 LAYERS_SRC_DIR],
 450                 working_dir=LAYERS_BUILD_DIR)
 451         run([which("cmake"), "--build", "."], working_dir=LAYERS_BUILD_DIR)
 452         LAYERS_PATH = str(LAYERS_BUILD_DIR / "layers")
 453         os.environ["VK_LAYER_PATH"] = LAYERS_PATH
 454     print(f"Tools cloned and built in: {PARENT_DIR}")
 455
 456 # Run cts with regres and move result files accordingly.
 457 def runCts():
 458     setVkIcdFilenames()
 459
 460     # Run cts and copy the resulting file to RESULT_DIR.
 461     print("Running cts...")
 462     runDeqp(str(DEQP_VK_BINARY), str(MUSTPASS_LIST))
 463     RESULT_DIR.mkdir(parents=True, exist_ok=True)
 464     copyfile(str(REGRES_DIR / "results.json"), str(RESULT_DIR / ARGS.result_output))
 465     print("Run completed.")
 466     print(f"Result file copied to: {RESULT_DIR}")
 467     exit(0)
 468
 469 # Recipe for running cts.
 470 if ARGS.recipe == "run-deqp":
 471     cloneSwsAndLayers()
 472     if ARGS.deqp_vk is None:
 473         buildCts()
 474     runCts()
 475
 476 # Recipe for only comparing the already existing result files.
 477 if ARGS.recipe == "check-comparison":
 478     if ARGS.files is None:
 479         raise RuntimeError("No comparable files provided. Please provide them with flag --files. Use --help for more info.")
 480     newFile, oldFile = ARGS.files
 481     compareRuns(str(newFile), str(oldFile))