Merge vk-gl-cts/vulkan-cts-1.2.7 into vk-gl-cts/vulkan-cts-1.2.8
[platform/upstream/VK-GL-CTS.git] / scripts / check_swiftshader_runtime.py
1 # Copyright 2021 Google LLC.
2 # Copyright 2021 The Khronos Group Inc.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     https://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 # Requirements to run the script:
17 # - Python3 (apt-get install -y python3.x)
18 # - GO      (apt-get install -y golang-go)
19 # - cmake   (version 3.13 or later)
20 # - ninja   (apt-get install -y ninja-build)
21 # - git     (sudo apt-get install -y git)
22
23 # GO dependencies needed:
24 # - crypto/openpgp (go get -u golang.org/x/crypto/openpgp...)
25
26 import os
27 import json
28 import tempfile
29 import subprocess
30 import sys
31
32 from argparse import ArgumentParser
33 from shutil import which, copyfile
34 from pathlib import Path
35 from datetime import datetime
36
37 # Check for correct python version (python3) before doing anything.
38 if sys.version_info.major < 3:
39         raise RuntimeError("Python version needs to be 3 or greater.")
40
41 AP = ArgumentParser()
42 AP.add_argument(
43     "-d",
44     "--directory",
45     metavar="DIRECTORY",
46     type=str,
47     help="Path to directory that will be used as root for cloning and file saving.",
48     default=str(Path(tempfile.gettempdir()) / "deqp-swiftshader")
49 )
50 AP.add_argument(
51     "-u",
52     "--url",
53     metavar="URL",
54     type=str,
55     help="URL of SwiftShader Git repository.",
56     default="https://swiftshader.googlesource.com/SwiftShader",
57 )
58 AP.add_argument(
59     "-l",
60     "--vlayer_url",
61     metavar="VURL",
62     type=str,
63     help="URL of Validation Layers Git repository.",
64     default="https://github.com/KhronosGroup/Vulkan-ValidationLayers.git",
65 )
66 AP.add_argument(
67     "-b",
68     "--sws_build_type",
69     metavar="SWS_BUILD_TYPE",
70     type=str,
71     help="SwiftShader build type.",
72     choices=["debug", "release"],
73     default="debug",
74 )
75 AP.add_argument(
76     "-q",
77     "--deqp_vk",
78     metavar="DEQP_VK",
79     type=str,
80     help="Path to deqp-vk binary.",
81 )
82 AP.add_argument(
83     "-v",
84     "--vk_gl_cts",
85     metavar="VK_GL_CTS",
86     type=str,
87     help="Path to vk-gl-cts source directory.",
88 )
89 AP.add_argument(
90     "-w",
91     "--vk_gl_cts_build",
92     metavar="VK_GL_CTS_BUILD",
93     type=str,
94     help="Path to vk-gl-cts build directory.",
95     default=str(Path(tempfile.gettempdir()) / "deqp-swiftshader" / "vk-gl-cts-build"),
96 )
97 AP.add_argument(
98     "-t",
99     "--vk_gl_cts_build_type",
100     metavar="VK_GL_CTS_BUILD_TYPE",
101     type=str,
102     help="vk-gl-cts build type.",
103     choices=["debug", "release"],
104     default="debug",
105 )
106 AP.add_argument(
107     "-r",
108     "--recipe",
109     metavar="RECIPE",
110     type=str,
111     help="Recipes to only run parts of script.",
112     choices=["run-deqp", "check-comparison"],
113     default="run-deqp",
114 )
115 AP.add_argument(
116     "-f",
117     "--files",
118     nargs=2,
119     metavar=("NEWER_FILE_PATH", "OLDER_FILE_PATH"),
120     type=str,
121     help="Compare two different run results.",
122 )
123 AP.add_argument(
124     "-a",
125     "--validation",
126     metavar="VALIDATION",
127     type=str,
128     help="Enable vulkan validation layers.",
129     choices=["true", "false"],
130     default="false",
131 )
132 AP.add_argument(
133     "-o",
134     "--result_output",
135     metavar="OUTPUT",
136     type=str,
137     help="Filename of the regres results.",
138     default=str("result_" + str(datetime.now().strftime('%m_%d_%Y_%H_%M_%S')) + ".json"),
139 )
140
141 ARGS = AP.parse_args()
142
143 # Check that we have everything needed to run the script when using recipe run-deqp.
144 if ARGS.recipe == "run-deqp":
145     if which("go") is None:
146         raise RuntimeError("go not found. (apt-get install -y golang-go)")
147     if which("cmake") is None:
148         raise RuntimeError("CMake not found. (version 3.13 or later needed)")
149     if which("ninja") is None:
150         raise RuntimeError("Ninja not found. (apt-get install -y ninja-build)")
151     if which("git") is None:
152         raise RuntimeError("Git not found. (apt-get install -y git)")
153     if ARGS.vk_gl_cts is None:
154         raise RuntimeError("vk-gl-cts source directory must be provided. Use --help for more info.")
155
156 PARENT_DIR = Path(ARGS.directory).resolve()
157
158 SWS_SRC_DIR = PARENT_DIR / "SwiftShader"
159 SWS_BUILD_DIR = SWS_SRC_DIR / "build"
160 SWIFTSHADER_URL = ARGS.url
161
162 LAYERS_PARENT_DIR = Path(ARGS.directory).resolve()
163 LAYERS_SRC_DIR = LAYERS_PARENT_DIR / "Vulkan_Validation_Layers"
164 LAYERS_URL = ARGS.vlayer_url
165 LAYERS_BUILD_DIR = LAYERS_SRC_DIR / "build"
166
167 LINUX_SWS_ICD_DIR = SWS_BUILD_DIR / "Linux"
168 REGRES_DIR = SWS_SRC_DIR / "tests" / "regres"
169 RESULT_DIR = PARENT_DIR / "regres_results"
170 COMP_RESULTS_DIR = PARENT_DIR / "comparison_results"
171
172 VK_GL_CTS_ROOT_DIR = Path(ARGS.vk_gl_cts)
173 VK_GL_CTS_BUILD_DIR = Path(ARGS.vk_gl_cts_build)
174 MUSTPASS_LIST = VK_GL_CTS_ROOT_DIR / "external" / "vulkancts" / "mustpass" / "master" / "vk-default.txt"
175 if ARGS.deqp_vk is None:
176     DEQP_VK_BINARY = VK_GL_CTS_BUILD_DIR / "external" / "vulkancts" / "modules" / "vulkan" / "deqp-vk"
177 else:
178     DEQP_VK_BINARY = str(ARGS.deqp_vk)
179
180 new_pass = []
181 new_fail = []
182 new_crash = []
183 new_notsupported = []
184 has_been_removed = []
185 status_change = []
186 compatibility_warning = []
187 quality_warning = []
188 internal_errors = []
189 waivers = []
190
191 class Result:
192     def __init__(self, filename):
193         self.filename = filename
194         self.f = open(filename)
195         # Skip the first four lines and check that the file order has not been changed.
196         tmp = ""
197         for i in range(4):
198             tmp = tmp + self.f.readline()
199         if "Tests" not in tmp:
200             raise RuntimeError("Skipped four lines, no starting line found. Has the file order changed?")
201
202     # Reads one test item from the file.
203     def readResult(self):
204         while True:
205             tmp = ""
206             while "}" not in tmp:
207                 tmp = tmp + self.f.readline()
208             if "Test" in tmp:
209                 tmp = tmp[tmp.find("{") : tmp.find("}") + 1]
210                 return json.loads(tmp)
211             else:
212                 return None
213
214     # Search for a test name. Returns the test data if found and otherwise False.
215     def searchTest(self, test):
216         line = self.f.readline()
217         while line:
218             if line.find(test) != -1:
219                 # Found the test.
220                 while "}" not in line:
221                     line = line + self.f.readline()
222
223                 line = line[line.find("{") : line.find("}") + 1]
224                 return json.loads(line)
225             line = self.f.readline()
226
227 # Run deqp-vk with regres.
228 def runDeqp(deqp_path, testlist_path):
229     deqpVkParam = "--deqp-vk=" + deqp_path
230     validationLayerParam = "--validation=" + ARGS.validation
231     testListParam = "--test-list=" + testlist_path
232     run(["./run_testlist.sh", deqpVkParam, validationLayerParam, testListParam], working_dir=REGRES_DIR)
233
234 # Run commands.
235 def run(command: str, working_dir: str = Path.cwd()) -> None:
236     """Run command using subprocess.run()"""
237     subprocess.run(command, cwd=working_dir, check=True)
238
239 # Set VK_ICD_FILENAMES
240 def setVkIcdFilenames():
241     os.environ["VK_ICD_FILENAMES"] = str(LINUX_SWS_ICD_DIR / "vk_swiftshader_icd.json")
242     print(f"VK_ICD_FILENAMES = {os.getenv('VK_ICD_FILENAMES')}")
243
244 # Choose the category/status to write results to.
245 def writeToStatus(test):
246     if test['Status'] == "PASS":
247         new_pass.append(test['Test'])
248     elif test['Status'] == "FAIL":
249         new_fail.append(test['Test'])
250     elif test['Status'] == "NOT_SUPPORTED" or test['Status'] == "UNSUPPORTED":
251         new_notsupported.append(test['Test'])
252     elif test['Status'] == "CRASH":
253         new_crash.append(test['Test'])
254     elif test['Status'] == "COMPATIBILITY_WARNING":
255         compatibility_warning.append(test['Test'])
256     elif test['Status'] == "QUALITY_WARNING":
257         quality_warning.append(test['Test'])
258     elif test['Status'] == "INTERNAL_ERROR":
259         internal_errors.append(test['Test'])
260     elif test['Status'] == "WAIVER":
261         waivers.append(test['Test'])
262     else:
263         raise RuntimeError(f"Expected PASS, FAIL, NOT_SUPPORTED, UNSUPPORTED, CRASH, COMPATIBILITY_WARNING, " +
264                            f"QUALITY_WARNING, INTERNAL_ERROR or WAIVER as status, " +
265                            f"got {test['Status']}. Is there an unhandled status case?")
266
267 # Compare two result.json files for regression.
268 def compareRuns(new_result, old_result):
269     print(f"Comparing files: {old_result} and {new_result}")
270
271     r0 = Result(new_result)
272     r1 = Result(old_result)
273
274     t0 = r0.readResult()
275     t1 = r1.readResult()
276
277     done = False
278
279     while not done:
280         # Old result file has ended, continue with new.
281         if t1 == None and t0 != None:
282             advance1 = False
283             writeToStatus(t0)
284         # New result file has ended, continue with old.
285         elif t0 == None and t1 != None:
286             advance0 = False
287             has_been_removed.append(t1['Test'])
288         # Both files have ended, stop iteration.
289         elif t1 == None and t0 == None:
290             done = True
291         # By default advance both files.
292         else:
293             advance0 = True
294             advance1 = True
295
296             if t0['Test'] == t1['Test']:
297                 # The normal case where both files are in sync. Just check if the status matches.
298                 if t0['Status'] != t1['Status']:
299                     status_change.append(f"{t0['Test']}, new status: {t0['Status']}, old status: {t1['Status']}")
300                     print(f"Status changed: {t0['Test']} {t0['Status']} vs {t1['Status']}")
301             else:
302                 # Create temporary objects for searching through the whole file.
303                 tmp0 = Result(r0.filename)
304                 tmp1 = Result(r1.filename)
305
306                 # Search the mismatching test cases from the opposite file.
307                 s0 = tmp0.searchTest(t1['Test'])
308                 s1 = tmp1.searchTest(t0['Test'])
309
310                 # Old test not in new results
311                 if not s0:
312                     print(f"Missing old test {t1['Test']} from new file: {r0.filename}\n")
313                     has_been_removed.append(t1['Test'])
314                     # Don't advance this file since we already read a test case other than the missing one.
315                     advance0 = False
316
317                 # New test not in old results
318                 if not s1:
319                     print(f"Missing new test {t0['Test']} from old file: {r1.filename}\n")
320                     writeToStatus(t0)
321                     # Don't advance this file since we already read a test case other than the missing one.
322                     advance1 = False
323
324                 if s0 and s1:
325                     # This should never happen because the test cases are in alphabetical order.
326                     # Print an error and bail out.
327                     raise RuntimeError(f"Tests in different locations: {t0['Test']}\n")
328
329             if not advance0 and not advance1:
330                 # An exotic case where both tests are missing from the other file.
331                 # Need to skip both.
332                 advance0 = True
333                 advance1 = True
334
335         if advance0:
336             t0 = r0.readResult()
337         if advance1:
338             t1 = r1.readResult()
339
340     result_file = str(COMP_RESULTS_DIR / "comparison_results_") + str(datetime.now().strftime('%m_%d_%Y_%H_%M_%S')) + ".txt"
341     print(f"Writing to file {result_file}")
342     COMP_RESULTS_DIR.mkdir(parents=True, exist_ok=True)
343
344     with open(result_file, "w") as log_file:
345         log_file.write("New passes:\n")
346         for line in new_pass:
347             log_file.write(line + "\n")
348         log_file.write("\n")
349
350         log_file.write("New fails:\n")
351         for line in new_fail:
352             log_file.write(line + "\n")
353         log_file.write("\n")
354
355         log_file.write("New crashes:\n")
356         for line in new_crash:
357             log_file.write(line + "\n")
358         log_file.write("\n")
359
360         log_file.write("New not_supported:\n")
361         for line in new_notsupported:
362             log_file.write(line + "\n")
363         log_file.write("\n")
364
365         log_file.write("Tests removed:\n")
366         for line in has_been_removed:
367             log_file.write(line + "\n")
368         log_file.write("\n")
369
370         log_file.write("Status changes:\n")
371         for line in status_change:
372             log_file.write(line + "\n")
373         log_file.write("\n")
374
375         log_file.write("Compatibility warnings:\n")
376         for line in compatibility_warning:
377             log_file.write(line + "\n")
378         log_file.write("\n")
379
380         log_file.write("Quality warnings:\n")
381         for line in quality_warning:
382             log_file.write(line + "\n")
383         log_file.write("\n")
384
385         log_file.write("Internal errors:\n")
386         for line in internal_errors:
387             log_file.write(line + "\n")
388         log_file.write("\n")
389
390         log_file.write("Waiver:\n")
391         for line in waivers:
392             log_file.write(line + "\n")
393
394     print(f"Comparison done. Results have been written to: {COMP_RESULTS_DIR}")
395
396 # Build VK-GL-CTS
397 def buildCts():
398     VK_GL_CTS_BUILD_DIR.mkdir(parents=True, exist_ok=True)
399
400     FETCH_SOURCES = str(VK_GL_CTS_ROOT_DIR / "external" / "fetch_sources.py")
401     run([which("python3"), FETCH_SOURCES], working_dir=VK_GL_CTS_ROOT_DIR)
402
403     # Build VK-GL-CTS
404     buildType = "-DCMAKE_BUILD_TYPE=" + ARGS.vk_gl_cts_build_type
405     run([which("cmake"), "-GNinja", str(VK_GL_CTS_ROOT_DIR), buildType], working_dir=VK_GL_CTS_BUILD_DIR)
406     run([which("ninja"), "deqp-vk"], working_dir=VK_GL_CTS_BUILD_DIR)
407     print(f"vk-gl-cts built to: {VK_GL_CTS_BUILD_DIR}")
408
409 # Clone and build SwiftShader and Vulkan validation layers.
410 def cloneSwsAndLayers():
411     # Clone SwiftShader or update if it already exists.
412     if not SWS_SRC_DIR.exists():
413         SWS_SRC_DIR.mkdir(parents=True, exist_ok=True)
414         run([which("git"), "clone", SWIFTSHADER_URL, SWS_SRC_DIR])
415     else:
416         run([which("git"), "pull", "origin"], working_dir=SWS_SRC_DIR)
417
418     # Build SwiftShader.
419     run([which("cmake"),
420             "-GNinja",
421             str(SWS_SRC_DIR),
422             "-DSWIFTSHADER_BUILD_EGL:BOOL=OFF",
423             "-DSWIFTSHADER_BUILD_GLESv2:BOOL=OFF",
424             "-DSWIFTSHADER_BUILD_TESTS:BOOL=OFF",
425             "-DINSTALL_GTEST=OFF",
426             "-DBUILD_TESTING:BOOL=OFF",
427             "-DENABLE_CTEST:BOOL=OFF",
428             "-DCMAKE_BUILD_TYPE=" + ARGS.sws_build_type],
429             working_dir=SWS_BUILD_DIR)
430     run([which("cmake"), "--build", ".", "--target", "vk_swiftshader"], working_dir=SWS_BUILD_DIR)
431
432     # Set Vulkan validation layers if flag is set.
433     if ARGS.validation == "true":
434         # Clone Vulkan validation layers or update if they already exist.
435         if not LAYERS_SRC_DIR.exists():
436             LAYERS_SRC_DIR.mkdir(parents=True, exist_ok=True)
437             run([which("git"), "clone", LAYERS_URL, LAYERS_SRC_DIR])
438         else:
439             run([which("git"), "pull", "origin"], working_dir=LAYERS_SRC_DIR)
440
441         # Build and set Vulkan validation layers.
442         LAYERS_BUILD_DIR.mkdir(parents=True, exist_ok=True)
443         UPDATE_DEPS = str(LAYERS_SRC_DIR / "scripts" / "update_deps.py")
444         run([which("python3"), UPDATE_DEPS], working_dir=LAYERS_BUILD_DIR)
445         run([which("cmake"),
446                 "-GNinja",
447                 "-C",
448                 "helper.cmake",
449                 LAYERS_SRC_DIR],
450                 working_dir=LAYERS_BUILD_DIR)
451         run([which("cmake"), "--build", "."], working_dir=LAYERS_BUILD_DIR)
452         LAYERS_PATH = str(LAYERS_BUILD_DIR / "layers")
453         os.environ["VK_LAYER_PATH"] = LAYERS_PATH
454     print(f"Tools cloned and built in: {PARENT_DIR}")
455
456 # Run cts with regres and move result files accordingly.
457 def runCts():
458     setVkIcdFilenames()
459
460     # Run cts and copy the resulting file to RESULT_DIR.
461     print("Running cts...")
462     runDeqp(str(DEQP_VK_BINARY), str(MUSTPASS_LIST))
463     RESULT_DIR.mkdir(parents=True, exist_ok=True)
464     copyfile(str(REGRES_DIR / "results.json"), str(RESULT_DIR / ARGS.result_output))
465     print("Run completed.")
466     print(f"Result file copied to: {RESULT_DIR}")
467     exit(0)
468
469 # Recipe for running cts.
470 if ARGS.recipe == "run-deqp":
471     cloneSwsAndLayers()
472     if ARGS.deqp_vk is None:
473         buildCts()
474     runCts()
475
476 # Recipe for only comparing the already existing result files.
477 if ARGS.recipe == "check-comparison":
478     if ARGS.files is None:
479         raise RuntimeError("No comparable files provided. Please provide them with flag --files. Use --help for more info.")
480     newFile, oldFile = ARGS.files
481     compareRuns(str(newFile), str(oldFile))