From 3b9dcc5bfd3b622b8fa0a75d88d205a425dd3603 Mon Sep 17 00:00:00 2001 From: Pyry Haulos Date: Thu, 10 Mar 2016 12:06:21 -0800 Subject: [PATCH] Clean up SPIR-V support code in framework * Return result codes instead of exceptions when GLSL compilation or assembling fails. Reserve exceptions for unexpected errors. * Move vk::disassembleSpirV() to vkSpirVAsm.cpp/hpp. * Use deUint32 vectors/pointers for passing around SPIR-V binary data. Consolidate endianness handling to vkPrograms.cpp. * Log disassembled GLSL shaders as . * Enable validation for glslang-generated SPIR-V binaries. Requires updating both glslang and spirv-tools. * Clean up vk::SpirVAsmSource and vk::SpirVProgramInfo. * Clean up some potential resource leaks in spirv-tools integration. * Add --validate-spv option to vk-build-programs. Change-Id: I0d5683c83c16b3e6d794e284c17ed65ea9541a35 --- external/fetch_sources.py | 4 +- external/spirv-tools/CMakeLists.txt | 2 +- external/vulkancts/build_spirv_binaries.py | 24 +-- .../framework/vulkan/vkBinaryRegistry.cpp | 16 +- .../vulkancts/framework/vulkan/vkGlslToSpirV.cpp | 91 +++------ .../vulkancts/framework/vulkan/vkGlslToSpirV.hpp | 19 +- external/vulkancts/framework/vulkan/vkPrograms.cpp | 142 ++++++++++++-- external/vulkancts/framework/vulkan/vkPrograms.hpp | 4 +- external/vulkancts/framework/vulkan/vkSpirVAsm.cpp | 132 +++++++++---- external/vulkancts/framework/vulkan/vkSpirVAsm.hpp | 9 +- .../vulkancts/framework/vulkan/vkSpirVProgram.cpp | 8 +- .../vulkancts/framework/vulkan/vkSpirVProgram.hpp | 40 ++-- .../vulkancts/modules/vulkan/vktBuildPrograms.cpp | 216 +++++++++++++-------- .../vulkancts/modules/vulkan/vktTestPackage.cpp | 6 +- 14 files changed, 462 insertions(+), 251 deletions(-) diff --git a/external/fetch_sources.py b/external/fetch_sources.py index 1c2fe44..67024d5 100644 --- a/external/fetch_sources.py +++ b/external/fetch_sources.py @@ -170,11 +170,11 @@ PACKAGES = [ postExtract = postExtractLibpng), GitRepo( "https://github.com/KhronosGroup/SPIRV-Tools.git", - "7ef6da7b7f9175da509b4d71a881c0a04e0b701b", + "f7e63786a919040cb2e0e572d960a0650f2c2881", "spirv-tools"), GitRepo( "https://github.com/KhronosGroup/glslang.git", - "f2d8a5c53fda69a7e19defbda7964f380da54ad1", + "5639f3aca5b75cbe5419a623eecf5e3794fab917", "glslang"), ] diff --git a/external/spirv-tools/CMakeLists.txt b/external/spirv-tools/CMakeLists.txt index c6166ec..879ce31 100644 --- a/external/spirv-tools/CMakeLists.txt +++ b/external/spirv-tools/CMakeLists.txt @@ -4,7 +4,7 @@ if (NOT DE_DEFS) message(FATAL_ERROR "Include Defs.cmake") endif () -if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/include/libspirv/libspirv.h") +if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/include/spirv-tools/libspirv.h") set(DEFAULT_SPIRV_TOOLS_SRC_PATH ${CMAKE_CURRENT_SOURCE_DIR}/src) else () set(DEFAULT_SPIRV_TOOLS_SRC_PATH "../spirv-tools") diff --git a/external/vulkancts/build_spirv_binaries.py b/external/vulkancts/build_spirv_binaries.py index 583a4de..8b1263f 100644 --- a/external/vulkancts/build_spirv_binaries.py +++ b/external/vulkancts/build_spirv_binaries.py @@ -60,20 +60,21 @@ def getBuildConfig (buildPathPtrn, targetName, buildType): return BuildConfig(buildPath, buildType, ["-DDEQP_TARGET=%s" % targetName]) def cleanDstDir (dstPath): - binFiles = [f for f in os.listdir(dstPath) if os.path.isfile(os.path.join(dstPath, f)) and fnmatch.fnmatch(f, "*.spirv")] + binFiles = [f for f in os.listdir(dstPath) if os.path.isfile(os.path.join(dstPath, f)) and fnmatch.fnmatch(f, "*.spv")] for binFile in binFiles: print "Removing %s" % os.path.join(dstPath, binFile) os.remove(os.path.join(dstPath, binFile)) -def execBuildPrograms (buildCfg, generator, module, mode, dstPath): - workDir = os.path.join(buildCfg.getBuildDir(), "modules", module.dirName) +def execBuildPrograms (buildCfg, generator, module, dstPath): + fullDstPath = os.path.realpath(dstPath) + workDir = os.path.join(buildCfg.getBuildDir(), "modules", module.dirName) pushWorkingDir(workDir) try: binPath = generator.getBinaryPath(buildCfg.getBuildType(), os.path.join(".", "vk-build-programs")) - execute([binPath, "--mode", mode, "--dst-path", dstPath]) + execute([binPath, "--validate-spv", "--dst-path", fullDstPath]) finally: popWorkingDir() @@ -95,10 +96,6 @@ def parseArgs (): dest="targetName", default=DEFAULT_TARGET, help="dEQP build target") - parser.add_argument("--mode", - dest="mode", - default="build", - help="Build mode (build or verify)") parser.add_argument("-d", "--dst-path", dest="dstPath", @@ -115,10 +112,9 @@ if __name__ == "__main__": build(buildCfg, generator, ["vk-build-programs"]) - if args.mode == "build": - if os.path.exists(args.dstPath): - cleanDstDir(args.dstPath) - else: - os.makedirs(args.dstPath) + if os.path.exists(args.dstPath): + cleanDstDir(args.dstPath) + else: + os.makedirs(args.dstPath) - execBuildPrograms(buildCfg, generator, module, args.mode, args.dstPath) + execBuildPrograms(buildCfg, generator, module, args.dstPath) diff --git a/external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp b/external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp index ce83b0c..b624093 100644 --- a/external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp +++ b/external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp @@ -284,10 +284,18 @@ void buildFinalIndex (std::vector* dst, const SparseIndexNode* { const deUint32 indexSize = getIndexSize(root); - DE_ASSERT(indexSize > 0); - - dst->resize(indexSize); - addAndCountNodes(&(*dst)[0], 0, root); + if (indexSize > 0) + { + dst->resize(indexSize); + addAndCountNodes(&(*dst)[0], 0, root); + } + else + { + // Generate empty index + dst->resize(1); + (*dst)[0].word = 0u; + (*dst)[0].index = 0u; + } } } // anonymous diff --git a/external/vulkancts/framework/vulkan/vkGlslToSpirV.cpp b/external/vulkancts/framework/vulkan/vkGlslToSpirV.cpp index e0f5256..87dd218 100644 --- a/external/vulkancts/framework/vulkan/vkGlslToSpirV.cpp +++ b/external/vulkancts/framework/vulkan/vkGlslToSpirV.cpp @@ -30,14 +30,13 @@ #include "vkGlslToSpirV.hpp" #include "deArrayUtil.hpp" +#include "deMutex.hpp" +#include "deSingleton.h" #include "deMemory.h" #include "deClock.h" #include "qpDebugOut.h" #if defined(DEQP_HAVE_GLSLANG) -# include "deSingleton.h" -# include "deMutex.hpp" - # include "SPIRV/GlslangToSpv.h" # include "SPIRV/disassemble.h" # include "SPIRV/doc.h" @@ -47,10 +46,6 @@ # include "glslang/Public/ShaderLang.h" #endif -#if defined(DEQP_HAVE_SPIRV_TOOLS) -# include "libspirv/libspirv.h" -#endif - namespace vk { @@ -206,10 +201,26 @@ void getDefaultBuiltInResources (TBuiltInResource* builtin) } // anonymous -void glslToSpirV (const glu::ProgramSources& program, std::vector* dst, glu::ShaderProgramInfo* buildInfo) +bool getNumShaderStages (const glu::ProgramSources& program) +{ + int numShaderStages = 0; + + for (int shaderType = 0; shaderType < glu::SHADERTYPE_LAST; ++shaderType) + { + if (!program.sources[shaderType].empty()) + numShaderStages += 1; + } + + return numShaderStages; +} + +bool compileGlslToSpirV (const glu::ProgramSources& program, std::vector* dst, glu::ShaderProgramInfo* buildInfo) { TBuiltInResource builtinRes; + if (getNumShaderStages(program) > 1) + TCU_THROW(InternalError, "Linking multiple shader stages into a single SPIR-V binary is not supported"); + prepareGlslang(); getDefaultBuiltInResources(&builtinRes); @@ -222,7 +233,6 @@ void glslToSpirV (const glu::ProgramSources& program, std::vector* dst, const std::string& srcText = program.sources[shaderType][0]; const char* srcPtrs[] = { srcText.c_str() }; const int srcLengths[] = { (int)srcText.size() }; - vector spvBlob; const EShLanguage shaderStage = getGlslangStage(glu::ShaderType(shaderType)); glslang::TShader shader (shaderStage); glslang::TProgram program; @@ -242,11 +252,10 @@ void glslToSpirV (const glu::ProgramSources& program, std::vector* dst, shaderBuildInfo.compileOk = (compileRes != 0); buildInfo->shaders.push_back(shaderBuildInfo); - - if (compileRes == 0) - TCU_FAIL("Failed to compile shader"); } + DE_ASSERT(buildInfo->shaders.size() == 1); + if (buildInfo->shaders[0].compileOk) { const deUint64 linkStartTime = deGetMicroseconds(); const int linkRes = program.link((EShMessages)(EShMsgSpvRules | EShMsgVulkanRules)); @@ -254,24 +263,15 @@ void glslToSpirV (const glu::ProgramSources& program, std::vector* dst, buildInfo->program.infoLog = program.getInfoLog(); // \todo [2015-11-05 scygan] Include debug log? buildInfo->program.linkOk = (linkRes != 0); buildInfo->program.linkTimeUs = deGetMicroseconds()-linkStartTime; - - if (linkRes == 0) - TCU_FAIL("Failed to link shader"); } + if (buildInfo->program.linkOk) { const glslang::TIntermediate* const intermediate = program.getIntermediate(shaderStage); - glslang::GlslangToSpv(*intermediate, spvBlob); + glslang::GlslangToSpv(*intermediate, *dst); } - dst->resize(spvBlob.size() * sizeof(deUint32)); -#if (DE_ENDIANNESS == DE_LITTLE_ENDIAN) - deMemcpy(&(*dst)[0], &spvBlob[0], dst->size()); -#else -# error "Big-endian not supported" -#endif - - return; + return buildInfo->program.linkOk; } } @@ -280,52 +280,11 @@ void glslToSpirV (const glu::ProgramSources& program, std::vector* dst, #else // defined(DEQP_HAVE_GLSLANG) -void glslToSpirV (const glu::ProgramSources&, std::vector*, glu::ShaderProgramInfo*) +bool compileGlslToSpirV (const glu::ProgramSources&, std::vector*, glu::ShaderProgramInfo*) { TCU_THROW(NotSupportedError, "GLSL to SPIR-V compilation not supported (DEQP_HAVE_GLSLANG not defined)"); } #endif // defined(DEQP_HAVE_GLSLANG) -#if defined(DEQP_HAVE_SPIRV_TOOLS) -void disassembleSpirV (size_t binarySize, const deUint8* binary, std::ostream* dst) -{ - std::vector binForDisasm (binarySize/4); - - DE_ASSERT(binarySize%4 == 0); - -#if (DE_ENDIANNESS == DE_LITTLE_ENDIAN) - deMemcpy(&binForDisasm[0], binary, binarySize); -#else -# error "Big-endian not supported" -#endif - - const spv_context context = spvContextCreate(); - if (!context) - throw std::bad_alloc(); - spv_text text = DE_NULL; - spv_diagnostic diagnostic; - try - { - const spv_result_t error = spvBinaryToText(context, &binForDisasm[0], binForDisasm.size() , 0, &text, &diagnostic); - TCU_CHECK_INTERNAL(!error); - *dst << text->str; - } - catch (...) - { - spvTextDestroy(text); - spvContextDestroy(context); - throw; - } - spvTextDestroy(text); - spvContextDestroy(context); -} -#else // defined(DEQP_HAVE_SPIRV_TOOLS) - -void disassembleSpirV (size_t, const deUint8*, std::ostream*) -{ - TCU_THROW(NotSupportedError, "SPIR-V disassembling not supported (DEQP_HAVE_SPIRV_TOOLS not defined)"); -} -#endif // defined(DEQP_HAVE_SPIRV_TOOLS) - } // vk diff --git a/external/vulkancts/framework/vulkan/vkGlslToSpirV.hpp b/external/vulkancts/framework/vulkan/vkGlslToSpirV.hpp index 547abf9..bae0553 100644 --- a/external/vulkancts/framework/vulkan/vkGlslToSpirV.hpp +++ b/external/vulkancts/framework/vulkan/vkGlslToSpirV.hpp @@ -39,11 +39,20 @@ namespace vk { -//! Compile GLSL program to SPIR-V. Will fail with NotSupportedError if compiler is not available. -void glslToSpirV (const glu::ProgramSources& src, std::vector* dst, glu::ShaderProgramInfo* buildInfo); - -//! Disassemble SPIR-V binary -void disassembleSpirV (size_t binarySize, const deUint8* binary, std::ostream* dst); +/*--------------------------------------------------------------------*//*! + * \brief Compile GLSL program to SPIR-V binary + * \param src + * \param dst + * \param buildInfo + * \return True if compilation and linking succeeded, false otherwise + * + * If deqp was built without glslang (and thus compiler is not available) + * tcu::NotSupportedError will be thrown instead. + * + * \note No linking is currently supported so src may contain source + * for only one shader stage. + *//*--------------------------------------------------------------------*/ +bool compileGlslToSpirV (const glu::ProgramSources& src, std::vector* dst, glu::ShaderProgramInfo* buildInfo); } // vk diff --git a/external/vulkancts/framework/vulkan/vkPrograms.cpp b/external/vulkancts/framework/vulkan/vkPrograms.cpp index e900cd9..2f72220 100644 --- a/external/vulkancts/framework/vulkan/vkPrograms.cpp +++ b/external/vulkancts/framework/vulkan/vkPrograms.cpp @@ -37,6 +37,7 @@ #include "deArrayUtil.hpp" #include "deMemory.h" +#include "deInt32.h" namespace vk { @@ -45,6 +46,14 @@ using std::string; using std::vector; using tcu::TestLog; +#if defined(DE_DEBUG) && defined(DEQP_HAVE_SPIRV_TOOLS) +# define VALIDATE_BINARIES true +#else +# define VALIDATE_BINARIES false +#endif + +#define SPIRV_BINARY_ENDIANNESS DE_LITTLE_ENDIAN + // ProgramBinary ProgramBinary::ProgramBinary (ProgramFormat format, size_t binarySize, const deUint8* binary) @@ -55,15 +64,76 @@ ProgramBinary::ProgramBinary (ProgramFormat format, size_t binarySize, const deU // Utils +namespace +{ + +bool isNativeSpirVBinaryEndianness (void) +{ +#if (DE_ENDIANNESS == SPIRV_BINARY_ENDIANNESS) + return true; +#else + return false; +#endif +} + +bool isSaneSpirVBinary (const ProgramBinary& binary) +{ + const deUint32 spirvMagicWord = 0x07230203; + const deUint32 spirvMagicBytes = isNativeSpirVBinaryEndianness() + ? spirvMagicWord + : deReverseBytes32(spirvMagicWord); + + DE_ASSERT(binary.getFormat() == PROGRAM_FORMAT_SPIRV); + + if (binary.getSize() % sizeof(deUint32) != 0) + return false; + + if (binary.getSize() < sizeof(deUint32)) + return false; + + if (*(const deUint32*)binary.getBinary() != spirvMagicBytes) + return false; + + return true; +} + +ProgramBinary* createProgramBinaryFromSpirV (const vector& binary) +{ + DE_ASSERT(!binary.empty()); + + if (isNativeSpirVBinaryEndianness()) + return new ProgramBinary(PROGRAM_FORMAT_SPIRV, binary.size()*sizeof(deUint32), (const deUint8*)&binary[0]); + else + TCU_THROW(InternalError, "SPIR-V endianness translation not supported"); +} + +} // anonymous + ProgramBinary* buildProgram (const glu::ProgramSources& program, ProgramFormat binaryFormat, glu::ShaderProgramInfo* buildInfo) { + const bool validateBinary = VALIDATE_BINARIES; + if (binaryFormat == PROGRAM_FORMAT_SPIRV) { - vector binary; - glslToSpirV(program, &binary, buildInfo); - // \todo[2016-02-10 dekimir]: Enable this when all glsl tests can pass it. - //validateSpirV(binary, &buildInfo->program.infoLog); - return new ProgramBinary(binaryFormat, binary.size(), &binary[0]); + vector binary; + + if (!compileGlslToSpirV(program, &binary, buildInfo)) + TCU_THROW(InternalError, "Compiling GLSL to SPIR-V failed"); + + if (validateBinary) + { + std::ostringstream validationLog; + + if (!validateSpirV(binary.size(), &binary[0], &validationLog)) + { + buildInfo->program.linkOk = false; + buildInfo->program.infoLog += "\n" + validationLog.str(); + + TCU_THROW(InternalError, "Validation failed for compiled SPIR-V binary"); + } + } + + return createProgramBinaryFromSpirV(binary); } else TCU_THROW(NotSupportedError, "Unsupported program format"); @@ -71,14 +141,60 @@ ProgramBinary* buildProgram (const glu::ProgramSources& program, ProgramFormat b ProgramBinary* assembleProgram (const SpirVAsmSource& program, SpirVProgramInfo* buildInfo) { - vector binary; - assembleSpirV(&program, &binary, buildInfo); -#if defined(DE_DEBUG) - buildInfo->compileOk &= validateSpirV(binary, &buildInfo->infoLog); - if (!buildInfo->compileOk) - TCU_FAIL("Failed to validate shader"); -#endif - return new ProgramBinary(PROGRAM_FORMAT_SPIRV, binary.size(), &binary[0]); + const bool validateBinary = VALIDATE_BINARIES; + vector binary; + + if (!assembleSpirV(&program, &binary, buildInfo)) + TCU_THROW(InternalError, "Failed to assemble SPIR-V"); + + if (validateBinary) + { + std::ostringstream validationLog; + + if (!validateSpirV(binary.size(), &binary[0], &validationLog)) + { + buildInfo->compileOk = false; + buildInfo->infoLog += "\n" + validationLog.str(); + + TCU_THROW(InternalError, "Validation failed for assembled SPIR-V binary"); + } + } + + return createProgramBinaryFromSpirV(binary); +} + +void disassembleProgram (const ProgramBinary& program, std::ostream* dst) +{ + if (program.getFormat() == PROGRAM_FORMAT_SPIRV) + { + TCU_CHECK_INTERNAL(isSaneSpirVBinary(program)); + + if (isNativeSpirVBinaryEndianness()) + disassembleSpirV(program.getSize()/sizeof(deUint32), (const deUint32*)program.getBinary(), dst); + else + TCU_THROW(InternalError, "SPIR-V endianness translation not supported"); + } + else + TCU_THROW(NotSupportedError, "Unsupported program format"); +} + +bool validateProgram (const ProgramBinary& program, std::ostream* dst) +{ + if (program.getFormat() == PROGRAM_FORMAT_SPIRV) + { + if (!isSaneSpirVBinary(program)) + { + *dst << "Binary doesn't look like SPIR-V at all"; + return false; + } + + if (isNativeSpirVBinaryEndianness()) + return validateSpirV(program.getSize()/sizeof(deUint32), (const deUint32*)program.getBinary(), dst); + else + TCU_THROW(InternalError, "SPIR-V endianness translation not supported"); + } + else + TCU_THROW(NotSupportedError, "Unsupported program format"); } Move createShaderModule (const DeviceInterface& deviceInterface, VkDevice device, const ProgramBinary& binary, VkShaderModuleCreateFlags flags) diff --git a/external/vulkancts/framework/vulkan/vkPrograms.hpp b/external/vulkancts/framework/vulkan/vkPrograms.hpp index 4936f18..b0a9ca8 100644 --- a/external/vulkancts/framework/vulkan/vkPrograms.hpp +++ b/external/vulkancts/framework/vulkan/vkPrograms.hpp @@ -176,9 +176,11 @@ struct SourceCollections typedef ProgramCollection BinaryCollection; -// \todo [2015-03-13 pyry] Likely need BinaryBuilder abstraction for this ProgramBinary* buildProgram (const glu::ProgramSources& program, ProgramFormat binaryFormat, glu::ShaderProgramInfo* buildInfo); ProgramBinary* assembleProgram (const vk::SpirVAsmSource& program, SpirVProgramInfo* buildInfo); +void disassembleProgram (const ProgramBinary& program, std::ostream* dst); +bool validateProgram (const ProgramBinary& program, std::ostream* dst); + Move createShaderModule (const DeviceInterface& deviceInterface, VkDevice device, const ProgramBinary& binary, VkShaderModuleCreateFlags flags); glu::ShaderType getGluShaderType (VkShaderStageFlagBits shaderStage); diff --git a/external/vulkancts/framework/vulkan/vkSpirVAsm.cpp b/external/vulkancts/framework/vulkan/vkSpirVAsm.cpp index bd5105d..e864c3d 100644 --- a/external/vulkancts/framework/vulkan/vkSpirVAsm.cpp +++ b/external/vulkancts/framework/vulkan/vkSpirVAsm.cpp @@ -30,13 +30,12 @@ #include "vkSpirVAsm.hpp" #include "vkSpirVProgram.hpp" -#include "deArrayUtil.hpp" -#include "deMemory.h" #include "deClock.h" -#include "qpDebugOut.h" + +#include #if defined(DEQP_HAVE_SPIRV_TOOLS) -# include "libspirv/libspirv.h" +# include "spirv-tools/libspirv.h" #endif namespace vk @@ -47,64 +46,117 @@ using std::vector; #if defined(DEQP_HAVE_SPIRV_TOOLS) - -void assembleSpirV (const SpirVAsmSource* program, std::vector* dst, SpirVProgramInfo* buildInfo) +bool assembleSpirV (const SpirVAsmSource* program, std::vector* dst, SpirVProgramInfo* buildInfo) { - spv_context context = spvContextCreate(); + const spv_context context = spvContextCreate(); + spv_binary binary = DE_NULL; + spv_diagnostic diagnostic = DE_NULL; - const std::string& spvSource = program->program.str(); - spv_binary binary = DE_NULL; - spv_diagnostic diagnostic = DE_NULL; - const deUint64 compileStartTime = deGetMicroseconds(); - const spv_result_t compileOk = spvTextToBinary(context, spvSource.c_str(), spvSource.size(), &binary, &diagnostic); + if (!context) + throw std::bad_alloc(); + try { - buildInfo->source = program; + const std::string& spvSource = program->source; + const deUint64 compileStartTime = deGetMicroseconds(); + const spv_result_t compileOk = spvTextToBinary(context, spvSource.c_str(), spvSource.size(), &binary, &diagnostic); + + buildInfo->source = spvSource; buildInfo->infoLog = diagnostic? diagnostic->error : ""; // \todo [2015-07-13 pyry] Include debug log? buildInfo->compileTimeUs = deGetMicroseconds() - compileStartTime; buildInfo->compileOk = (compileOk == SPV_SUCCESS); + + dst->resize(binary->wordCount); + std::copy(&binary->code[0], &binary->code[0] + binary->wordCount, dst->begin()); + + spvBinaryDestroy(binary); + spvDiagnosticDestroy(diagnostic); + spvContextDestroy(context); + + return compileOk == SPV_SUCCESS; + } + catch (...) + { + spvBinaryDestroy(binary); + spvDiagnosticDestroy(diagnostic); + spvContextDestroy(context); + + throw; } +} - if (compileOk != SPV_SUCCESS) - TCU_FAIL("Failed to compile shader"); +void disassembleSpirV (size_t binarySizeInWords, const deUint32* binary, std::ostream* dst) +{ + const spv_context context = spvContextCreate(); + spv_text text = DE_NULL; + spv_diagnostic diagnostic = DE_NULL; - dst->resize((int)binary->wordCount * sizeof(deUint32)); -#if (DE_ENDIANNESS == DE_LITTLE_ENDIAN) - deMemcpy(&(*dst)[0], &binary->code[0], dst->size()); -#else -# error "Big-endian not supported" -#endif - spvBinaryDestroy(binary); - spvDiagnosticDestroy(diagnostic); - spvContextDestroy(context); - return; + if (!context) + throw std::bad_alloc(); + + try + { + const spv_result_t result = spvBinaryToText(context, binary, binarySizeInWords, 0, &text, &diagnostic); + + if (result != SPV_SUCCESS) + TCU_THROW(InternalError, "Disassembling SPIR-V failed"); + + *dst << text->str; + + spvTextDestroy(text); + spvDiagnosticDestroy(diagnostic); + spvContextDestroy(context); + } + catch (...) + { + spvTextDestroy(text); + spvDiagnosticDestroy(diagnostic); + spvContextDestroy(context); + + throw; + } } -bool validateSpirV (const std::vector& spirv, std::string* infoLog) +bool validateSpirV (size_t binarySizeInWords, const deUint32* binary, std::ostream* infoLog) { - const size_t bytesPerWord = sizeof(uint32_t) / sizeof(deUint8); - DE_ASSERT(spirv.size() % bytesPerWord == 0); - std::vector words(spirv.size() / bytesPerWord); - deMemcpy(words.data(), spirv.data(), spirv.size()); - spv_const_binary_t cbinary = { words.data(), words.size() }; + const spv_context context = spvContextCreate(); spv_diagnostic diagnostic = DE_NULL; - spv_context context = spvContextCreate(); - const spv_result_t valid = spvValidate(context, &cbinary, SPV_VALIDATE_ALL, &diagnostic); - if (diagnostic) - *infoLog += diagnostic->error; - spvContextDestroy(context); - spvDiagnosticDestroy(diagnostic); - return valid == SPV_SUCCESS; + + try + { + spv_const_binary_t cbinary = { binary, binarySizeInWords }; + const spv_result_t valid = spvValidate(context, &cbinary, &diagnostic); + + if (diagnostic) + *infoLog << diagnostic->error; + + spvDiagnosticDestroy(diagnostic); + spvContextDestroy(context); + + return valid == SPV_SUCCESS; + } + catch (...) + { + spvDiagnosticDestroy(diagnostic); + spvContextDestroy(context); + + throw; + } } #else // defined(DEQP_HAVE_SPIRV_TOOLS) -void assembleSpirV (const SpirVAsmSource*, std::vector*, SpirVProgramInfo*) +bool assembleSpirV (const SpirVAsmSource*, std::vector*, SpirVProgramInfo*) { TCU_THROW(NotSupportedError, "SPIR-V assembly not supported (DEQP_HAVE_SPIRV_TOOLS not defined)"); } -bool validateSpirV (const std::vector&, std::string*) +void disassembleSpirV (size_t, const deUint32*, std::ostream*) +{ + TCU_THROW(NotSupportedError, "SPIR-V disassembling not supported (DEQP_HAVE_SPIRV_TOOLS not defined)"); +} + +bool validateSpirV (size_t, const deUint32*, std::ostream*) { TCU_THROW(NotSupportedError, "SPIR-V validation not supported (DEQP_HAVE_SPIRV_TOOLS not defined)"); } diff --git a/external/vulkancts/framework/vulkan/vkSpirVAsm.hpp b/external/vulkancts/framework/vulkan/vkSpirVAsm.hpp index 43348d9..23b3f8a 100644 --- a/external/vulkancts/framework/vulkan/vkSpirVAsm.hpp +++ b/external/vulkancts/framework/vulkan/vkSpirVAsm.hpp @@ -33,14 +33,19 @@ #include "vkDefs.hpp" #include "vkPrograms.hpp" +#include + namespace vk { //! Assemble SPIR-V program. Will fail with NotSupportedError if compiler is not available. -void assembleSpirV (const SpirVAsmSource* program, std::vector* dst, SpirVProgramInfo* buildInfo); +bool assembleSpirV (const SpirVAsmSource* program, std::vector* dst, SpirVProgramInfo* buildInfo); + +//! Disassemble SPIR-V binary. Throws tcu::NotSupportedError if disassembler is not available +void disassembleSpirV (size_t binarySizeInWords, const deUint32* binary, std::ostream* dst); //! Validate SPIR-V binary, returning true if validation succeeds. Will fail with NotSupportedError if compiler is not available. -bool validateSpirV (const std::vector& spirv, std::string* infoLog); +bool validateSpirV (size_t binarySizeInWords, const deUint32* binary, std::ostream* infoLog); } // vk diff --git a/external/vulkancts/framework/vulkan/vkSpirVProgram.cpp b/external/vulkancts/framework/vulkan/vkSpirVProgram.cpp index 0987887..0ef12ee 100644 --- a/external/vulkancts/framework/vulkan/vkSpirVProgram.cpp +++ b/external/vulkancts/framework/vulkan/vkSpirVProgram.cpp @@ -37,7 +37,9 @@ namespace vk tcu::TestLog& operator<< (tcu::TestLog& log, const SpirVProgramInfo& shaderInfo) { - log << tcu::TestLog::ShaderProgram(shaderInfo.compileOk , shaderInfo.infoLog) << tcu::TestLog::SpirVAssemblySource(shaderInfo.source->program.str()) << tcu::TestLog::EndShaderProgram; + log << tcu::TestLog::ShaderProgram(shaderInfo.compileOk , shaderInfo.infoLog) + << tcu::TestLog::SpirVAssemblySource(shaderInfo.source) + << tcu::TestLog::EndShaderProgram; // Write statistics log << tcu::TestLog::Float( "SpirVAssemblyTime", @@ -48,7 +50,9 @@ tcu::TestLog& operator<< (tcu::TestLog& log, const SpirVProgramInfo& shaderInfo) tcu::TestLog& operator<< (tcu::TestLog& log, const SpirVAsmSource& source) { - log << tcu::TestLog::KernelSource(source.program.str()); + log << tcu::TestLog::ShaderProgram(true , "") + << tcu::TestLog::SpirVAssemblySource(source.source) + << tcu::TestLog::EndShaderProgram; return log; } diff --git a/external/vulkancts/framework/vulkan/vkSpirVProgram.hpp b/external/vulkancts/framework/vulkan/vkSpirVProgram.hpp index f6adf1d..b551d2c 100644 --- a/external/vulkancts/framework/vulkan/vkSpirVProgram.hpp +++ b/external/vulkancts/framework/vulkan/vkSpirVProgram.hpp @@ -32,41 +32,51 @@ #include "vkDefs.hpp" #include "tcuTestLog.hpp" +#include "deStringUtil.hpp" #include -#include namespace vk { struct SpirVAsmSource { - template - SpirVAsmSource& operator<<(const T& val) + SpirVAsmSource (void) { - program << val; - return *this; } - std::ostringstream program; + + SpirVAsmSource (const std::string& source_) + : source(source_) + { + } + + std::string source; }; struct SpirVProgramInfo { - SpirVProgramInfo() - : source (DE_NULL) - , compileTimeUs (0) + SpirVProgramInfo (void) + : compileTimeUs (0) , compileOk (false) { } - const SpirVAsmSource* source; - std::string infoLog; - deUint64 compileTimeUs; - bool compileOk; + std::string source; + std::string infoLog; + deUint64 compileTimeUs; + bool compileOk; }; -tcu::TestLog& operator<< (tcu::TestLog& log, const SpirVProgramInfo& shaderInfo); -tcu::TestLog& operator<< (tcu::TestLog& log, const SpirVAsmSource& program); +tcu::TestLog& operator<< (tcu::TestLog& log, const SpirVProgramInfo& shaderInfo); +tcu::TestLog& operator<< (tcu::TestLog& log, const SpirVAsmSource& program); + +// Helper for constructing SpirVAsmSource +template +SpirVAsmSource& operator<< (SpirVAsmSource& src, const T& val) +{ + src.source += de::toString(val); + return src; +} } diff --git a/external/vulkancts/modules/vulkan/vktBuildPrograms.cpp b/external/vulkancts/modules/vulkan/vktBuildPrograms.cpp index 5f8348f..675c94b 100644 --- a/external/vulkancts/modules/vulkan/vktBuildPrograms.cpp +++ b/external/vulkancts/modules/vulkan/vktBuildPrograms.cpp @@ -41,6 +41,7 @@ #include "vktTestPackage.hpp" #include "deUniquePtr.hpp" #include "deCommandLine.hpp" +#include "deSharedPtr.hpp" #include @@ -48,6 +49,7 @@ using std::vector; using std::string; using de::UniquePtr; using de::MovePtr; +using de::SharedPtr; namespace vkt { @@ -59,14 +61,6 @@ tcu::TestPackageRoot* createRoot (tcu::TestContext& testCtx) return new tcu::TestPackageRoot(testCtx, children); } -enum BuildMode -{ - BUILDMODE_BUILD = 0, - BUILDMODE_VERIFY, - - BUILDMODE_LAST -}; - struct BuildStats { int numSucceeded; @@ -82,89 +76,122 @@ struct BuildStats namespace // anonymous { -vk::ProgramBinary* compileProgram (const glu::ProgramSources& source, glu::ShaderProgramInfo* buildInfo) -{ - return vk::buildProgram(source, vk::PROGRAM_FORMAT_SPIRV, buildInfo); -} - -vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, vk::SpirVProgramInfo* buildInfo) -{ - return vk::assembleProgram(source, buildInfo); -} - -void writeVerboseLogs (const glu::ShaderProgramInfo& buildInfo) +void writeBuildLogs (const glu::ShaderProgramInfo& buildInfo, std::ostream& dst) { for (size_t shaderNdx = 0; shaderNdx < buildInfo.shaders.size(); shaderNdx++) { const glu::ShaderInfo& shaderInfo = buildInfo.shaders[shaderNdx]; const char* const shaderName = getShaderTypeName(shaderInfo.type); - tcu::print("%s source:\n---\n%s\n---\n", shaderName, shaderInfo.source.c_str()); - tcu::print("%s compile log:\n---\n%s\n---\n", shaderName, shaderInfo.infoLog.c_str()); + dst << shaderName << " source:\n" + << "---\n" + << shaderInfo.source << "\n" + << "---\n" + << shaderName << " compile log:\n" + << "---\n" + << shaderInfo.infoLog << "\n" + << "---\n"; } + + dst << "link log:\n" + << "---\n" + << buildInfo.program.infoLog << "\n" + << "---\n"; } -void writeVerboseLogs (const vk::SpirVProgramInfo& buildInfo) +void writeBuildLogs (const vk::SpirVProgramInfo& buildInfo, std::ostream& dst) { - tcu::print("source:\n---\n%s\n---\n", buildInfo.source->program.str().c_str()); - tcu::print("compile log:\n---\n%s\n---\n", buildInfo.infoLog.c_str()); + dst << "source:\n" + << "---\n" + << buildInfo.source << "\n" + << "---\n"; } -template -void buildProgram (const std::string& casePath, - bool printLogs, - IteratorType iter, - BuildMode mode, - BuildStats* stats, - vk::BinaryRegistryReader* reader, - vk::BinaryRegistryWriter* writer) +vk::ProgramBinary* compileProgram (const glu::ProgramSources& source, std::ostream& buildLog) { - InfoType buildInfo; + glu::ShaderProgramInfo buildInfo; + try { - const vk::ProgramIdentifier progId (casePath, iter.getName()); - const UniquePtr binary (compileProgram(iter.getProgram(), &buildInfo)); + return vk::buildProgram(source, vk::PROGRAM_FORMAT_SPIRV, &buildInfo); + } + catch (const tcu::Exception&) + { + writeBuildLogs(buildInfo, buildLog); + throw; + } +} - if (mode == BUILDMODE_BUILD) - writer->storeProgram(progId, *binary); - else - { - DE_ASSERT(mode == BUILDMODE_VERIFY); +vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, std::ostream& buildLog) +{ + vk::SpirVProgramInfo buildInfo; + + try + { + return vk::assembleProgram(source, &buildInfo); + } + catch (const tcu::Exception&) + { + writeBuildLogs(buildInfo, buildLog); + throw; + } +} - const UniquePtr storedBinary (reader->loadProgram(progId)); +struct BuiltProgram +{ + vk::ProgramIdentifier id; + bool buildOk; + UniquePtr binary; // Null if build failed + std::string buildLog; + + BuiltProgram (const vk::ProgramIdentifier& id_, + bool buildOk_, + MovePtr binary_, + const std::string& buildLog_) + : id (id_) + , buildOk (buildOk_) + , binary (binary_) + , buildLog (buildLog_) + { + } +}; - if (binary->getSize() != storedBinary->getSize()) - throw tcu::Exception("Binary size doesn't match"); +typedef SharedPtr BuiltProgramSp; - if (deMemCmp(binary->getBinary(), storedBinary->getBinary(), binary->getSize())) - throw tcu::Exception("Binary contents don't match"); - } +template +BuiltProgramSp buildProgram (IteratorType progIter, const std::string& casePath) +{ + std::ostringstream buildLog; + MovePtr programBinary; + bool buildOk = false; - tcu::print(" OK: %s\n", iter.getName().c_str()); - stats->numSucceeded += 1; + try + { + programBinary = MovePtr(compileProgram(progIter.getProgram(), buildLog)); + buildOk = true; } - catch (const std::exception& e) + catch (const std::exception&) { - tcu::print(" ERROR: %s: %s\n", iter.getName().c_str(), e.what()); - if (printLogs) - { - writeVerboseLogs(buildInfo); - } - stats->numFailed += 1; + // Ignore, buildOk = false + DE_ASSERT(!programBinary); } + + return BuiltProgramSp(new BuiltProgram(vk::ProgramIdentifier(casePath, progIter.getName()), + buildOk, + programBinary, + buildLog.str())); } } // anonymous -BuildStats buildPrograms (tcu::TestContext& testCtx, const std::string& dstPath, BuildMode mode, bool verbose) + +BuildStats buildPrograms (tcu::TestContext& testCtx, const std::string& dstPath, bool validateBinaries) { const UniquePtr root (createRoot(testCtx)); tcu::DefaultHierarchyInflater inflater (testCtx); tcu::TestHierarchyIterator iterator (*root, inflater, testCtx.getCommandLine()); const tcu::DirArchive srcArchive (dstPath.c_str()); - UniquePtr writer (mode == BUILDMODE_BUILD ? new vk::BinaryRegistryWriter(dstPath) : DE_NULL); - UniquePtr reader (mode == BUILDMODE_VERIFY ? new vk::BinaryRegistryReader(srcArchive, "") : DE_NULL); + UniquePtr writer (new vk::BinaryRegistryWriter(dstPath)); BuildStats stats; - const bool printLogs = verbose; while (iterator.getState() != tcu::TestHierarchyIterator::STATE_FINISHED) { @@ -173,28 +200,63 @@ BuildStats buildPrograms (tcu::TestContext& testCtx, const std::string& dstPath, { const TestCase* const testCase = dynamic_cast(iterator.getNode()); const string casePath = iterator.getNodePath(); - vk::SourceCollections progs; + vk::SourceCollections sourcePrograms; + vector builtPrograms; tcu::print("%s\n", casePath.c_str()); - testCase->initPrograms(progs); + testCase->initPrograms(sourcePrograms); - for (vk::GlslSourceCollection::Iterator progIter = progs.glslSources.begin(); progIter != progs.glslSources.end(); ++progIter) + for (vk::GlslSourceCollection::Iterator progIter = sourcePrograms.glslSources.begin(); + progIter != sourcePrograms.glslSources.end(); + ++progIter) { - buildProgram(casePath, printLogs, progIter, mode, &stats, reader.get(), writer.get()); + builtPrograms.push_back(buildProgram(progIter, casePath)); } - for (vk::SpirVAsmCollection::Iterator progIter = progs.spirvAsmSources.begin(); progIter != progs.spirvAsmSources.end(); ++progIter) + for (vk::SpirVAsmCollection::Iterator progIter = sourcePrograms.spirvAsmSources.begin(); + progIter != sourcePrograms.spirvAsmSources.end(); + ++progIter) { - buildProgram(casePath, printLogs, progIter, mode, &stats, reader.get(), writer.get()); + builtPrograms.push_back(buildProgram(progIter, casePath)); + } + + // Process programs + for (vector::const_iterator progIter = builtPrograms.begin(); + progIter != builtPrograms.end(); + ++progIter) + { + const BuiltProgram& program = **progIter; + + if (program.buildOk) + { + std::ostringstream validationLog; + + writer->storeProgram(program.id, *program.binary); + + if (validateBinaries && + !vk::validateProgram(*program.binary, &validationLog)) + { + tcu::print("ERROR: validation failed for %s\n", program.id.programName.c_str()); + tcu::print("%s\n", validationLog.str().c_str()); + stats.numFailed += 1; + } + else + stats.numSucceeded += 1; + } + else + { + tcu::print("ERROR: failed to build %s\n", program.id.programName.c_str()); + tcu::print("%s\n", program.buildLog.c_str()); + stats.numFailed += 1; + } } } iterator.next(); } - if (mode == BUILDMODE_BUILD) - writer->writeIndex(); + writer->writeIndex(); return stats; } @@ -205,27 +267,18 @@ namespace opt { DE_DECLARE_COMMAND_LINE_OPT(DstPath, std::string); -DE_DECLARE_COMMAND_LINE_OPT(Mode, vkt::BuildMode); -DE_DECLARE_COMMAND_LINE_OPT(Verbose, bool); DE_DECLARE_COMMAND_LINE_OPT(Cases, std::string); +DE_DECLARE_COMMAND_LINE_OPT(Validate, bool); } // opt void registerOptions (de::cmdline::Parser& parser) { using de::cmdline::Option; - using de::cmdline::NamedValue; - static const NamedValue s_modes[] = - { - { "build", vkt::BUILDMODE_BUILD }, - { "verify", vkt::BUILDMODE_VERIFY } - }; - - parser << Option ("d", "dst-path", "Destination path", "out") - << Option ("m", "mode", "Build mode", s_modes, "build") - << Option ("v", "verbose", "Verbose output") - << Option ("n", "deqp-case", "Case path filter (works as in test binaries)"); + parser << Option ("d", "dst-path", "Destination path", "out") + << Option ("n", "deqp-case", "Case path filter (works as in test binaries)") + << Option ("v", "validate-spv", "Validate generated SPIR-V binaries"); } int main (int argc, const char* argv[]) @@ -267,8 +320,7 @@ int main (int argc, const char* argv[]) const vkt::BuildStats stats = vkt::buildPrograms(testCtx, cmdLine.getOption(), - cmdLine.getOption(), - cmdLine.getOption()); + cmdLine.getOption()); tcu::print("DONE: %d passed, %d failed\n", stats.numSucceeded, stats.numFailed); diff --git a/external/vulkancts/modules/vulkan/vktTestPackage.cpp b/external/vulkancts/modules/vulkan/vktTestPackage.cpp index 0d84b8a..3b50384 100644 --- a/external/vulkancts/modules/vulkan/vktTestPackage.cpp +++ b/external/vulkancts/modules/vulkan/vktTestPackage.cpp @@ -38,7 +38,6 @@ #include "vkPrograms.hpp" #include "vkBinaryRegistry.hpp" #include "vkGlslToSpirV.hpp" -#include "vkSpirVAsm.hpp" #include "deUniquePtr.hpp" @@ -203,9 +202,9 @@ void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePat { std::ostringstream disasm; - vk::disassembleSpirV(binProg->getSize(), binProg->getBinary(), &disasm); + vk::disassembleProgram(*binProg, &disasm); - log << TestLog::KernelSource(disasm.str()); + log << vk::SpirVAsmSource(disasm.str()); } catch (const tcu::NotSupportedError& err) { @@ -216,7 +215,6 @@ void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePat for (vk::SpirVAsmCollection::Iterator asmIterator = sourceProgs.spirvAsmSources.begin(); asmIterator != sourceProgs.spirvAsmSources.end(); ++asmIterator) { buildProgram(casePath, asmIterator, m_prebuiltBinRegistry, log, &m_progCollection); - log << TestLog::KernelSource((*asmIterator).program.str()); } DE_ASSERT(!m_instance); -- 2.7.4