From 501ac7ba5259b7e9f6d107fe1bed304886ef7e12 Mon Sep 17 00:00:00 2001 From: Pyry Haulos Date: Mon, 28 Mar 2016 17:35:44 -0700 Subject: [PATCH] Add --deqp-validation=enable/disable option Specifying --deqp-validation=enable when running Vulkan tests will result in: 1) vkt::TestCaseExecutor will create debug report callback on default instance. Any debug messages reported during test execution will be included as a separate log section in the test log. 2) vkt::Context will enable known validation layers on both default instance as well as default device. If any errors are reported (for example validation errors) test result will be set to InternalError. Change-Id: Icc89a3f57f0c19170988d7b518fe6b24dcf3df6e --- .../framework/vulkan/vkDebugReportUtil.cpp | 7 ++ .../framework/vulkan/vkDebugReportUtil.hpp | 2 + .../vulkancts/framework/vulkan/vkQueryUtil.cpp | 25 ++++++ .../vulkancts/framework/vulkan/vkQueryUtil.hpp | 38 +++++++++ external/vulkancts/modules/vulkan/vktTestCase.cpp | 96 ++++++++++++++++++++-- .../vulkancts/modules/vulkan/vktTestPackage.cpp | 71 +++++++++++++--- framework/common/tcuCommandLine.cpp | 5 +- framework/common/tcuCommandLine.hpp | 3 + 8 files changed, 230 insertions(+), 17 deletions(-) diff --git a/external/vulkancts/framework/vulkan/vkDebugReportUtil.cpp b/external/vulkancts/framework/vulkan/vkDebugReportUtil.cpp index d3f9ff7..c2f6335 100644 --- a/external/vulkancts/framework/vulkan/vkDebugReportUtil.cpp +++ b/external/vulkancts/framework/vulkan/vkDebugReportUtil.cpp @@ -23,6 +23,7 @@ #include "vkDebugReportUtil.hpp" #include "vkRefUtil.hpp" +#include "vkQueryUtil.hpp" #include "deArrayUtil.hpp" namespace vk @@ -156,4 +157,10 @@ DebugReportRecorder::~DebugReportRecorder (void) { } +bool isDebugReportSupported (const PlatformInterface& vkp) +{ + return isExtensionSupported(enumerateInstanceExtensionProperties(vkp, DE_NULL), + RequiredExtension("VK_EXT_debug_report")); +} + } // vk diff --git a/external/vulkancts/framework/vulkan/vkDebugReportUtil.hpp b/external/vulkancts/framework/vulkan/vkDebugReportUtil.hpp index 4463e27..6c4a8dd 100644 --- a/external/vulkancts/framework/vulkan/vkDebugReportUtil.hpp +++ b/external/vulkancts/framework/vulkan/vkDebugReportUtil.hpp @@ -85,6 +85,8 @@ private: const Unique m_callback; }; +bool isDebugReportSupported (const PlatformInterface& vkp); + } // vk #endif // _VKDEBUGREPORTUTIL_HPP diff --git a/external/vulkancts/framework/vulkan/vkQueryUtil.cpp b/external/vulkancts/framework/vulkan/vkQueryUtil.cpp index 9a7300a..c24f714 100644 --- a/external/vulkancts/framework/vulkan/vkQueryUtil.cpp +++ b/external/vulkancts/framework/vulkan/vkQueryUtil.cpp @@ -242,9 +242,34 @@ bool isCompatible (const VkExtensionProperties& extensionProperties, const Requi return true; } +bool isCompatible (const VkLayerProperties& layerProperties, const RequiredLayer& required) +{ + if (required.name != layerProperties.layerName) + return false; + + if (required.minSpecVersion && required.minSpecVersion.get() > layerProperties.specVersion) + return false; + + if (required.maxSpecVersion && required.maxSpecVersion.get() < layerProperties.specVersion) + return false; + + if (required.minImplVersion && required.minImplVersion.get() > layerProperties.implementationVersion) + return false; + + if (required.maxImplVersion && required.maxImplVersion.get() < layerProperties.implementationVersion) + return false; + + return true; +} + bool isExtensionSupported (const std::vector& extensions, const RequiredExtension& required) { return isExtensionSupported(extensions.begin(), extensions.end(), required); } +bool isLayerSupported (const std::vector& layers, const RequiredLayer& required) +{ + return isLayerSupported(layers.begin(), layers.end(), required); +} + } // vk diff --git a/external/vulkancts/framework/vulkan/vkQueryUtil.hpp b/external/vulkancts/framework/vulkan/vkQueryUtil.hpp index 4eb209e..0235f7b 100644 --- a/external/vulkancts/framework/vulkan/vkQueryUtil.hpp +++ b/external/vulkancts/framework/vulkan/vkQueryUtil.hpp @@ -70,12 +70,38 @@ struct RequiredExtension {} }; +struct RequiredLayer +{ + std::string name; + tcu::Maybe minSpecVersion; + tcu::Maybe maxSpecVersion; + tcu::Maybe minImplVersion; + tcu::Maybe maxImplVersion; + + explicit RequiredLayer (const std::string& name_, + tcu::Maybe minSpecVersion_ = tcu::nothing(), + tcu::Maybe maxSpecVersion_ = tcu::nothing(), + tcu::Maybe minImplVersion_ = tcu::nothing(), + tcu::Maybe maxImplVersion_ = tcu::nothing()) + : name (name_) + , minSpecVersion(minSpecVersion_) + , maxSpecVersion(maxSpecVersion_) + , minImplVersion(minImplVersion_) + , maxImplVersion(maxImplVersion_) + {} +}; + bool isCompatible (const VkExtensionProperties& extensionProperties, const RequiredExtension& required); +bool isCompatible (const VkLayerProperties& layerProperties, const RequiredLayer& required); template bool isExtensionSupported (ExtensionIterator begin, ExtensionIterator end, const RequiredExtension& required); bool isExtensionSupported (const std::vector& extensions, const RequiredExtension& required); +template +bool isLayerSupported (LayerIterator begin, LayerIterator end, const RequiredLayer& required); +bool isLayerSupported (const std::vector& layers, const RequiredLayer& required); + // Return variable initialization validation typedef struct @@ -83,6 +109,7 @@ typedef struct size_t offset; size_t size; } QueryMemberTableEntry; + template bool validateInitComplete(Context context, void (Interface::*Function)(Context, Type*)const, const Interface& interface, const QueryMemberTableEntry* queryMemberTableEntry) { @@ -116,6 +143,17 @@ bool isExtensionSupported (ExtensionIterator begin, ExtensionIterator end, const return false; } +template +bool isLayerSupported (LayerIterator begin, LayerIterator end, const RequiredLayer& required) +{ + for (LayerIterator cur = begin; cur != end; ++cur) + { + if (isCompatible(*cur, required)) + return true; + } + return false; +} + } // vk #endif // _VKQUERYUTIL_HPP diff --git a/external/vulkancts/modules/vulkan/vktTestCase.cpp b/external/vulkancts/modules/vulkan/vktTestCase.cpp index ddf6de9..42b066c 100644 --- a/external/vulkancts/modules/vulkan/vktTestCase.cpp +++ b/external/vulkancts/modules/vulkan/vktTestCase.cpp @@ -29,6 +29,9 @@ #include "vkDeviceUtil.hpp" #include "vkMemUtil.hpp" #include "vkPlatform.hpp" +#include "vkDebugReportUtil.hpp" + +#include "tcuCommandLine.hpp" #include "deMemory.h" @@ -38,8 +41,71 @@ namespace vkt // Default device utilities using std::vector; +using std::string; using namespace vk; +vector getValidationLayers (const vector& supportedLayers) +{ + static const char* s_magicLayer = "VK_LAYER_LUNARG_standard_validation"; + static const char* s_defaultLayers[] = + { + "VK_LAYER_GOOGLE_threading", + "VK_LAYER_LUNARG_parameter_validation", + "VK_LAYER_LUNARG_device_limits", + "VK_LAYER_LUNARG_object_tracker", + "VK_LAYER_LUNARG_image", + "VK_LAYER_LUNARG_core_validation", + "VK_LAYER_LUNARG_swapchain", + "VK_LAYER_GOOGLE_unique_objects" + }; + + vector enabledLayers; + + if (isLayerSupported(supportedLayers, RequiredLayer(s_magicLayer))) + enabledLayers.push_back(s_magicLayer); + else + { + for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_defaultLayers); ++ndx) + { + if (isLayerSupported(supportedLayers, RequiredLayer(s_defaultLayers[ndx]))) + enabledLayers.push_back(s_defaultLayers[ndx]); + } + } + + return enabledLayers; +} + +vector getValidationLayers (const PlatformInterface& vkp) +{ + return getValidationLayers(enumerateInstanceLayerProperties(vkp)); +} + +vector getValidationLayers (const InstanceInterface& vki, VkPhysicalDevice physicalDevice) +{ + return getValidationLayers(enumerateDeviceLayerProperties(vki, physicalDevice)); +} + +Move createInstance (const PlatformInterface& vkp, const tcu::CommandLine& cmdLine) +{ + const bool isValidationEnabled = cmdLine.isValidationEnabled(); + vector enabledExtensions; + vector enabledLayers; + + if (isValidationEnabled) + { + if (isDebugReportSupported(vkp)) + enabledExtensions.push_back("VK_EXT_debug_report"); + else + TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported"); + + enabledLayers = getValidationLayers(vkp); + if (enabledLayers.empty()) + TCU_THROW(NotSupportedError, "No validation layers found"); + } + + return createDefaultInstance(vkp, enabledLayers, enabledExtensions); +} + static deUint32 findQueueFamilyIndexWithCaps (const InstanceInterface& vkInstance, VkPhysicalDevice physicalDevice, VkQueueFlags requiredCaps) { const vector queueProps = getPhysicalDeviceQueueFamilyProperties(vkInstance, physicalDevice); @@ -53,15 +119,33 @@ static deUint32 findQueueFamilyIndexWithCaps (const InstanceInterface& vkInstanc TCU_THROW(NotSupportedError, "No matching queue found"); } -Move createDefaultDevice (const InstanceInterface& vki, VkPhysicalDevice physicalDevice, deUint32 queueIndex, const VkPhysicalDeviceFeatures& enabledFeatures) +Move createDefaultDevice (const InstanceInterface& vki, + VkPhysicalDevice physicalDevice, + deUint32 queueIndex, + const VkPhysicalDeviceFeatures& enabledFeatures, + const tcu::CommandLine& cmdLine) { VkDeviceQueueCreateInfo queueInfo; VkDeviceCreateInfo deviceInfo; + vector enabledLayers; + vector layerPtrs; const float queuePriority = 1.0f; deMemset(&queueInfo, 0, sizeof(queueInfo)); deMemset(&deviceInfo, 0, sizeof(deviceInfo)); + if (cmdLine.isValidationEnabled()) + { + enabledLayers = getValidationLayers(vki, physicalDevice); + if (enabledLayers.empty()) + TCU_THROW(NotSupportedError, "No validation layers found"); + } + + layerPtrs.resize(enabledLayers.size()); + + for (size_t ndx = 0; ndx < enabledLayers.size(); ++ndx) + layerPtrs[ndx] = enabledLayers[ndx].c_str(); + queueInfo.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; queueInfo.pNext = DE_NULL; queueInfo.flags = (VkDeviceQueueCreateFlags)0u; @@ -75,8 +159,8 @@ Move createDefaultDevice (const InstanceInterface& vki, VkPhysicalDevi deviceInfo.pQueueCreateInfos = &queueInfo; deviceInfo.enabledExtensionCount = 0u; deviceInfo.ppEnabledExtensionNames = DE_NULL; - deviceInfo.enabledLayerCount = 0u; - deviceInfo.ppEnabledLayerNames = DE_NULL; + deviceInfo.enabledLayerCount = (deUint32)layerPtrs.size(); + deviceInfo.ppEnabledLayerNames = (layerPtrs.empty() ? DE_NULL : &layerPtrs[0]); deviceInfo.pEnabledFeatures = &enabledFeatures; return createDevice(vki, physicalDevice, &deviceInfo); @@ -117,13 +201,13 @@ private: }; DefaultDevice::DefaultDevice (const PlatformInterface& vkPlatform, const tcu::CommandLine& cmdLine) - : m_instance (createDefaultInstance(vkPlatform)) + : m_instance (createInstance(vkPlatform, cmdLine)) , m_instanceInterface (vkPlatform, *m_instance) , m_physicalDevice (chooseDevice(m_instanceInterface, *m_instance, cmdLine)) , m_universalQueueFamilyIndex (findQueueFamilyIndexWithCaps(m_instanceInterface, m_physicalDevice, VK_QUEUE_GRAPHICS_BIT|VK_QUEUE_COMPUTE_BIT)) , m_deviceFeatures (filterDefaultDeviceFeatures(getPhysicalDeviceFeatures(m_instanceInterface, m_physicalDevice))) - , m_deviceProperties (getPhysicalDeviceProperties(m_instanceInterface, m_physicalDevice)) // \note All supported features are enabled - , m_device (createDefaultDevice(m_instanceInterface, m_physicalDevice, m_universalQueueFamilyIndex, m_deviceFeatures)) + , m_deviceProperties (getPhysicalDeviceProperties(m_instanceInterface, m_physicalDevice)) + , m_device (createDefaultDevice(m_instanceInterface, m_physicalDevice, m_universalQueueFamilyIndex, m_deviceFeatures, cmdLine)) , m_deviceInterface (m_instanceInterface, *m_device) { } diff --git a/external/vulkancts/modules/vulkan/vktTestPackage.cpp b/external/vulkancts/modules/vulkan/vktTestPackage.cpp index 4ed6efb..69d51fc 100644 --- a/external/vulkancts/modules/vulkan/vktTestPackage.cpp +++ b/external/vulkancts/modules/vulkan/vktTestPackage.cpp @@ -26,11 +26,14 @@ #include "tcuPlatform.hpp" #include "tcuTestCase.hpp" #include "tcuTestLog.hpp" +#include "tcuCommandLine.hpp" #include "vkPlatform.hpp" #include "vkPrograms.hpp" #include "vkBinaryRegistry.hpp" #include "vkGlslToSpirV.hpp" +#include "vkDebugReportUtil.hpp" +#include "vkQueryUtil.hpp" #include "deUniquePtr.hpp" @@ -134,27 +137,42 @@ using de::UniquePtr; using de::MovePtr; using tcu::TestLog; +namespace +{ + +MovePtr createDebugReportRecorder (const vk::PlatformInterface& vkp, const vk::InstanceInterface& vki, vk::VkInstance instance) +{ + if (isDebugReportSupported(vkp)) + return MovePtr(new vk::DebugReportRecorder(vki, instance)); + else + TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported"); +} + +} // anonymous + // TestCaseExecutor class TestCaseExecutor : public tcu::TestCaseExecutor { public: - TestCaseExecutor (tcu::TestContext& testCtx); - ~TestCaseExecutor (void); + TestCaseExecutor (tcu::TestContext& testCtx); + ~TestCaseExecutor (void); - virtual void init (tcu::TestCase* testCase, const std::string& path); - virtual void deinit (tcu::TestCase* testCase); + virtual void init (tcu::TestCase* testCase, const std::string& path); + virtual void deinit (tcu::TestCase* testCase); - virtual tcu::TestNode::IterateResult iterate (tcu::TestCase* testCase); + virtual tcu::TestNode::IterateResult iterate (tcu::TestCase* testCase); private: - vk::BinaryCollection m_progCollection; - vk::BinaryRegistryReader m_prebuiltBinRegistry; + vk::BinaryCollection m_progCollection; + vk::BinaryRegistryReader m_prebuiltBinRegistry; + + const UniquePtr m_library; + Context m_context; - de::UniquePtr m_library; - Context m_context; + const UniquePtr m_debugReportRecorder; - TestInstance* m_instance; //!< Current test case instance + TestInstance* m_instance; //!< Current test case instance }; static MovePtr createLibrary (tcu::TestContext& testCtx) @@ -166,6 +184,11 @@ TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx) : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt") , m_library (createLibrary(testCtx)) , m_context (testCtx, m_library->getPlatformInterface(), m_progCollection) + , m_debugReportRecorder (testCtx.getCommandLine().isValidationEnabled() + ? createDebugReportRecorder(m_library->getPlatformInterface(), + m_context.getInstanceInterface(), + m_context.getInstance()) + : MovePtr(DE_NULL)) , m_instance (DE_NULL) { } @@ -220,6 +243,34 @@ void TestCaseExecutor::deinit (tcu::TestCase*) { delete m_instance; m_instance = DE_NULL; + + // Collect and report any debug messages + if (m_debugReportRecorder) + { + typedef vk::DebugReportRecorder::MessageList DebugMessages; + + const DebugMessages& messages = m_debugReportRecorder->getMessages(); + tcu::TestLog& log = m_context.getTestContext().getLog(); + + if (messages.begin() != messages.end()) + { + const tcu::ScopedLogSection section (log, "DebugMessages", "Debug Messages"); + bool anyErrors = false; + + for (DebugMessages::const_iterator curMsg = messages.begin(); curMsg != messages.end(); ++curMsg) + { + log << tcu::TestLog::Message << *curMsg << tcu::TestLog::EndMessage; + + if ((curMsg->flags & vk::VK_DEBUG_REPORT_ERROR_BIT_EXT) != 0) + anyErrors = true; + } + + m_debugReportRecorder->clearMessages(); + + if (anyErrors) + m_context.getTestContext().setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, "API usage error found"); + } + } } tcu::TestNode::IterateResult TestCaseExecutor::iterate (tcu::TestCase*) diff --git a/framework/common/tcuCommandLine.cpp b/framework/common/tcuCommandLine.cpp index ba267d5..1582329 100644 --- a/framework/common/tcuCommandLine.cpp +++ b/framework/common/tcuCommandLine.cpp @@ -85,6 +85,7 @@ DE_DECLARE_COMMAND_LINE_OPT(LogShaderSources, bool); DE_DECLARE_COMMAND_LINE_OPT(TestOOM, bool); DE_DECLARE_COMMAND_LINE_OPT(VKDeviceID, int); DE_DECLARE_COMMAND_LINE_OPT(LogFlush, bool); +DE_DECLARE_COMMAND_LINE_OPT(Validation, bool); static void parseIntList (const char* src, std::vector* dst) { @@ -170,7 +171,8 @@ void registerOptions (de::cmdline::Parser& parser) << Option (DE_NULL, "deqp-log-images", "Enable or disable logging of result images", s_enableNames, "enable") << Option (DE_NULL, "deqp-log-shader-sources", "Enable or disable logging of shader sources", s_enableNames, "enable") << Option (DE_NULL, "deqp-test-oom", "Run tests that exhaust memory on purpose", s_enableNames, TEST_OOM_DEFAULT) - << Option (DE_NULL, "deqp-log-flush", "Enable or disable log file fflush", s_enableNames, "enable"); + << Option (DE_NULL, "deqp-log-flush", "Enable or disable log file fflush", s_enableNames, "enable") + << Option (DE_NULL, "deqp-validation", "Enable or disable test case validation", s_enableNames, "disable"); } void registerLegacyOptions (de::cmdline::Parser& parser) @@ -817,6 +819,7 @@ int CommandLine::getGLConfigId (void) const { return m_cmdLine.getOption int CommandLine::getCLPlatformId (void) const { return m_cmdLine.getOption(); } const std::vector& CommandLine::getCLDeviceIds (void) const { return m_cmdLine.getOption(); } int CommandLine::getVKDeviceId (void) const { return m_cmdLine.getOption(); } +bool CommandLine::isValidationEnabled (void) const { return m_cmdLine.getOption(); } bool CommandLine::isOutOfMemoryTestEnabled (void) const { return m_cmdLine.getOption(); } const char* CommandLine::getGLContextType (void) const diff --git a/framework/common/tcuCommandLine.hpp b/framework/common/tcuCommandLine.hpp index 6195bec..6f2bb5b 100644 --- a/framework/common/tcuCommandLine.hpp +++ b/framework/common/tcuCommandLine.hpp @@ -179,6 +179,9 @@ public: //! Get Vulkan device ID (--deqp-vk-device-id) int getVKDeviceId (void) const; + //! Enable development-time test case validation checks + bool isValidationEnabled (void) const; + //! Should we run tests that exhaust memory (--deqp-test-oom) bool isOutOfMemoryTestEnabled(void) const; -- 2.7.4