Add --deqp-validation=enable/disable option
authorPyry Haulos <phaulos@google.com>
Tue, 29 Mar 2016 00:35:44 +0000 (17:35 -0700)
committerPyry Haulos <phaulos@google.com>
Mon, 4 Apr 2016 19:47:20 +0000 (12:47 -0700)
Specifying --deqp-validation=enable when running Vulkan tests will
result in:

1) vkt::TestCaseExecutor will create debug report callback on default
instance. Any debug messages reported during test execution will be
included as a separate log section in the test log.

2) vkt::Context will enable known validation layers on both default
instance as well as default device.

If any errors are reported (for example validation errors) test result
will be set to InternalError.

Change-Id: Icc89a3f57f0c19170988d7b518fe6b24dcf3df6e

external/vulkancts/framework/vulkan/vkDebugReportUtil.cpp
external/vulkancts/framework/vulkan/vkDebugReportUtil.hpp
external/vulkancts/framework/vulkan/vkQueryUtil.cpp
external/vulkancts/framework/vulkan/vkQueryUtil.hpp
external/vulkancts/modules/vulkan/vktTestCase.cpp
external/vulkancts/modules/vulkan/vktTestPackage.cpp
framework/common/tcuCommandLine.cpp
framework/common/tcuCommandLine.hpp

index d3f9ff7..c2f6335 100644 (file)
@@ -23,6 +23,7 @@
 
 #include "vkDebugReportUtil.hpp"
 #include "vkRefUtil.hpp"
+#include "vkQueryUtil.hpp"
 #include "deArrayUtil.hpp"
 
 namespace vk
@@ -156,4 +157,10 @@ DebugReportRecorder::~DebugReportRecorder (void)
 {
 }
 
+bool isDebugReportSupported (const PlatformInterface& vkp)
+{
+       return isExtensionSupported(enumerateInstanceExtensionProperties(vkp, DE_NULL),
+                                                               RequiredExtension("VK_EXT_debug_report"));
+}
+
 } // vk
index 4463e27..6c4a8dd 100644 (file)
@@ -85,6 +85,8 @@ private:
        const Unique<VkDebugReportCallbackEXT>  m_callback;
 };
 
+bool   isDebugReportSupported          (const PlatformInterface& vkp);
+
 } // vk
 
 #endif // _VKDEBUGREPORTUTIL_HPP
index 9a7300a..c24f714 100644 (file)
@@ -242,9 +242,34 @@ bool isCompatible (const VkExtensionProperties& extensionProperties, const Requi
        return true;
 }
 
+bool isCompatible (const VkLayerProperties& layerProperties, const RequiredLayer& required)
+{
+       if (required.name != layerProperties.layerName)
+               return false;
+
+       if (required.minSpecVersion && required.minSpecVersion.get() > layerProperties.specVersion)
+               return false;
+
+       if (required.maxSpecVersion && required.maxSpecVersion.get() < layerProperties.specVersion)
+               return false;
+
+       if (required.minImplVersion && required.minImplVersion.get() > layerProperties.implementationVersion)
+               return false;
+
+       if (required.maxImplVersion && required.maxImplVersion.get() < layerProperties.implementationVersion)
+               return false;
+
+       return true;
+}
+
 bool isExtensionSupported (const std::vector<VkExtensionProperties>& extensions, const RequiredExtension& required)
 {
        return isExtensionSupported(extensions.begin(), extensions.end(), required);
 }
 
+bool isLayerSupported (const std::vector<VkLayerProperties>& layers, const RequiredLayer& required)
+{
+       return isLayerSupported(layers.begin(), layers.end(), required);
+}
+
 } // vk
index 4eb209e..0235f7b 100644 (file)
@@ -70,12 +70,38 @@ struct RequiredExtension
        {}
 };
 
+struct RequiredLayer
+{
+       std::string                             name;
+       tcu::Maybe<deUint32>    minSpecVersion;
+       tcu::Maybe<deUint32>    maxSpecVersion;
+       tcu::Maybe<deUint32>    minImplVersion;
+       tcu::Maybe<deUint32>    maxImplVersion;
+
+       explicit RequiredLayer (const std::string&                      name_,
+                                                       tcu::Maybe<deUint32>            minSpecVersion_         = tcu::nothing<deUint32>(),
+                                                       tcu::Maybe<deUint32>            maxSpecVersion_         = tcu::nothing<deUint32>(),
+                                                       tcu::Maybe<deUint32>            minImplVersion_         = tcu::nothing<deUint32>(),
+                                                       tcu::Maybe<deUint32>            maxImplVersion_         = tcu::nothing<deUint32>())
+               : name                  (name_)
+               , minSpecVersion(minSpecVersion_)
+               , maxSpecVersion(maxSpecVersion_)
+               , minImplVersion(minImplVersion_)
+               , maxImplVersion(maxImplVersion_)
+       {}
+};
+
 bool                                                                           isCompatible                                                    (const VkExtensionProperties& extensionProperties, const RequiredExtension& required);
+bool                                                                           isCompatible                                                    (const VkLayerProperties& layerProperties, const RequiredLayer& required);
 
 template<typename ExtensionIterator>
 bool                                                                           isExtensionSupported                                    (ExtensionIterator begin, ExtensionIterator end, const RequiredExtension& required);
 bool                                                                           isExtensionSupported                                    (const std::vector<VkExtensionProperties>& extensions, const RequiredExtension& required);
 
+template<typename LayerIterator>
+bool                                                                           isLayerSupported                                                (LayerIterator begin, LayerIterator end, const RequiredLayer& required);
+bool                                                                           isLayerSupported                                                (const std::vector<VkLayerProperties>& layers, const RequiredLayer& required);
+
 // Return variable initialization validation
 
 typedef struct
@@ -83,6 +109,7 @@ typedef struct
        size_t          offset;
        size_t          size;
 } QueryMemberTableEntry;
+
 template <typename Context, typename Interface, typename Type>
 bool validateInitComplete(Context context, void (Interface::*Function)(Context, Type*)const, const Interface& interface, const QueryMemberTableEntry* queryMemberTableEntry)
 {
@@ -116,6 +143,17 @@ bool isExtensionSupported (ExtensionIterator begin, ExtensionIterator end, const
        return false;
 }
 
+template<typename LayerIterator>
+bool isLayerSupported (LayerIterator begin, LayerIterator end, const RequiredLayer& required)
+{
+       for (LayerIterator cur = begin; cur != end; ++cur)
+       {
+               if (isCompatible(*cur, required))
+                       return true;
+       }
+       return false;
+}
+
 } // vk
 
 #endif // _VKQUERYUTIL_HPP
index ddf6de9..42b066c 100644 (file)
@@ -29,6 +29,9 @@
 #include "vkDeviceUtil.hpp"
 #include "vkMemUtil.hpp"
 #include "vkPlatform.hpp"
+#include "vkDebugReportUtil.hpp"
+
+#include "tcuCommandLine.hpp"
 
 #include "deMemory.h"
 
@@ -38,8 +41,71 @@ namespace vkt
 // Default device utilities
 
 using std::vector;
+using std::string;
 using namespace vk;
 
+vector<string> getValidationLayers (const vector<VkLayerProperties>& supportedLayers)
+{
+       static const char*      s_magicLayer            = "VK_LAYER_LUNARG_standard_validation";
+       static const char*      s_defaultLayers[]       =
+       {
+               "VK_LAYER_GOOGLE_threading",
+               "VK_LAYER_LUNARG_parameter_validation",
+               "VK_LAYER_LUNARG_device_limits",
+               "VK_LAYER_LUNARG_object_tracker",
+               "VK_LAYER_LUNARG_image",
+               "VK_LAYER_LUNARG_core_validation",
+               "VK_LAYER_LUNARG_swapchain",
+               "VK_LAYER_GOOGLE_unique_objects"
+       };
+
+       vector<string>          enabledLayers;
+
+       if (isLayerSupported(supportedLayers, RequiredLayer(s_magicLayer)))
+               enabledLayers.push_back(s_magicLayer);
+       else
+       {
+               for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_defaultLayers); ++ndx)
+               {
+                       if (isLayerSupported(supportedLayers, RequiredLayer(s_defaultLayers[ndx])))
+                               enabledLayers.push_back(s_defaultLayers[ndx]);
+               }
+       }
+
+       return enabledLayers;
+}
+
+vector<string> getValidationLayers (const PlatformInterface& vkp)
+{
+       return getValidationLayers(enumerateInstanceLayerProperties(vkp));
+}
+
+vector<string> getValidationLayers (const InstanceInterface& vki, VkPhysicalDevice physicalDevice)
+{
+       return getValidationLayers(enumerateDeviceLayerProperties(vki, physicalDevice));
+}
+
+Move<VkInstance> createInstance (const PlatformInterface& vkp, const tcu::CommandLine& cmdLine)
+{
+       const bool              isValidationEnabled     = cmdLine.isValidationEnabled();
+       vector<string>  enabledExtensions;
+       vector<string>  enabledLayers;
+
+       if (isValidationEnabled)
+       {
+               if (isDebugReportSupported(vkp))
+                       enabledExtensions.push_back("VK_EXT_debug_report");
+               else
+                       TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported");
+
+               enabledLayers = getValidationLayers(vkp);
+               if (enabledLayers.empty())
+                       TCU_THROW(NotSupportedError, "No validation layers found");
+       }
+
+       return createDefaultInstance(vkp, enabledLayers, enabledExtensions);
+}
+
 static deUint32 findQueueFamilyIndexWithCaps (const InstanceInterface& vkInstance, VkPhysicalDevice physicalDevice, VkQueueFlags requiredCaps)
 {
        const vector<VkQueueFamilyProperties>   queueProps      = getPhysicalDeviceQueueFamilyProperties(vkInstance, physicalDevice);
@@ -53,15 +119,33 @@ static deUint32 findQueueFamilyIndexWithCaps (const InstanceInterface& vkInstanc
        TCU_THROW(NotSupportedError, "No matching queue found");
 }
 
-Move<VkDevice> createDefaultDevice (const InstanceInterface& vki, VkPhysicalDevice physicalDevice, deUint32 queueIndex, const VkPhysicalDeviceFeatures& enabledFeatures)
+Move<VkDevice> createDefaultDevice (const InstanceInterface&           vki,
+                                                                       VkPhysicalDevice                                physicalDevice,
+                                                                       deUint32                                                queueIndex,
+                                                                       const VkPhysicalDeviceFeatures& enabledFeatures,
+                                                                       const tcu::CommandLine&                 cmdLine)
 {
        VkDeviceQueueCreateInfo         queueInfo;
        VkDeviceCreateInfo                      deviceInfo;
+       vector<string>                          enabledLayers;
+       vector<const char*>                     layerPtrs;
        const float                                     queuePriority   = 1.0f;
 
        deMemset(&queueInfo,    0, sizeof(queueInfo));
        deMemset(&deviceInfo,   0, sizeof(deviceInfo));
 
+       if (cmdLine.isValidationEnabled())
+       {
+               enabledLayers = getValidationLayers(vki, physicalDevice);
+               if (enabledLayers.empty())
+                       TCU_THROW(NotSupportedError, "No validation layers found");
+       }
+
+       layerPtrs.resize(enabledLayers.size());
+
+       for (size_t ndx = 0; ndx < enabledLayers.size(); ++ndx)
+               layerPtrs[ndx] = enabledLayers[ndx].c_str();
+
        queueInfo.sType                                                 = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
        queueInfo.pNext                                                 = DE_NULL;
        queueInfo.flags                                                 = (VkDeviceQueueCreateFlags)0u;
@@ -75,8 +159,8 @@ Move<VkDevice> createDefaultDevice (const InstanceInterface& vki, VkPhysicalDevi
        deviceInfo.pQueueCreateInfos                    = &queueInfo;
        deviceInfo.enabledExtensionCount                = 0u;
        deviceInfo.ppEnabledExtensionNames              = DE_NULL;
-       deviceInfo.enabledLayerCount                    = 0u;
-       deviceInfo.ppEnabledLayerNames                  = DE_NULL;
+       deviceInfo.enabledLayerCount                    = (deUint32)layerPtrs.size();
+       deviceInfo.ppEnabledLayerNames                  = (layerPtrs.empty() ? DE_NULL : &layerPtrs[0]);
        deviceInfo.pEnabledFeatures                             = &enabledFeatures;
 
        return createDevice(vki, physicalDevice, &deviceInfo);
@@ -117,13 +201,13 @@ private:
 };
 
 DefaultDevice::DefaultDevice (const PlatformInterface& vkPlatform, const tcu::CommandLine& cmdLine)
-       : m_instance                                    (createDefaultInstance(vkPlatform))
+       : m_instance                                    (createInstance(vkPlatform, cmdLine))
        , m_instanceInterface                   (vkPlatform, *m_instance)
        , m_physicalDevice                              (chooseDevice(m_instanceInterface, *m_instance, cmdLine))
        , m_universalQueueFamilyIndex   (findQueueFamilyIndexWithCaps(m_instanceInterface, m_physicalDevice, VK_QUEUE_GRAPHICS_BIT|VK_QUEUE_COMPUTE_BIT))
        , m_deviceFeatures                              (filterDefaultDeviceFeatures(getPhysicalDeviceFeatures(m_instanceInterface, m_physicalDevice)))
-       , m_deviceProperties                    (getPhysicalDeviceProperties(m_instanceInterface, m_physicalDevice)) // \note All supported features are enabled
-       , m_device                                              (createDefaultDevice(m_instanceInterface, m_physicalDevice, m_universalQueueFamilyIndex, m_deviceFeatures))
+       , m_deviceProperties                    (getPhysicalDeviceProperties(m_instanceInterface, m_physicalDevice))
+       , m_device                                              (createDefaultDevice(m_instanceInterface, m_physicalDevice, m_universalQueueFamilyIndex, m_deviceFeatures, cmdLine))
        , m_deviceInterface                             (m_instanceInterface, *m_device)
 {
 }
index 4ed6efb..69d51fc 100644 (file)
 #include "tcuPlatform.hpp"
 #include "tcuTestCase.hpp"
 #include "tcuTestLog.hpp"
+#include "tcuCommandLine.hpp"
 
 #include "vkPlatform.hpp"
 #include "vkPrograms.hpp"
 #include "vkBinaryRegistry.hpp"
 #include "vkGlslToSpirV.hpp"
+#include "vkDebugReportUtil.hpp"
+#include "vkQueryUtil.hpp"
 
 #include "deUniquePtr.hpp"
 
@@ -134,27 +137,42 @@ using de::UniquePtr;
 using de::MovePtr;
 using tcu::TestLog;
 
+namespace
+{
+
+MovePtr<vk::DebugReportRecorder> createDebugReportRecorder (const vk::PlatformInterface& vkp, const vk::InstanceInterface& vki, vk::VkInstance instance)
+{
+       if (isDebugReportSupported(vkp))
+               return MovePtr<vk::DebugReportRecorder>(new vk::DebugReportRecorder(vki, instance));
+       else
+               TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported");
+}
+
+} // anonymous
+
 // TestCaseExecutor
 
 class TestCaseExecutor : public tcu::TestCaseExecutor
 {
 public:
-                                                                                       TestCaseExecutor        (tcu::TestContext& testCtx);
-                                                                                       ~TestCaseExecutor       (void);
+                                                                                               TestCaseExecutor        (tcu::TestContext& testCtx);
+                                                                                               ~TestCaseExecutor       (void);
 
-       virtual void                                                    init                            (tcu::TestCase* testCase, const std::string& path);
-       virtual void                                                    deinit                          (tcu::TestCase* testCase);
+       virtual void                                                            init                            (tcu::TestCase* testCase, const std::string& path);
+       virtual void                                                            deinit                          (tcu::TestCase* testCase);
 
-       virtual tcu::TestNode::IterateResult    iterate                         (tcu::TestCase* testCase);
+       virtual tcu::TestNode::IterateResult            iterate                         (tcu::TestCase* testCase);
 
 private:
-       vk::BinaryCollection                                    m_progCollection;
-       vk::BinaryRegistryReader                                m_prebuiltBinRegistry;
+       vk::BinaryCollection                                            m_progCollection;
+       vk::BinaryRegistryReader                                        m_prebuiltBinRegistry;
+
+       const UniquePtr<vk::Library>                            m_library;
+       Context                                                                         m_context;
 
-       de::UniquePtr<vk::Library>                              m_library;
-       Context                                                                 m_context;
+       const UniquePtr<vk::DebugReportRecorder>        m_debugReportRecorder;
 
-       TestInstance*                                                   m_instance;                     //!< Current test case instance
+       TestInstance*                                                           m_instance;                     //!< Current test case instance
 };
 
 static MovePtr<vk::Library> createLibrary (tcu::TestContext& testCtx)
@@ -166,6 +184,11 @@ TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx)
        : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt")
        , m_library                             (createLibrary(testCtx))
        , m_context                             (testCtx, m_library->getPlatformInterface(), m_progCollection)
+       , m_debugReportRecorder (testCtx.getCommandLine().isValidationEnabled()
+                                                        ? createDebugReportRecorder(m_library->getPlatformInterface(),
+                                                                                                                m_context.getInstanceInterface(),
+                                                                                                                m_context.getInstance())
+                                                        : MovePtr<vk::DebugReportRecorder>(DE_NULL))
        , m_instance                    (DE_NULL)
 {
 }
@@ -220,6 +243,34 @@ void TestCaseExecutor::deinit (tcu::TestCase*)
 {
        delete m_instance;
        m_instance = DE_NULL;
+
+       // Collect and report any debug messages
+       if (m_debugReportRecorder)
+       {
+               typedef vk::DebugReportRecorder::MessageList    DebugMessages;
+
+               const DebugMessages&    messages        = m_debugReportRecorder->getMessages();
+               tcu::TestLog&                   log                     = m_context.getTestContext().getLog();
+
+               if (messages.begin() != messages.end())
+               {
+                       const tcu::ScopedLogSection     section         (log, "DebugMessages", "Debug Messages");
+                       bool                                            anyErrors       = false;
+
+                       for (DebugMessages::const_iterator curMsg = messages.begin(); curMsg != messages.end(); ++curMsg)
+                       {
+                               log << tcu::TestLog::Message << *curMsg << tcu::TestLog::EndMessage;
+
+                               if ((curMsg->flags & vk::VK_DEBUG_REPORT_ERROR_BIT_EXT) != 0)
+                                       anyErrors = true;
+                       }
+
+                       m_debugReportRecorder->clearMessages();
+
+                       if (anyErrors)
+                               m_context.getTestContext().setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, "API usage error found");
+               }
+       }
 }
 
 tcu::TestNode::IterateResult TestCaseExecutor::iterate (tcu::TestCase*)
index ba267d5..1582329 100644 (file)
@@ -85,6 +85,7 @@ DE_DECLARE_COMMAND_LINE_OPT(LogShaderSources,                 bool);
 DE_DECLARE_COMMAND_LINE_OPT(TestOOM,                                   bool);
 DE_DECLARE_COMMAND_LINE_OPT(VKDeviceID,                                        int);
 DE_DECLARE_COMMAND_LINE_OPT(LogFlush,                                  bool);
+DE_DECLARE_COMMAND_LINE_OPT(Validation,                                        bool);
 
 static void parseIntList (const char* src, std::vector<int>* dst)
 {
@@ -170,7 +171,8 @@ void registerOptions (de::cmdline::Parser& parser)
                << Option<LogImages>                    (DE_NULL,       "deqp-log-images",                              "Enable or disable logging of result images",           s_enableNames,          "enable")
                << Option<LogShaderSources>             (DE_NULL,       "deqp-log-shader-sources",              "Enable or disable logging of shader sources",          s_enableNames,          "enable")
                << Option<TestOOM>                              (DE_NULL,       "deqp-test-oom",                                "Run tests that exhaust memory on purpose",                     s_enableNames,          TEST_OOM_DEFAULT)
-               << Option<LogFlush>             (DE_NULL,   "deqp-log-flush",               "Enable or disable log file fflush",                s_enableNames,      "enable");
+               << Option<LogFlush>                             (DE_NULL,       "deqp-log-flush",                               "Enable or disable log file fflush",                            s_enableNames,          "enable")
+               << Option<Validation>                   (DE_NULL,       "deqp-validation",                              "Enable or disable test case validation",                       s_enableNames,          "disable");
 }
 
 void registerLegacyOptions (de::cmdline::Parser& parser)
@@ -817,6 +819,7 @@ int                                         CommandLine::getGLConfigId                              (void) const    { return m_cmdLine.getOption
 int                                            CommandLine::getCLPlatformId                    (void) const    { return m_cmdLine.getOption<opt::CLPlatformID>();                                      }
 const std::vector<int>&        CommandLine::getCLDeviceIds                             (void) const    { return m_cmdLine.getOption<opt::CLDeviceIDs>();                                       }
 int                                            CommandLine::getVKDeviceId                              (void) const    { return m_cmdLine.getOption<opt::VKDeviceID>();                                        }
+bool                                   CommandLine::isValidationEnabled                (void) const    { return m_cmdLine.getOption<opt::Validation>();                                        }
 bool                                   CommandLine::isOutOfMemoryTestEnabled   (void) const    { return m_cmdLine.getOption<opt::TestOOM>();                                           }
 
 const char* CommandLine::getGLContextType (void) const
index 6195bec..6f2bb5b 100644 (file)
@@ -179,6 +179,9 @@ public:
        //! Get Vulkan device ID (--deqp-vk-device-id)
        int                                                             getVKDeviceId                           (void) const;
 
+       //! Enable development-time test case validation checks
+       bool                                                    isValidationEnabled                     (void) const;
+
        //! Should we run tests that exhaust memory (--deqp-test-oom)
        bool                                                    isOutOfMemoryTestEnabled(void) const;