external/vulkancts/modules/vulkan/vktTestPackage.cpp

   1 /*-------------------------------------------------------------------------
   2  * Vulkan Conformance Tests
   3  * ------------------------
   4  *
   5  * Copyright (c) 2015 Google Inc.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Vulkan Test Package
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "vktTestPackage.hpp"
  25
  26 #include "tcuPlatform.hpp"
  27 #include "tcuTestCase.hpp"
  28 #include "tcuTestLog.hpp"
  29 #include "tcuCommandLine.hpp"
  30
  31 #include "vkPlatform.hpp"
  32 #include "vkPrograms.hpp"
  33 #include "vkBinaryRegistry.hpp"
  34 #include "vkGlslToSpirV.hpp"
  35 #include "vkDebugReportUtil.hpp"
  36 #include "vkQueryUtil.hpp"
  37
  38 #include "deUniquePtr.hpp"
  39
  40 #include "vktTestGroupUtil.hpp"
  41 #include "vktApiTests.hpp"
  42 #include "vktPipelineTests.hpp"
  43 #include "vktBindingModelTests.hpp"
  44 #include "vktSpvAsmTests.hpp"
  45 #include "vktShaderLibrary.hpp"
  46 #include "vktRenderPassTests.hpp"
  47 #include "vktMemoryTests.hpp"
  48 #include "vktShaderRenderDiscardTests.hpp"
  49 #include "vktShaderRenderIndexingTests.hpp"
  50 #include "vktShaderRenderLoopTests.hpp"
  51 #include "vktShaderRenderMatrixTests.hpp"
  52 #include "vktShaderRenderOperatorTests.hpp"
  53 #include "vktShaderRenderReturnTests.hpp"
  54 #include "vktShaderRenderStructTests.hpp"
  55 #include "vktShaderRenderSwitchTests.hpp"
  56 #include "vktShaderBuiltinTests.hpp"
  57 #include "vktOpaqueTypeIndexingTests.hpp"
  58 #include "vktUniformBlockTests.hpp"
  59 #include "vktDynamicStateTests.hpp"
  60 #include "vktSSBOLayoutTests.hpp"
  61 #include "vktQueryPoolTests.hpp"
  62 #include "vktDrawTests.hpp"
  63 #include "vktComputeTests.hpp"
  64 #include "vktImageTests.hpp"
  65 #include "vktInfoTests.hpp"
  66 #include "vktWsiTests.hpp"
  67 #include "vktSynchronization.hpp"
  68 #include "vktSparseResourcesTests.hpp"
  69 #include "vktTessellationTests.hpp"
  70
  71 #include <vector>
  72 #include <sstream>
  73
  74 namespace // compilation
  75 {
  76
  77 vk::ProgramBinary* compileProgram (const glu::ProgramSources& source, glu::ShaderProgramInfo* buildInfo)
  78 {
  79         return vk::buildProgram(source, vk::PROGRAM_FORMAT_SPIRV, buildInfo);
  80 }
  81
  82 vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, vk::SpirVProgramInfo* buildInfo)
  83 {
  84         return vk::assembleProgram(source, buildInfo);
  85 }
  86
  87 template <typename InfoType, typename IteratorType>
  88 vk::ProgramBinary* buildProgram (const std::string&                                     casePath,
  89                                                                  IteratorType                                           iter,
  90                                                                  const vk::BinaryRegistryReader&        prebuiltBinRegistry,
  91                                                                  tcu::TestLog&                                          log,
  92                                                                  vk::BinaryCollection*                          progCollection)
  93 {
  94         const vk::ProgramIdentifier             progId          (casePath, iter.getName());
  95         const tcu::ScopedLogSection             progSection     (log, iter.getName(), "Program: " + iter.getName());
  96         de::MovePtr<vk::ProgramBinary>  binProg;
  97         InfoType                                                buildInfo;
  98
  99         try
 100         {
 101                 binProg = de::MovePtr<vk::ProgramBinary>(compileProgram(iter.getProgram(), &buildInfo));
 102                 log << buildInfo;
 103         }
 104         catch (const tcu::NotSupportedError& err)
 105         {
 106                 // Try to load from cache
 107                 log << err << tcu::TestLog::Message << "Building from source not supported, loading stored binary instead" << tcu::TestLog::EndMessage;
 108
 109                 binProg = de::MovePtr<vk::ProgramBinary>(prebuiltBinRegistry.loadProgram(progId));
 110
 111                 log << iter.getProgram();
 112         }
 113         catch (const tcu::Exception&)
 114         {
 115                 // Build failed for other reason
 116                 log << buildInfo;
 117                 throw;
 118         }
 119
 120         TCU_CHECK_INTERNAL(binProg);
 121
 122         {
 123                 vk::ProgramBinary* const        returnBinary    = binProg.get();
 124
 125                 progCollection->add(progId.programName, binProg);
 126
 127                 return returnBinary;
 128         }
 129 }
 130
 131 } // anonymous(compilation)
 132
 133 namespace vkt
 134 {
 135
 136 using std::vector;
 137 using de::UniquePtr;
 138 using de::MovePtr;
 139 using tcu::TestLog;
 140
 141 namespace
 142 {
 143
 144 MovePtr<vk::DebugReportRecorder> createDebugReportRecorder (const vk::PlatformInterface& vkp, const vk::InstanceInterface& vki, vk::VkInstance instance)
 145 {
 146         if (isDebugReportSupported(vkp))
 147                 return MovePtr<vk::DebugReportRecorder>(new vk::DebugReportRecorder(vki, instance));
 148         else
 149                 TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported");
 150 }
 151
 152 } // anonymous
 153
 154 // TestCaseExecutor
 155
 156 class TestCaseExecutor : public tcu::TestCaseExecutor
 157 {
 158 public:
 159                                                                                                 TestCaseExecutor        (tcu::TestContext& testCtx);
 160                                                                                                 ~TestCaseExecutor       (void);
 161
 162         virtual void                                                            init                            (tcu::TestCase* testCase, const std::string& path);
 163         virtual void                                                            deinit                          (tcu::TestCase* testCase);
 164
 165         virtual tcu::TestNode::IterateResult            iterate                         (tcu::TestCase* testCase);
 166
 167 private:
 168         vk::BinaryCollection                                            m_progCollection;
 169         vk::BinaryRegistryReader                                        m_prebuiltBinRegistry;
 170
 171         const UniquePtr<vk::Library>                            m_library;
 172         Context                                                                         m_context;
 173
 174         const UniquePtr<vk::DebugReportRecorder>        m_debugReportRecorder;
 175
 176         TestInstance*                                                           m_instance;                     //!< Current test case instance
 177 };
 178
 179 static MovePtr<vk::Library> createLibrary (tcu::TestContext& testCtx)
 180 {
 181         return MovePtr<vk::Library>(testCtx.getPlatform().getVulkanPlatform().createLibrary());
 182 }
 183
 184 TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx)
 185         : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt")
 186         , m_library                             (createLibrary(testCtx))
 187         , m_context                             (testCtx, m_library->getPlatformInterface(), m_progCollection)
 188         , m_debugReportRecorder (testCtx.getCommandLine().isValidationEnabled()
 189                                                          ? createDebugReportRecorder(m_library->getPlatformInterface(),
 190                                                                                                                  m_context.getInstanceInterface(),
 191                                                                                                                  m_context.getInstance())
 192                                                          : MovePtr<vk::DebugReportRecorder>(DE_NULL))
 193         , m_instance                    (DE_NULL)
 194 {
 195 }
 196
 197 TestCaseExecutor::~TestCaseExecutor (void)
 198 {
 199         delete m_instance;
 200 }
 201
 202 void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePath)
 203 {
 204         const TestCase*                 vktCase         = dynamic_cast<TestCase*>(testCase);
 205         tcu::TestLog&                   log                     = m_context.getTestContext().getLog();
 206         vk::SourceCollections   sourceProgs;
 207
 208         DE_UNREF(casePath); // \todo [2015-03-13 pyry] Use this to identify ProgramCollection storage path
 209
 210         if (!vktCase)
 211                 TCU_THROW(InternalError, "Test node not an instance of vkt::TestCase");
 212
 213         m_progCollection.clear();
 214         vktCase->initPrograms(sourceProgs);
 215
 216         for (vk::GlslSourceCollection::Iterator progIter = sourceProgs.glslSources.begin(); progIter != sourceProgs.glslSources.end(); ++progIter)
 217         {
 218                 vk::ProgramBinary* binProg = buildProgram<glu::ShaderProgramInfo, vk::GlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection);
 219
 220                 try
 221                 {
 222                         std::ostringstream disasm;
 223
 224                         vk::disassembleProgram(*binProg, &disasm);
 225
 226                         log << vk::SpirVAsmSource(disasm.str());
 227                 }
 228                 catch (const tcu::NotSupportedError& err)
 229                 {
 230                         log << err;
 231                 }
 232         }
 233
 234         for (vk::SpirVAsmCollection::Iterator asmIterator = sourceProgs.spirvAsmSources.begin(); asmIterator != sourceProgs.spirvAsmSources.end(); ++asmIterator)
 235         {
 236                 buildProgram<vk::SpirVProgramInfo, vk::SpirVAsmCollection::Iterator>(casePath, asmIterator, m_prebuiltBinRegistry, log, &m_progCollection);
 237         }
 238
 239         DE_ASSERT(!m_instance);
 240         m_instance = vktCase->createInstance(m_context);
 241 }
 242
 243 void TestCaseExecutor::deinit (tcu::TestCase*)
 244 {
 245         delete m_instance;
 246         m_instance = DE_NULL;
 247
 248         // Collect and report any debug messages
 249         if (m_debugReportRecorder)
 250         {
 251                 // \note We are not logging INFORMATION and DEBUG messages
 252                 static const vk::VkDebugReportFlagsEXT                  errorFlags              = vk::VK_DEBUG_REPORT_ERROR_BIT_EXT;
 253                 static const vk::VkDebugReportFlagsEXT                  logFlags                = errorFlags
 254                                                                                                                                                 | vk::VK_DEBUG_REPORT_WARNING_BIT_EXT
 255                                                                                                                                                 | vk::VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT;
 256
 257                 typedef vk::DebugReportRecorder::MessageList    DebugMessages;
 258
 259                 const DebugMessages&    messages        = m_debugReportRecorder->getMessages();
 260                 tcu::TestLog&                   log                     = m_context.getTestContext().getLog();
 261
 262                 if (messages.begin() != messages.end())
 263                 {
 264                         const tcu::ScopedLogSection     section         (log, "DebugMessages", "Debug Messages");
 265                         int                                                     numErrors       = 0;
 266
 267                         for (DebugMessages::const_iterator curMsg = messages.begin(); curMsg != messages.end(); ++curMsg)
 268                         {
 269                                 if ((curMsg->flags & logFlags) != 0)
 270                                         log << tcu::TestLog::Message << *curMsg << tcu::TestLog::EndMessage;
 271
 272                                 if ((curMsg->flags & errorFlags) != 0)
 273                                         numErrors += 1;
 274                         }
 275
 276                         m_debugReportRecorder->clearMessages();
 277
 278                         if (numErrors > 0)
 279                                 m_context.getTestContext().setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, (de::toString(numErrors) + " API usage errors found").c_str());
 280                 }
 281         }
 282 }
 283
 284 tcu::TestNode::IterateResult TestCaseExecutor::iterate (tcu::TestCase*)
 285 {
 286         DE_ASSERT(m_instance);
 287
 288         const tcu::TestStatus   result  = m_instance->iterate();
 289
 290         if (result.isComplete())
 291         {
 292                 // Vulkan tests shouldn't set result directly
 293                 DE_ASSERT(m_context.getTestContext().getTestResult() == QP_TEST_RESULT_LAST);
 294                 m_context.getTestContext().setTestResult(result.getCode(), result.getDescription().c_str());
 295                 return tcu::TestNode::STOP;
 296         }
 297         else
 298                 return tcu::TestNode::CONTINUE;
 299 }
 300
 301 // GLSL shader tests
 302
 303 void createGlslTests (tcu::TestCaseGroup* glslTests)
 304 {
 305         tcu::TestContext&       testCtx         = glslTests->getTestContext();
 306
 307         // ShaderLibrary-based tests
 308         static const struct
 309         {
 310                 const char*             name;
 311                 const char*             description;
 312         } s_es310Tests[] =
 313         {
 314                 { "arrays",                                             "Arrays"                                        },
 315                 { "conditionals",                               "Conditional statements"        },
 316                 { "constant_expressions",               "Constant expressions"          },
 317                 { "constants",                                  "Constants"                                     },
 318                 { "conversions",                                "Type conversions"                      },
 319                 { "functions",                                  "Functions"                                     },
 320                 { "linkage",                                    "Linking"                                       },
 321                 { "scoping",                                    "Scoping"                                       },
 322                 { "swizzles",                                   "Swizzles"                                      },
 323         };
 324
 325         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_es310Tests); ndx++)
 326                 glslTests->addChild(createShaderLibraryGroup(testCtx,
 327                                                                                                          s_es310Tests[ndx].name,
 328                                                                                                          s_es310Tests[ndx].description,
 329                                                                                                          std::string("vulkan/glsl/es310/") + s_es310Tests[ndx].name + ".test").release());
 330
 331         // ShaderRenderCase-based tests
 332         glslTests->addChild(sr::createDiscardTests      (testCtx));
 333         glslTests->addChild(sr::createIndexingTests     (testCtx));
 334         glslTests->addChild(sr::createLoopTests         (testCtx));
 335         glslTests->addChild(sr::createMatrixTests       (testCtx));
 336         glslTests->addChild(sr::createOperatorTests     (testCtx));
 337         glslTests->addChild(sr::createReturnTests       (testCtx));
 338         glslTests->addChild(sr::createStructTests       (testCtx));
 339         glslTests->addChild(sr::createSwitchTests       (testCtx));
 340
 341         // ShaderExecutor-based tests
 342         glslTests->addChild(shaderexecutor::createBuiltinTests                          (testCtx));
 343         glslTests->addChild(shaderexecutor::createOpaqueTypeIndexingTests       (testCtx));
 344 }
 345
 346 // TestPackage
 347
 348 TestPackage::TestPackage (tcu::TestContext& testCtx)
 349         : tcu::TestPackage(testCtx, "dEQP-VK", "dEQP Vulkan Tests")
 350 {
 351 }
 352
 353 TestPackage::~TestPackage (void)
 354 {
 355 }
 356
 357 tcu::TestCaseExecutor* TestPackage::createExecutor (void) const
 358 {
 359         return new TestCaseExecutor(m_testCtx);
 360 }
 361
 362 void TestPackage::init (void)
 363 {
 364         addChild(createTestGroup                        (m_testCtx, "info", "Build and Device Info Tests", createInfoTests));
 365         addChild(api::createTests                       (m_testCtx));
 366         addChild(pipeline::createTests          (m_testCtx));
 367         addChild(BindingModel::createTests      (m_testCtx));
 368         addChild(SpirVAssembly::createTests     (m_testCtx));
 369         addChild(createTestGroup                        (m_testCtx, "glsl", "GLSL shader execution tests", createGlslTests));
 370         addChild(createRenderPassTests          (m_testCtx));
 371         addChild(memory::createTests            (m_testCtx));
 372         addChild(ubo::createTests                       (m_testCtx));
 373         addChild(DynamicState::createTests      (m_testCtx));
 374         addChild(ssbo::createTests                      (m_testCtx));
 375         addChild(QueryPool::createTests         (m_testCtx));
 376         addChild(Draw::createTests                      (m_testCtx));
 377         addChild(compute::createTests           (m_testCtx));
 378         addChild(image::createTests                     (m_testCtx));
 379         addChild(wsi::createTests                       (m_testCtx));
 380         addChild(createSynchronizationTests     (m_testCtx));
 381         addChild(sparse::createTests            (m_testCtx));
 382         addChild(tessellation::createTests      (m_testCtx));
 383 }
 384
 385 } // vkt