external/vulkancts/modules/vulkan/vktTestPackage.cpp

   1 /*-------------------------------------------------------------------------
   2  * Vulkan Conformance Tests
   3  * ------------------------
   4  *
   5  * Copyright (c) 2015 Google Inc.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Vulkan Test Package
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "vktTestPackage.hpp"
  25
  26 #include "tcuPlatform.hpp"
  27 #include "tcuTestCase.hpp"
  28 #include "tcuTestLog.hpp"
  29 #include "tcuCommandLine.hpp"
  30
  31 #include "vkPlatform.hpp"
  32 #include "vkPrograms.hpp"
  33 #include "vkBinaryRegistry.hpp"
  34 #include "vkShaderToSpirV.hpp"
  35 #include "vkDebugReportUtil.hpp"
  36 #include "vkQueryUtil.hpp"
  37 #include "vkApiVersion.hpp"
  38
  39 #include "deUniquePtr.hpp"
  40
  41 #include "vktTestGroupUtil.hpp"
  42 #include "vktApiTests.hpp"
  43 #include "vktPipelineTests.hpp"
  44 #include "vktBindingModelTests.hpp"
  45 #include "vktSpvAsmTests.hpp"
  46 #include "vktShaderLibrary.hpp"
  47 #include "vktRenderPassTests.hpp"
  48 #include "vktMemoryTests.hpp"
  49 #include "vktShaderRenderBuiltinVarTests.hpp"
  50 #include "vktShaderRenderDerivateTests.hpp"
  51 #include "vktShaderRenderDiscardTests.hpp"
  52 #include "vktShaderRenderIndexingTests.hpp"
  53 #include "vktShaderRenderLoopTests.hpp"
  54 #include "vktShaderRenderMatrixTests.hpp"
  55 #include "vktShaderRenderOperatorTests.hpp"
  56 #include "vktShaderRenderReturnTests.hpp"
  57 #include "vktShaderRenderStructTests.hpp"
  58 #include "vktShaderRenderSwitchTests.hpp"
  59 #include "vktShaderRenderTextureFunctionTests.hpp"
  60 #include "vktShaderRenderTextureGatherTests.hpp"
  61 #include "vktShaderBuiltinTests.hpp"
  62 #include "vktOpaqueTypeIndexingTests.hpp"
  63 #include "vktAtomicOperationTests.hpp"
  64 #include "vktUniformBlockTests.hpp"
  65 #include "vktDynamicStateTests.hpp"
  66 #include "vktSSBOLayoutTests.hpp"
  67 #include "vktQueryPoolTests.hpp"
  68 #include "vktDrawTests.hpp"
  69 #include "vktComputeTests.hpp"
  70 #include "vktImageTests.hpp"
  71 #include "vktInfoTests.hpp"
  72 #include "vktWsiTests.hpp"
  73 #include "vktSynchronizationTests.hpp"
  74 #include "vktSparseResourcesTests.hpp"
  75 #include "vktTessellationTests.hpp"
  76 #include "vktRasterizationTests.hpp"
  77 #include "vktClippingTests.hpp"
  78 #include "vktFragmentOperationsTests.hpp"
  79 #include "vktTextureTests.hpp"
  80 #include "vktGeometryTests.hpp"
  81 #include "vktRobustnessTests.hpp"
  82 #include "vktMultiViewTests.hpp"
  83 #include "vktSubgroupsTests.hpp"
  84 #include "vktYCbCrTests.hpp"
  85 #include "vktProtectedMemTests.hpp"
  86 #include "vktDeviceGroupTests.hpp"
  87
  88 #include <vector>
  89 #include <sstream>
  90
  91 namespace // compilation
  92 {
  93
  94 vk::ProgramBinary* compileProgram (const vk::GlslSource& source, glu::ShaderProgramInfo* buildInfo, const tcu::CommandLine& commandLine)
  95 {
  96         return vk::buildProgram(source, buildInfo, commandLine);
  97 }
  98
  99 vk::ProgramBinary* compileProgram (const vk::HlslSource& source, glu::ShaderProgramInfo* buildInfo, const tcu::CommandLine& commandLine)
 100 {
 101         return vk::buildProgram(source, buildInfo, commandLine);
 102 }
 103
 104 vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, vk::SpirVProgramInfo* buildInfo, const tcu::CommandLine& commandLine)
 105 {
 106         return vk::assembleProgram(source, buildInfo, commandLine);
 107 }
 108
 109 template <typename InfoType, typename IteratorType>
 110 vk::ProgramBinary* buildProgram (const std::string&                                     casePath,
 111                                                                  IteratorType                                           iter,
 112                                                                  const vk::BinaryRegistryReader&        prebuiltBinRegistry,
 113                                                                  tcu::TestLog&                                          log,
 114                                                                  vk::BinaryCollection*                          progCollection,
 115                                                                  const tcu::CommandLine&                        commandLine)
 116 {
 117         const vk::ProgramIdentifier             progId          (casePath, iter.getName());
 118         const tcu::ScopedLogSection             progSection     (log, iter.getName(), "Program: " + iter.getName());
 119         de::MovePtr<vk::ProgramBinary>  binProg;
 120         InfoType                                                buildInfo;
 121
 122         try
 123         {
 124                 binProg = de::MovePtr<vk::ProgramBinary>(compileProgram(iter.getProgram(), &buildInfo, commandLine));
 125                 log << buildInfo;
 126         }
 127         catch (const tcu::NotSupportedError& err)
 128         {
 129                 // Try to load from cache
 130                 log << err << tcu::TestLog::Message << "Building from source not supported, loading stored binary instead" << tcu::TestLog::EndMessage;
 131
 132                 binProg = de::MovePtr<vk::ProgramBinary>(prebuiltBinRegistry.loadProgram(progId));
 133
 134                 log << iter.getProgram();
 135         }
 136         catch (const tcu::Exception&)
 137         {
 138                 // Build failed for other reason
 139                 log << buildInfo;
 140                 throw;
 141         }
 142
 143         TCU_CHECK_INTERNAL(binProg);
 144
 145         {
 146                 vk::ProgramBinary* const        returnBinary    = binProg.get();
 147
 148                 progCollection->add(progId.programName, binProg);
 149
 150                 return returnBinary;
 151         }
 152 }
 153
 154 } // anonymous(compilation)
 155
 156 namespace vkt
 157 {
 158
 159 using std::vector;
 160 using de::UniquePtr;
 161 using de::MovePtr;
 162 using tcu::TestLog;
 163
 164 namespace
 165 {
 166
 167 MovePtr<vk::DebugReportRecorder> createDebugReportRecorder (const vk::PlatformInterface& vkp, const vk::InstanceInterface& vki, vk::VkInstance instance)
 168 {
 169         if (isDebugReportSupported(vkp))
 170                 return MovePtr<vk::DebugReportRecorder>(new vk::DebugReportRecorder(vki, instance));
 171         else
 172                 TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported");
 173 }
 174
 175 } // anonymous
 176
 177 // TestCaseExecutor
 178
 179 class TestCaseExecutor : public tcu::TestCaseExecutor
 180 {
 181 public:
 182                                                                                                 TestCaseExecutor        (tcu::TestContext& testCtx);
 183                                                                                                 ~TestCaseExecutor       (void);
 184
 185         virtual void                                                            init                            (tcu::TestCase* testCase, const std::string& path);
 186         virtual void                                                            deinit                          (tcu::TestCase* testCase);
 187
 188         virtual tcu::TestNode::IterateResult            iterate                         (tcu::TestCase* testCase);
 189
 190 private:
 191         vk::BinaryCollection                                            m_progCollection;
 192         vk::BinaryRegistryReader                                        m_prebuiltBinRegistry;
 193
 194         const UniquePtr<vk::Library>                            m_library;
 195         Context                                                                         m_context;
 196
 197         const UniquePtr<vk::DebugReportRecorder>        m_debugReportRecorder;
 198
 199         TestInstance*                                                           m_instance;                     //!< Current test case instance
 200 };
 201
 202 static MovePtr<vk::Library> createLibrary (tcu::TestContext& testCtx)
 203 {
 204         return MovePtr<vk::Library>(testCtx.getPlatform().getVulkanPlatform().createLibrary());
 205 }
 206
 207 TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx)
 208         : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt")
 209         , m_library                             (createLibrary(testCtx))
 210         , m_context                             (testCtx, m_library->getPlatformInterface(), m_progCollection)
 211         , m_debugReportRecorder (testCtx.getCommandLine().isValidationEnabled()
 212                                                          ? createDebugReportRecorder(m_library->getPlatformInterface(),
 213                                                                                                                  m_context.getInstanceInterface(),
 214                                                                                                                  m_context.getInstance())
 215                                                          : MovePtr<vk::DebugReportRecorder>(DE_NULL))
 216         , m_instance                    (DE_NULL)
 217 {
 218 }
 219
 220 TestCaseExecutor::~TestCaseExecutor (void)
 221 {
 222         delete m_instance;
 223 }
 224
 225 void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePath)
 226 {
 227         const TestCase*                         vktCase                                         = dynamic_cast<TestCase*>(testCase);
 228         tcu::TestLog&                           log                                                     = m_context.getTestContext().getLog();
 229         const deUint32                          usedVulkanVersion                       = m_context.getUsedApiVersion();
 230         const vk::SpirvVersion          spirvVersionForGlsl                     = vk::getSpirvVersionForGlsl(usedVulkanVersion);
 231         const vk::SpirvVersion          spirvVersionForAsm                      = vk::getSpirvVersionForAsm(usedVulkanVersion);
 232         vk::ShaderBuildOptions          defaultGlslBuildOptions         (spirvVersionForGlsl, 0u);
 233         vk::ShaderBuildOptions          defaultHlslBuildOptions         (spirvVersionForGlsl, 0u);
 234         vk::SpirVAsmBuildOptions        defaultSpirvAsmBuildOptions     (spirvVersionForAsm);
 235         vk::SourceCollections           sourceProgs                                     (usedVulkanVersion, defaultGlslBuildOptions, defaultHlslBuildOptions, defaultSpirvAsmBuildOptions);
 236         const bool                                      doShaderLog                                     = log.isShaderLoggingEnabled();
 237         const tcu::CommandLine&         commandLine                                     = m_context.getTestContext().getCommandLine();
 238
 239         DE_UNREF(casePath); // \todo [2015-03-13 pyry] Use this to identify ProgramCollection storage path
 240
 241         if (!vktCase)
 242                 TCU_THROW(InternalError, "Test node not an instance of vkt::TestCase");
 243
 244         vktCase->checkSupport(m_context);
 245
 246         m_progCollection.clear();
 247         vktCase->initPrograms(sourceProgs);
 248
 249         for (vk::GlslSourceCollection::Iterator progIter = sourceProgs.glslSources.begin(); progIter != sourceProgs.glslSources.end(); ++progIter)
 250         {
 251                 if (progIter.getProgram().buildOptions.targetVersion > vk::getSpirvVersionForGlsl(m_context.getUsedApiVersion()))
 252                         TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
 253
 254                 const vk::ProgramBinary* const binProg = buildProgram<glu::ShaderProgramInfo, vk::GlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection, commandLine);
 255
 256                 if (doShaderLog)
 257                 {
 258                         try
 259                         {
 260                                 std::ostringstream disasm;
 261
 262                                 vk::disassembleProgram(*binProg, &disasm, spirvVersionForGlsl);
 263
 264                                 log << vk::SpirVAsmSource(disasm.str());
 265                         }
 266                         catch (const tcu::NotSupportedError& err)
 267                         {
 268                                 log << err;
 269                         }
 270                 }
 271         }
 272
 273         for (vk::HlslSourceCollection::Iterator progIter = sourceProgs.hlslSources.begin(); progIter != sourceProgs.hlslSources.end(); ++progIter)
 274         {
 275                 if (progIter.getProgram().buildOptions.targetVersion > vk::getSpirvVersionForGlsl(m_context.getUsedApiVersion()))
 276                         TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
 277
 278                 const vk::ProgramBinary* const binProg = buildProgram<glu::ShaderProgramInfo, vk::HlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection, commandLine);
 279
 280                 if (doShaderLog)
 281                 {
 282                         try
 283                         {
 284                                 std::ostringstream disasm;
 285
 286                                 vk::disassembleProgram(*binProg, &disasm, spirvVersionForGlsl);
 287
 288                                 log << vk::SpirVAsmSource(disasm.str());
 289                         }
 290                         catch (const tcu::NotSupportedError& err)
 291                         {
 292                                 log << err;
 293                         }
 294                 }
 295         }
 296
 297         for (vk::SpirVAsmCollection::Iterator asmIterator = sourceProgs.spirvAsmSources.begin(); asmIterator != sourceProgs.spirvAsmSources.end(); ++asmIterator)
 298         {
 299                 if (asmIterator.getProgram().buildOptions.targetVersion > vk::getSpirvVersionForAsm(m_context.getUsedApiVersion()))
 300                         TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
 301
 302                 buildProgram<vk::SpirVProgramInfo, vk::SpirVAsmCollection::Iterator>(casePath, asmIterator, m_prebuiltBinRegistry, log, &m_progCollection, commandLine);
 303         }
 304
 305         DE_ASSERT(!m_instance);
 306         m_instance = vktCase->createInstance(m_context);
 307 }
 308
 309 void TestCaseExecutor::deinit (tcu::TestCase*)
 310 {
 311         delete m_instance;
 312         m_instance = DE_NULL;
 313
 314         // Collect and report any debug messages
 315         if (m_debugReportRecorder)
 316         {
 317                 // \note We are not logging INFORMATION and DEBUG messages
 318                 static const vk::VkDebugReportFlagsEXT                  errorFlags              = vk::VK_DEBUG_REPORT_ERROR_BIT_EXT;
 319                 static const vk::VkDebugReportFlagsEXT                  logFlags                = errorFlags
 320                                                                                                                                                 | vk::VK_DEBUG_REPORT_WARNING_BIT_EXT
 321                                                                                                                                                 | vk::VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT;
 322
 323                 typedef vk::DebugReportRecorder::MessageList    DebugMessages;
 324
 325                 const DebugMessages&    messages        = m_debugReportRecorder->getMessages();
 326                 tcu::TestLog&                   log                     = m_context.getTestContext().getLog();
 327
 328                 if (messages.begin() != messages.end())
 329                 {
 330                         const tcu::ScopedLogSection     section         (log, "DebugMessages", "Debug Messages");
 331                         int                                                     numErrors       = 0;
 332
 333                         for (DebugMessages::const_iterator curMsg = messages.begin(); curMsg != messages.end(); ++curMsg)
 334                         {
 335                                 if ((curMsg->flags & logFlags) != 0)
 336                                         log << tcu::TestLog::Message << *curMsg << tcu::TestLog::EndMessage;
 337
 338                                 if ((curMsg->flags & errorFlags) != 0)
 339                                         numErrors += 1;
 340                         }
 341
 342                         m_debugReportRecorder->clearMessages();
 343
 344                         if (numErrors > 0)
 345                                 m_context.getTestContext().setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, (de::toString(numErrors) + " API usage errors found").c_str());
 346                 }
 347         }
 348 }
 349
 350 tcu::TestNode::IterateResult TestCaseExecutor::iterate (tcu::TestCase*)
 351 {
 352         DE_ASSERT(m_instance);
 353
 354         const tcu::TestStatus   result  = m_instance->iterate();
 355
 356         if (result.isComplete())
 357         {
 358                 // Vulkan tests shouldn't set result directly
 359                 DE_ASSERT(m_context.getTestContext().getTestResult() == QP_TEST_RESULT_LAST);
 360                 m_context.getTestContext().setTestResult(result.getCode(), result.getDescription().c_str());
 361                 return tcu::TestNode::STOP;
 362         }
 363         else
 364                 return tcu::TestNode::CONTINUE;
 365 }
 366
 367 // GLSL shader tests
 368
 369 void createGlslTests (tcu::TestCaseGroup* glslTests)
 370 {
 371         tcu::TestContext&       testCtx         = glslTests->getTestContext();
 372
 373         // ShaderLibrary-based tests
 374         static const struct
 375         {
 376                 const char*             name;
 377                 const char*             description;
 378         } s_es310Tests[] =
 379         {
 380                 { "arrays",                                             "Arrays"                                        },
 381                 { "conditionals",                               "Conditional statements"        },
 382                 { "constant_expressions",               "Constant expressions"          },
 383                 { "constants",                                  "Constants"                                     },
 384                 { "conversions",                                "Type conversions"                      },
 385                 { "functions",                                  "Functions"                                     },
 386                 { "linkage",                                    "Linking"                                       },
 387                 { "scoping",                                    "Scoping"                                       },
 388                 { "swizzles",                                   "Swizzles"                                      },
 389         };
 390
 391         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_es310Tests); ndx++)
 392                 glslTests->addChild(createShaderLibraryGroup(testCtx,
 393                                                                                                          s_es310Tests[ndx].name,
 394                                                                                                          s_es310Tests[ndx].description,
 395                                                                                                          std::string("vulkan/glsl/es310/") + s_es310Tests[ndx].name + ".test").release());
 396
 397         static const struct
 398         {
 399                 const char*             name;
 400                 const char*             description;
 401         } s_440Tests[] =
 402         {
 403                 { "linkage",                                    "Linking"                                       },
 404         };
 405
 406         de::MovePtr<tcu::TestCaseGroup> glsl440Tests = de::MovePtr<tcu::TestCaseGroup>(new tcu::TestCaseGroup(testCtx, "440", ""));
 407
 408         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_440Tests); ndx++)
 409                 glsl440Tests->addChild(createShaderLibraryGroup(testCtx,
 410                                                                                                          s_440Tests[ndx].name,
 411                                                                                                          s_440Tests[ndx].description,
 412                                                                                                          std::string("vulkan/glsl/440/") + s_440Tests[ndx].name + ".test").release());
 413
 414         glslTests->addChild(glsl440Tests.release());
 415
 416         // ShaderRenderCase-based tests
 417         glslTests->addChild(sr::createDerivateTests                     (testCtx));
 418         glslTests->addChild(sr::createDiscardTests                      (testCtx));
 419         glslTests->addChild(sr::createIndexingTests                     (testCtx));
 420         glslTests->addChild(sr::createLoopTests                         (testCtx));
 421         glslTests->addChild(sr::createMatrixTests                       (testCtx));
 422         glslTests->addChild(sr::createOperatorTests                     (testCtx));
 423         glslTests->addChild(sr::createReturnTests                       (testCtx));
 424         glslTests->addChild(sr::createStructTests                       (testCtx));
 425         glslTests->addChild(sr::createSwitchTests                       (testCtx));
 426         glslTests->addChild(sr::createTextureFunctionTests      (testCtx));
 427         glslTests->addChild(sr::createTextureGatherTests        (testCtx));
 428         glslTests->addChild(sr::createBuiltinVarTests           (testCtx));
 429
 430         // ShaderExecutor-based tests
 431         glslTests->addChild(shaderexecutor::createBuiltinTests                          (testCtx));
 432         glslTests->addChild(shaderexecutor::createOpaqueTypeIndexingTests       (testCtx));
 433         glslTests->addChild(shaderexecutor::createAtomicOperationTests          (testCtx));
 434 }
 435
 436 // TestPackage
 437
 438 TestPackage::TestPackage (tcu::TestContext& testCtx)
 439         : tcu::TestPackage(testCtx, "dEQP-VK", "dEQP Vulkan Tests")
 440 {
 441 }
 442
 443 TestPackage::~TestPackage (void)
 444 {
 445 }
 446
 447 tcu::TestCaseExecutor* TestPackage::createExecutor (void) const
 448 {
 449         return new TestCaseExecutor(m_testCtx);
 450 }
 451
 452 void TestPackage::init (void)
 453 {
 454         addChild(createTestGroup                                (m_testCtx, "info", "Build and Device Info Tests", createInfoTests));
 455         addChild(api::createTests                               (m_testCtx));
 456         addChild(memory::createTests                    (m_testCtx));
 457         addChild(pipeline::createTests                  (m_testCtx));
 458         addChild(BindingModel::createTests              (m_testCtx));
 459         addChild(SpirVAssembly::createTests             (m_testCtx));
 460         addChild(createTestGroup                                (m_testCtx, "glsl", "GLSL shader execution tests", createGlslTests));
 461         addChild(createRenderPassTests                  (m_testCtx));
 462         addChild(ubo::createTests                               (m_testCtx));
 463         addChild(DynamicState::createTests              (m_testCtx));
 464         addChild(ssbo::createTests                              (m_testCtx));
 465         addChild(QueryPool::createTests                 (m_testCtx));
 466         addChild(Draw::createTests                              (m_testCtx));
 467         addChild(compute::createTests                   (m_testCtx));
 468         addChild(image::createTests                             (m_testCtx));
 469         addChild(wsi::createTests                               (m_testCtx));
 470         addChild(synchronization::createTests   (m_testCtx));
 471         addChild(sparse::createTests                    (m_testCtx));
 472         addChild(tessellation::createTests              (m_testCtx));
 473         addChild(rasterization::createTests             (m_testCtx));
 474         addChild(clipping::createTests                  (m_testCtx));
 475         addChild(FragmentOperations::createTests(m_testCtx));
 476         addChild(texture::createTests                   (m_testCtx));
 477         addChild(geometry::createTests                  (m_testCtx));
 478         addChild(robustness::createTests                (m_testCtx));
 479         addChild(MultiView::createTests                 (m_testCtx));
 480         addChild(subgroups::createTests                 (m_testCtx));
 481         addChild(ycbcr::createTests                             (m_testCtx));
 482         addChild(ProtectedMem::createTests              (m_testCtx));
 483         addChild(DeviceGroup::createTests               (m_testCtx));
 484 }
 485
 486 } // vkt