1 /*-------------------------------------------------------------------------
2 * Vulkan Conformance Tests
3 * ------------------------
5 * Copyright (c) 2015 Google Inc.
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 * \brief Vulkan Test Package
22 *//*--------------------------------------------------------------------*/
24 #include "vktTestPackage.hpp"
26 #include "tcuPlatform.hpp"
27 #include "tcuTestCase.hpp"
28 #include "tcuTestLog.hpp"
29 #include "tcuCommandLine.hpp"
31 #include "vkPlatform.hpp"
32 #include "vkPrograms.hpp"
33 #include "vkBinaryRegistry.hpp"
34 #include "vkShaderToSpirV.hpp"
35 #include "vkDebugReportUtil.hpp"
36 #include "vkQueryUtil.hpp"
37 #include "vkApiVersion.hpp"
39 #include "deUniquePtr.hpp"
41 #include "vktTestGroupUtil.hpp"
42 #include "vktApiTests.hpp"
43 #include "vktPipelineTests.hpp"
44 #include "vktBindingModelTests.hpp"
45 #include "vktSpvAsmTests.hpp"
46 #include "vktShaderLibrary.hpp"
47 #include "vktRenderPassTests.hpp"
48 #include "vktMemoryTests.hpp"
49 #include "vktShaderRenderBuiltinVarTests.hpp"
50 #include "vktShaderRenderDerivateTests.hpp"
51 #include "vktShaderRenderDiscardTests.hpp"
52 #include "vktShaderRenderIndexingTests.hpp"
53 #include "vktShaderRenderLoopTests.hpp"
54 #include "vktShaderRenderMatrixTests.hpp"
55 #include "vktShaderRenderOperatorTests.hpp"
56 #include "vktShaderRenderReturnTests.hpp"
57 #include "vktShaderRenderStructTests.hpp"
58 #include "vktShaderRenderSwitchTests.hpp"
59 #include "vktShaderRenderTextureFunctionTests.hpp"
60 #include "vktShaderRenderTextureGatherTests.hpp"
61 #include "vktShaderBuiltinTests.hpp"
62 #include "vktOpaqueTypeIndexingTests.hpp"
63 #include "vktAtomicOperationTests.hpp"
64 #include "vktUniformBlockTests.hpp"
65 #include "vktDynamicStateTests.hpp"
66 #include "vktSSBOLayoutTests.hpp"
67 #include "vktQueryPoolTests.hpp"
68 #include "vktDrawTests.hpp"
69 #include "vktComputeTests.hpp"
70 #include "vktImageTests.hpp"
71 #include "vktInfoTests.hpp"
72 #include "vktWsiTests.hpp"
73 #include "vktSynchronizationTests.hpp"
74 #include "vktSparseResourcesTests.hpp"
75 #include "vktTessellationTests.hpp"
76 #include "vktRasterizationTests.hpp"
77 #include "vktClippingTests.hpp"
78 #include "vktFragmentOperationsTests.hpp"
79 #include "vktTextureTests.hpp"
80 #include "vktGeometryTests.hpp"
81 #include "vktRobustnessTests.hpp"
82 #include "vktMultiViewTests.hpp"
83 #include "vktSubgroupsTests.hpp"
84 #include "vktYCbCrTests.hpp"
85 #include "vktProtectedMemTests.hpp"
90 namespace // compilation
93 vk::ProgramBinary* compileProgram (const vk::GlslSource& source, glu::ShaderProgramInfo* buildInfo)
95 return vk::buildProgram(source, buildInfo);
98 vk::ProgramBinary* compileProgram (const vk::HlslSource& source, glu::ShaderProgramInfo* buildInfo)
100 return vk::buildProgram(source, buildInfo);
103 vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, vk::SpirVProgramInfo* buildInfo)
105 return vk::assembleProgram(source, buildInfo);
108 template <typename InfoType, typename IteratorType>
109 vk::ProgramBinary* buildProgram (const std::string& casePath,
111 const vk::BinaryRegistryReader& prebuiltBinRegistry,
113 vk::BinaryCollection* progCollection)
115 const vk::ProgramIdentifier progId (casePath, iter.getName());
116 const tcu::ScopedLogSection progSection (log, iter.getName(), "Program: " + iter.getName());
117 de::MovePtr<vk::ProgramBinary> binProg;
122 binProg = de::MovePtr<vk::ProgramBinary>(compileProgram(iter.getProgram(), &buildInfo));
125 catch (const tcu::NotSupportedError& err)
127 // Try to load from cache
128 log << err << tcu::TestLog::Message << "Building from source not supported, loading stored binary instead" << tcu::TestLog::EndMessage;
130 binProg = de::MovePtr<vk::ProgramBinary>(prebuiltBinRegistry.loadProgram(progId));
132 log << iter.getProgram();
134 catch (const tcu::Exception&)
136 // Build failed for other reason
141 TCU_CHECK_INTERNAL(binProg);
144 vk::ProgramBinary* const returnBinary = binProg.get();
146 progCollection->add(progId.programName, binProg);
152 } // anonymous(compilation)
165 MovePtr<vk::DebugReportRecorder> createDebugReportRecorder (const vk::PlatformInterface& vkp, const vk::InstanceInterface& vki, vk::VkInstance instance)
167 if (isDebugReportSupported(vkp))
168 return MovePtr<vk::DebugReportRecorder>(new vk::DebugReportRecorder(vki, instance));
170 TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported");
177 class TestCaseExecutor : public tcu::TestCaseExecutor
180 TestCaseExecutor (tcu::TestContext& testCtx);
181 ~TestCaseExecutor (void);
183 virtual void init (tcu::TestCase* testCase, const std::string& path);
184 virtual void deinit (tcu::TestCase* testCase);
186 virtual tcu::TestNode::IterateResult iterate (tcu::TestCase* testCase);
189 vk::BinaryCollection m_progCollection;
190 vk::BinaryRegistryReader m_prebuiltBinRegistry;
192 const UniquePtr<vk::Library> m_library;
195 const UniquePtr<vk::DebugReportRecorder> m_debugReportRecorder;
197 TestInstance* m_instance; //!< Current test case instance
200 static MovePtr<vk::Library> createLibrary (tcu::TestContext& testCtx)
202 return MovePtr<vk::Library>(testCtx.getPlatform().getVulkanPlatform().createLibrary());
205 TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx)
206 : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt")
207 , m_library (createLibrary(testCtx))
208 , m_context (testCtx, m_library->getPlatformInterface(), m_progCollection)
209 , m_debugReportRecorder (testCtx.getCommandLine().isValidationEnabled()
210 ? createDebugReportRecorder(m_library->getPlatformInterface(),
211 m_context.getInstanceInterface(),
212 m_context.getInstance())
213 : MovePtr<vk::DebugReportRecorder>(DE_NULL))
214 , m_instance (DE_NULL)
218 TestCaseExecutor::~TestCaseExecutor (void)
223 void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePath)
225 const TestCase* vktCase = dynamic_cast<TestCase*>(testCase);
226 tcu::TestLog& log = m_context.getTestContext().getLog();
227 const deUint32 usedVulkanVersion = m_context.getUsedApiVersion();
228 const vk::SpirvVersion spirvVersionForGlsl = vk::getSpirvVersionForGlsl(usedVulkanVersion);
229 const vk::SpirvVersion spirvVersionForAsm = vk::getSpirvVersionForAsm(usedVulkanVersion);
230 vk::ShaderBuildOptions defaultGlslBuildOptions (spirvVersionForGlsl, 0u);
231 vk::ShaderBuildOptions defaultHlslBuildOptions (spirvVersionForGlsl, 0u);
232 vk::SpirVAsmBuildOptions defaultSpirvAsmBuildOptions (spirvVersionForAsm);
233 vk::SourceCollections sourceProgs (usedVulkanVersion, defaultGlslBuildOptions, defaultHlslBuildOptions, defaultSpirvAsmBuildOptions);
235 DE_UNREF(casePath); // \todo [2015-03-13 pyry] Use this to identify ProgramCollection storage path
238 TCU_THROW(InternalError, "Test node not an instance of vkt::TestCase");
240 m_progCollection.clear();
241 vktCase->initPrograms(sourceProgs);
243 for (vk::GlslSourceCollection::Iterator progIter = sourceProgs.glslSources.begin(); progIter != sourceProgs.glslSources.end(); ++progIter)
245 if (progIter.getProgram().buildOptions.targetVersion > vk::getSpirvVersionForGlsl(m_context.getUsedApiVersion()))
246 TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
248 const vk::ProgramBinary* const binProg = buildProgram<glu::ShaderProgramInfo, vk::GlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection);
252 std::ostringstream disasm;
254 vk::disassembleProgram(*binProg, &disasm, spirvVersionForGlsl);
256 log << vk::SpirVAsmSource(disasm.str());
258 catch (const tcu::NotSupportedError& err)
264 for (vk::HlslSourceCollection::Iterator progIter = sourceProgs.hlslSources.begin(); progIter != sourceProgs.hlslSources.end(); ++progIter)
266 if (progIter.getProgram().buildOptions.targetVersion > vk::getSpirvVersionForGlsl(m_context.getUsedApiVersion()))
267 TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
269 const vk::ProgramBinary* const binProg = buildProgram<glu::ShaderProgramInfo, vk::HlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection);
273 std::ostringstream disasm;
275 vk::disassembleProgram(*binProg, &disasm, spirvVersionForGlsl);
277 log << vk::SpirVAsmSource(disasm.str());
279 catch (const tcu::NotSupportedError& err)
285 for (vk::SpirVAsmCollection::Iterator asmIterator = sourceProgs.spirvAsmSources.begin(); asmIterator != sourceProgs.spirvAsmSources.end(); ++asmIterator)
287 if (asmIterator.getProgram().buildOptions.targetVersion > vk::getSpirvVersionForAsm(m_context.getUsedApiVersion()))
288 TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
290 buildProgram<vk::SpirVProgramInfo, vk::SpirVAsmCollection::Iterator>(casePath, asmIterator, m_prebuiltBinRegistry, log, &m_progCollection);
293 DE_ASSERT(!m_instance);
294 m_instance = vktCase->createInstance(m_context);
297 void TestCaseExecutor::deinit (tcu::TestCase*)
300 m_instance = DE_NULL;
302 // Collect and report any debug messages
303 if (m_debugReportRecorder)
305 // \note We are not logging INFORMATION and DEBUG messages
306 static const vk::VkDebugReportFlagsEXT errorFlags = vk::VK_DEBUG_REPORT_ERROR_BIT_EXT;
307 static const vk::VkDebugReportFlagsEXT logFlags = errorFlags
308 | vk::VK_DEBUG_REPORT_WARNING_BIT_EXT
309 | vk::VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT;
311 typedef vk::DebugReportRecorder::MessageList DebugMessages;
313 const DebugMessages& messages = m_debugReportRecorder->getMessages();
314 tcu::TestLog& log = m_context.getTestContext().getLog();
316 if (messages.begin() != messages.end())
318 const tcu::ScopedLogSection section (log, "DebugMessages", "Debug Messages");
321 for (DebugMessages::const_iterator curMsg = messages.begin(); curMsg != messages.end(); ++curMsg)
323 if ((curMsg->flags & logFlags) != 0)
324 log << tcu::TestLog::Message << *curMsg << tcu::TestLog::EndMessage;
326 if ((curMsg->flags & errorFlags) != 0)
330 m_debugReportRecorder->clearMessages();
333 m_context.getTestContext().setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, (de::toString(numErrors) + " API usage errors found").c_str());
338 tcu::TestNode::IterateResult TestCaseExecutor::iterate (tcu::TestCase*)
340 DE_ASSERT(m_instance);
342 const tcu::TestStatus result = m_instance->iterate();
344 if (result.isComplete())
346 // Vulkan tests shouldn't set result directly
347 DE_ASSERT(m_context.getTestContext().getTestResult() == QP_TEST_RESULT_LAST);
348 m_context.getTestContext().setTestResult(result.getCode(), result.getDescription().c_str());
349 return tcu::TestNode::STOP;
352 return tcu::TestNode::CONTINUE;
357 void createGlslTests (tcu::TestCaseGroup* glslTests)
359 tcu::TestContext& testCtx = glslTests->getTestContext();
361 // ShaderLibrary-based tests
365 const char* description;
368 { "arrays", "Arrays" },
369 { "conditionals", "Conditional statements" },
370 { "constant_expressions", "Constant expressions" },
371 { "constants", "Constants" },
372 { "conversions", "Type conversions" },
373 { "functions", "Functions" },
374 { "linkage", "Linking" },
375 { "scoping", "Scoping" },
376 { "swizzles", "Swizzles" },
379 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_es310Tests); ndx++)
380 glslTests->addChild(createShaderLibraryGroup(testCtx,
381 s_es310Tests[ndx].name,
382 s_es310Tests[ndx].description,
383 std::string("vulkan/glsl/es310/") + s_es310Tests[ndx].name + ".test").release());
385 // ShaderRenderCase-based tests
386 glslTests->addChild(sr::createDerivateTests (testCtx));
387 glslTests->addChild(sr::createDiscardTests (testCtx));
388 glslTests->addChild(sr::createIndexingTests (testCtx));
389 glslTests->addChild(sr::createLoopTests (testCtx));
390 glslTests->addChild(sr::createMatrixTests (testCtx));
391 glslTests->addChild(sr::createOperatorTests (testCtx));
392 glslTests->addChild(sr::createReturnTests (testCtx));
393 glslTests->addChild(sr::createStructTests (testCtx));
394 glslTests->addChild(sr::createSwitchTests (testCtx));
395 glslTests->addChild(sr::createTextureFunctionTests (testCtx));
396 glslTests->addChild(sr::createTextureGatherTests (testCtx));
397 glslTests->addChild(sr::createBuiltinVarTests (testCtx));
399 // ShaderExecutor-based tests
400 glslTests->addChild(shaderexecutor::createBuiltinTests (testCtx));
401 glslTests->addChild(shaderexecutor::createOpaqueTypeIndexingTests (testCtx));
402 glslTests->addChild(shaderexecutor::createAtomicOperationTests (testCtx));
407 TestPackage::TestPackage (tcu::TestContext& testCtx)
408 : tcu::TestPackage(testCtx, "dEQP-VK", "dEQP Vulkan Tests")
412 TestPackage::~TestPackage (void)
416 tcu::TestCaseExecutor* TestPackage::createExecutor (void) const
418 return new TestCaseExecutor(m_testCtx);
421 void TestPackage::init (void)
423 addChild(createTestGroup (m_testCtx, "info", "Build and Device Info Tests", createInfoTests));
424 addChild(api::createTests (m_testCtx));
425 addChild(memory::createTests (m_testCtx));
426 addChild(pipeline::createTests (m_testCtx));
427 addChild(BindingModel::createTests (m_testCtx));
428 addChild(SpirVAssembly::createTests (m_testCtx));
429 addChild(createTestGroup (m_testCtx, "glsl", "GLSL shader execution tests", createGlslTests));
430 addChild(createRenderPassTests (m_testCtx));
431 addChild(ubo::createTests (m_testCtx));
432 addChild(DynamicState::createTests (m_testCtx));
433 addChild(ssbo::createTests (m_testCtx));
434 addChild(QueryPool::createTests (m_testCtx));
435 addChild(Draw::createTests (m_testCtx));
436 addChild(compute::createTests (m_testCtx));
437 addChild(image::createTests (m_testCtx));
438 addChild(wsi::createTests (m_testCtx));
439 addChild(synchronization::createTests (m_testCtx));
440 addChild(sparse::createTests (m_testCtx));
441 addChild(tessellation::createTests (m_testCtx));
442 addChild(rasterization::createTests (m_testCtx));
443 addChild(clipping::createTests (m_testCtx));
444 addChild(FragmentOperations::createTests(m_testCtx));
445 addChild(texture::createTests (m_testCtx));
446 addChild(geometry::createTests (m_testCtx));
447 addChild(robustness::createTests (m_testCtx));
448 addChild(MultiView::createTests (m_testCtx));
449 addChild(subgroups::createTests (m_testCtx));
450 addChild(ycbcr::createTests (m_testCtx));
451 addChild(ProtectedMem::createTests (m_testCtx));