resolve merge conflicts of 59febde9 to deqp-dev
[platform/upstream/VK-GL-CTS.git] / external / vulkancts / modules / vulkan / vktTestPackage.cpp
1 /*-------------------------------------------------------------------------
2  * Vulkan Conformance Tests
3  * ------------------------
4  *
5  * Copyright (c) 2015 Google Inc.
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  *
19  *//*!
20  * \file
21  * \brief Vulkan Test Package
22  *//*--------------------------------------------------------------------*/
23
24 #include "vktTestPackage.hpp"
25
26 #include "tcuPlatform.hpp"
27 #include "tcuTestCase.hpp"
28 #include "tcuTestLog.hpp"
29 #include "tcuCommandLine.hpp"
30
31 #include "vkPlatform.hpp"
32 #include "vkPrograms.hpp"
33 #include "vkBinaryRegistry.hpp"
34 #include "vkGlslToSpirV.hpp"
35 #include "vkDebugReportUtil.hpp"
36 #include "vkQueryUtil.hpp"
37
38 #include "deUniquePtr.hpp"
39
40 #include "vktTestGroupUtil.hpp"
41 #include "vktApiTests.hpp"
42 #include "vktPipelineTests.hpp"
43 #include "vktBindingModelTests.hpp"
44 #include "vktSpvAsmTests.hpp"
45 #include "vktShaderLibrary.hpp"
46 #include "vktRenderPassTests.hpp"
47 #include "vktMemoryTests.hpp"
48 #include "vktShaderRenderBuiltinVarTests.hpp"
49 #include "vktShaderRenderDerivateTests.hpp"
50 #include "vktShaderRenderDiscardTests.hpp"
51 #include "vktShaderRenderIndexingTests.hpp"
52 #include "vktShaderRenderLoopTests.hpp"
53 #include "vktShaderRenderMatrixTests.hpp"
54 #include "vktShaderRenderOperatorTests.hpp"
55 #include "vktShaderRenderReturnTests.hpp"
56 #include "vktShaderRenderStructTests.hpp"
57 #include "vktShaderRenderSwitchTests.hpp"
58 #include "vktShaderRenderTextureFunctionTests.hpp"
59 #include "vktShaderRenderTextureGatherTests.hpp"
60 #include "vktShaderBuiltinTests.hpp"
61 #include "vktOpaqueTypeIndexingTests.hpp"
62 #include "vktAtomicOperationTests.hpp"
63 #include "vktUniformBlockTests.hpp"
64 #include "vktDynamicStateTests.hpp"
65 #include "vktSSBOLayoutTests.hpp"
66 #include "vktQueryPoolTests.hpp"
67 #include "vktDrawTests.hpp"
68 #include "vktComputeTests.hpp"
69 #include "vktImageTests.hpp"
70 #include "vktInfoTests.hpp"
71 #include "vktWsiTests.hpp"
72 #include "vktSynchronizationTests.hpp"
73 #include "vktSparseResourcesTests.hpp"
74 #include "vktTessellationTests.hpp"
75 #include "vktRasterizationTests.hpp"
76 #include "vktClippingTests.hpp"
77 #include "vktFragmentOperationsTests.hpp"
78 #include "vktTextureTests.hpp"
79 #include "vktGeometryTests.hpp"
80 #include "vktRobustnessTests.hpp"
81
82 #include <vector>
83 #include <sstream>
84
85 namespace // compilation
86 {
87
88 vk::ProgramBinary* compileProgram (const vk::GlslSource& source, glu::ShaderProgramInfo* buildInfo)
89 {
90         return vk::buildProgram(source, buildInfo);
91 }
92
93 vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, vk::SpirVProgramInfo* buildInfo)
94 {
95         return vk::assembleProgram(source, buildInfo);
96 }
97
98 template <typename InfoType, typename IteratorType>
99 vk::ProgramBinary* buildProgram (const std::string&                                     casePath,
100                                                                  IteratorType                                           iter,
101                                                                  const vk::BinaryRegistryReader&        prebuiltBinRegistry,
102                                                                  tcu::TestLog&                                          log,
103                                                                  vk::BinaryCollection*                          progCollection)
104 {
105         const vk::ProgramIdentifier             progId          (casePath, iter.getName());
106         const tcu::ScopedLogSection             progSection     (log, iter.getName(), "Program: " + iter.getName());
107         de::MovePtr<vk::ProgramBinary>  binProg;
108         InfoType                                                buildInfo;
109
110         try
111         {
112                 binProg = de::MovePtr<vk::ProgramBinary>(compileProgram(iter.getProgram(), &buildInfo));
113                 log << buildInfo;
114         }
115         catch (const tcu::NotSupportedError& err)
116         {
117                 // Try to load from cache
118                 log << err << tcu::TestLog::Message << "Building from source not supported, loading stored binary instead" << tcu::TestLog::EndMessage;
119
120                 binProg = de::MovePtr<vk::ProgramBinary>(prebuiltBinRegistry.loadProgram(progId));
121
122                 log << iter.getProgram();
123         }
124         catch (const tcu::Exception&)
125         {
126                 // Build failed for other reason
127                 log << buildInfo;
128                 throw;
129         }
130
131         TCU_CHECK_INTERNAL(binProg);
132
133         {
134                 vk::ProgramBinary* const        returnBinary    = binProg.get();
135
136                 progCollection->add(progId.programName, binProg);
137
138                 return returnBinary;
139         }
140 }
141
142 } // anonymous(compilation)
143
144 namespace vkt
145 {
146
147 using std::vector;
148 using de::UniquePtr;
149 using de::MovePtr;
150 using tcu::TestLog;
151
152 namespace
153 {
154
155 MovePtr<vk::DebugReportRecorder> createDebugReportRecorder (const vk::PlatformInterface& vkp, const vk::InstanceInterface& vki, vk::VkInstance instance)
156 {
157         if (isDebugReportSupported(vkp))
158                 return MovePtr<vk::DebugReportRecorder>(new vk::DebugReportRecorder(vki, instance));
159         else
160                 TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported");
161 }
162
163 } // anonymous
164
165 // TestCaseExecutor
166
167 class TestCaseExecutor : public tcu::TestCaseExecutor
168 {
169 public:
170                                                                                                 TestCaseExecutor        (tcu::TestContext& testCtx);
171                                                                                                 ~TestCaseExecutor       (void);
172
173         virtual void                                                            init                            (tcu::TestCase* testCase, const std::string& path);
174         virtual void                                                            deinit                          (tcu::TestCase* testCase);
175
176         virtual tcu::TestNode::IterateResult            iterate                         (tcu::TestCase* testCase);
177
178 private:
179         vk::BinaryCollection                                            m_progCollection;
180         vk::BinaryRegistryReader                                        m_prebuiltBinRegistry;
181
182         const UniquePtr<vk::Library>                            m_library;
183         Context                                                                         m_context;
184
185         const UniquePtr<vk::DebugReportRecorder>        m_debugReportRecorder;
186
187         TestInstance*                                                           m_instance;                     //!< Current test case instance
188 };
189
190 static MovePtr<vk::Library> createLibrary (tcu::TestContext& testCtx)
191 {
192         return MovePtr<vk::Library>(testCtx.getPlatform().getVulkanPlatform().createLibrary());
193 }
194
195 TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx)
196         : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt")
197         , m_library                             (createLibrary(testCtx))
198         , m_context                             (testCtx, m_library->getPlatformInterface(), m_progCollection)
199         , m_debugReportRecorder (testCtx.getCommandLine().isValidationEnabled()
200                                                          ? createDebugReportRecorder(m_library->getPlatformInterface(),
201                                                                                                                  m_context.getInstanceInterface(),
202                                                                                                                  m_context.getInstance())
203                                                          : MovePtr<vk::DebugReportRecorder>(DE_NULL))
204         , m_instance                    (DE_NULL)
205 {
206 }
207
208 TestCaseExecutor::~TestCaseExecutor (void)
209 {
210         delete m_instance;
211 }
212
213 void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePath)
214 {
215         const TestCase*                 vktCase         = dynamic_cast<TestCase*>(testCase);
216         tcu::TestLog&                   log                     = m_context.getTestContext().getLog();
217         vk::SourceCollections   sourceProgs;
218
219         DE_UNREF(casePath); // \todo [2015-03-13 pyry] Use this to identify ProgramCollection storage path
220
221         if (!vktCase)
222                 TCU_THROW(InternalError, "Test node not an instance of vkt::TestCase");
223
224         m_progCollection.clear();
225         vktCase->initPrograms(sourceProgs);
226
227         for (vk::GlslSourceCollection::Iterator progIter = sourceProgs.glslSources.begin(); progIter != sourceProgs.glslSources.end(); ++progIter)
228         {
229                 vk::ProgramBinary* binProg = buildProgram<glu::ShaderProgramInfo, vk::GlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection);
230
231                 try
232                 {
233                         std::ostringstream disasm;
234
235                         vk::disassembleProgram(*binProg, &disasm);
236
237                         log << vk::SpirVAsmSource(disasm.str());
238                 }
239                 catch (const tcu::NotSupportedError& err)
240                 {
241                         log << err;
242                 }
243         }
244
245         for (vk::SpirVAsmCollection::Iterator asmIterator = sourceProgs.spirvAsmSources.begin(); asmIterator != sourceProgs.spirvAsmSources.end(); ++asmIterator)
246         {
247                 buildProgram<vk::SpirVProgramInfo, vk::SpirVAsmCollection::Iterator>(casePath, asmIterator, m_prebuiltBinRegistry, log, &m_progCollection);
248         }
249
250         DE_ASSERT(!m_instance);
251         m_instance = vktCase->createInstance(m_context);
252 }
253
254 void TestCaseExecutor::deinit (tcu::TestCase*)
255 {
256         delete m_instance;
257         m_instance = DE_NULL;
258
259         // Collect and report any debug messages
260         if (m_debugReportRecorder)
261         {
262                 // \note We are not logging INFORMATION and DEBUG messages
263                 static const vk::VkDebugReportFlagsEXT                  errorFlags              = vk::VK_DEBUG_REPORT_ERROR_BIT_EXT;
264                 static const vk::VkDebugReportFlagsEXT                  logFlags                = errorFlags
265                                                                                                                                                 | vk::VK_DEBUG_REPORT_WARNING_BIT_EXT
266                                                                                                                                                 | vk::VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT;
267
268                 typedef vk::DebugReportRecorder::MessageList    DebugMessages;
269
270                 const DebugMessages&    messages        = m_debugReportRecorder->getMessages();
271                 tcu::TestLog&                   log                     = m_context.getTestContext().getLog();
272
273                 if (messages.begin() != messages.end())
274                 {
275                         const tcu::ScopedLogSection     section         (log, "DebugMessages", "Debug Messages");
276                         int                                                     numErrors       = 0;
277
278                         for (DebugMessages::const_iterator curMsg = messages.begin(); curMsg != messages.end(); ++curMsg)
279                         {
280                                 if ((curMsg->flags & logFlags) != 0)
281                                         log << tcu::TestLog::Message << *curMsg << tcu::TestLog::EndMessage;
282
283                                 if ((curMsg->flags & errorFlags) != 0)
284                                         numErrors += 1;
285                         }
286
287                         m_debugReportRecorder->clearMessages();
288
289                         if (numErrors > 0)
290                                 m_context.getTestContext().setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, (de::toString(numErrors) + " API usage errors found").c_str());
291                 }
292         }
293 }
294
295 tcu::TestNode::IterateResult TestCaseExecutor::iterate (tcu::TestCase*)
296 {
297         DE_ASSERT(m_instance);
298
299         const tcu::TestStatus   result  = m_instance->iterate();
300
301         if (result.isComplete())
302         {
303                 // Vulkan tests shouldn't set result directly
304                 DE_ASSERT(m_context.getTestContext().getTestResult() == QP_TEST_RESULT_LAST);
305                 m_context.getTestContext().setTestResult(result.getCode(), result.getDescription().c_str());
306                 return tcu::TestNode::STOP;
307         }
308         else
309                 return tcu::TestNode::CONTINUE;
310 }
311
312 // GLSL shader tests
313
314 void createGlslTests (tcu::TestCaseGroup* glslTests)
315 {
316         tcu::TestContext&       testCtx         = glslTests->getTestContext();
317
318         // ShaderLibrary-based tests
319         static const struct
320         {
321                 const char*             name;
322                 const char*             description;
323         } s_es310Tests[] =
324         {
325                 { "arrays",                                             "Arrays"                                        },
326                 { "conditionals",                               "Conditional statements"        },
327                 { "constant_expressions",               "Constant expressions"          },
328                 { "constants",                                  "Constants"                                     },
329                 { "conversions",                                "Type conversions"                      },
330                 { "functions",                                  "Functions"                                     },
331                 { "linkage",                                    "Linking"                                       },
332                 { "scoping",                                    "Scoping"                                       },
333                 { "swizzles",                                   "Swizzles"                                      },
334         };
335
336         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_es310Tests); ndx++)
337                 glslTests->addChild(createShaderLibraryGroup(testCtx,
338                                                                                                          s_es310Tests[ndx].name,
339                                                                                                          s_es310Tests[ndx].description,
340                                                                                                          std::string("vulkan/glsl/es310/") + s_es310Tests[ndx].name + ".test").release());
341
342         // ShaderRenderCase-based tests
343         glslTests->addChild(sr::createDerivateTests                     (testCtx));
344         glslTests->addChild(sr::createDiscardTests                      (testCtx));
345         glslTests->addChild(sr::createIndexingTests                     (testCtx));
346         glslTests->addChild(sr::createLoopTests                         (testCtx));
347         glslTests->addChild(sr::createMatrixTests                       (testCtx));
348         glslTests->addChild(sr::createOperatorTests                     (testCtx));
349         glslTests->addChild(sr::createReturnTests                       (testCtx));
350         glslTests->addChild(sr::createStructTests                       (testCtx));
351         glslTests->addChild(sr::createSwitchTests                       (testCtx));
352         glslTests->addChild(sr::createTextureFunctionTests      (testCtx));
353         glslTests->addChild(sr::createTextureGatherTests        (testCtx));
354         glslTests->addChild(sr::createBuiltinVarTests           (testCtx));
355
356         // ShaderExecutor-based tests
357         glslTests->addChild(shaderexecutor::createBuiltinTests                          (testCtx));
358         glslTests->addChild(shaderexecutor::createOpaqueTypeIndexingTests       (testCtx));
359         glslTests->addChild(shaderexecutor::createAtomicOperationTests          (testCtx));
360 }
361
362 // TestPackage
363
364 TestPackage::TestPackage (tcu::TestContext& testCtx)
365         : tcu::TestPackage(testCtx, "dEQP-VK", "dEQP Vulkan Tests")
366 {
367 }
368
369 TestPackage::~TestPackage (void)
370 {
371 }
372
373 tcu::TestCaseExecutor* TestPackage::createExecutor (void) const
374 {
375         return new TestCaseExecutor(m_testCtx);
376 }
377
378 void TestPackage::init (void)
379 {
380         addChild(createTestGroup                                (m_testCtx, "info", "Build and Device Info Tests", createInfoTests));
381         addChild(api::createTests                               (m_testCtx));
382         addChild(memory::createTests                    (m_testCtx));
383         addChild(pipeline::createTests                  (m_testCtx));
384         addChild(BindingModel::createTests              (m_testCtx));
385         addChild(SpirVAssembly::createTests             (m_testCtx));
386         addChild(createTestGroup                                (m_testCtx, "glsl", "GLSL shader execution tests", createGlslTests));
387         addChild(createRenderPassTests                  (m_testCtx));
388         addChild(ubo::createTests                               (m_testCtx));
389         addChild(DynamicState::createTests              (m_testCtx));
390         addChild(ssbo::createTests                              (m_testCtx));
391         addChild(QueryPool::createTests                 (m_testCtx));
392         addChild(Draw::createTests                              (m_testCtx));
393         addChild(compute::createTests                   (m_testCtx));
394         addChild(image::createTests                             (m_testCtx));
395         addChild(wsi::createTests                               (m_testCtx));
396         addChild(synchronization::createTests   (m_testCtx));
397         addChild(sparse::createTests                    (m_testCtx));
398         addChild(tessellation::createTests              (m_testCtx));
399         addChild(rasterization::createTests             (m_testCtx));
400         addChild(clipping::createTests                  (m_testCtx));
401         addChild(FragmentOperations::createTests(m_testCtx));
402         addChild(texture::createTests                   (m_testCtx));
403         addChild(geometry::createTests                  (m_testCtx));
404         addChild(robustness::createTests                (m_testCtx));
405 }
406
407 } // vkt