1 /*-------------------------------------------------------------------------
2 * Vulkan Conformance Tests
3 * ------------------------
4 *
5 * Copyright (c) 2015 Google Inc.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Vulkan Test Package
22 *//*--------------------------------------------------------------------*/
23
24 #include "vktTestPackage.hpp"
25
26 #include "tcuPlatform.hpp"
27 #include "tcuTestCase.hpp"
28 #include "tcuTestLog.hpp"
29 #include "tcuCommandLine.hpp"
30
31 #include "vkPlatform.hpp"
32 #include "vkPrograms.hpp"
33 #include "vkBinaryRegistry.hpp"
34 #include "vkShaderToSpirV.hpp"
35 #include "vkDebugReportUtil.hpp"
36 #include "vkQueryUtil.hpp"
37 #include "vkApiVersion.hpp"
38 #include "vkRenderDocUtil.hpp"
39
40 #include "deUniquePtr.hpp"
41
42 #include "vktTestGroupUtil.hpp"
43 #include "vktApiTests.hpp"
44 #include "vktPipelineTests.hpp"
45 #include "vktBindingModelTests.hpp"
46 #include "vktSpvAsmTests.hpp"
47 #include "vktShaderLibrary.hpp"
48 #include "vktRenderPassTests.hpp"
49 #include "vktMemoryTests.hpp"
50 #include "vktShaderRenderBuiltinVarTests.hpp"
51 #include "vktShaderRenderDerivateTests.hpp"
52 #include "vktShaderRenderDiscardTests.hpp"
53 #include "vktShaderRenderIndexingTests.hpp"
54 #include "vktShaderRenderLimitTests.hpp"
55 #include "vktShaderRenderLoopTests.hpp"
56 #include "vktShaderRenderMatrixTests.hpp"
57 #include "vktShaderRenderOperatorTests.hpp"
58 #include "vktShaderRenderReturnTests.hpp"
59 #include "vktShaderRenderStructTests.hpp"
60 #include "vktShaderRenderSwitchTests.hpp"
61 #include "vktShaderRenderTextureFunctionTests.hpp"
62 #include "vktShaderRenderTextureGatherTests.hpp"
63 #include "vktShaderBuiltinTests.hpp"
64 #include "vktOpaqueTypeIndexingTests.hpp"
65 #include "vktAtomicOperationTests.hpp"
66 #include "vktUniformBlockTests.hpp"
67 #include "vktDynamicStateTests.hpp"
68 #include "vktSSBOLayoutTests.hpp"
69 #include "vktQueryPoolTests.hpp"
70 #include "vktDrawTests.hpp"
71 #include "vktComputeTests.hpp"
72 #include "vktConditionalTests.hpp"
73 #include "vktImageTests.hpp"
74 #include "vktInfoTests.hpp"
75 #include "vktWsiTests.hpp"
76 #include "vktSynchronizationTests.hpp"
77 #include "vktSparseResourcesTests.hpp"
78 #include "vktTessellationTests.hpp"
79 #include "vktRasterizationTests.hpp"
80 #include "vktClippingTests.hpp"
81 #include "vktFragmentOperationsTests.hpp"
82 #include "vktTextureTests.hpp"
83 #include "vktGeometryTests.hpp"
84 #include "vktRobustnessTests.hpp"
85 #include "vktMultiViewTests.hpp"
86 #include "vktSubgroupsTests.hpp"
87 #include "vktYCbCrTests.hpp"
88 #include "vktProtectedMemTests.hpp"
89 #include "vktDeviceGroupTests.hpp"
90 #include "vktMemoryModelTests.hpp"
91 #include "vktVkRunnerExampleTests.hpp"
92
93 #include <vector>
94 #include <sstream>
95
96 namespace // compilation
97 {
98
compileProgram(const vk::GlslSource & source,glu::ShaderProgramInfo * buildInfo,const tcu::CommandLine & commandLine)99 vk::ProgramBinary* compileProgram (const vk::GlslSource& source, glu::ShaderProgramInfo* buildInfo, const tcu::CommandLine& commandLine)
100 {
101 return vk::buildProgram(source, buildInfo, commandLine);
102 }
103
compileProgram(const vk::HlslSource & source,glu::ShaderProgramInfo * buildInfo,const tcu::CommandLine & commandLine)104 vk::ProgramBinary* compileProgram (const vk::HlslSource& source, glu::ShaderProgramInfo* buildInfo, const tcu::CommandLine& commandLine)
105 {
106 return vk::buildProgram(source, buildInfo, commandLine);
107 }
108
compileProgram(const vk::SpirVAsmSource & source,vk::SpirVProgramInfo * buildInfo,const tcu::CommandLine & commandLine)109 vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, vk::SpirVProgramInfo* buildInfo, const tcu::CommandLine& commandLine)
110 {
111 return vk::assembleProgram(source, buildInfo, commandLine);
112 }
113
114 template <typename InfoType, typename IteratorType>
buildProgram(const std::string & casePath,IteratorType iter,const vk::BinaryRegistryReader & prebuiltBinRegistry,tcu::TestLog & log,vk::BinaryCollection * progCollection,const tcu::CommandLine & commandLine)115 vk::ProgramBinary* buildProgram (const std::string& casePath,
116 IteratorType iter,
117 const vk::BinaryRegistryReader& prebuiltBinRegistry,
118 tcu::TestLog& log,
119 vk::BinaryCollection* progCollection,
120 const tcu::CommandLine& commandLine)
121 {
122 const vk::ProgramIdentifier progId (casePath, iter.getName());
123 const tcu::ScopedLogSection progSection (log, iter.getName(), "Program: " + iter.getName());
124 de::MovePtr<vk::ProgramBinary> binProg;
125 InfoType buildInfo;
126
127 try
128 {
129 binProg = de::MovePtr<vk::ProgramBinary>(compileProgram(iter.getProgram(), &buildInfo, commandLine));
130 log << buildInfo;
131 }
132 catch (const tcu::NotSupportedError& err)
133 {
134 // Try to load from cache
135 log << err << tcu::TestLog::Message << "Building from source not supported, loading stored binary instead" << tcu::TestLog::EndMessage;
136
137 binProg = de::MovePtr<vk::ProgramBinary>(prebuiltBinRegistry.loadProgram(progId));
138
139 log << iter.getProgram();
140 }
141 catch (const tcu::Exception&)
142 {
143 // Build failed for other reason
144 log << buildInfo;
145 throw;
146 }
147
148 TCU_CHECK_INTERNAL(binProg);
149
150 {
151 vk::ProgramBinary* const returnBinary = binProg.get();
152
153 progCollection->add(progId.programName, binProg);
154
155 return returnBinary;
156 }
157 }
158
159 } // anonymous(compilation)
160
161 namespace vkt
162 {
163
164 using std::vector;
165 using de::UniquePtr;
166 using de::MovePtr;
167 using tcu::TestLog;
168
169 namespace
170 {
171
createDebugReportRecorder(const vk::PlatformInterface & vkp,const vk::InstanceInterface & vki,vk::VkInstance instance)172 MovePtr<vk::DebugReportRecorder> createDebugReportRecorder (const vk::PlatformInterface& vkp, const vk::InstanceInterface& vki, vk::VkInstance instance)
173 {
174 if (isDebugReportSupported(vkp))
175 return MovePtr<vk::DebugReportRecorder>(new vk::DebugReportRecorder(vki, instance));
176 else
177 TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported");
178 }
179
180 } // anonymous
181
182 // TestCaseExecutor
183
184 class TestCaseExecutor : public tcu::TestCaseExecutor
185 {
186 public:
187 TestCaseExecutor (tcu::TestContext& testCtx);
188 ~TestCaseExecutor (void);
189
190 virtual void init (tcu::TestCase* testCase, const std::string& path);
191 virtual void deinit (tcu::TestCase* testCase);
192
193 virtual tcu::TestNode::IterateResult iterate (tcu::TestCase* testCase);
194
195 private:
196 vk::BinaryCollection m_progCollection;
197 vk::BinaryRegistryReader m_prebuiltBinRegistry;
198
199 const UniquePtr<vk::Library> m_library;
200 Context m_context;
201
202 const UniquePtr<vk::DebugReportRecorder> m_debugReportRecorder;
203 const UniquePtr<vk::RenderDocUtil> m_renderDoc;
204
205 TestInstance* m_instance; //!< Current test case instance
206 };
207
createLibrary(tcu::TestContext & testCtx)208 static MovePtr<vk::Library> createLibrary (tcu::TestContext& testCtx)
209 {
210 return MovePtr<vk::Library>(testCtx.getPlatform().getVulkanPlatform().createLibrary());
211 }
212
TestCaseExecutor(tcu::TestContext & testCtx)213 TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx)
214 : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt")
215 , m_library (createLibrary(testCtx))
216 , m_context (testCtx, m_library->getPlatformInterface(), m_progCollection)
217 , m_debugReportRecorder (testCtx.getCommandLine().isValidationEnabled()
218 ? createDebugReportRecorder(m_library->getPlatformInterface(),
219 m_context.getInstanceInterface(),
220 m_context.getInstance())
221 : MovePtr<vk::DebugReportRecorder>(DE_NULL))
222 , m_renderDoc (testCtx.getCommandLine().isRenderDocEnabled()
223 ? MovePtr<vk::RenderDocUtil>(new vk::RenderDocUtil())
224 : MovePtr<vk::RenderDocUtil>(DE_NULL))
225 , m_instance (DE_NULL)
226 {
227 }
228
~TestCaseExecutor(void)229 TestCaseExecutor::~TestCaseExecutor (void)
230 {
231 delete m_instance;
232 }
233
init(tcu::TestCase * testCase,const std::string & casePath)234 void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePath)
235 {
236 const TestCase* vktCase = dynamic_cast<TestCase*>(testCase);
237 tcu::TestLog& log = m_context.getTestContext().getLog();
238 const deUint32 usedVulkanVersion = m_context.getUsedApiVersion();
239 const vk::SpirvVersion baselineSpirvVersion = vk::getBaselineSpirvVersion(usedVulkanVersion);
240 vk::ShaderBuildOptions defaultGlslBuildOptions (usedVulkanVersion, baselineSpirvVersion, 0u);
241 vk::ShaderBuildOptions defaultHlslBuildOptions (usedVulkanVersion, baselineSpirvVersion, 0u);
242 vk::SpirVAsmBuildOptions defaultSpirvAsmBuildOptions (usedVulkanVersion, baselineSpirvVersion);
243 vk::SourceCollections sourceProgs (usedVulkanVersion, defaultGlslBuildOptions, defaultHlslBuildOptions, defaultSpirvAsmBuildOptions);
244 const bool doShaderLog = log.isShaderLoggingEnabled();
245 const tcu::CommandLine& commandLine = m_context.getTestContext().getCommandLine();
246
247 DE_UNREF(casePath); // \todo [2015-03-13 pyry] Use this to identify ProgramCollection storage path
248
249 if (!vktCase)
250 TCU_THROW(InternalError, "Test node not an instance of vkt::TestCase");
251
252 vktCase->checkSupport(m_context);
253
254 m_progCollection.clear();
255 vktCase->initPrograms(sourceProgs);
256
257 for (vk::GlslSourceCollection::Iterator progIter = sourceProgs.glslSources.begin(); progIter != sourceProgs.glslSources.end(); ++progIter)
258 {
259 if (progIter.getProgram().buildOptions.targetVersion > vk::getMaxSpirvVersionForGlsl(m_context.getUsedApiVersion()))
260 TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
261
262 const vk::ProgramBinary* const binProg = buildProgram<glu::ShaderProgramInfo, vk::GlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection, commandLine);
263
264 if (doShaderLog)
265 {
266 try
267 {
268 std::ostringstream disasm;
269
270 vk::disassembleProgram(*binProg, &disasm);
271
272 log << vk::SpirVAsmSource(disasm.str());
273 }
274 catch (const tcu::NotSupportedError& err)
275 {
276 log << err;
277 }
278 }
279 }
280
281 for (vk::HlslSourceCollection::Iterator progIter = sourceProgs.hlslSources.begin(); progIter != sourceProgs.hlslSources.end(); ++progIter)
282 {
283 if (progIter.getProgram().buildOptions.targetVersion > vk::getMaxSpirvVersionForGlsl(m_context.getUsedApiVersion()))
284 TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
285
286 const vk::ProgramBinary* const binProg = buildProgram<glu::ShaderProgramInfo, vk::HlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection, commandLine);
287
288 if (doShaderLog)
289 {
290 try
291 {
292 std::ostringstream disasm;
293
294 vk::disassembleProgram(*binProg, &disasm);
295
296 log << vk::SpirVAsmSource(disasm.str());
297 }
298 catch (const tcu::NotSupportedError& err)
299 {
300 log << err;
301 }
302 }
303 }
304
305 for (vk::SpirVAsmCollection::Iterator asmIterator = sourceProgs.spirvAsmSources.begin(); asmIterator != sourceProgs.spirvAsmSources.end(); ++asmIterator)
306 {
307 if (asmIterator.getProgram().buildOptions.targetVersion > vk::getMaxSpirvVersionForAsm(m_context.getUsedApiVersion()))
308 TCU_THROW(NotSupportedError, "Shader requires SPIR-V higher than available");
309
310 buildProgram<vk::SpirVProgramInfo, vk::SpirVAsmCollection::Iterator>(casePath, asmIterator, m_prebuiltBinRegistry, log, &m_progCollection, commandLine);
311 }
312
313 if (m_renderDoc) m_renderDoc->startFrame(m_context.getInstance());
314
315 DE_ASSERT(!m_instance);
316 m_instance = vktCase->createInstance(m_context);
317 }
318
deinit(tcu::TestCase *)319 void TestCaseExecutor::deinit (tcu::TestCase*)
320 {
321 delete m_instance;
322 m_instance = DE_NULL;
323
324 if (m_renderDoc) m_renderDoc->endFrame(m_context.getInstance());
325
326 // Collect and report any debug messages
327 if (m_debugReportRecorder)
328 {
329 // \note We are not logging INFORMATION and DEBUG messages
330 static const vk::VkDebugReportFlagsEXT errorFlags = vk::VK_DEBUG_REPORT_ERROR_BIT_EXT;
331 static const vk::VkDebugReportFlagsEXT logFlags = errorFlags
332 | vk::VK_DEBUG_REPORT_WARNING_BIT_EXT
333 | vk::VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT;
334
335 typedef vk::DebugReportRecorder::MessageList DebugMessages;
336
337 const DebugMessages& messages = m_debugReportRecorder->getMessages();
338 tcu::TestLog& log = m_context.getTestContext().getLog();
339
340 if (messages.begin() != messages.end())
341 {
342 const tcu::ScopedLogSection section (log, "DebugMessages", "Debug Messages");
343 int numErrors = 0;
344
345 for (DebugMessages::const_iterator curMsg = messages.begin(); curMsg != messages.end(); ++curMsg)
346 {
347 if ((curMsg->flags & logFlags) != 0)
348 log << tcu::TestLog::Message << *curMsg << tcu::TestLog::EndMessage;
349
350 if ((curMsg->flags & errorFlags) != 0)
351 numErrors += 1;
352 }
353
354 m_debugReportRecorder->clearMessages();
355
356 if (numErrors > 0)
357 m_context.getTestContext().setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, (de::toString(numErrors) + " API usage errors found").c_str());
358 }
359 }
360 }
361
iterate(tcu::TestCase *)362 tcu::TestNode::IterateResult TestCaseExecutor::iterate (tcu::TestCase*)
363 {
364 DE_ASSERT(m_instance);
365
366 const tcu::TestStatus result = m_instance->iterate();
367
368 if (result.isComplete())
369 {
370 // Vulkan tests shouldn't set result directly
371 DE_ASSERT(m_context.getTestContext().getTestResult() == QP_TEST_RESULT_LAST);
372 m_context.getTestContext().setTestResult(result.getCode(), result.getDescription().c_str());
373 return tcu::TestNode::STOP;
374 }
375 else
376 return tcu::TestNode::CONTINUE;
377 }
378
379 // GLSL shader tests
380
createGlslTests(tcu::TestCaseGroup * glslTests)381 void createGlslTests (tcu::TestCaseGroup* glslTests)
382 {
383 tcu::TestContext& testCtx = glslTests->getTestContext();
384
385 // ShaderLibrary-based tests
386 static const struct
387 {
388 const char* name;
389 const char* description;
390 } s_es310Tests[] =
391 {
392 { "arrays", "Arrays" },
393 { "conditionals", "Conditional statements" },
394 { "constant_expressions", "Constant expressions" },
395 { "constants", "Constants" },
396 { "conversions", "Type conversions" },
397 { "functions", "Functions" },
398 { "linkage", "Linking" },
399 { "scoping", "Scoping" },
400 { "swizzles", "Swizzles" },
401 };
402
403 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_es310Tests); ndx++)
404 glslTests->addChild(createShaderLibraryGroup(testCtx,
405 s_es310Tests[ndx].name,
406 s_es310Tests[ndx].description,
407 std::string("vulkan/glsl/es310/") + s_es310Tests[ndx].name + ".test").release());
408
409 static const struct
410 {
411 const char* name;
412 const char* description;
413 } s_440Tests[] =
414 {
415 { "linkage", "Linking" },
416 };
417
418 de::MovePtr<tcu::TestCaseGroup> glsl440Tests = de::MovePtr<tcu::TestCaseGroup>(new tcu::TestCaseGroup(testCtx, "440", ""));
419
420 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_440Tests); ndx++)
421 glsl440Tests->addChild(createShaderLibraryGroup(testCtx,
422 s_440Tests[ndx].name,
423 s_440Tests[ndx].description,
424 std::string("vulkan/glsl/440/") + s_440Tests[ndx].name + ".test").release());
425
426 glslTests->addChild(glsl440Tests.release());
427
428 // ShaderRenderCase-based tests
429 glslTests->addChild(sr::createDerivateTests (testCtx));
430 glslTests->addChild(sr::createDiscardTests (testCtx));
431 glslTests->addChild(sr::createIndexingTests (testCtx));
432 glslTests->addChild(sr::createLimitTests (testCtx));
433 glslTests->addChild(sr::createLoopTests (testCtx));
434 glslTests->addChild(sr::createMatrixTests (testCtx));
435 glslTests->addChild(sr::createOperatorTests (testCtx));
436 glslTests->addChild(sr::createReturnTests (testCtx));
437 glslTests->addChild(sr::createStructTests (testCtx));
438 glslTests->addChild(sr::createSwitchTests (testCtx));
439 glslTests->addChild(sr::createTextureFunctionTests (testCtx));
440 glslTests->addChild(sr::createTextureGatherTests (testCtx));
441 glslTests->addChild(sr::createBuiltinVarTests (testCtx));
442
443 // ShaderExecutor-based tests
444 glslTests->addChild(shaderexecutor::createBuiltinTests (testCtx));
445 glslTests->addChild(shaderexecutor::createOpaqueTypeIndexingTests (testCtx));
446 glslTests->addChild(shaderexecutor::createAtomicOperationTests (testCtx));
447 }
448
449 // TestPackage
450
TestPackage(tcu::TestContext & testCtx)451 TestPackage::TestPackage (tcu::TestContext& testCtx)
452 : tcu::TestPackage(testCtx, "dEQP-VK", "dEQP Vulkan Tests")
453 {
454 }
455
~TestPackage(void)456 TestPackage::~TestPackage (void)
457 {
458 }
459
createExecutor(void) const460 tcu::TestCaseExecutor* TestPackage::createExecutor (void) const
461 {
462 return new TestCaseExecutor(m_testCtx);
463 }
464
init(void)465 void TestPackage::init (void)
466 {
467 addChild(createTestGroup (m_testCtx, "info", "Build and Device Info Tests", createInfoTests));
468 addChild(api::createTests (m_testCtx));
469 addChild(memory::createTests (m_testCtx));
470 addChild(pipeline::createTests (m_testCtx));
471 addChild(BindingModel::createTests (m_testCtx));
472 addChild(SpirVAssembly::createTests (m_testCtx));
473 addChild(createTestGroup (m_testCtx, "glsl", "GLSL shader execution tests", createGlslTests));
474 addChild(createRenderPassTests (m_testCtx));
475 addChild(createRenderPass2Tests (m_testCtx));
476 addChild(ubo::createTests (m_testCtx));
477 addChild(DynamicState::createTests (m_testCtx));
478 addChild(ssbo::createTests (m_testCtx));
479 addChild(QueryPool::createTests (m_testCtx));
480 addChild(Draw::createTests (m_testCtx));
481 addChild(compute::createTests (m_testCtx));
482 addChild(image::createTests (m_testCtx));
483 addChild(wsi::createTests (m_testCtx));
484 addChild(synchronization::createTests (m_testCtx));
485 addChild(sparse::createTests (m_testCtx));
486 addChild(tessellation::createTests (m_testCtx));
487 addChild(rasterization::createTests (m_testCtx));
488 addChild(clipping::createTests (m_testCtx));
489 addChild(FragmentOperations::createTests(m_testCtx));
490 addChild(texture::createTests (m_testCtx));
491 addChild(geometry::createTests (m_testCtx));
492 addChild(robustness::createTests (m_testCtx));
493 addChild(MultiView::createTests (m_testCtx));
494 addChild(subgroups::createTests (m_testCtx));
495 addChild(ycbcr::createTests (m_testCtx));
496 addChild(ProtectedMem::createTests (m_testCtx));
497 addChild(DeviceGroup::createTests (m_testCtx));
498 addChild(MemoryModel::createTests (m_testCtx));
499 addChild(conditional::createTests (m_testCtx));
500 addChild(vkrunner::createTests (m_testCtx));
501 }
502
503 } // vkt
504