1 /*------------------------------------------------------------------------
2 * Vulkan Conformance Tests
3 * ------------------------
4 *
5 * Copyright (c) 2019 Google LLC
6 * Copyright (c) 2019 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 *//*!
21 * \file
22 * \brief Functional tests using amber
23 *//*--------------------------------------------------------------------*/
24
25 #include <amber/amber.h>
26 #include "amber/recipe.h"
27
28 #include <iostream>
29
30 #include "deDefs.hpp"
31 #include "deUniquePtr.hpp"
32 #include "deFilePath.hpp"
33 #include "vktTestCaseUtil.hpp"
34 #include "tcuTestLog.hpp"
35 #include "vktAmberTestCase.hpp"
36 #include "vktAmberHelper.hpp"
37 #include "tcuResource.hpp"
38 #include "tcuTestLog.hpp"
39 #include "vkSpirVProgram.hpp"
40 #include "vkImageUtil.hpp"
41
42 namespace vkt
43 {
44 namespace cts_amber
45 {
46
AmberTestCase(tcu::TestContext & testCtx,const char * name,const char * description,const std::string & readFilename)47 AmberTestCase::AmberTestCase (tcu::TestContext& testCtx,
48 const char* name,
49 const char* description,
50 const std::string& readFilename)
51 : TestCase(testCtx, name),
52 m_recipe(DE_NULL),
53 m_readFilename(readFilename)
54 {
55 (void)description;
56 }
57
~AmberTestCase(void)58 AmberTestCase::~AmberTestCase (void)
59 {
60 delete m_recipe;
61 }
62
createInstance(Context & ctx) const63 TestInstance* AmberTestCase::createInstance (Context& ctx) const
64 {
65 return new AmberTestInstance(ctx, m_recipe, nullptr);
66 }
67
createEngineConfig(Context & ctx,vk::VkDevice customDevice)68 static amber::EngineConfig* createEngineConfig (Context& ctx, vk::VkDevice customDevice)
69 {
70 vk::VkDevice dev = customDevice != nullptr ? customDevice : ctx.getDevice();
71 vk::VkQueue queue;
72 vk::DeviceDriver vk(ctx.getPlatformInterface(), ctx.getInstance(), dev, ctx.getUsedApiVersion());
73 vk.getDeviceQueue(dev, ctx.getUniversalQueueFamilyIndex(), 0, &queue);
74
75 amber::EngineConfig* vkConfig = GetVulkanConfig(ctx.getInstance(),
76 ctx.getPhysicalDevice(), dev, &ctx.getDeviceFeatures(),
77 &ctx.getDeviceFeatures2(), ctx.getInstanceExtensions(),
78 ctx.getDeviceExtensions(), ctx.getUniversalQueueFamilyIndex(),
79 queue, ctx.getInstanceProcAddr());
80
81 return vkConfig;
82 }
83
84 // Returns true if the given feature is supported by the device.
85 // Throws an internal error If the feature is not recognized at all.
isFeatureSupported(const vkt::Context & ctx,const std::string & feature)86 static bool isFeatureSupported(const vkt::Context& ctx, const std::string& feature)
87 {
88 if (feature == "Storage16BitFeatures.storageBuffer16BitAccess")
89 return ctx.get16BitStorageFeatures().storageBuffer16BitAccess;
90 if (feature == "Float16Int8Features.shaderFloat16")
91 return ctx.getShaderFloat16Int8Features().shaderFloat16;
92 if (feature == "Float16Int8Features.shaderInt8")
93 return ctx.getShaderFloat16Int8Features().shaderInt8;
94 if (feature == "Features.shaderFloat64")
95 return ctx.getDeviceFeatures().shaderFloat64;
96 if (feature == "Features.shaderInt16")
97 return ctx.getDeviceFeatures().shaderInt16;
98 if (feature == "Features.shaderInt64")
99 return ctx.getDeviceFeatures().shaderInt64;
100 if (feature == "Features.depthClamp")
101 return ctx.getDeviceFeatures().depthClamp;
102 if (feature == "Features.tessellationShader")
103 return ctx.getDeviceFeatures().tessellationShader;
104 if (feature == "Features.shaderTessellationAndGeometryPointSize")
105 return ctx.getDeviceFeatures().shaderTessellationAndGeometryPointSize;
106 if (feature == "Features.geometryShader")
107 return ctx.getDeviceFeatures().geometryShader;
108 if (feature == "Features.fragmentStoresAndAtomics")
109 return ctx.getDeviceFeatures().fragmentStoresAndAtomics;
110 if (feature == "Features.vertexPipelineStoresAndAtomics")
111 return ctx.getDeviceFeatures().vertexPipelineStoresAndAtomics;
112 if (feature == "Features.fillModeNonSolid")
113 return ctx.getDeviceFeatures().fillModeNonSolid;
114 if (feature == "Features.shaderStorageImageMultisample")
115 return ctx.getDeviceFeatures().shaderStorageImageMultisample;
116 if (feature == "Features.sampleRateShading")
117 return ctx.getDeviceFeatures().sampleRateShading;
118 if (feature == "VariablePointerFeatures.variablePointersStorageBuffer")
119 return ctx.getVariablePointersFeatures().variablePointersStorageBuffer;
120 if (feature == "VariablePointerFeatures.variablePointers")
121 return ctx.getVariablePointersFeatures().variablePointers;
122 if (feature == "SubgroupSupportedStages.fragment")
123 return (ctx.getSubgroupProperties().supportedStages & vk::VK_SHADER_STAGE_FRAGMENT_BIT) != 0;
124 if (feature == "SubgroupSupportedOperations.vote")
125 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_VOTE_BIT) != 0;
126 if (feature == "SubgroupSupportedOperations.basic")
127 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_BASIC_BIT) != 0;
128 if (feature == "SubgroupSupportedOperations.ballot")
129 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_BALLOT_BIT) != 0;
130 if (feature == "Storage16BitFeatures.storageBuffer16BitAccess")
131 return ctx.get16BitStorageFeatures().storageBuffer16BitAccess;
132 if (feature == "Storage8BitFeatures.storageBuffer8BitAccess")
133 return ctx.get8BitStorageFeatures().storageBuffer8BitAccess;
134
135 std::string message = std::string("Unexpected feature name: ") + feature;
136 TCU_THROW(InternalError, message.c_str());
137 }
138
delayedInit(void)139 void AmberTestCase::delayedInit(void)
140 {
141 // Make sure the input can be parsed before we use it.
142 if (!parse(m_readFilename))
143 {
144 std::string message = "Failed to parse Amber file: " + m_readFilename;
145 TCU_THROW(InternalError, message.c_str());
146 }
147 }
148
checkSupport(Context & ctx) const149 void AmberTestCase::checkSupport(Context& ctx) const
150 {
151 // Check for instance and device extensions as declared by the test code.
152 if (m_required_extensions.size())
153 {
154 std::set<std::string> device_extensions(ctx.getDeviceExtensions().begin(),
155 ctx.getDeviceExtensions().end());
156 std::set<std::string> instance_extensions(ctx.getInstanceExtensions().begin(),
157 ctx.getInstanceExtensions().end());
158 std::string missing;
159 for (std::set<std::string>::iterator iter = m_required_extensions.begin();
160 iter != m_required_extensions.end();
161 ++iter)
162 {
163 const std::string extension = *iter;
164 if ((device_extensions.count(extension) == 0) &&
165 (instance_extensions.count(extension) == 0))
166 {
167 missing += " " + extension;
168 }
169 }
170 if (missing.size() > 0)
171 {
172 std::string message("Test requires unsupported extensions:");
173 message += missing;
174 TCU_THROW(NotSupportedError, message.c_str());
175 }
176 }
177
178 // Check for required features. Do this after extensions are checked because
179 // some feature checks are only valid when corresponding extensions are enabled.
180 if (m_required_features.size())
181 {
182 std::string missing;
183 for (std::set<std::string>::iterator iter = m_required_features.begin();
184 iter != m_required_features.end();
185 ++iter)
186 {
187 const std::string feature = *iter;
188 if (!isFeatureSupported(ctx, feature))
189 {
190 missing += " " + feature;
191 }
192 }
193 if (missing.size() > 0)
194 {
195 std::string message("Test requires unsupported features:");
196 message += missing;
197 TCU_THROW(NotSupportedError, message.c_str());
198 }
199 }
200
201 for (auto req : m_imageRequirements)
202 checkImageSupport(ctx.getInstanceInterface(), ctx.getPhysicalDevice(), req);
203
204 for (auto req : m_bufferRequirements)
205 {
206 vk::VkFormatProperties prop;
207 ctx.getInstanceInterface().getPhysicalDeviceFormatProperties(ctx.getPhysicalDevice(), req.m_format, &prop);
208
209 if ((req.m_featureFlags & prop.bufferFeatures) != req.m_featureFlags)
210 {
211 TCU_THROW(NotSupportedError, "Buffer format doesn't support required feature flags");
212 }
213 }
214
215 if (m_checkSupportCallback)
216 (m_checkSupportCallback)(ctx, m_name);
217 }
218
219 class Delegate : public amber::Delegate
220 {
221 public:
222 Delegate (tcu::TestContext& testCtx);
223
224 amber::Result LoadBufferData (const std::string file_name,
225 amber::BufferDataFileType file_type,
226 amber::BufferInfo* buffer) const override;
227
Log(const std::string &)228 void Log (const std::string& /*message*/) override { DE_FATAL("amber::Delegate::Log unimplemented"); }
LogGraphicsCalls(void) const229 bool LogGraphicsCalls (void) const override { return m_logGraphicsCalls; }
SetLogGraphicsCalls(bool log_graphics_calls)230 void SetLogGraphicsCalls (bool log_graphics_calls) { m_logGraphicsCalls = log_graphics_calls; }
LogExecuteCalls(void) const231 bool LogExecuteCalls (void) const override { return m_logExecuteCalls; }
SetLogExecuteCalls(bool log_execute_calls)232 void SetLogExecuteCalls (bool log_execute_calls) { m_logExecuteCalls = log_execute_calls; }
LogGraphicsCallsTime(void) const233 bool LogGraphicsCallsTime (void) const override { return m_logGraphicsCallsTime; }
SetLogGraphicsCallsTime(bool log_graphics_calls_time)234 void SetLogGraphicsCallsTime (bool log_graphics_calls_time) { m_logGraphicsCallsTime = log_graphics_calls_time; }
GetTimestampNs(void) const235 deUint64 GetTimestampNs (void) const override { DE_FATAL("amber::Delegate::GetTimestampNs unimplemented"); return 0; }
SetScriptPath(std::string path)236 void SetScriptPath (std::string path) { m_path = path; }
237
238 private:
239 tcu::TestContext& m_testCtx;
240 std::string m_path;
241 bool m_logGraphicsCalls;
242 bool m_logGraphicsCallsTime;
243 bool m_logExecuteCalls;
244 };
245
Delegate(tcu::TestContext & testCtx)246 Delegate::Delegate (tcu::TestContext& testCtx)
247 : m_testCtx (testCtx)
248 , m_path ("")
249 , m_logGraphicsCalls (false)
250 , m_logGraphicsCallsTime (false)
251 , m_logExecuteCalls (false)
252 {
253 }
254
LoadBufferData(const std::string file_name,amber::BufferDataFileType file_type,amber::BufferInfo * buffer) const255 amber::Result Delegate::LoadBufferData (const std::string file_name,
256 amber::BufferDataFileType file_type,
257 amber::BufferInfo* buffer) const
258 {
259 const tcu::Archive& archive = m_testCtx.getArchive();
260 const de::FilePath filePath = de::FilePath(m_path).join(file_name);
261 de::UniquePtr<tcu::Resource> file (archive.getResource(filePath.getPath()));
262 int numBytes = file->getSize();
263 std::vector<deUint8> bytes (numBytes);
264
265 if (file_type == amber::BufferDataFileType::kPng)
266 return amber::Result("Amber PNG loading unimplemented");
267
268 file->read(bytes.data(), numBytes);
269
270 if (bytes.empty())
271 return amber::Result("Failed to load buffer data " + file_name);
272
273 for (deUint8 byte : bytes)
274 {
275 amber::Value value;
276 value.SetIntValue(static_cast<deUint64>(byte));
277 buffer->values.push_back(value);
278 }
279
280 buffer->width = 1;
281 buffer->height = 1;
282
283 return {};
284 }
285
parse(const std::string & readFilename)286 bool AmberTestCase::parse (const std::string& readFilename)
287 {
288 std::string script = ShaderSourceProvider::getSource(m_testCtx.getArchive(), readFilename.c_str());
289 if (script.empty())
290 return false;
291
292 Delegate delegate (m_testCtx);
293 delegate.SetScriptPath(de::FilePath(readFilename).getDirName());
294
295 m_recipe = new amber::Recipe();
296
297 amber::Amber am (&delegate);
298 amber::Result r = am.Parse(script, m_recipe);
299
300 m_recipe->SetFenceTimeout(~0u); // infinity of miliseconds
301
302 if (!r.IsSuccess())
303 {
304 getTestContext().getLog()
305 << tcu::TestLog::Message
306 << "Failed to parse Amber test "
307 << readFilename
308 << ": "
309 << r.Error()
310 << "\n"
311 << tcu::TestLog::EndMessage;
312 // TODO(dneto): Enhance Amber to not require this.
313 m_recipe->SetImpl(DE_NULL);
314 return false;
315 }
316 return true;
317 }
318
initPrograms(vk::SourceCollections & programCollection) const319 void AmberTestCase::initPrograms (vk::SourceCollections& programCollection) const
320 {
321 std::vector<amber::ShaderInfo> shaders = m_recipe->GetShaderInfo();
322 for (size_t i = 0; i < shaders.size(); ++i)
323 {
324 const amber::ShaderInfo& shader = shaders[i];
325
326 vk::SpirvVersion spirvVersion = vk::SPIRV_VERSION_1_0;
327 DE_STATIC_ASSERT(vk::SPIRV_VERSION_LAST == vk::SPIRV_VERSION_1_6 + 1);
328 if (shader.target_env == "spv1.6")
329 spirvVersion = vk::SPIRV_VERSION_1_6;
330 else if (shader.target_env == "spv1.5")
331 spirvVersion = vk::SPIRV_VERSION_1_5;
332 else if (shader.target_env == "spv1.4")
333 spirvVersion = vk::SPIRV_VERSION_1_4;
334 else if (shader.target_env == "spv1.3")
335 spirvVersion = vk::SPIRV_VERSION_1_3;
336 else if (shader.target_env == "spv1.2")
337 spirvVersion = vk::SPIRV_VERSION_1_2;
338 else if (shader.target_env == "spv1.1")
339 spirvVersion = vk::SPIRV_VERSION_1_1;
340
341 /* Hex encoded shaders do not need to be pre-compiled */
342 if (shader.format == amber::kShaderFormatSpirvHex)
343 continue;
344
345 if (shader.format == amber::kShaderFormatSpirvAsm)
346 {
347 programCollection.spirvAsmSources.add(shader.shader_name) << shader.shader_source << m_asm_options;
348 }
349 else if (shader.format == amber::kShaderFormatGlsl)
350 {
351 bool allowSpirv14 = (spirvVersion == vk::SPIRV_VERSION_1_4);
352
353 switch (shader.type)
354 {
355 case amber::kShaderTypeCompute:
356 programCollection.glslSources.add(shader.shader_name)
357 << glu::ComputeSource(shader.shader_source)
358 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
359 break;
360 case amber::kShaderTypeGeometry:
361 programCollection.glslSources.add(shader.shader_name)
362 << glu::GeometrySource(shader.shader_source)
363 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
364 break;
365 case amber::kShaderTypeFragment:
366 programCollection.glslSources.add(shader.shader_name)
367 << glu::FragmentSource(shader.shader_source)
368 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
369 break;
370 case amber::kShaderTypeVertex:
371 programCollection.glslSources.add(shader.shader_name)
372 << glu::VertexSource(shader.shader_source)
373 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
374 break;
375 case amber::kShaderTypeTessellationControl:
376 programCollection.glslSources.add(shader.shader_name)
377 << glu::TessellationControlSource(shader.shader_source)
378 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
379 break;
380 case amber::kShaderTypeTessellationEvaluation:
381 programCollection.glslSources.add(shader.shader_name)
382 << glu::TessellationEvaluationSource(shader.shader_source)
383 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
384 break;
385 case amber::kShaderTypeMulti:
386 DE_ASSERT(false && "Multi shaders not supported");
387 break;
388 }
389 }
390 else
391 {
392 DE_ASSERT(false && "Shader format not supported");
393 }
394 }
395 }
396
iterate(void)397 tcu::TestStatus AmberTestInstance::iterate (void)
398 {
399 amber::Amber am (DE_NULL);
400 amber::Options amber_options;
401 amber::ShaderMap shaderMap;
402 amber::Result r;
403
404 amber_options.engine = amber::kEngineTypeVulkan;
405 amber_options.config = createEngineConfig(m_context, m_customDevice);
406 amber_options.execution_type = amber::ExecutionType::kExecute;
407
408 // Check for extensions as declared by the Amber script itself. Throw an internal
409 // error if that's more demanding.
410 r = am.AreAllRequirementsSupported(m_recipe, &amber_options);
411 if (!r.IsSuccess())
412 {
413 // dEQP does not to rely on external code to determine whether
414 // a test is supported. So throw an internal error here instead
415 // of a NotSupportedError. If an Amber test is not supported, then
416 // you must override this method and throw a NotSupported exception
417 // before reach here.
418 TCU_THROW(InternalError, r.Error().c_str());
419 }
420
421 std::vector<amber::ShaderInfo> shaders = m_recipe->GetShaderInfo();
422 for (size_t i = 0; i < shaders.size(); ++i)
423 {
424 const amber::ShaderInfo& shader = shaders[i];
425
426 if (!m_context.getBinaryCollection().contains(shader.shader_name))
427 continue;
428
429 size_t len = m_context.getBinaryCollection().get(shader.shader_name).getSize();
430 /* This is a compiled spir-v binary which must be made of 4-byte words. We
431 * are moving into a word sized vector so divide by 4
432 */
433 std::vector<deUint32> data;
434 data.resize(len >> 2);
435 deMemcpy(data.data(), m_context.getBinaryCollection().get(shader.shader_name).getBinary(), len);
436
437 shaderMap[shader.shader_name] = data;
438 }
439
440 r = am.ExecuteWithShaderData(m_recipe, &amber_options, shaderMap);
441 if (!r.IsSuccess()) {
442 m_context.getTestContext().getLog()
443 << tcu::TestLog::Message
444 << r.Error()
445 << "\n"
446 << tcu::TestLog::EndMessage;
447 }
448
449 delete amber_options.config;
450
451 return r.IsSuccess() ? tcu::TestStatus::pass("Pass") :tcu::TestStatus::fail("Fail");
452 }
453
setSpirVAsmBuildOptions(const vk::SpirVAsmBuildOptions & asm_options)454 void AmberTestCase::setSpirVAsmBuildOptions (const vk::SpirVAsmBuildOptions& asm_options)
455 {
456 m_asm_options = asm_options;
457 }
458
addRequirement(const std::string & requirement)459 void AmberTestCase::addRequirement (const std::string& requirement)
460 {
461 if (requirement.find(".") != std::string::npos)
462 m_required_features.insert(requirement);
463 else
464 m_required_extensions.insert(requirement);
465 }
466
addImageRequirement(vk::VkImageCreateInfo info)467 void AmberTestCase::addImageRequirement (vk::VkImageCreateInfo info)
468 {
469 m_imageRequirements.push_back(info);
470 }
471
addBufferRequirement(BufferRequirement req)472 void AmberTestCase::addBufferRequirement (BufferRequirement req)
473 {
474 m_bufferRequirements.push_back(req);
475 }
476
validateRequirements()477 bool AmberTestCase::validateRequirements()
478 {
479 if (!parse(m_readFilename))
480 {
481 std::string message = "Failed to parse Amber file: " + m_readFilename;
482 m_testCtx.getLog() << tcu::TestLog::Message << message << tcu::TestLog::EndMessage;
483 return false;
484 }
485
486 // Check if the list of required CTS features and extensions matches the
487 // one in the recipe. Throw InternalError if they do not match.
488
489 const auto& deviceExtensions = m_recipe->GetRequiredInstanceExtensions();
490 const auto& instanceExtensions = m_recipe->GetRequiredDeviceExtensions();
491 auto requiredFeatures = m_recipe->GetRequiredFeatures();
492
493 for (auto& req : requiredFeatures)
494 {
495 if (req.find(".") == std::string::npos)
496 req = "Features." + req;
497 }
498
499 std::set<std::string> allRequirements;
500 allRequirements.insert(begin(deviceExtensions), end(deviceExtensions));
501 allRequirements.insert(begin(instanceExtensions), end(instanceExtensions));
502 allRequirements.insert(begin(requiredFeatures), end(requiredFeatures));
503
504 std::set<std::string> ctsRequirements = m_required_features;
505 ctsRequirements.insert(begin(m_required_extensions), end(m_required_extensions));
506
507 if (allRequirements != ctsRequirements)
508 {
509 auto& log = m_testCtx.getLog();
510 log << tcu::TestLog::Message << "ERROR: CTS and Amber test requirement mismatch." << tcu::TestLog::EndMessage;
511 log << tcu::TestLog::Message << "Amber filename: " << m_readFilename << tcu::TestLog::EndMessage;
512 log << tcu::TestLog::Message << "CTS requirements:" << tcu::TestLog::EndMessage;
513 for (const auto& ctsReq : ctsRequirements)
514 log << tcu::TestLog::Message << " " << ctsReq << tcu::TestLog::EndMessage;
515
516 log << tcu::TestLog::Message << "Amber requirements:" << tcu::TestLog::EndMessage;
517 for (const auto& amberReq : allRequirements)
518 log << tcu::TestLog::Message << " " << amberReq << tcu::TestLog::EndMessage;
519
520 // Repeat message for cerr so it's visible in console log.
521 std::cerr << "ERROR: CTS and Amber test requirement mismatch.\n";
522 std::cerr << "Amber filename: " << m_readFilename << "\n";
523 std::cerr << "CTS requirements:\n";
524 for (const auto& ctsReq : ctsRequirements)
525 std::cerr << " " << ctsReq << "\n";
526
527 std::cerr << "Amber requirements:\n";
528 for (const auto& amberReq : allRequirements)
529 std::cerr << " " << amberReq << "\n";
530
531 return false;
532 }
533 return true;
534 }
535
536 } // cts_amber
537 } // vkt
538