1 /*------------------------------------------------------------------------
2 * Vulkan Conformance Tests
3 * ------------------------
4 *
5 * Copyright (c) 2019 Google LLC
6 * Copyright (c) 2019 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 *//*!
21 * \file
22 * \brief Functional tests using amber
23 *//*--------------------------------------------------------------------*/
24
25 #include <amber/amber.h>
26 #include "amber/recipe.h"
27
28 #include <iostream>
29
30 #include "deDefs.hpp"
31 #include "deUniquePtr.hpp"
32 #include "deFilePath.hpp"
33 #include "vktTestCaseUtil.hpp"
34 #include "tcuTestLog.hpp"
35 #include "vktAmberTestCase.hpp"
36 #include "vktAmberHelper.hpp"
37 #include "tcuResource.hpp"
38 #include "tcuTestLog.hpp"
39 #include "vkSpirVProgram.hpp"
40 #include "vkImageUtil.hpp"
41
42 namespace vkt
43 {
44 namespace cts_amber
45 {
46
AmberTestCase(tcu::TestContext & testCtx,const char * name,const char * description,const std::string & readFilename)47 AmberTestCase::AmberTestCase (tcu::TestContext& testCtx,
48 const char* name,
49 const char* description,
50 const std::string& readFilename)
51 : TestCase(testCtx, name, description),
52 m_recipe(DE_NULL),
53 m_readFilename(readFilename)
54 {
55 }
56
~AmberTestCase(void)57 AmberTestCase::~AmberTestCase (void)
58 {
59 delete m_recipe;
60 }
61
createInstance(Context & ctx) const62 TestInstance* AmberTestCase::createInstance (Context& ctx) const
63 {
64 return new AmberTestInstance(ctx, m_recipe, nullptr);
65 }
66
createEngineConfig(Context & ctx,vk::VkDevice customDevice)67 static amber::EngineConfig* createEngineConfig (Context& ctx, vk::VkDevice customDevice)
68 {
69 vk::VkDevice dev = customDevice != nullptr ? customDevice : ctx.getDevice();
70 vk::VkQueue queue;
71 vk::DeviceDriver vk(ctx.getPlatformInterface(), ctx.getInstance(), dev);
72 vk.getDeviceQueue(dev, ctx.getUniversalQueueFamilyIndex(), 0, &queue);
73
74 amber::EngineConfig* vkConfig = GetVulkanConfig(ctx.getInstance(),
75 ctx.getPhysicalDevice(), dev, &ctx.getDeviceFeatures(),
76 &ctx.getDeviceFeatures2(), ctx.getInstanceExtensions(),
77 ctx.getDeviceExtensions(), ctx.getUniversalQueueFamilyIndex(),
78 queue, ctx.getInstanceProcAddr());
79
80 return vkConfig;
81 }
82
83 // Returns true if the given feature is supported by the device.
84 // Throws an internal error If the feature is not recognized at all.
isFeatureSupported(const vkt::Context & ctx,const std::string & feature)85 static bool isFeatureSupported(const vkt::Context& ctx, const std::string& feature)
86 {
87 if (feature == "Storage16BitFeatures.storageBuffer16BitAccess")
88 return ctx.get16BitStorageFeatures().storageBuffer16BitAccess;
89 if (feature == "Float16Int8Features.shaderFloat16")
90 return ctx.getShaderFloat16Int8Features().shaderFloat16;
91 if (feature == "Float16Int8Features.shaderInt8")
92 return ctx.getShaderFloat16Int8Features().shaderInt8;
93 if (feature == "Features.shaderFloat64")
94 return ctx.getDeviceFeatures().shaderFloat64;
95 if (feature == "Features.shaderInt16")
96 return ctx.getDeviceFeatures().shaderInt16;
97 if (feature == "Features.shaderInt64")
98 return ctx.getDeviceFeatures().shaderInt64;
99 if (feature == "Features.depthClamp")
100 return ctx.getDeviceFeatures().depthClamp;
101 if (feature == "Features.tessellationShader")
102 return ctx.getDeviceFeatures().tessellationShader;
103 if (feature == "Features.shaderTessellationAndGeometryPointSize")
104 return ctx.getDeviceFeatures().shaderTessellationAndGeometryPointSize;
105 if (feature == "Features.geometryShader")
106 return ctx.getDeviceFeatures().geometryShader;
107 if (feature == "Features.fragmentStoresAndAtomics")
108 return ctx.getDeviceFeatures().fragmentStoresAndAtomics;
109 if (feature == "Features.vertexPipelineStoresAndAtomics")
110 return ctx.getDeviceFeatures().vertexPipelineStoresAndAtomics;
111 if (feature == "Features.fillModeNonSolid")
112 return ctx.getDeviceFeatures().fillModeNonSolid;
113 if (feature == "Features.shaderStorageImageMultisample")
114 return ctx.getDeviceFeatures().shaderStorageImageMultisample;
115 if (feature == "Features.sampleRateShading")
116 return ctx.getDeviceFeatures().sampleRateShading;
117 if (feature == "VariablePointerFeatures.variablePointersStorageBuffer")
118 return ctx.getVariablePointersFeatures().variablePointersStorageBuffer;
119 if (feature == "VariablePointerFeatures.variablePointers")
120 return ctx.getVariablePointersFeatures().variablePointers;
121 if (feature == "SubgroupSupportedStages.fragment")
122 return (ctx.getSubgroupProperties().supportedStages & vk::VK_SHADER_STAGE_FRAGMENT_BIT) != 0;
123 if (feature == "SubgroupSupportedOperations.vote")
124 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_VOTE_BIT) != 0;
125 if (feature == "SubgroupSupportedOperations.basic")
126 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_BASIC_BIT) != 0;
127 if (feature == "SubgroupSupportedOperations.ballot")
128 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_BALLOT_BIT) != 0;
129 if (feature == "Storage16BitFeatures.storageBuffer16BitAccess")
130 return ctx.get16BitStorageFeatures().storageBuffer16BitAccess;
131 if (feature == "Storage8BitFeatures.storageBuffer8BitAccess")
132 return ctx.get8BitStorageFeatures().storageBuffer8BitAccess;
133
134 std::string message = std::string("Unexpected feature name: ") + feature;
135 TCU_THROW(InternalError, message.c_str());
136 }
137
delayedInit(void)138 void AmberTestCase::delayedInit(void)
139 {
140 // Make sure the input can be parsed before we use it.
141 if (!parse(m_readFilename))
142 {
143 std::string message = "Failed to parse Amber file: " + m_readFilename;
144 TCU_THROW(InternalError, message.c_str());
145 }
146 }
147
checkSupport(Context & ctx) const148 void AmberTestCase::checkSupport(Context& ctx) const
149 {
150 // Check for instance and device extensions as declared by the test code.
151 if (m_required_extensions.size())
152 {
153 std::set<std::string> device_extensions(ctx.getDeviceExtensions().begin(),
154 ctx.getDeviceExtensions().end());
155 std::set<std::string> instance_extensions(ctx.getInstanceExtensions().begin(),
156 ctx.getInstanceExtensions().end());
157 std::string missing;
158 for (std::set<std::string>::iterator iter = m_required_extensions.begin();
159 iter != m_required_extensions.end();
160 ++iter)
161 {
162 const std::string extension = *iter;
163 if ((device_extensions.count(extension) == 0) &&
164 (instance_extensions.count(extension) == 0))
165 {
166 missing += " " + extension;
167 }
168 }
169 if (missing.size() > 0)
170 {
171 std::string message("Test requires unsupported extensions:");
172 message += missing;
173 TCU_THROW(NotSupportedError, message.c_str());
174 }
175 }
176
177 // Check for required features. Do this after extensions are checked because
178 // some feature checks are only valid when corresponding extensions are enabled.
179 if (m_required_features.size())
180 {
181 std::string missing;
182 for (std::set<std::string>::iterator iter = m_required_features.begin();
183 iter != m_required_features.end();
184 ++iter)
185 {
186 const std::string feature = *iter;
187 if (!isFeatureSupported(ctx, feature))
188 {
189 missing += " " + feature;
190 }
191 }
192 if (missing.size() > 0)
193 {
194 std::string message("Test requires unsupported features:");
195 message += missing;
196 TCU_THROW(NotSupportedError, message.c_str());
197 }
198 }
199
200 for (auto req : m_imageRequirements)
201 checkImageSupport(ctx.getInstanceInterface(), ctx.getPhysicalDevice(), req);
202
203 for (auto req : m_bufferRequirements)
204 {
205 vk::VkFormatProperties prop;
206 ctx.getInstanceInterface().getPhysicalDeviceFormatProperties(ctx.getPhysicalDevice(), req.m_format, &prop);
207
208 if ((req.m_featureFlags & prop.bufferFeatures) != req.m_featureFlags)
209 {
210 TCU_THROW(NotSupportedError, "Buffer format doesn't support required feature flags");
211 }
212 }
213
214 if (m_checkSupportCallback)
215 (m_checkSupportCallback)(ctx, m_name);
216 }
217
218 class Delegate : public amber::Delegate
219 {
220 public:
221 Delegate (tcu::TestContext& testCtx);
222
223 amber::Result LoadBufferData (const std::string file_name,
224 amber::BufferDataFileType file_type,
225 amber::BufferInfo* buffer) const override;
226
Log(const std::string &)227 void Log (const std::string& /*message*/) override { DE_FATAL("amber::Delegate::Log unimplemented"); }
LogGraphicsCalls(void) const228 bool LogGraphicsCalls (void) const override { return m_logGraphicsCalls; }
SetLogGraphicsCalls(bool log_graphics_calls)229 void SetLogGraphicsCalls (bool log_graphics_calls) { m_logGraphicsCalls = log_graphics_calls; }
LogExecuteCalls(void) const230 bool LogExecuteCalls (void) const override { return m_logExecuteCalls; }
SetLogExecuteCalls(bool log_execute_calls)231 void SetLogExecuteCalls (bool log_execute_calls) { m_logExecuteCalls = log_execute_calls; }
LogGraphicsCallsTime(void) const232 bool LogGraphicsCallsTime (void) const override { return m_logGraphicsCallsTime; }
SetLogGraphicsCallsTime(bool log_graphics_calls_time)233 void SetLogGraphicsCallsTime (bool log_graphics_calls_time) { m_logGraphicsCallsTime = log_graphics_calls_time; }
GetTimestampNs(void) const234 deUint64 GetTimestampNs (void) const override { DE_FATAL("amber::Delegate::GetTimestampNs unimplemented"); return 0; }
SetScriptPath(std::string path)235 void SetScriptPath (std::string path) { m_path = path; }
236
237 private:
238 tcu::TestContext& m_testCtx;
239 std::string m_path;
240 bool m_logGraphicsCalls;
241 bool m_logGraphicsCallsTime;
242 bool m_logExecuteCalls;
243 };
244
Delegate(tcu::TestContext & testCtx)245 Delegate::Delegate (tcu::TestContext& testCtx)
246 : m_testCtx (testCtx)
247 , m_path ("")
248 , m_logGraphicsCalls (false)
249 , m_logGraphicsCallsTime (false)
250 , m_logExecuteCalls (false)
251 {
252 }
253
LoadBufferData(const std::string file_name,amber::BufferDataFileType file_type,amber::BufferInfo * buffer) const254 amber::Result Delegate::LoadBufferData (const std::string file_name,
255 amber::BufferDataFileType file_type,
256 amber::BufferInfo* buffer) const
257 {
258 const tcu::Archive& archive = m_testCtx.getArchive();
259 const de::FilePath filePath = de::FilePath(m_path).join(file_name);
260 de::UniquePtr<tcu::Resource> file (archive.getResource(filePath.getPath()));
261 int numBytes = file->getSize();
262 std::vector<deUint8> bytes (numBytes);
263
264 if (file_type == amber::BufferDataFileType::kPng)
265 return amber::Result("Amber PNG loading unimplemented");
266
267 file->read(bytes.data(), numBytes);
268
269 if (bytes.empty())
270 return amber::Result("Failed to load buffer data " + file_name);
271
272 for (deUint8 byte : bytes)
273 {
274 amber::Value value;
275 value.SetIntValue(static_cast<deUint64>(byte));
276 buffer->values.push_back(value);
277 }
278
279 buffer->width = 1;
280 buffer->height = 1;
281
282 return {};
283 }
284
parse(const std::string & readFilename)285 bool AmberTestCase::parse (const std::string& readFilename)
286 {
287 std::string script = ShaderSourceProvider::getSource(m_testCtx.getArchive(), readFilename.c_str());
288 if (script.empty())
289 return false;
290
291 Delegate delegate (m_testCtx);
292 delegate.SetScriptPath(de::FilePath(readFilename).getDirName());
293
294 m_recipe = new amber::Recipe();
295
296 amber::Amber am (&delegate);
297 amber::Result r = am.Parse(script, m_recipe);
298
299 m_recipe->SetFenceTimeout(~0u); // infinity of miliseconds
300
301 if (!r.IsSuccess())
302 {
303 getTestContext().getLog()
304 << tcu::TestLog::Message
305 << "Failed to parse Amber test "
306 << readFilename
307 << ": "
308 << r.Error()
309 << "\n"
310 << tcu::TestLog::EndMessage;
311 // TODO(dneto): Enhance Amber to not require this.
312 m_recipe->SetImpl(DE_NULL);
313 return false;
314 }
315 return true;
316 }
317
initPrograms(vk::SourceCollections & programCollection) const318 void AmberTestCase::initPrograms (vk::SourceCollections& programCollection) const
319 {
320 std::vector<amber::ShaderInfo> shaders = m_recipe->GetShaderInfo();
321 for (size_t i = 0; i < shaders.size(); ++i)
322 {
323 const amber::ShaderInfo& shader = shaders[i];
324
325 vk::SpirvVersion spirvVersion = vk::SPIRV_VERSION_1_0;
326 DE_STATIC_ASSERT(vk::SPIRV_VERSION_LAST == vk::SPIRV_VERSION_1_6 + 1);
327 if (shader.target_env == "spv1.6")
328 spirvVersion = vk::SPIRV_VERSION_1_6;
329 else if (shader.target_env == "spv1.5")
330 spirvVersion = vk::SPIRV_VERSION_1_5;
331 else if (shader.target_env == "spv1.4")
332 spirvVersion = vk::SPIRV_VERSION_1_4;
333 else if (shader.target_env == "spv1.3")
334 spirvVersion = vk::SPIRV_VERSION_1_3;
335 else if (shader.target_env == "spv1.2")
336 spirvVersion = vk::SPIRV_VERSION_1_2;
337 else if (shader.target_env == "spv1.1")
338 spirvVersion = vk::SPIRV_VERSION_1_1;
339
340 /* Hex encoded shaders do not need to be pre-compiled */
341 if (shader.format == amber::kShaderFormatSpirvHex)
342 continue;
343
344 if (shader.format == amber::kShaderFormatSpirvAsm)
345 {
346 programCollection.spirvAsmSources.add(shader.shader_name) << shader.shader_source << m_asm_options;
347 }
348 else if (shader.format == amber::kShaderFormatGlsl)
349 {
350 bool allowSpirv14 = (spirvVersion == vk::SPIRV_VERSION_1_4);
351
352 switch (shader.type)
353 {
354 case amber::kShaderTypeCompute:
355 programCollection.glslSources.add(shader.shader_name)
356 << glu::ComputeSource(shader.shader_source)
357 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
358 break;
359 case amber::kShaderTypeGeometry:
360 programCollection.glslSources.add(shader.shader_name)
361 << glu::GeometrySource(shader.shader_source)
362 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
363 break;
364 case amber::kShaderTypeFragment:
365 programCollection.glslSources.add(shader.shader_name)
366 << glu::FragmentSource(shader.shader_source)
367 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
368 break;
369 case amber::kShaderTypeVertex:
370 programCollection.glslSources.add(shader.shader_name)
371 << glu::VertexSource(shader.shader_source)
372 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
373 break;
374 case amber::kShaderTypeTessellationControl:
375 programCollection.glslSources.add(shader.shader_name)
376 << glu::TessellationControlSource(shader.shader_source)
377 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
378 break;
379 case amber::kShaderTypeTessellationEvaluation:
380 programCollection.glslSources.add(shader.shader_name)
381 << glu::TessellationEvaluationSource(shader.shader_source)
382 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
383 break;
384 case amber::kShaderTypeMulti:
385 DE_ASSERT(false && "Multi shaders not supported");
386 break;
387 }
388 }
389 else
390 {
391 DE_ASSERT(false && "Shader format not supported");
392 }
393 }
394 }
395
iterate(void)396 tcu::TestStatus AmberTestInstance::iterate (void)
397 {
398 amber::Amber am (DE_NULL);
399 amber::Options amber_options;
400 amber::ShaderMap shaderMap;
401 amber::Result r;
402
403 amber_options.engine = amber::kEngineTypeVulkan;
404 amber_options.config = createEngineConfig(m_context, m_customDevice);
405 amber_options.execution_type = amber::ExecutionType::kExecute;
406
407 // Check for extensions as declared by the Amber script itself. Throw an internal
408 // error if that's more demanding.
409 r = am.AreAllRequirementsSupported(m_recipe, &amber_options);
410 if (!r.IsSuccess())
411 {
412 // dEQP does not to rely on external code to determine whether
413 // a test is supported. So throw an internal error here instead
414 // of a NotSupportedError. If an Amber test is not supported, then
415 // you must override this method and throw a NotSupported exception
416 // before reach here.
417 TCU_THROW(InternalError, r.Error().c_str());
418 }
419
420 std::vector<amber::ShaderInfo> shaders = m_recipe->GetShaderInfo();
421 for (size_t i = 0; i < shaders.size(); ++i)
422 {
423 const amber::ShaderInfo& shader = shaders[i];
424
425 if (!m_context.getBinaryCollection().contains(shader.shader_name))
426 continue;
427
428 size_t len = m_context.getBinaryCollection().get(shader.shader_name).getSize();
429 /* This is a compiled spir-v binary which must be made of 4-byte words. We
430 * are moving into a word sized vector so divide by 4
431 */
432 std::vector<deUint32> data;
433 data.resize(len >> 2);
434 deMemcpy(data.data(), m_context.getBinaryCollection().get(shader.shader_name).getBinary(), len);
435
436 shaderMap[shader.shader_name] = data;
437 }
438
439 r = am.ExecuteWithShaderData(m_recipe, &amber_options, shaderMap);
440 if (!r.IsSuccess()) {
441 m_context.getTestContext().getLog()
442 << tcu::TestLog::Message
443 << r.Error()
444 << "\n"
445 << tcu::TestLog::EndMessage;
446 }
447
448 delete amber_options.config;
449
450 return r.IsSuccess() ? tcu::TestStatus::pass("Pass") :tcu::TestStatus::fail("Fail");
451 }
452
setSpirVAsmBuildOptions(const vk::SpirVAsmBuildOptions & asm_options)453 void AmberTestCase::setSpirVAsmBuildOptions (const vk::SpirVAsmBuildOptions& asm_options)
454 {
455 m_asm_options = asm_options;
456 }
457
addRequirement(const std::string & requirement)458 void AmberTestCase::addRequirement (const std::string& requirement)
459 {
460 if (requirement.find(".") != std::string::npos)
461 m_required_features.insert(requirement);
462 else
463 m_required_extensions.insert(requirement);
464 }
465
addImageRequirement(vk::VkImageCreateInfo info)466 void AmberTestCase::addImageRequirement (vk::VkImageCreateInfo info)
467 {
468 m_imageRequirements.push_back(info);
469 }
470
addBufferRequirement(BufferRequirement req)471 void AmberTestCase::addBufferRequirement (BufferRequirement req)
472 {
473 m_bufferRequirements.push_back(req);
474 }
475
validateRequirements()476 bool AmberTestCase::validateRequirements()
477 {
478 if (!parse(m_readFilename))
479 {
480 std::string message = "Failed to parse Amber file: " + m_readFilename;
481 m_testCtx.getLog() << tcu::TestLog::Message << message << tcu::TestLog::EndMessage;
482 return false;
483 }
484
485 // Check if the list of required CTS features and extensions matches the
486 // one in the recipe. Throw InternalError if they do not match.
487
488 const auto& deviceExtensions = m_recipe->GetRequiredInstanceExtensions();
489 const auto& instanceExtensions = m_recipe->GetRequiredDeviceExtensions();
490 auto requiredFeatures = m_recipe->GetRequiredFeatures();
491
492 for (auto& req : requiredFeatures)
493 {
494 if (req.find(".") == std::string::npos)
495 req = "Features." + req;
496 }
497
498 std::set<std::string> allRequirements;
499 allRequirements.insert(begin(deviceExtensions), end(deviceExtensions));
500 allRequirements.insert(begin(instanceExtensions), end(instanceExtensions));
501 allRequirements.insert(begin(requiredFeatures), end(requiredFeatures));
502
503 std::set<std::string> ctsRequirements = m_required_features;
504 ctsRequirements.insert(begin(m_required_extensions), end(m_required_extensions));
505
506 if (allRequirements != ctsRequirements)
507 {
508 auto& log = m_testCtx.getLog();
509 log << tcu::TestLog::Message << "ERROR: CTS and Amber test requirement mismatch." << tcu::TestLog::EndMessage;
510 log << tcu::TestLog::Message << "Amber filename: " << m_readFilename << tcu::TestLog::EndMessage;
511 log << tcu::TestLog::Message << "CTS requirements:" << tcu::TestLog::EndMessage;
512 for (const auto& ctsReq : ctsRequirements)
513 log << tcu::TestLog::Message << " " << ctsReq << tcu::TestLog::EndMessage;
514
515 log << tcu::TestLog::Message << "Amber requirements:" << tcu::TestLog::EndMessage;
516 for (const auto& amberReq : allRequirements)
517 log << tcu::TestLog::Message << " " << amberReq << tcu::TestLog::EndMessage;
518
519 // Repeat message for cerr so it's visible in console log.
520 std::cerr << "ERROR: CTS and Amber test requirement mismatch.\n";
521 std::cerr << "Amber filename: " << m_readFilename << "\n";
522 std::cerr << "CTS requirements:\n";
523 for (const auto& ctsReq : ctsRequirements)
524 std::cerr << " " << ctsReq << "\n";
525
526 std::cerr << "Amber requirements:\n";
527 for (const auto& amberReq : allRequirements)
528 std::cerr << " " << amberReq << "\n";
529
530 return false;
531 }
532 return true;
533 }
534
535 } // cts_amber
536 } // vkt
537