1 /*------------------------------------------------------------------------
2 * Vulkan Conformance Tests
3 * ------------------------
4 *
5 * Copyright (c) 2019 Google LLC
6 * Copyright (c) 2019 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 *//*!
21 * \file
22 * \brief Functional tests using amber
23 *//*--------------------------------------------------------------------*/
24
25 #include <amber/amber.h>
26 #include "amber/recipe.h"
27
28 #include <iostream>
29
30 #include "deDefs.hpp"
31 #include "deUniquePtr.hpp"
32 #include "deFilePath.hpp"
33 #include "vktTestCaseUtil.hpp"
34 #include "tcuTestLog.hpp"
35 #include "vktAmberTestCase.hpp"
36 #include "vktAmberHelper.hpp"
37 #include "tcuResource.hpp"
38 #include "tcuTestLog.hpp"
39 #include "vkSpirVProgram.hpp"
40 #include "vkImageUtil.hpp"
41
42 namespace vkt
43 {
44 namespace cts_amber
45 {
46
AmberTestCase(tcu::TestContext & testCtx,const char * name,const char * description,const std::string & readFilename)47 AmberTestCase::AmberTestCase (tcu::TestContext& testCtx,
48 const char* name,
49 const char* description,
50 const std::string& readFilename)
51 : TestCase(testCtx, name, description),
52 m_recipe(DE_NULL),
53 m_readFilename(readFilename)
54 {
55 }
56
~AmberTestCase(void)57 AmberTestCase::~AmberTestCase (void)
58 {
59 delete m_recipe;
60 }
61
createInstance(Context & ctx) const62 TestInstance* AmberTestCase::createInstance (Context& ctx) const
63 {
64 return new AmberTestInstance(ctx, m_recipe);
65 }
66
createEngineConfig(Context & ctx)67 static amber::EngineConfig* createEngineConfig (Context& ctx)
68 {
69 amber::EngineConfig* vkConfig = GetVulkanConfig(ctx.getInstance(),
70 ctx.getPhysicalDevice(), ctx.getDevice(), &ctx.getDeviceFeatures(),
71 &ctx.getDeviceFeatures2(), ctx.getInstanceExtensions(),
72 ctx.getDeviceExtensions(), ctx.getUniversalQueueFamilyIndex(),
73 ctx.getUniversalQueue(), ctx.getInstanceProcAddr());
74
75 return vkConfig;
76 }
77
78 // Returns true if the given feature is supported by the device.
79 // Throws an internal error If the feature is not recognized at all.
isFeatureSupported(const vkt::Context & ctx,const std::string & feature)80 static bool isFeatureSupported(const vkt::Context& ctx, const std::string& feature)
81 {
82 if (feature == "Storage16BitFeatures.storageBuffer16BitAccess")
83 return ctx.get16BitStorageFeatures().storageBuffer16BitAccess;
84 if (feature == "Float16Int8Features.shaderFloat16")
85 return ctx.getShaderFloat16Int8Features().shaderFloat16;
86 if (feature == "Features.shaderFloat64")
87 return ctx.getDeviceFeatures().shaderFloat64;
88 if (feature == "Features.shaderInt16")
89 return ctx.getDeviceFeatures().shaderInt16;
90 if (feature == "Features.shaderInt64")
91 return ctx.getDeviceFeatures().shaderInt64;
92 if (feature == "Features.tessellationShader")
93 return ctx.getDeviceFeatures().tessellationShader;
94 if (feature == "Features.geometryShader")
95 return ctx.getDeviceFeatures().geometryShader;
96 if (feature == "Features.fragmentStoresAndAtomics")
97 return ctx.getDeviceFeatures().fragmentStoresAndAtomics;
98 if (feature == "Features.vertexPipelineStoresAndAtomics")
99 return ctx.getDeviceFeatures().vertexPipelineStoresAndAtomics;
100 if (feature == "Features.fillModeNonSolid")
101 return ctx.getDeviceFeatures().fillModeNonSolid;
102 if (feature == "Features.shaderStorageImageMultisample")
103 return ctx.getDeviceFeatures().shaderStorageImageMultisample;
104 if (feature == "Features.sampleRateShading")
105 return ctx.getDeviceFeatures().sampleRateShading;
106 if (feature == "VariablePointerFeatures.variablePointersStorageBuffer")
107 return ctx.getVariablePointersFeatures().variablePointersStorageBuffer;
108 if (feature == "VariablePointerFeatures.variablePointers")
109 return ctx.getVariablePointersFeatures().variablePointers;
110 if (feature == "SubgroupSupportedStages.fragment")
111 return (ctx.getSubgroupProperties().supportedStages & vk::VK_SHADER_STAGE_FRAGMENT_BIT) != 0;
112 if (feature == "SubgroupSupportedOperations.vote")
113 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_VOTE_BIT) != 0;
114 if (feature == "SubgroupSupportedOperations.ballot")
115 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_BALLOT_BIT) != 0;
116 if (feature == "Storage16BitFeatures.storageBuffer16BitAccess")
117 return ctx.get16BitStorageFeatures().storageBuffer16BitAccess;
118
119 std::string message = std::string("Unexpected feature name: ") + feature;
120 TCU_THROW(InternalError, message.c_str());
121 }
122
delayedInit(void)123 void AmberTestCase::delayedInit(void)
124 {
125 // Make sure the input can be parsed before we use it.
126 if (!parse(m_readFilename))
127 {
128 std::string message = "Failed to parse Amber file: " + m_readFilename;
129 TCU_THROW(InternalError, message.c_str());
130 }
131 }
132
checkSupport(Context & ctx) const133 void AmberTestCase::checkSupport(Context& ctx) const
134 {
135 // Check for instance and device extensions as declared by the test code.
136 if (m_required_extensions.size())
137 {
138 std::set<std::string> device_extensions(ctx.getDeviceExtensions().begin(),
139 ctx.getDeviceExtensions().end());
140 std::set<std::string> instance_extensions(ctx.getInstanceExtensions().begin(),
141 ctx.getInstanceExtensions().end());
142 std::string missing;
143 for (std::set<std::string>::iterator iter = m_required_extensions.begin();
144 iter != m_required_extensions.end();
145 ++iter)
146 {
147 const std::string extension = *iter;
148 if ((device_extensions.count(extension) == 0) &&
149 (instance_extensions.count(extension) == 0))
150 {
151 missing += " " + extension;
152 }
153 }
154 if (missing.size() > 0)
155 {
156 std::string message("Test requires unsupported extensions:");
157 message += missing;
158 TCU_THROW(NotSupportedError, message.c_str());
159 }
160 }
161
162 // Check for required features. Do this after extensions are checked because
163 // some feature checks are only valid when corresponding extensions are enabled.
164 if (m_required_features.size())
165 {
166 std::string missing;
167 for (std::set<std::string>::iterator iter = m_required_features.begin();
168 iter != m_required_features.end();
169 ++iter)
170 {
171 const std::string feature = *iter;
172 if (!isFeatureSupported(ctx, feature))
173 {
174 missing += " " + feature;
175 }
176 }
177 if (missing.size() > 0)
178 {
179 std::string message("Test requires unsupported features:");
180 message += missing;
181 TCU_THROW(NotSupportedError, message.c_str());
182 }
183 }
184
185 for (auto req : m_imageRequirements)
186 checkImageSupport(ctx.getInstanceInterface(), ctx.getPhysicalDevice(), req);
187
188 for (auto req : m_bufferRequirements)
189 {
190 vk::VkFormatProperties prop;
191 ctx.getInstanceInterface().getPhysicalDeviceFormatProperties(ctx.getPhysicalDevice(), req.m_format, &prop);
192
193 if ((req.m_featureFlags & prop.bufferFeatures) != req.m_featureFlags)
194 {
195 TCU_THROW(NotSupportedError, "Buffer format doesn't support required feature flags");
196 }
197 }
198
199 // when checkSupport is called script is not yet parsed so we need to determine
200 // unsupported tests by name ; in AmberTestCase we do not have access to actual
201 // m_recipe implementation - we can't scan it to see if test can be executed;
202 // alternatively portability extension and its features could be checked in amber.cc
203 if (ctx.isDeviceFunctionalitySupported("VK_KHR_portability_subset"))
204 {
205 if (m_name == "triangle_fan" && !ctx.getPortabilitySubsetFeatures().triangleFans)
206 TCU_THROW(NotSupportedError, "VK_KHR_portability_subset: Triangle fans are not supported by this implementation");
207
208 if (ctx.getPortabilitySubsetProperties().minVertexInputBindingStrideAlignment == 4)
209 {
210 const std::set<std::string> casesToSkip
211 {
212 "line-strip",
213 "polygon-mode-lines",
214 "r8g8-uint-highp",
215 "r8g8-uint-highp-output-uint",
216 "r8g8-uint-mediump",
217 "r8g8-uint-mediump-output-uint",
218 "inputs-outputs-mod",
219 };
220
221 if (casesToSkip.count(m_name))
222 TCU_THROW(NotSupportedError, "VK_KHR_portability_subset: Stride is not multiply of minVertexInputBindingStrideAlignment");
223 }
224 }
225 }
226
227 class Delegate : public amber::Delegate
228 {
229 public:
230 Delegate (tcu::TestContext& testCtx);
231
232 amber::Result LoadBufferData (const std::string file_name,
233 amber::BufferDataFileType file_type,
234 amber::BufferInfo* buffer) const override;
235
Log(const std::string &)236 void Log (const std::string& /*message*/) override { DE_FATAL("amber::Delegate::Log unimplemented"); }
LogGraphicsCalls(void) const237 bool LogGraphicsCalls (void) const override { return m_logGraphicsCalls; }
SetLogGraphicsCalls(bool log_graphics_calls)238 void SetLogGraphicsCalls (bool log_graphics_calls) { m_logGraphicsCalls = log_graphics_calls; }
LogExecuteCalls(void) const239 bool LogExecuteCalls (void) const override { return m_logExecuteCalls; }
SetLogExecuteCalls(bool log_execute_calls)240 void SetLogExecuteCalls (bool log_execute_calls) { m_logExecuteCalls = log_execute_calls; }
LogGraphicsCallsTime(void) const241 bool LogGraphicsCallsTime (void) const override { return m_logGraphicsCallsTime; }
SetLogGraphicsCallsTime(bool log_graphics_calls_time)242 void SetLogGraphicsCallsTime (bool log_graphics_calls_time) { m_logGraphicsCallsTime = log_graphics_calls_time; }
GetTimestampNs(void) const243 deUint64 GetTimestampNs (void) const override { DE_FATAL("amber::Delegate::GetTimestampNs unimplemented"); return 0; }
SetScriptPath(std::string path)244 void SetScriptPath (std::string path) { m_path = path; }
245
246 private:
247 tcu::TestContext& m_testCtx;
248 std::string m_path;
249 bool m_logGraphicsCalls;
250 bool m_logGraphicsCallsTime;
251 bool m_logExecuteCalls;
252 };
253
Delegate(tcu::TestContext & testCtx)254 Delegate::Delegate (tcu::TestContext& testCtx)
255 : m_testCtx (testCtx)
256 , m_path ("")
257 , m_logGraphicsCalls (false)
258 , m_logGraphicsCallsTime (false)
259 , m_logExecuteCalls (false)
260 {
261 }
262
LoadBufferData(const std::string file_name,amber::BufferDataFileType file_type,amber::BufferInfo * buffer) const263 amber::Result Delegate::LoadBufferData (const std::string file_name,
264 amber::BufferDataFileType file_type,
265 amber::BufferInfo* buffer) const
266 {
267 const tcu::Archive& archive = m_testCtx.getArchive();
268 const de::FilePath filePath = de::FilePath(m_path).join(file_name);
269 de::UniquePtr<tcu::Resource> file (archive.getResource(filePath.getPath()));
270 int numBytes = file->getSize();
271 std::vector<deUint8> bytes (numBytes);
272
273 if (file_type == amber::BufferDataFileType::kPng)
274 return amber::Result("Amber PNG loading unimplemented");
275
276 file->read(bytes.data(), numBytes);
277
278 if (bytes.empty())
279 return amber::Result("Failed to load buffer data " + file_name);
280
281 for (deUint8 byte : bytes)
282 {
283 amber::Value value;
284 value.SetIntValue(static_cast<deUint64>(byte));
285 buffer->values.push_back(value);
286 }
287
288 buffer->width = 1;
289 buffer->height = 1;
290
291 return {};
292 }
293
parse(const std::string & readFilename)294 bool AmberTestCase::parse (const std::string& readFilename)
295 {
296 std::string script = ShaderSourceProvider::getSource(m_testCtx.getArchive(), readFilename.c_str());
297 if (script.empty())
298 return false;
299
300 Delegate delegate (m_testCtx);
301 delegate.SetScriptPath(de::FilePath(readFilename).getDirName());
302
303 m_recipe = new amber::Recipe();
304
305 amber::Amber am (&delegate);
306 amber::Result r = am.Parse(script, m_recipe);
307
308 m_recipe->SetFenceTimeout(~0u); // infinity of miliseconds
309
310 if (!r.IsSuccess())
311 {
312 getTestContext().getLog()
313 << tcu::TestLog::Message
314 << "Failed to parse Amber test "
315 << readFilename
316 << ": "
317 << r.Error()
318 << "\n"
319 << tcu::TestLog::EndMessage;
320 // TODO(dneto): Enhance Amber to not require this.
321 m_recipe->SetImpl(DE_NULL);
322 return false;
323 }
324 return true;
325 }
326
initPrograms(vk::SourceCollections & programCollection) const327 void AmberTestCase::initPrograms (vk::SourceCollections& programCollection) const
328 {
329 std::vector<amber::ShaderInfo> shaders = m_recipe->GetShaderInfo();
330 for (size_t i = 0; i < shaders.size(); ++i)
331 {
332 const amber::ShaderInfo& shader = shaders[i];
333
334 vk::SpirvVersion spirvVersion = vk::SPIRV_VERSION_1_0;
335 DE_STATIC_ASSERT(vk::SPIRV_VERSION_LAST == vk::SPIRV_VERSION_1_5 + 1);
336 if (shader.target_env == "spv1.5")
337 spirvVersion = vk::SPIRV_VERSION_1_5;
338 else if (shader.target_env == "spv1.4")
339 spirvVersion = vk::SPIRV_VERSION_1_4;
340 else if (shader.target_env == "spv1.3")
341 spirvVersion = vk::SPIRV_VERSION_1_3;
342 else if (shader.target_env == "spv1.2")
343 spirvVersion = vk::SPIRV_VERSION_1_2;
344 else if (shader.target_env == "spv1.1")
345 spirvVersion = vk::SPIRV_VERSION_1_1;
346
347 /* Hex encoded shaders do not need to be pre-compiled */
348 if (shader.format == amber::kShaderFormatSpirvHex)
349 continue;
350
351 if (shader.format == amber::kShaderFormatSpirvAsm)
352 {
353 programCollection.spirvAsmSources.add(shader.shader_name) << shader.shader_source << m_asm_options;
354 }
355 else if (shader.format == amber::kShaderFormatGlsl)
356 {
357 bool allowSpirv14 = (spirvVersion == vk::SPIRV_VERSION_1_4);
358
359 switch (shader.type)
360 {
361 case amber::kShaderTypeCompute:
362 programCollection.glslSources.add(shader.shader_name)
363 << glu::ComputeSource(shader.shader_source)
364 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
365 break;
366 case amber::kShaderTypeGeometry:
367 programCollection.glslSources.add(shader.shader_name)
368 << glu::GeometrySource(shader.shader_source)
369 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
370 break;
371 case amber::kShaderTypeFragment:
372 programCollection.glslSources.add(shader.shader_name)
373 << glu::FragmentSource(shader.shader_source)
374 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
375 break;
376 case amber::kShaderTypeVertex:
377 programCollection.glslSources.add(shader.shader_name)
378 << glu::VertexSource(shader.shader_source)
379 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
380 break;
381 case amber::kShaderTypeTessellationControl:
382 programCollection.glslSources.add(shader.shader_name)
383 << glu::TessellationControlSource(shader.shader_source)
384 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
385 break;
386 case amber::kShaderTypeTessellationEvaluation:
387 programCollection.glslSources.add(shader.shader_name)
388 << glu::TessellationEvaluationSource(shader.shader_source)
389 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
390 break;
391 case amber::kShaderTypeMulti:
392 DE_ASSERT(false && "Multi shaders not supported");
393 break;
394 }
395 }
396 else
397 {
398 DE_ASSERT(false && "Shader format not supported");
399 }
400 }
401 }
402
iterate(void)403 tcu::TestStatus AmberTestInstance::iterate (void)
404 {
405 amber::Amber am (DE_NULL);
406 amber::Options amber_options;
407 amber::ShaderMap shaderMap;
408 amber::Result r;
409
410 amber_options.engine = amber::kEngineTypeVulkan;
411 amber_options.config = createEngineConfig(m_context);
412 amber_options.execution_type = amber::ExecutionType::kExecute;
413
414 // Check for extensions as declared by the Amber script itself. Throw an internal
415 // error if that's more demanding.
416 r = am.AreAllRequirementsSupported(m_recipe, &amber_options);
417 if (!r.IsSuccess())
418 {
419 // dEQP does not to rely on external code to determine whether
420 // a test is supported. So throw an internal error here instead
421 // of a NotSupportedError. If an Amber test is not supported, then
422 // you must override this method and throw a NotSupported exception
423 // before reach here.
424 TCU_THROW(InternalError, r.Error().c_str());
425 }
426
427 std::vector<amber::ShaderInfo> shaders = m_recipe->GetShaderInfo();
428 for (size_t i = 0; i < shaders.size(); ++i)
429 {
430 const amber::ShaderInfo& shader = shaders[i];
431
432 if (!m_context.getBinaryCollection().contains(shader.shader_name))
433 continue;
434
435 size_t len = m_context.getBinaryCollection().get(shader.shader_name).getSize();
436 /* This is a compiled spir-v binary which must be made of 4-byte words. We
437 * are moving into a word sized vector so divide by 4
438 */
439 std::vector<deUint32> data;
440 data.resize(len >> 2);
441 deMemcpy(data.data(), m_context.getBinaryCollection().get(shader.shader_name).getBinary(), len);
442
443 shaderMap[shader.shader_name] = data;
444 }
445
446 r = am.ExecuteWithShaderData(m_recipe, &amber_options, shaderMap);
447 if (!r.IsSuccess()) {
448 m_context.getTestContext().getLog()
449 << tcu::TestLog::Message
450 << r.Error()
451 << "\n"
452 << tcu::TestLog::EndMessage;
453 }
454
455 delete amber_options.config;
456
457 return r.IsSuccess() ? tcu::TestStatus::pass("Pass") :tcu::TestStatus::fail("Fail");
458 }
459
setSpirVAsmBuildOptions(const vk::SpirVAsmBuildOptions & asm_options)460 void AmberTestCase::setSpirVAsmBuildOptions (const vk::SpirVAsmBuildOptions& asm_options)
461 {
462 m_asm_options = asm_options;
463 }
464
addRequirement(const std::string & requirement)465 void AmberTestCase::addRequirement (const std::string& requirement)
466 {
467 if (requirement.find(".") != std::string::npos)
468 m_required_features.insert(requirement);
469 else
470 m_required_extensions.insert(requirement);
471 }
472
addImageRequirement(vk::VkImageCreateInfo info)473 void AmberTestCase::addImageRequirement (vk::VkImageCreateInfo info)
474 {
475 m_imageRequirements.push_back(info);
476 }
477
addBufferRequirement(BufferRequirement req)478 void AmberTestCase::addBufferRequirement (BufferRequirement req)
479 {
480 m_bufferRequirements.push_back(req);
481 }
482
validateRequirements()483 bool AmberTestCase::validateRequirements()
484 {
485 if (!parse(m_readFilename))
486 {
487 std::string message = "Failed to parse Amber file: " + m_readFilename;
488 m_testCtx.getLog() << tcu::TestLog::Message << message << tcu::TestLog::EndMessage;
489 return false;
490 }
491
492 // Check if the list of required CTS features and extensions matches the
493 // one in the recipe. Throw InternalError if they do not match.
494
495 const auto& deviceExtensions = m_recipe->GetRequiredInstanceExtensions();
496 const auto& instanceExtensions = m_recipe->GetRequiredDeviceExtensions();
497 auto requiredFeatures = m_recipe->GetRequiredFeatures();
498
499 for (auto& req : requiredFeatures)
500 {
501 if (req.find(".") == std::string::npos)
502 req = "Features." + req;
503 }
504
505 std::set<std::string> allRequirements;
506 allRequirements.insert(begin(deviceExtensions), end(deviceExtensions));
507 allRequirements.insert(begin(instanceExtensions), end(instanceExtensions));
508 allRequirements.insert(begin(requiredFeatures), end(requiredFeatures));
509
510 std::set<std::string> ctsRequirements = m_required_features;
511 ctsRequirements.insert(begin(m_required_extensions), end(m_required_extensions));
512
513 if (allRequirements != ctsRequirements)
514 {
515 auto& log = m_testCtx.getLog();
516 log << tcu::TestLog::Message << "ERROR: CTS and Amber test requirement mismatch." << tcu::TestLog::EndMessage;
517 log << tcu::TestLog::Message << "Amber filename: " << m_readFilename << tcu::TestLog::EndMessage;
518 log << tcu::TestLog::Message << "CTS requirements:" << tcu::TestLog::EndMessage;
519 for (const auto& ctsReq : ctsRequirements)
520 log << tcu::TestLog::Message << " " << ctsReq << tcu::TestLog::EndMessage;
521
522 log << tcu::TestLog::Message << "Amber requirements:" << tcu::TestLog::EndMessage;
523 for (const auto& amberReq : allRequirements)
524 log << tcu::TestLog::Message << " " << amberReq << tcu::TestLog::EndMessage;
525
526 return false;
527 }
528 return true;
529 }
530
531 } // cts_amber
532 } // vkt
533