1 /*-------------------------------------------------------------------------
2 * Vulkan CTS Framework
3 * --------------------
4 *
5 * Copyright (c) 2021 The Khronos Group Inc.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *-------------------------------------------------------------------------*/
20
21 #include <iostream>
22 #include <fstream>
23 #include <sstream>
24 #include <json/json.h>
25 #include "deCommandLine.hpp"
26 #include "deDirectoryIterator.hpp"
27 #include "tcuCommandLine.hpp"
28 #include "tcuPlatform.hpp"
29 #include "tcuTestContext.hpp"
30 #include "tcuResource.hpp"
31 #include "tcuTestLog.hpp"
32 #include "vkPlatform.hpp"
33 #include "vktTestCase.hpp"
34 #include "vksStructsVKSC.hpp"
35 #include "vksCacheBuilder.hpp"
36
37 namespace opt
38 {
39
40 DE_DECLARE_COMMAND_LINE_OPT(CompilerDataPath, std::string);
41 DE_DECLARE_COMMAND_LINE_OPT(CompilerOutputFile, std::string);
42 DE_DECLARE_COMMAND_LINE_OPT(LogFile, std::string);
43 DE_DECLARE_COMMAND_LINE_OPT(FilePrefix, std::string);
44
registerOptions(de::cmdline::Parser & parser)45 void registerOptions(de::cmdline::Parser &parser)
46 {
47 using de::cmdline::NamedValue;
48 using de::cmdline::Option;
49
50 parser << Option<CompilerDataPath>("p", "path", "Offline pipeline data directory", "");
51 parser << Option<CompilerOutputFile>("o", "out", "Output file with pipeline cache", "");
52 parser << Option<LogFile>("l", "log", "Log file", "dummy.log");
53 parser << Option<FilePrefix>("x", "prefix", "Prefix for input files", "");
54 }
55
56 } // namespace opt
57
58 enum PipelineType
59 {
60 PT_UNDEFINED_PIPELINE = 0,
61 PT_GRAPHICS_PIPELINE,
62 PT_COMPUTE_PIPELINE,
63 };
64
importFilesForExternalCompiler(vksc_server::VulkanPipelineCacheInput & input,const std::string & path,const std::string & filePrefix)65 void importFilesForExternalCompiler(vksc_server::VulkanPipelineCacheInput &input, const std::string &path,
66 const std::string &filePrefix)
67 {
68 vksc_server::json::Context context;
69
70 for (de::DirectoryIterator iter(path); iter.hasItem(); iter.next())
71 {
72 const de::FilePath filePath = iter.getItem();
73 if (filePath.getType() != de::FilePath::TYPE_FILE)
74 continue;
75 if (filePath.getFileExtension() != "json")
76 continue;
77 if (!filePrefix.empty() && filePath.getBaseName().find(filePrefix) != 0)
78 continue;
79
80 std::string fileContents;
81 {
82 std::ifstream file(filePath.getPath());
83 std::stringstream buffer;
84 buffer << file.rdbuf();
85 fileContents = buffer.str();
86 }
87
88 Json::Value jsonRoot;
89 std::string errors;
90 bool parsingSuccessful =
91 context.reader->parse(fileContents.c_str(), fileContents.c_str() + fileContents.size(), &jsonRoot, &errors);
92 if (!parsingSuccessful)
93 TCU_THROW(InternalError,
94 (std::string("JSON parsing error. File ") + filePath.getPath() + " Error : " + errors).c_str());
95
96 // decide what pipeline type will be created later
97 PipelineType pipelineType = PT_UNDEFINED_PIPELINE;
98 if (jsonRoot.isMember("GraphicsPipelineState"))
99 pipelineType = PT_GRAPHICS_PIPELINE;
100 else if (jsonRoot.isMember("ComputePipelineState"))
101 pipelineType = PT_COMPUTE_PIPELINE;
102 if (pipelineType == PT_UNDEFINED_PIPELINE)
103 TCU_THROW(InternalError, (std::string("JSON - unknown pipeline. File ") + filePath.getPath()).c_str());
104
105 const Json::Value &jsonGraphicsPipelineState = jsonRoot["GraphicsPipelineState"];
106 const Json::Value &jsonComputePipelineState = jsonRoot["ComputePipelineState"];
107 const Json::Value &jsonPipelineState =
108 (pipelineType == PT_GRAPHICS_PIPELINE) ? jsonGraphicsPipelineState : jsonComputePipelineState;
109 vksc_server::VulkanJsonPipelineDescription pipelineDescription;
110
111 {
112 const Json::Value &jsonSamplerYcbcrConversions = jsonPipelineState["YcbcrSamplers"];
113 if (!jsonSamplerYcbcrConversions.isNull())
114 {
115 for (Json::ArrayIndex i = 0; i < jsonSamplerYcbcrConversions.size(); ++i)
116 {
117 const Json::Value::Members membersNames = jsonSamplerYcbcrConversions[i].getMemberNames();
118 const Json::Value &value = jsonSamplerYcbcrConversions[i][membersNames[0]];
119 uint64_t index;
120 std::istringstream(membersNames[0]) >> index;
121 input.samplerYcbcrConversions[vk::VkSamplerYcbcrConversion(reinterpret_cast<void *>(index))] =
122 std::string(fileContents.begin() + value.getOffsetStart(),
123 fileContents.begin() + value.getOffsetLimit());
124 }
125 }
126
127 const Json::Value &jsonSamplers = jsonPipelineState["ImmutableSamplers"];
128 if (!jsonSamplers.isNull())
129 {
130 for (Json::ArrayIndex i = 0; i < jsonSamplers.size(); ++i)
131 {
132 const Json::Value::Members membersNames = jsonSamplers[i].getMemberNames();
133 const Json::Value &value = jsonSamplers[i][membersNames[0]];
134 uint64_t index;
135 std::istringstream(membersNames[0]) >> index;
136 input.samplers[vk::VkSampler(reinterpret_cast<void *>(index))] = std::string(
137 fileContents.begin() + value.getOffsetStart(), fileContents.begin() + value.getOffsetLimit());
138 }
139 }
140
141 const Json::Value &jsonDescriptorSetLayouts = jsonPipelineState["DescriptorSetLayouts"];
142 if (!jsonDescriptorSetLayouts.isNull())
143 {
144 for (Json::ArrayIndex i = 0; i < jsonDescriptorSetLayouts.size(); ++i)
145 {
146 const Json::Value::Members membersNames = jsonDescriptorSetLayouts[i].getMemberNames();
147 const Json::Value &value = jsonDescriptorSetLayouts[i][membersNames[0]];
148 uint64_t index;
149 std::istringstream(membersNames[0]) >> index;
150 input.descriptorSetLayouts[vk::VkDescriptorSetLayout(reinterpret_cast<void *>(index))] =
151 std::string(fileContents.begin() + value.getOffsetStart(),
152 fileContents.begin() + value.getOffsetLimit());
153 }
154 }
155
156 uint64_t pipelineLayoutHandle = 0u;
157 uint64_t renderPassHandle = 0u;
158 std::map<std::string, uint64_t> stages;
159
160 const Json::Value &jsonComputePipeline = jsonPipelineState["ComputePipeline"];
161 if (!jsonComputePipeline.isNull())
162 {
163 pipelineDescription.pipelineContents =
164 std::string(fileContents.begin() + jsonComputePipeline.getOffsetStart(),
165 fileContents.begin() + jsonComputePipeline.getOffsetLimit());
166 pipelineLayoutHandle = jsonComputePipeline["layout"].asUInt64();
167
168 const Json::Value &jsonStage = jsonComputePipeline["stage"];
169 stages[jsonStage["stage"].asString()] = jsonStage["module"].asUInt64();
170 }
171
172 const Json::Value &jsonGraphicsPipeline = jsonPipelineState["GraphicsPipeline"];
173 if (!jsonGraphicsPipeline.isNull())
174 {
175 pipelineDescription.pipelineContents =
176 std::string(fileContents.begin() + jsonGraphicsPipeline.getOffsetStart(),
177 fileContents.begin() + jsonGraphicsPipeline.getOffsetLimit());
178 pipelineLayoutHandle = jsonGraphicsPipeline["layout"].asUInt64();
179 renderPassHandle = jsonGraphicsPipeline["renderPass"].asUInt64();
180
181 const Json::Value &jsonStages = jsonGraphicsPipeline["pStages"];
182 for (Json::ArrayIndex i = 0; i < jsonStages.size(); ++i)
183 stages[jsonStages[i]["stage"].asString()] = jsonStages[i]["module"].asUInt64();
184 }
185
186 const Json::Value &jsonPipelineLayout = jsonPipelineState["PipelineLayout"];
187 if (!jsonPipelineLayout.isNull() && pipelineLayoutHandle != 0u)
188 {
189 input.pipelineLayouts[vk::VkPipelineLayout(reinterpret_cast<void *>(pipelineLayoutHandle))] =
190 std::string(fileContents.begin() + jsonPipelineLayout.getOffsetStart(),
191 fileContents.begin() + jsonPipelineLayout.getOffsetLimit());
192 }
193
194 const Json::Value &jsonRenderPass = jsonPipelineState["Renderpass"];
195 if (!jsonRenderPass.isNull() && renderPassHandle != 0u)
196 {
197 input.renderPasses[vk::VkRenderPass(reinterpret_cast<void *>(renderPassHandle))] =
198 std::string(fileContents.begin() + jsonRenderPass.getOffsetStart(),
199 fileContents.begin() + jsonRenderPass.getOffsetLimit());
200 }
201
202 const Json::Value &jsonRenderPass2 = jsonPipelineState["Renderpass2"];
203 if (!jsonRenderPass2.isNull() && renderPassHandle != 0u)
204 {
205 input.renderPasses[vk::VkRenderPass(reinterpret_cast<void *>(renderPassHandle))] =
206 std::string(fileContents.begin() + jsonRenderPass.getOffsetStart(),
207 fileContents.begin() + jsonRenderPass.getOffsetLimit());
208 }
209
210 const Json::Value &jsonShaderFileNames = jsonPipelineState["ShaderFileNames"];
211 if (!jsonShaderFileNames.isNull())
212 {
213 for (Json::ArrayIndex i = 0; i < jsonShaderFileNames.size(); ++i)
214 {
215 std::string stageName = jsonShaderFileNames[i]["stage"].asString();
216 std::string fileName = jsonShaderFileNames[i]["filename"].asString();
217 auto it = stages.find(stageName);
218 if (it == end(stages))
219 TCU_THROW(InternalError,
220 (std::string("JSON - missing shader stage. File ") + filePath.getPath()).c_str());
221
222 de::FilePath shaderPath(path);
223 shaderPath.join(de::FilePath(fileName));
224 std::ifstream iFile(shaderPath.getPath(), std::ios::in | std::ios::binary);
225 if (!iFile)
226 TCU_THROW(InternalError, (std::string("JSON - missing shader file ") + fileName + ". File " +
227 filePath.getPath())
228 .c_str());
229
230 auto fileBegin = iFile.tellg();
231 iFile.seekg(0, std::ios::end);
232 auto fileEnd = iFile.tellg();
233 iFile.seekg(0, std::ios::beg);
234 std::size_t fileSize = static_cast<std::size_t>(fileEnd - fileBegin);
235 std::vector<uint8_t> shaderData(fileSize);
236
237 iFile.read(reinterpret_cast<char *>(shaderData.data()), fileSize);
238 if (iFile.fail())
239 TCU_THROW(InternalError, (std::string("JSON - error reading shader file ") + fileName +
240 ". File " + filePath.getPath())
241 .c_str());
242
243 vk::VkShaderModuleCreateInfo smCI{
244 VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, // VkStructureType sType;
245 nullptr, // const void* pNext;
246 vk::VkShaderModuleCreateFlags(0u), // VkShaderModuleCreateFlags flags;
247 fileSize, // uintptr_t codeSize;
248 reinterpret_cast<uint32_t *>(shaderData.data()) // const uint32_t* pCode;
249 };
250
251 input.shaderModules[vk::VkShaderModule(reinterpret_cast<void *>(it->second))] =
252 vksc_server::json::writeJSON_VkShaderModuleCreateInfo(smCI);
253 }
254 }
255
256 const Json::Value &jsonPhysicalDeviceFeatures = jsonPipelineState["PhysicalDeviceFeatures"];
257 if (!jsonPhysicalDeviceFeatures.isNull())
258 {
259 pipelineDescription.deviceFeatures =
260 std::string(fileContents.begin() + jsonPhysicalDeviceFeatures.getOffsetStart(),
261 fileContents.begin() + jsonPhysicalDeviceFeatures.getOffsetLimit());
262 }
263 }
264
265 const Json::Value &jsonEnabledExtensions = jsonRoot["EnabledExtensions"];
266 if (!jsonEnabledExtensions.isNull())
267 {
268 for (Json::ArrayIndex i = 0; i < jsonEnabledExtensions.size(); ++i)
269 pipelineDescription.deviceExtensions.push_back(jsonEnabledExtensions[i].asString());
270 }
271
272 const Json::Value &jsonPipelineUUID = jsonRoot["PipelineUUID"];
273 if (!jsonPipelineUUID.isNull())
274 {
275 pipelineDescription.id.sType = VK_STRUCTURE_TYPE_PIPELINE_OFFLINE_CREATE_INFO;
276 pipelineDescription.id.pNext = nullptr;
277 for (Json::ArrayIndex i = 0; i < jsonPipelineUUID.size(); ++i)
278 pipelineDescription.id.pipelineIdentifier[i] = uint8_t(jsonPipelineUUID[i].asUInt());
279 pipelineDescription.id.matchControl = VK_PIPELINE_MATCH_CONTROL_APPLICATION_UUID_EXACT_MATCH;
280 pipelineDescription.id.poolEntrySize = 0u;
281 }
282 input.pipelines.push_back(pipelineDescription);
283 }
284 }
285
286 tcu::Platform *createPlatform(void);
287
main(int argc,char ** argv)288 int main(int argc, char **argv)
289 {
290 de::cmdline::CommandLine cmdLine;
291
292 // Parse command line.
293 {
294 de::cmdline::Parser parser;
295 opt::registerOptions(parser);
296
297 if (!parser.parse(argc, argv, &cmdLine, std::cerr))
298 {
299 parser.help(std::cout);
300 return EXIT_FAILURE;
301 }
302 }
303
304 try
305 {
306 // load JSON files into VulkanPipelineCacheInput
307 vksc_server::VulkanPipelineCacheInput input;
308 importFilesForExternalCompiler(input, cmdLine.getOption<opt::CompilerDataPath>(),
309 cmdLine.getOption<opt::FilePrefix>());
310
311 // create Vulkan instance
312 tcu::CommandLine cmdLineDummy{"--deqp-vk-device-id=0"};
313 tcu::DirArchive archive{""};
314 tcu::TestLog log{cmdLine.getOption<opt::LogFile>().c_str()};
315 log.supressLogging(true);
316 de::SharedPtr<tcu::Platform> platform{createPlatform()};
317 #ifdef DE_PLATFORM_USE_LIBRARY_TYPE
318 de::SharedPtr<vk::Library> library{
319 platform->getVulkanPlatform().createLibrary(vk::Platform::LIBRARY_TYPE_VULKAN, nullptr)};
320 #else
321 de::SharedPtr<vk::Library> library{platform->getVulkanPlatform().createLibrary(nullptr)};
322 #endif
323 tcu::TestContext tcx{*platform, archive, log, cmdLineDummy, nullptr};
324 vk::BinaryCollection collection{};
325 vkt::Context context(tcx, library->getPlatformInterface(), collection,
326 de::SharedPtr<vk::ResourceInterface>{new vk::ResourceInterfaceStandard{tcx}});
327
328 // create pipeline cache
329 std::vector<uint8_t> binary = vksc_server::buildPipelineCache(
330 input, library->getPlatformInterface(), context.getInstance(), context.getInstanceInterface(),
331 context.getPhysicalDevice(), context.getUniversalQueueFamilyIndex());
332
333 // write pipeline cache to output file
334 std::ofstream oFile(cmdLine.getOption<opt::CompilerOutputFile>().c_str(), std::ios::out | std::ios::binary);
335 if (!oFile)
336 TCU_THROW(InternalError,
337 (std::string("Cannot create file : ") + cmdLine.getOption<opt::CompilerOutputFile>().c_str()));
338 oFile.write(reinterpret_cast<char *>(binary.data()), binary.size());
339 }
340 catch (const std::exception &e)
341 {
342 std::cout << e.what() << std::endl;
343 }
344
345 return EXIT_SUCCESS;
346 }
347