• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2016 Google, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <array>
18 
19 #include <glm/gtc/type_ptr.hpp>
20 #include <glm/gtc/matrix_transform.hpp>
21 
22 #include "Helpers.h"
23 #include "Smoke.h"
24 #include "Meshes.h"
25 #include "Shell.h"
26 
27 namespace {
28 
29 // TODO do not rely on compiler to use std140 layout
30 // TODO move lower frequency data to another descriptor set
31 struct ShaderParamBlock {
32     float light_pos[4];
33     float light_color[4];
34     float model[4 * 4];
35     float view_projection[4 * 4];
36 };
37 
38 } // namespace
39 
Smoke(const std::vector<std::string> & args)40 Smoke::Smoke(const std::vector<std::string> &args)
41     : Game("Smoke", args), multithread_(true), use_push_constants_(false),
42       sim_paused_(false), sim_(5000), camera_(2.5f), frame_data_(),
43       render_pass_clear_value_({{ 0.0f, 0.1f, 0.2f, 1.0f }}),
44       render_pass_begin_info_(),
45       primary_cmd_begin_info_(), primary_cmd_submit_info_()
46 {
47     for (auto it = args.begin(); it != args.end(); ++it) {
48         if (*it == "-s")
49             multithread_ = false;
50         else if (*it == "-p")
51             use_push_constants_ = true;
52     }
53 
54     init_workers();
55 }
56 
~Smoke()57 Smoke::~Smoke()
58 {
59 }
60 
init_workers()61 void Smoke::init_workers()
62 {
63     int worker_count = std::thread::hardware_concurrency();
64 
65     // not enough cores
66     if (!multithread_ || worker_count < 2) {
67         multithread_ = false;
68         worker_count = 1;
69     }
70 
71     const int object_per_worker =
72         static_cast<int>(sim_.objects().size()) / worker_count;
73     int object_begin = 0, object_end = 0;
74 
75     workers_.reserve(worker_count);
76     for (int i = 0; i < worker_count; i++) {
77         object_begin = object_end;
78         if (i < worker_count - 1)
79             object_end += object_per_worker;
80         else
81             object_end = static_cast<int>(sim_.objects().size());
82 
83         Worker *worker = new Worker(*this, i, object_begin, object_end);
84         workers_.emplace_back(std::unique_ptr<Worker>(worker));
85     }
86 }
87 
attach_shell(Shell & sh)88 void Smoke::attach_shell(Shell &sh)
89 {
90     Game::attach_shell(sh);
91 
92     const Shell::Context &ctx = sh.context();
93     physical_dev_ = ctx.physical_dev;
94     dev_ = ctx.dev;
95     queue_ = ctx.game_queue;
96     queue_family_ = ctx.game_queue_family;
97     format_ = ctx.format.format;
98 
99     vk::GetPhysicalDeviceProperties(physical_dev_, &physical_dev_props_);
100 
101     if (use_push_constants_ &&
102         sizeof(ShaderParamBlock) > physical_dev_props_.limits.maxPushConstantsSize) {
103         shell_->log(Shell::LOG_WARN, "cannot enable push constants");
104         use_push_constants_ = false;
105     }
106 
107     VkPhysicalDeviceMemoryProperties mem_props;
108     vk::GetPhysicalDeviceMemoryProperties(physical_dev_, &mem_props);
109     mem_flags_.reserve(mem_props.memoryTypeCount);
110     for (uint32_t i = 0; i < mem_props.memoryTypeCount; i++)
111         mem_flags_.push_back(mem_props.memoryTypes[i].propertyFlags);
112 
113     meshes_ = new Meshes(dev_, mem_flags_);
114 
115     create_render_pass();
116     create_shader_modules();
117     create_descriptor_set_layout();
118     create_pipeline_layout();
119     create_pipeline();
120 
121     create_frame_data(2);
122 
123     render_pass_begin_info_.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
124     render_pass_begin_info_.renderPass = render_pass_;
125     render_pass_begin_info_.clearValueCount = 1;
126     render_pass_begin_info_.pClearValues = &render_pass_clear_value_;
127 
128     primary_cmd_begin_info_.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
129     primary_cmd_begin_info_.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
130 
131     // we will render to the swapchain images
132     primary_cmd_submit_wait_stages_ = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
133 
134     primary_cmd_submit_info_.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
135     primary_cmd_submit_info_.waitSemaphoreCount = 1;
136     primary_cmd_submit_info_.pWaitDstStageMask = &primary_cmd_submit_wait_stages_;
137     primary_cmd_submit_info_.commandBufferCount = 1;
138     primary_cmd_submit_info_.signalSemaphoreCount = 1;
139 
140     if (multithread_) {
141         for (auto &worker : workers_)
142             worker->start();
143     }
144 }
145 
detach_shell()146 void Smoke::detach_shell()
147 {
148     if (multithread_) {
149         for (auto &worker : workers_)
150             worker->stop();
151     }
152 
153     destroy_frame_data();
154 
155     vk::DestroyPipeline(dev_, pipeline_, nullptr);
156     vk::DestroyPipelineLayout(dev_, pipeline_layout_, nullptr);
157     if (!use_push_constants_)
158         vk::DestroyDescriptorSetLayout(dev_, desc_set_layout_, nullptr);
159     vk::DestroyShaderModule(dev_, fs_, nullptr);
160     vk::DestroyShaderModule(dev_, vs_, nullptr);
161     vk::DestroyRenderPass(dev_, render_pass_, nullptr);
162 
163     delete meshes_;
164 
165     Game::detach_shell();
166 }
167 
create_render_pass()168 void Smoke::create_render_pass()
169 {
170     VkAttachmentDescription attachment = {};
171     attachment.format = format_;
172     attachment.samples = VK_SAMPLE_COUNT_1_BIT;
173     attachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
174     attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
175     attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
176     attachment.finalLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
177 
178     VkAttachmentReference attachment_ref = {};
179     attachment_ref.attachment = 0;
180     attachment_ref.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
181 
182     VkSubpassDescription subpass = {};
183     subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
184     subpass.colorAttachmentCount = 1;
185     subpass.pColorAttachments = &attachment_ref;
186 
187     std::array<VkSubpassDependency, 2> subpass_deps;
188     subpass_deps[0].srcSubpass = VK_SUBPASS_EXTERNAL;
189     subpass_deps[0].dstSubpass = 0;
190     subpass_deps[0].srcStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
191     subpass_deps[0].dstStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
192     subpass_deps[0].srcAccessMask = VK_ACCESS_MEMORY_READ_BIT;
193     subpass_deps[0].dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
194                                     VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
195     subpass_deps[0].dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
196 
197     subpass_deps[1].srcSubpass = 0;
198     subpass_deps[1].dstSubpass = VK_SUBPASS_EXTERNAL;
199     subpass_deps[1].srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
200     subpass_deps[1].dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
201     subpass_deps[1].srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
202                                     VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
203     subpass_deps[1].dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
204     subpass_deps[1].dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
205 
206     VkRenderPassCreateInfo render_pass_info = {};
207     render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO;
208     render_pass_info.attachmentCount = 1;
209     render_pass_info.pAttachments = &attachment;
210     render_pass_info.subpassCount = 1;
211     render_pass_info.pSubpasses = &subpass;
212     render_pass_info.dependencyCount = (uint32_t)subpass_deps.size();
213     render_pass_info.pDependencies = subpass_deps.data();
214 
215     vk::assert_success(vk::CreateRenderPass(dev_, &render_pass_info, nullptr, &render_pass_));
216 }
217 
create_shader_modules()218 void Smoke::create_shader_modules()
219 {
220     VkShaderModuleCreateInfo sh_info = {};
221     sh_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO;
222     if (use_push_constants_) {
223 #include "Smoke.push_constant.vert.h"
224         sh_info.codeSize = sizeof(Smoke_push_constant_vert);
225         sh_info.pCode = Smoke_push_constant_vert;
226     } else {
227 #include "Smoke.vert.h"
228         sh_info.codeSize = sizeof(Smoke_vert);
229         sh_info.pCode = Smoke_vert;
230     }
231     vk::assert_success(vk::CreateShaderModule(dev_, &sh_info, nullptr, &vs_));
232 
233 #include "Smoke.frag.h"
234     sh_info.codeSize = sizeof(Smoke_frag);
235     sh_info.pCode = Smoke_frag;
236     vk::assert_success(vk::CreateShaderModule(dev_, &sh_info, nullptr, &fs_));
237 }
238 
create_descriptor_set_layout()239 void Smoke::create_descriptor_set_layout()
240 {
241     if (use_push_constants_)
242         return;
243 
244     VkDescriptorSetLayoutBinding layout_binding = {};
245     layout_binding.binding = 0;
246     layout_binding.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC;
247     layout_binding.descriptorCount = 1;
248     layout_binding.stageFlags = VK_SHADER_STAGE_VERTEX_BIT;
249 
250     VkDescriptorSetLayoutCreateInfo layout_info = {};
251     layout_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
252     layout_info.bindingCount = 1;
253     layout_info.pBindings = &layout_binding;
254 
255     vk::assert_success(vk::CreateDescriptorSetLayout(dev_, &layout_info,
256                 nullptr, &desc_set_layout_));
257 }
258 
create_pipeline_layout()259 void Smoke::create_pipeline_layout()
260 {
261     VkPushConstantRange push_const_range = {};
262 
263     VkPipelineLayoutCreateInfo pipeline_layout_info = {};
264     pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;
265 
266     if (use_push_constants_) {
267         push_const_range.stageFlags = VK_SHADER_STAGE_VERTEX_BIT;
268         push_const_range.offset = 0;
269         push_const_range.size = sizeof(ShaderParamBlock);
270 
271         pipeline_layout_info.pushConstantRangeCount = 1;
272         pipeline_layout_info.pPushConstantRanges = &push_const_range;
273     } else {
274         pipeline_layout_info.setLayoutCount = 1;
275         pipeline_layout_info.pSetLayouts = &desc_set_layout_;
276     }
277 
278     vk::assert_success(vk::CreatePipelineLayout(dev_, &pipeline_layout_info,
279                 nullptr, &pipeline_layout_));
280 }
281 
create_pipeline()282 void Smoke::create_pipeline()
283 {
284     VkPipelineShaderStageCreateInfo stage_info[2] = {};
285     stage_info[0].sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
286     stage_info[0].stage = VK_SHADER_STAGE_VERTEX_BIT;
287     stage_info[0].module = vs_;
288     stage_info[0].pName = "main";
289     stage_info[1].sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
290     stage_info[1].stage = VK_SHADER_STAGE_FRAGMENT_BIT;
291     stage_info[1].module = fs_;
292     stage_info[1].pName = "main";
293 
294     VkPipelineViewportStateCreateInfo viewport_info = {};
295     viewport_info.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO;
296     // both dynamic
297     viewport_info.viewportCount = 1;
298     viewport_info.scissorCount = 1;
299 
300     VkPipelineRasterizationStateCreateInfo rast_info = {};
301     rast_info.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO;
302     rast_info.depthClampEnable = false;
303     rast_info.rasterizerDiscardEnable = false;
304     rast_info.polygonMode = VK_POLYGON_MODE_FILL;
305     rast_info.cullMode = VK_CULL_MODE_NONE;
306     rast_info.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
307     rast_info.depthBiasEnable = false;
308     rast_info.lineWidth = 1.0f;
309 
310     VkPipelineMultisampleStateCreateInfo multisample_info = {};
311     multisample_info.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO;
312     multisample_info.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;
313     multisample_info.sampleShadingEnable = false;
314     multisample_info.pSampleMask = nullptr;
315     multisample_info.alphaToCoverageEnable = false;
316     multisample_info.alphaToOneEnable = false;
317 
318     VkPipelineColorBlendAttachmentState blend_attachment = {};
319     blend_attachment.blendEnable = true;
320     blend_attachment.srcColorBlendFactor = VK_BLEND_FACTOR_SRC_ALPHA;
321     blend_attachment.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
322     blend_attachment.colorBlendOp = VK_BLEND_OP_ADD;
323     blend_attachment.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
324     blend_attachment.dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO;
325     blend_attachment.alphaBlendOp = VK_BLEND_OP_ADD;
326     blend_attachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT |
327                                       VK_COLOR_COMPONENT_G_BIT |
328                                       VK_COLOR_COMPONENT_B_BIT |
329                                       VK_COLOR_COMPONENT_A_BIT;
330 
331     VkPipelineColorBlendStateCreateInfo blend_info = {};
332     blend_info.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO;
333     blend_info.logicOpEnable = false;
334     blend_info.attachmentCount = 1;
335     blend_info.pAttachments = &blend_attachment;
336 
337     std::array<VkDynamicState, 2> dynamic_states = {
338         VK_DYNAMIC_STATE_VIEWPORT,
339         VK_DYNAMIC_STATE_SCISSOR
340     };
341     struct VkPipelineDynamicStateCreateInfo dynamic_info = {};
342     dynamic_info.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO;
343     dynamic_info.dynamicStateCount = (uint32_t)dynamic_states.size();
344     dynamic_info.pDynamicStates = dynamic_states.data();
345 
346     VkGraphicsPipelineCreateInfo pipeline_info = {};
347     pipeline_info.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO;
348     pipeline_info.stageCount = 2;
349     pipeline_info.pStages = stage_info;
350     pipeline_info.pVertexInputState = &meshes_->vertex_input_state();
351     pipeline_info.pInputAssemblyState = &meshes_->input_assembly_state();
352     pipeline_info.pTessellationState = nullptr;
353     pipeline_info.pViewportState = &viewport_info;
354     pipeline_info.pRasterizationState = &rast_info;
355     pipeline_info.pMultisampleState = &multisample_info;
356     pipeline_info.pDepthStencilState = nullptr;
357     pipeline_info.pColorBlendState = &blend_info;
358     pipeline_info.pDynamicState = &dynamic_info;
359     pipeline_info.layout = pipeline_layout_;
360     pipeline_info.renderPass = render_pass_;
361     pipeline_info.subpass = 0;
362     vk::assert_success(vk::CreateGraphicsPipelines(dev_, VK_NULL_HANDLE, 1, &pipeline_info, nullptr, &pipeline_));
363 }
364 
create_frame_data(int count)365 void Smoke::create_frame_data(int count)
366 {
367     frame_data_.resize(count);
368 
369     create_fences();
370     create_command_buffers();
371 
372     if (!use_push_constants_) {
373         create_buffers();
374         create_buffer_memory();
375         create_descriptor_sets();
376     }
377 
378     frame_data_index_ = 0;
379 }
380 
destroy_frame_data()381 void Smoke::destroy_frame_data()
382 {
383     if (!use_push_constants_) {
384         vk::DestroyDescriptorPool(dev_, desc_pool_, nullptr);
385 
386         vk::UnmapMemory(dev_, frame_data_mem_);
387         vk::FreeMemory(dev_, frame_data_mem_, nullptr);
388 
389         for (auto &data : frame_data_)
390             vk::DestroyBuffer(dev_, data.buf, nullptr);
391     }
392 
393     for (auto cmd_pool : worker_cmd_pools_)
394         vk::DestroyCommandPool(dev_, cmd_pool, nullptr);
395     worker_cmd_pools_.clear();
396     vk::DestroyCommandPool(dev_, primary_cmd_pool_, nullptr);
397 
398     for (auto &data : frame_data_)
399         vk::DestroyFence(dev_, data.fence, nullptr);
400 
401     frame_data_.clear();
402 }
403 
create_fences()404 void Smoke::create_fences()
405 {
406     VkFenceCreateInfo fence_info = {};
407     fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
408     fence_info.flags = VK_FENCE_CREATE_SIGNALED_BIT;
409 
410     for (auto &data : frame_data_)
411         vk::assert_success(vk::CreateFence(dev_, &fence_info, nullptr, &data.fence));
412 }
413 
create_command_buffers()414 void Smoke::create_command_buffers()
415 {
416     VkCommandPoolCreateInfo cmd_pool_info = {};
417     cmd_pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
418     cmd_pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
419     cmd_pool_info.queueFamilyIndex = queue_family_;
420 
421     VkCommandBufferAllocateInfo cmd_info = {};
422     cmd_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
423     cmd_info.commandBufferCount = static_cast<uint32_t>(frame_data_.size());
424 
425     // create command pools and buffers
426     std::vector<VkCommandPool> cmd_pools(workers_.size() + 1, VK_NULL_HANDLE);
427     std::vector<std::vector<VkCommandBuffer>> cmds_vec(workers_.size() + 1,
428             std::vector<VkCommandBuffer>(frame_data_.size(), VK_NULL_HANDLE));
429     for (size_t i = 0; i < cmd_pools.size(); i++) {
430         auto &cmd_pool = cmd_pools[i];
431         auto &cmds = cmds_vec[i];
432 
433         vk::assert_success(vk::CreateCommandPool(dev_, &cmd_pool_info,
434                     nullptr, &cmd_pool));
435 
436         cmd_info.commandPool = cmd_pool;
437         cmd_info.level = (cmd_pool == cmd_pools.back()) ?
438             VK_COMMAND_BUFFER_LEVEL_PRIMARY : VK_COMMAND_BUFFER_LEVEL_SECONDARY;
439 
440         vk::assert_success(vk::AllocateCommandBuffers(dev_, &cmd_info, cmds.data()));
441     }
442 
443     // update frame_data_
444     for (size_t i = 0; i < frame_data_.size(); i++) {
445         for (const auto &cmds : cmds_vec) {
446             if (cmds == cmds_vec.back()) {
447                 frame_data_[i].primary_cmd = cmds[i];
448             } else {
449                 frame_data_[i].worker_cmds.push_back(cmds[i]);
450             }
451         }
452     }
453 
454     primary_cmd_pool_ = cmd_pools.back();
455     cmd_pools.pop_back();
456     worker_cmd_pools_ = cmd_pools;
457 }
458 
create_buffers()459 void Smoke::create_buffers()
460 {
461     VkDeviceSize object_data_size = sizeof(ShaderParamBlock);
462     // align object data to device limit
463     const VkDeviceSize &alignment =
464         physical_dev_props_.limits.minStorageBufferOffsetAlignment;
465     if (object_data_size % alignment)
466         object_data_size += alignment - (object_data_size % alignment);
467 
468     // update simulation
469     sim_.set_frame_data_size(static_cast<uint32_t>(object_data_size));
470 
471     VkBufferCreateInfo buf_info = {};
472     buf_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
473     buf_info.size = object_data_size * sim_.objects().size();
474     buf_info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
475     buf_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
476 
477     for (auto &data : frame_data_)
478         vk::assert_success(vk::CreateBuffer(dev_, &buf_info, nullptr, &data.buf));
479 }
480 
create_buffer_memory()481 void Smoke::create_buffer_memory()
482 {
483     VkMemoryRequirements mem_reqs;
484     vk::GetBufferMemoryRequirements(dev_, frame_data_[0].buf, &mem_reqs);
485 
486     VkDeviceSize aligned_size = mem_reqs.size;
487     if (aligned_size % mem_reqs.alignment)
488         aligned_size += mem_reqs.alignment - (aligned_size % mem_reqs.alignment);
489 
490     // allocate memory
491     VkMemoryAllocateInfo mem_info = {};
492     mem_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
493     mem_info.allocationSize = aligned_size * (frame_data_.size() - 1) +
494         mem_reqs.size;
495 
496     for (uint32_t idx = 0; idx < mem_flags_.size(); idx++) {
497         if ((mem_reqs.memoryTypeBits & (1 << idx)) &&
498             (mem_flags_[idx] & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) &&
499             (mem_flags_[idx] & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
500             // TODO is this guaranteed to exist?
501             mem_info.memoryTypeIndex = idx;
502             break;
503         }
504     }
505 
506     vk::AllocateMemory(dev_, &mem_info, nullptr, &frame_data_mem_);
507 
508     void *ptr;
509     vk::MapMemory(dev_, frame_data_mem_, 0, VK_WHOLE_SIZE, 0, &ptr);
510 
511     VkDeviceSize offset = 0;
512     for (auto &data : frame_data_) {
513         vk::BindBufferMemory(dev_, data.buf, frame_data_mem_, offset);
514         data.base = reinterpret_cast<uint8_t *>(ptr) + offset;
515         offset += aligned_size;
516     }
517 }
518 
create_descriptor_sets()519 void Smoke::create_descriptor_sets()
520 {
521     VkDescriptorPoolSize desc_pool_size = {};
522     desc_pool_size.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC;
523     desc_pool_size.descriptorCount = static_cast<uint32_t>(frame_data_.size());
524 
525     VkDescriptorPoolCreateInfo desc_pool_info = {};
526     desc_pool_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
527     desc_pool_info.maxSets = static_cast<uint32_t>(frame_data_.size());
528     desc_pool_info.poolSizeCount = 1;
529     desc_pool_info.pPoolSizes = &desc_pool_size;
530 
531     // create descriptor pool
532     vk::assert_success(vk::CreateDescriptorPool(dev_, &desc_pool_info,
533                 nullptr, &desc_pool_));
534 
535     std::vector<VkDescriptorSetLayout> set_layouts(frame_data_.size(), desc_set_layout_);
536     VkDescriptorSetAllocateInfo set_info = {};
537     set_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
538     set_info.descriptorPool = desc_pool_;
539     set_info.descriptorSetCount = static_cast<uint32_t>(set_layouts.size());
540     set_info.pSetLayouts = set_layouts.data();
541 
542     // create descriptor sets
543     std::vector<VkDescriptorSet> desc_sets(frame_data_.size(), VK_NULL_HANDLE);
544     vk::assert_success(vk::AllocateDescriptorSets(dev_, &set_info, desc_sets.data()));
545 
546     std::vector<VkDescriptorBufferInfo> desc_bufs(frame_data_.size());
547     std::vector<VkWriteDescriptorSet> desc_writes(frame_data_.size());
548 
549     for (size_t i = 0; i < frame_data_.size(); i++) {
550         auto &data = frame_data_[i];
551 
552         data.desc_set = desc_sets[i];
553 
554         VkDescriptorBufferInfo desc_buf = {};
555         desc_buf.buffer = data.buf;
556         desc_buf.offset = 0;
557         desc_buf.range = VK_WHOLE_SIZE;
558         desc_bufs[i] = desc_buf;
559 
560         VkWriteDescriptorSet desc_write = {};
561         desc_write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
562         desc_write.dstSet = data.desc_set;
563         desc_write.dstBinding = 0;
564         desc_write.dstArrayElement = 0;
565         desc_write.descriptorCount = 1;
566         desc_write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC;
567         desc_write.pBufferInfo = &desc_bufs[i];
568         desc_writes[i] = desc_write;
569     }
570 
571     vk::UpdateDescriptorSets(dev_,
572             static_cast<uint32_t>(desc_writes.size()),
573             desc_writes.data(), 0, nullptr);
574 }
575 
attach_swapchain()576 void Smoke::attach_swapchain()
577 {
578     const Shell::Context &ctx = shell_->context();
579 
580     prepare_viewport(ctx.extent);
581     prepare_framebuffers(ctx.swapchain);
582 
583     update_camera();
584 }
585 
detach_swapchain()586 void Smoke::detach_swapchain()
587 {
588     for (auto fb : framebuffers_)
589         vk::DestroyFramebuffer(dev_, fb, nullptr);
590     for (auto view : image_views_)
591         vk::DestroyImageView(dev_, view, nullptr);
592 
593     framebuffers_.clear();
594     image_views_.clear();
595     images_.clear();
596 }
597 
prepare_viewport(const VkExtent2D & extent)598 void Smoke::prepare_viewport(const VkExtent2D &extent)
599 {
600     extent_ = extent;
601 
602     viewport_.x = 0.0f;
603     viewport_.y = 0.0f;
604     viewport_.width = static_cast<float>(extent.width);
605     viewport_.height = static_cast<float>(extent.height);
606     viewport_.minDepth = 0.0f;
607     viewport_.maxDepth = 1.0f;
608 
609     scissor_.offset = { 0, 0 };
610     scissor_.extent = extent_;
611 }
612 
prepare_framebuffers(VkSwapchainKHR swapchain)613 void Smoke::prepare_framebuffers(VkSwapchainKHR swapchain)
614 {
615     // get swapchain images
616     vk::get(dev_, swapchain, images_);
617 
618     assert(framebuffers_.empty());
619     image_views_.reserve(images_.size());
620     framebuffers_.reserve(images_.size());
621     for (auto img : images_) {
622         VkImageViewCreateInfo view_info = {};
623         view_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
624         view_info.image = img;
625         view_info.viewType = VK_IMAGE_VIEW_TYPE_2D;
626         view_info.format = format_;
627         view_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
628         view_info.subresourceRange.levelCount = 1;
629         view_info.subresourceRange.layerCount = 1;
630 
631         VkImageView view;
632         vk::assert_success(vk::CreateImageView(dev_, &view_info, nullptr, &view));
633         image_views_.push_back(view);
634 
635         VkFramebufferCreateInfo fb_info = {};
636         fb_info.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO;
637         fb_info.renderPass = render_pass_;
638         fb_info.attachmentCount = 1;
639         fb_info.pAttachments = &view;
640         fb_info.width = extent_.width;
641         fb_info.height = extent_.height;
642         fb_info.layers = 1;
643 
644         VkFramebuffer fb;
645         vk::assert_success(vk::CreateFramebuffer(dev_, &fb_info, nullptr, &fb));
646         framebuffers_.push_back(fb);
647     }
648 }
649 
update_camera()650 void Smoke::update_camera()
651 {
652     const glm::vec3 center(0.0f);
653     const glm::vec3 up(0.f, 0.0f, 1.0f);
654     const glm::mat4 view = glm::lookAt(camera_.eye_pos, center, up);
655 
656     float aspect = static_cast<float>(extent_.width) / static_cast<float>(extent_.height);
657     const glm::mat4 projection = glm::perspective(0.4f, aspect, 0.1f, 100.0f);
658 
659     // Vulkan clip space has inverted Y and half Z.
660     const glm::mat4 clip(1.0f,  0.0f, 0.0f, 0.0f,
661                          0.0f, -1.0f, 0.0f, 0.0f,
662                          0.0f,  0.0f, 0.5f, 0.0f,
663                          0.0f,  0.0f, 0.5f, 1.0f);
664 
665     camera_.view_projection = clip * projection * view;
666 }
667 
draw_object(const Simulation::Object & obj,FrameData & data,VkCommandBuffer cmd) const668 void Smoke::draw_object(const Simulation::Object &obj, FrameData &data, VkCommandBuffer cmd) const
669 {
670     if (use_push_constants_) {
671         ShaderParamBlock params;
672         memcpy(params.light_pos, glm::value_ptr(obj.light_pos), sizeof(obj.light_pos));
673         memcpy(params.light_color, glm::value_ptr(obj.light_color), sizeof(obj.light_color));
674         memcpy(params.model, glm::value_ptr(obj.model), sizeof(obj.model));
675         memcpy(params.view_projection, glm::value_ptr(camera_.view_projection), sizeof(camera_.view_projection));
676 
677         vk::CmdPushConstants(cmd, pipeline_layout_, VK_SHADER_STAGE_VERTEX_BIT,
678                 0, sizeof(params), &params);
679     } else {
680         ShaderParamBlock *params =
681             reinterpret_cast<ShaderParamBlock *>(data.base + obj.frame_data_offset);
682         memcpy(params->light_pos, glm::value_ptr(obj.light_pos), sizeof(obj.light_pos));
683         memcpy(params->light_color, glm::value_ptr(obj.light_color), sizeof(obj.light_color));
684         memcpy(params->model, glm::value_ptr(obj.model), sizeof(obj.model));
685         memcpy(params->view_projection, glm::value_ptr(camera_.view_projection), sizeof(camera_.view_projection));
686 
687         vk::CmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS,
688                 pipeline_layout_, 0, 1, &data.desc_set, 1, &obj.frame_data_offset);
689     }
690 
691     meshes_->cmd_draw(cmd, obj.mesh);
692 }
693 
update_simulation(const Worker & worker)694 void Smoke::update_simulation(const Worker &worker)
695 {
696     sim_.update(worker.tick_interval_, worker.object_begin_, worker.object_end_);
697 }
698 
draw_objects(Worker & worker)699 void Smoke::draw_objects(Worker &worker)
700 {
701     auto &data = frame_data_[frame_data_index_];
702     auto cmd = data.worker_cmds[worker.index_];
703 
704     VkCommandBufferInheritanceInfo inherit_info = {};
705     inherit_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
706     inherit_info.renderPass = render_pass_;
707     inherit_info.framebuffer = worker.fb_;
708 
709     VkCommandBufferBeginInfo begin_info = {};
710     begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
711     begin_info.flags = VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT;
712     begin_info.pInheritanceInfo = &inherit_info;
713 
714     vk::BeginCommandBuffer(cmd, &begin_info);
715 
716     vk::CmdSetViewport(cmd, 0, 1, &viewport_);
717     vk::CmdSetScissor(cmd, 0, 1, &scissor_);
718 
719     vk::CmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_);
720 
721     meshes_->cmd_bind_buffers(cmd);
722 
723     for (int i = worker.object_begin_; i < worker.object_end_; i++) {
724         auto &obj = sim_.objects()[i];
725 
726         draw_object(obj, data, cmd);
727     }
728 
729     vk::EndCommandBuffer(cmd);
730 }
731 
on_key(Key key)732 void Smoke::on_key(Key key)
733 {
734     switch (key) {
735     case KEY_SHUTDOWN:
736     case KEY_ESC:
737         shell_->quit();
738         break;
739     case KEY_UP:
740         camera_.eye_pos -= glm::vec3(0.05f);
741         update_camera();
742         break;
743     case KEY_DOWN:
744         camera_.eye_pos += glm::vec3(0.05f);
745         update_camera();
746         break;
747     case KEY_SPACE:
748         sim_paused_ = !sim_paused_;
749         break;
750     default:
751         break;
752     }
753 }
754 
on_tick()755 void Smoke::on_tick()
756 {
757     if (sim_paused_)
758         return;
759 
760     for (auto &worker : workers_)
761         worker->update_simulation();
762 }
763 
on_frame(float frame_pred)764 void Smoke::on_frame(float frame_pred)
765 {
766     auto &data = frame_data_[frame_data_index_];
767 
768     // wait for the last submission since we reuse frame data
769     vk::assert_success(vk::WaitForFences(dev_, 1, &data.fence, true, UINT64_MAX));
770     vk::assert_success(vk::ResetFences(dev_, 1, &data.fence));
771 
772     const Shell::BackBuffer &back = shell_->context().acquired_back_buffer;
773 
774     // ignore frame_pred
775     for (auto &worker : workers_)
776         worker->draw_objects(framebuffers_[back.image_index]);
777 
778     VkResult res = vk::BeginCommandBuffer(data.primary_cmd, &primary_cmd_begin_info_);
779 
780     if (!use_push_constants_) {
781         VkBufferMemoryBarrier buf_barrier = {};
782         buf_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
783         buf_barrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT;
784         buf_barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
785         buf_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
786         buf_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
787         buf_barrier.buffer = data.buf;
788         buf_barrier.offset = 0;
789         buf_barrier.size = VK_WHOLE_SIZE;
790         vk::CmdPipelineBarrier(data.primary_cmd,
791                                VK_PIPELINE_STAGE_HOST_BIT,
792                                VK_PIPELINE_STAGE_VERTEX_SHADER_BIT,
793                                0, 0, nullptr, 1, &buf_barrier, 0, nullptr);
794     }
795 
796     render_pass_begin_info_.framebuffer = framebuffers_[back.image_index];
797     render_pass_begin_info_.renderArea.extent = extent_;
798     vk::CmdBeginRenderPass(data.primary_cmd, &render_pass_begin_info_,
799             VK_SUBPASS_CONTENTS_SECONDARY_COMMAND_BUFFERS);
800 
801     // record render pass commands
802     for (auto &worker : workers_)
803         worker->wait_idle();
804     vk::CmdExecuteCommands(data.primary_cmd,
805             static_cast<uint32_t>(data.worker_cmds.size()),
806             data.worker_cmds.data());
807 
808     vk::CmdEndRenderPass(data.primary_cmd);
809     vk::EndCommandBuffer(data.primary_cmd);
810 
811     // wait for the image to be owned and signal for render completion
812     primary_cmd_submit_info_.pWaitSemaphores = &back.acquire_semaphore;
813     primary_cmd_submit_info_.pCommandBuffers = &data.primary_cmd;
814     primary_cmd_submit_info_.pSignalSemaphores = &back.render_semaphore;
815 
816     res = vk::QueueSubmit(queue_, 1, &primary_cmd_submit_info_, data.fence);
817 
818     frame_data_index_ = (frame_data_index_ + 1) % frame_data_.size();
819 
820     (void) res;
821 }
822 
Worker(Smoke & smoke,int index,int object_begin,int object_end)823 Smoke::Worker::Worker(Smoke &smoke, int index, int object_begin, int object_end)
824     : smoke_(smoke), index_(index),
825       object_begin_(object_begin), object_end_(object_end),
826       tick_interval_(1.0f / smoke.settings_.ticks_per_second), state_(INIT)
827 {
828 }
829 
start()830 void Smoke::Worker::start()
831 {
832     state_ = IDLE;
833     thread_ = std::thread(Smoke::Worker::thread_loop, this);
834 }
835 
stop()836 void Smoke::Worker::stop()
837 {
838     {
839         std::lock_guard<std::mutex> lock(mutex_);
840         state_ = INIT;
841     }
842     state_cv_.notify_one();
843 
844     thread_.join();
845 }
846 
update_simulation()847 void Smoke::Worker::update_simulation()
848 {
849     {
850         std::lock_guard<std::mutex> lock(mutex_);
851         bool started = (state_ != INIT);
852 
853         state_ = STEP;
854 
855         // step directly
856         if (!started) {
857             smoke_.update_simulation(*this);
858             state_ = INIT;
859         }
860     }
861     state_cv_.notify_one();
862 }
863 
draw_objects(VkFramebuffer fb)864 void Smoke::Worker::draw_objects(VkFramebuffer fb)
865 {
866     // wait for step_objects first
867     wait_idle();
868 
869     {
870         std::lock_guard<std::mutex> lock(mutex_);
871         bool started = (state_ != INIT);
872 
873         fb_ = fb;
874         state_ = DRAW;
875 
876         // render directly
877         if (!started) {
878             smoke_.draw_objects(*this);
879             state_ = INIT;
880         }
881     }
882     state_cv_.notify_one();
883 }
884 
wait_idle()885 void Smoke::Worker::wait_idle()
886 {
887     std::unique_lock<std::mutex> lock(mutex_);
888     bool started = (state_ != INIT);
889 
890     if (started)
891         state_cv_.wait(lock, [this] { return (state_ == IDLE); });
892 }
893 
update_loop()894 void Smoke::Worker::update_loop()
895 {
896     while (true) {
897         std::unique_lock<std::mutex> lock(mutex_);
898 
899         state_cv_.wait(lock, [this] { return (state_ != IDLE); });
900         if (state_ == INIT)
901             break;
902 
903         assert(state_ == STEP || state_ == DRAW);
904         if (state_ == STEP)
905             smoke_.update_simulation(*this);
906         else
907             smoke_.draw_objects(*this);
908 
909         state_ = IDLE;
910         lock.unlock();
911         state_cv_.notify_one();
912     }
913 }
914