1 /*
2 * Copyright © 2016 Red Hat
3 * based on intel anv code:
4 * Copyright © 2015 Intel Corporation
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23 * IN THE SOFTWARE.
24 */
25
26 #include "radv_meta.h"
27
28 #include "vk_util.h"
29
30 #include <fcntl.h>
31 #include <limits.h>
32 #ifndef _WIN32
33 #include <pwd.h>
34 #endif
35 #include <sys/stat.h>
36
37 static void
radv_suspend_queries(struct radv_meta_saved_state * state,struct radv_cmd_buffer * cmd_buffer)38 radv_suspend_queries(struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer)
39 {
40 /* Pipeline statistics queries. */
41 if (cmd_buffer->state.active_pipeline_queries > 0) {
42 cmd_buffer->state.flush_bits &= ~RADV_CMD_FLAG_START_PIPELINE_STATS;
43 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_STOP_PIPELINE_STATS;
44 }
45
46 /* Occlusion queries. */
47 if (cmd_buffer->state.active_occlusion_queries > 0) {
48 radv_set_db_count_control(cmd_buffer, false);
49 }
50
51 /* Primitives generated queries. */
52 if (cmd_buffer->state.prims_gen_query_enabled) {
53 cmd_buffer->state.suspend_streamout = true;
54 radv_emit_streamout_enable(cmd_buffer);
55
56 /* Save the number of active GDS queries and reset it to make sure internal operations won't
57 * increment the counters via GDS.
58 */
59 state->active_pipeline_gds_queries = cmd_buffer->state.active_pipeline_gds_queries;
60 cmd_buffer->state.active_pipeline_gds_queries = 0;
61 }
62 }
63
64 static void
radv_resume_queries(const struct radv_meta_saved_state * state,struct radv_cmd_buffer * cmd_buffer)65 radv_resume_queries(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer)
66 {
67 /* Pipeline statistics queries. */
68 if (cmd_buffer->state.active_pipeline_queries > 0) {
69 cmd_buffer->state.flush_bits &= ~RADV_CMD_FLAG_STOP_PIPELINE_STATS;
70 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_START_PIPELINE_STATS;
71 }
72
73 /* Occlusion queries. */
74 if (cmd_buffer->state.active_occlusion_queries > 0) {
75 radv_set_db_count_control(cmd_buffer, true);
76 }
77
78 /* Primitives generated queries. */
79 if (cmd_buffer->state.prims_gen_query_enabled) {
80 cmd_buffer->state.suspend_streamout = false;
81 radv_emit_streamout_enable(cmd_buffer);
82
83 /* Restore the number of active GDS queries to resume counting. */
84 cmd_buffer->state.active_pipeline_gds_queries = state->active_pipeline_gds_queries;
85 }
86 }
87
88 void
radv_meta_save(struct radv_meta_saved_state * state,struct radv_cmd_buffer * cmd_buffer,uint32_t flags)89 radv_meta_save(struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer,
90 uint32_t flags)
91 {
92 VkPipelineBindPoint bind_point = flags & RADV_META_SAVE_GRAPHICS_PIPELINE
93 ? VK_PIPELINE_BIND_POINT_GRAPHICS
94 : VK_PIPELINE_BIND_POINT_COMPUTE;
95 struct radv_descriptor_state *descriptors_state =
96 radv_get_descriptors_state(cmd_buffer, bind_point);
97
98 assert(flags & (RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_COMPUTE_PIPELINE));
99
100 state->flags = flags;
101
102 if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
103 assert(!(state->flags & RADV_META_SAVE_COMPUTE_PIPELINE));
104
105 state->old_graphics_pipeline = cmd_buffer->state.graphics_pipeline;
106
107 /* Save all viewports. */
108 state->dynamic.viewport.count = cmd_buffer->state.dynamic.viewport.count;
109 typed_memcpy(state->dynamic.viewport.viewports, cmd_buffer->state.dynamic.viewport.viewports,
110 MAX_VIEWPORTS);
111 typed_memcpy(state->dynamic.viewport.xform, cmd_buffer->state.dynamic.viewport.xform,
112 MAX_VIEWPORTS);
113
114 /* Save all scissors. */
115 state->dynamic.scissor.count = cmd_buffer->state.dynamic.scissor.count;
116 typed_memcpy(state->dynamic.scissor.scissors, cmd_buffer->state.dynamic.scissor.scissors,
117 MAX_SCISSORS);
118
119 state->dynamic.line_stipple.factor = cmd_buffer->state.dynamic.line_stipple.factor;
120 state->dynamic.line_stipple.pattern = cmd_buffer->state.dynamic.line_stipple.pattern;
121
122 state->dynamic.cull_mode = cmd_buffer->state.dynamic.cull_mode;
123 state->dynamic.front_face = cmd_buffer->state.dynamic.front_face;
124
125 state->dynamic.primitive_topology = cmd_buffer->state.dynamic.primitive_topology;
126
127 state->dynamic.depth_test_enable = cmd_buffer->state.dynamic.depth_test_enable;
128 state->dynamic.depth_write_enable = cmd_buffer->state.dynamic.depth_write_enable;
129 state->dynamic.depth_compare_op = cmd_buffer->state.dynamic.depth_compare_op;
130 state->dynamic.depth_bounds_test_enable = cmd_buffer->state.dynamic.depth_bounds_test_enable;
131 state->dynamic.stencil_test_enable = cmd_buffer->state.dynamic.stencil_test_enable;
132
133 state->dynamic.stencil_op.front.compare_op = cmd_buffer->state.dynamic.stencil_op.front.compare_op;
134 state->dynamic.stencil_op.front.fail_op = cmd_buffer->state.dynamic.stencil_op.front.fail_op;
135 state->dynamic.stencil_op.front.pass_op = cmd_buffer->state.dynamic.stencil_op.front.pass_op;
136 state->dynamic.stencil_op.front.depth_fail_op =
137 cmd_buffer->state.dynamic.stencil_op.front.depth_fail_op;
138
139 state->dynamic.stencil_op.back.compare_op = cmd_buffer->state.dynamic.stencil_op.back.compare_op;
140 state->dynamic.stencil_op.back.fail_op = cmd_buffer->state.dynamic.stencil_op.back.fail_op;
141 state->dynamic.stencil_op.back.pass_op = cmd_buffer->state.dynamic.stencil_op.back.pass_op;
142 state->dynamic.stencil_op.back.depth_fail_op =
143 cmd_buffer->state.dynamic.stencil_op.back.depth_fail_op;
144
145 state->dynamic.line_width = cmd_buffer->state.dynamic.line_width;
146
147 state->dynamic.depth_bias.bias = cmd_buffer->state.dynamic.depth_bias.bias;
148 state->dynamic.depth_bias.clamp = cmd_buffer->state.dynamic.depth_bias.clamp;
149 state->dynamic.depth_bias.slope = cmd_buffer->state.dynamic.depth_bias.slope;
150
151 memcpy(state->dynamic.blend_constants, cmd_buffer->state.dynamic.blend_constants,
152 sizeof(state->dynamic.blend_constants));
153
154 state->dynamic.depth_bounds.min = cmd_buffer->state.dynamic.depth_bounds.min;
155 state->dynamic.depth_bounds.max = cmd_buffer->state.dynamic.depth_bounds.max;
156
157 state->dynamic.stencil_compare_mask.front = cmd_buffer->state.dynamic.stencil_compare_mask.front;
158 state->dynamic.stencil_compare_mask.back = cmd_buffer->state.dynamic.stencil_compare_mask.back;
159
160 state->dynamic.stencil_write_mask.front = cmd_buffer->state.dynamic.stencil_write_mask.front;
161 state->dynamic.stencil_write_mask.back = cmd_buffer->state.dynamic.stencil_write_mask.back;
162
163 state->dynamic.stencil_reference.front = cmd_buffer->state.dynamic.stencil_reference.front;
164 state->dynamic.stencil_reference.back = cmd_buffer->state.dynamic.stencil_reference.back;
165
166 state->dynamic.fragment_shading_rate.size = cmd_buffer->state.dynamic.fragment_shading_rate.size;
167 state->dynamic.fragment_shading_rate.combiner_ops[0] =
168 cmd_buffer->state.dynamic.fragment_shading_rate.combiner_ops[0];
169 state->dynamic.fragment_shading_rate.combiner_ops[1] =
170 cmd_buffer->state.dynamic.fragment_shading_rate.combiner_ops[1];
171
172 state->dynamic.depth_bias_enable = cmd_buffer->state.dynamic.depth_bias_enable;
173
174 state->dynamic.primitive_restart_enable = cmd_buffer->state.dynamic.primitive_restart_enable;
175
176 state->dynamic.rasterizer_discard_enable = cmd_buffer->state.dynamic.rasterizer_discard_enable;
177
178 state->dynamic.logic_op = cmd_buffer->state.dynamic.logic_op;
179
180 state->dynamic.color_write_enable = cmd_buffer->state.dynamic.color_write_enable;
181
182 state->dynamic.discard_rectangle.count = cmd_buffer->state.dynamic.discard_rectangle.count;
183 typed_memcpy(state->dynamic.discard_rectangle.rectangles,
184 cmd_buffer->state.dynamic.discard_rectangle.rectangles,
185 MAX_DISCARD_RECTANGLES);
186 }
187
188 if (state->flags & RADV_META_SAVE_SAMPLE_LOCATIONS) {
189 typed_memcpy(&state->dynamic.sample_location, &cmd_buffer->state.dynamic.sample_location, 1);
190 }
191
192 if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) {
193 assert(!(state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE));
194
195 state->old_compute_pipeline = cmd_buffer->state.compute_pipeline;
196 }
197
198 if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
199 state->old_descriptor_set0 = descriptors_state->sets[0];
200 if (!(descriptors_state->valid & 1) || !state->old_descriptor_set0)
201 state->flags &= ~RADV_META_SAVE_DESCRIPTORS;
202 }
203
204 if (state->flags & RADV_META_SAVE_CONSTANTS) {
205 memcpy(state->push_constants, cmd_buffer->push_constants, MAX_PUSH_CONSTANTS_SIZE);
206 }
207
208 if (state->flags & RADV_META_SAVE_PASS) {
209 state->pass = cmd_buffer->state.pass;
210 state->subpass = cmd_buffer->state.subpass;
211 state->framebuffer = cmd_buffer->state.framebuffer;
212 state->attachments = cmd_buffer->state.attachments;
213 state->render_area = cmd_buffer->state.render_area;
214 }
215
216 if (state->flags & RADV_META_SUSPEND_PREDICATING) {
217 state->predicating = cmd_buffer->state.predicating;
218 cmd_buffer->state.predicating = false;
219 }
220
221 radv_suspend_queries(state, cmd_buffer);
222 }
223
224 void
radv_meta_restore(const struct radv_meta_saved_state * state,struct radv_cmd_buffer * cmd_buffer)225 radv_meta_restore(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer)
226 {
227 VkPipelineBindPoint bind_point = state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE
228 ? VK_PIPELINE_BIND_POINT_GRAPHICS
229 : VK_PIPELINE_BIND_POINT_COMPUTE;
230
231 if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
232 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
233 radv_pipeline_to_handle(&state->old_graphics_pipeline->base));
234
235 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE;
236
237 /* Restore all viewports. */
238 cmd_buffer->state.dynamic.viewport.count = state->dynamic.viewport.count;
239 typed_memcpy(cmd_buffer->state.dynamic.viewport.viewports, state->dynamic.viewport.viewports,
240 MAX_VIEWPORTS);
241 typed_memcpy(cmd_buffer->state.dynamic.viewport.xform, state->dynamic.viewport.xform,
242 MAX_VIEWPORTS);
243
244 /* Restore all scissors. */
245 cmd_buffer->state.dynamic.scissor.count = state->dynamic.scissor.count;
246 typed_memcpy(cmd_buffer->state.dynamic.scissor.scissors, state->dynamic.scissor.scissors,
247 MAX_SCISSORS);
248
249 cmd_buffer->state.dynamic.line_stipple.factor = state->dynamic.line_stipple.factor;
250 cmd_buffer->state.dynamic.line_stipple.pattern = state->dynamic.line_stipple.pattern;
251
252 cmd_buffer->state.dynamic.cull_mode = state->dynamic.cull_mode;
253 cmd_buffer->state.dynamic.front_face = state->dynamic.front_face;
254
255 cmd_buffer->state.dynamic.primitive_topology = state->dynamic.primitive_topology;
256
257 cmd_buffer->state.dynamic.depth_test_enable = state->dynamic.depth_test_enable;
258 cmd_buffer->state.dynamic.depth_write_enable = state->dynamic.depth_write_enable;
259 cmd_buffer->state.dynamic.depth_compare_op = state->dynamic.depth_compare_op;
260 cmd_buffer->state.dynamic.depth_bounds_test_enable = state->dynamic.depth_bounds_test_enable;
261 cmd_buffer->state.dynamic.stencil_test_enable = state->dynamic.stencil_test_enable;
262
263 cmd_buffer->state.dynamic.stencil_op.front.compare_op = state->dynamic.stencil_op.front.compare_op;
264 cmd_buffer->state.dynamic.stencil_op.front.fail_op = state->dynamic.stencil_op.front.fail_op;
265 cmd_buffer->state.dynamic.stencil_op.front.pass_op = state->dynamic.stencil_op.front.pass_op;
266 cmd_buffer->state.dynamic.stencil_op.front.depth_fail_op =
267 state->dynamic.stencil_op.front.depth_fail_op;
268
269 cmd_buffer->state.dynamic.stencil_op.back.compare_op = state->dynamic.stencil_op.back.compare_op;
270 cmd_buffer->state.dynamic.stencil_op.back.fail_op = state->dynamic.stencil_op.back.fail_op;
271 cmd_buffer->state.dynamic.stencil_op.back.pass_op = state->dynamic.stencil_op.back.pass_op;
272 cmd_buffer->state.dynamic.stencil_op.back.depth_fail_op =
273 state->dynamic.stencil_op.back.depth_fail_op;
274
275 cmd_buffer->state.dynamic.line_width = state->dynamic.line_width;
276
277 cmd_buffer->state.dynamic.depth_bias.bias = state->dynamic.depth_bias.bias;
278 cmd_buffer->state.dynamic.depth_bias.clamp = state->dynamic.depth_bias.clamp;
279 cmd_buffer->state.dynamic.depth_bias.slope = state->dynamic.depth_bias.slope;
280
281 memcpy(cmd_buffer->state.dynamic.blend_constants, state->dynamic.blend_constants,
282 sizeof(state->dynamic.blend_constants));
283
284 cmd_buffer->state.dynamic.depth_bounds.min = state->dynamic.depth_bounds.min;
285 cmd_buffer->state.dynamic.depth_bounds.max = state->dynamic.depth_bounds.max;
286
287 cmd_buffer->state.dynamic.stencil_compare_mask.front = state->dynamic.stencil_compare_mask.front;
288 cmd_buffer->state.dynamic.stencil_compare_mask.back = state->dynamic.stencil_compare_mask.back;
289
290 cmd_buffer->state.dynamic.stencil_write_mask.front = state->dynamic.stencil_write_mask.front;
291 cmd_buffer->state.dynamic.stencil_write_mask.back = state->dynamic.stencil_write_mask.back;
292
293 cmd_buffer->state.dynamic.stencil_reference.front = state->dynamic.stencil_reference.front;
294 cmd_buffer->state.dynamic.stencil_reference.back = state->dynamic.stencil_reference.back;
295
296 cmd_buffer->state.dynamic.fragment_shading_rate.size = state->dynamic.fragment_shading_rate.size;
297 cmd_buffer->state.dynamic.fragment_shading_rate.combiner_ops[0] =
298 state->dynamic.fragment_shading_rate.combiner_ops[0];
299 cmd_buffer->state.dynamic.fragment_shading_rate.combiner_ops[1] =
300 state->dynamic.fragment_shading_rate.combiner_ops[1];
301
302 cmd_buffer->state.dynamic.depth_bias_enable = state->dynamic.depth_bias_enable;
303
304 cmd_buffer->state.dynamic.primitive_restart_enable = state->dynamic.primitive_restart_enable;
305
306 cmd_buffer->state.dynamic.rasterizer_discard_enable = state->dynamic.rasterizer_discard_enable;
307
308 cmd_buffer->state.dynamic.logic_op = state->dynamic.logic_op;
309
310 cmd_buffer->state.dynamic.color_write_enable = state->dynamic.color_write_enable;
311
312 cmd_buffer->state.dynamic.discard_rectangle.count = state->dynamic.discard_rectangle.count;
313 typed_memcpy(cmd_buffer->state.dynamic.discard_rectangle.rectangles,
314 state->dynamic.discard_rectangle.rectangles,
315 MAX_DISCARD_RECTANGLES);
316
317 cmd_buffer->state.dirty |=
318 RADV_CMD_DIRTY_DYNAMIC_VIEWPORT | RADV_CMD_DIRTY_DYNAMIC_SCISSOR |
319 RADV_CMD_DIRTY_DYNAMIC_CULL_MODE | RADV_CMD_DIRTY_DYNAMIC_FRONT_FACE |
320 RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_TOPOLOGY | RADV_CMD_DIRTY_DYNAMIC_DEPTH_TEST_ENABLE |
321 RADV_CMD_DIRTY_DYNAMIC_DEPTH_WRITE_ENABLE | RADV_CMD_DIRTY_DYNAMIC_DEPTH_COMPARE_OP |
322 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE |
323 RADV_CMD_DIRTY_DYNAMIC_STENCIL_TEST_ENABLE | RADV_CMD_DIRTY_DYNAMIC_STENCIL_OP |
324 RADV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK | RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE |
325 RADV_CMD_DIRTY_DYNAMIC_FRAGMENT_SHADING_RATE | RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS_ENABLE |
326 RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_RESTART_ENABLE |
327 RADV_CMD_DIRTY_DYNAMIC_RASTERIZER_DISCARD_ENABLE | RADV_CMD_DIRTY_DYNAMIC_LOGIC_OP |
328 RADV_CMD_DIRTY_DYNAMIC_COLOR_WRITE_ENABLE | RADV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE |
329 RADV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK | RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS |
330 RADV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS | RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH |
331 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS | RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE;
332 }
333
334 if (state->flags & RADV_META_SAVE_SAMPLE_LOCATIONS) {
335 typed_memcpy(&cmd_buffer->state.dynamic.sample_location.locations,
336 &state->dynamic.sample_location.locations, 1);
337
338 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_SAMPLE_LOCATIONS;
339 }
340
341 if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) {
342 if (state->old_compute_pipeline) {
343 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE,
344 radv_pipeline_to_handle(&state->old_compute_pipeline->base));
345 }
346 }
347
348 if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
349 radv_set_descriptor_set(cmd_buffer, bind_point, state->old_descriptor_set0, 0);
350 }
351
352 if (state->flags & RADV_META_SAVE_CONSTANTS) {
353 VkShaderStageFlags stages = VK_SHADER_STAGE_COMPUTE_BIT;
354
355 if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE)
356 stages |= VK_SHADER_STAGE_ALL_GRAPHICS;
357
358 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), VK_NULL_HANDLE, stages, 0,
359 MAX_PUSH_CONSTANTS_SIZE, state->push_constants);
360 }
361
362 if (state->flags & RADV_META_SAVE_PASS) {
363 cmd_buffer->state.pass = state->pass;
364 cmd_buffer->state.subpass = state->subpass;
365 cmd_buffer->state.framebuffer = state->framebuffer;
366 cmd_buffer->state.attachments = state->attachments;
367 cmd_buffer->state.render_area = state->render_area;
368 if (state->subpass)
369 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER;
370 }
371
372 if (state->flags & RADV_META_SUSPEND_PREDICATING)
373 cmd_buffer->state.predicating = state->predicating;
374
375 radv_resume_queries(state, cmd_buffer);
376 }
377
378 VkImageViewType
radv_meta_get_view_type(const struct radv_image * image)379 radv_meta_get_view_type(const struct radv_image *image)
380 {
381 switch (image->vk.image_type) {
382 case VK_IMAGE_TYPE_1D:
383 return VK_IMAGE_VIEW_TYPE_1D;
384 case VK_IMAGE_TYPE_2D:
385 return VK_IMAGE_VIEW_TYPE_2D;
386 case VK_IMAGE_TYPE_3D:
387 return VK_IMAGE_VIEW_TYPE_3D;
388 default:
389 unreachable("bad VkImageViewType");
390 }
391 }
392
393 /**
394 * When creating a destination VkImageView, this function provides the needed
395 * VkImageViewCreateInfo::subresourceRange::baseArrayLayer.
396 */
397 uint32_t
radv_meta_get_iview_layer(const struct radv_image * dest_image,const VkImageSubresourceLayers * dest_subresource,const VkOffset3D * dest_offset)398 radv_meta_get_iview_layer(const struct radv_image *dest_image,
399 const VkImageSubresourceLayers *dest_subresource,
400 const VkOffset3D *dest_offset)
401 {
402 switch (dest_image->vk.image_type) {
403 case VK_IMAGE_TYPE_1D:
404 case VK_IMAGE_TYPE_2D:
405 return dest_subresource->baseArrayLayer;
406 case VK_IMAGE_TYPE_3D:
407 /* HACK: Vulkan does not allow attaching a 3D image to a framebuffer,
408 * but meta does it anyway. When doing so, we translate the
409 * destination's z offset into an array offset.
410 */
411 return dest_offset->z;
412 default:
413 assert(!"bad VkImageType");
414 return 0;
415 }
416 }
417
418 static VKAPI_ATTR void * VKAPI_CALL
meta_alloc(void * _device,size_t size,size_t alignment,VkSystemAllocationScope allocationScope)419 meta_alloc(void *_device, size_t size, size_t alignment, VkSystemAllocationScope allocationScope)
420 {
421 struct radv_device *device = _device;
422 return device->vk.alloc.pfnAllocation(device->vk.alloc.pUserData, size, alignment,
423 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
424 }
425
426 static VKAPI_ATTR void * VKAPI_CALL
meta_realloc(void * _device,void * original,size_t size,size_t alignment,VkSystemAllocationScope allocationScope)427 meta_realloc(void *_device, void *original, size_t size, size_t alignment,
428 VkSystemAllocationScope allocationScope)
429 {
430 struct radv_device *device = _device;
431 return device->vk.alloc.pfnReallocation(device->vk.alloc.pUserData, original, size, alignment,
432 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
433 }
434
435 static VKAPI_ATTR void VKAPI_CALL
meta_free(void * _device,void * data)436 meta_free(void *_device, void *data)
437 {
438 struct radv_device *device = _device;
439 device->vk.alloc.pfnFree(device->vk.alloc.pUserData, data);
440 }
441
442 #ifndef _WIN32
443 static bool
radv_builtin_cache_path(char * path)444 radv_builtin_cache_path(char *path)
445 {
446 char *xdg_cache_home = getenv("XDG_CACHE_HOME");
447 const char *suffix = "/radv_builtin_shaders";
448 const char *suffix2 = "/.cache/radv_builtin_shaders";
449 struct passwd pwd, *result;
450 char path2[PATH_MAX + 1]; /* PATH_MAX is not a real max,but suffices here. */
451 int ret;
452
453 if (xdg_cache_home) {
454 ret = snprintf(path, PATH_MAX + 1, "%s%s%zd", xdg_cache_home, suffix, sizeof(void *) * 8);
455 return ret > 0 && ret < PATH_MAX + 1;
456 }
457
458 getpwuid_r(getuid(), &pwd, path2, PATH_MAX - strlen(suffix2), &result);
459 if (!result)
460 return false;
461
462 strcpy(path, pwd.pw_dir);
463 strcat(path, "/.cache");
464 if (mkdir(path, 0755) && errno != EEXIST)
465 return false;
466
467 ret = snprintf(path, PATH_MAX + 1, "%s%s%zd", pwd.pw_dir, suffix2, sizeof(void *) * 8);
468 return ret > 0 && ret < PATH_MAX + 1;
469 }
470 #endif
471
472 static bool
radv_load_meta_pipeline(struct radv_device * device)473 radv_load_meta_pipeline(struct radv_device *device)
474 {
475 #ifdef _WIN32
476 return false;
477 #else
478 char path[PATH_MAX + 1];
479 struct stat st;
480 void *data = NULL;
481 bool ret = false;
482
483 if (!radv_builtin_cache_path(path))
484 return false;
485
486 int fd = open(path, O_RDONLY);
487 if (fd < 0)
488 return false;
489 if (fstat(fd, &st))
490 goto fail;
491 data = malloc(st.st_size);
492 if (!data)
493 goto fail;
494 if (read(fd, data, st.st_size) == -1)
495 goto fail;
496
497 ret = radv_pipeline_cache_load(&device->meta_state.cache, data, st.st_size);
498 fail:
499 free(data);
500 close(fd);
501 return ret;
502 #endif
503 }
504
505 static void
radv_store_meta_pipeline(struct radv_device * device)506 radv_store_meta_pipeline(struct radv_device *device)
507 {
508 #ifndef _WIN32
509 char path[PATH_MAX + 1], path2[PATH_MAX + 7];
510 size_t size;
511 void *data = NULL;
512
513 if (!device->meta_state.cache.modified)
514 return;
515
516 if (radv_GetPipelineCacheData(radv_device_to_handle(device),
517 radv_pipeline_cache_to_handle(&device->meta_state.cache), &size,
518 NULL))
519 return;
520
521 if (!radv_builtin_cache_path(path))
522 return;
523
524 strcpy(path2, path);
525 strcat(path2, "XXXXXX");
526 int fd = mkstemp(path2); // open(path, O_WRONLY | O_CREAT, 0600);
527 if (fd < 0)
528 return;
529 data = malloc(size);
530 if (!data)
531 goto fail;
532
533 if (radv_GetPipelineCacheData(radv_device_to_handle(device),
534 radv_pipeline_cache_to_handle(&device->meta_state.cache), &size,
535 data))
536 goto fail;
537 if (write(fd, data, size) == -1)
538 goto fail;
539
540 rename(path2, path);
541 fail:
542 free(data);
543 close(fd);
544 unlink(path2);
545 #endif
546 }
547
548 VkResult
radv_device_init_meta(struct radv_device * device)549 radv_device_init_meta(struct radv_device *device)
550 {
551 VkResult result;
552
553 memset(&device->meta_state, 0, sizeof(device->meta_state));
554
555 device->meta_state.alloc = (VkAllocationCallbacks){
556 .pUserData = device,
557 .pfnAllocation = meta_alloc,
558 .pfnReallocation = meta_realloc,
559 .pfnFree = meta_free,
560 };
561
562 device->meta_state.cache.alloc = device->meta_state.alloc;
563 radv_pipeline_cache_init(&device->meta_state.cache, device);
564 bool loaded_cache = radv_load_meta_pipeline(device);
565 bool on_demand = !loaded_cache;
566
567 mtx_init(&device->meta_state.mtx, mtx_plain);
568
569 device->app_shaders_internal = true;
570
571 result = radv_device_init_meta_clear_state(device, on_demand);
572 if (result != VK_SUCCESS)
573 goto fail_clear;
574
575 result = radv_device_init_meta_resolve_state(device, on_demand);
576 if (result != VK_SUCCESS)
577 goto fail_resolve;
578
579 result = radv_device_init_meta_blit_state(device, on_demand);
580 if (result != VK_SUCCESS)
581 goto fail_blit;
582
583 result = radv_device_init_meta_blit2d_state(device, on_demand);
584 if (result != VK_SUCCESS)
585 goto fail_blit2d;
586
587 result = radv_device_init_meta_bufimage_state(device);
588 if (result != VK_SUCCESS)
589 goto fail_bufimage;
590
591 result = radv_device_init_meta_depth_decomp_state(device, on_demand);
592 if (result != VK_SUCCESS)
593 goto fail_depth_decomp;
594
595 result = radv_device_init_meta_buffer_state(device);
596 if (result != VK_SUCCESS)
597 goto fail_buffer;
598
599 result = radv_device_init_meta_query_state(device, on_demand);
600 if (result != VK_SUCCESS)
601 goto fail_query;
602
603 result = radv_device_init_meta_fast_clear_flush_state(device, on_demand);
604 if (result != VK_SUCCESS)
605 goto fail_fast_clear;
606
607 result = radv_device_init_meta_resolve_compute_state(device, on_demand);
608 if (result != VK_SUCCESS)
609 goto fail_resolve_compute;
610
611 result = radv_device_init_meta_resolve_fragment_state(device, on_demand);
612 if (result != VK_SUCCESS)
613 goto fail_resolve_fragment;
614
615 result = radv_device_init_meta_fmask_expand_state(device);
616 if (result != VK_SUCCESS)
617 goto fail_fmask_expand;
618
619 if (radv_enable_rt(device->physical_device, false)) {
620 result = radv_device_init_accel_struct_build_state(device);
621 if (result != VK_SUCCESS)
622 goto fail_accel_struct_build;
623 }
624
625 result = radv_device_init_meta_fmask_copy_state(device);
626 if (result != VK_SUCCESS)
627 goto fail_fmask_copy;
628
629 result = radv_device_init_meta_etc_decode_state(device, on_demand);
630 if (result != VK_SUCCESS)
631 goto fail_etc_decode;
632
633 if (device->uses_device_generated_commands) {
634 result = radv_device_init_dgc_prepare_state(device);
635 if (result != VK_SUCCESS)
636 goto fail_dgc;
637 }
638
639 device->app_shaders_internal = false;
640
641 return VK_SUCCESS;
642
643 fail_dgc:
644 radv_device_finish_dgc_prepare_state(device);
645 fail_etc_decode:
646 radv_device_finish_meta_etc_decode_state(device);
647 fail_fmask_copy:
648 radv_device_finish_meta_fmask_copy_state(device);
649 fail_accel_struct_build:
650 radv_device_finish_accel_struct_build_state(device);
651 fail_fmask_expand:
652 radv_device_finish_meta_fmask_expand_state(device);
653 fail_resolve_fragment:
654 radv_device_finish_meta_resolve_fragment_state(device);
655 fail_resolve_compute:
656 radv_device_finish_meta_resolve_compute_state(device);
657 fail_fast_clear:
658 radv_device_finish_meta_fast_clear_flush_state(device);
659 fail_query:
660 radv_device_finish_meta_query_state(device);
661 fail_buffer:
662 radv_device_finish_meta_buffer_state(device);
663 fail_depth_decomp:
664 radv_device_finish_meta_depth_decomp_state(device);
665 fail_bufimage:
666 radv_device_finish_meta_bufimage_state(device);
667 fail_blit2d:
668 radv_device_finish_meta_blit2d_state(device);
669 fail_blit:
670 radv_device_finish_meta_blit_state(device);
671 fail_resolve:
672 radv_device_finish_meta_resolve_state(device);
673 fail_clear:
674 radv_device_finish_meta_clear_state(device);
675
676 mtx_destroy(&device->meta_state.mtx);
677 radv_pipeline_cache_finish(&device->meta_state.cache);
678 return result;
679 }
680
681 void
radv_device_finish_meta(struct radv_device * device)682 radv_device_finish_meta(struct radv_device *device)
683 {
684 radv_device_finish_dgc_prepare_state(device);
685 radv_device_finish_meta_etc_decode_state(device);
686 radv_device_finish_accel_struct_build_state(device);
687 radv_device_finish_meta_clear_state(device);
688 radv_device_finish_meta_resolve_state(device);
689 radv_device_finish_meta_blit_state(device);
690 radv_device_finish_meta_blit2d_state(device);
691 radv_device_finish_meta_bufimage_state(device);
692 radv_device_finish_meta_depth_decomp_state(device);
693 radv_device_finish_meta_query_state(device);
694 radv_device_finish_meta_buffer_state(device);
695 radv_device_finish_meta_fast_clear_flush_state(device);
696 radv_device_finish_meta_resolve_compute_state(device);
697 radv_device_finish_meta_resolve_fragment_state(device);
698 radv_device_finish_meta_fmask_expand_state(device);
699 radv_device_finish_meta_dcc_retile_state(device);
700 radv_device_finish_meta_copy_vrs_htile_state(device);
701 radv_device_finish_meta_fmask_copy_state(device);
702
703 radv_store_meta_pipeline(device);
704 radv_pipeline_cache_finish(&device->meta_state.cache);
705 mtx_destroy(&device->meta_state.mtx);
706 }
707
708 nir_builder PRINTFLIKE(3, 4)
radv_meta_init_shader(struct radv_device * dev,gl_shader_stage stage,const char * name,...)709 radv_meta_init_shader(struct radv_device *dev, gl_shader_stage stage, const char *name, ...)
710 {
711 nir_builder b = nir_builder_init_simple_shader(stage, NULL, NULL);
712 if (name) {
713 va_list args;
714 va_start(args, name);
715 b.shader->info.name = ralloc_vasprintf(b.shader, name, args);
716 va_end(args);
717 }
718
719 b.shader->options = &dev->physical_device->nir_options[stage];
720 b.shader->info.workgroup_size[0] = 1;
721 b.shader->info.workgroup_size[1] = 1;
722 b.shader->info.workgroup_size[2] = 1;
723
724 return b;
725 }
726
727 /* vertex shader that generates vertices */
728 nir_shader *
radv_meta_build_nir_vs_generate_vertices(struct radv_device * dev)729 radv_meta_build_nir_vs_generate_vertices(struct radv_device *dev)
730 {
731 const struct glsl_type *vec4 = glsl_vec4_type();
732
733 nir_variable *v_position;
734
735 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_VERTEX, "meta_vs_gen_verts");
736
737 nir_ssa_def *outvec = nir_gen_rect_vertices(&b, NULL, NULL);
738
739 v_position = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
740 v_position->data.location = VARYING_SLOT_POS;
741
742 nir_store_var(&b, v_position, outvec, 0xf);
743
744 return b.shader;
745 }
746
747 nir_shader *
radv_meta_build_nir_fs_noop(struct radv_device * dev)748 radv_meta_build_nir_fs_noop(struct radv_device *dev)
749 {
750 return radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_noop_fs").shader;
751 }
752
753 void
radv_meta_build_resolve_shader_core(nir_builder * b,bool is_integer,int samples,nir_variable * input_img,nir_variable * color,nir_ssa_def * img_coord)754 radv_meta_build_resolve_shader_core(nir_builder *b, bool is_integer, int samples,
755 nir_variable *input_img, nir_variable *color,
756 nir_ssa_def *img_coord)
757 {
758 /* do a txf_ms on each sample */
759 nir_ssa_def *tmp;
760 bool inserted_if = false;
761
762 nir_ssa_def *input_img_deref = &nir_build_deref_var(b, input_img)->dest.ssa;
763
764 nir_tex_instr *tex = nir_tex_instr_create(b->shader, 3);
765 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
766 tex->op = nir_texop_txf_ms;
767 tex->src[0].src_type = nir_tex_src_coord;
768 tex->src[0].src = nir_src_for_ssa(img_coord);
769 tex->src[1].src_type = nir_tex_src_ms_index;
770 tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
771 tex->src[2].src_type = nir_tex_src_texture_deref;
772 tex->src[2].src = nir_src_for_ssa(input_img_deref);
773 tex->dest_type = nir_get_nir_type_for_glsl_base_type(glsl_get_sampler_result_type(input_img->type));
774 tex->is_array = false;
775 tex->coord_components = 2;
776
777 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
778 nir_builder_instr_insert(b, &tex->instr);
779
780 tmp = &tex->dest.ssa;
781
782 if (!is_integer && samples > 1) {
783 nir_tex_instr *tex_all_same = nir_tex_instr_create(b->shader, 2);
784 tex_all_same->sampler_dim = GLSL_SAMPLER_DIM_MS;
785 tex_all_same->op = nir_texop_samples_identical;
786 tex_all_same->src[0].src_type = nir_tex_src_coord;
787 tex_all_same->src[0].src = nir_src_for_ssa(img_coord);
788 tex_all_same->src[1].src_type = nir_tex_src_texture_deref;
789 tex_all_same->src[1].src = nir_src_for_ssa(input_img_deref);
790 tex_all_same->dest_type = nir_type_bool1;
791 tex_all_same->is_array = false;
792 tex_all_same->coord_components = 2;
793
794 nir_ssa_dest_init(&tex_all_same->instr, &tex_all_same->dest, 1, 1, "tex");
795 nir_builder_instr_insert(b, &tex_all_same->instr);
796
797 nir_ssa_def *not_all_same = nir_inot(b, &tex_all_same->dest.ssa);
798 nir_push_if(b, not_all_same);
799 for (int i = 1; i < samples; i++) {
800 nir_tex_instr *tex_add = nir_tex_instr_create(b->shader, 3);
801 tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS;
802 tex_add->op = nir_texop_txf_ms;
803 tex_add->src[0].src_type = nir_tex_src_coord;
804 tex_add->src[0].src = nir_src_for_ssa(img_coord);
805 tex_add->src[1].src_type = nir_tex_src_ms_index;
806 tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(b, i));
807 tex_add->src[2].src_type = nir_tex_src_texture_deref;
808 tex_add->src[2].src = nir_src_for_ssa(input_img_deref);
809 tex_add->dest_type = nir_type_float32;
810 tex_add->is_array = false;
811 tex_add->coord_components = 2;
812
813 nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex");
814 nir_builder_instr_insert(b, &tex_add->instr);
815
816 tmp = nir_fadd(b, tmp, &tex_add->dest.ssa);
817 }
818
819 tmp = nir_fdiv(b, tmp, nir_imm_float(b, samples));
820 nir_store_var(b, color, tmp, 0xf);
821 nir_push_else(b, NULL);
822 inserted_if = true;
823 }
824 nir_store_var(b, color, &tex->dest.ssa, 0xf);
825
826 if (inserted_if)
827 nir_pop_if(b, NULL);
828 }
829
830 nir_ssa_def *
radv_meta_load_descriptor(nir_builder * b,unsigned desc_set,unsigned binding)831 radv_meta_load_descriptor(nir_builder *b, unsigned desc_set, unsigned binding)
832 {
833 nir_ssa_def *rsrc = nir_vulkan_resource_index(b, 3, 32, nir_imm_int(b, 0), .desc_set = desc_set,
834 .binding = binding);
835 return nir_channels(b, rsrc, 0x3);
836 }
837
838 nir_ssa_def *
get_global_ids(nir_builder * b,unsigned num_components)839 get_global_ids(nir_builder *b, unsigned num_components)
840 {
841 unsigned mask = BITFIELD_MASK(num_components);
842
843 nir_ssa_def *local_ids = nir_channels(b, nir_load_local_invocation_id(b), mask);
844 nir_ssa_def *block_ids = nir_channels(b, nir_load_workgroup_id(b, 32), mask);
845 nir_ssa_def *block_size = nir_channels(
846 b,
847 nir_imm_ivec4(b, b->shader->info.workgroup_size[0], b->shader->info.workgroup_size[1],
848 b->shader->info.workgroup_size[2], 0),
849 mask);
850
851 return nir_iadd(b, nir_imul(b, block_ids, block_size), local_ids);
852 }
853
854 void
radv_break_on_count(nir_builder * b,nir_variable * var,nir_ssa_def * count)855 radv_break_on_count(nir_builder *b, nir_variable *var, nir_ssa_def *count)
856 {
857 nir_ssa_def *counter = nir_load_var(b, var);
858
859 nir_push_if(b, nir_uge(b, counter, count));
860 nir_jump(b, nir_jump_break);
861 nir_pop_if(b, NULL);
862
863 counter = nir_iadd_imm(b, counter, 1);
864 nir_store_var(b, var, counter, 0x1);
865 }
866