1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "radv_meta.h"
25 #include "vk_format.h"
26
27 static VkExtent3D
meta_image_block_size(const struct radv_image * image)28 meta_image_block_size(const struct radv_image *image)
29 {
30 const struct vk_format_description *desc = vk_format_description(image->vk_format);
31 return (VkExtent3D) { desc->block.width, desc->block.height, 1 };
32 }
33
34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
35 * elements rather than texels. One element equals one texel or one block
36 * if Image is uncompressed or compressed, respectively.
37 */
38 static struct VkExtent3D
meta_region_extent_el(const struct radv_image * image,const VkImageType imageType,const struct VkExtent3D * extent)39 meta_region_extent_el(const struct radv_image *image,
40 const VkImageType imageType,
41 const struct VkExtent3D *extent)
42 {
43 const VkExtent3D block = meta_image_block_size(image);
44 return radv_sanitize_image_extent(imageType, (VkExtent3D) {
45 .width = DIV_ROUND_UP(extent->width , block.width),
46 .height = DIV_ROUND_UP(extent->height, block.height),
47 .depth = DIV_ROUND_UP(extent->depth , block.depth),
48 });
49 }
50
51 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
52 * elements rather than texels. One element equals one texel or one block
53 * if Image is uncompressed or compressed, respectively.
54 */
55 static struct VkOffset3D
meta_region_offset_el(const struct radv_image * image,const struct VkOffset3D * offset)56 meta_region_offset_el(const struct radv_image *image,
57 const struct VkOffset3D *offset)
58 {
59 const VkExtent3D block = meta_image_block_size(image);
60 return radv_sanitize_image_offset(image->type, (VkOffset3D) {
61 .x = offset->x / block.width,
62 .y = offset->y / block.height,
63 .z = offset->z / block.depth,
64 });
65 }
66
67 static VkFormat
vk_format_for_size(int bs)68 vk_format_for_size(int bs)
69 {
70 switch (bs) {
71 case 1: return VK_FORMAT_R8_UINT;
72 case 2: return VK_FORMAT_R8G8_UINT;
73 case 4: return VK_FORMAT_R8G8B8A8_UINT;
74 case 8: return VK_FORMAT_R16G16B16A16_UINT;
75 case 16: return VK_FORMAT_R32G32B32A32_UINT;
76 default:
77 unreachable("Invalid format block size");
78 }
79 }
80
81 static struct radv_meta_blit2d_surf
blit_surf_for_image_level_layer(struct radv_image * image,VkImageLayout layout,const VkImageSubresourceLayers * subres)82 blit_surf_for_image_level_layer(struct radv_image *image,
83 VkImageLayout layout,
84 const VkImageSubresourceLayers *subres)
85 {
86 VkFormat format = image->vk_format;
87 if (subres->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT)
88 format = vk_format_depth_only(format);
89 else if (subres->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT)
90 format = vk_format_stencil_only(format);
91
92 if (!image->surface.dcc_size &&
93 !(image->surface.htile_size && image->tc_compatible_htile))
94 format = vk_format_for_size(vk_format_get_blocksize(format));
95
96 return (struct radv_meta_blit2d_surf) {
97 .format = format,
98 .bs = vk_format_get_blocksize(format),
99 .level = subres->mipLevel,
100 .layer = subres->baseArrayLayer,
101 .image = image,
102 .aspect_mask = subres->aspectMask,
103 .current_layout = layout,
104 };
105 }
106
107 static void
meta_copy_buffer_to_image(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer * buffer,struct radv_image * image,VkImageLayout layout,uint32_t regionCount,const VkBufferImageCopy * pRegions)108 meta_copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer,
109 struct radv_buffer* buffer,
110 struct radv_image* image,
111 VkImageLayout layout,
112 uint32_t regionCount,
113 const VkBufferImageCopy* pRegions)
114 {
115 bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
116 struct radv_meta_saved_state saved_state;
117
118 /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
119 * VK_SAMPLE_COUNT_1_BIT."
120 */
121 assert(image->info.samples == 1);
122
123 radv_meta_save(&saved_state, cmd_buffer,
124 (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
125 RADV_META_SAVE_GRAPHICS_PIPELINE) |
126 RADV_META_SAVE_CONSTANTS |
127 RADV_META_SAVE_DESCRIPTORS);
128
129 for (unsigned r = 0; r < regionCount; r++) {
130
131 /**
132 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
133 * extent is the size in texels of the source image to copy in width,
134 * height and depth. 1D images use only x and width. 2D images use x, y,
135 * width and height. 3D images use x, y, z, width, height and depth.
136 *
137 *
138 * Also, convert the offsets and extent from units of texels to units of
139 * blocks - which is the highest resolution accessible in this command.
140 */
141 const VkOffset3D img_offset_el =
142 meta_region_offset_el(image, &pRegions[r].imageOffset);
143 const VkExtent3D bufferExtent = {
144 .width = pRegions[r].bufferRowLength ?
145 pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
146 .height = pRegions[r].bufferImageHeight ?
147 pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
148 };
149 const VkExtent3D buf_extent_el =
150 meta_region_extent_el(image, image->type, &bufferExtent);
151
152 /* Start creating blit rect */
153 const VkExtent3D img_extent_el =
154 meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
155 struct radv_meta_blit2d_rect rect = {
156 .width = img_extent_el.width,
157 .height = img_extent_el.height,
158 };
159
160 /* Create blit surfaces */
161 struct radv_meta_blit2d_surf img_bsurf =
162 blit_surf_for_image_level_layer(image,
163 layout,
164 &pRegions[r].imageSubresource);
165
166 struct radv_meta_blit2d_buffer buf_bsurf = {
167 .bs = img_bsurf.bs,
168 .format = img_bsurf.format,
169 .buffer = buffer,
170 .offset = pRegions[r].bufferOffset,
171 .pitch = buf_extent_el.width,
172 };
173
174 if (image->type == VK_IMAGE_TYPE_3D)
175 img_bsurf.layer = img_offset_el.z;
176 /* Loop through each 3D or array slice */
177 unsigned num_slices_3d = img_extent_el.depth;
178 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
179 unsigned slice_3d = 0;
180 unsigned slice_array = 0;
181 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
182
183 rect.dst_x = img_offset_el.x;
184 rect.dst_y = img_offset_el.y;
185
186
187 /* Perform Blit */
188 if (cs)
189 radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, 1, &rect);
190 else
191 radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, 1, &rect);
192
193 /* Once we've done the blit, all of the actual information about
194 * the image is embedded in the command buffer so we can just
195 * increment the offset directly in the image effectively
196 * re-binding it to different backing memory.
197 */
198 buf_bsurf.offset += buf_extent_el.width *
199 buf_extent_el.height * buf_bsurf.bs;
200 img_bsurf.layer++;
201 if (image->type == VK_IMAGE_TYPE_3D)
202 slice_3d++;
203 else
204 slice_array++;
205 }
206 }
207
208 radv_meta_restore(&saved_state, cmd_buffer);
209 }
210
radv_CmdCopyBufferToImage(VkCommandBuffer commandBuffer,VkBuffer srcBuffer,VkImage destImage,VkImageLayout destImageLayout,uint32_t regionCount,const VkBufferImageCopy * pRegions)211 void radv_CmdCopyBufferToImage(
212 VkCommandBuffer commandBuffer,
213 VkBuffer srcBuffer,
214 VkImage destImage,
215 VkImageLayout destImageLayout,
216 uint32_t regionCount,
217 const VkBufferImageCopy* pRegions)
218 {
219 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
220 RADV_FROM_HANDLE(radv_image, dest_image, destImage);
221 RADV_FROM_HANDLE(radv_buffer, src_buffer, srcBuffer);
222
223 meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image, destImageLayout,
224 regionCount, pRegions);
225 }
226
227 static void
meta_copy_image_to_buffer(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer * buffer,struct radv_image * image,VkImageLayout layout,uint32_t regionCount,const VkBufferImageCopy * pRegions)228 meta_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
229 struct radv_buffer* buffer,
230 struct radv_image* image,
231 VkImageLayout layout,
232 uint32_t regionCount,
233 const VkBufferImageCopy* pRegions)
234 {
235 struct radv_meta_saved_state saved_state;
236
237 radv_meta_save(&saved_state, cmd_buffer,
238 RADV_META_SAVE_COMPUTE_PIPELINE |
239 RADV_META_SAVE_CONSTANTS |
240 RADV_META_SAVE_DESCRIPTORS);
241
242 for (unsigned r = 0; r < regionCount; r++) {
243
244 /**
245 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
246 * extent is the size in texels of the source image to copy in width,
247 * height and depth. 1D images use only x and width. 2D images use x, y,
248 * width and height. 3D images use x, y, z, width, height and depth.
249 *
250 *
251 * Also, convert the offsets and extent from units of texels to units of
252 * blocks - which is the highest resolution accessible in this command.
253 */
254 const VkOffset3D img_offset_el =
255 meta_region_offset_el(image, &pRegions[r].imageOffset);
256 const VkExtent3D bufferExtent = {
257 .width = pRegions[r].bufferRowLength ?
258 pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
259 .height = pRegions[r].bufferImageHeight ?
260 pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
261 };
262 const VkExtent3D buf_extent_el =
263 meta_region_extent_el(image, image->type, &bufferExtent);
264
265 /* Start creating blit rect */
266 const VkExtent3D img_extent_el =
267 meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
268 struct radv_meta_blit2d_rect rect = {
269 .width = img_extent_el.width,
270 .height = img_extent_el.height,
271 };
272
273 /* Create blit surfaces */
274 struct radv_meta_blit2d_surf img_info =
275 blit_surf_for_image_level_layer(image,
276 layout,
277 &pRegions[r].imageSubresource);
278
279 struct radv_meta_blit2d_buffer buf_info = {
280 .bs = img_info.bs,
281 .format = img_info.format,
282 .buffer = buffer,
283 .offset = pRegions[r].bufferOffset,
284 .pitch = buf_extent_el.width,
285 };
286
287 if (image->type == VK_IMAGE_TYPE_3D)
288 img_info.layer = img_offset_el.z;
289 /* Loop through each 3D or array slice */
290 unsigned num_slices_3d = img_extent_el.depth;
291 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
292 unsigned slice_3d = 0;
293 unsigned slice_array = 0;
294 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
295
296 rect.src_x = img_offset_el.x;
297 rect.src_y = img_offset_el.y;
298
299
300 /* Perform Blit */
301 radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, 1, &rect);
302
303 buf_info.offset += buf_extent_el.width *
304 buf_extent_el.height * buf_info.bs;
305 img_info.layer++;
306 if (image->type == VK_IMAGE_TYPE_3D)
307 slice_3d++;
308 else
309 slice_array++;
310 }
311 }
312
313 radv_meta_restore(&saved_state, cmd_buffer);
314 }
315
radv_CmdCopyImageToBuffer(VkCommandBuffer commandBuffer,VkImage srcImage,VkImageLayout srcImageLayout,VkBuffer destBuffer,uint32_t regionCount,const VkBufferImageCopy * pRegions)316 void radv_CmdCopyImageToBuffer(
317 VkCommandBuffer commandBuffer,
318 VkImage srcImage,
319 VkImageLayout srcImageLayout,
320 VkBuffer destBuffer,
321 uint32_t regionCount,
322 const VkBufferImageCopy* pRegions)
323 {
324 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
325 RADV_FROM_HANDLE(radv_image, src_image, srcImage);
326 RADV_FROM_HANDLE(radv_buffer, dst_buffer, destBuffer);
327
328 meta_copy_image_to_buffer(cmd_buffer, dst_buffer, src_image,
329 srcImageLayout,
330 regionCount, pRegions);
331 }
332
333 static void
meta_copy_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dest_image,VkImageLayout dest_image_layout,uint32_t regionCount,const VkImageCopy * pRegions)334 meta_copy_image(struct radv_cmd_buffer *cmd_buffer,
335 struct radv_image *src_image,
336 VkImageLayout src_image_layout,
337 struct radv_image *dest_image,
338 VkImageLayout dest_image_layout,
339 uint32_t regionCount,
340 const VkImageCopy *pRegions)
341 {
342 bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
343 struct radv_meta_saved_state saved_state;
344
345 /* From the Vulkan 1.0 spec:
346 *
347 * vkCmdCopyImage can be used to copy image data between multisample
348 * images, but both images must have the same number of samples.
349 */
350 assert(src_image->info.samples == dest_image->info.samples);
351
352 radv_meta_save(&saved_state, cmd_buffer,
353 (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
354 RADV_META_SAVE_GRAPHICS_PIPELINE) |
355 RADV_META_SAVE_CONSTANTS |
356 RADV_META_SAVE_DESCRIPTORS);
357
358 for (unsigned r = 0; r < regionCount; r++) {
359 assert(pRegions[r].srcSubresource.aspectMask ==
360 pRegions[r].dstSubresource.aspectMask);
361
362 /* Create blit surfaces */
363 struct radv_meta_blit2d_surf b_src =
364 blit_surf_for_image_level_layer(src_image,
365 src_image_layout,
366 &pRegions[r].srcSubresource);
367
368 struct radv_meta_blit2d_surf b_dst =
369 blit_surf_for_image_level_layer(dest_image,
370 dest_image_layout,
371 &pRegions[r].dstSubresource);
372
373 uint32_t dst_queue_mask = radv_image_queue_family_mask(dest_image,
374 cmd_buffer->queue_family_index,
375 cmd_buffer->queue_family_index);
376 bool dst_compressed = radv_layout_dcc_compressed(dest_image, dest_image_layout, dst_queue_mask);
377 uint32_t src_queue_mask = radv_image_queue_family_mask(src_image,
378 cmd_buffer->queue_family_index,
379 cmd_buffer->queue_family_index);
380 bool src_compressed = radv_layout_dcc_compressed(src_image, src_image_layout, src_queue_mask);
381
382 if (!src_compressed || radv_dcc_formats_compatible(b_src.format, b_dst.format)) {
383 b_src.format = b_dst.format;
384 } else if (!dst_compressed) {
385 b_dst.format = b_src.format;
386 } else {
387 radv_decompress_dcc(cmd_buffer, dest_image, &(VkImageSubresourceRange) {
388 .aspectMask = pRegions[r].dstSubresource.aspectMask,
389 .baseMipLevel = pRegions[r].dstSubresource.mipLevel,
390 .levelCount = 1,
391 .baseArrayLayer = pRegions[r].dstSubresource.baseArrayLayer,
392 .layerCount = pRegions[r].dstSubresource.layerCount,
393 });
394 b_dst.format = b_src.format;
395 b_dst.current_layout = VK_IMAGE_LAYOUT_GENERAL;
396 }
397
398
399 /**
400 * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
401 * imageExtent is the size in texels of the image to copy in width, height
402 * and depth. 1D images use only x and width. 2D images use x, y, width
403 * and height. 3D images use x, y, z, width, height and depth.
404 *
405 * Also, convert the offsets and extent from units of texels to units of
406 * blocks - which is the highest resolution accessible in this command.
407 */
408 const VkOffset3D dst_offset_el =
409 meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
410 const VkOffset3D src_offset_el =
411 meta_region_offset_el(src_image, &pRegions[r].srcOffset);
412
413 /*
414 * From Vulkan 1.0.68, "Copying Data Between Images":
415 * "When copying between compressed and uncompressed formats
416 * the extent members represent the texel dimensions of the
417 * source image and not the destination."
418 * However, we must use the destination image type to avoid
419 * clamping depth when copying multiple layers of a 2D image to
420 * a 3D image.
421 */
422 const VkExtent3D img_extent_el =
423 meta_region_extent_el(src_image, dest_image->type, &pRegions[r].extent);
424
425 /* Start creating blit rect */
426 struct radv_meta_blit2d_rect rect = {
427 .width = img_extent_el.width,
428 .height = img_extent_el.height,
429 };
430
431 if (src_image->type == VK_IMAGE_TYPE_3D)
432 b_src.layer = src_offset_el.z;
433
434 if (dest_image->type == VK_IMAGE_TYPE_3D)
435 b_dst.layer = dst_offset_el.z;
436
437 /* Loop through each 3D or array slice */
438 unsigned num_slices_3d = img_extent_el.depth;
439 unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
440 unsigned slice_3d = 0;
441 unsigned slice_array = 0;
442 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
443
444 /* Finish creating blit rect */
445 rect.dst_x = dst_offset_el.x;
446 rect.dst_y = dst_offset_el.y;
447 rect.src_x = src_offset_el.x;
448 rect.src_y = src_offset_el.y;
449
450 /* Perform Blit */
451 if (cs)
452 radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, 1, &rect);
453 else
454 radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, 1, &rect);
455
456 b_src.layer++;
457 b_dst.layer++;
458 if (dest_image->type == VK_IMAGE_TYPE_3D)
459 slice_3d++;
460 else
461 slice_array++;
462 }
463 }
464
465 radv_meta_restore(&saved_state, cmd_buffer);
466 }
467
radv_CmdCopyImage(VkCommandBuffer commandBuffer,VkImage srcImage,VkImageLayout srcImageLayout,VkImage destImage,VkImageLayout destImageLayout,uint32_t regionCount,const VkImageCopy * pRegions)468 void radv_CmdCopyImage(
469 VkCommandBuffer commandBuffer,
470 VkImage srcImage,
471 VkImageLayout srcImageLayout,
472 VkImage destImage,
473 VkImageLayout destImageLayout,
474 uint32_t regionCount,
475 const VkImageCopy* pRegions)
476 {
477 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
478 RADV_FROM_HANDLE(radv_image, src_image, srcImage);
479 RADV_FROM_HANDLE(radv_image, dest_image, destImage);
480
481 meta_copy_image(cmd_buffer,
482 src_image, srcImageLayout,
483 dest_image, destImageLayout,
484 regionCount, pRegions);
485 }
486
radv_blit_to_prime_linear(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,struct radv_image * linear_image)487 void radv_blit_to_prime_linear(struct radv_cmd_buffer *cmd_buffer,
488 struct radv_image *image,
489 struct radv_image *linear_image)
490 {
491 struct VkImageCopy image_copy = { 0 };
492
493 image_copy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
494 image_copy.srcSubresource.layerCount = 1;
495
496 image_copy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
497 image_copy.dstSubresource.layerCount = 1;
498
499 image_copy.extent.width = image->info.width;
500 image_copy.extent.height = image->info.height;
501 image_copy.extent.depth = 1;
502
503 meta_copy_image(cmd_buffer, image, VK_IMAGE_LAYOUT_GENERAL, linear_image,
504 VK_IMAGE_LAYOUT_GENERAL,
505 1, &image_copy);
506 }
507