• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2016 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 #include "radv_meta.h"
25 #include "vk_format.h"
26 
27 static VkExtent3D
meta_image_block_size(const struct radv_image * image)28 meta_image_block_size(const struct radv_image *image)
29 {
30 	const struct vk_format_description *desc = vk_format_description(image->vk_format);
31 	return (VkExtent3D) { desc->block.width, desc->block.height, 1 };
32 }
33 
34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
35  * elements rather than texels. One element equals one texel or one block
36  * if Image is uncompressed or compressed, respectively.
37  */
38 static struct VkExtent3D
meta_region_extent_el(const struct radv_image * image,const VkImageType imageType,const struct VkExtent3D * extent)39 meta_region_extent_el(const struct radv_image *image,
40                       const VkImageType imageType,
41                       const struct VkExtent3D *extent)
42 {
43 	const VkExtent3D block = meta_image_block_size(image);
44 	return radv_sanitize_image_extent(imageType, (VkExtent3D) {
45 			.width  = DIV_ROUND_UP(extent->width , block.width),
46 				.height = DIV_ROUND_UP(extent->height, block.height),
47 				.depth  = DIV_ROUND_UP(extent->depth , block.depth),
48 				});
49 }
50 
51 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
52  * elements rather than texels. One element equals one texel or one block
53  * if Image is uncompressed or compressed, respectively.
54  */
55 static struct VkOffset3D
meta_region_offset_el(const struct radv_image * image,const struct VkOffset3D * offset)56 meta_region_offset_el(const struct radv_image *image,
57                       const struct VkOffset3D *offset)
58 {
59 	const VkExtent3D block = meta_image_block_size(image);
60 	return radv_sanitize_image_offset(image->type, (VkOffset3D) {
61 			.x = offset->x / block.width,
62 				.y = offset->y / block.height,
63 				.z = offset->z / block.depth,
64 				});
65 }
66 
67 static VkFormat
vk_format_for_size(int bs)68 vk_format_for_size(int bs)
69 {
70 	switch (bs) {
71 	case 1: return VK_FORMAT_R8_UINT;
72 	case 2: return VK_FORMAT_R8G8_UINT;
73 	case 4: return VK_FORMAT_R8G8B8A8_UINT;
74 	case 8: return VK_FORMAT_R16G16B16A16_UINT;
75 	case 16: return VK_FORMAT_R32G32B32A32_UINT;
76 	default:
77 		unreachable("Invalid format block size");
78 	}
79 }
80 
81 static struct radv_meta_blit2d_surf
blit_surf_for_image_level_layer(struct radv_image * image,VkImageLayout layout,const VkImageSubresourceLayers * subres)82 blit_surf_for_image_level_layer(struct radv_image *image,
83 				VkImageLayout layout,
84 				const VkImageSubresourceLayers *subres)
85 {
86 	VkFormat format = image->vk_format;
87 	if (subres->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT)
88 		format = vk_format_depth_only(format);
89 	else if (subres->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT)
90 		format = vk_format_stencil_only(format);
91 
92 	if (!image->surface.dcc_size &&
93 	    !(image->surface.htile_size && image->tc_compatible_htile))
94 		format = vk_format_for_size(vk_format_get_blocksize(format));
95 
96 	return (struct radv_meta_blit2d_surf) {
97 		.format = format,
98 		.bs = vk_format_get_blocksize(format),
99 		.level = subres->mipLevel,
100 		.layer = subres->baseArrayLayer,
101 		.image = image,
102 		.aspect_mask = subres->aspectMask,
103 		.current_layout = layout,
104 	};
105 }
106 
107 static void
meta_copy_buffer_to_image(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer * buffer,struct radv_image * image,VkImageLayout layout,uint32_t regionCount,const VkBufferImageCopy * pRegions)108 meta_copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer,
109                           struct radv_buffer* buffer,
110                           struct radv_image* image,
111 			  VkImageLayout layout,
112                           uint32_t regionCount,
113                           const VkBufferImageCopy* pRegions)
114 {
115 	bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
116 	struct radv_meta_saved_state saved_state;
117 
118 	/* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
119 	 * VK_SAMPLE_COUNT_1_BIT."
120 	 */
121 	assert(image->info.samples == 1);
122 
123 	radv_meta_save(&saved_state, cmd_buffer,
124 		       (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
125 			RADV_META_SAVE_GRAPHICS_PIPELINE) |
126 		       RADV_META_SAVE_CONSTANTS |
127 		       RADV_META_SAVE_DESCRIPTORS);
128 
129 	for (unsigned r = 0; r < regionCount; r++) {
130 
131 		/**
132 		 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
133 		 *    extent is the size in texels of the source image to copy in width,
134 		 *    height and depth. 1D images use only x and width. 2D images use x, y,
135 		 *    width and height. 3D images use x, y, z, width, height and depth.
136 		 *
137 		 *
138 		 * Also, convert the offsets and extent from units of texels to units of
139 		 * blocks - which is the highest resolution accessible in this command.
140 		 */
141 		const VkOffset3D img_offset_el =
142 			meta_region_offset_el(image, &pRegions[r].imageOffset);
143 		const VkExtent3D bufferExtent = {
144 			.width  = pRegions[r].bufferRowLength ?
145 			pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
146 			.height = pRegions[r].bufferImageHeight ?
147 			pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
148 		};
149 		const VkExtent3D buf_extent_el =
150 			meta_region_extent_el(image, image->type, &bufferExtent);
151 
152 		/* Start creating blit rect */
153 		const VkExtent3D img_extent_el =
154 			meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
155 		struct radv_meta_blit2d_rect rect = {
156 			.width = img_extent_el.width,
157 			.height =  img_extent_el.height,
158 		};
159 
160 		/* Create blit surfaces */
161 		struct radv_meta_blit2d_surf img_bsurf =
162 			blit_surf_for_image_level_layer(image,
163 							layout,
164 							&pRegions[r].imageSubresource);
165 
166 		struct radv_meta_blit2d_buffer buf_bsurf = {
167 			.bs = img_bsurf.bs,
168 			.format = img_bsurf.format,
169 			.buffer = buffer,
170 			.offset = pRegions[r].bufferOffset,
171 			.pitch = buf_extent_el.width,
172 		};
173 
174 		if (image->type == VK_IMAGE_TYPE_3D)
175 			img_bsurf.layer = img_offset_el.z;
176 		/* Loop through each 3D or array slice */
177 		unsigned num_slices_3d = img_extent_el.depth;
178 		unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
179 		unsigned slice_3d = 0;
180 		unsigned slice_array = 0;
181 		while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
182 
183 			rect.dst_x = img_offset_el.x;
184 			rect.dst_y = img_offset_el.y;
185 
186 
187 			/* Perform Blit */
188 			if (cs)
189 				radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, 1, &rect);
190 			else
191 				radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, 1, &rect);
192 
193 			/* Once we've done the blit, all of the actual information about
194 			 * the image is embedded in the command buffer so we can just
195 			 * increment the offset directly in the image effectively
196 			 * re-binding it to different backing memory.
197 			 */
198 			buf_bsurf.offset += buf_extent_el.width *
199 			                    buf_extent_el.height * buf_bsurf.bs;
200 			img_bsurf.layer++;
201 			if (image->type == VK_IMAGE_TYPE_3D)
202 				slice_3d++;
203 			else
204 				slice_array++;
205 		}
206 	}
207 
208 	radv_meta_restore(&saved_state, cmd_buffer);
209 }
210 
radv_CmdCopyBufferToImage(VkCommandBuffer commandBuffer,VkBuffer srcBuffer,VkImage destImage,VkImageLayout destImageLayout,uint32_t regionCount,const VkBufferImageCopy * pRegions)211 void radv_CmdCopyBufferToImage(
212 	VkCommandBuffer                             commandBuffer,
213 	VkBuffer                                    srcBuffer,
214 	VkImage                                     destImage,
215 	VkImageLayout                               destImageLayout,
216 	uint32_t                                    regionCount,
217 	const VkBufferImageCopy*                    pRegions)
218 {
219 	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
220 	RADV_FROM_HANDLE(radv_image, dest_image, destImage);
221 	RADV_FROM_HANDLE(radv_buffer, src_buffer, srcBuffer);
222 
223 	meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image, destImageLayout,
224 				  regionCount, pRegions);
225 }
226 
227 static void
meta_copy_image_to_buffer(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer * buffer,struct radv_image * image,VkImageLayout layout,uint32_t regionCount,const VkBufferImageCopy * pRegions)228 meta_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
229                           struct radv_buffer* buffer,
230                           struct radv_image* image,
231 			  VkImageLayout layout,
232                           uint32_t regionCount,
233                           const VkBufferImageCopy* pRegions)
234 {
235 	struct radv_meta_saved_state saved_state;
236 
237 	radv_meta_save(&saved_state, cmd_buffer,
238 		       RADV_META_SAVE_COMPUTE_PIPELINE |
239 		       RADV_META_SAVE_CONSTANTS |
240 		       RADV_META_SAVE_DESCRIPTORS);
241 
242 	for (unsigned r = 0; r < regionCount; r++) {
243 
244 		/**
245 		 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
246 		 *    extent is the size in texels of the source image to copy in width,
247 		 *    height and depth. 1D images use only x and width. 2D images use x, y,
248 		 *    width and height. 3D images use x, y, z, width, height and depth.
249 		 *
250 		 *
251 		 * Also, convert the offsets and extent from units of texels to units of
252 		 * blocks - which is the highest resolution accessible in this command.
253 		 */
254 		const VkOffset3D img_offset_el =
255 			meta_region_offset_el(image, &pRegions[r].imageOffset);
256 		const VkExtent3D bufferExtent = {
257 			.width  = pRegions[r].bufferRowLength ?
258 			pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
259 			.height = pRegions[r].bufferImageHeight ?
260 			pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
261 		};
262 		const VkExtent3D buf_extent_el =
263 			meta_region_extent_el(image, image->type, &bufferExtent);
264 
265 		/* Start creating blit rect */
266 		const VkExtent3D img_extent_el =
267 			meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
268 		struct radv_meta_blit2d_rect rect = {
269 			.width = img_extent_el.width,
270 			.height =  img_extent_el.height,
271 		};
272 
273 		/* Create blit surfaces */
274 		struct radv_meta_blit2d_surf img_info =
275 			blit_surf_for_image_level_layer(image,
276 							layout,
277 							&pRegions[r].imageSubresource);
278 
279 		struct radv_meta_blit2d_buffer buf_info = {
280 			.bs = img_info.bs,
281 			.format = img_info.format,
282 			.buffer = buffer,
283 			.offset = pRegions[r].bufferOffset,
284 			.pitch = buf_extent_el.width,
285 		};
286 
287 		if (image->type == VK_IMAGE_TYPE_3D)
288 			img_info.layer = img_offset_el.z;
289 		/* Loop through each 3D or array slice */
290 		unsigned num_slices_3d = img_extent_el.depth;
291 		unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
292 		unsigned slice_3d = 0;
293 		unsigned slice_array = 0;
294 		while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
295 
296 			rect.src_x = img_offset_el.x;
297 			rect.src_y = img_offset_el.y;
298 
299 
300 			/* Perform Blit */
301 			radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, 1, &rect);
302 
303 			buf_info.offset += buf_extent_el.width *
304 			                    buf_extent_el.height * buf_info.bs;
305 			img_info.layer++;
306 			if (image->type == VK_IMAGE_TYPE_3D)
307 				slice_3d++;
308 			else
309 				slice_array++;
310 		}
311 	}
312 
313 	radv_meta_restore(&saved_state, cmd_buffer);
314 }
315 
radv_CmdCopyImageToBuffer(VkCommandBuffer commandBuffer,VkImage srcImage,VkImageLayout srcImageLayout,VkBuffer destBuffer,uint32_t regionCount,const VkBufferImageCopy * pRegions)316 void radv_CmdCopyImageToBuffer(
317 	VkCommandBuffer                             commandBuffer,
318 	VkImage                                     srcImage,
319 	VkImageLayout                               srcImageLayout,
320 	VkBuffer                                    destBuffer,
321 	uint32_t                                    regionCount,
322 	const VkBufferImageCopy*                    pRegions)
323 {
324 	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
325 	RADV_FROM_HANDLE(radv_image, src_image, srcImage);
326 	RADV_FROM_HANDLE(radv_buffer, dst_buffer, destBuffer);
327 
328 	meta_copy_image_to_buffer(cmd_buffer, dst_buffer, src_image,
329 				  srcImageLayout,
330 				  regionCount, pRegions);
331 }
332 
333 static void
meta_copy_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dest_image,VkImageLayout dest_image_layout,uint32_t regionCount,const VkImageCopy * pRegions)334 meta_copy_image(struct radv_cmd_buffer *cmd_buffer,
335 		struct radv_image *src_image,
336 		VkImageLayout src_image_layout,
337 		struct radv_image *dest_image,
338 		VkImageLayout dest_image_layout,
339 		uint32_t regionCount,
340 		const VkImageCopy *pRegions)
341 {
342 	bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
343 	struct radv_meta_saved_state saved_state;
344 
345 	/* From the Vulkan 1.0 spec:
346 	 *
347 	 *    vkCmdCopyImage can be used to copy image data between multisample
348 	 *    images, but both images must have the same number of samples.
349 	 */
350 	assert(src_image->info.samples == dest_image->info.samples);
351 
352 	radv_meta_save(&saved_state, cmd_buffer,
353 		       (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
354 			RADV_META_SAVE_GRAPHICS_PIPELINE) |
355 		       RADV_META_SAVE_CONSTANTS |
356 		       RADV_META_SAVE_DESCRIPTORS);
357 
358 	for (unsigned r = 0; r < regionCount; r++) {
359 		assert(pRegions[r].srcSubresource.aspectMask ==
360 		       pRegions[r].dstSubresource.aspectMask);
361 
362 		/* Create blit surfaces */
363 		struct radv_meta_blit2d_surf b_src =
364 			blit_surf_for_image_level_layer(src_image,
365 							src_image_layout,
366 							&pRegions[r].srcSubresource);
367 
368 		struct radv_meta_blit2d_surf b_dst =
369 			blit_surf_for_image_level_layer(dest_image,
370 							dest_image_layout,
371 							&pRegions[r].dstSubresource);
372 
373 		uint32_t dst_queue_mask = radv_image_queue_family_mask(dest_image,
374 		                                                       cmd_buffer->queue_family_index,
375 		                                                       cmd_buffer->queue_family_index);
376 		bool dst_compressed = radv_layout_dcc_compressed(dest_image, dest_image_layout, dst_queue_mask);
377 		uint32_t src_queue_mask = radv_image_queue_family_mask(src_image,
378 		                                                       cmd_buffer->queue_family_index,
379 		                                                       cmd_buffer->queue_family_index);
380 		bool src_compressed = radv_layout_dcc_compressed(src_image, src_image_layout, src_queue_mask);
381 
382 		if (!src_compressed || radv_dcc_formats_compatible(b_src.format, b_dst.format)) {
383 			b_src.format = b_dst.format;
384 		} else if (!dst_compressed) {
385 			b_dst.format = b_src.format;
386 		} else {
387 			radv_decompress_dcc(cmd_buffer, dest_image, &(VkImageSubresourceRange) {
388 			                        .aspectMask = pRegions[r].dstSubresource.aspectMask,
389 			                        .baseMipLevel = pRegions[r].dstSubresource.mipLevel,
390 			                        .levelCount = 1,
391 			                        .baseArrayLayer = pRegions[r].dstSubresource.baseArrayLayer,
392 			                        .layerCount = pRegions[r].dstSubresource.layerCount,
393 			                    });
394 			b_dst.format = b_src.format;
395 			b_dst.current_layout = VK_IMAGE_LAYOUT_GENERAL;
396 		}
397 
398 
399 		/**
400 		 * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
401 		 *    imageExtent is the size in texels of the image to copy in width, height
402 		 *    and depth. 1D images use only x and width. 2D images use x, y, width
403 		 *    and height. 3D images use x, y, z, width, height and depth.
404 		 *
405 		 * Also, convert the offsets and extent from units of texels to units of
406 		 * blocks - which is the highest resolution accessible in this command.
407 		 */
408 		const VkOffset3D dst_offset_el =
409 			meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
410 		const VkOffset3D src_offset_el =
411 			meta_region_offset_el(src_image, &pRegions[r].srcOffset);
412 
413 		/*
414 		 * From Vulkan 1.0.68, "Copying Data Between Images":
415 		 *    "When copying between compressed and uncompressed formats
416 		 *     the extent members represent the texel dimensions of the
417 		 *     source image and not the destination."
418 		 * However, we must use the destination image type to avoid
419 		 * clamping depth when copying multiple layers of a 2D image to
420 		 * a 3D image.
421 		 */
422 		const VkExtent3D img_extent_el =
423 			meta_region_extent_el(src_image, dest_image->type, &pRegions[r].extent);
424 
425 		/* Start creating blit rect */
426 		struct radv_meta_blit2d_rect rect = {
427 			.width = img_extent_el.width,
428 			.height = img_extent_el.height,
429 		};
430 
431 		if (src_image->type == VK_IMAGE_TYPE_3D)
432 			b_src.layer = src_offset_el.z;
433 
434 		if (dest_image->type == VK_IMAGE_TYPE_3D)
435 			b_dst.layer = dst_offset_el.z;
436 
437 		/* Loop through each 3D or array slice */
438 		unsigned num_slices_3d = img_extent_el.depth;
439 		unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
440 		unsigned slice_3d = 0;
441 		unsigned slice_array = 0;
442 		while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
443 
444 			/* Finish creating blit rect */
445 			rect.dst_x = dst_offset_el.x;
446 			rect.dst_y = dst_offset_el.y;
447 			rect.src_x = src_offset_el.x;
448 			rect.src_y = src_offset_el.y;
449 
450 			/* Perform Blit */
451 			if (cs)
452 				radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, 1, &rect);
453 			else
454 				radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, 1, &rect);
455 
456 			b_src.layer++;
457 			b_dst.layer++;
458 			if (dest_image->type == VK_IMAGE_TYPE_3D)
459 				slice_3d++;
460 			else
461 				slice_array++;
462 		}
463 	}
464 
465 	radv_meta_restore(&saved_state, cmd_buffer);
466 }
467 
radv_CmdCopyImage(VkCommandBuffer commandBuffer,VkImage srcImage,VkImageLayout srcImageLayout,VkImage destImage,VkImageLayout destImageLayout,uint32_t regionCount,const VkImageCopy * pRegions)468 void radv_CmdCopyImage(
469 	VkCommandBuffer                             commandBuffer,
470 	VkImage                                     srcImage,
471 	VkImageLayout                               srcImageLayout,
472 	VkImage                                     destImage,
473 	VkImageLayout                               destImageLayout,
474 	uint32_t                                    regionCount,
475 	const VkImageCopy*                          pRegions)
476 {
477 	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
478 	RADV_FROM_HANDLE(radv_image, src_image, srcImage);
479 	RADV_FROM_HANDLE(radv_image, dest_image, destImage);
480 
481 	meta_copy_image(cmd_buffer,
482 			src_image, srcImageLayout,
483 			dest_image, destImageLayout,
484 			regionCount, pRegions);
485 }
486 
radv_blit_to_prime_linear(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,struct radv_image * linear_image)487 void radv_blit_to_prime_linear(struct radv_cmd_buffer *cmd_buffer,
488 			       struct radv_image *image,
489 			       struct radv_image *linear_image)
490 {
491 	struct VkImageCopy image_copy = { 0 };
492 
493 	image_copy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
494 	image_copy.srcSubresource.layerCount = 1;
495 
496 	image_copy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
497 	image_copy.dstSubresource.layerCount = 1;
498 
499 	image_copy.extent.width = image->info.width;
500 	image_copy.extent.height = image->info.height;
501 	image_copy.extent.depth = 1;
502 
503 	meta_copy_image(cmd_buffer, image, VK_IMAGE_LAYOUT_GENERAL, linear_image,
504 			VK_IMAGE_LAYOUT_GENERAL,
505 			1, &image_copy);
506 }
507