Lines Matching refs:cs
65 r2d_coords(struct tu_cs *cs, in r2d_coords() argument
70 tu_cs_emit_regs(cs, in r2d_coords()
77 tu_cs_emit_regs(cs, in r2d_coords()
85 r2d_clear_value(struct tu_cs *cs, VkFormat format, const VkClearValue *val) in r2d_clear_value() argument
140 tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_SRC_SOLID_C0, 4); in r2d_clear_value()
141 tu_cs_emit_array(cs, clear_value, 4); in r2d_clear_value()
146 struct tu_cs *cs, in r2d_src() argument
155 tu_cs_emit_pkt4(cs, REG_A6XX_SP_PS_2D_SRC_INFO, 5); in r2d_src()
156 tu_cs_emit(cs, src_info); in r2d_src()
157 tu_cs_emit(cs, iview->SP_PS_2D_SRC_SIZE); in r2d_src()
158 tu_cs_image_ref_2d(cs, iview, layer, true); in r2d_src()
160 tu_cs_emit_pkt4(cs, REG_A6XX_SP_PS_2D_SRC_FLAGS, 3); in r2d_src()
161 tu_cs_image_flag_ref(cs, iview, layer); in r2d_src()
166 struct tu_cs *cs, in r2d_src_stencil() argument
171 tu_cs_emit_pkt4(cs, REG_A6XX_SP_PS_2D_SRC_INFO, 5); in r2d_src_stencil()
172 tu_cs_emit(cs, tu_image_view_stencil(iview, SP_PS_2D_SRC_INFO) & ~A6XX_SP_PS_2D_SRC_INFO_FLAGS); in r2d_src_stencil()
173 tu_cs_emit(cs, iview->SP_PS_2D_SRC_SIZE); in r2d_src_stencil()
174 tu_cs_emit_qw(cs, iview->stencil_base_addr + iview->stencil_layer_size * layer); in r2d_src_stencil()
176 tu_cs_emit(cs, iview->stencil_PITCH << 9); in r2d_src_stencil()
181 struct tu_cs *cs, in r2d_src_buffer() argument
188 tu_cs_emit_regs(cs, in r2d_src_buffer()
201 r2d_dst(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer) in r2d_dst() argument
203 tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_INFO, 4); in r2d_dst()
204 tu_cs_emit(cs, iview->RB_2D_DST_INFO); in r2d_dst()
205 tu_cs_image_ref_2d(cs, iview, layer, false); in r2d_dst()
207 tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_FLAGS, 3); in r2d_dst()
208 tu_cs_image_flag_ref(cs, iview, layer); in r2d_dst()
212 r2d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer) in r2d_dst_stencil() argument
214 tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_INFO, 4); in r2d_dst_stencil()
215 tu_cs_emit(cs, tu_image_view_stencil(iview, RB_2D_DST_INFO) & ~A6XX_RB_2D_DST_INFO_FLAGS); in r2d_dst_stencil()
216 tu_cs_emit_qw(cs, iview->stencil_base_addr + iview->stencil_layer_size * layer); in r2d_dst_stencil()
217 tu_cs_emit(cs, iview->stencil_PITCH); in r2d_dst_stencil()
221 r2d_dst_buffer(struct tu_cs *cs, VkFormat vk_format, uint64_t va, uint32_t pitch) in r2d_dst_buffer() argument
225 tu_cs_emit_regs(cs, in r2d_dst_buffer()
236 struct tu_cs *cs, in r2d_setup_common() argument
263 tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_UNKNOWN_8C01, 1); in r2d_setup_common()
264 tu_cs_emit(cs, unknown_8c01); in r2d_setup_common()
276 tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_BLIT_CNTL, 1); in r2d_setup_common()
277 tu_cs_emit(cs, blit_cntl); in r2d_setup_common()
279 tu_cs_emit_pkt4(cs, REG_A6XX_GRAS_2D_BLIT_CNTL, 1); in r2d_setup_common()
280 tu_cs_emit(cs, blit_cntl); in r2d_setup_common()
285 tu_cs_emit_regs(cs, A6XX_SP_2D_DST_FORMAT( in r2d_setup_common()
295 struct tu_cs *cs, in r2d_setup() argument
305 tu_emit_cache_flush_ccu(cmd, cs, TU_CMD_CCU_SYSMEM); in r2d_setup()
307 r2d_setup_common(cmd, cs, vk_format, aspect_mask, blit_param, clear, ubwc, false); in r2d_setup()
312 struct tu_cs *cs) in r2d_teardown() argument
318 r2d_run(struct tu_cmd_buffer *cmd, struct tu_cs *cs) in r2d_run() argument
320 tu_cs_emit_pkt7(cs, CP_BLIT, 1); in r2d_run()
321 tu_cs_emit(cs, CP_BLIT_0_OP(BLIT_OP_SCALE)); in r2d_run()
594 r3d_common(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool blit, in r3d_common() argument
617 tu_cs_emit_regs(cs, A6XX_HLSQ_INVALIDATE_CMD( in r3d_common()
630 tu6_emit_xs_config(cs, MESA_SHADER_VERTEX, vs); in r3d_common()
631 tu6_emit_xs_config(cs, MESA_SHADER_TESS_CTRL, NULL); in r3d_common()
632 tu6_emit_xs_config(cs, MESA_SHADER_TESS_EVAL, NULL); in r3d_common()
633 tu6_emit_xs_config(cs, MESA_SHADER_GEOMETRY, NULL); in r3d_common()
634 tu6_emit_xs_config(cs, MESA_SHADER_FRAGMENT, fs); in r3d_common()
637 tu6_emit_xs(cs, MESA_SHADER_VERTEX, vs, &pvtmem, vs_iova); in r3d_common()
638 tu6_emit_xs(cs, MESA_SHADER_FRAGMENT, fs, &pvtmem, fs_iova); in r3d_common()
640 tu_cs_emit_regs(cs, A6XX_PC_PRIMITIVE_CNTL_0()); in r3d_common()
641 tu_cs_emit_regs(cs, A6XX_VFD_CONTROL_0()); in r3d_common()
648 tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3); in r3d_common()
649 tu_cs_emit(cs, CP_REG_WRITE_0_TRACKER(UNK_EVENT_WRITE)); in r3d_common()
650 tu_cs_emit(cs, REG_A6XX_PC_MULTIVIEW_CNTL); in r3d_common()
651 tu_cs_emit(cs, 0); in r3d_common()
653 tu_cs_emit_regs(cs, A6XX_PC_MULTIVIEW_CNTL()); in r3d_common()
655 tu_cs_emit_regs(cs, A6XX_VFD_MULTIVIEW_CNTL()); in r3d_common()
657 tu6_emit_vpc(cs, vs, NULL, NULL, NULL, fs, 0); in r3d_common()
660 tu_cs_emit_regs(cs, A6XX_VPC_VARYING_INTERP_MODE(0, 0)); in r3d_common()
661 tu_cs_emit_regs(cs, A6XX_VPC_VARYING_PS_REPL_MODE(0, 2 << 2 | 1 << 0)); in r3d_common()
663 tu6_emit_fs_inputs(cs, fs); in r3d_common()
665 tu_cs_emit_regs(cs, in r3d_common()
671 tu_cs_emit_regs(cs, A6XX_GRAS_SU_CNTL()); // XXX msaa enable? in r3d_common()
673 tu_cs_emit_regs(cs, A6XX_PC_RASTER_CNTL()); in r3d_common()
674 tu_cs_emit_regs(cs, A6XX_VPC_UNKNOWN_9107()); in r3d_common()
676 tu_cs_emit_regs(cs, in r3d_common()
679 tu_cs_emit_regs(cs, in r3d_common()
683 tu_cs_emit_regs(cs, in r3d_common()
689 tu_cs_emit_pkt4(cs, REG_A6XX_SP_FS_OUTPUT_REG(0), rts_count); in r3d_common()
695 tu_cs_emit(cs, A6XX_SP_FS_OUTPUT_REG_REGID(regid)); in r3d_common()
700 tu6_emit_msaa(cs, samples, cmd->state.line_mode); in r3d_common()
704 r3d_coords_raw(struct tu_cs *cs, const float *coords) in r3d_coords_raw() argument
706 tu_cs_emit_pkt7(cs, CP_LOAD_STATE6_GEOM, 3 + 8); in r3d_coords_raw()
707 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(0) | in r3d_coords_raw()
712 tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0)); in r3d_coords_raw()
713 tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0)); in r3d_coords_raw()
714 tu_cs_emit_array(cs, (const uint32_t *) coords, 8); in r3d_coords_raw()
719 r3d_coord_z(struct tu_cs *cs, float z) in r3d_coord_z() argument
721 tu_cs_emit_pkt7(cs, CP_LOAD_STATE6_GEOM, 3 + 4); in r3d_coord_z()
722 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(2) | in r3d_coord_z()
727 tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0)); in r3d_coord_z()
728 tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0)); in r3d_coord_z()
729 tu_cs_emit(cs, fui(z)); in r3d_coord_z()
730 tu_cs_emit(cs, 0); in r3d_coord_z()
731 tu_cs_emit(cs, 0); in r3d_coord_z()
732 tu_cs_emit(cs, 0); in r3d_coord_z()
736 r3d_coords(struct tu_cs *cs, in r3d_coords() argument
743 r3d_coords_raw(cs, (float[]) { in r3d_coords()
752 r3d_clear_value(struct tu_cs *cs, VkFormat format, const VkClearValue *val) in r3d_clear_value() argument
754 tu_cs_emit_pkt7(cs, CP_LOAD_STATE6_FRAG, 3 + 4); in r3d_clear_value()
755 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(0) | in r3d_clear_value()
760 tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0)); in r3d_clear_value()
761 tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0)); in r3d_clear_value()
767 tu_cs_emit(cs, fui((tmp & 0xff) / 255.0f)); in r3d_clear_value()
768 tu_cs_emit(cs, fui((tmp >> 8 & 0xff) / 255.0f)); in r3d_clear_value()
769 tu_cs_emit(cs, fui((tmp >> 16 & 0xff) / 255.0f)); in r3d_clear_value()
770 tu_cs_emit(cs, fui((val->depthStencil.stencil & 0xff) / 255.0f)); in r3d_clear_value()
774 tu_cs_emit(cs, fui(val->depthStencil.depth)); in r3d_clear_value()
775 tu_cs_emit(cs, 0); in r3d_clear_value()
776 tu_cs_emit(cs, 0); in r3d_clear_value()
777 tu_cs_emit(cs, 0); in r3d_clear_value()
780 tu_cs_emit(cs, val->depthStencil.stencil & 0xff); in r3d_clear_value()
781 tu_cs_emit(cs, 0); in r3d_clear_value()
782 tu_cs_emit(cs, 0); in r3d_clear_value()
783 tu_cs_emit(cs, 0); in r3d_clear_value()
788 tu_cs_emit_array(cs, val->color.uint32, 4); in r3d_clear_value()
795 struct tu_cs *cs, in r3d_src_common() argument
832 tu_cs_emit_pkt7(cs, CP_LOAD_STATE6_FRAG, 3); in r3d_src_common()
833 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(0) | in r3d_src_common()
838 tu_cs_emit_qw(cs, texture.iova + A6XX_TEX_CONST_DWORDS * 4); in r3d_src_common()
840 tu_cs_emit_regs(cs, A6XX_SP_FS_TEX_SAMP(.qword = texture.iova + A6XX_TEX_CONST_DWORDS * 4)); in r3d_src_common()
842 tu_cs_emit_pkt7(cs, CP_LOAD_STATE6_FRAG, 3); in r3d_src_common()
843 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(0) | in r3d_src_common()
848 tu_cs_emit_qw(cs, texture.iova); in r3d_src_common()
850 tu_cs_emit_regs(cs, A6XX_SP_FS_TEX_CONST(.qword = texture.iova)); in r3d_src_common()
851 tu_cs_emit_regs(cs, A6XX_SP_FS_TEX_COUNT(1)); in r3d_src_common()
856 struct tu_cs *cs, in r3d_src() argument
861 r3d_src_common(cmd, cs, iview->descriptor, in r3d_src()
869 struct tu_cs *cs, in r3d_src_buffer() argument
897 r3d_src_common(cmd, cs, desc, 0, 0, VK_FILTER_NEAREST); in r3d_src_buffer()
902 struct tu_cs *cs, in r3d_src_gmem() argument
927 r3d_src_common(cmd, cs, desc, 0, 0, VK_FILTER_NEAREST); in r3d_src_gmem()
931 r3d_dst(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer) in r3d_dst() argument
933 tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_BUF_INFO(0), 6); in r3d_dst()
934 tu_cs_emit(cs, iview->RB_MRT_BUF_INFO); in r3d_dst()
935 tu_cs_image_ref(cs, iview, layer); in r3d_dst()
936 tu_cs_emit(cs, 0); in r3d_dst()
938 tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_FLAG_BUFFER(0), 3); in r3d_dst()
939 tu_cs_image_flag_ref(cs, iview, layer); in r3d_dst()
941 tu_cs_emit_regs(cs, A6XX_RB_RENDER_CNTL(.flag_mrts = iview->ubwc_enabled)); in r3d_dst()
945 r3d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer) in r3d_dst_stencil() argument
947 tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_BUF_INFO(0), 6); in r3d_dst_stencil()
948 tu_cs_emit(cs, tu_image_view_stencil(iview, RB_MRT_BUF_INFO)); in r3d_dst_stencil()
949 tu_cs_image_stencil_ref(cs, iview, layer); in r3d_dst_stencil()
950 tu_cs_emit(cs, 0); in r3d_dst_stencil()
952 tu_cs_emit_regs(cs, A6XX_RB_RENDER_CNTL()); in r3d_dst_stencil()
956 r3d_dst_buffer(struct tu_cs *cs, VkFormat vk_format, uint64_t va, uint32_t pitch) in r3d_dst_buffer() argument
960 tu_cs_emit_regs(cs, in r3d_dst_buffer()
967 tu_cs_emit_regs(cs, A6XX_RB_RENDER_CNTL()); in r3d_dst_buffer()
989 struct tu_cs *cs, in r3d_setup() argument
1005 tu_emit_cache_flush_ccu(cmd, cs, TU_CMD_CCU_SYSMEM); in r3d_setup()
1006 tu6_emit_window_scissor(cs, 0, 0, 0x3fff, 0x3fff); in r3d_setup()
1009 tu_cs_emit_regs(cs, A6XX_GRAS_BIN_CONTROL(.dword = 0xc00000)); in r3d_setup()
1010 tu_cs_emit_regs(cs, A6XX_RB_BIN_CONTROL(.dword = 0xc00000)); in r3d_setup()
1012 r3d_common(cmd, cs, !clear, 1, blit_param, samples); in r3d_setup()
1014 tu_cs_emit_pkt4(cs, REG_A6XX_SP_FS_OUTPUT_CNTL0, 2); in r3d_setup()
1015 tu_cs_emit(cs, A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID(0xfc) | in r3d_setup()
1018 tu_cs_emit(cs, A6XX_SP_FS_OUTPUT_CNTL1_MRT(1)); in r3d_setup()
1020 tu_cs_emit_regs(cs, in r3d_setup()
1024 tu_cs_emit_regs(cs, A6XX_SP_BLEND_CNTL()); in r3d_setup()
1025 tu_cs_emit_regs(cs, A6XX_RB_BLEND_CNTL(.sample_mask = 0xffff)); in r3d_setup()
1027 tu_cs_emit_regs(cs, A6XX_RB_DEPTH_PLANE_CNTL()); in r3d_setup()
1028 tu_cs_emit_regs(cs, A6XX_RB_DEPTH_CNTL()); in r3d_setup()
1029 tu_cs_emit_regs(cs, A6XX_GRAS_SU_DEPTH_PLANE_CNTL()); in r3d_setup()
1030 tu_cs_emit_regs(cs, A6XX_RB_STENCIL_CONTROL()); in r3d_setup()
1031 tu_cs_emit_regs(cs, A6XX_RB_STENCILMASK()); in r3d_setup()
1032 tu_cs_emit_regs(cs, A6XX_RB_STENCILWRMASK()); in r3d_setup()
1033 tu_cs_emit_regs(cs, A6XX_RB_STENCILREF()); in r3d_setup()
1035 tu_cs_emit_regs(cs, A6XX_RB_RENDER_COMPONENTS(.rt0 = 0xf)); in r3d_setup()
1036 tu_cs_emit_regs(cs, A6XX_SP_FS_RENDER_COMPONENTS(.rt0 = 0xf)); in r3d_setup()
1038 tu_cs_emit_regs(cs, A6XX_SP_FS_MRT_REG(0, in r3d_setup()
1043 tu_cs_emit_regs(cs, A6XX_RB_MRT_CONTROL(0, in r3d_setup()
1045 tu_cs_emit_regs(cs, A6XX_RB_SRGB_CNTL(vk_format_is_srgb(vk_format))); in r3d_setup()
1046 tu_cs_emit_regs(cs, A6XX_SP_SRGB_CNTL(vk_format_is_srgb(vk_format))); in r3d_setup()
1048 tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_CNTL(0)); in r3d_setup()
1049 tu_cs_emit_regs(cs, A6XX_RB_LRZ_CNTL(0)); in r3d_setup()
1051 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_SC_CNTL, in r3d_setup()
1055 tu_cs_emit_pkt7(cs, CP_DRAW_PRED_ENABLE_LOCAL, 1); in r3d_setup()
1056 tu_cs_emit(cs, 0); in r3d_setup()
1061 r3d_run(struct tu_cmd_buffer *cmd, struct tu_cs *cs) in r3d_run() argument
1063 tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3); in r3d_run()
1064 tu_cs_emit(cs, CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(DI_PT_RECTLIST) | in r3d_run()
1067 tu_cs_emit(cs, 1); /* instance count */ in r3d_run()
1068 tu_cs_emit(cs, 2); /* vertex count */ in r3d_run()
1072 r3d_teardown(struct tu_cmd_buffer *cmd, struct tu_cs *cs) in r3d_teardown() argument
1075 tu_cs_emit_pkt7(cs, CP_DRAW_PRED_ENABLE_LOCAL, 1); in r3d_teardown()
1076 tu_cs_emit(cs, 1); in r3d_teardown()
1083 void (*coords)(struct tu_cs *cs,
1087 void (*clear_value)(struct tu_cs *cs, VkFormat format, const VkClearValue *val);
1090 struct tu_cs *cs,
1094 void (*src_buffer)(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
1098 void (*dst)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1099 void (*dst_buffer)(struct tu_cs *cs, VkFormat vk_format, uint64_t va, uint32_t pitch);
1101 struct tu_cs *cs,
1108 void (*run)(struct tu_cmd_buffer *cmd, struct tu_cs *cs);
1110 struct tu_cs *cs);
1140 struct tu_cs *cs, in coords() argument
1145 ops->coords(cs, (const VkOffset2D*) dst, (const VkOffset2D*) src, (const VkExtent2D*) extent); in coords()
1231 struct tu_cs *cs, in tu6_clear_lrz() argument
1237 ops->setup(cmd, cs, VK_FORMAT_D16_UNORM, VK_IMAGE_ASPECT_DEPTH_BIT, 0, true, false, in tu6_clear_lrz()
1239 ops->clear_value(cs, VK_FORMAT_D16_UNORM, value); in tu6_clear_lrz()
1240 ops->dst_buffer(cs, VK_FORMAT_D16_UNORM, in tu6_clear_lrz()
1243 ops->coords(cs, &(VkOffset2D) {}, NULL, &(VkExtent2D) {image->lrz_pitch, image->lrz_height}); in tu6_clear_lrz()
1244 ops->run(cmd, cs); in tu6_clear_lrz()
1245 ops->teardown(cmd, cs); in tu6_clear_lrz()
1310 struct tu_cs *cs = &cmd->cs; in tu6_blit_image() local
1376 trace_start_blit(&cmd->trace, cs); in tu6_blit_image()
1378 ops->setup(cmd, cs, format, info->dstSubresource.aspectMask, in tu6_blit_image()
1383 r3d_coords_raw(cs, (float[]) { in tu6_blit_image()
1390 tu_cs_emit_regs(cs, in tu6_blit_image()
1395 tu_cs_emit_regs(cs, in tu6_blit_image()
1409 ops->src(cmd, cs, &src, 0, filter); in tu6_blit_image()
1417 r3d_coord_z(cs, t * (src1_z - src0_z) + src0_z); in tu6_blit_image()
1419 ops->src(cmd, cs, &src, i, filter); in tu6_blit_image()
1421 ops->dst(cs, &dst, i); in tu6_blit_image()
1422 ops->run(cmd, cs); in tu6_blit_image()
1425 ops->teardown(cmd, cs); in tu6_blit_image()
1427 trace_end_blit(&cmd->trace, cs, in tu6_blit_image()
1499 struct tu_cs *cs = &cmd->cs; in tu_copy_buffer_to_image() local
1525 ops->setup(cmd, cs, in tu_copy_buffer_to_image()
1534 ops->dst(cs, &dst, i); in tu_copy_buffer_to_image()
1540 ops->src_buffer(cmd, cs, src_format, src_va & ~63, pitch, in tu_copy_buffer_to_image()
1542 ops->coords(cs, &(VkOffset2D){offset.x, offset.y + y}, &(VkOffset2D){x}, in tu_copy_buffer_to_image()
1544 ops->run(cmd, cs); in tu_copy_buffer_to_image()
1548 ops->src_buffer(cmd, cs, src_format, src_va, pitch, extent.width, extent.height); in tu_copy_buffer_to_image()
1549 coords(ops, cs, &offset, &(VkOffset3D){}, &extent); in tu_copy_buffer_to_image()
1550 ops->run(cmd, cs); in tu_copy_buffer_to_image()
1554 ops->teardown(cmd, cs); in tu_copy_buffer_to_image()
1579 struct tu_cs *cs = &cmd->cs; in tu_copy_image_to_buffer() local
1601 ops->setup(cmd, cs, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, in tu_copy_image_to_buffer()
1608 ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); in tu_copy_image_to_buffer()
1614 ops->dst_buffer(cs, dst_format, dst_va & ~63, 0); in tu_copy_image_to_buffer()
1615 ops->coords(cs, &(VkOffset2D) {x}, &(VkOffset2D){offset.x, offset.y + y}, in tu_copy_image_to_buffer()
1617 ops->run(cmd, cs); in tu_copy_image_to_buffer()
1621 ops->dst_buffer(cs, dst_format, dst_va, pitch); in tu_copy_image_to_buffer()
1622 coords(ops, cs, &(VkOffset3D) {0, 0}, &offset, &extent); in tu_copy_image_to_buffer()
1623 ops->run(cmd, cs); in tu_copy_image_to_buffer()
1627 ops->teardown(cmd, cs); in tu_copy_image_to_buffer()
1682 struct tu_cs *cs = &cmd->cs; in tu_copy_image_to_image() local
1800 ops->setup(cmd, cs, src_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, in tu_copy_image_to_image()
1802 coords(ops, cs, &staging_offset, &src_offset, &extent); in tu_copy_image_to_image()
1805 ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); in tu_copy_image_to_image()
1806 ops->dst(cs, &staging, i); in tu_copy_image_to_image()
1807 ops->run(cmd, cs); in tu_copy_image_to_image()
1813 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS); in tu_copy_image_to_image()
1814 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE); in tu_copy_image_to_image()
1815 tu_cs_emit_wfi(cs); in tu_copy_image_to_image()
1820 ops->setup(cmd, cs, dst_format, info->dstSubresource.aspectMask, in tu_copy_image_to_image()
1823 coords(ops, cs, &dst_offset, &staging_offset, &extent); in tu_copy_image_to_image()
1826 ops->src(cmd, cs, &staging, i, VK_FILTER_NEAREST); in tu_copy_image_to_image()
1827 ops->dst(cs, &dst, i); in tu_copy_image_to_image()
1828 ops->run(cmd, cs); in tu_copy_image_to_image()
1834 ops->setup(cmd, cs, format, info->dstSubresource.aspectMask, in tu_copy_image_to_image()
1837 coords(ops, cs, &dst_offset, &src_offset, &extent); in tu_copy_image_to_image()
1840 ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); in tu_copy_image_to_image()
1841 ops->dst(cs, &dst, i); in tu_copy_image_to_image()
1842 ops->run(cmd, cs); in tu_copy_image_to_image()
1846 ops->teardown(cmd, cs); in tu_copy_image_to_image()
1885 struct tu_cs *cs = &cmd->cs; in copy_buffer() local
1889 ops->setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, in copy_buffer()
1897 ops->src_buffer(cmd, cs, format, src_va & ~63, 0, src_x + width, 1); in copy_buffer()
1898 ops->dst_buffer( cs, format, dst_va & ~63, 0); in copy_buffer()
1899 ops->coords(cs, &(VkOffset2D) {dst_x}, &(VkOffset2D) {src_x}, &(VkExtent2D) {width, 1}); in copy_buffer()
1900 ops->run(cmd, cs); in copy_buffer()
1907 ops->teardown(cmd, cs); in copy_buffer()
1960 struct tu_cs *cs = &cmd->cs; in tu_CmdFillBuffer() local
1968 ops->setup(cmd, cs, VK_FORMAT_R32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, in tu_CmdFillBuffer()
1970 ops->clear_value(cs, VK_FORMAT_R32_UINT, &(VkClearValue){.color = {.uint32[0] = data}}); in tu_CmdFillBuffer()
1976 ops->dst_buffer(cs, VK_FORMAT_R32_UINT, dst_va & ~63, 0); in tu_CmdFillBuffer()
1977 ops->coords(cs, &(VkOffset2D) {dst_x}, NULL, &(VkExtent2D) {width, 1}); in tu_CmdFillBuffer()
1978 ops->run(cmd, cs); in tu_CmdFillBuffer()
1984 ops->teardown(cmd, cs); in tu_CmdFillBuffer()
2000 struct tu_cs *cs = &cmd->cs; in tu_CmdResolveImage() local
2002 ops->setup(cmd, cs, dst_image->vk_format, VK_IMAGE_ASPECT_COLOR_BIT, in tu_CmdResolveImage()
2012 coords(ops, cs, &info->dstOffset, &info->srcOffset, &info->extent); in tu_CmdResolveImage()
2019 ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); in tu_CmdResolveImage()
2020 ops->dst(cs, &dst, i); in tu_CmdResolveImage()
2021 ops->run(cmd, cs); in tu_CmdResolveImage()
2025 ops->teardown(cmd, cs); in tu_CmdResolveImage()
2036 struct tu_cs *cs, in resolve_sysmem() argument
2047 trace_start_sysmem_resolve(&cmd->trace, cs); in resolve_sysmem()
2049 ops->setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, in resolve_sysmem()
2051 ops->coords(cs, &rect->offset, &rect->offset, &rect->extent); in resolve_sysmem()
2055 r2d_src_stencil(cmd, cs, src, i, VK_FILTER_NEAREST); in resolve_sysmem()
2056 r2d_dst_stencil(cs, dst, i); in resolve_sysmem()
2058 ops->src(cmd, cs, src, i, VK_FILTER_NEAREST); in resolve_sysmem()
2059 ops->dst(cs, dst, i); in resolve_sysmem()
2061 ops->run(cmd, cs); in resolve_sysmem()
2064 ops->teardown(cmd, cs); in resolve_sysmem()
2066 trace_end_sysmem_resolve(&cmd->trace, cs, format); in resolve_sysmem()
2071 struct tu_cs *cs, in tu_resolve_sysmem() argument
2081 resolve_sysmem(cmd, cs, VK_FORMAT_D32_SFLOAT, in tu_resolve_sysmem()
2083 resolve_sysmem(cmd, cs, VK_FORMAT_S8_UINT, in tu_resolve_sysmem()
2086 resolve_sysmem(cmd, cs, dst->image->vk_format, in tu_resolve_sysmem()
2100 struct tu_cs *cs = &cmd->cs; in clear_image() local
2112 ops->setup(cmd, cs, format, aspect_mask, 0, true, image->layout[0].ubwc, in clear_image()
2115 ops->clear_value(cs, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, clear_value); in clear_image()
2117 ops->clear_value(cs, format, clear_value); in clear_image()
2123 ops->coords(cs, &(VkOffset2D){}, NULL, &(VkExtent2D) { in clear_image()
2137 ops->dst(cs, &dst, i); in clear_image()
2138 ops->run(cmd, cs); in clear_image()
2142 ops->teardown(cmd, cs); in clear_image()
2195 struct tu_cs *cs = &cmd->draw_cs; in tu_clear_sysmem_attachments() local
2203 trace_start_sysmem_clear_all(&cmd->trace, cs); in tu_clear_sysmem_attachments()
2245 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * (TU_DRAW_STATE_COUNT - 2)); in tu_clear_sysmem_attachments()
2250 tu_cs_emit(cs, CP_SET_DRAW_STATE__0_GROUP_ID(i) | in tu_clear_sysmem_attachments()
2252 tu_cs_emit_qw(cs, 0); in tu_clear_sysmem_attachments()
2256 tu_cs_emit_pkt4(cs, REG_A6XX_SP_FS_OUTPUT_CNTL0, 2); in tu_clear_sysmem_attachments()
2257 tu_cs_emit(cs, A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID(0xfc) | in tu_clear_sysmem_attachments()
2260 tu_cs_emit(cs, A6XX_SP_FS_OUTPUT_CNTL1_MRT(mrt_count)); in tu_clear_sysmem_attachments()
2262 r3d_common(cmd, cs, false, clear_rts, false, cmd->state.subpass->samples); in tu_clear_sysmem_attachments()
2264 tu_cs_emit_regs(cs, in tu_clear_sysmem_attachments()
2266 tu_cs_emit_regs(cs, in tu_clear_sysmem_attachments()
2269 tu_cs_emit_regs(cs, in tu_clear_sysmem_attachments()
2273 tu_cs_emit_regs(cs, A6XX_SP_BLEND_CNTL()); in tu_clear_sysmem_attachments()
2274 tu_cs_emit_regs(cs, A6XX_RB_BLEND_CNTL(.independent_blend = 1, .sample_mask = 0xffff)); in tu_clear_sysmem_attachments()
2276 tu_cs_emit_regs(cs, A6XX_RB_MRT_CONTROL(i, in tu_clear_sysmem_attachments()
2280 tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_CNTL(0)); in tu_clear_sysmem_attachments()
2281 tu_cs_emit_regs(cs, A6XX_RB_LRZ_CNTL(0)); in tu_clear_sysmem_attachments()
2283 tu_cs_emit_regs(cs, A6XX_RB_DEPTH_PLANE_CNTL()); in tu_clear_sysmem_attachments()
2284 tu_cs_emit_regs(cs, A6XX_RB_DEPTH_CNTL( in tu_clear_sysmem_attachments()
2288 tu_cs_emit_regs(cs, A6XX_GRAS_SU_DEPTH_PLANE_CNTL()); in tu_clear_sysmem_attachments()
2289 tu_cs_emit_regs(cs, A6XX_RB_STENCIL_CONTROL( in tu_clear_sysmem_attachments()
2293 tu_cs_emit_regs(cs, A6XX_RB_STENCILMASK(.mask = 0xff)); in tu_clear_sysmem_attachments()
2294 tu_cs_emit_regs(cs, A6XX_RB_STENCILWRMASK(.wrmask = 0xff)); in tu_clear_sysmem_attachments()
2295 tu_cs_emit_regs(cs, A6XX_RB_STENCILREF(.ref = s_clear_val)); in tu_clear_sysmem_attachments()
2298 tu_cs_emit_pkt7(cs, CP_LOAD_STATE6_FRAG, 3 + 4 * num_rts); in tu_clear_sysmem_attachments()
2299 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(0) | in tu_clear_sysmem_attachments()
2304 tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0)); in tu_clear_sysmem_attachments()
2305 tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0)); in tu_clear_sysmem_attachments()
2307 tu_cs_emit_array(cs, clear_value[b], 4); in tu_clear_sysmem_attachments()
2325 r3d_coords_raw(cs, (float[]) { in tu_clear_sysmem_attachments()
2332 r3d_run(cmd, cs); in tu_clear_sysmem_attachments()
2337 cs, mrt_count, rect_count); in tu_clear_sysmem_attachments()
2419 struct tu_cs *cs, in clear_gmem_attachment() argument
2425 tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_DST_INFO, 1); in clear_gmem_attachment()
2426 tu_cs_emit(cs, A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT(tu6_base_format(format))); in clear_gmem_attachment()
2428 tu_cs_emit_regs(cs, A6XX_RB_BLIT_INFO(.gmem = 1, .clear_mask = clear_mask)); in clear_gmem_attachment()
2430 tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_BASE_GMEM, 1); in clear_gmem_attachment()
2431 tu_cs_emit(cs, gmem_offset); in clear_gmem_attachment()
2433 tu_cs_emit_pkt4(cs, REG_A6XX_RB_UNKNOWN_88D0, 1); in clear_gmem_attachment()
2434 tu_cs_emit(cs, 0); in clear_gmem_attachment()
2439 tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0, 4); in clear_gmem_attachment()
2440 tu_cs_emit_array(cs, clear_vals, 4); in clear_gmem_attachment()
2442 tu6_emit_event_write(cmd, cs, BLIT); in clear_gmem_attachment()
2447 struct tu_cs *cs, in tu_emit_clear_gmem_attachment() argument
2455 trace_start_gmem_clear(&cmd->trace, cs); in tu_emit_clear_gmem_attachment()
2459 clear_gmem_attachment(cmd, cs, VK_FORMAT_D32_SFLOAT, 0xf, att->gmem_offset, value); in tu_emit_clear_gmem_attachment()
2461 clear_gmem_attachment(cmd, cs, VK_FORMAT_S8_UINT, 0xf, att->gmem_offset_stencil, value); in tu_emit_clear_gmem_attachment()
2465 …clear_gmem_attachment(cmd, cs, att->format, aspect_write_mask(att->format, mask), att->gmem_offset… in tu_emit_clear_gmem_attachment()
2467 trace_end_gmem_clear(&cmd->trace, cs, att->format, att->samples); in tu_emit_clear_gmem_attachment()
2478 struct tu_cs *cs = &cmd->draw_cs; in tu_clear_gmem_attachments() local
2487 tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_SCISSOR_TL, 2); in tu_clear_gmem_attachments()
2488 tu_cs_emit(cs, A6XX_RB_BLIT_SCISSOR_TL_X(x1) | A6XX_RB_BLIT_SCISSOR_TL_Y(y1)); in tu_clear_gmem_attachments()
2489 tu_cs_emit(cs, A6XX_RB_BLIT_SCISSOR_BR_X(x2) | A6XX_RB_BLIT_SCISSOR_BR_Y(y2)); in tu_clear_gmem_attachments()
2501 tu_emit_clear_gmem_attachment(cmd, cs, a, attachments[j].aspectMask, in tu_clear_gmem_attachments()
2515 struct tu_cs *cs = &cmd->draw_cs; in tu_CmdClearAttachments() local
2520 tu_emit_cache_flush_renderpass(cmd, cs); in tu_CmdClearAttachments()
2538 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM | in tu_CmdClearAttachments()
2541 tu_cond_exec_end(cs); in tu_CmdClearAttachments()
2545 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM); in tu_CmdClearAttachments()
2547 tu_cond_exec_end(cs); in tu_CmdClearAttachments()
2549 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM); in tu_CmdClearAttachments()
2551 tu_cond_exec_end(cs); in tu_CmdClearAttachments()
2556 struct tu_cs *cs, in clear_sysmem_attachment() argument
2570 trace_start_sysmem_clear(&cmd->trace, cs); in clear_sysmem_attachment()
2572 ops->setup(cmd, cs, format, clear_mask, 0, true, iview->ubwc_enabled, in clear_sysmem_attachment()
2574 ops->coords(cs, &info->renderArea.offset, NULL, &info->renderArea.extent); in clear_sysmem_attachment()
2575 ops->clear_value(cs, format, &info->pClearValues[a]); in clear_sysmem_attachment()
2580 r3d_dst_stencil(cs, iview, i); in clear_sysmem_attachment()
2582 r2d_dst_stencil(cs, iview, i); in clear_sysmem_attachment()
2584 ops->dst(cs, iview, i); in clear_sysmem_attachment()
2586 ops->run(cmd, cs); in clear_sysmem_attachment()
2589 ops->teardown(cmd, cs); in clear_sysmem_attachment()
2591 trace_end_sysmem_clear(&cmd->trace, cs, in clear_sysmem_attachment()
2598 struct tu_cs *cs, in tu_clear_sysmem_attachment() argument
2610 clear_sysmem_attachment(cmd, cs, VK_FORMAT_D32_SFLOAT, VK_IMAGE_ASPECT_COLOR_BIT, in tu_clear_sysmem_attachment()
2614 clear_sysmem_attachment(cmd, cs, VK_FORMAT_S8_UINT, VK_IMAGE_ASPECT_COLOR_BIT, in tu_clear_sysmem_attachment()
2618 clear_sysmem_attachment(cmd, cs, attachment->format, attachment->clear_mask, in tu_clear_sysmem_attachment()
2632 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS); in tu_clear_sysmem_attachment()
2633 tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_DEPTH); in tu_clear_sysmem_attachment()
2635 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS); in tu_clear_sysmem_attachment()
2636 tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_COLOR); in tu_clear_sysmem_attachment()
2640 tu_cs_emit_wfi(cs); in tu_clear_sysmem_attachment()
2645 struct tu_cs *cs, in tu_clear_gmem_attachment() argument
2655 tu_cs_emit_regs(cs, A6XX_RB_MSAA_CNTL(tu_msaa_samples(attachment->samples))); in tu_clear_gmem_attachment()
2657 tu_emit_clear_gmem_attachment(cmd, cs, a, attachment->clear_mask, in tu_clear_gmem_attachment()
2663 struct tu_cs *cs, in tu_emit_blit() argument
2669 tu_cs_emit_regs(cs, in tu_emit_blit()
2672 tu_cs_emit_regs(cs, A6XX_RB_BLIT_INFO( in tu_emit_blit()
2678 tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_DST_INFO, 4); in tu_emit_blit()
2680 tu_cs_emit(cs, tu_image_view_stencil(iview, RB_BLIT_DST_INFO) & ~A6XX_RB_BLIT_DST_INFO_FLAGS); in tu_emit_blit()
2681 tu_cs_emit_qw(cs, iview->stencil_base_addr); in tu_emit_blit()
2682 tu_cs_emit(cs, iview->stencil_PITCH); in tu_emit_blit()
2684 tu_cs_emit_regs(cs, in tu_emit_blit()
2687 tu_cs_emit(cs, iview->RB_BLIT_DST_INFO); in tu_emit_blit()
2688 tu_cs_image_ref_2d(cs, iview, 0, false); in tu_emit_blit()
2690 tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_FLAG_DST, 3); in tu_emit_blit()
2691 tu_cs_image_flag_ref(cs, iview, 0); in tu_emit_blit()
2693 tu_cs_emit_regs(cs, in tu_emit_blit()
2697 tu6_emit_event_write(cmd, cs, BLIT); in tu_emit_blit()
2737 struct tu_cs *cs, in tu_load_gmem_attachment() argument
2745 trace_start_gmem_load(&cmd->trace, cs); in tu_load_gmem_attachment()
2748 tu_emit_blit(cmd, cs, iview, attachment, false, false); in tu_load_gmem_attachment()
2751 tu_emit_blit(cmd, cs, iview, attachment, false, true); in tu_load_gmem_attachment()
2753 trace_end_gmem_load(&cmd->trace, cs, attachment->format, force_load); in tu_load_gmem_attachment()
2758 struct tu_cs *cs, in store_cp_blit() argument
2766 r2d_setup_common(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, in store_cp_blit()
2769 r2d_dst_stencil(cs, iview, 0); in store_cp_blit()
2771 r2d_dst(cs, iview, 0); in store_cp_blit()
2773 tu_cs_emit_regs(cs, in store_cp_blit()
2789 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE); in store_cp_blit()
2792 tu_cs_emit_wfi(cs); in store_cp_blit()
2794 tu_cs_emit_pkt7(cs, CP_BLIT, 1); in store_cp_blit()
2795 tu_cs_emit(cs, CP_BLIT_0_OP(BLIT_OP_SCALE)); in store_cp_blit()
2801 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS); in store_cp_blit()
2806 struct tu_cs *cs, in store_3d_blit() argument
2815 r3d_setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, in store_3d_blit()
2818 r3d_coords(cs, &render_area->offset, &render_area->offset, &render_area->extent); in store_3d_blit()
2821 r3d_dst_stencil(cs, iview, 0); in store_3d_blit()
2823 r3d_dst(cs, iview, 0); in store_3d_blit()
2825 r3d_src_gmem(cmd, cs, iview, format, gmem_offset, cpp); in store_3d_blit()
2828 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE); in store_3d_blit()
2830 r3d_run(cmd, cs); in store_3d_blit()
2837 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS); in store_3d_blit()
2842 struct tu_cs *cs, in tu_store_gmem_attachment() argument
2880 trace_start_gmem_store(&cmd->trace, cs); in tu_store_gmem_attachment()
2885 tu_emit_blit(cmd, cs, iview, src, true, resolve_d32s8_s8); in tu_store_gmem_attachment()
2887 tu_emit_blit(cmd, cs, iview, src, true, true); in tu_store_gmem_attachment()
2889 trace_end_gmem_store(&cmd->trace, cs, dst->format, true, false); in tu_store_gmem_attachment()
2906 tu_disable_draw_states(cmd, cs); in tu_store_gmem_attachment()
2909 store_3d_blit(cmd, cs, iview, dst->samples, resolve_d32s8_s8, format, in tu_store_gmem_attachment()
2913 store_3d_blit(cmd, cs, iview, dst->samples, true, VK_FORMAT_S8_UINT, in tu_store_gmem_attachment()
2917 r2d_coords(cs, &render_area->offset, &render_area->offset, &render_area->extent); in tu_store_gmem_attachment()
2920 store_cp_blit(cmd, cs, iview, src->samples, resolve_d32s8_s8, format, in tu_store_gmem_attachment()
2924 store_cp_blit(cmd, cs, iview, src->samples, true, VK_FORMAT_S8_UINT, in tu_store_gmem_attachment()
2929 trace_end_gmem_store(&cmd->trace, cs, dst->format, false, unaligned); in tu_store_gmem_attachment()