1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /*
25 * NOTE: The header can be included multiple times, from the same file.
26 */
27
28 /*
29 * Gen-specific function declarations. This header must *not* be included
30 * directly. Instead, it is included multiple times by anv_private.h.
31 *
32 * In this header file, the usual genx() macro is available.
33 */
34
35 #ifndef ANV_PRIVATE_H
36 #error This file is included by means other than anv_private.h
37 #endif
38
39 struct intel_sample_positions;
40 struct intel_urb_config;
41
42 typedef struct nir_builder nir_builder;
43 typedef struct nir_shader nir_shader;
44
45 extern const uint32_t genX(vk_to_intel_cullmode)[];
46
47 extern const uint32_t genX(vk_to_intel_front_face)[];
48
49 extern const uint32_t genX(vk_to_intel_primitive_type)[];
50
51 extern const uint32_t genX(vk_to_intel_compare_op)[];
52
53 extern const uint32_t genX(vk_to_intel_stencil_op)[];
54
55 extern const uint32_t genX(vk_to_intel_logic_op)[];
56
57 extern const uint32_t genX(vk_to_intel_fillmode)[];
58
59 void genX(init_physical_device_state)(struct anv_physical_device *device);
60
61 VkResult genX(init_device_state)(struct anv_device *device);
62
63 void genX(init_cps_device_state)(struct anv_device *device);
64
65 nir_shader *genX(load_libanv_shader)(struct anv_device *device, void *mem_ctx);
66
67 uint32_t genX(call_internal_shader)(nir_builder *b,
68 enum anv_internal_kernel_name shader_name);
69
70 void
71 genX(set_fast_clear_state)(struct anv_cmd_buffer *cmd_buffer,
72 const struct anv_image *image,
73 const enum isl_format format,
74 union isl_color_value clear_color);
75
76 void
77 genX(load_image_clear_color)(struct anv_cmd_buffer *cmd_buffer,
78 struct anv_state surface_state,
79 const struct anv_image *image);
80
81 void genX(cmd_buffer_emit_state_base_address)(struct anv_cmd_buffer *cmd_buffer);
82
83 void genX(cmd_buffer_apply_pipe_flushes)(struct anv_cmd_buffer *cmd_buffer);
84
85 void genX(cmd_buffer_emit_gfx12_depth_wa)(struct anv_cmd_buffer *cmd_buffer,
86 const struct isl_surf *surf);
87
88 void genX(cmd_buffer_set_binding_for_gfx8_vb_flush)(struct anv_cmd_buffer *cmd_buffer,
89 int vb_index,
90 struct anv_address vb_address,
91 uint32_t vb_size);
92 void genX(cmd_buffer_update_dirty_vbs_for_gfx8_vb_flush)(struct anv_cmd_buffer *cmd_buffer,
93 uint32_t access_type,
94 uint64_t vb_used);
95
96 void genX(cmd_buffer_emit_hashing_mode)(struct anv_cmd_buffer *cmd_buffer,
97 unsigned width, unsigned height,
98 unsigned scale);
99
100 void genX(urb_workaround)(struct anv_cmd_buffer *cmd_buffer,
101 const struct intel_urb_config *urb_cfg);
102
103 void genX(flush_pipeline_select_3d)(struct anv_cmd_buffer *cmd_buffer);
104 void genX(flush_pipeline_select_gpgpu)(struct anv_cmd_buffer *cmd_buffer);
105 void genX(emit_pipeline_select)(struct anv_batch *batch, uint32_t pipeline,
106 const struct anv_device *device);
107
108 void genX(apply_task_urb_workaround)(struct anv_cmd_buffer *cmd_buffer);
109
110 void genX(emit_vertex_input)(struct anv_batch *batch,
111 uint32_t *vertex_element_dws,
112 struct anv_graphics_pipeline *pipeline,
113 const struct vk_vertex_input_state *vi,
114 bool emit_in_pipeline);
115
116 enum anv_pipe_bits
117 genX(emit_apply_pipe_flushes)(struct anv_batch *batch,
118 struct anv_device *device,
119 uint32_t current_pipeline,
120 enum anv_pipe_bits bits,
121 enum anv_pipe_bits *emitted_flush_bits);
122 void
123 genX(invalidate_aux_map)(struct anv_batch *batch,
124 struct anv_device *device,
125 enum intel_engine_class engine_class,
126 enum anv_pipe_bits bits);
127
128
129 void genX(emit_so_memcpy_init)(struct anv_memcpy_state *state,
130 struct anv_device *device,
131 struct anv_batch *batch);
132
133 void genX(emit_so_memcpy_fini)(struct anv_memcpy_state *state);
134
135 void genX(emit_so_memcpy_end)(struct anv_memcpy_state *state);
136
137 void genX(emit_so_memcpy)(struct anv_memcpy_state *state,
138 struct anv_address dst, struct anv_address src,
139 uint32_t size);
140
141 void genX(emit_l3_config)(struct anv_batch *batch,
142 const struct anv_device *device,
143 const struct intel_l3_config *cfg);
144
145 void genX(cmd_buffer_config_l3)(struct anv_cmd_buffer *cmd_buffer,
146 const struct intel_l3_config *cfg);
147
148 uint32_t
149 genX(cmd_buffer_flush_descriptor_sets)(struct anv_cmd_buffer *cmd_buffer,
150 struct anv_cmd_pipeline_state *pipe_state,
151 const VkShaderStageFlags dirty,
152 struct anv_shader_bin **shaders,
153 uint32_t num_shaders);
154
155 void genX(cmd_buffer_flush_gfx_hw_state)(struct anv_cmd_buffer *cmd_buffer);
156
157 void genX(cmd_buffer_flush_gfx_runtime_state)(struct anv_cmd_buffer *cmd_buffer);
158
159 void genX(cmd_buffer_flush_gfx_hw_state)(struct anv_cmd_buffer *cmd_buffer);
160
161 void genX(cmd_buffer_enable_pma_fix)(struct anv_cmd_buffer *cmd_buffer,
162 bool enable);
163
164 void genX(cmd_buffer_mark_image_written)(struct anv_cmd_buffer *cmd_buffer,
165 const struct anv_image *image,
166 VkImageAspectFlagBits aspect,
167 enum isl_aux_usage aux_usage,
168 uint32_t level,
169 uint32_t base_layer,
170 uint32_t layer_count);
171
172 void genX(cmd_emit_conditional_render_predicate)(struct anv_cmd_buffer *cmd_buffer);
173
174 struct anv_state genX(cmd_buffer_ray_query_globals)(struct anv_cmd_buffer *cmd_buffer);
175
176 void genX(cmd_buffer_ensure_cfe_state)(struct anv_cmd_buffer *cmd_buffer,
177 uint32_t total_scratch);
178
179 void
180 genX(emit_urb_setup)(struct anv_device *device, struct anv_batch *batch,
181 const struct intel_l3_config *l3_config,
182 VkShaderStageFlags active_stages,
183 const struct intel_urb_config *urb_cfg_in,
184 struct intel_urb_config *urb_cfg_out,
185 enum intel_urb_deref_block_size *deref_block_size);
186
187 void genX(emit_sample_pattern)(struct anv_batch *batch,
188 const struct vk_sample_locations_state *sl);
189
190 void genX(cmd_buffer_so_memcpy)(struct anv_cmd_buffer *cmd_buffer,
191 struct anv_address dst, struct anv_address src,
192 uint32_t size);
193
194 void genX(cmd_buffer_dispatch_kernel)(struct anv_cmd_buffer *cmd_buffer,
195 struct anv_kernel *kernel,
196 const uint32_t *global_size, /* NULL for indirect */
197 uint32_t arg_count,
198 const struct anv_kernel_arg *args);
199
200 void genX(blorp_exec)(struct blorp_batch *batch,
201 const struct blorp_params *params);
202
203 void genX(batch_emit_secondary_call)(struct anv_batch *batch,
204 struct anv_address secondary_addr,
205 struct anv_address secondary_return_addr);
206
207 void *genX(batch_emit_return)(struct anv_batch *batch);
208
209 void genX(cmd_emit_timestamp)(struct anv_batch *batch,
210 struct anv_device *device,
211 struct anv_address addr,
212 enum anv_timestamp_capture_type type,
213 void *data);
214
215 void
216 genX(batch_emit_post_3dprimitive_was)(struct anv_batch *batch,
217 const struct anv_device *device,
218 uint32_t primitive_topology,
219 uint32_t vertex_count);
220
221 void genX(batch_emit_fast_color_dummy_blit)(struct anv_batch *batch,
222 struct anv_device *device);
223
224 VkPolygonMode
225 genX(raster_polygon_mode)(const struct anv_graphics_pipeline *pipeline,
226 VkPolygonMode polygon_mode,
227 VkPrimitiveTopology primitive_topology);
228
229 void
230 genX(graphics_pipeline_emit)(struct anv_graphics_pipeline *pipeline,
231 const struct vk_graphics_pipeline_state *state);
232
233 void
234 genX(compute_pipeline_emit)(struct anv_compute_pipeline *pipeline);
235
236 void
237 genX(ray_tracing_pipeline_emit)(struct anv_ray_tracing_pipeline *pipeline);
238
239 #define anv_shader_bin_get_bsr(bin, local_arg_offset) ({ \
240 assert((local_arg_offset) % 8 == 0); \
241 const struct brw_bs_prog_data *prog_data = \
242 brw_bs_prog_data_const(bin->prog_data); \
243 assert(prog_data->simd_size == 8 || prog_data->simd_size == 16); \
244 \
245 (struct GENX(BINDLESS_SHADER_RECORD)) { \
246 .OffsetToLocalArguments = (local_arg_offset) / 8, \
247 .BindlessShaderDispatchMode = \
248 prog_data->simd_size == 16 ? RT_SIMD16 : RT_SIMD8, \
249 .KernelStartPointer = bin->kernel.offset, \
250 }; \
251 })
252
253 void
254 genX(batch_set_preemption)(struct anv_batch *batch,
255 const struct intel_device_info *devinfo,
256 uint32_t current_pipeline,
257 bool value);
258
259 void
260 genX(cmd_buffer_set_preemption)(struct anv_cmd_buffer *cmd_buffer, bool value);
261
262 void
263 genX(batch_emit_pipe_control)(struct anv_batch *batch,
264 const struct intel_device_info *devinfo,
265 uint32_t current_pipeline,
266 enum anv_pipe_bits bits,
267 const char *reason);
268
269 void
270 genX(batch_emit_pipe_control_write)(struct anv_batch *batch,
271 const struct intel_device_info *devinfo,
272 uint32_t current_pipeline,
273 uint32_t post_sync_op,
274 struct anv_address address,
275 uint32_t imm_data,
276 enum anv_pipe_bits bits,
277 const char *reason);
278
279 #define genx_batch_emit_pipe_control(a, b, c, d) \
280 genX(batch_emit_pipe_control) (a, b, c, d, __func__)
281
282 #define genx_batch_emit_pipe_control_write(a, b, c, d, e, f, g) \
283 genX(batch_emit_pipe_control_write) (a, b, c, d, e, f, g, __func__)
284
285 void genX(batch_emit_breakpoint)(struct anv_batch *batch,
286 struct anv_device *device,
287 bool emit_before_draw);
288
289 static inline void
genX(emit_breakpoint)290 genX(emit_breakpoint)(struct anv_batch *batch,
291 struct anv_device *device,
292 bool emit_before_draw)
293 {
294 if (INTEL_DEBUG(DEBUG_DRAW_BKP))
295 genX(batch_emit_breakpoint)(batch, device, emit_before_draw);
296 }
297
298 struct anv_state
299 genX(cmd_buffer_begin_companion_rcs_syncpoint)(struct anv_cmd_buffer *cmd_buffer);
300
301 void
302 genX(cmd_buffer_end_companion_rcs_syncpoint)(struct anv_cmd_buffer *cmd_buffer,
303 struct anv_state syncpoint);
304
305 void
306 genX(emit_simple_shader_init)(struct anv_simple_shader *state);
307
308 void
309 genX(emit_simple_shader_dispatch)(struct anv_simple_shader *state,
310 uint32_t num_threads,
311 struct anv_state push_state);
312
313 struct anv_state
314 genX(simple_shader_alloc_push)(struct anv_simple_shader *state, uint32_t size);
315
316 struct anv_address
317 genX(simple_shader_push_state_address)(struct anv_simple_shader *state,
318 struct anv_state push_state);
319
320 void
321 genX(emit_simple_shader_end)(struct anv_simple_shader *state);
322
323 VkResult genX(init_trtt_context_state)(struct anv_queue *queue);
324
325 VkResult genX(write_trtt_entries)(struct anv_trtt_submission *submit);
326
327 void
328 genX(cmd_buffer_emit_push_descriptor_buffer_surface)(struct anv_cmd_buffer *cmd_buffer,
329 struct anv_descriptor_set *set);
330
331 void
332 genX(cmd_buffer_emit_push_descriptor_surfaces)(struct anv_cmd_buffer *cmd_buffer,
333 struct anv_descriptor_set *set);
334
335 static inline VkShaderStageFlags
genX(cmd_buffer_flush_push_descriptors)336 genX(cmd_buffer_flush_push_descriptors)(struct anv_cmd_buffer *cmd_buffer,
337 struct anv_cmd_pipeline_state *state,
338 struct anv_pipeline *pipeline)
339 {
340 if (!pipeline->use_push_descriptor && !pipeline->use_push_descriptor_buffer)
341 return 0;
342
343 assert(pipeline->layout.push_descriptor_set_index != -1);
344 struct anv_descriptor_set *set =
345 state->descriptors[pipeline->layout.push_descriptor_set_index];
346 assert(set->is_push);
347
348 const VkShaderStageFlags push_buffer_dirty =
349 cmd_buffer->state.push_descriptors_dirty &
350 pipeline->use_push_descriptor_buffer;
351 if (push_buffer_dirty) {
352 if (set->desc_surface_state.map == NULL)
353 genX(cmd_buffer_emit_push_descriptor_buffer_surface)(cmd_buffer, set);
354
355 /* Force the next push descriptor update to allocate a new descriptor set. */
356 state->push_descriptor.set_used_on_gpu = true;
357 }
358
359 const VkShaderStageFlags push_descriptor_dirty =
360 cmd_buffer->state.push_descriptors_dirty & pipeline->use_push_descriptor;
361 if (push_descriptor_dirty) {
362 genX(cmd_buffer_emit_push_descriptor_surfaces)(cmd_buffer, set);
363
364 /* Force the next push descriptor update to allocate a new descriptor set. */
365 state->push_descriptor.set_used_on_gpu = true;
366 }
367
368 /* Clear the dirty stages now that we've generated the surface states for
369 * them.
370 */
371 cmd_buffer->state.push_descriptors_dirty &=
372 ~(push_descriptor_dirty | push_buffer_dirty);
373
374 /* Return the binding table stages that need to be updated */
375 return push_buffer_dirty | push_descriptor_dirty;
376 }
377