1 /*
2 * Copyright © 2014-2017 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "util/u_blitter.h"
25 #include "util/u_draw.h"
26 #include "util/u_prim.h"
27 #include "util/format/u_format.h"
28 #include "util/u_helpers.h"
29 #include "util/u_pack_color.h"
30 #include "util/u_prim_restart.h"
31 #include "util/u_upload_mgr.h"
32
33 #include "v3d_context.h"
34 #include "v3d_resource.h"
35 #include "v3d_cl.h"
36 #include "broadcom/compiler/v3d_compiler.h"
37 #include "broadcom/common/v3d_macros.h"
38 #include "broadcom/common/v3d_util.h"
39 #include "broadcom/common/v3d_csd.h"
40 #include "broadcom/cle/v3dx_pack.h"
41
42 void
v3dX(start_binning)43 v3dX(start_binning)(struct v3d_context *v3d, struct v3d_job *job)
44 {
45 assert(job->needs_flush);
46
47 /* Get space to emit our BCL state, using a branch to jump to a new BO
48 * if necessary.
49 */
50
51 v3d_cl_ensure_space_with_branch(&job->bcl, 256 /* XXX */);
52
53 job->submit.bcl_start = job->bcl.bo->offset;
54 v3d_job_add_bo(job, job->bcl.bo);
55
56 /* The PTB will request the tile alloc initial size per tile at start
57 * of tile binning.
58 */
59 uint32_t tile_alloc_size =
60 MAX2(job->num_layers, 1) * job->draw_tiles_x * job->draw_tiles_y * 64;
61
62 /* The PTB allocates in aligned 4k chunks after the initial setup. */
63 tile_alloc_size = align(tile_alloc_size, 4096);
64
65 /* Include the first two chunk allocations that the PTB does so that
66 * we definitely clear the OOM condition before triggering one (the HW
67 * won't trigger OOM during the first allocations).
68 */
69 tile_alloc_size += 8192;
70
71 /* For performance, allocate some extra initial memory after the PTB's
72 * minimal allocations, so that we hopefully don't have to block the
73 * GPU on the kernel handling an OOM signal.
74 */
75 tile_alloc_size += 512 * 1024;
76
77 job->tile_alloc = v3d_bo_alloc(v3d->screen, tile_alloc_size,
78 "tile_alloc");
79 uint32_t tsda_per_tile_size = 256;
80 job->tile_state = v3d_bo_alloc(v3d->screen,
81 MAX2(job->num_layers, 1) *
82 job->draw_tiles_y *
83 job->draw_tiles_x *
84 tsda_per_tile_size,
85 "TSDA");
86
87 /* This must go before the binning mode configuration. It is
88 * required for layered framebuffers to work.
89 */
90 if (job->num_layers > 0) {
91 cl_emit(&job->bcl, NUMBER_OF_LAYERS, config) {
92 config.number_of_layers = job->num_layers;
93 }
94 }
95
96 assert(!job->msaa || !job->double_buffer);
97 #if V3D_VERSION >= 71
98 cl_emit(&job->bcl, TILE_BINNING_MODE_CFG, config) {
99 config.width_in_pixels = job->draw_width;
100 config.height_in_pixels = job->draw_height;
101
102 config.log2_tile_width = log2_tile_size(job->tile_width);
103 config.log2_tile_height = log2_tile_size(job->tile_height);
104
105 /* FIXME: ideallly we would like next assert on the packet header (as is
106 * general, so also applies to GL). We would need to expand
107 * gen_pack_header for that.
108 */
109 assert(config.log2_tile_width == config.log2_tile_height ||
110 config.log2_tile_width == config.log2_tile_height + 1);
111 }
112
113 #endif
114
115 #if V3D_VERSION == 42
116 cl_emit(&job->bcl, TILE_BINNING_MODE_CFG, config) {
117 config.width_in_pixels = job->draw_width;
118 config.height_in_pixels = job->draw_height;
119 config.number_of_render_targets =
120 MAX2(job->nr_cbufs, 1);
121
122 config.multisample_mode_4x = job->msaa;
123 config.double_buffer_in_non_ms_mode = job->double_buffer;
124
125 config.maximum_bpp_of_all_render_targets = job->internal_bpp;
126 }
127 #endif
128
129 /* There's definitely nothing in the VCD cache we want. */
130 cl_emit(&job->bcl, FLUSH_VCD_CACHE, bin);
131
132 /* Disable any leftover OQ state from another job. */
133 cl_emit(&job->bcl, OCCLUSION_QUERY_COUNTER, counter);
134
135 /* "Binning mode lists must have a Start Tile Binning item (6) after
136 * any prefix state data before the binning list proper starts."
137 */
138 cl_emit(&job->bcl, START_TILE_BINNING, bin);
139 }
140 /**
141 * Does the initial bining command list setup for drawing to a given FBO.
142 */
143 static void
v3d_start_draw(struct v3d_context * v3d)144 v3d_start_draw(struct v3d_context *v3d)
145 {
146 struct v3d_job *job = v3d->job;
147
148 if (job->needs_flush)
149 return;
150
151 job->needs_flush = true;
152 job->draw_width = v3d->framebuffer.width;
153 job->draw_height = v3d->framebuffer.height;
154 job->num_layers = util_framebuffer_get_num_layers(&v3d->framebuffer);
155
156 v3dX(start_binning)(v3d, job);
157 }
158
159 static void
v3d_predraw_check_stage_inputs(struct pipe_context * pctx,enum pipe_shader_type s)160 v3d_predraw_check_stage_inputs(struct pipe_context *pctx,
161 enum pipe_shader_type s)
162 {
163 struct v3d_context *v3d = v3d_context(pctx);
164
165 /* Flush writes to textures we're sampling. */
166 for (int i = 0; i < v3d->tex[s].num_textures; i++) {
167 struct pipe_sampler_view *pview = v3d->tex[s].textures[i];
168 if (!pview)
169 continue;
170 struct v3d_sampler_view *view = v3d_sampler_view(pview);
171
172 if (view->texture != view->base.texture &&
173 view->base.format != PIPE_FORMAT_X32_S8X24_UINT)
174 v3d_update_shadow_texture(pctx, &view->base);
175
176 v3d_flush_jobs_writing_resource(v3d, view->texture,
177 V3D_FLUSH_NOT_CURRENT_JOB,
178 s == PIPE_SHADER_COMPUTE);
179 }
180
181 /* Flush writes to UBOs. */
182 u_foreach_bit(i, v3d->constbuf[s].enabled_mask) {
183 struct pipe_constant_buffer *cb = &v3d->constbuf[s].cb[i];
184 if (cb->buffer) {
185 v3d_flush_jobs_writing_resource(v3d, cb->buffer,
186 V3D_FLUSH_DEFAULT,
187 s == PIPE_SHADER_COMPUTE);
188 }
189 }
190
191 /* Flush reads/writes to our SSBOs */
192 u_foreach_bit(i, v3d->ssbo[s].enabled_mask) {
193 struct pipe_shader_buffer *sb = &v3d->ssbo[s].sb[i];
194 if (sb->buffer) {
195 v3d_flush_jobs_reading_resource(v3d, sb->buffer,
196 V3D_FLUSH_NOT_CURRENT_JOB,
197 s == PIPE_SHADER_COMPUTE);
198 }
199 }
200
201 /* Flush reads/writes to our image views */
202 u_foreach_bit(i, v3d->shaderimg[s].enabled_mask) {
203 struct v3d_image_view *view = &v3d->shaderimg[s].si[i];
204
205 v3d_flush_jobs_reading_resource(v3d, view->base.resource,
206 V3D_FLUSH_NOT_CURRENT_JOB,
207 s == PIPE_SHADER_COMPUTE);
208 }
209
210 /* Flush writes to our vertex buffers (i.e. from transform feedback) */
211 if (s == PIPE_SHADER_VERTEX) {
212 u_foreach_bit(i, v3d->vertexbuf.enabled_mask) {
213 struct pipe_vertex_buffer *vb = &v3d->vertexbuf.vb[i];
214
215 v3d_flush_jobs_writing_resource(v3d, vb->buffer.resource,
216 V3D_FLUSH_DEFAULT,
217 false);
218 }
219 }
220 }
221
222 static void
v3d_predraw_check_outputs(struct pipe_context * pctx)223 v3d_predraw_check_outputs(struct pipe_context *pctx)
224 {
225 struct v3d_context *v3d = v3d_context(pctx);
226
227 /* Flush jobs reading from TF buffers that we are about to write. */
228 if (v3d_transform_feedback_enabled(v3d)) {
229 struct v3d_streamout_stateobj *so = &v3d->streamout;
230
231 for (int i = 0; i < so->num_targets; i++) {
232 if (!so->targets[i])
233 continue;
234
235 const struct pipe_stream_output_target *target =
236 so->targets[i];
237 v3d_flush_jobs_reading_resource(v3d, target->buffer,
238 V3D_FLUSH_DEFAULT,
239 false);
240 }
241 }
242 }
243
244 /**
245 * Checks if the state for the current draw reads a particular resource in
246 * in the given shader stage.
247 */
248 static bool
v3d_state_reads_resource(struct v3d_context * v3d,struct pipe_resource * prsc,enum pipe_shader_type s)249 v3d_state_reads_resource(struct v3d_context *v3d,
250 struct pipe_resource *prsc,
251 enum pipe_shader_type s)
252 {
253 struct v3d_resource *rsc = v3d_resource(prsc);
254
255 /* Vertex buffers */
256 if (s == PIPE_SHADER_VERTEX) {
257 u_foreach_bit(i, v3d->vertexbuf.enabled_mask) {
258 struct pipe_vertex_buffer *vb = &v3d->vertexbuf.vb[i];
259 if (!vb->buffer.resource)
260 continue;
261
262 struct v3d_resource *vb_rsc =
263 v3d_resource(vb->buffer.resource);
264 if (rsc->bo == vb_rsc->bo)
265 return true;
266 }
267 }
268
269 /* Constant buffers */
270 u_foreach_bit(i, v3d->constbuf[s].enabled_mask) {
271 struct pipe_constant_buffer *cb = &v3d->constbuf[s].cb[i];
272 if (!cb->buffer)
273 continue;
274
275 struct v3d_resource *cb_rsc = v3d_resource(cb->buffer);
276 if (rsc->bo == cb_rsc->bo)
277 return true;
278 }
279
280 /* Shader storage buffers */
281 u_foreach_bit(i, v3d->ssbo[s].enabled_mask) {
282 struct pipe_shader_buffer *sb = &v3d->ssbo[s].sb[i];
283 if (!sb->buffer)
284 continue;
285
286 struct v3d_resource *sb_rsc = v3d_resource(sb->buffer);
287 if (rsc->bo == sb_rsc->bo)
288 return true;
289 }
290
291 /* Textures */
292 for (int i = 0; i < v3d->tex[s].num_textures; i++) {
293 struct pipe_sampler_view *pview = v3d->tex[s].textures[i];
294 if (!pview)
295 continue;
296
297 struct v3d_sampler_view *view = v3d_sampler_view(pview);
298 struct v3d_resource *v_rsc = v3d_resource(view->texture);
299 if (rsc->bo == v_rsc->bo)
300 return true;
301 }
302
303 return false;
304 }
305
306 static void
v3d_emit_wait_for_tf(struct v3d_job * job)307 v3d_emit_wait_for_tf(struct v3d_job *job)
308 {
309 /* XXX: we might be able to skip this in some cases, for now we
310 * always emit it.
311 */
312 cl_emit(&job->bcl, FLUSH_TRANSFORM_FEEDBACK_DATA, flush);
313
314 cl_emit(&job->bcl, WAIT_FOR_TRANSFORM_FEEDBACK, wait) {
315 /* XXX: Wait for all outstanding writes... maybe we can do
316 * better in some cases.
317 */
318 wait.block_count = 255;
319 }
320
321 /* We have just flushed all our outstanding TF work in this job so make
322 * sure we don't emit TF flushes again for any of it again.
323 */
324 _mesa_set_clear(job->tf_write_prscs, NULL);
325 }
326
327 static void
v3d_emit_wait_for_tf_if_needed(struct v3d_context * v3d,struct v3d_job * job)328 v3d_emit_wait_for_tf_if_needed(struct v3d_context *v3d, struct v3d_job *job)
329 {
330 if (!job->tf_enabled)
331 return;
332
333 set_foreach(job->tf_write_prscs, entry) {
334 struct pipe_resource *prsc = (struct pipe_resource *)entry->key;
335 for (int s = 0; s < PIPE_SHADER_COMPUTE; s++) {
336 /* Fragment shaders can only start executing after all
337 * binning (and thus TF) is complete.
338 *
339 * XXX: For VS/GS/TES, if the binning shader does not
340 * read the resource then we could also avoid emitting
341 * the wait.
342 */
343 if (s == PIPE_SHADER_FRAGMENT)
344 continue;
345
346 if (v3d_state_reads_resource(v3d, prsc, s)) {
347 v3d_emit_wait_for_tf(job);
348 return;
349 }
350 }
351 }
352 }
353
354 static void
v3d_emit_gs_state_record(struct v3d_job * job,struct v3d_compiled_shader * gs_bin,struct v3d_cl_reloc gs_bin_uniforms,struct v3d_compiled_shader * gs,struct v3d_cl_reloc gs_render_uniforms)355 v3d_emit_gs_state_record(struct v3d_job *job,
356 struct v3d_compiled_shader *gs_bin,
357 struct v3d_cl_reloc gs_bin_uniforms,
358 struct v3d_compiled_shader *gs,
359 struct v3d_cl_reloc gs_render_uniforms)
360 {
361 cl_emit(&job->indirect, GEOMETRY_SHADER_STATE_RECORD, shader) {
362 shader.geometry_bin_mode_shader_code_address =
363 cl_address(v3d_resource(gs_bin->resource)->bo,
364 gs_bin->offset);
365 shader.geometry_bin_mode_shader_4_way_threadable =
366 gs_bin->prog_data.gs->base.threads == 4;
367 shader.geometry_bin_mode_shader_start_in_final_thread_section =
368 gs_bin->prog_data.gs->base.single_seg;
369 #if V3D_VERSION == 42
370 shader.geometry_bin_mode_shader_propagate_nans = true;
371 #endif
372 shader.geometry_bin_mode_shader_uniforms_address =
373 gs_bin_uniforms;
374
375 shader.geometry_render_mode_shader_code_address =
376 cl_address(v3d_resource(gs->resource)->bo, gs->offset);
377 shader.geometry_render_mode_shader_4_way_threadable =
378 gs->prog_data.gs->base.threads == 4;
379 shader.geometry_render_mode_shader_start_in_final_thread_section =
380 gs->prog_data.gs->base.single_seg;
381 #if V3D_VERSION == 42
382 shader.geometry_render_mode_shader_propagate_nans = true;
383 #endif
384 shader.geometry_render_mode_shader_uniforms_address =
385 gs_render_uniforms;
386 }
387 }
388
389 static uint8_t
v3d_gs_output_primitive(enum mesa_prim prim_type)390 v3d_gs_output_primitive(enum mesa_prim prim_type)
391 {
392 switch (prim_type) {
393 case MESA_PRIM_POINTS:
394 return GEOMETRY_SHADER_POINTS;
395 case MESA_PRIM_LINE_STRIP:
396 return GEOMETRY_SHADER_LINE_STRIP;
397 case MESA_PRIM_TRIANGLE_STRIP:
398 return GEOMETRY_SHADER_TRI_STRIP;
399 default:
400 unreachable("Unsupported primitive type");
401 }
402 }
403
404 static void
v3d_emit_tes_gs_common_params(struct v3d_job * job,uint8_t gs_out_prim_type,uint8_t gs_num_invocations)405 v3d_emit_tes_gs_common_params(struct v3d_job *job,
406 uint8_t gs_out_prim_type,
407 uint8_t gs_num_invocations)
408 {
409 /* This, and v3d_emit_tes_gs_shader_params below, fill in default
410 * values for tessellation fields even though we don't support
411 * tessellation yet because our packing functions (and the simulator)
412 * complain if we don't.
413 */
414 cl_emit(&job->indirect, TESSELLATION_GEOMETRY_COMMON_PARAMS, shader) {
415 shader.tessellation_type = TESSELLATION_TYPE_TRIANGLE;
416 shader.tessellation_point_mode = false;
417 shader.tessellation_edge_spacing = TESSELLATION_EDGE_SPACING_EVEN;
418 shader.tessellation_clockwise = true;
419 shader.tessellation_invocations = 1;
420
421 shader.geometry_shader_output_format =
422 v3d_gs_output_primitive(gs_out_prim_type);
423 shader.geometry_shader_instances = gs_num_invocations & 0x1F;
424 }
425 }
426
427 static uint8_t
simd_width_to_gs_pack_mode(uint32_t width)428 simd_width_to_gs_pack_mode(uint32_t width)
429 {
430 switch (width) {
431 case 16:
432 return V3D_PACK_MODE_16_WAY;
433 case 8:
434 return V3D_PACK_MODE_8_WAY;
435 case 4:
436 return V3D_PACK_MODE_4_WAY;
437 case 1:
438 return V3D_PACK_MODE_1_WAY;
439 default:
440 unreachable("Invalid SIMD width");
441 };
442 }
443
444 static void
v3d_emit_tes_gs_shader_params(struct v3d_job * job,uint32_t gs_simd,uint32_t gs_vpm_output_size,uint32_t gs_max_vpm_input_size_per_batch)445 v3d_emit_tes_gs_shader_params(struct v3d_job *job,
446 uint32_t gs_simd,
447 uint32_t gs_vpm_output_size,
448 uint32_t gs_max_vpm_input_size_per_batch)
449 {
450 cl_emit(&job->indirect, TESSELLATION_GEOMETRY_SHADER_PARAMS, shader) {
451 shader.tcs_batch_flush_mode = V3D_TCS_FLUSH_MODE_FULLY_PACKED;
452 shader.per_patch_data_column_depth = 1;
453 shader.tcs_output_segment_size_in_sectors = 1;
454 shader.tcs_output_segment_pack_mode = V3D_PACK_MODE_16_WAY;
455 shader.tes_output_segment_size_in_sectors = 1;
456 shader.tes_output_segment_pack_mode = V3D_PACK_MODE_16_WAY;
457 shader.gs_output_segment_size_in_sectors = gs_vpm_output_size;
458 shader.gs_output_segment_pack_mode =
459 simd_width_to_gs_pack_mode(gs_simd);
460 shader.tbg_max_patches_per_tcs_batch = 1;
461 shader.tbg_max_extra_vertex_segs_for_patches_after_first = 0;
462 shader.tbg_min_tcs_output_segments_required_in_play = 1;
463 shader.tbg_min_per_patch_data_segments_required_in_play = 1;
464 shader.tpg_max_patches_per_tes_batch = 1;
465 shader.tpg_max_vertex_segments_per_tes_batch = 0;
466 shader.tpg_max_tcs_output_segments_per_tes_batch = 1;
467 shader.tpg_min_tes_output_segments_required_in_play = 1;
468 shader.gbg_max_tes_output_vertex_segments_per_gs_batch =
469 gs_max_vpm_input_size_per_batch;
470 shader.gbg_min_gs_output_segments_required_in_play = 1;
471 }
472 }
473
474 static void
v3d_emit_gl_shader_state(struct v3d_context * v3d,const struct pipe_draw_info * info)475 v3d_emit_gl_shader_state(struct v3d_context *v3d,
476 const struct pipe_draw_info *info)
477 {
478 struct v3d_job *job = v3d->job;
479 /* V3D_DIRTY_VTXSTATE */
480 struct v3d_vertex_stateobj *vtx = v3d->vtx;
481 /* V3D_DIRTY_VTXBUF */
482 struct v3d_vertexbuf_stateobj *vertexbuf = &v3d->vertexbuf;
483
484 /* Upload the uniforms to the indirect CL first */
485 struct v3d_cl_reloc fs_uniforms =
486 v3d_write_uniforms(v3d, job, v3d->prog.fs,
487 PIPE_SHADER_FRAGMENT);
488
489 struct v3d_cl_reloc gs_uniforms = { NULL, 0 };
490 struct v3d_cl_reloc gs_bin_uniforms = { NULL, 0 };
491 if (v3d->prog.gs) {
492 gs_uniforms = v3d_write_uniforms(v3d, job, v3d->prog.gs,
493 PIPE_SHADER_GEOMETRY);
494 }
495 if (v3d->prog.gs_bin) {
496 gs_bin_uniforms = v3d_write_uniforms(v3d, job, v3d->prog.gs_bin,
497 PIPE_SHADER_GEOMETRY);
498 }
499
500 struct v3d_cl_reloc vs_uniforms =
501 v3d_write_uniforms(v3d, job, v3d->prog.vs,
502 PIPE_SHADER_VERTEX);
503 struct v3d_cl_reloc cs_uniforms =
504 v3d_write_uniforms(v3d, job, v3d->prog.cs,
505 PIPE_SHADER_VERTEX);
506
507 /* Update the cache dirty flag based on the shader progs data */
508 job->tmu_dirty_rcl |= v3d->prog.cs->prog_data.vs->base.tmu_dirty_rcl;
509 job->tmu_dirty_rcl |= v3d->prog.vs->prog_data.vs->base.tmu_dirty_rcl;
510 if (v3d->prog.gs_bin) {
511 job->tmu_dirty_rcl |=
512 v3d->prog.gs_bin->prog_data.gs->base.tmu_dirty_rcl;
513 }
514 if (v3d->prog.gs) {
515 job->tmu_dirty_rcl |=
516 v3d->prog.gs->prog_data.gs->base.tmu_dirty_rcl;
517 }
518 job->tmu_dirty_rcl |= v3d->prog.fs->prog_data.fs->base.tmu_dirty_rcl;
519
520 uint32_t num_elements_to_emit = 0;
521 for (int i = 0; i < vtx->num_elements; i++) {
522 struct pipe_vertex_element *elem = &vtx->pipe[i];
523 struct pipe_vertex_buffer *vb =
524 &vertexbuf->vb[elem->vertex_buffer_index];
525 if (vb->buffer.resource)
526 num_elements_to_emit++;
527 }
528
529 uint32_t shader_state_record_length =
530 cl_packet_length(GL_SHADER_STATE_RECORD);
531 if (v3d->prog.gs) {
532 shader_state_record_length +=
533 cl_packet_length(GEOMETRY_SHADER_STATE_RECORD) +
534 cl_packet_length(TESSELLATION_GEOMETRY_COMMON_PARAMS) +
535 2 * cl_packet_length(TESSELLATION_GEOMETRY_SHADER_PARAMS);
536 }
537
538 /* See GFXH-930 workaround below */
539 uint32_t shader_rec_offset =
540 v3d_cl_ensure_space(&job->indirect,
541 shader_state_record_length +
542 MAX2(num_elements_to_emit, 1) *
543 cl_packet_length(GL_SHADER_STATE_ATTRIBUTE_RECORD),
544 32);
545
546 /* XXX perf: We should move most of the SHADER_STATE_RECORD setup to
547 * compile time, so that we mostly just have to OR the VS and FS
548 * records together at draw time.
549 */
550
551 struct vpm_config vpm_cfg_bin, vpm_cfg;
552 v3d_compute_vpm_config(&v3d->screen->devinfo,
553 v3d->prog.cs->prog_data.vs,
554 v3d->prog.vs->prog_data.vs,
555 v3d->prog.gs ? v3d->prog.gs_bin->prog_data.gs : NULL,
556 v3d->prog.gs ? v3d->prog.gs->prog_data.gs : NULL,
557 &vpm_cfg_bin,
558 &vpm_cfg);
559
560 if (v3d->prog.gs) {
561 v3d_emit_gs_state_record(v3d->job,
562 v3d->prog.gs_bin, gs_bin_uniforms,
563 v3d->prog.gs, gs_uniforms);
564
565 struct v3d_gs_prog_data *gs = v3d->prog.gs->prog_data.gs;
566 v3d_emit_tes_gs_common_params(v3d->job,
567 gs->out_prim_type,
568 gs->num_invocations);
569
570 /* Bin Tes/Gs params */
571 v3d_emit_tes_gs_shader_params(v3d->job,
572 vpm_cfg_bin.gs_width,
573 vpm_cfg_bin.Gd,
574 vpm_cfg_bin.Gv);
575
576 /* Render Tes/Gs params */
577 v3d_emit_tes_gs_shader_params(v3d->job,
578 vpm_cfg.gs_width,
579 vpm_cfg.Gd,
580 vpm_cfg.Gv);
581 }
582
583 cl_emit(&job->indirect, GL_SHADER_STATE_RECORD, shader) {
584 shader.enable_clipping = true;
585 /* V3D_DIRTY_PRIM_MODE | V3D_DIRTY_RASTERIZER */
586 shader.point_size_in_shaded_vertex_data =
587 (info->mode == MESA_PRIM_POINTS &&
588 v3d->rasterizer->base.point_size_per_vertex);
589
590 /* Must be set if the shader modifies Z, discards, or modifies
591 * the sample mask. For any of these cases, the fragment
592 * shader needs to write the Z value (even just discards).
593 */
594 shader.fragment_shader_does_z_writes =
595 v3d->prog.fs->prog_data.fs->writes_z;
596
597 /* Set if the EZ test must be disabled (due to shader side
598 * effects and the early_z flag not being present in the
599 * shader).
600 */
601 shader.turn_off_early_z_test =
602 v3d->prog.fs->prog_data.fs->disable_ez;
603
604 shader.fragment_shader_uses_real_pixel_centre_w_in_addition_to_centroid_w2 =
605 v3d->prog.fs->prog_data.fs->uses_center_w;
606
607 shader.any_shader_reads_hardware_written_primitive_id =
608 (v3d->prog.gs && v3d->prog.gs->prog_data.gs->uses_pid) ||
609 v3d->prog.fs->prog_data.fs->uses_pid;
610 shader.insert_primitive_id_as_first_varying_to_fragment_shader =
611 !v3d->prog.gs && v3d->prog.fs->prog_data.fs->uses_pid;
612
613 shader.do_scoreboard_wait_on_first_thread_switch =
614 v3d->prog.fs->prog_data.fs->lock_scoreboard_on_first_thrsw;
615 shader.disable_implicit_point_line_varyings =
616 !v3d->prog.fs->prog_data.fs->uses_implicit_point_line_varyings;
617
618 shader.number_of_varyings_in_fragment_shader =
619 v3d->prog.fs->prog_data.fs->num_inputs;
620
621 shader.coordinate_shader_code_address =
622 cl_address(v3d_resource(v3d->prog.cs->resource)->bo,
623 v3d->prog.cs->offset);
624 shader.vertex_shader_code_address =
625 cl_address(v3d_resource(v3d->prog.vs->resource)->bo,
626 v3d->prog.vs->offset);
627 shader.fragment_shader_code_address =
628 cl_address(v3d_resource(v3d->prog.fs->resource)->bo,
629 v3d->prog.fs->offset);
630
631 #if V3D_VERSION == 42
632 shader.coordinate_shader_propagate_nans = true;
633 shader.vertex_shader_propagate_nans = true;
634 shader.fragment_shader_propagate_nans = true;
635
636 /* XXX: Use combined input/output size flag in the common
637 * case.
638 */
639 shader.coordinate_shader_has_separate_input_and_output_vpm_blocks =
640 v3d->prog.cs->prog_data.vs->separate_segments;
641 shader.vertex_shader_has_separate_input_and_output_vpm_blocks =
642 v3d->prog.vs->prog_data.vs->separate_segments;
643 shader.coordinate_shader_input_vpm_segment_size =
644 v3d->prog.cs->prog_data.vs->separate_segments ?
645 v3d->prog.cs->prog_data.vs->vpm_input_size : 1;
646 shader.vertex_shader_input_vpm_segment_size =
647 v3d->prog.vs->prog_data.vs->separate_segments ?
648 v3d->prog.vs->prog_data.vs->vpm_input_size : 1;
649 #endif
650 /* On V3D 7.1 there isn't a specific flag to set if we are using
651 * shared/separate segments or not. We just set the value of
652 * vpm_input_size to 0, and set output to the max needed. That should be
653 * already properly set on prog_data_vs_bin
654 */
655 #if V3D_VERSION == 71
656 shader.coordinate_shader_input_vpm_segment_size =
657 v3d->prog.cs->prog_data.vs->vpm_input_size;
658 shader.vertex_shader_input_vpm_segment_size =
659 v3d->prog.vs->prog_data.vs->vpm_input_size;
660 #endif
661
662 shader.coordinate_shader_output_vpm_segment_size =
663 v3d->prog.cs->prog_data.vs->vpm_output_size;
664 shader.vertex_shader_output_vpm_segment_size =
665 v3d->prog.vs->prog_data.vs->vpm_output_size;
666
667 shader.coordinate_shader_uniforms_address = cs_uniforms;
668 shader.vertex_shader_uniforms_address = vs_uniforms;
669 shader.fragment_shader_uniforms_address = fs_uniforms;
670
671 shader.min_coord_shader_input_segments_required_in_play =
672 vpm_cfg_bin.As;
673 shader.min_vertex_shader_input_segments_required_in_play =
674 vpm_cfg.As;
675
676 shader.min_coord_shader_output_segments_required_in_play_in_addition_to_vcm_cache_size =
677 vpm_cfg_bin.Ve;
678 shader.min_vertex_shader_output_segments_required_in_play_in_addition_to_vcm_cache_size =
679 vpm_cfg.Ve;
680
681 shader.coordinate_shader_4_way_threadable =
682 v3d->prog.cs->prog_data.vs->base.threads == 4;
683 shader.vertex_shader_4_way_threadable =
684 v3d->prog.vs->prog_data.vs->base.threads == 4;
685 shader.fragment_shader_4_way_threadable =
686 v3d->prog.fs->prog_data.fs->base.threads == 4;
687
688 shader.coordinate_shader_start_in_final_thread_section =
689 v3d->prog.cs->prog_data.vs->base.single_seg;
690 shader.vertex_shader_start_in_final_thread_section =
691 v3d->prog.vs->prog_data.vs->base.single_seg;
692 shader.fragment_shader_start_in_final_thread_section =
693 v3d->prog.fs->prog_data.fs->base.single_seg;
694
695 shader.vertex_id_read_by_coordinate_shader =
696 v3d->prog.cs->prog_data.vs->uses_vid;
697 shader.instance_id_read_by_coordinate_shader =
698 v3d->prog.cs->prog_data.vs->uses_iid;
699 shader.vertex_id_read_by_vertex_shader =
700 v3d->prog.vs->prog_data.vs->uses_vid;
701 shader.instance_id_read_by_vertex_shader =
702 v3d->prog.vs->prog_data.vs->uses_iid;
703
704 #if V3D_VERSION == 42
705 shader.address_of_default_attribute_values =
706 cl_address(v3d_resource(vtx->defaults)->bo,
707 vtx->defaults_offset);
708 #endif
709 }
710
711 bool cs_loaded_any = false;
712 for (int i = 0; i < vtx->num_elements; i++) {
713 struct pipe_vertex_element *elem = &vtx->pipe[i];
714 struct pipe_vertex_buffer *vb =
715 &vertexbuf->vb[elem->vertex_buffer_index];
716 struct v3d_resource *rsc = v3d_resource(vb->buffer.resource);
717
718 if (!rsc)
719 continue;
720
721 enum { size = cl_packet_length(GL_SHADER_STATE_ATTRIBUTE_RECORD) };
722 cl_emit_with_prepacked(&job->indirect,
723 GL_SHADER_STATE_ATTRIBUTE_RECORD,
724 &vtx->attrs[i * size], attr) {
725 attr.stride = elem->src_stride;
726 attr.address = cl_address(rsc->bo,
727 vb->buffer_offset +
728 elem->src_offset);
729 attr.number_of_values_read_by_coordinate_shader =
730 v3d->prog.cs->prog_data.vs->vattr_sizes[i];
731 attr.number_of_values_read_by_vertex_shader =
732 v3d->prog.vs->prog_data.vs->vattr_sizes[i];
733
734 /* GFXH-930: At least one attribute must be enabled
735 * and read by CS and VS. If we have attributes being
736 * consumed by the VS but not the CS, then set up a
737 * dummy load of the last attribute into the CS's VPM
738 * inputs. (Since CS is just dead-code-elimination
739 * compared to VS, we can't have CS loading but not
740 * VS).
741 */
742 if (v3d->prog.cs->prog_data.vs->vattr_sizes[i])
743 cs_loaded_any = true;
744 if (i == vtx->num_elements - 1 && !cs_loaded_any) {
745 attr.number_of_values_read_by_coordinate_shader = 1;
746 }
747 attr.maximum_index = 0xffffff;
748 }
749 STATIC_ASSERT(sizeof(vtx->attrs) >= V3D_MAX_VS_INPUTS / 4 * size);
750 }
751
752 if (num_elements_to_emit == 0) {
753 /* GFXH-930: At least one attribute must be enabled and read
754 * by CS and VS. If we have no attributes being consumed by
755 * the shader, set up a dummy to be loaded into the VPM.
756 */
757 cl_emit(&job->indirect, GL_SHADER_STATE_ATTRIBUTE_RECORD, attr) {
758 /* Valid address of data whose value will be unused. */
759 attr.address = cl_address(job->indirect.bo, 0);
760
761 attr.type = ATTRIBUTE_FLOAT;
762 attr.stride = 0;
763 attr.vec_size = 1;
764
765 attr.number_of_values_read_by_coordinate_shader = 1;
766 attr.number_of_values_read_by_vertex_shader = 1;
767 }
768 num_elements_to_emit = 1;
769 }
770
771 cl_emit(&job->bcl, VCM_CACHE_SIZE, vcm) {
772 vcm.number_of_16_vertex_batches_for_binning = vpm_cfg_bin.Vc;
773 vcm.number_of_16_vertex_batches_for_rendering = vpm_cfg.Vc;
774 }
775
776 if (v3d->prog.gs) {
777 cl_emit(&job->bcl, GL_SHADER_STATE_INCLUDING_GS, state) {
778 state.address = cl_address(job->indirect.bo,
779 shader_rec_offset);
780 state.number_of_attribute_arrays = num_elements_to_emit;
781 }
782 } else {
783 cl_emit(&job->bcl, GL_SHADER_STATE, state) {
784 state.address = cl_address(job->indirect.bo,
785 shader_rec_offset);
786 state.number_of_attribute_arrays = num_elements_to_emit;
787 }
788 }
789
790 v3d_bo_unreference(&cs_uniforms.bo);
791 v3d_bo_unreference(&vs_uniforms.bo);
792 if (gs_uniforms.bo)
793 v3d_bo_unreference(&gs_uniforms.bo);
794 if (gs_bin_uniforms.bo)
795 v3d_bo_unreference(&gs_bin_uniforms.bo);
796 v3d_bo_unreference(&fs_uniforms.bo);
797 }
798
799 /**
800 * Updates the number of primitives generated from the number of vertices
801 * to draw. This only works when no GS is present, since otherwise the number
802 * of primitives generated cannot be determined in advance and we need to
803 * use the PRIMITIVE_COUNTS_FEEDBACK command instead, however, that requires
804 * a sync wait for the draw to complete, so we only use that when GS is present.
805 */
806 static void
v3d_update_primitives_generated_counter(struct v3d_context * v3d,const struct pipe_draw_info * info,const struct pipe_draw_start_count_bias * draw)807 v3d_update_primitives_generated_counter(struct v3d_context *v3d,
808 const struct pipe_draw_info *info,
809 const struct pipe_draw_start_count_bias *draw)
810 {
811 assert(!v3d->prog.gs);
812
813 if (!v3d->active_queries)
814 return;
815
816 uint32_t prims = u_prims_for_vertices(info->mode, draw->count);
817 v3d->prims_generated += prims;
818 }
819
820 static void
v3d_update_job_ez(struct v3d_context * v3d,struct v3d_job * job)821 v3d_update_job_ez(struct v3d_context *v3d, struct v3d_job *job)
822 {
823 /* If first_ez_state is V3D_EZ_DISABLED it means that we have already
824 * determined that we should disable EZ completely for all draw calls
825 * in this job. This will cause us to disable EZ for the entire job in
826 * the Tile Rendering Mode RCL packet and when we do that we need to
827 * make sure we never emit a draw call in the job with EZ enabled in
828 * the CFG_BITS packet, so ez_state must also be V3D_EZ_DISABLED.
829 */
830 if (job->first_ez_state == V3D_EZ_DISABLED) {
831 assert(job->ez_state == V3D_EZ_DISABLED);
832 return;
833 }
834
835 /* If this is the first time we update EZ state for this job we first
836 * check if there is anything that requires disabling it completely
837 * for the entire job (based on state that is not related to the
838 * current draw call and pipeline state).
839 */
840 if (!job->decided_global_ez_enable) {
841 job->decided_global_ez_enable = true;
842
843 if (!job->zsbuf) {
844 job->first_ez_state = V3D_EZ_DISABLED;
845 job->ez_state = V3D_EZ_DISABLED;
846 return;
847 }
848
849 /* GFXH-1918: the early-Z buffer may load incorrect depth
850 * values if the frame has odd width or height. Disable early-Z
851 * in this case.
852 */
853 bool needs_depth_load = v3d->zsa && job->zsbuf &&
854 v3d->zsa->base.depth_enabled &&
855 (PIPE_CLEAR_DEPTH & ~job->clear);
856 if (needs_depth_load &&
857 ((job->draw_width % 2 != 0) || (job->draw_height % 2 != 0))) {
858 perf_debug("Loading depth buffer for framebuffer with odd width "
859 "or height disables early-Z tests\n");
860 job->first_ez_state = V3D_EZ_DISABLED;
861 job->ez_state = V3D_EZ_DISABLED;
862 return;
863 }
864 }
865
866 switch (v3d->zsa->ez_state) {
867 case V3D_EZ_UNDECIDED:
868 /* If the Z/S state didn't pick a direction but didn't
869 * disable, then go along with the current EZ state. This
870 * allows EZ optimization for Z func == EQUAL or NEVER.
871 */
872 break;
873
874 case V3D_EZ_LT_LE:
875 case V3D_EZ_GT_GE:
876 /* If the Z/S state picked a direction, then it needs to match
877 * the current direction if we've decided on one.
878 */
879 if (job->ez_state == V3D_EZ_UNDECIDED)
880 job->ez_state = v3d->zsa->ez_state;
881 else if (job->ez_state != v3d->zsa->ez_state)
882 job->ez_state = V3D_EZ_DISABLED;
883 break;
884
885 case V3D_EZ_DISABLED:
886 /* If the current Z/S state disables EZ because of a bad Z
887 * func or stencil operation, then we can't do any more EZ in
888 * this frame.
889 */
890 job->ez_state = V3D_EZ_DISABLED;
891 break;
892 }
893
894 /* If the FS affects the Z of the pixels, then it may update against
895 * the chosen EZ direction (though we could use
896 * ARB_conservative_depth's hints to avoid this)
897 */
898 if (v3d->prog.fs->prog_data.fs->writes_z &&
899 !v3d->prog.fs->prog_data.fs->writes_z_from_fep) {
900 job->ez_state = V3D_EZ_DISABLED;
901 }
902
903 if (job->first_ez_state == V3D_EZ_UNDECIDED &&
904 (job->ez_state != V3D_EZ_DISABLED || job->draw_calls_queued == 0))
905 job->first_ez_state = job->ez_state;
906 }
907
908 static bool
v3d_check_compiled_shaders(struct v3d_context * v3d)909 v3d_check_compiled_shaders(struct v3d_context *v3d)
910 {
911 static bool warned[5] = { 0 };
912
913 uint32_t failed_stage = MESA_SHADER_NONE;
914 if (!v3d->prog.vs->resource || !v3d->prog.cs->resource) {
915 failed_stage = MESA_SHADER_VERTEX;
916 } else if ((v3d->prog.gs_bin && !v3d->prog.gs_bin->resource) ||
917 (v3d->prog.gs && !v3d->prog.gs->resource)) {
918 failed_stage = MESA_SHADER_GEOMETRY;
919 } else if (v3d->prog.fs && !v3d->prog.fs->resource) {
920 failed_stage = MESA_SHADER_FRAGMENT;
921 }
922
923 if (likely(failed_stage == MESA_SHADER_NONE))
924 return true;
925
926 if (!warned[failed_stage]) {
927 fprintf(stderr,
928 "%s shader failed to compile. Expect corruption.\n",
929 _mesa_shader_stage_to_string(failed_stage));
930 warned[failed_stage] = true;
931 }
932 return false;
933 }
934
935 static void
v3d_draw_vbo(struct pipe_context * pctx,const struct pipe_draw_info * info,unsigned drawid_offset,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)936 v3d_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info,
937 unsigned drawid_offset,
938 const struct pipe_draw_indirect_info *indirect,
939 const struct pipe_draw_start_count_bias *draws,
940 unsigned num_draws)
941 {
942 if (num_draws > 1) {
943 util_draw_multi(pctx, info, drawid_offset, indirect, draws, num_draws);
944 return;
945 }
946
947 if (!indirect && (!draws[0].count || !info->instance_count))
948 return;
949
950 struct v3d_context *v3d = v3d_context(pctx);
951
952 if (!indirect &&
953 !info->primitive_restart &&
954 !u_trim_pipe_prim(info->mode, (unsigned*)&draws[0].count))
955 return;
956
957 if (!v3d_render_condition_check(v3d))
958 return;
959
960 /* Fall back for weird desktop GL primitive restart values. */
961 if (info->primitive_restart &&
962 info->index_size) {
963 uint32_t mask = util_prim_restart_index_from_size(info->index_size);
964 if (info->restart_index != mask) {
965 util_draw_vbo_without_prim_restart(pctx, info, drawid_offset, indirect, &draws[0]);
966 return;
967 }
968 }
969
970 /* Before setting up the draw, flush anything writing to the resources
971 * that we read from or reading from resources we write to.
972 */
973 for (int s = 0; s < PIPE_SHADER_COMPUTE; s++)
974 v3d_predraw_check_stage_inputs(pctx, s);
975
976 if (indirect && indirect->buffer) {
977 v3d_flush_jobs_writing_resource(v3d, indirect->buffer,
978 V3D_FLUSH_DEFAULT, false);
979 }
980
981 v3d_predraw_check_outputs(pctx);
982
983 /* If transform feedback is active and we are switching primitive type
984 * we need to submit the job before drawing and update the vertex count
985 * written to TF based on the primitive type since we will need to
986 * know the exact vertex count if the application decides to call
987 * glDrawTransformFeedback() later.
988 */
989 if (v3d->streamout.num_targets > 0 &&
990 u_base_prim_type(info->mode) != u_base_prim_type(v3d->prim_mode)) {
991 v3d_update_primitive_counters(v3d);
992 }
993
994 struct v3d_job *job = v3d_get_job_for_fbo(v3d);
995
996 /* If vertex texturing depends on the output of rendering, we need to
997 * ensure that that rendering is complete before we run a coordinate
998 * shader that depends on it.
999 *
1000 * Given that doing that is unusual, for now we just block the binner
1001 * on the last submitted render, rather than tracking the last
1002 * rendering to each texture's BO.
1003 */
1004 if (v3d->tex[PIPE_SHADER_VERTEX].num_textures || (indirect && indirect->buffer)) {
1005 static bool warned = false;
1006 if (!warned) {
1007 perf_debug("Blocking binner on last render due to "
1008 "vertex texturing or indirect drawing.\n");
1009 warned = true;
1010 }
1011 job->submit.in_sync_bcl = v3d->out_sync;
1012 }
1013
1014 /* We also need to ensure that compute is complete when render depends
1015 * on resources written by it.
1016 */
1017 if (v3d->sync_on_last_compute_job) {
1018 job->submit.in_sync_bcl = v3d->out_sync;
1019 v3d->sync_on_last_compute_job = false;
1020 }
1021
1022 /* Mark SSBOs and images as being written. We don't actually know
1023 * which ones are read vs written, so just assume the worst.
1024 */
1025 for (int s = 0; s < PIPE_SHADER_COMPUTE; s++) {
1026 u_foreach_bit(i, v3d->ssbo[s].enabled_mask) {
1027 v3d_job_add_write_resource(job,
1028 v3d->ssbo[s].sb[i].buffer);
1029 struct v3d_resource *rsc= v3d_resource(v3d->ssbo[s].sb[i].buffer);
1030 rsc->graphics_written = true;
1031 job->tmu_dirty_rcl = true;
1032 }
1033
1034 u_foreach_bit(i, v3d->shaderimg[s].enabled_mask) {
1035 v3d_job_add_write_resource(job,
1036 v3d->shaderimg[s].si[i].base.resource);
1037 struct v3d_resource *rsc= v3d_resource(v3d->shaderimg[s].si[i].base.resource);
1038 rsc->graphics_written = true;
1039 job->tmu_dirty_rcl = true;
1040 }
1041 }
1042
1043 /* Get space to emit our draw call into the BCL, using a branch to
1044 * jump to a new BO if necessary.
1045 */
1046 v3d_cl_ensure_space_with_branch(&job->bcl, 256 /* XXX */);
1047
1048 if (v3d->prim_mode != info->mode) {
1049 v3d->prim_mode = info->mode;
1050 v3d->dirty |= V3D_DIRTY_PRIM_MODE;
1051 }
1052
1053 v3d_start_draw(v3d);
1054 v3d_update_compiled_shaders(v3d, info->mode);
1055 if (!v3d_check_compiled_shaders(v3d))
1056 return;
1057 v3d_update_job_ez(v3d, job);
1058
1059 /* If this job was writing to transform feedback buffers before this
1060 * draw and we are reading from them here, then we need to wait for TF
1061 * to complete before we emit this draw.
1062 *
1063 * Notice this check needs to happen before we emit state for the
1064 * current draw call, where we update job->tf_enabled, so we can ensure
1065 * that we only check TF writes for prior draws.
1066 */
1067 v3d_emit_wait_for_tf_if_needed(v3d, job);
1068
1069 v3dX(emit_state)(pctx);
1070
1071 if (v3d->dirty & (V3D_DIRTY_VTXBUF |
1072 V3D_DIRTY_VTXSTATE |
1073 V3D_DIRTY_PRIM_MODE |
1074 V3D_DIRTY_RASTERIZER |
1075 V3D_DIRTY_COMPILED_CS |
1076 V3D_DIRTY_COMPILED_VS |
1077 V3D_DIRTY_COMPILED_GS_BIN |
1078 V3D_DIRTY_COMPILED_GS |
1079 V3D_DIRTY_COMPILED_FS |
1080 v3d->prog.cs->uniform_dirty_bits |
1081 v3d->prog.vs->uniform_dirty_bits |
1082 (v3d->prog.gs_bin ?
1083 v3d->prog.gs_bin->uniform_dirty_bits : 0) |
1084 (v3d->prog.gs ?
1085 v3d->prog.gs->uniform_dirty_bits : 0) |
1086 v3d->prog.fs->uniform_dirty_bits)) {
1087 v3d_emit_gl_shader_state(v3d, info);
1088 }
1089
1090 v3d->dirty = 0;
1091
1092 /* The Base Vertex/Base Instance packet sets those values to nonzero
1093 * for the next draw call only.
1094 */
1095 if ((info->index_size && draws->index_bias) || info->start_instance) {
1096 cl_emit(&job->bcl, BASE_VERTEX_BASE_INSTANCE, base) {
1097 base.base_instance = info->start_instance;
1098 base.base_vertex = info->index_size ? draws->index_bias : 0;
1099 }
1100 }
1101
1102 uint32_t prim_tf_enable = 0;
1103
1104 v3d->prim_restart = info->primitive_restart;
1105
1106 if (!v3d->prog.gs && !v3d->prim_restart)
1107 v3d_update_primitives_generated_counter(v3d, info, &draws[0]);
1108
1109 uint32_t hw_prim_type = v3d_hw_prim_type(info->mode);
1110 if (info->index_size) {
1111 uint32_t index_size = info->index_size;
1112 uint32_t offset = draws[0].start * index_size;
1113 struct pipe_resource *prsc;
1114 if (info->has_user_indices) {
1115 unsigned start_offset = draws[0].start * info->index_size;
1116 prsc = NULL;
1117 u_upload_data(v3d->uploader, start_offset,
1118 draws[0].count * info->index_size, 4,
1119 (char*)info->index.user + start_offset,
1120 &offset, &prsc);
1121 } else {
1122 prsc = info->index.resource;
1123 }
1124 struct v3d_resource *rsc = v3d_resource(prsc);
1125
1126 cl_emit(&job->bcl, INDEX_BUFFER_SETUP, ib) {
1127 ib.address = cl_address(rsc->bo, 0);
1128 ib.size = rsc->bo->size;
1129 }
1130
1131 if (indirect && indirect->buffer) {
1132 cl_emit(&job->bcl, INDIRECT_INDEXED_INSTANCED_PRIM_LIST, prim) {
1133 prim.index_type = ffs(info->index_size) - 1;
1134 prim.mode = hw_prim_type | prim_tf_enable;
1135 prim.enable_primitive_restarts = info->primitive_restart;
1136
1137 prim.number_of_draw_indirect_indexed_records = indirect->draw_count;
1138
1139 prim.stride_in_multiples_of_4_bytes = indirect->stride >> 2;
1140 prim.address = cl_address(v3d_resource(indirect->buffer)->bo,
1141 indirect->offset);
1142 }
1143 } else if (info->instance_count > 1) {
1144 cl_emit(&job->bcl, INDEXED_INSTANCED_PRIM_LIST, prim) {
1145 prim.index_type = ffs(info->index_size) - 1;
1146 prim.index_offset = offset;
1147 prim.mode = hw_prim_type | prim_tf_enable;
1148 prim.enable_primitive_restarts = info->primitive_restart;
1149
1150 prim.number_of_instances = info->instance_count;
1151 prim.instance_length = draws[0].count;
1152 }
1153 } else {
1154 cl_emit(&job->bcl, INDEXED_PRIM_LIST, prim) {
1155 prim.index_type = ffs(info->index_size) - 1;
1156 prim.length = draws[0].count;
1157 prim.index_offset = offset;
1158 prim.mode = hw_prim_type | prim_tf_enable;
1159 prim.enable_primitive_restarts = info->primitive_restart;
1160 }
1161 }
1162
1163 if (info->has_user_indices)
1164 pipe_resource_reference(&prsc, NULL);
1165 } else {
1166 if (indirect && indirect->buffer) {
1167 cl_emit(&job->bcl, INDIRECT_VERTEX_ARRAY_INSTANCED_PRIMS, prim) {
1168 prim.mode = hw_prim_type | prim_tf_enable;
1169 prim.number_of_draw_indirect_array_records = indirect->draw_count;
1170
1171 prim.stride_in_multiples_of_4_bytes = indirect->stride >> 2;
1172 prim.address = cl_address(v3d_resource(indirect->buffer)->bo,
1173 indirect->offset);
1174 }
1175 } else if (info->instance_count > 1) {
1176 struct pipe_stream_output_target *so =
1177 indirect && indirect->count_from_stream_output ?
1178 indirect->count_from_stream_output : NULL;
1179 uint32_t vert_count = so ?
1180 v3d_stream_output_target_get_vertex_count(so) :
1181 draws[0].count;
1182 cl_emit(&job->bcl, VERTEX_ARRAY_INSTANCED_PRIMS, prim) {
1183 prim.mode = hw_prim_type | prim_tf_enable;
1184 prim.index_of_first_vertex = draws[0].start;
1185 prim.number_of_instances = info->instance_count;
1186 prim.instance_length = vert_count;
1187 }
1188 } else {
1189 struct pipe_stream_output_target *so =
1190 indirect && indirect->count_from_stream_output ?
1191 indirect->count_from_stream_output : NULL;
1192 uint32_t vert_count = so ?
1193 v3d_stream_output_target_get_vertex_count(so) :
1194 draws[0].count;
1195 cl_emit(&job->bcl, VERTEX_ARRAY_PRIMS, prim) {
1196 prim.mode = hw_prim_type | prim_tf_enable;
1197 prim.length = vert_count;
1198 prim.index_of_first_vertex = draws[0].start;
1199 }
1200 }
1201 }
1202
1203 /* A flush is required in between a TF draw and any following TF specs
1204 * packet, or the GPU may hang. Just flush each time for now.
1205 */
1206 if (v3d->streamout.num_targets)
1207 cl_emit(&job->bcl, TRANSFORM_FEEDBACK_FLUSH_AND_COUNT, flush);
1208
1209 job->draw_calls_queued++;
1210 if (v3d->streamout.num_targets)
1211 job->tf_draw_calls_queued++;
1212
1213 /* Increment the TF offsets by how many verts we wrote. XXX: This
1214 * needs some clamping to the buffer size.
1215 *
1216 * If primitive restart is enabled or we have a geometry shader, we
1217 * update it later, when we can query the device to know how many
1218 * vertices were written.
1219 */
1220 if (!v3d->prog.gs && !v3d->prim_restart) {
1221 for (int i = 0; i < v3d->streamout.num_targets; i++)
1222 v3d_stream_output_target(v3d->streamout.targets[i])->offset +=
1223 u_stream_outputs_for_vertices(info->mode, draws[0].count);
1224 }
1225
1226 if (v3d->zsa && job->zsbuf && v3d->zsa->base.depth_enabled) {
1227 struct v3d_resource *rsc = v3d_resource(job->zsbuf->texture);
1228 v3d_job_add_bo(job, rsc->bo);
1229
1230 job->load |= PIPE_CLEAR_DEPTH & ~job->clear;
1231 if (v3d->zsa->base.depth_writemask)
1232 job->store |= PIPE_CLEAR_DEPTH;
1233 rsc->initialized_buffers = PIPE_CLEAR_DEPTH;
1234 }
1235
1236 if (v3d->zsa && job->zsbuf && v3d->zsa->base.stencil[0].enabled) {
1237 struct v3d_resource *rsc = v3d_resource(job->zsbuf->texture);
1238 if (rsc->separate_stencil)
1239 rsc = rsc->separate_stencil;
1240
1241 v3d_job_add_bo(job, rsc->bo);
1242
1243 job->load |= PIPE_CLEAR_STENCIL & ~job->clear;
1244 if (v3d->zsa->base.stencil[0].writemask ||
1245 v3d->zsa->base.stencil[1].writemask) {
1246 job->store |= PIPE_CLEAR_STENCIL;
1247 }
1248 rsc->initialized_buffers |= PIPE_CLEAR_STENCIL;
1249 }
1250
1251 for (int i = 0; i < job->nr_cbufs; i++) {
1252 uint32_t bit = PIPE_CLEAR_COLOR0 << i;
1253 int blend_rt = v3d->blend->base.independent_blend_enable ? i : 0;
1254
1255 if (job->store & bit || !job->cbufs[i])
1256 continue;
1257 struct v3d_resource *rsc = v3d_resource(job->cbufs[i]->texture);
1258
1259 job->load |= bit & ~job->clear;
1260 if (v3d->blend->base.rt[blend_rt].colormask)
1261 job->store |= bit;
1262 v3d_job_add_bo(job, rsc->bo);
1263 }
1264
1265 if (job->referenced_size > 768 * 1024 * 1024) {
1266 perf_debug("Flushing job with %dkb to try to free up memory\n",
1267 job->referenced_size / 1024);
1268 v3d_flush(pctx);
1269 }
1270
1271 if (V3D_DBG(ALWAYS_FLUSH))
1272 v3d_flush(pctx);
1273 }
1274
1275 static void
v3d_launch_grid(struct pipe_context * pctx,const struct pipe_grid_info * info)1276 v3d_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info)
1277 {
1278 struct v3d_context *v3d = v3d_context(pctx);
1279 struct v3d_screen *screen = v3d->screen;
1280
1281 v3d_predraw_check_stage_inputs(pctx, PIPE_SHADER_COMPUTE);
1282
1283 v3d_update_compiled_cs(v3d);
1284
1285 if (!v3d->prog.compute->resource) {
1286 static bool warned = false;
1287 if (!warned) {
1288 fprintf(stderr,
1289 "Compute shader failed to compile. "
1290 "Expect corruption.\n");
1291 warned = true;
1292 }
1293 return;
1294 }
1295
1296 /* Some of the units of scale:
1297 *
1298 * - Batches of 16 work items (shader invocations) that will be queued
1299 * to the run on a QPU at once.
1300 *
1301 * - Workgroups composed of work items based on the shader's layout
1302 * declaration.
1303 *
1304 * - Supergroups of 1-16 workgroups. There can only be 16 supergroups
1305 * running at a time on the core, so we want to keep them large to
1306 * keep the QPUs busy, but a whole supergroup will sync at a barrier
1307 * so we want to keep them small if one is present.
1308 */
1309 struct drm_v3d_submit_csd submit = { 0 };
1310 struct v3d_job *job = v3d_job_create(v3d);
1311
1312 /* Set up the actual number of workgroups, synchronously mapping the
1313 * indirect buffer if necessary to get the dimensions.
1314 */
1315 if (info->indirect) {
1316 struct pipe_transfer *transfer;
1317 uint32_t *map = pipe_buffer_map_range(pctx, info->indirect,
1318 info->indirect_offset,
1319 3 * sizeof(uint32_t),
1320 PIPE_MAP_READ,
1321 &transfer);
1322 memcpy(v3d->compute_num_workgroups, map, 3 * sizeof(uint32_t));
1323 pipe_buffer_unmap(pctx, transfer);
1324
1325 if (v3d->compute_num_workgroups[0] == 0 ||
1326 v3d->compute_num_workgroups[1] == 0 ||
1327 v3d->compute_num_workgroups[2] == 0) {
1328 /* Nothing to dispatch, so skip the draw (CSD can't
1329 * handle 0 workgroups).
1330 */
1331 return;
1332 }
1333 } else {
1334 v3d->compute_num_workgroups[0] = info->grid[0];
1335 v3d->compute_num_workgroups[1] = info->grid[1];
1336 v3d->compute_num_workgroups[2] = info->grid[2];
1337 }
1338
1339 uint32_t num_wgs = 1;
1340 for (int i = 0; i < 3; i++) {
1341 num_wgs *= v3d->compute_num_workgroups[i];
1342 submit.cfg[i] |= (v3d->compute_num_workgroups[i] <<
1343 V3D_CSD_CFG012_WG_COUNT_SHIFT);
1344 }
1345
1346 uint32_t wg_size = info->block[0] * info->block[1] * info->block[2];
1347
1348 struct v3d_compute_prog_data *compute =
1349 v3d->prog.compute->prog_data.compute;
1350 uint32_t wgs_per_sg =
1351 v3d_csd_choose_workgroups_per_supergroup(
1352 &v3d->screen->devinfo,
1353 compute->has_subgroups,
1354 compute->base.has_control_barrier,
1355 compute->base.threads,
1356 num_wgs, wg_size);
1357
1358 uint32_t batches_per_sg = DIV_ROUND_UP(wgs_per_sg * wg_size, 16);
1359 uint32_t whole_sgs = num_wgs / wgs_per_sg;
1360 uint32_t rem_wgs = num_wgs - whole_sgs * wgs_per_sg;
1361 uint32_t num_batches = batches_per_sg * whole_sgs +
1362 DIV_ROUND_UP(rem_wgs * wg_size, 16);
1363
1364 submit.cfg[3] |= (wgs_per_sg & 0xf) << V3D_CSD_CFG3_WGS_PER_SG_SHIFT;
1365 submit.cfg[3] |=
1366 (batches_per_sg - 1) << V3D_CSD_CFG3_BATCHES_PER_SG_M1_SHIFT;
1367 submit.cfg[3] |= (wg_size & 0xff) << V3D_CSD_CFG3_WG_SIZE_SHIFT;
1368
1369
1370 /* Number of batches the dispatch will invoke.
1371 * V3D 7.1.6 and later don't subtract 1 from the number of batches
1372 */
1373 if (v3d->screen->devinfo.ver < 71 ||
1374 (v3d->screen->devinfo.ver == 71 && v3d->screen->devinfo.rev < 6)) {
1375 submit.cfg[4] = num_batches - 1;
1376 } else {
1377 submit.cfg[4] = num_batches;
1378 }
1379
1380 /* Make sure we didn't accidentally underflow. */
1381 assert(submit.cfg[4] != ~0);
1382
1383 v3d_job_add_bo(job, v3d_resource(v3d->prog.compute->resource)->bo);
1384 submit.cfg[5] = (v3d_resource(v3d->prog.compute->resource)->bo->offset +
1385 v3d->prog.compute->offset);
1386 if (v3d->screen->devinfo.ver < 71)
1387 submit.cfg[5] |= V3D_CSD_CFG5_PROPAGATE_NANS;
1388 if (v3d->prog.compute->prog_data.base->single_seg)
1389 submit.cfg[5] |= V3D_CSD_CFG5_SINGLE_SEG;
1390 if (v3d->prog.compute->prog_data.base->threads == 4)
1391 submit.cfg[5] |= V3D_CSD_CFG5_THREADING;
1392
1393 if (v3d->prog.compute->prog_data.compute->shared_size) {
1394 v3d->compute_shared_memory =
1395 v3d_bo_alloc(v3d->screen,
1396 v3d->prog.compute->prog_data.compute->shared_size *
1397 num_wgs,
1398 "shared_vars");
1399 }
1400
1401 struct v3d_cl_reloc uniforms = v3d_write_uniforms(v3d, job,
1402 v3d->prog.compute,
1403 PIPE_SHADER_COMPUTE);
1404 v3d_job_add_bo(job, uniforms.bo);
1405 submit.cfg[6] = uniforms.bo->offset + uniforms.offset;
1406
1407 /* Pull some job state that was stored in a SUBMIT_CL struct out to
1408 * our SUBMIT_CSD struct
1409 */
1410 submit.bo_handles = job->submit.bo_handles;
1411 submit.bo_handle_count = job->submit.bo_handle_count;
1412
1413 /* Serialize this in the rest of our command stream. */
1414 submit.in_sync = v3d->out_sync;
1415 submit.out_sync = v3d->out_sync;
1416
1417 if (v3d->active_perfmon) {
1418 assert(screen->has_perfmon);
1419 submit.perfmon_id = v3d->active_perfmon->kperfmon_id;
1420 }
1421
1422 v3d->last_perfmon = v3d->active_perfmon;
1423
1424 if (!V3D_DBG(NORAST)) {
1425 int ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_SUBMIT_CSD,
1426 &submit);
1427 static bool warned = false;
1428 if (ret && !warned) {
1429 fprintf(stderr, "CSD submit call returned %s. "
1430 "Expect corruption.\n", strerror(errno));
1431 warned = true;
1432 } else if (!ret) {
1433 if (v3d->active_perfmon)
1434 v3d->active_perfmon->job_submitted = true;
1435 }
1436 }
1437
1438 v3d_job_free(v3d, job);
1439
1440 /* Mark SSBOs as being written.. we don't actually know which ones are
1441 * read vs written, so just assume the worst
1442 */
1443 u_foreach_bit(i, v3d->ssbo[PIPE_SHADER_COMPUTE].enabled_mask) {
1444 struct v3d_resource *rsc = v3d_resource(
1445 v3d->ssbo[PIPE_SHADER_COMPUTE].sb[i].buffer);
1446 rsc->writes++;
1447 rsc->compute_written = true;
1448 }
1449
1450 u_foreach_bit(i, v3d->shaderimg[PIPE_SHADER_COMPUTE].enabled_mask) {
1451 struct v3d_resource *rsc = v3d_resource(
1452 v3d->shaderimg[PIPE_SHADER_COMPUTE].si[i].base.resource);
1453 rsc->writes++;
1454 rsc->compute_written = true;
1455 }
1456
1457 v3d_bo_unreference(&uniforms.bo);
1458 v3d_bo_unreference(&v3d->compute_shared_memory);
1459 }
1460
1461 /**
1462 * Implements gallium's clear() hook (glClear()) by drawing a pair of triangles.
1463 */
1464 static void
v3d_draw_clear(struct v3d_context * v3d,unsigned buffers,const union pipe_color_union * color,double depth,unsigned stencil)1465 v3d_draw_clear(struct v3d_context *v3d,
1466 unsigned buffers,
1467 const union pipe_color_union *color,
1468 double depth, unsigned stencil)
1469 {
1470 v3d_blitter_save(v3d, false, true);
1471 util_blitter_clear(v3d->blitter,
1472 v3d->framebuffer.width,
1473 v3d->framebuffer.height,
1474 util_framebuffer_get_num_layers(&v3d->framebuffer),
1475 buffers, color, depth, stencil,
1476 util_framebuffer_get_num_samples(&v3d->framebuffer) > 1);
1477 }
1478
1479 /**
1480 * Attempts to perform the GL clear by using the TLB's fast clear at the start
1481 * of the frame.
1482 */
1483 static unsigned
v3d_tlb_clear(struct v3d_job * job,unsigned buffers,const union pipe_color_union * color,double depth,unsigned stencil)1484 v3d_tlb_clear(struct v3d_job *job, unsigned buffers,
1485 const union pipe_color_union *color,
1486 double depth, unsigned stencil)
1487 {
1488 struct v3d_context *v3d = job->v3d;
1489
1490 if (job->draw_calls_queued) {
1491 /* If anything in the CL has drawn using the buffer, then the
1492 * TLB clear we're trying to add now would happen before that
1493 * drawing.
1494 */
1495 buffers &= ~(job->load | job->store);
1496 }
1497
1498 /* GFXH-1461: If we were to emit a load of just depth or just stencil,
1499 * then the clear for the other may get lost. We need to decide now
1500 * if it would be possible to need to emit a load of just one after
1501 * we've set up our TLB clears. This issue is fixed since V3D 4.3.18.
1502 */
1503 if (v3d->screen->devinfo.ver == 42 &&
1504 buffers & PIPE_CLEAR_DEPTHSTENCIL &&
1505 (buffers & PIPE_CLEAR_DEPTHSTENCIL) != PIPE_CLEAR_DEPTHSTENCIL &&
1506 job->zsbuf &&
1507 util_format_is_depth_and_stencil(job->zsbuf->texture->format)) {
1508 buffers &= ~PIPE_CLEAR_DEPTHSTENCIL;
1509 }
1510
1511 for (int i = 0; i < job->nr_cbufs; i++) {
1512 uint32_t bit = PIPE_CLEAR_COLOR0 << i;
1513 if (!(buffers & bit))
1514 continue;
1515
1516 struct pipe_surface *psurf = v3d->framebuffer.cbufs[i];
1517 struct v3d_surface *surf = v3d_surface(psurf);
1518 struct v3d_resource *rsc = v3d_resource(psurf->texture);
1519
1520 union util_color uc;
1521 uint32_t internal_size = 4 << surf->internal_bpp;
1522
1523 /* While hardware supports clamping, this is not applied on
1524 * the clear values, so we need to do it manually.
1525 *
1526 * "Clamping is performed on color values immediately as they
1527 * enter the TLB and after blending. Clamping is not
1528 * performed on the clear color."
1529 */
1530 union pipe_color_union clamped_color =
1531 util_clamp_color(psurf->format, color);
1532
1533 if (v3d->swap_color_rb & (1 << i)) {
1534 union pipe_color_union orig_color = clamped_color;
1535 clamped_color.f[0] = orig_color.f[2];
1536 clamped_color.f[1] = orig_color.f[1];
1537 clamped_color.f[2] = orig_color.f[0];
1538 clamped_color.f[3] = orig_color.f[3];
1539 }
1540
1541 if (util_format_is_alpha(psurf->format))
1542 clamped_color.f[0] = clamped_color.f[3];
1543
1544 switch (surf->internal_type) {
1545 case V3D_INTERNAL_TYPE_8:
1546 util_pack_color(clamped_color.f, PIPE_FORMAT_R8G8B8A8_UNORM,
1547 &uc);
1548 memcpy(job->clear_color[i], uc.ui, internal_size);
1549 break;
1550 case V3D_INTERNAL_TYPE_8I:
1551 case V3D_INTERNAL_TYPE_8UI:
1552 job->clear_color[i][0] = ((clamped_color.ui[0] & 0xff) |
1553 (clamped_color.ui[1] & 0xff) << 8 |
1554 (clamped_color.ui[2] & 0xff) << 16 |
1555 (clamped_color.ui[3] & 0xff) << 24);
1556 break;
1557 case V3D_INTERNAL_TYPE_16F:
1558 util_pack_color(clamped_color.f, PIPE_FORMAT_R16G16B16A16_FLOAT,
1559 &uc);
1560 memcpy(job->clear_color[i], uc.ui, internal_size);
1561 break;
1562 case V3D_INTERNAL_TYPE_16I:
1563 case V3D_INTERNAL_TYPE_16UI:
1564 job->clear_color[i][0] = ((clamped_color.ui[0] & 0xffff) |
1565 clamped_color.ui[1] << 16);
1566 job->clear_color[i][1] = ((clamped_color.ui[2] & 0xffff) |
1567 clamped_color.ui[3] << 16);
1568 break;
1569 case V3D_INTERNAL_TYPE_32F:
1570 case V3D_INTERNAL_TYPE_32I:
1571 case V3D_INTERNAL_TYPE_32UI:
1572 memcpy(job->clear_color[i], clamped_color.ui, internal_size);
1573 break;
1574 }
1575
1576 rsc->initialized_buffers |= bit;
1577 }
1578
1579 unsigned zsclear = buffers & PIPE_CLEAR_DEPTHSTENCIL;
1580 if (zsclear) {
1581 struct v3d_resource *rsc =
1582 v3d_resource(v3d->framebuffer.zsbuf->texture);
1583
1584 if (zsclear & PIPE_CLEAR_DEPTH)
1585 job->clear_z = depth;
1586 if (zsclear & PIPE_CLEAR_STENCIL)
1587 job->clear_s = stencil;
1588
1589 rsc->initialized_buffers |= zsclear;
1590 }
1591
1592 job->draw_min_x = 0;
1593 job->draw_min_y = 0;
1594 job->draw_max_x = v3d->framebuffer.width;
1595 job->draw_max_y = v3d->framebuffer.height;
1596 job->clear |= buffers;
1597 job->store |= buffers;
1598 job->scissor.disabled = true;
1599
1600 v3d_start_draw(v3d);
1601
1602 return buffers;
1603 }
1604
1605 static void
v3d_clear(struct pipe_context * pctx,unsigned buffers,const struct pipe_scissor_state * scissor_state,const union pipe_color_union * color,double depth,unsigned stencil)1606 v3d_clear(struct pipe_context *pctx, unsigned buffers, const struct pipe_scissor_state *scissor_state,
1607 const union pipe_color_union *color, double depth, unsigned stencil)
1608 {
1609 struct v3d_context *v3d = v3d_context(pctx);
1610 struct v3d_job *job = v3d_get_job_for_fbo(v3d);
1611
1612 buffers &= ~v3d_tlb_clear(job, buffers, color, depth, stencil);
1613
1614 if (!buffers || !v3d_render_condition_check(v3d))
1615 return;
1616
1617 v3d_draw_clear(v3d, buffers, color, depth, stencil);
1618 }
1619
1620 static void
v3d_clear_render_target(struct pipe_context * pctx,struct pipe_surface * ps,const union pipe_color_union * color,unsigned x,unsigned y,unsigned w,unsigned h,bool render_condition_enabled)1621 v3d_clear_render_target(struct pipe_context *pctx, struct pipe_surface *ps,
1622 const union pipe_color_union *color,
1623 unsigned x, unsigned y, unsigned w, unsigned h,
1624 bool render_condition_enabled)
1625 {
1626 struct v3d_context *v3d = v3d_context(pctx);
1627
1628 if (render_condition_enabled && !v3d_render_condition_check(v3d))
1629 return;
1630
1631 v3d_blitter_save(v3d, false, render_condition_enabled);
1632 util_blitter_clear_render_target(v3d->blitter, ps, color, x, y, w, h);
1633 }
1634
1635 static void
v3d_clear_depth_stencil(struct pipe_context * pctx,struct pipe_surface * ps,unsigned buffers,double depth,unsigned stencil,unsigned x,unsigned y,unsigned w,unsigned h,bool render_condition_enabled)1636 v3d_clear_depth_stencil(struct pipe_context *pctx, struct pipe_surface *ps,
1637 unsigned buffers, double depth, unsigned stencil,
1638 unsigned x, unsigned y, unsigned w, unsigned h,
1639 bool render_condition_enabled)
1640 {
1641 struct v3d_context *v3d = v3d_context(pctx);
1642
1643 if (render_condition_enabled && !v3d_render_condition_check(v3d))
1644 return;
1645
1646 v3d_blitter_save(v3d, false, render_condition_enabled);
1647 util_blitter_clear_depth_stencil(v3d->blitter, ps, buffers, depth,
1648 stencil, x, y, w, h);
1649 }
1650
1651 void
v3dX(draw_init)1652 v3dX(draw_init)(struct pipe_context *pctx)
1653 {
1654 pctx->draw_vbo = v3d_draw_vbo;
1655 pctx->clear = v3d_clear;
1656 pctx->clear_render_target = v3d_clear_render_target;
1657 pctx->clear_depth_stencil = v3d_clear_depth_stencil;
1658 if (v3d_context(pctx)->screen->has_csd)
1659 pctx->launch_grid = v3d_launch_grid;
1660 }
1661