1 /*
2 * Copyright (C) 2016 Rob Clark <robclark@freedesktop.org>
3 * Copyright © 2018 Google, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 * Authors:
25 * Rob Clark <robclark@freedesktop.org>
26 */
27
28 #include "pipe/p_state.h"
29 #include "util/bitset.h"
30 #include "util/format/u_format.h"
31 #include "util/u_inlines.h"
32 #include "util/u_memory.h"
33 #include "util/u_string.h"
34
35 #include "freedreno_program.h"
36
37 #include "fd6_const.h"
38 #include "fd6_emit.h"
39 #include "fd6_format.h"
40 #include "fd6_pack.h"
41 #include "fd6_program.h"
42 #include "fd6_texture.h"
43
44 void
fd6_emit_shader(struct fd_context * ctx,struct fd_ringbuffer * ring,const struct ir3_shader_variant * so)45 fd6_emit_shader(struct fd_context *ctx, struct fd_ringbuffer *ring,
46 const struct ir3_shader_variant *so)
47 {
48 enum a6xx_state_block sb = fd6_stage2shadersb(so->type);
49
50 uint32_t first_exec_offset = 0;
51 uint32_t instrlen = 0;
52 uint32_t hw_stack_offset = 0;
53
54 switch (so->type) {
55 case MESA_SHADER_VERTEX:
56 first_exec_offset = REG_A6XX_SP_VS_OBJ_FIRST_EXEC_OFFSET;
57 instrlen = REG_A6XX_SP_VS_INSTRLEN;
58 hw_stack_offset = REG_A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET;
59 break;
60 case MESA_SHADER_TESS_CTRL:
61 first_exec_offset = REG_A6XX_SP_HS_OBJ_FIRST_EXEC_OFFSET;
62 instrlen = REG_A6XX_SP_HS_INSTRLEN;
63 hw_stack_offset = REG_A6XX_SP_HS_PVT_MEM_HW_STACK_OFFSET;
64 break;
65 case MESA_SHADER_TESS_EVAL:
66 first_exec_offset = REG_A6XX_SP_DS_OBJ_FIRST_EXEC_OFFSET;
67 instrlen = REG_A6XX_SP_DS_INSTRLEN;
68 hw_stack_offset = REG_A6XX_SP_DS_PVT_MEM_HW_STACK_OFFSET;
69 break;
70 case MESA_SHADER_GEOMETRY:
71 first_exec_offset = REG_A6XX_SP_GS_OBJ_FIRST_EXEC_OFFSET;
72 instrlen = REG_A6XX_SP_GS_INSTRLEN;
73 hw_stack_offset = REG_A6XX_SP_GS_PVT_MEM_HW_STACK_OFFSET;
74 break;
75 case MESA_SHADER_FRAGMENT:
76 first_exec_offset = REG_A6XX_SP_FS_OBJ_FIRST_EXEC_OFFSET;
77 instrlen = REG_A6XX_SP_FS_INSTRLEN;
78 hw_stack_offset = REG_A6XX_SP_FS_PVT_MEM_HW_STACK_OFFSET;
79 break;
80 case MESA_SHADER_COMPUTE:
81 case MESA_SHADER_KERNEL:
82 first_exec_offset = REG_A6XX_SP_CS_OBJ_FIRST_EXEC_OFFSET;
83 instrlen = REG_A6XX_SP_CS_INSTRLEN;
84 hw_stack_offset = REG_A6XX_SP_CS_PVT_MEM_HW_STACK_OFFSET;
85 break;
86 case MESA_SHADER_TASK:
87 case MESA_SHADER_MESH:
88 case MESA_SHADER_RAYGEN:
89 case MESA_SHADER_ANY_HIT:
90 case MESA_SHADER_CLOSEST_HIT:
91 case MESA_SHADER_MISS:
92 case MESA_SHADER_INTERSECTION:
93 case MESA_SHADER_CALLABLE:
94 unreachable("Unsupported shader stage");
95 case MESA_SHADER_NONE:
96 unreachable("");
97 }
98
99 #ifdef DEBUG
100 /* Name should generally match what you get with MESA_SHADER_CAPTURE_PATH: */
101 const char *name = so->shader->nir->info.name;
102 if (name)
103 fd_emit_string5(ring, name, strlen(name));
104 #endif
105
106 uint32_t fibers_per_sp = ctx->screen->info->a6xx.fibers_per_sp;
107 uint32_t num_sp_cores = ctx->screen->info->num_sp_cores;
108
109 uint32_t per_fiber_size = ALIGN(so->pvtmem_size, 512);
110 if (per_fiber_size > ctx->pvtmem[so->pvtmem_per_wave].per_fiber_size) {
111 if (ctx->pvtmem[so->pvtmem_per_wave].bo)
112 fd_bo_del(ctx->pvtmem[so->pvtmem_per_wave].bo);
113 ctx->pvtmem[so->pvtmem_per_wave].per_fiber_size = per_fiber_size;
114 uint32_t total_size =
115 ALIGN(per_fiber_size * fibers_per_sp, 1 << 12) * num_sp_cores;
116 ctx->pvtmem[so->pvtmem_per_wave].bo = fd_bo_new(
117 ctx->screen->dev, total_size, 0,
118 "pvtmem_%s_%d", so->pvtmem_per_wave ? "per_wave" : "per_fiber",
119 per_fiber_size);
120 } else {
121 per_fiber_size = ctx->pvtmem[so->pvtmem_per_wave].per_fiber_size;
122 }
123
124 uint32_t per_sp_size = ALIGN(per_fiber_size * fibers_per_sp, 1 << 12);
125
126 OUT_PKT4(ring, instrlen, 1);
127 OUT_RING(ring, so->instrlen);
128
129 OUT_PKT4(ring, first_exec_offset, 7);
130 OUT_RING(ring, 0); /* SP_xS_OBJ_FIRST_EXEC_OFFSET */
131 OUT_RELOC(ring, so->bo, 0, 0, 0); /* SP_xS_OBJ_START_LO */
132 OUT_RING(ring, A6XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM(per_fiber_size));
133 if (so->pvtmem_size > 0) { /* SP_xS_PVT_MEM_ADDR */
134 OUT_RELOC(ring, ctx->pvtmem[so->pvtmem_per_wave].bo, 0, 0, 0);
135 } else {
136 OUT_RING(ring, 0);
137 OUT_RING(ring, 0);
138 }
139 OUT_RING(ring, A6XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(per_sp_size) |
140 COND(so->pvtmem_per_wave,
141 A6XX_SP_VS_PVT_MEM_SIZE_PERWAVEMEMLAYOUT));
142
143 OUT_PKT4(ring, hw_stack_offset, 1);
144 OUT_RING(ring, A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET_OFFSET(per_sp_size));
145
146 OUT_PKT7(ring, fd6_stage2opcode(so->type), 3);
147 OUT_RING(ring, CP_LOAD_STATE6_0_DST_OFF(0) |
148 CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
149 CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
150 CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
151 CP_LOAD_STATE6_0_NUM_UNIT(so->instrlen));
152 OUT_RELOC(ring, so->bo, 0, 0, 0);
153 }
154
155 /**
156 * Build a pre-baked state-obj to disable SO, so that we aren't dynamically
157 * building this at draw time whenever we transition from SO enabled->disabled
158 */
159 static void
setup_stream_out_disable(struct fd_context * ctx)160 setup_stream_out_disable(struct fd_context *ctx)
161 {
162 unsigned sizedw = 4;
163
164 if (ctx->screen->info->a6xx.tess_use_shared)
165 sizedw += 2;
166
167 struct fd_ringbuffer *ring =
168 fd_ringbuffer_new_object(ctx->pipe, (1 + sizedw) * 4);
169
170 OUT_PKT7(ring, CP_CONTEXT_REG_BUNCH, sizedw);
171 OUT_RING(ring, REG_A6XX_VPC_SO_CNTL);
172 OUT_RING(ring, 0);
173 OUT_RING(ring, REG_A6XX_VPC_SO_STREAM_CNTL);
174 OUT_RING(ring, 0);
175
176 if (ctx->screen->info->a6xx.tess_use_shared) {
177 OUT_RING(ring, REG_A6XX_PC_SO_STREAM_CNTL);
178 OUT_RING(ring, 0);
179 }
180
181 fd6_context(ctx)->streamout_disable_stateobj = ring;
182 }
183
184 static void
setup_stream_out(struct fd_context * ctx,struct fd6_program_state * state,const struct ir3_shader_variant * v,struct ir3_shader_linkage * l)185 setup_stream_out(struct fd_context *ctx, struct fd6_program_state *state,
186 const struct ir3_shader_variant *v,
187 struct ir3_shader_linkage *l)
188 {
189 const struct ir3_stream_output_info *strmout = &v->shader->stream_output;
190
191 uint32_t ncomp[PIPE_MAX_SO_BUFFERS];
192 uint32_t prog[256 / 2];
193 uint32_t prog_count;
194
195 memset(ncomp, 0, sizeof(ncomp));
196 memset(prog, 0, sizeof(prog));
197
198 prog_count = align(l->max_loc, 2) / 2;
199
200 debug_assert(prog_count < ARRAY_SIZE(prog));
201
202 for (unsigned i = 0; i < strmout->num_outputs; i++) {
203 const struct ir3_stream_output *out = &strmout->output[i];
204 unsigned k = out->register_index;
205 unsigned idx;
206
207 ncomp[out->output_buffer] += out->num_components;
208
209 /* linkage map sorted by order frag shader wants things, so
210 * a bit less ideal here..
211 */
212 for (idx = 0; idx < l->cnt; idx++)
213 if (l->var[idx].regid == v->outputs[k].regid)
214 break;
215
216 debug_assert(idx < l->cnt);
217
218 for (unsigned j = 0; j < out->num_components; j++) {
219 unsigned c = j + out->start_component;
220 unsigned loc = l->var[idx].loc + c;
221 unsigned off = j + out->dst_offset; /* in dwords */
222
223 if (loc & 1) {
224 prog[loc / 2] |= A6XX_VPC_SO_PROG_B_EN |
225 A6XX_VPC_SO_PROG_B_BUF(out->output_buffer) |
226 A6XX_VPC_SO_PROG_B_OFF(off * 4);
227 } else {
228 prog[loc / 2] |= A6XX_VPC_SO_PROG_A_EN |
229 A6XX_VPC_SO_PROG_A_BUF(out->output_buffer) |
230 A6XX_VPC_SO_PROG_A_OFF(off * 4);
231 }
232 }
233 }
234
235 unsigned sizedw = 12 + (2 * prog_count);
236 if (ctx->screen->info->a6xx.tess_use_shared)
237 sizedw += 2;
238
239 struct fd_ringbuffer *ring =
240 fd_ringbuffer_new_object(ctx->pipe, (1 + sizedw) * 4);
241
242 OUT_PKT7(ring, CP_CONTEXT_REG_BUNCH, sizedw);
243 OUT_RING(ring, REG_A6XX_VPC_SO_STREAM_CNTL);
244 OUT_RING(ring,
245 A6XX_VPC_SO_STREAM_CNTL_STREAM_ENABLE(0x1) |
246 COND(ncomp[0] > 0, A6XX_VPC_SO_STREAM_CNTL_BUF0_STREAM(1)) |
247 COND(ncomp[1] > 0, A6XX_VPC_SO_STREAM_CNTL_BUF1_STREAM(1)) |
248 COND(ncomp[2] > 0, A6XX_VPC_SO_STREAM_CNTL_BUF2_STREAM(1)) |
249 COND(ncomp[3] > 0, A6XX_VPC_SO_STREAM_CNTL_BUF3_STREAM(1)));
250 OUT_RING(ring, REG_A6XX_VPC_SO_NCOMP(0));
251 OUT_RING(ring, ncomp[0]);
252 OUT_RING(ring, REG_A6XX_VPC_SO_NCOMP(1));
253 OUT_RING(ring, ncomp[1]);
254 OUT_RING(ring, REG_A6XX_VPC_SO_NCOMP(2));
255 OUT_RING(ring, ncomp[2]);
256 OUT_RING(ring, REG_A6XX_VPC_SO_NCOMP(3));
257 OUT_RING(ring, ncomp[3]);
258 OUT_RING(ring, REG_A6XX_VPC_SO_CNTL);
259 OUT_RING(ring, A6XX_VPC_SO_CNTL_RESET);
260 for (unsigned i = 0; i < prog_count; i++) {
261 OUT_RING(ring, REG_A6XX_VPC_SO_PROG);
262 OUT_RING(ring, prog[i]);
263 }
264 if (ctx->screen->info->a6xx.tess_use_shared) {
265 /* Possibly not tess_use_shared related, but the combination of
266 * tess + xfb fails some tests if we don't emit this.
267 */
268 OUT_RING(ring, REG_A6XX_PC_SO_STREAM_CNTL);
269 OUT_RING(ring, A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE);
270 }
271
272 state->streamout_stateobj = ring;
273 }
274
275 static void
setup_config_stateobj(struct fd_context * ctx,struct fd6_program_state * state)276 setup_config_stateobj(struct fd_context *ctx, struct fd6_program_state *state)
277 {
278 struct fd_ringbuffer *ring = fd_ringbuffer_new_object(ctx->pipe, 100 * 4);
279
280 OUT_REG(ring, A6XX_HLSQ_INVALIDATE_CMD(.vs_state = true, .hs_state = true,
281 .ds_state = true, .gs_state = true,
282 .fs_state = true, .cs_state = true,
283 .gfx_ibo = true, .cs_ibo = true, ));
284
285 debug_assert(state->vs->constlen >= state->bs->constlen);
286
287 OUT_PKT4(ring, REG_A6XX_HLSQ_VS_CNTL, 4);
288 OUT_RING(ring, A6XX_HLSQ_VS_CNTL_CONSTLEN(state->vs->constlen) |
289 A6XX_HLSQ_VS_CNTL_ENABLED);
290 OUT_RING(ring, COND(state->hs,
291 A6XX_HLSQ_HS_CNTL_ENABLED |
292 A6XX_HLSQ_HS_CNTL_CONSTLEN(state->hs->constlen)));
293 OUT_RING(ring, COND(state->ds,
294 A6XX_HLSQ_DS_CNTL_ENABLED |
295 A6XX_HLSQ_DS_CNTL_CONSTLEN(state->ds->constlen)));
296 OUT_RING(ring, COND(state->gs,
297 A6XX_HLSQ_GS_CNTL_ENABLED |
298 A6XX_HLSQ_GS_CNTL_CONSTLEN(state->gs->constlen)));
299 OUT_PKT4(ring, REG_A6XX_HLSQ_FS_CNTL, 1);
300 OUT_RING(ring, A6XX_HLSQ_FS_CNTL_CONSTLEN(state->fs->constlen) |
301 A6XX_HLSQ_FS_CNTL_ENABLED);
302
303 OUT_PKT4(ring, REG_A6XX_SP_VS_CONFIG, 1);
304 OUT_RING(ring, COND(state->vs, A6XX_SP_VS_CONFIG_ENABLED) |
305 A6XX_SP_VS_CONFIG_NIBO(ir3_shader_nibo(state->vs)) |
306 A6XX_SP_VS_CONFIG_NTEX(state->vs->num_samp) |
307 A6XX_SP_VS_CONFIG_NSAMP(state->vs->num_samp));
308
309 OUT_PKT4(ring, REG_A6XX_SP_HS_CONFIG, 1);
310 OUT_RING(ring, COND(state->hs,
311 A6XX_SP_HS_CONFIG_ENABLED |
312 A6XX_SP_HS_CONFIG_NIBO(ir3_shader_nibo(state->hs)) |
313 A6XX_SP_HS_CONFIG_NTEX(state->hs->num_samp) |
314 A6XX_SP_HS_CONFIG_NSAMP(state->hs->num_samp)));
315
316 OUT_PKT4(ring, REG_A6XX_SP_DS_CONFIG, 1);
317 OUT_RING(ring, COND(state->ds,
318 A6XX_SP_DS_CONFIG_ENABLED |
319 A6XX_SP_DS_CONFIG_NIBO(ir3_shader_nibo(state->ds)) |
320 A6XX_SP_DS_CONFIG_NTEX(state->ds->num_samp) |
321 A6XX_SP_DS_CONFIG_NSAMP(state->ds->num_samp)));
322
323 OUT_PKT4(ring, REG_A6XX_SP_GS_CONFIG, 1);
324 OUT_RING(ring, COND(state->gs,
325 A6XX_SP_GS_CONFIG_ENABLED |
326 A6XX_SP_GS_CONFIG_NIBO(ir3_shader_nibo(state->gs)) |
327 A6XX_SP_GS_CONFIG_NTEX(state->gs->num_samp) |
328 A6XX_SP_GS_CONFIG_NSAMP(state->gs->num_samp)));
329
330 OUT_PKT4(ring, REG_A6XX_SP_FS_CONFIG, 1);
331 OUT_RING(ring, COND(state->fs, A6XX_SP_FS_CONFIG_ENABLED) |
332 A6XX_SP_FS_CONFIG_NIBO(ir3_shader_nibo(state->fs)) |
333 A6XX_SP_FS_CONFIG_NTEX(state->fs->num_samp) |
334 A6XX_SP_FS_CONFIG_NSAMP(state->fs->num_samp));
335
336 OUT_PKT4(ring, REG_A6XX_SP_IBO_COUNT, 1);
337 OUT_RING(ring, ir3_shader_nibo(state->fs));
338
339 state->config_stateobj = ring;
340 }
341
342 static inline uint32_t
next_regid(uint32_t reg,uint32_t increment)343 next_regid(uint32_t reg, uint32_t increment)
344 {
345 if (VALIDREG(reg))
346 return reg + increment;
347 else
348 return regid(63, 0);
349 }
350
351 static void
setup_stateobj(struct fd_ringbuffer * ring,struct fd_context * ctx,struct fd6_program_state * state,const struct ir3_shader_key * key,bool binning_pass)352 setup_stateobj(struct fd_ringbuffer *ring, struct fd_context *ctx,
353 struct fd6_program_state *state,
354 const struct ir3_shader_key *key, bool binning_pass) assert_dt
355 {
356 uint32_t pos_regid, psize_regid, color_regid[8], posz_regid;
357 uint32_t clip0_regid, clip1_regid;
358 uint32_t face_regid, coord_regid, zwcoord_regid, samp_id_regid;
359 uint32_t smask_in_regid, smask_regid;
360 uint32_t stencilref_regid;
361 uint32_t vertex_regid, instance_regid, layer_regid, vs_primitive_regid;
362 uint32_t hs_invocation_regid;
363 uint32_t tess_coord_x_regid, tess_coord_y_regid, hs_rel_patch_regid,
364 ds_rel_patch_regid, ds_primitive_regid;
365 uint32_t ij_regid[IJ_COUNT];
366 uint32_t gs_header_regid;
367 enum a6xx_threadsize fssz;
368 uint8_t psize_loc = ~0, pos_loc = ~0, layer_loc = ~0;
369 uint8_t clip0_loc, clip1_loc;
370 int i, j;
371
372 static const struct ir3_shader_variant dummy_fs = {0};
373 const struct ir3_shader_variant *vs = binning_pass ? state->bs : state->vs;
374 const struct ir3_shader_variant *hs = state->hs;
375 const struct ir3_shader_variant *ds = state->ds;
376 const struct ir3_shader_variant *gs = state->gs;
377 const struct ir3_shader_variant *fs = binning_pass ? &dummy_fs : state->fs;
378
379 /* binning VS is wrong when GS is present, so use nonbinning VS
380 * TODO: compile both binning VS/GS variants correctly
381 */
382 if (binning_pass && state->gs)
383 vs = state->vs;
384
385 bool sample_shading = fs->per_samp | key->sample_shading;
386
387 fssz = fs->info.double_threadsize ? THREAD128 : THREAD64;
388
389 pos_regid = ir3_find_output_regid(vs, VARYING_SLOT_POS);
390 psize_regid = ir3_find_output_regid(vs, VARYING_SLOT_PSIZ);
391 clip0_regid = ir3_find_output_regid(vs, VARYING_SLOT_CLIP_DIST0);
392 clip1_regid = ir3_find_output_regid(vs, VARYING_SLOT_CLIP_DIST1);
393 layer_regid = ir3_find_output_regid(vs, VARYING_SLOT_LAYER);
394 vertex_regid = ir3_find_sysval_regid(vs, SYSTEM_VALUE_VERTEX_ID);
395 instance_regid = ir3_find_sysval_regid(vs, SYSTEM_VALUE_INSTANCE_ID);
396 if (hs)
397 vs_primitive_regid = ir3_find_sysval_regid(hs, SYSTEM_VALUE_PRIMITIVE_ID);
398 else if (gs)
399 vs_primitive_regid = ir3_find_sysval_regid(gs, SYSTEM_VALUE_PRIMITIVE_ID);
400 else
401 vs_primitive_regid = regid(63, 0);
402
403 bool hs_reads_primid = false, ds_reads_primid = false;
404 if (hs) {
405 tess_coord_x_regid = ir3_find_sysval_regid(ds, SYSTEM_VALUE_TESS_COORD);
406 tess_coord_y_regid = next_regid(tess_coord_x_regid, 1);
407 hs_reads_primid = VALIDREG(ir3_find_sysval_regid(hs, SYSTEM_VALUE_PRIMITIVE_ID));
408 ds_reads_primid = VALIDREG(ir3_find_sysval_regid(ds, SYSTEM_VALUE_PRIMITIVE_ID));
409 hs_rel_patch_regid = ir3_find_sysval_regid(hs, SYSTEM_VALUE_REL_PATCH_ID_IR3);
410 ds_rel_patch_regid = ir3_find_sysval_regid(ds, SYSTEM_VALUE_REL_PATCH_ID_IR3);
411 ds_primitive_regid = ir3_find_sysval_regid(ds, SYSTEM_VALUE_PRIMITIVE_ID);
412 hs_invocation_regid =
413 ir3_find_sysval_regid(hs, SYSTEM_VALUE_TCS_HEADER_IR3);
414
415 pos_regid = ir3_find_output_regid(ds, VARYING_SLOT_POS);
416 psize_regid = ir3_find_output_regid(ds, VARYING_SLOT_PSIZ);
417 clip0_regid = ir3_find_output_regid(ds, VARYING_SLOT_CLIP_DIST0);
418 clip1_regid = ir3_find_output_regid(ds, VARYING_SLOT_CLIP_DIST1);
419 } else {
420 tess_coord_x_regid = regid(63, 0);
421 tess_coord_y_regid = regid(63, 0);
422 hs_rel_patch_regid = regid(63, 0);
423 ds_rel_patch_regid = regid(63, 0);
424 ds_primitive_regid = regid(63, 0);
425 hs_invocation_regid = regid(63, 0);
426 }
427
428 bool gs_reads_primid = false;
429 if (gs) {
430 gs_header_regid = ir3_find_sysval_regid(gs, SYSTEM_VALUE_GS_HEADER_IR3);
431 gs_reads_primid = VALIDREG(ir3_find_sysval_regid(gs, SYSTEM_VALUE_PRIMITIVE_ID));
432 pos_regid = ir3_find_output_regid(gs, VARYING_SLOT_POS);
433 psize_regid = ir3_find_output_regid(gs, VARYING_SLOT_PSIZ);
434 clip0_regid = ir3_find_output_regid(gs, VARYING_SLOT_CLIP_DIST0);
435 clip1_regid = ir3_find_output_regid(gs, VARYING_SLOT_CLIP_DIST1);
436 layer_regid = ir3_find_output_regid(gs, VARYING_SLOT_LAYER);
437 } else {
438 gs_header_regid = regid(63, 0);
439 }
440
441 if (fs->color0_mrt) {
442 color_regid[0] = color_regid[1] = color_regid[2] = color_regid[3] =
443 color_regid[4] = color_regid[5] = color_regid[6] = color_regid[7] =
444 ir3_find_output_regid(fs, FRAG_RESULT_COLOR);
445 } else {
446 color_regid[0] = ir3_find_output_regid(fs, FRAG_RESULT_DATA0);
447 color_regid[1] = ir3_find_output_regid(fs, FRAG_RESULT_DATA1);
448 color_regid[2] = ir3_find_output_regid(fs, FRAG_RESULT_DATA2);
449 color_regid[3] = ir3_find_output_regid(fs, FRAG_RESULT_DATA3);
450 color_regid[4] = ir3_find_output_regid(fs, FRAG_RESULT_DATA4);
451 color_regid[5] = ir3_find_output_regid(fs, FRAG_RESULT_DATA5);
452 color_regid[6] = ir3_find_output_regid(fs, FRAG_RESULT_DATA6);
453 color_regid[7] = ir3_find_output_regid(fs, FRAG_RESULT_DATA7);
454 }
455
456 samp_id_regid = ir3_find_sysval_regid(fs, SYSTEM_VALUE_SAMPLE_ID);
457 smask_in_regid = ir3_find_sysval_regid(fs, SYSTEM_VALUE_SAMPLE_MASK_IN);
458 face_regid = ir3_find_sysval_regid(fs, SYSTEM_VALUE_FRONT_FACE);
459 coord_regid = ir3_find_sysval_regid(fs, SYSTEM_VALUE_FRAG_COORD);
460 zwcoord_regid = next_regid(coord_regid, 2);
461 posz_regid = ir3_find_output_regid(fs, FRAG_RESULT_DEPTH);
462 smask_regid = ir3_find_output_regid(fs, FRAG_RESULT_SAMPLE_MASK);
463 stencilref_regid = ir3_find_output_regid(fs, FRAG_RESULT_STENCIL);
464 for (unsigned i = 0; i < ARRAY_SIZE(ij_regid); i++)
465 ij_regid[i] =
466 ir3_find_sysval_regid(fs, SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL + i);
467
468 /* If we have pre-dispatch texture fetches, then ij_pix should not
469 * be DCE'd, even if not actually used in the shader itself:
470 */
471 if (fs->num_sampler_prefetch > 0) {
472 assert(VALIDREG(ij_regid[IJ_PERSP_PIXEL]));
473 /* also, it seems like ij_pix is *required* to be r0.x */
474 assert(ij_regid[IJ_PERSP_PIXEL] == regid(0, 0));
475 }
476
477 /* we can't write gl_SampleMask for !msaa.. if b0 is zero then we
478 * end up masking the single sample!!
479 */
480 if (!key->msaa)
481 smask_regid = regid(63, 0);
482
483 /* we could probably divide this up into things that need to be
484 * emitted if frag-prog is dirty vs if vert-prog is dirty..
485 */
486
487 OUT_PKT4(ring, REG_A6XX_SP_FS_PREFETCH_CNTL, 1 + fs->num_sampler_prefetch);
488 OUT_RING(ring, A6XX_SP_FS_PREFETCH_CNTL_COUNT(fs->num_sampler_prefetch) |
489 A6XX_SP_FS_PREFETCH_CNTL_UNK4(regid(63, 0)) |
490 0x7000); // XXX
491 for (int i = 0; i < fs->num_sampler_prefetch; i++) {
492 const struct ir3_sampler_prefetch *prefetch = &fs->sampler_prefetch[i];
493 OUT_RING(ring,
494 A6XX_SP_FS_PREFETCH_CMD_SRC(prefetch->src) |
495 A6XX_SP_FS_PREFETCH_CMD_SAMP_ID(prefetch->samp_id) |
496 A6XX_SP_FS_PREFETCH_CMD_TEX_ID(prefetch->tex_id) |
497 A6XX_SP_FS_PREFETCH_CMD_DST(prefetch->dst) |
498 A6XX_SP_FS_PREFETCH_CMD_WRMASK(prefetch->wrmask) |
499 COND(prefetch->half_precision, A6XX_SP_FS_PREFETCH_CMD_HALF) |
500 A6XX_SP_FS_PREFETCH_CMD_CMD(prefetch->cmd));
501 }
502
503 OUT_PKT4(ring, REG_A6XX_SP_UNKNOWN_A9A8, 1);
504 OUT_RING(ring, 0);
505
506 OUT_PKT4(ring, REG_A6XX_SP_MODE_CONTROL, 1);
507 OUT_RING(ring, A6XX_SP_MODE_CONTROL_CONSTANT_DEMOTION_ENABLE | 4);
508
509 bool fs_has_dual_src_color =
510 !binning_pass && fs->shader->nir->info.fs.color_is_dual_source;
511
512 OUT_PKT4(ring, REG_A6XX_SP_FS_OUTPUT_CNTL0, 1);
513 OUT_RING(ring,
514 A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID(posz_regid) |
515 A6XX_SP_FS_OUTPUT_CNTL0_SAMPMASK_REGID(smask_regid) |
516 A6XX_SP_FS_OUTPUT_CNTL0_STENCILREF_REGID(stencilref_regid) |
517 COND(fs_has_dual_src_color,
518 A6XX_SP_FS_OUTPUT_CNTL0_DUAL_COLOR_IN_ENABLE));
519
520 OUT_PKT4(ring, REG_A6XX_SP_VS_CTRL_REG0, 1);
521 OUT_RING(
522 ring,
523 A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(vs->info.max_reg + 1) |
524 A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(vs->info.max_half_reg + 1) |
525 COND(vs->mergedregs, A6XX_SP_VS_CTRL_REG0_MERGEDREGS) |
526 A6XX_SP_VS_CTRL_REG0_BRANCHSTACK(ir3_shader_branchstack_hw(vs)));
527
528 fd6_emit_shader(ctx, ring, vs);
529 fd6_emit_immediates(ctx->screen, vs, ring);
530
531 struct ir3_shader_linkage l = {0};
532 const struct ir3_shader_variant *last_shader = fd6_last_shader(state);
533
534 bool do_streamout = (last_shader->shader->stream_output.num_outputs > 0);
535 uint8_t clip_mask = last_shader->clip_mask,
536 cull_mask = last_shader->cull_mask;
537 uint8_t clip_cull_mask = clip_mask | cull_mask;
538
539 /* If we have streamout, link against the real FS, rather than the
540 * dummy FS used for binning pass state, to ensure the OUTLOC's
541 * match. Depending on whether we end up doing sysmem or gmem,
542 * the actual streamout could happen with either the binning pass
543 * or draw pass program, but the same streamout stateobj is used
544 * in either case:
545 */
546 ir3_link_shaders(&l, last_shader, do_streamout ? state->fs : fs, true);
547
548 bool primid_passthru = l.primid_loc != 0xff;
549 clip0_loc = l.clip0_loc;
550 clip1_loc = l.clip1_loc;
551
552 OUT_PKT4(ring, REG_A6XX_VPC_VAR_DISABLE(0), 4);
553 OUT_RING(ring, ~l.varmask[0]); /* VPC_VAR[0].DISABLE */
554 OUT_RING(ring, ~l.varmask[1]); /* VPC_VAR[1].DISABLE */
555 OUT_RING(ring, ~l.varmask[2]); /* VPC_VAR[2].DISABLE */
556 OUT_RING(ring, ~l.varmask[3]); /* VPC_VAR[3].DISABLE */
557
558 /* Add stream out outputs after computing the VPC_VAR_DISABLE bitmask. */
559 ir3_link_stream_out(&l, last_shader);
560
561 if (VALIDREG(layer_regid)) {
562 layer_loc = l.max_loc;
563 ir3_link_add(&l, layer_regid, 0x1, l.max_loc);
564 }
565
566 if (VALIDREG(pos_regid)) {
567 pos_loc = l.max_loc;
568 ir3_link_add(&l, pos_regid, 0xf, l.max_loc);
569 }
570
571 if (VALIDREG(psize_regid)) {
572 psize_loc = l.max_loc;
573 ir3_link_add(&l, psize_regid, 0x1, l.max_loc);
574 }
575
576 /* Handle the case where clip/cull distances aren't read by the FS. Make
577 * sure to avoid adding an output with an empty writemask if the user
578 * disables all the clip distances in the API so that the slot is unused.
579 */
580 if (clip0_loc == 0xff && VALIDREG(clip0_regid) &&
581 (clip_cull_mask & 0xf) != 0) {
582 clip0_loc = l.max_loc;
583 ir3_link_add(&l, clip0_regid, clip_cull_mask & 0xf, l.max_loc);
584 }
585
586 if (clip1_loc == 0xff && VALIDREG(clip1_regid) &&
587 (clip_cull_mask >> 4) != 0) {
588 clip1_loc = l.max_loc;
589 ir3_link_add(&l, clip1_regid, clip_cull_mask >> 4, l.max_loc);
590 }
591
592 /* If we have stream-out, we use the full shader for binning
593 * pass, rather than the optimized binning pass one, so that we
594 * have all the varying outputs available for xfb. So streamout
595 * state should always be derived from the non-binning pass
596 * program:
597 */
598 if (do_streamout && !binning_pass) {
599 setup_stream_out(ctx, state, last_shader, &l);
600
601 if (!fd6_context(ctx)->streamout_disable_stateobj)
602 setup_stream_out_disable(ctx);
603 }
604
605 debug_assert(l.cnt <= 32);
606 if (gs)
607 OUT_PKT4(ring, REG_A6XX_SP_GS_OUT_REG(0), DIV_ROUND_UP(l.cnt, 2));
608 else if (ds)
609 OUT_PKT4(ring, REG_A6XX_SP_DS_OUT_REG(0), DIV_ROUND_UP(l.cnt, 2));
610 else
611 OUT_PKT4(ring, REG_A6XX_SP_VS_OUT_REG(0), DIV_ROUND_UP(l.cnt, 2));
612
613 for (j = 0; j < l.cnt;) {
614 uint32_t reg = 0;
615
616 reg |= A6XX_SP_VS_OUT_REG_A_REGID(l.var[j].regid);
617 reg |= A6XX_SP_VS_OUT_REG_A_COMPMASK(l.var[j].compmask);
618 j++;
619
620 reg |= A6XX_SP_VS_OUT_REG_B_REGID(l.var[j].regid);
621 reg |= A6XX_SP_VS_OUT_REG_B_COMPMASK(l.var[j].compmask);
622 j++;
623
624 OUT_RING(ring, reg);
625 }
626
627 if (gs)
628 OUT_PKT4(ring, REG_A6XX_SP_GS_VPC_DST_REG(0), DIV_ROUND_UP(l.cnt, 4));
629 else if (ds)
630 OUT_PKT4(ring, REG_A6XX_SP_DS_VPC_DST_REG(0), DIV_ROUND_UP(l.cnt, 4));
631 else
632 OUT_PKT4(ring, REG_A6XX_SP_VS_VPC_DST_REG(0), DIV_ROUND_UP(l.cnt, 4));
633
634 for (j = 0; j < l.cnt;) {
635 uint32_t reg = 0;
636
637 reg |= A6XX_SP_VS_VPC_DST_REG_OUTLOC0(l.var[j++].loc);
638 reg |= A6XX_SP_VS_VPC_DST_REG_OUTLOC1(l.var[j++].loc);
639 reg |= A6XX_SP_VS_VPC_DST_REG_OUTLOC2(l.var[j++].loc);
640 reg |= A6XX_SP_VS_VPC_DST_REG_OUTLOC3(l.var[j++].loc);
641
642 OUT_RING(ring, reg);
643 }
644
645 if (hs) {
646 assert(vs->mergedregs == hs->mergedregs);
647 OUT_PKT4(ring, REG_A6XX_SP_HS_CTRL_REG0, 1);
648 OUT_RING(
649 ring,
650 A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT(hs->info.max_reg + 1) |
651 A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT(hs->info.max_half_reg + 1) |
652 A6XX_SP_HS_CTRL_REG0_BRANCHSTACK(ir3_shader_branchstack_hw(hs)));
653
654 fd6_emit_shader(ctx, ring, hs);
655 fd6_emit_immediates(ctx->screen, hs, ring);
656 fd6_emit_link_map(ctx->screen, vs, hs, ring);
657
658 OUT_PKT4(ring, REG_A6XX_SP_DS_CTRL_REG0, 1);
659 OUT_RING(
660 ring,
661 A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT(ds->info.max_reg + 1) |
662 A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT(ds->info.max_half_reg + 1) |
663 COND(ds->mergedregs, A6XX_SP_DS_CTRL_REG0_MERGEDREGS) |
664 A6XX_SP_DS_CTRL_REG0_BRANCHSTACK(ir3_shader_branchstack_hw(ds)));
665
666 fd6_emit_shader(ctx, ring, ds);
667 fd6_emit_immediates(ctx->screen, ds, ring);
668 fd6_emit_link_map(ctx->screen, hs, ds, ring);
669
670 shader_info *hs_info = &hs->shader->nir->info;
671 OUT_PKT4(ring, REG_A6XX_PC_TESS_NUM_VERTEX, 1);
672 OUT_RING(ring, hs_info->tess.tcs_vertices_out);
673
674 if (ctx->screen->info->a6xx.tess_use_shared) {
675 unsigned hs_input_size = 6 + (3 * (vs->output_size - 1));
676 unsigned wave_input_size =
677 MIN2(64, DIV_ROUND_UP(hs_input_size * 4,
678 hs_info->tess.tcs_vertices_out));
679
680 OUT_PKT4(ring, REG_A6XX_PC_HS_INPUT_SIZE, 1);
681 OUT_RING(ring, hs_input_size);
682
683 OUT_PKT4(ring, REG_A6XX_SP_HS_WAVE_INPUT_SIZE, 1);
684 OUT_RING(ring, wave_input_size);
685 } else {
686 uint32_t hs_input_size =
687 hs_info->tess.tcs_vertices_out * vs->output_size / 4;
688
689 /* Total attribute slots in HS incoming patch. */
690 OUT_PKT4(ring, REG_A6XX_PC_HS_INPUT_SIZE, 1);
691 OUT_RING(ring, hs_input_size);
692
693 const uint32_t wavesize = 64;
694 const uint32_t max_wave_input_size = 64;
695 const uint32_t patch_control_points = hs_info->tess.tcs_vertices_out;
696
697 /* note: if HS is really just the VS extended, then this
698 * should be by MAX2(patch_control_points, hs_info->tess.tcs_vertices_out)
699 * however that doesn't match the blob, and fails some dEQP tests.
700 */
701 uint32_t prims_per_wave = wavesize / hs_info->tess.tcs_vertices_out;
702 uint32_t max_prims_per_wave = max_wave_input_size * wavesize /
703 (vs->output_size * patch_control_points);
704 prims_per_wave = MIN2(prims_per_wave, max_prims_per_wave);
705
706 uint32_t total_size =
707 vs->output_size * patch_control_points * prims_per_wave;
708 uint32_t wave_input_size = DIV_ROUND_UP(total_size, wavesize);
709
710 OUT_PKT4(ring, REG_A6XX_SP_HS_WAVE_INPUT_SIZE, 1);
711 OUT_RING(ring, wave_input_size);
712 }
713
714 shader_info *ds_info = &ds->shader->nir->info;
715 OUT_PKT4(ring, REG_A6XX_PC_TESS_CNTL, 1);
716 uint32_t output;
717 if (ds_info->tess.point_mode)
718 output = TESS_POINTS;
719 else if (ds_info->tess.primitive_mode == GL_ISOLINES)
720 output = TESS_LINES;
721 else if (ds_info->tess.ccw)
722 output = TESS_CCW_TRIS;
723 else
724 output = TESS_CW_TRIS;
725
726 OUT_RING(ring, A6XX_PC_TESS_CNTL_SPACING(
727 fd6_gl2spacing(ds_info->tess.spacing)) |
728 A6XX_PC_TESS_CNTL_OUTPUT(output));
729
730 OUT_PKT4(ring, REG_A6XX_VPC_DS_CLIP_CNTL, 1);
731 OUT_RING(ring, A6XX_VPC_DS_CLIP_CNTL_CLIP_MASK(clip_cull_mask) |
732 A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_03_LOC(clip0_loc) |
733 A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_47_LOC(clip1_loc));
734
735 OUT_PKT4(ring, REG_A6XX_VPC_DS_LAYER_CNTL, 1);
736 OUT_RING(ring, 0x0000ffff);
737
738 OUT_PKT4(ring, REG_A6XX_GRAS_DS_LAYER_CNTL, 1);
739 OUT_RING(ring, 0x0);
740
741 OUT_PKT4(ring, REG_A6XX_GRAS_DS_CL_CNTL, 1);
742 OUT_RING(ring, A6XX_GRAS_DS_CL_CNTL_CLIP_MASK(clip_mask) |
743 A6XX_GRAS_DS_CL_CNTL_CULL_MASK(cull_mask));
744
745 OUT_PKT4(ring, REG_A6XX_VPC_VS_PACK, 1);
746 OUT_RING(ring, A6XX_VPC_VS_PACK_POSITIONLOC(pos_loc) |
747 A6XX_VPC_VS_PACK_PSIZELOC(255) |
748 A6XX_VPC_VS_PACK_STRIDE_IN_VPC(l.max_loc));
749
750 OUT_PKT4(ring, REG_A6XX_VPC_DS_PACK, 1);
751 OUT_RING(ring, A6XX_VPC_DS_PACK_POSITIONLOC(pos_loc) |
752 A6XX_VPC_DS_PACK_PSIZELOC(psize_loc) |
753 A6XX_VPC_DS_PACK_STRIDE_IN_VPC(l.max_loc));
754
755 OUT_PKT4(ring, REG_A6XX_SP_DS_PRIMITIVE_CNTL, 1);
756 OUT_RING(ring, A6XX_SP_DS_PRIMITIVE_CNTL_OUT(l.cnt));
757
758 OUT_PKT4(ring, REG_A6XX_PC_DS_OUT_CNTL, 1);
759 OUT_RING(ring, A6XX_PC_DS_OUT_CNTL_STRIDE_IN_VPC(l.max_loc) |
760 CONDREG(psize_regid, A6XX_PC_DS_OUT_CNTL_PSIZE) |
761 COND(ds_reads_primid, A6XX_PC_DS_OUT_CNTL_PRIMITIVE_ID) |
762 A6XX_PC_DS_OUT_CNTL_CLIP_MASK(clip_cull_mask));
763
764 OUT_PKT4(ring, REG_A6XX_PC_HS_OUT_CNTL, 1);
765 OUT_RING(ring, COND(hs_reads_primid, A6XX_PC_HS_OUT_CNTL_PRIMITIVE_ID));
766 } else {
767 OUT_PKT4(ring, REG_A6XX_SP_HS_WAVE_INPUT_SIZE, 1);
768 OUT_RING(ring, 0);
769 }
770
771 OUT_PKT4(ring, REG_A6XX_SP_VS_PRIMITIVE_CNTL, 1);
772 OUT_RING(ring, A6XX_SP_VS_PRIMITIVE_CNTL_OUT(l.cnt));
773
774 bool enable_varyings = fs->total_in > 0;
775
776 OUT_PKT4(ring, REG_A6XX_VPC_CNTL_0, 1);
777 OUT_RING(ring, A6XX_VPC_CNTL_0_NUMNONPOSVAR(fs->total_in) |
778 COND(enable_varyings, A6XX_VPC_CNTL_0_VARYING) |
779 A6XX_VPC_CNTL_0_PRIMIDLOC(l.primid_loc) |
780 A6XX_VPC_CNTL_0_VIEWIDLOC(0xff));
781
782 OUT_PKT4(ring, REG_A6XX_PC_VS_OUT_CNTL, 1);
783 OUT_RING(ring, A6XX_PC_VS_OUT_CNTL_STRIDE_IN_VPC(l.max_loc) |
784 CONDREG(psize_regid, A6XX_PC_VS_OUT_CNTL_PSIZE) |
785 CONDREG(layer_regid, A6XX_PC_VS_OUT_CNTL_LAYER) |
786 A6XX_PC_VS_OUT_CNTL_CLIP_MASK(clip_cull_mask));
787
788 OUT_PKT4(ring, REG_A6XX_HLSQ_CONTROL_1_REG, 5);
789 OUT_RING(ring, 0x7); /* XXX */
790 OUT_RING(ring, A6XX_HLSQ_CONTROL_2_REG_FACEREGID(face_regid) |
791 A6XX_HLSQ_CONTROL_2_REG_SAMPLEID(samp_id_regid) |
792 A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(smask_in_regid) |
793 A6XX_HLSQ_CONTROL_2_REG_SIZE(ij_regid[IJ_PERSP_SIZE]));
794 OUT_RING(
795 ring,
796 A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL(ij_regid[IJ_PERSP_PIXEL]) |
797 A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL(ij_regid[IJ_LINEAR_PIXEL]) |
798 A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID(
799 ij_regid[IJ_PERSP_CENTROID]) |
800 A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID(
801 ij_regid[IJ_LINEAR_CENTROID]));
802 OUT_RING(
803 ring,
804 A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(coord_regid) |
805 A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(zwcoord_regid) |
806 A6XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE(ij_regid[IJ_PERSP_SAMPLE]) |
807 A6XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE(ij_regid[IJ_LINEAR_SAMPLE]));
808 OUT_RING(ring, 0xfcfc); /* line length (?), foveation quality */
809
810 OUT_PKT4(ring, REG_A6XX_HLSQ_FS_CNTL_0, 1);
811 OUT_RING(ring, A6XX_HLSQ_FS_CNTL_0_THREADSIZE(fssz) |
812 COND(enable_varyings, A6XX_HLSQ_FS_CNTL_0_VARYINGS));
813
814 OUT_PKT4(ring, REG_A6XX_SP_FS_CTRL_REG0, 1);
815 OUT_RING(
816 ring,
817 A6XX_SP_FS_CTRL_REG0_THREADSIZE(fssz) |
818 COND(enable_varyings, A6XX_SP_FS_CTRL_REG0_VARYING) | 0x1000000 |
819 A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(fs->info.max_reg + 1) |
820 A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(fs->info.max_half_reg + 1) |
821 COND(fs->mergedregs, A6XX_SP_FS_CTRL_REG0_MERGEDREGS) |
822 A6XX_SP_FS_CTRL_REG0_BRANCHSTACK(ir3_shader_branchstack_hw(fs)) |
823 COND(fs->need_pixlod, A6XX_SP_FS_CTRL_REG0_PIXLODENABLE));
824
825 OUT_PKT4(ring, REG_A6XX_VPC_VS_LAYER_CNTL, 1);
826 OUT_RING(ring, A6XX_VPC_VS_LAYER_CNTL_LAYERLOC(layer_loc) |
827 A6XX_VPC_VS_LAYER_CNTL_VIEWLOC(0xff));
828
829 bool need_size = fs->frag_face || fs->fragcoord_compmask != 0;
830 bool need_size_persamp = false;
831 if (VALIDREG(ij_regid[IJ_PERSP_SIZE])) {
832 if (sample_shading)
833 need_size_persamp = true;
834 else
835 need_size = true;
836 }
837
838 OUT_PKT4(ring, REG_A6XX_GRAS_CNTL, 1);
839 OUT_RING(
840 ring,
841 CONDREG(ij_regid[IJ_PERSP_PIXEL], A6XX_GRAS_CNTL_IJ_PERSP_PIXEL) |
842 CONDREG(ij_regid[IJ_PERSP_CENTROID],
843 A6XX_GRAS_CNTL_IJ_PERSP_CENTROID) |
844 CONDREG(ij_regid[IJ_PERSP_SAMPLE], A6XX_GRAS_CNTL_IJ_PERSP_SAMPLE) |
845 CONDREG(ij_regid[IJ_LINEAR_PIXEL], A6XX_GRAS_CNTL_IJ_LINEAR_PIXEL) |
846 CONDREG(ij_regid[IJ_LINEAR_CENTROID],
847 A6XX_GRAS_CNTL_IJ_LINEAR_CENTROID) |
848 CONDREG(ij_regid[IJ_LINEAR_SAMPLE], A6XX_GRAS_CNTL_IJ_LINEAR_SAMPLE) |
849 COND(need_size, A6XX_GRAS_CNTL_IJ_LINEAR_PIXEL) |
850 COND(need_size_persamp, A6XX_GRAS_CNTL_IJ_LINEAR_SAMPLE) |
851 COND(fs->fragcoord_compmask != 0,
852 A6XX_GRAS_CNTL_COORD_MASK(fs->fragcoord_compmask)));
853
854 OUT_PKT4(ring, REG_A6XX_RB_RENDER_CONTROL0, 2);
855 OUT_RING(
856 ring,
857 CONDREG(ij_regid[IJ_PERSP_PIXEL],
858 A6XX_RB_RENDER_CONTROL0_IJ_PERSP_PIXEL) |
859 CONDREG(ij_regid[IJ_PERSP_CENTROID],
860 A6XX_RB_RENDER_CONTROL0_IJ_PERSP_CENTROID) |
861 CONDREG(ij_regid[IJ_PERSP_SAMPLE],
862 A6XX_RB_RENDER_CONTROL0_IJ_PERSP_SAMPLE) |
863 CONDREG(ij_regid[IJ_LINEAR_PIXEL],
864 A6XX_RB_RENDER_CONTROL0_IJ_LINEAR_PIXEL) |
865 CONDREG(ij_regid[IJ_LINEAR_CENTROID],
866 A6XX_RB_RENDER_CONTROL0_IJ_LINEAR_CENTROID) |
867 CONDREG(ij_regid[IJ_LINEAR_SAMPLE],
868 A6XX_RB_RENDER_CONTROL0_IJ_LINEAR_SAMPLE) |
869 COND(need_size, A6XX_RB_RENDER_CONTROL0_IJ_LINEAR_PIXEL) |
870 COND(enable_varyings, A6XX_RB_RENDER_CONTROL0_UNK10) |
871 COND(need_size_persamp, A6XX_RB_RENDER_CONTROL0_IJ_LINEAR_SAMPLE) |
872 COND(fs->fragcoord_compmask != 0,
873 A6XX_RB_RENDER_CONTROL0_COORD_MASK(fs->fragcoord_compmask)));
874
875 OUT_RING(ring,
876 CONDREG(smask_in_regid, A6XX_RB_RENDER_CONTROL1_SAMPLEMASK) |
877 CONDREG(samp_id_regid, A6XX_RB_RENDER_CONTROL1_SAMPLEID) |
878 CONDREG(ij_regid[IJ_PERSP_SIZE], A6XX_RB_RENDER_CONTROL1_SIZE) |
879 COND(fs->frag_face, A6XX_RB_RENDER_CONTROL1_FACENESS));
880
881 OUT_PKT4(ring, REG_A6XX_RB_SAMPLE_CNTL, 1);
882 OUT_RING(ring, COND(sample_shading, A6XX_RB_SAMPLE_CNTL_PER_SAMP_MODE));
883
884 OUT_PKT4(ring, REG_A6XX_GRAS_LRZ_PS_INPUT_CNTL, 1);
885 OUT_RING(ring,
886 CONDREG(samp_id_regid, A6XX_GRAS_LRZ_PS_INPUT_CNTL_SAMPLEID) |
887 A6XX_GRAS_LRZ_PS_INPUT_CNTL_FRAGCOORDSAMPLEMODE(
888 sample_shading ? FRAGCOORD_SAMPLE : FRAGCOORD_CENTER));
889
890 OUT_PKT4(ring, REG_A6XX_GRAS_SAMPLE_CNTL, 1);
891 OUT_RING(ring, COND(sample_shading, A6XX_GRAS_SAMPLE_CNTL_PER_SAMP_MODE));
892
893 OUT_PKT4(ring, REG_A6XX_SP_FS_OUTPUT_REG(0), 8);
894 for (i = 0; i < 8; i++) {
895 OUT_RING(ring, A6XX_SP_FS_OUTPUT_REG_REGID(color_regid[i]) |
896 COND(color_regid[i] & HALF_REG_ID,
897 A6XX_SP_FS_OUTPUT_REG_HALF_PRECISION));
898 if (VALIDREG(color_regid[i])) {
899 state->mrt_components |= 0xf << (i * 4);
900 }
901 }
902
903 /* dual source blending has an extra fs output in the 2nd slot */
904 if (fs_has_dual_src_color) {
905 state->mrt_components |= 0xf << 4;
906 }
907
908 OUT_PKT4(ring, REG_A6XX_VPC_VS_PACK, 1);
909 OUT_RING(ring, A6XX_VPC_VS_PACK_POSITIONLOC(pos_loc) |
910 A6XX_VPC_VS_PACK_PSIZELOC(psize_loc) |
911 A6XX_VPC_VS_PACK_STRIDE_IN_VPC(l.max_loc));
912
913 if (gs) {
914 assert(gs->mergedregs == (ds ? ds->mergedregs : vs->mergedregs));
915 OUT_PKT4(ring, REG_A6XX_SP_GS_CTRL_REG0, 1);
916 OUT_RING(
917 ring,
918 A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT(gs->info.max_reg + 1) |
919 A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT(gs->info.max_half_reg + 1) |
920 A6XX_SP_GS_CTRL_REG0_BRANCHSTACK(ir3_shader_branchstack_hw(gs)));
921
922 fd6_emit_shader(ctx, ring, gs);
923 fd6_emit_immediates(ctx->screen, gs, ring);
924 if (ds)
925 fd6_emit_link_map(ctx->screen, ds, gs, ring);
926 else
927 fd6_emit_link_map(ctx->screen, vs, gs, ring);
928
929 OUT_PKT4(ring, REG_A6XX_VPC_GS_PACK, 1);
930 OUT_RING(ring, A6XX_VPC_GS_PACK_POSITIONLOC(pos_loc) |
931 A6XX_VPC_GS_PACK_PSIZELOC(psize_loc) |
932 A6XX_VPC_GS_PACK_STRIDE_IN_VPC(l.max_loc));
933
934 OUT_PKT4(ring, REG_A6XX_VPC_GS_LAYER_CNTL, 1);
935 OUT_RING(ring, A6XX_VPC_GS_LAYER_CNTL_LAYERLOC(layer_loc) | 0xff00);
936
937 OUT_PKT4(ring, REG_A6XX_GRAS_GS_LAYER_CNTL, 1);
938 OUT_RING(ring,
939 CONDREG(layer_regid, A6XX_GRAS_GS_LAYER_CNTL_WRITES_LAYER));
940
941 uint32_t flags_regid =
942 ir3_find_output_regid(gs, VARYING_SLOT_GS_VERTEX_FLAGS_IR3);
943
944 OUT_PKT4(ring, REG_A6XX_SP_GS_PRIMITIVE_CNTL, 1);
945 OUT_RING(ring, A6XX_SP_GS_PRIMITIVE_CNTL_OUT(l.cnt) |
946 A6XX_SP_GS_PRIMITIVE_CNTL_FLAGS_REGID(flags_regid));
947
948 OUT_PKT4(ring, REG_A6XX_PC_GS_OUT_CNTL, 1);
949 OUT_RING(ring,
950 A6XX_PC_GS_OUT_CNTL_STRIDE_IN_VPC(l.max_loc) |
951 CONDREG(psize_regid, A6XX_PC_GS_OUT_CNTL_PSIZE) |
952 CONDREG(layer_regid, A6XX_PC_GS_OUT_CNTL_LAYER) |
953 COND(gs_reads_primid, A6XX_PC_GS_OUT_CNTL_PRIMITIVE_ID) |
954 A6XX_PC_GS_OUT_CNTL_CLIP_MASK(clip_cull_mask));
955
956 uint32_t output;
957 switch (gs->shader->nir->info.gs.output_primitive) {
958 case GL_POINTS:
959 output = TESS_POINTS;
960 break;
961 case GL_LINE_STRIP:
962 output = TESS_LINES;
963 break;
964 case GL_TRIANGLE_STRIP:
965 output = TESS_CW_TRIS;
966 break;
967 default:
968 unreachable("");
969 }
970 OUT_PKT4(ring, REG_A6XX_PC_PRIMITIVE_CNTL_5, 1);
971 OUT_RING(ring, A6XX_PC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT(
972 gs->shader->nir->info.gs.vertices_out - 1) |
973 A6XX_PC_PRIMITIVE_CNTL_5_GS_OUTPUT(output) |
974 A6XX_PC_PRIMITIVE_CNTL_5_GS_INVOCATIONS(
975 gs->shader->nir->info.gs.invocations - 1));
976
977 OUT_PKT4(ring, REG_A6XX_GRAS_GS_CL_CNTL, 1);
978 OUT_RING(ring, A6XX_GRAS_GS_CL_CNTL_CLIP_MASK(clip_mask) |
979 A6XX_GRAS_GS_CL_CNTL_CULL_MASK(cull_mask));
980
981 OUT_PKT4(ring, REG_A6XX_VPC_GS_PARAM, 1);
982 OUT_RING(ring, 0xff);
983
984 OUT_PKT4(ring, REG_A6XX_VPC_GS_CLIP_CNTL, 1);
985 OUT_RING(ring, A6XX_VPC_GS_CLIP_CNTL_CLIP_MASK(clip_cull_mask) |
986 A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_03_LOC(clip0_loc) |
987 A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_47_LOC(clip1_loc));
988
989 const struct ir3_shader_variant *prev = state->ds ? state->ds : state->vs;
990
991 /* Size of per-primitive alloction in ldlw memory in vec4s. */
992 uint32_t vec4_size = gs->shader->nir->info.gs.vertices_in *
993 DIV_ROUND_UP(prev->output_size, 4);
994 OUT_PKT4(ring, REG_A6XX_PC_PRIMITIVE_CNTL_6, 1);
995 OUT_RING(ring, A6XX_PC_PRIMITIVE_CNTL_6_STRIDE_IN_VPC(vec4_size));
996
997 OUT_PKT4(ring, REG_A6XX_PC_MULTIVIEW_CNTL, 1);
998 OUT_RING(ring, 0);
999
1000 uint32_t prim_size = prev->output_size;
1001 if (prim_size > 64)
1002 prim_size = 64;
1003 else if (prim_size == 64)
1004 prim_size = 63;
1005 OUT_PKT4(ring, REG_A6XX_SP_GS_PRIM_SIZE, 1);
1006 OUT_RING(ring, prim_size);
1007 } else {
1008 OUT_PKT4(ring, REG_A6XX_PC_PRIMITIVE_CNTL_6, 1);
1009 OUT_RING(ring, 0);
1010 OUT_PKT4(ring, REG_A6XX_SP_GS_PRIM_SIZE, 1);
1011 OUT_RING(ring, 0);
1012
1013 OUT_PKT4(ring, REG_A6XX_GRAS_VS_LAYER_CNTL, 1);
1014 OUT_RING(ring,
1015 CONDREG(layer_regid, A6XX_GRAS_VS_LAYER_CNTL_WRITES_LAYER));
1016 }
1017
1018 OUT_PKT4(ring, REG_A6XX_VPC_VS_CLIP_CNTL, 1);
1019 OUT_RING(ring, A6XX_VPC_VS_CLIP_CNTL_CLIP_MASK(clip_cull_mask) |
1020 A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_03_LOC(clip0_loc) |
1021 A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_47_LOC(clip1_loc));
1022
1023 OUT_PKT4(ring, REG_A6XX_GRAS_VS_CL_CNTL, 1);
1024 OUT_RING(ring, A6XX_GRAS_VS_CL_CNTL_CLIP_MASK(clip_mask) |
1025 A6XX_GRAS_VS_CL_CNTL_CULL_MASK(cull_mask));
1026
1027 OUT_PKT4(ring, REG_A6XX_VPC_UNKNOWN_9107, 1);
1028 OUT_RING(ring, 0);
1029
1030 if (fs->instrlen)
1031 fd6_emit_shader(ctx, ring, fs);
1032
1033 OUT_REG(ring, A6XX_PC_PRIMID_PASSTHRU(primid_passthru));
1034
1035 uint32_t non_sysval_input_count = 0;
1036 for (uint32_t i = 0; i < vs->inputs_count; i++)
1037 if (!vs->inputs[i].sysval)
1038 non_sysval_input_count++;
1039
1040 OUT_PKT4(ring, REG_A6XX_VFD_CONTROL_0, 1);
1041 OUT_RING(ring, A6XX_VFD_CONTROL_0_FETCH_CNT(non_sysval_input_count) |
1042 A6XX_VFD_CONTROL_0_DECODE_CNT(non_sysval_input_count));
1043
1044 OUT_PKT4(ring, REG_A6XX_VFD_DEST_CNTL(0), non_sysval_input_count);
1045 for (uint32_t i = 0; i < non_sysval_input_count; i++) {
1046 assert(vs->inputs[i].compmask);
1047 OUT_RING(ring,
1048 A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK(vs->inputs[i].compmask) |
1049 A6XX_VFD_DEST_CNTL_INSTR_REGID(vs->inputs[i].regid));
1050 }
1051
1052 OUT_PKT4(ring, REG_A6XX_VFD_CONTROL_1, 6);
1053 OUT_RING(ring, A6XX_VFD_CONTROL_1_REGID4VTX(vertex_regid) |
1054 A6XX_VFD_CONTROL_1_REGID4INST(instance_regid) |
1055 A6XX_VFD_CONTROL_1_REGID4PRIMID(vs_primitive_regid) |
1056 0xfc000000);
1057 OUT_RING(ring,
1058 A6XX_VFD_CONTROL_2_REGID_HSRELPATCHID(hs_rel_patch_regid) |
1059 A6XX_VFD_CONTROL_2_REGID_INVOCATIONID(hs_invocation_regid));
1060 OUT_RING(ring, A6XX_VFD_CONTROL_3_REGID_DSRELPATCHID(ds_rel_patch_regid) |
1061 A6XX_VFD_CONTROL_3_REGID_TESSX(tess_coord_x_regid) |
1062 A6XX_VFD_CONTROL_3_REGID_TESSY(tess_coord_y_regid) |
1063 A6XX_VFD_CONTROL_3_REGID_DSPRIMID(ds_primitive_regid));
1064 OUT_RING(ring, 0x000000fc); /* VFD_CONTROL_4 */
1065 OUT_RING(ring, A6XX_VFD_CONTROL_5_REGID_GSHEADER(gs_header_regid) |
1066 0xfc00); /* VFD_CONTROL_5 */
1067 OUT_RING(ring, COND(primid_passthru,
1068 A6XX_VFD_CONTROL_6_PRIMID_PASSTHRU)); /* VFD_CONTROL_6 */
1069
1070 if (!binning_pass)
1071 fd6_emit_immediates(ctx->screen, fs, ring);
1072 }
1073
1074 static void emit_interp_state(struct fd_ringbuffer *ring,
1075 struct ir3_shader_variant *fs, bool rasterflat,
1076 bool sprite_coord_mode,
1077 uint32_t sprite_coord_enable);
1078
1079 static struct fd_ringbuffer *
create_interp_stateobj(struct fd_context * ctx,struct fd6_program_state * state)1080 create_interp_stateobj(struct fd_context *ctx, struct fd6_program_state *state)
1081 {
1082 struct fd_ringbuffer *ring = fd_ringbuffer_new_object(ctx->pipe, 18 * 4);
1083
1084 emit_interp_state(ring, state->fs, false, false, 0);
1085
1086 return ring;
1087 }
1088
1089 /* build the program streaming state which is not part of the pre-
1090 * baked stateobj because of dependency on other gl state (rasterflat
1091 * or sprite-coord-replacement)
1092 */
1093 struct fd_ringbuffer *
fd6_program_interp_state(struct fd6_emit * emit)1094 fd6_program_interp_state(struct fd6_emit *emit)
1095 {
1096 const struct fd6_program_state *state = fd6_emit_get_prog(emit);
1097
1098 if (!unlikely(emit->rasterflat || emit->sprite_coord_enable)) {
1099 /* fastpath: */
1100 return fd_ringbuffer_ref(state->interp_stateobj);
1101 } else {
1102 struct fd_ringbuffer *ring = fd_submit_new_ringbuffer(
1103 emit->ctx->batch->submit, 18 * 4, FD_RINGBUFFER_STREAMING);
1104
1105 emit_interp_state(ring, state->fs, emit->rasterflat,
1106 emit->sprite_coord_mode, emit->sprite_coord_enable);
1107
1108 return ring;
1109 }
1110 }
1111
1112 static void
emit_interp_state(struct fd_ringbuffer * ring,struct ir3_shader_variant * fs,bool rasterflat,bool sprite_coord_mode,uint32_t sprite_coord_enable)1113 emit_interp_state(struct fd_ringbuffer *ring, struct ir3_shader_variant *fs,
1114 bool rasterflat, bool sprite_coord_mode,
1115 uint32_t sprite_coord_enable)
1116 {
1117 uint32_t vinterp[8], vpsrepl[8];
1118
1119 memset(vinterp, 0, sizeof(vinterp));
1120 memset(vpsrepl, 0, sizeof(vpsrepl));
1121
1122 for (int j = -1; (j = ir3_next_varying(fs, j)) < (int)fs->inputs_count;) {
1123
1124 /* NOTE: varyings are packed, so if compmask is 0xb
1125 * then first, third, and fourth component occupy
1126 * three consecutive varying slots:
1127 */
1128 unsigned compmask = fs->inputs[j].compmask;
1129
1130 uint32_t inloc = fs->inputs[j].inloc;
1131
1132 if (fs->inputs[j].flat || (fs->inputs[j].rasterflat && rasterflat)) {
1133 uint32_t loc = inloc;
1134
1135 for (int i = 0; i < 4; i++) {
1136 if (compmask & (1 << i)) {
1137 vinterp[loc / 16] |= 1 << ((loc % 16) * 2);
1138 loc++;
1139 }
1140 }
1141 }
1142
1143 bool coord_mode = sprite_coord_mode;
1144 if (ir3_point_sprite(fs, j, sprite_coord_enable, &coord_mode)) {
1145 /* mask is two 2-bit fields, where:
1146 * '01' -> S
1147 * '10' -> T
1148 * '11' -> 1 - T (flip mode)
1149 */
1150 unsigned mask = coord_mode ? 0b1101 : 0b1001;
1151 uint32_t loc = inloc;
1152 if (compmask & 0x1) {
1153 vpsrepl[loc / 16] |= ((mask >> 0) & 0x3) << ((loc % 16) * 2);
1154 loc++;
1155 }
1156 if (compmask & 0x2) {
1157 vpsrepl[loc / 16] |= ((mask >> 2) & 0x3) << ((loc % 16) * 2);
1158 loc++;
1159 }
1160 if (compmask & 0x4) {
1161 /* .z <- 0.0f */
1162 vinterp[loc / 16] |= 0b10 << ((loc % 16) * 2);
1163 loc++;
1164 }
1165 if (compmask & 0x8) {
1166 /* .w <- 1.0f */
1167 vinterp[loc / 16] |= 0b11 << ((loc % 16) * 2);
1168 loc++;
1169 }
1170 }
1171 }
1172
1173 OUT_PKT4(ring, REG_A6XX_VPC_VARYING_INTERP_MODE(0), 8);
1174 for (int i = 0; i < 8; i++)
1175 OUT_RING(ring, vinterp[i]); /* VPC_VARYING_INTERP[i].MODE */
1176
1177 OUT_PKT4(ring, REG_A6XX_VPC_VARYING_PS_REPL_MODE(0), 8);
1178 for (int i = 0; i < 8; i++)
1179 OUT_RING(ring, vpsrepl[i]); /* VPC_VARYING_PS_REPL[i] */
1180 }
1181
1182 static struct ir3_program_state *
fd6_program_create(void * data,struct ir3_shader_variant * bs,struct ir3_shader_variant * vs,struct ir3_shader_variant * hs,struct ir3_shader_variant * ds,struct ir3_shader_variant * gs,struct ir3_shader_variant * fs,const struct ir3_shader_key * key)1183 fd6_program_create(void *data, struct ir3_shader_variant *bs,
1184 struct ir3_shader_variant *vs, struct ir3_shader_variant *hs,
1185 struct ir3_shader_variant *ds, struct ir3_shader_variant *gs,
1186 struct ir3_shader_variant *fs,
1187 const struct ir3_shader_key *key) in_dt
1188 {
1189 struct fd_context *ctx = fd_context(data);
1190 struct fd6_program_state *state = CALLOC_STRUCT(fd6_program_state);
1191
1192 tc_assert_driver_thread(ctx->tc);
1193
1194 /* if we have streamout, use full VS in binning pass, as the
1195 * binning pass VS will have outputs on other than position/psize
1196 * stripped out:
1197 */
1198 state->bs = vs->shader->stream_output.num_outputs ? vs : bs;
1199 state->vs = vs;
1200 state->hs = hs;
1201 state->ds = ds;
1202 state->gs = gs;
1203 state->fs = fs;
1204 state->binning_stateobj = fd_ringbuffer_new_object(ctx->pipe, 0x1000);
1205 state->stateobj = fd_ringbuffer_new_object(ctx->pipe, 0x1000);
1206
1207 #ifdef DEBUG
1208 if (!ds) {
1209 for (unsigned i = 0; i < bs->inputs_count; i++) {
1210 if (vs->inputs[i].sysval)
1211 continue;
1212 debug_assert(bs->inputs[i].regid == vs->inputs[i].regid);
1213 }
1214 }
1215 #endif
1216
1217 setup_config_stateobj(ctx, state);
1218 setup_stateobj(state->binning_stateobj, ctx, state, key, true);
1219 setup_stateobj(state->stateobj, ctx, state, key, false);
1220 state->interp_stateobj = create_interp_stateobj(ctx, state);
1221
1222 struct ir3_stream_output_info *stream_output =
1223 &fd6_last_shader(state)->shader->stream_output;
1224 if (stream_output->num_outputs > 0)
1225 state->stream_output = stream_output;
1226
1227 return &state->base;
1228 }
1229
1230 static void
fd6_program_destroy(void * data,struct ir3_program_state * state)1231 fd6_program_destroy(void *data, struct ir3_program_state *state)
1232 {
1233 struct fd6_program_state *so = fd6_program_state(state);
1234 fd_ringbuffer_del(so->stateobj);
1235 fd_ringbuffer_del(so->binning_stateobj);
1236 fd_ringbuffer_del(so->config_stateobj);
1237 fd_ringbuffer_del(so->interp_stateobj);
1238 if (so->streamout_stateobj)
1239 fd_ringbuffer_del(so->streamout_stateobj);
1240 free(so);
1241 }
1242
1243 static const struct ir3_cache_funcs cache_funcs = {
1244 .create_state = fd6_program_create,
1245 .destroy_state = fd6_program_destroy,
1246 };
1247
1248 void
fd6_prog_init(struct pipe_context * pctx)1249 fd6_prog_init(struct pipe_context *pctx)
1250 {
1251 struct fd_context *ctx = fd_context(pctx);
1252
1253 ctx->shader_cache = ir3_cache_create(&cache_funcs, ctx);
1254
1255 ir3_prog_init(pctx);
1256
1257 fd_prog_init(pctx);
1258 }
1259