1
2 #include "pipe/p_context.h"
3 #include "pipe/p_state.h"
4 #include "util/u_inlines.h"
5 #include "util/format/u_format.h"
6 #include "translate/translate.h"
7
8 #include "nvc0/nvc0_context.h"
9 #include "nvc0/nvc0_resource.h"
10
11 #include "nvc0/nvc0_3d.xml.h"
12
13 struct push_context {
14 struct nouveau_pushbuf *push;
15
16 struct translate *translate;
17 void *dest;
18 const void *idxbuf;
19
20 uint32_t vertex_size;
21 uint32_t restart_index;
22 uint32_t start_instance;
23 uint32_t instance_id;
24
25 bool prim_restart;
26 bool need_vertex_id;
27
28 struct {
29 bool enabled;
30 bool value;
31 uint8_t width;
32 unsigned stride;
33 const uint8_t *data;
34 } edgeflag;
35 };
36
37 static void nvc0_push_upload_vertex_ids(struct push_context *,
38 struct nvc0_context *,
39 const struct pipe_draw_info *);
40
41 static void
nvc0_push_context_init(struct nvc0_context * nvc0,struct push_context * ctx)42 nvc0_push_context_init(struct nvc0_context *nvc0, struct push_context *ctx)
43 {
44 ctx->push = nvc0->base.pushbuf;
45
46 ctx->translate = nvc0->vertex->translate;
47 ctx->vertex_size = nvc0->vertex->size;
48 ctx->instance_id = 0;
49
50 ctx->need_vertex_id =
51 nvc0->vertprog->vp.need_vertex_id && (nvc0->vertex->num_elements < 32);
52
53 ctx->edgeflag.value = true;
54 ctx->edgeflag.enabled = nvc0->vertprog->vp.edgeflag < PIPE_MAX_ATTRIBS;
55
56 /* silence warnings */
57 ctx->edgeflag.data = NULL;
58 ctx->edgeflag.stride = 0;
59 ctx->edgeflag.width = 0;
60 }
61
62 static inline void
nvc0_vertex_configure_translate(struct nvc0_context * nvc0,int32_t index_bias)63 nvc0_vertex_configure_translate(struct nvc0_context *nvc0, int32_t index_bias)
64 {
65 struct translate *translate = nvc0->vertex->translate;
66 unsigned i;
67
68 for (i = 0; i < nvc0->num_vtxbufs; ++i) {
69 const uint8_t *map;
70 const struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[i];
71
72 if (likely(vb->is_user_buffer))
73 map = (const uint8_t *)vb->buffer.user;
74 else {
75 if (!vb->buffer.resource)
76 continue;
77
78 map = nouveau_resource_map_offset(&nvc0->base,
79 nv04_resource(vb->buffer.resource), vb->buffer_offset, NOUVEAU_BO_RD);
80 }
81
82 if (index_bias && !unlikely(nvc0->vertex->instance_bufs & (1 << i)))
83 map += (intptr_t)index_bias * vb->stride;
84
85 translate->set_buffer(translate, i, map, vb->stride, ~0);
86 }
87 }
88
89 static inline void
nvc0_push_map_idxbuf(struct push_context * ctx,struct nvc0_context * nvc0,const struct pipe_draw_info * info)90 nvc0_push_map_idxbuf(struct push_context *ctx, struct nvc0_context *nvc0,
91 const struct pipe_draw_info *info)
92 {
93 if (!info->has_user_indices) {
94 struct nv04_resource *buf = nv04_resource(info->index.resource);
95 ctx->idxbuf = nouveau_resource_map_offset(
96 &nvc0->base, buf, 0, NOUVEAU_BO_RD);
97 } else {
98 ctx->idxbuf = info->index.user;
99 }
100 }
101
102 static inline void
nvc0_push_map_edgeflag(struct push_context * ctx,struct nvc0_context * nvc0,int32_t index_bias)103 nvc0_push_map_edgeflag(struct push_context *ctx, struct nvc0_context *nvc0,
104 int32_t index_bias)
105 {
106 unsigned attr = nvc0->vertprog->vp.edgeflag;
107 struct pipe_vertex_element *ve = &nvc0->vertex->element[attr].pipe;
108 struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[ve->vertex_buffer_index];
109 struct nv04_resource *buf = nv04_resource(vb->buffer.resource);
110
111 ctx->edgeflag.stride = vb->stride;
112 ctx->edgeflag.width = util_format_get_blocksize(ve->src_format);
113 if (!vb->is_user_buffer) {
114 unsigned offset = vb->buffer_offset + ve->src_offset;
115 ctx->edgeflag.data = nouveau_resource_map_offset(&nvc0->base,
116 buf, offset, NOUVEAU_BO_RD);
117 } else {
118 ctx->edgeflag.data = (const uint8_t *)vb->buffer.user + ve->src_offset;
119 }
120
121 if (index_bias)
122 ctx->edgeflag.data += (intptr_t)index_bias * vb->stride;
123 }
124
125 static inline unsigned
prim_restart_search_i08(const uint8_t * elts,unsigned push,uint8_t index)126 prim_restart_search_i08(const uint8_t *elts, unsigned push, uint8_t index)
127 {
128 unsigned i;
129 for (i = 0; i < push && elts[i] != index; ++i);
130 return i;
131 }
132
133 static inline unsigned
prim_restart_search_i16(const uint16_t * elts,unsigned push,uint16_t index)134 prim_restart_search_i16(const uint16_t *elts, unsigned push, uint16_t index)
135 {
136 unsigned i;
137 for (i = 0; i < push && elts[i] != index; ++i);
138 return i;
139 }
140
141 static inline unsigned
prim_restart_search_i32(const uint32_t * elts,unsigned push,uint32_t index)142 prim_restart_search_i32(const uint32_t *elts, unsigned push, uint32_t index)
143 {
144 unsigned i;
145 for (i = 0; i < push && elts[i] != index; ++i);
146 return i;
147 }
148
149 static inline bool
ef_value_8(const struct push_context * ctx,uint32_t index)150 ef_value_8(const struct push_context *ctx, uint32_t index)
151 {
152 uint8_t *pf = (uint8_t *)&ctx->edgeflag.data[index * ctx->edgeflag.stride];
153 return !!*pf;
154 }
155
156 static inline bool
ef_value_32(const struct push_context * ctx,uint32_t index)157 ef_value_32(const struct push_context *ctx, uint32_t index)
158 {
159 uint32_t *pf = (uint32_t *)&ctx->edgeflag.data[index * ctx->edgeflag.stride];
160 return !!*pf;
161 }
162
163 static inline bool
ef_toggle(struct push_context * ctx)164 ef_toggle(struct push_context *ctx)
165 {
166 ctx->edgeflag.value = !ctx->edgeflag.value;
167 return ctx->edgeflag.value;
168 }
169
170 static inline unsigned
ef_toggle_search_i08(struct push_context * ctx,const uint8_t * elts,unsigned n)171 ef_toggle_search_i08(struct push_context *ctx, const uint8_t *elts, unsigned n)
172 {
173 unsigned i;
174 bool ef = ctx->edgeflag.value;
175 if (ctx->edgeflag.width == 1)
176 for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
177 else
178 for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
179 return i;
180 }
181
182 static inline unsigned
ef_toggle_search_i16(struct push_context * ctx,const uint16_t * elts,unsigned n)183 ef_toggle_search_i16(struct push_context *ctx, const uint16_t *elts, unsigned n)
184 {
185 unsigned i;
186 bool ef = ctx->edgeflag.value;
187 if (ctx->edgeflag.width == 1)
188 for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
189 else
190 for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
191 return i;
192 }
193
194 static inline unsigned
ef_toggle_search_i32(struct push_context * ctx,const uint32_t * elts,unsigned n)195 ef_toggle_search_i32(struct push_context *ctx, const uint32_t *elts, unsigned n)
196 {
197 unsigned i;
198 bool ef = ctx->edgeflag.value;
199 if (ctx->edgeflag.width == 1)
200 for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
201 else
202 for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
203 return i;
204 }
205
206 static inline unsigned
ef_toggle_search_seq(struct push_context * ctx,unsigned start,unsigned n)207 ef_toggle_search_seq(struct push_context *ctx, unsigned start, unsigned n)
208 {
209 unsigned i;
210 bool ef = ctx->edgeflag.value;
211 if (ctx->edgeflag.width == 1)
212 for (i = 0; i < n && ef_value_8(ctx, start++) == ef; ++i);
213 else
214 for (i = 0; i < n && ef_value_32(ctx, start++) == ef; ++i);
215 return i;
216 }
217
218 static inline void *
nvc0_push_setup_vertex_array(struct nvc0_context * nvc0,const unsigned count)219 nvc0_push_setup_vertex_array(struct nvc0_context *nvc0, const unsigned count)
220 {
221 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
222 struct nouveau_bo *bo;
223 uint64_t va;
224 const unsigned size = count * nvc0->vertex->size;
225
226 void *const dest = nouveau_scratch_get(&nvc0->base, size, &va, &bo);
227
228 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_START_HIGH(0)), 2);
229 PUSH_DATAh(push, va);
230 PUSH_DATA (push, va);
231
232 if (nvc0->screen->eng3d->oclass < TU102_3D_CLASS)
233 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(0)), 2);
234 else
235 BEGIN_NVC0(push, SUBC_3D(TU102_3D_VERTEX_ARRAY_LIMIT_HIGH(0)), 2);
236 PUSH_DATAh(push, va + size - 1);
237 PUSH_DATA (push, va + size - 1);
238
239 BCTX_REFN_bo(nvc0->bufctx_3d, 3D_VTX_TMP, NOUVEAU_BO_GART | NOUVEAU_BO_RD,
240 bo);
241 nouveau_pushbuf_validate(push);
242
243 return dest;
244 }
245
246 static void
disp_vertices_i08(struct push_context * ctx,unsigned start,unsigned count)247 disp_vertices_i08(struct push_context *ctx, unsigned start, unsigned count)
248 {
249 struct nouveau_pushbuf *push = ctx->push;
250 struct translate *translate = ctx->translate;
251 const uint8_t *restrict elts = (uint8_t *)ctx->idxbuf + start;
252 unsigned pos = 0;
253
254 do {
255 unsigned nR = count;
256
257 if (unlikely(ctx->prim_restart))
258 nR = prim_restart_search_i08(elts, nR, ctx->restart_index);
259
260 translate->run_elts8(translate, elts, nR,
261 ctx->start_instance, ctx->instance_id, ctx->dest);
262 count -= nR;
263 ctx->dest += nR * ctx->vertex_size;
264
265 while (nR) {
266 unsigned nE = nR;
267
268 if (unlikely(ctx->edgeflag.enabled))
269 nE = ef_toggle_search_i08(ctx, elts, nR);
270
271 PUSH_SPACE(push, 4);
272 if (likely(nE >= 2)) {
273 BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
274 PUSH_DATA (push, pos);
275 PUSH_DATA (push, nE);
276 } else
277 if (nE) {
278 if (pos <= 0xff) {
279 IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
280 } else {
281 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
282 PUSH_DATA (push, pos);
283 }
284 }
285 if (unlikely(nE != nR))
286 IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
287
288 pos += nE;
289 elts += nE;
290 nR -= nE;
291 }
292 if (count) {
293 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
294 PUSH_DATA (push, 0xffffffff);
295 ++elts;
296 ctx->dest += ctx->vertex_size;
297 ++pos;
298 --count;
299 }
300 } while (count);
301 }
302
303 static void
disp_vertices_i16(struct push_context * ctx,unsigned start,unsigned count)304 disp_vertices_i16(struct push_context *ctx, unsigned start, unsigned count)
305 {
306 struct nouveau_pushbuf *push = ctx->push;
307 struct translate *translate = ctx->translate;
308 const uint16_t *restrict elts = (uint16_t *)ctx->idxbuf + start;
309 unsigned pos = 0;
310
311 do {
312 unsigned nR = count;
313
314 if (unlikely(ctx->prim_restart))
315 nR = prim_restart_search_i16(elts, nR, ctx->restart_index);
316
317 translate->run_elts16(translate, elts, nR,
318 ctx->start_instance, ctx->instance_id, ctx->dest);
319 count -= nR;
320 ctx->dest += nR * ctx->vertex_size;
321
322 while (nR) {
323 unsigned nE = nR;
324
325 if (unlikely(ctx->edgeflag.enabled))
326 nE = ef_toggle_search_i16(ctx, elts, nR);
327
328 PUSH_SPACE(push, 4);
329 if (likely(nE >= 2)) {
330 BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
331 PUSH_DATA (push, pos);
332 PUSH_DATA (push, nE);
333 } else
334 if (nE) {
335 if (pos <= 0xff) {
336 IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
337 } else {
338 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
339 PUSH_DATA (push, pos);
340 }
341 }
342 if (unlikely(nE != nR))
343 IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
344
345 pos += nE;
346 elts += nE;
347 nR -= nE;
348 }
349 if (count) {
350 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
351 PUSH_DATA (push, 0xffffffff);
352 ++elts;
353 ctx->dest += ctx->vertex_size;
354 ++pos;
355 --count;
356 }
357 } while (count);
358 }
359
360 static void
disp_vertices_i32(struct push_context * ctx,unsigned start,unsigned count)361 disp_vertices_i32(struct push_context *ctx, unsigned start, unsigned count)
362 {
363 struct nouveau_pushbuf *push = ctx->push;
364 struct translate *translate = ctx->translate;
365 const uint32_t *restrict elts = (uint32_t *)ctx->idxbuf + start;
366 unsigned pos = 0;
367
368 do {
369 unsigned nR = count;
370
371 if (unlikely(ctx->prim_restart))
372 nR = prim_restart_search_i32(elts, nR, ctx->restart_index);
373
374 translate->run_elts(translate, elts, nR,
375 ctx->start_instance, ctx->instance_id, ctx->dest);
376 count -= nR;
377 ctx->dest += nR * ctx->vertex_size;
378
379 while (nR) {
380 unsigned nE = nR;
381
382 if (unlikely(ctx->edgeflag.enabled))
383 nE = ef_toggle_search_i32(ctx, elts, nR);
384
385 PUSH_SPACE(push, 4);
386 if (likely(nE >= 2)) {
387 BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
388 PUSH_DATA (push, pos);
389 PUSH_DATA (push, nE);
390 } else
391 if (nE) {
392 if (pos <= 0xff) {
393 IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
394 } else {
395 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
396 PUSH_DATA (push, pos);
397 }
398 }
399 if (unlikely(nE != nR))
400 IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
401
402 pos += nE;
403 elts += nE;
404 nR -= nE;
405 }
406 if (count) {
407 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
408 PUSH_DATA (push, 0xffffffff);
409 ++elts;
410 ctx->dest += ctx->vertex_size;
411 ++pos;
412 --count;
413 }
414 } while (count);
415 }
416
417 static void
disp_vertices_seq(struct push_context * ctx,unsigned start,unsigned count)418 disp_vertices_seq(struct push_context *ctx, unsigned start, unsigned count)
419 {
420 struct nouveau_pushbuf *push = ctx->push;
421 struct translate *translate = ctx->translate;
422 unsigned pos = 0;
423
424 /* XXX: This will read the data corresponding to the primitive restart index,
425 * maybe we should avoid that ?
426 */
427 translate->run(translate, start, count,
428 ctx->start_instance, ctx->instance_id, ctx->dest);
429 do {
430 unsigned nr = count;
431
432 if (unlikely(ctx->edgeflag.enabled))
433 nr = ef_toggle_search_seq(ctx, start + pos, nr);
434
435 PUSH_SPACE(push, 4);
436 if (likely(nr)) {
437 BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
438 PUSH_DATA (push, pos);
439 PUSH_DATA (push, nr);
440 }
441 if (unlikely(nr != count))
442 IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
443
444 pos += nr;
445 count -= nr;
446 } while (count);
447 }
448
449
450 #define NVC0_PRIM_GL_CASE(n) \
451 case PIPE_PRIM_##n: return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_##n
452
453 static inline unsigned
nvc0_prim_gl(unsigned prim)454 nvc0_prim_gl(unsigned prim)
455 {
456 switch (prim) {
457 NVC0_PRIM_GL_CASE(POINTS);
458 NVC0_PRIM_GL_CASE(LINES);
459 NVC0_PRIM_GL_CASE(LINE_LOOP);
460 NVC0_PRIM_GL_CASE(LINE_STRIP);
461 NVC0_PRIM_GL_CASE(TRIANGLES);
462 NVC0_PRIM_GL_CASE(TRIANGLE_STRIP);
463 NVC0_PRIM_GL_CASE(TRIANGLE_FAN);
464 NVC0_PRIM_GL_CASE(QUADS);
465 NVC0_PRIM_GL_CASE(QUAD_STRIP);
466 NVC0_PRIM_GL_CASE(POLYGON);
467 NVC0_PRIM_GL_CASE(LINES_ADJACENCY);
468 NVC0_PRIM_GL_CASE(LINE_STRIP_ADJACENCY);
469 NVC0_PRIM_GL_CASE(TRIANGLES_ADJACENCY);
470 NVC0_PRIM_GL_CASE(TRIANGLE_STRIP_ADJACENCY);
471 NVC0_PRIM_GL_CASE(PATCHES);
472 default:
473 return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS;
474 }
475 }
476
477 typedef struct {
478 uint32_t count;
479 uint32_t primCount;
480 uint32_t first;
481 uint32_t baseInstance;
482 } DrawArraysIndirectCommand;
483
484 typedef struct {
485 uint32_t count;
486 uint32_t primCount;
487 uint32_t firstIndex;
488 int32_t baseVertex;
489 uint32_t baseInstance;
490 } DrawElementsIndirectCommand;
491
492 void
nvc0_push_vbo_indirect(struct nvc0_context * nvc0,const struct pipe_draw_info * info)493 nvc0_push_vbo_indirect(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
494 {
495 /* The strategy here is to just read the commands from the indirect buffer
496 * and do the draws. This is suboptimal, but will only happen in the case
497 * that conversion is required for FIXED or DOUBLE inputs.
498 */
499 struct nvc0_screen *screen = nvc0->screen;
500 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
501 struct nv04_resource *buf = nv04_resource(info->indirect->buffer);
502 struct nv04_resource *buf_count = nv04_resource(info->indirect->indirect_draw_count);
503 unsigned i;
504
505 unsigned draw_count = info->indirect->draw_count;
506 if (buf_count) {
507 uint32_t *count = nouveau_resource_map_offset(
508 &nvc0->base, buf_count, info->indirect->indirect_draw_count_offset,
509 NOUVEAU_BO_RD);
510 draw_count = *count;
511 }
512
513 uint8_t *buf_data = nouveau_resource_map_offset(
514 &nvc0->base, buf, info->indirect->offset, NOUVEAU_BO_RD);
515 struct pipe_draw_info single = *info;
516 single.indirect = NULL;
517 for (i = 0; i < draw_count; i++, buf_data += info->indirect->stride) {
518 if (info->index_size) {
519 DrawElementsIndirectCommand *cmd = (void *)buf_data;
520 single.start = info->start + cmd->firstIndex;
521 single.count = cmd->count;
522 single.start_instance = cmd->baseInstance;
523 single.instance_count = cmd->primCount;
524 single.index_bias = cmd->baseVertex;
525 } else {
526 DrawArraysIndirectCommand *cmd = (void *)buf_data;
527 single.start = cmd->first;
528 single.count = cmd->count;
529 single.start_instance = cmd->baseInstance;
530 single.instance_count = cmd->primCount;
531 }
532
533 if (nvc0->vertprog->vp.need_draw_parameters) {
534 PUSH_SPACE(push, 9);
535 BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
536 PUSH_DATA (push, NVC0_CB_AUX_SIZE);
537 PUSH_DATAh(push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(0));
538 PUSH_DATA (push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(0));
539 BEGIN_1IC0(push, NVC0_3D(CB_POS), 1 + 3);
540 PUSH_DATA (push, NVC0_CB_AUX_DRAW_INFO);
541 PUSH_DATA (push, single.index_bias);
542 PUSH_DATA (push, single.start_instance);
543 PUSH_DATA (push, single.drawid + i);
544 }
545
546 nvc0_push_vbo(nvc0, &single);
547 }
548
549 nouveau_resource_unmap(buf);
550 if (buf_count)
551 nouveau_resource_unmap(buf_count);
552 }
553
554 void
nvc0_push_vbo(struct nvc0_context * nvc0,const struct pipe_draw_info * info)555 nvc0_push_vbo(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
556 {
557 struct push_context ctx;
558 unsigned i, index_size;
559 unsigned inst_count = info->instance_count;
560 unsigned vert_count = info->count;
561 unsigned prim;
562
563 nvc0_push_context_init(nvc0, &ctx);
564
565 nvc0_vertex_configure_translate(nvc0, info->index_bias);
566
567 if (nvc0->state.index_bias) {
568 /* this is already taken care of by translate */
569 IMMED_NVC0(ctx.push, NVC0_3D(VB_ELEMENT_BASE), 0);
570 nvc0->state.index_bias = 0;
571 }
572
573 if (unlikely(ctx.edgeflag.enabled))
574 nvc0_push_map_edgeflag(&ctx, nvc0, info->index_bias);
575
576 ctx.prim_restart = info->primitive_restart;
577 ctx.restart_index = info->restart_index;
578
579 if (info->primitive_restart) {
580 /* NOTE: I hope we won't ever need that last index (~0).
581 * If we do, we have to disable primitive restart here always and
582 * use END,BEGIN to restart. (XXX: would that affect PrimitiveID ?)
583 * We could also deactive PRIM_RESTART_WITH_DRAW_ARRAYS temporarily,
584 * and add manual restart to disp_vertices_seq.
585 */
586 BEGIN_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 2);
587 PUSH_DATA (ctx.push, 1);
588 PUSH_DATA (ctx.push, info->index_size ? 0xffffffff : info->restart_index);
589 } else
590 if (nvc0->state.prim_restart) {
591 IMMED_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 0);
592 }
593 nvc0->state.prim_restart = info->primitive_restart;
594
595 if (info->index_size) {
596 nvc0_push_map_idxbuf(&ctx, nvc0, info);
597 index_size = info->index_size;
598 } else {
599 if (unlikely(info->count_from_stream_output)) {
600 struct pipe_context *pipe = &nvc0->base.pipe;
601 struct nvc0_so_target *targ;
602 targ = nvc0_so_target(info->count_from_stream_output);
603 pipe->get_query_result(pipe, targ->pq, true, (void *)&vert_count);
604 vert_count /= targ->stride;
605 }
606 ctx.idxbuf = NULL; /* shut up warnings */
607 index_size = 0;
608 }
609
610 ctx.start_instance = info->start_instance;
611
612 prim = nvc0_prim_gl(info->mode);
613 do {
614 PUSH_SPACE(ctx.push, 9);
615
616 ctx.dest = nvc0_push_setup_vertex_array(nvc0, vert_count);
617 if (unlikely(!ctx.dest))
618 break;
619
620 if (unlikely(ctx.need_vertex_id))
621 nvc0_push_upload_vertex_ids(&ctx, nvc0, info);
622
623 if (nvc0->screen->eng3d->oclass < GM107_3D_CLASS)
624 IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
625 BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_BEGIN_GL), 1);
626 PUSH_DATA (ctx.push, prim);
627 switch (index_size) {
628 case 1:
629 disp_vertices_i08(&ctx, info->start, vert_count);
630 break;
631 case 2:
632 disp_vertices_i16(&ctx, info->start, vert_count);
633 break;
634 case 4:
635 disp_vertices_i32(&ctx, info->start, vert_count);
636 break;
637 default:
638 assert(index_size == 0);
639 disp_vertices_seq(&ctx, info->start, vert_count);
640 break;
641 }
642 PUSH_SPACE(ctx.push, 1);
643 IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_END_GL), 0);
644
645 if (--inst_count) {
646 prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
647 ++ctx.instance_id;
648 }
649 nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_VTX_TMP);
650 nouveau_scratch_done(&nvc0->base);
651 } while (inst_count);
652
653
654 /* reset state and unmap buffers (no-op) */
655
656 if (unlikely(!ctx.edgeflag.value)) {
657 PUSH_SPACE(ctx.push, 1);
658 IMMED_NVC0(ctx.push, NVC0_3D(EDGEFLAG), 1);
659 }
660
661 if (unlikely(ctx.need_vertex_id)) {
662 PUSH_SPACE(ctx.push, 4);
663 IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ID_REPLACE), 0);
664 BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_ATTRIB_FORMAT(1)), 1);
665 PUSH_DATA (ctx.push,
666 NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST |
667 NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT |
668 NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32);
669 IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FETCH(1)), 0);
670 }
671
672 if (info->index_size && !info->has_user_indices)
673 nouveau_resource_unmap(nv04_resource(info->index.resource));
674 for (i = 0; i < nvc0->num_vtxbufs; ++i)
675 nouveau_resource_unmap(nv04_resource(nvc0->vtxbuf[i].buffer.resource));
676
677 NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_fallback_count, 1);
678 }
679
680 static inline void
copy_indices_u8(uint32_t * dst,const uint8_t * elts,uint32_t bias,unsigned n)681 copy_indices_u8(uint32_t *dst, const uint8_t *elts, uint32_t bias, unsigned n)
682 {
683 unsigned i;
684 for (i = 0; i < n; ++i)
685 dst[i] = elts[i] + bias;
686 }
687
688 static inline void
copy_indices_u16(uint32_t * dst,const uint16_t * elts,uint32_t bias,unsigned n)689 copy_indices_u16(uint32_t *dst, const uint16_t *elts, uint32_t bias, unsigned n)
690 {
691 unsigned i;
692 for (i = 0; i < n; ++i)
693 dst[i] = elts[i] + bias;
694 }
695
696 static inline void
copy_indices_u32(uint32_t * dst,const uint32_t * elts,uint32_t bias,unsigned n)697 copy_indices_u32(uint32_t *dst, const uint32_t *elts, uint32_t bias, unsigned n)
698 {
699 unsigned i;
700 for (i = 0; i < n; ++i)
701 dst[i] = elts[i] + bias;
702 }
703
704 static void
nvc0_push_upload_vertex_ids(struct push_context * ctx,struct nvc0_context * nvc0,const struct pipe_draw_info * info)705 nvc0_push_upload_vertex_ids(struct push_context *ctx,
706 struct nvc0_context *nvc0,
707 const struct pipe_draw_info *info)
708
709 {
710 struct nouveau_pushbuf *push = ctx->push;
711 struct nouveau_bo *bo;
712 uint64_t va;
713 uint32_t *data;
714 uint32_t format;
715 unsigned index_size = info->index_size;
716 unsigned i;
717 unsigned a = nvc0->vertex->num_elements;
718
719 if (!index_size || info->index_bias)
720 index_size = 4;
721 data = (uint32_t *)nouveau_scratch_get(&nvc0->base,
722 info->count * index_size, &va, &bo);
723
724 BCTX_REFN_bo(nvc0->bufctx_3d, 3D_VTX_TMP, NOUVEAU_BO_GART | NOUVEAU_BO_RD,
725 bo);
726 nouveau_pushbuf_validate(push);
727
728 if (info->index_size) {
729 if (!info->index_bias) {
730 memcpy(data, ctx->idxbuf, info->count * index_size);
731 } else {
732 switch (info->index_size) {
733 case 1:
734 copy_indices_u8(data, ctx->idxbuf, info->index_bias, info->count);
735 break;
736 case 2:
737 copy_indices_u16(data, ctx->idxbuf, info->index_bias, info->count);
738 break;
739 default:
740 copy_indices_u32(data, ctx->idxbuf, info->index_bias, info->count);
741 break;
742 }
743 }
744 } else {
745 for (i = 0; i < info->count; ++i)
746 data[i] = i + (info->start + info->index_bias);
747 }
748
749 format = (1 << NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__SHIFT) |
750 NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_UINT;
751
752 switch (index_size) {
753 case 1:
754 format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_8;
755 break;
756 case 2:
757 format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_16;
758 break;
759 default:
760 format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32;
761 break;
762 }
763
764 PUSH_SPACE(push, 12);
765
766 if (unlikely(nvc0->state.instance_elts & 2)) {
767 nvc0->state.instance_elts &= ~2;
768 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_PER_INSTANCE(1)), 0);
769 }
770
771 BEGIN_NVC0(push, NVC0_3D(VERTEX_ATTRIB_FORMAT(a)), 1);
772 PUSH_DATA (push, format);
773
774 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(1)), 3);
775 PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | index_size);
776 PUSH_DATAh(push, va);
777 PUSH_DATA (push, va);
778
779 if (nvc0->screen->eng3d->oclass < TU102_3D_CLASS)
780 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(1)), 2);
781 else
782 BEGIN_NVC0(push, SUBC_3D(TU102_3D_VERTEX_ARRAY_LIMIT_HIGH(1)), 2);
783 PUSH_DATAh(push, va + info->count * index_size - 1);
784 PUSH_DATA (push, va + info->count * index_size - 1);
785
786 #define NVC0_3D_VERTEX_ID_REPLACE_SOURCE_ATTR_X(a) \
787 (((0x80 + (a) * 0x10) / 4) << NVC0_3D_VERTEX_ID_REPLACE_SOURCE__SHIFT)
788
789 BEGIN_NVC0(push, NVC0_3D(VERTEX_ID_REPLACE), 1);
790 PUSH_DATA (push, NVC0_3D_VERTEX_ID_REPLACE_SOURCE_ATTR_X(a) | 1);
791 }
792