1 /**************************************************************************
2 *
3 * Copyright 2011 Marek Olšák <maraeo@gmail.com>
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 /**
29 * This module uploads user buffers and translates the vertex buffers which
30 * contain incompatible vertices (i.e. not supported by the driver/hardware)
31 * into compatible ones, based on the Gallium CAPs.
32 *
33 * It does not upload index buffers.
34 *
35 * The module heavily uses bitmasks to represent per-buffer and
36 * per-vertex-element flags to avoid looping over the list of buffers just
37 * to see if there's a non-zero stride, or user buffer, or unsupported format,
38 * etc.
39 *
40 * There are 3 categories of vertex elements, which are processed separately:
41 * - per-vertex attribs (stride != 0, instance_divisor == 0)
42 * - instanced attribs (stride != 0, instance_divisor > 0)
43 * - constant attribs (stride == 0)
44 *
45 * All needed uploads and translations are performed every draw command, but
46 * only the subset of vertices needed for that draw command is uploaded or
47 * translated. (the module never translates whole buffers)
48 *
49 *
50 * The module consists of two main parts:
51 *
52 *
53 * 1) Translate (u_vbuf_translate_begin/end)
54 *
55 * This is pretty much a vertex fetch fallback. It translates vertices from
56 * one vertex buffer to another in an unused vertex buffer slot. It does
57 * whatever is needed to make the vertices readable by the hardware (changes
58 * vertex formats and aligns offsets and strides). The translate module is
59 * used here.
60 *
61 * Each of the 3 categories is translated to a separate buffer.
62 * Only the [min_index, max_index] range is translated. For instanced attribs,
63 * the range is [start_instance, start_instance+instance_count]. For constant
64 * attribs, the range is [0, 1].
65 *
66 *
67 * 2) User buffer uploading (u_vbuf_upload_buffers)
68 *
69 * Only the [min_index, max_index] range is uploaded (just like Translate)
70 * with a single memcpy.
71 *
72 * This method works best for non-indexed draw operations or indexed draw
73 * operations where the [min_index, max_index] range is not being way bigger
74 * than the vertex count.
75 *
76 * If the range is too big (e.g. one triangle with indices {0, 1, 10000}),
77 * the per-vertex attribs are uploaded via the translate module, all packed
78 * into one vertex buffer, and the indexed draw call is turned into
79 * a non-indexed one in the process. This adds additional complexity
80 * to the translate part, but it prevents bad apps from bringing your frame
81 * rate down.
82 *
83 *
84 * If there is nothing to do, it forwards every command to the driver.
85 * The module also has its own CSO cache of vertex element states.
86 */
87
88 #include "util/u_vbuf.h"
89
90 #include "util/u_dump.h"
91 #include "util/format/u_format.h"
92 #include "util/u_helpers.h"
93 #include "util/u_inlines.h"
94 #include "util/u_memory.h"
95 #include "util/u_prim_restart.h"
96 #include "util/u_screen.h"
97 #include "util/u_upload_mgr.h"
98 #include "indices/u_primconvert.h"
99 #include "translate/translate.h"
100 #include "translate/translate_cache.h"
101 #include "cso_cache/cso_cache.h"
102 #include "cso_cache/cso_hash.h"
103
104 struct u_vbuf_elements {
105 unsigned count;
106 struct pipe_vertex_element ve[PIPE_MAX_ATTRIBS];
107
108 unsigned src_format_size[PIPE_MAX_ATTRIBS];
109
110 /* If (velem[i].src_format != native_format[i]), the vertex buffer
111 * referenced by the vertex element cannot be used for rendering and
112 * its vertex data must be translated to native_format[i]. */
113 enum pipe_format native_format[PIPE_MAX_ATTRIBS];
114 unsigned native_format_size[PIPE_MAX_ATTRIBS];
115 unsigned component_size[PIPE_MAX_ATTRIBS];
116 /* buffer-indexed */
117 unsigned strides[PIPE_MAX_ATTRIBS];
118
119 /* Which buffers are used by the vertex element state. */
120 uint32_t used_vb_mask;
121 /* This might mean two things:
122 * - src_format != native_format, as discussed above.
123 * - src_offset % 4 != 0 (if the caps don't allow such an offset). */
124 uint32_t incompatible_elem_mask; /* each bit describes a corresp. attrib */
125 /* Which buffer has at least one vertex element referencing it
126 * incompatible. */
127 uint32_t incompatible_vb_mask_any;
128 /* Which buffer has all vertex elements referencing it incompatible. */
129 uint32_t incompatible_vb_mask_all;
130 /* Which buffer has at least one vertex element referencing it
131 * compatible. */
132 uint32_t compatible_vb_mask_any;
133 uint32_t vb_align_mask[2]; //which buffers require 2/4 byte alignments
134 /* Which buffer has all vertex elements referencing it compatible. */
135 uint32_t compatible_vb_mask_all;
136
137 /* Which buffer has at least one vertex element referencing it
138 * non-instanced. */
139 uint32_t noninstance_vb_mask_any;
140
141 /* Which buffers are used by multiple vertex attribs. */
142 uint32_t interleaved_vb_mask;
143
144 /* Which buffer has a non-zero stride. */
145 uint32_t nonzero_stride_vb_mask; /* each bit describes a corresp. buffer */
146
147 /* Which buffer is incompatible (unaligned). */
148 uint32_t incompatible_vb_mask; /* each bit describes a corresp. buffer */
149
150 void *driver_cso;
151 };
152
153 enum {
154 VB_VERTEX = 0,
155 VB_INSTANCE = 1,
156 VB_CONST = 2,
157 VB_NUM = 3
158 };
159
160 struct u_vbuf {
161 struct u_vbuf_caps caps;
162 bool has_signed_vb_offset;
163
164 struct pipe_context *pipe;
165 struct translate_cache *translate_cache;
166 struct cso_cache cso_cache;
167
168 struct primconvert_context *pc;
169 bool flatshade_first;
170
171 /* This is what was set in set_vertex_buffers.
172 * May contain user buffers. */
173 struct pipe_vertex_buffer vertex_buffer[PIPE_MAX_ATTRIBS];
174 uint8_t num_vertex_buffers;
175 uint8_t num_real_vertex_buffers;
176 bool vertex_buffers_dirty;
177 uint32_t enabled_vb_mask;
178
179 uint32_t unaligned_vb_mask[2]; //16/32bit
180
181 /* Vertex buffers for the driver.
182 * There are usually no user buffers. */
183 struct pipe_vertex_buffer real_vertex_buffer[PIPE_MAX_ATTRIBS];
184
185 /* Vertex elements. */
186 struct u_vbuf_elements *ve, *ve_saved;
187
188 /* Vertex elements used for the translate fallback. */
189 struct cso_velems_state fallback_velems;
190 /* If non-NULL, this is a vertex element state used for the translate
191 * fallback and therefore used for rendering too. */
192 bool using_translate;
193 /* The vertex buffer slot index where translated vertices have been
194 * stored in. */
195 unsigned fallback_vbs[VB_NUM];
196 unsigned fallback_vbs_mask;
197
198 /* Which buffer is a user buffer. */
199 uint32_t user_vb_mask; /* each bit describes a corresp. buffer */
200 /* Which buffer is incompatible (unaligned). */
201 uint32_t incompatible_vb_mask; /* each bit describes a corresp. buffer */
202 /* Which buffers are allowed (supported by hardware). */
203 uint32_t allowed_vb_mask;
204 };
205
206 static void *
207 u_vbuf_create_vertex_elements(struct u_vbuf *mgr, unsigned count,
208 const struct pipe_vertex_element *attribs);
209 static void u_vbuf_delete_vertex_elements(void *ctx, void *state,
210 enum cso_cache_type type);
211
212 static const struct {
213 enum pipe_format from, to;
214 } vbuf_format_fallbacks[] = {
215 { PIPE_FORMAT_R32_FIXED, PIPE_FORMAT_R32_FLOAT },
216 { PIPE_FORMAT_R32G32_FIXED, PIPE_FORMAT_R32G32_FLOAT },
217 { PIPE_FORMAT_R32G32B32_FIXED, PIPE_FORMAT_R32G32B32_FLOAT },
218 { PIPE_FORMAT_R32G32B32A32_FIXED, PIPE_FORMAT_R32G32B32A32_FLOAT },
219 { PIPE_FORMAT_R16_FLOAT, PIPE_FORMAT_R32_FLOAT },
220 { PIPE_FORMAT_R16G16_FLOAT, PIPE_FORMAT_R32G32_FLOAT },
221 { PIPE_FORMAT_R16G16B16_FLOAT, PIPE_FORMAT_R32G32B32_FLOAT },
222 { PIPE_FORMAT_R16G16B16A16_FLOAT, PIPE_FORMAT_R32G32B32A32_FLOAT },
223 { PIPE_FORMAT_R64_FLOAT, PIPE_FORMAT_R32_FLOAT },
224 { PIPE_FORMAT_R64G64_FLOAT, PIPE_FORMAT_R32G32_FLOAT },
225 { PIPE_FORMAT_R64G64B64_FLOAT, PIPE_FORMAT_R32G32B32_FLOAT },
226 { PIPE_FORMAT_R64G64B64A64_FLOAT, PIPE_FORMAT_R32G32B32A32_FLOAT },
227 { PIPE_FORMAT_R32_UNORM, PIPE_FORMAT_R32_FLOAT },
228 { PIPE_FORMAT_R32G32_UNORM, PIPE_FORMAT_R32G32_FLOAT },
229 { PIPE_FORMAT_R32G32B32_UNORM, PIPE_FORMAT_R32G32B32_FLOAT },
230 { PIPE_FORMAT_R32G32B32A32_UNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
231 { PIPE_FORMAT_R32_SNORM, PIPE_FORMAT_R32_FLOAT },
232 { PIPE_FORMAT_R32G32_SNORM, PIPE_FORMAT_R32G32_FLOAT },
233 { PIPE_FORMAT_R32G32B32_SNORM, PIPE_FORMAT_R32G32B32_FLOAT },
234 { PIPE_FORMAT_R32G32B32A32_SNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
235 { PIPE_FORMAT_R32_USCALED, PIPE_FORMAT_R32_FLOAT },
236 { PIPE_FORMAT_R32G32_USCALED, PIPE_FORMAT_R32G32_FLOAT },
237 { PIPE_FORMAT_R32G32B32_USCALED, PIPE_FORMAT_R32G32B32_FLOAT },
238 { PIPE_FORMAT_R32G32B32A32_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
239 { PIPE_FORMAT_R32_SSCALED, PIPE_FORMAT_R32_FLOAT },
240 { PIPE_FORMAT_R32G32_SSCALED, PIPE_FORMAT_R32G32_FLOAT },
241 { PIPE_FORMAT_R32G32B32_SSCALED, PIPE_FORMAT_R32G32B32_FLOAT },
242 { PIPE_FORMAT_R32G32B32A32_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
243 { PIPE_FORMAT_R16_UNORM, PIPE_FORMAT_R32_FLOAT },
244 { PIPE_FORMAT_R16G16_UNORM, PIPE_FORMAT_R32G32_FLOAT },
245 { PIPE_FORMAT_R16G16B16_UNORM, PIPE_FORMAT_R32G32B32_FLOAT },
246 { PIPE_FORMAT_R16G16B16A16_UNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
247 { PIPE_FORMAT_R16_SNORM, PIPE_FORMAT_R32_FLOAT },
248 { PIPE_FORMAT_R16G16_SNORM, PIPE_FORMAT_R32G32_FLOAT },
249 { PIPE_FORMAT_R16G16B16_SNORM, PIPE_FORMAT_R32G32B32_FLOAT },
250 { PIPE_FORMAT_R16G16B16_SINT, PIPE_FORMAT_R32G32B32_SINT },
251 { PIPE_FORMAT_R16G16B16_UINT, PIPE_FORMAT_R32G32B32_UINT },
252 { PIPE_FORMAT_R16G16B16A16_SNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
253 { PIPE_FORMAT_R16_USCALED, PIPE_FORMAT_R32_FLOAT },
254 { PIPE_FORMAT_R16G16_USCALED, PIPE_FORMAT_R32G32_FLOAT },
255 { PIPE_FORMAT_R16G16B16_USCALED, PIPE_FORMAT_R32G32B32_FLOAT },
256 { PIPE_FORMAT_R16G16B16A16_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
257 { PIPE_FORMAT_R16_SSCALED, PIPE_FORMAT_R32_FLOAT },
258 { PIPE_FORMAT_R16G16_SSCALED, PIPE_FORMAT_R32G32_FLOAT },
259 { PIPE_FORMAT_R16G16B16_SSCALED, PIPE_FORMAT_R32G32B32_FLOAT },
260 { PIPE_FORMAT_R16G16B16A16_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
261 { PIPE_FORMAT_R8_UNORM, PIPE_FORMAT_R32_FLOAT },
262 { PIPE_FORMAT_R8G8_UNORM, PIPE_FORMAT_R32G32_FLOAT },
263 { PIPE_FORMAT_R8G8B8_UNORM, PIPE_FORMAT_R32G32B32_FLOAT },
264 { PIPE_FORMAT_R8G8B8A8_UNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
265 { PIPE_FORMAT_R8_SNORM, PIPE_FORMAT_R32_FLOAT },
266 { PIPE_FORMAT_R8G8_SNORM, PIPE_FORMAT_R32G32_FLOAT },
267 { PIPE_FORMAT_R8G8B8_SNORM, PIPE_FORMAT_R32G32B32_FLOAT },
268 { PIPE_FORMAT_R8G8B8A8_SNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
269 { PIPE_FORMAT_R8_USCALED, PIPE_FORMAT_R32_FLOAT },
270 { PIPE_FORMAT_R8G8_USCALED, PIPE_FORMAT_R32G32_FLOAT },
271 { PIPE_FORMAT_R8G8B8_USCALED, PIPE_FORMAT_R32G32B32_FLOAT },
272 { PIPE_FORMAT_R8G8B8A8_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
273 { PIPE_FORMAT_R8_SSCALED, PIPE_FORMAT_R32_FLOAT },
274 { PIPE_FORMAT_R8G8_SSCALED, PIPE_FORMAT_R32G32_FLOAT },
275 { PIPE_FORMAT_R8G8B8_SSCALED, PIPE_FORMAT_R32G32B32_FLOAT },
276 { PIPE_FORMAT_R8G8B8A8_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
277 };
278
u_vbuf_get_caps(struct pipe_screen * screen,struct u_vbuf_caps * caps,bool needs64b)279 void u_vbuf_get_caps(struct pipe_screen *screen, struct u_vbuf_caps *caps,
280 bool needs64b)
281 {
282 unsigned i;
283
284 memset(caps, 0, sizeof(*caps));
285
286 /* I'd rather have a bitfield of which formats are supported and a static
287 * table of the translations indexed by format, but since we don't have C99
288 * we can't easily make a sparsely-populated table indexed by format. So,
289 * we construct the sparse table here.
290 */
291 for (i = 0; i < PIPE_FORMAT_COUNT; i++)
292 caps->format_translation[i] = i;
293
294 for (i = 0; i < ARRAY_SIZE(vbuf_format_fallbacks); i++) {
295 enum pipe_format format = vbuf_format_fallbacks[i].from;
296 unsigned comp_bits = util_format_get_component_bits(format, 0, 0);
297
298 if ((comp_bits > 32) && !needs64b)
299 continue;
300
301 if (!screen->is_format_supported(screen, format, PIPE_BUFFER, 0, 0,
302 PIPE_BIND_VERTEX_BUFFER)) {
303 caps->format_translation[format] = vbuf_format_fallbacks[i].to;
304 caps->fallback_always = true;
305 }
306 }
307
308 /* by default, all of these are supported */
309 caps->attrib_4byte_unaligned = 1;
310 caps->attrib_element_unaligned = 1;
311
312 /* pipe cap removes capabilities */
313 switch (screen->caps.vertex_input_alignment) {
314 case PIPE_VERTEX_INPUT_ALIGNMENT_4BYTE:
315 caps->attrib_4byte_unaligned = 0;
316 break;
317 case PIPE_VERTEX_INPUT_ALIGNMENT_ELEMENT:
318 caps->attrib_element_unaligned = 0;
319 break;
320 default:
321 break;
322 }
323
324 caps->user_vertex_buffers =
325 screen->caps.user_vertex_buffers;
326 caps->max_vertex_buffers =
327 screen->caps.max_vertex_buffers;
328
329 if (screen->caps.primitive_restart ||
330 screen->caps.primitive_restart_fixed_index) {
331 caps->rewrite_restart_index = screen->caps.emulate_nonfixed_primitive_restart;
332 caps->supported_restart_modes = screen->caps.supported_prim_modes_with_restart;
333 caps->supported_restart_modes |= BITFIELD_BIT(MESA_PRIM_PATCHES);
334 if (caps->supported_restart_modes != BITFIELD_MASK(MESA_PRIM_COUNT))
335 caps->fallback_always = true;
336 caps->fallback_always |= caps->rewrite_restart_index;
337 }
338 caps->supported_prim_modes = screen->caps.supported_prim_modes;
339 if (caps->supported_prim_modes != BITFIELD_MASK(MESA_PRIM_COUNT))
340 caps->fallback_always = true;
341
342 if (!screen->is_format_supported(screen, PIPE_FORMAT_R8_UINT, PIPE_BUFFER, 0, 0, PIPE_BIND_INDEX_BUFFER))
343 caps->fallback_always = caps->rewrite_ubyte_ibs = true;
344
345 /* OpenGL 2.0 requires a minimum of 16 vertex buffers */
346 if (caps->max_vertex_buffers < 16)
347 caps->fallback_always = true;
348
349 if (!caps->attrib_4byte_unaligned || !caps->attrib_element_unaligned)
350 caps->fallback_always = true;
351
352 if (!caps->fallback_always && !caps->user_vertex_buffers)
353 caps->fallback_only_for_user_vbuffers = true;
354 }
355
356 struct u_vbuf *
u_vbuf_create(struct pipe_context * pipe,struct u_vbuf_caps * caps)357 u_vbuf_create(struct pipe_context *pipe, struct u_vbuf_caps *caps)
358 {
359 struct u_vbuf *mgr = CALLOC_STRUCT(u_vbuf);
360
361 mgr->caps = *caps;
362 mgr->pipe = pipe;
363 if (caps->rewrite_ubyte_ibs || caps->rewrite_restart_index ||
364 /* require all but patches */
365 ((caps->supported_prim_modes & caps->supported_restart_modes & BITFIELD_MASK(MESA_PRIM_COUNT))) !=
366 BITFIELD_MASK(MESA_PRIM_COUNT)) {
367 struct primconvert_config cfg;
368 cfg.fixed_prim_restart = caps->rewrite_restart_index;
369 cfg.primtypes_mask = caps->supported_prim_modes;
370 cfg.restart_primtypes_mask = caps->supported_restart_modes;
371 mgr->pc = util_primconvert_create_config(pipe, &cfg);
372 }
373 mgr->translate_cache = translate_cache_create();
374 memset(mgr->fallback_vbs, ~0, sizeof(mgr->fallback_vbs));
375 mgr->allowed_vb_mask = u_bit_consecutive(0, mgr->caps.max_vertex_buffers);
376
377 mgr->has_signed_vb_offset =
378 pipe->screen->caps.signed_vertex_buffer_offset;
379
380 cso_cache_init(&mgr->cso_cache, pipe);
381 cso_cache_set_delete_cso_callback(&mgr->cso_cache,
382 u_vbuf_delete_vertex_elements, pipe);
383
384 return mgr;
385 }
386
387 /* u_vbuf uses its own caching for vertex elements, because it needs to keep
388 * its own preprocessed state per vertex element CSO. */
389 static struct u_vbuf_elements *
u_vbuf_set_vertex_elements_internal(struct u_vbuf * mgr,const struct cso_velems_state * velems)390 u_vbuf_set_vertex_elements_internal(struct u_vbuf *mgr,
391 const struct cso_velems_state *velems)
392 {
393 struct pipe_context *pipe = mgr->pipe;
394 unsigned key_size, hash_key;
395 struct cso_hash_iter iter;
396 struct u_vbuf_elements *ve;
397
398 /* need to include the count into the stored state data too. */
399 key_size = sizeof(struct pipe_vertex_element) * velems->count +
400 sizeof(unsigned);
401 hash_key = cso_construct_key(velems, key_size);
402 iter = cso_find_state_template(&mgr->cso_cache, hash_key, CSO_VELEMENTS,
403 velems, key_size);
404
405 if (cso_hash_iter_is_null(iter)) {
406 struct cso_velements *cso = MALLOC_STRUCT(cso_velements);
407 memcpy(&cso->state, velems, key_size);
408 cso->data = u_vbuf_create_vertex_elements(mgr, velems->count,
409 velems->velems);
410
411 iter = cso_insert_state(&mgr->cso_cache, hash_key, CSO_VELEMENTS, cso);
412 ve = cso->data;
413 } else {
414 ve = ((struct cso_velements *)cso_hash_iter_data(iter))->data;
415 }
416
417 assert(ve);
418
419 if (ve != mgr->ve)
420 pipe->bind_vertex_elements_state(pipe, ve->driver_cso);
421
422 return ve;
423 }
424
u_vbuf_set_vertex_elements(struct u_vbuf * mgr,const struct cso_velems_state * velems)425 void u_vbuf_set_vertex_elements(struct u_vbuf *mgr,
426 const struct cso_velems_state *velems)
427 {
428 mgr->ve = u_vbuf_set_vertex_elements_internal(mgr, velems);
429 }
430
u_vbuf_set_flatshade_first(struct u_vbuf * mgr,bool flatshade_first)431 void u_vbuf_set_flatshade_first(struct u_vbuf *mgr, bool flatshade_first)
432 {
433 mgr->flatshade_first = flatshade_first;
434 }
435
u_vbuf_unset_vertex_elements(struct u_vbuf * mgr)436 void u_vbuf_unset_vertex_elements(struct u_vbuf *mgr)
437 {
438 mgr->ve = NULL;
439 }
440
u_vbuf_destroy(struct u_vbuf * mgr)441 void u_vbuf_destroy(struct u_vbuf *mgr)
442 {
443 unsigned i;
444
445 mgr->pipe->set_vertex_buffers(mgr->pipe, 0, NULL);
446
447 for (i = 0; i < PIPE_MAX_ATTRIBS; i++)
448 pipe_vertex_buffer_unreference(&mgr->vertex_buffer[i]);
449 for (i = 0; i < PIPE_MAX_ATTRIBS; i++)
450 pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[i]);
451
452 if (mgr->pc)
453 util_primconvert_destroy(mgr->pc);
454
455 translate_cache_destroy(mgr->translate_cache);
456 cso_cache_delete(&mgr->cso_cache);
457 FREE(mgr);
458 }
459
460 static enum pipe_error
u_vbuf_translate_buffers(struct u_vbuf * mgr,struct translate_key * key,const struct pipe_draw_info * info,const struct pipe_draw_start_count_bias * draw,unsigned vb_mask,unsigned out_vb,int start_vertex,unsigned num_vertices,int min_index,bool unroll_indices)461 u_vbuf_translate_buffers(struct u_vbuf *mgr, struct translate_key *key,
462 const struct pipe_draw_info *info,
463 const struct pipe_draw_start_count_bias *draw,
464 unsigned vb_mask, unsigned out_vb,
465 int start_vertex, unsigned num_vertices,
466 int min_index, bool unroll_indices)
467 {
468 struct translate *tr;
469 struct pipe_transfer *vb_transfer[PIPE_MAX_ATTRIBS] = {0};
470 struct pipe_resource *out_buffer = NULL;
471 uint8_t *out_map;
472 unsigned out_offset, mask;
473
474 /* Get a translate object. */
475 tr = translate_cache_find(mgr->translate_cache, key);
476
477 /* Map buffers we want to translate. */
478 mask = vb_mask;
479 while (mask) {
480 struct pipe_vertex_buffer *vb;
481 unsigned offset;
482 uint8_t *map;
483 unsigned i = u_bit_scan(&mask);
484 unsigned stride = mgr->ve->strides[i];
485
486 vb = &mgr->vertex_buffer[i];
487 offset = vb->buffer_offset + stride * start_vertex;
488
489 if (vb->is_user_buffer) {
490 map = (uint8_t*)vb->buffer.user + offset;
491 } else {
492 unsigned size = stride ? num_vertices * stride
493 : sizeof(double)*4;
494
495 if (!vb->buffer.resource) {
496 static uint64_t dummy_buf[4] = { 0 };
497 tr->set_buffer(tr, i, dummy_buf, 0, 0);
498 continue;
499 }
500
501 if (stride) {
502 /* the stride cannot be used to calculate the map size of the buffer,
503 * as it only determines the bytes between elements, not the size of elements
504 * themselves, meaning that if stride < element_size, the mapped size will
505 * be too small and conversion will overrun the map buffer
506 *
507 * instead, add the size of the largest possible attribute to the final attribute's offset
508 * in order to ensure the map is large enough
509 */
510 unsigned last_offset = size - stride;
511 size = MAX2(size, last_offset + sizeof(double)*4);
512 }
513
514 if (offset + size > vb->buffer.resource->width0) {
515 /* Don't try to map past end of buffer. This often happens when
516 * we're translating an attribute that's at offset > 0 from the
517 * start of the vertex. If we'd subtract attrib's offset from
518 * the size, this probably wouldn't happen.
519 */
520 size = vb->buffer.resource->width0 - offset;
521
522 /* Also adjust num_vertices. A common user error is to call
523 * glDrawRangeElements() with incorrect 'end' argument. The 'end
524 * value should be the max index value, but people often
525 * accidentally add one to this value. This adjustment avoids
526 * crashing (by reading past the end of a hardware buffer mapping)
527 * when people do that.
528 */
529 num_vertices = (size + stride - 1) / stride;
530 }
531
532 map = pipe_buffer_map_range(mgr->pipe, vb->buffer.resource, offset, size,
533 PIPE_MAP_READ, &vb_transfer[i]);
534 }
535
536 /* Subtract min_index so that indexing with the index buffer works. */
537 if (unroll_indices) {
538 map -= (ptrdiff_t)stride * min_index;
539 }
540
541 tr->set_buffer(tr, i, map, stride, info->max_index);
542 }
543
544 /* Translate. */
545 if (unroll_indices) {
546 struct pipe_transfer *transfer = NULL;
547 const unsigned offset = draw->start * info->index_size;
548 uint8_t *map;
549
550 /* Create and map the output buffer. */
551 u_upload_alloc(mgr->pipe->stream_uploader, 0,
552 key->output_stride * draw->count, 4,
553 &out_offset, &out_buffer,
554 (void**)&out_map);
555 if (!out_buffer)
556 return PIPE_ERROR_OUT_OF_MEMORY;
557
558 if (info->has_user_indices) {
559 map = (uint8_t*)info->index.user + offset;
560 } else {
561 map = pipe_buffer_map_range(mgr->pipe, info->index.resource, offset,
562 draw->count * info->index_size,
563 PIPE_MAP_READ, &transfer);
564 }
565
566 switch (info->index_size) {
567 case 4:
568 tr->run_elts(tr, (unsigned*)map, draw->count, 0, 0, out_map);
569 break;
570 case 2:
571 tr->run_elts16(tr, (uint16_t*)map, draw->count, 0, 0, out_map);
572 break;
573 case 1:
574 tr->run_elts8(tr, map, draw->count, 0, 0, out_map);
575 break;
576 }
577
578 if (transfer) {
579 pipe_buffer_unmap(mgr->pipe, transfer);
580 }
581 } else {
582 /* Create and map the output buffer. */
583 u_upload_alloc(mgr->pipe->stream_uploader,
584 mgr->has_signed_vb_offset ?
585 0 : key->output_stride * start_vertex,
586 key->output_stride * num_vertices, 4,
587 &out_offset, &out_buffer,
588 (void**)&out_map);
589 if (!out_buffer)
590 return PIPE_ERROR_OUT_OF_MEMORY;
591
592 out_offset -= key->output_stride * start_vertex;
593
594 tr->run(tr, 0, num_vertices, 0, 0, out_map);
595 }
596
597 /* Unmap all buffers. */
598 mask = vb_mask;
599 while (mask) {
600 unsigned i = u_bit_scan(&mask);
601
602 if (vb_transfer[i]) {
603 pipe_buffer_unmap(mgr->pipe, vb_transfer[i]);
604 }
605 }
606
607 /* Setup the new vertex buffer. */
608 mgr->real_vertex_buffer[out_vb].buffer_offset = out_offset;
609
610 /* Move the buffer reference. */
611 pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[out_vb]);
612 mgr->real_vertex_buffer[out_vb].buffer.resource = out_buffer;
613 mgr->real_vertex_buffer[out_vb].is_user_buffer = false;
614
615 return PIPE_OK;
616 }
617
618 static bool
u_vbuf_translate_find_free_vb_slots(struct u_vbuf * mgr,unsigned mask[VB_NUM])619 u_vbuf_translate_find_free_vb_slots(struct u_vbuf *mgr,
620 unsigned mask[VB_NUM])
621 {
622 unsigned type;
623 unsigned fallback_vbs[VB_NUM];
624 /* Set the bit for each buffer which is incompatible, or isn't set. */
625 uint32_t unused_vb_mask =
626 mgr->ve->incompatible_vb_mask_all | mgr->incompatible_vb_mask | mgr->ve->incompatible_vb_mask |
627 ~mgr->enabled_vb_mask;
628 uint32_t unused_vb_mask_orig;
629 bool insufficient_buffers = false;
630
631 /* No vertex buffers available at all */
632 if (!unused_vb_mask)
633 return false;
634
635 memset(fallback_vbs, ~0, sizeof(fallback_vbs));
636 mgr->fallback_vbs_mask = 0;
637
638 /* Find free slots for each type if needed. */
639 unused_vb_mask_orig = unused_vb_mask;
640 for (type = 0; type < VB_NUM; type++) {
641 if (mask[type]) {
642 uint32_t index;
643
644 if (!unused_vb_mask) {
645 insufficient_buffers = true;
646 break;
647 }
648
649 index = ffs(unused_vb_mask) - 1;
650 fallback_vbs[type] = index;
651 mgr->fallback_vbs_mask |= 1 << index;
652 unused_vb_mask &= ~(1 << index);
653 /*printf("found slot=%i for type=%i\n", index, type);*/
654 }
655 }
656
657 if (insufficient_buffers) {
658 /* not enough vbs for all types supported by the hardware, they will have to share one
659 * buffer */
660 uint32_t index = ffs(unused_vb_mask_orig) - 1;
661 /* When sharing one vertex buffer use per-vertex frequency for everything. */
662 fallback_vbs[VB_VERTEX] = index;
663 mgr->fallback_vbs_mask = 1 << index;
664 mask[VB_VERTEX] = mask[VB_VERTEX] | mask[VB_CONST] | mask[VB_INSTANCE];
665 mask[VB_CONST] = 0;
666 mask[VB_INSTANCE] = 0;
667 }
668
669 for (type = 0; type < VB_NUM; type++) {
670 if (mask[type]) {
671 mgr->num_real_vertex_buffers =
672 MAX2(mgr->num_real_vertex_buffers, fallback_vbs[type] + 1);
673 mgr->vertex_buffers_dirty = true;
674 }
675 }
676
677 memcpy(mgr->fallback_vbs, fallback_vbs, sizeof(fallback_vbs));
678 return true;
679 }
680
681 static bool
u_vbuf_translate_begin(struct u_vbuf * mgr,const struct pipe_draw_info * info,const struct pipe_draw_start_count_bias * draw,int start_vertex,unsigned num_vertices,int min_index,bool unroll_indices,uint32_t misaligned)682 u_vbuf_translate_begin(struct u_vbuf *mgr,
683 const struct pipe_draw_info *info,
684 const struct pipe_draw_start_count_bias *draw,
685 int start_vertex, unsigned num_vertices,
686 int min_index, bool unroll_indices,
687 uint32_t misaligned)
688 {
689 unsigned mask[VB_NUM] = {0};
690 struct translate_key key[VB_NUM];
691 unsigned elem_index[VB_NUM][PIPE_MAX_ATTRIBS]; /* ... into key.elements */
692 unsigned i, type;
693 const unsigned incompatible_vb_mask = (misaligned | mgr->incompatible_vb_mask | mgr->ve->incompatible_vb_mask) &
694 mgr->ve->used_vb_mask;
695
696 const int start[VB_NUM] = {
697 start_vertex, /* VERTEX */
698 info->start_instance, /* INSTANCE */
699 0 /* CONST */
700 };
701
702 const unsigned num[VB_NUM] = {
703 num_vertices, /* VERTEX */
704 info->instance_count, /* INSTANCE */
705 1 /* CONST */
706 };
707
708 memset(key, 0, sizeof(key));
709 memset(elem_index, ~0, sizeof(elem_index));
710
711 /* See if there are vertex attribs of each type to translate and
712 * which ones. */
713 for (i = 0; i < mgr->ve->count; i++) {
714 unsigned vb_index = mgr->ve->ve[i].vertex_buffer_index;
715
716 if (!mgr->ve->ve[i].src_stride) {
717 if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&
718 !(incompatible_vb_mask & (1 << vb_index))) {
719 continue;
720 }
721 mask[VB_CONST] |= 1 << vb_index;
722 } else if (mgr->ve->ve[i].instance_divisor) {
723 if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&
724 !(incompatible_vb_mask & (1 << vb_index))) {
725 continue;
726 }
727 mask[VB_INSTANCE] |= 1 << vb_index;
728 } else {
729 if (!unroll_indices &&
730 !(mgr->ve->incompatible_elem_mask & (1 << i)) &&
731 !(incompatible_vb_mask & (1 << vb_index))) {
732 continue;
733 }
734 mask[VB_VERTEX] |= 1 << vb_index;
735 }
736 }
737
738 assert(mask[VB_VERTEX] || mask[VB_INSTANCE] || mask[VB_CONST]);
739
740 /* Find free vertex buffer slots. */
741 if (!u_vbuf_translate_find_free_vb_slots(mgr, mask)) {
742 return false;
743 }
744
745 unsigned min_alignment[VB_NUM] = {0};
746 /* Initialize the translate keys. */
747 for (i = 0; i < mgr->ve->count; i++) {
748 struct translate_key *k;
749 struct translate_element *te;
750 enum pipe_format output_format = mgr->ve->native_format[i];
751 unsigned bit, vb_index = mgr->ve->ve[i].vertex_buffer_index;
752 bit = 1 << vb_index;
753
754 if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&
755 !(incompatible_vb_mask & (1 << vb_index)) &&
756 (!unroll_indices || !(mask[VB_VERTEX] & bit))) {
757 continue;
758 }
759
760 /* Set type to what we will translate.
761 * Whether vertex, instance, or constant attribs. */
762 for (type = 0; type < VB_NUM; type++) {
763 if (mask[type] & bit) {
764 break;
765 }
766 }
767 assert(type < VB_NUM);
768 if (mgr->ve->ve[i].src_format != output_format)
769 assert(translate_is_output_format_supported(output_format));
770 /*printf("velem=%i type=%i\n", i, type);*/
771
772 /* Add the vertex element. */
773 k = &key[type];
774 elem_index[type][i] = k->nr_elements;
775
776 te = &k->element[k->nr_elements];
777 te->type = TRANSLATE_ELEMENT_NORMAL;
778 te->instance_divisor = 0;
779 te->input_buffer = vb_index;
780 te->input_format = mgr->ve->ve[i].src_format;
781 te->input_offset = mgr->ve->ve[i].src_offset;
782 te->output_format = output_format;
783 te->output_offset = k->output_stride;
784 unsigned adjustment = 0;
785 if (!mgr->caps.attrib_element_unaligned &&
786 te->output_offset % mgr->ve->component_size[i] != 0) {
787 unsigned aligned = align(te->output_offset, mgr->ve->component_size[i]);
788 adjustment = aligned - te->output_offset;
789 te->output_offset = aligned;
790 }
791
792 k->output_stride += mgr->ve->native_format_size[i] + adjustment;
793 k->nr_elements++;
794 min_alignment[type] = MAX2(min_alignment[type], mgr->ve->component_size[i]);
795 }
796
797 /* Translate buffers. */
798 for (type = 0; type < VB_NUM; type++) {
799 if (key[type].nr_elements) {
800 enum pipe_error err;
801 if (!mgr->caps.attrib_element_unaligned)
802 key[type].output_stride = align(key[type].output_stride, min_alignment[type]);
803 err = u_vbuf_translate_buffers(mgr, &key[type], info, draw,
804 mask[type], mgr->fallback_vbs[type],
805 start[type], num[type], min_index,
806 unroll_indices && type == VB_VERTEX);
807 if (err != PIPE_OK)
808 return false;
809 }
810 }
811
812 /* Setup new vertex elements. */
813 for (i = 0; i < mgr->ve->count; i++) {
814 for (type = 0; type < VB_NUM; type++) {
815 if (elem_index[type][i] < key[type].nr_elements) {
816 struct translate_element *te = &key[type].element[elem_index[type][i]];
817 mgr->fallback_velems.velems[i].instance_divisor = mgr->ve->ve[i].instance_divisor;
818 mgr->fallback_velems.velems[i].src_format = te->output_format;
819 mgr->fallback_velems.velems[i].src_offset = te->output_offset;
820 mgr->fallback_velems.velems[i].vertex_buffer_index = mgr->fallback_vbs[type];
821
822 /* Fixup the stride for constant attribs. */
823 if (type == VB_CONST)
824 mgr->fallback_velems.velems[i].src_stride = 0;
825 else
826 mgr->fallback_velems.velems[i].src_stride = key[type].output_stride;
827
828 /* elem_index[type][i] can only be set for one type. */
829 assert(type > VB_INSTANCE || elem_index[type+1][i] == ~0u);
830 assert(type > VB_VERTEX || elem_index[type+2][i] == ~0u);
831 break;
832 }
833 }
834 /* No translating, just copy the original vertex element over. */
835 if (type == VB_NUM) {
836 memcpy(&mgr->fallback_velems.velems[i], &mgr->ve->ve[i],
837 sizeof(struct pipe_vertex_element));
838 }
839 }
840
841 mgr->fallback_velems.count = mgr->ve->count;
842
843 u_vbuf_set_vertex_elements_internal(mgr, &mgr->fallback_velems);
844 mgr->using_translate = true;
845 return true;
846 }
847
u_vbuf_translate_end(struct u_vbuf * mgr)848 static void u_vbuf_translate_end(struct u_vbuf *mgr)
849 {
850 unsigned i;
851
852 /* Restore vertex elements. */
853 mgr->pipe->bind_vertex_elements_state(mgr->pipe, mgr->ve->driver_cso);
854 mgr->using_translate = false;
855
856 /* Unreference the now-unused VBOs. */
857 for (i = 0; i < VB_NUM; i++) {
858 unsigned vb = mgr->fallback_vbs[i];
859 if (vb != ~0u) {
860 pipe_resource_reference(&mgr->real_vertex_buffer[vb].buffer.resource, NULL);
861 mgr->fallback_vbs[i] = ~0;
862 }
863 }
864 /* This will cause the fallback buffers above num_vertex_buffers to be
865 * unbound.
866 */
867 mgr->num_real_vertex_buffers = mgr->num_vertex_buffers;
868 mgr->vertex_buffers_dirty = true;
869 mgr->fallback_vbs_mask = 0;
870 }
871
872 static void *
u_vbuf_create_vertex_elements(struct u_vbuf * mgr,unsigned count,const struct pipe_vertex_element * attribs)873 u_vbuf_create_vertex_elements(struct u_vbuf *mgr, unsigned count,
874 const struct pipe_vertex_element *attribs)
875 {
876 struct pipe_vertex_element tmp[PIPE_MAX_ATTRIBS];
877 util_lower_uint64_vertex_elements(&attribs, &count, tmp);
878
879 struct pipe_context *pipe = mgr->pipe;
880 unsigned i;
881 struct pipe_vertex_element driver_attribs[PIPE_MAX_ATTRIBS];
882 struct u_vbuf_elements *ve = CALLOC_STRUCT(u_vbuf_elements);
883 uint32_t used_buffers = 0;
884
885 ve->count = count;
886
887 memcpy(ve->ve, attribs, sizeof(struct pipe_vertex_element) * count);
888 memcpy(driver_attribs, attribs, sizeof(struct pipe_vertex_element) * count);
889
890 /* Set the best native format in case the original format is not
891 * supported. */
892 for (i = 0; i < count; i++) {
893 enum pipe_format format = ve->ve[i].src_format;
894 unsigned vb_index_bit = 1 << ve->ve[i].vertex_buffer_index;
895
896 ve->src_format_size[i] = util_format_get_blocksize(format);
897
898 if (used_buffers & vb_index_bit)
899 ve->interleaved_vb_mask |= vb_index_bit;
900
901 used_buffers |= vb_index_bit;
902
903 if (!ve->ve[i].instance_divisor) {
904 ve->noninstance_vb_mask_any |= vb_index_bit;
905 }
906
907 format = mgr->caps.format_translation[format];
908
909 driver_attribs[i].src_format = format;
910 ve->native_format[i] = format;
911 ve->native_format_size[i] =
912 util_format_get_blocksize(ve->native_format[i]);
913
914 const struct util_format_description *desc = util_format_description(format);
915 bool is_packed = false;
916 for (unsigned c = 0; c < desc->nr_channels; c++)
917 is_packed |= desc->channel[c].size != desc->channel[0].size || desc->channel[c].size % 8 != 0;
918 unsigned component_size = is_packed ?
919 ve->native_format_size[i] : (ve->native_format_size[i] / desc->nr_channels);
920 ve->component_size[i] = component_size;
921
922 if (ve->ve[i].src_format != format ||
923 (!mgr->caps.attrib_4byte_unaligned &&
924 ve->ve[i].src_offset % 4 != 0) ||
925 (!mgr->caps.attrib_element_unaligned &&
926 ve->ve[i].src_offset % component_size != 0)) {
927 ve->incompatible_elem_mask |= 1 << i;
928 ve->incompatible_vb_mask_any |= vb_index_bit;
929 } else {
930 ve->compatible_vb_mask_any |= vb_index_bit;
931 if (component_size == 2) {
932 ve->vb_align_mask[0] |= vb_index_bit;
933 }
934 else if (component_size == 4) {
935 ve->vb_align_mask[1] |= vb_index_bit;
936 }
937 }
938 ve->strides[ve->ve[i].vertex_buffer_index] = ve->ve[i].src_stride;
939 if (ve->ve[i].src_stride) {
940 ve->nonzero_stride_vb_mask |= 1 << ve->ve[i].vertex_buffer_index;
941 }
942 if ((!mgr->caps.attrib_4byte_unaligned && ve->ve[i].src_stride % 4 != 0) ||
943 (!mgr->caps.attrib_element_unaligned && ve->ve[i].src_stride % component_size != 0))
944 ve->incompatible_vb_mask |= vb_index_bit;
945 }
946
947 if (used_buffers & ~mgr->allowed_vb_mask) {
948 /* More vertex buffers are used than the hardware supports. In
949 * principle, we only need to make sure that less vertex buffers are
950 * used, and mark some of the latter vertex buffers as incompatible.
951 * For now, mark all vertex buffers as incompatible.
952 */
953 ve->incompatible_vb_mask_any = used_buffers;
954 ve->compatible_vb_mask_any = 0;
955 ve->incompatible_elem_mask = u_bit_consecutive(0, count);
956 }
957
958 ve->used_vb_mask = used_buffers;
959 ve->compatible_vb_mask_all = ~ve->incompatible_vb_mask_any & used_buffers;
960 ve->incompatible_vb_mask_all = ~ve->compatible_vb_mask_any & used_buffers;
961
962 /* Align the formats and offsets to the size of DWORD if needed. */
963 if (!mgr->caps.attrib_4byte_unaligned) {
964 for (i = 0; i < count; i++) {
965 ve->native_format_size[i] = align(ve->native_format_size[i], 4);
966 driver_attribs[i].src_offset = align(ve->ve[i].src_offset, 4);
967 }
968 }
969
970 /* Only create driver CSO if no incompatible elements */
971 if (!ve->incompatible_elem_mask) {
972 ve->driver_cso =
973 pipe->create_vertex_elements_state(pipe, count, driver_attribs);
974 }
975
976 return ve;
977 }
978
u_vbuf_delete_vertex_elements(void * ctx,void * state,enum cso_cache_type type)979 static void u_vbuf_delete_vertex_elements(void *ctx, void *state,
980 enum cso_cache_type type)
981 {
982 struct pipe_context *pipe = (struct pipe_context*)ctx;
983 struct cso_velements *cso = (struct cso_velements*)state;
984 struct u_vbuf_elements *ve = (struct u_vbuf_elements*)cso->data;
985
986 if (ve->driver_cso)
987 pipe->delete_vertex_elements_state(pipe, ve->driver_cso);
988 FREE(ve);
989 FREE(cso);
990 }
991
u_vbuf_set_vertex_buffers(struct u_vbuf * mgr,unsigned count,bool take_ownership,const struct pipe_vertex_buffer * bufs)992 void u_vbuf_set_vertex_buffers(struct u_vbuf *mgr,
993 unsigned count,
994 bool take_ownership,
995 const struct pipe_vertex_buffer *bufs)
996 {
997 if (!count) {
998 struct pipe_context *pipe = mgr->pipe;
999 unsigned last_count = mgr->num_vertex_buffers;
1000
1001 /* Unbind. */
1002 mgr->num_vertex_buffers = 0;
1003 mgr->num_real_vertex_buffers = 0;
1004 mgr->user_vb_mask = 0;
1005 mgr->incompatible_vb_mask = 0;
1006 mgr->enabled_vb_mask = 0;
1007 mgr->unaligned_vb_mask[0] = 0;
1008 mgr->unaligned_vb_mask[1] = 0;
1009 mgr->vertex_buffers_dirty = false;
1010
1011 for (unsigned i = 0; i < last_count; i++) {
1012 pipe_vertex_buffer_unreference(&mgr->vertex_buffer[i]);
1013 pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[i]);
1014 }
1015
1016 pipe->set_vertex_buffers(pipe, 0, NULL);
1017 return;
1018 }
1019
1020 assert(bufs);
1021
1022 unsigned i;
1023 /* which buffers are enabled */
1024 uint32_t enabled_vb_mask = 0;
1025 /* which buffers are in user memory */
1026 uint32_t user_vb_mask = 0;
1027 /* which buffers are incompatible with the driver */
1028 uint32_t incompatible_vb_mask = 0;
1029 /* which buffers are unaligned to 2/4 bytes */
1030 uint32_t unaligned_vb_mask[2] = {0};
1031 unsigned num_identical = 0;
1032
1033 for (i = 0; i < count; i++) {
1034 const struct pipe_vertex_buffer *vb = &bufs[i];
1035 struct pipe_vertex_buffer *orig_vb = &mgr->vertex_buffer[i];
1036 struct pipe_vertex_buffer *real_vb = &mgr->real_vertex_buffer[i];
1037
1038 if (!vb->buffer.resource) {
1039 pipe_vertex_buffer_unreference(orig_vb);
1040 pipe_vertex_buffer_unreference(real_vb);
1041 continue;
1042 }
1043
1044 /* The structure has holes: do not use memcmp. */
1045 if (orig_vb->is_user_buffer == vb->is_user_buffer &&
1046 orig_vb->buffer_offset == vb->buffer_offset &&
1047 orig_vb->buffer.resource == vb->buffer.resource)
1048 num_identical++;
1049
1050 if (take_ownership) {
1051 pipe_vertex_buffer_unreference(orig_vb);
1052 memcpy(orig_vb, vb, sizeof(*vb));
1053 } else {
1054 pipe_vertex_buffer_reference(orig_vb, vb);
1055 }
1056
1057 enabled_vb_mask |= 1 << i;
1058
1059 if ((!mgr->caps.attrib_4byte_unaligned && vb->buffer_offset % 4 != 0)) {
1060 incompatible_vb_mask |= 1 << i;
1061 real_vb->buffer_offset = vb->buffer_offset;
1062 pipe_vertex_buffer_unreference(real_vb);
1063 real_vb->is_user_buffer = false;
1064 continue;
1065 }
1066
1067 if (!mgr->caps.attrib_element_unaligned) {
1068 if (vb->buffer_offset % 2 != 0)
1069 unaligned_vb_mask[0] |= BITFIELD_BIT(i);
1070 if (vb->buffer_offset % 4 != 0)
1071 unaligned_vb_mask[1] |= BITFIELD_BIT(i);
1072 }
1073
1074 if (!mgr->caps.user_vertex_buffers && vb->is_user_buffer) {
1075 user_vb_mask |= 1 << i;
1076 real_vb->buffer_offset = vb->buffer_offset;
1077 pipe_vertex_buffer_unreference(real_vb);
1078 real_vb->is_user_buffer = false;
1079 continue;
1080 }
1081
1082 pipe_vertex_buffer_reference(real_vb, vb);
1083 }
1084
1085 unsigned last_count = mgr->num_vertex_buffers;
1086
1087 if (num_identical == count && count == last_count)
1088 return;
1089
1090 for (; i < last_count; i++) {
1091 pipe_vertex_buffer_unreference(&mgr->vertex_buffer[i]);
1092 pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[i]);
1093 }
1094
1095 mgr->num_vertex_buffers = count;
1096 mgr->num_real_vertex_buffers = count;
1097 mgr->user_vb_mask = user_vb_mask;
1098 mgr->incompatible_vb_mask = incompatible_vb_mask;
1099 mgr->enabled_vb_mask = enabled_vb_mask;
1100 mgr->unaligned_vb_mask[0] = unaligned_vb_mask[0];
1101 mgr->unaligned_vb_mask[1] = unaligned_vb_mask[1];
1102 mgr->vertex_buffers_dirty = true;
1103 }
1104
1105 static ALWAYS_INLINE bool
get_upload_offset_size(struct u_vbuf * mgr,const struct pipe_vertex_buffer * vb,struct u_vbuf_elements * ve,const struct pipe_vertex_element * velem,unsigned vb_index,unsigned velem_index,int start_vertex,unsigned num_vertices,int start_instance,unsigned num_instances,unsigned * offset,unsigned * size)1106 get_upload_offset_size(struct u_vbuf *mgr,
1107 const struct pipe_vertex_buffer *vb,
1108 struct u_vbuf_elements *ve,
1109 const struct pipe_vertex_element *velem,
1110 unsigned vb_index, unsigned velem_index,
1111 int start_vertex, unsigned num_vertices,
1112 int start_instance, unsigned num_instances,
1113 unsigned *offset, unsigned *size)
1114 {
1115 /* Skip the buffers generated by translate. */
1116 if ((1 << vb_index) & mgr->fallback_vbs_mask || !vb->is_user_buffer)
1117 return false;
1118
1119 unsigned instance_div = velem->instance_divisor;
1120 *offset = vb->buffer_offset + velem->src_offset;
1121
1122 if (!velem->src_stride) {
1123 /* Constant attrib. */
1124 *size = ve->src_format_size[velem_index];
1125 } else if (instance_div) {
1126 /* Per-instance attrib. */
1127
1128 /* Figure out how many instances we'll render given instance_div. We
1129 * can't use the typical div_round_up() pattern because the CTS uses
1130 * instance_div = ~0 for a test, which overflows div_round_up()'s
1131 * addition.
1132 */
1133 unsigned count = num_instances / instance_div;
1134 if (count * instance_div != num_instances)
1135 count++;
1136
1137 *offset += velem->src_stride * start_instance;
1138 *size = velem->src_stride * (count - 1) + ve->src_format_size[velem_index];
1139 } else {
1140 /* Per-vertex attrib. */
1141 *offset += velem->src_stride * start_vertex;
1142 *size = velem->src_stride * (num_vertices - 1) + ve->src_format_size[velem_index];
1143 }
1144 return true;
1145 }
1146
1147
1148 static enum pipe_error
u_vbuf_upload_buffers(struct u_vbuf * mgr,int start_vertex,unsigned num_vertices,int start_instance,unsigned num_instances)1149 u_vbuf_upload_buffers(struct u_vbuf *mgr,
1150 int start_vertex, unsigned num_vertices,
1151 int start_instance, unsigned num_instances)
1152 {
1153 unsigned i;
1154 struct u_vbuf_elements *ve = mgr->ve;
1155 unsigned nr_velems = ve->count;
1156 const struct pipe_vertex_element *velems =
1157 mgr->using_translate ? mgr->fallback_velems.velems : ve->ve;
1158
1159 /* Faster path when no vertex attribs are interleaved. */
1160 if ((ve->interleaved_vb_mask & mgr->user_vb_mask) == 0) {
1161 for (i = 0; i < nr_velems; i++) {
1162 const struct pipe_vertex_element *velem = &velems[i];
1163 unsigned index = velem->vertex_buffer_index;
1164 struct pipe_vertex_buffer *vb = &mgr->vertex_buffer[index];
1165 unsigned offset, size;
1166
1167 if (!get_upload_offset_size(mgr, vb, ve, velem, index, i, start_vertex,
1168 num_vertices, start_instance, num_instances,
1169 &offset, &size))
1170 continue;
1171
1172 struct pipe_vertex_buffer *real_vb = &mgr->real_vertex_buffer[index];
1173 const uint8_t *ptr = mgr->vertex_buffer[index].buffer.user;
1174
1175 u_upload_data(mgr->pipe->stream_uploader,
1176 mgr->has_signed_vb_offset ? 0 : offset,
1177 size, 4, ptr + offset, &real_vb->buffer_offset,
1178 &real_vb->buffer.resource);
1179 if (!real_vb->buffer.resource)
1180 return PIPE_ERROR_OUT_OF_MEMORY;
1181
1182 real_vb->buffer_offset -= offset;
1183 }
1184 return PIPE_OK;
1185 }
1186
1187 unsigned start_offset[PIPE_MAX_ATTRIBS];
1188 unsigned end_offset[PIPE_MAX_ATTRIBS];
1189 uint32_t buffer_mask = 0;
1190
1191 /* Slower path supporting interleaved vertex attribs using 2 loops. */
1192 /* Determine how much data needs to be uploaded. */
1193 for (i = 0; i < nr_velems; i++) {
1194 const struct pipe_vertex_element *velem = &velems[i];
1195 unsigned index = velem->vertex_buffer_index;
1196 struct pipe_vertex_buffer *vb = &mgr->vertex_buffer[index];
1197 unsigned first, size, index_bit;
1198
1199 if (!get_upload_offset_size(mgr, vb, ve, velem, index, i, start_vertex,
1200 num_vertices, start_instance, num_instances,
1201 &first, &size))
1202 continue;
1203
1204 index_bit = 1 << index;
1205
1206 /* Update offsets. */
1207 if (!(buffer_mask & index_bit)) {
1208 start_offset[index] = first;
1209 end_offset[index] = first + size;
1210 } else {
1211 if (first < start_offset[index])
1212 start_offset[index] = first;
1213 if (first + size > end_offset[index])
1214 end_offset[index] = first + size;
1215 }
1216
1217 buffer_mask |= index_bit;
1218 }
1219
1220 /* Upload buffers. */
1221 while (buffer_mask) {
1222 unsigned start, end;
1223 struct pipe_vertex_buffer *real_vb;
1224 const uint8_t *ptr;
1225
1226 i = u_bit_scan(&buffer_mask);
1227
1228 start = start_offset[i];
1229 end = end_offset[i];
1230 assert(start < end);
1231
1232 real_vb = &mgr->real_vertex_buffer[i];
1233 ptr = mgr->vertex_buffer[i].buffer.user;
1234
1235 u_upload_data(mgr->pipe->stream_uploader,
1236 mgr->has_signed_vb_offset ? 0 : start,
1237 end - start, 4,
1238 ptr + start, &real_vb->buffer_offset, &real_vb->buffer.resource);
1239 if (!real_vb->buffer.resource)
1240 return PIPE_ERROR_OUT_OF_MEMORY;
1241
1242 real_vb->buffer_offset -= start;
1243 }
1244
1245 return PIPE_OK;
1246 }
1247
u_vbuf_need_minmax_index(const struct u_vbuf * mgr,uint32_t misaligned)1248 static bool u_vbuf_need_minmax_index(const struct u_vbuf *mgr, uint32_t misaligned)
1249 {
1250 /* See if there are any per-vertex attribs which will be uploaded or
1251 * translated. Use bitmasks to get the info instead of looping over vertex
1252 * elements. */
1253 return (mgr->ve->used_vb_mask &
1254 ((mgr->user_vb_mask |
1255 mgr->incompatible_vb_mask | mgr->ve->incompatible_vb_mask |
1256 misaligned |
1257 mgr->ve->incompatible_vb_mask_any) &
1258 mgr->ve->noninstance_vb_mask_any &
1259 mgr->ve->nonzero_stride_vb_mask)) != 0;
1260 }
1261
u_vbuf_mapping_vertex_buffer_blocks(const struct u_vbuf * mgr,uint32_t misaligned)1262 static bool u_vbuf_mapping_vertex_buffer_blocks(const struct u_vbuf *mgr, uint32_t misaligned)
1263 {
1264 /* Return true if there are hw buffers which don't need to be translated.
1265 *
1266 * We could query whether each buffer is busy, but that would
1267 * be way more costly than this. */
1268 return (mgr->ve->used_vb_mask &
1269 (~mgr->user_vb_mask &
1270 ~mgr->incompatible_vb_mask &
1271 ~mgr->ve->incompatible_vb_mask &
1272 ~misaligned &
1273 mgr->ve->compatible_vb_mask_all &
1274 mgr->ve->noninstance_vb_mask_any &
1275 mgr->ve->nonzero_stride_vb_mask)) != 0;
1276 }
1277
1278 static void
u_vbuf_get_minmax_index_mapped(const struct pipe_draw_info * info,unsigned count,const void * indices,unsigned * out_min_index,unsigned * out_max_index)1279 u_vbuf_get_minmax_index_mapped(const struct pipe_draw_info *info,
1280 unsigned count,
1281 const void *indices, unsigned *out_min_index,
1282 unsigned *out_max_index)
1283 {
1284 if (!count) {
1285 *out_min_index = 0;
1286 *out_max_index = 0;
1287 return;
1288 }
1289
1290 switch (info->index_size) {
1291 case 4: {
1292 const unsigned *ui_indices = (const unsigned*)indices;
1293 unsigned max = 0;
1294 unsigned min = ~0u;
1295 if (info->primitive_restart) {
1296 for (unsigned i = 0; i < count; i++) {
1297 if (ui_indices[i] != info->restart_index) {
1298 if (ui_indices[i] > max) max = ui_indices[i];
1299 if (ui_indices[i] < min) min = ui_indices[i];
1300 }
1301 }
1302 }
1303 else {
1304 for (unsigned i = 0; i < count; i++) {
1305 if (ui_indices[i] > max) max = ui_indices[i];
1306 if (ui_indices[i] < min) min = ui_indices[i];
1307 }
1308 }
1309 *out_min_index = min;
1310 *out_max_index = max;
1311 break;
1312 }
1313 case 2: {
1314 const unsigned short *us_indices = (const unsigned short*)indices;
1315 unsigned short max = 0;
1316 unsigned short min = ~((unsigned short)0);
1317 if (info->primitive_restart) {
1318 for (unsigned i = 0; i < count; i++) {
1319 if (us_indices[i] != info->restart_index) {
1320 if (us_indices[i] > max) max = us_indices[i];
1321 if (us_indices[i] < min) min = us_indices[i];
1322 }
1323 }
1324 }
1325 else {
1326 for (unsigned i = 0; i < count; i++) {
1327 if (us_indices[i] > max) max = us_indices[i];
1328 if (us_indices[i] < min) min = us_indices[i];
1329 }
1330 }
1331 *out_min_index = min;
1332 *out_max_index = max;
1333 break;
1334 }
1335 case 1: {
1336 const unsigned char *ub_indices = (const unsigned char*)indices;
1337 unsigned char max = 0;
1338 unsigned char min = ~((unsigned char)0);
1339 if (info->primitive_restart) {
1340 for (unsigned i = 0; i < count; i++) {
1341 if (ub_indices[i] != info->restart_index) {
1342 if (ub_indices[i] > max) max = ub_indices[i];
1343 if (ub_indices[i] < min) min = ub_indices[i];
1344 }
1345 }
1346 }
1347 else {
1348 for (unsigned i = 0; i < count; i++) {
1349 if (ub_indices[i] > max) max = ub_indices[i];
1350 if (ub_indices[i] < min) min = ub_indices[i];
1351 }
1352 }
1353 *out_min_index = min;
1354 *out_max_index = max;
1355 break;
1356 }
1357 default:
1358 unreachable("bad index size");
1359 }
1360 }
1361
u_vbuf_get_minmax_index(struct pipe_context * pipe,const struct pipe_draw_info * info,const struct pipe_draw_start_count_bias * draw,unsigned * out_min_index,unsigned * out_max_index)1362 void u_vbuf_get_minmax_index(struct pipe_context *pipe,
1363 const struct pipe_draw_info *info,
1364 const struct pipe_draw_start_count_bias *draw,
1365 unsigned *out_min_index, unsigned *out_max_index)
1366 {
1367 struct pipe_transfer *transfer = NULL;
1368 const void *indices;
1369
1370 if (info->has_user_indices) {
1371 indices = (uint8_t*)info->index.user +
1372 draw->start * info->index_size;
1373 } else {
1374 indices = pipe_buffer_map_range(pipe, info->index.resource,
1375 draw->start * info->index_size,
1376 draw->count * info->index_size,
1377 PIPE_MAP_READ, &transfer);
1378 }
1379
1380 u_vbuf_get_minmax_index_mapped(info, draw->count, indices,
1381 out_min_index, out_max_index);
1382
1383 if (transfer) {
1384 pipe_buffer_unmap(pipe, transfer);
1385 }
1386 }
1387
u_vbuf_set_driver_vertex_buffers(struct u_vbuf * mgr)1388 static void u_vbuf_set_driver_vertex_buffers(struct u_vbuf *mgr)
1389 {
1390 struct pipe_context *pipe = mgr->pipe;
1391 unsigned count = mgr->num_real_vertex_buffers;
1392
1393 assert(mgr->vertex_buffers_dirty);
1394
1395 if (mgr->user_vb_mask == BITFIELD_MASK(count)) {
1396 /* Fast path that allows us to transfer the VBO references to the driver
1397 * to skip atomic reference counting there. These are freshly uploaded
1398 * user buffers that can be discarded after this call.
1399 */
1400 pipe->set_vertex_buffers(pipe, count, mgr->real_vertex_buffer);
1401
1402 /* We don't own the VBO references now. Set them to NULL. */
1403 for (unsigned i = 0; i < count; i++) {
1404 assert(!mgr->real_vertex_buffer[i].is_user_buffer);
1405 mgr->real_vertex_buffer[i].buffer.resource = NULL;
1406 }
1407 } else {
1408 /* Slow path where we have to keep VBO references. */
1409 util_set_vertex_buffers(pipe, count, false, mgr->real_vertex_buffer);
1410 }
1411 mgr->vertex_buffers_dirty = false;
1412 }
1413
1414 static void
u_vbuf_split_indexed_multidraw(struct u_vbuf * mgr,struct pipe_draw_info * info,unsigned drawid_offset,unsigned * indirect_data,unsigned stride,unsigned draw_count)1415 u_vbuf_split_indexed_multidraw(struct u_vbuf *mgr, struct pipe_draw_info *info,
1416 unsigned drawid_offset,
1417 unsigned *indirect_data, unsigned stride,
1418 unsigned draw_count)
1419 {
1420 /* Increase refcount to be able to use take_index_buffer_ownership with
1421 * all draws.
1422 */
1423 if (draw_count > 1 && info->take_index_buffer_ownership)
1424 p_atomic_add(&info->index.resource->reference.count, draw_count - 1);
1425
1426 assert(info->index_size);
1427
1428 for (unsigned i = 0; i < draw_count; i++) {
1429 struct pipe_draw_start_count_bias draw;
1430 unsigned offset = i * stride / 4;
1431
1432 draw.count = indirect_data[offset + 0];
1433 info->instance_count = indirect_data[offset + 1];
1434 draw.start = indirect_data[offset + 2];
1435 draw.index_bias = indirect_data[offset + 3];
1436 info->start_instance = indirect_data[offset + 4];
1437
1438 u_vbuf_draw_vbo(mgr->pipe, info, drawid_offset, NULL, &draw, 1);
1439 }
1440 }
1441
u_vbuf_draw_vbo(struct pipe_context * pipe,const struct pipe_draw_info * info,unsigned drawid_offset,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)1442 void u_vbuf_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info,
1443 unsigned drawid_offset,
1444 const struct pipe_draw_indirect_info *indirect,
1445 const struct pipe_draw_start_count_bias *draws,
1446 unsigned num_draws)
1447 {
1448 struct u_vbuf *mgr = pipe->vbuf;
1449 int start_vertex;
1450 unsigned min_index;
1451 unsigned num_vertices;
1452 bool unroll_indices = false;
1453 const uint32_t used_vb_mask = mgr->ve->used_vb_mask;
1454 uint32_t user_vb_mask = mgr->user_vb_mask & used_vb_mask;
1455 unsigned fixed_restart_index = info->index_size ? util_prim_restart_index_from_size(info->index_size) : 0;
1456
1457 uint32_t misaligned = 0;
1458 if (!mgr->caps.attrib_element_unaligned) {
1459 for (unsigned i = 0; i < ARRAY_SIZE(mgr->unaligned_vb_mask); i++) {
1460 misaligned |= mgr->ve->vb_align_mask[i] & mgr->unaligned_vb_mask[i];
1461 }
1462 }
1463 const uint32_t incompatible_vb_mask =
1464 (mgr->incompatible_vb_mask | mgr->ve->incompatible_vb_mask | misaligned) & used_vb_mask;
1465
1466 /* Normal draw. No fallback and no user buffers. */
1467 if (!incompatible_vb_mask &&
1468 !mgr->ve->incompatible_elem_mask &&
1469 !user_vb_mask &&
1470 (info->index_size != 1 || !mgr->caps.rewrite_ubyte_ibs) &&
1471 (!info->primitive_restart ||
1472 info->restart_index == fixed_restart_index ||
1473 !mgr->caps.rewrite_restart_index) &&
1474 (!info->primitive_restart || mgr->caps.supported_restart_modes & BITFIELD_BIT(info->mode)) &&
1475 mgr->caps.supported_prim_modes & BITFIELD_BIT(info->mode)) {
1476
1477 /* Set vertex buffers if needed. */
1478 if (mgr->vertex_buffers_dirty) {
1479 u_vbuf_set_driver_vertex_buffers(mgr);
1480 }
1481
1482 pipe->draw_vbo(pipe, info, drawid_offset, indirect, draws, num_draws);
1483 return;
1484 }
1485
1486 /* Increase refcount to be able to use take_index_buffer_ownership with
1487 * all draws.
1488 */
1489 if (num_draws > 1 && info->take_index_buffer_ownership)
1490 p_atomic_add(&info->index.resource->reference.count, num_draws - 1);
1491
1492 for (unsigned d = 0; d < num_draws; d++) {
1493 struct pipe_draw_info new_info = *info;
1494 struct pipe_draw_start_count_bias new_draw = draws[d];
1495
1496 /* Handle indirect (multi)draws. */
1497 if (indirect && indirect->buffer) {
1498 unsigned draw_count = 0;
1499
1500 /* num_draws can only be 1 with indirect draws. */
1501 assert(num_draws == 1);
1502
1503 /* Get the number of draws. */
1504 if (indirect->indirect_draw_count) {
1505 pipe_buffer_read(pipe, indirect->indirect_draw_count,
1506 indirect->indirect_draw_count_offset,
1507 4, &draw_count);
1508 } else {
1509 draw_count = indirect->draw_count;
1510 }
1511
1512 if (!draw_count)
1513 goto cleanup;
1514
1515 unsigned data_size = (draw_count - 1) * indirect->stride +
1516 (new_info.index_size ? 20 : 16);
1517 unsigned *data = malloc(data_size);
1518 if (!data)
1519 goto cleanup; /* report an error? */
1520
1521 /* Read the used buffer range only once, because the read can be
1522 * uncached.
1523 */
1524 pipe_buffer_read(pipe, indirect->buffer, indirect->offset, data_size,
1525 data);
1526
1527 if (info->index_size) {
1528 /* Indexed multidraw. */
1529 unsigned index_bias0 = data[3];
1530 bool index_bias_same = true;
1531
1532 /* If we invoke the translate path, we have to split the multidraw. */
1533 if (incompatible_vb_mask ||
1534 mgr->ve->incompatible_elem_mask) {
1535 u_vbuf_split_indexed_multidraw(mgr, &new_info, drawid_offset, data,
1536 indirect->stride, draw_count);
1537 free(data);
1538 /* We're done (as num_draws is 1), so return early. */
1539 return;
1540 }
1541
1542 /* See if index_bias is the same for all draws. */
1543 for (unsigned i = 1; i < draw_count; i++) {
1544 if (data[i * indirect->stride / 4 + 3] != index_bias0) {
1545 index_bias_same = false;
1546 break;
1547 }
1548 }
1549
1550 /* Split the multidraw if index_bias is different. */
1551 if (!index_bias_same) {
1552 u_vbuf_split_indexed_multidraw(mgr, &new_info, drawid_offset, data,
1553 indirect->stride, draw_count);
1554 free(data);
1555 /* We're done (as num_draws is 1), so return early. */
1556 return;
1557 }
1558
1559 /* If we don't need to use the translate path and index_bias is
1560 * the same, we can process the multidraw with the time complexity
1561 * equal to 1 draw call (except for the index range computation).
1562 * We only need to compute the index range covering all draw calls
1563 * of the multidraw.
1564 *
1565 * The driver will not look at these values because indirect != NULL.
1566 * These values determine the user buffer bounds to upload.
1567 */
1568 new_draw.index_bias = index_bias0;
1569 new_info.index_bounds_valid = true;
1570 new_info.min_index = ~0u;
1571 new_info.max_index = 0;
1572 new_info.start_instance = ~0u;
1573 unsigned end_instance = 0;
1574
1575 struct pipe_transfer *transfer = NULL;
1576 const uint8_t *indices;
1577
1578 if (info->has_user_indices) {
1579 indices = (uint8_t*)info->index.user;
1580 } else {
1581 indices = (uint8_t*)pipe_buffer_map(pipe, info->index.resource,
1582 PIPE_MAP_READ, &transfer);
1583 }
1584
1585 for (unsigned i = 0; i < draw_count; i++) {
1586 unsigned offset = i * indirect->stride / 4;
1587 unsigned start = data[offset + 2];
1588 unsigned count = data[offset + 0];
1589 unsigned start_instance = data[offset + 4];
1590 unsigned instance_count = data[offset + 1];
1591
1592 if (!count || !instance_count)
1593 continue;
1594
1595 /* Update the ranges of instances. */
1596 new_info.start_instance = MIN2(new_info.start_instance,
1597 start_instance);
1598 end_instance = MAX2(end_instance, start_instance + instance_count);
1599
1600 /* Update the index range. */
1601 unsigned min, max;
1602 u_vbuf_get_minmax_index_mapped(&new_info, count,
1603 indices +
1604 new_info.index_size * start,
1605 &min, &max);
1606
1607 new_info.min_index = MIN2(new_info.min_index, min);
1608 new_info.max_index = MAX2(new_info.max_index, max);
1609 }
1610 free(data);
1611
1612 if (transfer)
1613 pipe_buffer_unmap(pipe, transfer);
1614
1615 /* Set the final instance count. */
1616 new_info.instance_count = end_instance - new_info.start_instance;
1617
1618 if (new_info.start_instance == ~0u || !new_info.instance_count)
1619 goto cleanup;
1620 } else {
1621 /* Non-indexed multidraw.
1622 *
1623 * Keep the draw call indirect and compute minimums & maximums,
1624 * which will determine the user buffer bounds to upload, but
1625 * the driver will not look at these values because indirect != NULL.
1626 *
1627 * This efficiently processes the multidraw with the time complexity
1628 * equal to 1 draw call.
1629 */
1630 new_draw.start = ~0u;
1631 new_info.start_instance = ~0u;
1632 unsigned end_vertex = 0;
1633 unsigned end_instance = 0;
1634
1635 for (unsigned i = 0; i < draw_count; i++) {
1636 unsigned offset = i * indirect->stride / 4;
1637 unsigned start = data[offset + 2];
1638 unsigned count = data[offset + 0];
1639 unsigned start_instance = data[offset + 3];
1640 unsigned instance_count = data[offset + 1];
1641
1642 new_draw.start = MIN2(new_draw.start, start);
1643 new_info.start_instance = MIN2(new_info.start_instance,
1644 start_instance);
1645
1646 end_vertex = MAX2(end_vertex, start + count);
1647 end_instance = MAX2(end_instance, start_instance + instance_count);
1648 }
1649 free(data);
1650
1651 /* Set the final counts. */
1652 new_draw.count = end_vertex - new_draw.start;
1653 new_info.instance_count = end_instance - new_info.start_instance;
1654
1655 if (new_draw.start == ~0u || !new_draw.count || !new_info.instance_count)
1656 goto cleanup;
1657 }
1658 } else {
1659 if ((!indirect && !new_draw.count) || !new_info.instance_count)
1660 goto cleanup;
1661 }
1662
1663 if (new_info.index_size) {
1664 /* See if anything needs to be done for per-vertex attribs. */
1665 if (u_vbuf_need_minmax_index(mgr, misaligned)) {
1666 unsigned max_index;
1667
1668 if (new_info.index_bounds_valid) {
1669 min_index = new_info.min_index;
1670 max_index = new_info.max_index;
1671 } else {
1672 u_vbuf_get_minmax_index(mgr->pipe, &new_info, &new_draw,
1673 &min_index, &max_index);
1674 }
1675
1676 assert(min_index <= max_index);
1677
1678 start_vertex = min_index + new_draw.index_bias;
1679 num_vertices = max_index + 1 - min_index;
1680
1681 /* Primitive restart doesn't work when unrolling indices.
1682 * We would have to break this drawing operation into several ones. */
1683 /* Use some heuristic to see if unrolling indices improves
1684 * performance. */
1685 if (!indirect &&
1686 !new_info.primitive_restart &&
1687 util_is_vbo_upload_ratio_too_large(new_draw.count, num_vertices) &&
1688 !u_vbuf_mapping_vertex_buffer_blocks(mgr, misaligned)) {
1689 unroll_indices = true;
1690 user_vb_mask &= ~(mgr->ve->nonzero_stride_vb_mask &
1691 mgr->ve->noninstance_vb_mask_any);
1692 }
1693 } else {
1694 /* Nothing to do for per-vertex attribs. */
1695 start_vertex = 0;
1696 num_vertices = 0;
1697 min_index = 0;
1698 }
1699 } else {
1700 start_vertex = new_draw.start;
1701 num_vertices = new_draw.count;
1702 min_index = 0;
1703 }
1704
1705 /* Translate vertices with non-native layouts or formats. */
1706 if (unroll_indices ||
1707 incompatible_vb_mask ||
1708 mgr->ve->incompatible_elem_mask) {
1709 if (!u_vbuf_translate_begin(mgr, &new_info, &new_draw,
1710 start_vertex, num_vertices,
1711 min_index, unroll_indices, misaligned)) {
1712 debug_warn_once("u_vbuf_translate_begin() failed");
1713 goto cleanup;
1714 }
1715
1716 if (unroll_indices) {
1717 if (!new_info.has_user_indices && info->take_index_buffer_ownership)
1718 pipe_drop_resource_references(new_info.index.resource, 1);
1719 new_info.index_size = 0;
1720 new_draw.index_bias = 0;
1721 new_info.index_bounds_valid = true;
1722 new_info.min_index = 0;
1723 new_info.max_index = new_draw.count - 1;
1724 new_draw.start = 0;
1725 }
1726
1727 user_vb_mask &= ~(incompatible_vb_mask |
1728 mgr->ve->incompatible_vb_mask_all);
1729 mgr->vertex_buffers_dirty = true;
1730 }
1731
1732 /* Upload user buffers. */
1733 if (user_vb_mask) {
1734 if (u_vbuf_upload_buffers(mgr, start_vertex, num_vertices,
1735 new_info.start_instance,
1736 new_info.instance_count) != PIPE_OK) {
1737 debug_warn_once("u_vbuf_upload_buffers() failed");
1738 goto cleanup;
1739 }
1740
1741 mgr->vertex_buffers_dirty = true;
1742 }
1743
1744 /*
1745 if (unroll_indices) {
1746 printf("unrolling indices: start_vertex = %i, num_vertices = %i\n",
1747 start_vertex, num_vertices);
1748 util_dump_draw_info(stdout, info);
1749 printf("\n");
1750 }
1751
1752 unsigned i;
1753 for (i = 0; i < mgr->nr_vertex_buffers; i++) {
1754 printf("input %i: ", i);
1755 util_dump_vertex_buffer(stdout, mgr->vertex_buffer+i);
1756 printf("\n");
1757 }
1758 for (i = 0; i < mgr->nr_real_vertex_buffers; i++) {
1759 printf("real %i: ", i);
1760 util_dump_vertex_buffer(stdout, mgr->real_vertex_buffer+i);
1761 printf("\n");
1762 }
1763 */
1764
1765 u_upload_unmap(pipe->stream_uploader);
1766 if (mgr->vertex_buffers_dirty)
1767 u_vbuf_set_driver_vertex_buffers(mgr);
1768
1769 if ((new_info.index_size == 1 && mgr->caps.rewrite_ubyte_ibs) ||
1770 (new_info.primitive_restart &&
1771 ((new_info.restart_index != fixed_restart_index && mgr->caps.rewrite_restart_index) ||
1772 !(mgr->caps.supported_restart_modes & BITFIELD_BIT(new_info.mode)))) ||
1773 !(mgr->caps.supported_prim_modes & BITFIELD_BIT(new_info.mode))) {
1774 util_primconvert_save_flatshade_first(mgr->pc, mgr->flatshade_first);
1775 util_primconvert_draw_vbo(mgr->pc, &new_info, drawid_offset, indirect, &new_draw, 1);
1776 } else
1777 pipe->draw_vbo(pipe, &new_info, drawid_offset, indirect, &new_draw, 1);
1778 if (info->increment_draw_id)
1779 drawid_offset++;
1780 }
1781
1782 if (mgr->using_translate) {
1783 u_vbuf_translate_end(mgr);
1784 }
1785 return;
1786
1787 cleanup:
1788 if (info->take_index_buffer_ownership) {
1789 struct pipe_resource *indexbuf = info->index.resource;
1790 pipe_resource_reference(&indexbuf, NULL);
1791 }
1792 }
1793
u_vbuf_save_vertex_elements(struct u_vbuf * mgr)1794 void u_vbuf_save_vertex_elements(struct u_vbuf *mgr)
1795 {
1796 assert(!mgr->ve_saved);
1797 mgr->ve_saved = mgr->ve;
1798 }
1799
u_vbuf_restore_vertex_elements(struct u_vbuf * mgr)1800 void u_vbuf_restore_vertex_elements(struct u_vbuf *mgr)
1801 {
1802 if (mgr->ve != mgr->ve_saved) {
1803 struct pipe_context *pipe = mgr->pipe;
1804
1805 mgr->ve = mgr->ve_saved;
1806 pipe->bind_vertex_elements_state(pipe,
1807 mgr->ve ? mgr->ve->driver_cso : NULL);
1808 }
1809 mgr->ve_saved = NULL;
1810 }
1811