196 u_vbuf_create_vertex_elements(struct u_vbuf *mgr, unsigned count,
343    struct u_vbuf *mgr = CALLOC_STRUCT(u_vbuf);  in u_vbuf_create()  local
345    mgr->caps = *caps;  in u_vbuf_create()
346    mgr->pipe = pipe;  in u_vbuf_create()
355       mgr->pc = util_primconvert_create_config(pipe, &cfg);  in u_vbuf_create()
357    mgr->translate_cache = translate_cache_create();  in u_vbuf_create()
358    memset(mgr->fallback_vbs, ~0, sizeof(mgr->fallback_vbs));  in u_vbuf_create()
359    mgr->allowed_vb_mask = u_bit_consecutive(0, mgr->caps.max_vertex_buffers);  in u_vbuf_create()
361    mgr->has_signed_vb_offset =  in u_vbuf_create()
365    cso_cache_init(&mgr->cso_cache, pipe);  in u_vbuf_create()
366    cso_cache_set_delete_cso_callback(&mgr->cso_cache,  in u_vbuf_create()
369    return mgr;  in u_vbuf_create()
375 u_vbuf_set_vertex_elements_internal(struct u_vbuf *mgr,  in u_vbuf_set_vertex_elements_internal()  argument
378    struct pipe_context *pipe = mgr->pipe;  in u_vbuf_set_vertex_elements_internal()
387    iter = cso_find_state_template(&mgr->cso_cache, hash_key, CSO_VELEMENTS,  in u_vbuf_set_vertex_elements_internal()
393       cso->data = u_vbuf_create_vertex_elements(mgr, velems->count,  in u_vbuf_set_vertex_elements_internal()
396       iter = cso_insert_state(&mgr->cso_cache, hash_key, CSO_VELEMENTS, cso);  in u_vbuf_set_vertex_elements_internal()
404    if (ve != mgr->ve)  in u_vbuf_set_vertex_elements_internal()
410 void u_vbuf_set_vertex_elements(struct u_vbuf *mgr,  in u_vbuf_set_vertex_elements()  argument
413    mgr->ve = u_vbuf_set_vertex_elements_internal(mgr, velems);  in u_vbuf_set_vertex_elements()
416 void u_vbuf_set_flatshade_first(struct u_vbuf *mgr, bool flatshade_first)  in u_vbuf_set_flatshade_first()  argument
418    mgr->flatshade_first = flatshade_first;  in u_vbuf_set_flatshade_first()
421 void u_vbuf_unset_vertex_elements(struct u_vbuf *mgr)  in u_vbuf_unset_vertex_elements()  argument
423    mgr->ve = NULL;  in u_vbuf_unset_vertex_elements()
426 void u_vbuf_destroy(struct u_vbuf *mgr)  in u_vbuf_destroy()  argument
428    struct pipe_screen *screen = mgr->pipe->screen;  in u_vbuf_destroy()
433    mgr->pipe->set_vertex_buffers(mgr->pipe, 0, 0, num_vb, false, NULL);  in u_vbuf_destroy()
436       pipe_vertex_buffer_unreference(&mgr->vertex_buffer[i]);  in u_vbuf_destroy()
438       pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[i]);  in u_vbuf_destroy()
440    if (mgr->pc)  in u_vbuf_destroy()
441       util_primconvert_destroy(mgr->pc);  in u_vbuf_destroy()
443    translate_cache_destroy(mgr->translate_cache);  in u_vbuf_destroy()
444    cso_cache_delete(&mgr->cso_cache);  in u_vbuf_destroy()
445    FREE(mgr);  in u_vbuf_destroy()
449 u_vbuf_translate_buffers(struct u_vbuf *mgr, struct translate_key *key,  in u_vbuf_translate_buffers()  argument
463    tr = translate_cache_find(mgr->translate_cache, key);  in u_vbuf_translate_buffers()
473       vb = &mgr->vertex_buffer[i];  in u_vbuf_translate_buffers()
518          map = pipe_buffer_map_range(mgr->pipe, vb->buffer.resource, offset, size,  in u_vbuf_translate_buffers()
537       u_upload_alloc(mgr->pipe->stream_uploader, 0,  in u_vbuf_translate_buffers()
547          map = pipe_buffer_map_range(mgr->pipe, info->index.resource, offset,  in u_vbuf_translate_buffers()
565          pipe_buffer_unmap(mgr->pipe, transfer);  in u_vbuf_translate_buffers()
569       u_upload_alloc(mgr->pipe->stream_uploader,  in u_vbuf_translate_buffers()
570                      mgr->has_signed_vb_offset ?  in u_vbuf_translate_buffers()
589          pipe_buffer_unmap(mgr->pipe, vb_transfer[i]);  in u_vbuf_translate_buffers()
594    mgr->real_vertex_buffer[out_vb].buffer_offset = out_offset;  in u_vbuf_translate_buffers()
595    mgr->real_vertex_buffer[out_vb].stride = key->output_stride;  in u_vbuf_translate_buffers()
598    pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[out_vb]);  in u_vbuf_translate_buffers()
599    mgr->real_vertex_buffer[out_vb].buffer.resource = out_buffer;  in u_vbuf_translate_buffers()
600    mgr->real_vertex_buffer[out_vb].is_user_buffer = false;  in u_vbuf_translate_buffers()
606 u_vbuf_translate_find_free_vb_slots(struct u_vbuf *mgr,  in u_vbuf_translate_find_free_vb_slots()  argument
613       mgr->ve->incompatible_vb_mask_all | mgr->incompatible_vb_mask |  in u_vbuf_translate_find_free_vb_slots()
614       ~mgr->enabled_vb_mask;  in u_vbuf_translate_find_free_vb_slots()
623    mgr->fallback_vbs_mask = 0;  in u_vbuf_translate_find_free_vb_slots()
638          mgr->fallback_vbs_mask |= 1 << index;  in u_vbuf_translate_find_free_vb_slots()
650       mgr->fallback_vbs_mask = 1 << index;  in u_vbuf_translate_find_free_vb_slots()
658          mgr->dirty_real_vb_mask |= 1 << fallback_vbs[type];  in u_vbuf_translate_find_free_vb_slots()
662    memcpy(mgr->fallback_vbs, fallback_vbs, sizeof(fallback_vbs));  in u_vbuf_translate_find_free_vb_slots()
667 u_vbuf_translate_begin(struct u_vbuf *mgr,  in u_vbuf_translate_begin()  argument
677    const unsigned incompatible_vb_mask = mgr->incompatible_vb_mask &  in u_vbuf_translate_begin()
678                                          mgr->ve->used_vb_mask;  in u_vbuf_translate_begin()
697    for (i = 0; i < mgr->ve->count; i++) {  in u_vbuf_translate_begin()
698       unsigned vb_index = mgr->ve->ve[i].vertex_buffer_index;  in u_vbuf_translate_begin()
700       if (!mgr->vertex_buffer[vb_index].stride) {  in u_vbuf_translate_begin()
701          if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&  in u_vbuf_translate_begin()
706       } else if (mgr->ve->ve[i].instance_divisor) {  in u_vbuf_translate_begin()
707          if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&  in u_vbuf_translate_begin()
714              !(mgr->ve->incompatible_elem_mask & (1 << i)) &&  in u_vbuf_translate_begin()
725    if (!u_vbuf_translate_find_free_vb_slots(mgr, mask)) {  in u_vbuf_translate_begin()
730    for (i = 0; i < mgr->ve->count; i++) {  in u_vbuf_translate_begin()
733       enum pipe_format output_format = mgr->ve->native_format[i];  in u_vbuf_translate_begin()
734       unsigned bit, vb_index = mgr->ve->ve[i].vertex_buffer_index;  in u_vbuf_translate_begin()
737       if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&  in u_vbuf_translate_begin()
751       if (mgr->ve->ve[i].src_format != output_format)  in u_vbuf_translate_begin()
763       te->input_format = mgr->ve->ve[i].src_format;  in u_vbuf_translate_begin()
764       te->input_offset = mgr->ve->ve[i].src_offset;  in u_vbuf_translate_begin()
768       k->output_stride += mgr->ve->native_format_size[i];  in u_vbuf_translate_begin()
776          err = u_vbuf_translate_buffers(mgr, &key[type], info, draw,  in u_vbuf_translate_begin()
777                                         mask[type], mgr->fallback_vbs[type],  in u_vbuf_translate_begin()
785             mgr->real_vertex_buffer[mgr->fallback_vbs[VB_CONST]].stride = 0;  in u_vbuf_translate_begin()
791    for (i = 0; i < mgr->ve->count; i++) {  in u_vbuf_translate_begin()
795             mgr->fallback_velems.velems[i].instance_divisor = mgr->ve->ve[i].instance_divisor;  in u_vbuf_translate_begin()
796             mgr->fallback_velems.velems[i].src_format = te->output_format;  in u_vbuf_translate_begin()
797             mgr->fallback_velems.velems[i].src_offset = te->output_offset;  in u_vbuf_translate_begin()
798             mgr->fallback_velems.velems[i].vertex_buffer_index = mgr->fallback_vbs[type];  in u_vbuf_translate_begin()
808          memcpy(&mgr->fallback_velems.velems[i], &mgr->ve->ve[i],  in u_vbuf_translate_begin()
813    mgr->fallback_velems.count = mgr->ve->count;  in u_vbuf_translate_begin()
815    u_vbuf_set_vertex_elements_internal(mgr, &mgr->fallback_velems);  in u_vbuf_translate_begin()
816    mgr->using_translate = TRUE;  in u_vbuf_translate_begin()
820 static void u_vbuf_translate_end(struct u_vbuf *mgr)  in u_vbuf_translate_end()  argument
825    mgr->pipe->bind_vertex_elements_state(mgr->pipe, mgr->ve->driver_cso);  in u_vbuf_translate_end()
826    mgr->using_translate = FALSE;  in u_vbuf_translate_end()
830       unsigned vb = mgr->fallback_vbs[i];  in u_vbuf_translate_end()
832          pipe_resource_reference(&mgr->real_vertex_buffer[vb].buffer.resource, NULL);  in u_vbuf_translate_end()
833          mgr->fallback_vbs[i] = ~0;  in u_vbuf_translate_end()
837    mgr->dirty_real_vb_mask |= mgr->fallback_vbs_mask;  in u_vbuf_translate_end()
838    mgr->fallback_vbs_mask = 0;  in u_vbuf_translate_end()
842 u_vbuf_create_vertex_elements(struct u_vbuf *mgr, unsigned count,  in u_vbuf_create_vertex_elements()  argument
848    struct pipe_context *pipe = mgr->pipe;  in u_vbuf_create_vertex_elements()
876       format = mgr->caps.format_translation[format];  in u_vbuf_create_vertex_elements()
884           (!mgr->caps.velem_src_offset_unaligned &&  in u_vbuf_create_vertex_elements()
893    if (used_buffers & ~mgr->allowed_vb_mask) {  in u_vbuf_create_vertex_elements()
909    if (!mgr->caps.velem_src_offset_unaligned) {  in u_vbuf_create_vertex_elements()
938 void u_vbuf_set_vertex_buffers(struct u_vbuf *mgr,  in u_vbuf_set_vertex_buffers()  argument
957    mgr->user_vb_mask &= mask;  in u_vbuf_set_vertex_buffers()
958    mgr->incompatible_vb_mask &= mask;  in u_vbuf_set_vertex_buffers()
959    mgr->nonzero_stride_vb_mask &= mask;  in u_vbuf_set_vertex_buffers()
960    mgr->enabled_vb_mask &= mask;  in u_vbuf_set_vertex_buffers()
963       struct pipe_context *pipe = mgr->pipe;  in u_vbuf_set_vertex_buffers()
966       mgr->dirty_real_vb_mask &= mask;  in u_vbuf_set_vertex_buffers()
971          pipe_vertex_buffer_unreference(&mgr->vertex_buffer[dst_index]);  in u_vbuf_set_vertex_buffers()
972          pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[dst_index]);  in u_vbuf_set_vertex_buffers()
983       struct pipe_vertex_buffer *orig_vb = &mgr->vertex_buffer[dst_index];  in u_vbuf_set_vertex_buffers()
984       struct pipe_vertex_buffer *real_vb = &mgr->real_vertex_buffer[dst_index];  in u_vbuf_set_vertex_buffers()
1004       if ((!mgr->caps.buffer_offset_unaligned && vb->buffer_offset % 4 != 0) ||  in u_vbuf_set_vertex_buffers()
1005           (!mgr->caps.buffer_stride_unaligned && vb->stride % 4 != 0)) {  in u_vbuf_set_vertex_buffers()
1014       if (!mgr->caps.user_vertex_buffers && vb->is_user_buffer) {  in u_vbuf_set_vertex_buffers()
1029       pipe_vertex_buffer_unreference(&mgr->vertex_buffer[dst_index]);  in u_vbuf_set_vertex_buffers()
1030       pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[dst_index]);  in u_vbuf_set_vertex_buffers()
1033    mgr->user_vb_mask |= user_vb_mask;  in u_vbuf_set_vertex_buffers()
1034    mgr->incompatible_vb_mask |= incompatible_vb_mask;  in u_vbuf_set_vertex_buffers()
1035    mgr->nonzero_stride_vb_mask |= nonzero_stride_vb_mask;  in u_vbuf_set_vertex_buffers()
1036    mgr->enabled_vb_mask |= enabled_vb_mask;  in u_vbuf_set_vertex_buffers()
1040    mgr->dirty_real_vb_mask |= ~mask;  in u_vbuf_set_vertex_buffers()
1044 get_upload_offset_size(struct u_vbuf *mgr,  in get_upload_offset_size()  argument
1054    if ((1 << vb_index) & mgr->fallback_vbs_mask || !vb->is_user_buffer)  in get_upload_offset_size()
1087 u_vbuf_upload_buffers(struct u_vbuf *mgr,  in u_vbuf_upload_buffers()  argument
1092    struct u_vbuf_elements *ve = mgr->ve;  in u_vbuf_upload_buffers()
1095          mgr->using_translate ? mgr->fallback_velems.velems : ve->ve;  in u_vbuf_upload_buffers()
1098    if ((ve->interleaved_vb_mask & mgr->user_vb_mask) == 0) {  in u_vbuf_upload_buffers()
1102          struct pipe_vertex_buffer *vb = &mgr->vertex_buffer[index];  in u_vbuf_upload_buffers()
1105          if (!get_upload_offset_size(mgr, vb, ve, velem, index, i, start_vertex,  in u_vbuf_upload_buffers()
1110          struct pipe_vertex_buffer *real_vb = &mgr->real_vertex_buffer[index];  in u_vbuf_upload_buffers()
1111          const uint8_t *ptr = mgr->vertex_buffer[index].buffer.user;  in u_vbuf_upload_buffers()
1113          u_upload_data(mgr->pipe->stream_uploader,  in u_vbuf_upload_buffers()
1114                        mgr->has_signed_vb_offset ? 0 : offset,  in u_vbuf_upload_buffers()
1134       struct pipe_vertex_buffer *vb = &mgr->vertex_buffer[index];  in u_vbuf_upload_buffers()
1137       if (!get_upload_offset_size(mgr, vb, ve, velem, index, i, start_vertex,  in u_vbuf_upload_buffers()
1170       real_vb = &mgr->real_vertex_buffer[i];  in u_vbuf_upload_buffers()
1171       ptr = mgr->vertex_buffer[i].buffer.user;  in u_vbuf_upload_buffers()
1173       u_upload_data(mgr->pipe->stream_uploader,  in u_vbuf_upload_buffers()
1174                     mgr->has_signed_vb_offset ? 0 : start,  in u_vbuf_upload_buffers()
1186 static boolean u_vbuf_need_minmax_index(const struct u_vbuf *mgr)  in u_vbuf_need_minmax_index()  argument
1191    return (mgr->ve->used_vb_mask &  in u_vbuf_need_minmax_index()
1192            ((mgr->user_vb_mask |  in u_vbuf_need_minmax_index()
1193              mgr->incompatible_vb_mask |  in u_vbuf_need_minmax_index()
1194              mgr->ve->incompatible_vb_mask_any) &  in u_vbuf_need_minmax_index()
1195             mgr->ve->noninstance_vb_mask_any &  in u_vbuf_need_minmax_index()
1196             mgr->nonzero_stride_vb_mask)) != 0;  in u_vbuf_need_minmax_index()
1199 static boolean u_vbuf_mapping_vertex_buffer_blocks(const struct u_vbuf *mgr)  in u_vbuf_mapping_vertex_buffer_blocks()  argument
1205    return (mgr->ve->used_vb_mask &  in u_vbuf_mapping_vertex_buffer_blocks()
1206            (~mgr->user_vb_mask &  in u_vbuf_mapping_vertex_buffer_blocks()
1207             ~mgr->incompatible_vb_mask &  in u_vbuf_mapping_vertex_buffer_blocks()
1208             mgr->ve->compatible_vb_mask_all &  in u_vbuf_mapping_vertex_buffer_blocks()
1209             mgr->ve->noninstance_vb_mask_any &  in u_vbuf_mapping_vertex_buffer_blocks()
1210             mgr->nonzero_stride_vb_mask)) != 0;  in u_vbuf_mapping_vertex_buffer_blocks()
1323 static void u_vbuf_set_driver_vertex_buffers(struct u_vbuf *mgr)  in u_vbuf_set_driver_vertex_buffers()  argument
1325    struct pipe_context *pipe = mgr->pipe;  in u_vbuf_set_driver_vertex_buffers()
1328    start_slot = ffs(mgr->dirty_real_vb_mask) - 1;  in u_vbuf_set_driver_vertex_buffers()
1329    count = util_last_bit(mgr->dirty_real_vb_mask >> start_slot);  in u_vbuf_set_driver_vertex_buffers()
1331    if (mgr->dirty_real_vb_mask == mgr->enabled_vb_mask &&  in u_vbuf_set_driver_vertex_buffers()
1332        mgr->dirty_real_vb_mask == mgr->user_vb_mask) {  in u_vbuf_set_driver_vertex_buffers()
1338                                mgr->real_vertex_buffer + start_slot);  in u_vbuf_set_driver_vertex_buffers()
1342          assert(!mgr->real_vertex_buffer[start_slot + i].is_user_buffer);  in u_vbuf_set_driver_vertex_buffers()
1343          mgr->real_vertex_buffer[start_slot + i].buffer.resource = NULL;  in u_vbuf_set_driver_vertex_buffers()
1348                                mgr->real_vertex_buffer + start_slot);  in u_vbuf_set_driver_vertex_buffers()
1350    mgr->dirty_real_vb_mask = 0;  in u_vbuf_set_driver_vertex_buffers()
1354 u_vbuf_split_indexed_multidraw(struct u_vbuf *mgr, struct pipe_draw_info *info,  in u_vbuf_split_indexed_multidraw()  argument
1377       u_vbuf_draw_vbo(mgr, info, drawid_offset, NULL, draw);  in u_vbuf_split_indexed_multidraw()
1381 void u_vbuf_draw_vbo(struct u_vbuf *mgr, const struct pipe_draw_info *info,  in u_vbuf_draw_vbo()  argument
1386    struct pipe_context *pipe = mgr->pipe;  in u_vbuf_draw_vbo()
1391    const uint32_t used_vb_mask = mgr->ve->used_vb_mask;  in u_vbuf_draw_vbo()
1392    uint32_t user_vb_mask = mgr->user_vb_mask & used_vb_mask;  in u_vbuf_draw_vbo()
1394       mgr->incompatible_vb_mask & used_vb_mask;  in u_vbuf_draw_vbo()
1401        !mgr->ve->incompatible_elem_mask &&  in u_vbuf_draw_vbo()
1403        (info->index_size != 1 || !mgr->caps.rewrite_ubyte_ibs) &&  in u_vbuf_draw_vbo()
1406         !mgr->caps.rewrite_restart_index) &&  in u_vbuf_draw_vbo()
1407        (!info->primitive_restart || mgr->caps.supported_restart_modes & BITFIELD_BIT(info->mode)) &&  in u_vbuf_draw_vbo()
1408        mgr->caps.supported_prim_modes & BITFIELD_BIT(info->mode)) {  in u_vbuf_draw_vbo()
1411       if (mgr->dirty_real_vb_mask & used_vb_mask) {  in u_vbuf_draw_vbo()
1412          u_vbuf_set_driver_vertex_buffers(mgr);  in u_vbuf_draw_vbo()
1457              mgr->ve->incompatible_elem_mask) {  in u_vbuf_draw_vbo()
1458             u_vbuf_split_indexed_multidraw(mgr, &new_info, drawid_offset, data,  in u_vbuf_draw_vbo()
1474             u_vbuf_split_indexed_multidraw(mgr, &new_info, drawid_offset, data,  in u_vbuf_draw_vbo()
1586       if (u_vbuf_need_minmax_index(mgr)) {  in u_vbuf_draw_vbo()
1593             u_vbuf_get_minmax_index(mgr->pipe, &new_info, &new_draw,  in u_vbuf_draw_vbo()
1609              !u_vbuf_mapping_vertex_buffer_blocks(mgr)) {  in u_vbuf_draw_vbo()
1611             user_vb_mask &= ~(mgr->nonzero_stride_vb_mask &  in u_vbuf_draw_vbo()
1612                               mgr->ve->noninstance_vb_mask_any);  in u_vbuf_draw_vbo()
1629        mgr->ve->incompatible_elem_mask) {  in u_vbuf_draw_vbo()
1630       if (!u_vbuf_translate_begin(mgr, &new_info, &new_draw,  in u_vbuf_draw_vbo()
1647                         mgr->ve->incompatible_vb_mask_all);  in u_vbuf_draw_vbo()
1652       if (u_vbuf_upload_buffers(mgr, start_vertex, num_vertices,  in u_vbuf_draw_vbo()
1659       mgr->dirty_real_vb_mask |= user_vb_mask;  in u_vbuf_draw_vbo()
1684    if (mgr->dirty_real_vb_mask)  in u_vbuf_draw_vbo()
1685       u_vbuf_set_driver_vertex_buffers(mgr);  in u_vbuf_draw_vbo()
1687    if ((new_info.index_size == 1 && mgr->caps.rewrite_ubyte_ibs) ||  in u_vbuf_draw_vbo()
1689         ((new_info.restart_index != fixed_restart_index && mgr->caps.rewrite_restart_index) ||  in u_vbuf_draw_vbo()
1690         !(mgr->caps.supported_restart_modes & BITFIELD_BIT(new_info.mode)))) ||  in u_vbuf_draw_vbo()
1691        !(mgr->caps.supported_prim_modes & BITFIELD_BIT(new_info.mode))) {  in u_vbuf_draw_vbo()
1692       util_primconvert_save_flatshade_first(mgr->pc, mgr->flatshade_first);  in u_vbuf_draw_vbo()
1693       util_primconvert_draw_vbo(mgr->pc, &new_info, drawid_offset, indirect, &new_draw, 1);  in u_vbuf_draw_vbo()
1697    if (mgr->using_translate) {  in u_vbuf_draw_vbo()
1698       u_vbuf_translate_end(mgr);  in u_vbuf_draw_vbo()
1709 void u_vbuf_save_vertex_elements(struct u_vbuf *mgr)  in u_vbuf_save_vertex_elements()  argument
1711    assert(!mgr->ve_saved);  in u_vbuf_save_vertex_elements()
1712    mgr->ve_saved = mgr->ve;  in u_vbuf_save_vertex_elements()
1715 void u_vbuf_restore_vertex_elements(struct u_vbuf *mgr)  in u_vbuf_restore_vertex_elements()  argument
1717    if (mgr->ve != mgr->ve_saved) {  in u_vbuf_restore_vertex_elements()
1718       struct pipe_context *pipe = mgr->pipe;  in u_vbuf_restore_vertex_elements()
1720       mgr->ve = mgr->ve_saved;  in u_vbuf_restore_vertex_elements()
1722                                        mgr->ve ? mgr->ve->driver_cso : NULL);  in u_vbuf_restore_vertex_elements()
1724    mgr->ve_saved = NULL;  in u_vbuf_restore_vertex_elements()