1 /**************************************************************************
2 *
3 * Copyright 2017 Advanced Micro Devices, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * on the rights to use, copy, modify, merge, publish, distribute, sub
10 * license, and/or sell copies of the Software, and to permit persons to whom
11 * the Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
23 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 *
25 **************************************************************************/
26
27 #include "util/u_threaded_context.h"
28 #include "util/u_cpu_detect.h"
29 #include "util/format/u_format.h"
30 #include "util/u_inlines.h"
31 #include "util/u_memory.h"
32 #include "util/u_upload_mgr.h"
33 #include "compiler/shader_info.h"
34
35 /* 0 = disabled, 1 = assertions, 2 = printfs */
36 #define TC_DEBUG 0
37
38 #if TC_DEBUG >= 1
39 #define tc_assert assert
40 #else
41 #define tc_assert(x)
42 #endif
43
44 #if TC_DEBUG >= 2
45 #define tc_printf printf
46 #define tc_asprintf asprintf
47 #define tc_strcmp strcmp
48 #else
49 #define tc_printf(...)
50 #define tc_asprintf(...) 0
51 #define tc_strcmp(...) 0
52 #endif
53
54 #define TC_SENTINEL 0x5ca1ab1e
55
56 enum tc_call_id {
57 #define CALL(name) TC_CALL_##name,
58 #include "u_threaded_context_calls.h"
59 #undef CALL
60 TC_NUM_CALLS,
61 };
62
63 /* This is actually variable-sized, because indirect isn't allocated if it's
64 * not needed. */
65 struct tc_full_draw_info {
66 struct pipe_draw_info draw;
67 struct pipe_draw_indirect_info indirect;
68 };
69
70 typedef void (*tc_execute)(struct pipe_context *pipe, union tc_payload *payload);
71
72 static const tc_execute execute_func[TC_NUM_CALLS];
73
74 static void
tc_batch_check(UNUSED struct tc_batch * batch)75 tc_batch_check(UNUSED struct tc_batch *batch)
76 {
77 tc_assert(batch->sentinel == TC_SENTINEL);
78 tc_assert(batch->num_total_call_slots <= TC_CALLS_PER_BATCH);
79 }
80
81 static void
tc_debug_check(struct threaded_context * tc)82 tc_debug_check(struct threaded_context *tc)
83 {
84 for (unsigned i = 0; i < TC_MAX_BATCHES; i++) {
85 tc_batch_check(&tc->batch_slots[i]);
86 tc_assert(tc->batch_slots[i].pipe == tc->pipe);
87 }
88 }
89
90 static bool
is_next_call_a_mergeable_draw(struct tc_full_draw_info * first_info,struct tc_call * next,struct tc_full_draw_info ** next_info)91 is_next_call_a_mergeable_draw(struct tc_full_draw_info *first_info,
92 struct tc_call *next,
93 struct tc_full_draw_info **next_info)
94 {
95 return next->call_id == TC_CALL_draw_vbo &&
96 (*next_info = (struct tc_full_draw_info*)&next->payload) &&
97 /* All fields must be the same except start and count. */
98 memcmp((uint32_t*)&first_info->draw + 2,
99 (uint32_t*)&(*next_info)->draw + 2,
100 sizeof(struct pipe_draw_info) - 8) == 0;
101 }
102
103 static void
tc_batch_execute(void * job,UNUSED int thread_index)104 tc_batch_execute(void *job, UNUSED int thread_index)
105 {
106 struct tc_batch *batch = job;
107 struct pipe_context *pipe = batch->pipe;
108 struct tc_call *last = &batch->call[batch->num_total_call_slots];
109
110 tc_batch_check(batch);
111
112 assert(!batch->token);
113
114 for (struct tc_call *iter = batch->call; iter != last;) {
115 tc_assert(iter->sentinel == TC_SENTINEL);
116
117 /* Draw call merging. */
118 if (iter->call_id == TC_CALL_draw_vbo) {
119 struct tc_call *first = iter;
120 struct tc_call *next = first + first->num_call_slots;
121 struct tc_full_draw_info *first_info =
122 (struct tc_full_draw_info*)&first->payload;
123 struct tc_full_draw_info *next_info;
124
125 /* If at least 2 consecutive draw calls can be merged... */
126 if (next != last && next->call_id == TC_CALL_draw_vbo &&
127 first_info->draw.drawid == 0 &&
128 !first_info->draw.indirect &&
129 !first_info->draw.count_from_stream_output &&
130 is_next_call_a_mergeable_draw(first_info, next, &next_info)) {
131 /* Merge up to 256 draw calls. */
132 struct pipe_draw_start_count multi[256];
133 unsigned num_draws = 2;
134
135 multi[0].start = first_info->draw.start;
136 multi[0].count = first_info->draw.count;
137 multi[1].start = next_info->draw.start;
138 multi[1].count = next_info->draw.count;
139
140 if (next_info->draw.index_size)
141 pipe_resource_reference(&next_info->draw.index.resource, NULL);
142
143 /* Find how many other draws can be merged. */
144 next = next + next->num_call_slots;
145 for (; next != last && num_draws < ARRAY_SIZE(multi) &&
146 is_next_call_a_mergeable_draw(first_info, next, &next_info);
147 next += next->num_call_slots, num_draws++) {
148 multi[num_draws].start = next_info->draw.start;
149 multi[num_draws].count = next_info->draw.count;
150
151 if (next_info->draw.index_size)
152 pipe_resource_reference(&next_info->draw.index.resource, NULL);
153 }
154
155 pipe->multi_draw(pipe, &first_info->draw, multi, num_draws);
156 if (first_info->draw.index_size)
157 pipe_resource_reference(&first_info->draw.index.resource, NULL);
158 iter = next;
159 continue;
160 }
161 }
162
163 execute_func[iter->call_id](pipe, &iter->payload);
164 iter += iter->num_call_slots;
165 }
166
167 tc_batch_check(batch);
168 batch->num_total_call_slots = 0;
169 }
170
171 static void
tc_batch_flush(struct threaded_context * tc)172 tc_batch_flush(struct threaded_context *tc)
173 {
174 struct tc_batch *next = &tc->batch_slots[tc->next];
175
176 tc_assert(next->num_total_call_slots != 0);
177 tc_batch_check(next);
178 tc_debug_check(tc);
179 tc->bytes_mapped_estimate = 0;
180 p_atomic_add(&tc->num_offloaded_slots, next->num_total_call_slots);
181
182 if (next->token) {
183 next->token->tc = NULL;
184 tc_unflushed_batch_token_reference(&next->token, NULL);
185 }
186
187 util_queue_add_job(&tc->queue, next, &next->fence, tc_batch_execute,
188 NULL, 0);
189 tc->last = tc->next;
190 tc->next = (tc->next + 1) % TC_MAX_BATCHES;
191 }
192
193 /* This is the function that adds variable-sized calls into the current
194 * batch. It also flushes the batch if there is not enough space there.
195 * All other higher-level "add" functions use it.
196 */
197 static union tc_payload *
tc_add_sized_call(struct threaded_context * tc,enum tc_call_id id,unsigned payload_size)198 tc_add_sized_call(struct threaded_context *tc, enum tc_call_id id,
199 unsigned payload_size)
200 {
201 struct tc_batch *next = &tc->batch_slots[tc->next];
202 unsigned total_size = offsetof(struct tc_call, payload) + payload_size;
203 unsigned num_call_slots = DIV_ROUND_UP(total_size, sizeof(struct tc_call));
204
205 tc_debug_check(tc);
206
207 if (unlikely(next->num_total_call_slots + num_call_slots > TC_CALLS_PER_BATCH)) {
208 tc_batch_flush(tc);
209 next = &tc->batch_slots[tc->next];
210 tc_assert(next->num_total_call_slots == 0);
211 }
212
213 tc_assert(util_queue_fence_is_signalled(&next->fence));
214
215 struct tc_call *call = &next->call[next->num_total_call_slots];
216 next->num_total_call_slots += num_call_slots;
217
218 call->sentinel = TC_SENTINEL;
219 call->call_id = id;
220 call->num_call_slots = num_call_slots;
221
222 tc_debug_check(tc);
223 return &call->payload;
224 }
225
226 #define tc_add_struct_typed_call(tc, execute, type) \
227 ((struct type*)tc_add_sized_call(tc, execute, sizeof(struct type)))
228
229 #define tc_add_slot_based_call(tc, execute, type, num_slots) \
230 ((struct type*)tc_add_sized_call(tc, execute, \
231 sizeof(struct type) + \
232 sizeof(((struct type*)NULL)->slot[0]) * \
233 (num_slots)))
234
235 static union tc_payload *
tc_add_small_call(struct threaded_context * tc,enum tc_call_id id)236 tc_add_small_call(struct threaded_context *tc, enum tc_call_id id)
237 {
238 return tc_add_sized_call(tc, id, 0);
239 }
240
241 static bool
tc_is_sync(struct threaded_context * tc)242 tc_is_sync(struct threaded_context *tc)
243 {
244 struct tc_batch *last = &tc->batch_slots[tc->last];
245 struct tc_batch *next = &tc->batch_slots[tc->next];
246
247 return util_queue_fence_is_signalled(&last->fence) &&
248 !next->num_total_call_slots;
249 }
250
251 static void
_tc_sync(struct threaded_context * tc,UNUSED const char * info,UNUSED const char * func)252 _tc_sync(struct threaded_context *tc, UNUSED const char *info, UNUSED const char *func)
253 {
254 struct tc_batch *last = &tc->batch_slots[tc->last];
255 struct tc_batch *next = &tc->batch_slots[tc->next];
256 bool synced = false;
257
258 tc_debug_check(tc);
259
260 /* Only wait for queued calls... */
261 if (!util_queue_fence_is_signalled(&last->fence)) {
262 util_queue_fence_wait(&last->fence);
263 synced = true;
264 }
265
266 tc_debug_check(tc);
267
268 if (next->token) {
269 next->token->tc = NULL;
270 tc_unflushed_batch_token_reference(&next->token, NULL);
271 }
272
273 /* .. and execute unflushed calls directly. */
274 if (next->num_total_call_slots) {
275 p_atomic_add(&tc->num_direct_slots, next->num_total_call_slots);
276 tc->bytes_mapped_estimate = 0;
277 tc_batch_execute(next, 0);
278 synced = true;
279 }
280
281 if (synced) {
282 p_atomic_inc(&tc->num_syncs);
283
284 if (tc_strcmp(func, "tc_destroy") != 0) {
285 tc_printf("sync %s %s\n", func, info);
286 }
287 }
288
289 tc_debug_check(tc);
290 }
291
292 #define tc_sync(tc) _tc_sync(tc, "", __func__)
293 #define tc_sync_msg(tc, info) _tc_sync(tc, info, __func__)
294
295 /**
296 * Call this from fence_finish for same-context fence waits of deferred fences
297 * that haven't been flushed yet.
298 *
299 * The passed pipe_context must be the one passed to pipe_screen::fence_finish,
300 * i.e., the wrapped one.
301 */
302 void
threaded_context_flush(struct pipe_context * _pipe,struct tc_unflushed_batch_token * token,bool prefer_async)303 threaded_context_flush(struct pipe_context *_pipe,
304 struct tc_unflushed_batch_token *token,
305 bool prefer_async)
306 {
307 struct threaded_context *tc = threaded_context(_pipe);
308
309 /* This is called from the gallium frontend / application thread. */
310 if (token->tc && token->tc == tc) {
311 struct tc_batch *last = &tc->batch_slots[tc->last];
312
313 /* Prefer to do the flush in the driver thread if it is already
314 * running. That should be better for cache locality.
315 */
316 if (prefer_async || !util_queue_fence_is_signalled(&last->fence))
317 tc_batch_flush(tc);
318 else
319 tc_sync(token->tc);
320 }
321 }
322
323 static void
tc_set_resource_reference(struct pipe_resource ** dst,struct pipe_resource * src)324 tc_set_resource_reference(struct pipe_resource **dst, struct pipe_resource *src)
325 {
326 *dst = NULL;
327 pipe_resource_reference(dst, src);
328 }
329
330 void
threaded_resource_init(struct pipe_resource * res)331 threaded_resource_init(struct pipe_resource *res)
332 {
333 struct threaded_resource *tres = threaded_resource(res);
334
335 tres->latest = &tres->b;
336 util_range_init(&tres->valid_buffer_range);
337 tres->base_valid_buffer_range = &tres->valid_buffer_range;
338 tres->is_shared = false;
339 tres->is_user_ptr = false;
340 }
341
342 void
threaded_resource_deinit(struct pipe_resource * res)343 threaded_resource_deinit(struct pipe_resource *res)
344 {
345 struct threaded_resource *tres = threaded_resource(res);
346
347 if (tres->latest != &tres->b)
348 pipe_resource_reference(&tres->latest, NULL);
349 util_range_destroy(&tres->valid_buffer_range);
350 }
351
352 struct pipe_context *
threaded_context_unwrap_sync(struct pipe_context * pipe)353 threaded_context_unwrap_sync(struct pipe_context *pipe)
354 {
355 if (!pipe || !pipe->priv)
356 return pipe;
357
358 tc_sync(threaded_context(pipe));
359 return (struct pipe_context*)pipe->priv;
360 }
361
362
363 /********************************************************************
364 * simple functions
365 */
366
367 #define TC_FUNC1(func, m_payload, qualifier, type, deref, deref2) \
368 static void \
369 tc_call_##func(struct pipe_context *pipe, union tc_payload *payload) \
370 { \
371 pipe->func(pipe, deref2((type*)payload)); \
372 } \
373 \
374 static void \
375 tc_##func(struct pipe_context *_pipe, qualifier type deref param) \
376 { \
377 struct threaded_context *tc = threaded_context(_pipe); \
378 type *p = (type*)tc_add_sized_call(tc, TC_CALL_##func, sizeof(type)); \
379 *p = deref(param); \
380 }
381
382 TC_FUNC1(set_active_query_state, flags, , bool, , *)
383
384 TC_FUNC1(set_blend_color, blend_color, const, struct pipe_blend_color, *, )
385 TC_FUNC1(set_stencil_ref, stencil_ref, const, struct pipe_stencil_ref, *, )
386 TC_FUNC1(set_clip_state, clip_state, const, struct pipe_clip_state, *, )
387 TC_FUNC1(set_sample_mask, sample_mask, , unsigned, , *)
388 TC_FUNC1(set_min_samples, min_samples, , unsigned, , *)
389 TC_FUNC1(set_polygon_stipple, polygon_stipple, const, struct pipe_poly_stipple, *, )
390
391 TC_FUNC1(texture_barrier, flags, , unsigned, , *)
392 TC_FUNC1(memory_barrier, flags, , unsigned, , *)
393
394
395 /********************************************************************
396 * queries
397 */
398
399 static struct pipe_query *
tc_create_query(struct pipe_context * _pipe,unsigned query_type,unsigned index)400 tc_create_query(struct pipe_context *_pipe, unsigned query_type,
401 unsigned index)
402 {
403 struct threaded_context *tc = threaded_context(_pipe);
404 struct pipe_context *pipe = tc->pipe;
405
406 return pipe->create_query(pipe, query_type, index);
407 }
408
409 static struct pipe_query *
tc_create_batch_query(struct pipe_context * _pipe,unsigned num_queries,unsigned * query_types)410 tc_create_batch_query(struct pipe_context *_pipe, unsigned num_queries,
411 unsigned *query_types)
412 {
413 struct threaded_context *tc = threaded_context(_pipe);
414 struct pipe_context *pipe = tc->pipe;
415
416 return pipe->create_batch_query(pipe, num_queries, query_types);
417 }
418
419 static void
tc_call_destroy_query(struct pipe_context * pipe,union tc_payload * payload)420 tc_call_destroy_query(struct pipe_context *pipe, union tc_payload *payload)
421 {
422 struct threaded_query *tq = threaded_query(payload->query);
423
424 if (tq->head_unflushed.next)
425 list_del(&tq->head_unflushed);
426
427 pipe->destroy_query(pipe, payload->query);
428 }
429
430 static void
tc_destroy_query(struct pipe_context * _pipe,struct pipe_query * query)431 tc_destroy_query(struct pipe_context *_pipe, struct pipe_query *query)
432 {
433 struct threaded_context *tc = threaded_context(_pipe);
434
435 tc_add_small_call(tc, TC_CALL_destroy_query)->query = query;
436 }
437
438 static void
tc_call_begin_query(struct pipe_context * pipe,union tc_payload * payload)439 tc_call_begin_query(struct pipe_context *pipe, union tc_payload *payload)
440 {
441 pipe->begin_query(pipe, payload->query);
442 }
443
444 static bool
tc_begin_query(struct pipe_context * _pipe,struct pipe_query * query)445 tc_begin_query(struct pipe_context *_pipe, struct pipe_query *query)
446 {
447 struct threaded_context *tc = threaded_context(_pipe);
448 union tc_payload *payload = tc_add_small_call(tc, TC_CALL_begin_query);
449
450 payload->query = query;
451 return true; /* we don't care about the return value for this call */
452 }
453
454 struct tc_end_query_payload {
455 struct threaded_context *tc;
456 struct pipe_query *query;
457 };
458
459 static void
tc_call_end_query(struct pipe_context * pipe,union tc_payload * payload)460 tc_call_end_query(struct pipe_context *pipe, union tc_payload *payload)
461 {
462 struct tc_end_query_payload *p = (struct tc_end_query_payload *)payload;
463 struct threaded_query *tq = threaded_query(p->query);
464
465 if (!tq->head_unflushed.next)
466 list_add(&tq->head_unflushed, &p->tc->unflushed_queries);
467
468 pipe->end_query(pipe, p->query);
469 }
470
471 static bool
tc_end_query(struct pipe_context * _pipe,struct pipe_query * query)472 tc_end_query(struct pipe_context *_pipe, struct pipe_query *query)
473 {
474 struct threaded_context *tc = threaded_context(_pipe);
475 struct threaded_query *tq = threaded_query(query);
476 struct tc_end_query_payload *payload =
477 tc_add_struct_typed_call(tc, TC_CALL_end_query, tc_end_query_payload);
478
479 payload->tc = tc;
480 payload->query = query;
481
482 tq->flushed = false;
483
484 return true; /* we don't care about the return value for this call */
485 }
486
487 static bool
tc_get_query_result(struct pipe_context * _pipe,struct pipe_query * query,bool wait,union pipe_query_result * result)488 tc_get_query_result(struct pipe_context *_pipe,
489 struct pipe_query *query, bool wait,
490 union pipe_query_result *result)
491 {
492 struct threaded_context *tc = threaded_context(_pipe);
493 struct threaded_query *tq = threaded_query(query);
494 struct pipe_context *pipe = tc->pipe;
495
496 if (!tq->flushed)
497 tc_sync_msg(tc, wait ? "wait" : "nowait");
498
499 bool success = pipe->get_query_result(pipe, query, wait, result);
500
501 if (success) {
502 tq->flushed = true;
503 if (tq->head_unflushed.next) {
504 /* This is safe because it can only happen after we sync'd. */
505 list_del(&tq->head_unflushed);
506 }
507 }
508 return success;
509 }
510
511 struct tc_query_result_resource {
512 struct pipe_query *query;
513 bool wait;
514 enum pipe_query_value_type result_type;
515 int index;
516 struct pipe_resource *resource;
517 unsigned offset;
518 };
519
520 static void
tc_call_get_query_result_resource(struct pipe_context * pipe,union tc_payload * payload)521 tc_call_get_query_result_resource(struct pipe_context *pipe,
522 union tc_payload *payload)
523 {
524 struct tc_query_result_resource *p = (struct tc_query_result_resource *)payload;
525
526 pipe->get_query_result_resource(pipe, p->query, p->wait, p->result_type,
527 p->index, p->resource, p->offset);
528 pipe_resource_reference(&p->resource, NULL);
529 }
530
531 static void
tc_get_query_result_resource(struct pipe_context * _pipe,struct pipe_query * query,bool wait,enum pipe_query_value_type result_type,int index,struct pipe_resource * resource,unsigned offset)532 tc_get_query_result_resource(struct pipe_context *_pipe,
533 struct pipe_query *query, bool wait,
534 enum pipe_query_value_type result_type, int index,
535 struct pipe_resource *resource, unsigned offset)
536 {
537 struct threaded_context *tc = threaded_context(_pipe);
538 struct tc_query_result_resource *p =
539 tc_add_struct_typed_call(tc, TC_CALL_get_query_result_resource,
540 tc_query_result_resource);
541
542 p->query = query;
543 p->wait = wait;
544 p->result_type = result_type;
545 p->index = index;
546 tc_set_resource_reference(&p->resource, resource);
547 p->offset = offset;
548 }
549
550 struct tc_render_condition {
551 struct pipe_query *query;
552 bool condition;
553 unsigned mode;
554 };
555
556 static void
tc_call_render_condition(struct pipe_context * pipe,union tc_payload * payload)557 tc_call_render_condition(struct pipe_context *pipe, union tc_payload *payload)
558 {
559 struct tc_render_condition *p = (struct tc_render_condition *)payload;
560 pipe->render_condition(pipe, p->query, p->condition, p->mode);
561 }
562
563 static void
tc_render_condition(struct pipe_context * _pipe,struct pipe_query * query,bool condition,enum pipe_render_cond_flag mode)564 tc_render_condition(struct pipe_context *_pipe,
565 struct pipe_query *query, bool condition,
566 enum pipe_render_cond_flag mode)
567 {
568 struct threaded_context *tc = threaded_context(_pipe);
569 struct tc_render_condition *p =
570 tc_add_struct_typed_call(tc, TC_CALL_render_condition, tc_render_condition);
571
572 p->query = query;
573 p->condition = condition;
574 p->mode = mode;
575 }
576
577
578 /********************************************************************
579 * constant (immutable) states
580 */
581
582 #define TC_CSO_CREATE(name, sname) \
583 static void * \
584 tc_create_##name##_state(struct pipe_context *_pipe, \
585 const struct pipe_##sname##_state *state) \
586 { \
587 struct pipe_context *pipe = threaded_context(_pipe)->pipe; \
588 return pipe->create_##name##_state(pipe, state); \
589 }
590
591 #define TC_CSO_BIND(name) TC_FUNC1(bind_##name##_state, cso, , void *, , *)
592 #define TC_CSO_DELETE(name) TC_FUNC1(delete_##name##_state, cso, , void *, , *)
593
594 #define TC_CSO_WHOLE2(name, sname) \
595 TC_CSO_CREATE(name, sname) \
596 TC_CSO_BIND(name) \
597 TC_CSO_DELETE(name)
598
599 #define TC_CSO_WHOLE(name) TC_CSO_WHOLE2(name, name)
600
601 TC_CSO_WHOLE(blend)
TC_CSO_WHOLE(rasterizer)602 TC_CSO_WHOLE(rasterizer)
603 TC_CSO_WHOLE(depth_stencil_alpha)
604 TC_CSO_WHOLE(compute)
605 TC_CSO_WHOLE2(fs, shader)
606 TC_CSO_WHOLE2(vs, shader)
607 TC_CSO_WHOLE2(gs, shader)
608 TC_CSO_WHOLE2(tcs, shader)
609 TC_CSO_WHOLE2(tes, shader)
610 TC_CSO_CREATE(sampler, sampler)
611 TC_CSO_DELETE(sampler)
612 TC_CSO_BIND(vertex_elements)
613 TC_CSO_DELETE(vertex_elements)
614
615 static void *
616 tc_create_vertex_elements_state(struct pipe_context *_pipe, unsigned count,
617 const struct pipe_vertex_element *elems)
618 {
619 struct pipe_context *pipe = threaded_context(_pipe)->pipe;
620
621 return pipe->create_vertex_elements_state(pipe, count, elems);
622 }
623
624 struct tc_sampler_states {
625 ubyte shader, start, count;
626 void *slot[0]; /* more will be allocated if needed */
627 };
628
629 static void
tc_call_bind_sampler_states(struct pipe_context * pipe,union tc_payload * payload)630 tc_call_bind_sampler_states(struct pipe_context *pipe, union tc_payload *payload)
631 {
632 struct tc_sampler_states *p = (struct tc_sampler_states *)payload;
633 pipe->bind_sampler_states(pipe, p->shader, p->start, p->count, p->slot);
634 }
635
636 static void
tc_bind_sampler_states(struct pipe_context * _pipe,enum pipe_shader_type shader,unsigned start,unsigned count,void ** states)637 tc_bind_sampler_states(struct pipe_context *_pipe,
638 enum pipe_shader_type shader,
639 unsigned start, unsigned count, void **states)
640 {
641 if (!count)
642 return;
643
644 struct threaded_context *tc = threaded_context(_pipe);
645 struct tc_sampler_states *p =
646 tc_add_slot_based_call(tc, TC_CALL_bind_sampler_states, tc_sampler_states, count);
647
648 p->shader = shader;
649 p->start = start;
650 p->count = count;
651 memcpy(p->slot, states, count * sizeof(states[0]));
652 }
653
654
655 /********************************************************************
656 * immediate states
657 */
658
659 static void
tc_call_set_framebuffer_state(struct pipe_context * pipe,union tc_payload * payload)660 tc_call_set_framebuffer_state(struct pipe_context *pipe, union tc_payload *payload)
661 {
662 struct pipe_framebuffer_state *p = (struct pipe_framebuffer_state *)payload;
663
664 pipe->set_framebuffer_state(pipe, p);
665
666 unsigned nr_cbufs = p->nr_cbufs;
667 for (unsigned i = 0; i < nr_cbufs; i++)
668 pipe_surface_reference(&p->cbufs[i], NULL);
669 pipe_surface_reference(&p->zsbuf, NULL);
670 }
671
672 static void
tc_set_framebuffer_state(struct pipe_context * _pipe,const struct pipe_framebuffer_state * fb)673 tc_set_framebuffer_state(struct pipe_context *_pipe,
674 const struct pipe_framebuffer_state *fb)
675 {
676 struct threaded_context *tc = threaded_context(_pipe);
677 struct pipe_framebuffer_state *p =
678 tc_add_struct_typed_call(tc, TC_CALL_set_framebuffer_state,
679 pipe_framebuffer_state);
680 unsigned nr_cbufs = fb->nr_cbufs;
681
682 p->width = fb->width;
683 p->height = fb->height;
684 p->samples = fb->samples;
685 p->layers = fb->layers;
686 p->nr_cbufs = nr_cbufs;
687
688 for (unsigned i = 0; i < nr_cbufs; i++) {
689 p->cbufs[i] = NULL;
690 pipe_surface_reference(&p->cbufs[i], fb->cbufs[i]);
691 }
692 p->zsbuf = NULL;
693 pipe_surface_reference(&p->zsbuf, fb->zsbuf);
694 }
695
696 static void
tc_call_set_tess_state(struct pipe_context * pipe,union tc_payload * payload)697 tc_call_set_tess_state(struct pipe_context *pipe, union tc_payload *payload)
698 {
699 float *p = (float*)payload;
700 pipe->set_tess_state(pipe, p, p + 4);
701 }
702
703 static void
tc_set_tess_state(struct pipe_context * _pipe,const float default_outer_level[4],const float default_inner_level[2])704 tc_set_tess_state(struct pipe_context *_pipe,
705 const float default_outer_level[4],
706 const float default_inner_level[2])
707 {
708 struct threaded_context *tc = threaded_context(_pipe);
709 float *p = (float*)tc_add_sized_call(tc, TC_CALL_set_tess_state,
710 sizeof(float) * 6);
711
712 memcpy(p, default_outer_level, 4 * sizeof(float));
713 memcpy(p + 4, default_inner_level, 2 * sizeof(float));
714 }
715
716 struct tc_constant_buffer {
717 ubyte shader, index;
718 struct pipe_constant_buffer cb;
719 };
720
721 static void
tc_call_set_constant_buffer(struct pipe_context * pipe,union tc_payload * payload)722 tc_call_set_constant_buffer(struct pipe_context *pipe, union tc_payload *payload)
723 {
724 struct tc_constant_buffer *p = (struct tc_constant_buffer *)payload;
725
726 pipe->set_constant_buffer(pipe,
727 p->shader,
728 p->index,
729 &p->cb);
730 pipe_resource_reference(&p->cb.buffer, NULL);
731 }
732
733 static void
tc_set_constant_buffer(struct pipe_context * _pipe,enum pipe_shader_type shader,uint index,const struct pipe_constant_buffer * cb)734 tc_set_constant_buffer(struct pipe_context *_pipe,
735 enum pipe_shader_type shader, uint index,
736 const struct pipe_constant_buffer *cb)
737 {
738 struct threaded_context *tc = threaded_context(_pipe);
739 struct pipe_resource *buffer = NULL;
740 unsigned offset;
741
742 /* This must be done before adding set_constant_buffer, because it could
743 * generate e.g. transfer_unmap and flush partially-uninitialized
744 * set_constant_buffer to the driver if it was done afterwards.
745 */
746 if (cb && cb->user_buffer) {
747 u_upload_data(tc->base.const_uploader, 0, cb->buffer_size, 64,
748 cb->user_buffer, &offset, &buffer);
749 u_upload_unmap(tc->base.const_uploader);
750 }
751
752 struct tc_constant_buffer *p =
753 tc_add_struct_typed_call(tc, TC_CALL_set_constant_buffer,
754 tc_constant_buffer);
755 p->shader = shader;
756 p->index = index;
757
758 if (cb) {
759 if (cb->user_buffer) {
760 p->cb.buffer_size = cb->buffer_size;
761 p->cb.user_buffer = NULL;
762 p->cb.buffer_offset = offset;
763 p->cb.buffer = buffer;
764 } else {
765 tc_set_resource_reference(&p->cb.buffer,
766 cb->buffer);
767 p->cb.user_buffer = NULL;
768 p->cb.buffer_offset = cb->buffer_offset;
769 p->cb.buffer_size = cb->buffer_size;
770 }
771 } else {
772 memset(&p->cb, 0, sizeof(*cb));
773 }
774 }
775
776 struct tc_inlinable_constants {
777 ubyte shader;
778 ubyte num_values;
779 uint32_t values[MAX_INLINABLE_UNIFORMS];
780 };
781
782 static void
tc_call_set_inlinable_constants(struct pipe_context * pipe,union tc_payload * payload)783 tc_call_set_inlinable_constants(struct pipe_context *pipe, union tc_payload *payload)
784 {
785 struct tc_inlinable_constants *p = (struct tc_inlinable_constants *)payload;
786
787 pipe->set_inlinable_constants(pipe, p->shader, p->num_values, p->values);
788 }
789
790 static void
tc_set_inlinable_constants(struct pipe_context * _pipe,enum pipe_shader_type shader,uint num_values,uint32_t * values)791 tc_set_inlinable_constants(struct pipe_context *_pipe,
792 enum pipe_shader_type shader,
793 uint num_values, uint32_t *values)
794 {
795 struct threaded_context *tc = threaded_context(_pipe);
796 struct tc_inlinable_constants *p =
797 tc_add_struct_typed_call(tc, TC_CALL_set_inlinable_constants,
798 tc_inlinable_constants);
799 p->shader = shader;
800 p->num_values = num_values;
801 memcpy(p->values, values, num_values * 4);
802 }
803
804 struct tc_scissors {
805 ubyte start, count;
806 struct pipe_scissor_state slot[0]; /* more will be allocated if needed */
807 };
808
809 static void
tc_call_set_scissor_states(struct pipe_context * pipe,union tc_payload * payload)810 tc_call_set_scissor_states(struct pipe_context *pipe, union tc_payload *payload)
811 {
812 struct tc_scissors *p = (struct tc_scissors *)payload;
813 pipe->set_scissor_states(pipe, p->start, p->count, p->slot);
814 }
815
816 static void
tc_set_scissor_states(struct pipe_context * _pipe,unsigned start,unsigned count,const struct pipe_scissor_state * states)817 tc_set_scissor_states(struct pipe_context *_pipe,
818 unsigned start, unsigned count,
819 const struct pipe_scissor_state *states)
820 {
821 struct threaded_context *tc = threaded_context(_pipe);
822 struct tc_scissors *p =
823 tc_add_slot_based_call(tc, TC_CALL_set_scissor_states, tc_scissors, count);
824
825 p->start = start;
826 p->count = count;
827 memcpy(&p->slot, states, count * sizeof(states[0]));
828 }
829
830 struct tc_viewports {
831 ubyte start, count;
832 struct pipe_viewport_state slot[0]; /* more will be allocated if needed */
833 };
834
835 static void
tc_call_set_viewport_states(struct pipe_context * pipe,union tc_payload * payload)836 tc_call_set_viewport_states(struct pipe_context *pipe, union tc_payload *payload)
837 {
838 struct tc_viewports *p = (struct tc_viewports *)payload;
839 pipe->set_viewport_states(pipe, p->start, p->count, p->slot);
840 }
841
842 static void
tc_set_viewport_states(struct pipe_context * _pipe,unsigned start,unsigned count,const struct pipe_viewport_state * states)843 tc_set_viewport_states(struct pipe_context *_pipe,
844 unsigned start, unsigned count,
845 const struct pipe_viewport_state *states)
846 {
847 if (!count)
848 return;
849
850 struct threaded_context *tc = threaded_context(_pipe);
851 struct tc_viewports *p =
852 tc_add_slot_based_call(tc, TC_CALL_set_viewport_states, tc_viewports, count);
853
854 p->start = start;
855 p->count = count;
856 memcpy(&p->slot, states, count * sizeof(states[0]));
857 }
858
859 struct tc_window_rects {
860 bool include;
861 ubyte count;
862 struct pipe_scissor_state slot[0]; /* more will be allocated if needed */
863 };
864
865 static void
tc_call_set_window_rectangles(struct pipe_context * pipe,union tc_payload * payload)866 tc_call_set_window_rectangles(struct pipe_context *pipe,
867 union tc_payload *payload)
868 {
869 struct tc_window_rects *p = (struct tc_window_rects *)payload;
870 pipe->set_window_rectangles(pipe, p->include, p->count, p->slot);
871 }
872
873 static void
tc_set_window_rectangles(struct pipe_context * _pipe,bool include,unsigned count,const struct pipe_scissor_state * rects)874 tc_set_window_rectangles(struct pipe_context *_pipe, bool include,
875 unsigned count,
876 const struct pipe_scissor_state *rects)
877 {
878 struct threaded_context *tc = threaded_context(_pipe);
879 struct tc_window_rects *p =
880 tc_add_slot_based_call(tc, TC_CALL_set_window_rectangles, tc_window_rects, count);
881
882 p->include = include;
883 p->count = count;
884 memcpy(p->slot, rects, count * sizeof(rects[0]));
885 }
886
887 struct tc_sampler_views {
888 ubyte shader, start, count;
889 struct pipe_sampler_view *slot[0]; /* more will be allocated if needed */
890 };
891
892 static void
tc_call_set_sampler_views(struct pipe_context * pipe,union tc_payload * payload)893 tc_call_set_sampler_views(struct pipe_context *pipe, union tc_payload *payload)
894 {
895 struct tc_sampler_views *p = (struct tc_sampler_views *)payload;
896 unsigned count = p->count;
897
898 pipe->set_sampler_views(pipe, p->shader, p->start, p->count, p->slot);
899 for (unsigned i = 0; i < count; i++)
900 pipe_sampler_view_reference(&p->slot[i], NULL);
901 }
902
903 static void
tc_set_sampler_views(struct pipe_context * _pipe,enum pipe_shader_type shader,unsigned start,unsigned count,struct pipe_sampler_view ** views)904 tc_set_sampler_views(struct pipe_context *_pipe,
905 enum pipe_shader_type shader,
906 unsigned start, unsigned count,
907 struct pipe_sampler_view **views)
908 {
909 if (!count)
910 return;
911
912 struct threaded_context *tc = threaded_context(_pipe);
913 struct tc_sampler_views *p =
914 tc_add_slot_based_call(tc, TC_CALL_set_sampler_views, tc_sampler_views, count);
915
916 p->shader = shader;
917 p->start = start;
918 p->count = count;
919
920 if (views) {
921 for (unsigned i = 0; i < count; i++) {
922 p->slot[i] = NULL;
923 pipe_sampler_view_reference(&p->slot[i], views[i]);
924 }
925 } else {
926 memset(p->slot, 0, count * sizeof(views[0]));
927 }
928 }
929
930 struct tc_shader_images {
931 ubyte shader, start, count;
932 bool unbind;
933 struct pipe_image_view slot[0]; /* more will be allocated if needed */
934 };
935
936 static void
tc_call_set_shader_images(struct pipe_context * pipe,union tc_payload * payload)937 tc_call_set_shader_images(struct pipe_context *pipe, union tc_payload *payload)
938 {
939 struct tc_shader_images *p = (struct tc_shader_images *)payload;
940 unsigned count = p->count;
941
942 if (p->unbind) {
943 pipe->set_shader_images(pipe, p->shader, p->start, p->count, NULL);
944 return;
945 }
946
947 pipe->set_shader_images(pipe, p->shader, p->start, p->count, p->slot);
948
949 for (unsigned i = 0; i < count; i++)
950 pipe_resource_reference(&p->slot[i].resource, NULL);
951 }
952
953 static void
tc_set_shader_images(struct pipe_context * _pipe,enum pipe_shader_type shader,unsigned start,unsigned count,const struct pipe_image_view * images)954 tc_set_shader_images(struct pipe_context *_pipe,
955 enum pipe_shader_type shader,
956 unsigned start, unsigned count,
957 const struct pipe_image_view *images)
958 {
959 if (!count)
960 return;
961
962 struct threaded_context *tc = threaded_context(_pipe);
963 struct tc_shader_images *p =
964 tc_add_slot_based_call(tc, TC_CALL_set_shader_images, tc_shader_images,
965 images ? count : 0);
966
967 p->shader = shader;
968 p->start = start;
969 p->count = count;
970 p->unbind = images == NULL;
971
972 if (images) {
973 for (unsigned i = 0; i < count; i++) {
974 tc_set_resource_reference(&p->slot[i].resource, images[i].resource);
975
976 if (images[i].access & PIPE_IMAGE_ACCESS_WRITE &&
977 images[i].resource &&
978 images[i].resource->target == PIPE_BUFFER) {
979 struct threaded_resource *tres =
980 threaded_resource(images[i].resource);
981
982 util_range_add(&tres->b, &tres->valid_buffer_range,
983 images[i].u.buf.offset,
984 images[i].u.buf.offset + images[i].u.buf.size);
985 }
986 }
987 memcpy(p->slot, images, count * sizeof(images[0]));
988 }
989 }
990
991 struct tc_shader_buffers {
992 ubyte shader, start, count;
993 bool unbind;
994 unsigned writable_bitmask;
995 struct pipe_shader_buffer slot[0]; /* more will be allocated if needed */
996 };
997
998 static void
tc_call_set_shader_buffers(struct pipe_context * pipe,union tc_payload * payload)999 tc_call_set_shader_buffers(struct pipe_context *pipe, union tc_payload *payload)
1000 {
1001 struct tc_shader_buffers *p = (struct tc_shader_buffers *)payload;
1002 unsigned count = p->count;
1003
1004 if (p->unbind) {
1005 pipe->set_shader_buffers(pipe, p->shader, p->start, p->count, NULL, 0);
1006 return;
1007 }
1008
1009 pipe->set_shader_buffers(pipe, p->shader, p->start, p->count, p->slot,
1010 p->writable_bitmask);
1011
1012 for (unsigned i = 0; i < count; i++)
1013 pipe_resource_reference(&p->slot[i].buffer, NULL);
1014 }
1015
1016 static void
tc_set_shader_buffers(struct pipe_context * _pipe,enum pipe_shader_type shader,unsigned start,unsigned count,const struct pipe_shader_buffer * buffers,unsigned writable_bitmask)1017 tc_set_shader_buffers(struct pipe_context *_pipe,
1018 enum pipe_shader_type shader,
1019 unsigned start, unsigned count,
1020 const struct pipe_shader_buffer *buffers,
1021 unsigned writable_bitmask)
1022 {
1023 if (!count)
1024 return;
1025
1026 struct threaded_context *tc = threaded_context(_pipe);
1027 struct tc_shader_buffers *p =
1028 tc_add_slot_based_call(tc, TC_CALL_set_shader_buffers, tc_shader_buffers,
1029 buffers ? count : 0);
1030
1031 p->shader = shader;
1032 p->start = start;
1033 p->count = count;
1034 p->unbind = buffers == NULL;
1035 p->writable_bitmask = writable_bitmask;
1036
1037 if (buffers) {
1038 for (unsigned i = 0; i < count; i++) {
1039 struct pipe_shader_buffer *dst = &p->slot[i];
1040 const struct pipe_shader_buffer *src = buffers + i;
1041
1042 tc_set_resource_reference(&dst->buffer, src->buffer);
1043 dst->buffer_offset = src->buffer_offset;
1044 dst->buffer_size = src->buffer_size;
1045
1046 if (src->buffer) {
1047 struct threaded_resource *tres = threaded_resource(src->buffer);
1048
1049 util_range_add(&tres->b, &tres->valid_buffer_range,
1050 src->buffer_offset,
1051 src->buffer_offset + src->buffer_size);
1052 }
1053 }
1054 }
1055 }
1056
1057 struct tc_vertex_buffers {
1058 ubyte start, count;
1059 bool unbind;
1060 struct pipe_vertex_buffer slot[0]; /* more will be allocated if needed */
1061 };
1062
1063 static void
tc_call_set_vertex_buffers(struct pipe_context * pipe,union tc_payload * payload)1064 tc_call_set_vertex_buffers(struct pipe_context *pipe, union tc_payload *payload)
1065 {
1066 struct tc_vertex_buffers *p = (struct tc_vertex_buffers *)payload;
1067 unsigned count = p->count;
1068
1069 if (p->unbind) {
1070 pipe->set_vertex_buffers(pipe, p->start, count, NULL);
1071 return;
1072 }
1073
1074 for (unsigned i = 0; i < count; i++)
1075 tc_assert(!p->slot[i].is_user_buffer);
1076
1077 pipe->set_vertex_buffers(pipe, p->start, count, p->slot);
1078 for (unsigned i = 0; i < count; i++)
1079 pipe_resource_reference(&p->slot[i].buffer.resource, NULL);
1080 }
1081
1082 static void
tc_set_vertex_buffers(struct pipe_context * _pipe,unsigned start,unsigned count,const struct pipe_vertex_buffer * buffers)1083 tc_set_vertex_buffers(struct pipe_context *_pipe,
1084 unsigned start, unsigned count,
1085 const struct pipe_vertex_buffer *buffers)
1086 {
1087 struct threaded_context *tc = threaded_context(_pipe);
1088
1089 if (!count)
1090 return;
1091
1092 if (buffers) {
1093 struct tc_vertex_buffers *p =
1094 tc_add_slot_based_call(tc, TC_CALL_set_vertex_buffers, tc_vertex_buffers, count);
1095 p->start = start;
1096 p->count = count;
1097 p->unbind = false;
1098
1099 for (unsigned i = 0; i < count; i++) {
1100 struct pipe_vertex_buffer *dst = &p->slot[i];
1101 const struct pipe_vertex_buffer *src = buffers + i;
1102
1103 tc_assert(!src->is_user_buffer);
1104 dst->stride = src->stride;
1105 dst->is_user_buffer = false;
1106 tc_set_resource_reference(&dst->buffer.resource,
1107 src->buffer.resource);
1108 dst->buffer_offset = src->buffer_offset;
1109 }
1110 } else {
1111 struct tc_vertex_buffers *p =
1112 tc_add_slot_based_call(tc, TC_CALL_set_vertex_buffers, tc_vertex_buffers, 0);
1113 p->start = start;
1114 p->count = count;
1115 p->unbind = true;
1116 }
1117 }
1118
1119 struct tc_stream_outputs {
1120 unsigned count;
1121 struct pipe_stream_output_target *targets[PIPE_MAX_SO_BUFFERS];
1122 unsigned offsets[PIPE_MAX_SO_BUFFERS];
1123 };
1124
1125 static void
tc_call_set_stream_output_targets(struct pipe_context * pipe,union tc_payload * payload)1126 tc_call_set_stream_output_targets(struct pipe_context *pipe, union tc_payload *payload)
1127 {
1128 struct tc_stream_outputs *p = (struct tc_stream_outputs *)payload;
1129 unsigned count = p->count;
1130
1131 pipe->set_stream_output_targets(pipe, count, p->targets, p->offsets);
1132 for (unsigned i = 0; i < count; i++)
1133 pipe_so_target_reference(&p->targets[i], NULL);
1134 }
1135
1136 static void
tc_set_stream_output_targets(struct pipe_context * _pipe,unsigned count,struct pipe_stream_output_target ** tgs,const unsigned * offsets)1137 tc_set_stream_output_targets(struct pipe_context *_pipe,
1138 unsigned count,
1139 struct pipe_stream_output_target **tgs,
1140 const unsigned *offsets)
1141 {
1142 struct threaded_context *tc = threaded_context(_pipe);
1143 struct tc_stream_outputs *p =
1144 tc_add_struct_typed_call(tc, TC_CALL_set_stream_output_targets,
1145 tc_stream_outputs);
1146
1147 for (unsigned i = 0; i < count; i++) {
1148 p->targets[i] = NULL;
1149 pipe_so_target_reference(&p->targets[i], tgs[i]);
1150 }
1151 p->count = count;
1152 memcpy(p->offsets, offsets, count * sizeof(unsigned));
1153 }
1154
1155 static void
tc_set_compute_resources(struct pipe_context * _pipe,unsigned start,unsigned count,struct pipe_surface ** resources)1156 tc_set_compute_resources(struct pipe_context *_pipe, unsigned start,
1157 unsigned count, struct pipe_surface **resources)
1158 {
1159 struct threaded_context *tc = threaded_context(_pipe);
1160 struct pipe_context *pipe = tc->pipe;
1161
1162 tc_sync(tc);
1163 pipe->set_compute_resources(pipe, start, count, resources);
1164 }
1165
1166 static void
tc_set_global_binding(struct pipe_context * _pipe,unsigned first,unsigned count,struct pipe_resource ** resources,uint32_t ** handles)1167 tc_set_global_binding(struct pipe_context *_pipe, unsigned first,
1168 unsigned count, struct pipe_resource **resources,
1169 uint32_t **handles)
1170 {
1171 struct threaded_context *tc = threaded_context(_pipe);
1172 struct pipe_context *pipe = tc->pipe;
1173
1174 tc_sync(tc);
1175 pipe->set_global_binding(pipe, first, count, resources, handles);
1176 }
1177
1178
1179 /********************************************************************
1180 * views
1181 */
1182
1183 static struct pipe_surface *
tc_create_surface(struct pipe_context * _pipe,struct pipe_resource * resource,const struct pipe_surface * surf_tmpl)1184 tc_create_surface(struct pipe_context *_pipe,
1185 struct pipe_resource *resource,
1186 const struct pipe_surface *surf_tmpl)
1187 {
1188 struct pipe_context *pipe = threaded_context(_pipe)->pipe;
1189 struct pipe_surface *view =
1190 pipe->create_surface(pipe, resource, surf_tmpl);
1191
1192 if (view)
1193 view->context = _pipe;
1194 return view;
1195 }
1196
1197 static void
tc_surface_destroy(struct pipe_context * _pipe,struct pipe_surface * surf)1198 tc_surface_destroy(struct pipe_context *_pipe,
1199 struct pipe_surface *surf)
1200 {
1201 struct pipe_context *pipe = threaded_context(_pipe)->pipe;
1202
1203 pipe->surface_destroy(pipe, surf);
1204 }
1205
1206 static struct pipe_sampler_view *
tc_create_sampler_view(struct pipe_context * _pipe,struct pipe_resource * resource,const struct pipe_sampler_view * templ)1207 tc_create_sampler_view(struct pipe_context *_pipe,
1208 struct pipe_resource *resource,
1209 const struct pipe_sampler_view *templ)
1210 {
1211 struct pipe_context *pipe = threaded_context(_pipe)->pipe;
1212 struct pipe_sampler_view *view =
1213 pipe->create_sampler_view(pipe, resource, templ);
1214
1215 if (view)
1216 view->context = _pipe;
1217 return view;
1218 }
1219
1220 static void
tc_sampler_view_destroy(struct pipe_context * _pipe,struct pipe_sampler_view * view)1221 tc_sampler_view_destroy(struct pipe_context *_pipe,
1222 struct pipe_sampler_view *view)
1223 {
1224 struct pipe_context *pipe = threaded_context(_pipe)->pipe;
1225
1226 pipe->sampler_view_destroy(pipe, view);
1227 }
1228
1229 static struct pipe_stream_output_target *
tc_create_stream_output_target(struct pipe_context * _pipe,struct pipe_resource * res,unsigned buffer_offset,unsigned buffer_size)1230 tc_create_stream_output_target(struct pipe_context *_pipe,
1231 struct pipe_resource *res,
1232 unsigned buffer_offset,
1233 unsigned buffer_size)
1234 {
1235 struct pipe_context *pipe = threaded_context(_pipe)->pipe;
1236 struct threaded_resource *tres = threaded_resource(res);
1237 struct pipe_stream_output_target *view;
1238
1239 tc_sync(threaded_context(_pipe));
1240 util_range_add(&tres->b, &tres->valid_buffer_range, buffer_offset,
1241 buffer_offset + buffer_size);
1242
1243 view = pipe->create_stream_output_target(pipe, res, buffer_offset,
1244 buffer_size);
1245 if (view)
1246 view->context = _pipe;
1247 return view;
1248 }
1249
1250 static void
tc_stream_output_target_destroy(struct pipe_context * _pipe,struct pipe_stream_output_target * target)1251 tc_stream_output_target_destroy(struct pipe_context *_pipe,
1252 struct pipe_stream_output_target *target)
1253 {
1254 struct pipe_context *pipe = threaded_context(_pipe)->pipe;
1255
1256 pipe->stream_output_target_destroy(pipe, target);
1257 }
1258
1259
1260 /********************************************************************
1261 * bindless
1262 */
1263
1264 static uint64_t
tc_create_texture_handle(struct pipe_context * _pipe,struct pipe_sampler_view * view,const struct pipe_sampler_state * state)1265 tc_create_texture_handle(struct pipe_context *_pipe,
1266 struct pipe_sampler_view *view,
1267 const struct pipe_sampler_state *state)
1268 {
1269 struct threaded_context *tc = threaded_context(_pipe);
1270 struct pipe_context *pipe = tc->pipe;
1271
1272 tc_sync(tc);
1273 return pipe->create_texture_handle(pipe, view, state);
1274 }
1275
1276 static void
tc_call_delete_texture_handle(struct pipe_context * pipe,union tc_payload * payload)1277 tc_call_delete_texture_handle(struct pipe_context *pipe,
1278 union tc_payload *payload)
1279 {
1280 pipe->delete_texture_handle(pipe, payload->handle);
1281 }
1282
1283 static void
tc_delete_texture_handle(struct pipe_context * _pipe,uint64_t handle)1284 tc_delete_texture_handle(struct pipe_context *_pipe, uint64_t handle)
1285 {
1286 struct threaded_context *tc = threaded_context(_pipe);
1287 union tc_payload *payload =
1288 tc_add_small_call(tc, TC_CALL_delete_texture_handle);
1289
1290 payload->handle = handle;
1291 }
1292
1293 struct tc_make_texture_handle_resident
1294 {
1295 uint64_t handle;
1296 bool resident;
1297 };
1298
1299 static void
tc_call_make_texture_handle_resident(struct pipe_context * pipe,union tc_payload * payload)1300 tc_call_make_texture_handle_resident(struct pipe_context *pipe,
1301 union tc_payload *payload)
1302 {
1303 struct tc_make_texture_handle_resident *p =
1304 (struct tc_make_texture_handle_resident *)payload;
1305
1306 pipe->make_texture_handle_resident(pipe, p->handle, p->resident);
1307 }
1308
1309 static void
tc_make_texture_handle_resident(struct pipe_context * _pipe,uint64_t handle,bool resident)1310 tc_make_texture_handle_resident(struct pipe_context *_pipe, uint64_t handle,
1311 bool resident)
1312 {
1313 struct threaded_context *tc = threaded_context(_pipe);
1314 struct tc_make_texture_handle_resident *p =
1315 tc_add_struct_typed_call(tc, TC_CALL_make_texture_handle_resident,
1316 tc_make_texture_handle_resident);
1317
1318 p->handle = handle;
1319 p->resident = resident;
1320 }
1321
1322 static uint64_t
tc_create_image_handle(struct pipe_context * _pipe,const struct pipe_image_view * image)1323 tc_create_image_handle(struct pipe_context *_pipe,
1324 const struct pipe_image_view *image)
1325 {
1326 struct threaded_context *tc = threaded_context(_pipe);
1327 struct pipe_context *pipe = tc->pipe;
1328
1329 tc_sync(tc);
1330 return pipe->create_image_handle(pipe, image);
1331 }
1332
1333 static void
tc_call_delete_image_handle(struct pipe_context * pipe,union tc_payload * payload)1334 tc_call_delete_image_handle(struct pipe_context *pipe,
1335 union tc_payload *payload)
1336 {
1337 pipe->delete_image_handle(pipe, payload->handle);
1338 }
1339
1340 static void
tc_delete_image_handle(struct pipe_context * _pipe,uint64_t handle)1341 tc_delete_image_handle(struct pipe_context *_pipe, uint64_t handle)
1342 {
1343 struct threaded_context *tc = threaded_context(_pipe);
1344 union tc_payload *payload =
1345 tc_add_small_call(tc, TC_CALL_delete_image_handle);
1346
1347 payload->handle = handle;
1348 }
1349
1350 struct tc_make_image_handle_resident
1351 {
1352 uint64_t handle;
1353 unsigned access;
1354 bool resident;
1355 };
1356
1357 static void
tc_call_make_image_handle_resident(struct pipe_context * pipe,union tc_payload * payload)1358 tc_call_make_image_handle_resident(struct pipe_context *pipe,
1359 union tc_payload *payload)
1360 {
1361 struct tc_make_image_handle_resident *p =
1362 (struct tc_make_image_handle_resident *)payload;
1363
1364 pipe->make_image_handle_resident(pipe, p->handle, p->access, p->resident);
1365 }
1366
1367 static void
tc_make_image_handle_resident(struct pipe_context * _pipe,uint64_t handle,unsigned access,bool resident)1368 tc_make_image_handle_resident(struct pipe_context *_pipe, uint64_t handle,
1369 unsigned access, bool resident)
1370 {
1371 struct threaded_context *tc = threaded_context(_pipe);
1372 struct tc_make_image_handle_resident *p =
1373 tc_add_struct_typed_call(tc, TC_CALL_make_image_handle_resident,
1374 tc_make_image_handle_resident);
1375
1376 p->handle = handle;
1377 p->access = access;
1378 p->resident = resident;
1379 }
1380
1381
1382 /********************************************************************
1383 * transfer
1384 */
1385
1386 struct tc_replace_buffer_storage {
1387 struct pipe_resource *dst;
1388 struct pipe_resource *src;
1389 tc_replace_buffer_storage_func func;
1390 };
1391
1392 static void
tc_call_replace_buffer_storage(struct pipe_context * pipe,union tc_payload * payload)1393 tc_call_replace_buffer_storage(struct pipe_context *pipe,
1394 union tc_payload *payload)
1395 {
1396 struct tc_replace_buffer_storage *p =
1397 (struct tc_replace_buffer_storage *)payload;
1398
1399 p->func(pipe, p->dst, p->src);
1400 pipe_resource_reference(&p->dst, NULL);
1401 pipe_resource_reference(&p->src, NULL);
1402 }
1403
1404 static bool
tc_invalidate_buffer(struct threaded_context * tc,struct threaded_resource * tbuf)1405 tc_invalidate_buffer(struct threaded_context *tc,
1406 struct threaded_resource *tbuf)
1407 {
1408 /* We can't check if the buffer is idle, so we invalidate it
1409 * unconditionally. */
1410 struct pipe_screen *screen = tc->base.screen;
1411 struct pipe_resource *new_buf;
1412
1413 /* Shared, pinned, and sparse buffers can't be reallocated. */
1414 if (tbuf->is_shared ||
1415 tbuf->is_user_ptr ||
1416 tbuf->b.flags & PIPE_RESOURCE_FLAG_SPARSE)
1417 return false;
1418
1419 /* Allocate a new one. */
1420 new_buf = screen->resource_create(screen, &tbuf->b);
1421 if (!new_buf)
1422 return false;
1423
1424 /* Replace the "latest" pointer. */
1425 if (tbuf->latest != &tbuf->b)
1426 pipe_resource_reference(&tbuf->latest, NULL);
1427
1428 tbuf->latest = new_buf;
1429 util_range_set_empty(&tbuf->valid_buffer_range);
1430
1431 /* The valid range should point to the original buffer. */
1432 threaded_resource(new_buf)->base_valid_buffer_range =
1433 &tbuf->valid_buffer_range;
1434
1435 /* Enqueue storage replacement of the original buffer. */
1436 struct tc_replace_buffer_storage *p =
1437 tc_add_struct_typed_call(tc, TC_CALL_replace_buffer_storage,
1438 tc_replace_buffer_storage);
1439
1440 p->func = tc->replace_buffer_storage;
1441 tc_set_resource_reference(&p->dst, &tbuf->b);
1442 tc_set_resource_reference(&p->src, new_buf);
1443 return true;
1444 }
1445
1446 static unsigned
tc_improve_map_buffer_flags(struct threaded_context * tc,struct threaded_resource * tres,unsigned usage,unsigned offset,unsigned size)1447 tc_improve_map_buffer_flags(struct threaded_context *tc,
1448 struct threaded_resource *tres, unsigned usage,
1449 unsigned offset, unsigned size)
1450 {
1451 /* Never invalidate inside the driver and never infer "unsynchronized". */
1452 unsigned tc_flags = TC_TRANSFER_MAP_NO_INVALIDATE |
1453 TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED;
1454
1455 /* Prevent a reentry. */
1456 if (usage & tc_flags)
1457 return usage;
1458
1459 /* Use the staging upload if it's preferred. */
1460 if (usage & (PIPE_MAP_DISCARD_RANGE |
1461 PIPE_MAP_DISCARD_WHOLE_RESOURCE) &&
1462 !(usage & PIPE_MAP_PERSISTENT) &&
1463 /* Try not to decrement the counter if it's not positive. Still racy,
1464 * but it makes it harder to wrap the counter from INT_MIN to INT_MAX. */
1465 tres->max_forced_staging_uploads > 0 &&
1466 p_atomic_dec_return(&tres->max_forced_staging_uploads) >= 0) {
1467 usage &= ~(PIPE_MAP_DISCARD_WHOLE_RESOURCE |
1468 PIPE_MAP_UNSYNCHRONIZED);
1469
1470 return usage | tc_flags | PIPE_MAP_DISCARD_RANGE;
1471 }
1472
1473 /* Sparse buffers can't be mapped directly and can't be reallocated
1474 * (fully invalidated). That may just be a radeonsi limitation, but
1475 * the threaded context must obey it with radeonsi.
1476 */
1477 if (tres->b.flags & PIPE_RESOURCE_FLAG_SPARSE) {
1478 /* We can use DISCARD_RANGE instead of full discard. This is the only
1479 * fast path for sparse buffers that doesn't need thread synchronization.
1480 */
1481 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE)
1482 usage |= PIPE_MAP_DISCARD_RANGE;
1483
1484 /* Allow DISCARD_WHOLE_RESOURCE and infering UNSYNCHRONIZED in drivers.
1485 * The threaded context doesn't do unsychronized mappings and invalida-
1486 * tions of sparse buffers, therefore a correct driver behavior won't
1487 * result in an incorrect behavior with the threaded context.
1488 */
1489 return usage;
1490 }
1491
1492 usage |= tc_flags;
1493
1494 /* Handle CPU reads trivially. */
1495 if (usage & PIPE_MAP_READ) {
1496 if (usage & PIPE_MAP_UNSYNCHRONIZED)
1497 usage |= TC_TRANSFER_MAP_THREADED_UNSYNC; /* don't sync */
1498
1499 /* Drivers aren't allowed to do buffer invalidations. */
1500 return usage & ~PIPE_MAP_DISCARD_WHOLE_RESOURCE;
1501 }
1502
1503 /* See if the buffer range being mapped has never been initialized,
1504 * in which case it can be mapped unsynchronized. */
1505 if (!(usage & PIPE_MAP_UNSYNCHRONIZED) &&
1506 !tres->is_shared &&
1507 !util_ranges_intersect(&tres->valid_buffer_range, offset, offset + size))
1508 usage |= PIPE_MAP_UNSYNCHRONIZED;
1509
1510 if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
1511 /* If discarding the entire range, discard the whole resource instead. */
1512 if (usage & PIPE_MAP_DISCARD_RANGE &&
1513 offset == 0 && size == tres->b.width0)
1514 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
1515
1516 /* Discard the whole resource if needed. */
1517 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
1518 if (tc_invalidate_buffer(tc, tres))
1519 usage |= PIPE_MAP_UNSYNCHRONIZED;
1520 else
1521 usage |= PIPE_MAP_DISCARD_RANGE; /* fallback */
1522 }
1523 }
1524
1525 /* We won't need this flag anymore. */
1526 /* TODO: We might not need TC_TRANSFER_MAP_NO_INVALIDATE with this. */
1527 usage &= ~PIPE_MAP_DISCARD_WHOLE_RESOURCE;
1528
1529 /* GL_AMD_pinned_memory and persistent mappings can't use staging
1530 * buffers. */
1531 if (usage & (PIPE_MAP_UNSYNCHRONIZED |
1532 PIPE_MAP_PERSISTENT) ||
1533 tres->is_user_ptr)
1534 usage &= ~PIPE_MAP_DISCARD_RANGE;
1535
1536 /* Unsychronized buffer mappings don't have to synchronize the thread. */
1537 if (usage & PIPE_MAP_UNSYNCHRONIZED) {
1538 usage &= ~PIPE_MAP_DISCARD_RANGE;
1539 usage |= TC_TRANSFER_MAP_THREADED_UNSYNC; /* notify the driver */
1540 }
1541
1542 return usage;
1543 }
1544
1545 static void *
tc_transfer_map(struct pipe_context * _pipe,struct pipe_resource * resource,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** transfer)1546 tc_transfer_map(struct pipe_context *_pipe,
1547 struct pipe_resource *resource, unsigned level,
1548 unsigned usage, const struct pipe_box *box,
1549 struct pipe_transfer **transfer)
1550 {
1551 struct threaded_context *tc = threaded_context(_pipe);
1552 struct threaded_resource *tres = threaded_resource(resource);
1553 struct pipe_context *pipe = tc->pipe;
1554
1555 if (resource->target == PIPE_BUFFER) {
1556 usage = tc_improve_map_buffer_flags(tc, tres, usage, box->x, box->width);
1557
1558 /* Do a staging transfer within the threaded context. The driver should
1559 * only get resource_copy_region.
1560 */
1561 if (usage & PIPE_MAP_DISCARD_RANGE) {
1562 struct threaded_transfer *ttrans = slab_alloc(&tc->pool_transfers);
1563 uint8_t *map;
1564
1565 ttrans->staging = NULL;
1566
1567 u_upload_alloc(tc->base.stream_uploader, 0,
1568 box->width + (box->x % tc->map_buffer_alignment),
1569 tc->map_buffer_alignment, &ttrans->offset,
1570 &ttrans->staging, (void**)&map);
1571 if (!map) {
1572 slab_free(&tc->pool_transfers, ttrans);
1573 return NULL;
1574 }
1575
1576 tc_set_resource_reference(&ttrans->b.resource, resource);
1577 ttrans->b.level = 0;
1578 ttrans->b.usage = usage;
1579 ttrans->b.box = *box;
1580 ttrans->b.stride = 0;
1581 ttrans->b.layer_stride = 0;
1582 *transfer = &ttrans->b;
1583 return map + (box->x % tc->map_buffer_alignment);
1584 }
1585 }
1586
1587 /* Unsychronized buffer mappings don't have to synchronize the thread. */
1588 if (!(usage & TC_TRANSFER_MAP_THREADED_UNSYNC))
1589 tc_sync_msg(tc, resource->target != PIPE_BUFFER ? " texture" :
1590 usage & PIPE_MAP_DISCARD_RANGE ? " discard_range" :
1591 usage & PIPE_MAP_READ ? " read" : " ??");
1592
1593 tc->bytes_mapped_estimate += box->width;
1594
1595 return pipe->transfer_map(pipe, tres->latest ? tres->latest : resource,
1596 level, usage, box, transfer);
1597 }
1598
1599 struct tc_transfer_flush_region {
1600 struct pipe_transfer *transfer;
1601 struct pipe_box box;
1602 };
1603
1604 static void
tc_call_transfer_flush_region(struct pipe_context * pipe,union tc_payload * payload)1605 tc_call_transfer_flush_region(struct pipe_context *pipe,
1606 union tc_payload *payload)
1607 {
1608 struct tc_transfer_flush_region *p =
1609 (struct tc_transfer_flush_region *)payload;
1610
1611 pipe->transfer_flush_region(pipe, p->transfer, &p->box);
1612 }
1613
1614 struct tc_resource_copy_region {
1615 struct pipe_resource *dst;
1616 unsigned dst_level;
1617 unsigned dstx, dsty, dstz;
1618 struct pipe_resource *src;
1619 unsigned src_level;
1620 struct pipe_box src_box;
1621 };
1622
1623 static void
1624 tc_resource_copy_region(struct pipe_context *_pipe,
1625 struct pipe_resource *dst, unsigned dst_level,
1626 unsigned dstx, unsigned dsty, unsigned dstz,
1627 struct pipe_resource *src, unsigned src_level,
1628 const struct pipe_box *src_box);
1629
1630 static void
tc_buffer_do_flush_region(struct threaded_context * tc,struct threaded_transfer * ttrans,const struct pipe_box * box)1631 tc_buffer_do_flush_region(struct threaded_context *tc,
1632 struct threaded_transfer *ttrans,
1633 const struct pipe_box *box)
1634 {
1635 struct threaded_resource *tres = threaded_resource(ttrans->b.resource);
1636
1637 if (ttrans->staging) {
1638 struct pipe_box src_box;
1639
1640 u_box_1d(ttrans->offset + ttrans->b.box.x % tc->map_buffer_alignment +
1641 (box->x - ttrans->b.box.x),
1642 box->width, &src_box);
1643
1644 /* Copy the staging buffer into the original one. */
1645 tc_resource_copy_region(&tc->base, ttrans->b.resource, 0, box->x, 0, 0,
1646 ttrans->staging, 0, &src_box);
1647 }
1648
1649 util_range_add(&tres->b, tres->base_valid_buffer_range,
1650 box->x, box->x + box->width);
1651 }
1652
1653 static void
tc_transfer_flush_region(struct pipe_context * _pipe,struct pipe_transfer * transfer,const struct pipe_box * rel_box)1654 tc_transfer_flush_region(struct pipe_context *_pipe,
1655 struct pipe_transfer *transfer,
1656 const struct pipe_box *rel_box)
1657 {
1658 struct threaded_context *tc = threaded_context(_pipe);
1659 struct threaded_transfer *ttrans = threaded_transfer(transfer);
1660 struct threaded_resource *tres = threaded_resource(transfer->resource);
1661 unsigned required_usage = PIPE_MAP_WRITE |
1662 PIPE_MAP_FLUSH_EXPLICIT;
1663
1664 if (tres->b.target == PIPE_BUFFER) {
1665 if ((transfer->usage & required_usage) == required_usage) {
1666 struct pipe_box box;
1667
1668 u_box_1d(transfer->box.x + rel_box->x, rel_box->width, &box);
1669 tc_buffer_do_flush_region(tc, ttrans, &box);
1670 }
1671
1672 /* Staging transfers don't send the call to the driver. */
1673 if (ttrans->staging)
1674 return;
1675 }
1676
1677 struct tc_transfer_flush_region *p =
1678 tc_add_struct_typed_call(tc, TC_CALL_transfer_flush_region,
1679 tc_transfer_flush_region);
1680 p->transfer = transfer;
1681 p->box = *rel_box;
1682 }
1683
1684 static void
tc_call_transfer_unmap(struct pipe_context * pipe,union tc_payload * payload)1685 tc_call_transfer_unmap(struct pipe_context *pipe, union tc_payload *payload)
1686 {
1687 pipe->transfer_unmap(pipe, payload->transfer);
1688 }
1689
1690 static void
1691 tc_flush(struct pipe_context *_pipe, struct pipe_fence_handle **fence,
1692 unsigned flags);
1693
1694 static void
tc_transfer_unmap(struct pipe_context * _pipe,struct pipe_transfer * transfer)1695 tc_transfer_unmap(struct pipe_context *_pipe, struct pipe_transfer *transfer)
1696 {
1697 struct threaded_context *tc = threaded_context(_pipe);
1698 struct threaded_transfer *ttrans = threaded_transfer(transfer);
1699 struct threaded_resource *tres = threaded_resource(transfer->resource);
1700
1701 /* PIPE_MAP_THREAD_SAFE is only valid with UNSYNCHRONIZED. It can be
1702 * called from any thread and bypasses all multithreaded queues.
1703 */
1704 if (transfer->usage & PIPE_MAP_THREAD_SAFE) {
1705 assert(transfer->usage & PIPE_MAP_UNSYNCHRONIZED);
1706 assert(!(transfer->usage & (PIPE_MAP_FLUSH_EXPLICIT |
1707 PIPE_MAP_DISCARD_RANGE)));
1708
1709 struct pipe_context *pipe = tc->pipe;
1710 util_range_add(&tres->b, tres->base_valid_buffer_range,
1711 transfer->box.x, transfer->box.x + transfer->box.width);
1712 pipe->transfer_unmap(pipe, transfer);
1713 return;
1714 }
1715
1716 if (tres->b.target == PIPE_BUFFER) {
1717 if (transfer->usage & PIPE_MAP_WRITE &&
1718 !(transfer->usage & PIPE_MAP_FLUSH_EXPLICIT))
1719 tc_buffer_do_flush_region(tc, ttrans, &transfer->box);
1720
1721 /* Staging transfers don't send the call to the driver. */
1722 if (ttrans->staging) {
1723 pipe_resource_reference(&ttrans->staging, NULL);
1724 pipe_resource_reference(&ttrans->b.resource, NULL);
1725 slab_free(&tc->pool_transfers, ttrans);
1726 return;
1727 }
1728 }
1729
1730 tc_add_small_call(tc, TC_CALL_transfer_unmap)->transfer = transfer;
1731
1732 /* tc_transfer_map directly maps the buffers, but tc_transfer_unmap
1733 * defers the unmap operation to the batch execution.
1734 * bytes_mapped_estimate is an estimation of the map/unmap bytes delta
1735 * and if it goes over an optional limit the current batch is flushed,
1736 * to reclaim some RAM. */
1737 if (!ttrans->staging && tc->bytes_mapped_limit &&
1738 tc->bytes_mapped_estimate > tc->bytes_mapped_limit) {
1739 tc_flush(_pipe, NULL, PIPE_FLUSH_ASYNC);
1740 }
1741 }
1742
1743 struct tc_buffer_subdata {
1744 struct pipe_resource *resource;
1745 unsigned usage, offset, size;
1746 char slot[0]; /* more will be allocated if needed */
1747 };
1748
1749 static void
tc_call_buffer_subdata(struct pipe_context * pipe,union tc_payload * payload)1750 tc_call_buffer_subdata(struct pipe_context *pipe, union tc_payload *payload)
1751 {
1752 struct tc_buffer_subdata *p = (struct tc_buffer_subdata *)payload;
1753
1754 pipe->buffer_subdata(pipe, p->resource, p->usage, p->offset, p->size,
1755 p->slot);
1756 pipe_resource_reference(&p->resource, NULL);
1757 }
1758
1759 static void
tc_buffer_subdata(struct pipe_context * _pipe,struct pipe_resource * resource,unsigned usage,unsigned offset,unsigned size,const void * data)1760 tc_buffer_subdata(struct pipe_context *_pipe,
1761 struct pipe_resource *resource,
1762 unsigned usage, unsigned offset,
1763 unsigned size, const void *data)
1764 {
1765 struct threaded_context *tc = threaded_context(_pipe);
1766 struct threaded_resource *tres = threaded_resource(resource);
1767
1768 if (!size)
1769 return;
1770
1771 usage |= PIPE_MAP_WRITE;
1772
1773 /* PIPE_MAP_DIRECTLY supresses implicit DISCARD_RANGE. */
1774 if (!(usage & PIPE_MAP_DIRECTLY))
1775 usage |= PIPE_MAP_DISCARD_RANGE;
1776
1777 usage = tc_improve_map_buffer_flags(tc, tres, usage, offset, size);
1778
1779 /* Unsychronized and big transfers should use transfer_map. Also handle
1780 * full invalidations, because drivers aren't allowed to do them.
1781 */
1782 if (usage & (PIPE_MAP_UNSYNCHRONIZED |
1783 PIPE_MAP_DISCARD_WHOLE_RESOURCE) ||
1784 size > TC_MAX_SUBDATA_BYTES) {
1785 struct pipe_transfer *transfer;
1786 struct pipe_box box;
1787 uint8_t *map = NULL;
1788
1789 u_box_1d(offset, size, &box);
1790
1791 map = tc_transfer_map(_pipe, resource, 0, usage, &box, &transfer);
1792 if (map) {
1793 memcpy(map, data, size);
1794 tc_transfer_unmap(_pipe, transfer);
1795 }
1796 return;
1797 }
1798
1799 util_range_add(&tres->b, &tres->valid_buffer_range, offset, offset + size);
1800
1801 /* The upload is small. Enqueue it. */
1802 struct tc_buffer_subdata *p =
1803 tc_add_slot_based_call(tc, TC_CALL_buffer_subdata, tc_buffer_subdata, size);
1804
1805 tc_set_resource_reference(&p->resource, resource);
1806 p->usage = usage;
1807 p->offset = offset;
1808 p->size = size;
1809 memcpy(p->slot, data, size);
1810 }
1811
1812 struct tc_texture_subdata {
1813 struct pipe_resource *resource;
1814 unsigned level, usage, stride, layer_stride;
1815 struct pipe_box box;
1816 char slot[0]; /* more will be allocated if needed */
1817 };
1818
1819 static void
tc_call_texture_subdata(struct pipe_context * pipe,union tc_payload * payload)1820 tc_call_texture_subdata(struct pipe_context *pipe, union tc_payload *payload)
1821 {
1822 struct tc_texture_subdata *p = (struct tc_texture_subdata *)payload;
1823
1824 pipe->texture_subdata(pipe, p->resource, p->level, p->usage, &p->box,
1825 p->slot, p->stride, p->layer_stride);
1826 pipe_resource_reference(&p->resource, NULL);
1827 }
1828
1829 static void
tc_texture_subdata(struct pipe_context * _pipe,struct pipe_resource * resource,unsigned level,unsigned usage,const struct pipe_box * box,const void * data,unsigned stride,unsigned layer_stride)1830 tc_texture_subdata(struct pipe_context *_pipe,
1831 struct pipe_resource *resource,
1832 unsigned level, unsigned usage,
1833 const struct pipe_box *box,
1834 const void *data, unsigned stride,
1835 unsigned layer_stride)
1836 {
1837 struct threaded_context *tc = threaded_context(_pipe);
1838 unsigned size;
1839
1840 assert(box->height >= 1);
1841 assert(box->depth >= 1);
1842
1843 size = (box->depth - 1) * layer_stride +
1844 (box->height - 1) * stride +
1845 box->width * util_format_get_blocksize(resource->format);
1846 if (!size)
1847 return;
1848
1849 /* Small uploads can be enqueued, big uploads must sync. */
1850 if (size <= TC_MAX_SUBDATA_BYTES) {
1851 struct tc_texture_subdata *p =
1852 tc_add_slot_based_call(tc, TC_CALL_texture_subdata, tc_texture_subdata, size);
1853
1854 tc_set_resource_reference(&p->resource, resource);
1855 p->level = level;
1856 p->usage = usage;
1857 p->box = *box;
1858 p->stride = stride;
1859 p->layer_stride = layer_stride;
1860 memcpy(p->slot, data, size);
1861 } else {
1862 struct pipe_context *pipe = tc->pipe;
1863
1864 tc_sync(tc);
1865 pipe->texture_subdata(pipe, resource, level, usage, box, data,
1866 stride, layer_stride);
1867 }
1868 }
1869
1870
1871 /********************************************************************
1872 * miscellaneous
1873 */
1874
1875 #define TC_FUNC_SYNC_RET0(ret_type, func) \
1876 static ret_type \
1877 tc_##func(struct pipe_context *_pipe) \
1878 { \
1879 struct threaded_context *tc = threaded_context(_pipe); \
1880 struct pipe_context *pipe = tc->pipe; \
1881 tc_sync(tc); \
1882 return pipe->func(pipe); \
1883 }
1884
TC_FUNC_SYNC_RET0(enum pipe_reset_status,get_device_reset_status)1885 TC_FUNC_SYNC_RET0(enum pipe_reset_status, get_device_reset_status)
1886 TC_FUNC_SYNC_RET0(uint64_t, get_timestamp)
1887
1888 static void
1889 tc_get_sample_position(struct pipe_context *_pipe,
1890 unsigned sample_count, unsigned sample_index,
1891 float *out_value)
1892 {
1893 struct threaded_context *tc = threaded_context(_pipe);
1894 struct pipe_context *pipe = tc->pipe;
1895
1896 tc_sync(tc);
1897 pipe->get_sample_position(pipe, sample_count, sample_index,
1898 out_value);
1899 }
1900
1901 static void
tc_set_device_reset_callback(struct pipe_context * _pipe,const struct pipe_device_reset_callback * cb)1902 tc_set_device_reset_callback(struct pipe_context *_pipe,
1903 const struct pipe_device_reset_callback *cb)
1904 {
1905 struct threaded_context *tc = threaded_context(_pipe);
1906 struct pipe_context *pipe = tc->pipe;
1907
1908 tc_sync(tc);
1909 pipe->set_device_reset_callback(pipe, cb);
1910 }
1911
1912 struct tc_string_marker {
1913 int len;
1914 char slot[0]; /* more will be allocated if needed */
1915 };
1916
1917 static void
tc_call_emit_string_marker(struct pipe_context * pipe,union tc_payload * payload)1918 tc_call_emit_string_marker(struct pipe_context *pipe, union tc_payload *payload)
1919 {
1920 struct tc_string_marker *p = (struct tc_string_marker *)payload;
1921 pipe->emit_string_marker(pipe, p->slot, p->len);
1922 }
1923
1924 static void
tc_emit_string_marker(struct pipe_context * _pipe,const char * string,int len)1925 tc_emit_string_marker(struct pipe_context *_pipe,
1926 const char *string, int len)
1927 {
1928 struct threaded_context *tc = threaded_context(_pipe);
1929
1930 if (len <= TC_MAX_STRING_MARKER_BYTES) {
1931 struct tc_string_marker *p =
1932 tc_add_slot_based_call(tc, TC_CALL_emit_string_marker, tc_string_marker, len);
1933
1934 memcpy(p->slot, string, len);
1935 p->len = len;
1936 } else {
1937 struct pipe_context *pipe = tc->pipe;
1938
1939 tc_sync(tc);
1940 pipe->emit_string_marker(pipe, string, len);
1941 }
1942 }
1943
1944 static void
tc_dump_debug_state(struct pipe_context * _pipe,FILE * stream,unsigned flags)1945 tc_dump_debug_state(struct pipe_context *_pipe, FILE *stream,
1946 unsigned flags)
1947 {
1948 struct threaded_context *tc = threaded_context(_pipe);
1949 struct pipe_context *pipe = tc->pipe;
1950
1951 tc_sync(tc);
1952 pipe->dump_debug_state(pipe, stream, flags);
1953 }
1954
1955 static void
tc_set_debug_callback(struct pipe_context * _pipe,const struct pipe_debug_callback * cb)1956 tc_set_debug_callback(struct pipe_context *_pipe,
1957 const struct pipe_debug_callback *cb)
1958 {
1959 struct threaded_context *tc = threaded_context(_pipe);
1960 struct pipe_context *pipe = tc->pipe;
1961
1962 /* Drop all synchronous debug callbacks. Drivers are expected to be OK
1963 * with this. shader-db will use an environment variable to disable
1964 * the threaded context.
1965 */
1966 if (cb && cb->debug_message && !cb->async)
1967 return;
1968
1969 tc_sync(tc);
1970 pipe->set_debug_callback(pipe, cb);
1971 }
1972
1973 static void
tc_set_log_context(struct pipe_context * _pipe,struct u_log_context * log)1974 tc_set_log_context(struct pipe_context *_pipe, struct u_log_context *log)
1975 {
1976 struct threaded_context *tc = threaded_context(_pipe);
1977 struct pipe_context *pipe = tc->pipe;
1978
1979 tc_sync(tc);
1980 pipe->set_log_context(pipe, log);
1981 }
1982
1983 static void
tc_create_fence_fd(struct pipe_context * _pipe,struct pipe_fence_handle ** fence,int fd,enum pipe_fd_type type)1984 tc_create_fence_fd(struct pipe_context *_pipe,
1985 struct pipe_fence_handle **fence, int fd,
1986 enum pipe_fd_type type)
1987 {
1988 struct threaded_context *tc = threaded_context(_pipe);
1989 struct pipe_context *pipe = tc->pipe;
1990
1991 tc_sync(tc);
1992 pipe->create_fence_fd(pipe, fence, fd, type);
1993 }
1994
1995 static void
tc_call_fence_server_sync(struct pipe_context * pipe,union tc_payload * payload)1996 tc_call_fence_server_sync(struct pipe_context *pipe, union tc_payload *payload)
1997 {
1998 pipe->fence_server_sync(pipe, payload->fence);
1999 pipe->screen->fence_reference(pipe->screen, &payload->fence, NULL);
2000 }
2001
2002 static void
tc_fence_server_sync(struct pipe_context * _pipe,struct pipe_fence_handle * fence)2003 tc_fence_server_sync(struct pipe_context *_pipe,
2004 struct pipe_fence_handle *fence)
2005 {
2006 struct threaded_context *tc = threaded_context(_pipe);
2007 struct pipe_screen *screen = tc->pipe->screen;
2008 union tc_payload *payload = tc_add_small_call(tc, TC_CALL_fence_server_sync);
2009
2010 payload->fence = NULL;
2011 screen->fence_reference(screen, &payload->fence, fence);
2012 }
2013
2014 static void
tc_call_fence_server_signal(struct pipe_context * pipe,union tc_payload * payload)2015 tc_call_fence_server_signal(struct pipe_context *pipe, union tc_payload *payload)
2016 {
2017 pipe->fence_server_signal(pipe, payload->fence);
2018 pipe->screen->fence_reference(pipe->screen, &payload->fence, NULL);
2019 }
2020
2021 static void
tc_fence_server_signal(struct pipe_context * _pipe,struct pipe_fence_handle * fence)2022 tc_fence_server_signal(struct pipe_context *_pipe,
2023 struct pipe_fence_handle *fence)
2024 {
2025 struct threaded_context *tc = threaded_context(_pipe);
2026 struct pipe_screen *screen = tc->pipe->screen;
2027 union tc_payload *payload = tc_add_small_call(tc, TC_CALL_fence_server_signal);
2028
2029 payload->fence = NULL;
2030 screen->fence_reference(screen, &payload->fence, fence);
2031 }
2032
2033 static struct pipe_video_codec *
tc_create_video_codec(UNUSED struct pipe_context * _pipe,UNUSED const struct pipe_video_codec * templ)2034 tc_create_video_codec(UNUSED struct pipe_context *_pipe,
2035 UNUSED const struct pipe_video_codec *templ)
2036 {
2037 unreachable("Threaded context should not be enabled for video APIs");
2038 return NULL;
2039 }
2040
2041 static struct pipe_video_buffer *
tc_create_video_buffer(UNUSED struct pipe_context * _pipe,UNUSED const struct pipe_video_buffer * templ)2042 tc_create_video_buffer(UNUSED struct pipe_context *_pipe,
2043 UNUSED const struct pipe_video_buffer *templ)
2044 {
2045 unreachable("Threaded context should not be enabled for video APIs");
2046 return NULL;
2047 }
2048
2049 struct tc_context_param {
2050 enum pipe_context_param param;
2051 unsigned value;
2052 };
2053
2054 static void
tc_call_set_context_param(struct pipe_context * pipe,union tc_payload * payload)2055 tc_call_set_context_param(struct pipe_context *pipe,
2056 union tc_payload *payload)
2057 {
2058 struct tc_context_param *p = (struct tc_context_param*)payload;
2059
2060 if (pipe->set_context_param)
2061 pipe->set_context_param(pipe, p->param, p->value);
2062 }
2063
2064 static void
tc_set_context_param(struct pipe_context * _pipe,enum pipe_context_param param,unsigned value)2065 tc_set_context_param(struct pipe_context *_pipe,
2066 enum pipe_context_param param,
2067 unsigned value)
2068 {
2069 struct threaded_context *tc = threaded_context(_pipe);
2070
2071 if (param == PIPE_CONTEXT_PARAM_PIN_THREADS_TO_L3_CACHE) {
2072 /* Pin the gallium thread as requested. */
2073 util_set_thread_affinity(tc->queue.threads[0],
2074 util_cpu_caps.L3_affinity_mask[value],
2075 NULL, UTIL_MAX_CPUS);
2076
2077 /* Execute this immediately (without enqueuing).
2078 * It's required to be thread-safe.
2079 */
2080 struct pipe_context *pipe = tc->pipe;
2081 if (pipe->set_context_param)
2082 pipe->set_context_param(pipe, param, value);
2083 return;
2084 }
2085
2086 if (tc->pipe->set_context_param) {
2087 struct tc_context_param *payload =
2088 tc_add_struct_typed_call(tc, TC_CALL_set_context_param,
2089 tc_context_param);
2090
2091 payload->param = param;
2092 payload->value = value;
2093 }
2094 }
2095
2096 static void
tc_call_set_frontend_noop(struct pipe_context * pipe,union tc_payload * payload)2097 tc_call_set_frontend_noop(struct pipe_context *pipe, union tc_payload *payload)
2098 {
2099 pipe->set_frontend_noop(pipe, payload->boolean);
2100 }
2101
2102 static void
tc_set_frontend_noop(struct pipe_context * _pipe,bool enable)2103 tc_set_frontend_noop(struct pipe_context *_pipe, bool enable)
2104 {
2105 struct threaded_context *tc = threaded_context(_pipe);
2106
2107 tc_add_small_call(tc, TC_CALL_set_frontend_noop)->boolean = enable;
2108 }
2109
2110
2111 /********************************************************************
2112 * draw, launch, clear, blit, copy, flush
2113 */
2114
2115 struct tc_flush_payload {
2116 struct threaded_context *tc;
2117 struct pipe_fence_handle *fence;
2118 unsigned flags;
2119 };
2120
2121 static void
tc_flush_queries(struct threaded_context * tc)2122 tc_flush_queries(struct threaded_context *tc)
2123 {
2124 struct threaded_query *tq, *tmp;
2125 LIST_FOR_EACH_ENTRY_SAFE(tq, tmp, &tc->unflushed_queries, head_unflushed) {
2126 list_del(&tq->head_unflushed);
2127
2128 /* Memory release semantics: due to a possible race with
2129 * tc_get_query_result, we must ensure that the linked list changes
2130 * are visible before setting tq->flushed.
2131 */
2132 p_atomic_set(&tq->flushed, true);
2133 }
2134 }
2135
2136 static void
tc_call_flush(struct pipe_context * pipe,union tc_payload * payload)2137 tc_call_flush(struct pipe_context *pipe, union tc_payload *payload)
2138 {
2139 struct tc_flush_payload *p = (struct tc_flush_payload *)payload;
2140 struct pipe_screen *screen = pipe->screen;
2141
2142 pipe->flush(pipe, p->fence ? &p->fence : NULL, p->flags);
2143 screen->fence_reference(screen, &p->fence, NULL);
2144
2145 if (!(p->flags & PIPE_FLUSH_DEFERRED))
2146 tc_flush_queries(p->tc);
2147 }
2148
2149 static void
tc_flush(struct pipe_context * _pipe,struct pipe_fence_handle ** fence,unsigned flags)2150 tc_flush(struct pipe_context *_pipe, struct pipe_fence_handle **fence,
2151 unsigned flags)
2152 {
2153 struct threaded_context *tc = threaded_context(_pipe);
2154 struct pipe_context *pipe = tc->pipe;
2155 struct pipe_screen *screen = pipe->screen;
2156 bool async = flags & (PIPE_FLUSH_DEFERRED | PIPE_FLUSH_ASYNC);
2157
2158 if (async && tc->create_fence) {
2159 if (fence) {
2160 struct tc_batch *next = &tc->batch_slots[tc->next];
2161
2162 if (!next->token) {
2163 next->token = malloc(sizeof(*next->token));
2164 if (!next->token)
2165 goto out_of_memory;
2166
2167 pipe_reference_init(&next->token->ref, 1);
2168 next->token->tc = tc;
2169 }
2170
2171 screen->fence_reference(screen, fence, tc->create_fence(pipe, next->token));
2172 if (!*fence)
2173 goto out_of_memory;
2174 }
2175
2176 struct tc_flush_payload *p =
2177 tc_add_struct_typed_call(tc, TC_CALL_flush, tc_flush_payload);
2178 p->tc = tc;
2179 p->fence = fence ? *fence : NULL;
2180 p->flags = flags | TC_FLUSH_ASYNC;
2181
2182 if (!(flags & PIPE_FLUSH_DEFERRED))
2183 tc_batch_flush(tc);
2184 return;
2185 }
2186
2187 out_of_memory:
2188 tc_sync_msg(tc, flags & PIPE_FLUSH_END_OF_FRAME ? "end of frame" :
2189 flags & PIPE_FLUSH_DEFERRED ? "deferred fence" : "normal");
2190
2191 if (!(flags & PIPE_FLUSH_DEFERRED))
2192 tc_flush_queries(tc);
2193 pipe->flush(pipe, fence, flags);
2194 }
2195
2196 static void
tc_call_draw_vbo(struct pipe_context * pipe,union tc_payload * payload)2197 tc_call_draw_vbo(struct pipe_context *pipe, union tc_payload *payload)
2198 {
2199 struct tc_full_draw_info *info = (struct tc_full_draw_info*)payload;
2200
2201 pipe->draw_vbo(pipe, &info->draw);
2202 pipe_so_target_reference(&info->draw.count_from_stream_output, NULL);
2203 if (info->draw.index_size)
2204 pipe_resource_reference(&info->draw.index.resource, NULL);
2205 if (info->draw.indirect) {
2206 pipe_resource_reference(&info->indirect.buffer, NULL);
2207 pipe_resource_reference(&info->indirect.indirect_draw_count, NULL);
2208 }
2209 }
2210
2211 static struct tc_full_draw_info *
tc_add_draw_vbo(struct pipe_context * _pipe,bool indirect)2212 tc_add_draw_vbo(struct pipe_context *_pipe, bool indirect)
2213 {
2214 return (struct tc_full_draw_info*)
2215 tc_add_sized_call(threaded_context(_pipe), TC_CALL_draw_vbo,
2216 indirect ? sizeof(struct tc_full_draw_info) :
2217 sizeof(struct pipe_draw_info));
2218 }
2219
2220 static void
tc_draw_vbo(struct pipe_context * _pipe,const struct pipe_draw_info * info)2221 tc_draw_vbo(struct pipe_context *_pipe, const struct pipe_draw_info *info)
2222 {
2223 struct threaded_context *tc = threaded_context(_pipe);
2224 struct pipe_draw_indirect_info *indirect = info->indirect;
2225 unsigned index_size = info->index_size;
2226 bool has_user_indices = info->has_user_indices;
2227
2228 if (index_size && has_user_indices) {
2229 unsigned size = info->count * index_size;
2230 struct pipe_resource *buffer = NULL;
2231 unsigned offset;
2232
2233 tc_assert(!indirect);
2234
2235 /* This must be done before adding draw_vbo, because it could generate
2236 * e.g. transfer_unmap and flush partially-uninitialized draw_vbo
2237 * to the driver if it was done afterwards.
2238 */
2239 u_upload_data(tc->base.stream_uploader, 0, size, 4,
2240 (uint8_t*)info->index.user + info->start * index_size,
2241 &offset, &buffer);
2242 if (unlikely(!buffer))
2243 return;
2244
2245 struct tc_full_draw_info *p = tc_add_draw_vbo(_pipe, false);
2246 p->draw.count_from_stream_output = NULL;
2247 pipe_so_target_reference(&p->draw.count_from_stream_output,
2248 info->count_from_stream_output);
2249 memcpy(&p->draw, info, sizeof(*info));
2250 p->draw.has_user_indices = false;
2251 p->draw.index.resource = buffer;
2252 p->draw.start = offset >> util_logbase2(index_size);
2253 } else {
2254 /* Non-indexed call or indexed with a real index buffer. */
2255 struct tc_full_draw_info *p = tc_add_draw_vbo(_pipe, indirect != NULL);
2256 p->draw.count_from_stream_output = NULL;
2257 pipe_so_target_reference(&p->draw.count_from_stream_output,
2258 info->count_from_stream_output);
2259 if (index_size) {
2260 tc_set_resource_reference(&p->draw.index.resource,
2261 info->index.resource);
2262 }
2263 memcpy(&p->draw, info, sizeof(*info));
2264
2265 if (indirect) {
2266 tc_set_resource_reference(&p->draw.indirect->buffer, indirect->buffer);
2267 tc_set_resource_reference(&p->indirect.indirect_draw_count,
2268 indirect->indirect_draw_count);
2269 memcpy(&p->indirect, indirect, sizeof(*indirect));
2270 p->draw.indirect = &p->indirect;
2271 }
2272 }
2273 }
2274
2275 static void
tc_call_launch_grid(struct pipe_context * pipe,union tc_payload * payload)2276 tc_call_launch_grid(struct pipe_context *pipe, union tc_payload *payload)
2277 {
2278 struct pipe_grid_info *p = (struct pipe_grid_info *)payload;
2279
2280 pipe->launch_grid(pipe, p);
2281 pipe_resource_reference(&p->indirect, NULL);
2282 }
2283
2284 static void
tc_launch_grid(struct pipe_context * _pipe,const struct pipe_grid_info * info)2285 tc_launch_grid(struct pipe_context *_pipe,
2286 const struct pipe_grid_info *info)
2287 {
2288 struct threaded_context *tc = threaded_context(_pipe);
2289 struct pipe_grid_info *p = tc_add_struct_typed_call(tc, TC_CALL_launch_grid,
2290 pipe_grid_info);
2291 assert(info->input == NULL);
2292
2293 tc_set_resource_reference(&p->indirect, info->indirect);
2294 memcpy(p, info, sizeof(*info));
2295 }
2296
2297 static void
tc_call_resource_copy_region(struct pipe_context * pipe,union tc_payload * payload)2298 tc_call_resource_copy_region(struct pipe_context *pipe, union tc_payload *payload)
2299 {
2300 struct tc_resource_copy_region *p = (struct tc_resource_copy_region *)payload;
2301
2302 pipe->resource_copy_region(pipe, p->dst, p->dst_level, p->dstx, p->dsty,
2303 p->dstz, p->src, p->src_level, &p->src_box);
2304 pipe_resource_reference(&p->dst, NULL);
2305 pipe_resource_reference(&p->src, NULL);
2306 }
2307
2308 static void
tc_resource_copy_region(struct pipe_context * _pipe,struct pipe_resource * dst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * src,unsigned src_level,const struct pipe_box * src_box)2309 tc_resource_copy_region(struct pipe_context *_pipe,
2310 struct pipe_resource *dst, unsigned dst_level,
2311 unsigned dstx, unsigned dsty, unsigned dstz,
2312 struct pipe_resource *src, unsigned src_level,
2313 const struct pipe_box *src_box)
2314 {
2315 struct threaded_context *tc = threaded_context(_pipe);
2316 struct threaded_resource *tdst = threaded_resource(dst);
2317 struct tc_resource_copy_region *p =
2318 tc_add_struct_typed_call(tc, TC_CALL_resource_copy_region,
2319 tc_resource_copy_region);
2320
2321 tc_set_resource_reference(&p->dst, dst);
2322 p->dst_level = dst_level;
2323 p->dstx = dstx;
2324 p->dsty = dsty;
2325 p->dstz = dstz;
2326 tc_set_resource_reference(&p->src, src);
2327 p->src_level = src_level;
2328 p->src_box = *src_box;
2329
2330 if (dst->target == PIPE_BUFFER)
2331 util_range_add(&tdst->b, &tdst->valid_buffer_range,
2332 dstx, dstx + src_box->width);
2333 }
2334
2335 static void
tc_call_blit(struct pipe_context * pipe,union tc_payload * payload)2336 tc_call_blit(struct pipe_context *pipe, union tc_payload *payload)
2337 {
2338 struct pipe_blit_info *blit = (struct pipe_blit_info*)payload;
2339
2340 pipe->blit(pipe, blit);
2341 pipe_resource_reference(&blit->dst.resource, NULL);
2342 pipe_resource_reference(&blit->src.resource, NULL);
2343 }
2344
2345 static void
tc_blit(struct pipe_context * _pipe,const struct pipe_blit_info * info)2346 tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info)
2347 {
2348 struct threaded_context *tc = threaded_context(_pipe);
2349 struct pipe_blit_info *blit =
2350 tc_add_struct_typed_call(tc, TC_CALL_blit, pipe_blit_info);
2351
2352 tc_set_resource_reference(&blit->dst.resource, info->dst.resource);
2353 tc_set_resource_reference(&blit->src.resource, info->src.resource);
2354 memcpy(blit, info, sizeof(*info));
2355 }
2356
2357 struct tc_generate_mipmap {
2358 struct pipe_resource *res;
2359 enum pipe_format format;
2360 unsigned base_level;
2361 unsigned last_level;
2362 unsigned first_layer;
2363 unsigned last_layer;
2364 };
2365
2366 static void
tc_call_generate_mipmap(struct pipe_context * pipe,union tc_payload * payload)2367 tc_call_generate_mipmap(struct pipe_context *pipe, union tc_payload *payload)
2368 {
2369 struct tc_generate_mipmap *p = (struct tc_generate_mipmap *)payload;
2370 ASSERTED bool result = pipe->generate_mipmap(pipe, p->res, p->format,
2371 p->base_level,
2372 p->last_level,
2373 p->first_layer,
2374 p->last_layer);
2375 assert(result);
2376 pipe_resource_reference(&p->res, NULL);
2377 }
2378
2379 static bool
tc_generate_mipmap(struct pipe_context * _pipe,struct pipe_resource * res,enum pipe_format format,unsigned base_level,unsigned last_level,unsigned first_layer,unsigned last_layer)2380 tc_generate_mipmap(struct pipe_context *_pipe,
2381 struct pipe_resource *res,
2382 enum pipe_format format,
2383 unsigned base_level,
2384 unsigned last_level,
2385 unsigned first_layer,
2386 unsigned last_layer)
2387 {
2388 struct threaded_context *tc = threaded_context(_pipe);
2389 struct pipe_context *pipe = tc->pipe;
2390 struct pipe_screen *screen = pipe->screen;
2391 unsigned bind = PIPE_BIND_SAMPLER_VIEW;
2392
2393 if (util_format_is_depth_or_stencil(format))
2394 bind = PIPE_BIND_DEPTH_STENCIL;
2395 else
2396 bind = PIPE_BIND_RENDER_TARGET;
2397
2398 if (!screen->is_format_supported(screen, format, res->target,
2399 res->nr_samples, res->nr_storage_samples,
2400 bind))
2401 return false;
2402
2403 struct tc_generate_mipmap *p =
2404 tc_add_struct_typed_call(tc, TC_CALL_generate_mipmap, tc_generate_mipmap);
2405
2406 tc_set_resource_reference(&p->res, res);
2407 p->format = format;
2408 p->base_level = base_level;
2409 p->last_level = last_level;
2410 p->first_layer = first_layer;
2411 p->last_layer = last_layer;
2412 return true;
2413 }
2414
2415 static void
tc_call_flush_resource(struct pipe_context * pipe,union tc_payload * payload)2416 tc_call_flush_resource(struct pipe_context *pipe, union tc_payload *payload)
2417 {
2418 pipe->flush_resource(pipe, payload->resource);
2419 pipe_resource_reference(&payload->resource, NULL);
2420 }
2421
2422 static void
tc_flush_resource(struct pipe_context * _pipe,struct pipe_resource * resource)2423 tc_flush_resource(struct pipe_context *_pipe,
2424 struct pipe_resource *resource)
2425 {
2426 struct threaded_context *tc = threaded_context(_pipe);
2427 union tc_payload *payload = tc_add_small_call(tc, TC_CALL_flush_resource);
2428
2429 tc_set_resource_reference(&payload->resource, resource);
2430 }
2431
2432 static void
tc_call_invalidate_resource(struct pipe_context * pipe,union tc_payload * payload)2433 tc_call_invalidate_resource(struct pipe_context *pipe, union tc_payload *payload)
2434 {
2435 pipe->invalidate_resource(pipe, payload->resource);
2436 pipe_resource_reference(&payload->resource, NULL);
2437 }
2438
2439 static void
tc_invalidate_resource(struct pipe_context * _pipe,struct pipe_resource * resource)2440 tc_invalidate_resource(struct pipe_context *_pipe,
2441 struct pipe_resource *resource)
2442 {
2443 struct threaded_context *tc = threaded_context(_pipe);
2444
2445 if (resource->target == PIPE_BUFFER) {
2446 tc_invalidate_buffer(tc, threaded_resource(resource));
2447 return;
2448 }
2449
2450 union tc_payload *payload = tc_add_small_call(tc, TC_CALL_invalidate_resource);
2451 tc_set_resource_reference(&payload->resource, resource);
2452 }
2453
2454 struct tc_clear {
2455 unsigned buffers;
2456 struct pipe_scissor_state scissor_state;
2457 union pipe_color_union color;
2458 double depth;
2459 unsigned stencil;
2460 bool scissor_state_set;
2461 };
2462
2463 static void
tc_call_clear(struct pipe_context * pipe,union tc_payload * payload)2464 tc_call_clear(struct pipe_context *pipe, union tc_payload *payload)
2465 {
2466 struct tc_clear *p = (struct tc_clear *)payload;
2467 pipe->clear(pipe, p->buffers, p->scissor_state_set ? &p->scissor_state : NULL, &p->color, p->depth, p->stencil);
2468 }
2469
2470 static void
tc_clear(struct pipe_context * _pipe,unsigned buffers,const struct pipe_scissor_state * scissor_state,const union pipe_color_union * color,double depth,unsigned stencil)2471 tc_clear(struct pipe_context *_pipe, unsigned buffers, const struct pipe_scissor_state *scissor_state,
2472 const union pipe_color_union *color, double depth,
2473 unsigned stencil)
2474 {
2475 struct threaded_context *tc = threaded_context(_pipe);
2476 struct tc_clear *p = tc_add_struct_typed_call(tc, TC_CALL_clear, tc_clear);
2477
2478 p->buffers = buffers;
2479 if (scissor_state)
2480 p->scissor_state = *scissor_state;
2481 p->scissor_state_set = !!scissor_state;
2482 p->color = *color;
2483 p->depth = depth;
2484 p->stencil = stencil;
2485 }
2486
2487 static void
tc_clear_render_target(struct pipe_context * _pipe,struct pipe_surface * dst,const union pipe_color_union * color,unsigned dstx,unsigned dsty,unsigned width,unsigned height,bool render_condition_enabled)2488 tc_clear_render_target(struct pipe_context *_pipe,
2489 struct pipe_surface *dst,
2490 const union pipe_color_union *color,
2491 unsigned dstx, unsigned dsty,
2492 unsigned width, unsigned height,
2493 bool render_condition_enabled)
2494 {
2495 struct threaded_context *tc = threaded_context(_pipe);
2496 struct pipe_context *pipe = tc->pipe;
2497
2498 tc_sync(tc);
2499 pipe->clear_render_target(pipe, dst, color, dstx, dsty, width, height,
2500 render_condition_enabled);
2501 }
2502
2503 static void
tc_clear_depth_stencil(struct pipe_context * _pipe,struct pipe_surface * dst,unsigned clear_flags,double depth,unsigned stencil,unsigned dstx,unsigned dsty,unsigned width,unsigned height,bool render_condition_enabled)2504 tc_clear_depth_stencil(struct pipe_context *_pipe,
2505 struct pipe_surface *dst, unsigned clear_flags,
2506 double depth, unsigned stencil, unsigned dstx,
2507 unsigned dsty, unsigned width, unsigned height,
2508 bool render_condition_enabled)
2509 {
2510 struct threaded_context *tc = threaded_context(_pipe);
2511 struct pipe_context *pipe = tc->pipe;
2512
2513 tc_sync(tc);
2514 pipe->clear_depth_stencil(pipe, dst, clear_flags, depth, stencil,
2515 dstx, dsty, width, height,
2516 render_condition_enabled);
2517 }
2518
2519 struct tc_clear_buffer {
2520 struct pipe_resource *res;
2521 unsigned offset;
2522 unsigned size;
2523 char clear_value[16];
2524 int clear_value_size;
2525 };
2526
2527 static void
tc_call_clear_buffer(struct pipe_context * pipe,union tc_payload * payload)2528 tc_call_clear_buffer(struct pipe_context *pipe, union tc_payload *payload)
2529 {
2530 struct tc_clear_buffer *p = (struct tc_clear_buffer *)payload;
2531
2532 pipe->clear_buffer(pipe, p->res, p->offset, p->size, p->clear_value,
2533 p->clear_value_size);
2534 pipe_resource_reference(&p->res, NULL);
2535 }
2536
2537 static void
tc_clear_buffer(struct pipe_context * _pipe,struct pipe_resource * res,unsigned offset,unsigned size,const void * clear_value,int clear_value_size)2538 tc_clear_buffer(struct pipe_context *_pipe, struct pipe_resource *res,
2539 unsigned offset, unsigned size,
2540 const void *clear_value, int clear_value_size)
2541 {
2542 struct threaded_context *tc = threaded_context(_pipe);
2543 struct threaded_resource *tres = threaded_resource(res);
2544 struct tc_clear_buffer *p =
2545 tc_add_struct_typed_call(tc, TC_CALL_clear_buffer, tc_clear_buffer);
2546
2547 tc_set_resource_reference(&p->res, res);
2548 p->offset = offset;
2549 p->size = size;
2550 memcpy(p->clear_value, clear_value, clear_value_size);
2551 p->clear_value_size = clear_value_size;
2552
2553 util_range_add(&tres->b, &tres->valid_buffer_range, offset, offset + size);
2554 }
2555
2556 struct tc_clear_texture {
2557 struct pipe_resource *res;
2558 unsigned level;
2559 struct pipe_box box;
2560 char data[16];
2561 };
2562
2563 static void
tc_call_clear_texture(struct pipe_context * pipe,union tc_payload * payload)2564 tc_call_clear_texture(struct pipe_context *pipe, union tc_payload *payload)
2565 {
2566 struct tc_clear_texture *p = (struct tc_clear_texture *)payload;
2567
2568 pipe->clear_texture(pipe, p->res, p->level, &p->box, p->data);
2569 pipe_resource_reference(&p->res, NULL);
2570 }
2571
2572 static void
tc_clear_texture(struct pipe_context * _pipe,struct pipe_resource * res,unsigned level,const struct pipe_box * box,const void * data)2573 tc_clear_texture(struct pipe_context *_pipe, struct pipe_resource *res,
2574 unsigned level, const struct pipe_box *box, const void *data)
2575 {
2576 struct threaded_context *tc = threaded_context(_pipe);
2577 struct tc_clear_texture *p =
2578 tc_add_struct_typed_call(tc, TC_CALL_clear_texture, tc_clear_texture);
2579
2580 tc_set_resource_reference(&p->res, res);
2581 p->level = level;
2582 p->box = *box;
2583 memcpy(p->data, data,
2584 util_format_get_blocksize(res->format));
2585 }
2586
2587 struct tc_resource_commit {
2588 struct pipe_resource *res;
2589 unsigned level;
2590 struct pipe_box box;
2591 bool commit;
2592 };
2593
2594 static void
tc_call_resource_commit(struct pipe_context * pipe,union tc_payload * payload)2595 tc_call_resource_commit(struct pipe_context *pipe, union tc_payload *payload)
2596 {
2597 struct tc_resource_commit *p = (struct tc_resource_commit *)payload;
2598
2599 pipe->resource_commit(pipe, p->res, p->level, &p->box, p->commit);
2600 pipe_resource_reference(&p->res, NULL);
2601 }
2602
2603 static bool
tc_resource_commit(struct pipe_context * _pipe,struct pipe_resource * res,unsigned level,struct pipe_box * box,bool commit)2604 tc_resource_commit(struct pipe_context *_pipe, struct pipe_resource *res,
2605 unsigned level, struct pipe_box *box, bool commit)
2606 {
2607 struct threaded_context *tc = threaded_context(_pipe);
2608 struct tc_resource_commit *p =
2609 tc_add_struct_typed_call(tc, TC_CALL_resource_commit, tc_resource_commit);
2610
2611 tc_set_resource_reference(&p->res, res);
2612 p->level = level;
2613 p->box = *box;
2614 p->commit = commit;
2615 return true; /* we don't care about the return value for this call */
2616 }
2617
2618
2619 /********************************************************************
2620 * callback
2621 */
2622
2623 struct tc_callback_payload {
2624 void (*fn)(void *data);
2625 void *data;
2626 };
2627
2628 static void
tc_call_callback(UNUSED struct pipe_context * pipe,union tc_payload * payload)2629 tc_call_callback(UNUSED struct pipe_context *pipe, union tc_payload *payload)
2630 {
2631 struct tc_callback_payload *p = (struct tc_callback_payload *)payload;
2632
2633 p->fn(p->data);
2634 }
2635
2636 static void
tc_callback(struct pipe_context * _pipe,void (* fn)(void *),void * data,bool asap)2637 tc_callback(struct pipe_context *_pipe, void (*fn)(void *), void *data,
2638 bool asap)
2639 {
2640 struct threaded_context *tc = threaded_context(_pipe);
2641
2642 if (asap && tc_is_sync(tc)) {
2643 fn(data);
2644 return;
2645 }
2646
2647 struct tc_callback_payload *p =
2648 tc_add_struct_typed_call(tc, TC_CALL_callback, tc_callback_payload);
2649 p->fn = fn;
2650 p->data = data;
2651 }
2652
2653
2654 /********************************************************************
2655 * create & destroy
2656 */
2657
2658 static void
tc_destroy(struct pipe_context * _pipe)2659 tc_destroy(struct pipe_context *_pipe)
2660 {
2661 struct threaded_context *tc = threaded_context(_pipe);
2662 struct pipe_context *pipe = tc->pipe;
2663
2664 if (tc->base.const_uploader &&
2665 tc->base.stream_uploader != tc->base.const_uploader)
2666 u_upload_destroy(tc->base.const_uploader);
2667
2668 if (tc->base.stream_uploader)
2669 u_upload_destroy(tc->base.stream_uploader);
2670
2671 tc_sync(tc);
2672
2673 if (util_queue_is_initialized(&tc->queue)) {
2674 util_queue_destroy(&tc->queue);
2675
2676 for (unsigned i = 0; i < TC_MAX_BATCHES; i++) {
2677 util_queue_fence_destroy(&tc->batch_slots[i].fence);
2678 assert(!tc->batch_slots[i].token);
2679 }
2680 }
2681
2682 slab_destroy_child(&tc->pool_transfers);
2683 assert(tc->batch_slots[tc->next].num_total_call_slots == 0);
2684 pipe->destroy(pipe);
2685 os_free_aligned(tc);
2686 }
2687
2688 static const tc_execute execute_func[TC_NUM_CALLS] = {
2689 #define CALL(name) tc_call_##name,
2690 #include "u_threaded_context_calls.h"
2691 #undef CALL
2692 };
2693
2694 /**
2695 * Wrap an existing pipe_context into a threaded_context.
2696 *
2697 * \param pipe pipe_context to wrap
2698 * \param parent_transfer_pool parent slab pool set up for creating pipe_-
2699 * transfer objects; the driver should have one
2700 * in pipe_screen.
2701 * \param replace_buffer callback for replacing a pipe_resource's storage
2702 * with another pipe_resource's storage.
2703 * \param out if successful, the threaded_context will be returned here in
2704 * addition to the return value if "out" != NULL
2705 */
2706 struct pipe_context *
threaded_context_create(struct pipe_context * pipe,struct slab_parent_pool * parent_transfer_pool,tc_replace_buffer_storage_func replace_buffer,tc_create_fence_func create_fence,struct threaded_context ** out)2707 threaded_context_create(struct pipe_context *pipe,
2708 struct slab_parent_pool *parent_transfer_pool,
2709 tc_replace_buffer_storage_func replace_buffer,
2710 tc_create_fence_func create_fence,
2711 struct threaded_context **out)
2712 {
2713 struct threaded_context *tc;
2714
2715 STATIC_ASSERT(sizeof(union tc_payload) <= 8);
2716 STATIC_ASSERT(sizeof(struct tc_call) <= 16);
2717
2718 if (!pipe)
2719 return NULL;
2720
2721 util_cpu_detect();
2722
2723 if (!debug_get_bool_option("GALLIUM_THREAD", util_cpu_caps.nr_cpus > 1))
2724 return pipe;
2725
2726 tc = os_malloc_aligned(sizeof(struct threaded_context), 16);
2727 if (!tc) {
2728 pipe->destroy(pipe);
2729 return NULL;
2730 }
2731 memset(tc, 0, sizeof(*tc));
2732
2733 assert((uintptr_t)tc % 16 == 0);
2734 /* These should be static asserts, but they don't work with MSVC */
2735 assert(offsetof(struct threaded_context, batch_slots) % 16 == 0);
2736 assert(offsetof(struct threaded_context, batch_slots[0].call) % 16 == 0);
2737 assert(offsetof(struct threaded_context, batch_slots[0].call[1]) % 16 == 0);
2738 assert(offsetof(struct threaded_context, batch_slots[1].call) % 16 == 0);
2739
2740 /* The driver context isn't wrapped, so set its "priv" to NULL. */
2741 pipe->priv = NULL;
2742
2743 tc->pipe = pipe;
2744 tc->replace_buffer_storage = replace_buffer;
2745 tc->create_fence = create_fence;
2746 tc->map_buffer_alignment =
2747 pipe->screen->get_param(pipe->screen, PIPE_CAP_MIN_MAP_BUFFER_ALIGNMENT);
2748 tc->base.priv = pipe; /* priv points to the wrapped driver context */
2749 tc->base.screen = pipe->screen;
2750 tc->base.destroy = tc_destroy;
2751 tc->base.callback = tc_callback;
2752
2753 tc->base.stream_uploader = u_upload_clone(&tc->base, pipe->stream_uploader);
2754 if (pipe->stream_uploader == pipe->const_uploader)
2755 tc->base.const_uploader = tc->base.stream_uploader;
2756 else
2757 tc->base.const_uploader = u_upload_clone(&tc->base, pipe->const_uploader);
2758
2759 if (!tc->base.stream_uploader || !tc->base.const_uploader)
2760 goto fail;
2761
2762 /* The queue size is the number of batches "waiting". Batches are removed
2763 * from the queue before being executed, so keep one tc_batch slot for that
2764 * execution. Also, keep one unused slot for an unflushed batch.
2765 */
2766 if (!util_queue_init(&tc->queue, "gdrv", TC_MAX_BATCHES - 2, 1, 0))
2767 goto fail;
2768
2769 for (unsigned i = 0; i < TC_MAX_BATCHES; i++) {
2770 tc->batch_slots[i].sentinel = TC_SENTINEL;
2771 tc->batch_slots[i].pipe = pipe;
2772 util_queue_fence_init(&tc->batch_slots[i].fence);
2773 }
2774
2775 list_inithead(&tc->unflushed_queries);
2776
2777 slab_create_child(&tc->pool_transfers, parent_transfer_pool);
2778
2779 tc->base.set_context_param = tc_set_context_param; /* always set this */
2780
2781 #define CTX_INIT(_member) \
2782 tc->base._member = tc->pipe->_member ? tc_##_member : NULL
2783
2784 CTX_INIT(flush);
2785 CTX_INIT(draw_vbo);
2786 CTX_INIT(launch_grid);
2787 CTX_INIT(resource_copy_region);
2788 CTX_INIT(blit);
2789 CTX_INIT(clear);
2790 CTX_INIT(clear_render_target);
2791 CTX_INIT(clear_depth_stencil);
2792 CTX_INIT(clear_buffer);
2793 CTX_INIT(clear_texture);
2794 CTX_INIT(flush_resource);
2795 CTX_INIT(generate_mipmap);
2796 CTX_INIT(render_condition);
2797 CTX_INIT(create_query);
2798 CTX_INIT(create_batch_query);
2799 CTX_INIT(destroy_query);
2800 CTX_INIT(begin_query);
2801 CTX_INIT(end_query);
2802 CTX_INIT(get_query_result);
2803 CTX_INIT(get_query_result_resource);
2804 CTX_INIT(set_active_query_state);
2805 CTX_INIT(create_blend_state);
2806 CTX_INIT(bind_blend_state);
2807 CTX_INIT(delete_blend_state);
2808 CTX_INIT(create_sampler_state);
2809 CTX_INIT(bind_sampler_states);
2810 CTX_INIT(delete_sampler_state);
2811 CTX_INIT(create_rasterizer_state);
2812 CTX_INIT(bind_rasterizer_state);
2813 CTX_INIT(delete_rasterizer_state);
2814 CTX_INIT(create_depth_stencil_alpha_state);
2815 CTX_INIT(bind_depth_stencil_alpha_state);
2816 CTX_INIT(delete_depth_stencil_alpha_state);
2817 CTX_INIT(create_fs_state);
2818 CTX_INIT(bind_fs_state);
2819 CTX_INIT(delete_fs_state);
2820 CTX_INIT(create_vs_state);
2821 CTX_INIT(bind_vs_state);
2822 CTX_INIT(delete_vs_state);
2823 CTX_INIT(create_gs_state);
2824 CTX_INIT(bind_gs_state);
2825 CTX_INIT(delete_gs_state);
2826 CTX_INIT(create_tcs_state);
2827 CTX_INIT(bind_tcs_state);
2828 CTX_INIT(delete_tcs_state);
2829 CTX_INIT(create_tes_state);
2830 CTX_INIT(bind_tes_state);
2831 CTX_INIT(delete_tes_state);
2832 CTX_INIT(create_compute_state);
2833 CTX_INIT(bind_compute_state);
2834 CTX_INIT(delete_compute_state);
2835 CTX_INIT(create_vertex_elements_state);
2836 CTX_INIT(bind_vertex_elements_state);
2837 CTX_INIT(delete_vertex_elements_state);
2838 CTX_INIT(set_blend_color);
2839 CTX_INIT(set_stencil_ref);
2840 CTX_INIT(set_sample_mask);
2841 CTX_INIT(set_min_samples);
2842 CTX_INIT(set_clip_state);
2843 CTX_INIT(set_constant_buffer);
2844 CTX_INIT(set_inlinable_constants);
2845 CTX_INIT(set_framebuffer_state);
2846 CTX_INIT(set_polygon_stipple);
2847 CTX_INIT(set_scissor_states);
2848 CTX_INIT(set_viewport_states);
2849 CTX_INIT(set_window_rectangles);
2850 CTX_INIT(set_sampler_views);
2851 CTX_INIT(set_tess_state);
2852 CTX_INIT(set_shader_buffers);
2853 CTX_INIT(set_shader_images);
2854 CTX_INIT(set_vertex_buffers);
2855 CTX_INIT(create_stream_output_target);
2856 CTX_INIT(stream_output_target_destroy);
2857 CTX_INIT(set_stream_output_targets);
2858 CTX_INIT(create_sampler_view);
2859 CTX_INIT(sampler_view_destroy);
2860 CTX_INIT(create_surface);
2861 CTX_INIT(surface_destroy);
2862 CTX_INIT(transfer_map);
2863 CTX_INIT(transfer_flush_region);
2864 CTX_INIT(transfer_unmap);
2865 CTX_INIT(buffer_subdata);
2866 CTX_INIT(texture_subdata);
2867 CTX_INIT(texture_barrier);
2868 CTX_INIT(memory_barrier);
2869 CTX_INIT(resource_commit);
2870 CTX_INIT(create_video_codec);
2871 CTX_INIT(create_video_buffer);
2872 CTX_INIT(set_compute_resources);
2873 CTX_INIT(set_global_binding);
2874 CTX_INIT(get_sample_position);
2875 CTX_INIT(invalidate_resource);
2876 CTX_INIT(get_device_reset_status);
2877 CTX_INIT(set_device_reset_callback);
2878 CTX_INIT(dump_debug_state);
2879 CTX_INIT(set_log_context);
2880 CTX_INIT(emit_string_marker);
2881 CTX_INIT(set_debug_callback);
2882 CTX_INIT(create_fence_fd);
2883 CTX_INIT(fence_server_sync);
2884 CTX_INIT(fence_server_signal);
2885 CTX_INIT(get_timestamp);
2886 CTX_INIT(create_texture_handle);
2887 CTX_INIT(delete_texture_handle);
2888 CTX_INIT(make_texture_handle_resident);
2889 CTX_INIT(create_image_handle);
2890 CTX_INIT(delete_image_handle);
2891 CTX_INIT(make_image_handle_resident);
2892 CTX_INIT(set_frontend_noop);
2893 #undef CTX_INIT
2894
2895 if (out)
2896 *out = tc;
2897
2898 return &tc->base;
2899
2900 fail:
2901 tc_destroy(&tc->base);
2902 return NULL;
2903 }
2904