1 /**************************************************************************
2 *
3 * Copyright 2012 Marek Olšák <maraeo@gmail.com>
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL THE AUTHORS AND/OR THEIR SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "util/u_cpu_detect.h"
29 #include "util/u_helpers.h"
30 #include "util/u_inlines.h"
31 #include "util/u_upload_mgr.h"
32 #include "util/u_thread.h"
33 #include "util/os_time.h"
34 #include <inttypes.h>
35
36 /**
37 * This function is used to copy an array of pipe_vertex_buffer structures,
38 * while properly referencing the pipe_vertex_buffer::buffer member.
39 *
40 * enabled_buffers is updated such that the bits corresponding to the indices
41 * of disabled buffers are set to 0 and the enabled ones are set to 1.
42 *
43 * \sa util_copy_framebuffer_state
44 */
util_set_vertex_buffers_mask(struct pipe_vertex_buffer * dst,uint32_t * enabled_buffers,const struct pipe_vertex_buffer * src,unsigned start_slot,unsigned count)45 void util_set_vertex_buffers_mask(struct pipe_vertex_buffer *dst,
46 uint32_t *enabled_buffers,
47 const struct pipe_vertex_buffer *src,
48 unsigned start_slot, unsigned count)
49 {
50 unsigned i;
51 uint32_t bitmask = 0;
52
53 dst += start_slot;
54
55 *enabled_buffers &= ~u_bit_consecutive(start_slot, count);
56
57 if (src) {
58 for (i = 0; i < count; i++) {
59 if (src[i].buffer.resource)
60 bitmask |= 1 << i;
61
62 pipe_vertex_buffer_unreference(&dst[i]);
63
64 if (!src[i].is_user_buffer)
65 pipe_resource_reference(&dst[i].buffer.resource, src[i].buffer.resource);
66 }
67
68 /* Copy over the other members of pipe_vertex_buffer. */
69 memcpy(dst, src, count * sizeof(struct pipe_vertex_buffer));
70
71 *enabled_buffers |= bitmask << start_slot;
72 }
73 else {
74 /* Unreference the buffers. */
75 for (i = 0; i < count; i++)
76 pipe_vertex_buffer_unreference(&dst[i]);
77 }
78 }
79
80 /**
81 * Same as util_set_vertex_buffers_mask, but it only returns the number
82 * of bound buffers.
83 */
util_set_vertex_buffers_count(struct pipe_vertex_buffer * dst,unsigned * dst_count,const struct pipe_vertex_buffer * src,unsigned start_slot,unsigned count)84 void util_set_vertex_buffers_count(struct pipe_vertex_buffer *dst,
85 unsigned *dst_count,
86 const struct pipe_vertex_buffer *src,
87 unsigned start_slot, unsigned count)
88 {
89 unsigned i;
90 uint32_t enabled_buffers = 0;
91
92 for (i = 0; i < *dst_count; i++) {
93 if (dst[i].buffer.resource)
94 enabled_buffers |= (1ull << i);
95 }
96
97 util_set_vertex_buffers_mask(dst, &enabled_buffers, src, start_slot,
98 count);
99
100 *dst_count = util_last_bit(enabled_buffers);
101 }
102
103 /**
104 * This function is used to copy an array of pipe_shader_buffer structures,
105 * while properly referencing the pipe_shader_buffer::buffer member.
106 *
107 * \sa util_set_vertex_buffer_mask
108 */
util_set_shader_buffers_mask(struct pipe_shader_buffer * dst,uint32_t * enabled_buffers,const struct pipe_shader_buffer * src,unsigned start_slot,unsigned count)109 void util_set_shader_buffers_mask(struct pipe_shader_buffer *dst,
110 uint32_t *enabled_buffers,
111 const struct pipe_shader_buffer *src,
112 unsigned start_slot, unsigned count)
113 {
114 unsigned i;
115
116 dst += start_slot;
117
118 if (src) {
119 for (i = 0; i < count; i++) {
120 pipe_resource_reference(&dst[i].buffer, src[i].buffer);
121
122 if (src[i].buffer)
123 *enabled_buffers |= (1ull << (start_slot + i));
124 else
125 *enabled_buffers &= ~(1ull << (start_slot + i));
126 }
127
128 /* Copy over the other members of pipe_shader_buffer. */
129 memcpy(dst, src, count * sizeof(struct pipe_shader_buffer));
130 }
131 else {
132 /* Unreference the buffers. */
133 for (i = 0; i < count; i++)
134 pipe_resource_reference(&dst[i].buffer, NULL);
135
136 *enabled_buffers &= ~(((1ull << count) - 1) << start_slot);
137 }
138 }
139
140 /**
141 * Given a user index buffer, save the structure to "saved", and upload it.
142 */
143 bool
util_upload_index_buffer(struct pipe_context * pipe,const struct pipe_draw_info * info,struct pipe_resource ** out_buffer,unsigned * out_offset,unsigned alignment)144 util_upload_index_buffer(struct pipe_context *pipe,
145 const struct pipe_draw_info *info,
146 struct pipe_resource **out_buffer,
147 unsigned *out_offset, unsigned alignment)
148 {
149 unsigned start_offset = info->start * info->index_size;
150
151 u_upload_data(pipe->stream_uploader, start_offset,
152 info->count * info->index_size, alignment,
153 (char*)info->index.user + start_offset,
154 out_offset, out_buffer);
155 u_upload_unmap(pipe->stream_uploader);
156 *out_offset -= start_offset;
157 return *out_buffer != NULL;
158 }
159
160 /* This is a helper for hardware bring-up. Don't remove. */
161 struct pipe_query *
util_begin_pipestat_query(struct pipe_context * ctx)162 util_begin_pipestat_query(struct pipe_context *ctx)
163 {
164 struct pipe_query *q =
165 ctx->create_query(ctx, PIPE_QUERY_PIPELINE_STATISTICS, 0);
166 if (!q)
167 return NULL;
168
169 ctx->begin_query(ctx, q);
170 return q;
171 }
172
173 /* This is a helper for hardware bring-up. Don't remove. */
174 void
util_end_pipestat_query(struct pipe_context * ctx,struct pipe_query * q,FILE * f)175 util_end_pipestat_query(struct pipe_context *ctx, struct pipe_query *q,
176 FILE *f)
177 {
178 static unsigned counter;
179 struct pipe_query_data_pipeline_statistics stats;
180
181 ctx->end_query(ctx, q);
182 ctx->get_query_result(ctx, q, true, (void*)&stats);
183 ctx->destroy_query(ctx, q);
184
185 fprintf(f,
186 "Draw call %u:\n"
187 " ia_vertices = %"PRIu64"\n"
188 " ia_primitives = %"PRIu64"\n"
189 " vs_invocations = %"PRIu64"\n"
190 " gs_invocations = %"PRIu64"\n"
191 " gs_primitives = %"PRIu64"\n"
192 " c_invocations = %"PRIu64"\n"
193 " c_primitives = %"PRIu64"\n"
194 " ps_invocations = %"PRIu64"\n"
195 " hs_invocations = %"PRIu64"\n"
196 " ds_invocations = %"PRIu64"\n"
197 " cs_invocations = %"PRIu64"\n",
198 (unsigned)p_atomic_inc_return(&counter),
199 stats.ia_vertices,
200 stats.ia_primitives,
201 stats.vs_invocations,
202 stats.gs_invocations,
203 stats.gs_primitives,
204 stats.c_invocations,
205 stats.c_primitives,
206 stats.ps_invocations,
207 stats.hs_invocations,
208 stats.ds_invocations,
209 stats.cs_invocations);
210 }
211
212 /* This is a helper for hardware bring-up. Don't remove. */
213 void
util_wait_for_idle(struct pipe_context * ctx)214 util_wait_for_idle(struct pipe_context *ctx)
215 {
216 struct pipe_fence_handle *fence = NULL;
217
218 ctx->flush(ctx, &fence, 0);
219 ctx->screen->fence_finish(ctx->screen, NULL, fence, PIPE_TIMEOUT_INFINITE);
220 }
221
222 void
util_throttle_init(struct util_throttle * t,uint64_t max_mem_usage)223 util_throttle_init(struct util_throttle *t, uint64_t max_mem_usage)
224 {
225 t->max_mem_usage = max_mem_usage;
226 }
227
228 void
util_throttle_deinit(struct pipe_screen * screen,struct util_throttle * t)229 util_throttle_deinit(struct pipe_screen *screen, struct util_throttle *t)
230 {
231 for (unsigned i = 0; i < ARRAY_SIZE(t->ring); i++)
232 screen->fence_reference(screen, &t->ring[i].fence, NULL);
233 }
234
235 static uint64_t
util_get_throttle_total_memory_usage(struct util_throttle * t)236 util_get_throttle_total_memory_usage(struct util_throttle *t)
237 {
238 uint64_t total_usage = 0;
239
240 for (unsigned i = 0; i < ARRAY_SIZE(t->ring); i++)
241 total_usage += t->ring[i].mem_usage;
242 return total_usage;
243 }
244
util_dump_throttle_ring(struct util_throttle * t)245 static void util_dump_throttle_ring(struct util_throttle *t)
246 {
247 printf("Throttle:\n");
248 for (unsigned i = 0; i < ARRAY_SIZE(t->ring); i++) {
249 printf(" ring[%u]: fence = %s, mem_usage = %"PRIu64"%s%s\n",
250 i, t->ring[i].fence ? "yes" : " no",
251 t->ring[i].mem_usage,
252 t->flush_index == i ? " [flush]" : "",
253 t->wait_index == i ? " [wait]" : "");
254 }
255 }
256
257 /**
258 * Notify util_throttle that the next operation allocates memory.
259 * util_throttle tracks memory usage and waits for fences until its tracked
260 * memory usage decreases.
261 *
262 * Example:
263 * util_throttle_memory_usage(..., w*h*d*Bpp);
264 * TexSubImage(..., w, h, d, ...);
265 *
266 * This means that TexSubImage can't allocate more memory its maximum limit
267 * set during initialization.
268 */
269 void
util_throttle_memory_usage(struct pipe_context * pipe,struct util_throttle * t,uint64_t memory_size)270 util_throttle_memory_usage(struct pipe_context *pipe,
271 struct util_throttle *t, uint64_t memory_size)
272 {
273 (void)util_dump_throttle_ring; /* silence warning */
274
275 if (!t->max_mem_usage)
276 return;
277
278 struct pipe_screen *screen = pipe->screen;
279 struct pipe_fence_handle **fence = NULL;
280 unsigned ring_size = ARRAY_SIZE(t->ring);
281 uint64_t total = util_get_throttle_total_memory_usage(t);
282
283 /* If there is not enough memory, walk the list of fences and find
284 * the latest one that we need to wait for.
285 */
286 while (t->wait_index != t->flush_index &&
287 total && total + memory_size > t->max_mem_usage) {
288 assert(t->ring[t->wait_index].fence);
289
290 /* Release an older fence if we need to wait for a newer one. */
291 if (fence)
292 screen->fence_reference(screen, fence, NULL);
293
294 fence = &t->ring[t->wait_index].fence;
295 t->ring[t->wait_index].mem_usage = 0;
296 t->wait_index = (t->wait_index + 1) % ring_size;
297
298 total = util_get_throttle_total_memory_usage(t);
299 }
300
301 /* Wait for the fence to decrease memory usage. */
302 if (fence) {
303 screen->fence_finish(screen, pipe, *fence, PIPE_TIMEOUT_INFINITE);
304 screen->fence_reference(screen, fence, NULL);
305 }
306
307 /* Flush and get a fence if we've exhausted memory usage for the current
308 * slot.
309 */
310 if (t->ring[t->flush_index].mem_usage &&
311 t->ring[t->flush_index].mem_usage + memory_size >
312 t->max_mem_usage / (ring_size / 2)) {
313 struct pipe_fence_handle **fence =
314 &t->ring[t->flush_index].fence;
315
316 /* Expect that the current flush slot doesn't have a fence yet. */
317 assert(!*fence);
318
319 pipe->flush(pipe, fence, PIPE_FLUSH_ASYNC);
320 t->flush_index = (t->flush_index + 1) % ring_size;
321
322 /* Vacate the next slot if it's occupied. This should be rare. */
323 if (t->flush_index == t->wait_index) {
324 struct pipe_fence_handle **fence =
325 &t->ring[t->wait_index].fence;
326
327 t->ring[t->wait_index].mem_usage = 0;
328 t->wait_index = (t->wait_index + 1) % ring_size;
329
330 assert(*fence);
331 screen->fence_finish(screen, pipe, *fence, PIPE_TIMEOUT_INFINITE);
332 screen->fence_reference(screen, fence, NULL);
333 }
334
335 assert(!t->ring[t->flush_index].mem_usage);
336 assert(!t->ring[t->flush_index].fence);
337 }
338
339 t->ring[t->flush_index].mem_usage += memory_size;
340 }
341