1 /*
2 * Copyright 2011 Christoph Bumiller
3 * Copyright 2015 Samuel Pitoiset
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #define NV50_PUSH_EXPLICIT_SPACE_CHECKING
25
26 #include "nv50/nv50_context.h"
27 #include "nv50/nv50_query_hw.h"
28 #include "nv50/nv50_query_hw_metric.h"
29 #include "nv50/nv50_query_hw_sm.h"
30 #include "nv_object.xml.h"
31
32 #define NV50_HW_QUERY_STATE_READY 0
33 #define NV50_HW_QUERY_STATE_ACTIVE 1
34 #define NV50_HW_QUERY_STATE_ENDED 2
35 #define NV50_HW_QUERY_STATE_FLUSHED 3
36
37 /* XXX: Nested queries, and simultaneous queries on multiple gallium contexts
38 * (since we use only a single GPU channel per screen) will not work properly.
39 *
40 * The first is not that big of an issue because OpenGL does not allow nested
41 * queries anyway.
42 */
43
44 #define NV50_HW_QUERY_ALLOC_SPACE 256
45
46 bool
nv50_hw_query_allocate(struct nv50_context * nv50,struct nv50_query * q,int size)47 nv50_hw_query_allocate(struct nv50_context *nv50, struct nv50_query *q,
48 int size)
49 {
50 struct nv50_screen *screen = nv50->screen;
51 struct nv50_hw_query *hq = nv50_hw_query(q);
52 int ret;
53
54 if (hq->bo) {
55 nouveau_bo_ref(NULL, &hq->bo);
56 if (hq->mm) {
57 if (hq->state == NV50_HW_QUERY_STATE_READY)
58 nouveau_mm_free(hq->mm);
59 else
60 nouveau_fence_work(screen->base.fence.current,
61 nouveau_mm_free_work, hq->mm);
62 }
63 }
64 if (size) {
65 hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size,
66 &hq->bo, &hq->base_offset);
67 if (!hq->bo)
68 return false;
69 hq->offset = hq->base_offset;
70
71 ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
72 if (ret) {
73 nv50_hw_query_allocate(nv50, q, 0);
74 return false;
75 }
76 hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
77 }
78 return true;
79 }
80
81 static void
nv50_hw_query_get(struct nouveau_pushbuf * push,struct nv50_query * q,unsigned offset,uint32_t get)82 nv50_hw_query_get(struct nouveau_pushbuf *push, struct nv50_query *q,
83 unsigned offset, uint32_t get)
84 {
85 struct nv50_hw_query *hq = nv50_hw_query(q);
86
87 offset += hq->offset;
88
89 PUSH_SPACE(push, 5);
90 PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
91 BEGIN_NV04(push, NV50_3D(QUERY_ADDRESS_HIGH), 4);
92 PUSH_DATAh(push, hq->bo->offset + offset);
93 PUSH_DATA (push, hq->bo->offset + offset);
94 PUSH_DATA (push, hq->sequence);
95 PUSH_DATA (push, get);
96 }
97
98 static inline void
nv50_hw_query_update(struct nv50_query * q)99 nv50_hw_query_update(struct nv50_query *q)
100 {
101 struct nv50_hw_query *hq = nv50_hw_query(q);
102
103 if (hq->is64bit) {
104 if (nouveau_fence_signalled(hq->fence))
105 hq->state = NV50_HW_QUERY_STATE_READY;
106 } else {
107 if (hq->data[0] == hq->sequence)
108 hq->state = NV50_HW_QUERY_STATE_READY;
109 }
110 }
111
112 static void
nv50_hw_destroy_query(struct nv50_context * nv50,struct nv50_query * q)113 nv50_hw_destroy_query(struct nv50_context *nv50, struct nv50_query *q)
114 {
115 struct nv50_hw_query *hq = nv50_hw_query(q);
116
117 if (hq->funcs && hq->funcs->destroy_query) {
118 hq->funcs->destroy_query(nv50, hq);
119 return;
120 }
121
122 nv50_hw_query_allocate(nv50, q, 0);
123 nouveau_fence_ref(NULL, &hq->fence);
124 FREE(hq);
125 }
126
127 static boolean
nv50_hw_begin_query(struct nv50_context * nv50,struct nv50_query * q)128 nv50_hw_begin_query(struct nv50_context *nv50, struct nv50_query *q)
129 {
130 struct nouveau_pushbuf *push = nv50->base.pushbuf;
131 struct nv50_hw_query *hq = nv50_hw_query(q);
132
133 if (hq->funcs && hq->funcs->begin_query)
134 return hq->funcs->begin_query(nv50, hq);
135
136 /* For occlusion queries we have to change the storage, because a previous
137 * query might set the initial render condition to false even *after* we re-
138 * initialized it to true.
139 */
140 if (hq->rotate) {
141 hq->offset += hq->rotate;
142 hq->data += hq->rotate / sizeof(*hq->data);
143 if (hq->offset - hq->base_offset == NV50_HW_QUERY_ALLOC_SPACE)
144 nv50_hw_query_allocate(nv50, q, NV50_HW_QUERY_ALLOC_SPACE);
145
146 /* XXX: can we do this with the GPU, and sync with respect to a previous
147 * query ?
148 */
149 hq->data[0] = hq->sequence; /* initialize sequence */
150 hq->data[1] = 1; /* initial render condition = true */
151 hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
152 hq->data[5] = 0;
153 }
154 if (!hq->is64bit)
155 hq->data[0] = hq->sequence++; /* the previously used one */
156
157 switch (q->type) {
158 case PIPE_QUERY_OCCLUSION_COUNTER:
159 case PIPE_QUERY_OCCLUSION_PREDICATE:
160 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
161 hq->nesting = nv50->screen->num_occlusion_queries_active++;
162 if (hq->nesting) {
163 nv50_hw_query_get(push, q, 0x10, 0x0100f002);
164 } else {
165 PUSH_SPACE(push, 4);
166 BEGIN_NV04(push, NV50_3D(COUNTER_RESET), 1);
167 PUSH_DATA (push, NV50_3D_COUNTER_RESET_SAMPLECNT);
168 BEGIN_NV04(push, NV50_3D(SAMPLECNT_ENABLE), 1);
169 PUSH_DATA (push, 1);
170 }
171 break;
172 case PIPE_QUERY_PRIMITIVES_GENERATED:
173 nv50_hw_query_get(push, q, 0x10, 0x06805002);
174 break;
175 case PIPE_QUERY_PRIMITIVES_EMITTED:
176 nv50_hw_query_get(push, q, 0x10, 0x05805002);
177 break;
178 case PIPE_QUERY_SO_STATISTICS:
179 nv50_hw_query_get(push, q, 0x20, 0x05805002);
180 nv50_hw_query_get(push, q, 0x30, 0x06805002);
181 break;
182 case PIPE_QUERY_PIPELINE_STATISTICS:
183 nv50_hw_query_get(push, q, 0x80, 0x00801002); /* VFETCH, VERTICES */
184 nv50_hw_query_get(push, q, 0x90, 0x01801002); /* VFETCH, PRIMS */
185 nv50_hw_query_get(push, q, 0xa0, 0x02802002); /* VP, LAUNCHES */
186 nv50_hw_query_get(push, q, 0xb0, 0x03806002); /* GP, LAUNCHES */
187 nv50_hw_query_get(push, q, 0xc0, 0x04806002); /* GP, PRIMS_OUT */
188 nv50_hw_query_get(push, q, 0xd0, 0x07804002); /* RAST, PRIMS_IN */
189 nv50_hw_query_get(push, q, 0xe0, 0x08804002); /* RAST, PRIMS_OUT */
190 nv50_hw_query_get(push, q, 0xf0, 0x0980a002); /* ROP, PIXELS */
191 break;
192 case PIPE_QUERY_TIME_ELAPSED:
193 nv50_hw_query_get(push, q, 0x10, 0x00005002);
194 break;
195 default:
196 assert(0);
197 return false;
198 }
199 hq->state = NV50_HW_QUERY_STATE_ACTIVE;
200 return true;
201 }
202
203 static void
nv50_hw_end_query(struct nv50_context * nv50,struct nv50_query * q)204 nv50_hw_end_query(struct nv50_context *nv50, struct nv50_query *q)
205 {
206 struct nouveau_pushbuf *push = nv50->base.pushbuf;
207 struct nv50_hw_query *hq = nv50_hw_query(q);
208
209 if (hq->funcs && hq->funcs->end_query) {
210 hq->funcs->end_query(nv50, hq);
211 return;
212 }
213
214 hq->state = NV50_HW_QUERY_STATE_ENDED;
215
216 switch (q->type) {
217 case PIPE_QUERY_OCCLUSION_COUNTER:
218 case PIPE_QUERY_OCCLUSION_PREDICATE:
219 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
220 nv50_hw_query_get(push, q, 0, 0x0100f002);
221 if (--nv50->screen->num_occlusion_queries_active == 0) {
222 PUSH_SPACE(push, 2);
223 BEGIN_NV04(push, NV50_3D(SAMPLECNT_ENABLE), 1);
224 PUSH_DATA (push, 0);
225 }
226 break;
227 case PIPE_QUERY_PRIMITIVES_GENERATED:
228 nv50_hw_query_get(push, q, 0, 0x06805002);
229 break;
230 case PIPE_QUERY_PRIMITIVES_EMITTED:
231 nv50_hw_query_get(push, q, 0, 0x05805002);
232 break;
233 case PIPE_QUERY_SO_STATISTICS:
234 nv50_hw_query_get(push, q, 0x00, 0x05805002);
235 nv50_hw_query_get(push, q, 0x10, 0x06805002);
236 break;
237 case PIPE_QUERY_PIPELINE_STATISTICS:
238 nv50_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
239 nv50_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
240 nv50_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
241 nv50_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
242 nv50_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
243 nv50_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
244 nv50_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
245 nv50_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
246 break;
247 case PIPE_QUERY_TIMESTAMP:
248 hq->sequence++;
249 /* fall through */
250 case PIPE_QUERY_TIME_ELAPSED:
251 nv50_hw_query_get(push, q, 0, 0x00005002);
252 break;
253 case PIPE_QUERY_GPU_FINISHED:
254 hq->sequence++;
255 nv50_hw_query_get(push, q, 0, 0x1000f010);
256 break;
257 case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
258 hq->sequence++;
259 nv50_hw_query_get(push, q, 0, 0x0d005002 | (q->index << 5));
260 break;
261 case PIPE_QUERY_TIMESTAMP_DISJOINT:
262 /* This query is not issued on GPU because disjoint is forced to false */
263 hq->state = NV50_HW_QUERY_STATE_READY;
264 break;
265 default:
266 assert(0);
267 break;
268 }
269 if (hq->is64bit)
270 nouveau_fence_ref(nv50->screen->base.fence.current, &hq->fence);
271 }
272
273 static boolean
nv50_hw_get_query_result(struct nv50_context * nv50,struct nv50_query * q,boolean wait,union pipe_query_result * result)274 nv50_hw_get_query_result(struct nv50_context *nv50, struct nv50_query *q,
275 boolean wait, union pipe_query_result *result)
276 {
277 struct nv50_hw_query *hq = nv50_hw_query(q);
278 uint64_t *res64 = (uint64_t *)result;
279 uint32_t *res32 = (uint32_t *)result;
280 uint8_t *res8 = (uint8_t *)result;
281 uint64_t *data64 = (uint64_t *)hq->data;
282 int i;
283
284 if (hq->funcs && hq->funcs->get_query_result)
285 return hq->funcs->get_query_result(nv50, hq, wait, result);
286
287 if (hq->state != NV50_HW_QUERY_STATE_READY)
288 nv50_hw_query_update(q);
289
290 if (hq->state != NV50_HW_QUERY_STATE_READY) {
291 if (!wait) {
292 /* for broken apps that spin on GL_QUERY_RESULT_AVAILABLE */
293 if (hq->state != NV50_HW_QUERY_STATE_FLUSHED) {
294 hq->state = NV50_HW_QUERY_STATE_FLUSHED;
295 PUSH_KICK(nv50->base.pushbuf);
296 }
297 return false;
298 }
299 if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nv50->screen->base.client))
300 return false;
301 }
302 hq->state = NV50_HW_QUERY_STATE_READY;
303
304 switch (q->type) {
305 case PIPE_QUERY_GPU_FINISHED:
306 res8[0] = true;
307 break;
308 case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
309 res64[0] = hq->data[1] - hq->data[5];
310 break;
311 case PIPE_QUERY_OCCLUSION_PREDICATE:
312 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
313 res8[0] = hq->data[1] != hq->data[5];
314 break;
315 case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
316 case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
317 res64[0] = data64[0] - data64[2];
318 break;
319 case PIPE_QUERY_SO_STATISTICS:
320 res64[0] = data64[0] - data64[4];
321 res64[1] = data64[2] - data64[6];
322 break;
323 case PIPE_QUERY_PIPELINE_STATISTICS:
324 for (i = 0; i < 8; ++i)
325 res64[i] = data64[i * 2] - data64[16 + i * 2];
326 break;
327 case PIPE_QUERY_TIMESTAMP:
328 res64[0] = data64[1];
329 break;
330 case PIPE_QUERY_TIMESTAMP_DISJOINT:
331 res64[0] = 1000000000;
332 res8[8] = false;
333 break;
334 case PIPE_QUERY_TIME_ELAPSED:
335 res64[0] = data64[1] - data64[3];
336 break;
337 case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
338 res32[0] = hq->data[1];
339 break;
340 default:
341 assert(0);
342 return false;
343 }
344
345 return true;
346 }
347
348 static const struct nv50_query_funcs hw_query_funcs = {
349 .destroy_query = nv50_hw_destroy_query,
350 .begin_query = nv50_hw_begin_query,
351 .end_query = nv50_hw_end_query,
352 .get_query_result = nv50_hw_get_query_result,
353 };
354
355 struct nv50_query *
nv50_hw_create_query(struct nv50_context * nv50,unsigned type,unsigned index)356 nv50_hw_create_query(struct nv50_context *nv50, unsigned type, unsigned index)
357 {
358 struct nv50_hw_query *hq;
359 struct nv50_query *q;
360
361 hq = nv50_hw_sm_create_query(nv50, type);
362 if (hq) {
363 hq->base.funcs = &hw_query_funcs;
364 return (struct nv50_query *)hq;
365 }
366
367 hq = nv50_hw_metric_create_query(nv50, type);
368 if (hq) {
369 hq->base.funcs = &hw_query_funcs;
370 return (struct nv50_query *)hq;
371 }
372
373 hq = CALLOC_STRUCT(nv50_hw_query);
374 if (!hq)
375 return NULL;
376
377 q = &hq->base;
378 q->funcs = &hw_query_funcs;
379 q->type = type;
380
381 switch (q->type) {
382 case PIPE_QUERY_OCCLUSION_COUNTER:
383 case PIPE_QUERY_OCCLUSION_PREDICATE:
384 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
385 hq->rotate = 32;
386 break;
387 case PIPE_QUERY_PRIMITIVES_GENERATED:
388 case PIPE_QUERY_PRIMITIVES_EMITTED:
389 case PIPE_QUERY_SO_STATISTICS:
390 case PIPE_QUERY_PIPELINE_STATISTICS:
391 hq->is64bit = true;
392 break;
393 case PIPE_QUERY_TIME_ELAPSED:
394 case PIPE_QUERY_TIMESTAMP:
395 case PIPE_QUERY_TIMESTAMP_DISJOINT:
396 case PIPE_QUERY_GPU_FINISHED:
397 case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
398 break;
399 default:
400 debug_printf("invalid query type: %u\n", type);
401 FREE(q);
402 return NULL;
403 }
404
405 if (!nv50_hw_query_allocate(nv50, q, NV50_HW_QUERY_ALLOC_SPACE)) {
406 FREE(hq);
407 return NULL;
408 }
409
410 if (hq->rotate) {
411 /* we advance before query_begin ! */
412 hq->offset -= hq->rotate;
413 hq->data -= hq->rotate / sizeof(*hq->data);
414 }
415
416 return q;
417 }
418
419 int
nv50_hw_get_driver_query_info(struct nv50_screen * screen,unsigned id,struct pipe_driver_query_info * info)420 nv50_hw_get_driver_query_info(struct nv50_screen *screen, unsigned id,
421 struct pipe_driver_query_info *info)
422 {
423 int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
424
425 num_hw_sm_queries = nv50_hw_sm_get_driver_query_info(screen, 0, NULL);
426 num_hw_metric_queries =
427 nv50_hw_metric_get_driver_query_info(screen, 0, NULL);
428
429 if (!info)
430 return num_hw_sm_queries + num_hw_metric_queries;
431
432 if (id < num_hw_sm_queries)
433 return nv50_hw_sm_get_driver_query_info(screen, id, info);
434
435 return nv50_hw_metric_get_driver_query_info(screen,
436 id - num_hw_sm_queries, info);
437 }
438
439 void
nv50_hw_query_pushbuf_submit(struct nouveau_pushbuf * push,uint16_t method,struct nv50_query * q,unsigned result_offset)440 nv50_hw_query_pushbuf_submit(struct nouveau_pushbuf *push, uint16_t method,
441 struct nv50_query *q, unsigned result_offset)
442 {
443 struct nv50_hw_query *hq = nv50_hw_query(q);
444
445 nv50_hw_query_update(q);
446 if (hq->state != NV50_HW_QUERY_STATE_READY)
447 nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, push->client);
448 hq->state = NV50_HW_QUERY_STATE_READY;
449
450 BEGIN_NV04(push, SUBC_3D(method), 1);
451 PUSH_DATA (push, hq->data[result_offset / 4]);
452 }
453
454 void
nv84_hw_query_fifo_wait(struct nouveau_pushbuf * push,struct nv50_query * q)455 nv84_hw_query_fifo_wait(struct nouveau_pushbuf *push, struct nv50_query *q)
456 {
457 struct nv50_hw_query *hq = nv50_hw_query(q);
458 unsigned offset = hq->offset;
459
460 PUSH_SPACE(push, 5);
461 PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
462 BEGIN_NV04(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
463 PUSH_DATAh(push, hq->bo->offset + offset);
464 PUSH_DATA (push, hq->bo->offset + offset);
465 PUSH_DATA (push, hq->sequence);
466 PUSH_DATA (push, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
467 }
468