1 /*
2 * Copyright 2011 Christoph Bumiller
3 * Copyright 2015 Samuel Pitoiset
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
25
26 #include "nvc0/nvc0_context.h"
27 #include "nvc0/nvc0_query_hw.h"
28 #include "nvc0/nvc0_query_hw_metric.h"
29 #include "nvc0/nvc0_query_hw_sm.h"
30
31 #define NVC0_HW_QUERY_STATE_READY 0
32 #define NVC0_HW_QUERY_STATE_ACTIVE 1
33 #define NVC0_HW_QUERY_STATE_ENDED 2
34 #define NVC0_HW_QUERY_STATE_FLUSHED 3
35
36 #define NVC0_HW_QUERY_ALLOC_SPACE 256
37
38 bool
nvc0_hw_query_allocate(struct nvc0_context * nvc0,struct nvc0_query * q,int size)39 nvc0_hw_query_allocate(struct nvc0_context *nvc0, struct nvc0_query *q,
40 int size)
41 {
42 struct nvc0_hw_query *hq = nvc0_hw_query(q);
43 struct nvc0_screen *screen = nvc0->screen;
44 int ret;
45
46 if (hq->bo) {
47 nouveau_bo_ref(NULL, &hq->bo);
48 if (hq->mm) {
49 if (hq->state == NVC0_HW_QUERY_STATE_READY)
50 nouveau_mm_free(hq->mm);
51 else
52 nouveau_fence_work(screen->base.fence.current,
53 nouveau_mm_free_work, hq->mm);
54 }
55 }
56 if (size) {
57 hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size, &hq->bo,
58 &hq->base_offset);
59 if (!hq->bo)
60 return false;
61 hq->offset = hq->base_offset;
62
63 ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
64 if (ret) {
65 nvc0_hw_query_allocate(nvc0, q, 0);
66 return false;
67 }
68 hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
69 }
70 return true;
71 }
72
73 static void
nvc0_hw_query_get(struct nouveau_pushbuf * push,struct nvc0_query * q,unsigned offset,uint32_t get)74 nvc0_hw_query_get(struct nouveau_pushbuf *push, struct nvc0_query *q,
75 unsigned offset, uint32_t get)
76 {
77 struct nvc0_hw_query *hq = nvc0_hw_query(q);
78
79 offset += hq->offset;
80
81 PUSH_SPACE(push, 5);
82 PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
83 BEGIN_NVC0(push, NVC0_3D(QUERY_ADDRESS_HIGH), 4);
84 PUSH_DATAh(push, hq->bo->offset + offset);
85 PUSH_DATA (push, hq->bo->offset + offset);
86 PUSH_DATA (push, hq->sequence);
87 PUSH_DATA (push, get);
88 }
89
90 static void
nvc0_hw_query_rotate(struct nvc0_context * nvc0,struct nvc0_query * q)91 nvc0_hw_query_rotate(struct nvc0_context *nvc0, struct nvc0_query *q)
92 {
93 struct nvc0_hw_query *hq = nvc0_hw_query(q);
94
95 hq->offset += hq->rotate;
96 hq->data += hq->rotate / sizeof(*hq->data);
97 if (hq->offset - hq->base_offset == NVC0_HW_QUERY_ALLOC_SPACE)
98 nvc0_hw_query_allocate(nvc0, q, NVC0_HW_QUERY_ALLOC_SPACE);
99 }
100
101 static inline void
nvc0_hw_query_update(struct nouveau_client * cli,struct nvc0_query * q)102 nvc0_hw_query_update(struct nouveau_client *cli, struct nvc0_query *q)
103 {
104 struct nvc0_hw_query *hq = nvc0_hw_query(q);
105
106 if (hq->is64bit) {
107 if (nouveau_fence_signalled(hq->fence))
108 hq->state = NVC0_HW_QUERY_STATE_READY;
109 } else {
110 if (hq->data[0] == hq->sequence)
111 hq->state = NVC0_HW_QUERY_STATE_READY;
112 }
113 }
114
115 static void
nvc0_hw_destroy_query(struct nvc0_context * nvc0,struct nvc0_query * q)116 nvc0_hw_destroy_query(struct nvc0_context *nvc0, struct nvc0_query *q)
117 {
118 struct nvc0_hw_query *hq = nvc0_hw_query(q);
119
120 if (hq->funcs && hq->funcs->destroy_query) {
121 hq->funcs->destroy_query(nvc0, hq);
122 return;
123 }
124
125 nvc0_hw_query_allocate(nvc0, q, 0);
126 nouveau_fence_ref(NULL, &hq->fence);
127 FREE(hq);
128 }
129
130 static boolean
nvc0_hw_begin_query(struct nvc0_context * nvc0,struct nvc0_query * q)131 nvc0_hw_begin_query(struct nvc0_context *nvc0, struct nvc0_query *q)
132 {
133 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
134 struct nvc0_hw_query *hq = nvc0_hw_query(q);
135 bool ret = true;
136
137 if (hq->funcs && hq->funcs->begin_query)
138 return hq->funcs->begin_query(nvc0, hq);
139
140 /* For occlusion queries we have to change the storage, because a previous
141 * query might set the initial render conition to false even *after* we re-
142 * initialized it to true.
143 */
144 if (hq->rotate) {
145 nvc0_hw_query_rotate(nvc0, q);
146
147 /* XXX: can we do this with the GPU, and sync with respect to a previous
148 * query ?
149 */
150 hq->data[0] = hq->sequence; /* initialize sequence */
151 hq->data[1] = 1; /* initial render condition = true */
152 hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
153 hq->data[5] = 0;
154 }
155 hq->sequence++;
156
157 switch (q->type) {
158 case PIPE_QUERY_OCCLUSION_COUNTER:
159 case PIPE_QUERY_OCCLUSION_PREDICATE:
160 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
161 hq->nesting = nvc0->screen->num_occlusion_queries_active++;
162 if (hq->nesting) {
163 nvc0_hw_query_get(push, q, 0x10, 0x0100f002);
164 } else {
165 PUSH_SPACE(push, 3);
166 BEGIN_NVC0(push, NVC0_3D(COUNTER_RESET), 1);
167 PUSH_DATA (push, NVC0_3D_COUNTER_RESET_SAMPLECNT);
168 IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 1);
169 }
170 break;
171 case PIPE_QUERY_PRIMITIVES_GENERATED:
172 nvc0_hw_query_get(push, q, 0x10, 0x09005002 | (q->index << 5));
173 break;
174 case PIPE_QUERY_PRIMITIVES_EMITTED:
175 nvc0_hw_query_get(push, q, 0x10, 0x05805002 | (q->index << 5));
176 break;
177 case PIPE_QUERY_SO_STATISTICS:
178 nvc0_hw_query_get(push, q, 0x20, 0x05805002 | (q->index << 5));
179 nvc0_hw_query_get(push, q, 0x30, 0x06805002 | (q->index << 5));
180 break;
181 case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
182 nvc0_hw_query_get(push, q, 0x10, 0x03005002 | (q->index << 5));
183 break;
184 case PIPE_QUERY_TIME_ELAPSED:
185 nvc0_hw_query_get(push, q, 0x10, 0x00005002);
186 break;
187 case PIPE_QUERY_PIPELINE_STATISTICS:
188 nvc0_hw_query_get(push, q, 0xc0 + 0x00, 0x00801002); /* VFETCH, VERTICES */
189 nvc0_hw_query_get(push, q, 0xc0 + 0x10, 0x01801002); /* VFETCH, PRIMS */
190 nvc0_hw_query_get(push, q, 0xc0 + 0x20, 0x02802002); /* VP, LAUNCHES */
191 nvc0_hw_query_get(push, q, 0xc0 + 0x30, 0x03806002); /* GP, LAUNCHES */
192 nvc0_hw_query_get(push, q, 0xc0 + 0x40, 0x04806002); /* GP, PRIMS_OUT */
193 nvc0_hw_query_get(push, q, 0xc0 + 0x50, 0x07804002); /* RAST, PRIMS_IN */
194 nvc0_hw_query_get(push, q, 0xc0 + 0x60, 0x08804002); /* RAST, PRIMS_OUT */
195 nvc0_hw_query_get(push, q, 0xc0 + 0x70, 0x0980a002); /* ROP, PIXELS */
196 nvc0_hw_query_get(push, q, 0xc0 + 0x80, 0x0d808002); /* TCP, LAUNCHES */
197 nvc0_hw_query_get(push, q, 0xc0 + 0x90, 0x0e809002); /* TEP, LAUNCHES */
198 break;
199 default:
200 break;
201 }
202 hq->state = NVC0_HW_QUERY_STATE_ACTIVE;
203 return ret;
204 }
205
206 static void
nvc0_hw_end_query(struct nvc0_context * nvc0,struct nvc0_query * q)207 nvc0_hw_end_query(struct nvc0_context *nvc0, struct nvc0_query *q)
208 {
209 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
210 struct nvc0_hw_query *hq = nvc0_hw_query(q);
211
212 if (hq->funcs && hq->funcs->end_query) {
213 hq->funcs->end_query(nvc0, hq);
214 return;
215 }
216
217 if (hq->state != NVC0_HW_QUERY_STATE_ACTIVE) {
218 /* some queries don't require 'begin' to be called (e.g. GPU_FINISHED) */
219 if (hq->rotate)
220 nvc0_hw_query_rotate(nvc0, q);
221 hq->sequence++;
222 }
223 hq->state = NVC0_HW_QUERY_STATE_ENDED;
224
225 switch (q->type) {
226 case PIPE_QUERY_OCCLUSION_COUNTER:
227 case PIPE_QUERY_OCCLUSION_PREDICATE:
228 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
229 nvc0_hw_query_get(push, q, 0, 0x0100f002);
230 if (--nvc0->screen->num_occlusion_queries_active == 0) {
231 PUSH_SPACE(push, 1);
232 IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 0);
233 }
234 break;
235 case PIPE_QUERY_PRIMITIVES_GENERATED:
236 nvc0_hw_query_get(push, q, 0, 0x09005002 | (q->index << 5));
237 break;
238 case PIPE_QUERY_PRIMITIVES_EMITTED:
239 nvc0_hw_query_get(push, q, 0, 0x05805002 | (q->index << 5));
240 break;
241 case PIPE_QUERY_SO_STATISTICS:
242 nvc0_hw_query_get(push, q, 0x00, 0x05805002 | (q->index << 5));
243 nvc0_hw_query_get(push, q, 0x10, 0x06805002 | (q->index << 5));
244 break;
245 case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
246 /* TODO: How do we sum over all streams for render condition ? */
247 /* PRIMS_DROPPED doesn't write sequence, use a ZERO query to sync on */
248 nvc0_hw_query_get(push, q, 0x00, 0x03005002 | (q->index << 5));
249 nvc0_hw_query_get(push, q, 0x20, 0x00005002);
250 break;
251 case PIPE_QUERY_TIMESTAMP:
252 case PIPE_QUERY_TIME_ELAPSED:
253 nvc0_hw_query_get(push, q, 0, 0x00005002);
254 break;
255 case PIPE_QUERY_GPU_FINISHED:
256 nvc0_hw_query_get(push, q, 0, 0x1000f010);
257 break;
258 case PIPE_QUERY_PIPELINE_STATISTICS:
259 nvc0_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
260 nvc0_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
261 nvc0_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
262 nvc0_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
263 nvc0_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
264 nvc0_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
265 nvc0_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
266 nvc0_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
267 nvc0_hw_query_get(push, q, 0x80, 0x0d808002); /* TCP, LAUNCHES */
268 nvc0_hw_query_get(push, q, 0x90, 0x0e809002); /* TEP, LAUNCHES */
269 break;
270 case PIPE_QUERY_TIMESTAMP_DISJOINT:
271 /* This query is not issued on GPU because disjoint is forced to false */
272 hq->state = NVC0_HW_QUERY_STATE_READY;
273 break;
274 case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
275 /* indexed by TFB buffer instead of by vertex stream */
276 nvc0_hw_query_get(push, q, 0x00, 0x0d005002 | (q->index << 5));
277 break;
278 default:
279 break;
280 }
281 if (hq->is64bit)
282 nouveau_fence_ref(nvc0->screen->base.fence.current, &hq->fence);
283 }
284
285 static boolean
nvc0_hw_get_query_result(struct nvc0_context * nvc0,struct nvc0_query * q,boolean wait,union pipe_query_result * result)286 nvc0_hw_get_query_result(struct nvc0_context *nvc0, struct nvc0_query *q,
287 boolean wait, union pipe_query_result *result)
288 {
289 struct nvc0_hw_query *hq = nvc0_hw_query(q);
290 uint64_t *res64 = (uint64_t*)result;
291 uint32_t *res32 = (uint32_t*)result;
292 uint8_t *res8 = (uint8_t*)result;
293 uint64_t *data64 = (uint64_t *)hq->data;
294 unsigned i;
295
296 if (hq->funcs && hq->funcs->get_query_result)
297 return hq->funcs->get_query_result(nvc0, hq, wait, result);
298
299 if (hq->state != NVC0_HW_QUERY_STATE_READY)
300 nvc0_hw_query_update(nvc0->screen->base.client, q);
301
302 if (hq->state != NVC0_HW_QUERY_STATE_READY) {
303 if (!wait) {
304 if (hq->state != NVC0_HW_QUERY_STATE_FLUSHED) {
305 hq->state = NVC0_HW_QUERY_STATE_FLUSHED;
306 /* flush for silly apps that spin on GL_QUERY_RESULT_AVAILABLE */
307 PUSH_KICK(nvc0->base.pushbuf);
308 }
309 return false;
310 }
311 if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nvc0->screen->base.client))
312 return false;
313 NOUVEAU_DRV_STAT(&nvc0->screen->base, query_sync_count, 1);
314 }
315 hq->state = NVC0_HW_QUERY_STATE_READY;
316
317 switch (q->type) {
318 case PIPE_QUERY_GPU_FINISHED:
319 res8[0] = true;
320 break;
321 case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
322 res64[0] = hq->data[1] - hq->data[5];
323 break;
324 case PIPE_QUERY_OCCLUSION_PREDICATE:
325 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
326 res8[0] = hq->data[1] != hq->data[5];
327 break;
328 case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
329 case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
330 res64[0] = data64[0] - data64[2];
331 break;
332 case PIPE_QUERY_SO_STATISTICS:
333 res64[0] = data64[0] - data64[4];
334 res64[1] = data64[2] - data64[6];
335 break;
336 case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
337 res8[0] = data64[0] != data64[2];
338 break;
339 case PIPE_QUERY_TIMESTAMP:
340 res64[0] = data64[1];
341 break;
342 case PIPE_QUERY_TIMESTAMP_DISJOINT:
343 res64[0] = 1000000000;
344 res8[8] = false;
345 break;
346 case PIPE_QUERY_TIME_ELAPSED:
347 res64[0] = data64[1] - data64[3];
348 break;
349 case PIPE_QUERY_PIPELINE_STATISTICS:
350 for (i = 0; i < 10; ++i)
351 res64[i] = data64[i * 2] - data64[24 + i * 2];
352 result->pipeline_statistics.cs_invocations = 0;
353 break;
354 case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
355 res32[0] = hq->data[1];
356 break;
357 default:
358 assert(0); /* can't happen, we don't create queries with invalid type */
359 return false;
360 }
361
362 return true;
363 }
364
365 static void
nvc0_hw_get_query_result_resource(struct nvc0_context * nvc0,struct nvc0_query * q,boolean wait,enum pipe_query_value_type result_type,int index,struct pipe_resource * resource,unsigned offset)366 nvc0_hw_get_query_result_resource(struct nvc0_context *nvc0,
367 struct nvc0_query *q,
368 boolean wait,
369 enum pipe_query_value_type result_type,
370 int index,
371 struct pipe_resource *resource,
372 unsigned offset)
373 {
374 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
375 struct nvc0_hw_query *hq = nvc0_hw_query(q);
376 struct nv04_resource *buf = nv04_resource(resource);
377 unsigned qoffset = 0, stride;
378
379 assert(!hq->funcs || !hq->funcs->get_query_result);
380
381 if (index == -1) {
382 /* TODO: Use a macro to write the availability of the query */
383 if (hq->state != NVC0_HW_QUERY_STATE_READY)
384 nvc0_hw_query_update(nvc0->screen->base.client, q);
385 uint32_t ready[2] = {hq->state == NVC0_HW_QUERY_STATE_READY};
386 nvc0->base.push_cb(&nvc0->base, buf, offset,
387 result_type >= PIPE_QUERY_TYPE_I64 ? 2 : 1,
388 ready);
389 return;
390 }
391
392 /* If the fence guarding this query has not been emitted, that makes a lot
393 * of the following logic more complicated.
394 */
395 if (hq->is64bit && hq->fence->state < NOUVEAU_FENCE_STATE_EMITTED)
396 nouveau_fence_emit(hq->fence);
397
398 /* We either need to compute a 32- or 64-bit difference between 2 values,
399 * and then store the result as either a 32- or 64-bit value. As such let's
400 * treat all inputs as 64-bit (and just push an extra 0 for the 32-bit
401 * ones), and have one macro that clamps result to i32, u32, or just
402 * outputs the difference (no need to worry about 64-bit clamping).
403 */
404 if (hq->state != NVC0_HW_QUERY_STATE_READY)
405 nvc0_hw_query_update(nvc0->screen->base.client, q);
406
407 if (wait && hq->state != NVC0_HW_QUERY_STATE_READY)
408 nvc0_hw_query_fifo_wait(nvc0, q);
409
410 nouveau_pushbuf_space(push, 32, 2, 0);
411 PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
412 PUSH_REFN (push, buf->bo, buf->domain | NOUVEAU_BO_WR);
413 BEGIN_1IC0(push, NVC0_3D(MACRO_QUERY_BUFFER_WRITE), 9);
414 if (q->type == PIPE_QUERY_OCCLUSION_PREDICATE ||
415 q->type == PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE) /* XXX what if 64-bit? */
416 PUSH_DATA(push, 0x00000001);
417 else if (result_type == PIPE_QUERY_TYPE_I32)
418 PUSH_DATA(push, 0x7fffffff);
419 else if (result_type == PIPE_QUERY_TYPE_U32)
420 PUSH_DATA(push, 0xffffffff);
421 else
422 PUSH_DATA(push, 0x00000000);
423
424 switch (q->type) {
425 case PIPE_QUERY_SO_STATISTICS:
426 stride = 2;
427 break;
428 case PIPE_QUERY_PIPELINE_STATISTICS:
429 stride = 12;
430 break;
431 case PIPE_QUERY_TIME_ELAPSED:
432 case PIPE_QUERY_TIMESTAMP:
433 qoffset = 8;
434 /* fallthrough */
435 default:
436 assert(index == 0);
437 stride = 1;
438 break;
439 }
440
441 if (hq->is64bit || qoffset) {
442 nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset + 16 * index,
443 8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
444 if (q->type == PIPE_QUERY_TIMESTAMP) {
445 PUSH_DATA(push, 0);
446 PUSH_DATA(push, 0);
447 } else {
448 nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset +
449 16 * (index + stride),
450 8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
451 }
452 } else {
453 nouveau_pushbuf_data(push, hq->bo, hq->offset + 4,
454 4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
455 PUSH_DATA(push, 0);
456 nouveau_pushbuf_data(push, hq->bo, hq->offset + 16 + 4,
457 4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
458 PUSH_DATA(push, 0);
459 }
460
461 if (wait || hq->state == NVC0_HW_QUERY_STATE_READY) {
462 PUSH_DATA(push, 0);
463 PUSH_DATA(push, 0);
464 } else if (hq->is64bit) {
465 PUSH_DATA(push, hq->fence->sequence);
466 nouveau_pushbuf_data(push, nvc0->screen->fence.bo, 0,
467 4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
468 } else {
469 PUSH_DATA(push, hq->sequence);
470 nouveau_pushbuf_data(push, hq->bo, hq->offset,
471 4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
472 }
473 PUSH_DATAh(push, buf->address + offset);
474 PUSH_DATA (push, buf->address + offset);
475
476 if (buf->mm) {
477 nouveau_fence_ref(nvc0->screen->base.fence.current, &buf->fence);
478 nouveau_fence_ref(nvc0->screen->base.fence.current, &buf->fence_wr);
479 }
480 }
481
482 static const struct nvc0_query_funcs hw_query_funcs = {
483 .destroy_query = nvc0_hw_destroy_query,
484 .begin_query = nvc0_hw_begin_query,
485 .end_query = nvc0_hw_end_query,
486 .get_query_result = nvc0_hw_get_query_result,
487 .get_query_result_resource = nvc0_hw_get_query_result_resource,
488 };
489
490 struct nvc0_query *
nvc0_hw_create_query(struct nvc0_context * nvc0,unsigned type,unsigned index)491 nvc0_hw_create_query(struct nvc0_context *nvc0, unsigned type, unsigned index)
492 {
493 struct nvc0_hw_query *hq;
494 struct nvc0_query *q;
495 unsigned space = NVC0_HW_QUERY_ALLOC_SPACE;
496
497 hq = nvc0_hw_sm_create_query(nvc0, type);
498 if (hq) {
499 hq->base.funcs = &hw_query_funcs;
500 return (struct nvc0_query *)hq;
501 }
502
503 hq = nvc0_hw_metric_create_query(nvc0, type);
504 if (hq) {
505 hq->base.funcs = &hw_query_funcs;
506 return (struct nvc0_query *)hq;
507 }
508
509 hq = CALLOC_STRUCT(nvc0_hw_query);
510 if (!hq)
511 return NULL;
512
513 q = &hq->base;
514 q->funcs = &hw_query_funcs;
515 q->type = type;
516
517 switch (q->type) {
518 case PIPE_QUERY_OCCLUSION_COUNTER:
519 case PIPE_QUERY_OCCLUSION_PREDICATE:
520 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
521 hq->rotate = 32;
522 space = NVC0_HW_QUERY_ALLOC_SPACE;
523 break;
524 case PIPE_QUERY_PIPELINE_STATISTICS:
525 hq->is64bit = true;
526 space = 512;
527 break;
528 case PIPE_QUERY_SO_STATISTICS:
529 case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
530 hq->is64bit = true;
531 space = 64;
532 break;
533 case PIPE_QUERY_PRIMITIVES_GENERATED:
534 case PIPE_QUERY_PRIMITIVES_EMITTED:
535 hq->is64bit = true;
536 q->index = index;
537 space = 32;
538 break;
539 case PIPE_QUERY_TIME_ELAPSED:
540 case PIPE_QUERY_TIMESTAMP:
541 case PIPE_QUERY_TIMESTAMP_DISJOINT:
542 case PIPE_QUERY_GPU_FINISHED:
543 space = 32;
544 break;
545 case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
546 space = 16;
547 break;
548 default:
549 debug_printf("invalid query type: %u\n", type);
550 FREE(q);
551 return NULL;
552 }
553
554 if (!nvc0_hw_query_allocate(nvc0, q, space)) {
555 FREE(hq);
556 return NULL;
557 }
558
559 if (hq->rotate) {
560 /* we advance before query_begin ! */
561 hq->offset -= hq->rotate;
562 hq->data -= hq->rotate / sizeof(*hq->data);
563 } else
564 if (!hq->is64bit)
565 hq->data[0] = 0; /* initialize sequence */
566
567 return q;
568 }
569
570 int
nvc0_hw_get_driver_query_info(struct nvc0_screen * screen,unsigned id,struct pipe_driver_query_info * info)571 nvc0_hw_get_driver_query_info(struct nvc0_screen *screen, unsigned id,
572 struct pipe_driver_query_info *info)
573 {
574 int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
575
576 num_hw_sm_queries = nvc0_hw_sm_get_driver_query_info(screen, 0, NULL);
577 num_hw_metric_queries =
578 nvc0_hw_metric_get_driver_query_info(screen, 0, NULL);
579
580 if (!info)
581 return num_hw_sm_queries + num_hw_metric_queries;
582
583 if (id < num_hw_sm_queries)
584 return nvc0_hw_sm_get_driver_query_info(screen, id, info);
585
586 return nvc0_hw_metric_get_driver_query_info(screen,
587 id - num_hw_sm_queries, info);
588 }
589
590 void
nvc0_hw_query_pushbuf_submit(struct nouveau_pushbuf * push,struct nvc0_query * q,unsigned result_offset)591 nvc0_hw_query_pushbuf_submit(struct nouveau_pushbuf *push,
592 struct nvc0_query *q, unsigned result_offset)
593 {
594 struct nvc0_hw_query *hq = nvc0_hw_query(q);
595
596 PUSH_REFN(push, hq->bo, NOUVEAU_BO_RD | NOUVEAU_BO_GART);
597 nouveau_pushbuf_data(push, hq->bo, hq->offset + result_offset, 4 |
598 NVC0_IB_ENTRY_1_NO_PREFETCH);
599 }
600
601 void
nvc0_hw_query_fifo_wait(struct nvc0_context * nvc0,struct nvc0_query * q)602 nvc0_hw_query_fifo_wait(struct nvc0_context *nvc0, struct nvc0_query *q)
603 {
604 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
605 struct nvc0_hw_query *hq = nvc0_hw_query(q);
606 unsigned offset = hq->offset;
607
608 if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) offset += 0x20;
609
610 PUSH_SPACE(push, 5);
611 PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
612 BEGIN_NVC0(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
613 if (hq->is64bit) {
614 PUSH_DATAh(push, nvc0->screen->fence.bo->offset);
615 PUSH_DATA (push, nvc0->screen->fence.bo->offset);
616 PUSH_DATA (push, hq->fence->sequence);
617 } else {
618 PUSH_DATAh(push, hq->bo->offset + offset);
619 PUSH_DATA (push, hq->bo->offset + offset);
620 PUSH_DATA (push, hq->sequence);
621 }
622 PUSH_DATA (push, (1 << 12) |
623 NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
624 }
625