• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2011 Christoph Bumiller
3  * Copyright 2015 Samuel Pitoiset
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  */
23 
24 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
25 
26 #include "nvc0/nvc0_context.h"
27 #include "nvc0/nvc0_query_hw.h"
28 #include "nvc0/nvc0_query_hw_metric.h"
29 #include "nvc0/nvc0_query_hw_sm.h"
30 
31 #define NVC0_HW_QUERY_STATE_READY   0
32 #define NVC0_HW_QUERY_STATE_ACTIVE  1
33 #define NVC0_HW_QUERY_STATE_ENDED   2
34 #define NVC0_HW_QUERY_STATE_FLUSHED 3
35 
36 #define NVC0_HW_QUERY_ALLOC_SPACE 256
37 
38 bool
nvc0_hw_query_allocate(struct nvc0_context * nvc0,struct nvc0_query * q,int size)39 nvc0_hw_query_allocate(struct nvc0_context *nvc0, struct nvc0_query *q,
40                        int size)
41 {
42    struct nvc0_hw_query *hq = nvc0_hw_query(q);
43    struct nvc0_screen *screen = nvc0->screen;
44    int ret;
45 
46    if (hq->bo) {
47       nouveau_bo_ref(NULL, &hq->bo);
48       if (hq->mm) {
49          if (hq->state == NVC0_HW_QUERY_STATE_READY)
50             nouveau_mm_free(hq->mm);
51          else
52             nouveau_fence_work(screen->base.fence.current,
53                                nouveau_mm_free_work, hq->mm);
54       }
55    }
56    if (size) {
57       hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size, &hq->bo,
58                                    &hq->base_offset);
59       if (!hq->bo)
60          return false;
61       hq->offset = hq->base_offset;
62 
63       ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
64       if (ret) {
65          nvc0_hw_query_allocate(nvc0, q, 0);
66          return false;
67       }
68       hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
69    }
70    return true;
71 }
72 
73 static void
nvc0_hw_query_get(struct nouveau_pushbuf * push,struct nvc0_query * q,unsigned offset,uint32_t get)74 nvc0_hw_query_get(struct nouveau_pushbuf *push, struct nvc0_query *q,
75                   unsigned offset, uint32_t get)
76 {
77    struct nvc0_hw_query *hq = nvc0_hw_query(q);
78 
79    offset += hq->offset;
80 
81    PUSH_SPACE(push, 5);
82    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
83    BEGIN_NVC0(push, NVC0_3D(QUERY_ADDRESS_HIGH), 4);
84    PUSH_DATAh(push, hq->bo->offset + offset);
85    PUSH_DATA (push, hq->bo->offset + offset);
86    PUSH_DATA (push, hq->sequence);
87    PUSH_DATA (push, get);
88 }
89 
90 static void
nvc0_hw_query_rotate(struct nvc0_context * nvc0,struct nvc0_query * q)91 nvc0_hw_query_rotate(struct nvc0_context *nvc0, struct nvc0_query *q)
92 {
93    struct nvc0_hw_query *hq = nvc0_hw_query(q);
94 
95    hq->offset += hq->rotate;
96    hq->data += hq->rotate / sizeof(*hq->data);
97    if (hq->offset - hq->base_offset == NVC0_HW_QUERY_ALLOC_SPACE)
98       nvc0_hw_query_allocate(nvc0, q, NVC0_HW_QUERY_ALLOC_SPACE);
99 }
100 
101 static inline void
nvc0_hw_query_update(struct nouveau_client * cli,struct nvc0_query * q)102 nvc0_hw_query_update(struct nouveau_client *cli, struct nvc0_query *q)
103 {
104    struct nvc0_hw_query *hq = nvc0_hw_query(q);
105 
106    if (hq->is64bit) {
107       if (nouveau_fence_signalled(hq->fence))
108          hq->state = NVC0_HW_QUERY_STATE_READY;
109    } else {
110       if (hq->data[0] == hq->sequence)
111          hq->state = NVC0_HW_QUERY_STATE_READY;
112    }
113 }
114 
115 static void
nvc0_hw_destroy_query(struct nvc0_context * nvc0,struct nvc0_query * q)116 nvc0_hw_destroy_query(struct nvc0_context *nvc0, struct nvc0_query *q)
117 {
118    struct nvc0_hw_query *hq = nvc0_hw_query(q);
119 
120    if (hq->funcs && hq->funcs->destroy_query) {
121       hq->funcs->destroy_query(nvc0, hq);
122       return;
123    }
124 
125    nvc0_hw_query_allocate(nvc0, q, 0);
126    nouveau_fence_ref(NULL, &hq->fence);
127    FREE(hq);
128 }
129 
130 static boolean
nvc0_hw_begin_query(struct nvc0_context * nvc0,struct nvc0_query * q)131 nvc0_hw_begin_query(struct nvc0_context *nvc0, struct nvc0_query *q)
132 {
133    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
134    struct nvc0_hw_query *hq = nvc0_hw_query(q);
135    bool ret = true;
136 
137    if (hq->funcs && hq->funcs->begin_query)
138       return hq->funcs->begin_query(nvc0, hq);
139 
140    /* For occlusion queries we have to change the storage, because a previous
141     * query might set the initial render conition to false even *after* we re-
142     * initialized it to true.
143     */
144    if (hq->rotate) {
145       nvc0_hw_query_rotate(nvc0, q);
146 
147       /* XXX: can we do this with the GPU, and sync with respect to a previous
148        *  query ?
149        */
150       hq->data[0] = hq->sequence; /* initialize sequence */
151       hq->data[1] = 1; /* initial render condition = true */
152       hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
153       hq->data[5] = 0;
154    }
155    hq->sequence++;
156 
157    switch (q->type) {
158    case PIPE_QUERY_OCCLUSION_COUNTER:
159    case PIPE_QUERY_OCCLUSION_PREDICATE:
160       hq->nesting = nvc0->screen->num_occlusion_queries_active++;
161       if (hq->nesting) {
162          nvc0_hw_query_get(push, q, 0x10, 0x0100f002);
163       } else {
164          PUSH_SPACE(push, 3);
165          BEGIN_NVC0(push, NVC0_3D(COUNTER_RESET), 1);
166          PUSH_DATA (push, NVC0_3D_COUNTER_RESET_SAMPLECNT);
167          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 1);
168       }
169       break;
170    case PIPE_QUERY_PRIMITIVES_GENERATED:
171       nvc0_hw_query_get(push, q, 0x10, 0x09005002 | (q->index << 5));
172       break;
173    case PIPE_QUERY_PRIMITIVES_EMITTED:
174       nvc0_hw_query_get(push, q, 0x10, 0x05805002 | (q->index << 5));
175       break;
176    case PIPE_QUERY_SO_STATISTICS:
177       nvc0_hw_query_get(push, q, 0x20, 0x05805002 | (q->index << 5));
178       nvc0_hw_query_get(push, q, 0x30, 0x06805002 | (q->index << 5));
179       break;
180    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
181       nvc0_hw_query_get(push, q, 0x10, 0x03005002 | (q->index << 5));
182       break;
183    case PIPE_QUERY_TIME_ELAPSED:
184       nvc0_hw_query_get(push, q, 0x10, 0x00005002);
185       break;
186    case PIPE_QUERY_PIPELINE_STATISTICS:
187       nvc0_hw_query_get(push, q, 0xc0 + 0x00, 0x00801002); /* VFETCH, VERTICES */
188       nvc0_hw_query_get(push, q, 0xc0 + 0x10, 0x01801002); /* VFETCH, PRIMS */
189       nvc0_hw_query_get(push, q, 0xc0 + 0x20, 0x02802002); /* VP, LAUNCHES */
190       nvc0_hw_query_get(push, q, 0xc0 + 0x30, 0x03806002); /* GP, LAUNCHES */
191       nvc0_hw_query_get(push, q, 0xc0 + 0x40, 0x04806002); /* GP, PRIMS_OUT */
192       nvc0_hw_query_get(push, q, 0xc0 + 0x50, 0x07804002); /* RAST, PRIMS_IN */
193       nvc0_hw_query_get(push, q, 0xc0 + 0x60, 0x08804002); /* RAST, PRIMS_OUT */
194       nvc0_hw_query_get(push, q, 0xc0 + 0x70, 0x0980a002); /* ROP, PIXELS */
195       nvc0_hw_query_get(push, q, 0xc0 + 0x80, 0x0d808002); /* TCP, LAUNCHES */
196       nvc0_hw_query_get(push, q, 0xc0 + 0x90, 0x0e809002); /* TEP, LAUNCHES */
197       break;
198    default:
199       break;
200    }
201    hq->state = NVC0_HW_QUERY_STATE_ACTIVE;
202    return ret;
203 }
204 
205 static void
nvc0_hw_end_query(struct nvc0_context * nvc0,struct nvc0_query * q)206 nvc0_hw_end_query(struct nvc0_context *nvc0, struct nvc0_query *q)
207 {
208    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
209    struct nvc0_hw_query *hq = nvc0_hw_query(q);
210 
211    if (hq->funcs && hq->funcs->end_query) {
212       hq->funcs->end_query(nvc0, hq);
213       return;
214    }
215 
216    if (hq->state != NVC0_HW_QUERY_STATE_ACTIVE) {
217       /* some queries don't require 'begin' to be called (e.g. GPU_FINISHED) */
218       if (hq->rotate)
219          nvc0_hw_query_rotate(nvc0, q);
220       hq->sequence++;
221    }
222    hq->state = NVC0_HW_QUERY_STATE_ENDED;
223 
224    switch (q->type) {
225    case PIPE_QUERY_OCCLUSION_COUNTER:
226    case PIPE_QUERY_OCCLUSION_PREDICATE:
227       nvc0_hw_query_get(push, q, 0, 0x0100f002);
228       if (--nvc0->screen->num_occlusion_queries_active == 0) {
229          PUSH_SPACE(push, 1);
230          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 0);
231       }
232       break;
233    case PIPE_QUERY_PRIMITIVES_GENERATED:
234       nvc0_hw_query_get(push, q, 0, 0x09005002 | (q->index << 5));
235       break;
236    case PIPE_QUERY_PRIMITIVES_EMITTED:
237       nvc0_hw_query_get(push, q, 0, 0x05805002 | (q->index << 5));
238       break;
239    case PIPE_QUERY_SO_STATISTICS:
240       nvc0_hw_query_get(push, q, 0x00, 0x05805002 | (q->index << 5));
241       nvc0_hw_query_get(push, q, 0x10, 0x06805002 | (q->index << 5));
242       break;
243    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
244       /* TODO: How do we sum over all streams for render condition ? */
245       /* PRIMS_DROPPED doesn't write sequence, use a ZERO query to sync on */
246       nvc0_hw_query_get(push, q, 0x00, 0x03005002 | (q->index << 5));
247       nvc0_hw_query_get(push, q, 0x20, 0x00005002);
248       break;
249    case PIPE_QUERY_TIMESTAMP:
250    case PIPE_QUERY_TIME_ELAPSED:
251       nvc0_hw_query_get(push, q, 0, 0x00005002);
252       break;
253    case PIPE_QUERY_GPU_FINISHED:
254       nvc0_hw_query_get(push, q, 0, 0x1000f010);
255       break;
256    case PIPE_QUERY_PIPELINE_STATISTICS:
257       nvc0_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
258       nvc0_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
259       nvc0_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
260       nvc0_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
261       nvc0_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
262       nvc0_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
263       nvc0_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
264       nvc0_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
265       nvc0_hw_query_get(push, q, 0x80, 0x0d808002); /* TCP, LAUNCHES */
266       nvc0_hw_query_get(push, q, 0x90, 0x0e809002); /* TEP, LAUNCHES */
267       break;
268    case PIPE_QUERY_TIMESTAMP_DISJOINT:
269       /* This query is not issued on GPU because disjoint is forced to false */
270       hq->state = NVC0_HW_QUERY_STATE_READY;
271       break;
272    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
273       /* indexed by TFB buffer instead of by vertex stream */
274       nvc0_hw_query_get(push, q, 0x00, 0x0d005002 | (q->index << 5));
275       break;
276    default:
277       break;
278    }
279    if (hq->is64bit)
280       nouveau_fence_ref(nvc0->screen->base.fence.current, &hq->fence);
281 }
282 
283 static boolean
nvc0_hw_get_query_result(struct nvc0_context * nvc0,struct nvc0_query * q,boolean wait,union pipe_query_result * result)284 nvc0_hw_get_query_result(struct nvc0_context *nvc0, struct nvc0_query *q,
285                          boolean wait, union pipe_query_result *result)
286 {
287    struct nvc0_hw_query *hq = nvc0_hw_query(q);
288    uint64_t *res64 = (uint64_t*)result;
289    uint32_t *res32 = (uint32_t*)result;
290    uint8_t *res8 = (uint8_t*)result;
291    uint64_t *data64 = (uint64_t *)hq->data;
292    unsigned i;
293 
294    if (hq->funcs && hq->funcs->get_query_result)
295       return hq->funcs->get_query_result(nvc0, hq, wait, result);
296 
297    if (hq->state != NVC0_HW_QUERY_STATE_READY)
298       nvc0_hw_query_update(nvc0->screen->base.client, q);
299 
300    if (hq->state != NVC0_HW_QUERY_STATE_READY) {
301       if (!wait) {
302          if (hq->state != NVC0_HW_QUERY_STATE_FLUSHED) {
303             hq->state = NVC0_HW_QUERY_STATE_FLUSHED;
304             /* flush for silly apps that spin on GL_QUERY_RESULT_AVAILABLE */
305             PUSH_KICK(nvc0->base.pushbuf);
306          }
307          return false;
308       }
309       if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nvc0->screen->base.client))
310          return false;
311       NOUVEAU_DRV_STAT(&nvc0->screen->base, query_sync_count, 1);
312    }
313    hq->state = NVC0_HW_QUERY_STATE_READY;
314 
315    switch (q->type) {
316    case PIPE_QUERY_GPU_FINISHED:
317       res8[0] = true;
318       break;
319    case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
320       res64[0] = hq->data[1] - hq->data[5];
321       break;
322    case PIPE_QUERY_OCCLUSION_PREDICATE:
323       res8[0] = hq->data[1] != hq->data[5];
324       break;
325    case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
326    case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
327       res64[0] = data64[0] - data64[2];
328       break;
329    case PIPE_QUERY_SO_STATISTICS:
330       res64[0] = data64[0] - data64[4];
331       res64[1] = data64[2] - data64[6];
332       break;
333    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
334       res8[0] = data64[0] != data64[2];
335       break;
336    case PIPE_QUERY_TIMESTAMP:
337       res64[0] = data64[1];
338       break;
339    case PIPE_QUERY_TIMESTAMP_DISJOINT:
340       res64[0] = 1000000000;
341       res8[8] = false;
342       break;
343    case PIPE_QUERY_TIME_ELAPSED:
344       res64[0] = data64[1] - data64[3];
345       break;
346    case PIPE_QUERY_PIPELINE_STATISTICS:
347       for (i = 0; i < 10; ++i)
348          res64[i] = data64[i * 2] - data64[24 + i * 2];
349       break;
350    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
351       res32[0] = hq->data[1];
352       break;
353    default:
354       assert(0); /* can't happen, we don't create queries with invalid type */
355       return false;
356    }
357 
358    return true;
359 }
360 
361 static void
nvc0_hw_get_query_result_resource(struct nvc0_context * nvc0,struct nvc0_query * q,boolean wait,enum pipe_query_value_type result_type,int index,struct pipe_resource * resource,unsigned offset)362 nvc0_hw_get_query_result_resource(struct nvc0_context *nvc0,
363                                   struct nvc0_query *q,
364                                   boolean wait,
365                                   enum pipe_query_value_type result_type,
366                                   int index,
367                                   struct pipe_resource *resource,
368                                   unsigned offset)
369 {
370    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
371    struct nvc0_hw_query *hq = nvc0_hw_query(q);
372    struct nv04_resource *buf = nv04_resource(resource);
373    unsigned qoffset = 0, stride;
374 
375    assert(!hq->funcs || !hq->funcs->get_query_result);
376 
377    if (index == -1) {
378       /* TODO: Use a macro to write the availability of the query */
379       if (hq->state != NVC0_HW_QUERY_STATE_READY)
380          nvc0_hw_query_update(nvc0->screen->base.client, q);
381       uint32_t ready[2] = {hq->state == NVC0_HW_QUERY_STATE_READY};
382       nvc0->base.push_cb(&nvc0->base, buf, offset,
383                          result_type >= PIPE_QUERY_TYPE_I64 ? 2 : 1,
384                          ready);
385       return;
386    }
387 
388    /* If the fence guarding this query has not been emitted, that makes a lot
389     * of the following logic more complicated.
390     */
391    if (hq->is64bit && hq->fence->state < NOUVEAU_FENCE_STATE_EMITTED)
392       nouveau_fence_emit(hq->fence);
393 
394    /* We either need to compute a 32- or 64-bit difference between 2 values,
395     * and then store the result as either a 32- or 64-bit value. As such let's
396     * treat all inputs as 64-bit (and just push an extra 0 for the 32-bit
397     * ones), and have one macro that clamps result to i32, u32, or just
398     * outputs the difference (no need to worry about 64-bit clamping).
399     */
400    if (hq->state != NVC0_HW_QUERY_STATE_READY)
401       nvc0_hw_query_update(nvc0->screen->base.client, q);
402 
403    if (wait && hq->state != NVC0_HW_QUERY_STATE_READY)
404       nvc0_hw_query_fifo_wait(nvc0, q);
405 
406    nouveau_pushbuf_space(push, 32, 2, 0);
407    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
408    PUSH_REFN (push, buf->bo, buf->domain | NOUVEAU_BO_WR);
409    BEGIN_NVC0(push, NVC0_3D(QUERY_ADDRESS_HIGH), 2);
410    PUSH_DATAh(push, buf->address + offset);
411    PUSH_DATA (push, buf->address + offset);
412    BEGIN_1IC0(push, NVC0_3D(MACRO_QUERY_BUFFER_WRITE), 7);
413    if (q->type == PIPE_QUERY_OCCLUSION_PREDICATE) /* XXX what if 64-bit? */
414       PUSH_DATA(push, 0x00000001);
415    else if (result_type == PIPE_QUERY_TYPE_I32)
416       PUSH_DATA(push, 0x7fffffff);
417    else if (result_type == PIPE_QUERY_TYPE_U32)
418       PUSH_DATA(push, 0xffffffff);
419    else
420       PUSH_DATA(push, 0x00000000);
421 
422    switch (q->type) {
423    case PIPE_QUERY_SO_STATISTICS:
424       stride = 2;
425       break;
426    case PIPE_QUERY_PIPELINE_STATISTICS:
427       stride = 12;
428       break;
429    case PIPE_QUERY_TIME_ELAPSED:
430    case PIPE_QUERY_TIMESTAMP:
431       qoffset = 8;
432       /* fallthrough */
433    default:
434       assert(index == 0);
435       stride = 1;
436       break;
437    }
438 
439    if (hq->is64bit || qoffset) {
440       nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset + 16 * index,
441                            8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
442       if (q->type == PIPE_QUERY_TIMESTAMP) {
443          PUSH_DATA(push, 0);
444          PUSH_DATA(push, 0);
445       } else {
446          nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset +
447                               16 * (index + stride),
448                               8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
449       }
450    } else {
451       nouveau_pushbuf_data(push, hq->bo, hq->offset + 4,
452                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
453       PUSH_DATA(push, 0);
454       nouveau_pushbuf_data(push, hq->bo, hq->offset + 16 + 4,
455                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
456       PUSH_DATA(push, 0);
457    }
458 
459    if (wait || hq->state == NVC0_HW_QUERY_STATE_READY) {
460       PUSH_DATA(push, 0);
461       PUSH_DATA(push, 0);
462    } else if (hq->is64bit) {
463       PUSH_DATA(push, hq->fence->sequence);
464       nouveau_pushbuf_data(push, nvc0->screen->fence.bo, 0,
465                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
466    } else {
467       PUSH_DATA(push, hq->sequence);
468       nouveau_pushbuf_data(push, hq->bo, hq->offset,
469                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
470    }
471 
472    if (buf->mm) {
473       nouveau_fence_ref(nvc0->screen->base.fence.current, &buf->fence);
474       nouveau_fence_ref(nvc0->screen->base.fence.current, &buf->fence_wr);
475    }
476 }
477 
478 static const struct nvc0_query_funcs hw_query_funcs = {
479    .destroy_query = nvc0_hw_destroy_query,
480    .begin_query = nvc0_hw_begin_query,
481    .end_query = nvc0_hw_end_query,
482    .get_query_result = nvc0_hw_get_query_result,
483    .get_query_result_resource = nvc0_hw_get_query_result_resource,
484 };
485 
486 struct nvc0_query *
nvc0_hw_create_query(struct nvc0_context * nvc0,unsigned type,unsigned index)487 nvc0_hw_create_query(struct nvc0_context *nvc0, unsigned type, unsigned index)
488 {
489    struct nvc0_hw_query *hq;
490    struct nvc0_query *q;
491    unsigned space = NVC0_HW_QUERY_ALLOC_SPACE;
492 
493    hq = nvc0_hw_sm_create_query(nvc0, type);
494    if (hq) {
495       hq->base.funcs = &hw_query_funcs;
496       return (struct nvc0_query *)hq;
497    }
498 
499    hq = nvc0_hw_metric_create_query(nvc0, type);
500    if (hq) {
501       hq->base.funcs = &hw_query_funcs;
502       return (struct nvc0_query *)hq;
503    }
504 
505    hq = CALLOC_STRUCT(nvc0_hw_query);
506    if (!hq)
507       return NULL;
508 
509    q = &hq->base;
510    q->funcs = &hw_query_funcs;
511    q->type = type;
512 
513    switch (q->type) {
514    case PIPE_QUERY_OCCLUSION_COUNTER:
515    case PIPE_QUERY_OCCLUSION_PREDICATE:
516       hq->rotate = 32;
517       space = NVC0_HW_QUERY_ALLOC_SPACE;
518       break;
519    case PIPE_QUERY_PIPELINE_STATISTICS:
520       hq->is64bit = true;
521       space = 512;
522       break;
523    case PIPE_QUERY_SO_STATISTICS:
524    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
525       hq->is64bit = true;
526       space = 64;
527       break;
528    case PIPE_QUERY_PRIMITIVES_GENERATED:
529    case PIPE_QUERY_PRIMITIVES_EMITTED:
530       hq->is64bit = true;
531       q->index = index;
532       space = 32;
533       break;
534    case PIPE_QUERY_TIME_ELAPSED:
535    case PIPE_QUERY_TIMESTAMP:
536    case PIPE_QUERY_TIMESTAMP_DISJOINT:
537    case PIPE_QUERY_GPU_FINISHED:
538       space = 32;
539       break;
540    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
541       space = 16;
542       break;
543    default:
544       debug_printf("invalid query type: %u\n", type);
545       FREE(q);
546       return NULL;
547    }
548 
549    if (!nvc0_hw_query_allocate(nvc0, q, space)) {
550       FREE(hq);
551       return NULL;
552    }
553 
554    if (hq->rotate) {
555       /* we advance before query_begin ! */
556       hq->offset -= hq->rotate;
557       hq->data -= hq->rotate / sizeof(*hq->data);
558    } else
559    if (!hq->is64bit)
560       hq->data[0] = 0; /* initialize sequence */
561 
562    return q;
563 }
564 
565 int
nvc0_hw_get_driver_query_info(struct nvc0_screen * screen,unsigned id,struct pipe_driver_query_info * info)566 nvc0_hw_get_driver_query_info(struct nvc0_screen *screen, unsigned id,
567                               struct pipe_driver_query_info *info)
568 {
569    int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
570 
571    num_hw_sm_queries = nvc0_hw_sm_get_driver_query_info(screen, 0, NULL);
572    num_hw_metric_queries =
573       nvc0_hw_metric_get_driver_query_info(screen, 0, NULL);
574 
575    if (!info)
576       return num_hw_sm_queries + num_hw_metric_queries;
577 
578    if (id < num_hw_sm_queries)
579       return nvc0_hw_sm_get_driver_query_info(screen, id, info);
580 
581    return nvc0_hw_metric_get_driver_query_info(screen,
582                                                id - num_hw_sm_queries, info);
583 }
584 
585 void
nvc0_hw_query_pushbuf_submit(struct nouveau_pushbuf * push,struct nvc0_query * q,unsigned result_offset)586 nvc0_hw_query_pushbuf_submit(struct nouveau_pushbuf *push,
587                              struct nvc0_query *q, unsigned result_offset)
588 {
589    struct nvc0_hw_query *hq = nvc0_hw_query(q);
590 
591    PUSH_REFN(push, hq->bo, NOUVEAU_BO_RD | NOUVEAU_BO_GART);
592    nouveau_pushbuf_data(push, hq->bo, hq->offset + result_offset, 4 |
593                         NVC0_IB_ENTRY_1_NO_PREFETCH);
594 }
595 
596 void
nvc0_hw_query_fifo_wait(struct nvc0_context * nvc0,struct nvc0_query * q)597 nvc0_hw_query_fifo_wait(struct nvc0_context *nvc0, struct nvc0_query *q)
598 {
599    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
600    struct nvc0_hw_query *hq = nvc0_hw_query(q);
601    unsigned offset = hq->offset;
602 
603    if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) offset += 0x20;
604 
605    PUSH_SPACE(push, 5);
606    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
607    BEGIN_NVC0(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
608    if (hq->is64bit) {
609       PUSH_DATAh(push, nvc0->screen->fence.bo->offset);
610       PUSH_DATA (push, nvc0->screen->fence.bo->offset);
611       PUSH_DATA (push, hq->fence->sequence);
612    } else {
613       PUSH_DATAh(push, hq->bo->offset + offset);
614       PUSH_DATA (push, hq->bo->offset + offset);
615       PUSH_DATA (push, hq->sequence);
616    }
617    PUSH_DATA (push, (1 << 12) |
618               NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
619 }
620