src/gallium/drivers/nouveau/nvc0/nvc0_query_hw.c

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  * Copyright 2015 Samuel Pitoiset
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
  25
  26 #include "nvc0/nvc0_context.h"
  27 #include "nvc0/nvc0_query_hw.h"
  28 #include "nvc0/nvc0_query_hw_metric.h"
  29 #include "nvc0/nvc0_query_hw_sm.h"
  30
  31 #define NVC0_HW_QUERY_STATE_READY   0
  32 #define NVC0_HW_QUERY_STATE_ACTIVE  1
  33 #define NVC0_HW_QUERY_STATE_ENDED   2
  34 #define NVC0_HW_QUERY_STATE_FLUSHED 3
  35
  36 #define NVC0_HW_QUERY_ALLOC_SPACE 256
  37
  38 bool
  39 nvc0_hw_query_allocate(struct nvc0_context *nvc0, struct nvc0_query *q,
  40                        int size)
  41 {
  42    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  43    struct nvc0_screen *screen = nvc0->screen;
  44    int ret;
  45
  46    if (hq->bo) {
  47       nouveau_bo_ref(NULL, &hq->bo);
  48       if (hq->mm) {
  49          if (hq->state == NVC0_HW_QUERY_STATE_READY)
  50             nouveau_mm_free(hq->mm);
  51          else
  52             nouveau_fence_work(screen->base.fence.current,
  53                                nouveau_mm_free_work, hq->mm);
  54       }
  55    }
  56    if (size) {
  57       hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size, &hq->bo,
  58                                    &hq->base_offset);
  59       if (!hq->bo)
  60          return false;
  61       hq->offset = hq->base_offset;
  62
  63       ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
  64       if (ret) {
  65          nvc0_hw_query_allocate(nvc0, q, 0);
  66          return false;
  67       }
  68       hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
  69    }
  70    return true;
  71 }
  72
  73 static void
  74 nvc0_hw_query_get(struct nouveau_pushbuf *push, struct nvc0_query *q,
  75                   unsigned offset, uint32_t get)
  76 {
  77    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  78
  79    offset += hq->offset;
  80
  81    PUSH_SPACE(push, 5);
  82    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
  83    BEGIN_NVC0(push, NVC0_3D(QUERY_ADDRESS_HIGH), 4);
  84    PUSH_DATAh(push, hq->bo->offset + offset);
  85    PUSH_DATA (push, hq->bo->offset + offset);
  86    PUSH_DATA (push, hq->sequence);
  87    PUSH_DATA (push, get);
  88 }
  89
  90 static void
  91 nvc0_hw_query_rotate(struct nvc0_context *nvc0, struct nvc0_query *q)
  92 {
  93    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  94
  95    hq->offset += hq->rotate;
  96    hq->data += hq->rotate / sizeof(*hq->data);
  97    if (hq->offset - hq->base_offset == NVC0_HW_QUERY_ALLOC_SPACE)
  98       nvc0_hw_query_allocate(nvc0, q, NVC0_HW_QUERY_ALLOC_SPACE);
  99 }
 100
 101 static inline void
 102 nvc0_hw_query_update(struct nouveau_client *cli, struct nvc0_query *q)
 103 {
 104    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 105
 106    if (hq->is64bit) {
 107       if (nouveau_fence_signalled(hq->fence))
 108          hq->state = NVC0_HW_QUERY_STATE_READY;
 109    } else {
 110       if (hq->data[0] == hq->sequence)
 111          hq->state = NVC0_HW_QUERY_STATE_READY;
 112    }
 113 }
 114
 115 static void
 116 nvc0_hw_destroy_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 117 {
 118    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 119
 120    if (hq->funcs && hq->funcs->destroy_query) {
 121       hq->funcs->destroy_query(nvc0, hq);
 122       return;
 123    }
 124
 125    nvc0_hw_query_allocate(nvc0, q, 0);
 126    nouveau_fence_ref(NULL, &hq->fence);
 127    FREE(hq);
 128 }
 129
 130 static boolean
 131 nvc0_hw_begin_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 132 {
 133    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 134    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 135    bool ret = true;
 136
 137    if (hq->funcs && hq->funcs->begin_query)
 138       return hq->funcs->begin_query(nvc0, hq);
 139
 140    /* For occlusion queries we have to change the storage, because a previous
 141     * query might set the initial render conition to false even *after* we re-
 142     * initialized it to true.
 143     */
 144    if (hq->rotate) {
 145       nvc0_hw_query_rotate(nvc0, q);
 146
 147       /* XXX: can we do this with the GPU, and sync with respect to a previous
 148        *  query ?
 149        */
 150       hq->data[0] = hq->sequence; /* initialize sequence */
 151       hq->data[1] = 1; /* initial render condition = true */
 152       hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
 153       hq->data[5] = 0;
 154    }
 155    hq->sequence++;
 156
 157    switch (q->type) {
 158    case PIPE_QUERY_OCCLUSION_COUNTER:
 159    case PIPE_QUERY_OCCLUSION_PREDICATE:
 160       hq->nesting = nvc0->screen->num_occlusion_queries_active++;
 161       if (hq->nesting) {
 162          nvc0_hw_query_get(push, q, 0x10, 0x0100f002);
 163       } else {
 164          PUSH_SPACE(push, 3);
 165          BEGIN_NVC0(push, NVC0_3D(COUNTER_RESET), 1);
 166          PUSH_DATA (push, NVC0_3D_COUNTER_RESET_SAMPLECNT);
 167          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 1);
 168       }
 169       break;
 170    case PIPE_QUERY_PRIMITIVES_GENERATED:
 171       nvc0_hw_query_get(push, q, 0x10, 0x09005002 | (q->index << 5));
 172       break;
 173    case PIPE_QUERY_PRIMITIVES_EMITTED:
 174       nvc0_hw_query_get(push, q, 0x10, 0x05805002 | (q->index << 5));
 175       break;
 176    case PIPE_QUERY_SO_STATISTICS:
 177       nvc0_hw_query_get(push, q, 0x20, 0x05805002 | (q->index << 5));
 178       nvc0_hw_query_get(push, q, 0x30, 0x06805002 | (q->index << 5));
 179       break;
 180    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 181       nvc0_hw_query_get(push, q, 0x10, 0x03005002 | (q->index << 5));
 182       break;
 183    case PIPE_QUERY_TIME_ELAPSED:
 184       nvc0_hw_query_get(push, q, 0x10, 0x00005002);
 185       break;
 186    case PIPE_QUERY_PIPELINE_STATISTICS:
 187       nvc0_hw_query_get(push, q, 0xc0 + 0x00, 0x00801002); /* VFETCH, VERTICES */
 188       nvc0_hw_query_get(push, q, 0xc0 + 0x10, 0x01801002); /* VFETCH, PRIMS */
 189       nvc0_hw_query_get(push, q, 0xc0 + 0x20, 0x02802002); /* VP, LAUNCHES */
 190       nvc0_hw_query_get(push, q, 0xc0 + 0x30, 0x03806002); /* GP, LAUNCHES */
 191       nvc0_hw_query_get(push, q, 0xc0 + 0x40, 0x04806002); /* GP, PRIMS_OUT */
 192       nvc0_hw_query_get(push, q, 0xc0 + 0x50, 0x07804002); /* RAST, PRIMS_IN */
 193       nvc0_hw_query_get(push, q, 0xc0 + 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 194       nvc0_hw_query_get(push, q, 0xc0 + 0x70, 0x0980a002); /* ROP, PIXELS */
 195       nvc0_hw_query_get(push, q, 0xc0 + 0x80, 0x0d808002); /* TCP, LAUNCHES */
 196       nvc0_hw_query_get(push, q, 0xc0 + 0x90, 0x0e809002); /* TEP, LAUNCHES */
 197       break;
 198    default:
 199       break;
 200    }
 201    hq->state = NVC0_HW_QUERY_STATE_ACTIVE;
 202    return ret;
 203 }
 204
 205 static void
 206 nvc0_hw_end_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 207 {
 208    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 209    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 210
 211    if (hq->funcs && hq->funcs->end_query) {
 212       hq->funcs->end_query(nvc0, hq);
 213       return;
 214    }
 215
 216    if (hq->state != NVC0_HW_QUERY_STATE_ACTIVE) {
 217       /* some queries don't require 'begin' to be called (e.g. GPU_FINISHED) */
 218       if (hq->rotate)
 219          nvc0_hw_query_rotate(nvc0, q);
 220       hq->sequence++;
 221    }
 222    hq->state = NVC0_HW_QUERY_STATE_ENDED;
 223
 224    switch (q->type) {
 225    case PIPE_QUERY_OCCLUSION_COUNTER:
 226    case PIPE_QUERY_OCCLUSION_PREDICATE:
 227       nvc0_hw_query_get(push, q, 0, 0x0100f002);
 228       if (--nvc0->screen->num_occlusion_queries_active == 0) {
 229          PUSH_SPACE(push, 1);
 230          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 0);
 231       }
 232       break;
 233    case PIPE_QUERY_PRIMITIVES_GENERATED:
 234       nvc0_hw_query_get(push, q, 0, 0x09005002 | (q->index << 5));
 235       break;
 236    case PIPE_QUERY_PRIMITIVES_EMITTED:
 237       nvc0_hw_query_get(push, q, 0, 0x05805002 | (q->index << 5));
 238       break;
 239    case PIPE_QUERY_SO_STATISTICS:
 240       nvc0_hw_query_get(push, q, 0x00, 0x05805002 | (q->index << 5));
 241       nvc0_hw_query_get(push, q, 0x10, 0x06805002 | (q->index << 5));
 242       break;
 243    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 244       /* TODO: How do we sum over all streams for render condition ? */
 245       /* PRIMS_DROPPED doesn't write sequence, use a ZERO query to sync on */
 246       nvc0_hw_query_get(push, q, 0x00, 0x03005002 | (q->index << 5));
 247       nvc0_hw_query_get(push, q, 0x20, 0x00005002);
 248       break;
 249    case PIPE_QUERY_TIMESTAMP:
 250    case PIPE_QUERY_TIME_ELAPSED:
 251       nvc0_hw_query_get(push, q, 0, 0x00005002);
 252       break;
 253    case PIPE_QUERY_GPU_FINISHED:
 254       nvc0_hw_query_get(push, q, 0, 0x1000f010);
 255       break;
 256    case PIPE_QUERY_PIPELINE_STATISTICS:
 257       nvc0_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
 258       nvc0_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
 259       nvc0_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
 260       nvc0_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
 261       nvc0_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
 262       nvc0_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
 263       nvc0_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 264       nvc0_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
 265       nvc0_hw_query_get(push, q, 0x80, 0x0d808002); /* TCP, LAUNCHES */
 266       nvc0_hw_query_get(push, q, 0x90, 0x0e809002); /* TEP, LAUNCHES */
 267       break;
 268    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 269       /* This query is not issued on GPU because disjoint is forced to false */
 270       hq->state = NVC0_HW_QUERY_STATE_READY;
 271       break;
 272    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 273       /* indexed by TFB buffer instead of by vertex stream */
 274       nvc0_hw_query_get(push, q, 0x00, 0x0d005002 | (q->index << 5));
 275       break;
 276    default:
 277       break;
 278    }
 279    if (hq->is64bit)
 280       nouveau_fence_ref(nvc0->screen->base.fence.current, &hq->fence);
 281 }
 282
 283 static boolean
 284 nvc0_hw_get_query_result(struct nvc0_context *nvc0, struct nvc0_query *q,
 285                          boolean wait, union pipe_query_result *result)
 286 {
 287    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 288    uint64_t *res64 = (uint64_t*)result;
 289    uint32_t *res32 = (uint32_t*)result;
 290    uint8_t *res8 = (uint8_t*)result;
 291    uint64_t *data64 = (uint64_t *)hq->data;
 292    unsigned i;
 293
 294    if (hq->funcs && hq->funcs->get_query_result)
 295       return hq->funcs->get_query_result(nvc0, hq, wait, result);
 296
 297    if (hq->state != NVC0_HW_QUERY_STATE_READY)
 298       nvc0_hw_query_update(nvc0->screen->base.client, q);
 299
 300    if (hq->state != NVC0_HW_QUERY_STATE_READY) {
 301       if (!wait) {
 302          if (hq->state != NVC0_HW_QUERY_STATE_FLUSHED) {
 303             hq->state = NVC0_HW_QUERY_STATE_FLUSHED;
 304             /* flush for silly apps that spin on GL_QUERY_RESULT_AVAILABLE */
 305             PUSH_KICK(nvc0->base.pushbuf);
 306          }
 307          return false;
 308       }
 309       if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nvc0->screen->base.client))
 310          return false;
 311       NOUVEAU_DRV_STAT(&nvc0->screen->base, query_sync_count, 1);
 312    }
 313    hq->state = NVC0_HW_QUERY_STATE_READY;
 314
 315    switch (q->type) {
 316    case PIPE_QUERY_GPU_FINISHED:
 317       res8[0] = true;
 318       break;
 319    case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
 320       res64[0] = hq->data[1] - hq->data[5];
 321       break;
 322    case PIPE_QUERY_OCCLUSION_PREDICATE:
 323       res8[0] = hq->data[1] != hq->data[5];
 324       break;
 325    case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
 326    case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
 327       res64[0] = data64[0] - data64[2];
 328       break;
 329    case PIPE_QUERY_SO_STATISTICS:
 330       res64[0] = data64[0] - data64[4];
 331       res64[1] = data64[2] - data64[6];
 332       break;
 333    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 334       res8[0] = data64[0] != data64[2];
 335       break;
 336    case PIPE_QUERY_TIMESTAMP:
 337       res64[0] = data64[1];
 338       break;
 339    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 340       res64[0] = 1000000000;
 341       res8[8] = false;
 342       break;
 343    case PIPE_QUERY_TIME_ELAPSED:
 344       res64[0] = data64[1] - data64[3];
 345       break;
 346    case PIPE_QUERY_PIPELINE_STATISTICS:
 347       for (i = 0; i < 10; ++i)
 348          res64[i] = data64[i * 2] - data64[24 + i * 2];
 349       break;
 350    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 351       res32[0] = hq->data[1];
 352       break;
 353    default:
 354       assert(0); /* can't happen, we don't create queries with invalid type */
 355       return false;
 356    }
 357
 358    return true;
 359 }
 360
 361 static void
 362 nvc0_hw_get_query_result_resource(struct nvc0_context *nvc0,
 363                                   struct nvc0_query *q,
 364                                   boolean wait,
 365                                   enum pipe_query_value_type result_type,
 366                                   int index,
 367                                   struct pipe_resource *resource,
 368                                   unsigned offset)
 369 {
 370    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 371    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 372    struct nv04_resource *buf = nv04_resource(resource);
 373    unsigned qoffset = 0, stride;
 374
 375    assert(!hq->funcs || !hq->funcs->get_query_result);
 376
 377    if (index == -1) {
 378       /* TODO: Use a macro to write the availability of the query */
 379       if (hq->state != NVC0_HW_QUERY_STATE_READY)
 380          nvc0_hw_query_update(nvc0->screen->base.client, q);
 381       uint32_t ready[2] = {hq->state == NVC0_HW_QUERY_STATE_READY};
 382       nvc0->base.push_cb(&nvc0->base, buf, offset,
 383                          result_type >= PIPE_QUERY_TYPE_I64 ? 2 : 1,
 384                          ready);
 385       return;
 386    }
 387
 388    /* If the fence guarding this query has not been emitted, that makes a lot
 389     * of the following logic more complicated.
 390     */
 391    if (hq->is64bit && hq->fence->state < NOUVEAU_FENCE_STATE_EMITTED)
 392       nouveau_fence_emit(hq->fence);
 393
 394    /* We either need to compute a 32- or 64-bit difference between 2 values,
 395     * and then store the result as either a 32- or 64-bit value. As such let's
 396     * treat all inputs as 64-bit (and just push an extra 0 for the 32-bit
 397     * ones), and have one macro that clamps result to i32, u32, or just
 398     * outputs the difference (no need to worry about 64-bit clamping).
 399     */
 400    if (hq->state != NVC0_HW_QUERY_STATE_READY)
 401       nvc0_hw_query_update(nvc0->screen->base.client, q);
 402
 403    if (wait && hq->state != NVC0_HW_QUERY_STATE_READY)
 404       nvc0_hw_query_fifo_wait(nvc0, q);
 405
 406    nouveau_pushbuf_space(push, 32, 2, 0);
 407    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 408    PUSH_REFN (push, buf->bo, buf->domain | NOUVEAU_BO_WR);
 409    BEGIN_NVC0(push, NVC0_3D(QUERY_ADDRESS_HIGH), 2);
 410    PUSH_DATAh(push, buf->address + offset);
 411    PUSH_DATA (push, buf->address + offset);
 412    BEGIN_1IC0(push, NVC0_3D(MACRO_QUERY_BUFFER_WRITE), 7);
 413    if (q->type == PIPE_QUERY_OCCLUSION_PREDICATE) /* XXX what if 64-bit? */
 414       PUSH_DATA(push, 0x00000001);
 415    else if (result_type == PIPE_QUERY_TYPE_I32)
 416       PUSH_DATA(push, 0x7fffffff);
 417    else if (result_type == PIPE_QUERY_TYPE_U32)
 418       PUSH_DATA(push, 0xffffffff);
 419    else
 420       PUSH_DATA(push, 0x00000000);
 421
 422    switch (q->type) {
 423    case PIPE_QUERY_SO_STATISTICS:
 424       stride = 2;
 425       break;
 426    case PIPE_QUERY_PIPELINE_STATISTICS:
 427       stride = 12;
 428       break;
 429    case PIPE_QUERY_TIME_ELAPSED:
 430    case PIPE_QUERY_TIMESTAMP:
 431       qoffset = 8;
 432       /* fallthrough */
 433    default:
 434       assert(index == 0);
 435       stride = 1;
 436       break;
 437    }
 438
 439    if (hq->is64bit || qoffset) {
 440       nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset + 16 * index,
 441                            8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 442       if (q->type == PIPE_QUERY_TIMESTAMP) {
 443          PUSH_DATA(push, 0);
 444          PUSH_DATA(push, 0);
 445       } else {
 446          nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset +
 447                               16 * (index + stride),
 448                               8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 449       }
 450    } else {
 451       nouveau_pushbuf_data(push, hq->bo, hq->offset + 4,
 452                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 453       PUSH_DATA(push, 0);
 454       nouveau_pushbuf_data(push, hq->bo, hq->offset + 16 + 4,
 455                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 456       PUSH_DATA(push, 0);
 457    }
 458
 459    if (wait || hq->state == NVC0_HW_QUERY_STATE_READY) {
 460       PUSH_DATA(push, 0);
 461       PUSH_DATA(push, 0);
 462    } else if (hq->is64bit) {
 463       PUSH_DATA(push, hq->fence->sequence);
 464       nouveau_pushbuf_data(push, nvc0->screen->fence.bo, 0,
 465                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 466    } else {
 467       PUSH_DATA(push, hq->sequence);
 468       nouveau_pushbuf_data(push, hq->bo, hq->offset,
 469                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 470    }
 471
 472    if (buf->mm) {
 473       nouveau_fence_ref(nvc0->screen->base.fence.current, &buf->fence);
 474       nouveau_fence_ref(nvc0->screen->base.fence.current, &buf->fence_wr);
 475    }
 476 }
 477
 478 static const struct nvc0_query_funcs hw_query_funcs = {
 479    .destroy_query = nvc0_hw_destroy_query,
 480    .begin_query = nvc0_hw_begin_query,
 481    .end_query = nvc0_hw_end_query,
 482    .get_query_result = nvc0_hw_get_query_result,
 483    .get_query_result_resource = nvc0_hw_get_query_result_resource,
 484 };
 485
 486 struct nvc0_query *
 487 nvc0_hw_create_query(struct nvc0_context *nvc0, unsigned type, unsigned index)
 488 {
 489    struct nvc0_hw_query *hq;
 490    struct nvc0_query *q;
 491    unsigned space = NVC0_HW_QUERY_ALLOC_SPACE;
 492
 493    hq = nvc0_hw_sm_create_query(nvc0, type);
 494    if (hq) {
 495       hq->base.funcs = &hw_query_funcs;
 496       return (struct nvc0_query *)hq;
 497    }
 498
 499    hq = nvc0_hw_metric_create_query(nvc0, type);
 500    if (hq) {
 501       hq->base.funcs = &hw_query_funcs;
 502       return (struct nvc0_query *)hq;
 503    }
 504
 505    hq = CALLOC_STRUCT(nvc0_hw_query);
 506    if (!hq)
 507       return NULL;
 508
 509    q = &hq->base;
 510    q->funcs = &hw_query_funcs;
 511    q->type = type;
 512
 513    switch (q->type) {
 514    case PIPE_QUERY_OCCLUSION_COUNTER:
 515    case PIPE_QUERY_OCCLUSION_PREDICATE:
 516       hq->rotate = 32;
 517       space = NVC0_HW_QUERY_ALLOC_SPACE;
 518       break;
 519    case PIPE_QUERY_PIPELINE_STATISTICS:
 520       hq->is64bit = true;
 521       space = 512;
 522       break;
 523    case PIPE_QUERY_SO_STATISTICS:
 524    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 525       hq->is64bit = true;
 526       space = 64;
 527       break;
 528    case PIPE_QUERY_PRIMITIVES_GENERATED:
 529    case PIPE_QUERY_PRIMITIVES_EMITTED:
 530       hq->is64bit = true;
 531       q->index = index;
 532       space = 32;
 533       break;
 534    case PIPE_QUERY_TIME_ELAPSED:
 535    case PIPE_QUERY_TIMESTAMP:
 536    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 537    case PIPE_QUERY_GPU_FINISHED:
 538       space = 32;
 539       break;
 540    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 541       space = 16;
 542       break;
 543    default:
 544       debug_printf("invalid query type: %u\n", type);
 545       FREE(q);
 546       return NULL;
 547    }
 548
 549    if (!nvc0_hw_query_allocate(nvc0, q, space)) {
 550       FREE(hq);
 551       return NULL;
 552    }
 553
 554    if (hq->rotate) {
 555       /* we advance before query_begin ! */
 556       hq->offset -= hq->rotate;
 557       hq->data -= hq->rotate / sizeof(*hq->data);
 558    } else
 559    if (!hq->is64bit)
 560       hq->data[0] = 0; /* initialize sequence */
 561
 562    return q;
 563 }
 564
 565 int
 566 nvc0_hw_get_driver_query_info(struct nvc0_screen *screen, unsigned id,
 567                               struct pipe_driver_query_info *info)
 568 {
 569    int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
 570
 571    num_hw_sm_queries = nvc0_hw_sm_get_driver_query_info(screen, 0, NULL);
 572    num_hw_metric_queries =
 573       nvc0_hw_metric_get_driver_query_info(screen, 0, NULL);
 574
 575    if (!info)
 576       return num_hw_sm_queries + num_hw_metric_queries;
 577
 578    if (id < num_hw_sm_queries)
 579       return nvc0_hw_sm_get_driver_query_info(screen, id, info);
 580
 581    return nvc0_hw_metric_get_driver_query_info(screen,
 582                                                id - num_hw_sm_queries, info);
 583 }
 584
 585 void
 586 nvc0_hw_query_pushbuf_submit(struct nouveau_pushbuf *push,
 587                              struct nvc0_query *q, unsigned result_offset)
 588 {
 589    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 590
 591    PUSH_REFN(push, hq->bo, NOUVEAU_BO_RD | NOUVEAU_BO_GART);
 592    nouveau_pushbuf_data(push, hq->bo, hq->offset + result_offset, 4 |
 593                         NVC0_IB_ENTRY_1_NO_PREFETCH);
 594 }
 595
 596 void
 597 nvc0_hw_query_fifo_wait(struct nvc0_context *nvc0, struct nvc0_query *q)
 598 {
 599    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 600    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 601    unsigned offset = hq->offset;
 602
 603    if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) offset += 0x20;
 604
 605    PUSH_SPACE(push, 5);
 606    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 607    BEGIN_NVC0(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
 608    if (hq->is64bit) {
 609       PUSH_DATAh(push, nvc0->screen->fence.bo->offset);
 610       PUSH_DATA (push, nvc0->screen->fence.bo->offset);
 611       PUSH_DATA (push, hq->fence->sequence);
 612    } else {
 613       PUSH_DATAh(push, hq->bo->offset + offset);
 614       PUSH_DATA (push, hq->bo->offset + offset);
 615       PUSH_DATA (push, hq->sequence);
 616    }
 617    PUSH_DATA (push, (1 << 12) |
 618               NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
 619 }