shared-core/nouveau_object.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drv.h"
  36 #include "nouveau_drm.h"
  37
  38 /* NVidia uses context objects to drive drawing operations.
  39
  40    Context objects can be selected into 8 subchannels in the FIFO,
  41    and then used via DMA command buffers.
  42
  43    A context object is referenced by a user defined handle (CARD32). The HW
  44    looks up graphics objects in a hash table in the instance RAM.
  45
  46    An entry in the hash table consists of 2 CARD32. The first CARD32 contains
  47    the handle, the second one a bitfield, that contains the address of the
  48    object in instance RAM.
  49
  50    The format of the second CARD32 seems to be:
  51
  52    NV4 to NV30:
  53
  54    15: 0  instance_addr >> 4
  55    17:16  engine (here uses 1 = graphics)
  56    28:24  channel id (here uses 0)
  57    31     valid (use 1)
  58
  59    NV40:
  60
  61    15: 0  instance_addr >> 4   (maybe 19-0)
  62    21:20  engine (here uses 1 = graphics)
  63    I'm unsure about the other bits, but using 0 seems to work.
  64
  65    The key into the hash table depends on the object handle and channel id and
  66    is given as:
  67 */
  68 static uint32_t
  69 nouveau_ramht_hash_handle(struct drm_device *dev, int channel, uint32_t handle)
  70 {
  71         struct drm_nouveau_private *dev_priv=dev->dev_private;
  72         uint32_t hash = 0;
  73         int i;
  74
  75         for (i=32;i>0;i-=dev_priv->ramht_bits) {
  76                 hash ^= (handle & ((1 << dev_priv->ramht_bits) - 1));
  77                 handle >>= dev_priv->ramht_bits;
  78         }
  79         if (dev_priv->card_type < NV_50)
  80                 hash ^= channel << (dev_priv->ramht_bits - 4);
  81         hash <<= 3;
  82
  83         DRM_DEBUG("ch%d handle=0x%08x hash=0x%08x\n", channel, handle, hash);
  84         return hash;
  85 }
  86
  87 static int
  88 nouveau_ramht_entry_valid(struct drm_device *dev, struct nouveau_gpuobj *ramht,
  89                           uint32_t offset)
  90 {
  91         struct drm_nouveau_private *dev_priv=dev->dev_private;
  92         uint32_t ctx = INSTANCE_RD(ramht, (offset + 4)/4);
  93
  94         if (dev_priv->card_type < NV_40)
  95                 return ((ctx & NV_RAMHT_CONTEXT_VALID) != 0);
  96         return (ctx != 0);
  97 }
  98
  99 static int
 100 nouveau_ramht_insert(struct drm_device *dev, struct nouveau_gpuobj_ref *ref)
 101 {
 102         struct drm_nouveau_private *dev_priv=dev->dev_private;
 103         struct nouveau_channel *chan = dev_priv->fifos[ref->channel];
 104         struct nouveau_gpuobj *ramht = chan->ramht ? chan->ramht->gpuobj : NULL;
 105         struct nouveau_gpuobj *gpuobj = ref->gpuobj;
 106         uint32_t ctx, co, ho;
 107
 108         if (!ramht) {
 109                 DRM_ERROR("No hash table!\n");
 110                 return -EINVAL;
 111         }
 112
 113         if (dev_priv->card_type < NV_40) {
 114                 ctx = NV_RAMHT_CONTEXT_VALID | (ref->instance >> 4) |
 115                       (ref->channel   << NV_RAMHT_CONTEXT_CHANNEL_SHIFT) |
 116                       (gpuobj->engine << NV_RAMHT_CONTEXT_ENGINE_SHIFT);
 117         } else
 118         if (dev_priv->card_type < NV_50) {
 119                 ctx = (ref->instance >> 4) |
 120                       (ref->channel   << NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) |
 121                       (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
 122         } else {
 123                 ctx = (ref->instance  >> 4) |
 124                       (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
 125         }
 126
 127         co = ho = nouveau_ramht_hash_handle(dev, ref->channel, ref->handle);
 128         do {
 129                 if (!nouveau_ramht_entry_valid(dev, ramht, co)) {
 130                         DRM_DEBUG("insert ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
 131                                   ref->channel, co, ref->handle, ctx);
 132                         INSTANCE_WR(ramht, (co + 0)/4, ref->handle);
 133                         INSTANCE_WR(ramht, (co + 4)/4, ctx);
 134
 135                         list_add_tail(&ref->list, &chan->ramht_refs);
 136                         return 0;
 137                 }
 138                 DRM_DEBUG("collision ch%d 0x%08x: h=0x%08x\n",
 139                           ref->channel, co, INSTANCE_RD(ramht, co/4));
 140
 141                 co += 8;
 142                 if (co >= dev_priv->ramht_size)
 143                         co = 0;
 144         } while (co != ho);
 145
 146         DRM_ERROR("RAMHT space exhausted. ch=%d\n", ref->channel);
 147         return -ENOMEM;
 148 }
 149
 150 static void
 151 nouveau_ramht_remove(struct drm_device *dev, struct nouveau_gpuobj_ref *ref)
 152 {
 153         struct drm_nouveau_private *dev_priv = dev->dev_private;
 154         struct nouveau_channel *chan = dev_priv->fifos[ref->channel];
 155         struct nouveau_gpuobj *ramht = chan->ramht ? chan->ramht->gpuobj : NULL;
 156         uint32_t co, ho;
 157
 158         if (!ramht) {
 159                 DRM_ERROR("No hash table!\n");
 160                 return;
 161         }
 162
 163         co = ho = nouveau_ramht_hash_handle(dev, ref->channel, ref->handle);
 164         do {
 165                 if (nouveau_ramht_entry_valid(dev, ramht, co) &&
 166                     (ref->handle == INSTANCE_RD(ramht, (co/4)))) {
 167                         DRM_DEBUG("remove ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
 168                                   ref->channel, co, ref->handle,
 169                                   INSTANCE_RD(ramht, (co + 4)));
 170                         INSTANCE_WR(ramht, (co + 0)/4, 0x00000000);
 171                         INSTANCE_WR(ramht, (co + 4)/4, 0x00000000);
 172
 173                         list_del(&ref->list);
 174                         return;
 175                 }
 176
 177                 co += 8;
 178                 if (co >= dev_priv->ramht_size)
 179                         co = 0;
 180         } while (co != ho);
 181
 182         DRM_ERROR("RAMHT entry not found. ch=%d, handle=0x%08x\n",
 183                   ref->channel, ref->handle);
 184 }
 185
 186 int
 187 nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
 188                    int size, int align, uint32_t flags,
 189                    struct nouveau_gpuobj **gpuobj_ret)
 190 {
 191         struct drm_nouveau_private *dev_priv = dev->dev_private;
 192         struct nouveau_engine *engine = &dev_priv->Engine;
 193         struct nouveau_gpuobj *gpuobj;
 194         struct mem_block *pramin = NULL;
 195         int ret;
 196
 197         DRM_DEBUG("ch%d size=%d align=%d flags=0x%08x\n",
 198                   chan ? chan->id : -1, size, align, flags);
 199
 200         if (!dev_priv || !gpuobj_ret || *gpuobj_ret != NULL)
 201                 return -EINVAL;
 202
 203         gpuobj = drm_calloc(1, sizeof(*gpuobj), DRM_MEM_DRIVER);
 204         if (!gpuobj)
 205                 return -ENOMEM;
 206         DRM_DEBUG("gpuobj %p\n", gpuobj);
 207         gpuobj->flags = flags;
 208         gpuobj->im_channel = chan ? chan->id : -1;
 209
 210         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 211
 212         /* Choose between global instmem heap, and per-channel private
 213          * instmem heap.  On <NV50 allow requests for private instmem
 214          * to be satisfied from global heap if no per-channel area
 215          * available.
 216          */
 217         if (chan) {
 218                 if (chan->ramin_heap) {
 219                         DRM_DEBUG("private heap\n");
 220                         pramin = chan->ramin_heap;
 221                 } else
 222                 if (dev_priv->card_type < NV_50) {
 223                         DRM_DEBUG("global heap fallback\n");
 224                         pramin = dev_priv->ramin_heap;
 225                 }
 226         } else {
 227                 DRM_DEBUG("global heap\n");
 228                 pramin = dev_priv->ramin_heap;
 229         }
 230
 231         if (!pramin) {
 232                 DRM_ERROR("No PRAMIN heap!\n");
 233                 return -EINVAL;
 234         }
 235
 236         if (!chan && (ret = engine->instmem.populate(dev, gpuobj, &size))) {
 237                 nouveau_gpuobj_del(dev, &gpuobj);
 238                 return ret;
 239         }
 240
 241         /* Allocate a chunk of the PRAMIN aperture */
 242         gpuobj->im_pramin = nouveau_mem_alloc_block(pramin, size,
 243                                                     drm_order(align),
 244                                                     (struct drm_file *)-2);
 245         if (!gpuobj->im_pramin) {
 246                 nouveau_gpuobj_del(dev, &gpuobj);
 247                 return -ENOMEM;
 248         }
 249         gpuobj->im_pramin->flags = NOUVEAU_MEM_INSTANCE;
 250
 251         if (!chan && (ret = engine->instmem.bind(dev, gpuobj))) {
 252                 nouveau_gpuobj_del(dev, &gpuobj);
 253                 return ret;
 254         }
 255
 256         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 257                 int i;
 258
 259                 for (i = 0; i < gpuobj->im_pramin->size; i += 4)
 260                         INSTANCE_WR(gpuobj, i/4, 0);
 261         }
 262
 263         *gpuobj_ret = gpuobj;
 264         return 0;
 265 }
 266
 267 int
 268 nouveau_gpuobj_early_init(struct drm_device *dev)
 269 {
 270         struct drm_nouveau_private *dev_priv = dev->dev_private;
 271
 272         DRM_DEBUG("\n");
 273
 274         INIT_LIST_HEAD(&dev_priv->gpuobj_list);
 275
 276         return 0;
 277 }
 278
 279 int
 280 nouveau_gpuobj_init(struct drm_device *dev)
 281 {
 282         struct drm_nouveau_private *dev_priv = dev->dev_private;
 283         int ret;
 284
 285         DRM_DEBUG("\n");
 286
 287         if (dev_priv->card_type < NV_50) {
 288                 if ((ret = nouveau_gpuobj_new_fake(dev, dev_priv->ramht_offset,
 289                                                    dev_priv->ramht_size,
 290                                                    NVOBJ_FLAG_ZERO_ALLOC |
 291                                                    NVOBJ_FLAG_ALLOW_NO_REFS,
 292                                                    &dev_priv->ramht, NULL)))
 293                         return ret;
 294         }
 295
 296         return 0;
 297 }
 298
 299 void
 300 nouveau_gpuobj_takedown(struct drm_device *dev)
 301 {
 302         struct drm_nouveau_private *dev_priv = dev->dev_private;
 303
 304         DRM_DEBUG("\n");
 305
 306         nouveau_gpuobj_del(dev, &dev_priv->ramht);
 307 }
 308
 309 void
 310 nouveau_gpuobj_late_takedown(struct drm_device *dev)
 311 {
 312         struct drm_nouveau_private *dev_priv = dev->dev_private;
 313         struct nouveau_gpuobj *gpuobj = NULL;
 314         struct list_head *entry, *tmp;
 315
 316         DRM_DEBUG("\n");
 317
 318         list_for_each_safe(entry, tmp, &dev_priv->gpuobj_list) {
 319                 gpuobj = list_entry(entry, struct nouveau_gpuobj, list);
 320
 321                 DRM_ERROR("gpuobj %p still exists at takedown, refs=%d\n",
 322                           gpuobj, gpuobj->refcount);
 323                 gpuobj->refcount = 0;
 324                 nouveau_gpuobj_del(dev, &gpuobj);
 325         }
 326 }
 327
 328 int
 329 nouveau_gpuobj_del(struct drm_device *dev, struct nouveau_gpuobj **pgpuobj)
 330 {
 331         struct drm_nouveau_private *dev_priv = dev->dev_private;
 332         struct nouveau_engine *engine = &dev_priv->Engine;
 333         struct nouveau_gpuobj *gpuobj;
 334
 335         DRM_DEBUG("gpuobj %p\n", pgpuobj ? *pgpuobj : NULL);
 336
 337         if (!dev_priv || !pgpuobj || !(*pgpuobj))
 338                 return -EINVAL;
 339         gpuobj = *pgpuobj;
 340
 341         if (gpuobj->refcount != 0) {
 342                 DRM_ERROR("gpuobj refcount is %d\n", gpuobj->refcount);
 343                 return -EINVAL;
 344         }
 345
 346         if (gpuobj->dtor)
 347                 gpuobj->dtor(dev, gpuobj);
 348
 349         engine->instmem.clear(dev, gpuobj);
 350
 351         if (gpuobj->im_pramin) {
 352                 if (gpuobj->flags & NVOBJ_FLAG_FAKE)
 353                         drm_free(gpuobj->im_pramin, sizeof(*gpuobj->im_pramin),
 354                                  DRM_MEM_DRIVER);
 355                 else
 356                         nouveau_mem_free_block(gpuobj->im_pramin);
 357         }
 358
 359         list_del(&gpuobj->list);
 360
 361         *pgpuobj = NULL;
 362         drm_free(gpuobj, sizeof(*gpuobj), DRM_MEM_DRIVER);
 363         return 0;
 364 }
 365
 366 static int
 367 nouveau_gpuobj_instance_get(struct drm_device *dev,
 368                             struct nouveau_channel *chan,
 369                             struct nouveau_gpuobj *gpuobj, uint32_t *inst)
 370 {
 371         struct drm_nouveau_private *dev_priv = dev->dev_private;
 372         struct nouveau_gpuobj *cpramin;
 373
 374         /* <NV50 use PRAMIN address everywhere */
 375         if (dev_priv->card_type < NV_50) {
 376                 *inst = gpuobj->im_pramin->start;
 377                 return 0;
 378         }
 379
 380         if (chan && gpuobj->im_channel != chan->id) {
 381                 DRM_ERROR("Channel mismatch: obj %d, ref %d\n",
 382                           gpuobj->im_channel, chan->id);
 383                 return -EINVAL;
 384         }
 385
 386         /* NV50 channel-local instance */
 387         if (chan > 0) {
 388                 cpramin = chan->ramin->gpuobj;
 389                 *inst = gpuobj->im_pramin->start - cpramin->im_pramin->start;
 390                 return 0;
 391         }
 392
 393         /* NV50 global (VRAM) instance */
 394         if (gpuobj->im_channel < 0) {
 395                 /* ...from global heap */
 396                 if (!gpuobj->im_backing) {
 397                         DRM_ERROR("AII, no VRAM backing gpuobj\n");
 398                         return -EINVAL;
 399                 }
 400                 *inst = gpuobj->im_backing->start;
 401                 return 0;
 402         } else {
 403                 /* ...from local heap */
 404                 cpramin = dev_priv->fifos[gpuobj->im_channel]->ramin->gpuobj;
 405                 *inst = cpramin->im_backing->start +
 406                         (gpuobj->im_pramin->start - cpramin->im_pramin->start);
 407                 return 0;
 408         }
 409
 410         return -EINVAL;
 411 }
 412
 413 int
 414 nouveau_gpuobj_ref_add(struct drm_device *dev, struct nouveau_channel *chan,
 415                        uint32_t handle, struct nouveau_gpuobj *gpuobj,
 416                        struct nouveau_gpuobj_ref **ref_ret)
 417 {
 418         struct drm_nouveau_private *dev_priv = dev->dev_private;
 419         struct nouveau_gpuobj_ref *ref;
 420         uint32_t instance;
 421         int ret;
 422
 423         DRM_DEBUG("ch%d h=0x%08x gpuobj=%p\n",
 424                   chan ? chan->id : -1, handle, gpuobj);
 425
 426         if (!dev_priv || !gpuobj || (ref_ret && *ref_ret != NULL))
 427                 return -EINVAL;
 428
 429         if (!chan && !ref_ret)
 430                 return -EINVAL;
 431
 432         ret = nouveau_gpuobj_instance_get(dev, chan, gpuobj, &instance);
 433         if (ret)
 434                 return ret;
 435
 436         ref = drm_calloc(1, sizeof(*ref), DRM_MEM_DRIVER);
 437         if (!ref)
 438                 return -ENOMEM;
 439         ref->gpuobj   = gpuobj;
 440         ref->channel  = chan ? chan->id : -1;
 441         ref->instance = instance;
 442
 443         if (!ref_ret) {
 444                 ref->handle = handle;
 445
 446                 ret = nouveau_ramht_insert(dev, ref);
 447                 if (ret) {
 448                         drm_free(ref, sizeof(*ref), DRM_MEM_DRIVER);
 449                         return ret;
 450                 }
 451         } else {
 452                 ref->handle = ~0;
 453                 *ref_ret = ref;
 454         }
 455
 456         ref->gpuobj->refcount++;
 457         return 0;
 458 }
 459
 460 int nouveau_gpuobj_ref_del(struct drm_device *dev, struct nouveau_gpuobj_ref **pref)
 461 {
 462         struct nouveau_gpuobj_ref *ref;
 463
 464         DRM_DEBUG("ref %p\n", pref ? *pref : NULL);
 465
 466         if (!dev || !pref || *pref == NULL)
 467                 return -EINVAL;
 468         ref = *pref;
 469
 470         if (ref->handle != ~0)
 471                 nouveau_ramht_remove(dev, ref);
 472
 473         if (ref->gpuobj) {
 474                 ref->gpuobj->refcount--;
 475
 476                 if (ref->gpuobj->refcount == 0) {
 477                         if (!(ref->gpuobj->flags & NVOBJ_FLAG_ALLOW_NO_REFS))
 478                                 nouveau_gpuobj_del(dev, &ref->gpuobj);
 479                 }
 480         }
 481
 482         *pref = NULL;
 483         drm_free(ref, sizeof(ref), DRM_MEM_DRIVER);
 484         return 0;
 485 }
 486
 487 int
 488 nouveau_gpuobj_new_ref(struct drm_device *dev,
 489                        struct nouveau_channel *oc, struct nouveau_channel *rc,
 490                        uint32_t handle, int size, int align, uint32_t flags,
 491                        struct nouveau_gpuobj_ref **ref)
 492 {
 493         struct nouveau_gpuobj *gpuobj = NULL;
 494         int ret;
 495
 496         if ((ret = nouveau_gpuobj_new(dev, oc, size, align, flags, &gpuobj)))
 497                 return ret;
 498
 499         if ((ret = nouveau_gpuobj_ref_add(dev, rc, handle, gpuobj, ref))) {
 500                 nouveau_gpuobj_del(dev, &gpuobj);
 501                 return ret;
 502         }
 503
 504         return 0;
 505 }
 506
 507 int
 508 nouveau_gpuobj_ref_find(struct nouveau_channel *chan, uint32_t handle,
 509                         struct nouveau_gpuobj_ref **ref_ret)
 510 {
 511         struct nouveau_gpuobj_ref *ref;
 512         struct list_head *entry, *tmp;
 513
 514         list_for_each_safe(entry, tmp, &chan->ramht_refs) {
 515                 ref = list_entry(entry, struct nouveau_gpuobj_ref, list);
 516
 517                 if (ref->handle == handle) {
 518                         if (ref_ret)
 519                                 *ref_ret = ref;
 520                         return 0;
 521                 }
 522         }
 523
 524         return -EINVAL;
 525 }
 526
 527 int
 528 nouveau_gpuobj_new_fake(struct drm_device *dev, uint32_t offset, uint32_t size,
 529                         uint32_t flags, struct nouveau_gpuobj **pgpuobj,
 530                         struct nouveau_gpuobj_ref **pref)
 531 {
 532         struct drm_nouveau_private *dev_priv = dev->dev_private;
 533         struct nouveau_gpuobj *gpuobj = NULL;
 534         int i;
 535
 536         DRM_DEBUG("offset=0x%08x size=0x%08x flags=0x%08x\n",
 537                   offset, size, flags);
 538
 539         gpuobj = drm_calloc(1, sizeof(*gpuobj), DRM_MEM_DRIVER);
 540         if (!gpuobj)
 541                 return -ENOMEM;
 542         DRM_DEBUG("gpuobj %p\n", gpuobj);
 543         gpuobj->im_channel = -1;
 544         gpuobj->flags      = flags | NVOBJ_FLAG_FAKE;
 545
 546         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 547
 548         gpuobj->im_pramin = drm_calloc(1, sizeof(struct mem_block),
 549                                        DRM_MEM_DRIVER);
 550         if (!gpuobj->im_pramin) {
 551                 nouveau_gpuobj_del(dev, &gpuobj);
 552                 return -ENOMEM;
 553         }
 554         gpuobj->im_pramin->start = offset;
 555         gpuobj->im_pramin->size  = size;
 556
 557         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 558                 for (i = 0; i < gpuobj->im_pramin->size; i += 4)
 559                         INSTANCE_WR(gpuobj, i/4, 0);
 560         }
 561
 562         if (pref) {
 563                 if ((i = nouveau_gpuobj_ref_add(dev, NULL, 0, gpuobj, pref))) {
 564                         nouveau_gpuobj_del(dev, &gpuobj);
 565                         return i;
 566                 }
 567         }
 568
 569         if (pgpuobj)
 570                 *pgpuobj = gpuobj;
 571         return 0;
 572 }
 573
 574
 575 static int
 576 nouveau_gpuobj_class_instmem_size(struct drm_device *dev, int class)
 577 {
 578         struct drm_nouveau_private *dev_priv = dev->dev_private;
 579
 580         /*XXX: dodgy hack for now */
 581         if (dev_priv->card_type >= NV_50)
 582                 return 24;
 583         if (dev_priv->card_type >= NV_40)
 584                 return 32;
 585         return 16;
 586 }
 587
 588 /*
 589    DMA objects are used to reference a piece of memory in the
 590    framebuffer, PCI or AGP address space. Each object is 16 bytes big
 591    and looks as follows:
 592
 593    entry[0]
 594    11:0  class (seems like I can always use 0 here)
 595    12    page table present?
 596    13    page entry linear?
 597    15:14 access: 0 rw, 1 ro, 2 wo
 598    17:16 target: 0 NV memory, 1 NV memory tiled, 2 PCI, 3 AGP
 599    31:20 dma adjust (bits 0-11 of the address)
 600    entry[1]
 601    dma limit (size of transfer)
 602    entry[X]
 603    1     0 readonly, 1 readwrite
 604    31:12 dma frame address of the page (bits 12-31 of the address)
 605    entry[N]
 606    page table terminator, same value as the first pte, as does nvidia
 607    rivatv uses 0xffffffff
 608
 609    Non linear page tables need a list of frame addresses afterwards,
 610    the rivatv project has some info on this.
 611
 612    The method below creates a DMA object in instance RAM and returns a handle
 613    to it that can be used to set up context objects.
 614 */
 615 int
 616 nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class,
 617                        uint64_t offset, uint64_t size, int access,
 618                        int target, struct nouveau_gpuobj **gpuobj)
 619 {
 620         struct drm_device *dev = chan->dev;
 621         struct drm_nouveau_private *dev_priv = dev->dev_private;
 622         int ret;
 623         uint32_t is_scatter_gather = 0;
 624
 625         /* Total number of pages covered by the request.
 626          */
 627         const unsigned int page_count = (size + PAGE_SIZE - 1) / PAGE_SIZE;
 628
 629
 630         DRM_DEBUG("ch%d class=0x%04x offset=0x%llx size=0x%llx\n",
 631                   chan->id, class, offset, size);
 632         DRM_DEBUG("access=%d target=%d\n", access, target);
 633
 634         switch (target) {
 635         case NV_DMA_TARGET_AGP:
 636                  offset += dev_priv->gart_info.aper_base;
 637                  break;
 638         case NV_DMA_TARGET_PCI_NONLINEAR:
 639                 /*assume the "offset" is a virtual memory address*/
 640                 is_scatter_gather = 1;
 641                 /*put back the right value*/
 642                 target = NV_DMA_TARGET_PCI;
 643                 break;
 644         default:
 645                 break;
 646         }
 647
 648         ret = nouveau_gpuobj_new(dev, chan,
 649                                  is_scatter_gather ? ((page_count << 2) + 12) : nouveau_gpuobj_class_instmem_size(dev, class),
 650                                  16,
 651                                  NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
 652                                  gpuobj);
 653         if (ret) {
 654                 DRM_ERROR("Error creating gpuobj: %d\n", ret);
 655                 return ret;
 656         }
 657
 658         if (dev_priv->card_type < NV_50) {
 659                 uint32_t frame, adjust, pte_flags = 0;
 660                 adjust = offset &  0x00000fff;
 661                 if (access != NV_DMA_ACCESS_RO)
 662                                 pte_flags |= (1<<1);
 663
 664                 if ( ! is_scatter_gather )
 665                         {
 666                         frame  = offset & ~0x00000fff;
 667
 668                         INSTANCE_WR(*gpuobj, 0, ((1<<12) | (1<<13) |
 669                                         (adjust << 20) |
 670                                          (access << 14) |
 671                                          (target << 16) |
 672                                           class));
 673                         INSTANCE_WR(*gpuobj, 1, size - 1);
 674                         INSTANCE_WR(*gpuobj, 2, frame | pte_flags);
 675                         INSTANCE_WR(*gpuobj, 3, frame | pte_flags);
 676                         }
 677                 else
 678                         {
 679                         /* Intial page entry in the scatter-gather area that
 680                          * corresponds to the base offset
 681                          */
 682                         unsigned int idx = offset / PAGE_SIZE;
 683
 684                         uint32_t instance_offset;
 685                         unsigned int i;
 686
 687                         if ((idx + page_count) > dev->sg->pages) {
 688                                 DRM_ERROR("Requested page range exceedes "
 689                                           "allocated scatter-gather range!");
 690                                 return -E2BIG;
 691                         }
 692
 693                         DRM_DEBUG("Creating PCI DMA object using virtual zone starting at %#llx, size %d\n", offset, (uint32_t)size);
 694                         INSTANCE_WR(*gpuobj, 0, ((1<<12) | (0<<13) |
 695                                 (adjust << 20) |
 696                                 (access << 14) |
 697                                 (target << 16) |
 698                                 class));
 699                         INSTANCE_WR(*gpuobj, 1, (uint32_t) size-1);
 700
 701
 702                         /*write starting at the third dword*/
 703                         instance_offset = 2;
 704
 705                         /*for each PAGE, get its bus address, fill in the page table entry, and advance*/
 706                         for (i = 0; i < page_count; i++) {
 707                                 if (dev->sg->busaddr[idx] == 0) {
 708                                         dev->sg->busaddr[idx] =
 709                                                 pci_map_page(dev->pdev,
 710                                                              dev->sg->pagelist[idx],
 711                                                              0,
 712                                                              PAGE_SIZE,
 713                                                              DMA_BIDIRECTIONAL);
 714
 715                                         if (dma_mapping_error(dev->sg->busaddr[idx])) {
 716                                                 return -ENOMEM;
 717                                         }
 718                                 }
 719
 720                                 frame = (uint32_t) dev->sg->busaddr[idx];
 721                                 INSTANCE_WR(*gpuobj, instance_offset,
 722                                             frame | pte_flags);
 723
 724                                 idx++;
 725                                 instance_offset ++;
 726                         }
 727                         }
 728         } else {
 729                 uint32_t flags0, flags5;
 730
 731                 if (target == NV_DMA_TARGET_VIDMEM) {
 732                         flags0 = 0x00190000;
 733                         flags5 = 0x00010000;
 734                 } else {
 735                         flags0 = 0x7fc00000;
 736                         flags5 = 0x00080000;
 737                 }
 738
 739                 INSTANCE_WR(*gpuobj, 0, flags0 | class);
 740                 INSTANCE_WR(*gpuobj, 1, offset + size - 1);
 741                 INSTANCE_WR(*gpuobj, 2, offset);
 742                 INSTANCE_WR(*gpuobj, 5, flags5);
 743         }
 744
 745         (*gpuobj)->engine = NVOBJ_ENGINE_SW;
 746         (*gpuobj)->class  = class;
 747         return 0;
 748 }
 749
 750 int
 751 nouveau_gpuobj_gart_dma_new(struct nouveau_channel *chan,
 752                             uint64_t offset, uint64_t size, int access,
 753                             struct nouveau_gpuobj **gpuobj,
 754                             uint32_t *o_ret)
 755 {
 756         struct drm_device *dev = chan->dev;
 757         struct drm_nouveau_private *dev_priv = dev->dev_private;
 758         int ret;
 759
 760         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP ||
 761             (dev_priv->card_type >= NV_50 &&
 762              dev_priv->gart_info.type == NOUVEAU_GART_SGDMA)) {
 763                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 764                                              offset, size, access,
 765                                              NV_DMA_TARGET_AGP, gpuobj);
 766                 if (o_ret)
 767                         *o_ret = 0;
 768         } else
 769         if (dev_priv->gart_info.type == NOUVEAU_GART_SGDMA) {
 770                 *gpuobj = dev_priv->gart_info.sg_ctxdma;
 771                 if (offset & ~0xffffffffULL) {
 772                         DRM_ERROR("obj offset exceeds 32-bits\n");
 773                         return -EINVAL;
 774                 }
 775                 if (o_ret)
 776                         *o_ret = (uint32_t)offset;
 777                 ret = (*gpuobj != NULL) ? 0 : -EINVAL;
 778         } else {
 779                 DRM_ERROR("Invalid GART type %d\n", dev_priv->gart_info.type);
 780                 return -EINVAL;
 781         }
 782
 783         return ret;
 784 }
 785
 786 /* Context objects in the instance RAM have the following structure.
 787  * On NV40 they are 32 byte long, on NV30 and smaller 16 bytes.
 788
 789    NV4 - NV30:
 790
 791    entry[0]
 792    11:0 class
 793    12   chroma key enable
 794    13   user clip enable
 795    14   swizzle enable
 796    17:15 patch config:
 797        scrcopy_and, rop_and, blend_and, scrcopy, srccopy_pre, blend_pre
 798    18   synchronize enable
 799    19   endian: 1 big, 0 little
 800    21:20 dither mode
 801    23    single step enable
 802    24    patch status: 0 invalid, 1 valid
 803    25    context_surface 0: 1 valid
 804    26    context surface 1: 1 valid
 805    27    context pattern: 1 valid
 806    28    context rop: 1 valid
 807    29,30 context beta, beta4
 808    entry[1]
 809    7:0   mono format
 810    15:8  color format
 811    31:16 notify instance address
 812    entry[2]
 813    15:0  dma 0 instance address
 814    31:16 dma 1 instance address
 815    entry[3]
 816    dma method traps
 817
 818    NV40:
 819    No idea what the exact format is. Here's what can be deducted:
 820
 821    entry[0]:
 822    11:0  class  (maybe uses more bits here?)
 823    17    user clip enable
 824    21:19 patch config
 825    25    patch status valid ?
 826    entry[1]:
 827    15:0  DMA notifier  (maybe 20:0)
 828    entry[2]:
 829    15:0  DMA 0 instance (maybe 20:0)
 830    24    big endian
 831    entry[3]:
 832    15:0  DMA 1 instance (maybe 20:0)
 833    entry[4]:
 834    entry[5]:
 835    set to 0?
 836 */
 837 int
 838 nouveau_gpuobj_gr_new(struct nouveau_channel *chan, int class,
 839                       struct nouveau_gpuobj **gpuobj)
 840 {
 841         struct drm_device *dev = chan->dev;
 842         struct drm_nouveau_private *dev_priv = dev->dev_private;
 843         int ret;
 844
 845         DRM_DEBUG("ch%d class=0x%04x\n", chan->id, class);
 846
 847         ret = nouveau_gpuobj_new(dev, chan,
 848                                  nouveau_gpuobj_class_instmem_size(dev, class),
 849                                  16,
 850                                  NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
 851                                  gpuobj);
 852         if (ret) {
 853                 DRM_ERROR("Error creating gpuobj: %d\n", ret);
 854                 return ret;
 855         }
 856
 857         if (dev_priv->card_type >= NV_50) {
 858                 INSTANCE_WR(*gpuobj, 0, class);
 859                 INSTANCE_WR(*gpuobj, 5, 0x00010000);
 860         } else {
 861         switch (class) {
 862         case NV_CLASS_NULL:
 863                 INSTANCE_WR(*gpuobj, 0, 0x00001030);
 864                 INSTANCE_WR(*gpuobj, 1, 0xFFFFFFFF);
 865                 break;
 866         default:
 867                 if (dev_priv->card_type >= NV_40) {
 868                         INSTANCE_WR(*gpuobj, 0, class);
 869 #ifdef __BIG_ENDIAN
 870                         INSTANCE_WR(*gpuobj, 2, 0x01000000);
 871 #endif
 872                 } else {
 873 #ifdef __BIG_ENDIAN
 874                         INSTANCE_WR(*gpuobj, 0, class | 0x00080000);
 875 #else
 876                         INSTANCE_WR(*gpuobj, 0, class);
 877 #endif
 878                 }
 879         }
 880         }
 881
 882         (*gpuobj)->engine = NVOBJ_ENGINE_GR;
 883         (*gpuobj)->class  = class;
 884         return 0;
 885 }
 886
 887 static int
 888 nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
 889 {
 890         struct drm_device *dev = chan->dev;
 891         struct drm_nouveau_private *dev_priv = dev->dev_private;
 892         struct nouveau_gpuobj *pramin = NULL;
 893         int size, base, ret;
 894
 895         DRM_DEBUG("ch%d\n", chan->id);
 896
 897         /* Base amount for object storage (4KiB enough?) */
 898         size = 0x1000;
 899         base = 0;
 900
 901         /* PGRAPH context */
 902
 903         if (dev_priv->card_type == NV_50) {
 904                 /* Various fixed table thingos */
 905                 size += 0x1400; /* mostly unknown stuff */
 906                 size += 0x4000; /* vm pd */
 907                 base  = 0x6000;
 908                 /* RAMHT, not sure about setting size yet, 32KiB to be safe */
 909                 size += 0x8000;
 910                 /* RAMFC */
 911                 size += 0x1000;
 912                 /* PGRAPH context */
 913                 size += 0x60000;
 914         }
 915
 916         DRM_DEBUG("ch%d PRAMIN size: 0x%08x bytes, base alloc=0x%08x\n",
 917                   chan->id, size, base);
 918         ret = nouveau_gpuobj_new_ref(dev, NULL, NULL, 0, size, 0x1000, 0,
 919                                      &chan->ramin);
 920         if (ret) {
 921                 DRM_ERROR("Error allocating channel PRAMIN: %d\n", ret);
 922                 return ret;
 923         }
 924         pramin = chan->ramin->gpuobj;
 925
 926         ret = nouveau_mem_init_heap(&chan->ramin_heap,
 927                                     pramin->im_pramin->start + base, size);
 928         if (ret) {
 929                 DRM_ERROR("Error creating PRAMIN heap: %d\n", ret);
 930                 nouveau_gpuobj_ref_del(dev, &chan->ramin);
 931                 return ret;
 932         }
 933
 934         return 0;
 935 }
 936
 937 int
 938 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 939                             uint32_t vram_h, uint32_t tt_h)
 940 {
 941         struct drm_device *dev = chan->dev;
 942         struct drm_nouveau_private *dev_priv = dev->dev_private;
 943         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
 944         int ret, i;
 945
 946         INIT_LIST_HEAD(&chan->ramht_refs);
 947
 948         DRM_DEBUG("ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
 949
 950         /* Reserve a block of PRAMIN for the channel
 951          *XXX: maybe on <NV50 too at some point
 952          */
 953         if (0 || dev_priv->card_type == NV_50) {
 954                 ret = nouveau_gpuobj_channel_init_pramin(chan);
 955                 if (ret)
 956                         return ret;
 957         }
 958
 959         /* NV50 VM, point offset 0-512MiB at shared PCIEGART table  */
 960         if (dev_priv->card_type >= NV_50) {
 961                 uint32_t vm_offset;
 962
 963                 vm_offset = (dev_priv->chipset & 0xf0) == 0x50 ? 0x1400 : 0x200;
 964                 vm_offset += chan->ramin->gpuobj->im_pramin->start;
 965                 if ((ret = nouveau_gpuobj_new_fake(dev, vm_offset, 0x4000,
 966                                                    0, &chan->vm_pd, NULL)))
 967                         return ret;
 968                 for (i=0; i<0x4000; i+=8) {
 969                         INSTANCE_WR(chan->vm_pd, (i+0)/4, 0x00000000);
 970                         INSTANCE_WR(chan->vm_pd, (i+4)/4, 0xdeadcafe);
 971                 }
 972
 973                 if ((ret = nouveau_gpuobj_ref_add(dev, NULL, 0,
 974                                                   dev_priv->gart_info.sg_ctxdma,
 975                                                   &chan->vm_gart_pt)))
 976                         return ret;
 977                 INSTANCE_WR(chan->vm_pd, (0+0)/4,
 978                             chan->vm_gart_pt->instance | 0x03);
 979                 INSTANCE_WR(chan->vm_pd, (0+4)/4, 0x00000000);
 980         }
 981
 982         /* RAMHT */
 983         if (dev_priv->card_type < NV_50) {
 984                 ret = nouveau_gpuobj_ref_add(dev, NULL, 0, dev_priv->ramht,
 985                                              &chan->ramht);
 986                 if (ret)
 987                         return ret;
 988         } else {
 989                 ret = nouveau_gpuobj_new_ref(dev, chan, chan, 0,
 990                                              0x8000, 16,
 991                                              NVOBJ_FLAG_ZERO_ALLOC,
 992                                              &chan->ramht);
 993                 if (ret)
 994                         return ret;
 995         }
 996
 997         /* VRAM ctxdma */
 998         if ((ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 999                                           0, dev_priv->fb_available_size,
1000                                           NV_DMA_ACCESS_RW,
1001                                           NV_DMA_TARGET_VIDMEM, &vram))) {
1002                 DRM_ERROR("Error creating VRAM ctxdma: %d\n", ret);
1003                 return ret;
1004         }
1005
1006         if ((ret = nouveau_gpuobj_ref_add(dev, chan, vram_h, vram, NULL))) {
1007                 DRM_ERROR("Error referencing VRAM ctxdma: %d\n", ret);
1008                 return ret;
1009         }
1010
1011         /* TT memory ctxdma */
1012         if (dev_priv->gart_info.type != NOUVEAU_GART_NONE) {
1013                 ret = nouveau_gpuobj_gart_dma_new(chan, 0,
1014                                                   dev_priv->gart_info.aper_size,
1015                                                   NV_DMA_ACCESS_RW, &tt, NULL);
1016         } else
1017         if (dev_priv->pci_heap) {
1018                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
1019                                              0, dev->sg->pages * PAGE_SIZE,
1020                                              NV_DMA_ACCESS_RW,
1021                                              NV_DMA_TARGET_PCI_NONLINEAR, &tt);
1022         } else {
1023                 DRM_ERROR("Invalid GART type %d\n", dev_priv->gart_info.type);
1024                 ret = -EINVAL;
1025         }
1026
1027         if (ret) {
1028                 DRM_ERROR("Error creating TT ctxdma: %d\n", ret);
1029                 return ret;
1030         }
1031
1032         ret = nouveau_gpuobj_ref_add(dev, chan, tt_h, tt, NULL);
1033         if (ret) {
1034                 DRM_ERROR("Error referencing TT ctxdma: %d\n", ret);
1035                 return ret;
1036         }
1037
1038         return 0;
1039 }
1040
1041 void
1042 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
1043 {
1044         struct drm_device *dev = chan->dev;
1045         struct list_head *entry, *tmp;
1046         struct nouveau_gpuobj_ref *ref;
1047
1048         DRM_DEBUG("ch%d\n", chan->id);
1049
1050         list_for_each_safe(entry, tmp, &chan->ramht_refs) {
1051                 ref = list_entry(entry, struct nouveau_gpuobj_ref, list);
1052
1053                 nouveau_gpuobj_ref_del(dev, &ref);
1054         }
1055
1056         nouveau_gpuobj_ref_del(dev, &chan->ramht);
1057
1058         nouveau_gpuobj_del(dev, &chan->vm_pd);
1059         nouveau_gpuobj_ref_del(dev, &chan->vm_gart_pt);
1060
1061         if (chan->ramin_heap)
1062                 nouveau_mem_takedown(&chan->ramin_heap);
1063         if (chan->ramin)
1064                 nouveau_gpuobj_ref_del(dev, &chan->ramin);
1065
1066 }
1067
1068 int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
1069                               struct drm_file *file_priv)
1070 {
1071         struct nouveau_channel *chan;
1072         struct drm_nouveau_grobj_alloc *init = data;
1073         struct nouveau_gpuobj *gr = NULL;
1074         int ret;
1075
1076         NOUVEAU_CHECK_INITIALISED_WITH_RETURN;
1077         NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(init->channel, file_priv, chan);
1078
1079         //FIXME: check args, only allow trusted objects to be created
1080
1081         if (init->handle == ~0)
1082                 return -EINVAL;
1083
1084         if (nouveau_gpuobj_ref_find(chan, init->handle, NULL) == 0)
1085                 return -EEXIST;
1086
1087         ret = nouveau_gpuobj_gr_new(chan, init->class, &gr);
1088         if (ret) {
1089                 DRM_ERROR("Error creating gr object: %d (%d/0x%08x)\n",
1090                           ret, init->channel, init->handle);
1091                 return ret;
1092         }
1093
1094         if ((ret = nouveau_gpuobj_ref_add(dev, chan, init->handle, gr, NULL))) {
1095                 DRM_ERROR("Error referencing gr object: %d (%d/0x%08x\n)",
1096                           ret, init->channel, init->handle);
1097                 nouveau_gpuobj_del(dev, &gr);
1098                 return ret;
1099         }
1100
1101         return 0;
1102 }
1103
1104 int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
1105                               struct drm_file *file_priv)
1106 {
1107         struct drm_nouveau_gpuobj_free *objfree = data;
1108         struct nouveau_gpuobj_ref *ref;
1109         struct nouveau_channel *chan;
1110         int ret;
1111
1112         NOUVEAU_CHECK_INITIALISED_WITH_RETURN;
1113         NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(objfree->channel, file_priv, chan);
1114
1115         if ((ret = nouveau_gpuobj_ref_find(chan, objfree->handle, &ref)))
1116                 return ret;
1117         nouveau_gpuobj_ref_del(dev, &ref);
1118
1119         return 0;
1120 }
1121