shared-core/nouveau_object.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drv.h"
  36 #include "nouveau_drm.h"
  37
  38 /* NVidia uses context objects to drive drawing operations.
  39
  40    Context objects can be selected into 8 subchannels in the FIFO,
  41    and then used via DMA command buffers.
  42
  43    A context object is referenced by a user defined handle (CARD32). The HW
  44    looks up graphics objects in a hash table in the instance RAM.
  45
  46    An entry in the hash table consists of 2 CARD32. The first CARD32 contains
  47    the handle, the second one a bitfield, that contains the address of the
  48    object in instance RAM.
  49
  50    The format of the second CARD32 seems to be:
  51
  52    NV4 to NV30:
  53
  54    15: 0  instance_addr >> 4
  55    17:16  engine (here uses 1 = graphics)
  56    28:24  channel id (here uses 0)
  57    31     valid (use 1)
  58
  59    NV40:
  60
  61    15: 0  instance_addr >> 4   (maybe 19-0)
  62    21:20  engine (here uses 1 = graphics)
  63    I'm unsure about the other bits, but using 0 seems to work.
  64
  65    The key into the hash table depends on the object handle and channel id and
  66    is given as:
  67 */
  68 static uint32_t
  69 nouveau_ramht_hash_handle(struct drm_device *dev, int channel, uint32_t handle)
  70 {
  71         struct drm_nouveau_private *dev_priv=dev->dev_private;
  72         uint32_t hash = 0;
  73         int i;
  74
  75         DRM_DEBUG("ch%d handle=0x%08x\n", channel, handle);
  76
  77         for (i=32;i>0;i-=dev_priv->ramht_bits) {
  78                 hash ^= (handle & ((1 << dev_priv->ramht_bits) - 1));
  79                 handle >>= dev_priv->ramht_bits;
  80         }
  81         if (dev_priv->card_type < NV_50)
  82                 hash ^= channel << (dev_priv->ramht_bits - 4);
  83         hash <<= 3;
  84
  85         DRM_DEBUG("hash=0x%08x\n", hash);
  86         return hash;
  87 }
  88
  89 static int
  90 nouveau_ramht_entry_valid(struct drm_device *dev, struct nouveau_gpuobj *ramht,
  91                           uint32_t offset)
  92 {
  93         struct drm_nouveau_private *dev_priv=dev->dev_private;
  94         uint32_t ctx = INSTANCE_RD(ramht, (offset + 4)/4);
  95
  96         if (dev_priv->card_type < NV_40)
  97                 return ((ctx & NV_RAMHT_CONTEXT_VALID) != 0);
  98         return (ctx != 0);
  99 }
 100
 101 static int
 102 nouveau_ramht_insert(struct drm_device *dev, struct nouveau_gpuobj_ref *ref)
 103 {
 104         struct drm_nouveau_private *dev_priv=dev->dev_private;
 105         struct nouveau_channel *chan = dev_priv->fifos[ref->channel];
 106         struct nouveau_gpuobj *ramht = chan->ramht ? chan->ramht->gpuobj : NULL;
 107         struct nouveau_gpuobj *gpuobj = ref->gpuobj;
 108         uint32_t ctx, co, ho;
 109
 110         if (!ramht) {
 111                 DRM_ERROR("No hash table!\n");
 112                 return -EINVAL;
 113         }
 114
 115         if (dev_priv->card_type < NV_40) {
 116                 ctx = NV_RAMHT_CONTEXT_VALID | (ref->instance >> 4) |
 117                       (ref->channel   << NV_RAMHT_CONTEXT_CHANNEL_SHIFT) |
 118                       (gpuobj->engine << NV_RAMHT_CONTEXT_ENGINE_SHIFT);
 119         } else
 120         if (dev_priv->card_type < NV_50) {
 121                 ctx = (ref->instance >> 4) |
 122                       (ref->channel   << NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) |
 123                       (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
 124         } else {
 125                 ctx = (ref->instance  >> 4) |
 126                       (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
 127         }
 128
 129         co = ho = nouveau_ramht_hash_handle(dev, ref->channel, ref->handle);
 130         do {
 131                 if (!nouveau_ramht_entry_valid(dev, ramht, co)) {
 132                         DRM_DEBUG("insert ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
 133                                   ref->channel, co, ref->handle, ctx);
 134                         INSTANCE_WR(ramht, (co + 0)/4, ref->handle);
 135                         INSTANCE_WR(ramht, (co + 4)/4, ctx);
 136
 137                         list_add_tail(&ref->list, &chan->ramht_refs);
 138                         return 0;
 139                 }
 140                 DRM_DEBUG("collision ch%d 0x%08x: h=0x%08x\n",
 141                           ref->channel, co, INSTANCE_RD(ramht, co/4));
 142
 143                 co += 8;
 144                 if (co >= dev_priv->ramht_size) {
 145                         DRM_INFO("no space left after collision\n");
 146                         co = 0;
 147                         /* exit as it seems to cause crash with nouveau_demo and
 148                          * 0xdead0001 object */
 149                         break;
 150                 }
 151         } while (co != ho);
 152
 153         DRM_ERROR("RAMHT space exhausted. ch=%d\n", ref->channel);
 154         return -ENOMEM;
 155 }
 156
 157 static void
 158 nouveau_ramht_remove(struct drm_device *dev, struct nouveau_gpuobj_ref *ref)
 159 {
 160         struct drm_nouveau_private *dev_priv = dev->dev_private;
 161         struct nouveau_channel *chan = dev_priv->fifos[ref->channel];
 162         struct nouveau_gpuobj *ramht = chan->ramht ? chan->ramht->gpuobj : NULL;
 163         uint32_t co, ho;
 164
 165         if (!ramht) {
 166                 DRM_ERROR("No hash table!\n");
 167                 return;
 168         }
 169
 170         co = ho = nouveau_ramht_hash_handle(dev, ref->channel, ref->handle);
 171         do {
 172                 if (nouveau_ramht_entry_valid(dev, ramht, co) &&
 173                     (ref->handle == INSTANCE_RD(ramht, (co/4)))) {
 174                         DRM_DEBUG("remove ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
 175                                   ref->channel, co, ref->handle,
 176                                   INSTANCE_RD(ramht, (co + 4)));
 177                         INSTANCE_WR(ramht, (co + 0)/4, 0x00000000);
 178                         INSTANCE_WR(ramht, (co + 4)/4, 0x00000000);
 179
 180                         list_del(&ref->list);
 181                         return;
 182                 }
 183
 184                 co += 8;
 185                 if (co >= dev_priv->ramht_size)
 186                         co = 0;
 187         } while (co != ho);
 188
 189         DRM_ERROR("RAMHT entry not found. ch=%d, handle=0x%08x\n",
 190                   ref->channel, ref->handle);
 191 }
 192
 193 int
 194 nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
 195                    int size, int align, uint32_t flags,
 196                    struct nouveau_gpuobj **gpuobj_ret)
 197 {
 198         struct drm_nouveau_private *dev_priv = dev->dev_private;
 199         struct nouveau_engine *engine = &dev_priv->Engine;
 200         struct nouveau_gpuobj *gpuobj;
 201         struct mem_block *pramin = NULL;
 202         int ret;
 203
 204         DRM_DEBUG("ch%d size=%d align=%d flags=0x%08x\n",
 205                   chan ? chan->id : -1, size, align, flags);
 206
 207         if (!dev_priv || !gpuobj_ret || *gpuobj_ret != NULL)
 208                 return -EINVAL;
 209
 210         gpuobj = drm_calloc(1, sizeof(*gpuobj), DRM_MEM_DRIVER);
 211         if (!gpuobj)
 212                 return -ENOMEM;
 213         DRM_DEBUG("gpuobj %p\n", gpuobj);
 214         gpuobj->flags = flags;
 215         gpuobj->im_channel = chan ? chan->id : -1;
 216
 217         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 218
 219         /* Choose between global instmem heap, and per-channel private
 220          * instmem heap.  On <NV50 allow requests for private instmem
 221          * to be satisfied from global heap if no per-channel area
 222          * available.
 223          */
 224         if (chan) {
 225                 if (chan->ramin_heap) {
 226                         DRM_DEBUG("private heap\n");
 227                         pramin = chan->ramin_heap;
 228                 } else
 229                 if (dev_priv->card_type < NV_50) {
 230                         DRM_DEBUG("global heap fallback\n");
 231                         pramin = dev_priv->ramin_heap;
 232                 }
 233         } else {
 234                 DRM_DEBUG("global heap\n");
 235                 pramin = dev_priv->ramin_heap;
 236         }
 237
 238         if (!pramin) {
 239                 DRM_ERROR("No PRAMIN heap!\n");
 240                 return -EINVAL;
 241         }
 242
 243         if (!chan && (ret = engine->instmem.populate(dev, gpuobj, &size))) {
 244                 nouveau_gpuobj_del(dev, &gpuobj);
 245                 return ret;
 246         }
 247
 248         /* Allocate a chunk of the PRAMIN aperture */
 249         gpuobj->im_pramin = nouveau_mem_alloc_block(pramin, size,
 250                                                     drm_order(align),
 251                                                     (struct drm_file *)-2, 0);
 252         if (!gpuobj->im_pramin) {
 253                 nouveau_gpuobj_del(dev, &gpuobj);
 254                 return -ENOMEM;
 255         }
 256         gpuobj->im_pramin->flags = NOUVEAU_MEM_INSTANCE;
 257
 258         if (!chan && (ret = engine->instmem.bind(dev, gpuobj))) {
 259                 nouveau_gpuobj_del(dev, &gpuobj);
 260                 return ret;
 261         }
 262
 263         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 264                 int i;
 265
 266                 for (i = 0; i < gpuobj->im_pramin->size; i += 4)
 267                         INSTANCE_WR(gpuobj, i/4, 0);
 268         }
 269
 270         *gpuobj_ret = gpuobj;
 271         return 0;
 272 }
 273
 274 int
 275 nouveau_gpuobj_early_init(struct drm_device *dev)
 276 {
 277         struct drm_nouveau_private *dev_priv = dev->dev_private;
 278
 279         DRM_DEBUG("\n");
 280
 281         INIT_LIST_HEAD(&dev_priv->gpuobj_list);
 282
 283         return 0;
 284 }
 285
 286 int
 287 nouveau_gpuobj_init(struct drm_device *dev)
 288 {
 289         struct drm_nouveau_private *dev_priv = dev->dev_private;
 290         int ret;
 291
 292         DRM_DEBUG("\n");
 293
 294         if (dev_priv->card_type < NV_50) {
 295                 if ((ret = nouveau_gpuobj_new_fake(dev, dev_priv->ramht_offset,
 296                                                    ~0, dev_priv->ramht_size,
 297                                                    NVOBJ_FLAG_ZERO_ALLOC |
 298                                                    NVOBJ_FLAG_ALLOW_NO_REFS,
 299                                                    &dev_priv->ramht, NULL)))
 300                         return ret;
 301         }
 302
 303         return 0;
 304 }
 305
 306 void
 307 nouveau_gpuobj_takedown(struct drm_device *dev)
 308 {
 309         struct drm_nouveau_private *dev_priv = dev->dev_private;
 310
 311         DRM_DEBUG("\n");
 312
 313         nouveau_gpuobj_del(dev, &dev_priv->ramht);
 314 }
 315
 316 void
 317 nouveau_gpuobj_late_takedown(struct drm_device *dev)
 318 {
 319         struct drm_nouveau_private *dev_priv = dev->dev_private;
 320         struct nouveau_gpuobj *gpuobj = NULL;
 321         struct list_head *entry, *tmp;
 322
 323         DRM_DEBUG("\n");
 324
 325         list_for_each_safe(entry, tmp, &dev_priv->gpuobj_list) {
 326                 gpuobj = list_entry(entry, struct nouveau_gpuobj, list);
 327
 328                 DRM_ERROR("gpuobj %p still exists at takedown, refs=%d\n",
 329                           gpuobj, gpuobj->refcount);
 330                 gpuobj->refcount = 0;
 331                 nouveau_gpuobj_del(dev, &gpuobj);
 332         }
 333 }
 334
 335 int
 336 nouveau_gpuobj_del(struct drm_device *dev, struct nouveau_gpuobj **pgpuobj)
 337 {
 338         struct drm_nouveau_private *dev_priv = dev->dev_private;
 339         struct nouveau_engine *engine = &dev_priv->Engine;
 340         struct nouveau_gpuobj *gpuobj;
 341
 342         DRM_DEBUG("gpuobj %p\n", pgpuobj ? *pgpuobj : NULL);
 343
 344         if (!dev_priv || !pgpuobj || !(*pgpuobj))
 345                 return -EINVAL;
 346         gpuobj = *pgpuobj;
 347
 348         if (gpuobj->refcount != 0) {
 349                 DRM_ERROR("gpuobj refcount is %d\n", gpuobj->refcount);
 350                 return -EINVAL;
 351         }
 352
 353         if (gpuobj->dtor)
 354                 gpuobj->dtor(dev, gpuobj);
 355
 356         if (gpuobj->im_backing) {
 357                 if (gpuobj->flags & NVOBJ_FLAG_FAKE)
 358                         drm_free(gpuobj->im_backing,
 359                                  sizeof(*gpuobj->im_backing), DRM_MEM_DRIVER);
 360                 else
 361                         engine->instmem.clear(dev, gpuobj);
 362         }
 363
 364         if (gpuobj->im_pramin) {
 365                 if (gpuobj->flags & NVOBJ_FLAG_FAKE)
 366                         drm_free(gpuobj->im_pramin, sizeof(*gpuobj->im_pramin),
 367                                  DRM_MEM_DRIVER);
 368                 else
 369                         nouveau_mem_free_block(gpuobj->im_pramin);
 370         }
 371
 372         list_del(&gpuobj->list);
 373
 374         *pgpuobj = NULL;
 375         drm_free(gpuobj, sizeof(*gpuobj), DRM_MEM_DRIVER);
 376         return 0;
 377 }
 378
 379 static int
 380 nouveau_gpuobj_instance_get(struct drm_device *dev,
 381                             struct nouveau_channel *chan,
 382                             struct nouveau_gpuobj *gpuobj, uint32_t *inst)
 383 {
 384         struct drm_nouveau_private *dev_priv = dev->dev_private;
 385         struct nouveau_gpuobj *cpramin;
 386
 387         /* <NV50 use PRAMIN address everywhere */
 388         if (dev_priv->card_type < NV_50) {
 389                 *inst = gpuobj->im_pramin->start;
 390                 return 0;
 391         }
 392
 393         if (chan && gpuobj->im_channel != chan->id) {
 394                 DRM_ERROR("Channel mismatch: obj %d, ref %d\n",
 395                           gpuobj->im_channel, chan->id);
 396                 return -EINVAL;
 397         }
 398
 399         /* NV50 channel-local instance */
 400         if (chan > 0) {
 401                 cpramin = chan->ramin->gpuobj;
 402                 *inst = gpuobj->im_pramin->start - cpramin->im_pramin->start;
 403                 return 0;
 404         }
 405
 406         /* NV50 global (VRAM) instance */
 407         if (gpuobj->im_channel < 0) {
 408                 /* ...from global heap */
 409                 if (!gpuobj->im_backing) {
 410                         DRM_ERROR("AII, no VRAM backing gpuobj\n");
 411                         return -EINVAL;
 412                 }
 413                 *inst = gpuobj->im_backing->start;
 414                 return 0;
 415         } else {
 416                 /* ...from local heap */
 417                 cpramin = dev_priv->fifos[gpuobj->im_channel]->ramin->gpuobj;
 418                 *inst = cpramin->im_backing->start +
 419                         (gpuobj->im_pramin->start - cpramin->im_pramin->start);
 420                 return 0;
 421         }
 422
 423         return -EINVAL;
 424 }
 425
 426 int
 427 nouveau_gpuobj_ref_add(struct drm_device *dev, struct nouveau_channel *chan,
 428                        uint32_t handle, struct nouveau_gpuobj *gpuobj,
 429                        struct nouveau_gpuobj_ref **ref_ret)
 430 {
 431         struct drm_nouveau_private *dev_priv = dev->dev_private;
 432         struct nouveau_gpuobj_ref *ref;
 433         uint32_t instance;
 434         int ret;
 435
 436         DRM_DEBUG("ch%d h=0x%08x gpuobj=%p\n",
 437                   chan ? chan->id : -1, handle, gpuobj);
 438
 439         if (!dev_priv || !gpuobj || (ref_ret && *ref_ret != NULL))
 440                 return -EINVAL;
 441
 442         if (!chan && !ref_ret)
 443                 return -EINVAL;
 444
 445         ret = nouveau_gpuobj_instance_get(dev, chan, gpuobj, &instance);
 446         if (ret)
 447                 return ret;
 448
 449         ref = drm_calloc(1, sizeof(*ref), DRM_MEM_DRIVER);
 450         if (!ref)
 451                 return -ENOMEM;
 452         ref->gpuobj   = gpuobj;
 453         ref->channel  = chan ? chan->id : -1;
 454         ref->instance = instance;
 455
 456         if (!ref_ret) {
 457                 ref->handle = handle;
 458
 459                 ret = nouveau_ramht_insert(dev, ref);
 460                 if (ret) {
 461                         drm_free(ref, sizeof(*ref), DRM_MEM_DRIVER);
 462                         return ret;
 463                 }
 464         } else {
 465                 ref->handle = ~0;
 466                 *ref_ret = ref;
 467         }
 468
 469         ref->gpuobj->refcount++;
 470         return 0;
 471 }
 472
 473 int nouveau_gpuobj_ref_del(struct drm_device *dev, struct nouveau_gpuobj_ref **pref)
 474 {
 475         struct nouveau_gpuobj_ref *ref;
 476
 477         DRM_DEBUG("ref %p\n", pref ? *pref : NULL);
 478
 479         if (!dev || !pref || *pref == NULL)
 480                 return -EINVAL;
 481         ref = *pref;
 482
 483         if (ref->handle != ~0)
 484                 nouveau_ramht_remove(dev, ref);
 485
 486         if (ref->gpuobj) {
 487                 ref->gpuobj->refcount--;
 488
 489                 if (ref->gpuobj->refcount == 0) {
 490                         if (!(ref->gpuobj->flags & NVOBJ_FLAG_ALLOW_NO_REFS))
 491                                 nouveau_gpuobj_del(dev, &ref->gpuobj);
 492                 }
 493         }
 494
 495         *pref = NULL;
 496         drm_free(ref, sizeof(ref), DRM_MEM_DRIVER);
 497         return 0;
 498 }
 499
 500 int
 501 nouveau_gpuobj_new_ref(struct drm_device *dev,
 502                        struct nouveau_channel *oc, struct nouveau_channel *rc,
 503                        uint32_t handle, int size, int align, uint32_t flags,
 504                        struct nouveau_gpuobj_ref **ref)
 505 {
 506         struct nouveau_gpuobj *gpuobj = NULL;
 507         int ret;
 508
 509         if ((ret = nouveau_gpuobj_new(dev, oc, size, align, flags, &gpuobj)))
 510                 return ret;
 511
 512         if ((ret = nouveau_gpuobj_ref_add(dev, rc, handle, gpuobj, ref))) {
 513                 nouveau_gpuobj_del(dev, &gpuobj);
 514                 return ret;
 515         }
 516
 517         return 0;
 518 }
 519
 520 int
 521 nouveau_gpuobj_ref_find(struct nouveau_channel *chan, uint32_t handle,
 522                         struct nouveau_gpuobj_ref **ref_ret)
 523 {
 524         struct nouveau_gpuobj_ref *ref;
 525         struct list_head *entry, *tmp;
 526
 527         list_for_each_safe(entry, tmp, &chan->ramht_refs) {
 528                 ref = list_entry(entry, struct nouveau_gpuobj_ref, list);
 529
 530                 if (ref->handle == handle) {
 531                         if (ref_ret)
 532                                 *ref_ret = ref;
 533                         return 0;
 534                 }
 535         }
 536
 537         return -EINVAL;
 538 }
 539
 540 int
 541 nouveau_gpuobj_new_fake(struct drm_device *dev, uint32_t p_offset,
 542                         uint32_t b_offset, uint32_t size,
 543                         uint32_t flags, struct nouveau_gpuobj **pgpuobj,
 544                         struct nouveau_gpuobj_ref **pref)
 545 {
 546         struct drm_nouveau_private *dev_priv = dev->dev_private;
 547         struct nouveau_gpuobj *gpuobj = NULL;
 548         int i;
 549
 550         DRM_DEBUG("p_offset=0x%08x b_offset=0x%08x size=0x%08x flags=0x%08x\n",
 551                   p_offset, b_offset, size, flags);
 552
 553         gpuobj = drm_calloc(1, sizeof(*gpuobj), DRM_MEM_DRIVER);
 554         if (!gpuobj)
 555                 return -ENOMEM;
 556         DRM_DEBUG("gpuobj %p\n", gpuobj);
 557         gpuobj->im_channel = -1;
 558         gpuobj->flags      = flags | NVOBJ_FLAG_FAKE;
 559
 560         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 561
 562         if (p_offset != ~0) {
 563                 gpuobj->im_pramin = drm_calloc(1, sizeof(struct mem_block),
 564                                                DRM_MEM_DRIVER);
 565                 if (!gpuobj->im_pramin) {
 566                         nouveau_gpuobj_del(dev, &gpuobj);
 567                         return -ENOMEM;
 568                 }
 569                 gpuobj->im_pramin->start = p_offset;
 570                 gpuobj->im_pramin->size  = size;
 571         }
 572
 573         if (b_offset != ~0) {
 574                 gpuobj->im_backing = drm_calloc(1, sizeof(struct mem_block),
 575                                                DRM_MEM_DRIVER);
 576                 if (!gpuobj->im_backing) {
 577                         nouveau_gpuobj_del(dev, &gpuobj);
 578                         return -ENOMEM;
 579                 }
 580                 gpuobj->im_backing->start = b_offset;
 581                 gpuobj->im_backing->size  = size;
 582         }
 583
 584         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 585                 for (i = 0; i < gpuobj->im_pramin->size; i += 4)
 586                         INSTANCE_WR(gpuobj, i/4, 0);
 587         }
 588
 589         if (pref) {
 590                 if ((i = nouveau_gpuobj_ref_add(dev, NULL, 0, gpuobj, pref))) {
 591                         nouveau_gpuobj_del(dev, &gpuobj);
 592                         return i;
 593                 }
 594         }
 595
 596         if (pgpuobj)
 597                 *pgpuobj = gpuobj;
 598         return 0;
 599 }
 600
 601
 602 static int
 603 nouveau_gpuobj_class_instmem_size(struct drm_device *dev, int class)
 604 {
 605         struct drm_nouveau_private *dev_priv = dev->dev_private;
 606
 607         /*XXX: dodgy hack for now */
 608         if (dev_priv->card_type >= NV_50)
 609                 return 24;
 610         if (dev_priv->card_type >= NV_40)
 611                 return 32;
 612         return 16;
 613 }
 614
 615 /*
 616    DMA objects are used to reference a piece of memory in the
 617    framebuffer, PCI or AGP address space. Each object is 16 bytes big
 618    and looks as follows:
 619
 620    entry[0]
 621    11:0  class (seems like I can always use 0 here)
 622    12    page table present?
 623    13    page entry linear?
 624    15:14 access: 0 rw, 1 ro, 2 wo
 625    17:16 target: 0 NV memory, 1 NV memory tiled, 2 PCI, 3 AGP
 626    31:20 dma adjust (bits 0-11 of the address)
 627    entry[1]
 628    dma limit (size of transfer)
 629    entry[X]
 630    1     0 readonly, 1 readwrite
 631    31:12 dma frame address of the page (bits 12-31 of the address)
 632    entry[N]
 633    page table terminator, same value as the first pte, as does nvidia
 634    rivatv uses 0xffffffff
 635
 636    Non linear page tables need a list of frame addresses afterwards,
 637    the rivatv project has some info on this.
 638
 639    The method below creates a DMA object in instance RAM and returns a handle
 640    to it that can be used to set up context objects.
 641 */
 642 int
 643 nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class,
 644                        uint64_t offset, uint64_t size, int access,
 645                        int target, struct nouveau_gpuobj **gpuobj)
 646 {
 647         struct drm_device *dev = chan->dev;
 648         struct drm_nouveau_private *dev_priv = dev->dev_private;
 649         int ret;
 650         uint32_t is_scatter_gather = 0;
 651
 652         /* Total number of pages covered by the request.
 653          */
 654         const unsigned int page_count = (size + PAGE_SIZE - 1) / PAGE_SIZE;
 655
 656
 657         DRM_DEBUG("ch%d class=0x%04x offset=0x%llx size=0x%llx\n",
 658                   chan->id, class, offset, size);
 659         DRM_DEBUG("access=%d target=%d\n", access, target);
 660
 661         switch (target) {
 662         case NV_DMA_TARGET_AGP:
 663                  offset += dev_priv->gart_info.aper_base;
 664                  break;
 665         case NV_DMA_TARGET_PCI_NONLINEAR:
 666                 /*assume the "offset" is a virtual memory address*/
 667                 is_scatter_gather = 1;
 668                 /*put back the right value*/
 669                 target = NV_DMA_TARGET_PCI;
 670                 break;
 671         default:
 672                 break;
 673         }
 674
 675         ret = nouveau_gpuobj_new(dev, chan,
 676                                  is_scatter_gather ? ((page_count << 2) + 12) : nouveau_gpuobj_class_instmem_size(dev, class),
 677                                  16,
 678                                  NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
 679                                  gpuobj);
 680         if (ret) {
 681                 DRM_ERROR("Error creating gpuobj: %d\n", ret);
 682                 return ret;
 683         }
 684
 685         if (dev_priv->card_type < NV_50) {
 686                 uint32_t frame, adjust, pte_flags = 0;
 687                 adjust = offset &  0x00000fff;
 688                 if (access != NV_DMA_ACCESS_RO)
 689                                 pte_flags |= (1<<1);
 690
 691                 if ( ! is_scatter_gather )
 692                         {
 693                         frame  = offset & ~0x00000fff;
 694
 695                         INSTANCE_WR(*gpuobj, 0, ((1<<12) | (1<<13) |
 696                                         (adjust << 20) |
 697                                          (access << 14) |
 698                                          (target << 16) |
 699                                           class));
 700                         INSTANCE_WR(*gpuobj, 1, size - 1);
 701                         INSTANCE_WR(*gpuobj, 2, frame | pte_flags);
 702                         INSTANCE_WR(*gpuobj, 3, frame | pte_flags);
 703                         }
 704                 else
 705                         {
 706                         /* Intial page entry in the scatter-gather area that
 707                          * corresponds to the base offset
 708                          */
 709                         unsigned int idx = offset / PAGE_SIZE;
 710
 711                         uint32_t instance_offset;
 712                         unsigned int i;
 713
 714                         if ((idx + page_count) > dev->sg->pages) {
 715                                 DRM_ERROR("Requested page range exceedes "
 716                                           "allocated scatter-gather range!");
 717                                 return -E2BIG;
 718                         }
 719
 720                         DRM_DEBUG("Creating PCI DMA object using virtual zone starting at %#llx, size %d\n", offset, (uint32_t)size);
 721                         INSTANCE_WR(*gpuobj, 0, ((1<<12) | (0<<13) |
 722                                 (adjust << 20) |
 723                                 (access << 14) |
 724                                 (target << 16) |
 725                                 class));
 726                         INSTANCE_WR(*gpuobj, 1, (uint32_t) size-1);
 727
 728
 729                         /*write starting at the third dword*/
 730                         instance_offset = 2;
 731
 732                         /*for each PAGE, get its bus address, fill in the page table entry, and advance*/
 733                         for (i = 0; i < page_count; i++) {
 734                                 if (dev->sg->busaddr[idx] == 0) {
 735                                         dev->sg->busaddr[idx] =
 736                                                 pci_map_page(dev->pdev,
 737                                                              dev->sg->pagelist[idx],
 738                                                              0,
 739                                                              PAGE_SIZE,
 740                                                              DMA_BIDIRECTIONAL);
 741
 742 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,27))
 743                                         /* Not a 100% sure this is the right kdev in all cases. */
 744                                         if (dma_mapping_error(&dev->primary->kdev, dev->sg->busaddr[idx])) {
 745 #else
 746                                         if (dma_mapping_error(dev->sg->busaddr[idx])) {
 747 #endif
 748                                                 return -ENOMEM;
 749                                         }
 750                                 }
 751
 752                                 frame = (uint32_t) dev->sg->busaddr[idx];
 753                                 INSTANCE_WR(*gpuobj, instance_offset,
 754                                             frame | pte_flags);
 755
 756                                 idx++;
 757                                 instance_offset ++;
 758                         }
 759                         }
 760         } else {
 761                 uint32_t flags0, flags5;
 762
 763                 if (target == NV_DMA_TARGET_VIDMEM) {
 764                         flags0 = 0x00190000;
 765                         flags5 = 0x00010000;
 766                 } else {
 767                         flags0 = 0x7fc00000;
 768                         flags5 = 0x00080000;
 769                 }
 770
 771                 INSTANCE_WR(*gpuobj, 0, flags0 | class);
 772                 INSTANCE_WR(*gpuobj, 1, offset + size - 1);
 773                 INSTANCE_WR(*gpuobj, 2, offset);
 774                 INSTANCE_WR(*gpuobj, 5, flags5);
 775         }
 776
 777         (*gpuobj)->engine = NVOBJ_ENGINE_SW;
 778         (*gpuobj)->class  = class;
 779         return 0;
 780 }
 781
 782 int
 783 nouveau_gpuobj_gart_dma_new(struct nouveau_channel *chan,
 784                             uint64_t offset, uint64_t size, int access,
 785                             struct nouveau_gpuobj **gpuobj,
 786                             uint32_t *o_ret)
 787 {
 788         struct drm_device *dev = chan->dev;
 789         struct drm_nouveau_private *dev_priv = dev->dev_private;
 790         int ret;
 791
 792         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP ||
 793             (dev_priv->card_type >= NV_50 &&
 794              dev_priv->gart_info.type == NOUVEAU_GART_SGDMA)) {
 795                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 796                                              offset, size, access,
 797                                              NV_DMA_TARGET_AGP, gpuobj);
 798                 if (o_ret)
 799                         *o_ret = 0;
 800         } else
 801         if (dev_priv->gart_info.type == NOUVEAU_GART_SGDMA) {
 802                 *gpuobj = dev_priv->gart_info.sg_ctxdma;
 803                 if (offset & ~0xffffffffULL) {
 804                         DRM_ERROR("obj offset exceeds 32-bits\n");
 805                         return -EINVAL;
 806                 }
 807                 if (o_ret)
 808                         *o_ret = (uint32_t)offset;
 809                 ret = (*gpuobj != NULL) ? 0 : -EINVAL;
 810         } else {
 811                 DRM_ERROR("Invalid GART type %d\n", dev_priv->gart_info.type);
 812                 return -EINVAL;
 813         }
 814
 815         return ret;
 816 }
 817
 818 /* Context objects in the instance RAM have the following structure.
 819  * On NV40 they are 32 byte long, on NV30 and smaller 16 bytes.
 820
 821    NV4 - NV30:
 822
 823    entry[0]
 824    11:0 class
 825    12   chroma key enable
 826    13   user clip enable
 827    14   swizzle enable
 828    17:15 patch config:
 829        scrcopy_and, rop_and, blend_and, scrcopy, srccopy_pre, blend_pre
 830    18   synchronize enable
 831    19   endian: 1 big, 0 little
 832    21:20 dither mode
 833    23    single step enable
 834    24    patch status: 0 invalid, 1 valid
 835    25    context_surface 0: 1 valid
 836    26    context surface 1: 1 valid
 837    27    context pattern: 1 valid
 838    28    context rop: 1 valid
 839    29,30 context beta, beta4
 840    entry[1]
 841    7:0   mono format
 842    15:8  color format
 843    31:16 notify instance address
 844    entry[2]
 845    15:0  dma 0 instance address
 846    31:16 dma 1 instance address
 847    entry[3]
 848    dma method traps
 849
 850    NV40:
 851    No idea what the exact format is. Here's what can be deducted:
 852
 853    entry[0]:
 854    11:0  class  (maybe uses more bits here?)
 855    17    user clip enable
 856    21:19 patch config
 857    25    patch status valid ?
 858    entry[1]:
 859    15:0  DMA notifier  (maybe 20:0)
 860    entry[2]:
 861    15:0  DMA 0 instance (maybe 20:0)
 862    24    big endian
 863    entry[3]:
 864    15:0  DMA 1 instance (maybe 20:0)
 865    entry[4]:
 866    entry[5]:
 867    set to 0?
 868 */
 869 int
 870 nouveau_gpuobj_gr_new(struct nouveau_channel *chan, int class,
 871                       struct nouveau_gpuobj **gpuobj)
 872 {
 873         struct drm_device *dev = chan->dev;
 874         struct drm_nouveau_private *dev_priv = dev->dev_private;
 875         int ret;
 876
 877         DRM_DEBUG("ch%d class=0x%04x\n", chan->id, class);
 878
 879         ret = nouveau_gpuobj_new(dev, chan,
 880                                  nouveau_gpuobj_class_instmem_size(dev, class),
 881                                  16,
 882                                  NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
 883                                  gpuobj);
 884         if (ret) {
 885                 DRM_ERROR("Error creating gpuobj: %d\n", ret);
 886                 return ret;
 887         }
 888
 889         if (dev_priv->card_type >= NV_50) {
 890                 INSTANCE_WR(*gpuobj, 0, class);
 891                 INSTANCE_WR(*gpuobj, 5, 0x00010000);
 892         } else {
 893         switch (class) {
 894         case NV_CLASS_NULL:
 895                 INSTANCE_WR(*gpuobj, 0, 0x00001030);
 896                 INSTANCE_WR(*gpuobj, 1, 0xFFFFFFFF);
 897                 break;
 898         default:
 899                 if (dev_priv->card_type >= NV_40) {
 900                         INSTANCE_WR(*gpuobj, 0, class);
 901 #ifdef __BIG_ENDIAN
 902                         INSTANCE_WR(*gpuobj, 2, 0x01000000);
 903 #endif
 904                 } else {
 905 #ifdef __BIG_ENDIAN
 906                         INSTANCE_WR(*gpuobj, 0, class | 0x00080000);
 907 #else
 908                         INSTANCE_WR(*gpuobj, 0, class);
 909 #endif
 910                 }
 911         }
 912         }
 913
 914         (*gpuobj)->engine = NVOBJ_ENGINE_GR;
 915         (*gpuobj)->class  = class;
 916         return 0;
 917 }
 918
 919 static int
 920 nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
 921 {
 922         struct drm_device *dev = chan->dev;
 923         struct drm_nouveau_private *dev_priv = dev->dev_private;
 924         struct nouveau_gpuobj *pramin = NULL;
 925         int size, base, ret;
 926
 927         DRM_DEBUG("ch%d\n", chan->id);
 928
 929         /* Base amount for object storage (4KiB enough?) */
 930         size = 0x1000;
 931         base = 0;
 932
 933         /* PGRAPH context */
 934
 935         if (dev_priv->card_type == NV_50) {
 936                 /* Various fixed table thingos */
 937                 size += 0x1400; /* mostly unknown stuff */
 938                 size += 0x4000; /* vm pd */
 939                 base  = 0x6000;
 940                 /* RAMHT, not sure about setting size yet, 32KiB to be safe */
 941                 size += 0x8000;
 942                 /* RAMFC */
 943                 size += 0x1000;
 944                 /* PGRAPH context */
 945                 size += 0x70000;
 946         }
 947
 948         DRM_DEBUG("ch%d PRAMIN size: 0x%08x bytes, base alloc=0x%08x\n",
 949                   chan->id, size, base);
 950         ret = nouveau_gpuobj_new_ref(dev, NULL, NULL, 0, size, 0x1000, 0,
 951                                      &chan->ramin);
 952         if (ret) {
 953                 DRM_ERROR("Error allocating channel PRAMIN: %d\n", ret);
 954                 return ret;
 955         }
 956         pramin = chan->ramin->gpuobj;
 957
 958         ret = nouveau_mem_init_heap(&chan->ramin_heap,
 959                                     pramin->im_pramin->start + base, size);
 960         if (ret) {
 961                 DRM_ERROR("Error creating PRAMIN heap: %d\n", ret);
 962                 nouveau_gpuobj_ref_del(dev, &chan->ramin);
 963                 return ret;
 964         }
 965
 966         return 0;
 967 }
 968
 969 int
 970 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 971                             uint32_t vram_h, uint32_t tt_h)
 972 {
 973         struct drm_device *dev = chan->dev;
 974         struct drm_nouveau_private *dev_priv = dev->dev_private;
 975         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
 976         int ret, i;
 977
 978         INIT_LIST_HEAD(&chan->ramht_refs);
 979
 980         DRM_DEBUG("ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
 981
 982         /* Reserve a block of PRAMIN for the channel
 983          *XXX: maybe on <NV50 too at some point
 984          */
 985         if (0 || dev_priv->card_type == NV_50) {
 986                 ret = nouveau_gpuobj_channel_init_pramin(chan);
 987                 if (ret)
 988                         return ret;
 989         }
 990
 991         /* NV50 VM
 992          *  - Allocate per-channel page-directory
 993          *  - Point offset 0-512MiB at shared PCIEGART table
 994          *  - Point offset 512-1024MiB at shared VRAM table
 995          */
 996         if (dev_priv->card_type >= NV_50) {
 997                 uint32_t vm_offset;
 998
 999                 vm_offset = (dev_priv->chipset & 0xf0) == 0x50 ? 0x1400 : 0x200;
1000                 vm_offset += chan->ramin->gpuobj->im_pramin->start;
1001                 if ((ret = nouveau_gpuobj_new_fake(dev, vm_offset, ~0, 0x4000,
1002                                                    0, &chan->vm_pd, NULL)))
1003                         return ret;
1004                 for (i=0; i<0x4000; i+=8) {
1005                         INSTANCE_WR(chan->vm_pd, (i+0)/4, 0x00000000);
1006                         INSTANCE_WR(chan->vm_pd, (i+4)/4, 0xdeadcafe);
1007                 }
1008
1009                 if ((ret = nouveau_gpuobj_ref_add(dev, NULL, 0,
1010                                                   dev_priv->gart_info.sg_ctxdma,
1011                                                   &chan->vm_gart_pt)))
1012                         return ret;
1013                 INSTANCE_WR(chan->vm_pd, (0+0)/4,
1014                             chan->vm_gart_pt->instance | 0x03);
1015                 INSTANCE_WR(chan->vm_pd, (0+4)/4, 0x00000000);
1016
1017                 if ((ret = nouveau_gpuobj_ref_add(dev, NULL, 0,
1018                                                   dev_priv->vm_vram_pt,
1019                                                   &chan->vm_vram_pt)))
1020                         return ret;
1021                 INSTANCE_WR(chan->vm_pd, (8+0)/4,
1022                             chan->vm_vram_pt->instance | 0x61);
1023                 INSTANCE_WR(chan->vm_pd, (8+4)/4, 0x00000000);
1024         }
1025
1026         /* RAMHT */
1027         if (dev_priv->card_type < NV_50) {
1028                 ret = nouveau_gpuobj_ref_add(dev, NULL, 0, dev_priv->ramht,
1029                                              &chan->ramht);
1030                 if (ret)
1031                         return ret;
1032         } else {
1033                 ret = nouveau_gpuobj_new_ref(dev, chan, chan, 0,
1034                                              0x8000, 16,
1035                                              NVOBJ_FLAG_ZERO_ALLOC,
1036                                              &chan->ramht);
1037                 if (ret)
1038                         return ret;
1039         }
1040
1041         /* VRAM ctxdma */
1042         if (dev_priv->card_type >= NV_50) {
1043                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
1044                                              0, 0x100000000ULL,
1045                                              NV_DMA_ACCESS_RW,
1046                                              NV_DMA_TARGET_AGP, &vram);
1047                 if (ret) {
1048                         DRM_ERROR("Error creating VRAM ctxdma: %d\n", ret);
1049                         return ret;
1050                 }
1051         } else
1052         if ((ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
1053                                           0, dev_priv->fb_available_size,
1054                                           NV_DMA_ACCESS_RW,
1055                                           NV_DMA_TARGET_VIDMEM, &vram))) {
1056                 DRM_ERROR("Error creating VRAM ctxdma: %d\n", ret);
1057                 return ret;
1058         }
1059
1060         if ((ret = nouveau_gpuobj_ref_add(dev, chan, vram_h, vram, NULL))) {
1061                 DRM_ERROR("Error referencing VRAM ctxdma: %d\n", ret);
1062                 return ret;
1063         }
1064
1065         /* TT memory ctxdma */
1066         if (dev_priv->card_type >= NV_50) {
1067                 tt = vram;
1068         } else
1069         if (dev_priv->gart_info.type != NOUVEAU_GART_NONE) {
1070                 ret = nouveau_gpuobj_gart_dma_new(chan, 0,
1071                                                   dev_priv->gart_info.aper_size,
1072                                                   NV_DMA_ACCESS_RW, &tt, NULL);
1073         } else
1074         if (dev_priv->pci_heap) {
1075                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
1076                                              0, dev->sg->pages * PAGE_SIZE,
1077                                              NV_DMA_ACCESS_RW,
1078                                              NV_DMA_TARGET_PCI_NONLINEAR, &tt);
1079         } else {
1080                 DRM_ERROR("Invalid GART type %d\n", dev_priv->gart_info.type);
1081                 ret = -EINVAL;
1082         }
1083
1084         if (ret) {
1085                 DRM_ERROR("Error creating TT ctxdma: %d\n", ret);
1086                 return ret;
1087         }
1088
1089         ret = nouveau_gpuobj_ref_add(dev, chan, tt_h, tt, NULL);
1090         if (ret) {
1091                 DRM_ERROR("Error referencing TT ctxdma: %d\n", ret);
1092                 return ret;
1093         }
1094
1095         return 0;
1096 }
1097
1098 void
1099 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
1100 {
1101         struct drm_device *dev = chan->dev;
1102         struct list_head *entry, *tmp;
1103         struct nouveau_gpuobj_ref *ref;
1104
1105         DRM_DEBUG("ch%d\n", chan->id);
1106
1107         list_for_each_safe(entry, tmp, &chan->ramht_refs) {
1108                 ref = list_entry(entry, struct nouveau_gpuobj_ref, list);
1109
1110                 nouveau_gpuobj_ref_del(dev, &ref);
1111         }
1112
1113         nouveau_gpuobj_ref_del(dev, &chan->ramht);
1114
1115         nouveau_gpuobj_del(dev, &chan->vm_pd);
1116         nouveau_gpuobj_ref_del(dev, &chan->vm_gart_pt);
1117         nouveau_gpuobj_ref_del(dev, &chan->vm_vram_pt);
1118
1119         if (chan->ramin_heap)
1120                 nouveau_mem_takedown(&chan->ramin_heap);
1121         if (chan->ramin)
1122                 nouveau_gpuobj_ref_del(dev, &chan->ramin);
1123
1124 }
1125
1126 int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
1127                               struct drm_file *file_priv)
1128 {
1129         struct nouveau_channel *chan;
1130         struct drm_nouveau_grobj_alloc *init = data;
1131         struct nouveau_gpuobj *gr = NULL;
1132         int ret;
1133
1134         NOUVEAU_CHECK_INITIALISED_WITH_RETURN;
1135         NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(init->channel, file_priv, chan);
1136
1137         //FIXME: check args, only allow trusted objects to be created
1138
1139         if (init->handle == ~0)
1140                 return -EINVAL;
1141
1142         if (nouveau_gpuobj_ref_find(chan, init->handle, NULL) == 0)
1143                 return -EEXIST;
1144
1145         ret = nouveau_gpuobj_gr_new(chan, init->class, &gr);
1146         if (ret) {
1147                 DRM_ERROR("Error creating gr object: %d (%d/0x%08x)\n",
1148                           ret, init->channel, init->handle);
1149                 return ret;
1150         }
1151
1152         if ((ret = nouveau_gpuobj_ref_add(dev, chan, init->handle, gr, NULL))) {
1153                 DRM_ERROR("Error referencing gr object: %d (%d/0x%08x\n)",
1154                           ret, init->channel, init->handle);
1155                 nouveau_gpuobj_del(dev, &gr);
1156                 return ret;
1157         }
1158
1159         return 0;
1160 }
1161
1162 int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
1163                               struct drm_file *file_priv)
1164 {
1165         struct drm_nouveau_gpuobj_free *objfree = data;
1166         struct nouveau_gpuobj_ref *ref;
1167         struct nouveau_channel *chan;
1168         int ret;
1169
1170         NOUVEAU_CHECK_INITIALISED_WITH_RETURN;
1171         NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(objfree->channel, file_priv, chan);
1172
1173         if ((ret = nouveau_gpuobj_ref_find(chan, objfree->handle, &ref)))
1174                 return ret;
1175         nouveau_gpuobj_ref_del(dev, &ref);
1176
1177         return 0;
1178 }