drivers/gpu/drm/nouveau/nouveau_bo.c

   1 /*
   2  * Copyright 2007 Dave Airlied
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24 /*
  25  * Authors: Dave Airlied <airlied@linux.ie>
  26  *          Ben Skeggs   <darktama@iinet.net.au>
  27  *          Jeremy Kolb  <jkolb@brandeis.edu>
  28  */
  29
  30 #include "drmP.h"
  31
  32 #include "nouveau_drm.h"
  33 #include "nouveau_drv.h"
  34 #include "nouveau_dma.h"
  35
  36 #include <linux/log2.h>
  37 #include <linux/slab.h>
  38
  39 static void
  40 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
  41 {
  42         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
  43         struct drm_device *dev = dev_priv->dev;
  44         struct nouveau_bo *nvbo = nouveau_bo(bo);
  45
  46         if (unlikely(nvbo->gem))
  47                 DRM_ERROR("bo %p still attached to GEM object\n", bo);
  48
  49         if (nvbo->tile)
  50                 nv10_mem_expire_tiling(dev, nvbo->tile, NULL);
  51
  52         kfree(nvbo);
  53 }
  54
  55 static void
  56 nouveau_bo_fixup_align(struct drm_device *dev,
  57                        uint32_t tile_mode, uint32_t tile_flags,
  58                        int *align, int *size)
  59 {
  60         struct drm_nouveau_private *dev_priv = dev->dev_private;
  61
  62         /*
  63          * Some of the tile_flags have a periodic structure of N*4096 bytes,
  64          * align to to that as well as the page size. Align the size to the
  65          * appropriate boundaries. This does imply that sizes are rounded up
  66          * 3-7 pages, so be aware of this and do not waste memory by allocating
  67          * many small buffers.
  68          */
  69         if (dev_priv->card_type == NV_50) {
  70                 uint32_t block_size = dev_priv->vram_size >> 15;
  71                 int i;
  72
  73                 switch (tile_flags) {
  74                 case 0x1800:
  75                 case 0x2800:
  76                 case 0x4800:
  77                 case 0x7a00:
  78                         if (is_power_of_2(block_size)) {
  79                                 for (i = 1; i < 10; i++) {
  80                                         *align = 12 * i * block_size;
  81                                         if (!(*align % 65536))
  82                                                 break;
  83                                 }
  84                         } else {
  85                                 for (i = 1; i < 10; i++) {
  86                                         *align = 8 * i * block_size;
  87                                         if (!(*align % 65536))
  88                                                 break;
  89                                 }
  90                         }
  91                         *size = roundup(*size, *align);
  92                         break;
  93                 default:
  94                         break;
  95                 }
  96
  97         } else {
  98                 if (tile_mode) {
  99                         if (dev_priv->chipset >= 0x40) {
 100                                 *align = 65536;
 101                                 *size = roundup(*size, 64 * tile_mode);
 102
 103                         } else if (dev_priv->chipset >= 0x30) {
 104                                 *align = 32768;
 105                                 *size = roundup(*size, 64 * tile_mode);
 106
 107                         } else if (dev_priv->chipset >= 0x20) {
 108                                 *align = 16384;
 109                                 *size = roundup(*size, 64 * tile_mode);
 110
 111                         } else if (dev_priv->chipset >= 0x10) {
 112                                 *align = 16384;
 113                                 *size = roundup(*size, 32 * tile_mode);
 114                         }
 115                 }
 116         }
 117
 118         /* ALIGN works only on powers of two. */
 119         *size = roundup(*size, PAGE_SIZE);
 120
 121         if (dev_priv->card_type == NV_50) {
 122                 *size = roundup(*size, 65536);
 123                 *align = max(65536, *align);
 124         }
 125 }
 126
 127 int
 128 nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
 129                int size, int align, uint32_t flags, uint32_t tile_mode,
 130                uint32_t tile_flags, bool no_vm, bool mappable,
 131                struct nouveau_bo **pnvbo)
 132 {
 133         struct drm_nouveau_private *dev_priv = dev->dev_private;
 134         struct nouveau_bo *nvbo;
 135         int ret = 0;
 136
 137         nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
 138         if (!nvbo)
 139                 return -ENOMEM;
 140         INIT_LIST_HEAD(&nvbo->head);
 141         INIT_LIST_HEAD(&nvbo->entry);
 142         nvbo->mappable = mappable;
 143         nvbo->no_vm = no_vm;
 144         nvbo->tile_mode = tile_mode;
 145         nvbo->tile_flags = tile_flags;
 146
 147         nouveau_bo_fixup_align(dev, tile_mode, nouveau_bo_tile_layout(nvbo),
 148                                &align, &size);
 149         align >>= PAGE_SHIFT;
 150
 151         nouveau_bo_placement_set(nvbo, flags, 0);
 152
 153         nvbo->channel = chan;
 154         ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size,
 155                           ttm_bo_type_device, &nvbo->placement, align, 0,
 156                           false, NULL, size, nouveau_bo_del_ttm);
 157         if (ret) {
 158                 /* ttm will call nouveau_bo_del_ttm if it fails.. */
 159                 return ret;
 160         }
 161         nvbo->channel = NULL;
 162
 163         *pnvbo = nvbo;
 164         return 0;
 165 }
 166
 167 static void
 168 set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags)
 169 {
 170         *n = 0;
 171
 172         if (type & TTM_PL_FLAG_VRAM)
 173                 pl[(*n)++] = TTM_PL_FLAG_VRAM | flags;
 174         if (type & TTM_PL_FLAG_TT)
 175                 pl[(*n)++] = TTM_PL_FLAG_TT | flags;
 176         if (type & TTM_PL_FLAG_SYSTEM)
 177                 pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags;
 178 }
 179
 180 void
 181 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
 182 {
 183         struct ttm_placement *pl = &nvbo->placement;
 184         uint32_t flags = TTM_PL_MASK_CACHING |
 185                 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
 186
 187         pl->placement = nvbo->placements;
 188         set_placement_list(nvbo->placements, &pl->num_placement,
 189                            type, flags);
 190
 191         pl->busy_placement = nvbo->busy_placements;
 192         set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
 193                            type | busy, flags);
 194 }
 195
 196 int
 197 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
 198 {
 199         struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 200         struct ttm_buffer_object *bo = &nvbo->bo;
 201         int ret;
 202
 203         if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) {
 204                 NV_ERROR(nouveau_bdev(bo->bdev)->dev,
 205                          "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
 206                          1 << bo->mem.mem_type, memtype);
 207                 return -EINVAL;
 208         }
 209
 210         if (nvbo->pin_refcnt++)
 211                 return 0;
 212
 213         ret = ttm_bo_reserve(bo, false, false, false, 0);
 214         if (ret)
 215                 goto out;
 216
 217         nouveau_bo_placement_set(nvbo, memtype, 0);
 218
 219         ret = ttm_bo_validate(bo, &nvbo->placement, false, false, false);
 220         if (ret == 0) {
 221                 switch (bo->mem.mem_type) {
 222                 case TTM_PL_VRAM:
 223                         dev_priv->fb_aper_free -= bo->mem.size;
 224                         break;
 225                 case TTM_PL_TT:
 226                         dev_priv->gart_info.aper_free -= bo->mem.size;
 227                         break;
 228                 default:
 229                         break;
 230                 }
 231         }
 232         ttm_bo_unreserve(bo);
 233 out:
 234         if (unlikely(ret))
 235                 nvbo->pin_refcnt--;
 236         return ret;
 237 }
 238
 239 int
 240 nouveau_bo_unpin(struct nouveau_bo *nvbo)
 241 {
 242         struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 243         struct ttm_buffer_object *bo = &nvbo->bo;
 244         int ret;
 245
 246         if (--nvbo->pin_refcnt)
 247                 return 0;
 248
 249         ret = ttm_bo_reserve(bo, false, false, false, 0);
 250         if (ret)
 251                 return ret;
 252
 253         nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
 254
 255         ret = ttm_bo_validate(bo, &nvbo->placement, false, false, false);
 256         if (ret == 0) {
 257                 switch (bo->mem.mem_type) {
 258                 case TTM_PL_VRAM:
 259                         dev_priv->fb_aper_free += bo->mem.size;
 260                         break;
 261                 case TTM_PL_TT:
 262                         dev_priv->gart_info.aper_free += bo->mem.size;
 263                         break;
 264                 default:
 265                         break;
 266                 }
 267         }
 268
 269         ttm_bo_unreserve(bo);
 270         return ret;
 271 }
 272
 273 int
 274 nouveau_bo_map(struct nouveau_bo *nvbo)
 275 {
 276         int ret;
 277
 278         ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
 279         if (ret)
 280                 return ret;
 281
 282         ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
 283         ttm_bo_unreserve(&nvbo->bo);
 284         return ret;
 285 }
 286
 287 void
 288 nouveau_bo_unmap(struct nouveau_bo *nvbo)
 289 {
 290         if (nvbo)
 291                 ttm_bo_kunmap(&nvbo->kmap);
 292 }
 293
 294 u16
 295 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
 296 {
 297         bool is_iomem;
 298         u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 299         mem = &mem[index];
 300         if (is_iomem)
 301                 return ioread16_native((void __force __iomem *)mem);
 302         else
 303                 return *mem;
 304 }
 305
 306 void
 307 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
 308 {
 309         bool is_iomem;
 310         u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 311         mem = &mem[index];
 312         if (is_iomem)
 313                 iowrite16_native(val, (void __force __iomem *)mem);
 314         else
 315                 *mem = val;
 316 }
 317
 318 u32
 319 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
 320 {
 321         bool is_iomem;
 322         u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 323         mem = &mem[index];
 324         if (is_iomem)
 325                 return ioread32_native((void __force __iomem *)mem);
 326         else
 327                 return *mem;
 328 }
 329
 330 void
 331 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
 332 {
 333         bool is_iomem;
 334         u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 335         mem = &mem[index];
 336         if (is_iomem)
 337                 iowrite32_native(val, (void __force __iomem *)mem);
 338         else
 339                 *mem = val;
 340 }
 341
 342 static struct ttm_backend *
 343 nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev)
 344 {
 345         struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
 346         struct drm_device *dev = dev_priv->dev;
 347
 348         switch (dev_priv->gart_info.type) {
 349 #if __OS_HAS_AGP
 350         case NOUVEAU_GART_AGP:
 351                 return ttm_agp_backend_init(bdev, dev->agp->bridge);
 352 #endif
 353         case NOUVEAU_GART_SGDMA:
 354                 return nouveau_sgdma_init_ttm(dev);
 355         default:
 356                 NV_ERROR(dev, "Unknown GART type %d\n",
 357                          dev_priv->gart_info.type);
 358                 break;
 359         }
 360
 361         return NULL;
 362 }
 363
 364 static int
 365 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
 366 {
 367         /* We'll do this from user space. */
 368         return 0;
 369 }
 370
 371 static int
 372 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 373                          struct ttm_mem_type_manager *man)
 374 {
 375         struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
 376         struct drm_device *dev = dev_priv->dev;
 377
 378         switch (type) {
 379         case TTM_PL_SYSTEM:
 380                 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
 381                 man->available_caching = TTM_PL_MASK_CACHING;
 382                 man->default_caching = TTM_PL_FLAG_CACHED;
 383                 break;
 384         case TTM_PL_VRAM:
 385                 man->func = &ttm_bo_manager_func;
 386                 man->flags = TTM_MEMTYPE_FLAG_FIXED |
 387                              TTM_MEMTYPE_FLAG_MAPPABLE;
 388                 man->available_caching = TTM_PL_FLAG_UNCACHED |
 389                                          TTM_PL_FLAG_WC;
 390                 man->default_caching = TTM_PL_FLAG_WC;
 391                 if (dev_priv->card_type == NV_50)
 392                         man->gpu_offset = 0x40000000;
 393                 else
 394                         man->gpu_offset = 0;
 395                 break;
 396         case TTM_PL_TT:
 397                 man->func = &ttm_bo_manager_func;
 398                 switch (dev_priv->gart_info.type) {
 399                 case NOUVEAU_GART_AGP:
 400                         man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
 401                         man->available_caching = TTM_PL_FLAG_UNCACHED;
 402                         man->default_caching = TTM_PL_FLAG_UNCACHED;
 403                         break;
 404                 case NOUVEAU_GART_SGDMA:
 405                         man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
 406                                      TTM_MEMTYPE_FLAG_CMA;
 407                         man->available_caching = TTM_PL_MASK_CACHING;
 408                         man->default_caching = TTM_PL_FLAG_CACHED;
 409                         break;
 410                 default:
 411                         NV_ERROR(dev, "Unknown GART type: %d\n",
 412                                  dev_priv->gart_info.type);
 413                         return -EINVAL;
 414                 }
 415                 man->gpu_offset = dev_priv->vm_gart_base;
 416                 break;
 417         default:
 418                 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type);
 419                 return -EINVAL;
 420         }
 421         return 0;
 422 }
 423
 424 static void
 425 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
 426 {
 427         struct nouveau_bo *nvbo = nouveau_bo(bo);
 428
 429         switch (bo->mem.mem_type) {
 430         case TTM_PL_VRAM:
 431                 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
 432                                          TTM_PL_FLAG_SYSTEM);
 433                 break;
 434         default:
 435                 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
 436                 break;
 437         }
 438
 439         *pl = nvbo->placement;
 440 }
 441
 442
 443 /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access
 444  * TTM_PL_{VRAM,TT} directly.
 445  */
 446
 447 static int
 448 nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan,
 449                               struct nouveau_bo *nvbo, bool evict,
 450                               bool no_wait_reserve, bool no_wait_gpu,
 451                               struct ttm_mem_reg *new_mem)
 452 {
 453         struct nouveau_fence *fence = NULL;
 454         int ret;
 455
 456         ret = nouveau_fence_new(chan, &fence, true);
 457         if (ret)
 458                 return ret;
 459
 460         if (nvbo->channel) {
 461                 ret = nouveau_fence_sync(fence, nvbo->channel);
 462                 if (ret)
 463                         goto out;
 464         }
 465
 466         ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict,
 467                                         no_wait_reserve, no_wait_gpu, new_mem);
 468 out:
 469         nouveau_fence_unref((void *)&fence);
 470         return ret;
 471 }
 472
 473 static inline uint32_t
 474 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
 475                       struct nouveau_channel *chan, struct ttm_mem_reg *mem)
 476 {
 477         struct nouveau_bo *nvbo = nouveau_bo(bo);
 478
 479         if (nvbo->no_vm) {
 480                 if (mem->mem_type == TTM_PL_TT)
 481                         return NvDmaGART;
 482                 return NvDmaVRAM;
 483         }
 484
 485         if (mem->mem_type == TTM_PL_TT)
 486                 return chan->gart_handle;
 487         return chan->vram_handle;
 488 }
 489
 490 static int
 491 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 492                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 493 {
 494         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 495         struct nouveau_bo *nvbo = nouveau_bo(bo);
 496         u64 length = (new_mem->num_pages << PAGE_SHIFT);
 497         u64 src_offset, dst_offset;
 498         int ret;
 499
 500         src_offset = old_mem->start << PAGE_SHIFT;
 501         dst_offset = new_mem->start << PAGE_SHIFT;
 502         if (!nvbo->no_vm) {
 503                 if (old_mem->mem_type == TTM_PL_VRAM)
 504                         src_offset += dev_priv->vm_vram_base;
 505                 else
 506                         src_offset += dev_priv->vm_gart_base;
 507
 508                 if (new_mem->mem_type == TTM_PL_VRAM)
 509                         dst_offset += dev_priv->vm_vram_base;
 510                 else
 511                         dst_offset += dev_priv->vm_gart_base;
 512         }
 513
 514         ret = RING_SPACE(chan, 3);
 515         if (ret)
 516                 return ret;
 517
 518         BEGIN_RING(chan, NvSubM2MF, 0x0184, 2);
 519         OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
 520         OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
 521
 522         while (length) {
 523                 u32 amount, stride, height;
 524
 525                 amount  = min(length, (u64)(4 * 1024 * 1024));
 526                 stride  = 16 * 4;
 527                 height  = amount / stride;
 528
 529                 if (new_mem->mem_type == TTM_PL_VRAM &&
 530                     nouveau_bo_tile_layout(nvbo)) {
 531                         ret = RING_SPACE(chan, 8);
 532                         if (ret)
 533                                 return ret;
 534
 535                         BEGIN_RING(chan, NvSubM2MF, 0x0200, 7);
 536                         OUT_RING  (chan, 0);
 537                         OUT_RING  (chan, 0);
 538                         OUT_RING  (chan, stride);
 539                         OUT_RING  (chan, height);
 540                         OUT_RING  (chan, 1);
 541                         OUT_RING  (chan, 0);
 542                         OUT_RING  (chan, 0);
 543                 } else {
 544                         ret = RING_SPACE(chan, 2);
 545                         if (ret)
 546                                 return ret;
 547
 548                         BEGIN_RING(chan, NvSubM2MF, 0x0200, 1);
 549                         OUT_RING  (chan, 1);
 550                 }
 551                 if (old_mem->mem_type == TTM_PL_VRAM &&
 552                     nouveau_bo_tile_layout(nvbo)) {
 553                         ret = RING_SPACE(chan, 8);
 554                         if (ret)
 555                                 return ret;
 556
 557                         BEGIN_RING(chan, NvSubM2MF, 0x021c, 7);
 558                         OUT_RING  (chan, 0);
 559                         OUT_RING  (chan, 0);
 560                         OUT_RING  (chan, stride);
 561                         OUT_RING  (chan, height);
 562                         OUT_RING  (chan, 1);
 563                         OUT_RING  (chan, 0);
 564                         OUT_RING  (chan, 0);
 565                 } else {
 566                         ret = RING_SPACE(chan, 2);
 567                         if (ret)
 568                                 return ret;
 569
 570                         BEGIN_RING(chan, NvSubM2MF, 0x021c, 1);
 571                         OUT_RING  (chan, 1);
 572                 }
 573
 574                 ret = RING_SPACE(chan, 14);
 575                 if (ret)
 576                         return ret;
 577
 578                 BEGIN_RING(chan, NvSubM2MF, 0x0238, 2);
 579                 OUT_RING  (chan, upper_32_bits(src_offset));
 580                 OUT_RING  (chan, upper_32_bits(dst_offset));
 581                 BEGIN_RING(chan, NvSubM2MF, 0x030c, 8);
 582                 OUT_RING  (chan, lower_32_bits(src_offset));
 583                 OUT_RING  (chan, lower_32_bits(dst_offset));
 584                 OUT_RING  (chan, stride);
 585                 OUT_RING  (chan, stride);
 586                 OUT_RING  (chan, stride);
 587                 OUT_RING  (chan, height);
 588                 OUT_RING  (chan, 0x00000101);
 589                 OUT_RING  (chan, 0x00000000);
 590                 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
 591                 OUT_RING  (chan, 0);
 592
 593                 length -= amount;
 594                 src_offset += amount;
 595                 dst_offset += amount;
 596         }
 597
 598         return 0;
 599 }
 600
 601 static int
 602 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 603                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 604 {
 605         u32 src_offset = old_mem->start << PAGE_SHIFT;
 606         u32 dst_offset = new_mem->start << PAGE_SHIFT;
 607         u32 page_count = new_mem->num_pages;
 608         int ret;
 609
 610         ret = RING_SPACE(chan, 3);
 611         if (ret)
 612                 return ret;
 613
 614         BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
 615         OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
 616         OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
 617
 618         page_count = new_mem->num_pages;
 619         while (page_count) {
 620                 int line_count = (page_count > 2047) ? 2047 : page_count;
 621
 622                 ret = RING_SPACE(chan, 11);
 623                 if (ret)
 624                         return ret;
 625
 626                 BEGIN_RING(chan, NvSubM2MF,
 627                                  NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
 628                 OUT_RING  (chan, src_offset);
 629                 OUT_RING  (chan, dst_offset);
 630                 OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
 631                 OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
 632                 OUT_RING  (chan, PAGE_SIZE); /* line_length */
 633                 OUT_RING  (chan, line_count);
 634                 OUT_RING  (chan, 0x00000101);
 635                 OUT_RING  (chan, 0x00000000);
 636                 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
 637                 OUT_RING  (chan, 0);
 638
 639                 page_count -= line_count;
 640                 src_offset += (PAGE_SIZE * line_count);
 641                 dst_offset += (PAGE_SIZE * line_count);
 642         }
 643
 644         return 0;
 645 }
 646
 647 static int
 648 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
 649                      bool no_wait_reserve, bool no_wait_gpu,
 650                      struct ttm_mem_reg *new_mem)
 651 {
 652         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 653         struct nouveau_bo *nvbo = nouveau_bo(bo);
 654         struct nouveau_channel *chan;
 655         int ret;
 656
 657         chan = nvbo->channel;
 658         if (!chan || nvbo->no_vm)
 659                 chan = dev_priv->channel;
 660
 661         if (dev_priv->card_type < NV_50)
 662                 ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
 663         else
 664                 ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
 665         if (ret)
 666                 return ret;
 667
 668         return nouveau_bo_move_accel_cleanup(chan, nvbo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 669 }
 670
 671 static int
 672 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
 673                       bool no_wait_reserve, bool no_wait_gpu,
 674                       struct ttm_mem_reg *new_mem)
 675 {
 676         u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
 677         struct ttm_placement placement;
 678         struct ttm_mem_reg tmp_mem;
 679         int ret;
 680
 681         placement.fpfn = placement.lpfn = 0;
 682         placement.num_placement = placement.num_busy_placement = 1;
 683         placement.placement = placement.busy_placement = &placement_memtype;
 684
 685         tmp_mem = *new_mem;
 686         tmp_mem.mm_node = NULL;
 687         ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu);
 688         if (ret)
 689                 return ret;
 690
 691         ret = ttm_tt_bind(bo->ttm, &tmp_mem);
 692         if (ret)
 693                 goto out;
 694
 695         ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);
 696         if (ret)
 697                 goto out;
 698
 699         ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 700 out:
 701         ttm_bo_mem_put(bo, &tmp_mem);
 702         return ret;
 703 }
 704
 705 static int
 706 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
 707                       bool no_wait_reserve, bool no_wait_gpu,
 708                       struct ttm_mem_reg *new_mem)
 709 {
 710         u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
 711         struct ttm_placement placement;
 712         struct ttm_mem_reg tmp_mem;
 713         int ret;
 714
 715         placement.fpfn = placement.lpfn = 0;
 716         placement.num_placement = placement.num_busy_placement = 1;
 717         placement.placement = placement.busy_placement = &placement_memtype;
 718
 719         tmp_mem = *new_mem;
 720         tmp_mem.mm_node = NULL;
 721         ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu);
 722         if (ret)
 723                 return ret;
 724
 725         ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, &tmp_mem);
 726         if (ret)
 727                 goto out;
 728
 729         ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
 730         if (ret)
 731                 goto out;
 732
 733 out:
 734         ttm_bo_mem_put(bo, &tmp_mem);
 735         return ret;
 736 }
 737
 738 static int
 739 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
 740                    struct nouveau_tile_reg **new_tile)
 741 {
 742         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 743         struct drm_device *dev = dev_priv->dev;
 744         struct nouveau_bo *nvbo = nouveau_bo(bo);
 745         uint64_t offset;
 746         int ret;
 747
 748         if (nvbo->no_vm || new_mem->mem_type != TTM_PL_VRAM) {
 749                 /* Nothing to do. */
 750                 *new_tile = NULL;
 751                 return 0;
 752         }
 753
 754         offset = new_mem->start << PAGE_SHIFT;
 755
 756         if (dev_priv->card_type == NV_50) {
 757                 ret = nv50_mem_vm_bind_linear(dev,
 758                                               offset + dev_priv->vm_vram_base,
 759                                               new_mem->size,
 760                                               nouveau_bo_tile_layout(nvbo),
 761                                               offset);
 762                 if (ret)
 763                         return ret;
 764
 765         } else if (dev_priv->card_type >= NV_10) {
 766                 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size,
 767                                                 nvbo->tile_mode);
 768         }
 769
 770         return 0;
 771 }
 772
 773 static void
 774 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
 775                       struct nouveau_tile_reg *new_tile,
 776                       struct nouveau_tile_reg **old_tile)
 777 {
 778         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 779         struct drm_device *dev = dev_priv->dev;
 780
 781         if (dev_priv->card_type >= NV_10 &&
 782             dev_priv->card_type < NV_50) {
 783                 if (*old_tile)
 784                         nv10_mem_expire_tiling(dev, *old_tile, bo->sync_obj);
 785
 786                 *old_tile = new_tile;
 787         }
 788 }
 789
 790 static int
 791 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
 792                 bool no_wait_reserve, bool no_wait_gpu,
 793                 struct ttm_mem_reg *new_mem)
 794 {
 795         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 796         struct nouveau_bo *nvbo = nouveau_bo(bo);
 797         struct ttm_mem_reg *old_mem = &bo->mem;
 798         struct nouveau_tile_reg *new_tile = NULL;
 799         int ret = 0;
 800
 801         ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
 802         if (ret)
 803                 return ret;
 804
 805         /* Fake bo copy. */
 806         if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
 807                 BUG_ON(bo->mem.mm_node != NULL);
 808                 bo->mem = *new_mem;
 809                 new_mem->mm_node = NULL;
 810                 goto out;
 811         }
 812
 813         /* Software copy if the card isn't up and running yet. */
 814         if (!dev_priv->channel) {
 815                 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 816                 goto out;
 817         }
 818
 819         /* Hardware assisted copy. */
 820         if (new_mem->mem_type == TTM_PL_SYSTEM)
 821                 ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
 822         else if (old_mem->mem_type == TTM_PL_SYSTEM)
 823                 ret = nouveau_bo_move_flips(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
 824         else
 825                 ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
 826
 827         if (!ret)
 828                 goto out;
 829
 830         /* Fallback to software copy. */
 831         ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 832
 833 out:
 834         if (ret)
 835                 nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
 836         else
 837                 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
 838
 839         return ret;
 840 }
 841
 842 static int
 843 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 844 {
 845         return 0;
 846 }
 847
 848 static int
 849 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 850 {
 851         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 852         struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
 853         struct drm_device *dev = dev_priv->dev;
 854
 855         mem->bus.addr = NULL;
 856         mem->bus.offset = 0;
 857         mem->bus.size = mem->num_pages << PAGE_SHIFT;
 858         mem->bus.base = 0;
 859         mem->bus.is_iomem = false;
 860         if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
 861                 return -EINVAL;
 862         switch (mem->mem_type) {
 863         case TTM_PL_SYSTEM:
 864                 /* System memory */
 865                 return 0;
 866         case TTM_PL_TT:
 867 #if __OS_HAS_AGP
 868                 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
 869                         mem->bus.offset = mem->start << PAGE_SHIFT;
 870                         mem->bus.base = dev_priv->gart_info.aper_base;
 871                         mem->bus.is_iomem = true;
 872                 }
 873 #endif
 874                 break;
 875         case TTM_PL_VRAM:
 876                 mem->bus.offset = mem->start << PAGE_SHIFT;
 877                 mem->bus.base = pci_resource_start(dev->pdev, 1);
 878                 mem->bus.is_iomem = true;
 879                 break;
 880         default:
 881                 return -EINVAL;
 882         }
 883         return 0;
 884 }
 885
 886 static void
 887 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 888 {
 889 }
 890
 891 static int
 892 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
 893 {
 894         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 895         struct nouveau_bo *nvbo = nouveau_bo(bo);
 896
 897         /* as long as the bo isn't in vram, and isn't tiled, we've got
 898          * nothing to do here.
 899          */
 900         if (bo->mem.mem_type != TTM_PL_VRAM) {
 901                 if (dev_priv->card_type < NV_50 ||
 902                     !nouveau_bo_tile_layout(nvbo))
 903                         return 0;
 904         }
 905
 906         /* make sure bo is in mappable vram */
 907         if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages)
 908                 return 0;
 909
 910
 911         nvbo->placement.fpfn = 0;
 912         nvbo->placement.lpfn = dev_priv->fb_mappable_pages;
 913         nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0);
 914         return ttm_bo_validate(bo, &nvbo->placement, false, true, false);
 915 }
 916
 917 struct ttm_bo_driver nouveau_bo_driver = {
 918         .create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry,
 919         .invalidate_caches = nouveau_bo_invalidate_caches,
 920         .init_mem_type = nouveau_bo_init_mem_type,
 921         .evict_flags = nouveau_bo_evict_flags,
 922         .move = nouveau_bo_move,
 923         .verify_access = nouveau_bo_verify_access,
 924         .sync_obj_signaled = nouveau_fence_signalled,
 925         .sync_obj_wait = nouveau_fence_wait,
 926         .sync_obj_flush = nouveau_fence_flush,
 927         .sync_obj_unref = nouveau_fence_unref,
 928         .sync_obj_ref = nouveau_fence_ref,
 929         .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
 930         .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
 931         .io_mem_free = &nouveau_ttm_io_mem_free,
 932 };
 933