shared-core/i915_dma.c

   1 /* i915_dma.c -- DMA support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #include "drmP.h"
  30 #include "drm.h"
  31 #include "i915_drm.h"
  32 #include "i915_drv.h"
  33
  34 /* Really want an OS-independent resettable timer.  Would like to have
  35  * this loop run for (eg) 3 sec, but have the timer reset every time
  36  * the head pointer changes, so that EBUSY only happens if the ring
  37  * actually stalls for (eg) 3 seconds.
  38  */
  39 int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
  40 {
  41         drm_i915_private_t *dev_priv = dev->dev_private;
  42         drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
  43         u32 last_head = I915_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
  44         int i;
  45
  46         for (i = 0; i < 10000; i++) {
  47                 ring->head = I915_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
  48                 ring->space = ring->head - (ring->tail + 8);
  49                 if (ring->space < 0)
  50                         ring->space += ring->Size;
  51                 if (ring->space >= n)
  52                         return 0;
  53
  54                 if (ring->head != last_head)
  55                         i = 0;
  56
  57                 last_head = ring->head;
  58                 DRM_UDELAY(1);
  59         }
  60
  61         return -EBUSY;
  62 }
  63
  64 void i915_kernel_lost_context(struct drm_device * dev)
  65 {
  66         drm_i915_private_t *dev_priv = dev->dev_private;
  67         drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
  68
  69         ring->head = I915_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
  70         ring->tail = I915_READ(LP_RING + RING_TAIL) & TAIL_ADDR;
  71         ring->space = ring->head - (ring->tail + 8);
  72         if (ring->space < 0)
  73                 ring->space += ring->Size;
  74 }
  75
  76 static int i915_dma_cleanup(struct drm_device * dev)
  77 {
  78         drm_i915_private_t *dev_priv = dev->dev_private;
  79         /* Make sure interrupts are disabled here because the uninstall ioctl
  80          * may not have been called from userspace and after dev_private
  81          * is freed, it's too late.
  82          */
  83         if (dev->irq)
  84                 drm_irq_uninstall(dev);
  85
  86         if (dev_priv->ring.virtual_start) {
  87                 drm_core_ioremapfree(&dev_priv->ring.map, dev);
  88                 dev_priv->ring.virtual_start = 0;
  89                 dev_priv->ring.map.handle = 0;
  90                 dev_priv->ring.map.size = 0;
  91         }
  92
  93         if (dev_priv->status_page_dmah) {
  94                 drm_pci_free(dev, dev_priv->status_page_dmah);
  95                 dev_priv->status_page_dmah = NULL;
  96                 /* Need to rewrite hardware status page */
  97                 I915_WRITE(0x02080, 0x1ffff000);
  98         }
  99
 100         if (dev_priv->status_gfx_addr) {
 101                 dev_priv->status_gfx_addr = 0;
 102                 drm_core_ioremapfree(&dev_priv->hws_map, dev);
 103                 I915_WRITE(0x02080, 0x1ffff000);
 104         }
 105
 106         return 0;
 107 }
 108
 109 static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 110 {
 111         drm_i915_private_t *dev_priv = dev->dev_private;
 112
 113         dev_priv->sarea = drm_getsarea(dev);
 114         if (!dev_priv->sarea) {
 115                 DRM_ERROR("can not find sarea!\n");
 116                 i915_dma_cleanup(dev);
 117                 return -EINVAL;
 118         }
 119
 120         dev_priv->mmio_map = drm_core_findmap(dev, init->mmio_offset);
 121         if (!dev_priv->mmio_map) {
 122                 i915_dma_cleanup(dev);
 123                 DRM_ERROR("can not find mmio map!\n");
 124                 return -EINVAL;
 125         }
 126
 127 #ifdef I915_HAVE_BUFFER
 128         dev_priv->max_validate_buffers = I915_MAX_VALIDATE_BUFFERS;
 129 #endif
 130
 131         dev_priv->sarea_priv = (drm_i915_sarea_t *)
 132             ((u8 *) dev_priv->sarea->handle + init->sarea_priv_offset);
 133
 134         dev_priv->ring.Start = init->ring_start;
 135         dev_priv->ring.End = init->ring_end;
 136         dev_priv->ring.Size = init->ring_size;
 137         dev_priv->ring.tail_mask = dev_priv->ring.Size - 1;
 138
 139         dev_priv->ring.map.offset = init->ring_start;
 140         dev_priv->ring.map.size = init->ring_size;
 141         dev_priv->ring.map.type = 0;
 142         dev_priv->ring.map.flags = 0;
 143         dev_priv->ring.map.mtrr = 0;
 144
 145         drm_core_ioremap(&dev_priv->ring.map, dev);
 146
 147         if (dev_priv->ring.map.handle == NULL) {
 148                 i915_dma_cleanup(dev);
 149                 DRM_ERROR("can not ioremap virtual address for"
 150                           " ring buffer\n");
 151                 return -ENOMEM;
 152         }
 153
 154         dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
 155
 156         dev_priv->cpp = init->cpp;
 157         dev_priv->sarea_priv->pf_current_page = 0;
 158
 159         /* We are using separate values as placeholders for mechanisms for
 160          * private backbuffer/depthbuffer usage.
 161          */
 162         dev_priv->use_mi_batchbuffer_start = 0;
 163         if (IS_I965G(dev)) /* 965 doesn't support older method */
 164                 dev_priv->use_mi_batchbuffer_start = 1;
 165
 166         /* Allow hardware batchbuffers unless told otherwise.
 167          */
 168         dev_priv->allow_batchbuffer = 1;
 169
 170         /* Enable vblank on pipe A for older X servers
 171          */
 172         dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A;
 173
 174         /* Program Hardware Status Page */
 175         if (!IS_G33(dev)) {
 176                 dev_priv->status_page_dmah =
 177                         drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 0xffffffff);
 178
 179                 if (!dev_priv->status_page_dmah) {
 180                         i915_dma_cleanup(dev);
 181                         DRM_ERROR("Can not allocate hardware status page\n");
 182                         return -ENOMEM;
 183                 }
 184                 dev_priv->hw_status_page = dev_priv->status_page_dmah->vaddr;
 185                 dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
 186
 187                 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
 188
 189                 I915_WRITE(0x02080, dev_priv->dma_status_page);
 190         }
 191         DRM_DEBUG("Enabled hardware status page\n");
 192 #ifdef I915_HAVE_BUFFER
 193         mutex_init(&dev_priv->cmdbuf_mutex);
 194 #endif
 195         return 0;
 196 }
 197
 198 static int i915_dma_resume(struct drm_device * dev)
 199 {
 200         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 201
 202         DRM_DEBUG("\n");
 203
 204         if (!dev_priv->sarea) {
 205                 DRM_ERROR("can not find sarea!\n");
 206                 return -EINVAL;
 207         }
 208
 209         if (!dev_priv->mmio_map) {
 210                 DRM_ERROR("can not find mmio map!\n");
 211                 return -EINVAL;
 212         }
 213
 214         if (dev_priv->ring.map.handle == NULL) {
 215                 DRM_ERROR("can not ioremap virtual address for"
 216                           " ring buffer\n");
 217                 return -ENOMEM;
 218         }
 219
 220         /* Program Hardware Status Page */
 221         if (!dev_priv->hw_status_page) {
 222                 DRM_ERROR("Can not find hardware status page\n");
 223                 return -EINVAL;
 224         }
 225         DRM_DEBUG("hw status page @ %p\n", dev_priv->hw_status_page);
 226
 227         if (dev_priv->status_gfx_addr != 0)
 228                 I915_WRITE(0x02080, dev_priv->status_gfx_addr);
 229         else
 230                 I915_WRITE(0x02080, dev_priv->dma_status_page);
 231         DRM_DEBUG("Enabled hardware status page\n");
 232
 233         return 0;
 234 }
 235
 236 static int i915_dma_init(struct drm_device *dev, void *data,
 237                          struct drm_file *file_priv)
 238 {
 239         drm_i915_init_t *init = data;
 240         int retcode = 0;
 241
 242         switch (init->func) {
 243         case I915_INIT_DMA:
 244                 retcode = i915_initialize(dev, init);
 245                 break;
 246         case I915_CLEANUP_DMA:
 247                 retcode = i915_dma_cleanup(dev);
 248                 break;
 249         case I915_RESUME_DMA:
 250                 retcode = i915_dma_resume(dev);
 251                 break;
 252         default:
 253                 retcode = -EINVAL;
 254                 break;
 255         }
 256
 257         return retcode;
 258 }
 259
 260 /* Implement basically the same security restrictions as hardware does
 261  * for MI_BATCH_NON_SECURE.  These can be made stricter at any time.
 262  *
 263  * Most of the calculations below involve calculating the size of a
 264  * particular instruction.  It's important to get the size right as
 265  * that tells us where the next instruction to check is.  Any illegal
 266  * instruction detected will be given a size of zero, which is a
 267  * signal to abort the rest of the buffer.
 268  */
 269 static int do_validate_cmd(int cmd)
 270 {
 271         switch (((cmd >> 29) & 0x7)) {
 272         case 0x0:
 273                 switch ((cmd >> 23) & 0x3f) {
 274                 case 0x0:
 275                         return 1;       /* MI_NOOP */
 276                 case 0x4:
 277                         return 1;       /* MI_FLUSH */
 278                 default:
 279                         return 0;       /* disallow everything else */
 280                 }
 281                 break;
 282         case 0x1:
 283                 return 0;       /* reserved */
 284         case 0x2:
 285                 return (cmd & 0xff) + 2;        /* 2d commands */
 286         case 0x3:
 287                 if (((cmd >> 24) & 0x1f) <= 0x18)
 288                         return 1;
 289
 290                 switch ((cmd >> 24) & 0x1f) {
 291                 case 0x1c:
 292                         return 1;
 293                 case 0x1d:
 294                         switch ((cmd >> 16) & 0xff) {
 295                         case 0x3:
 296                                 return (cmd & 0x1f) + 2;
 297                         case 0x4:
 298                                 return (cmd & 0xf) + 2;
 299                         default:
 300                                 return (cmd & 0xffff) + 2;
 301                         }
 302                 case 0x1e:
 303                         if (cmd & (1 << 23))
 304                                 return (cmd & 0xffff) + 1;
 305                         else
 306                                 return 1;
 307                 case 0x1f:
 308                         if ((cmd & (1 << 23)) == 0)     /* inline vertices */
 309                                 return (cmd & 0x1ffff) + 2;
 310                         else if (cmd & (1 << 17))       /* indirect random */
 311                                 if ((cmd & 0xffff) == 0)
 312                                         return 0;       /* unknown length, too hard */
 313                                 else
 314                                         return (((cmd & 0xffff) + 1) / 2) + 1;
 315                         else
 316                                 return 2;       /* indirect sequential */
 317                 default:
 318                         return 0;
 319                 }
 320         default:
 321                 return 0;
 322         }
 323
 324         return 0;
 325 }
 326
 327 static int validate_cmd(int cmd)
 328 {
 329         int ret = do_validate_cmd(cmd);
 330
 331 /*      printk("validate_cmd( %x ): %d\n", cmd, ret); */
 332
 333         return ret;
 334 }
 335
 336 static int i915_emit_cmds(struct drm_device *dev, int __user *buffer,
 337                           int dwords)
 338 {
 339         drm_i915_private_t *dev_priv = dev->dev_private;
 340         int i;
 341         RING_LOCALS;
 342
 343         if ((dwords+1) * sizeof(int) >= dev_priv->ring.Size - 8)
 344                 return -EINVAL;
 345
 346         BEGIN_LP_RING((dwords+1)&~1);
 347
 348         for (i = 0; i < dwords;) {
 349                 int cmd, sz;
 350
 351                 if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd)))
 352                         return -EINVAL;
 353
 354                 if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
 355                         return -EINVAL;
 356
 357                 OUT_RING(cmd);
 358
 359                 while (++i, --sz) {
 360                         if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i],
 361                                                          sizeof(cmd))) {
 362                                 return -EINVAL;
 363                         }
 364                         OUT_RING(cmd);
 365                 }
 366         }
 367
 368         if (dwords & 1)
 369                 OUT_RING(0);
 370
 371         ADVANCE_LP_RING();
 372
 373         return 0;
 374 }
 375
 376 static int i915_emit_box(struct drm_device * dev,
 377                          struct drm_clip_rect __user * boxes,
 378                          int i, int DR1, int DR4)
 379 {
 380         drm_i915_private_t *dev_priv = dev->dev_private;
 381         struct drm_clip_rect box;
 382         RING_LOCALS;
 383
 384         if (DRM_COPY_FROM_USER_UNCHECKED(&box, &boxes[i], sizeof(box))) {
 385                 return -EFAULT;
 386         }
 387
 388         if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) {
 389                 DRM_ERROR("Bad box %d,%d..%d,%d\n",
 390                           box.x1, box.y1, box.x2, box.y2);
 391                 return -EINVAL;
 392         }
 393
 394         if (IS_I965G(dev)) {
 395                 BEGIN_LP_RING(4);
 396                 OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
 397                 OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
 398                 OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
 399                 OUT_RING(DR4);
 400                 ADVANCE_LP_RING();
 401         } else {
 402                 BEGIN_LP_RING(6);
 403                 OUT_RING(GFX_OP_DRAWRECT_INFO);
 404                 OUT_RING(DR1);
 405                 OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
 406                 OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
 407                 OUT_RING(DR4);
 408                 OUT_RING(0);
 409                 ADVANCE_LP_RING();
 410         }
 411
 412         return 0;
 413 }
 414
 415 /* XXX: Emitting the counter should really be moved to part of the IRQ
 416  * emit. For now, do it in both places:
 417  */
 418
 419 void i915_emit_breadcrumb(struct drm_device *dev)
 420 {
 421         drm_i915_private_t *dev_priv = dev->dev_private;
 422         RING_LOCALS;
 423
 424         if (++dev_priv->counter > BREADCRUMB_MASK) {
 425                  dev_priv->counter = 1;
 426                  DRM_DEBUG("Breadcrumb counter wrapped around\n");
 427         }
 428
 429         dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
 430
 431         BEGIN_LP_RING(4);
 432         OUT_RING(CMD_STORE_DWORD_IDX);
 433         OUT_RING(20);
 434         OUT_RING(dev_priv->counter);
 435         OUT_RING(0);
 436         ADVANCE_LP_RING();
 437 }
 438
 439
 440 int i915_emit_mi_flush(struct drm_device *dev, uint32_t flush)
 441 {
 442         drm_i915_private_t *dev_priv = dev->dev_private;
 443         uint32_t flush_cmd = CMD_MI_FLUSH;
 444         RING_LOCALS;
 445
 446         flush_cmd |= flush;
 447
 448         i915_kernel_lost_context(dev);
 449
 450         BEGIN_LP_RING(4);
 451         OUT_RING(flush_cmd);
 452         OUT_RING(0);
 453         OUT_RING(0);
 454         OUT_RING(0);
 455         ADVANCE_LP_RING();
 456
 457         return 0;
 458 }
 459
 460
 461 static int i915_dispatch_cmdbuffer(struct drm_device * dev,
 462                                    drm_i915_cmdbuffer_t * cmd)
 463 {
 464 #ifdef I915_HAVE_FENCE
 465         drm_i915_private_t *dev_priv = dev->dev_private;
 466 #endif
 467         int nbox = cmd->num_cliprects;
 468         int i = 0, count, ret;
 469
 470         if (cmd->sz & 0x3) {
 471                 DRM_ERROR("alignment\n");
 472                 return -EINVAL;
 473         }
 474
 475         i915_kernel_lost_context(dev);
 476
 477         count = nbox ? nbox : 1;
 478
 479         for (i = 0; i < count; i++) {
 480                 if (i < nbox) {
 481                         ret = i915_emit_box(dev, cmd->cliprects, i,
 482                                             cmd->DR1, cmd->DR4);
 483                         if (ret)
 484                                 return ret;
 485                 }
 486
 487                 ret = i915_emit_cmds(dev, (int __user *)cmd->buf, cmd->sz / 4);
 488                 if (ret)
 489                         return ret;
 490         }
 491
 492         i915_emit_breadcrumb(dev);
 493 #ifdef I915_HAVE_FENCE
 494         if (unlikely((dev_priv->counter & 0xFF) == 0))
 495                 drm_fence_flush_old(dev, 0, dev_priv->counter);
 496 #endif
 497         return 0;
 498 }
 499
 500 static int i915_dispatch_batchbuffer(struct drm_device * dev,
 501                                      drm_i915_batchbuffer_t * batch)
 502 {
 503         drm_i915_private_t *dev_priv = dev->dev_private;
 504         struct drm_clip_rect __user *boxes = batch->cliprects;
 505         int nbox = batch->num_cliprects;
 506         int i = 0, count;
 507         RING_LOCALS;
 508
 509         if ((batch->start | batch->used) & 0x7) {
 510                 DRM_ERROR("alignment\n");
 511                 return -EINVAL;
 512         }
 513
 514         i915_kernel_lost_context(dev);
 515
 516         count = nbox ? nbox : 1;
 517
 518         for (i = 0; i < count; i++) {
 519                 if (i < nbox) {
 520                         int ret = i915_emit_box(dev, boxes, i,
 521                                                 batch->DR1, batch->DR4);
 522                         if (ret)
 523                                 return ret;
 524                 }
 525
 526                 if (dev_priv->use_mi_batchbuffer_start) {
 527                         BEGIN_LP_RING(2);
 528                         if (IS_I965G(dev)) {
 529                                 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
 530                                 OUT_RING(batch->start);
 531                         } else {
 532                                 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
 533                                 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
 534                         }
 535                         ADVANCE_LP_RING();
 536
 537                 } else {
 538                         BEGIN_LP_RING(4);
 539                         OUT_RING(MI_BATCH_BUFFER);
 540                         OUT_RING(batch->start | MI_BATCH_NON_SECURE);
 541                         OUT_RING(batch->start + batch->used - 4);
 542                         OUT_RING(0);
 543                         ADVANCE_LP_RING();
 544                 }
 545         }
 546
 547         i915_emit_breadcrumb(dev);
 548 #ifdef I915_HAVE_FENCE
 549         if (unlikely((dev_priv->counter & 0xFF) == 0))
 550                 drm_fence_flush_old(dev, 0, dev_priv->counter);
 551 #endif
 552         return 0;
 553 }
 554
 555 static void i915_do_dispatch_flip(struct drm_device * dev, int plane, int sync)
 556 {
 557         drm_i915_private_t *dev_priv = dev->dev_private;
 558         u32 num_pages, current_page, next_page, dspbase;
 559         int shift = 2 * plane, x, y;
 560         RING_LOCALS;
 561
 562         /* Calculate display base offset */
 563         num_pages = dev_priv->sarea_priv->third_handle ? 3 : 2;
 564         current_page = (dev_priv->sarea_priv->pf_current_page >> shift) & 0x3;
 565         next_page = (current_page + 1) % num_pages;
 566
 567         switch (next_page) {
 568         default:
 569         case 0:
 570                 dspbase = dev_priv->sarea_priv->front_offset;
 571                 break;
 572         case 1:
 573                 dspbase = dev_priv->sarea_priv->back_offset;
 574                 break;
 575         case 2:
 576                 dspbase = dev_priv->sarea_priv->third_offset;
 577                 break;
 578         }
 579
 580         if (plane == 0) {
 581                 x = dev_priv->sarea_priv->planeA_x;
 582                 y = dev_priv->sarea_priv->planeA_y;
 583         } else {
 584                 x = dev_priv->sarea_priv->planeB_x;
 585                 y = dev_priv->sarea_priv->planeB_y;
 586         }
 587
 588         dspbase += (y * dev_priv->sarea_priv->pitch + x) * dev_priv->cpp;
 589
 590         DRM_DEBUG("plane=%d current_page=%d dspbase=0x%x\n", plane, current_page,
 591                   dspbase);
 592
 593         BEGIN_LP_RING(4);
 594         OUT_RING(sync ? 0 :
 595                  (MI_WAIT_FOR_EVENT | (plane ? MI_WAIT_FOR_PLANE_B_FLIP :
 596                                        MI_WAIT_FOR_PLANE_A_FLIP)));
 597         OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | (sync ? 0 : ASYNC_FLIP) |
 598                  (plane ? DISPLAY_PLANE_B : DISPLAY_PLANE_A));
 599         OUT_RING(dev_priv->sarea_priv->pitch * dev_priv->cpp);
 600         OUT_RING(dspbase);
 601         ADVANCE_LP_RING();
 602
 603         dev_priv->sarea_priv->pf_current_page &= ~(0x3 << shift);
 604         dev_priv->sarea_priv->pf_current_page |= next_page << shift;
 605 }
 606
 607 void i915_dispatch_flip(struct drm_device * dev, int planes, int sync)
 608 {
 609         drm_i915_private_t *dev_priv = dev->dev_private;
 610         int i;
 611
 612         DRM_DEBUG("planes=0x%x pfCurrentPage=%d\n",
 613                   planes, dev_priv->sarea_priv->pf_current_page);
 614
 615         i915_emit_mi_flush(dev, MI_READ_FLUSH | MI_EXE_FLUSH);
 616
 617         for (i = 0; i < 2; i++)
 618                 if (planes & (1 << i))
 619                         i915_do_dispatch_flip(dev, i, sync);
 620
 621         i915_emit_breadcrumb(dev);
 622 #ifdef I915_HAVE_FENCE
 623         if (unlikely(!sync && ((dev_priv->counter & 0xFF) == 0)))
 624                 drm_fence_flush_old(dev, 0, dev_priv->counter);
 625 #endif
 626 }
 627
 628 static int i915_quiescent(struct drm_device *dev)
 629 {
 630         drm_i915_private_t *dev_priv = dev->dev_private;
 631
 632         i915_kernel_lost_context(dev);
 633         return i915_wait_ring(dev, dev_priv->ring.Size - 8, __FUNCTION__);
 634 }
 635
 636 static int i915_flush_ioctl(struct drm_device *dev, void *data,
 637                             struct drm_file *file_priv)
 638 {
 639
 640         LOCK_TEST_WITH_RETURN(dev, file_priv);
 641
 642         return i915_quiescent(dev);
 643 }
 644
 645 static int i915_batchbuffer(struct drm_device *dev, void *data,
 646                             struct drm_file *file_priv)
 647 {
 648         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 649         drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 650             dev_priv->sarea_priv;
 651         drm_i915_batchbuffer_t *batch = data;
 652         int ret;
 653
 654         if (!dev_priv->allow_batchbuffer) {
 655                 DRM_ERROR("Batchbuffer ioctl disabled\n");
 656                 return -EINVAL;
 657         }
 658
 659         DRM_DEBUG("i915 batchbuffer, start %x used %d cliprects %d\n",
 660                   batch->start, batch->used, batch->num_cliprects);
 661
 662         LOCK_TEST_WITH_RETURN(dev, file_priv);
 663
 664         if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects,
 665                                                         batch->num_cliprects *
 666                                                         sizeof(struct drm_clip_rect)))
 667                 return -EFAULT;
 668
 669         ret = i915_dispatch_batchbuffer(dev, batch);
 670
 671         sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 672         return ret;
 673 }
 674
 675 static int i915_cmdbuffer(struct drm_device *dev, void *data,
 676                           struct drm_file *file_priv)
 677 {
 678         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 679         drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 680             dev_priv->sarea_priv;
 681         drm_i915_cmdbuffer_t *cmdbuf = data;
 682         int ret;
 683
 684         DRM_DEBUG("i915 cmdbuffer, buf %p sz %d cliprects %d\n",
 685                   cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects);
 686
 687         LOCK_TEST_WITH_RETURN(dev, file_priv);
 688
 689         if (cmdbuf->num_cliprects &&
 690             DRM_VERIFYAREA_READ(cmdbuf->cliprects,
 691                                 cmdbuf->num_cliprects *
 692                                 sizeof(struct drm_clip_rect))) {
 693                 DRM_ERROR("Fault accessing cliprects\n");
 694                 return -EFAULT;
 695         }
 696
 697         ret = i915_dispatch_cmdbuffer(dev, cmdbuf);
 698         if (ret) {
 699                 DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
 700                 return ret;
 701         }
 702
 703         sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 704         return 0;
 705 }
 706
 707 #if DRM_DEBUG_CODE
 708 #define DRM_DEBUG_RELOCATION    (drm_debug != 0)
 709 #else
 710 #define DRM_DEBUG_RELOCATION    0
 711 #endif
 712
 713 #ifdef I915_HAVE_BUFFER
 714
 715 struct i915_relocatee_info {
 716         struct drm_buffer_object *buf;
 717         unsigned long offset;
 718         u32 *data_page;
 719         unsigned page_offset;
 720         struct drm_bo_kmap_obj kmap;
 721         int is_iomem;
 722 };
 723
 724 struct drm_i915_validate_buffer {
 725         struct drm_buffer_object *buffer;
 726         int presumed_offset_correct;
 727 };
 728
 729 static void i915_dereference_buffers_locked(struct drm_i915_validate_buffer *buffers,
 730                                             unsigned num_buffers)
 731 {
 732         while (num_buffers--)
 733                 drm_bo_usage_deref_locked(&buffers[num_buffers].buffer);
 734 }
 735
 736 int i915_apply_reloc(struct drm_file *file_priv, int num_buffers,
 737                      struct drm_i915_validate_buffer *buffers,
 738                      struct i915_relocatee_info *relocatee,
 739                      uint32_t *reloc)
 740 {
 741         unsigned index;
 742         unsigned long new_cmd_offset;
 743         u32 val;
 744         int ret, i;
 745         int buf_index = -1;
 746
 747         for (i = 0; i <= num_buffers; i++)
 748                 if (buffers[i].buffer)
 749                         if (reloc[2] == buffers[i].buffer->base.hash.key)
 750                                 buf_index = i;
 751
 752         if (buf_index == -1) {
 753                 DRM_ERROR("Illegal relocation buffer %08X\n", reloc[2]);
 754                 return -EINVAL;
 755         }
 756
 757         /*
 758          * Short-circuit relocations that were correctly
 759          * guessed by the client
 760          */
 761         if (buffers[buf_index].presumed_offset_correct && !DRM_DEBUG_RELOCATION)
 762                 return 0;
 763
 764         new_cmd_offset = reloc[0];
 765         if (!relocatee->data_page ||
 766             !drm_bo_same_page(relocatee->offset, new_cmd_offset)) {
 767                 drm_bo_kunmap(&relocatee->kmap);
 768                 relocatee->offset = new_cmd_offset;
 769                 mutex_lock (&relocatee->buf->mutex);
 770                 ret = drm_bo_wait (relocatee->buf, 0, 0, FALSE);
 771                 mutex_unlock (&relocatee->buf->mutex);
 772                 if (ret) {
 773                         DRM_ERROR("Could not wait for buffer to apply relocs\n %08lx", new_cmd_offset);
 774                         return ret;
 775                 }
 776                 ret = drm_bo_kmap(relocatee->buf, new_cmd_offset >> PAGE_SHIFT,
 777                                   1, &relocatee->kmap);
 778                 if (ret) {
 779                         DRM_ERROR("Could not map command buffer to apply relocs\n %08lx", new_cmd_offset);
 780                         return ret;
 781                 }
 782
 783                 relocatee->data_page = drm_bmo_virtual(&relocatee->kmap,
 784                                                        &relocatee->is_iomem);
 785                 relocatee->page_offset = (relocatee->offset & PAGE_MASK);
 786         }
 787
 788         val = buffers[buf_index].buffer->offset;
 789         index = (reloc[0] - relocatee->page_offset) >> 2;
 790
 791         /* add in validate */
 792         val = val + reloc[1];
 793
 794         if (DRM_DEBUG_RELOCATION) {
 795                 if (buffers[buf_index].presumed_offset_correct &&
 796                     relocatee->data_page[index] != val) {
 797                         DRM_DEBUG ("Relocation mismatch source %d target %d buffer %d user %08x kernel %08x\n",
 798                                    reloc[0], reloc[1], buf_index, relocatee->data_page[index], val);
 799                 }
 800         }
 801         relocatee->data_page[index] = val;
 802         return 0;
 803 }
 804
 805 int i915_process_relocs(struct drm_file *file_priv,
 806                         uint32_t buf_handle,
 807                         uint32_t __user **reloc_user_ptr,
 808                         struct i915_relocatee_info *relocatee,
 809                         struct drm_i915_validate_buffer *buffers,
 810                         uint32_t num_buffers)
 811 {
 812         int ret, reloc_stride;
 813         uint32_t cur_offset;
 814         uint32_t reloc_count;
 815         uint32_t reloc_type;
 816         uint32_t reloc_buf_size;
 817         uint32_t *reloc_buf = NULL;
 818         int i;
 819
 820         /* do a copy from user from the user ptr */
 821         ret = get_user(reloc_count, *reloc_user_ptr);
 822         if (ret) {
 823                 DRM_ERROR("Could not map relocation buffer.\n");
 824                 goto out;
 825         }
 826
 827         ret = get_user(reloc_type, (*reloc_user_ptr)+1);
 828         if (ret) {
 829                 DRM_ERROR("Could not map relocation buffer.\n");
 830                 goto out;
 831         }
 832
 833         if (reloc_type != 0) {
 834                 DRM_ERROR("Unsupported relocation type requested\n");
 835                 ret = -EINVAL;
 836                 goto out;
 837         }
 838
 839         reloc_buf_size = (I915_RELOC_HEADER + (reloc_count * I915_RELOC0_STRIDE)) * sizeof(uint32_t);
 840         reloc_buf = kmalloc(reloc_buf_size, GFP_KERNEL);
 841         if (!reloc_buf) {
 842                 DRM_ERROR("Out of memory for reloc buffer\n");
 843                 ret = -ENOMEM;
 844                 goto out;
 845         }
 846
 847         if (copy_from_user(reloc_buf, *reloc_user_ptr, reloc_buf_size)) {
 848                 ret = -EFAULT;
 849                 goto out;
 850         }
 851
 852         /* get next relocate buffer handle */
 853         *reloc_user_ptr = (uint32_t *)*(unsigned long *)&reloc_buf[2];
 854
 855         reloc_stride = I915_RELOC0_STRIDE * sizeof(uint32_t); /* may be different for other types of relocs */
 856
 857         DRM_DEBUG("num relocs is %d, next is %p\n", reloc_count, *reloc_user_ptr);
 858
 859         for (i = 0; i < reloc_count; i++) {
 860                 cur_offset = I915_RELOC_HEADER + (i * I915_RELOC0_STRIDE);
 861
 862                 ret = i915_apply_reloc(file_priv, num_buffers, buffers,
 863                                        relocatee, reloc_buf + cur_offset);
 864                 if (ret)
 865                         goto out;
 866         }
 867
 868 out:
 869
 870         if (reloc_buf)
 871                 kfree(reloc_buf);
 872         drm_bo_kunmap(&relocatee->kmap);
 873         relocatee->data_page = NULL;
 874
 875         return ret;
 876 }
 877
 878 static int i915_exec_reloc(struct drm_file *file_priv, drm_handle_t buf_handle,
 879                            uint32_t __user *reloc_user_ptr,
 880                            struct drm_i915_validate_buffer *buffers,
 881                            uint32_t buf_count)
 882 {
 883         struct drm_device *dev = file_priv->head->dev;
 884         struct i915_relocatee_info relocatee;
 885         int ret = 0;
 886         int b;
 887
 888         /*
 889          * Short circuit relocations when all previous
 890          * buffers offsets were correctly guessed by
 891          * the client
 892          */
 893         if (!DRM_DEBUG_RELOCATION) {
 894                 for (b = 0; b < buf_count; b++)
 895                         if (!buffers[b].presumed_offset_correct)
 896                                 break;
 897
 898                 if (b == buf_count)
 899                         return 0;
 900         }
 901
 902         memset(&relocatee, 0, sizeof(relocatee));
 903
 904         mutex_lock(&dev->struct_mutex);
 905         relocatee.buf = drm_lookup_buffer_object(file_priv, buf_handle, 1);
 906         mutex_unlock(&dev->struct_mutex);
 907         if (!relocatee.buf) {
 908                 DRM_DEBUG("relocatee buffer invalid %08x\n", buf_handle);
 909                 ret = -EINVAL;
 910                 goto out_err;
 911         }
 912
 913         while (reloc_user_ptr) {
 914                 ret = i915_process_relocs(file_priv, buf_handle, &reloc_user_ptr, &relocatee, buffers, buf_count);
 915                 if (ret) {
 916                         DRM_ERROR("process relocs failed\n");
 917                         break;
 918                 }
 919         }
 920
 921         mutex_lock(&dev->struct_mutex);
 922         drm_bo_usage_deref_locked(&relocatee.buf);
 923         mutex_unlock(&dev->struct_mutex);
 924
 925 out_err:
 926         return ret;
 927 }
 928
 929 /*
 930  * Validate, add fence and relocate a block of bos from a userspace list
 931  */
 932 int i915_validate_buffer_list(struct drm_file *file_priv,
 933                               unsigned int fence_class, uint64_t data,
 934                               struct drm_i915_validate_buffer *buffers,
 935                               uint32_t *num_buffers)
 936 {
 937         struct drm_i915_op_arg arg;
 938         struct drm_bo_op_req *req = &arg.d.req;
 939         struct drm_bo_arg_rep rep;
 940         unsigned long next = 0;
 941         int ret = 0;
 942         unsigned buf_count = 0;
 943         struct drm_device *dev = file_priv->head->dev;
 944         uint32_t buf_handle;
 945         uint32_t __user *reloc_user_ptr;
 946
 947         do {
 948                 if (buf_count >= *num_buffers) {
 949                         DRM_ERROR("Buffer count exceeded %d\n.", *num_buffers);
 950                         ret = -EINVAL;
 951                         goto out_err;
 952                 }
 953
 954                 buffers[buf_count].buffer = NULL;
 955                 buffers[buf_count].presumed_offset_correct = 0;
 956
 957                 if (copy_from_user(&arg, (void __user *)(unsigned long)data, sizeof(arg))) {
 958                         ret = -EFAULT;
 959                         goto out_err;
 960                 }
 961
 962                 if (arg.handled) {
 963                         data = arg.next;
 964                         mutex_lock(&dev->struct_mutex);
 965                         buffers[buf_count].buffer = drm_lookup_buffer_object(file_priv, req->arg_handle, 1);
 966                         mutex_unlock(&dev->struct_mutex);
 967                         buf_count++;
 968                         continue;
 969                 }
 970
 971                 rep.ret = 0;
 972                 if (req->op != drm_bo_validate) {
 973                         DRM_ERROR
 974                             ("Buffer object operation wasn't \"validate\".\n");
 975                         rep.ret = -EINVAL;
 976                         goto out_err;
 977                 }
 978
 979                 buf_handle = req->bo_req.handle;
 980                 reloc_user_ptr = (uint32_t *)(unsigned long)arg.reloc_ptr;
 981
 982                 if (reloc_user_ptr) {
 983                         ret = i915_exec_reloc(file_priv, buf_handle, reloc_user_ptr, buffers, buf_count);
 984                         if (ret)
 985                                 goto out_err;
 986                         DRM_MEMORYBARRIER();
 987                 }
 988
 989                 rep.ret = drm_bo_handle_validate(file_priv, req->bo_req.handle,
 990                                                  req->bo_req.flags, req->bo_req.mask,
 991                                                  req->bo_req.hint,
 992                                                  req->bo_req.fence_class, 0,
 993                                                  &rep.bo_info,
 994                                                  &buffers[buf_count].buffer);
 995
 996                 if (rep.ret) {
 997                         DRM_ERROR("error on handle validate %d\n", rep.ret);
 998                         goto out_err;
 999                 }
1000                 /*
1001                  * If the user provided a presumed offset hint, check whether
1002                  * the buffer is in the same place, if so, relocations relative to
1003                  * this buffer need not be performed
1004                  */
1005                 if ((req->bo_req.hint & DRM_BO_HINT_PRESUMED_OFFSET) &&
1006                     buffers[buf_count].buffer->offset == req->bo_req.presumed_offset) {
1007                         buffers[buf_count].presumed_offset_correct = 1;
1008                 }
1009
1010                 next = arg.next;
1011                 arg.handled = 1;
1012                 arg.d.rep = rep;
1013
1014                 if (copy_to_user((void __user *)(unsigned long)data, &arg, sizeof(arg)))
1015                         return -EFAULT;
1016
1017                 data = next;
1018                 buf_count++;
1019
1020         } while (next != 0);
1021         *num_buffers = buf_count;
1022         return 0;
1023 out_err:
1024         mutex_lock(&dev->struct_mutex);
1025         i915_dereference_buffers_locked(buffers, buf_count);
1026         mutex_unlock(&dev->struct_mutex);
1027         *num_buffers = 0;
1028         return (ret) ? ret : rep.ret;
1029 }
1030
1031 static int i915_execbuffer(struct drm_device *dev, void *data,
1032                            struct drm_file *file_priv)
1033 {
1034         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1035         drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
1036                 dev_priv->sarea_priv;
1037         struct drm_i915_execbuffer *exec_buf = data;
1038         struct _drm_i915_batchbuffer *batch = &exec_buf->batch;
1039         struct drm_fence_arg *fence_arg = &exec_buf->fence_arg;
1040         int num_buffers;
1041         int ret;
1042         struct drm_i915_validate_buffer *buffers;
1043         struct drm_fence_object *fence;
1044
1045         if (!dev_priv->allow_batchbuffer) {
1046                 DRM_ERROR("Batchbuffer ioctl disabled\n");
1047                 return -EINVAL;
1048         }
1049
1050
1051         if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects,
1052                                                         batch->num_cliprects *
1053                                                         sizeof(struct drm_clip_rect)))
1054                 return -EFAULT;
1055
1056         if (exec_buf->num_buffers > dev_priv->max_validate_buffers)
1057                 return -EINVAL;
1058
1059
1060         ret = drm_bo_read_lock(&dev->bm.bm_lock);
1061         if (ret)
1062                 return ret;
1063
1064         /*
1065          * The cmdbuf_mutex makes sure the validate-submit-fence
1066          * operation is atomic.
1067          */
1068
1069         ret = mutex_lock_interruptible(&dev_priv->cmdbuf_mutex);
1070         if (ret) {
1071                 drm_bo_read_unlock(&dev->bm.bm_lock);
1072                 return -EAGAIN;
1073         }
1074
1075         num_buffers = exec_buf->num_buffers;
1076
1077         buffers = drm_calloc(num_buffers, sizeof(struct drm_i915_validate_buffer), DRM_MEM_DRIVER);
1078         if (!buffers) {
1079                 drm_bo_read_unlock(&dev->bm.bm_lock);
1080                 mutex_unlock(&dev_priv->cmdbuf_mutex);
1081                 return -ENOMEM;
1082         }
1083
1084         /* validate buffer list + fixup relocations */
1085         ret = i915_validate_buffer_list(file_priv, 0, exec_buf->ops_list,
1086                                         buffers, &num_buffers);
1087         if (ret)
1088                 goto out_free;
1089
1090         /* make sure all previous memory operations have passed */
1091         DRM_MEMORYBARRIER();
1092         drm_agp_chipset_flush(dev);
1093
1094         /* submit buffer */
1095         batch->start = buffers[num_buffers-1].buffer->offset;
1096
1097         DRM_DEBUG("i915 exec batchbuffer, start %x used %d cliprects %d\n",
1098                   batch->start, batch->used, batch->num_cliprects);
1099
1100         ret = i915_dispatch_batchbuffer(dev, batch);
1101         if (ret)
1102                 goto out_err0;
1103
1104         sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
1105
1106         /* fence */
1107         ret = drm_fence_buffer_objects(dev, NULL, fence_arg->flags,
1108                                        NULL, &fence);
1109         if (ret)
1110                 goto out_err0;
1111
1112         if (!(fence_arg->flags & DRM_FENCE_FLAG_NO_USER)) {
1113                 ret = drm_fence_add_user_object(file_priv, fence, fence_arg->flags & DRM_FENCE_FLAG_SHAREABLE);
1114                 if (!ret) {
1115                         fence_arg->handle = fence->base.hash.key;
1116                         fence_arg->fence_class = fence->fence_class;
1117                         fence_arg->type = fence->type;
1118                         fence_arg->signaled = fence->signaled_types;
1119                 }
1120         }
1121         drm_fence_usage_deref_unlocked(&fence);
1122 out_err0:
1123
1124         /* handle errors */
1125         mutex_lock(&dev->struct_mutex);
1126         i915_dereference_buffers_locked(buffers, num_buffers);
1127         mutex_unlock(&dev->struct_mutex);
1128
1129 out_free:
1130         drm_free(buffers, (exec_buf->num_buffers * sizeof(struct drm_buffer_object *)), DRM_MEM_DRIVER);
1131
1132         mutex_unlock(&dev_priv->cmdbuf_mutex);
1133         drm_bo_read_unlock(&dev->bm.bm_lock);
1134         return ret;
1135 }
1136 #endif
1137
1138 static int i915_do_cleanup_pageflip(struct drm_device * dev)
1139 {
1140         drm_i915_private_t *dev_priv = dev->dev_private;
1141         int i, planes, num_pages = dev_priv->sarea_priv->third_handle ? 3 : 2;
1142
1143         DRM_DEBUG("\n");
1144
1145         for (i = 0, planes = 0; i < 2; i++)
1146                 if (dev_priv->sarea_priv->pf_current_page & (0x3 << (2 * i))) {
1147                         dev_priv->sarea_priv->pf_current_page =
1148                                 (dev_priv->sarea_priv->pf_current_page &
1149                                  ~(0x3 << (2 * i))) | ((num_pages - 1) << (2 * i));
1150
1151                         planes |= 1 << i;
1152                 }
1153
1154         if (planes)
1155                 i915_dispatch_flip(dev, planes, 0);
1156
1157         return 0;
1158 }
1159
1160 static int i915_flip_bufs(struct drm_device *dev, void *data, struct drm_file *file_priv)
1161 {
1162         drm_i915_flip_t *param = data;
1163
1164         DRM_DEBUG("\n");
1165
1166         LOCK_TEST_WITH_RETURN(dev, file_priv);
1167
1168         /* This is really planes */
1169         if (param->pipes & ~0x3) {
1170                 DRM_ERROR("Invalid planes 0x%x, only <= 0x3 is valid\n",
1171                           param->pipes);
1172                 return -EINVAL;
1173         }
1174
1175         i915_dispatch_flip(dev, param->pipes, 0);
1176
1177         return 0;
1178 }
1179
1180
1181 static int i915_getparam(struct drm_device *dev, void *data,
1182                          struct drm_file *file_priv)
1183 {
1184         drm_i915_private_t *dev_priv = dev->dev_private;
1185         drm_i915_getparam_t *param = data;
1186         int value;
1187
1188         if (!dev_priv) {
1189                 DRM_ERROR("called with no initialization\n");
1190                 return -EINVAL;
1191         }
1192
1193         switch (param->param) {
1194         case I915_PARAM_IRQ_ACTIVE:
1195                 value = dev->irq ? 1 : 0;
1196                 break;
1197         case I915_PARAM_ALLOW_BATCHBUFFER:
1198                 value = dev_priv->allow_batchbuffer ? 1 : 0;
1199                 break;
1200         case I915_PARAM_LAST_DISPATCH:
1201                 value = READ_BREADCRUMB(dev_priv);
1202                 break;
1203         default:
1204                 DRM_ERROR("Unknown parameter %d\n", param->param);
1205                 return -EINVAL;
1206         }
1207
1208         if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
1209                 DRM_ERROR("DRM_COPY_TO_USER failed\n");
1210                 return -EFAULT;
1211         }
1212
1213         return 0;
1214 }
1215
1216 static int i915_setparam(struct drm_device *dev, void *data,
1217                          struct drm_file *file_priv)
1218 {
1219         drm_i915_private_t *dev_priv = dev->dev_private;
1220         drm_i915_setparam_t *param = data;
1221
1222         if (!dev_priv) {
1223                 DRM_ERROR("called with no initialization\n");
1224                 return -EINVAL;
1225         }
1226
1227         switch (param->param) {
1228         case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
1229                 if (!IS_I965G(dev))
1230                         dev_priv->use_mi_batchbuffer_start = param->value;
1231                 break;
1232         case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
1233                 dev_priv->tex_lru_log_granularity = param->value;
1234                 break;
1235         case I915_SETPARAM_ALLOW_BATCHBUFFER:
1236                 dev_priv->allow_batchbuffer = param->value;
1237                 break;
1238         default:
1239                 DRM_ERROR("unknown parameter %d\n", param->param);
1240                 return -EINVAL;
1241         }
1242
1243         return 0;
1244 }
1245
1246 drm_i915_mmio_entry_t mmio_table[] = {
1247         [MMIO_REGS_PS_DEPTH_COUNT] = {
1248                 I915_MMIO_MAY_READ|I915_MMIO_MAY_WRITE,
1249                 0x2350,
1250                 8
1251         }
1252 };
1253
1254 static int mmio_table_size = sizeof(mmio_table)/sizeof(drm_i915_mmio_entry_t);
1255
1256 static int i915_mmio(struct drm_device *dev, void *data,
1257                      struct drm_file *file_priv)
1258 {
1259         uint32_t buf[8];
1260         drm_i915_private_t *dev_priv = dev->dev_private;
1261         drm_i915_mmio_entry_t *e;
1262         drm_i915_mmio_t *mmio = data;
1263         void __iomem *base;
1264         int i;
1265
1266         if (!dev_priv) {
1267                 DRM_ERROR("called with no initialization\n");
1268                 return -EINVAL;
1269         }
1270
1271         if (mmio->reg >= mmio_table_size)
1272                 return -EINVAL;
1273
1274         e = &mmio_table[mmio->reg];
1275         base = (u8 *) dev_priv->mmio_map->handle + e->offset;
1276
1277         switch (mmio->read_write) {
1278         case I915_MMIO_READ:
1279                 if (!(e->flag & I915_MMIO_MAY_READ))
1280                         return -EINVAL;
1281                 for (i = 0; i < e->size / 4; i++)
1282                         buf[i] = I915_READ(e->offset + i * 4);
1283                 if (DRM_COPY_TO_USER(mmio->data, buf, e->size)) {
1284                         DRM_ERROR("DRM_COPY_TO_USER failed\n");
1285                         return -EFAULT;
1286                 }
1287                 break;
1288
1289         case I915_MMIO_WRITE:
1290                 if (!(e->flag & I915_MMIO_MAY_WRITE))
1291                         return -EINVAL;
1292                 if (DRM_COPY_FROM_USER(buf, mmio->data, e->size)) {
1293                         DRM_ERROR("DRM_COPY_TO_USER failed\n");
1294                         return -EFAULT;
1295                 }
1296                 for (i = 0; i < e->size / 4; i++)
1297                         I915_WRITE(e->offset + i * 4, buf[i]);
1298                 break;
1299         }
1300         return 0;
1301 }
1302
1303 static int i915_set_status_page(struct drm_device *dev, void *data,
1304                                 struct drm_file *file_priv)
1305 {
1306         drm_i915_private_t *dev_priv = dev->dev_private;
1307         drm_i915_hws_addr_t *hws = data;
1308
1309         if (!dev_priv) {
1310                 DRM_ERROR("called with no initialization\n");
1311                 return -EINVAL;
1312         }
1313         DRM_DEBUG("set status page addr 0x%08x\n", (u32)hws->addr);
1314
1315         dev_priv->status_gfx_addr = hws->addr & (0x1ffff<<12);
1316
1317         dev_priv->hws_map.offset = dev->agp->base + hws->addr;
1318         dev_priv->hws_map.size = 4*1024;
1319         dev_priv->hws_map.type = 0;
1320         dev_priv->hws_map.flags = 0;
1321         dev_priv->hws_map.mtrr = 0;
1322
1323         drm_core_ioremap(&dev_priv->hws_map, dev);
1324         if (dev_priv->hws_map.handle == NULL) {
1325                 i915_dma_cleanup(dev);
1326                 dev_priv->status_gfx_addr = 0;
1327                 DRM_ERROR("can not ioremap virtual address for"
1328                                 " G33 hw status page\n");
1329                 return -ENOMEM;
1330         }
1331         dev_priv->hw_status_page = dev_priv->hws_map.handle;
1332
1333         memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
1334         I915_WRITE(0x02080, dev_priv->status_gfx_addr);
1335         DRM_DEBUG("load hws 0x2080 with gfx mem 0x%x\n",
1336                         dev_priv->status_gfx_addr);
1337         DRM_DEBUG("load hws at %p\n", dev_priv->hw_status_page);
1338         return 0;
1339 }
1340
1341 int i915_driver_load(struct drm_device *dev, unsigned long flags)
1342 {
1343         struct drm_i915_private *dev_priv = dev->dev_private;
1344         unsigned long base, size;
1345         int ret = 0, mmio_bar = IS_I9XX(dev) ? 0 : 1;
1346
1347         /* i915 has 4 more counters */
1348         dev->counters += 4;
1349         dev->types[6] = _DRM_STAT_IRQ;
1350         dev->types[7] = _DRM_STAT_PRIMARY;
1351         dev->types[8] = _DRM_STAT_SECONDARY;
1352         dev->types[9] = _DRM_STAT_DMA;
1353
1354         dev_priv = drm_alloc(sizeof(drm_i915_private_t), DRM_MEM_DRIVER);
1355         if (dev_priv == NULL)
1356                 return -ENOMEM;
1357
1358         memset(dev_priv, 0, sizeof(drm_i915_private_t));
1359
1360         dev->dev_private = (void *)dev_priv;
1361
1362         /* Add register map (needed for suspend/resume) */
1363         base = drm_get_resource_start(dev, mmio_bar);
1364         size = drm_get_resource_len(dev, mmio_bar);
1365
1366         ret = drm_addmap(dev, base, size, _DRM_REGISTERS,
1367                 _DRM_KERNEL | _DRM_DRIVER, &dev_priv->mmio_map);
1368
1369 #ifdef __linux__
1370 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)
1371         intel_init_chipset_flush_compat(dev);
1372 #endif
1373 #endif
1374
1375         return ret;
1376 }
1377
1378 int i915_driver_unload(struct drm_device *dev)
1379 {
1380         struct drm_i915_private *dev_priv = dev->dev_private;
1381
1382         if (dev_priv->mmio_map)
1383                 drm_rmmap(dev, dev_priv->mmio_map);
1384
1385         drm_free(dev->dev_private, sizeof(drm_i915_private_t),
1386                  DRM_MEM_DRIVER);
1387 #ifdef __linux__
1388 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)
1389         intel_fini_chipset_flush_compat(dev);
1390 #endif
1391 #endif
1392         return 0;
1393 }
1394
1395 void i915_driver_lastclose(struct drm_device * dev)
1396 {
1397         drm_i915_private_t *dev_priv = dev->dev_private;
1398
1399         if (drm_getsarea(dev) && dev_priv->sarea_priv)
1400                 i915_do_cleanup_pageflip(dev);
1401         if (dev_priv->agp_heap)
1402                 i915_mem_takedown(&(dev_priv->agp_heap));
1403
1404         i915_dma_cleanup(dev);
1405 }
1406
1407 void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
1408 {
1409         drm_i915_private_t *dev_priv = dev->dev_private;
1410         i915_mem_release(dev, file_priv, dev_priv->agp_heap);
1411 }
1412
1413 struct drm_ioctl_desc i915_ioctls[] = {
1414         DRM_IOCTL_DEF(DRM_I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1415         DRM_IOCTL_DEF(DRM_I915_FLUSH, i915_flush_ioctl, DRM_AUTH),
1416         DRM_IOCTL_DEF(DRM_I915_FLIP, i915_flip_bufs, DRM_AUTH),
1417         DRM_IOCTL_DEF(DRM_I915_BATCHBUFFER, i915_batchbuffer, DRM_AUTH),
1418         DRM_IOCTL_DEF(DRM_I915_IRQ_EMIT, i915_irq_emit, DRM_AUTH),
1419         DRM_IOCTL_DEF(DRM_I915_IRQ_WAIT, i915_irq_wait, DRM_AUTH),
1420         DRM_IOCTL_DEF(DRM_I915_GETPARAM, i915_getparam, DRM_AUTH),
1421         DRM_IOCTL_DEF(DRM_I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1422         DRM_IOCTL_DEF(DRM_I915_ALLOC, i915_mem_alloc, DRM_AUTH),
1423         DRM_IOCTL_DEF(DRM_I915_FREE, i915_mem_free, DRM_AUTH),
1424         DRM_IOCTL_DEF(DRM_I915_INIT_HEAP, i915_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1425         DRM_IOCTL_DEF(DRM_I915_CMDBUFFER, i915_cmdbuffer, DRM_AUTH),
1426         DRM_IOCTL_DEF(DRM_I915_DESTROY_HEAP,  i915_mem_destroy_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
1427         DRM_IOCTL_DEF(DRM_I915_SET_VBLANK_PIPE,  i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
1428         DRM_IOCTL_DEF(DRM_I915_GET_VBLANK_PIPE,  i915_vblank_pipe_get, DRM_AUTH ),
1429         DRM_IOCTL_DEF(DRM_I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH),
1430         DRM_IOCTL_DEF(DRM_I915_MMIO, i915_mmio, DRM_AUTH),
1431         DRM_IOCTL_DEF(DRM_I915_HWS_ADDR, i915_set_status_page, DRM_AUTH),
1432 #ifdef I915_HAVE_BUFFER
1433         DRM_IOCTL_DEF(DRM_I915_EXECBUFFER, i915_execbuffer, DRM_AUTH),
1434 #endif
1435 };
1436
1437 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
1438
1439 /**
1440  * Determine if the device really is AGP or not.
1441  *
1442  * All Intel graphics chipsets are treated as AGP, even if they are really
1443  * PCI-e.
1444  *
1445  * \param dev   The device to be tested.
1446  *
1447  * \returns
1448  * A value of 1 is always retured to indictate every i9x5 is AGP.
1449  */
1450 int i915_driver_device_is_agp(struct drm_device * dev)
1451 {
1452         return 1;
1453 }
1454
1455 int i915_driver_firstopen(struct drm_device *dev)
1456 {
1457 #ifdef I915_HAVE_BUFFER
1458         drm_bo_driver_init(dev);
1459 #endif
1460         return 0;
1461 }