intel/intel_bufmgr_fake.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /* Originally a fake version of the buffer manager so that we can
  29  * prototype the changes in a driver fairly quickly, has been fleshed
  30  * out to a fully functional interim solution.
  31  *
  32  * Basically wraps the old style memory management in the new
  33  * programming interface, but is more expressive and avoids many of
  34  * the bugs in the old texture manager.
  35  */
  36
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <assert.h>
  40 #include <errno.h>
  41 #include <strings.h>
  42 #include <xf86drm.h>
  43 #include <pthread.h>
  44 #include "intel_bufmgr.h"
  45 #include "intel_bufmgr_priv.h"
  46 #include "drm.h"
  47 #include "i915_drm.h"
  48 #include "mm.h"
  49 #include "libdrm_macros.h"
  50 #include "libdrm_lists.h"
  51
  52 #define DBG(...) do {                                   \
  53         if (bufmgr_fake->bufmgr.debug)                  \
  54                 drmMsg(__VA_ARGS__);                    \
  55 } while (0)
  56
  57 /* Internal flags:
  58  */
  59 #define BM_NO_BACKING_STORE                     0x00000001
  60 #define BM_NO_FENCE_SUBDATA                     0x00000002
  61 #define BM_PINNED                               0x00000004
  62
  63 /* Wrapper around mm.c's mem_block, which understands that you must
  64  * wait for fences to expire before memory can be freed.  This is
  65  * specific to our use of memcpy for uploads - an upload that was
  66  * processed through the command queue wouldn't need to care about
  67  * fences.
  68  */
  69 #define MAX_RELOCS 4096
  70
  71 struct fake_buffer_reloc {
  72         /** Buffer object that the relocation points at. */
  73         drm_intel_bo *target_buf;
  74         /** Offset of the relocation entry within reloc_buf. */
  75         uint32_t offset;
  76         /**
  77          * Cached value of the offset when we last performed this relocation.
  78          */
  79         uint32_t last_target_offset;
  80         /** Value added to target_buf's offset to get the relocation entry. */
  81         uint32_t delta;
  82         /** Cache domains the target buffer is read into. */
  83         uint32_t read_domains;
  84         /** Cache domain the target buffer will have dirty cachelines in. */
  85         uint32_t write_domain;
  86 };
  87
  88 struct block {
  89         struct block *next, *prev;
  90         struct mem_block *mem;  /* BM_MEM_AGP */
  91
  92         /**
  93          * Marks that the block is currently in the aperture and has yet to be
  94          * fenced.
  95          */
  96         unsigned on_hardware:1;
  97         /**
  98          * Marks that the block is currently fenced (being used by rendering)
  99          * and can't be freed until @fence is passed.
 100          */
 101         unsigned fenced:1;
 102
 103         /** Fence cookie for the block. */
 104         unsigned fence;         /* Split to read_fence, write_fence */
 105
 106         drm_intel_bo *bo;
 107         void *virtual;
 108 };
 109
 110 typedef struct _bufmgr_fake {
 111         drm_intel_bufmgr bufmgr;
 112
 113         pthread_mutex_t lock;
 114
 115         unsigned long low_offset;
 116         unsigned long size;
 117         void *virtual;
 118
 119         struct mem_block *heap;
 120
 121         unsigned buf_nr;        /* for generating ids */
 122
 123         /**
 124          * List of blocks which are currently in the GART but haven't been
 125          * fenced yet.
 126          */
 127         struct block on_hardware;
 128         /**
 129          * List of blocks which are in the GART and have an active fence on
 130          * them.
 131          */
 132         struct block fenced;
 133         /**
 134          * List of blocks which have an expired fence and are ready to be
 135          * evicted.
 136          */
 137         struct block lru;
 138
 139         unsigned int last_fence;
 140
 141         unsigned fail:1;
 142         unsigned need_fence:1;
 143         int thrashing;
 144
 145         /**
 146          * Driver callback to emit a fence, returning the cookie.
 147          *
 148          * This allows the driver to hook in a replacement for the DRM usage in
 149          * bufmgr_fake.
 150          *
 151          * Currently, this also requires that a write flush be emitted before
 152          * emitting the fence, but this should change.
 153          */
 154         unsigned int (*fence_emit) (void *private);
 155         /** Driver callback to wait for a fence cookie to have passed. */
 156         void (*fence_wait) (unsigned int fence, void *private);
 157         void *fence_priv;
 158
 159         /**
 160          * Driver callback to execute a buffer.
 161          *
 162          * This allows the driver to hook in a replacement for the DRM usage in
 163          * bufmgr_fake.
 164          */
 165         int (*exec) (drm_intel_bo *bo, unsigned int used, void *priv);
 166         void *exec_priv;
 167
 168         /** Driver-supplied argument to driver callbacks */
 169         void *driver_priv;
 170         /**
 171          * Pointer to kernel-updated sarea data for the last completed user irq
 172          */
 173         volatile int *last_dispatch;
 174
 175         int fd;
 176
 177         int debug;
 178
 179         int performed_rendering;
 180 } drm_intel_bufmgr_fake;
 181
 182 typedef struct _drm_intel_bo_fake {
 183         drm_intel_bo bo;
 184
 185         unsigned id;            /* debug only */
 186         const char *name;
 187
 188         unsigned dirty:1;
 189         /**
 190          * has the card written to this buffer - we make need to copy it back
 191          */
 192         unsigned card_dirty:1;
 193         unsigned int refcount;
 194         /* Flags may consist of any of the DRM_BO flags, plus
 195          * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the
 196          * first two driver private flags.
 197          */
 198         uint64_t flags;
 199         /** Cache domains the target buffer is read into. */
 200         uint32_t read_domains;
 201         /** Cache domain the target buffer will have dirty cachelines in. */
 202         uint32_t write_domain;
 203
 204         unsigned int alignment;
 205         int is_static, validated;
 206         unsigned int map_count;
 207
 208         /** relocation list */
 209         struct fake_buffer_reloc *relocs;
 210         int nr_relocs;
 211         /**
 212          * Total size of the target_bos of this buffer.
 213          *
 214          * Used for estimation in check_aperture.
 215          */
 216         unsigned int child_size;
 217
 218         struct block *block;
 219         void *backing_store;
 220         void (*invalidate_cb) (drm_intel_bo *bo, void *ptr);
 221         void *invalidate_ptr;
 222 } drm_intel_bo_fake;
 223
 224 static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake,
 225                         unsigned int fence_cookie);
 226
 227 #define MAXFENCE 0x7fffffff
 228
 229 static int
 230 FENCE_LTE(unsigned a, unsigned b)
 231 {
 232         if (a == b)
 233                 return 1;
 234
 235         if (a < b && b - a < (1 << 24))
 236                 return 1;
 237
 238         if (a > b && MAXFENCE - a + b < (1 << 24))
 239                 return 1;
 240
 241         return 0;
 242 }
 243
 244 drm_public void
 245 drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr *bufmgr,
 246                                          unsigned int (*emit) (void *priv),
 247                                          void (*wait) (unsigned int fence,
 248                                                        void *priv),
 249                                          void *priv)
 250 {
 251         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
 252
 253         bufmgr_fake->fence_emit = emit;
 254         bufmgr_fake->fence_wait = wait;
 255         bufmgr_fake->fence_priv = priv;
 256 }
 257
 258 static unsigned int
 259 _fence_emit_internal(drm_intel_bufmgr_fake *bufmgr_fake)
 260 {
 261         struct drm_i915_irq_emit ie;
 262         int ret, seq = 1;
 263
 264         if (bufmgr_fake->fence_emit != NULL) {
 265                 seq = bufmgr_fake->fence_emit(bufmgr_fake->fence_priv);
 266                 return seq;
 267         }
 268
 269         ie.irq_seq = &seq;
 270         ret = drmCommandWriteRead(bufmgr_fake->fd, DRM_I915_IRQ_EMIT,
 271                                   &ie, sizeof(ie));
 272         if (ret) {
 273                 drmMsg("%s: drm_i915_irq_emit: %d\n", __func__, ret);
 274                 abort();
 275         }
 276
 277         DBG("emit 0x%08x\n", seq);
 278         return seq;
 279 }
 280
 281 static void
 282 _fence_wait_internal(drm_intel_bufmgr_fake *bufmgr_fake, int seq)
 283 {
 284         struct drm_i915_irq_wait iw;
 285         int hw_seq, busy_count = 0;
 286         int ret;
 287         int kernel_lied;
 288
 289         if (bufmgr_fake->fence_wait != NULL) {
 290                 bufmgr_fake->fence_wait(seq, bufmgr_fake->fence_priv);
 291                 clear_fenced(bufmgr_fake, seq);
 292                 return;
 293         }
 294
 295         iw.irq_seq = seq;
 296
 297         DBG("wait 0x%08x\n", iw.irq_seq);
 298
 299         /* The kernel IRQ_WAIT implementation is all sorts of broken.
 300          * 1) It returns 1 to 0x7fffffff instead of using the full 32-bit
 301          *    unsigned range.
 302          * 2) It returns 0 if hw_seq >= seq, not seq - hw_seq < 0 on the 32-bit
 303          *    signed range.
 304          * 3) It waits if seq < hw_seq, not seq - hw_seq > 0 on the 32-bit
 305          *    signed range.
 306          * 4) It returns -EBUSY in 3 seconds even if the hardware is still
 307          *    successfully chewing through buffers.
 308          *
 309          * Assume that in userland we treat sequence numbers as ints, which
 310          * makes some of the comparisons convenient, since the sequence
 311          * numbers are all positive signed integers.
 312          *
 313          * From this we get several cases we need to handle.  Here's a timeline.
 314          * 0x2   0x7                                    0x7ffffff8   0x7ffffffd
 315          *   |    |                                             |    |
 316          * ------------------------------------------------------------
 317          *
 318          * A) Normal wait for hw to catch up
 319          * hw_seq seq
 320          *   |    |
 321          * ------------------------------------------------------------
 322          * seq - hw_seq = 5.  If we call IRQ_WAIT, it will wait for hw to
 323          * catch up.
 324          *
 325          * B) Normal wait for a sequence number that's already passed.
 326          * seq    hw_seq
 327          *   |    |
 328          * ------------------------------------------------------------
 329          * seq - hw_seq = -5.  If we call IRQ_WAIT, it returns 0 quickly.
 330          *
 331          * C) Hardware has already wrapped around ahead of us
 332          * hw_seq                                                    seq
 333          *   |                                                       |
 334          * ------------------------------------------------------------
 335          * seq - hw_seq = 0x80000000 - 5.  If we called IRQ_WAIT, it would wait
 336          * for hw_seq >= seq, which may never occur.  Thus, we want to catch
 337          * this in userland and return 0.
 338          *
 339          * D) We've wrapped around ahead of the hardware.
 340          * seq                                                      hw_seq
 341          *   |                                                       |
 342          * ------------------------------------------------------------
 343          * seq - hw_seq = -(0x80000000 - 5).  If we called IRQ_WAIT, it would
 344          * return 0 quickly because hw_seq >= seq, even though the hardware
 345          * isn't caught up. Thus, we need to catch this early return in
 346          * userland and bother the kernel until the hardware really does
 347          * catch up.
 348          *
 349          * E) Hardware might wrap after we test in userland.
 350          *                                                  hw_seq  seq
 351          *                                                      |    |
 352          * ------------------------------------------------------------
 353          * seq - hw_seq = 5.  If we call IRQ_WAIT, it will likely see seq >=
 354          * hw_seq and wait.  However, suppose hw_seq wraps before we make it
 355          * into the kernel.  The kernel sees hw_seq >= seq and waits for 3
 356          * seconds then returns -EBUSY.  This is case C).  We should catch
 357          * this and then return successfully.
 358          *
 359          * F) Hardware might take a long time on a buffer.
 360          * hw_seq seq
 361          *   |    |
 362          * -------------------------------------------------------------------
 363          * seq - hw_seq = 5.  If we call IRQ_WAIT, if sequence 2 through 5
 364          * take too long, it will return -EBUSY.  Batchbuffers in the
 365          * gltestperf demo were seen to take up to 7 seconds.  We should
 366          * catch early -EBUSY return and keep trying.
 367          */
 368
 369         do {
 370                 /* Keep a copy of last_dispatch so that if the wait -EBUSYs
 371                  * because the hardware didn't catch up in 3 seconds, we can
 372                  * see if it at least made progress and retry.
 373                  */
 374                 hw_seq = *bufmgr_fake->last_dispatch;
 375
 376                 /* Catch case C */
 377                 if (seq - hw_seq > 0x40000000)
 378                         return;
 379
 380                 ret = drmCommandWrite(bufmgr_fake->fd, DRM_I915_IRQ_WAIT,
 381                                       &iw, sizeof(iw));
 382                 /* Catch case D */
 383                 kernel_lied = (ret == 0) && (seq - *bufmgr_fake->last_dispatch <
 384                                              -0x40000000);
 385
 386                 /* Catch case E */
 387                 if (ret == -EBUSY
 388                     && (seq - *bufmgr_fake->last_dispatch > 0x40000000))
 389                         ret = 0;
 390
 391                 /* Catch case F: Allow up to 15 seconds chewing on one buffer. */
 392                 if ((ret == -EBUSY) && (hw_seq != *bufmgr_fake->last_dispatch))
 393                         busy_count = 0;
 394                 else
 395                         busy_count++;
 396         } while (kernel_lied || ret == -EAGAIN || ret == -EINTR ||
 397                  (ret == -EBUSY && busy_count < 5));
 398
 399         if (ret != 0) {
 400                 drmMsg("%s:%d: Error waiting for fence: %s.\n", __FILE__,
 401                        __LINE__, strerror(-ret));
 402                 abort();
 403         }
 404         clear_fenced(bufmgr_fake, seq);
 405 }
 406
 407 static int
 408 _fence_test(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)
 409 {
 410         /* Slight problem with wrap-around:
 411          */
 412         return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence);
 413 }
 414
 415 /**
 416  * Allocate a memory manager block for the buffer.
 417  */
 418 static int
 419 alloc_block(drm_intel_bo *bo)
 420 {
 421         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 422         drm_intel_bufmgr_fake *bufmgr_fake =
 423             (drm_intel_bufmgr_fake *) bo->bufmgr;
 424         struct block *block = (struct block *)calloc(sizeof *block, 1);
 425         unsigned int align_log2 = ffs(bo_fake->alignment) - 1;
 426         unsigned int sz;
 427
 428         if (!block)
 429                 return 1;
 430
 431         sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1);
 432
 433         block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0);
 434         if (!block->mem) {
 435                 free(block);
 436                 return 0;
 437         }
 438
 439         DRMINITLISTHEAD(block);
 440
 441         /* Insert at head or at tail??? */
 442         DRMLISTADDTAIL(block, &bufmgr_fake->lru);
 443
 444         block->virtual = (uint8_t *) bufmgr_fake->virtual +
 445             block->mem->ofs - bufmgr_fake->low_offset;
 446         block->bo = bo;
 447
 448         bo_fake->block = block;
 449
 450         return 1;
 451 }
 452
 453 /* Release the card storage associated with buf:
 454  */
 455 static void
 456 free_block(drm_intel_bufmgr_fake *bufmgr_fake, struct block *block,
 457            int skip_dirty_copy)
 458 {
 459         drm_intel_bo_fake *bo_fake;
 460         DBG("free block %p %08x %d %d\n", block, block->mem->ofs,
 461             block->on_hardware, block->fenced);
 462
 463         if (!block)
 464                 return;
 465
 466         bo_fake = (drm_intel_bo_fake *) block->bo;
 467
 468         if (bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))
 469                 skip_dirty_copy = 1;
 470
 471         if (!skip_dirty_copy && (bo_fake->card_dirty == 1)) {
 472                 memcpy(bo_fake->backing_store, block->virtual, block->bo->size);
 473                 bo_fake->card_dirty = 0;
 474                 bo_fake->dirty = 1;
 475         }
 476
 477         if (block->on_hardware) {
 478                 block->bo = NULL;
 479         } else if (block->fenced) {
 480                 block->bo = NULL;
 481         } else {
 482                 DBG("    - free immediately\n");
 483                 DRMLISTDEL(block);
 484
 485                 mmFreeMem(block->mem);
 486                 free(block);
 487         }
 488 }
 489
 490 static void
 491 alloc_backing_store(drm_intel_bo *bo)
 492 {
 493         drm_intel_bufmgr_fake *bufmgr_fake =
 494             (drm_intel_bufmgr_fake *) bo->bufmgr;
 495         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 496         assert(!bo_fake->backing_store);
 497         assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)));
 498
 499         bo_fake->backing_store = malloc(bo->size);
 500
 501         DBG("alloc_backing - buf %d %p %lu\n", bo_fake->id,
 502             bo_fake->backing_store, bo->size);
 503         assert(bo_fake->backing_store);
 504 }
 505
 506 static void
 507 free_backing_store(drm_intel_bo *bo)
 508 {
 509         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 510
 511         if (bo_fake->backing_store) {
 512                 assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)));
 513                 free(bo_fake->backing_store);
 514                 bo_fake->backing_store = NULL;
 515         }
 516 }
 517
 518 static void
 519 set_dirty(drm_intel_bo *bo)
 520 {
 521         drm_intel_bufmgr_fake *bufmgr_fake =
 522             (drm_intel_bufmgr_fake *) bo->bufmgr;
 523         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 524
 525         if (bo_fake->flags & BM_NO_BACKING_STORE
 526             && bo_fake->invalidate_cb != NULL)
 527                 bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr);
 528
 529         assert(!(bo_fake->flags & BM_PINNED));
 530
 531         DBG("set_dirty - buf %d\n", bo_fake->id);
 532         bo_fake->dirty = 1;
 533 }
 534
 535 static int
 536 evict_lru(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int max_fence)
 537 {
 538         struct block *block, *tmp;
 539
 540         DBG("%s\n", __func__);
 541
 542         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
 543                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
 544
 545                 if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA))
 546                         continue;
 547
 548                 if (block->fence && max_fence && !FENCE_LTE(block->fence,
 549                                                             max_fence))
 550                         return 0;
 551
 552                 set_dirty(&bo_fake->bo);
 553                 bo_fake->block = NULL;
 554
 555                 free_block(bufmgr_fake, block, 0);
 556                 return 1;
 557         }
 558
 559         return 0;
 560 }
 561
 562 static int
 563 evict_mru(drm_intel_bufmgr_fake *bufmgr_fake)
 564 {
 565         struct block *block, *tmp;
 566
 567         DBG("%s\n", __func__);
 568
 569         DRMLISTFOREACHSAFEREVERSE(block, tmp, &bufmgr_fake->lru) {
 570                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
 571
 572                 if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA))
 573                         continue;
 574
 575                 set_dirty(&bo_fake->bo);
 576                 bo_fake->block = NULL;
 577
 578                 free_block(bufmgr_fake, block, 0);
 579                 return 1;
 580         }
 581
 582         return 0;
 583 }
 584
 585 /**
 586  * Removes all objects from the fenced list older than the given fence.
 587  */
 588 static int
 589 clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int fence_cookie)
 590 {
 591         struct block *block, *tmp;
 592         int ret = 0;
 593
 594         bufmgr_fake->last_fence = fence_cookie;
 595         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->fenced) {
 596                 assert(block->fenced);
 597
 598                 if (_fence_test(bufmgr_fake, block->fence)) {
 599
 600                         block->fenced = 0;
 601
 602                         if (!block->bo) {
 603                                 DBG("delayed free: offset %x sz %x\n",
 604                                     block->mem->ofs, block->mem->size);
 605                                 DRMLISTDEL(block);
 606                                 mmFreeMem(block->mem);
 607                                 free(block);
 608                         } else {
 609                                 DBG("return to lru: offset %x sz %x\n",
 610                                     block->mem->ofs, block->mem->size);
 611                                 DRMLISTDEL(block);
 612                                 DRMLISTADDTAIL(block, &bufmgr_fake->lru);
 613                         }
 614
 615                         ret = 1;
 616                 } else {
 617                         /* Blocks are ordered by fence, so if one fails, all
 618                          * from here will fail also:
 619                          */
 620                         DBG("fence not passed: offset %x sz %x %d %d \n",
 621                             block->mem->ofs, block->mem->size, block->fence,
 622                             bufmgr_fake->last_fence);
 623                         break;
 624                 }
 625         }
 626
 627         DBG("%s: %d\n", __func__, ret);
 628         return ret;
 629 }
 630
 631 static void
 632 fence_blocks(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)
 633 {
 634         struct block *block, *tmp;
 635
 636         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {
 637                 DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n",
 638                     block, block->mem->size, block->mem->ofs, block->bo, fence);
 639                 block->fence = fence;
 640
 641                 block->on_hardware = 0;
 642                 block->fenced = 1;
 643
 644                 /* Move to tail of pending list here
 645                  */
 646                 DRMLISTDEL(block);
 647                 DRMLISTADDTAIL(block, &bufmgr_fake->fenced);
 648         }
 649
 650         assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
 651 }
 652
 653 static int
 654 evict_and_alloc_block(drm_intel_bo *bo)
 655 {
 656         drm_intel_bufmgr_fake *bufmgr_fake =
 657             (drm_intel_bufmgr_fake *) bo->bufmgr;
 658         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 659
 660         assert(bo_fake->block == NULL);
 661
 662         /* Search for already free memory:
 663          */
 664         if (alloc_block(bo))
 665                 return 1;
 666
 667         /* If we're not thrashing, allow lru eviction to dig deeper into
 668          * recently used textures.  We'll probably be thrashing soon:
 669          */
 670         if (!bufmgr_fake->thrashing) {
 671                 while (evict_lru(bufmgr_fake, 0))
 672                         if (alloc_block(bo))
 673                                 return 1;
 674         }
 675
 676         /* Keep thrashing counter alive?
 677          */
 678         if (bufmgr_fake->thrashing)
 679                 bufmgr_fake->thrashing = 20;
 680
 681         /* Wait on any already pending fences - here we are waiting for any
 682          * freed memory that has been submitted to hardware and fenced to
 683          * become available:
 684          */
 685         while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {
 686                 uint32_t fence = bufmgr_fake->fenced.next->fence;
 687                 _fence_wait_internal(bufmgr_fake, fence);
 688
 689                 if (alloc_block(bo))
 690                         return 1;
 691         }
 692
 693         if (!DRMLISTEMPTY(&bufmgr_fake->on_hardware)) {
 694                 while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {
 695                         uint32_t fence = bufmgr_fake->fenced.next->fence;
 696                         _fence_wait_internal(bufmgr_fake, fence);
 697                 }
 698
 699                 if (!bufmgr_fake->thrashing) {
 700                         DBG("thrashing\n");
 701                 }
 702                 bufmgr_fake->thrashing = 20;
 703
 704                 if (alloc_block(bo))
 705                         return 1;
 706         }
 707
 708         while (evict_mru(bufmgr_fake))
 709                 if (alloc_block(bo))
 710                         return 1;
 711
 712         DBG("%s 0x%lx bytes failed\n", __func__, bo->size);
 713
 714         return 0;
 715 }
 716
 717 /***********************************************************************
 718  * Public functions
 719  */
 720
 721 /**
 722  * Wait for hardware idle by emitting a fence and waiting for it.
 723  */
 724 static void
 725 drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake *bufmgr_fake)
 726 {
 727         unsigned int cookie;
 728
 729         cookie = _fence_emit_internal(bufmgr_fake);
 730         _fence_wait_internal(bufmgr_fake, cookie);
 731 }
 732
 733 /**
 734  * Wait for rendering to a buffer to complete.
 735  *
 736  * It is assumed that the batchbuffer which performed the rendering included
 737  * the necessary flushing.
 738  */
 739 static void
 740 drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo *bo)
 741 {
 742         drm_intel_bufmgr_fake *bufmgr_fake =
 743             (drm_intel_bufmgr_fake *) bo->bufmgr;
 744         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 745
 746         if (bo_fake->block == NULL || !bo_fake->block->fenced)
 747                 return;
 748
 749         _fence_wait_internal(bufmgr_fake, bo_fake->block->fence);
 750 }
 751
 752 static void
 753 drm_intel_fake_bo_wait_rendering(drm_intel_bo *bo)
 754 {
 755         drm_intel_bufmgr_fake *bufmgr_fake =
 756             (drm_intel_bufmgr_fake *) bo->bufmgr;
 757
 758         pthread_mutex_lock(&bufmgr_fake->lock);
 759         drm_intel_fake_bo_wait_rendering_locked(bo);
 760         pthread_mutex_unlock(&bufmgr_fake->lock);
 761 }
 762
 763 /* Specifically ignore texture memory sharing.
 764  *  -- just evict everything
 765  *  -- and wait for idle
 766  */
 767 drm_public void
 768 drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr *bufmgr)
 769 {
 770         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
 771         struct block *block, *tmp;
 772
 773         pthread_mutex_lock(&bufmgr_fake->lock);
 774
 775         bufmgr_fake->need_fence = 1;
 776         bufmgr_fake->fail = 0;
 777
 778         /* Wait for hardware idle.  We don't know where acceleration has been
 779          * happening, so we'll need to wait anyway before letting anything get
 780          * put on the card again.
 781          */
 782         drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
 783
 784         /* Check that we hadn't released the lock without having fenced the last
 785          * set of buffers.
 786          */
 787         assert(DRMLISTEMPTY(&bufmgr_fake->fenced));
 788         assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
 789
 790         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
 791                 assert(_fence_test(bufmgr_fake, block->fence));
 792                 set_dirty(block->bo);
 793         }
 794
 795         pthread_mutex_unlock(&bufmgr_fake->lock);
 796 }
 797
 798 static drm_intel_bo *
 799 drm_intel_fake_bo_alloc(drm_intel_bufmgr *bufmgr,
 800                         const char *name,
 801                         unsigned long size,
 802                         unsigned int alignment)
 803 {
 804         drm_intel_bufmgr_fake *bufmgr_fake;
 805         drm_intel_bo_fake *bo_fake;
 806
 807         bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
 808
 809         assert(size != 0);
 810
 811         bo_fake = calloc(1, sizeof(*bo_fake));
 812         if (!bo_fake)
 813                 return NULL;
 814
 815         bo_fake->bo.size = size;
 816         bo_fake->bo.offset = -1;
 817         bo_fake->bo.virtual = NULL;
 818         bo_fake->bo.bufmgr = bufmgr;
 819         bo_fake->refcount = 1;
 820
 821         /* Alignment must be a power of two */
 822         assert((alignment & (alignment - 1)) == 0);
 823         if (alignment == 0)
 824                 alignment = 1;
 825         bo_fake->alignment = alignment;
 826         bo_fake->id = ++bufmgr_fake->buf_nr;
 827         bo_fake->name = name;
 828         bo_fake->flags = 0;
 829         bo_fake->is_static = 0;
 830
 831         DBG("drm_bo_alloc: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name,
 832             bo_fake->bo.size / 1024);
 833
 834         return &bo_fake->bo;
 835 }
 836
 837 static drm_intel_bo *
 838 drm_intel_fake_bo_alloc_tiled(drm_intel_bufmgr * bufmgr,
 839                               const char *name,
 840                               int x, int y, int cpp,
 841                               uint32_t *tiling_mode,
 842                               unsigned long *pitch,
 843                               unsigned long flags)
 844 {
 845         unsigned long stride, aligned_y;
 846
 847         /* No runtime tiling support for fake. */
 848         *tiling_mode = I915_TILING_NONE;
 849
 850         /* Align it for being a render target.  Shouldn't need anything else. */
 851         stride = x * cpp;
 852         stride = ROUND_UP_TO(stride, 64);
 853
 854         /* 965 subspan loading alignment */
 855         aligned_y = ALIGN(y, 2);
 856
 857         *pitch = stride;
 858
 859         return drm_intel_fake_bo_alloc(bufmgr, name, stride * aligned_y,
 860                                        4096);
 861 }
 862
 863 drm_public drm_intel_bo *
 864 drm_intel_bo_fake_alloc_static(drm_intel_bufmgr *bufmgr,
 865                                const char *name,
 866                                unsigned long offset,
 867                                unsigned long size, void *virtual)
 868 {
 869         drm_intel_bufmgr_fake *bufmgr_fake;
 870         drm_intel_bo_fake *bo_fake;
 871
 872         bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
 873
 874         assert(size != 0);
 875
 876         bo_fake = calloc(1, sizeof(*bo_fake));
 877         if (!bo_fake)
 878                 return NULL;
 879
 880         bo_fake->bo.size = size;
 881         bo_fake->bo.offset = offset;
 882         bo_fake->bo.virtual = virtual;
 883         bo_fake->bo.bufmgr = bufmgr;
 884         bo_fake->refcount = 1;
 885         bo_fake->id = ++bufmgr_fake->buf_nr;
 886         bo_fake->name = name;
 887         bo_fake->flags = BM_PINNED;
 888         bo_fake->is_static = 1;
 889
 890         DBG("drm_bo_alloc_static: (buf %d: %s, %lu kb)\n", bo_fake->id,
 891             bo_fake->name, bo_fake->bo.size / 1024);
 892
 893         return &bo_fake->bo;
 894 }
 895
 896 static void
 897 drm_intel_fake_bo_reference(drm_intel_bo *bo)
 898 {
 899         drm_intel_bufmgr_fake *bufmgr_fake =
 900             (drm_intel_bufmgr_fake *) bo->bufmgr;
 901         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 902
 903         pthread_mutex_lock(&bufmgr_fake->lock);
 904         bo_fake->refcount++;
 905         pthread_mutex_unlock(&bufmgr_fake->lock);
 906 }
 907
 908 static void
 909 drm_intel_fake_bo_reference_locked(drm_intel_bo *bo)
 910 {
 911         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 912
 913         bo_fake->refcount++;
 914 }
 915
 916 static void
 917 drm_intel_fake_bo_unreference_locked(drm_intel_bo *bo)
 918 {
 919         drm_intel_bufmgr_fake *bufmgr_fake =
 920             (drm_intel_bufmgr_fake *) bo->bufmgr;
 921         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 922         int i;
 923
 924         if (--bo_fake->refcount == 0) {
 925                 assert(bo_fake->map_count == 0);
 926                 /* No remaining references, so free it */
 927                 if (bo_fake->block)
 928                         free_block(bufmgr_fake, bo_fake->block, 1);
 929                 free_backing_store(bo);
 930
 931                 for (i = 0; i < bo_fake->nr_relocs; i++)
 932                         drm_intel_fake_bo_unreference_locked(bo_fake->relocs[i].
 933                                                              target_buf);
 934
 935                 DBG("drm_bo_unreference: free buf %d %s\n", bo_fake->id,
 936                     bo_fake->name);
 937
 938                 free(bo_fake->relocs);
 939                 free(bo);
 940         }
 941 }
 942
 943 static void
 944 drm_intel_fake_bo_unreference(drm_intel_bo *bo)
 945 {
 946         drm_intel_bufmgr_fake *bufmgr_fake =
 947             (drm_intel_bufmgr_fake *) bo->bufmgr;
 948
 949         pthread_mutex_lock(&bufmgr_fake->lock);
 950         drm_intel_fake_bo_unreference_locked(bo);
 951         pthread_mutex_unlock(&bufmgr_fake->lock);
 952 }
 953
 954 /**
 955  * Set the buffer as not requiring backing store, and instead get the callback
 956  * invoked whenever it would be set dirty.
 957  */
 958 drm_public void
 959 drm_intel_bo_fake_disable_backing_store(drm_intel_bo *bo,
 960                                         void (*invalidate_cb) (drm_intel_bo *bo,
 961                                                                void *ptr),
 962                                         void *ptr)
 963 {
 964         drm_intel_bufmgr_fake *bufmgr_fake =
 965             (drm_intel_bufmgr_fake *) bo->bufmgr;
 966         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 967
 968         pthread_mutex_lock(&bufmgr_fake->lock);
 969
 970         if (bo_fake->backing_store)
 971                 free_backing_store(bo);
 972
 973         bo_fake->flags |= BM_NO_BACKING_STORE;
 974
 975         DBG("disable_backing_store set buf %d dirty\n", bo_fake->id);
 976         bo_fake->dirty = 1;
 977         bo_fake->invalidate_cb = invalidate_cb;
 978         bo_fake->invalidate_ptr = ptr;
 979
 980         /* Note that it is invalid right from the start.  Also note
 981          * invalidate_cb is called with the bufmgr locked, so cannot
 982          * itself make bufmgr calls.
 983          */
 984         if (invalidate_cb != NULL)
 985                 invalidate_cb(bo, ptr);
 986
 987         pthread_mutex_unlock(&bufmgr_fake->lock);
 988 }
 989
 990 /**
 991  * Map a buffer into bo->virtual, allocating either card memory space (If
 992  * BM_NO_BACKING_STORE or BM_PINNED) or backing store, as necessary.
 993  */
 994 static int
 995  drm_intel_fake_bo_map_locked(drm_intel_bo *bo, int write_enable)
 996 {
 997         drm_intel_bufmgr_fake *bufmgr_fake =
 998             (drm_intel_bufmgr_fake *) bo->bufmgr;
 999         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1000
1001         /* Static buffers are always mapped. */
1002         if (bo_fake->is_static) {
1003                 if (bo_fake->card_dirty) {
1004                         drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1005                         bo_fake->card_dirty = 0;
1006                 }
1007                 return 0;
1008         }
1009
1010         /* Allow recursive mapping.  Mesa may recursively map buffers with
1011          * nested display loops, and it is used internally in bufmgr_fake
1012          * for relocation.
1013          */
1014         if (bo_fake->map_count++ != 0)
1015                 return 0;
1016
1017         {
1018                 DBG("drm_bo_map: (buf %d: %s, %lu kb)\n", bo_fake->id,
1019                     bo_fake->name, bo_fake->bo.size / 1024);
1020
1021                 if (bo->virtual != NULL) {
1022                         drmMsg("%s: already mapped\n", __func__);
1023                         abort();
1024                 } else if (bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED)) {
1025
1026                         if (!bo_fake->block && !evict_and_alloc_block(bo)) {
1027                                 DBG("%s: alloc failed\n", __func__);
1028                                 bufmgr_fake->fail = 1;
1029                                 return 1;
1030                         } else {
1031                                 assert(bo_fake->block);
1032                                 bo_fake->dirty = 0;
1033
1034                                 if (!(bo_fake->flags & BM_NO_FENCE_SUBDATA) &&
1035                                     bo_fake->block->fenced) {
1036                                         drm_intel_fake_bo_wait_rendering_locked
1037                                             (bo);
1038                                 }
1039
1040                                 bo->virtual = bo_fake->block->virtual;
1041                         }
1042                 } else {
1043                         if (write_enable)
1044                                 set_dirty(bo);
1045
1046                         if (bo_fake->backing_store == 0)
1047                                 alloc_backing_store(bo);
1048
1049                         if ((bo_fake->card_dirty == 1) && bo_fake->block) {
1050                                 if (bo_fake->block->fenced)
1051                                         drm_intel_fake_bo_wait_rendering_locked
1052                                             (bo);
1053
1054                                 memcpy(bo_fake->backing_store,
1055                                        bo_fake->block->virtual,
1056                                        bo_fake->block->bo->size);
1057                                 bo_fake->card_dirty = 0;
1058                         }
1059
1060                         bo->virtual = bo_fake->backing_store;
1061                 }
1062         }
1063
1064         return 0;
1065 }
1066
1067 static int
1068  drm_intel_fake_bo_map(drm_intel_bo *bo, int write_enable)
1069 {
1070         drm_intel_bufmgr_fake *bufmgr_fake =
1071             (drm_intel_bufmgr_fake *) bo->bufmgr;
1072         int ret;
1073
1074         pthread_mutex_lock(&bufmgr_fake->lock);
1075         ret = drm_intel_fake_bo_map_locked(bo, write_enable);
1076         pthread_mutex_unlock(&bufmgr_fake->lock);
1077
1078         return ret;
1079 }
1080
1081 static int
1082  drm_intel_fake_bo_unmap_locked(drm_intel_bo *bo)
1083 {
1084         drm_intel_bufmgr_fake *bufmgr_fake =
1085             (drm_intel_bufmgr_fake *) bo->bufmgr;
1086         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1087
1088         /* Static buffers are always mapped. */
1089         if (bo_fake->is_static)
1090                 return 0;
1091
1092         assert(bo_fake->map_count != 0);
1093         if (--bo_fake->map_count != 0)
1094                 return 0;
1095
1096         DBG("drm_bo_unmap: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name,
1097             bo_fake->bo.size / 1024);
1098
1099         bo->virtual = NULL;
1100
1101         return 0;
1102 }
1103
1104 static int drm_intel_fake_bo_unmap(drm_intel_bo *bo)
1105 {
1106         drm_intel_bufmgr_fake *bufmgr_fake =
1107             (drm_intel_bufmgr_fake *) bo->bufmgr;
1108         int ret;
1109
1110         pthread_mutex_lock(&bufmgr_fake->lock);
1111         ret = drm_intel_fake_bo_unmap_locked(bo);
1112         pthread_mutex_unlock(&bufmgr_fake->lock);
1113
1114         return ret;
1115 }
1116
1117 static int
1118 drm_intel_fake_bo_subdata(drm_intel_bo *bo, unsigned long offset,
1119                           unsigned long size, const void *data)
1120 {
1121         int ret;
1122
1123         if (size == 0 || data == NULL)
1124                 return 0;
1125
1126         ret = drm_intel_bo_map(bo, 1);
1127         if (ret)
1128                 return ret;
1129         memcpy((unsigned char *)bo->virtual + offset, data, size);
1130         drm_intel_bo_unmap(bo);
1131         return 0;
1132 }
1133
1134 static void
1135  drm_intel_fake_kick_all_locked(drm_intel_bufmgr_fake *bufmgr_fake)
1136 {
1137         struct block *block, *tmp;
1138
1139         bufmgr_fake->performed_rendering = 0;
1140         /* okay for ever BO that is on the HW kick it off.
1141            seriously not afraid of the POLICE right now */
1142         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {
1143                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
1144
1145                 block->on_hardware = 0;
1146                 free_block(bufmgr_fake, block, 0);
1147                 bo_fake->block = NULL;
1148                 bo_fake->validated = 0;
1149                 if (!(bo_fake->flags & BM_NO_BACKING_STORE))
1150                         bo_fake->dirty = 1;
1151         }
1152
1153 }
1154
1155 static int
1156  drm_intel_fake_bo_validate(drm_intel_bo *bo)
1157 {
1158         drm_intel_bufmgr_fake *bufmgr_fake;
1159         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1160
1161         bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr;
1162
1163         DBG("drm_bo_validate: (buf %d: %s, %lu kb)\n", bo_fake->id,
1164             bo_fake->name, bo_fake->bo.size / 1024);
1165
1166         /* Sanity check: Buffers should be unmapped before being validated.
1167          * This is not so much of a problem for bufmgr_fake, but TTM refuses,
1168          * and the problem is harder to debug there.
1169          */
1170         assert(bo_fake->map_count == 0);
1171
1172         if (bo_fake->is_static) {
1173                 /* Add it to the needs-fence list */
1174                 bufmgr_fake->need_fence = 1;
1175                 return 0;
1176         }
1177
1178         /* Allocate the card memory */
1179         if (!bo_fake->block && !evict_and_alloc_block(bo)) {
1180                 bufmgr_fake->fail = 1;
1181                 DBG("Failed to validate buf %d:%s\n", bo_fake->id,
1182                     bo_fake->name);
1183                 return -1;
1184         }
1185
1186         assert(bo_fake->block);
1187         assert(bo_fake->block->bo == &bo_fake->bo);
1188
1189         bo->offset = bo_fake->block->mem->ofs;
1190
1191         /* Upload the buffer contents if necessary */
1192         if (bo_fake->dirty) {
1193                 DBG("Upload dirty buf %d:%s, sz %lu offset 0x%x\n", bo_fake->id,
1194                     bo_fake->name, bo->size, bo_fake->block->mem->ofs);
1195
1196                 assert(!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED)));
1197
1198                 /* Actually, should be able to just wait for a fence on the
1199                  * memory, which we would be tracking when we free it. Waiting
1200                  * for idle is a sufficiently large hammer for now.
1201                  */
1202                 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1203
1204                 /* we may never have mapped this BO so it might not have any
1205                  * backing store if this happens it should be rare, but 0 the
1206                  * card memory in any case */
1207                 if (bo_fake->backing_store)
1208                         memcpy(bo_fake->block->virtual, bo_fake->backing_store,
1209                                bo->size);
1210                 else
1211                         memset(bo_fake->block->virtual, 0, bo->size);
1212
1213                 bo_fake->dirty = 0;
1214         }
1215
1216         bo_fake->block->fenced = 0;
1217         bo_fake->block->on_hardware = 1;
1218         DRMLISTDEL(bo_fake->block);
1219         DRMLISTADDTAIL(bo_fake->block, &bufmgr_fake->on_hardware);
1220
1221         bo_fake->validated = 1;
1222         bufmgr_fake->need_fence = 1;
1223
1224         return 0;
1225 }
1226
1227 static void
1228 drm_intel_fake_fence_validated(drm_intel_bufmgr *bufmgr)
1229 {
1230         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1231         unsigned int cookie;
1232
1233         cookie = _fence_emit_internal(bufmgr_fake);
1234         fence_blocks(bufmgr_fake, cookie);
1235
1236         DBG("drm_fence_validated: 0x%08x cookie\n", cookie);
1237 }
1238
1239 static void
1240 drm_intel_fake_destroy(drm_intel_bufmgr *bufmgr)
1241 {
1242         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1243
1244         pthread_mutex_destroy(&bufmgr_fake->lock);
1245         mmDestroy(bufmgr_fake->heap);
1246         free(bufmgr);
1247 }
1248
1249 static int
1250 drm_intel_fake_emit_reloc(drm_intel_bo *bo, uint32_t offset,
1251                           drm_intel_bo *target_bo, uint32_t target_offset,
1252                           uint32_t read_domains, uint32_t write_domain)
1253 {
1254         drm_intel_bufmgr_fake *bufmgr_fake =
1255             (drm_intel_bufmgr_fake *) bo->bufmgr;
1256         struct fake_buffer_reloc *r;
1257         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1258         drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *) target_bo;
1259         int i;
1260
1261         pthread_mutex_lock(&bufmgr_fake->lock);
1262
1263         assert(bo);
1264         assert(target_bo);
1265
1266         if (bo_fake->relocs == NULL) {
1267                 bo_fake->relocs =
1268                     malloc(sizeof(struct fake_buffer_reloc) * MAX_RELOCS);
1269         }
1270
1271         r = &bo_fake->relocs[bo_fake->nr_relocs++];
1272
1273         assert(bo_fake->nr_relocs <= MAX_RELOCS);
1274
1275         drm_intel_fake_bo_reference_locked(target_bo);
1276
1277         if (!target_fake->is_static) {
1278                 bo_fake->child_size +=
1279                     ALIGN(target_bo->size, target_fake->alignment);
1280                 bo_fake->child_size += target_fake->child_size;
1281         }
1282         r->target_buf = target_bo;
1283         r->offset = offset;
1284         r->last_target_offset = target_bo->offset;
1285         r->delta = target_offset;
1286         r->read_domains = read_domains;
1287         r->write_domain = write_domain;
1288
1289         if (bufmgr_fake->debug) {
1290                 /* Check that a conflicting relocation hasn't already been
1291                  * emitted.
1292                  */
1293                 for (i = 0; i < bo_fake->nr_relocs - 1; i++) {
1294                         struct fake_buffer_reloc *r2 = &bo_fake->relocs[i];
1295
1296                         assert(r->offset != r2->offset);
1297                 }
1298         }
1299
1300         pthread_mutex_unlock(&bufmgr_fake->lock);
1301
1302         return 0;
1303 }
1304
1305 /**
1306  * Incorporates the validation flags associated with each relocation into
1307  * the combined validation flags for the buffer on this batchbuffer submission.
1308  */
1309 static void
1310 drm_intel_fake_calculate_domains(drm_intel_bo *bo)
1311 {
1312         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1313         int i;
1314
1315         for (i = 0; i < bo_fake->nr_relocs; i++) {
1316                 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1317                 drm_intel_bo_fake *target_fake =
1318                     (drm_intel_bo_fake *) r->target_buf;
1319
1320                 /* Do the same for the tree of buffers we depend on */
1321                 drm_intel_fake_calculate_domains(r->target_buf);
1322
1323                 target_fake->read_domains |= r->read_domains;
1324                 target_fake->write_domain |= r->write_domain;
1325         }
1326 }
1327
1328 static int
1329 drm_intel_fake_reloc_and_validate_buffer(drm_intel_bo *bo)
1330 {
1331         drm_intel_bufmgr_fake *bufmgr_fake =
1332             (drm_intel_bufmgr_fake *) bo->bufmgr;
1333         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1334         int i, ret;
1335
1336         assert(bo_fake->map_count == 0);
1337
1338         for (i = 0; i < bo_fake->nr_relocs; i++) {
1339                 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1340                 drm_intel_bo_fake *target_fake =
1341                     (drm_intel_bo_fake *) r->target_buf;
1342                 uint32_t reloc_data;
1343
1344                 /* Validate the target buffer if that hasn't been done. */
1345                 if (!target_fake->validated) {
1346                         ret =
1347                             drm_intel_fake_reloc_and_validate_buffer(r->target_buf);
1348                         if (ret != 0) {
1349                                 if (bo->virtual != NULL)
1350                                         drm_intel_fake_bo_unmap_locked(bo);
1351                                 return ret;
1352                         }
1353                 }
1354
1355                 /* Calculate the value of the relocation entry. */
1356                 if (r->target_buf->offset != r->last_target_offset) {
1357                         reloc_data = r->target_buf->offset + r->delta;
1358
1359                         if (bo->virtual == NULL)
1360                                 drm_intel_fake_bo_map_locked(bo, 1);
1361
1362                         *(uint32_t *) ((uint8_t *) bo->virtual + r->offset) =
1363                             reloc_data;
1364
1365                         r->last_target_offset = r->target_buf->offset;
1366                 }
1367         }
1368
1369         if (bo->virtual != NULL)
1370                 drm_intel_fake_bo_unmap_locked(bo);
1371
1372         if (bo_fake->write_domain != 0) {
1373                 if (!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED))) {
1374                         if (bo_fake->backing_store == 0)
1375                                 alloc_backing_store(bo);
1376                 }
1377                 bo_fake->card_dirty = 1;
1378                 bufmgr_fake->performed_rendering = 1;
1379         }
1380
1381         return drm_intel_fake_bo_validate(bo);
1382 }
1383
1384 static void
1385 drm_intel_bo_fake_post_submit(drm_intel_bo *bo)
1386 {
1387         drm_intel_bufmgr_fake *bufmgr_fake =
1388             (drm_intel_bufmgr_fake *) bo->bufmgr;
1389         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1390         int i;
1391
1392         for (i = 0; i < bo_fake->nr_relocs; i++) {
1393                 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1394                 drm_intel_bo_fake *target_fake =
1395                     (drm_intel_bo_fake *) r->target_buf;
1396
1397                 if (target_fake->validated)
1398                         drm_intel_bo_fake_post_submit(r->target_buf);
1399
1400                 DBG("%s@0x%08x + 0x%08x -> %s@0x%08x + 0x%08x\n",
1401                     bo_fake->name, (uint32_t) bo->offset, r->offset,
1402                     target_fake->name, (uint32_t) r->target_buf->offset,
1403                     r->delta);
1404         }
1405
1406         assert(bo_fake->map_count == 0);
1407         bo_fake->validated = 0;
1408         bo_fake->read_domains = 0;
1409         bo_fake->write_domain = 0;
1410 }
1411
1412 drm_public void
1413 drm_intel_bufmgr_fake_set_exec_callback(drm_intel_bufmgr *bufmgr,
1414                                              int (*exec) (drm_intel_bo *bo,
1415                                                           unsigned int used,
1416                                                           void *priv),
1417                                              void *priv)
1418 {
1419         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1420
1421         bufmgr_fake->exec = exec;
1422         bufmgr_fake->exec_priv = priv;
1423 }
1424
1425 static int
1426 drm_intel_fake_bo_exec(drm_intel_bo *bo, int used,
1427                        drm_clip_rect_t * cliprects, int num_cliprects, int DR4)
1428 {
1429         drm_intel_bufmgr_fake *bufmgr_fake =
1430             (drm_intel_bufmgr_fake *) bo->bufmgr;
1431         drm_intel_bo_fake *batch_fake = (drm_intel_bo_fake *) bo;
1432         struct drm_i915_batchbuffer batch;
1433         int ret;
1434         int retry_count = 0;
1435
1436         pthread_mutex_lock(&bufmgr_fake->lock);
1437
1438         bufmgr_fake->performed_rendering = 0;
1439
1440         drm_intel_fake_calculate_domains(bo);
1441
1442         batch_fake->read_domains = I915_GEM_DOMAIN_COMMAND;
1443
1444         /* we've ran out of RAM so blow the whole lot away and retry */
1445 restart:
1446         ret = drm_intel_fake_reloc_and_validate_buffer(bo);
1447         if (bufmgr_fake->fail == 1) {
1448                 if (retry_count == 0) {
1449                         retry_count++;
1450                         drm_intel_fake_kick_all_locked(bufmgr_fake);
1451                         bufmgr_fake->fail = 0;
1452                         goto restart;
1453                 } else          /* dump out the memory here */
1454                         mmDumpMemInfo(bufmgr_fake->heap);
1455         }
1456
1457         assert(ret == 0);
1458
1459         if (bufmgr_fake->exec != NULL) {
1460                 ret = bufmgr_fake->exec(bo, used, bufmgr_fake->exec_priv);
1461                 if (ret != 0) {
1462                         pthread_mutex_unlock(&bufmgr_fake->lock);
1463                         return ret;
1464                 }
1465         } else {
1466                 batch.start = bo->offset;
1467                 batch.used = used;
1468                 batch.cliprects = cliprects;
1469                 batch.num_cliprects = num_cliprects;
1470                 batch.DR1 = 0;
1471                 batch.DR4 = DR4;
1472
1473                 if (drmCommandWrite
1474                     (bufmgr_fake->fd, DRM_I915_BATCHBUFFER, &batch,
1475                      sizeof(batch))) {
1476                         drmMsg("DRM_I915_BATCHBUFFER: %d\n", -errno);
1477                         pthread_mutex_unlock(&bufmgr_fake->lock);
1478                         return -errno;
1479                 }
1480         }
1481
1482         drm_intel_fake_fence_validated(bo->bufmgr);
1483
1484         drm_intel_bo_fake_post_submit(bo);
1485
1486         pthread_mutex_unlock(&bufmgr_fake->lock);
1487
1488         return 0;
1489 }
1490
1491 /**
1492  * Return an error if the list of BOs will exceed the aperture size.
1493  *
1494  * This is a rough guess and likely to fail, as during the validate sequence we
1495  * may place a buffer in an inopportune spot early on and then fail to fit
1496  * a set smaller than the aperture.
1497  */
1498 static int
1499 drm_intel_fake_check_aperture_space(drm_intel_bo ** bo_array, int count)
1500 {
1501         drm_intel_bufmgr_fake *bufmgr_fake =
1502             (drm_intel_bufmgr_fake *) bo_array[0]->bufmgr;
1503         unsigned int sz = 0;
1504         int i;
1505
1506         for (i = 0; i < count; i++) {
1507                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo_array[i];
1508
1509                 if (bo_fake == NULL)
1510                         continue;
1511
1512                 if (!bo_fake->is_static)
1513                         sz += ALIGN(bo_array[i]->size, bo_fake->alignment);
1514                 sz += bo_fake->child_size;
1515         }
1516
1517         if (sz > bufmgr_fake->size) {
1518                 DBG("check_space: overflowed bufmgr size, %ukb vs %lukb\n",
1519                     sz / 1024, bufmgr_fake->size / 1024);
1520                 return -1;
1521         }
1522
1523         DBG("drm_check_space: sz %ukb vs bufgr %lukb\n", sz / 1024,
1524             bufmgr_fake->size / 1024);
1525         return 0;
1526 }
1527
1528 /**
1529  * Evicts all buffers, waiting for fences to pass and copying contents out
1530  * as necessary.
1531  *
1532  * Used by the X Server on LeaveVT, when the card memory is no longer our
1533  * own.
1534  */
1535 drm_public void
1536 drm_intel_bufmgr_fake_evict_all(drm_intel_bufmgr *bufmgr)
1537 {
1538         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1539         struct block *block, *tmp;
1540
1541         pthread_mutex_lock(&bufmgr_fake->lock);
1542
1543         bufmgr_fake->need_fence = 1;
1544         bufmgr_fake->fail = 0;
1545
1546         /* Wait for hardware idle.  We don't know where acceleration has been
1547          * happening, so we'll need to wait anyway before letting anything get
1548          * put on the card again.
1549          */
1550         drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1551
1552         /* Check that we hadn't released the lock without having fenced the last
1553          * set of buffers.
1554          */
1555         assert(DRMLISTEMPTY(&bufmgr_fake->fenced));
1556         assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
1557
1558         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
1559                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
1560                 /* Releases the memory, and memcpys dirty contents out if
1561                  * necessary.
1562                  */
1563                 free_block(bufmgr_fake, block, 0);
1564                 bo_fake->block = NULL;
1565         }
1566
1567         pthread_mutex_unlock(&bufmgr_fake->lock);
1568 }
1569
1570 drm_public void
1571 drm_intel_bufmgr_fake_set_last_dispatch(drm_intel_bufmgr *bufmgr,
1572                                         volatile unsigned int
1573                                         *last_dispatch)
1574 {
1575         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1576
1577         bufmgr_fake->last_dispatch = (volatile int *)last_dispatch;
1578 }
1579
1580 drm_public drm_intel_bufmgr *
1581 drm_intel_bufmgr_fake_init(int fd, unsigned long low_offset,
1582                            void *low_virtual, unsigned long size,
1583                            volatile unsigned int *last_dispatch)
1584 {
1585         drm_intel_bufmgr_fake *bufmgr_fake;
1586
1587         bufmgr_fake = calloc(1, sizeof(*bufmgr_fake));
1588
1589         if (pthread_mutex_init(&bufmgr_fake->lock, NULL) != 0) {
1590                 free(bufmgr_fake);
1591                 return NULL;
1592         }
1593
1594         /* Initialize allocator */
1595         DRMINITLISTHEAD(&bufmgr_fake->fenced);
1596         DRMINITLISTHEAD(&bufmgr_fake->on_hardware);
1597         DRMINITLISTHEAD(&bufmgr_fake->lru);
1598
1599         bufmgr_fake->low_offset = low_offset;
1600         bufmgr_fake->virtual = low_virtual;
1601         bufmgr_fake->size = size;
1602         bufmgr_fake->heap = mmInit(low_offset, size);
1603
1604         /* Hook in methods */
1605         bufmgr_fake->bufmgr.bo_alloc = drm_intel_fake_bo_alloc;
1606         bufmgr_fake->bufmgr.bo_alloc_for_render = drm_intel_fake_bo_alloc;
1607         bufmgr_fake->bufmgr.bo_alloc_tiled = drm_intel_fake_bo_alloc_tiled;
1608         bufmgr_fake->bufmgr.bo_reference = drm_intel_fake_bo_reference;
1609         bufmgr_fake->bufmgr.bo_unreference = drm_intel_fake_bo_unreference;
1610         bufmgr_fake->bufmgr.bo_map = drm_intel_fake_bo_map;
1611         bufmgr_fake->bufmgr.bo_unmap = drm_intel_fake_bo_unmap;
1612         bufmgr_fake->bufmgr.bo_subdata = drm_intel_fake_bo_subdata;
1613         bufmgr_fake->bufmgr.bo_wait_rendering =
1614             drm_intel_fake_bo_wait_rendering;
1615         bufmgr_fake->bufmgr.bo_emit_reloc = drm_intel_fake_emit_reloc;
1616         bufmgr_fake->bufmgr.destroy = drm_intel_fake_destroy;
1617         bufmgr_fake->bufmgr.bo_exec = drm_intel_fake_bo_exec;
1618         bufmgr_fake->bufmgr.check_aperture_space =
1619             drm_intel_fake_check_aperture_space;
1620         bufmgr_fake->bufmgr.debug = 0;
1621
1622         bufmgr_fake->fd = fd;
1623         bufmgr_fake->last_dispatch = (volatile int *)last_dispatch;
1624
1625         return &bufmgr_fake->bufmgr;
1626 }