1 /**************************************************************************
3 * Copyright © 2007 Red Hat Inc.
4 * Copyright © 2007 Intel Corporation
5 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 * The above copyright notice and this permission notice (including the
25 * next paragraph) shall be included in all copies or substantial portions
29 **************************************************************************/
31 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
32 * Keith Whitwell <keithw-at-tungstengraphics-dot-com>
33 * Eric Anholt <eric@anholt.net>
34 * Dave Airlie <airlied@linux.ie>
48 #include <sys/ioctl.h>
52 #include "intel_bufmgr.h"
53 #include "intel_bufmgr_priv.h"
58 #define DBG(...) do { \
59 if (bufmgr_gem->bufmgr.debug) \
60 fprintf(stderr, __VA_ARGS__); \
63 typedef struct _dri_bo_gem dri_bo_gem;
65 struct dri_gem_bo_bucket {
66 dri_bo_gem *head, **tail;
68 * Limit on the number of entries in this bucket.
70 * 0 means that this caching at this bucket size is disabled.
71 * -1 means that there is no limit to caching at this size.
77 /* Arbitrarily chosen, 16 means that the maximum size we'll cache for reuse
78 * is 1 << 16 pages, or 256MB.
80 #define INTEL_GEM_BO_BUCKETS 16
81 typedef struct _dri_bufmgr_gem {
90 struct drm_i915_gem_exec_object *exec_objects;
95 /** Array of lists of cached gem objects of power-of-two sizes */
96 struct dri_gem_bo_bucket cache_bucket[INTEL_GEM_BO_BUCKETS];
105 /** Boolean whether the mmap ioctl has been called for this buffer yet. */
111 * Kenel-assigned global name for this object
113 unsigned int global_name;
116 * Index of the buffer within the validation list while preparing a
117 * batchbuffer execution.
122 * Boolean whether we've started swrast
123 * Set when the buffer has been mapped
124 * Cleared when the buffer is unmapped
128 /** Array passed to the DRM containing relocation information. */
129 struct drm_i915_gem_relocation_entry *relocs;
130 /** Array of bos corresponding to relocs[i].target_handle */
131 dri_bo **reloc_target_bo;
132 /** Number of entries in relocs */
134 /** Mapped address for the buffer */
141 * Boolean of whether this BO and its children have been included in
142 * the current dri_bufmgr_check_aperture_space() total.
144 char included_in_check_aperture;
147 * Boolean of whether this buffer has been used as a relocation
148 * target and had its size accounted for, and thus can't have any
149 * further relocations added to it.
151 char used_as_reloc_target;
154 * Size in bytes of this buffer and its relocation descendents.
156 * Used to avoid costly tree walking in dri_bufmgr_check_aperture in
162 static void dri_gem_bo_reference_locked(dri_bo *bo);
178 static struct dri_gem_bo_bucket *
179 dri_gem_bo_bucket_for_size(dri_bufmgr_gem *bufmgr_gem, unsigned long size)
183 /* We only do buckets in power of two increments */
184 if ((size & (size - 1)) != 0)
187 /* We should only see sizes rounded to pages. */
188 assert((size % 4096) == 0);
190 /* We always allocate in units of pages */
191 i = ffs(size / 4096) - 1;
192 if (i >= INTEL_GEM_BO_BUCKETS)
195 return &bufmgr_gem->cache_bucket[i];
199 static void dri_gem_dump_validation_list(dri_bufmgr_gem *bufmgr_gem)
203 for (i = 0; i < bufmgr_gem->exec_count; i++) {
204 dri_bo *bo = bufmgr_gem->exec_bos[i];
205 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
207 if (bo_gem->relocs == NULL) {
208 DBG("%2d: %d (%s)\n", i, bo_gem->gem_handle, bo_gem->name);
212 for (j = 0; j < bo_gem->reloc_count; j++) {
213 dri_bo *target_bo = bo_gem->reloc_target_bo[j];
214 dri_bo_gem *target_gem = (dri_bo_gem *)target_bo;
216 DBG("%2d: %d (%s)@0x%08llx -> %d (%s)@0x%08lx + 0x%08x\n",
218 bo_gem->gem_handle, bo_gem->name, bo_gem->relocs[j].offset,
219 target_gem->gem_handle, target_gem->name, target_bo->offset,
220 bo_gem->relocs[j].delta);
226 * Adds the given buffer to the list of buffers to be validated (moved into the
227 * appropriate memory type) with the next batch submission.
229 * If a buffer is validated multiple times in a batch submission, it ends up
230 * with the intersection of the memory type flags and the union of the
234 intel_add_validate_buffer(dri_bo *bo)
236 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
237 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
240 if (bo_gem->validate_index != -1)
243 /* Extend the array of validation entries as necessary. */
244 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
245 int new_size = bufmgr_gem->exec_size * 2;
250 bufmgr_gem->exec_objects =
251 realloc(bufmgr_gem->exec_objects,
252 sizeof(*bufmgr_gem->exec_objects) * new_size);
253 bufmgr_gem->exec_bos =
254 realloc(bufmgr_gem->exec_bos,
255 sizeof(*bufmgr_gem->exec_bos) * new_size);
256 bufmgr_gem->exec_size = new_size;
259 index = bufmgr_gem->exec_count;
260 bo_gem->validate_index = index;
261 /* Fill in array entry */
262 bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;
263 bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;
264 bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
265 bufmgr_gem->exec_objects[index].alignment = 0;
266 bufmgr_gem->exec_objects[index].offset = 0;
267 bufmgr_gem->exec_bos[index] = bo;
268 dri_gem_bo_reference_locked(bo);
269 bufmgr_gem->exec_count++;
273 #define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \
277 intel_setup_reloc_list(dri_bo *bo)
279 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
280 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
282 bo_gem->relocs = malloc(bufmgr_gem->max_relocs *
283 sizeof(struct drm_i915_gem_relocation_entry));
284 bo_gem->reloc_target_bo = malloc(bufmgr_gem->max_relocs * sizeof(dri_bo *));
290 dri_gem_bo_alloc(dri_bufmgr *bufmgr, const char *name,
291 unsigned long size, unsigned int alignment)
293 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bufmgr;
295 unsigned int page_size = getpagesize();
297 struct dri_gem_bo_bucket *bucket;
298 int alloc_from_cache = 0;
299 unsigned long bo_size;
301 /* Round the allocated size up to a power of two number of pages. */
302 bo_size = 1 << logbase2(size);
303 if (bo_size < page_size)
305 bucket = dri_gem_bo_bucket_for_size(bufmgr_gem, bo_size);
307 /* If we don't have caching at this size, don't actually round the
310 if (bucket == NULL || bucket->max_entries == 0) {
312 if (bo_size < page_size)
316 pthread_mutex_lock(&bufmgr_gem->lock);
317 /* Get a buffer out of the cache if available */
318 if (bucket != NULL && bucket->num_entries > 0) {
319 struct drm_i915_gem_busy busy;
321 bo_gem = bucket->head;
322 busy.handle = bo_gem->gem_handle;
324 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
325 alloc_from_cache = (ret == 0 && busy.busy == 0);
327 if (alloc_from_cache) {
328 bucket->head = bo_gem->next;
329 if (bo_gem->next == NULL)
330 bucket->tail = &bucket->head;
331 bucket->num_entries--;
334 pthread_mutex_unlock(&bufmgr_gem->lock);
336 if (!alloc_from_cache) {
337 struct drm_i915_gem_create create;
339 bo_gem = calloc(1, sizeof(*bo_gem));
343 bo_gem->bo.size = bo_size;
344 memset(&create, 0, sizeof(create));
345 create.size = bo_size;
347 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CREATE, &create);
348 bo_gem->gem_handle = create.handle;
353 bo_gem->bo.bufmgr = bufmgr;
357 bo_gem->refcount = 1;
358 bo_gem->validate_index = -1;
359 bo_gem->reloc_tree_size = bo_gem->bo.size;
360 bo_gem->used_as_reloc_target = 0;
362 DBG("bo_create: buf %d (%s) %ldb\n",
363 bo_gem->gem_handle, bo_gem->name, size);
369 * Returns a dri_bo wrapping the given buffer object handle.
371 * This can be used when one application needs to pass a buffer object
375 intel_bo_gem_create_from_name(dri_bufmgr *bufmgr, const char *name,
378 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bufmgr;
381 struct drm_gem_open open_arg;
383 bo_gem = calloc(1, sizeof(*bo_gem));
387 memset(&open_arg, 0, sizeof(open_arg));
388 open_arg.name = handle;
389 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_GEM_OPEN, &open_arg);
391 fprintf(stderr, "Couldn't reference %s handle 0x%08x: %s\n",
392 name, handle, strerror(errno));
396 bo_gem->bo.size = open_arg.size;
397 bo_gem->bo.offset = 0;
398 bo_gem->bo.virtual = NULL;
399 bo_gem->bo.bufmgr = bufmgr;
401 bo_gem->refcount = 1;
402 bo_gem->validate_index = -1;
403 bo_gem->gem_handle = open_arg.handle;
404 bo_gem->global_name = handle;
406 DBG("bo_create_from_handle: %d (%s)\n", handle, bo_gem->name);
412 dri_gem_bo_reference(dri_bo *bo)
414 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
415 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
417 pthread_mutex_lock(&bufmgr_gem->lock);
419 pthread_mutex_unlock(&bufmgr_gem->lock);
423 dri_gem_bo_reference_locked(dri_bo *bo)
425 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
431 dri_gem_bo_free(dri_bo *bo)
433 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
434 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
435 struct drm_gem_close close;
439 munmap (bo_gem->virtual, bo_gem->bo.size);
441 /* Close this object */
442 close.handle = bo_gem->gem_handle;
443 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close);
446 "DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
447 bo_gem->gem_handle, bo_gem->name, strerror(errno));
453 dri_gem_bo_unreference_locked(dri_bo *bo)
455 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
456 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
458 if (--bo_gem->refcount == 0) {
459 struct dri_gem_bo_bucket *bucket;
461 if (bo_gem->relocs != NULL) {
464 /* Unreference all the target buffers */
465 for (i = 0; i < bo_gem->reloc_count; i++)
466 dri_gem_bo_unreference_locked(bo_gem->reloc_target_bo[i]);
467 free(bo_gem->reloc_target_bo);
468 free(bo_gem->relocs);
471 DBG("bo_unreference final: %d (%s)\n",
472 bo_gem->gem_handle, bo_gem->name);
474 bucket = dri_gem_bo_bucket_for_size(bufmgr_gem, bo->size);
475 /* Put the buffer into our internal cache for reuse if we can. */
476 if (bucket != NULL &&
477 (bucket->max_entries == -1 ||
478 (bucket->max_entries > 0 &&
479 bucket->num_entries < bucket->max_entries)))
482 bo_gem->validate_index = -1;
483 bo_gem->relocs = NULL;
484 bo_gem->reloc_target_bo = NULL;
485 bo_gem->reloc_count = 0;
488 *bucket->tail = bo_gem;
489 bucket->tail = &bo_gem->next;
490 bucket->num_entries++;
498 dri_gem_bo_unreference(dri_bo *bo)
500 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
502 pthread_mutex_lock(&bufmgr_gem->lock);
503 dri_gem_bo_unreference_locked(bo);
504 pthread_mutex_unlock(&bufmgr_gem->lock);
508 dri_gem_bo_map(dri_bo *bo, int write_enable)
510 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
511 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
512 struct drm_i915_gem_set_domain set_domain;
515 pthread_mutex_lock(&bufmgr_gem->lock);
517 /* Allow recursive mapping. Mesa may recursively map buffers with
518 * nested display loops.
520 if (!bo_gem->mapped) {
522 assert(bo->virtual == NULL);
524 DBG("bo_map: %d (%s)\n", bo_gem->gem_handle, bo_gem->name);
526 if (bo_gem->virtual == NULL) {
527 struct drm_i915_gem_mmap mmap_arg;
529 memset(&mmap_arg, 0, sizeof(mmap_arg));
530 mmap_arg.handle = bo_gem->gem_handle;
532 mmap_arg.size = bo->size;
533 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg);
535 fprintf(stderr, "%s:%d: Error mapping buffer %d (%s): %s .\n",
537 bo_gem->gem_handle, bo_gem->name, strerror(errno));
539 bo_gem->virtual = (void *)(uintptr_t)mmap_arg.addr_ptr;
541 bo->virtual = bo_gem->virtual;
544 DBG("bo_map: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name, bo_gem->virtual);
547 if (!bo_gem->swrast) {
548 set_domain.handle = bo_gem->gem_handle;
549 set_domain.read_domains = I915_GEM_DOMAIN_CPU;
551 set_domain.write_domain = I915_GEM_DOMAIN_CPU;
553 set_domain.write_domain = 0;
555 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN,
557 } while (ret == -1 && errno == EINTR);
559 fprintf (stderr, "%s:%d: Error setting swrast %d: %s\n",
560 __FILE__, __LINE__, bo_gem->gem_handle, strerror (errno));
565 pthread_mutex_unlock(&bufmgr_gem->lock);
571 dri_gem_bo_unmap(dri_bo *bo)
573 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
574 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
575 struct drm_i915_gem_sw_finish sw_finish;
581 assert(bo_gem->mapped);
583 pthread_mutex_lock(&bufmgr_gem->lock);
584 if (bo_gem->swrast) {
585 sw_finish.handle = bo_gem->gem_handle;
587 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_SW_FINISH,
589 } while (ret == -1 && errno == EINTR);
592 pthread_mutex_unlock(&bufmgr_gem->lock);
597 dri_gem_bo_subdata (dri_bo *bo, unsigned long offset,
598 unsigned long size, const void *data)
600 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
601 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
602 struct drm_i915_gem_pwrite pwrite;
605 memset (&pwrite, 0, sizeof (pwrite));
606 pwrite.handle = bo_gem->gem_handle;
607 pwrite.offset = offset;
609 pwrite.data_ptr = (uint64_t) (uintptr_t) data;
611 ret = ioctl (bufmgr_gem->fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite);
612 } while (ret == -1 && errno == EINTR);
614 fprintf (stderr, "%s:%d: Error writing data to buffer %d: (%d %d) %s .\n",
616 bo_gem->gem_handle, (int) offset, (int) size,
623 dri_gem_bo_get_subdata (dri_bo *bo, unsigned long offset,
624 unsigned long size, void *data)
626 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
627 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
628 struct drm_i915_gem_pread pread;
631 memset (&pread, 0, sizeof (pread));
632 pread.handle = bo_gem->gem_handle;
633 pread.offset = offset;
635 pread.data_ptr = (uint64_t) (uintptr_t) data;
637 ret = ioctl (bufmgr_gem->fd, DRM_IOCTL_I915_GEM_PREAD, &pread);
638 } while (ret == -1 && errno == EINTR);
640 fprintf (stderr, "%s:%d: Error reading data from buffer %d: (%d %d) %s .\n",
642 bo_gem->gem_handle, (int) offset, (int) size,
649 dri_gem_bo_wait_rendering(dri_bo *bo)
651 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
652 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
653 struct drm_i915_gem_set_domain set_domain;
656 set_domain.handle = bo_gem->gem_handle;
657 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
658 set_domain.write_domain = 0;
659 ret = ioctl (bufmgr_gem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);
661 fprintf (stderr, "%s:%d: Error setting memory domains %d (%08x %08x): %s .\n",
663 bo_gem->gem_handle, set_domain.read_domains, set_domain.write_domain,
669 dri_bufmgr_gem_destroy(dri_bufmgr *bufmgr)
671 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bufmgr;
674 free(bufmgr_gem->exec_objects);
675 free(bufmgr_gem->exec_bos);
677 pthread_mutex_destroy(&bufmgr_gem->lock);
679 /* Free any cached buffer objects we were going to reuse */
680 for (i = 0; i < INTEL_GEM_BO_BUCKETS; i++) {
681 struct dri_gem_bo_bucket *bucket = &bufmgr_gem->cache_bucket[i];
684 while ((bo_gem = bucket->head) != NULL) {
685 bucket->head = bo_gem->next;
686 if (bo_gem->next == NULL)
687 bucket->tail = &bucket->head;
688 bucket->num_entries--;
690 dri_gem_bo_free(&bo_gem->bo);
698 * Adds the target buffer to the validation list and adds the relocation
699 * to the reloc_buffer's relocation list.
701 * The relocation entry at the given offset must already contain the
702 * precomputed relocation value, because the kernel will optimize out
703 * the relocation entry write when the buffer hasn't moved from the
704 * last known offset in target_bo.
707 dri_gem_bo_emit_reloc(dri_bo *bo, uint32_t read_domains, uint32_t write_domain,
708 uint32_t delta, uint32_t offset, dri_bo *target_bo)
710 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
711 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
712 dri_bo_gem *target_bo_gem = (dri_bo_gem *)target_bo;
714 pthread_mutex_lock(&bufmgr_gem->lock);
716 /* Create a new relocation list if needed */
717 if (bo_gem->relocs == NULL)
718 intel_setup_reloc_list(bo);
721 assert(bo_gem->reloc_count < bufmgr_gem->max_relocs);
724 assert (offset <= bo->size - 4);
725 assert ((write_domain & (write_domain-1)) == 0);
727 /* Make sure that we're not adding a reloc to something whose size has
728 * already been accounted for.
730 assert(!bo_gem->used_as_reloc_target);
731 bo_gem->reloc_tree_size += target_bo_gem->reloc_tree_size;
733 /* Flag the target to disallow further relocations in it. */
734 target_bo_gem->used_as_reloc_target = 1;
736 bo_gem->relocs[bo_gem->reloc_count].offset = offset;
737 bo_gem->relocs[bo_gem->reloc_count].delta = delta;
738 bo_gem->relocs[bo_gem->reloc_count].target_handle =
739 target_bo_gem->gem_handle;
740 bo_gem->relocs[bo_gem->reloc_count].read_domains = read_domains;
741 bo_gem->relocs[bo_gem->reloc_count].write_domain = write_domain;
742 bo_gem->relocs[bo_gem->reloc_count].presumed_offset = target_bo->offset;
744 bo_gem->reloc_target_bo[bo_gem->reloc_count] = target_bo;
745 dri_gem_bo_reference_locked(target_bo);
747 bo_gem->reloc_count++;
749 pthread_mutex_unlock(&bufmgr_gem->lock);
755 * Walk the tree of relocations rooted at BO and accumulate the list of
756 * validations to be performed and update the relocation buffers with
757 * index values into the validation list.
760 dri_gem_bo_process_reloc(dri_bo *bo)
762 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
765 if (bo_gem->relocs == NULL)
768 for (i = 0; i < bo_gem->reloc_count; i++) {
769 dri_bo *target_bo = bo_gem->reloc_target_bo[i];
771 /* Continue walking the tree depth-first. */
772 dri_gem_bo_process_reloc(target_bo);
774 /* Add the target to the validate list */
775 intel_add_validate_buffer(target_bo);
780 intel_update_buffer_offsets (dri_bufmgr_gem *bufmgr_gem)
784 for (i = 0; i < bufmgr_gem->exec_count; i++) {
785 dri_bo *bo = bufmgr_gem->exec_bos[i];
786 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
788 /* Update the buffer offset */
789 if (bufmgr_gem->exec_objects[i].offset != bo->offset) {
790 DBG("BO %d (%s) migrated: 0x%08lx -> 0x%08llx\n",
791 bo_gem->gem_handle, bo_gem->name, bo->offset,
792 bufmgr_gem->exec_objects[i].offset);
793 bo->offset = bufmgr_gem->exec_objects[i].offset;
799 dri_gem_bo_exec(dri_bo *bo, int used,
800 drm_clip_rect_t *cliprects, int num_cliprects,
803 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
804 struct drm_i915_gem_execbuffer execbuf;
807 pthread_mutex_lock(&bufmgr_gem->lock);
808 /* Update indices and set up the validate list. */
809 dri_gem_bo_process_reloc(bo);
811 /* Add the batch buffer to the validation list. There are no relocations
814 intel_add_validate_buffer(bo);
816 execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec_objects;
817 execbuf.buffer_count = bufmgr_gem->exec_count;
818 execbuf.batch_start_offset = 0;
819 execbuf.batch_len = used;
820 execbuf.cliprects_ptr = (uintptr_t)cliprects;
821 execbuf.num_cliprects = num_cliprects;
826 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_EXECBUFFER, &execbuf);
827 } while (ret != 0 && errno == EAGAIN);
829 intel_update_buffer_offsets (bufmgr_gem);
831 if (bufmgr_gem->bufmgr.debug)
832 dri_gem_dump_validation_list(bufmgr_gem);
834 for (i = 0; i < bufmgr_gem->exec_count; i++) {
835 dri_bo *bo = bufmgr_gem->exec_bos[i];
836 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
838 /* Need to call swrast on next bo_map */
841 /* Disconnect the buffer from the validate list */
842 bo_gem->validate_index = -1;
843 dri_gem_bo_unreference_locked(bo);
844 bufmgr_gem->exec_bos[i] = NULL;
846 bufmgr_gem->exec_count = 0;
847 pthread_mutex_unlock(&bufmgr_gem->lock);
853 dri_gem_bo_pin(dri_bo *bo, uint32_t alignment)
855 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
856 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
857 struct drm_i915_gem_pin pin;
860 pin.handle = bo_gem->gem_handle;
861 pin.alignment = alignment;
863 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_PIN, &pin);
867 bo->offset = pin.offset;
872 dri_gem_bo_unpin(dri_bo *bo)
874 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
875 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
876 struct drm_i915_gem_unpin unpin;
879 unpin.handle = bo_gem->gem_handle;
881 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_UNPIN, &unpin);
889 dri_gem_bo_set_tiling(dri_bo *bo, uint32_t *tiling_mode)
891 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
892 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
893 struct drm_i915_gem_set_tiling set_tiling;
896 set_tiling.handle = bo_gem->gem_handle;
897 set_tiling.tiling_mode = *tiling_mode;
899 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling);
901 *tiling_mode = I915_TILING_NONE;
905 *tiling_mode = set_tiling.tiling_mode;
910 dri_gem_bo_get_tiling(dri_bo *bo, uint32_t *tiling_mode, uint32_t *swizzle_mode)
912 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
913 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
914 struct drm_i915_gem_get_tiling get_tiling;
917 get_tiling.handle = bo_gem->gem_handle;
919 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &get_tiling);
921 *tiling_mode = I915_TILING_NONE;
922 *swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
926 *tiling_mode = get_tiling.tiling_mode;
927 *swizzle_mode = get_tiling.swizzle_mode;
932 dri_gem_bo_flink(dri_bo *bo, uint32_t *name)
934 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr;
935 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
936 struct drm_gem_flink flink;
939 if (!bo_gem->global_name) {
940 flink.handle = bo_gem->gem_handle;
942 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_GEM_FLINK, &flink);
945 bo_gem->global_name = flink.name;
948 *name = bo_gem->global_name;
953 * Enables unlimited caching of buffer objects for reuse.
955 * This is potentially very memory expensive, as the cache at each bucket
956 * size is only bounded by how many buffers of that size we've managed to have
960 intel_bufmgr_gem_enable_reuse(dri_bufmgr *bufmgr)
962 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bufmgr;
965 for (i = 0; i < INTEL_GEM_BO_BUCKETS; i++) {
966 bufmgr_gem->cache_bucket[i].max_entries = -1;
971 * Return the additional aperture space required by the tree of buffer objects
975 dri_gem_bo_get_aperture_space(dri_bo *bo)
977 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
981 if (bo == NULL || bo_gem->included_in_check_aperture)
985 bo_gem->included_in_check_aperture = 1;
987 for (i = 0; i < bo_gem->reloc_count; i++)
988 total += dri_gem_bo_get_aperture_space(bo_gem->reloc_target_bo[i]);
994 * Clear the flag set by dri_gem_bo_get_aperture_space() so we're ready for
995 * the next dri_bufmgr_check_aperture_space() call.
998 dri_gem_bo_clear_aperture_space_flag(dri_bo *bo)
1000 dri_bo_gem *bo_gem = (dri_bo_gem *)bo;
1003 if (bo == NULL || !bo_gem->included_in_check_aperture)
1006 bo_gem->included_in_check_aperture = 0;
1008 for (i = 0; i < bo_gem->reloc_count; i++)
1009 dri_gem_bo_clear_aperture_space_flag(bo_gem->reloc_target_bo[i]);
1013 * Return -1 if the batchbuffer should be flushed before attempting to
1014 * emit rendering referencing the buffers pointed to by bo_array.
1016 * This is required because if we try to emit a batchbuffer with relocations
1017 * to a tree of buffers that won't simultaneously fit in the aperture,
1018 * the rendering will return an error at a point where the software is not
1019 * prepared to recover from it.
1021 * However, we also want to emit the batchbuffer significantly before we reach
1022 * the limit, as a series of batchbuffers each of which references buffers
1023 * covering almost all of the aperture means that at each emit we end up
1024 * waiting to evict a buffer from the last rendering, and we get synchronous
1025 * performance. By emitting smaller batchbuffers, we eat some CPU overhead to
1026 * get better parallelism.
1029 dri_gem_check_aperture_space(dri_bo **bo_array, int count)
1031 dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo_array[0]->bufmgr;
1032 unsigned int total = 0;
1033 unsigned int threshold = bufmgr_gem->gtt_size * 3 / 4;
1036 for (i = 0; i < count; i++) {
1037 dri_bo_gem *bo_gem = (dri_bo_gem *)bo_array[i];
1039 total += bo_gem->reloc_tree_size;
1042 if (total > threshold) {
1044 for (i = 0; i < count; i++)
1045 total += dri_gem_bo_get_aperture_space(bo_array[i]);
1047 for (i = 0; i < count; i++)
1048 dri_gem_bo_clear_aperture_space_flag(bo_array[i]);
1051 if (total > bufmgr_gem->gtt_size * 3 / 4) {
1052 DBG("check_space: overflowed available aperture, %dkb vs %dkb\n",
1053 total / 1024, (int)bufmgr_gem->gtt_size / 1024);
1056 DBG("drm_check_space: total %dkb vs bufgr %dkb\n", total / 1024 ,
1057 (int)bufmgr_gem->gtt_size / 1024);
1063 * Initializes the GEM buffer manager, which uses the kernel to allocate, map,
1064 * and manage map buffer objections.
1066 * \param fd File descriptor of the opened DRM device.
1069 intel_bufmgr_gem_init(int fd, int batch_size)
1071 dri_bufmgr_gem *bufmgr_gem;
1072 struct drm_i915_gem_get_aperture aperture;
1075 bufmgr_gem = calloc(1, sizeof(*bufmgr_gem));
1076 bufmgr_gem->fd = fd;
1078 if (pthread_mutex_init(&bufmgr_gem->lock, NULL) != 0) {
1083 ret = ioctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
1086 bufmgr_gem->gtt_size = aperture.aper_available_size;
1088 fprintf(stderr, "DRM_IOCTL_I915_GEM_APERTURE failed: %s\n",
1090 bufmgr_gem->gtt_size = 128 * 1024 * 1024;
1091 fprintf(stderr, "Assuming %dkB available aperture size.\n"
1092 "May lead to reduced performance or incorrect rendering.\n",
1093 (int)bufmgr_gem->gtt_size / 1024);
1096 /* Let's go with one relocation per every 2 dwords (but round down a bit
1097 * since a power of two will mean an extra page allocation for the reloc
1100 * Every 4 was too few for the blender benchmark.
1102 bufmgr_gem->max_relocs = batch_size / sizeof(uint32_t) / 2 - 2;
1104 bufmgr_gem->bufmgr.bo_alloc = dri_gem_bo_alloc;
1105 bufmgr_gem->bufmgr.bo_reference = dri_gem_bo_reference;
1106 bufmgr_gem->bufmgr.bo_unreference = dri_gem_bo_unreference;
1107 bufmgr_gem->bufmgr.bo_map = dri_gem_bo_map;
1108 bufmgr_gem->bufmgr.bo_unmap = dri_gem_bo_unmap;
1109 bufmgr_gem->bufmgr.bo_subdata = dri_gem_bo_subdata;
1110 bufmgr_gem->bufmgr.bo_get_subdata = dri_gem_bo_get_subdata;
1111 bufmgr_gem->bufmgr.bo_wait_rendering = dri_gem_bo_wait_rendering;
1112 bufmgr_gem->bufmgr.bo_emit_reloc = dri_gem_bo_emit_reloc;
1113 bufmgr_gem->bufmgr.bo_pin = dri_gem_bo_pin;
1114 bufmgr_gem->bufmgr.bo_unpin = dri_gem_bo_unpin;
1115 bufmgr_gem->bufmgr.bo_get_tiling = dri_gem_bo_get_tiling;
1116 bufmgr_gem->bufmgr.bo_set_tiling = dri_gem_bo_set_tiling;
1117 bufmgr_gem->bufmgr.bo_flink = dri_gem_bo_flink;
1118 bufmgr_gem->bufmgr.bo_exec = dri_gem_bo_exec;
1119 bufmgr_gem->bufmgr.destroy = dri_bufmgr_gem_destroy;
1120 bufmgr_gem->bufmgr.debug = 0;
1121 bufmgr_gem->bufmgr.check_aperture_space = dri_gem_check_aperture_space;
1122 /* Initialize the linked lists for BO reuse cache. */
1123 for (i = 0; i < INTEL_GEM_BO_BUCKETS; i++)
1124 bufmgr_gem->cache_bucket[i].tail = &bufmgr_gem->cache_bucket[i].head;
1126 return &bufmgr_gem->bufmgr;