1 /**************************************************************************
3 * Copyright © 2007 Red Hat Inc.
4 * Copyright © 2007 Intel Corporation
5 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 * The above copyright notice and this permission notice (including the
25 * next paragraph) shall be included in all copies or substantial portions
29 **************************************************************************/
31 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
32 * Keith Whitwell <keithw-at-tungstengraphics-dot-com>
33 * Eric Anholt <eric@anholt.net>
34 * Dave Airlie <airlied@linux.ie>
43 #include "dri_bufmgr.h"
49 #include "intel_bufmgr_ttm.h"
51 #define DBG(...) do { \
52 if (bufmgr_ttm->bufmgr.debug) \
53 _mesa_printf(__VA_ARGS__); \
56 /* Buffer validation list */
57 struct intel_bo_list {
62 typedef struct _dri_bufmgr_ttm {
66 unsigned int fence_type;
67 unsigned int fence_type_flush;
70 struct intel_bo_list list; /* list of buffers to be validated */
73 typedef struct _dri_bo_ttm {
80 /** DRM buffer object containing relocation list */
85 typedef struct _dri_fence_ttm
94 /* Validation list node */
99 struct drm_i915_op_arg bo_arg;
105 intel_init_validate_list(struct intel_bo_list *list)
107 DRMINITLISTHEAD(&list->list);
108 list->numCurrent = 0;
112 * Empties the validation list and clears the relocations
115 intel_free_validate_list(dri_bufmgr_ttm *bufmgr_ttm)
117 struct intel_bo_list *list = &bufmgr_ttm->list;
120 for (l = list->list.next; l != &list->list; l = list->list.next) {
121 struct intel_bo_node *node =
122 DRMLISTENTRY(struct intel_bo_node, l, head);
123 dri_bo_ttm *bo_ttm = (dri_bo_ttm *)node->bo;
127 /* Clear relocation list */
128 if (bo_ttm->relocs != NULL)
129 bo_ttm->relocs[0] = bo_ttm->relocs[0] & ~0xffff;
131 dri_bo_unreference(node->bo);
138 static void dri_ttm_dump_validation_list(dri_bufmgr_ttm *bufmgr_ttm)
140 struct intel_bo_list *list = &bufmgr_ttm->list;
144 for (l = list->list.next; l != &list->list; l = l->next) {
146 struct intel_bo_node *node =
147 DRMLISTENTRY(struct intel_bo_node, l, head);
148 dri_bo_ttm *bo_ttm = (dri_bo_ttm *)node->bo;
150 if (bo_ttm->relocs != NULL) {
151 for (j = 0; j < (bo_ttm->relocs[0] & 0xffff); j++) {
152 uint32_t *reloc_entry = bo_ttm->relocs + I915_RELOC_HEADER +
153 j * I915_RELOC0_STRIDE;
155 DBG("%2d: %s@0x%08x -> %d + 0x%08x\n",
157 reloc_entry[0], reloc_entry[2], reloc_entry[1]);
160 DBG("%2d: %s\n", i, bo_ttm->name);
166 static struct drm_i915_op_arg *
167 intel_setup_validate_list(dri_bufmgr_ttm *bufmgr_ttm, GLuint *count_p)
169 struct intel_bo_list *list = &bufmgr_ttm->list;
171 struct drm_i915_op_arg *first;
172 uint64_t *prevNext = NULL;
177 for (l = list->list.next; l != &list->list; l = l->next) {
178 struct intel_bo_node *node =
179 DRMLISTENTRY(struct intel_bo_node, l, head);
180 dri_bo_ttm *ttm_buf = (dri_bo_ttm *)node->bo;
181 struct drm_i915_op_arg *arg = &node->bo_arg;
182 struct drm_bo_op_req *req = &arg->d.req;
188 *prevNext = (unsigned long) arg;
190 memset(arg, 0, sizeof(*arg));
191 prevNext = &arg->next;
192 req->bo_req.handle = ttm_buf->drm_bo.handle;
193 req->op = drm_bo_validate;
194 req->bo_req.flags = node->flags;
195 req->bo_req.hint = 0;
196 #ifdef DRM_BO_HINT_PRESUMED_OFFSET
197 req->bo_req.hint |= DRM_BO_HINT_PRESUMED_OFFSET;
198 req->bo_req.presumed_offset = node->bo->offset;
200 req->bo_req.mask = node->mask;
201 req->bo_req.fence_class = 0; /* Backwards compat. */
203 if (ttm_buf->reloc_buf != NULL)
204 arg->reloc_handle = ttm_buf->reloc_buf->handle;
206 arg->reloc_handle = 0;
219 * Adds the given buffer to the list of buffers to be validated (moved into the
220 * appropriate memory type) with the next batch submission.
222 * If a buffer is validated multiple times in a batch submission, it ends up
223 * with the intersection of the memory type flags and the union of the
226 static struct intel_bo_node *
227 intel_add_validate_buffer(dri_bufmgr_ttm *bufmgr_ttm,
229 uint64_t flags, uint64_t mask,
232 struct intel_bo_list *list = &bufmgr_ttm->list;
233 struct intel_bo_node *cur;
234 dri_bo_ttm *ttm_buf = (dri_bo_ttm *)buf;
240 /* Find the buffer in the validation list if it's already there. */
241 for (l = list->list.next; l != &list->list; l = l->next) {
242 struct intel_bo_node *node =
243 DRMLISTENTRY(struct intel_bo_node, l, head);
245 if (((dri_bo_ttm *)node->bo)->drm_bo.handle == ttm_buf->drm_bo.handle) {
253 cur = drmMalloc(sizeof(*cur));
258 dri_bo_reference(buf);
263 DRMLISTADDTAIL(&cur->head, &list->list);
265 uint64_t memMask = (cur->mask | mask) & DRM_BO_MASK_MEM;
266 uint64_t memFlags = cur->flags & flags & memMask;
270 "%s: No shared memory types between "
271 "0x%16llx and 0x%16llx\n",
272 __FUNCTION__, cur->flags, flags);
275 if (mask & cur->mask & ~DRM_BO_MASK_MEM & (cur->flags ^ flags)) {
277 "%s: Incompatible flags between 0x%16llx and 0x%16llx "
278 "(0x%16llx, 0x%16llx masks)\n",
279 __FUNCTION__, cur->flags, flags, cur->mask, mask);
283 cur->flags = memFlags | ((cur->flags | flags) &
284 cur->mask & ~DRM_BO_MASK_MEM);
292 #define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \
296 intel_setup_reloc_list(dri_bo *bo)
298 dri_bo_ttm *bo_ttm = (dri_bo_ttm *)bo;
299 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)bo->bufmgr;
302 /* If the buffer exists, then it was just created, or it was reintialized
303 * at the last intel_free_validate_list().
305 if (bo_ttm->reloc_buf != NULL)
308 bo_ttm->reloc_buf = malloc(sizeof(bo_ttm->drm_bo));
310 ret = drmBOCreate(bufmgr_ttm->fd,
311 RELOC_BUF_SIZE(bufmgr_ttm->max_relocs), 0,
313 DRM_BO_FLAG_MEM_LOCAL |
316 DRM_BO_FLAG_MAPPABLE |
318 0, bo_ttm->reloc_buf);
320 fprintf(stderr, "Failed to create relocation BO: %s\n",
325 ret = drmBOMap(bufmgr_ttm->fd, bo_ttm->reloc_buf,
326 DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE,
327 0, (void **)&bo_ttm->relocs);
329 fprintf(stderr, "Failed to map relocation BO: %s\n",
334 /* Initialize the relocation list with the header:
335 * DWORD 0: relocation type, relocation count
336 * DWORD 1: handle to next relocation list (currently none)
340 bo_ttm->relocs[0] = I915_RELOC_TYPE_0 << 16;
341 bo_ttm->relocs[1] = 0;
342 bo_ttm->relocs[2] = 0;
343 bo_ttm->relocs[3] = 0;
350 driFenceSignaled(DriFenceObject * fence, unsigned type)
358 ret = drmFenceSignaled(bufmgr_ttm->fd, &fence->fence, type, &signaled);
365 dri_ttm_alloc(dri_bufmgr *bufmgr, const char *name,
366 unsigned long size, unsigned int alignment,
367 uint64_t location_mask)
369 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)bufmgr;
371 unsigned int pageSize = getpagesize();
373 unsigned int flags, hint;
375 ttm_buf = malloc(sizeof(*ttm_buf));
379 /* The mask argument doesn't do anything for us that we want other than
380 * determine which pool (TTM or local) the buffer is allocated into, so
381 * just pass all of the allocation class flags.
383 flags = location_mask | DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE |
385 /* No hints we want to use. */
388 ret = drmBOCreate(bufmgr_ttm->fd, size, alignment / pageSize,
389 NULL, flags, hint, &ttm_buf->drm_bo);
394 ttm_buf->bo.size = ttm_buf->drm_bo.size;
395 ttm_buf->bo.offset = ttm_buf->drm_bo.offset;
396 ttm_buf->bo.virtual = NULL;
397 ttm_buf->bo.bufmgr = bufmgr;
398 ttm_buf->name = name;
399 ttm_buf->refcount = 1;
400 ttm_buf->reloc_buf = NULL;
401 ttm_buf->relocs = NULL;
403 DBG("bo_create: %p (%s)\n", &ttm_buf->bo, ttm_buf->name);
408 /* Our TTM backend doesn't allow creation of static buffers, as that requires
409 * privelege for the non-fake case, and the lock in the fake case where we were
410 * working around the X Server not creating buffers and passing handles to us.
413 dri_ttm_alloc_static(dri_bufmgr *bufmgr, const char *name,
414 unsigned long offset, unsigned long size, void *virtual,
415 uint64_t location_mask)
421 * Returns a dri_bo wrapping the given buffer object handle.
423 * This can be used when one application needs to pass a buffer object
427 intel_ttm_bo_create_from_handle(dri_bufmgr *bufmgr, const char *name,
430 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)bufmgr;
434 ttm_buf = malloc(sizeof(*ttm_buf));
438 ret = drmBOReference(bufmgr_ttm->fd, handle, &ttm_buf->drm_bo);
440 fprintf(stderr, "Couldn't reference %s handle 0x%08x: %s\n",
441 name, handle, strerror(-ret));
445 ttm_buf->bo.size = ttm_buf->drm_bo.size;
446 ttm_buf->bo.offset = ttm_buf->drm_bo.offset;
447 ttm_buf->bo.virtual = NULL;
448 ttm_buf->bo.bufmgr = bufmgr;
449 ttm_buf->name = name;
450 ttm_buf->refcount = 1;
451 ttm_buf->reloc_buf = NULL;
452 ttm_buf->relocs = NULL;
454 DBG("bo_create_from_handle: %p %08x (%s)\n",
455 &ttm_buf->bo, handle, ttm_buf->name);
461 dri_ttm_bo_reference(dri_bo *buf)
463 dri_bo_ttm *ttm_buf = (dri_bo_ttm *)buf;
469 dri_ttm_bo_unreference(dri_bo *buf)
471 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)buf->bufmgr;
472 dri_bo_ttm *ttm_buf = (dri_bo_ttm *)buf;
477 if (--ttm_buf->refcount == 0) {
480 if (ttm_buf->reloc_buf) {
481 drmBOUnmap(bufmgr_ttm->fd, ttm_buf->reloc_buf);
482 drmBOUnreference(bufmgr_ttm->fd, ttm_buf->reloc_buf);
483 free(ttm_buf->reloc_buf);
486 ret = drmBOUnreference(bufmgr_ttm->fd, &ttm_buf->drm_bo);
488 fprintf(stderr, "drmBOUnreference failed (%s): %s\n",
489 ttm_buf->name, strerror(-ret));
491 DBG("bo_unreference final: %p (%s)\n", &ttm_buf->bo, ttm_buf->name);
499 dri_ttm_bo_map(dri_bo *buf, GLboolean write_enable)
501 dri_bufmgr_ttm *bufmgr_ttm;
502 dri_bo_ttm *ttm_buf = (dri_bo_ttm *)buf;
505 bufmgr_ttm = (dri_bufmgr_ttm *)buf->bufmgr;
507 flags = DRM_BO_FLAG_READ;
509 flags |= DRM_BO_FLAG_WRITE;
511 assert(buf->virtual == NULL);
513 DBG("bo_map: %p (%s)\n", &ttm_buf->bo, ttm_buf->name);
515 return drmBOMap(bufmgr_ttm->fd, &ttm_buf->drm_bo, flags, 0, &buf->virtual);
519 dri_ttm_bo_unmap(dri_bo *buf)
521 dri_bufmgr_ttm *bufmgr_ttm;
522 dri_bo_ttm *ttm_buf = (dri_bo_ttm *)buf;
527 bufmgr_ttm = (dri_bufmgr_ttm *)buf->bufmgr;
529 assert(buf->virtual != NULL);
533 DBG("bo_unmap: %p (%s)\n", &ttm_buf->bo, ttm_buf->name);
535 return drmBOUnmap(bufmgr_ttm->fd, &ttm_buf->drm_bo);
539 * Returns a dri_bo wrapping the given buffer object handle.
541 * This can be used when one application needs to pass a buffer object
545 intel_ttm_fence_create_from_arg(dri_bufmgr *bufmgr, const char *name,
546 drm_fence_arg_t *arg)
548 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)bufmgr;
549 dri_fence_ttm *ttm_fence;
551 ttm_fence = malloc(sizeof(*ttm_fence));
555 ttm_fence->drm_fence.handle = arg->handle;
556 ttm_fence->drm_fence.fence_class = arg->fence_class;
557 ttm_fence->drm_fence.type = arg->type;
558 ttm_fence->drm_fence.flags = arg->flags;
559 ttm_fence->drm_fence.signaled = 0;
560 ttm_fence->drm_fence.sequence = arg->sequence;
562 ttm_fence->fence.bufmgr = bufmgr;
563 ttm_fence->name = name;
564 ttm_fence->refcount = 1;
566 DBG("fence_create_from_handle: %p (%s)\n",
567 &ttm_fence->fence, ttm_fence->name);
569 return &ttm_fence->fence;
574 dri_ttm_fence_reference(dri_fence *fence)
576 dri_fence_ttm *fence_ttm = (dri_fence_ttm *)fence;
577 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)fence->bufmgr;
579 ++fence_ttm->refcount;
580 DBG("fence_reference: %p (%s)\n", &fence_ttm->fence, fence_ttm->name);
584 dri_ttm_fence_unreference(dri_fence *fence)
586 dri_fence_ttm *fence_ttm = (dri_fence_ttm *)fence;
587 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)fence->bufmgr;
592 DBG("fence_unreference: %p (%s)\n", &fence_ttm->fence, fence_ttm->name);
594 if (--fence_ttm->refcount == 0) {
597 ret = drmFenceUnreference(bufmgr_ttm->fd, &fence_ttm->drm_fence);
599 fprintf(stderr, "drmFenceUnreference failed (%s): %s\n",
600 fence_ttm->name, strerror(-ret));
609 dri_ttm_fence_wait(dri_fence *fence)
611 dri_fence_ttm *fence_ttm = (dri_fence_ttm *)fence;
612 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)fence->bufmgr;
615 ret = drmFenceWait(bufmgr_ttm->fd, DRM_FENCE_FLAG_WAIT_LAZY, &fence_ttm->drm_fence, 0);
617 _mesa_printf("%s:%d: Error %d waiting for fence %s.\n",
618 __FILE__, __LINE__, ret, fence_ttm->name);
622 DBG("fence_wait: %p (%s)\n", &fence_ttm->fence, fence_ttm->name);
626 dri_bufmgr_ttm_destroy(dri_bufmgr *bufmgr)
628 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)bufmgr;
630 intel_free_validate_list(bufmgr_ttm);
636 * Adds the target buffer to the validation list and adds the relocation
637 * to the reloc_buffer's relocation list.
639 * The relocation entry at the given offset must already contain the
640 * precomputed relocation value, because the kernel will optimize out
641 * the relocation entry write when the buffer hasn't moved from the
642 * last known offset in target_buf.
645 dri_ttm_emit_reloc(dri_bo *reloc_buf, uint64_t flags, GLuint delta,
646 GLuint offset, dri_bo *target_buf)
648 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)reloc_buf->bufmgr;
649 dri_bo_ttm *reloc_buf_ttm = (dri_bo_ttm *)reloc_buf;
650 struct intel_bo_node *node;
654 uint32_t *this_reloc;
656 mask = DRM_BO_MASK_MEM;
657 mask |= flags & (DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_EXE);
659 node = intel_add_validate_buffer(bufmgr_ttm, target_buf, flags, mask,
662 intel_setup_reloc_list(reloc_buf);
664 num_relocs = (reloc_buf_ttm->relocs[0] & 0xffff);
667 assert((reloc_buf_ttm->relocs[0] & 0xffff) < bufmgr_ttm->max_relocs);
669 this_reloc = reloc_buf_ttm->relocs + I915_RELOC_HEADER +
670 num_relocs * I915_RELOC0_STRIDE;
672 this_reloc[0] = offset;
673 this_reloc[1] = delta;
674 this_reloc[2] = index;
677 reloc_buf_ttm->relocs[0]++; /* Increment relocation count */
678 /* Check wraparound */
679 assert((reloc_buf_ttm->relocs[0] & 0xffff) != 0);
684 dri_ttm_process_reloc(dri_bo *batch_buf, GLuint *count)
686 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)batch_buf->bufmgr;
690 /* Add the batch buffer to the validation list. There are no relocations
693 intel_add_validate_buffer(bufmgr_ttm, batch_buf,
694 DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_EXE,
695 DRM_BO_MASK_MEM | DRM_BO_FLAG_EXE,
698 ptr = intel_setup_validate_list(bufmgr_ttm, count);
704 intel_update_buffer_offsets (dri_bufmgr_ttm *bufmgr_ttm)
706 struct intel_bo_list *list = &bufmgr_ttm->list;
707 struct intel_bo_node *node;
709 struct drm_i915_op_arg *arg;
710 struct drm_bo_arg_rep *rep;
712 for (l = list->list.next; l != &list->list; l = l->next) {
713 node = DRMLISTENTRY(struct intel_bo_node, l, head);
716 node->bo->offset = rep->bo_info.offset;
721 dri_ttm_post_submit(dri_bo *batch_buf, dri_fence **last_fence)
723 dri_bufmgr_ttm *bufmgr_ttm = (dri_bufmgr_ttm *)batch_buf->bufmgr;
725 intel_update_buffer_offsets (bufmgr_ttm);
727 if (bufmgr_ttm->bufmgr.debug)
728 dri_ttm_dump_validation_list(bufmgr_ttm);
730 intel_free_validate_list(bufmgr_ttm);
734 * Initializes the TTM buffer manager, which uses the kernel to allocate, map,
735 * and manage map buffer objections.
737 * \param fd File descriptor of the opened DRM device.
738 * \param fence_type Driver-specific fence type used for fences with no flush.
739 * \param fence_type_flush Driver-specific fence type used for fences with a
743 intel_bufmgr_ttm_init(int fd, unsigned int fence_type,
744 unsigned int fence_type_flush, int batch_size)
746 dri_bufmgr_ttm *bufmgr_ttm;
748 bufmgr_ttm = malloc(sizeof(*bufmgr_ttm));
750 bufmgr_ttm->fence_type = fence_type;
751 bufmgr_ttm->fence_type_flush = fence_type_flush;
753 /* lets go with one relocation per every four dwords - purely heuristic */
754 bufmgr_ttm->max_relocs = batch_size / sizeof(uint32_t) / 4;
756 intel_init_validate_list(&bufmgr_ttm->list);
758 bufmgr_ttm->bufmgr.bo_alloc = dri_ttm_alloc;
759 bufmgr_ttm->bufmgr.bo_alloc_static = dri_ttm_alloc_static;
760 bufmgr_ttm->bufmgr.bo_reference = dri_ttm_bo_reference;
761 bufmgr_ttm->bufmgr.bo_unreference = dri_ttm_bo_unreference;
762 bufmgr_ttm->bufmgr.bo_map = dri_ttm_bo_map;
763 bufmgr_ttm->bufmgr.bo_unmap = dri_ttm_bo_unmap;
764 bufmgr_ttm->bufmgr.fence_reference = dri_ttm_fence_reference;
765 bufmgr_ttm->bufmgr.fence_unreference = dri_ttm_fence_unreference;
766 bufmgr_ttm->bufmgr.fence_wait = dri_ttm_fence_wait;
767 bufmgr_ttm->bufmgr.destroy = dri_bufmgr_ttm_destroy;
768 bufmgr_ttm->bufmgr.emit_reloc = dri_ttm_emit_reloc;
769 bufmgr_ttm->bufmgr.process_relocs = dri_ttm_process_reloc;
770 bufmgr_ttm->bufmgr.post_submit = dri_ttm_post_submit;
771 bufmgr_ttm->bufmgr.debug = GL_FALSE;
773 return &bufmgr_ttm->bufmgr;