From 3ef1616b63507db01f54efa882a9cf28839cfdf3 Mon Sep 17 00:00:00 2001 From: Christoph Bumiller Date: Thu, 9 Dec 2010 15:01:37 +0100 Subject: [PATCH] nvc0: buffer suballocation with a primitive slab allocator --- src/gallium/drivers/nvc0/Makefile | 3 +- src/gallium/drivers/nvc0/SConscript | 1 + src/gallium/drivers/nvc0/nvc0_buffer.c | 203 ++++++++++++++++++---- src/gallium/drivers/nvc0/nvc0_context.c | 2 + src/gallium/drivers/nvc0/nvc0_context.h | 2 + src/gallium/drivers/nvc0/nvc0_fence.c | 57 +++++-- src/gallium/drivers/nvc0/nvc0_fence.h | 29 ++-- src/gallium/drivers/nvc0/nvc0_graph_macros.h | 2 +- src/gallium/drivers/nvc0/nvc0_mm.c | 245 +++++++++++++++++++++++++++ src/gallium/drivers/nvc0/nvc0_push.c | 4 +- src/gallium/drivers/nvc0/nvc0_resource.h | 46 ++++- src/gallium/drivers/nvc0/nvc0_screen.c | 55 ++++-- src/gallium/drivers/nvc0/nvc0_screen.h | 61 +++++++ src/gallium/drivers/nvc0/nvc0_shader_state.c | 8 +- src/gallium/drivers/nvc0/nvc0_tgsi_to_nc.c | 6 +- src/gallium/drivers/nvc0/nvc0_transfer.c | 119 +++++++++---- src/gallium/drivers/nvc0/nvc0_vbo.c | 97 ++++++++--- src/gallium/drivers/nvc0/nvc0_winsys.h | 16 ++ 18 files changed, 807 insertions(+), 149 deletions(-) create mode 100644 src/gallium/drivers/nvc0/nvc0_mm.c diff --git a/src/gallium/drivers/nvc0/Makefile b/src/gallium/drivers/nvc0/Makefile index 7aefd6f1f3e..5c3d46d9eab 100644 --- a/src/gallium/drivers/nvc0/Makefile +++ b/src/gallium/drivers/nvc0/Makefile @@ -27,6 +27,7 @@ C_SOURCES = \ nvc0_pc_regalloc.c \ nvc0_push.c \ nvc0_push2.c \ - nvc0_fence.c + nvc0_fence.c \ + nvc0_mm.c include ../../Makefile.template diff --git a/src/gallium/drivers/nvc0/SConscript b/src/gallium/drivers/nvc0/SConscript index 808d6895f11..ec529d44f5f 100644 --- a/src/gallium/drivers/nvc0/SConscript +++ b/src/gallium/drivers/nvc0/SConscript @@ -29,6 +29,7 @@ nvc0 = env.ConvenienceLibrary( 'nvc0_push.c', 'nvc0_push2.c', 'nvc0_fence.c', + 'nvc0_mm' ]) Export('nvc0') diff --git a/src/gallium/drivers/nvc0/nvc0_buffer.c b/src/gallium/drivers/nvc0/nvc0_buffer.c index 873016f0d5f..93d7f5d3033 100644 --- a/src/gallium/drivers/nvc0/nvc0_buffer.c +++ b/src/gallium/drivers/nvc0/nvc0_buffer.c @@ -11,46 +11,116 @@ #include "nvc0_context.h" #include "nvc0_resource.h" +#define NVC0_BUFFER_STATUS_USER_MEMORY 0xff + +static INLINE boolean +nvc0_buffer_allocate(struct nvc0_screen *screen, struct nvc0_resource *buf, + unsigned domain) +{ + if (domain == NOUVEAU_BO_VRAM) { + buf->mm = nvc0_mm_allocate(screen->mm_VRAM, buf->base.width0, &buf->bo, + &buf->offset); + if (!buf->bo) + return nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_GART); + } else + if (domain == NOUVEAU_BO_GART) { + buf->mm = nvc0_mm_allocate(screen->mm_GART, buf->base.width0, &buf->bo, + &buf->offset); + if (!buf->bo) + return FALSE; + } else { + assert(!domain); + if (!buf->data) + buf->data = MALLOC(buf->base.width0); + if (!buf->data) + return FALSE; + } + buf->domain = domain; + return TRUE; +} + +static INLINE void +release_allocation(struct nvc0_mm_allocation **mm, struct nvc0_fence *fence) +{ + (*mm)->next = fence->buffers; + fence->buffers = (*mm); + (*mm) = NULL; +} + static void nvc0_buffer_destroy(struct pipe_screen *pscreen, struct pipe_resource *presource) { + struct nvc0_screen *screen = nvc0_screen(pscreen); struct nvc0_resource *res = nvc0_resource(presource); - if (res->bo) - nouveau_screen_bo_release(pscreen, res->bo); + nouveau_bo_ref(NULL, &res->bo); - if (res->data) + if (res->mm) + release_allocation(&res->mm, screen->fence.current); + + if (res->status != NVC0_BUFFER_STATUS_USER_MEMORY && res->data) FREE(res->data); FREE(res); } +static INLINE uint32_t +nouveau_buffer_rw_flags(unsigned pipe) +{ + uint32_t flags = 0; + + if (pipe & PIPE_TRANSFER_READ) + flags = NOUVEAU_BO_RD; + if (pipe & PIPE_TRANSFER_WRITE) + flags |= NOUVEAU_BO_WR; + + return flags; +} + static void * nvc0_buffer_transfer_map(struct pipe_context *pipe, struct pipe_transfer *transfer) { struct nvc0_resource *res = nvc0_resource(transfer->resource); + struct nvc0_fence *fence; uint8_t *map; - uint32_t flags; + int ret; + uint32_t flags = nouveau_buffer_rw_flags(transfer->usage); - if (res->base.bind & PIPE_BIND_VERTEX_BUFFER) + if ((res->base.bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER)) && + (flags & NOUVEAU_BO_WR)) nvc0_context(pipe)->vbo_dirty = TRUE; -// #ifdef NOUVEAU_USERPSACE_MM - if (res->base.bind & PIPE_BIND_CONSTANT_BUFFER) + if (res->domain == 0) return res->data + transfer->box.x; -// #endif - flags = nouveau_screen_transfer_flags(transfer->usage); - map = nouveau_screen_bo_map_range(pipe->screen, - res->bo, - transfer->box.x, transfer->box.width, - flags); - if (!map) + if (res->domain == NOUVEAU_BO_VRAM) { + NOUVEAU_ERR("transfers to/from VRAM buffers are not allowed\n"); + /* if this happens, migrate back to GART */ return NULL; + } + + if (res->score > -1024) + --res->score; + + ret = nouveau_bo_map(res->bo, flags | NOUVEAU_BO_NOSYNC); + if (ret) + return NULL; + map = res->bo->map; + nouveau_bo_unmap(res->bo); + + fence = (flags == NOUVEAU_BO_RD) ? res->fence_wr : res->fence; - return map + transfer->box.x; + if (fence) { + if (nvc0_fence_wait(fence) == FALSE) + NOUVEAU_ERR("failed to fence buffer\n"); + + nvc0_fence_reference(&res->fence, NULL); + nvc0_fence_reference(&res->fence_wr, NULL); + } + + return map + transfer->box.x + res->offset; } @@ -62,13 +132,12 @@ nvc0_buffer_transfer_flush_region(struct pipe_context *pipe, { struct nvc0_resource *res = nvc0_resource(transfer->resource); -#ifdef NOUVEAU_USERPSACE_MM if (!res->bo) return; -#endif + nouveau_screen_bo_map_flush_range(pipe->screen, res->bo, - transfer->box.x + box->x, + res->offset + transfer->box.x + box->x, box->width); } @@ -78,11 +147,10 @@ nvc0_buffer_transfer_unmap(struct pipe_context *pipe, { struct nvc0_resource *res = nvc0_resource(transfer->resource); -// #ifdef NOUVEAU_USERPSACE_MM if (res->data) return; -// #endif - nouveau_screen_bo_unmap(pipe->screen, res->bo); + + /* nouveau_screen_bo_unmap(pipe->screen, res->bo); */ } const struct u_resource_vtbl nvc0_buffer_vtbl = @@ -102,7 +170,9 @@ struct pipe_resource * nvc0_buffer_create(struct pipe_screen *pscreen, const struct pipe_resource *templ) { + struct nvc0_screen *screen = nvc0_screen(pscreen); struct nvc0_resource *buffer; + boolean ret; buffer = CALLOC_STRUCT(nvc0_resource); if (!buffer) @@ -114,14 +184,11 @@ nvc0_buffer_create(struct pipe_screen *pscreen, buffer->base.screen = pscreen; if (buffer->base.bind & PIPE_BIND_CONSTANT_BUFFER) - buffer->data = MALLOC(buffer->base.width0); - - buffer->bo = nouveau_screen_bo_new(pscreen, - 16, - buffer->base.usage, - buffer->base.bind, - buffer->base.width0); - if (buffer->bo == NULL) + ret = nvc0_buffer_allocate(screen, buffer, 0); + else + ret = nvc0_buffer_allocate(screen, buffer, NOUVEAU_BO_GART); + + if (ret == FALSE) goto fail; return &buffer->base; @@ -154,13 +221,77 @@ nvc0_user_buffer_create(struct pipe_screen *pscreen, buffer->base.height0 = 1; buffer->base.depth0 = 1; - buffer->bo = nouveau_screen_bo_user(pscreen, ptr, bytes); - if (!buffer->bo) - goto fail; - + buffer->data = ptr; + buffer->status = NVC0_BUFFER_STATUS_USER_MEMORY; + return &buffer->base; +} -fail: - FREE(buffer); - return NULL; +/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */ +boolean +nvc0_buffer_migrate(struct nvc0_context *nvc0, + struct nvc0_resource *buf, unsigned domain) +{ + struct nvc0_screen *screen = nvc0_screen(buf->base.screen); + struct nouveau_bo *bo; + unsigned size = buf->base.width0; + int ret; + + if (domain == NOUVEAU_BO_GART && buf->domain == 0) { + if (!nvc0_buffer_allocate(screen, buf, domain)) + return FALSE; + ret = nouveau_bo_map(buf->bo, NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC); + if (ret) + return ret; + memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size); + nouveau_bo_unmap(buf->bo); + } else + if (domain == NOUVEAU_BO_VRAM && buf->domain == NOUVEAU_BO_GART) { + struct nvc0_mm_allocation *mm = buf->mm; + + bo = buf->bo; + buf->bo = NULL; + buf->mm = NULL; + nvc0_buffer_allocate(screen, buf, domain); + + nvc0_m2mf_copy_linear(nvc0, buf->bo, 0, NOUVEAU_BO_VRAM, + bo, 0, NOUVEAU_BO_GART, buf->base.width0); + + release_allocation(&mm, screen->fence.current); + nouveau_bo_ref(NULL, &bo); + } else + if (domain == NOUVEAU_BO_VRAM && buf->domain == 0) { + /* should use a scratch buffer instead here */ + if (!nvc0_buffer_migrate(nvc0, buf, NOUVEAU_BO_GART)) + return FALSE; + return nvc0_buffer_migrate(nvc0, buf, NOUVEAU_BO_VRAM); + } else + return -1; + + buf->domain = domain; + + return TRUE; +} + +/* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART. + * MUST NOT FLUSH THE PUSH BUFFER, we could be in the middle of a method. + */ +boolean +nvc0_migrate_vertices(struct nvc0_resource *buf, unsigned base, unsigned size) +{ + struct nvc0_screen *screen = nvc0_screen(buf->base.screen); + int ret; + + assert(buf->data && !buf->domain); + + if (!nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_GART)) + return FALSE; + ret = nouveau_bo_map_range(buf->bo, base + buf->offset, size, + NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC); + if (ret) + return FALSE; + memcpy(buf->bo->map, buf->data + base, size); + nouveau_bo_unmap(buf->bo); + + return TRUE; } diff --git a/src/gallium/drivers/nvc0/nvc0_context.c b/src/gallium/drivers/nvc0/nvc0_context.c index f718feccd74..a05408a6780 100644 --- a/src/gallium/drivers/nvc0/nvc0_context.c +++ b/src/gallium/drivers/nvc0/nvc0_context.c @@ -49,6 +49,8 @@ nvc0_flush(struct pipe_context *pipe, unsigned flags, if (flags & (PIPE_FLUSH_SWAPBUFFERS | PIPE_FLUSH_FRAME)) { FIRE_RING(chan); + + nvc0_screen_fence_next(nvc0->screen); } } diff --git a/src/gallium/drivers/nvc0/nvc0_context.h b/src/gallium/drivers/nvc0/nvc0_context.h index d0f78a564d7..26263309d8e 100644 --- a/src/gallium/drivers/nvc0/nvc0_context.h +++ b/src/gallium/drivers/nvc0/nvc0_context.h @@ -118,6 +118,7 @@ struct nvc0_context { unsigned sample_mask; boolean vbo_dirty; + boolean vbo_push_hint; struct draw_context *draw; }; @@ -150,6 +151,7 @@ static INLINE void nvc0_make_buffer_resident(struct nvc0_context *nvc0, struct nvc0_resource *res, unsigned flags) { + nvc0_resource_validate(res, flags); nvc0_make_bo_resident(nvc0, res->bo, flags); } diff --git a/src/gallium/drivers/nvc0/nvc0_fence.c b/src/gallium/drivers/nvc0/nvc0_fence.c index 3587097449c..dc2abe45bde 100644 --- a/src/gallium/drivers/nvc0/nvc0_fence.c +++ b/src/gallium/drivers/nvc0/nvc0_fence.c @@ -30,14 +30,14 @@ boolean nvc0_screen_fence_new(struct nvc0_screen *screen, struct nvc0_fence **fence, - boolean emit) + boolean emit) { *fence = CALLOC_STRUCT(nvc0_fence); if (!*fence) return FALSE; (*fence)->screen = screen; - pipe_reference_init(&(*fence)->reference, 1); + (*fence)->ref = 1; if (emit) nvc0_fence_emit(*fence); @@ -53,15 +53,15 @@ nvc0_fence_emit(struct nvc0_fence *fence) fence->sequence = ++screen->fence.sequence; - assert(!(fence->state & NVC0_FENCE_STATE_EMITTED)); + assert(fence->state == NVC0_FENCE_STATE_AVAILABLE); BEGIN_RING(chan, RING_3D(QUERY_ADDRESS_HIGH), 4); OUT_RELOCh(chan, screen->fence.bo, 0, NOUVEAU_BO_WR); OUT_RELOCl(chan, screen->fence.bo, 0, NOUVEAU_BO_WR); OUT_RING (chan, fence->sequence); - OUT_RING (chan, 0x1000f010); + OUT_RING (chan, NVC0_3D_QUERY_GET_FENCE); - pipe_reference(NULL, &fence->reference); + ++fence->ref; if (screen->fence.tail) screen->fence.tail->next = fence; @@ -95,6 +95,18 @@ nvc0_fence_del(struct nvc0_fence *fence) } static void +nvc0_fence_trigger_release_buffers(struct nvc0_fence *fence) +{ + struct nvc0_mm_allocation *alloc = fence->buffers; + + while (alloc) { + struct nvc0_mm_allocation *next = alloc->next; + nvc0_mm_free(alloc); + alloc = next; + }; +} + +static void nvc0_screen_fence_update(struct nvc0_screen *screen) { struct nvc0_fence *fence; @@ -110,10 +122,12 @@ nvc0_screen_fence_update(struct nvc0_screen *screen) sequence = fence->sequence; fence->state = NVC0_FENCE_STATE_SIGNALLED; - if (fence->trigger.func) - fence->trigger.func(fence->trigger.arg); + + if (fence->buffers) + nvc0_fence_trigger_release_buffers(fence); nvc0_fence_reference(&fence, NULL); + if (sequence == screen->fence.sequence_ack) break; } @@ -122,24 +136,45 @@ nvc0_screen_fence_update(struct nvc0_screen *screen) screen->fence.tail = NULL; } +#define NVC0_FENCE_MAX_SPINS (1 << 17) + boolean nvc0_fence_wait(struct nvc0_fence *fence) { struct nvc0_screen *screen = fence->screen; int spins = 0; - if (fence->state != NVC0_FENCE_STATE_EMITTED) - return TRUE; + if (fence->state == NVC0_FENCE_STATE_AVAILABLE) { + nvc0_fence_emit(fence); + + FIRE_RING(screen->base.channel); + + if (fence == screen->fence.current) + nvc0_screen_fence_new(screen, &screen->fence.current, FALSE); + } + do { nvc0_screen_fence_update(screen); if (fence->state == NVC0_FENCE_STATE_SIGNALLED) return TRUE; + spins++; #ifdef PIPE_OS_UNIX - if ((spins & 7) == 7) /* spend a few cycles */ + if (!(spins % 8)) /* donate a few cycles */ sched_yield(); #endif - } while (++spins < 10000); + } while (spins < NVC0_FENCE_MAX_SPINS); + + if (spins > 9000) + NOUVEAU_ERR("fence %x: been spinning too long\n", fence->sequence); return FALSE; } + +void +nvc0_screen_fence_next(struct nvc0_screen *screen) +{ + nvc0_fence_emit(screen->fence.current); + nvc0_screen_fence_new(screen, &screen->fence.current, FALSE); + nvc0_screen_fence_update(screen); +} diff --git a/src/gallium/drivers/nvc0/nvc0_fence.h b/src/gallium/drivers/nvc0/nvc0_fence.h index 513ac07c1a8..7b31f28808c 100644 --- a/src/gallium/drivers/nvc0/nvc0_fence.h +++ b/src/gallium/drivers/nvc0/nvc0_fence.h @@ -3,24 +3,21 @@ #define __NVC0_FENCE_H__ #include "util/u_inlines.h" +#include "util/u_double_list.h" -struct nvc0_fence_trigger { - void (*func)(void *); - void *arg; - struct nvc0_fence_trigger *next; -}; - +#define NVC0_FENCE_STATE_AVAILABLE 0 #define NVC0_FENCE_STATE_EMITTED 1 #define NVC0_FENCE_STATE_SIGNALLED 2 -/* reference first, so pipe_reference works directly */ +struct nvc0_mm_allocation; + struct nvc0_fence { - struct pipe_reference reference; struct nvc0_fence *next; struct nvc0_screen *screen; int state; + int ref; uint32_t sequence; - struct nvc0_fence_trigger trigger; + struct nvc0_mm_allocation *buffers; }; void nvc0_fence_emit(struct nvc0_fence *); @@ -31,10 +28,20 @@ boolean nvc0_fence_wait(struct nvc0_fence *); static INLINE void nvc0_fence_reference(struct nvc0_fence **ref, struct nvc0_fence *fence) { - if (pipe_reference(&(*ref)->reference, &fence->reference)) - nvc0_fence_del(*ref); + if (*ref) { + if (--(*ref)->ref == 0) + nvc0_fence_del(*ref); + } + if (fence) + ++fence->ref; *ref = fence; } +static INLINE struct nvc0_fence * +nvc0_fence(struct pipe_fence_handle *fence) +{ + return (struct nvc0_fence *)fence; +} + #endif // __NVC0_FENCE_H__ diff --git a/src/gallium/drivers/nvc0/nvc0_graph_macros.h b/src/gallium/drivers/nvc0/nvc0_graph_macros.h index b8b89de28e0..7db09287ab3 100644 --- a/src/gallium/drivers/nvc0/nvc0_graph_macros.h +++ b/src/gallium/drivers/nvc0/nvc0_graph_macros.h @@ -63,7 +63,7 @@ static const uint32_t nvc0_9097_vertex_array_select[] = static const uint32_t nvc0_9097_color_mask_brdc[] = { - 0x05a00021, /* maddr [0x1680] */ + 0x05a00021, /* maddr [0x1a00, increment = 4] */ 0x00000841, /* send $r1 */ 0x00000841, /* send $r1 */ 0x00000841, /* send $r1 */ diff --git a/src/gallium/drivers/nvc0/nvc0_mm.c b/src/gallium/drivers/nvc0/nvc0_mm.c new file mode 100644 index 00000000000..e031fb393ac --- /dev/null +++ b/src/gallium/drivers/nvc0/nvc0_mm.c @@ -0,0 +1,245 @@ + +#include "util/u_inlines.h" +#include "util/u_memory.h" +#include "util/u_double_list.h" + +#include "nvc0_screen.h" + +#define MM_MIN_ORDER 7 +#define MM_MAX_ORDER 20 + +#define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1) + +#define MM_MIN_SIZE (1 << MM_MIN_ORDER) +#define MM_MAX_SIZE (1 << MM_MAX_ORDER) + +struct mm_bucket { + struct list_head free; + struct list_head used; + struct list_head full; + int num_free; +}; + +struct nvc0_mman { + struct nouveau_device *dev; + struct mm_bucket bucket[MM_NUM_BUCKETS]; + uint32_t storage_type; + uint32_t domain; + uint64_t allocated; +}; + +struct mm_slab { + struct list_head head; + struct nouveau_bo *bo; + struct nvc0_mman *cache; + int order; + int count; + int free; + uint32_t bits[0]; +}; + +static int +mm_slab_alloc(struct mm_slab *slab) +{ + int i, n, b; + + if (slab->free == 0) + return -1; + + for (i = 0; i < (slab->count + 31) / 32; ++i) { + b = ffs(slab->bits[i]) - 1; + if (b >= 0) { + n = i * 32 + b; + assert(n < slab->count); + slab->free--; + slab->bits[i] &= ~(1 << b); + return n; + } + } + return -1; +} + +static INLINE void +mm_slab_free(struct mm_slab *slab, int i) +{ + assert(i < slab->count); + slab->bits[i / 32] |= 1 << (i % 32); + slab->free++; + assert(slab->free <= slab->count); +} + +static INLINE int +mm_get_order(uint32_t size) +{ + int s = __builtin_clz(size) ^ 31; + + if (size > (1 << s)) + s += 1; + return s; +} + +static struct mm_bucket * +mm_bucket_by_order(struct nvc0_mman *cache, int order) +{ + if (order > MM_MAX_ORDER) + return NULL; + return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER]; +} + +static struct mm_bucket * +mm_bucket_by_size(struct nvc0_mman *cache, unsigned size) +{ + return mm_bucket_by_order(cache, mm_get_order(size)); +} + +/* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */ +static INLINE uint32_t +mm_default_slab_size(unsigned chunk_order) +{ + assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER); + + static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] = + { + 12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22 + }; + + return 1 << slab_order[chunk_order - MM_MIN_ORDER]; +} + +static int +mm_slab_new(struct nvc0_mman *cache, int chunk_order) +{ + struct mm_slab *slab; + int words, ret; + const uint32_t size = mm_default_slab_size(chunk_order); + + words = ((size >> chunk_order) + 31) / 32; + assert(words); + + slab = MALLOC(sizeof(struct mm_slab) + words * 4); + if (!slab) + return PIPE_ERROR_OUT_OF_MEMORY; + + memset(&slab->bits[0], ~0, words * 4); + + slab->bo = NULL; + ret = nouveau_bo_new_tile(cache->dev, cache->domain, 0, size, + 0, cache->storage_type, &slab->bo); + if (ret) { + FREE(slab); + return PIPE_ERROR_OUT_OF_MEMORY; + } + + LIST_INITHEAD(&slab->head); + + slab->cache = cache; + slab->order = chunk_order; + slab->count = slab->free = size >> chunk_order; + + LIST_ADD(&slab->head, &mm_bucket_by_order(cache, chunk_order)->free); + + cache->allocated += size; + + debug_printf("MM: new slab, total memory = %lu KiB\n", + cache->allocated / 1024); + + return PIPE_OK; +} + +/* @return token to identify slab or NULL if we just allocated a new bo */ +struct nvc0_mm_allocation * +nvc0_mm_allocate(struct nvc0_mman *cache, + uint32_t size, struct nouveau_bo **bo, uint32_t *offset) +{ + struct mm_bucket *bucket; + struct mm_slab *slab; + struct nvc0_mm_allocation *alloc; + int ret; + + bucket = mm_bucket_by_size(cache, size); + if (!bucket) { + ret = nouveau_bo_new_tile(cache->dev, cache->domain, 0, size, + 0, cache->storage_type, bo); + if (ret) + debug_printf("bo_new(%x, %x): %i\n", size, cache->storage_type, ret); + + *offset = 0; + return NULL; + } + + if (!LIST_IS_EMPTY(&bucket->used)) { + slab = LIST_ENTRY(struct mm_slab, bucket->used.next, head); + } else { + if (LIST_IS_EMPTY(&bucket->free)) { + mm_slab_new(cache, MAX2(mm_get_order(size), MM_MIN_ORDER)); + } + slab = LIST_ENTRY(struct mm_slab, bucket->free.next, head); + + LIST_DEL(&slab->head); + LIST_ADD(&slab->head, &bucket->used); + } + + *offset = mm_slab_alloc(slab) << slab->order; + + alloc = MALLOC_STRUCT(nvc0_mm_allocation); + if (!alloc) + return NULL; + + nouveau_bo_ref(slab->bo, bo); + + if (slab->free == 0) { + LIST_DEL(&slab->head); + LIST_ADD(&slab->head, &bucket->full); + } + + alloc->next = NULL; + alloc->offset = *offset; + alloc->priv = (void *)slab; + + return alloc; +} + +void +nvc0_mm_free(struct nvc0_mm_allocation *alloc) +{ + struct mm_slab *slab = (struct mm_slab *)alloc->priv; + struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order); + + mm_slab_free(slab, alloc->offset >> slab->order); + + if (slab->free == 1) { + LIST_DEL(&slab->head); + + if (slab->count > 1) + LIST_ADDTAIL(&slab->head, &bucket->used); + else + LIST_ADDTAIL(&slab->head, &bucket->free); + } + + FREE(alloc); +} + +struct nvc0_mman * +nvc0_mm_create(struct nouveau_device *dev, uint32_t domain, + uint32_t storage_type) +{ + struct nvc0_mman *cache = MALLOC_STRUCT(nvc0_mman); + int i; + + if (!cache) + return NULL; + + cache->dev = dev; + cache->domain = domain; + cache->storage_type = storage_type; + cache->allocated = 0; + + for (i = 0; i < MM_NUM_BUCKETS; ++i) { + LIST_INITHEAD(&cache->bucket[i].free); + LIST_INITHEAD(&cache->bucket[i].used); + LIST_INITHEAD(&cache->bucket[i].full); + } + + return cache; +} + diff --git a/src/gallium/drivers/nvc0/nvc0_push.c b/src/gallium/drivers/nvc0/nvc0_push.c index 8b8fe610e2d..1bdc8e88a72 100644 --- a/src/gallium/drivers/nvc0/nvc0_push.c +++ b/src/gallium/drivers/nvc0/nvc0_push.c @@ -215,9 +215,7 @@ nvc0_push_vbo(struct nvc0_context *nvc0, const struct pipe_draw_info *info) struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[i]; struct nvc0_resource *res = nvc0_resource(vb->buffer); - if (nouveau_bo_map(res->bo, NOUVEAU_BO_RD)) - return; - data = (uint8_t *)res->bo->map + vb->buffer_offset; + data = nvc0_resource_map_offset(res, vb->buffer_offset, NOUVEAU_BO_RD); if (info->indexed) data += info->index_bias * vb->stride; diff --git a/src/gallium/drivers/nvc0/nvc0_resource.h b/src/gallium/drivers/nvc0/nvc0_resource.h index 9c6895ea81d..b9f3f7b5d81 100644 --- a/src/gallium/drivers/nvc0/nvc0_resource.h +++ b/src/gallium/drivers/nvc0/nvc0_resource.h @@ -29,10 +29,43 @@ struct nvc0_resource { uint8_t status; uint8_t domain; + + int16_t score; /* low if mapped very often, if high can move to VRAM */ + struct nvc0_fence *fence; - struct list_head list; + struct nvc0_fence *fence_wr; + + struct nvc0_mm_allocation *mm; }; +/* XXX: wait for fence (atm only using this for vertex push) */ +static INLINE void * +nvc0_resource_map_offset(struct nvc0_resource *res, uint32_t offset, + uint32_t flags) +{ + void *map; + + if (res->domain == 0) + return res->data + offset; + + if (nouveau_bo_map_range(res->bo, res->offset + offset, + res->base.width0, flags | NOUVEAU_BO_NOSYNC)) + return NULL; + + /* With suballocation, the same bo can be mapped several times, so unmap + * immediately. Maps are guaranteed to persist. */ + map = res->bo->map; + nouveau_bo_unmap(res->bo); + return map; +} + +static INLINE void +nvc0_resource_unmap(struct nvc0_resource *res) +{ + if (res->domain != 0 && 0) + nouveau_bo_unmap(res->bo); +} + #define NVC0_TILE_H(m) (8 << ((m >> 4) & 0xf)) #define NVC0_TILE_D(m) (1 << (m >> 8)) @@ -67,7 +100,7 @@ nvc0_resource(struct pipe_resource *resource) static INLINE boolean nvc0_resource_mapped_by_gpu(struct pipe_resource *resource) { - return nvc0_resource(resource)->bo->offset != 0ULL; + return nvc0_resource(resource)->domain != 0; } void @@ -106,4 +139,13 @@ nvc0_miptree_surface_new(struct pipe_screen *pscreen, struct pipe_resource *pt, void nvc0_miptree_surface_del(struct pipe_surface *ps); +struct nvc0_context; + +boolean +nvc0_buffer_migrate(struct nvc0_context *, + struct nvc0_resource *, unsigned domain); + +boolean +nvc0_migrate_vertices(struct nvc0_resource *buf, unsigned base, unsigned size); + #endif diff --git a/src/gallium/drivers/nvc0/nvc0_screen.c b/src/gallium/drivers/nvc0/nvc0_screen.c index 616a9903371..107d50d7f2b 100644 --- a/src/gallium/drivers/nvc0/nvc0_screen.c +++ b/src/gallium/drivers/nvc0/nvc0_screen.c @@ -229,12 +229,28 @@ nvc0_graph_set_macro(struct nvc0_screen *screen, uint32_t m, unsigned pos, return pos + size; } +static void +nvc0_screen_fence_reference(struct pipe_screen *pscreen, + struct pipe_fence_handle **ptr, + struct pipe_fence_handle *fence) +{ + nvc0_fence_reference((struct nvc0_fence **)ptr, nvc0_fence(fence)); +} + +static int +nvc0_screen_fence_signalled(struct pipe_screen *pscreen, + struct pipe_fence_handle *fence, + unsigned flags) +{ + return !(((struct nvc0_fence *)fence)->state == NVC0_FENCE_STATE_SIGNALLED); +} + static int nvc0_screen_fence_finish(struct pipe_screen *pscreen, - struct pipe_fence_handle *pfence, + struct pipe_fence_handle *fence, unsigned flags) { - return nvc0_fence_wait((struct nvc0_fence *)pfence) != TRUE; + return nvc0_fence_wait((struct nvc0_fence *)fence) != TRUE; } static void @@ -339,6 +355,8 @@ nvc0_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev) pscreen->get_param = nvc0_screen_get_param; pscreen->get_shader_param = nvc0_screen_get_shader_param; pscreen->get_paramf = nvc0_screen_get_paramf; + pscreen->fence_reference = nvc0_screen_fence_reference; + pscreen->fence_signalled = nvc0_screen_fence_signalled; pscreen->fence_finish = nvc0_screen_fence_finish; nvc0_screen_init_resource_functions(pscreen); @@ -353,6 +371,18 @@ nvc0_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev) screen->fence.map = screen->fence.bo->map; nouveau_bo_unmap(screen->fence.bo); + for (i = 0; i < NVC0_SCRATCH_NR_BUFFERS; ++i) { + ret = nouveau_bo_new(dev, NOUVEAU_BO_GART, 0, NVC0_SCRATCH_SIZE, + &screen->scratch.bo[i]); + if (ret) + goto fail; + } + + for (i = 0; i < 8; ++i) { + BEGIN_RING(chan, (i << 13) | (0x0000 >> 2), 1); + OUT_RING (chan, 0x0000); + } + BEGIN_RING(chan, RING_MF_(0x0000), 1); OUT_RING (chan, 0x9039); BEGIN_RING(chan, RING_MF(NOTIFY_ADDRESS_HIGH), 3); @@ -510,15 +540,11 @@ nvc0_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev) BEGIN_RING(chan, RING_3D(RASTERIZE_ENABLE), 1); OUT_RING (chan, 1); - // BEGIN_RING(chan, RING_3D(GP_SELECT), 1); - // OUT_RING (chan, 0x40); - BEGIN_RING(chan, RING_3D(SP_SELECT(4)), 1); + BEGIN_RING(chan, RING_3D(GP_SELECT), 1); OUT_RING (chan, 0x40); BEGIN_RING(chan, RING_3D(GP_BUILTIN_RESULT_EN), 1); OUT_RING (chan, 0); - // BEGIN_RING(chan, RING_3D(TEP_SELECT), 1); - // OUT_RING (chan, 0x30); - BEGIN_RING(chan, RING_3D(SP_SELECT(3)), 1); + BEGIN_RING(chan, RING_3D(TEP_SELECT), 1); OUT_RING (chan, 0x30); BEGIN_RING(chan, RING_3D(PATCH_VERTICES), 1); OUT_RING (chan, 3); @@ -538,18 +564,19 @@ nvc0_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev) BEGIN_RING(chan, RING_3D(VERTEX_RUNOUT_ADDRESS_HIGH), 2); OUT_RING (chan, 0xab); OUT_RING (chan, 0x00000000); - BEGIN_RING(chan, RING_3D_(0x07e8), 2); - OUT_RING (chan, 0xac); - OUT_RING (chan, 0x00000000); - BEGIN_RING(chan, RING_3D_(0x07f0), 2); - OUT_RING (chan, 0xac); - OUT_RING (chan, 0x00000000); FIRE_RING (chan); screen->tic.entries = CALLOC(4096, sizeof(void *)); screen->tsc.entries = screen->tic.entries + 2048; + screen->mm_GART = nvc0_mm_create(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, + 0x000); + screen->mm_VRAM = nvc0_mm_create(dev, NOUVEAU_BO_VRAM, 0x000); + screen->mm_VRAM_fe0 = nvc0_mm_create(dev, NOUVEAU_BO_VRAM, 0xfe0); + + nvc0_screen_fence_new(screen, &screen->fence.current, FALSE); + return pscreen; fail: diff --git a/src/gallium/drivers/nvc0/nvc0_screen.h b/src/gallium/drivers/nvc0/nvc0_screen.h index 989e183bdef..12cea658a5a 100644 --- a/src/gallium/drivers/nvc0/nvc0_screen.h +++ b/src/gallium/drivers/nvc0/nvc0_screen.h @@ -10,9 +10,13 @@ #define NVC0_TIC_MAX_ENTRIES 2048 #define NVC0_TSC_MAX_ENTRIES 2048 +struct nvc0_mman; struct nvc0_context; struct nvc0_fence; +#define NVC0_SCRATCH_SIZE (2 << 20) +#define NVC0_SCRATCH_NR_BUFFERS 2 + struct nvc0_screen { struct nouveau_screen base; struct nouveau_winsys *nvws; @@ -30,6 +34,13 @@ struct nvc0_screen { struct nouveau_resource *text_heap; struct { + struct nouveau_bo *bo[NVC0_SCRATCH_NR_BUFFERS]; + uint8_t *buf; + int index; + uint32_t offset; + } scratch; + + struct { void **entries; int next; uint32_t lock[NVC0_TIC_MAX_ENTRIES / 32]; @@ -50,6 +61,10 @@ struct nvc0_screen { uint32_t sequence_ack; struct nouveau_bo *bo; } fence; + + struct nvc0_mman *mm_GART; + struct nvc0_mman *mm_VRAM; + struct nvc0_mman *mm_VRAM_fe0; }; static INLINE struct nvc0_screen * @@ -58,14 +73,60 @@ nvc0_screen(struct pipe_screen *screen) return (struct nvc0_screen *)screen; } +/* Since a resource can be migrated, we need to decouple allocations from + * them. This struct is linked with fences for delayed freeing of allocs. + */ +struct nvc0_mm_allocation { + struct nvc0_mm_allocation *next; + void *priv; + uint32_t offset; +}; + +extern struct nvc0_mman * +nvc0_mm_create(struct nouveau_device *, uint32_t domain, uint32_t storage_type); + +extern struct nvc0_mm_allocation * +nvc0_mm_allocate(struct nvc0_mman *, + uint32_t size, struct nouveau_bo **, uint32_t *offset); +extern void +nvc0_mm_free(struct nvc0_mm_allocation *); + void nvc0_screen_make_buffers_resident(struct nvc0_screen *); int nvc0_screen_tic_alloc(struct nvc0_screen *, void *); int nvc0_screen_tsc_alloc(struct nvc0_screen *, void *); +static INLINE void +nvc0_resource_validate(struct nvc0_resource *res, uint32_t flags) +{ + struct nvc0_screen *screen = nvc0_screen(res->base.screen); + + assert(res->mm); + + nvc0_fence_reference(&res->fence, screen->fence.current); + + if (flags & NOUVEAU_BO_WR) + nvc0_fence_reference(&res->fence_wr, screen->fence.current); + + nouveau_reloc_emit(screen->base.channel, + NULL, 0, NULL, res->bo, 0, 0, NOUVEAU_BO_RDWR, 0, 0); +} + + boolean nvc0_screen_fence_new(struct nvc0_screen *, struct nvc0_fence **, boolean emit); +void +nvc0_screen_fence_next(struct nvc0_screen *); + +static INLINE boolean +nvc0_screen_fence_emit(struct nvc0_screen *screen) +{ + nvc0_fence_emit(screen->fence.current); + + return nvc0_screen_fence_new(screen, &screen->fence.current, FALSE); +} + struct nvc0_format { uint32_t rt; uint32_t tic; diff --git a/src/gallium/drivers/nvc0/nvc0_shader_state.c b/src/gallium/drivers/nvc0/nvc0_shader_state.c index 932848ea94b..25a9ef722b5 100644 --- a/src/gallium/drivers/nvc0/nvc0_shader_state.c +++ b/src/gallium/drivers/nvc0/nvc0_shader_state.c @@ -76,10 +76,10 @@ nvc0_vertprog_validate(struct nvc0_context *nvc0) BEGIN_RING(chan, RING_3D(SP_GPR_ALLOC(1)), 1); OUT_RING (chan, vp->max_gpr); - BEGIN_RING(chan, RING_3D_(0x163c), 1); - OUT_RING (chan, 0); - BEGIN_RING(chan, RING_3D_(0x2600), 1); - OUT_RING (chan, 1); + // BEGIN_RING(chan, RING_3D_(0x163c), 1); + // OUT_RING (chan, 0); + // BEGIN_RING(chan, RING_3D_(0x2600), 1); + // OUT_RING (chan, 1); } void diff --git a/src/gallium/drivers/nvc0/nvc0_tgsi_to_nc.c b/src/gallium/drivers/nvc0/nvc0_tgsi_to_nc.c index 1b161f66dc1..a6797db9c54 100644 --- a/src/gallium/drivers/nvc0/nvc0_tgsi_to_nc.c +++ b/src/gallium/drivers/nvc0/nvc0_tgsi_to_nc.c @@ -22,6 +22,8 @@ #include +#define NOUVEAU_DEBUG 1 + #include "pipe/p_shader_tokens.h" #include "tgsi/tgsi_parse.h" #include "tgsi/tgsi_util.h" @@ -194,7 +196,7 @@ static INLINE void bld_warn_uninitialized(struct bld_context *bld, int kind, struct bld_register *reg, struct nv_basic_block *b) { -#ifdef NOUVEAU_DEBUG_BITS +#ifdef NOUVEAU_DEBUG long i = (reg - &bld->tvs[0][0]) / 4; long c = (reg - &bld->tvs[0][0]) & 3; @@ -1359,7 +1361,7 @@ bld_instruction(struct bld_context *bld, uint opcode = translate_opcode(insn->Instruction.Opcode); uint8_t mask = insn->Dst[0].Register.WriteMask; -#ifdef NOUVEAU_DEBUG_BITS +#ifdef NOUVEAU_DEBUG debug_printf("bld_instruction:"); tgsi_dump_instruction(insn, 1); #endif diff --git a/src/gallium/drivers/nvc0/nvc0_transfer.c b/src/gallium/drivers/nvc0/nvc0_transfer.c index adf9192943a..56c5fe12c20 100644 --- a/src/gallium/drivers/nvc0/nvc0_transfer.c +++ b/src/gallium/drivers/nvc0/nvc0_transfer.c @@ -111,6 +111,8 @@ nvc0_m2mf_push_linear(struct nvc0_context *nvc0, uint32_t *src = (uint32_t *)data; unsigned count = (size + 3) / 4; + MARK_RING (chan, 8, 2); + BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2); OUT_RELOCh(chan, dst, offset, domain | NOUVEAU_BO_WR); OUT_RELOCl(chan, dst, offset, domain | NOUVEAU_BO_WR); @@ -125,6 +127,7 @@ nvc0_m2mf_push_linear(struct nvc0_context *nvc0, if (nr < 9) { FIRE_RING(chan); + nvc0_make_bo_resident(nvc0, dst, NOUVEAU_BO_WR); continue; } nr = MIN2(count, nr - 1); @@ -138,53 +141,90 @@ nvc0_m2mf_push_linear(struct nvc0_context *nvc0, } } +void +nvc0_m2mf_copy_linear(struct nvc0_context *nvc0, + struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom, + struct nouveau_bo *src, unsigned srcoff, unsigned srcdom, + unsigned size) +{ + struct nouveau_channel *chan = nvc0->screen->base.channel; + + while (size) { + unsigned bytes = MIN2(size, 1 << 17); + + MARK_RING (chan, 11, 4); + + BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2); + OUT_RELOCh(chan, dst, dstoff, dstdom | NOUVEAU_BO_WR); + OUT_RELOCl(chan, dst, dstoff, dstdom | NOUVEAU_BO_WR); + BEGIN_RING(chan, RING_MF(OFFSET_IN_HIGH), 2); + OUT_RELOCh(chan, src, srcoff, srcdom | NOUVEAU_BO_RD); + OUT_RELOCl(chan, src, srcoff, srcdom | NOUVEAU_BO_RD); + BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2); + OUT_RING (chan, bytes); + OUT_RING (chan, 1); + BEGIN_RING(chan, RING_MF(EXEC), 1); + OUT_RING (chan, (1 << NVC0_M2MF_EXEC_INC__SHIFT) | + NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT); + + srcoff += bytes; + dstoff += bytes; + size -= bytes; + } +} + static void -nvc0_sifc_push_rect(struct pipe_screen *pscreen, - const struct nvc0_m2mf_rect *dst, unsigned dst_format, - unsigned src_format, unsigned src_pitch, void *src, +nvc0_m2mf_push_rect(struct pipe_screen *pscreen, + const struct nvc0_m2mf_rect *dst, + const void *data, unsigned nblocksx, unsigned nblocksy) { struct nouveau_channel *chan; + const uint8_t *src = (const uint8_t *)data; + const int cpp = dst->cpp; + const int line_len = nblocksx * cpp; + int dy = dst->y; - if (dst->bo->tile_flags) { - BEGIN_RING(chan, RING_2D(DST_FORMAT), 5); - OUT_RING (chan, dst_format); - OUT_RING (chan, 0); - OUT_RING (chan, dst->tile_mode); - OUT_RING (chan, 1); - OUT_RING (chan, 0); - } else { - BEGIN_RING(chan, RING_2D(DST_FORMAT), 2); - OUT_RING (chan, NV50_SURFACE_FORMAT_A8R8G8B8_UNORM); - OUT_RING (chan, 1); - BEGIN_RING(chan, RING_2D(DST_PITCH), 1); - OUT_RING (chan, dst->pitch); - } + assert(dst->bo->tile_flags); - BEGIN_RING(chan, RING_2D(DST_WIDTH), 4); - OUT_RING (chan, dst->width); + BEGIN_RING(chan, RING_MF(TILING_MODE_OUT), 5); + OUT_RING (chan, dst->tile_mode); + OUT_RING (chan, dst->width * cpp); OUT_RING (chan, dst->height); - OUT_RELOCh(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR); - OUT_RELOCl(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR); - - BEGIN_RING(chan, RING_2D(SIFC_BITMAP_ENABLE), 2); - OUT_RING (chan, 0); - OUT_RING (chan, src_format); - BEGIN_RING(chan, RING_2D(SIFC_WIDTH), 10); - OUT_RING (chan, nblocksx); - OUT_RING (chan, nblocksy); - OUT_RING (chan, 0); - OUT_RING (chan, 1); - OUT_RING (chan, 0); - OUT_RING (chan, 1); - OUT_RING (chan, 0); - OUT_RING (chan, dst->x); - OUT_RING (chan, 0); - OUT_RING (chan, dst->y); + OUT_RING (chan, dst->depth); + OUT_RING (chan, dst->z); while (nblocksy) { + int line_count, words; + int size = MIN2(AVAIL_RING(chan), NV04_PFIFO_MAX_PACKET_LEN); - src = (uint8_t *)src + src_pitch; + if (size < (12 + words)) { + FIRE_RING(chan); + continue; + } + line_count = (size * 4) / line_len; + words = (line_count * line_len + 3) / 4; + + BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2); + OUT_RELOCh(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR); + OUT_RELOCl(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR); + + BEGIN_RING(chan, RING_MF(TILING_POSITION_OUT_X), 2); + OUT_RING (chan, dst->x * cpp); + OUT_RING (chan, dy); + BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2); + OUT_RING (chan, line_len); + OUT_RING (chan, line_count); + BEGIN_RING(chan, RING_MF(EXEC), 1); + OUT_RING (chan, (1 << NVC0_M2MF_EXEC_INC__SHIFT) | + NVC0_M2MF_EXEC_PUSH | NVC0_M2MF_EXEC_LINEAR_IN); + + BEGIN_RING(chan, RING_MF(DATA), words); + OUT_RINGp (chan, src, words); + + dy += line_count; + src += line_len * line_count; + nblocksy -= line_count; } } @@ -242,6 +282,11 @@ nvc0_miptree_transfer_new(struct pipe_context *pctx, tx->rect[0].pitch = lvl->pitch; tx->rect[0].domain = NOUVEAU_BO_VRAM; + if (!(usage & PIPE_TRANSFER_READ) && + (res->depth0 == 1) && (tx->nblocksy * tx->base.stride < 512 * 4)) { + /* don't allocate scratch buffer, upload through FIFO */ + } + ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0, tx->nblocksy * tx->base.stride, &tx->rect[1].bo); if (ret) { diff --git a/src/gallium/drivers/nvc0/nvc0_vbo.c b/src/gallium/drivers/nvc0/nvc0_vbo.c index b8529e632d2..84951ed9458 100644 --- a/src/gallium/drivers/nvc0/nvc0_vbo.c +++ b/src/gallium/drivers/nvc0/nvc0_vbo.c @@ -131,8 +131,16 @@ nvc0_vertex_arrays_validate(struct nvc0_context *nvc0) ve = &vertex->element[i]; vb = &nvc0->vtxbuf[ve->pipe.vertex_buffer_index]; - if (!nvc0_resource_mapped_by_gpu(vb->buffer)) - nvc0->vbo_fifo |= 1 << i; + if (!nvc0_resource_mapped_by_gpu(vb->buffer)) { + if (nvc0->vbo_push_hint) { + nvc0->vbo_fifo |= 1 << i; + } else { + nvc0_migrate_vertices(nvc0_resource(vb->buffer), + vb->buffer_offset, + vb->buffer->width0 - vb->buffer_offset); + nvc0->vbo_dirty = TRUE; + } + } if (1 || likely(vb->stride)) { OUT_RING(chan, ve->state); @@ -142,7 +150,7 @@ nvc0_vertex_arrays_validate(struct nvc0_context *nvc0) } for (i = 0; i < vertex->num_elements; ++i) { - struct nouveau_bo *bo; + struct nvc0_resource *res; unsigned size, offset; ve = &vertex->element[i]; @@ -158,7 +166,7 @@ nvc0_vertex_arrays_validate(struct nvc0_context *nvc0) continue; } - bo = nvc0_resource(vb->buffer)->bo; + res = nvc0_resource(vb->buffer); size = vb->buffer->width0; offset = ve->pipe.src_offset + vb->buffer_offset; @@ -173,17 +181,16 @@ nvc0_vertex_arrays_validate(struct nvc0_context *nvc0) INLIN_RING(chan, RING_3D(VERTEX_ARRAY_PER_INSTANCE(i)), 0); } - nvc0_bufctx_add_resident(nvc0, NVC0_BUFCTX_VERTEX, - nvc0_resource(vb->buffer), NOUVEAU_BO_RD); + nvc0_bufctx_add_resident(nvc0, NVC0_BUFCTX_VERTEX, res, NOUVEAU_BO_RD); BEGIN_RING(chan, RING_3D(VERTEX_ARRAY_FETCH(i)), 1); OUT_RING (chan, (1 << 12) | vb->stride); BEGIN_RING_1I(chan, RING_3D(VERTEX_ARRAY_SELECT), 5); OUT_RING (chan, i); - OUT_RELOCh(chan, bo, size, NOUVEAU_BO_GART | NOUVEAU_BO_RD); - OUT_RELOCl(chan, bo, size, NOUVEAU_BO_GART | NOUVEAU_BO_RD); - OUT_RELOCh(chan, bo, offset, NOUVEAU_BO_GART | NOUVEAU_BO_RD); - OUT_RELOCl(chan, bo, offset, NOUVEAU_BO_GART | NOUVEAU_BO_RD); + OUT_RESRCh(chan, res, size, NOUVEAU_BO_RD); + OUT_RESRCl(chan, res, size, NOUVEAU_BO_RD); + OUT_RESRCh(chan, res, offset, NOUVEAU_BO_RD); + OUT_RESRCl(chan, res, offset, NOUVEAU_BO_RD); } for (; i < nvc0->state.num_vtxelts; ++i) { BEGIN_RING(chan, RING_3D(VERTEX_ATTRIB_FORMAT(i)), 1); @@ -231,8 +238,6 @@ nvc0_draw_vbo_flush_notify(struct nouveau_channel *chan) struct nvc0_context *nvc0 = chan->user_private; nvc0_bufctx_emit_relocs(nvc0); - - debug_printf("%s(%p)\n", __FUNCTION__, nvc0); } #if 0 @@ -325,7 +330,7 @@ nvc0_draw_elements_inline_u08(struct nouveau_channel *chan, uint8_t *map, count &= ~3; } while (count) { - unsigned i, nr = MIN2(count, (NV04_PFIFO_MAX_PACKET_LEN & ~3) * 4) / 4; + unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 4) / 4; BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U8), nr); for (i = 0; i < nr; ++i) { @@ -333,7 +338,7 @@ nvc0_draw_elements_inline_u08(struct nouveau_channel *chan, uint8_t *map, (map[3] << 24) | (map[2] << 16) | (map[1] << 8) | map[0]); map += 4; } - count -= nr; + count -= nr * 4; } } @@ -349,14 +354,14 @@ nvc0_draw_elements_inline_u16(struct nouveau_channel *chan, uint16_t *map, OUT_RING (chan, *map++); } while (count) { - unsigned i, nr = MIN2(count, (NV04_PFIFO_MAX_PACKET_LEN & ~1) * 2) / 2; + unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 2) / 2; BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U16), nr); for (i = 0; i < nr; ++i) { OUT_RING(chan, (map[1] << 16) | map[0]); map += 2; } - count -= nr; + count -= nr * 2; } } @@ -367,18 +372,41 @@ nvc0_draw_elements_inline_u32(struct nouveau_channel *chan, uint32_t *map, map += start; while (count) { - unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN); + const unsigned nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN); BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U32), nr); - for (i = 0; i < nr; ++i) - OUT_RING(chan, *map++); + OUT_RINGp (chan, map, nr); + map += nr; count -= nr; } } static void -nvc0_draw_elements(struct nvc0_context *nvc0, +nvc0_draw_elements_inline_u32_short(struct nouveau_channel *chan, uint32_t *map, + unsigned start, unsigned count) +{ + map += start; + + if (count & 1) { + count--; + BEGIN_RING(chan, RING_3D(VB_ELEMENT_U32), 1); + OUT_RING (chan, *map++); + } + while (count) { + unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 2) / 2; + + BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U16), nr); + for (i = 0; i < nr; ++i) { + OUT_RING(chan, (map[1] << 16) | map[0]); + map += 2; + } + count -= nr * 2; + } +} + +static void +nvc0_draw_elements(struct nvc0_context *nvc0, boolean shorten, unsigned mode, unsigned start, unsigned count, unsigned instance_count, int32_t index_bias) { @@ -400,7 +428,7 @@ nvc0_draw_elements(struct nvc0_context *nvc0, } if (nvc0_resource_mapped_by_gpu(nvc0->idxbuf.buffer)) { - struct nouveau_bo *bo = nvc0_resource(nvc0->idxbuf.buffer)->bo; + struct nvc0_resource *res = nvc0_resource(nvc0->idxbuf.buffer); unsigned offset = nvc0->idxbuf.offset; unsigned limit = nvc0->idxbuf.buffer->width0 - 1; @@ -415,10 +443,10 @@ nvc0_draw_elements(struct nvc0_context *nvc0, BEGIN_RING(chan, RING_3D(VERTEX_BEGIN_GL), 1); OUT_RING (chan, mode); BEGIN_RING(chan, RING_3D(INDEX_ARRAY_START_HIGH), 7); - OUT_RELOCh(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_GART); - OUT_RELOCl(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_GART); - OUT_RELOCh(chan, bo, limit, NOUVEAU_BO_VRAM | NOUVEAU_BO_GART); - OUT_RELOCl(chan, bo, limit, NOUVEAU_BO_VRAM | NOUVEAU_BO_GART); + OUT_RESRCh(chan, res, offset, NOUVEAU_BO_RD); + OUT_RESRCl(chan, res, offset, NOUVEAU_BO_RD); + OUT_RESRCh(chan, res, limit, NOUVEAU_BO_RD); + OUT_RESRCl(chan, res, limit, NOUVEAU_BO_RD); OUT_RING (chan, index_size); OUT_RING (chan, start); OUT_RING (chan, count); @@ -443,7 +471,10 @@ nvc0_draw_elements(struct nvc0_context *nvc0, nvc0_draw_elements_inline_u16(chan, data, start, count); break; case 4: - nvc0_draw_elements_inline_u32(chan, data, start, count); + if (shorten) + nvc0_draw_elements_inline_u32_short(chan, data, start, count); + else + nvc0_draw_elements_inline_u32(chan, data, start, count); break; default: assert(0); @@ -464,6 +495,13 @@ nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info) struct nvc0_context *nvc0 = nvc0_context(pipe); struct nouveau_channel *chan = nvc0->screen->base.channel; + /* For picking only a few vertices from a large user buffer, push is better, + * if index count is larger and we expect repeated vertices, suggest upload. + */ + nvc0->vbo_push_hint = /* the 64 is heuristic */ + !(info->indexed && + ((info->max_index - info->min_index + 64) < info->count)); + nvc0_state_validate(nvc0); if (nvc0->state.instance_base != info->start_instance) { @@ -488,6 +526,8 @@ nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info) info->mode, info->start, info->count, info->instance_count); } else { + boolean shorten = info->max_index <= 65535; + assert(nvc0->idxbuf.buffer); if (info->primitive_restart != nvc0->state.prim_restart) { @@ -495,6 +535,9 @@ nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info) BEGIN_RING(chan, RING_3D(PRIM_RESTART_ENABLE), 2); OUT_RING (chan, 1); OUT_RING (chan, info->restart_index); + + if (info->restart_index > 65535) + shorten = FALSE; } else { INLIN_RING(chan, RING_3D(PRIM_RESTART_ENABLE), 0); } @@ -505,7 +548,7 @@ nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info) OUT_RING (chan, info->restart_index); } - nvc0_draw_elements(nvc0, + nvc0_draw_elements(nvc0, shorten, info->mode, info->start, info->count, info->instance_count, info->index_bias); } diff --git a/src/gallium/drivers/nvc0/nvc0_winsys.h b/src/gallium/drivers/nvc0/nvc0_winsys.h index 48ea87613b5..e97ca8e90da 100644 --- a/src/gallium/drivers/nvc0/nvc0_winsys.h +++ b/src/gallium/drivers/nvc0/nvc0_winsys.h @@ -12,6 +12,8 @@ #include "nouveau/nouveau_resource.h" #include "nouveau/nouveau_reloc.h" +#include "nvc0_resource.h" /* OUT_RESRC */ + #ifndef NV04_PFIFO_MAX_PACKET_LEN #define NV04_PFIFO_MAX_PACKET_LEN 2047 #endif @@ -143,6 +145,20 @@ OUT_RELOCh(struct nouveau_channel *chan, struct nouveau_bo *bo, return OUT_RELOC(chan, bo, delta, flags | NOUVEAU_BO_HIGH, 0, 0); } +static INLINE int +OUT_RESRCh(struct nouveau_channel *chan, struct nvc0_resource *res, + unsigned delta, unsigned flags) +{ + return OUT_RELOCh(chan, res->bo, res->offset + delta, res->domain | flags); +} + +static INLINE int +OUT_RESRCl(struct nouveau_channel *chan, struct nvc0_resource *res, + unsigned delta, unsigned flags) +{ + return OUT_RELOCl(chan, res->bo, res->offset + delta, res->domain | flags); +} + static INLINE void FIRE_RING(struct nouveau_channel *chan) { -- 2.11.0