2 #include "util/u_inlines.h"
3 #include "util/u_memory.h"
4 #include "util/u_math.h"
5 #include "util/u_surface.h"
7 #include "nouveau_screen.h"
8 #include "nouveau_context.h"
9 #include "nouveau_winsys.h"
10 #include "nouveau_fence.h"
11 #include "nouveau_buffer.h"
12 #include "nouveau_mm.h"
14 #define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192
16 struct nouveau_transfer {
17 struct pipe_transfer base;
20 struct nouveau_bo *bo;
21 struct nouveau_mm_allocation *mm;
25 static inline struct nouveau_transfer *
26 nouveau_transfer(struct pipe_transfer *transfer)
28 return (struct nouveau_transfer *)transfer;
32 nouveau_buffer_malloc(struct nv04_resource *buf)
35 buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
40 nouveau_buffer_allocate(struct nouveau_screen *screen,
41 struct nv04_resource *buf, unsigned domain)
43 uint32_t size = align(buf->base.width0, 0x100);
45 if (domain == NOUVEAU_BO_VRAM) {
46 buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
47 &buf->bo, &buf->offset);
49 return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
50 NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
52 if (domain == NOUVEAU_BO_GART) {
53 buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
54 &buf->bo, &buf->offset);
57 NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
60 if (!nouveau_buffer_malloc(buf))
65 buf->address = buf->bo->offset + buf->offset;
67 util_range_set_empty(&buf->valid_buffer_range);
73 release_allocation(struct nouveau_mm_allocation **mm,
74 struct nouveau_fence *fence)
76 nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
81 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
83 if (buf->fence && buf->fence->state < NOUVEAU_FENCE_STATE_FLUSHED) {
84 nouveau_fence_work(buf->fence, nouveau_fence_unref_bo, buf->bo);
87 nouveau_bo_ref(NULL, &buf->bo);
91 release_allocation(&buf->mm, buf->fence);
93 if (buf->domain == NOUVEAU_BO_VRAM)
94 NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0);
95 if (buf->domain == NOUVEAU_BO_GART)
96 NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0);
102 nouveau_buffer_reallocate(struct nouveau_screen *screen,
103 struct nv04_resource *buf, unsigned domain)
105 nouveau_buffer_release_gpu_storage(buf);
107 nouveau_fence_ref(NULL, &buf->fence);
108 nouveau_fence_ref(NULL, &buf->fence_wr);
110 buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
112 return nouveau_buffer_allocate(screen, buf, domain);
116 nouveau_buffer_destroy(struct pipe_screen *pscreen,
117 struct pipe_resource *presource)
119 struct nv04_resource *res = nv04_resource(presource);
121 nouveau_buffer_release_gpu_storage(res);
123 if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
124 align_free(res->data);
126 nouveau_fence_ref(NULL, &res->fence);
127 nouveau_fence_ref(NULL, &res->fence_wr);
129 util_range_destroy(&res->valid_buffer_range);
133 NOUVEAU_DRV_STAT(nouveau_screen(pscreen), buf_obj_current_count, -1);
136 /* Set up a staging area for the transfer. This is either done in "regular"
137 * system memory if the driver supports push_data (nv50+) and the data is
138 * small enough (and permit_pb == true), or in GART memory.
141 nouveau_transfer_staging(struct nouveau_context *nv,
142 struct nouveau_transfer *tx, bool permit_pb)
144 const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK;
145 const unsigned size = align(tx->base.box.width, 4) + adj;
150 if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) {
151 tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
156 nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset);
159 if (!nouveau_bo_map(tx->bo, 0, NULL))
160 tx->map = (uint8_t *)tx->bo->map + tx->offset;
166 /* Copies data from the resource into the transfer's temporary GART
167 * buffer. Also updates buf->data if present.
169 * Maybe just migrate to GART right away if we actually need to do this. */
171 nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx)
173 struct nv04_resource *buf = nv04_resource(tx->base.resource);
174 const unsigned base = tx->base.box.x;
175 const unsigned size = tx->base.box.width;
177 NOUVEAU_DRV_STAT(nv->screen, buf_read_bytes_staging_vid, size);
179 nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART,
180 buf->bo, buf->offset + base, buf->domain, size);
182 if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client))
186 memcpy(buf->data + base, tx->map, size);
192 nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx,
193 unsigned offset, unsigned size)
195 struct nv04_resource *buf = nv04_resource(tx->base.resource);
196 uint8_t *data = tx->map + offset;
197 const unsigned base = tx->base.box.x + offset;
198 const bool can_cb = !((base | size) & 3);
201 memcpy(data, buf->data + base, size);
203 buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY;
205 if (buf->domain == NOUVEAU_BO_VRAM)
206 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_vid, size);
207 if (buf->domain == NOUVEAU_BO_GART)
208 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_sys, size);
211 nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain,
212 tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size);
214 if (nv->push_cb && can_cb)
216 base, size / 4, (const uint32_t *)data);
218 nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data);
220 nouveau_fence_ref(nv->screen->fence.current, &buf->fence);
221 nouveau_fence_ref(nv->screen->fence.current, &buf->fence_wr);
224 /* Does a CPU wait for the buffer's backing data to become reliably accessible
225 * for write/read by waiting on the buffer's relevant fences.
228 nouveau_buffer_sync(struct nouveau_context *nv,
229 struct nv04_resource *buf, unsigned rw)
231 if (rw == PIPE_TRANSFER_READ) {
234 NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
235 !nouveau_fence_signalled(buf->fence_wr));
236 if (!nouveau_fence_wait(buf->fence_wr, &nv->debug))
241 NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
242 !nouveau_fence_signalled(buf->fence));
243 if (!nouveau_fence_wait(buf->fence, &nv->debug))
246 nouveau_fence_ref(NULL, &buf->fence);
248 nouveau_fence_ref(NULL, &buf->fence_wr);
254 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
256 if (rw == PIPE_TRANSFER_READ)
257 return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
259 return (buf->fence && !nouveau_fence_signalled(buf->fence));
263 nouveau_buffer_transfer_init(struct nouveau_transfer *tx,
264 struct pipe_resource *resource,
265 const struct pipe_box *box,
268 tx->base.resource = resource;
270 tx->base.usage = usage;
271 tx->base.box.x = box->x;
274 tx->base.box.width = box->width;
275 tx->base.box.height = 1;
276 tx->base.box.depth = 1;
278 tx->base.layer_stride = 0;
285 nouveau_buffer_transfer_del(struct nouveau_context *nv,
286 struct nouveau_transfer *tx)
289 if (likely(tx->bo)) {
290 nouveau_fence_work(nv->screen->fence.current,
291 nouveau_fence_unref_bo, tx->bo);
293 release_allocation(&tx->mm, nv->screen->fence.current);
296 (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK));
301 /* Creates a cache in system memory of the buffer data. */
303 nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf)
305 struct nouveau_transfer tx;
307 tx.base.resource = &buf->base;
309 tx.base.box.width = buf->base.width0;
314 if (!nouveau_buffer_malloc(buf))
316 if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY))
318 nv->stats.buf_cache_count++;
320 if (!nouveau_transfer_staging(nv, &tx, false))
323 ret = nouveau_transfer_read(nv, &tx);
325 buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY;
326 memcpy(buf->data, tx.map, buf->base.width0);
328 nouveau_buffer_transfer_del(nv, &tx);
333 #define NOUVEAU_TRANSFER_DISCARD \
334 (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
336 /* Checks whether it is possible to completely discard the memory backing this
337 * resource. This can be useful if we would otherwise have to wait for a read
338 * operation to complete on this data.
341 nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage)
343 if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE))
345 if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
347 if (unlikely(usage & PIPE_TRANSFER_PERSISTENT))
349 return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE);
352 /* Returns a pointer to a memory area representing a window into the
355 * This may or may not be the _actual_ memory area of the resource. However
356 * when calling nouveau_buffer_transfer_unmap, if it wasn't the actual memory
357 * area, the contents of the returned map are copied over to the resource.
359 * The usage indicates what the caller plans to do with the map:
361 * WRITE means that the user plans to write to it
363 * READ means that the user plans on reading from it
365 * DISCARD_WHOLE_RESOURCE means that the whole resource is going to be
366 * potentially overwritten, and even if it isn't, the bits that aren't don't
367 * need to be maintained.
369 * DISCARD_RANGE means that all the data in the specified range is going to
372 * The strategy for determining what kind of memory area to return is complex,
373 * see comments inside of the function.
376 nouveau_buffer_transfer_map(struct pipe_context *pipe,
377 struct pipe_resource *resource,
378 unsigned level, unsigned usage,
379 const struct pipe_box *box,
380 struct pipe_transfer **ptransfer)
382 struct nouveau_context *nv = nouveau_context(pipe);
383 struct nv04_resource *buf = nv04_resource(resource);
384 struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer);
390 nouveau_buffer_transfer_init(tx, resource, box, usage);
391 *ptransfer = &tx->base;
393 if (usage & PIPE_TRANSFER_READ)
394 NOUVEAU_DRV_STAT(nv->screen, buf_transfers_rd, 1);
395 if (usage & PIPE_TRANSFER_WRITE)
396 NOUVEAU_DRV_STAT(nv->screen, buf_transfers_wr, 1);
398 /* If we are trying to write to an uninitialized range, the user shouldn't
399 * care what was there before. So we can treat the write as if the target
400 * range were being discarded. Furthermore, since we know that even if this
401 * buffer is busy due to GPU activity, because the contents were
402 * uninitialized, the GPU can't care what was there, and so we can treat
403 * the write as being unsynchronized.
405 if ((usage & PIPE_TRANSFER_WRITE) &&
406 !util_ranges_intersect(&buf->valid_buffer_range, box->x, box->x + box->width))
407 usage |= PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_UNSYNCHRONIZED;
409 if (buf->domain == NOUVEAU_BO_VRAM) {
410 if (usage & NOUVEAU_TRANSFER_DISCARD) {
411 /* Set up a staging area for the user to write to. It will be copied
412 * back into VRAM on unmap. */
413 if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
414 buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
415 nouveau_transfer_staging(nv, tx, true);
417 if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
418 /* The GPU is currently writing to this buffer. Copy its current
419 * contents to a staging area in the GART. This is necessary since
420 * not the whole area being mapped is being discarded.
423 align_free(buf->data);
426 nouveau_transfer_staging(nv, tx, false);
427 nouveau_transfer_read(nv, tx);
429 /* The buffer is currently idle. Create a staging area for writes,
430 * and make sure that the cached data is up-to-date. */
431 if (usage & PIPE_TRANSFER_WRITE)
432 nouveau_transfer_staging(nv, tx, true);
434 nouveau_buffer_cache(nv, buf);
437 return buf->data ? (buf->data + box->x) : tx->map;
439 if (unlikely(buf->domain == 0)) {
440 return buf->data + box->x;
443 /* At this point, buf->domain == GART */
445 if (nouveau_buffer_should_discard(buf, usage)) {
446 int ref = buf->base.reference.count - 1;
447 nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
448 if (ref > 0) /* any references inside context possible ? */
449 nv->invalidate_resource_storage(nv, &buf->base, ref);
452 /* Note that nouveau_bo_map ends up doing a nouveau_bo_wait with the
453 * relevant flags. If buf->mm is set, that means this resource is part of a
454 * larger slab bo that holds multiple resources. So in that case, don't
455 * wait on the whole slab and instead use the logic below to return a
456 * reasonable buffer for that case.
458 ret = nouveau_bo_map(buf->bo,
459 buf->mm ? 0 : nouveau_screen_transfer_flags(usage),
465 map = (uint8_t *)buf->bo->map + buf->offset + box->x;
467 /* using kernel fences only if !buf->mm */
468 if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm)
471 /* If the GPU is currently reading/writing this buffer, we shouldn't
472 * interfere with its progress. So instead we either wait for the GPU to
473 * complete its operation, or set up a staging area to perform our work in.
475 if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) {
476 if (unlikely(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE |
477 PIPE_TRANSFER_PERSISTENT))) {
478 /* Discarding was not possible, must sync because
479 * subsequent transfers might use UNSYNCHRONIZED. */
480 nouveau_buffer_sync(nv, buf, usage & PIPE_TRANSFER_READ_WRITE);
482 if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
483 /* The whole range is being discarded, so it doesn't matter what was
484 * there before. No need to copy anything over. */
485 nouveau_transfer_staging(nv, tx, true);
488 if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) {
489 if (usage & PIPE_TRANSFER_DONTBLOCK)
492 nouveau_buffer_sync(nv, buf, usage & PIPE_TRANSFER_READ_WRITE);
494 /* It is expected that the returned buffer be a representation of the
495 * data in question, so we must copy it over from the buffer. */
496 nouveau_transfer_staging(nv, tx, true);
498 memcpy(tx->map, map, box->width);
510 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
511 struct pipe_transfer *transfer,
512 const struct pipe_box *box)
514 struct nouveau_transfer *tx = nouveau_transfer(transfer);
515 struct nv04_resource *buf = nv04_resource(transfer->resource);
518 nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width);
520 util_range_add(&buf->valid_buffer_range,
521 tx->base.box.x + box->x,
522 tx->base.box.x + box->x + box->width);
525 /* Unmap stage of the transfer. If it was a WRITE transfer and the map that
526 * was returned was not the real resource's data, this needs to transfer the
527 * data back to the resource.
529 * Also marks vbo dirty based on the buffer's binding
532 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
533 struct pipe_transfer *transfer)
535 struct nouveau_context *nv = nouveau_context(pipe);
536 struct nouveau_transfer *tx = nouveau_transfer(transfer);
537 struct nv04_resource *buf = nv04_resource(transfer->resource);
539 if (tx->base.usage & PIPE_TRANSFER_WRITE) {
540 if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT)) {
542 nouveau_transfer_write(nv, tx, 0, tx->base.box.width);
544 util_range_add(&buf->valid_buffer_range,
545 tx->base.box.x, tx->base.box.x + tx->base.box.width);
548 if (likely(buf->domain)) {
549 const uint8_t bind = buf->base.bind;
550 /* make sure we invalidate dedicated caches */
551 if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
552 nv->vbo_dirty = true;
556 if (!tx->bo && (tx->base.usage & PIPE_TRANSFER_WRITE))
557 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_direct, tx->base.box.width);
559 nouveau_buffer_transfer_del(nv, tx);
565 nouveau_copy_buffer(struct nouveau_context *nv,
566 struct nv04_resource *dst, unsigned dstx,
567 struct nv04_resource *src, unsigned srcx, unsigned size)
569 assert(dst->base.target == PIPE_BUFFER && src->base.target == PIPE_BUFFER);
571 if (likely(dst->domain) && likely(src->domain)) {
573 dst->bo, dst->offset + dstx, dst->domain,
574 src->bo, src->offset + srcx, src->domain, size);
576 dst->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING;
577 nouveau_fence_ref(nv->screen->fence.current, &dst->fence);
578 nouveau_fence_ref(nv->screen->fence.current, &dst->fence_wr);
580 src->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
581 nouveau_fence_ref(nv->screen->fence.current, &src->fence);
583 struct pipe_box src_box;
587 src_box.width = size;
590 util_resource_copy_region(&nv->pipe,
591 &dst->base, 0, dstx, 0, 0,
592 &src->base, 0, &src_box);
595 util_range_add(&dst->valid_buffer_range, dstx, dstx + size);
600 nouveau_resource_map_offset(struct nouveau_context *nv,
601 struct nv04_resource *res, uint32_t offset,
604 if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
605 return res->data + offset;
607 if (res->domain == NOUVEAU_BO_VRAM) {
608 if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
609 nouveau_buffer_cache(nv, res);
611 if (res->domain != NOUVEAU_BO_GART)
612 return res->data + offset;
616 rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
617 nouveau_buffer_sync(nv, res, rw);
618 if (nouveau_bo_map(res->bo, 0, NULL))
621 if (nouveau_bo_map(res->bo, flags, nv->client))
624 return (uint8_t *)res->bo->map + res->offset + offset;
628 const struct u_resource_vtbl nouveau_buffer_vtbl =
630 u_default_resource_get_handle, /* get_handle */
631 nouveau_buffer_destroy, /* resource_destroy */
632 nouveau_buffer_transfer_map, /* transfer_map */
633 nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
634 nouveau_buffer_transfer_unmap, /* transfer_unmap */
637 struct pipe_resource *
638 nouveau_buffer_create(struct pipe_screen *pscreen,
639 const struct pipe_resource *templ)
641 struct nouveau_screen *screen = nouveau_screen(pscreen);
642 struct nv04_resource *buffer;
645 buffer = CALLOC_STRUCT(nv04_resource);
649 buffer->base = *templ;
650 buffer->vtbl = &nouveau_buffer_vtbl;
651 pipe_reference_init(&buffer->base.reference, 1);
652 buffer->base.screen = pscreen;
654 if (buffer->base.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
655 PIPE_RESOURCE_FLAG_MAP_COHERENT)) {
656 buffer->domain = NOUVEAU_BO_GART;
657 } else if (buffer->base.bind == 0 || (buffer->base.bind &
658 (screen->vidmem_bindings & screen->sysmem_bindings))) {
659 switch (buffer->base.usage) {
660 case PIPE_USAGE_DEFAULT:
661 case PIPE_USAGE_IMMUTABLE:
662 buffer->domain = NV_VRAM_DOMAIN(screen);
664 case PIPE_USAGE_DYNAMIC:
665 /* For most apps, we'd have to do staging transfers to avoid sync
666 * with this usage, and GART -> GART copies would be suboptimal.
668 buffer->domain = NV_VRAM_DOMAIN(screen);
670 case PIPE_USAGE_STAGING:
671 case PIPE_USAGE_STREAM:
672 buffer->domain = NOUVEAU_BO_GART;
679 if (buffer->base.bind & screen->vidmem_bindings)
680 buffer->domain = NV_VRAM_DOMAIN(screen);
682 if (buffer->base.bind & screen->sysmem_bindings)
683 buffer->domain = NOUVEAU_BO_GART;
686 ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
691 if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy)
692 nouveau_buffer_cache(NULL, buffer);
694 NOUVEAU_DRV_STAT(screen, buf_obj_current_count, 1);
696 util_range_init(&buffer->valid_buffer_range);
698 return &buffer->base;
706 struct pipe_resource *
707 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
708 unsigned bytes, unsigned bind)
710 struct nv04_resource *buffer;
712 buffer = CALLOC_STRUCT(nv04_resource);
716 pipe_reference_init(&buffer->base.reference, 1);
717 buffer->vtbl = &nouveau_buffer_vtbl;
718 buffer->base.screen = pscreen;
719 buffer->base.format = PIPE_FORMAT_R8_UNORM;
720 buffer->base.usage = PIPE_USAGE_IMMUTABLE;
721 buffer->base.bind = bind;
722 buffer->base.width0 = bytes;
723 buffer->base.height0 = 1;
724 buffer->base.depth0 = 1;
727 buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
729 util_range_init(&buffer->valid_buffer_range);
730 util_range_add(&buffer->valid_buffer_range, 0, bytes);
732 return &buffer->base;
736 nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
737 struct nouveau_bo *bo, unsigned offset, unsigned size)
739 if (!nouveau_buffer_malloc(buf))
741 if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client))
743 memcpy(buf->data, (uint8_t *)bo->map + offset, size);
747 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
749 nouveau_buffer_migrate(struct nouveau_context *nv,
750 struct nv04_resource *buf, const unsigned new_domain)
752 struct nouveau_screen *screen = nv->screen;
753 struct nouveau_bo *bo;
754 const unsigned old_domain = buf->domain;
755 unsigned size = buf->base.width0;
759 assert(new_domain != old_domain);
761 if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
762 if (!nouveau_buffer_allocate(screen, buf, new_domain))
764 ret = nouveau_bo_map(buf->bo, 0, nv->client);
767 memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
768 align_free(buf->data);
770 if (old_domain != 0 && new_domain != 0) {
771 struct nouveau_mm_allocation *mm = buf->mm;
773 if (new_domain == NOUVEAU_BO_VRAM) {
774 /* keep a system memory copy of our data in case we hit a fallback */
775 if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
777 if (nouveau_mesa_debug)
778 debug_printf("migrating %u KiB to VRAM\n", size / 1024);
781 offset = buf->offset;
785 nouveau_buffer_allocate(screen, buf, new_domain);
787 nv->copy_data(nv, buf->bo, buf->offset, new_domain,
788 bo, offset, old_domain, buf->base.width0);
790 nouveau_fence_work(screen->fence.current, nouveau_fence_unref_bo, bo);
792 release_allocation(&mm, screen->fence.current);
794 if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
795 struct nouveau_transfer tx;
796 if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
798 tx.base.resource = &buf->base;
800 tx.base.box.width = buf->base.width0;
803 if (!nouveau_transfer_staging(nv, &tx, false))
805 nouveau_transfer_write(nv, &tx, 0, tx.base.box.width);
806 nouveau_buffer_transfer_del(nv, &tx);
810 assert(buf->domain == new_domain);
814 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
815 * We'd like to only allocate @size bytes here, but then we'd have to rebase
816 * the vertex indices ...
819 nouveau_user_buffer_upload(struct nouveau_context *nv,
820 struct nv04_resource *buf,
821 unsigned base, unsigned size)
823 struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
826 assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
828 buf->base.width0 = base + size;
829 if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
832 ret = nouveau_bo_map(buf->bo, 0, nv->client);
835 memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
840 /* Invalidate underlying buffer storage, reset fences, reallocate to non-busy
844 nouveau_buffer_invalidate(struct pipe_context *pipe,
845 struct pipe_resource *resource)
847 struct nouveau_context *nv = nouveau_context(pipe);
848 struct nv04_resource *buf = nv04_resource(resource);
849 int ref = buf->base.reference.count - 1;
851 /* Shared buffers shouldn't get reallocated */
852 if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
855 /* We can't touch persistent/coherent buffers */
856 if (buf->base.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
857 PIPE_RESOURCE_FLAG_MAP_COHERENT))
860 /* If the buffer is sub-allocated and not currently being written, just
861 * wipe the valid buffer range. Otherwise we have to create fresh
862 * storage. (We don't keep track of fences for non-sub-allocated BO's.)
864 if (buf->mm && !nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE)) {
865 util_range_set_empty(&buf->valid_buffer_range);
867 nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
868 if (ref > 0) /* any references inside context possible ? */
869 nv->invalidate_resource_storage(nv, &buf->base, ref);
874 /* Scratch data allocation. */
877 nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
880 return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
881 4096, size, NULL, pbo);
885 nouveau_scratch_unref_bos(void *d)
887 struct runout *b = d;
890 for (i = 0; i < b->nr; ++i)
891 nouveau_bo_ref(NULL, &b->bo[i]);
897 nouveau_scratch_runout_release(struct nouveau_context *nv)
899 if (!nv->scratch.runout)
902 if (!nouveau_fence_work(nv->screen->fence.current, nouveau_scratch_unref_bos,
907 nv->scratch.runout = NULL;
910 /* Allocate an extra bo if we can't fit everything we need simultaneously.
911 * (Could happen for very large user arrays.)
914 nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
919 if (nv->scratch.runout)
920 n = nv->scratch.runout->nr;
923 nv->scratch.runout = REALLOC(nv->scratch.runout, n == 0 ? 0 :
924 (sizeof(*nv->scratch.runout) + (n + 0) * sizeof(void *)),
925 sizeof(*nv->scratch.runout) + (n + 1) * sizeof(void *));
926 nv->scratch.runout->nr = n + 1;
927 nv->scratch.runout->bo[n] = NULL;
929 ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout->bo[n], size);
931 ret = nouveau_bo_map(nv->scratch.runout->bo[n], 0, NULL);
933 nouveau_bo_ref(NULL, &nv->scratch.runout->bo[--nv->scratch.runout->nr]);
936 nv->scratch.current = nv->scratch.runout->bo[n];
937 nv->scratch.offset = 0;
938 nv->scratch.end = size;
939 nv->scratch.map = nv->scratch.current->map;
944 /* Continue to next scratch buffer, if available (no wrapping, large enough).
945 * Allocate it if it has not yet been created.
948 nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
950 struct nouveau_bo *bo;
952 const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
954 if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
958 bo = nv->scratch.bo[i];
960 ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
963 nv->scratch.bo[i] = bo;
965 nv->scratch.current = bo;
966 nv->scratch.offset = 0;
967 nv->scratch.end = nv->scratch.bo_size;
969 ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
971 nv->scratch.map = bo->map;
976 nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
980 ret = nouveau_scratch_next(nv, min_size);
982 ret = nouveau_scratch_runout(nv, min_size);
987 /* Copy data to a scratch buffer and return address & bo the data resides in. */
989 nouveau_scratch_data(struct nouveau_context *nv,
990 const void *data, unsigned base, unsigned size,
991 struct nouveau_bo **bo)
993 unsigned bgn = MAX2(base, nv->scratch.offset);
994 unsigned end = bgn + size;
996 if (end >= nv->scratch.end) {
998 if (!nouveau_scratch_more(nv, end))
1002 nv->scratch.offset = align(end, 4);
1004 memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
1006 *bo = nv->scratch.current;
1007 return (*bo)->offset + (bgn - base);
1011 nouveau_scratch_get(struct nouveau_context *nv,
1012 unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
1014 unsigned bgn = nv->scratch.offset;
1015 unsigned end = nv->scratch.offset + size;
1017 if (end >= nv->scratch.end) {
1019 if (!nouveau_scratch_more(nv, end))
1023 nv->scratch.offset = align(end, 4);
1025 *pbo = nv->scratch.current;
1026 *gpu_addr = nv->scratch.current->offset + bgn;
1027 return nv->scratch.map + bgn;