2 * Copyright © 2014-2015 Broadcom
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 #include <xf86drmMode.h>
31 #include "util/u_hash_table.h"
32 #include "util/u_memory.h"
33 #include "util/ralloc.h"
35 #include "vc4_context.h"
36 #include "vc4_screen.h"
38 static bool dump_stats = false;
41 vc4_bo_cache_free_all(struct vc4_bo_cache *cache);
44 vc4_bo_dump_stats(struct vc4_screen *screen)
46 struct vc4_bo_cache *cache = &screen->bo_cache;
48 fprintf(stderr, " BOs allocated: %d\n", screen->bo_count);
49 fprintf(stderr, " BOs size: %dkb\n", screen->bo_size / 1024);
50 fprintf(stderr, " BOs cached: %d\n", cache->bo_count);
51 fprintf(stderr, " BOs cached size: %dkb\n", cache->bo_size / 1024);
53 if (!list_empty(&cache->time_list)) {
54 struct vc4_bo *first = LIST_ENTRY(struct vc4_bo,
55 cache->time_list.next,
57 struct vc4_bo *last = LIST_ENTRY(struct vc4_bo,
58 cache->time_list.prev,
61 fprintf(stderr, " oldest cache time: %ld\n",
62 (long)first->free_time);
63 fprintf(stderr, " newest cache time: %ld\n",
64 (long)last->free_time);
67 clock_gettime(CLOCK_MONOTONIC, &time);
68 fprintf(stderr, " now: %ld\n",
74 vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo)
76 list_del(&bo->time_list);
77 list_del(&bo->size_list);
79 cache->bo_size -= bo->size;
82 static struct vc4_bo *
83 vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name)
85 struct vc4_bo_cache *cache = &screen->bo_cache;
86 uint32_t page_index = size / 4096 - 1;
88 if (cache->size_list_size <= page_index)
91 struct vc4_bo *bo = NULL;
92 pipe_mutex_lock(cache->lock);
93 if (!list_empty(&cache->size_list[page_index])) {
94 bo = LIST_ENTRY(struct vc4_bo, cache->size_list[page_index].next,
97 /* Check that the BO has gone idle. If not, then we want to
98 * allocate something new instead, since we assume that the
99 * user will proceed to CPU map it and fill it with stuff.
101 if (!vc4_bo_wait(bo, 0, NULL)) {
102 pipe_mutex_unlock(cache->lock);
106 pipe_reference_init(&bo->reference, 1);
107 vc4_bo_remove_from_cache(cache, bo);
111 pipe_mutex_unlock(cache->lock);
116 vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name)
121 size = align(size, 4096);
123 bo = vc4_bo_from_cache(screen, size, name);
126 fprintf(stderr, "Allocated %s %dkb from cache:\n",
128 vc4_bo_dump_stats(screen);
133 bo = CALLOC_STRUCT(vc4_bo);
137 pipe_reference_init(&bo->reference, 1);
143 bool cleared_and_retried = false;
145 if (!using_vc4_simulator) {
146 struct drm_vc4_create_bo create;
147 memset(&create, 0, sizeof(create));
151 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create);
152 bo->handle = create.handle;
154 struct drm_mode_create_dumb create;
155 memset(&create, 0, sizeof(create));
159 create.height = (size + 127) / 128;
161 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
162 bo->handle = create.handle;
163 assert(create.size >= size);
166 if (!list_empty(&screen->bo_cache.time_list) &&
167 !cleared_and_retried) {
168 cleared_and_retried = true;
169 vc4_bo_cache_free_all(&screen->bo_cache);
178 screen->bo_size += bo->size;
180 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
181 vc4_bo_dump_stats(screen);
188 vc4_bo_last_unreference(struct vc4_bo *bo)
190 struct vc4_screen *screen = bo->screen;
192 struct timespec time;
193 clock_gettime(CLOCK_MONOTONIC, &time);
194 pipe_mutex_lock(screen->bo_cache.lock);
195 vc4_bo_last_unreference_locked_timed(bo, time.tv_sec);
196 pipe_mutex_unlock(screen->bo_cache.lock);
200 vc4_bo_free(struct vc4_bo *bo)
202 struct vc4_screen *screen = bo->screen;
205 #ifdef USE_VC4_SIMULATOR
206 if (bo->simulator_winsys_map) {
208 bo->map = bo->simulator_winsys_map;
211 munmap(bo->map, bo->size);
214 struct drm_gem_close c;
215 memset(&c, 0, sizeof(c));
216 c.handle = bo->handle;
217 int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
219 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
222 screen->bo_size -= bo->size;
225 fprintf(stderr, "Freed %s%s%dkb:\n",
226 bo->name ? bo->name : "",
229 vc4_bo_dump_stats(screen);
236 free_stale_bos(struct vc4_screen *screen, time_t time)
238 struct vc4_bo_cache *cache = &screen->bo_cache;
239 bool freed_any = false;
241 list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
243 if (dump_stats && !freed_any) {
244 fprintf(stderr, "Freeing stale BOs:\n");
245 vc4_bo_dump_stats(screen);
249 /* If it's more than a second old, free it. */
250 if (time - bo->free_time > 2) {
251 vc4_bo_remove_from_cache(cache, bo);
258 if (dump_stats && freed_any) {
259 fprintf(stderr, "Freed stale BOs:\n");
260 vc4_bo_dump_stats(screen);
265 vc4_bo_cache_free_all(struct vc4_bo_cache *cache)
267 pipe_mutex_lock(cache->lock);
268 list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
270 vc4_bo_remove_from_cache(cache, bo);
273 pipe_mutex_unlock(cache->lock);
277 vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time)
279 struct vc4_screen *screen = bo->screen;
280 struct vc4_bo_cache *cache = &screen->bo_cache;
281 uint32_t page_index = bo->size / 4096 - 1;
288 if (cache->size_list_size <= page_index) {
289 struct list_head *new_list =
290 ralloc_array(screen, struct list_head, page_index + 1);
292 /* Move old list contents over (since the array has moved, and
293 * therefore the pointers to the list heads have to change).
295 for (int i = 0; i < cache->size_list_size; i++) {
296 struct list_head *old_head = &cache->size_list[i];
297 if (list_empty(old_head))
298 list_inithead(&new_list[i]);
300 new_list[i].next = old_head->next;
301 new_list[i].prev = old_head->prev;
302 new_list[i].next->prev = &new_list[i];
303 new_list[i].prev->next = &new_list[i];
306 for (int i = cache->size_list_size; i < page_index + 1; i++)
307 list_inithead(&new_list[i]);
309 cache->size_list = new_list;
310 cache->size_list_size = page_index + 1;
313 bo->free_time = time;
314 list_addtail(&bo->size_list, &cache->size_list[page_index]);
315 list_addtail(&bo->time_list, &cache->time_list);
317 cache->bo_size += bo->size;
319 fprintf(stderr, "Freed %s %dkb to cache:\n",
320 bo->name, bo->size / 1024);
321 vc4_bo_dump_stats(screen);
325 free_stale_bos(screen, time);
328 static struct vc4_bo *
329 vc4_bo_open_handle(struct vc4_screen *screen,
330 uint32_t winsys_stride,
331 uint32_t handle, uint32_t size)
337 pipe_mutex_lock(screen->bo_handles_mutex);
339 bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
341 pipe_reference(NULL, &bo->reference);
345 bo = CALLOC_STRUCT(vc4_bo);
346 pipe_reference_init(&bo->reference, 1);
353 #ifdef USE_VC4_SIMULATOR
355 bo->simulator_winsys_map = bo->map;
356 bo->simulator_winsys_stride = winsys_stride;
357 bo->map = malloc(bo->size);
360 util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo);
363 pipe_mutex_unlock(screen->bo_handles_mutex);
368 vc4_bo_open_name(struct vc4_screen *screen, uint32_t name,
369 uint32_t winsys_stride)
371 struct drm_gem_open o = {
374 int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
376 fprintf(stderr, "Failed to open bo %d: %s\n",
377 name, strerror(errno));
381 return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size);
385 vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride)
388 int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
391 fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd);
395 /* Determine the size of the bo we were handed. */
396 size = lseek(fd, 0, SEEK_END);
398 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
402 return vc4_bo_open_handle(screen, winsys_stride, handle, size);
406 vc4_bo_get_dmabuf(struct vc4_bo *bo)
409 int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
412 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
417 pipe_mutex_lock(bo->screen->bo_handles_mutex);
419 util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
420 pipe_mutex_unlock(bo->screen->bo_handles_mutex);
426 vc4_bo_alloc_shader(struct vc4_screen *screen, const void *data, uint32_t size)
431 bo = CALLOC_STRUCT(vc4_bo);
435 pipe_reference_init(&bo->reference, 1);
437 bo->size = align(size, 4096);
439 bo->private = false; /* Make sure it doesn't go back to the cache. */
441 if (!using_vc4_simulator) {
442 struct drm_vc4_create_shader_bo create = {
444 .data = (uintptr_t)data,
447 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_SHADER_BO,
449 bo->handle = create.handle;
451 struct drm_mode_create_dumb create;
452 memset(&create, 0, sizeof(create));
456 create.height = (size + 127) / 128;
458 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
459 bo->handle = create.handle;
460 assert(create.size >= size);
463 memcpy(bo->map, data, size);
466 fprintf(stderr, "create shader ioctl failure\n");
471 screen->bo_size += bo->size;
473 fprintf(stderr, "Allocated shader %dkb:\n", bo->size / 1024);
474 vc4_bo_dump_stats(screen);
481 vc4_bo_flink(struct vc4_bo *bo, uint32_t *name)
483 struct drm_gem_flink flink = {
484 .handle = bo->handle,
486 int ret = drmIoctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
488 fprintf(stderr, "Failed to flink bo %d: %s\n",
489 bo->handle, strerror(errno));
500 static int vc4_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns)
502 if (using_vc4_simulator)
505 struct drm_vc4_wait_seqno wait = {
507 .timeout_ns = timeout_ns,
509 int ret = drmIoctl(fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
518 vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns,
521 if (screen->finished_seqno >= seqno)
524 if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
525 if (vc4_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) {
526 fprintf(stderr, "Blocking on seqno %lld for %s\n",
527 (long long)seqno, reason);
531 int ret = vc4_wait_seqno_ioctl(screen->fd, seqno, timeout_ns);
534 fprintf(stderr, "wait failed: %d\n", ret);
541 screen->finished_seqno = seqno;
545 static int vc4_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
547 if (using_vc4_simulator)
550 struct drm_vc4_wait_bo wait = {
552 .timeout_ns = timeout_ns,
554 int ret = drmIoctl(fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
563 vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns, const char *reason)
565 struct vc4_screen *screen = bo->screen;
567 if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
568 if (vc4_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
569 fprintf(stderr, "Blocking on %s BO for %s\n",
574 int ret = vc4_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
577 fprintf(stderr, "wait failed: %d\n", ret);
588 vc4_bo_map_unsynchronized(struct vc4_bo *bo)
596 if (!using_vc4_simulator) {
597 struct drm_vc4_mmap_bo map;
598 memset(&map, 0, sizeof(map));
599 map.handle = bo->handle;
600 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map);
603 struct drm_mode_map_dumb map;
604 memset(&map, 0, sizeof(map));
605 map.handle = bo->handle;
606 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_MODE_MAP_DUMB, &map);
610 fprintf(stderr, "map ioctl failure\n");
614 bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
615 bo->screen->fd, offset);
616 if (bo->map == MAP_FAILED) {
617 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
618 bo->handle, (long long)offset, bo->size);
626 vc4_bo_map(struct vc4_bo *bo)
628 void *map = vc4_bo_map_unsynchronized(bo);
630 bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
632 fprintf(stderr, "BO wait for map failed\n");
640 vc4_bufmgr_destroy(struct pipe_screen *pscreen)
642 struct vc4_screen *screen = vc4_screen(pscreen);
643 struct vc4_bo_cache *cache = &screen->bo_cache;
645 vc4_bo_cache_free_all(cache);
648 fprintf(stderr, "BO stats after screen destroy:\n");
649 vc4_bo_dump_stats(screen);