freedreno/msm/msm_ringbuffer.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2013 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #ifdef HAVE_CONFIG_H
  30 # include <config.h>
  31 #endif
  32
  33 #include <assert.h>
  34 #include <inttypes.h>
  35
  36 #include "freedreno_ringbuffer.h"
  37 #include "msm_priv.h"
  38
  39 /* represents a single cmd buffer in the submit ioctl.  Each cmd buffer has
  40  * a backing bo, and a reloc table.
  41  */
  42 struct msm_cmd {
  43         struct list_head list;
  44
  45         struct fd_ringbuffer *ring;
  46         struct fd_bo *ring_bo;
  47
  48         /* reloc's table: */
  49         struct drm_msm_gem_submit_reloc *relocs;
  50         uint32_t nr_relocs, max_relocs;
  51
  52         uint32_t size;
  53 };
  54
  55 struct msm_ringbuffer {
  56         struct fd_ringbuffer base;
  57
  58         /* submit ioctl related tables:
  59          * Note that bos and cmds are tracked by the parent ringbuffer, since
  60          * that is global to the submit ioctl call.  The reloc's table is tracked
  61          * per cmd-buffer.
  62          */
  63         struct {
  64                 /* bo's table: */
  65                 struct drm_msm_gem_submit_bo *bos;
  66                 uint32_t nr_bos, max_bos;
  67
  68                 /* cmd's table: */
  69                 struct drm_msm_gem_submit_cmd *cmds;
  70                 uint32_t nr_cmds, max_cmds;
  71         } submit;
  72
  73         /* should have matching entries in submit.bos: */
  74         /* Note, only in parent ringbuffer */
  75         struct fd_bo **bos;
  76         uint32_t nr_bos, max_bos;
  77
  78         /* should have matching entries in submit.cmds: */
  79         struct msm_cmd **cmds;
  80         uint32_t nr_cmds, max_cmds;
  81
  82         /* List of physical cmdstream buffers (msm_cmd) assocated with this
  83          * logical fd_ringbuffer.
  84          *
  85          * Note that this is different from msm_ringbuffer::cmds (which
  86          * shadows msm_ringbuffer::submit::cmds for tracking submit ioctl
  87          * related stuff, and *only* is tracked in the parent ringbuffer.
  88          * And only has "completed" cmd buffers (ie. we already know the
  89          * size) added via get_cmd().
  90          */
  91         struct list_head cmd_list;
  92
  93         int is_growable;
  94         unsigned cmd_count;
  95
  96         unsigned seqno;
  97
  98         /* maps fd_bo to idx: */
  99         void *bo_table;
 100 };
 101
 102 static inline struct msm_ringbuffer * to_msm_ringbuffer(struct fd_ringbuffer *x)
 103 {
 104         return (struct msm_ringbuffer *)x;
 105 }
 106
 107 #define INIT_SIZE 0x1000
 108
 109 static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
 110 drm_private extern pthread_mutex_t table_lock;
 111
 112 static void ring_bo_del(struct fd_device *dev, struct fd_bo *bo)
 113 {
 114         int ret;
 115
 116         pthread_mutex_lock(&table_lock);
 117         ret = fd_bo_cache_free(&to_msm_device(dev)->ring_cache, bo);
 118         pthread_mutex_unlock(&table_lock);
 119
 120         if (ret == 0)
 121                 return;
 122
 123         fd_bo_del(bo);
 124 }
 125
 126 static struct fd_bo * ring_bo_new(struct fd_device *dev, uint32_t size)
 127 {
 128         struct fd_bo *bo;
 129
 130         bo = fd_bo_cache_alloc(&to_msm_device(dev)->ring_cache, &size, 0);
 131         if (bo)
 132                 return bo;
 133
 134         bo = fd_bo_new(dev, size, 0);
 135         if (!bo)
 136                 return NULL;
 137
 138         /* keep ringbuffer bo's out of the normal bo cache: */
 139         bo->bo_reuse = FALSE;
 140
 141         return bo;
 142 }
 143
 144 static void ring_cmd_del(struct msm_cmd *cmd)
 145 {
 146         if (cmd->ring_bo)
 147                 ring_bo_del(cmd->ring->pipe->dev, cmd->ring_bo);
 148         list_del(&cmd->list);
 149         to_msm_ringbuffer(cmd->ring)->cmd_count--;
 150         free(cmd->relocs);
 151         free(cmd);
 152 }
 153
 154 static struct msm_cmd * ring_cmd_new(struct fd_ringbuffer *ring, uint32_t size)
 155 {
 156         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 157         struct msm_cmd *cmd = calloc(1, sizeof(*cmd));
 158
 159         if (!cmd)
 160                 return NULL;
 161
 162         cmd->ring = ring;
 163         cmd->ring_bo = ring_bo_new(ring->pipe->dev, size);
 164         if (!cmd->ring_bo)
 165                 goto fail;
 166
 167         list_addtail(&cmd->list, &msm_ring->cmd_list);
 168         msm_ring->cmd_count++;
 169
 170         return cmd;
 171
 172 fail:
 173         ring_cmd_del(cmd);
 174         return NULL;
 175 }
 176
 177 static void *grow(void *ptr, uint32_t nr, uint32_t *max, uint32_t sz)
 178 {
 179         if ((nr + 1) > *max) {
 180                 if ((*max * 2) < (nr + 1))
 181                         *max = nr + 5;
 182                 else
 183                         *max = *max * 2;
 184                 ptr = realloc(ptr, *max * sz);
 185         }
 186         return ptr;
 187 }
 188
 189 #define APPEND(x, name) ({ \
 190         (x)->name = grow((x)->name, (x)->nr_ ## name, &(x)->max_ ## name, sizeof((x)->name[0])); \
 191         (x)->nr_ ## name ++; \
 192 })
 193
 194 static struct msm_cmd *current_cmd(struct fd_ringbuffer *ring)
 195 {
 196         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 197         assert(!LIST_IS_EMPTY(&msm_ring->cmd_list));
 198         return LIST_LAST_ENTRY(&msm_ring->cmd_list, struct msm_cmd, list);
 199 }
 200
 201 static uint32_t append_bo(struct fd_ringbuffer *ring, struct fd_bo *bo)
 202 {
 203         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 204         uint32_t idx;
 205
 206         idx = APPEND(&msm_ring->submit, bos);
 207         idx = APPEND(msm_ring, bos);
 208
 209         msm_ring->submit.bos[idx].flags = 0;
 210         msm_ring->submit.bos[idx].handle = bo->handle;
 211         msm_ring->submit.bos[idx].presumed = to_msm_bo(bo)->presumed;
 212
 213         msm_ring->bos[idx] = fd_bo_ref(bo);
 214
 215         return idx;
 216 }
 217
 218 /* add (if needed) bo, return idx: */
 219 static uint32_t bo2idx(struct fd_ringbuffer *ring, struct fd_bo *bo, uint32_t flags)
 220 {
 221         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 222         struct msm_bo *msm_bo = to_msm_bo(bo);
 223         uint32_t idx;
 224         pthread_mutex_lock(&idx_lock);
 225         if (msm_bo->current_ring_seqno == msm_ring->seqno) {
 226                 idx = msm_bo->idx;
 227         } else {
 228                 void *val;
 229
 230                 if (!msm_ring->bo_table)
 231                         msm_ring->bo_table = drmHashCreate();
 232
 233                 if (!drmHashLookup(msm_ring->bo_table, bo->handle, &val)) {
 234                         /* found */
 235                         idx = (uint32_t)(uintptr_t)val;
 236                 } else {
 237                         idx = append_bo(ring, bo);
 238                         val = (void *)(uintptr_t)idx;
 239                         drmHashInsert(msm_ring->bo_table, bo->handle, val);
 240                 }
 241                 msm_bo->current_ring_seqno = msm_ring->seqno;
 242                 msm_bo->idx = idx;
 243         }
 244         pthread_mutex_unlock(&idx_lock);
 245         if (flags & FD_RELOC_READ)
 246                 msm_ring->submit.bos[idx].flags |= MSM_SUBMIT_BO_READ;
 247         if (flags & FD_RELOC_WRITE)
 248                 msm_ring->submit.bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
 249         return idx;
 250 }
 251
 252 static int check_cmd_bo(struct fd_ringbuffer *ring,
 253                 struct drm_msm_gem_submit_cmd *cmd, struct fd_bo *bo)
 254 {
 255         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 256         return msm_ring->submit.bos[cmd->submit_idx].handle == bo->handle;
 257 }
 258
 259 /* Ensure that submit has corresponding entry in cmds table for the
 260  * target cmdstream buffer:
 261  */
 262 static void get_cmd(struct fd_ringbuffer *ring, struct msm_cmd *target_cmd,
 263                 uint32_t submit_offset, uint32_t size, uint32_t type)
 264 {
 265         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 266         struct drm_msm_gem_submit_cmd *cmd;
 267         uint32_t i;
 268
 269         /* figure out if we already have a cmd buf: */
 270         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 271                 cmd = &msm_ring->submit.cmds[i];
 272                 if ((cmd->submit_offset == submit_offset) &&
 273                                 (cmd->size == size) &&
 274                                 (cmd->type == type) &&
 275                                 check_cmd_bo(ring, cmd, target_cmd->ring_bo))
 276                         return;
 277         }
 278
 279         /* create cmd buf if not: */
 280         i = APPEND(&msm_ring->submit, cmds);
 281         APPEND(msm_ring, cmds);
 282         msm_ring->cmds[i] = target_cmd;
 283         cmd = &msm_ring->submit.cmds[i];
 284         cmd->type = type;
 285         cmd->submit_idx = bo2idx(ring, target_cmd->ring_bo, FD_RELOC_READ);
 286         cmd->submit_offset = submit_offset;
 287         cmd->size = size;
 288         cmd->pad = 0;
 289
 290         target_cmd->size = size;
 291 }
 292
 293 static void * msm_ringbuffer_hostptr(struct fd_ringbuffer *ring)
 294 {
 295         return fd_bo_map(current_cmd(ring)->ring_bo);
 296 }
 297
 298 static uint32_t find_next_reloc_idx(struct msm_cmd *msm_cmd,
 299                 uint32_t start, uint32_t offset)
 300 {
 301         uint32_t i;
 302
 303         /* a binary search would be more clever.. */
 304         for (i = start; i < msm_cmd->nr_relocs; i++) {
 305                 struct drm_msm_gem_submit_reloc *reloc = &msm_cmd->relocs[i];
 306                 if (reloc->submit_offset >= offset)
 307                         return i;
 308         }
 309
 310         return i;
 311 }
 312
 313 static void delete_cmds(struct msm_ringbuffer *msm_ring)
 314 {
 315         struct msm_cmd *cmd, *tmp;
 316
 317         LIST_FOR_EACH_ENTRY_SAFE(cmd, tmp, &msm_ring->cmd_list, list) {
 318                 ring_cmd_del(cmd);
 319         }
 320 }
 321
 322 static void flush_reset(struct fd_ringbuffer *ring)
 323 {
 324         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 325         unsigned i;
 326
 327         for (i = 0; i < msm_ring->nr_bos; i++) {
 328                 struct msm_bo *msm_bo = to_msm_bo(msm_ring->bos[i]);
 329                 msm_bo->current_ring_seqno = 0;
 330                 fd_bo_del(&msm_bo->base);
 331         }
 332
 333         /* for each of the cmd buffers, clear their reloc's: */
 334         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 335                 struct msm_cmd *target_cmd = msm_ring->cmds[i];
 336                 target_cmd->nr_relocs = 0;
 337         }
 338
 339         msm_ring->submit.nr_cmds = 0;
 340         msm_ring->submit.nr_bos = 0;
 341         msm_ring->nr_cmds = 0;
 342         msm_ring->nr_bos = 0;
 343
 344         if (msm_ring->bo_table) {
 345                 drmHashDestroy(msm_ring->bo_table);
 346                 msm_ring->bo_table = NULL;
 347         }
 348
 349         if (msm_ring->is_growable) {
 350                 delete_cmds(msm_ring);
 351         } else {
 352                 /* in old mode, just reset the # of relocs: */
 353                 current_cmd(ring)->nr_relocs = 0;
 354         }
 355 }
 356
 357 static void finalize_current_cmd(struct fd_ringbuffer *ring, uint32_t *last_start)
 358 {
 359         uint32_t submit_offset, size, type;
 360         struct fd_ringbuffer *parent;
 361
 362         if (ring->parent) {
 363                 parent = ring->parent;
 364                 type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
 365         } else {
 366                 parent = ring;
 367                 type = MSM_SUBMIT_CMD_BUF;
 368         }
 369
 370         submit_offset = offset_bytes(last_start, ring->start);
 371         size = offset_bytes(ring->cur, last_start);
 372
 373         get_cmd(parent, current_cmd(ring), submit_offset, size, type);
 374 }
 375
 376 static void dump_submit(struct msm_ringbuffer *msm_ring)
 377 {
 378         uint32_t i, j;
 379
 380         for (i = 0; i < msm_ring->submit.nr_bos; i++) {
 381                 struct drm_msm_gem_submit_bo *bo = &msm_ring->submit.bos[i];
 382                 ERROR_MSG("  bos[%d]: handle=%u, flags=%x", i, bo->handle, bo->flags);
 383         }
 384         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 385                 struct drm_msm_gem_submit_cmd *cmd = &msm_ring->submit.cmds[i];
 386                 struct drm_msm_gem_submit_reloc *relocs = U642VOID(cmd->relocs);
 387                 ERROR_MSG("  cmd[%d]: type=%u, submit_idx=%u, submit_offset=%u, size=%u",
 388                                 i, cmd->type, cmd->submit_idx, cmd->submit_offset, cmd->size);
 389                 for (j = 0; j < cmd->nr_relocs; j++) {
 390                         struct drm_msm_gem_submit_reloc *r = &relocs[j];
 391                         ERROR_MSG("    reloc[%d]: submit_offset=%u, or=%08x, shift=%d, reloc_idx=%u"
 392                                         ", reloc_offset=%"PRIu64, j, r->submit_offset, r->or, r->shift,
 393                                         r->reloc_idx, r->reloc_offset);
 394                 }
 395         }
 396 }
 397
 398 static int msm_ringbuffer_flush(struct fd_ringbuffer *ring, uint32_t *last_start)
 399 {
 400         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 401         struct drm_msm_gem_submit req = {
 402                         .pipe = to_msm_pipe(ring->pipe)->pipe,
 403         };
 404         uint32_t i;
 405         int ret;
 406
 407         finalize_current_cmd(ring, last_start);
 408
 409         /* needs to be after get_cmd() as that could create bos/cmds table: */
 410         req.bos = VOID2U64(msm_ring->submit.bos),
 411         req.nr_bos = msm_ring->submit.nr_bos;
 412         req.cmds = VOID2U64(msm_ring->submit.cmds),
 413         req.nr_cmds = msm_ring->submit.nr_cmds;
 414
 415         /* for each of the cmd's fix up their reloc's: */
 416         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 417                 struct drm_msm_gem_submit_cmd *cmd = &msm_ring->submit.cmds[i];
 418                 struct msm_cmd *msm_cmd = msm_ring->cmds[i];
 419                 uint32_t a = find_next_reloc_idx(msm_cmd, 0, cmd->submit_offset);
 420                 uint32_t b = find_next_reloc_idx(msm_cmd, a, cmd->submit_offset + cmd->size);
 421                 cmd->relocs = VOID2U64(&msm_cmd->relocs[a]);
 422                 cmd->nr_relocs = (b > a) ? b - a : 0;
 423         }
 424
 425         DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos);
 426
 427         ret = drmCommandWriteRead(ring->pipe->dev->fd, DRM_MSM_GEM_SUBMIT,
 428                         &req, sizeof(req));
 429         if (ret) {
 430                 ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
 431                 dump_submit(msm_ring);
 432         } else if (!ret) {
 433                 /* update timestamp on all rings associated with submit: */
 434                 for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 435                         struct msm_cmd *msm_cmd = msm_ring->cmds[i];
 436                         msm_cmd->ring->last_timestamp = req.fence;
 437                 }
 438         }
 439
 440         flush_reset(ring);
 441
 442         return ret;
 443 }
 444
 445 static void msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size)
 446 {
 447         assert(to_msm_ringbuffer(ring)->is_growable);
 448         finalize_current_cmd(ring, ring->last_start);
 449         ring_cmd_new(ring, size);
 450 }
 451
 452 static void msm_ringbuffer_reset(struct fd_ringbuffer *ring)
 453 {
 454         flush_reset(ring);
 455 }
 456
 457 static void msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring,
 458                 const struct fd_reloc *r)
 459 {
 460         struct fd_ringbuffer *parent = ring->parent ? ring->parent : ring;
 461         struct msm_bo *msm_bo = to_msm_bo(r->bo);
 462         struct drm_msm_gem_submit_reloc *reloc;
 463         struct msm_cmd *cmd = current_cmd(ring);
 464         uint32_t idx = APPEND(cmd, relocs);
 465         uint32_t addr;
 466
 467         reloc = &cmd->relocs[idx];
 468
 469         reloc->reloc_idx = bo2idx(parent, r->bo, r->flags);
 470         reloc->reloc_offset = r->offset;
 471         reloc->or = r->or;
 472         reloc->shift = r->shift;
 473         reloc->submit_offset = offset_bytes(ring->cur, ring->start);
 474
 475         addr = msm_bo->presumed;
 476         if (r->shift < 0)
 477                 addr >>= -r->shift;
 478         else
 479                 addr <<= r->shift;
 480         (*ring->cur++) = addr | r->or;
 481 }
 482
 483 static uint32_t msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring,
 484                 struct fd_ringbuffer *target, uint32_t cmd_idx,
 485                 uint32_t submit_offset, uint32_t size)
 486 {
 487         struct msm_cmd *cmd = NULL;
 488         uint32_t idx = 0;
 489
 490         LIST_FOR_EACH_ENTRY(cmd, &to_msm_ringbuffer(target)->cmd_list, list) {
 491                 if (idx == cmd_idx)
 492                         break;
 493                 idx++;
 494         }
 495
 496         assert(cmd && (idx == cmd_idx));
 497
 498         if (idx < (to_msm_ringbuffer(target)->cmd_count - 1)) {
 499                 /* All but the last cmd buffer is fully "baked" (ie. already has
 500                  * done get_cmd() to add it to the cmds table).  But in this case,
 501                  * the size we get is invalid (since it is calculated from the
 502                  * last cmd buffer):
 503                  */
 504                 size = cmd->size;
 505         } else {
 506                 get_cmd(ring, cmd, submit_offset, size, MSM_SUBMIT_CMD_IB_TARGET_BUF);
 507         }
 508
 509         msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){
 510                 .bo = cmd->ring_bo,
 511                 .flags = FD_RELOC_READ,
 512                 .offset = submit_offset,
 513         });
 514
 515         return size;
 516 }
 517
 518 static uint32_t msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring)
 519 {
 520         return to_msm_ringbuffer(ring)->cmd_count;
 521 }
 522
 523 static void msm_ringbuffer_destroy(struct fd_ringbuffer *ring)
 524 {
 525         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 526
 527         flush_reset(ring);
 528         delete_cmds(msm_ring);
 529
 530         free(msm_ring->submit.cmds);
 531         free(msm_ring->submit.bos);
 532         free(msm_ring->bos);
 533         free(msm_ring->cmds);
 534         free(msm_ring);
 535 }
 536
 537 static const struct fd_ringbuffer_funcs funcs = {
 538                 .hostptr = msm_ringbuffer_hostptr,
 539                 .flush = msm_ringbuffer_flush,
 540                 .grow = msm_ringbuffer_grow,
 541                 .reset = msm_ringbuffer_reset,
 542                 .emit_reloc = msm_ringbuffer_emit_reloc,
 543                 .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring,
 544                 .cmd_count = msm_ringbuffer_cmd_count,
 545                 .destroy = msm_ringbuffer_destroy,
 546 };
 547
 548 drm_private struct fd_ringbuffer * msm_ringbuffer_new(struct fd_pipe *pipe,
 549                 uint32_t size)
 550 {
 551         struct msm_ringbuffer *msm_ring;
 552         struct fd_ringbuffer *ring = NULL;
 553
 554         msm_ring = calloc(1, sizeof(*msm_ring));
 555         if (!msm_ring) {
 556                 ERROR_MSG("allocation failed");
 557                 goto fail;
 558         }
 559
 560         if (size == 0) {
 561                 assert(pipe->dev->version >= FD_VERSION_UNLIMITED_CMDS);
 562                 size = INIT_SIZE;
 563                 msm_ring->is_growable = TRUE;
 564         }
 565
 566         list_inithead(&msm_ring->cmd_list);
 567         msm_ring->seqno = ++to_msm_device(pipe->dev)->ring_cnt;
 568
 569         ring = &msm_ring->base;
 570         ring->funcs = &funcs;
 571         ring->size = size;
 572         ring->pipe = pipe;   /* needed in ring_cmd_new() */
 573
 574         ring_cmd_new(ring, size);
 575
 576         return ring;
 577 fail:
 578         if (ring)
 579                 fd_ringbuffer_del(ring);
 580         return NULL;
 581 }