drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c

   1 /*
   2  * Copyright 2008 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Jerome Glisse <glisse@freedesktop.org>
  26  */
  27
  28 #include <linux/file.h>
  29 #include <linux/pagemap.h>
  30 #include <linux/sync_file.h>
  31 #include <linux/dma-buf.h>
  32
  33 #include <drm/amdgpu_drm.h>
  34 #include <drm/drm_syncobj.h>
  35 #include <drm/ttm/ttm_tt.h>
  36
  37 #include "amdgpu_cs.h"
  38 #include "amdgpu.h"
  39 #include "amdgpu_trace.h"
  40 #include "amdgpu_gmc.h"
  41 #include "amdgpu_gem.h"
  42 #include "amdgpu_ras.h"
  43
  44 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p,
  45                                  struct amdgpu_device *adev,
  46                                  struct drm_file *filp,
  47                                  union drm_amdgpu_cs *cs)
  48 {
  49         struct amdgpu_fpriv *fpriv = filp->driver_priv;
  50
  51         if (cs->in.num_chunks == 0)
  52                 return -EINVAL;
  53
  54         memset(p, 0, sizeof(*p));
  55         p->adev = adev;
  56         p->filp = filp;
  57
  58         p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
  59         if (!p->ctx)
  60                 return -EINVAL;
  61
  62         if (atomic_read(&p->ctx->guilty)) {
  63                 amdgpu_ctx_put(p->ctx);
  64                 return -ECANCELED;
  65         }
  66
  67         amdgpu_sync_create(&p->sync);
  68         drm_exec_init(&p->exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
  69         return 0;
  70 }
  71
  72 static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p,
  73                              struct drm_amdgpu_cs_chunk_ib *chunk_ib)
  74 {
  75         struct drm_sched_entity *entity;
  76         unsigned int i;
  77         int r;
  78
  79         r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type,
  80                                   chunk_ib->ip_instance,
  81                                   chunk_ib->ring, &entity);
  82         if (r)
  83                 return r;
  84
  85         /*
  86          * Abort if there is no run queue associated with this entity.
  87          * Possibly because of disabled HW IP.
  88          */
  89         if (entity->rq == NULL)
  90                 return -EINVAL;
  91
  92         /* Check if we can add this IB to some existing job */
  93         for (i = 0; i < p->gang_size; ++i)
  94                 if (p->entities[i] == entity)
  95                         return i;
  96
  97         /* If not increase the gang size if possible */
  98         if (i == AMDGPU_CS_GANG_SIZE)
  99                 return -EINVAL;
 100
 101         p->entities[i] = entity;
 102         p->gang_size = i + 1;
 103         return i;
 104 }
 105
 106 static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,
 107                            struct drm_amdgpu_cs_chunk_ib *chunk_ib,
 108                            unsigned int *num_ibs)
 109 {
 110         int r;
 111
 112         r = amdgpu_cs_job_idx(p, chunk_ib);
 113         if (r < 0)
 114                 return r;
 115
 116         if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type))
 117                 return -EINVAL;
 118
 119         ++(num_ibs[r]);
 120         p->gang_leader_idx = r;
 121         return 0;
 122 }
 123
 124 static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p,
 125                                    struct drm_amdgpu_cs_chunk_fence *data,
 126                                    uint32_t *offset)
 127 {
 128         struct drm_gem_object *gobj;
 129         unsigned long size;
 130
 131         gobj = drm_gem_object_lookup(p->filp, data->handle);
 132         if (gobj == NULL)
 133                 return -EINVAL;
 134
 135         p->uf_bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
 136         drm_gem_object_put(gobj);
 137
 138         size = amdgpu_bo_size(p->uf_bo);
 139         if (size != PAGE_SIZE || data->offset > (size - 8))
 140                 return -EINVAL;
 141
 142         if (amdgpu_ttm_tt_get_usermm(p->uf_bo->tbo.ttm))
 143                 return -EINVAL;
 144
 145         *offset = data->offset;
 146         return 0;
 147 }
 148
 149 static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p,
 150                                    struct drm_amdgpu_bo_list_in *data)
 151 {
 152         struct drm_amdgpu_bo_list_entry *info;
 153         int r;
 154
 155         r = amdgpu_bo_create_list_entry_array(data, &info);
 156         if (r)
 157                 return r;
 158
 159         r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number,
 160                                   &p->bo_list);
 161         if (r)
 162                 goto error_free;
 163
 164         kvfree(info);
 165         return 0;
 166
 167 error_free:
 168         kvfree(info);
 169
 170         return r;
 171 }
 172
 173 /* Copy the data from userspace and go over it the first time */
 174 static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
 175                            union drm_amdgpu_cs *cs)
 176 {
 177         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
 178         unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { };
 179         struct amdgpu_vm *vm = &fpriv->vm;
 180         uint64_t *chunk_array_user;
 181         uint64_t *chunk_array;
 182         uint32_t uf_offset = 0;
 183         size_t size;
 184         int ret;
 185         int i;
 186
 187         chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t),
 188                                      GFP_KERNEL);
 189         if (!chunk_array)
 190                 return -ENOMEM;
 191
 192         /* get chunks */
 193         chunk_array_user = u64_to_user_ptr(cs->in.chunks);
 194         if (copy_from_user(chunk_array, chunk_array_user,
 195                            sizeof(uint64_t)*cs->in.num_chunks)) {
 196                 ret = -EFAULT;
 197                 goto free_chunk;
 198         }
 199
 200         p->nchunks = cs->in.num_chunks;
 201         p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
 202                             GFP_KERNEL);
 203         if (!p->chunks) {
 204                 ret = -ENOMEM;
 205                 goto free_chunk;
 206         }
 207
 208         for (i = 0; i < p->nchunks; i++) {
 209                 struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
 210                 struct drm_amdgpu_cs_chunk user_chunk;
 211                 uint32_t __user *cdata;
 212
 213                 chunk_ptr = u64_to_user_ptr(chunk_array[i]);
 214                 if (copy_from_user(&user_chunk, chunk_ptr,
 215                                        sizeof(struct drm_amdgpu_cs_chunk))) {
 216                         ret = -EFAULT;
 217                         i--;
 218                         goto free_partial_kdata;
 219                 }
 220                 p->chunks[i].chunk_id = user_chunk.chunk_id;
 221                 p->chunks[i].length_dw = user_chunk.length_dw;
 222
 223                 size = p->chunks[i].length_dw;
 224                 cdata = u64_to_user_ptr(user_chunk.chunk_data);
 225
 226                 p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t),
 227                                                     GFP_KERNEL);
 228                 if (p->chunks[i].kdata == NULL) {
 229                         ret = -ENOMEM;
 230                         i--;
 231                         goto free_partial_kdata;
 232                 }
 233                 size *= sizeof(uint32_t);
 234                 if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
 235                         ret = -EFAULT;
 236                         goto free_partial_kdata;
 237                 }
 238
 239                 /* Assume the worst on the following checks */
 240                 ret = -EINVAL;
 241                 switch (p->chunks[i].chunk_id) {
 242                 case AMDGPU_CHUNK_ID_IB:
 243                         if (size < sizeof(struct drm_amdgpu_cs_chunk_ib))
 244                                 goto free_partial_kdata;
 245
 246                         ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs);
 247                         if (ret)
 248                                 goto free_partial_kdata;
 249                         break;
 250
 251                 case AMDGPU_CHUNK_ID_FENCE:
 252                         if (size < sizeof(struct drm_amdgpu_cs_chunk_fence))
 253                                 goto free_partial_kdata;
 254
 255                         ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata,
 256                                                       &uf_offset);
 257                         if (ret)
 258                                 goto free_partial_kdata;
 259                         break;
 260
 261                 case AMDGPU_CHUNK_ID_BO_HANDLES:
 262                         if (size < sizeof(struct drm_amdgpu_bo_list_in))
 263                                 goto free_partial_kdata;
 264
 265                         ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata);
 266                         if (ret)
 267                                 goto free_partial_kdata;
 268                         break;
 269
 270                 case AMDGPU_CHUNK_ID_DEPENDENCIES:
 271                 case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
 272                 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
 273                 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
 274                 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
 275                 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
 276                 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:
 277                         break;
 278
 279                 default:
 280                         goto free_partial_kdata;
 281                 }
 282         }
 283
 284         if (!p->gang_size) {
 285                 ret = -EINVAL;
 286                 goto free_all_kdata;
 287         }
 288
 289         for (i = 0; i < p->gang_size; ++i) {
 290                 ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,
 291                                        num_ibs[i], &p->jobs[i]);
 292                 if (ret)
 293                         goto free_all_kdata;
 294         }
 295         p->gang_leader = p->jobs[p->gang_leader_idx];
 296
 297         if (p->ctx->generation != p->gang_leader->generation) {
 298                 ret = -ECANCELED;
 299                 goto free_all_kdata;
 300         }
 301
 302         if (p->uf_bo)
 303                 p->gang_leader->uf_addr = uf_offset;
 304         kvfree(chunk_array);
 305
 306         /* Use this opportunity to fill in task info for the vm */
 307         amdgpu_vm_set_task_info(vm);
 308
 309         return 0;
 310
 311 free_all_kdata:
 312         i = p->nchunks - 1;
 313 free_partial_kdata:
 314         for (; i >= 0; i--)
 315                 kvfree(p->chunks[i].kdata);
 316         kvfree(p->chunks);
 317         p->chunks = NULL;
 318         p->nchunks = 0;
 319 free_chunk:
 320         kvfree(chunk_array);
 321
 322         return ret;
 323 }
 324
 325 static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
 326                            struct amdgpu_cs_chunk *chunk,
 327                            unsigned int *ce_preempt,
 328                            unsigned int *de_preempt)
 329 {
 330         struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata;
 331         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
 332         struct amdgpu_vm *vm = &fpriv->vm;
 333         struct amdgpu_ring *ring;
 334         struct amdgpu_job *job;
 335         struct amdgpu_ib *ib;
 336         int r;
 337
 338         r = amdgpu_cs_job_idx(p, chunk_ib);
 339         if (r < 0)
 340                 return r;
 341
 342         job = p->jobs[r];
 343         ring = amdgpu_job_ring(job);
 344         ib = &job->ibs[job->num_ibs++];
 345
 346         /* MM engine doesn't support user fences */
 347         if (p->uf_bo && ring->funcs->no_user_fence)
 348                 return -EINVAL;
 349
 350         if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
 351             chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
 352                 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
 353                         (*ce_preempt)++;
 354                 else
 355                         (*de_preempt)++;
 356
 357                 /* Each GFX command submit allows only 1 IB max
 358                  * preemptible for CE & DE */
 359                 if (*ce_preempt > 1 || *de_preempt > 1)
 360                         return -EINVAL;
 361         }
 362
 363         if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)
 364                 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;
 365
 366         r =  amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ?
 367                            chunk_ib->ib_bytes : 0,
 368                            AMDGPU_IB_POOL_DELAYED, ib);
 369         if (r) {
 370                 DRM_ERROR("Failed to get ib !\n");
 371                 return r;
 372         }
 373
 374         ib->gpu_addr = chunk_ib->va_start;
 375         ib->length_dw = chunk_ib->ib_bytes / 4;
 376         ib->flags = chunk_ib->flags;
 377         return 0;
 378 }
 379
 380 static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p,
 381                                      struct amdgpu_cs_chunk *chunk)
 382 {
 383         struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata;
 384         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
 385         unsigned int num_deps;
 386         int i, r;
 387
 388         num_deps = chunk->length_dw * 4 /
 389                 sizeof(struct drm_amdgpu_cs_chunk_dep);
 390
 391         for (i = 0; i < num_deps; ++i) {
 392                 struct amdgpu_ctx *ctx;
 393                 struct drm_sched_entity *entity;
 394                 struct dma_fence *fence;
 395
 396                 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
 397                 if (ctx == NULL)
 398                         return -EINVAL;
 399
 400                 r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type,
 401                                           deps[i].ip_instance,
 402                                           deps[i].ring, &entity);
 403                 if (r) {
 404                         amdgpu_ctx_put(ctx);
 405                         return r;
 406                 }
 407
 408                 fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);
 409                 amdgpu_ctx_put(ctx);
 410
 411                 if (IS_ERR(fence))
 412                         return PTR_ERR(fence);
 413                 else if (!fence)
 414                         continue;
 415
 416                 if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
 417                         struct drm_sched_fence *s_fence;
 418                         struct dma_fence *old = fence;
 419
 420                         s_fence = to_drm_sched_fence(fence);
 421                         fence = dma_fence_get(&s_fence->scheduled);
 422                         dma_fence_put(old);
 423                 }
 424
 425                 r = amdgpu_sync_fence(&p->sync, fence);
 426                 dma_fence_put(fence);
 427                 if (r)
 428                         return r;
 429         }
 430         return 0;
 431 }
 432
 433 static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p,
 434                                          uint32_t handle, u64 point,
 435                                          u64 flags)
 436 {
 437         struct dma_fence *fence;
 438         int r;
 439
 440         r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);
 441         if (r) {
 442                 DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",
 443                           handle, point, r);
 444                 return r;
 445         }
 446
 447         r = amdgpu_sync_fence(&p->sync, fence);
 448         dma_fence_put(fence);
 449         return r;
 450 }
 451
 452 static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p,
 453                                    struct amdgpu_cs_chunk *chunk)
 454 {
 455         struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
 456         unsigned int num_deps;
 457         int i, r;
 458
 459         num_deps = chunk->length_dw * 4 /
 460                 sizeof(struct drm_amdgpu_cs_chunk_sem);
 461         for (i = 0; i < num_deps; ++i) {
 462                 r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0);
 463                 if (r)
 464                         return r;
 465         }
 466
 467         return 0;
 468 }
 469
 470 static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p,
 471                                               struct amdgpu_cs_chunk *chunk)
 472 {
 473         struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
 474         unsigned int num_deps;
 475         int i, r;
 476
 477         num_deps = chunk->length_dw * 4 /
 478                 sizeof(struct drm_amdgpu_cs_chunk_syncobj);
 479         for (i = 0; i < num_deps; ++i) {
 480                 r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle,
 481                                                   syncobj_deps[i].point,
 482                                                   syncobj_deps[i].flags);
 483                 if (r)
 484                         return r;
 485         }
 486
 487         return 0;
 488 }
 489
 490 static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p,
 491                                     struct amdgpu_cs_chunk *chunk)
 492 {
 493         struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
 494         unsigned int num_deps;
 495         int i;
 496
 497         num_deps = chunk->length_dw * 4 /
 498                 sizeof(struct drm_amdgpu_cs_chunk_sem);
 499
 500         if (p->post_deps)
 501                 return -EINVAL;
 502
 503         p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
 504                                      GFP_KERNEL);
 505         p->num_post_deps = 0;
 506
 507         if (!p->post_deps)
 508                 return -ENOMEM;
 509
 510
 511         for (i = 0; i < num_deps; ++i) {
 512                 p->post_deps[i].syncobj =
 513                         drm_syncobj_find(p->filp, deps[i].handle);
 514                 if (!p->post_deps[i].syncobj)
 515                         return -EINVAL;
 516                 p->post_deps[i].chain = NULL;
 517                 p->post_deps[i].point = 0;
 518                 p->num_post_deps++;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p,
 525                                                 struct amdgpu_cs_chunk *chunk)
 526 {
 527         struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
 528         unsigned int num_deps;
 529         int i;
 530
 531         num_deps = chunk->length_dw * 4 /
 532                 sizeof(struct drm_amdgpu_cs_chunk_syncobj);
 533
 534         if (p->post_deps)
 535                 return -EINVAL;
 536
 537         p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
 538                                      GFP_KERNEL);
 539         p->num_post_deps = 0;
 540
 541         if (!p->post_deps)
 542                 return -ENOMEM;
 543
 544         for (i = 0; i < num_deps; ++i) {
 545                 struct amdgpu_cs_post_dep *dep = &p->post_deps[i];
 546
 547                 dep->chain = NULL;
 548                 if (syncobj_deps[i].point) {
 549                         dep->chain = dma_fence_chain_alloc();
 550                         if (!dep->chain)
 551                                 return -ENOMEM;
 552                 }
 553
 554                 dep->syncobj = drm_syncobj_find(p->filp,
 555                                                 syncobj_deps[i].handle);
 556                 if (!dep->syncobj) {
 557                         dma_fence_chain_free(dep->chain);
 558                         return -EINVAL;
 559                 }
 560                 dep->point = syncobj_deps[i].point;
 561                 p->num_post_deps++;
 562         }
 563
 564         return 0;
 565 }
 566
 567 static int amdgpu_cs_p2_shadow(struct amdgpu_cs_parser *p,
 568                                struct amdgpu_cs_chunk *chunk)
 569 {
 570         struct drm_amdgpu_cs_chunk_cp_gfx_shadow *shadow = chunk->kdata;
 571         int i;
 572
 573         if (shadow->flags & ~AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW)
 574                 return -EINVAL;
 575
 576         for (i = 0; i < p->gang_size; ++i) {
 577                 p->jobs[i]->shadow_va = shadow->shadow_va;
 578                 p->jobs[i]->csa_va = shadow->csa_va;
 579                 p->jobs[i]->gds_va = shadow->gds_va;
 580                 p->jobs[i]->init_shadow =
 581                         shadow->flags & AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW;
 582         }
 583
 584         return 0;
 585 }
 586
 587 static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p)
 588 {
 589         unsigned int ce_preempt = 0, de_preempt = 0;
 590         int i, r;
 591
 592         for (i = 0; i < p->nchunks; ++i) {
 593                 struct amdgpu_cs_chunk *chunk;
 594
 595                 chunk = &p->chunks[i];
 596
 597                 switch (chunk->chunk_id) {
 598                 case AMDGPU_CHUNK_ID_IB:
 599                         r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt);
 600                         if (r)
 601                                 return r;
 602                         break;
 603                 case AMDGPU_CHUNK_ID_DEPENDENCIES:
 604                 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
 605                         r = amdgpu_cs_p2_dependencies(p, chunk);
 606                         if (r)
 607                                 return r;
 608                         break;
 609                 case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
 610                         r = amdgpu_cs_p2_syncobj_in(p, chunk);
 611                         if (r)
 612                                 return r;
 613                         break;
 614                 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
 615                         r = amdgpu_cs_p2_syncobj_out(p, chunk);
 616                         if (r)
 617                                 return r;
 618                         break;
 619                 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
 620                         r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk);
 621                         if (r)
 622                                 return r;
 623                         break;
 624                 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
 625                         r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk);
 626                         if (r)
 627                                 return r;
 628                         break;
 629                 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:
 630                         r = amdgpu_cs_p2_shadow(p, chunk);
 631                         if (r)
 632                                 return r;
 633                         break;
 634                 }
 635         }
 636
 637         return 0;
 638 }
 639
 640 /* Convert microseconds to bytes. */
 641 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
 642 {
 643         if (us <= 0 || !adev->mm_stats.log2_max_MBps)
 644                 return 0;
 645
 646         /* Since accum_us is incremented by a million per second, just
 647          * multiply it by the number of MB/s to get the number of bytes.
 648          */
 649         return us << adev->mm_stats.log2_max_MBps;
 650 }
 651
 652 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
 653 {
 654         if (!adev->mm_stats.log2_max_MBps)
 655                 return 0;
 656
 657         return bytes >> adev->mm_stats.log2_max_MBps;
 658 }
 659
 660 /* Returns how many bytes TTM can move right now. If no bytes can be moved,
 661  * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
 662  * which means it can go over the threshold once. If that happens, the driver
 663  * will be in debt and no other buffer migrations can be done until that debt
 664  * is repaid.
 665  *
 666  * This approach allows moving a buffer of any size (it's important to allow
 667  * that).
 668  *
 669  * The currency is simply time in microseconds and it increases as the clock
 670  * ticks. The accumulated microseconds (us) are converted to bytes and
 671  * returned.
 672  */
 673 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
 674                                               u64 *max_bytes,
 675                                               u64 *max_vis_bytes)
 676 {
 677         s64 time_us, increment_us;
 678         u64 free_vram, total_vram, used_vram;
 679         /* Allow a maximum of 200 accumulated ms. This is basically per-IB
 680          * throttling.
 681          *
 682          * It means that in order to get full max MBps, at least 5 IBs per
 683          * second must be submitted and not more than 200ms apart from each
 684          * other.
 685          */
 686         const s64 us_upper_bound = 200000;
 687
 688         if (!adev->mm_stats.log2_max_MBps) {
 689                 *max_bytes = 0;
 690                 *max_vis_bytes = 0;
 691                 return;
 692         }
 693
 694         total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size);
 695         used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager);
 696         free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
 697
 698         spin_lock(&adev->mm_stats.lock);
 699
 700         /* Increase the amount of accumulated us. */
 701         time_us = ktime_to_us(ktime_get());
 702         increment_us = time_us - adev->mm_stats.last_update_us;
 703         adev->mm_stats.last_update_us = time_us;
 704         adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
 705                                       us_upper_bound);
 706
 707         /* This prevents the short period of low performance when the VRAM
 708          * usage is low and the driver is in debt or doesn't have enough
 709          * accumulated us to fill VRAM quickly.
 710          *
 711          * The situation can occur in these cases:
 712          * - a lot of VRAM is freed by userspace
 713          * - the presence of a big buffer causes a lot of evictions
 714          *   (solution: split buffers into smaller ones)
 715          *
 716          * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
 717          * accum_us to a positive number.
 718          */
 719         if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
 720                 s64 min_us;
 721
 722                 /* Be more aggressive on dGPUs. Try to fill a portion of free
 723                  * VRAM now.
 724                  */
 725                 if (!(adev->flags & AMD_IS_APU))
 726                         min_us = bytes_to_us(adev, free_vram / 4);
 727                 else
 728                         min_us = 0; /* Reset accum_us on APUs. */
 729
 730                 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
 731         }
 732
 733         /* This is set to 0 if the driver is in debt to disallow (optional)
 734          * buffer moves.
 735          */
 736         *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
 737
 738         /* Do the same for visible VRAM if half of it is free */
 739         if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) {
 740                 u64 total_vis_vram = adev->gmc.visible_vram_size;
 741                 u64 used_vis_vram =
 742                   amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr);
 743
 744                 if (used_vis_vram < total_vis_vram) {
 745                         u64 free_vis_vram = total_vis_vram - used_vis_vram;
 746
 747                         adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +
 748                                                           increment_us, us_upper_bound);
 749
 750                         if (free_vis_vram >= total_vis_vram / 2)
 751                                 adev->mm_stats.accum_us_vis =
 752                                         max(bytes_to_us(adev, free_vis_vram / 2),
 753                                             adev->mm_stats.accum_us_vis);
 754                 }
 755
 756                 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);
 757         } else {
 758                 *max_vis_bytes = 0;
 759         }
 760
 761         spin_unlock(&adev->mm_stats.lock);
 762 }
 763
 764 /* Report how many bytes have really been moved for the last command
 765  * submission. This can result in a debt that can stop buffer migrations
 766  * temporarily.
 767  */
 768 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
 769                                   u64 num_vis_bytes)
 770 {
 771         spin_lock(&adev->mm_stats.lock);
 772         adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
 773         adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);
 774         spin_unlock(&adev->mm_stats.lock);
 775 }
 776
 777 static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo)
 778 {
 779         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 780         struct amdgpu_cs_parser *p = param;
 781         struct ttm_operation_ctx ctx = {
 782                 .interruptible = true,
 783                 .no_wait_gpu = false,
 784                 .resv = bo->tbo.base.resv
 785         };
 786         uint32_t domain;
 787         int r;
 788
 789         if (bo->tbo.pin_count)
 790                 return 0;
 791
 792         /* Don't move this buffer if we have depleted our allowance
 793          * to move it. Don't move anything if the threshold is zero.
 794          */
 795         if (p->bytes_moved < p->bytes_moved_threshold &&
 796             (!bo->tbo.base.dma_buf ||
 797             list_empty(&bo->tbo.base.dma_buf->attachments))) {
 798                 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 799                     (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
 800                         /* And don't move a CPU_ACCESS_REQUIRED BO to limited
 801                          * visible VRAM if we've depleted our allowance to do
 802                          * that.
 803                          */
 804                         if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)
 805                                 domain = bo->preferred_domains;
 806                         else
 807                                 domain = bo->allowed_domains;
 808                 } else {
 809                         domain = bo->preferred_domains;
 810                 }
 811         } else {
 812                 domain = bo->allowed_domains;
 813         }
 814
 815 retry:
 816         amdgpu_bo_placement_from_domain(bo, domain);
 817         r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 818
 819         p->bytes_moved += ctx.bytes_moved;
 820         if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 821             amdgpu_bo_in_cpu_visible_vram(bo))
 822                 p->bytes_moved_vis += ctx.bytes_moved;
 823
 824         if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
 825                 domain = bo->allowed_domains;
 826                 goto retry;
 827         }
 828
 829         return r;
 830 }
 831
 832 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 833                                 union drm_amdgpu_cs *cs)
 834 {
 835         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
 836         struct ttm_operation_ctx ctx = { true, false };
 837         struct amdgpu_vm *vm = &fpriv->vm;
 838         struct amdgpu_bo_list_entry *e;
 839         struct drm_gem_object *obj;
 840         unsigned long index;
 841         unsigned int i;
 842         int r;
 843
 844         /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
 845         if (cs->in.bo_list_handle) {
 846                 if (p->bo_list)
 847                         return -EINVAL;
 848
 849                 r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle,
 850                                        &p->bo_list);
 851                 if (r)
 852                         return r;
 853         } else if (!p->bo_list) {
 854                 /* Create a empty bo_list when no handle is provided */
 855                 r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0,
 856                                           &p->bo_list);
 857                 if (r)
 858                         return r;
 859         }
 860
 861         mutex_lock(&p->bo_list->bo_list_mutex);
 862
 863         /* Get userptr backing pages. If pages are updated after registered
 864          * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do
 865          * amdgpu_ttm_backend_bind() to flush and invalidate new pages
 866          */
 867         amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
 868                 bool userpage_invalidated = false;
 869                 struct amdgpu_bo *bo = e->bo;
 870                 int i;
 871
 872                 e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
 873                                         sizeof(struct page *),
 874                                         GFP_KERNEL | __GFP_ZERO);
 875                 if (!e->user_pages) {
 876                         DRM_ERROR("kvmalloc_array failure\n");
 877                         r = -ENOMEM;
 878                         goto out_free_user_pages;
 879                 }
 880
 881                 r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages, &e->range);
 882                 if (r) {
 883                         kvfree(e->user_pages);
 884                         e->user_pages = NULL;
 885                         goto out_free_user_pages;
 886                 }
 887
 888                 for (i = 0; i < bo->tbo.ttm->num_pages; i++) {
 889                         if (bo->tbo.ttm->pages[i] != e->user_pages[i]) {
 890                                 userpage_invalidated = true;
 891                                 break;
 892                         }
 893                 }
 894                 e->user_invalidated = userpage_invalidated;
 895         }
 896
 897         drm_exec_until_all_locked(&p->exec) {
 898                 r = amdgpu_vm_lock_pd(&fpriv->vm, &p->exec, 1 + p->gang_size);
 899                 drm_exec_retry_on_contention(&p->exec);
 900                 if (unlikely(r))
 901                         goto out_free_user_pages;
 902
 903                 amdgpu_bo_list_for_each_entry(e, p->bo_list) {
 904                         /* One fence for TTM and one for each CS job */
 905                         r = drm_exec_prepare_obj(&p->exec, &e->bo->tbo.base,
 906                                                  1 + p->gang_size);
 907                         drm_exec_retry_on_contention(&p->exec);
 908                         if (unlikely(r))
 909                                 goto out_free_user_pages;
 910
 911                         e->bo_va = amdgpu_vm_bo_find(vm, e->bo);
 912                 }
 913
 914                 if (p->uf_bo) {
 915                         r = drm_exec_prepare_obj(&p->exec, &p->uf_bo->tbo.base,
 916                                                  1 + p->gang_size);
 917                         drm_exec_retry_on_contention(&p->exec);
 918                         if (unlikely(r))
 919                                 goto out_free_user_pages;
 920                 }
 921         }
 922
 923         amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
 924                 struct mm_struct *usermm;
 925
 926                 usermm = amdgpu_ttm_tt_get_usermm(e->bo->tbo.ttm);
 927                 if (usermm && usermm != current->mm) {
 928                         r = -EPERM;
 929                         goto out_free_user_pages;
 930                 }
 931
 932                 if (amdgpu_ttm_tt_is_userptr(e->bo->tbo.ttm) &&
 933                     e->user_invalidated && e->user_pages) {
 934                         amdgpu_bo_placement_from_domain(e->bo,
 935                                                         AMDGPU_GEM_DOMAIN_CPU);
 936                         r = ttm_bo_validate(&e->bo->tbo, &e->bo->placement,
 937                                             &ctx);
 938                         if (r)
 939                                 goto out_free_user_pages;
 940
 941                         amdgpu_ttm_tt_set_user_pages(e->bo->tbo.ttm,
 942                                                      e->user_pages);
 943                 }
 944
 945                 kvfree(e->user_pages);
 946                 e->user_pages = NULL;
 947         }
 948
 949         amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
 950                                           &p->bytes_moved_vis_threshold);
 951         p->bytes_moved = 0;
 952         p->bytes_moved_vis = 0;
 953
 954         r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm,
 955                                       amdgpu_cs_bo_validate, p);
 956         if (r) {
 957                 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
 958                 goto out_free_user_pages;
 959         }
 960
 961         drm_exec_for_each_locked_object(&p->exec, index, obj) {
 962                 r = amdgpu_cs_bo_validate(p, gem_to_amdgpu_bo(obj));
 963                 if (unlikely(r))
 964                         goto out_free_user_pages;
 965         }
 966
 967         if (p->uf_bo) {
 968                 r = amdgpu_ttm_alloc_gart(&p->uf_bo->tbo);
 969                 if (unlikely(r))
 970                         goto out_free_user_pages;
 971
 972                 p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(p->uf_bo);
 973         }
 974
 975         amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
 976                                      p->bytes_moved_vis);
 977
 978         for (i = 0; i < p->gang_size; ++i)
 979                 amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj,
 980                                          p->bo_list->gws_obj,
 981                                          p->bo_list->oa_obj);
 982         return 0;
 983
 984 out_free_user_pages:
 985         amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
 986                 struct amdgpu_bo *bo = e->bo;
 987
 988                 if (!e->user_pages)
 989                         continue;
 990                 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, e->range);
 991                 kvfree(e->user_pages);
 992                 e->user_pages = NULL;
 993                 e->range = NULL;
 994         }
 995         mutex_unlock(&p->bo_list->bo_list_mutex);
 996         return r;
 997 }
 998
 999 static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p)
1000 {
1001         int i, j;
1002
1003         if (!trace_amdgpu_cs_enabled())
1004                 return;
1005
1006         for (i = 0; i < p->gang_size; ++i) {
1007                 struct amdgpu_job *job = p->jobs[i];
1008
1009                 for (j = 0; j < job->num_ibs; ++j)
1010                         trace_amdgpu_cs(p, job, &job->ibs[j]);
1011         }
1012 }
1013
1014 static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p,
1015                                struct amdgpu_job *job)
1016 {
1017         struct amdgpu_ring *ring = amdgpu_job_ring(job);
1018         unsigned int i;
1019         int r;
1020
1021         /* Only for UVD/VCE VM emulation */
1022         if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place)
1023                 return 0;
1024
1025         for (i = 0; i < job->num_ibs; ++i) {
1026                 struct amdgpu_ib *ib = &job->ibs[i];
1027                 struct amdgpu_bo_va_mapping *m;
1028                 struct amdgpu_bo *aobj;
1029                 uint64_t va_start;
1030                 uint8_t *kptr;
1031
1032                 va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK;
1033                 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
1034                 if (r) {
1035                         DRM_ERROR("IB va_start is invalid\n");
1036                         return r;
1037                 }
1038
1039                 if ((va_start + ib->length_dw * 4) >
1040                     (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
1041                         DRM_ERROR("IB va_start+ib_bytes is invalid\n");
1042                         return -EINVAL;
1043                 }
1044
1045                 /* the IB should be reserved at this point */
1046                 r = amdgpu_bo_kmap(aobj, (void **)&kptr);
1047                 if (r)
1048                         return r;
1049
1050                 kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE);
1051
1052                 if (ring->funcs->parse_cs) {
1053                         memcpy(ib->ptr, kptr, ib->length_dw * 4);
1054                         amdgpu_bo_kunmap(aobj);
1055
1056                         r = amdgpu_ring_parse_cs(ring, p, job, ib);
1057                         if (r)
1058                                 return r;
1059                 } else {
1060                         ib->ptr = (uint32_t *)kptr;
1061                         r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib);
1062                         amdgpu_bo_kunmap(aobj);
1063                         if (r)
1064                                 return r;
1065                 }
1066         }
1067
1068         return 0;
1069 }
1070
1071 static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p)
1072 {
1073         unsigned int i;
1074         int r;
1075
1076         for (i = 0; i < p->gang_size; ++i) {
1077                 r = amdgpu_cs_patch_ibs(p, p->jobs[i]);
1078                 if (r)
1079                         return r;
1080         }
1081         return 0;
1082 }
1083
1084 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
1085 {
1086         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1087         struct amdgpu_job *job = p->gang_leader;
1088         struct amdgpu_device *adev = p->adev;
1089         struct amdgpu_vm *vm = &fpriv->vm;
1090         struct amdgpu_bo_list_entry *e;
1091         struct amdgpu_bo_va *bo_va;
1092         unsigned int i;
1093         int r;
1094
1095         r = amdgpu_vm_clear_freed(adev, vm, NULL);
1096         if (r)
1097                 return r;
1098
1099         r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
1100         if (r)
1101                 return r;
1102
1103         r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update);
1104         if (r)
1105                 return r;
1106
1107         if (fpriv->csa_va) {
1108                 bo_va = fpriv->csa_va;
1109                 BUG_ON(!bo_va);
1110                 r = amdgpu_vm_bo_update(adev, bo_va, false);
1111                 if (r)
1112                         return r;
1113
1114                 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);
1115                 if (r)
1116                         return r;
1117         }
1118
1119         amdgpu_bo_list_for_each_entry(e, p->bo_list) {
1120                 bo_va = e->bo_va;
1121                 if (bo_va == NULL)
1122                         continue;
1123
1124                 r = amdgpu_vm_bo_update(adev, bo_va, false);
1125                 if (r)
1126                         return r;
1127
1128                 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);
1129                 if (r)
1130                         return r;
1131         }
1132
1133         r = amdgpu_vm_handle_moved(adev, vm);
1134         if (r)
1135                 return r;
1136
1137         r = amdgpu_vm_update_pdes(adev, vm, false);
1138         if (r)
1139                 return r;
1140
1141         r = amdgpu_sync_fence(&p->sync, vm->last_update);
1142         if (r)
1143                 return r;
1144
1145         for (i = 0; i < p->gang_size; ++i) {
1146                 job = p->jobs[i];
1147
1148                 if (!job->vm)
1149                         continue;
1150
1151                 job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);
1152         }
1153
1154         if (amdgpu_vm_debug) {
1155                 /* Invalidate all BOs to test for userspace bugs */
1156                 amdgpu_bo_list_for_each_entry(e, p->bo_list) {
1157                         struct amdgpu_bo *bo = e->bo;
1158
1159                         /* ignore duplicates */
1160                         if (!bo)
1161                                 continue;
1162
1163                         amdgpu_vm_bo_invalidate(adev, bo, false);
1164                 }
1165         }
1166
1167         return 0;
1168 }
1169
1170 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
1171 {
1172         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1173         struct drm_gpu_scheduler *sched;
1174         struct drm_gem_object *obj;
1175         struct dma_fence *fence;
1176         unsigned long index;
1177         unsigned int i;
1178         int r;
1179
1180         r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_leader_idx]);
1181         if (r) {
1182                 if (r != -ERESTARTSYS)
1183                         DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n");
1184                 return r;
1185         }
1186
1187         drm_exec_for_each_locked_object(&p->exec, index, obj) {
1188                 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
1189
1190                 struct dma_resv *resv = bo->tbo.base.resv;
1191                 enum amdgpu_sync_mode sync_mode;
1192
1193                 sync_mode = amdgpu_bo_explicit_sync(bo) ?
1194                         AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER;
1195                 r = amdgpu_sync_resv(p->adev, &p->sync, resv, sync_mode,
1196                                      &fpriv->vm);
1197                 if (r)
1198                         return r;
1199         }
1200
1201         for (i = 0; i < p->gang_size; ++i) {
1202                 r = amdgpu_sync_push_to_job(&p->sync, p->jobs[i]);
1203                 if (r)
1204                         return r;
1205         }
1206
1207         sched = p->gang_leader->base.entity->rq->sched;
1208         while ((fence = amdgpu_sync_get_fence(&p->sync))) {
1209                 struct drm_sched_fence *s_fence = to_drm_sched_fence(fence);
1210
1211                 /*
1212                  * When we have an dependency it might be necessary to insert a
1213                  * pipeline sync to make sure that all caches etc are flushed and the
1214                  * next job actually sees the results from the previous one
1215                  * before we start executing on the same scheduler ring.
1216                  */
1217                 if (!s_fence || s_fence->sched != sched) {
1218                         dma_fence_put(fence);
1219                         continue;
1220                 }
1221
1222                 r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence);
1223                 dma_fence_put(fence);
1224                 if (r)
1225                         return r;
1226         }
1227         return 0;
1228 }
1229
1230 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
1231 {
1232         int i;
1233
1234         for (i = 0; i < p->num_post_deps; ++i) {
1235                 if (p->post_deps[i].chain && p->post_deps[i].point) {
1236                         drm_syncobj_add_point(p->post_deps[i].syncobj,
1237                                               p->post_deps[i].chain,
1238                                               p->fence, p->post_deps[i].point);
1239                         p->post_deps[i].chain = NULL;
1240                 } else {
1241                         drm_syncobj_replace_fence(p->post_deps[i].syncobj,
1242                                                   p->fence);
1243                 }
1244         }
1245 }
1246
1247 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
1248                             union drm_amdgpu_cs *cs)
1249 {
1250         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1251         struct amdgpu_job *leader = p->gang_leader;
1252         struct amdgpu_bo_list_entry *e;
1253         struct drm_gem_object *gobj;
1254         unsigned long index;
1255         unsigned int i;
1256         uint64_t seq;
1257         int r;
1258
1259         for (i = 0; i < p->gang_size; ++i)
1260                 drm_sched_job_arm(&p->jobs[i]->base);
1261
1262         for (i = 0; i < p->gang_size; ++i) {
1263                 struct dma_fence *fence;
1264
1265                 if (p->jobs[i] == leader)
1266                         continue;
1267
1268                 fence = &p->jobs[i]->base.s_fence->scheduled;
1269                 dma_fence_get(fence);
1270                 r = drm_sched_job_add_dependency(&leader->base, fence);
1271                 if (r) {
1272                         dma_fence_put(fence);
1273                         return r;
1274                 }
1275         }
1276
1277         if (p->gang_size > 1) {
1278                 for (i = 0; i < p->gang_size; ++i)
1279                         amdgpu_job_set_gang_leader(p->jobs[i], leader);
1280         }
1281
1282         /* No memory allocation is allowed while holding the notifier lock.
1283          * The lock is held until amdgpu_cs_submit is finished and fence is
1284          * added to BOs.
1285          */
1286         mutex_lock(&p->adev->notifier_lock);
1287
1288         /* If userptr are invalidated after amdgpu_cs_parser_bos(), return
1289          * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
1290          */
1291         r = 0;
1292         amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
1293                 r |= !amdgpu_ttm_tt_get_user_pages_done(e->bo->tbo.ttm,
1294                                                         e->range);
1295                 e->range = NULL;
1296         }
1297         if (r) {
1298                 r = -EAGAIN;
1299                 mutex_unlock(&p->adev->notifier_lock);
1300                 return r;
1301         }
1302
1303         p->fence = dma_fence_get(&leader->base.s_fence->finished);
1304         drm_exec_for_each_locked_object(&p->exec, index, gobj) {
1305
1306                 ttm_bo_move_to_lru_tail_unlocked(&gem_to_amdgpu_bo(gobj)->tbo);
1307
1308                 /* Everybody except for the gang leader uses READ */
1309                 for (i = 0; i < p->gang_size; ++i) {
1310                         if (p->jobs[i] == leader)
1311                                 continue;
1312
1313                         dma_resv_add_fence(gobj->resv,
1314                                            &p->jobs[i]->base.s_fence->finished,
1315                                            DMA_RESV_USAGE_READ);
1316                 }
1317
1318                 /* The gang leader as remembered as writer */
1319                 dma_resv_add_fence(gobj->resv, p->fence, DMA_RESV_USAGE_WRITE);
1320         }
1321
1322         seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_leader_idx],
1323                                    p->fence);
1324         amdgpu_cs_post_dependencies(p);
1325
1326         if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&
1327             !p->ctx->preamble_presented) {
1328                 leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
1329                 p->ctx->preamble_presented = true;
1330         }
1331
1332         cs->out.handle = seq;
1333         leader->uf_sequence = seq;
1334
1335         amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->exec.ticket);
1336         for (i = 0; i < p->gang_size; ++i) {
1337                 amdgpu_job_free_resources(p->jobs[i]);
1338                 trace_amdgpu_cs_ioctl(p->jobs[i]);
1339                 drm_sched_entity_push_job(&p->jobs[i]->base);
1340                 p->jobs[i] = NULL;
1341         }
1342
1343         amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);
1344
1345         mutex_unlock(&p->adev->notifier_lock);
1346         mutex_unlock(&p->bo_list->bo_list_mutex);
1347         return 0;
1348 }
1349
1350 /* Cleanup the parser structure */
1351 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser)
1352 {
1353         unsigned int i;
1354
1355         amdgpu_sync_free(&parser->sync);
1356         drm_exec_fini(&parser->exec);
1357
1358         for (i = 0; i < parser->num_post_deps; i++) {
1359                 drm_syncobj_put(parser->post_deps[i].syncobj);
1360                 kfree(parser->post_deps[i].chain);
1361         }
1362         kfree(parser->post_deps);
1363
1364         dma_fence_put(parser->fence);
1365
1366         if (parser->ctx)
1367                 amdgpu_ctx_put(parser->ctx);
1368         if (parser->bo_list)
1369                 amdgpu_bo_list_put(parser->bo_list);
1370
1371         for (i = 0; i < parser->nchunks; i++)
1372                 kvfree(parser->chunks[i].kdata);
1373         kvfree(parser->chunks);
1374         for (i = 0; i < parser->gang_size; ++i) {
1375                 if (parser->jobs[i])
1376                         amdgpu_job_free(parser->jobs[i]);
1377         }
1378         amdgpu_bo_unref(&parser->uf_bo);
1379 }
1380
1381 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
1382 {
1383         struct amdgpu_device *adev = drm_to_adev(dev);
1384         struct amdgpu_cs_parser parser;
1385         int r;
1386
1387         if (amdgpu_ras_intr_triggered())
1388                 return -EHWPOISON;
1389
1390         if (!adev->accel_working)
1391                 return -EBUSY;
1392
1393         r = amdgpu_cs_parser_init(&parser, adev, filp, data);
1394         if (r) {
1395                 if (printk_ratelimit())
1396                         DRM_ERROR("Failed to initialize parser %d!\n", r);
1397                 return r;
1398         }
1399
1400         r = amdgpu_cs_pass1(&parser, data);
1401         if (r)
1402                 goto error_fini;
1403
1404         r = amdgpu_cs_pass2(&parser);
1405         if (r)
1406                 goto error_fini;
1407
1408         r = amdgpu_cs_parser_bos(&parser, data);
1409         if (r) {
1410                 if (r == -ENOMEM)
1411                         DRM_ERROR("Not enough memory for command submission!\n");
1412                 else if (r != -ERESTARTSYS && r != -EAGAIN)
1413                         DRM_ERROR("Failed to process the buffer list %d!\n", r);
1414                 goto error_fini;
1415         }
1416
1417         r = amdgpu_cs_patch_jobs(&parser);
1418         if (r)
1419                 goto error_backoff;
1420
1421         r = amdgpu_cs_vm_handling(&parser);
1422         if (r)
1423                 goto error_backoff;
1424
1425         r = amdgpu_cs_sync_rings(&parser);
1426         if (r)
1427                 goto error_backoff;
1428
1429         trace_amdgpu_cs_ibs(&parser);
1430
1431         r = amdgpu_cs_submit(&parser, data);
1432         if (r)
1433                 goto error_backoff;
1434
1435         amdgpu_cs_parser_fini(&parser);
1436         return 0;
1437
1438 error_backoff:
1439         mutex_unlock(&parser.bo_list->bo_list_mutex);
1440
1441 error_fini:
1442         amdgpu_cs_parser_fini(&parser);
1443         return r;
1444 }
1445
1446 /**
1447  * amdgpu_cs_wait_ioctl - wait for a command submission to finish
1448  *
1449  * @dev: drm device
1450  * @data: data from userspace
1451  * @filp: file private
1452  *
1453  * Wait for the command submission identified by handle to finish.
1454  */
1455 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
1456                          struct drm_file *filp)
1457 {
1458         union drm_amdgpu_wait_cs *wait = data;
1459         unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
1460         struct drm_sched_entity *entity;
1461         struct amdgpu_ctx *ctx;
1462         struct dma_fence *fence;
1463         long r;
1464
1465         ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
1466         if (ctx == NULL)
1467                 return -EINVAL;
1468
1469         r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance,
1470                                   wait->in.ring, &entity);
1471         if (r) {
1472                 amdgpu_ctx_put(ctx);
1473                 return r;
1474         }
1475
1476         fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle);
1477         if (IS_ERR(fence))
1478                 r = PTR_ERR(fence);
1479         else if (fence) {
1480                 r = dma_fence_wait_timeout(fence, true, timeout);
1481                 if (r > 0 && fence->error)
1482                         r = fence->error;
1483                 dma_fence_put(fence);
1484         } else
1485                 r = 1;
1486
1487         amdgpu_ctx_put(ctx);
1488         if (r < 0)
1489                 return r;
1490
1491         memset(wait, 0, sizeof(*wait));
1492         wait->out.status = (r == 0);
1493
1494         return 0;
1495 }
1496
1497 /**
1498  * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
1499  *
1500  * @adev: amdgpu device
1501  * @filp: file private
1502  * @user: drm_amdgpu_fence copied from user space
1503  */
1504 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
1505                                              struct drm_file *filp,
1506                                              struct drm_amdgpu_fence *user)
1507 {
1508         struct drm_sched_entity *entity;
1509         struct amdgpu_ctx *ctx;
1510         struct dma_fence *fence;
1511         int r;
1512
1513         ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);
1514         if (ctx == NULL)
1515                 return ERR_PTR(-EINVAL);
1516
1517         r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance,
1518                                   user->ring, &entity);
1519         if (r) {
1520                 amdgpu_ctx_put(ctx);
1521                 return ERR_PTR(r);
1522         }
1523
1524         fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no);
1525         amdgpu_ctx_put(ctx);
1526
1527         return fence;
1528 }
1529
1530 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
1531                                     struct drm_file *filp)
1532 {
1533         struct amdgpu_device *adev = drm_to_adev(dev);
1534         union drm_amdgpu_fence_to_handle *info = data;
1535         struct dma_fence *fence;
1536         struct drm_syncobj *syncobj;
1537         struct sync_file *sync_file;
1538         int fd, r;
1539
1540         fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
1541         if (IS_ERR(fence))
1542                 return PTR_ERR(fence);
1543
1544         if (!fence)
1545                 fence = dma_fence_get_stub();
1546
1547         switch (info->in.what) {
1548         case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
1549                 r = drm_syncobj_create(&syncobj, 0, fence);
1550                 dma_fence_put(fence);
1551                 if (r)
1552                         return r;
1553                 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
1554                 drm_syncobj_put(syncobj);
1555                 return r;
1556
1557         case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
1558                 r = drm_syncobj_create(&syncobj, 0, fence);
1559                 dma_fence_put(fence);
1560                 if (r)
1561                         return r;
1562                 r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle);
1563                 drm_syncobj_put(syncobj);
1564                 return r;
1565
1566         case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
1567                 fd = get_unused_fd_flags(O_CLOEXEC);
1568                 if (fd < 0) {
1569                         dma_fence_put(fence);
1570                         return fd;
1571                 }
1572
1573                 sync_file = sync_file_create(fence);
1574                 dma_fence_put(fence);
1575                 if (!sync_file) {
1576                         put_unused_fd(fd);
1577                         return -ENOMEM;
1578                 }
1579
1580                 fd_install(fd, sync_file->file);
1581                 info->out.handle = fd;
1582                 return 0;
1583
1584         default:
1585                 dma_fence_put(fence);
1586                 return -EINVAL;
1587         }
1588 }
1589
1590 /**
1591  * amdgpu_cs_wait_all_fences - wait on all fences to signal
1592  *
1593  * @adev: amdgpu device
1594  * @filp: file private
1595  * @wait: wait parameters
1596  * @fences: array of drm_amdgpu_fence
1597  */
1598 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
1599                                      struct drm_file *filp,
1600                                      union drm_amdgpu_wait_fences *wait,
1601                                      struct drm_amdgpu_fence *fences)
1602 {
1603         uint32_t fence_count = wait->in.fence_count;
1604         unsigned int i;
1605         long r = 1;
1606
1607         for (i = 0; i < fence_count; i++) {
1608                 struct dma_fence *fence;
1609                 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1610
1611                 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1612                 if (IS_ERR(fence))
1613                         return PTR_ERR(fence);
1614                 else if (!fence)
1615                         continue;
1616
1617                 r = dma_fence_wait_timeout(fence, true, timeout);
1618                 if (r > 0 && fence->error)
1619                         r = fence->error;
1620
1621                 dma_fence_put(fence);
1622                 if (r < 0)
1623                         return r;
1624
1625                 if (r == 0)
1626                         break;
1627         }
1628
1629         memset(wait, 0, sizeof(*wait));
1630         wait->out.status = (r > 0);
1631
1632         return 0;
1633 }
1634
1635 /**
1636  * amdgpu_cs_wait_any_fence - wait on any fence to signal
1637  *
1638  * @adev: amdgpu device
1639  * @filp: file private
1640  * @wait: wait parameters
1641  * @fences: array of drm_amdgpu_fence
1642  */
1643 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
1644                                     struct drm_file *filp,
1645                                     union drm_amdgpu_wait_fences *wait,
1646                                     struct drm_amdgpu_fence *fences)
1647 {
1648         unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1649         uint32_t fence_count = wait->in.fence_count;
1650         uint32_t first = ~0;
1651         struct dma_fence **array;
1652         unsigned int i;
1653         long r;
1654
1655         /* Prepare the fence array */
1656         array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL);
1657
1658         if (array == NULL)
1659                 return -ENOMEM;
1660
1661         for (i = 0; i < fence_count; i++) {
1662                 struct dma_fence *fence;
1663
1664                 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1665                 if (IS_ERR(fence)) {
1666                         r = PTR_ERR(fence);
1667                         goto err_free_fence_array;
1668                 } else if (fence) {
1669                         array[i] = fence;
1670                 } else { /* NULL, the fence has been already signaled */
1671                         r = 1;
1672                         first = i;
1673                         goto out;
1674                 }
1675         }
1676
1677         r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
1678                                        &first);
1679         if (r < 0)
1680                 goto err_free_fence_array;
1681
1682 out:
1683         memset(wait, 0, sizeof(*wait));
1684         wait->out.status = (r > 0);
1685         wait->out.first_signaled = first;
1686
1687         if (first < fence_count && array[first])
1688                 r = array[first]->error;
1689         else
1690                 r = 0;
1691
1692 err_free_fence_array:
1693         for (i = 0; i < fence_count; i++)
1694                 dma_fence_put(array[i]);
1695         kfree(array);
1696
1697         return r;
1698 }
1699
1700 /**
1701  * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
1702  *
1703  * @dev: drm device
1704  * @data: data from userspace
1705  * @filp: file private
1706  */
1707 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1708                                 struct drm_file *filp)
1709 {
1710         struct amdgpu_device *adev = drm_to_adev(dev);
1711         union drm_amdgpu_wait_fences *wait = data;
1712         uint32_t fence_count = wait->in.fence_count;
1713         struct drm_amdgpu_fence *fences_user;
1714         struct drm_amdgpu_fence *fences;
1715         int r;
1716
1717         /* Get the fences from userspace */
1718         fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
1719                         GFP_KERNEL);
1720         if (fences == NULL)
1721                 return -ENOMEM;
1722
1723         fences_user = u64_to_user_ptr(wait->in.fences);
1724         if (copy_from_user(fences, fences_user,
1725                 sizeof(struct drm_amdgpu_fence) * fence_count)) {
1726                 r = -EFAULT;
1727                 goto err_free_fences;
1728         }
1729
1730         if (wait->in.wait_all)
1731                 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);
1732         else
1733                 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);
1734
1735 err_free_fences:
1736         kfree(fences);
1737
1738         return r;
1739 }
1740
1741 /**
1742  * amdgpu_cs_find_mapping - find bo_va for VM address
1743  *
1744  * @parser: command submission parser context
1745  * @addr: VM address
1746  * @bo: resulting BO of the mapping found
1747  * @map: Placeholder to return found BO mapping
1748  *
1749  * Search the buffer objects in the command submission context for a certain
1750  * virtual memory address. Returns allocation structure when found, NULL
1751  * otherwise.
1752  */
1753 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1754                            uint64_t addr, struct amdgpu_bo **bo,
1755                            struct amdgpu_bo_va_mapping **map)
1756 {
1757         struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
1758         struct ttm_operation_ctx ctx = { false, false };
1759         struct amdgpu_vm *vm = &fpriv->vm;
1760         struct amdgpu_bo_va_mapping *mapping;
1761         int r;
1762
1763         addr /= AMDGPU_GPU_PAGE_SIZE;
1764
1765         mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
1766         if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
1767                 return -EINVAL;
1768
1769         *bo = mapping->bo_va->base.bo;
1770         *map = mapping;
1771
1772         /* Double check that the BO is reserved by this CS */
1773         if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->exec.ticket)
1774                 return -EINVAL;
1775
1776         if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
1777                 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1778                 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains);
1779                 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
1780                 if (r)
1781                         return r;
1782         }
1783
1784         return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
1785 }