drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/kthread.h>
  25 #include <linux/wait.h>
  26 #include <linux/sched.h>
  27 #include <drm/drmP.h>
  28 #include "gpu_scheduler.h"
  29
  30 /* Initialize a given run queue struct */
  31 static void init_rq(struct amd_run_queue *rq)
  32 {
  33         INIT_LIST_HEAD(&rq->head.list);
  34         rq->head.belongto_rq = rq;
  35         mutex_init(&rq->lock);
  36         atomic_set(&rq->nr_entity, 0);
  37         rq->current_entity = &rq->head;
  38 }
  39
  40 /* Note: caller must hold the lock or in a atomic context */
  41 static void rq_remove_entity(struct amd_run_queue *rq,
  42                              struct amd_sched_entity *entity)
  43 {
  44         if (rq->current_entity == entity)
  45                 rq->current_entity = list_entry(entity->list.prev,
  46                                                 typeof(*entity), list);
  47         list_del_init(&entity->list);
  48         atomic_dec(&rq->nr_entity);
  49 }
  50
  51 static void rq_add_entity(struct amd_run_queue *rq,
  52                           struct amd_sched_entity *entity)
  53 {
  54         list_add_tail(&entity->list, &rq->head.list);
  55         atomic_inc(&rq->nr_entity);
  56 }
  57
  58 /**
  59  * Select next entity from a specified run queue with round robin policy.
  60  * It could return the same entity as current one if current is the only
  61  * available one in the queue. Return NULL if nothing available.
  62  */
  63 static struct amd_sched_entity *rq_select_entity(struct amd_run_queue *rq)
  64 {
  65         struct amd_sched_entity *p = rq->current_entity;
  66         int i = atomic_read(&rq->nr_entity) + 1; /*real count + dummy head*/
  67         while (i) {
  68                 p = list_entry(p->list.next, typeof(*p), list);
  69                 if (!rq->check_entity_status(p)) {
  70                         rq->current_entity = p;
  71                         break;
  72                 }
  73                 i--;
  74         }
  75         return i ? p : NULL;
  76 }
  77
  78 static bool context_entity_is_waiting(struct amd_context_entity *entity)
  79 {
  80         /* TODO: sync obj for multi-ring synchronization */
  81         return false;
  82 }
  83
  84 static int gpu_entity_check_status(struct amd_sched_entity *entity)
  85 {
  86         struct amd_context_entity *tmp = NULL;
  87
  88         if (entity == &entity->belongto_rq->head)
  89                 return -1;
  90
  91         tmp = container_of(entity, typeof(*tmp), generic_entity);
  92         if (kfifo_is_empty(&tmp->job_queue) ||
  93             context_entity_is_waiting(tmp))
  94                 return -1;
  95
  96         return 0;
  97 }
  98
  99 /**
 100  * Note: This function should only been called inside scheduler main
 101  * function for thread safety, there is no other protection here.
 102  * return ture if scheduler has something ready to run.
 103  *
 104  * For active_hw_rq, there is only one producer(scheduler thread) and
 105  * one consumer(ISR). It should be safe to use this function in scheduler
 106  * main thread to decide whether to continue emit more IBs.
 107 */
 108 static bool is_scheduler_ready(struct amd_gpu_scheduler *sched)
 109 {
 110         return !kfifo_is_full(&sched->active_hw_rq);
 111 }
 112
 113 /**
 114  * Select next entity from the kernel run queue, if not available,
 115  * return null.
 116 */
 117 static struct amd_context_entity *kernel_rq_select_context(
 118         struct amd_gpu_scheduler *sched)
 119 {
 120         struct amd_sched_entity *sched_entity = NULL;
 121         struct amd_context_entity *tmp = NULL;
 122         struct amd_run_queue *rq = &sched->kernel_rq;
 123
 124         mutex_lock(&rq->lock);
 125         sched_entity = rq_select_entity(rq);
 126         if (sched_entity)
 127                 tmp = container_of(sched_entity,
 128                                    typeof(*tmp),
 129                                    generic_entity);
 130         mutex_unlock(&rq->lock);
 131         return tmp;
 132 }
 133
 134 /**
 135  * Select next entity containing real IB submissions
 136 */
 137 static struct amd_context_entity *select_context(
 138         struct amd_gpu_scheduler *sched)
 139 {
 140         struct amd_context_entity *wake_entity = NULL;
 141         struct amd_context_entity *tmp;
 142         struct amd_run_queue *rq;
 143
 144         if (!is_scheduler_ready(sched))
 145                 return NULL;
 146
 147         /* Kernel run queue has higher priority than normal run queue*/
 148         tmp = kernel_rq_select_context(sched);
 149         if (tmp != NULL)
 150                 goto exit;
 151
 152         WARN_ON(offsetof(struct amd_context_entity, generic_entity) != 0);
 153
 154         rq = &sched->sched_rq;
 155         mutex_lock(&rq->lock);
 156         tmp = container_of(rq_select_entity(rq),
 157                            typeof(*tmp), generic_entity);
 158         mutex_unlock(&rq->lock);
 159 exit:
 160         if (sched->current_entity && (sched->current_entity != tmp))
 161                 wake_entity = sched->current_entity;
 162         sched->current_entity = tmp;
 163         if (wake_entity)
 164                 wake_up(&wake_entity->wait_queue);
 165         return tmp;
 166 }
 167
 168 /**
 169  * Init a context entity used by scheduler when submit to HW ring.
 170  *
 171  * @sched       The pointer to the scheduler
 172  * @entity      The pointer to a valid amd_context_entity
 173  * @parent      The parent entity of this amd_context_entity
 174  * @rq          The run queue this entity belongs
 175  * @context_id  The context id for this entity
 176  * @jobs        The max number of jobs in the job queue
 177  *
 178  * return 0 if succeed. negative error code on failure
 179 */
 180 int amd_context_entity_init(struct amd_gpu_scheduler *sched,
 181                             struct amd_context_entity *entity,
 182                             struct amd_sched_entity *parent,
 183                             struct amd_run_queue *rq,
 184                             uint32_t context_id,
 185                             uint32_t jobs)
 186 {
 187         uint64_t seq_ring = 0;
 188
 189         if (!(sched && entity && rq))
 190                 return -EINVAL;
 191
 192         memset(entity, 0, sizeof(struct amd_context_entity));
 193         seq_ring = ((uint64_t)sched->ring_id) << 60;
 194         spin_lock_init(&entity->lock);
 195         entity->generic_entity.belongto_rq = rq;
 196         entity->generic_entity.parent = parent;
 197         entity->scheduler = sched;
 198         init_waitqueue_head(&entity->wait_queue);
 199         init_waitqueue_head(&entity->wait_emit);
 200         if(kfifo_alloc(&entity->job_queue,
 201                        jobs * sizeof(void *),
 202                        GFP_KERNEL))
 203                 return -EINVAL;
 204
 205         spin_lock_init(&entity->queue_lock);
 206         entity->tgid = (context_id == AMD_KERNEL_CONTEXT_ID) ?
 207                 AMD_KERNEL_PROCESS_ID : current->tgid;
 208         entity->context_id = context_id;
 209         atomic64_set(&entity->last_emitted_v_seq, seq_ring);
 210         atomic64_set(&entity->last_queued_v_seq, seq_ring);
 211
 212         /* Add the entity to the run queue */
 213         mutex_lock(&rq->lock);
 214         rq_add_entity(rq, &entity->generic_entity);
 215         mutex_unlock(&rq->lock);
 216         return 0;
 217 }
 218
 219 /**
 220  * Query if entity is initialized
 221  *
 222  * @sched       Pointer to scheduler instance
 223  * @entity      The pointer to a valid scheduler entity
 224  *
 225  * return true if entity is initialized, false otherwise
 226 */
 227 static bool is_context_entity_initialized(struct amd_gpu_scheduler *sched,
 228                                           struct amd_context_entity *entity)
 229 {
 230         return entity->scheduler == sched &&
 231                 entity->generic_entity.belongto_rq != NULL;
 232 }
 233
 234 static bool is_context_entity_idle(struct amd_gpu_scheduler *sched,
 235                                    struct amd_context_entity *entity)
 236 {
 237         /**
 238          * Idle means no pending IBs, and the entity is not
 239          * currently being used.
 240         */
 241         barrier();
 242         if ((sched->current_entity != entity) &&
 243             kfifo_is_empty(&entity->job_queue))
 244                 return true;
 245
 246         return false;
 247 }
 248
 249 /**
 250  * Destroy a context entity
 251  *
 252  * @sched       Pointer to scheduler instance
 253  * @entity      The pointer to a valid scheduler entity
 254  *
 255  * return 0 if succeed. negative error code on failure
 256  */
 257 int amd_context_entity_fini(struct amd_gpu_scheduler *sched,
 258                             struct amd_context_entity *entity)
 259 {
 260         int r = 0;
 261         struct amd_run_queue *rq = entity->generic_entity.belongto_rq;
 262
 263         if (!is_context_entity_initialized(sched, entity))
 264                 return 0;
 265
 266         /**
 267          * The client will not queue more IBs during this fini, consume existing
 268          * queued IBs
 269         */
 270         r = wait_event_timeout(
 271                 entity->wait_queue,
 272                 is_context_entity_idle(sched, entity),
 273                 msecs_to_jiffies(AMD_GPU_WAIT_IDLE_TIMEOUT_IN_MS)
 274                 ) ?  0 : -1;
 275
 276         if (r) {
 277                 if (entity->is_pending)
 278                         DRM_INFO("Entity %u is in waiting state during fini,\
 279                                 all pending ibs will be canceled.\n",
 280                                  entity->context_id);
 281         }
 282
 283         mutex_lock(&rq->lock);
 284         rq_remove_entity(rq, &entity->generic_entity);
 285         mutex_unlock(&rq->lock);
 286         kfifo_free(&entity->job_queue);
 287         return r;
 288 }
 289
 290 /**
 291  * Submit a normal job to the job queue
 292  *
 293  * @sched       The pointer to the scheduler
 294  * @c_entity    The pointer to amd_context_entity
 295  * @job         The pointer to job required to submit
 296  * return the virtual sequence number
 297 */
 298 uint64_t amd_sched_push_job(struct amd_gpu_scheduler *sched,
 299                        struct amd_context_entity *c_entity,
 300                        void *job)
 301 {
 302         while (kfifo_in_spinlocked(&c_entity->job_queue, &job, sizeof(void *),
 303                                    &c_entity->queue_lock) != sizeof(void *)) {
 304                 /**
 305                  * Current context used up all its IB slots
 306                  * wait here, or need to check whether GPU is hung
 307                 */
 308                 schedule();
 309         }
 310
 311         wake_up_interruptible(&sched->wait_queue);
 312
 313         return atomic64_inc_return(&c_entity->last_queued_v_seq);
 314 }
 315
 316 /**
 317  * Wait for a virtual sequence number to be emitted.
 318  *
 319  * @c_entity    The pointer to a valid context entity
 320  * @seq         The virtual sequence number to wait
 321  * @intr        Interruptible or not
 322  * @timeout     Timeout in ms, wait infinitely if <0
 323  * @emit        wait for emit or signal
 324  *
 325  * return =0 signaled ,  <0 failed
 326 */
 327 int amd_sched_wait_emit(struct amd_context_entity *c_entity,
 328                         uint64_t seq,
 329                         bool intr,
 330                         long timeout)
 331 {
 332         atomic64_t *v_seq = &c_entity->last_emitted_v_seq;
 333         wait_queue_head_t *wait_queue = &c_entity->wait_emit;
 334
 335         if (intr && (timeout < 0)) {
 336                 wait_event_interruptible(
 337                         *wait_queue,
 338                         seq <= atomic64_read(v_seq));
 339                 return 0;
 340         } else if (intr && (timeout >= 0)) {
 341                 wait_event_interruptible_timeout(
 342                         *wait_queue,
 343                         seq <= atomic64_read(v_seq),
 344                         msecs_to_jiffies(timeout));
 345                 return (seq <= atomic64_read(v_seq)) ?
 346                         0 : -1;
 347         } else if (!intr && (timeout < 0)) {
 348                 wait_event(
 349                         *wait_queue,
 350                         seq <= atomic64_read(v_seq));
 351                 return 0;
 352         } else if (!intr && (timeout >= 0)) {
 353                 wait_event_timeout(
 354                         *wait_queue,
 355                         seq <= atomic64_read(v_seq),
 356                         msecs_to_jiffies(timeout));
 357                 return (seq <= atomic64_read(v_seq)) ?
 358                         0 : -1;
 359         }
 360         return 0;
 361 }
 362
 363 static int amd_sched_main(void *param)
 364 {
 365         int r;
 366         void *job;
 367         struct sched_param sparam = {.sched_priority = 1};
 368         struct amd_context_entity *c_entity = NULL;
 369         struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
 370
 371         sched_setscheduler(current, SCHED_FIFO, &sparam);
 372
 373         while (!kthread_should_stop()) {
 374                 wait_event_interruptible(sched->wait_queue,
 375                                          is_scheduler_ready(sched) &&
 376                                          (c_entity = select_context(sched)));
 377                 r = kfifo_out(&c_entity->job_queue, &job, sizeof(void *));
 378                 if (r != sizeof(void *))
 379                         continue;
 380                 r = sched->ops->prepare_job(sched, c_entity, job);
 381                 if (!r)
 382                         WARN_ON(kfifo_in_spinlocked(
 383                                         &sched->active_hw_rq,
 384                                         &job,
 385                                         sizeof(void *),
 386                                         &sched->queue_lock) != sizeof(void *));
 387                 mutex_lock(&sched->sched_lock);
 388                 sched->ops->run_job(sched, c_entity, job);
 389                 mutex_unlock(&sched->sched_lock);
 390         }
 391         return 0;
 392 }
 393
 394 uint64_t amd_sched_get_handled_seq(struct amd_gpu_scheduler *sched)
 395 {
 396         return sched->last_handled_seq;
 397 }
 398
 399 /**
 400  * ISR to handle EOP inetrrupts
 401  *
 402  * @sched: gpu scheduler
 403  *
 404 */
 405 void amd_sched_isr(struct amd_gpu_scheduler *sched)
 406 {
 407         int r;
 408         void *job;
 409         r = kfifo_out_spinlocked(&sched->active_hw_rq,
 410                                  &job, sizeof(void *),
 411                                  &sched->queue_lock);
 412
 413         if (r != sizeof(void *))
 414                 job = NULL;
 415
 416         sched->ops->process_job(sched, job);
 417         sched->last_handled_seq++;
 418         wake_up_interruptible(&sched->wait_queue);
 419 }
 420
 421 /**
 422  * Create a gpu scheduler
 423  *
 424  * @device      The device context for this scheduler
 425  * @ops         The backend operations for this scheduler.
 426  * @id          The scheduler is per ring, here is ring id.
 427  * @granularity The minumum ms unit the scheduler will scheduled.
 428  * @preemption  Indicate whether this ring support preemption, 0 is no.
 429  *
 430  * return the pointer to scheduler for success, otherwise return NULL
 431 */
 432 struct amd_gpu_scheduler *amd_sched_create(void *device,
 433                                            struct amd_sched_backend_ops *ops,
 434                                            unsigned ring,
 435                                            unsigned granularity,
 436                                            unsigned preemption,
 437                                            unsigned hw_submission)
 438 {
 439         struct amd_gpu_scheduler *sched;
 440         char name[20] = "gpu_sched[0]";
 441
 442         sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
 443         if (!sched)
 444                 return NULL;
 445
 446         sched->device = device;
 447         sched->ops = ops;
 448         sched->granularity = granularity;
 449         sched->ring_id = ring;
 450         sched->preemption = preemption;
 451         sched->last_handled_seq = 0;
 452
 453         snprintf(name, sizeof(name), "gpu_sched[%d]", ring);
 454         mutex_init(&sched->sched_lock);
 455         spin_lock_init(&sched->queue_lock);
 456         init_rq(&sched->sched_rq);
 457         sched->sched_rq.check_entity_status = gpu_entity_check_status;
 458
 459         init_rq(&sched->kernel_rq);
 460         sched->kernel_rq.check_entity_status = gpu_entity_check_status;
 461
 462         init_waitqueue_head(&sched->wait_queue);
 463         if(kfifo_alloc(&sched->active_hw_rq,
 464                        hw_submission * sizeof(void *),
 465                        GFP_KERNEL)) {
 466                 kfree(sched);
 467                 return NULL;
 468         }
 469
 470         /* Each scheduler will run on a seperate kernel thread */
 471         sched->thread = kthread_create(amd_sched_main, sched, name);
 472         if (sched->thread) {
 473                 wake_up_process(sched->thread);
 474                 return sched;
 475         }
 476
 477         DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
 478         kfifo_free(&sched->active_hw_rq);
 479         kfree(sched);
 480         return NULL;
 481 }
 482
 483 /**
 484  * Destroy a gpu scheduler
 485  *
 486  * @sched       The pointer to the scheduler
 487  *
 488  * return 0 if succeed. -1 if failed.
 489  */
 490 int amd_sched_destroy(struct amd_gpu_scheduler *sched)
 491 {
 492         kthread_stop(sched->thread);
 493         kfifo_free(&sched->active_hw_rq);
 494         kfree(sched);
 495         return  0;
 496 }
 497
 498 /**
 499  * Update emitted sequence and wake up the waiters, called by run_job
 500  * in driver side
 501  *
 502  * @entity The context entity
 503  * @seq The sequence number for the latest emitted job
 504 */
 505 void amd_sched_emit(struct amd_context_entity *c_entity, uint64_t seq)
 506 {
 507         atomic64_set(&c_entity->last_emitted_v_seq, seq);
 508         wake_up_all(&c_entity->wait_emit);
 509 }