block/blk-mq.c

   1 /*
   2  * Block multiqueue core code
   3  *
   4  * Copyright (C) 2013-2014 Jens Axboe
   5  * Copyright (C) 2013-2014 Christoph Hellwig
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/backing-dev.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/mm.h>
  13 #include <linux/init.h>
  14 #include <linux/slab.h>
  15 #include <linux/workqueue.h>
  16 #include <linux/smp.h>
  17 #include <linux/llist.h>
  18 #include <linux/list_sort.h>
  19 #include <linux/cpu.h>
  20 #include <linux/cache.h>
  21 #include <linux/sched/sysctl.h>
  22 #include <linux/delay.h>
  23 #include <linux/crash_dump.h>
  24
  25 #include <trace/events/block.h>
  26
  27 #include <linux/blk-mq.h>
  28 #include "blk.h"
  29 #include "blk-mq.h"
  30 #include "blk-mq-tag.h"
  31
  32 static DEFINE_MUTEX(all_q_mutex);
  33 static LIST_HEAD(all_q_list);
  34
  35 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
  36
  37 /*
  38  * Check if any of the ctx's have pending work in this hardware queue
  39  */
  40 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
  41 {
  42         unsigned int i;
  43
  44         for (i = 0; i < hctx->ctx_map.map_size; i++)
  45                 if (hctx->ctx_map.map[i].word)
  46                         return true;
  47
  48         return false;
  49 }
  50
  51 static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
  52                                               struct blk_mq_ctx *ctx)
  53 {
  54         return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
  55 }
  56
  57 #define CTX_TO_BIT(hctx, ctx)   \
  58         ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
  59
  60 /*
  61  * Mark this ctx as having pending work in this hardware queue
  62  */
  63 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
  64                                      struct blk_mq_ctx *ctx)
  65 {
  66         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  67
  68         if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
  69                 set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  70 }
  71
  72 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
  73                                       struct blk_mq_ctx *ctx)
  74 {
  75         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  76
  77         clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  78 }
  79
  80 static int blk_mq_queue_enter(struct request_queue *q)
  81 {
  82         while (true) {
  83                 int ret;
  84
  85                 if (percpu_ref_tryget_live(&q->mq_usage_counter))
  86                         return 0;
  87
  88                 ret = wait_event_interruptible(q->mq_freeze_wq,
  89                                 !q->mq_freeze_depth || blk_queue_dying(q));
  90                 if (blk_queue_dying(q))
  91                         return -ENODEV;
  92                 if (ret)
  93                         return ret;
  94         }
  95 }
  96
  97 static void blk_mq_queue_exit(struct request_queue *q)
  98 {
  99         percpu_ref_put(&q->mq_usage_counter);
 100 }
 101
 102 static void blk_mq_usage_counter_release(struct percpu_ref *ref)
 103 {
 104         struct request_queue *q =
 105                 container_of(ref, struct request_queue, mq_usage_counter);
 106
 107         wake_up_all(&q->mq_freeze_wq);
 108 }
 109
 110 static void blk_mq_freeze_queue_start(struct request_queue *q)
 111 {
 112         bool freeze;
 113
 114         spin_lock_irq(q->queue_lock);
 115         freeze = !q->mq_freeze_depth++;
 116         spin_unlock_irq(q->queue_lock);
 117
 118         if (freeze) {
 119                 percpu_ref_kill(&q->mq_usage_counter);
 120                 blk_mq_run_queues(q, false);
 121         }
 122 }
 123
 124 static void blk_mq_freeze_queue_wait(struct request_queue *q)
 125 {
 126         wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->mq_usage_counter));
 127 }
 128
 129 /*
 130  * Guarantee no request is in use, so we can change any data structure of
 131  * the queue afterward.
 132  */
 133 void blk_mq_freeze_queue(struct request_queue *q)
 134 {
 135         blk_mq_freeze_queue_start(q);
 136         blk_mq_freeze_queue_wait(q);
 137 }
 138
 139 static void blk_mq_unfreeze_queue(struct request_queue *q)
 140 {
 141         bool wake;
 142
 143         spin_lock_irq(q->queue_lock);
 144         wake = !--q->mq_freeze_depth;
 145         WARN_ON_ONCE(q->mq_freeze_depth < 0);
 146         spin_unlock_irq(q->queue_lock);
 147         if (wake) {
 148                 percpu_ref_reinit(&q->mq_usage_counter);
 149                 wake_up_all(&q->mq_freeze_wq);
 150         }
 151 }
 152
 153 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
 154 {
 155         return blk_mq_has_free_tags(hctx->tags);
 156 }
 157 EXPORT_SYMBOL(blk_mq_can_queue);
 158
 159 static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
 160                                struct request *rq, unsigned int rw_flags)
 161 {
 162         if (blk_queue_io_stat(q))
 163                 rw_flags |= REQ_IO_STAT;
 164
 165         INIT_LIST_HEAD(&rq->queuelist);
 166         /* csd/requeue_work/fifo_time is initialized before use */
 167         rq->q = q;
 168         rq->mq_ctx = ctx;
 169         rq->cmd_flags |= rw_flags;
 170         /* do not touch atomic flags, it needs atomic ops against the timer */
 171         rq->cpu = -1;
 172         INIT_HLIST_NODE(&rq->hash);
 173         RB_CLEAR_NODE(&rq->rb_node);
 174         rq->rq_disk = NULL;
 175         rq->part = NULL;
 176         rq->start_time = jiffies;
 177 #ifdef CONFIG_BLK_CGROUP
 178         rq->rl = NULL;
 179         set_start_time_ns(rq);
 180         rq->io_start_time_ns = 0;
 181 #endif
 182         rq->nr_phys_segments = 0;
 183 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 184         rq->nr_integrity_segments = 0;
 185 #endif
 186         rq->special = NULL;
 187         /* tag was already set */
 188         rq->errors = 0;
 189
 190         rq->cmd = rq->__cmd;
 191
 192         rq->extra_len = 0;
 193         rq->sense_len = 0;
 194         rq->resid_len = 0;
 195         rq->sense = NULL;
 196
 197         INIT_LIST_HEAD(&rq->timeout_list);
 198         rq->timeout = 0;
 199
 200         rq->end_io = NULL;
 201         rq->end_io_data = NULL;
 202         rq->next_rq = NULL;
 203
 204         ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
 205 }
 206
 207 static struct request *
 208 __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw)
 209 {
 210         struct request *rq;
 211         unsigned int tag;
 212
 213         tag = blk_mq_get_tag(data);
 214         if (tag != BLK_MQ_TAG_FAIL) {
 215                 rq = data->hctx->tags->rqs[tag];
 216
 217                 if (blk_mq_tag_busy(data->hctx)) {
 218                         rq->cmd_flags = REQ_MQ_INFLIGHT;
 219                         atomic_inc(&data->hctx->nr_active);
 220                 }
 221
 222                 rq->tag = tag;
 223                 blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw);
 224                 return rq;
 225         }
 226
 227         return NULL;
 228 }
 229
 230 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
 231                 bool reserved)
 232 {
 233         struct blk_mq_ctx *ctx;
 234         struct blk_mq_hw_ctx *hctx;
 235         struct request *rq;
 236         struct blk_mq_alloc_data alloc_data;
 237         int ret;
 238
 239         ret = blk_mq_queue_enter(q);
 240         if (ret)
 241                 return ERR_PTR(ret);
 242
 243         ctx = blk_mq_get_ctx(q);
 244         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 245         blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_WAIT,
 246                         reserved, ctx, hctx);
 247
 248         rq = __blk_mq_alloc_request(&alloc_data, rw);
 249         if (!rq && (gfp & __GFP_WAIT)) {
 250                 __blk_mq_run_hw_queue(hctx);
 251                 blk_mq_put_ctx(ctx);
 252
 253                 ctx = blk_mq_get_ctx(q);
 254                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
 255                 blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx,
 256                                 hctx);
 257                 rq =  __blk_mq_alloc_request(&alloc_data, rw);
 258                 ctx = alloc_data.ctx;
 259         }
 260         blk_mq_put_ctx(ctx);
 261         if (!rq)
 262                 return ERR_PTR(-EWOULDBLOCK);
 263         return rq;
 264 }
 265 EXPORT_SYMBOL(blk_mq_alloc_request);
 266
 267 static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
 268                                   struct blk_mq_ctx *ctx, struct request *rq)
 269 {
 270         const int tag = rq->tag;
 271         struct request_queue *q = rq->q;
 272
 273         if (rq->cmd_flags & REQ_MQ_INFLIGHT)
 274                 atomic_dec(&hctx->nr_active);
 275         rq->cmd_flags = 0;
 276
 277         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 278         blk_mq_put_tag(hctx, tag, &ctx->last_tag);
 279         blk_mq_queue_exit(q);
 280 }
 281
 282 void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *hctx, struct request *rq)
 283 {
 284         struct blk_mq_ctx *ctx = rq->mq_ctx;
 285
 286         ctx->rq_completed[rq_is_sync(rq)]++;
 287         __blk_mq_free_request(hctx, ctx, rq);
 288
 289 }
 290 EXPORT_SYMBOL_GPL(blk_mq_free_hctx_request);
 291
 292 void blk_mq_free_request(struct request *rq)
 293 {
 294         struct blk_mq_hw_ctx *hctx;
 295         struct request_queue *q = rq->q;
 296
 297         hctx = q->mq_ops->map_queue(q, rq->mq_ctx->cpu);
 298         blk_mq_free_hctx_request(hctx, rq);
 299 }
 300 EXPORT_SYMBOL_GPL(blk_mq_free_request);
 301
 302 inline void __blk_mq_end_request(struct request *rq, int error)
 303 {
 304         blk_account_io_done(rq);
 305
 306         if (rq->end_io) {
 307                 rq->end_io(rq, error);
 308         } else {
 309                 if (unlikely(blk_bidi_rq(rq)))
 310                         blk_mq_free_request(rq->next_rq);
 311                 blk_mq_free_request(rq);
 312         }
 313 }
 314 EXPORT_SYMBOL(__blk_mq_end_request);
 315
 316 void blk_mq_end_request(struct request *rq, int error)
 317 {
 318         if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 319                 BUG();
 320         __blk_mq_end_request(rq, error);
 321 }
 322 EXPORT_SYMBOL(blk_mq_end_request);
 323
 324 static void __blk_mq_complete_request_remote(void *data)
 325 {
 326         struct request *rq = data;
 327
 328         rq->q->softirq_done_fn(rq);
 329 }
 330
 331 static void blk_mq_ipi_complete_request(struct request *rq)
 332 {
 333         struct blk_mq_ctx *ctx = rq->mq_ctx;
 334         bool shared = false;
 335         int cpu;
 336
 337         if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
 338                 rq->q->softirq_done_fn(rq);
 339                 return;
 340         }
 341
 342         cpu = get_cpu();
 343         if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
 344                 shared = cpus_share_cache(cpu, ctx->cpu);
 345
 346         if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
 347                 rq->csd.func = __blk_mq_complete_request_remote;
 348                 rq->csd.info = rq;
 349                 rq->csd.flags = 0;
 350                 smp_call_function_single_async(ctx->cpu, &rq->csd);
 351         } else {
 352                 rq->q->softirq_done_fn(rq);
 353         }
 354         put_cpu();
 355 }
 356
 357 void __blk_mq_complete_request(struct request *rq)
 358 {
 359         struct request_queue *q = rq->q;
 360
 361         if (!q->softirq_done_fn)
 362                 blk_mq_end_request(rq, rq->errors);
 363         else
 364                 blk_mq_ipi_complete_request(rq);
 365 }
 366
 367 /**
 368  * blk_mq_complete_request - end I/O on a request
 369  * @rq:         the request being processed
 370  *
 371  * Description:
 372  *      Ends all I/O on a request. It does not handle partial completions.
 373  *      The actual completion happens out-of-order, through a IPI handler.
 374  **/
 375 void blk_mq_complete_request(struct request *rq)
 376 {
 377         struct request_queue *q = rq->q;
 378
 379         if (unlikely(blk_should_fake_timeout(q)))
 380                 return;
 381         if (!blk_mark_rq_complete(rq))
 382                 __blk_mq_complete_request(rq);
 383 }
 384 EXPORT_SYMBOL(blk_mq_complete_request);
 385
 386 void blk_mq_start_request(struct request *rq)
 387 {
 388         struct request_queue *q = rq->q;
 389
 390         trace_block_rq_issue(q, rq);
 391
 392         rq->resid_len = blk_rq_bytes(rq);
 393         if (unlikely(blk_bidi_rq(rq)))
 394                 rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 395
 396         blk_add_timer(rq);
 397
 398         /*
 399          * Ensure that ->deadline is visible before set the started
 400          * flag and clear the completed flag.
 401          */
 402         smp_mb__before_atomic();
 403
 404         /*
 405          * Mark us as started and clear complete. Complete might have been
 406          * set if requeue raced with timeout, which then marked it as
 407          * complete. So be sure to clear complete again when we start
 408          * the request, otherwise we'll ignore the completion event.
 409          */
 410         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 411                 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 412         if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
 413                 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
 414
 415         if (q->dma_drain_size && blk_rq_bytes(rq)) {
 416                 /*
 417                  * Make sure space for the drain appears.  We know we can do
 418                  * this because max_hw_segments has been adjusted to be one
 419                  * fewer than the device can handle.
 420                  */
 421                 rq->nr_phys_segments++;
 422         }
 423 }
 424 EXPORT_SYMBOL(blk_mq_start_request);
 425
 426 static void __blk_mq_requeue_request(struct request *rq)
 427 {
 428         struct request_queue *q = rq->q;
 429
 430         trace_block_rq_requeue(q, rq);
 431
 432         if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
 433                 if (q->dma_drain_size && blk_rq_bytes(rq))
 434                         rq->nr_phys_segments--;
 435         }
 436 }
 437
 438 void blk_mq_requeue_request(struct request *rq)
 439 {
 440         __blk_mq_requeue_request(rq);
 441
 442         BUG_ON(blk_queued_rq(rq));
 443         blk_mq_add_to_requeue_list(rq, true);
 444 }
 445 EXPORT_SYMBOL(blk_mq_requeue_request);
 446
 447 static void blk_mq_requeue_work(struct work_struct *work)
 448 {
 449         struct request_queue *q =
 450                 container_of(work, struct request_queue, requeue_work);
 451         LIST_HEAD(rq_list);
 452         struct request *rq, *next;
 453         unsigned long flags;
 454
 455         spin_lock_irqsave(&q->requeue_lock, flags);
 456         list_splice_init(&q->requeue_list, &rq_list);
 457         spin_unlock_irqrestore(&q->requeue_lock, flags);
 458
 459         list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
 460                 if (!(rq->cmd_flags & REQ_SOFTBARRIER))
 461                         continue;
 462
 463                 rq->cmd_flags &= ~REQ_SOFTBARRIER;
 464                 list_del_init(&rq->queuelist);
 465                 blk_mq_insert_request(rq, true, false, false);
 466         }
 467
 468         while (!list_empty(&rq_list)) {
 469                 rq = list_entry(rq_list.next, struct request, queuelist);
 470                 list_del_init(&rq->queuelist);
 471                 blk_mq_insert_request(rq, false, false, false);
 472         }
 473
 474         /*
 475          * Use the start variant of queue running here, so that running
 476          * the requeue work will kick stopped queues.
 477          */
 478         blk_mq_start_hw_queues(q);
 479 }
 480
 481 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
 482 {
 483         struct request_queue *q = rq->q;
 484         unsigned long flags;
 485
 486         /*
 487          * We abuse this flag that is otherwise used by the I/O scheduler to
 488          * request head insertation from the workqueue.
 489          */
 490         BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
 491
 492         spin_lock_irqsave(&q->requeue_lock, flags);
 493         if (at_head) {
 494                 rq->cmd_flags |= REQ_SOFTBARRIER;
 495                 list_add(&rq->queuelist, &q->requeue_list);
 496         } else {
 497                 list_add_tail(&rq->queuelist, &q->requeue_list);
 498         }
 499         spin_unlock_irqrestore(&q->requeue_lock, flags);
 500 }
 501 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 502
 503 void blk_mq_kick_requeue_list(struct request_queue *q)
 504 {
 505         kblockd_schedule_work(&q->requeue_work);
 506 }
 507 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 508
 509 static inline bool is_flush_request(struct request *rq,
 510                 struct blk_flush_queue *fq, unsigned int tag)
 511 {
 512         return ((rq->cmd_flags & REQ_FLUSH_SEQ) &&
 513                         fq->flush_rq->tag == tag);
 514 }
 515
 516 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
 517 {
 518         struct request *rq = tags->rqs[tag];
 519         /* mq_ctx of flush rq is always cloned from the corresponding req */
 520         struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx);
 521
 522         if (!is_flush_request(rq, fq, tag))
 523                 return rq;
 524
 525         return fq->flush_rq;
 526 }
 527 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 528
 529 struct blk_mq_timeout_data {
 530         unsigned long next;
 531         unsigned int next_set;
 532 };
 533
 534 void blk_mq_rq_timed_out(struct request *req, bool reserved)
 535 {
 536         struct blk_mq_ops *ops = req->q->mq_ops;
 537         enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
 538
 539         /*
 540          * We know that complete is set at this point. If STARTED isn't set
 541          * anymore, then the request isn't active and the "timeout" should
 542          * just be ignored. This can happen due to the bitflag ordering.
 543          * Timeout first checks if STARTED is set, and if it is, assumes
 544          * the request is active. But if we race with completion, then
 545          * we both flags will get cleared. So check here again, and ignore
 546          * a timeout event with a request that isn't active.
 547          */
 548         if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
 549                 return;
 550
 551         if (ops->timeout)
 552                 ret = ops->timeout(req, reserved);
 553
 554         switch (ret) {
 555         case BLK_EH_HANDLED:
 556                 __blk_mq_complete_request(req);
 557                 break;
 558         case BLK_EH_RESET_TIMER:
 559                 blk_add_timer(req);
 560                 blk_clear_rq_complete(req);
 561                 break;
 562         case BLK_EH_NOT_HANDLED:
 563                 break;
 564         default:
 565                 printk(KERN_ERR "block: bad eh return: %d\n", ret);
 566                 break;
 567         }
 568 }
 569
 570 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
 571                 struct request *rq, void *priv, bool reserved)
 572 {
 573         struct blk_mq_timeout_data *data = priv;
 574
 575         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 576                 return;
 577
 578         if (time_after_eq(jiffies, rq->deadline)) {
 579                 if (!blk_mark_rq_complete(rq))
 580                         blk_mq_rq_timed_out(rq, reserved);
 581         } else if (!data->next_set || time_after(data->next, rq->deadline)) {
 582                 data->next = rq->deadline;
 583                 data->next_set = 1;
 584         }
 585 }
 586
 587 static void blk_mq_rq_timer(unsigned long priv)
 588 {
 589         struct request_queue *q = (struct request_queue *)priv;
 590         struct blk_mq_timeout_data data = {
 591                 .next           = 0,
 592                 .next_set       = 0,
 593         };
 594         struct blk_mq_hw_ctx *hctx;
 595         int i;
 596
 597         queue_for_each_hw_ctx(q, hctx, i) {
 598                 /*
 599                  * If not software queues are currently mapped to this
 600                  * hardware queue, there's nothing to check
 601                  */
 602                 if (!hctx->nr_ctx || !hctx->tags)
 603                         continue;
 604
 605                 blk_mq_tag_busy_iter(hctx, blk_mq_check_expired, &data);
 606         }
 607
 608         if (data.next_set) {
 609                 data.next = blk_rq_timeout(round_jiffies_up(data.next));
 610                 mod_timer(&q->timeout, data.next);
 611         } else {
 612                 queue_for_each_hw_ctx(q, hctx, i)
 613                         blk_mq_tag_idle(hctx);
 614         }
 615 }
 616
 617 /*
 618  * Reverse check our software queue for entries that we could potentially
 619  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 620  * too much time checking for merges.
 621  */
 622 static bool blk_mq_attempt_merge(struct request_queue *q,
 623                                  struct blk_mq_ctx *ctx, struct bio *bio)
 624 {
 625         struct request *rq;
 626         int checked = 8;
 627
 628         list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
 629                 int el_ret;
 630
 631                 if (!checked--)
 632                         break;
 633
 634                 if (!blk_rq_merge_ok(rq, bio))
 635                         continue;
 636
 637                 el_ret = blk_try_merge(rq, bio);
 638                 if (el_ret == ELEVATOR_BACK_MERGE) {
 639                         if (bio_attempt_back_merge(q, rq, bio)) {
 640                                 ctx->rq_merged++;
 641                                 return true;
 642                         }
 643                         break;
 644                 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
 645                         if (bio_attempt_front_merge(q, rq, bio)) {
 646                                 ctx->rq_merged++;
 647                                 return true;
 648                         }
 649                         break;
 650                 }
 651         }
 652
 653         return false;
 654 }
 655
 656 /*
 657  * Process software queues that have been marked busy, splicing them
 658  * to the for-dispatch
 659  */
 660 static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 661 {
 662         struct blk_mq_ctx *ctx;
 663         int i;
 664
 665         for (i = 0; i < hctx->ctx_map.map_size; i++) {
 666                 struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
 667                 unsigned int off, bit;
 668
 669                 if (!bm->word)
 670                         continue;
 671
 672                 bit = 0;
 673                 off = i * hctx->ctx_map.bits_per_word;
 674                 do {
 675                         bit = find_next_bit(&bm->word, bm->depth, bit);
 676                         if (bit >= bm->depth)
 677                                 break;
 678
 679                         ctx = hctx->ctxs[bit + off];
 680                         clear_bit(bit, &bm->word);
 681                         spin_lock(&ctx->lock);
 682                         list_splice_tail_init(&ctx->rq_list, list);
 683                         spin_unlock(&ctx->lock);
 684
 685                         bit++;
 686                 } while (1);
 687         }
 688 }
 689
 690 /*
 691  * Run this hardware queue, pulling any software queues mapped to it in.
 692  * Note that this function currently has various problems around ordering
 693  * of IO. In particular, we'd like FIFO behaviour on handling existing
 694  * items on the hctx->dispatch list. Ignore that for now.
 695  */
 696 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 697 {
 698         struct request_queue *q = hctx->queue;
 699         struct request *rq;
 700         LIST_HEAD(rq_list);
 701         LIST_HEAD(driver_list);
 702         struct list_head *dptr;
 703         int queued;
 704
 705         WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 706
 707         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 708                 return;
 709
 710         hctx->run++;
 711
 712         /*
 713          * Touch any software queue that has pending entries.
 714          */
 715         flush_busy_ctxs(hctx, &rq_list);
 716
 717         /*
 718          * If we have previous entries on our dispatch list, grab them
 719          * and stuff them at the front for more fair dispatch.
 720          */
 721         if (!list_empty_careful(&hctx->dispatch)) {
 722                 spin_lock(&hctx->lock);
 723                 if (!list_empty(&hctx->dispatch))
 724                         list_splice_init(&hctx->dispatch, &rq_list);
 725                 spin_unlock(&hctx->lock);
 726         }
 727
 728         /*
 729          * Start off with dptr being NULL, so we start the first request
 730          * immediately, even if we have more pending.
 731          */
 732         dptr = NULL;
 733
 734         /*
 735          * Now process all the entries, sending them to the driver.
 736          */
 737         queued = 0;
 738         while (!list_empty(&rq_list)) {
 739                 struct blk_mq_queue_data bd;
 740                 int ret;
 741
 742                 rq = list_first_entry(&rq_list, struct request, queuelist);
 743                 list_del_init(&rq->queuelist);
 744
 745                 bd.rq = rq;
 746                 bd.list = dptr;
 747                 bd.last = list_empty(&rq_list);
 748
 749                 ret = q->mq_ops->queue_rq(hctx, &bd);
 750                 switch (ret) {
 751                 case BLK_MQ_RQ_QUEUE_OK:
 752                         queued++;
 753                         continue;
 754                 case BLK_MQ_RQ_QUEUE_BUSY:
 755                         list_add(&rq->queuelist, &rq_list);
 756                         __blk_mq_requeue_request(rq);
 757                         break;
 758                 default:
 759                         pr_err("blk-mq: bad return on queue: %d\n", ret);
 760                 case BLK_MQ_RQ_QUEUE_ERROR:
 761                         rq->errors = -EIO;
 762                         blk_mq_end_request(rq, rq->errors);
 763                         break;
 764                 }
 765
 766                 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
 767                         break;
 768
 769                 /*
 770                  * We've done the first request. If we have more than 1
 771                  * left in the list, set dptr to defer issue.
 772                  */
 773                 if (!dptr && rq_list.next != rq_list.prev)
 774                         dptr = &driver_list;
 775         }
 776
 777         if (!queued)
 778                 hctx->dispatched[0]++;
 779         else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
 780                 hctx->dispatched[ilog2(queued) + 1]++;
 781
 782         /*
 783          * Any items that need requeuing? Stuff them into hctx->dispatch,
 784          * that is where we will continue on next queue run.
 785          */
 786         if (!list_empty(&rq_list)) {
 787                 spin_lock(&hctx->lock);
 788                 list_splice(&rq_list, &hctx->dispatch);
 789                 spin_unlock(&hctx->lock);
 790         }
 791 }
 792
 793 /*
 794  * It'd be great if the workqueue API had a way to pass
 795  * in a mask and had some smarts for more clever placement.
 796  * For now we just round-robin here, switching for every
 797  * BLK_MQ_CPU_WORK_BATCH queued items.
 798  */
 799 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 800 {
 801         if (hctx->queue->nr_hw_queues == 1)
 802                 return WORK_CPU_UNBOUND;
 803
 804         if (--hctx->next_cpu_batch <= 0) {
 805                 int cpu = hctx->next_cpu, next_cpu;
 806
 807                 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
 808                 if (next_cpu >= nr_cpu_ids)
 809                         next_cpu = cpumask_first(hctx->cpumask);
 810
 811                 hctx->next_cpu = next_cpu;
 812                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 813
 814                 return cpu;
 815         }
 816
 817         return hctx->next_cpu;
 818 }
 819
 820 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 821 {
 822         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 823                 return;
 824
 825         if (!async) {
 826                 int cpu = get_cpu();
 827                 if (cpumask_test_cpu(cpu, hctx->cpumask)) {
 828                         __blk_mq_run_hw_queue(hctx);
 829                         put_cpu();
 830                         return;
 831                 }
 832
 833                 put_cpu();
 834         }
 835
 836         kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
 837                         &hctx->run_work, 0);
 838 }
 839
 840 void blk_mq_run_queues(struct request_queue *q, bool async)
 841 {
 842         struct blk_mq_hw_ctx *hctx;
 843         int i;
 844
 845         queue_for_each_hw_ctx(q, hctx, i) {
 846                 if ((!blk_mq_hctx_has_pending(hctx) &&
 847                     list_empty_careful(&hctx->dispatch)) ||
 848                     test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 849                         continue;
 850
 851                 blk_mq_run_hw_queue(hctx, async);
 852         }
 853 }
 854 EXPORT_SYMBOL(blk_mq_run_queues);
 855
 856 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 857 {
 858         cancel_delayed_work(&hctx->run_work);
 859         cancel_delayed_work(&hctx->delay_work);
 860         set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 861 }
 862 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
 863
 864 void blk_mq_stop_hw_queues(struct request_queue *q)
 865 {
 866         struct blk_mq_hw_ctx *hctx;
 867         int i;
 868
 869         queue_for_each_hw_ctx(q, hctx, i)
 870                 blk_mq_stop_hw_queue(hctx);
 871 }
 872 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
 873
 874 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 875 {
 876         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 877
 878         blk_mq_run_hw_queue(hctx, false);
 879 }
 880 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 881
 882 void blk_mq_start_hw_queues(struct request_queue *q)
 883 {
 884         struct blk_mq_hw_ctx *hctx;
 885         int i;
 886
 887         queue_for_each_hw_ctx(q, hctx, i)
 888                 blk_mq_start_hw_queue(hctx);
 889 }
 890 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 891
 892
 893 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 894 {
 895         struct blk_mq_hw_ctx *hctx;
 896         int i;
 897
 898         queue_for_each_hw_ctx(q, hctx, i) {
 899                 if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 900                         continue;
 901
 902                 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 903                 blk_mq_run_hw_queue(hctx, async);
 904         }
 905 }
 906 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 907
 908 static void blk_mq_run_work_fn(struct work_struct *work)
 909 {
 910         struct blk_mq_hw_ctx *hctx;
 911
 912         hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
 913
 914         __blk_mq_run_hw_queue(hctx);
 915 }
 916
 917 static void blk_mq_delay_work_fn(struct work_struct *work)
 918 {
 919         struct blk_mq_hw_ctx *hctx;
 920
 921         hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
 922
 923         if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
 924                 __blk_mq_run_hw_queue(hctx);
 925 }
 926
 927 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
 928 {
 929         kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
 930                         &hctx->delay_work, msecs_to_jiffies(msecs));
 931 }
 932 EXPORT_SYMBOL(blk_mq_delay_queue);
 933
 934 static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
 935                                     struct request *rq, bool at_head)
 936 {
 937         struct blk_mq_ctx *ctx = rq->mq_ctx;
 938
 939         trace_block_rq_insert(hctx->queue, rq);
 940
 941         if (at_head)
 942                 list_add(&rq->queuelist, &ctx->rq_list);
 943         else
 944                 list_add_tail(&rq->queuelist, &ctx->rq_list);
 945
 946         blk_mq_hctx_mark_pending(hctx, ctx);
 947 }
 948
 949 void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
 950                 bool async)
 951 {
 952         struct request_queue *q = rq->q;
 953         struct blk_mq_hw_ctx *hctx;
 954         struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;
 955
 956         current_ctx = blk_mq_get_ctx(q);
 957         if (!cpu_online(ctx->cpu))
 958                 rq->mq_ctx = ctx = current_ctx;
 959
 960         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 961
 962         spin_lock(&ctx->lock);
 963         __blk_mq_insert_request(hctx, rq, at_head);
 964         spin_unlock(&ctx->lock);
 965
 966         if (run_queue)
 967                 blk_mq_run_hw_queue(hctx, async);
 968
 969         blk_mq_put_ctx(current_ctx);
 970 }
 971
 972 static void blk_mq_insert_requests(struct request_queue *q,
 973                                      struct blk_mq_ctx *ctx,
 974                                      struct list_head *list,
 975                                      int depth,
 976                                      bool from_schedule)
 977
 978 {
 979         struct blk_mq_hw_ctx *hctx;
 980         struct blk_mq_ctx *current_ctx;
 981
 982         trace_block_unplug(q, depth, !from_schedule);
 983
 984         current_ctx = blk_mq_get_ctx(q);
 985
 986         if (!cpu_online(ctx->cpu))
 987                 ctx = current_ctx;
 988         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 989
 990         /*
 991          * preemption doesn't flush plug list, so it's possible ctx->cpu is
 992          * offline now
 993          */
 994         spin_lock(&ctx->lock);
 995         while (!list_empty(list)) {
 996                 struct request *rq;
 997
 998                 rq = list_first_entry(list, struct request, queuelist);
 999                 list_del_init(&rq->queuelist);
1000                 rq->mq_ctx = ctx;
1001                 __blk_mq_insert_request(hctx, rq, false);
1002         }
1003         spin_unlock(&ctx->lock);
1004
1005         blk_mq_run_hw_queue(hctx, from_schedule);
1006         blk_mq_put_ctx(current_ctx);
1007 }
1008
1009 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1010 {
1011         struct request *rqa = container_of(a, struct request, queuelist);
1012         struct request *rqb = container_of(b, struct request, queuelist);
1013
1014         return !(rqa->mq_ctx < rqb->mq_ctx ||
1015                  (rqa->mq_ctx == rqb->mq_ctx &&
1016                   blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1017 }
1018
1019 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1020 {
1021         struct blk_mq_ctx *this_ctx;
1022         struct request_queue *this_q;
1023         struct request *rq;
1024         LIST_HEAD(list);
1025         LIST_HEAD(ctx_list);
1026         unsigned int depth;
1027
1028         list_splice_init(&plug->mq_list, &list);
1029
1030         list_sort(NULL, &list, plug_ctx_cmp);
1031
1032         this_q = NULL;
1033         this_ctx = NULL;
1034         depth = 0;
1035
1036         while (!list_empty(&list)) {
1037                 rq = list_entry_rq(list.next);
1038                 list_del_init(&rq->queuelist);
1039                 BUG_ON(!rq->q);
1040                 if (rq->mq_ctx != this_ctx) {
1041                         if (this_ctx) {
1042                                 blk_mq_insert_requests(this_q, this_ctx,
1043                                                         &ctx_list, depth,
1044                                                         from_schedule);
1045                         }
1046
1047                         this_ctx = rq->mq_ctx;
1048                         this_q = rq->q;
1049                         depth = 0;
1050                 }
1051
1052                 depth++;
1053                 list_add_tail(&rq->queuelist, &ctx_list);
1054         }
1055
1056         /*
1057          * If 'this_ctx' is set, we know we have entries to complete
1058          * on 'ctx_list'. Do those.
1059          */
1060         if (this_ctx) {
1061                 blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1062                                        from_schedule);
1063         }
1064 }
1065
1066 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1067 {
1068         init_request_from_bio(rq, bio);
1069
1070         if (blk_do_io_stat(rq))
1071                 blk_account_io_start(rq, 1);
1072 }
1073
1074 static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
1075 {
1076         return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
1077                 !blk_queue_nomerges(hctx->queue);
1078 }
1079
1080 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1081                                          struct blk_mq_ctx *ctx,
1082                                          struct request *rq, struct bio *bio)
1083 {
1084         if (!hctx_allow_merges(hctx)) {
1085                 blk_mq_bio_to_request(rq, bio);
1086                 spin_lock(&ctx->lock);
1087 insert_rq:
1088                 __blk_mq_insert_request(hctx, rq, false);
1089                 spin_unlock(&ctx->lock);
1090                 return false;
1091         } else {
1092                 struct request_queue *q = hctx->queue;
1093
1094                 spin_lock(&ctx->lock);
1095                 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1096                         blk_mq_bio_to_request(rq, bio);
1097                         goto insert_rq;
1098                 }
1099
1100                 spin_unlock(&ctx->lock);
1101                 __blk_mq_free_request(hctx, ctx, rq);
1102                 return true;
1103         }
1104 }
1105
1106 struct blk_map_ctx {
1107         struct blk_mq_hw_ctx *hctx;
1108         struct blk_mq_ctx *ctx;
1109 };
1110
1111 static struct request *blk_mq_map_request(struct request_queue *q,
1112                                           struct bio *bio,
1113                                           struct blk_map_ctx *data)
1114 {
1115         struct blk_mq_hw_ctx *hctx;
1116         struct blk_mq_ctx *ctx;
1117         struct request *rq;
1118         int rw = bio_data_dir(bio);
1119         struct blk_mq_alloc_data alloc_data;
1120
1121         if (unlikely(blk_mq_queue_enter(q))) {
1122                 bio_endio(bio, -EIO);
1123                 return NULL;
1124         }
1125
1126         ctx = blk_mq_get_ctx(q);
1127         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1128
1129         if (rw_is_sync(bio->bi_rw))
1130                 rw |= REQ_SYNC;
1131
1132         trace_block_getrq(q, bio, rw);
1133         blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx,
1134                         hctx);
1135         rq = __blk_mq_alloc_request(&alloc_data, rw);
1136         if (unlikely(!rq)) {
1137                 __blk_mq_run_hw_queue(hctx);
1138                 blk_mq_put_ctx(ctx);
1139                 trace_block_sleeprq(q, bio, rw);
1140
1141                 ctx = blk_mq_get_ctx(q);
1142                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1143                 blk_mq_set_alloc_data(&alloc_data, q,
1144                                 __GFP_WAIT|GFP_ATOMIC, false, ctx, hctx);
1145                 rq = __blk_mq_alloc_request(&alloc_data, rw);
1146                 ctx = alloc_data.ctx;
1147                 hctx = alloc_data.hctx;
1148         }
1149
1150         hctx->queued++;
1151         data->hctx = hctx;
1152         data->ctx = ctx;
1153         return rq;
1154 }
1155
1156 /*
1157  * Multiple hardware queue variant. This will not use per-process plugs,
1158  * but will attempt to bypass the hctx queueing if we can go straight to
1159  * hardware for SYNC IO.
1160  */
1161 static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1162 {
1163         const int is_sync = rw_is_sync(bio->bi_rw);
1164         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1165         struct blk_map_ctx data;
1166         struct request *rq;
1167
1168         blk_queue_bounce(q, &bio);
1169
1170         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1171                 bio_endio(bio, -EIO);
1172                 return;
1173         }
1174
1175         rq = blk_mq_map_request(q, bio, &data);
1176         if (unlikely(!rq))
1177                 return;
1178
1179         if (unlikely(is_flush_fua)) {
1180                 blk_mq_bio_to_request(rq, bio);
1181                 blk_insert_flush(rq);
1182                 goto run_queue;
1183         }
1184
1185         /*
1186          * If the driver supports defer issued based on 'last', then
1187          * queue it up like normal since we can potentially save some
1188          * CPU this way.
1189          */
1190         if (is_sync && !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
1191                 struct blk_mq_queue_data bd = {
1192                         .rq = rq,
1193                         .list = NULL,
1194                         .last = 1
1195                 };
1196                 int ret;
1197
1198                 blk_mq_bio_to_request(rq, bio);
1199
1200                 /*
1201                  * For OK queue, we are done. For error, kill it. Any other
1202                  * error (busy), just add it to our list as we previously
1203                  * would have done
1204                  */
1205                 ret = q->mq_ops->queue_rq(data.hctx, &bd);
1206                 if (ret == BLK_MQ_RQ_QUEUE_OK)
1207                         goto done;
1208                 else {
1209                         __blk_mq_requeue_request(rq);
1210
1211                         if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1212                                 rq->errors = -EIO;
1213                                 blk_mq_end_request(rq, rq->errors);
1214                                 goto done;
1215                         }
1216                 }
1217         }
1218
1219         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1220                 /*
1221                  * For a SYNC request, send it to the hardware immediately. For
1222                  * an ASYNC request, just ensure that we run it later on. The
1223                  * latter allows for merging opportunities and more efficient
1224                  * dispatching.
1225                  */
1226 run_queue:
1227                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1228         }
1229 done:
1230         blk_mq_put_ctx(data.ctx);
1231 }
1232
1233 /*
1234  * Single hardware queue variant. This will attempt to use any per-process
1235  * plug for merging and IO deferral.
1236  */
1237 static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1238 {
1239         const int is_sync = rw_is_sync(bio->bi_rw);
1240         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1241         unsigned int use_plug, request_count = 0;
1242         struct blk_map_ctx data;
1243         struct request *rq;
1244
1245         /*
1246          * If we have multiple hardware queues, just go directly to
1247          * one of those for sync IO.
1248          */
1249         use_plug = !is_flush_fua && !is_sync;
1250
1251         blk_queue_bounce(q, &bio);
1252
1253         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1254                 bio_endio(bio, -EIO);
1255                 return;
1256         }
1257
1258         if (use_plug && !blk_queue_nomerges(q) &&
1259             blk_attempt_plug_merge(q, bio, &request_count))
1260                 return;
1261
1262         rq = blk_mq_map_request(q, bio, &data);
1263         if (unlikely(!rq))
1264                 return;
1265
1266         if (unlikely(is_flush_fua)) {
1267                 blk_mq_bio_to_request(rq, bio);
1268                 blk_insert_flush(rq);
1269                 goto run_queue;
1270         }
1271
1272         /*
1273          * A task plug currently exists. Since this is completely lockless,
1274          * utilize that to temporarily store requests until the task is
1275          * either done or scheduled away.
1276          */
1277         if (use_plug) {
1278                 struct blk_plug *plug = current->plug;
1279
1280                 if (plug) {
1281                         blk_mq_bio_to_request(rq, bio);
1282                         if (list_empty(&plug->mq_list))
1283                                 trace_block_plug(q);
1284                         else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1285                                 blk_flush_plug_list(plug, false);
1286                                 trace_block_plug(q);
1287                         }
1288                         list_add_tail(&rq->queuelist, &plug->mq_list);
1289                         blk_mq_put_ctx(data.ctx);
1290                         return;
1291                 }
1292         }
1293
1294         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1295                 /*
1296                  * For a SYNC request, send it to the hardware immediately. For
1297                  * an ASYNC request, just ensure that we run it later on. The
1298                  * latter allows for merging opportunities and more efficient
1299                  * dispatching.
1300                  */
1301 run_queue:
1302                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1303         }
1304
1305         blk_mq_put_ctx(data.ctx);
1306 }
1307
1308 /*
1309  * Default mapping to a software queue, since we use one per CPU.
1310  */
1311 struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
1312 {
1313         return q->queue_hw_ctx[q->mq_map[cpu]];
1314 }
1315 EXPORT_SYMBOL(blk_mq_map_queue);
1316
1317 static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
1318                 struct blk_mq_tags *tags, unsigned int hctx_idx)
1319 {
1320         struct page *page;
1321
1322         if (tags->rqs && set->ops->exit_request) {
1323                 int i;
1324
1325                 for (i = 0; i < tags->nr_tags; i++) {
1326                         if (!tags->rqs[i])
1327                                 continue;
1328                         set->ops->exit_request(set->driver_data, tags->rqs[i],
1329                                                 hctx_idx, i);
1330                         tags->rqs[i] = NULL;
1331                 }
1332         }
1333
1334         while (!list_empty(&tags->page_list)) {
1335                 page = list_first_entry(&tags->page_list, struct page, lru);
1336                 list_del_init(&page->lru);
1337                 __free_pages(page, page->private);
1338         }
1339
1340         kfree(tags->rqs);
1341
1342         blk_mq_free_tags(tags);
1343 }
1344
1345 static size_t order_to_size(unsigned int order)
1346 {
1347         return (size_t)PAGE_SIZE << order;
1348 }
1349
1350 static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
1351                 unsigned int hctx_idx)
1352 {
1353         struct blk_mq_tags *tags;
1354         unsigned int i, j, entries_per_page, max_order = 4;
1355         size_t rq_size, left;
1356
1357         tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1358                                 set->numa_node);
1359         if (!tags)
1360                 return NULL;
1361
1362         INIT_LIST_HEAD(&tags->page_list);
1363
1364         tags->rqs = kzalloc_node(set->queue_depth * sizeof(struct request *),
1365                                  GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY,
1366                                  set->numa_node);
1367         if (!tags->rqs) {
1368                 blk_mq_free_tags(tags);
1369                 return NULL;
1370         }
1371
1372         /*
1373          * rq_size is the size of the request plus driver payload, rounded
1374          * to the cacheline size
1375          */
1376         rq_size = round_up(sizeof(struct request) + set->cmd_size,
1377                                 cache_line_size());
1378         left = rq_size * set->queue_depth;
1379
1380         for (i = 0; i < set->queue_depth; ) {
1381                 int this_order = max_order;
1382                 struct page *page;
1383                 int to_do;
1384                 void *p;
1385
1386                 while (left < order_to_size(this_order - 1) && this_order)
1387                         this_order--;
1388
1389                 do {
1390                         page = alloc_pages_node(set->numa_node,
1391                                 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY,
1392                                 this_order);
1393                         if (page)
1394                                 break;
1395                         if (!this_order--)
1396                                 break;
1397                         if (order_to_size(this_order) < rq_size)
1398                                 break;
1399                 } while (1);
1400
1401                 if (!page)
1402                         goto fail;
1403
1404                 page->private = this_order;
1405                 list_add_tail(&page->lru, &tags->page_list);
1406
1407                 p = page_address(page);
1408                 entries_per_page = order_to_size(this_order) / rq_size;
1409                 to_do = min(entries_per_page, set->queue_depth - i);
1410                 left -= to_do * rq_size;
1411                 for (j = 0; j < to_do; j++) {
1412                         tags->rqs[i] = p;
1413                         tags->rqs[i]->atomic_flags = 0;
1414                         tags->rqs[i]->cmd_flags = 0;
1415                         if (set->ops->init_request) {
1416                                 if (set->ops->init_request(set->driver_data,
1417                                                 tags->rqs[i], hctx_idx, i,
1418                                                 set->numa_node)) {
1419                                         tags->rqs[i] = NULL;
1420                                         goto fail;
1421                                 }
1422                         }
1423
1424                         p += rq_size;
1425                         i++;
1426                 }
1427         }
1428
1429         return tags;
1430
1431 fail:
1432         blk_mq_free_rq_map(set, tags, hctx_idx);
1433         return NULL;
1434 }
1435
1436 static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
1437 {
1438         kfree(bitmap->map);
1439 }
1440
1441 static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
1442 {
1443         unsigned int bpw = 8, total, num_maps, i;
1444
1445         bitmap->bits_per_word = bpw;
1446
1447         num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
1448         bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
1449                                         GFP_KERNEL, node);
1450         if (!bitmap->map)
1451                 return -ENOMEM;
1452
1453         bitmap->map_size = num_maps;
1454
1455         total = nr_cpu_ids;
1456         for (i = 0; i < num_maps; i++) {
1457                 bitmap->map[i].depth = min(total, bitmap->bits_per_word);
1458                 total -= bitmap->map[i].depth;
1459         }
1460
1461         return 0;
1462 }
1463
1464 static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
1465 {
1466         struct request_queue *q = hctx->queue;
1467         struct blk_mq_ctx *ctx;
1468         LIST_HEAD(tmp);
1469
1470         /*
1471          * Move ctx entries to new CPU, if this one is going away.
1472          */
1473         ctx = __blk_mq_get_ctx(q, cpu);
1474
1475         spin_lock(&ctx->lock);
1476         if (!list_empty(&ctx->rq_list)) {
1477                 list_splice_init(&ctx->rq_list, &tmp);
1478                 blk_mq_hctx_clear_pending(hctx, ctx);
1479         }
1480         spin_unlock(&ctx->lock);
1481
1482         if (list_empty(&tmp))
1483                 return NOTIFY_OK;
1484
1485         ctx = blk_mq_get_ctx(q);
1486         spin_lock(&ctx->lock);
1487
1488         while (!list_empty(&tmp)) {
1489                 struct request *rq;
1490
1491                 rq = list_first_entry(&tmp, struct request, queuelist);
1492                 rq->mq_ctx = ctx;
1493                 list_move_tail(&rq->queuelist, &ctx->rq_list);
1494         }
1495
1496         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1497         blk_mq_hctx_mark_pending(hctx, ctx);
1498
1499         spin_unlock(&ctx->lock);
1500
1501         blk_mq_run_hw_queue(hctx, true);
1502         blk_mq_put_ctx(ctx);
1503         return NOTIFY_OK;
1504 }
1505
1506 static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
1507 {
1508         struct request_queue *q = hctx->queue;
1509         struct blk_mq_tag_set *set = q->tag_set;
1510
1511         if (set->tags[hctx->queue_num])
1512                 return NOTIFY_OK;
1513
1514         set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
1515         if (!set->tags[hctx->queue_num])
1516                 return NOTIFY_STOP;
1517
1518         hctx->tags = set->tags[hctx->queue_num];
1519         return NOTIFY_OK;
1520 }
1521
1522 static int blk_mq_hctx_notify(void *data, unsigned long action,
1523                               unsigned int cpu)
1524 {
1525         struct blk_mq_hw_ctx *hctx = data;
1526
1527         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1528                 return blk_mq_hctx_cpu_offline(hctx, cpu);
1529         else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
1530                 return blk_mq_hctx_cpu_online(hctx, cpu);
1531
1532         return NOTIFY_OK;
1533 }
1534
1535 static void blk_mq_exit_hctx(struct request_queue *q,
1536                 struct blk_mq_tag_set *set,
1537                 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1538 {
1539         unsigned flush_start_tag = set->queue_depth;
1540
1541         blk_mq_tag_idle(hctx);
1542
1543         if (set->ops->exit_request)
1544                 set->ops->exit_request(set->driver_data,
1545                                        hctx->fq->flush_rq, hctx_idx,
1546                                        flush_start_tag + hctx_idx);
1547
1548         if (set->ops->exit_hctx)
1549                 set->ops->exit_hctx(hctx, hctx_idx);
1550
1551         blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1552         blk_free_flush_queue(hctx->fq);
1553         kfree(hctx->ctxs);
1554         blk_mq_free_bitmap(&hctx->ctx_map);
1555 }
1556
1557 static void blk_mq_exit_hw_queues(struct request_queue *q,
1558                 struct blk_mq_tag_set *set, int nr_queue)
1559 {
1560         struct blk_mq_hw_ctx *hctx;
1561         unsigned int i;
1562
1563         queue_for_each_hw_ctx(q, hctx, i) {
1564                 if (i == nr_queue)
1565                         break;
1566                 blk_mq_exit_hctx(q, set, hctx, i);
1567         }
1568 }
1569
1570 static void blk_mq_free_hw_queues(struct request_queue *q,
1571                 struct blk_mq_tag_set *set)
1572 {
1573         struct blk_mq_hw_ctx *hctx;
1574         unsigned int i;
1575
1576         queue_for_each_hw_ctx(q, hctx, i) {
1577                 free_cpumask_var(hctx->cpumask);
1578                 kfree(hctx);
1579         }
1580 }
1581
1582 static int blk_mq_init_hctx(struct request_queue *q,
1583                 struct blk_mq_tag_set *set,
1584                 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1585 {
1586         int node;
1587         unsigned flush_start_tag = set->queue_depth;
1588
1589         node = hctx->numa_node;
1590         if (node == NUMA_NO_NODE)
1591                 node = hctx->numa_node = set->numa_node;
1592
1593         INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
1594         INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1595         spin_lock_init(&hctx->lock);
1596         INIT_LIST_HEAD(&hctx->dispatch);
1597         hctx->queue = q;
1598         hctx->queue_num = hctx_idx;
1599         hctx->flags = set->flags;
1600         hctx->cmd_size = set->cmd_size;
1601
1602         blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1603                                         blk_mq_hctx_notify, hctx);
1604         blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1605
1606         hctx->tags = set->tags[hctx_idx];
1607
1608         /*
1609          * Allocate space for all possible cpus to avoid allocation at
1610          * runtime
1611          */
1612         hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1613                                         GFP_KERNEL, node);
1614         if (!hctx->ctxs)
1615                 goto unregister_cpu_notifier;
1616
1617         if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
1618                 goto free_ctxs;
1619
1620         hctx->nr_ctx = 0;
1621
1622         if (set->ops->init_hctx &&
1623             set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
1624                 goto free_bitmap;
1625
1626         hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
1627         if (!hctx->fq)
1628                 goto exit_hctx;
1629
1630         if (set->ops->init_request &&
1631             set->ops->init_request(set->driver_data,
1632                                    hctx->fq->flush_rq, hctx_idx,
1633                                    flush_start_tag + hctx_idx, node))
1634                 goto free_fq;
1635
1636         return 0;
1637
1638  free_fq:
1639         kfree(hctx->fq);
1640  exit_hctx:
1641         if (set->ops->exit_hctx)
1642                 set->ops->exit_hctx(hctx, hctx_idx);
1643  free_bitmap:
1644         blk_mq_free_bitmap(&hctx->ctx_map);
1645  free_ctxs:
1646         kfree(hctx->ctxs);
1647  unregister_cpu_notifier:
1648         blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1649
1650         return -1;
1651 }
1652
1653 static int blk_mq_init_hw_queues(struct request_queue *q,
1654                 struct blk_mq_tag_set *set)
1655 {
1656         struct blk_mq_hw_ctx *hctx;
1657         unsigned int i;
1658
1659         /*
1660          * Initialize hardware queues
1661          */
1662         queue_for_each_hw_ctx(q, hctx, i) {
1663                 if (blk_mq_init_hctx(q, set, hctx, i))
1664                         break;
1665         }
1666
1667         if (i == q->nr_hw_queues)
1668                 return 0;
1669
1670         /*
1671          * Init failed
1672          */
1673         blk_mq_exit_hw_queues(q, set, i);
1674
1675         return 1;
1676 }
1677
1678 static void blk_mq_init_cpu_queues(struct request_queue *q,
1679                                    unsigned int nr_hw_queues)
1680 {
1681         unsigned int i;
1682
1683         for_each_possible_cpu(i) {
1684                 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1685                 struct blk_mq_hw_ctx *hctx;
1686
1687                 memset(__ctx, 0, sizeof(*__ctx));
1688                 __ctx->cpu = i;
1689                 spin_lock_init(&__ctx->lock);
1690                 INIT_LIST_HEAD(&__ctx->rq_list);
1691                 __ctx->queue = q;
1692
1693                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1694                 if (!cpu_online(i))
1695                         continue;
1696
1697                 hctx = q->mq_ops->map_queue(q, i);
1698                 cpumask_set_cpu(i, hctx->cpumask);
1699                 hctx->nr_ctx++;
1700
1701                 /*
1702                  * Set local node, IFF we have more than one hw queue. If
1703                  * not, we remain on the home node of the device
1704                  */
1705                 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1706                         hctx->numa_node = cpu_to_node(i);
1707         }
1708 }
1709
1710 static void blk_mq_map_swqueue(struct request_queue *q)
1711 {
1712         unsigned int i;
1713         struct blk_mq_hw_ctx *hctx;
1714         struct blk_mq_ctx *ctx;
1715
1716         queue_for_each_hw_ctx(q, hctx, i) {
1717                 cpumask_clear(hctx->cpumask);
1718                 hctx->nr_ctx = 0;
1719         }
1720
1721         /*
1722          * Map software to hardware queues
1723          */
1724         queue_for_each_ctx(q, ctx, i) {
1725                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1726                 if (!cpu_online(i))
1727                         continue;
1728
1729                 hctx = q->mq_ops->map_queue(q, i);
1730                 cpumask_set_cpu(i, hctx->cpumask);
1731                 ctx->index_hw = hctx->nr_ctx;
1732                 hctx->ctxs[hctx->nr_ctx++] = ctx;
1733         }
1734
1735         queue_for_each_hw_ctx(q, hctx, i) {
1736                 /*
1737                  * If no software queues are mapped to this hardware queue,
1738                  * disable it and free the request entries.
1739                  */
1740                 if (!hctx->nr_ctx) {
1741                         struct blk_mq_tag_set *set = q->tag_set;
1742
1743                         if (set->tags[i]) {
1744                                 blk_mq_free_rq_map(set, set->tags[i], i);
1745                                 set->tags[i] = NULL;
1746                                 hctx->tags = NULL;
1747                         }
1748                         continue;
1749                 }
1750
1751                 /*
1752                  * Initialize batch roundrobin counts
1753                  */
1754                 hctx->next_cpu = cpumask_first(hctx->cpumask);
1755                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1756         }
1757 }
1758
1759 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
1760 {
1761         struct blk_mq_hw_ctx *hctx;
1762         struct request_queue *q;
1763         bool shared;
1764         int i;
1765
1766         if (set->tag_list.next == set->tag_list.prev)
1767                 shared = false;
1768         else
1769                 shared = true;
1770
1771         list_for_each_entry(q, &set->tag_list, tag_set_list) {
1772                 blk_mq_freeze_queue(q);
1773
1774                 queue_for_each_hw_ctx(q, hctx, i) {
1775                         if (shared)
1776                                 hctx->flags |= BLK_MQ_F_TAG_SHARED;
1777                         else
1778                                 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
1779                 }
1780                 blk_mq_unfreeze_queue(q);
1781         }
1782 }
1783
1784 static void blk_mq_del_queue_tag_set(struct request_queue *q)
1785 {
1786         struct blk_mq_tag_set *set = q->tag_set;
1787
1788         mutex_lock(&set->tag_list_lock);
1789         list_del_init(&q->tag_set_list);
1790         blk_mq_update_tag_set_depth(set);
1791         mutex_unlock(&set->tag_list_lock);
1792 }
1793
1794 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
1795                                      struct request_queue *q)
1796 {
1797         q->tag_set = set;
1798
1799         mutex_lock(&set->tag_list_lock);
1800         list_add_tail(&q->tag_set_list, &set->tag_list);
1801         blk_mq_update_tag_set_depth(set);
1802         mutex_unlock(&set->tag_list_lock);
1803 }
1804
1805 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1806 {
1807         struct blk_mq_hw_ctx **hctxs;
1808         struct blk_mq_ctx __percpu *ctx;
1809         struct request_queue *q;
1810         unsigned int *map;
1811         int i;
1812
1813         ctx = alloc_percpu(struct blk_mq_ctx);
1814         if (!ctx)
1815                 return ERR_PTR(-ENOMEM);
1816
1817         /*
1818          * If a crashdump is active, then we are potentially in a very
1819          * memory constrained environment. Limit us to 1 queue and
1820          * 64 tags to prevent using too much memory.
1821          */
1822         if (is_kdump_kernel()) {
1823                 set->nr_hw_queues = 1;
1824                 set->queue_depth = min(64U, set->queue_depth);
1825         }
1826
1827         hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1828                         set->numa_node);
1829
1830         if (!hctxs)
1831                 goto err_percpu;
1832
1833         map = blk_mq_make_queue_map(set);
1834         if (!map)
1835                 goto err_map;
1836
1837         for (i = 0; i < set->nr_hw_queues; i++) {
1838                 int node = blk_mq_hw_queue_to_node(map, i);
1839
1840                 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
1841                                         GFP_KERNEL, node);
1842                 if (!hctxs[i])
1843                         goto err_hctxs;
1844
1845                 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
1846                                                 node))
1847                         goto err_hctxs;
1848
1849                 atomic_set(&hctxs[i]->nr_active, 0);
1850                 hctxs[i]->numa_node = node;
1851                 hctxs[i]->queue_num = i;
1852         }
1853
1854         q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
1855         if (!q)
1856                 goto err_hctxs;
1857
1858         /*
1859          * Init percpu_ref in atomic mode so that it's faster to shutdown.
1860          * See blk_register_queue() for details.
1861          */
1862         if (percpu_ref_init(&q->mq_usage_counter, blk_mq_usage_counter_release,
1863                             PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
1864                 goto err_map;
1865
1866         setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1867         blk_queue_rq_timeout(q, 30000);
1868
1869         q->nr_queues = nr_cpu_ids;
1870         q->nr_hw_queues = set->nr_hw_queues;
1871         q->mq_map = map;
1872
1873         q->queue_ctx = ctx;
1874         q->queue_hw_ctx = hctxs;
1875
1876         q->mq_ops = set->ops;
1877         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1878
1879         if (!(set->flags & BLK_MQ_F_SG_MERGE))
1880                 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
1881
1882         q->sg_reserved_size = INT_MAX;
1883
1884         INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
1885         INIT_LIST_HEAD(&q->requeue_list);
1886         spin_lock_init(&q->requeue_lock);
1887
1888         if (q->nr_hw_queues > 1)
1889                 blk_queue_make_request(q, blk_mq_make_request);
1890         else
1891                 blk_queue_make_request(q, blk_sq_make_request);
1892
1893         if (set->timeout)
1894                 blk_queue_rq_timeout(q, set->timeout);
1895
1896         /*
1897          * Do this after blk_queue_make_request() overrides it...
1898          */
1899         q->nr_requests = set->queue_depth;
1900
1901         if (set->ops->complete)
1902                 blk_queue_softirq_done(q, set->ops->complete);
1903
1904         blk_mq_init_cpu_queues(q, set->nr_hw_queues);
1905
1906         if (blk_mq_init_hw_queues(q, set))
1907                 goto err_hw;
1908
1909         mutex_lock(&all_q_mutex);
1910         list_add_tail(&q->all_q_node, &all_q_list);
1911         mutex_unlock(&all_q_mutex);
1912
1913         blk_mq_add_queue_tag_set(set, q);
1914
1915         blk_mq_map_swqueue(q);
1916
1917         return q;
1918
1919 err_hw:
1920         blk_cleanup_queue(q);
1921 err_hctxs:
1922         kfree(map);
1923         for (i = 0; i < set->nr_hw_queues; i++) {
1924                 if (!hctxs[i])
1925                         break;
1926                 free_cpumask_var(hctxs[i]->cpumask);
1927                 kfree(hctxs[i]);
1928         }
1929 err_map:
1930         kfree(hctxs);
1931 err_percpu:
1932         free_percpu(ctx);
1933         return ERR_PTR(-ENOMEM);
1934 }
1935 EXPORT_SYMBOL(blk_mq_init_queue);
1936
1937 void blk_mq_free_queue(struct request_queue *q)
1938 {
1939         struct blk_mq_tag_set   *set = q->tag_set;
1940
1941         blk_mq_del_queue_tag_set(q);
1942
1943         blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
1944         blk_mq_free_hw_queues(q, set);
1945
1946         percpu_ref_exit(&q->mq_usage_counter);
1947
1948         free_percpu(q->queue_ctx);
1949         kfree(q->queue_hw_ctx);
1950         kfree(q->mq_map);
1951
1952         q->queue_ctx = NULL;
1953         q->queue_hw_ctx = NULL;
1954         q->mq_map = NULL;
1955
1956         mutex_lock(&all_q_mutex);
1957         list_del_init(&q->all_q_node);
1958         mutex_unlock(&all_q_mutex);
1959 }
1960
1961 /* Basically redo blk_mq_init_queue with queue frozen */
1962 static void blk_mq_queue_reinit(struct request_queue *q)
1963 {
1964         WARN_ON_ONCE(!q->mq_freeze_depth);
1965
1966         blk_mq_sysfs_unregister(q);
1967
1968         blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1969
1970         /*
1971          * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1972          * we should change hctx numa_node according to new topology (this
1973          * involves free and re-allocate memory, worthy doing?)
1974          */
1975
1976         blk_mq_map_swqueue(q);
1977
1978         blk_mq_sysfs_register(q);
1979 }
1980
1981 static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1982                                       unsigned long action, void *hcpu)
1983 {
1984         struct request_queue *q;
1985
1986         /*
1987          * Before new mappings are established, hotadded cpu might already
1988          * start handling requests. This doesn't break anything as we map
1989          * offline CPUs to first hardware queue. We will re-init the queue
1990          * below to get optimal settings.
1991          */
1992         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1993             action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1994                 return NOTIFY_OK;
1995
1996         mutex_lock(&all_q_mutex);
1997
1998         /*
1999          * We need to freeze and reinit all existing queues.  Freezing
2000          * involves synchronous wait for an RCU grace period and doing it
2001          * one by one may take a long time.  Start freezing all queues in
2002          * one swoop and then wait for the completions so that freezing can
2003          * take place in parallel.
2004          */
2005         list_for_each_entry(q, &all_q_list, all_q_node)
2006                 blk_mq_freeze_queue_start(q);
2007         list_for_each_entry(q, &all_q_list, all_q_node)
2008                 blk_mq_freeze_queue_wait(q);
2009
2010         list_for_each_entry(q, &all_q_list, all_q_node)
2011                 blk_mq_queue_reinit(q);
2012
2013         list_for_each_entry(q, &all_q_list, all_q_node)
2014                 blk_mq_unfreeze_queue(q);
2015
2016         mutex_unlock(&all_q_mutex);
2017         return NOTIFY_OK;
2018 }
2019
2020 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2021 {
2022         int i;
2023
2024         for (i = 0; i < set->nr_hw_queues; i++) {
2025                 set->tags[i] = blk_mq_init_rq_map(set, i);
2026                 if (!set->tags[i])
2027                         goto out_unwind;
2028         }
2029
2030         return 0;
2031
2032 out_unwind:
2033         while (--i >= 0)
2034                 blk_mq_free_rq_map(set, set->tags[i], i);
2035
2036         return -ENOMEM;
2037 }
2038
2039 /*
2040  * Allocate the request maps associated with this tag_set. Note that this
2041  * may reduce the depth asked for, if memory is tight. set->queue_depth
2042  * will be updated to reflect the allocated depth.
2043  */
2044 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2045 {
2046         unsigned int depth;
2047         int err;
2048
2049         depth = set->queue_depth;
2050         do {
2051                 err = __blk_mq_alloc_rq_maps(set);
2052                 if (!err)
2053                         break;
2054
2055                 set->queue_depth >>= 1;
2056                 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
2057                         err = -ENOMEM;
2058                         break;
2059                 }
2060         } while (set->queue_depth);
2061
2062         if (!set->queue_depth || err) {
2063                 pr_err("blk-mq: failed to allocate request map\n");
2064                 return -ENOMEM;
2065         }
2066
2067         if (depth != set->queue_depth)
2068                 pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
2069                                                 depth, set->queue_depth);
2070
2071         return 0;
2072 }
2073
2074 /*
2075  * Alloc a tag set to be associated with one or more request queues.
2076  * May fail with EINVAL for various error conditions. May adjust the
2077  * requested depth down, if if it too large. In that case, the set
2078  * value will be stored in set->queue_depth.
2079  */
2080 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
2081 {
2082         if (!set->nr_hw_queues)
2083                 return -EINVAL;
2084         if (!set->queue_depth)
2085                 return -EINVAL;
2086         if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
2087                 return -EINVAL;
2088
2089         if (!set->nr_hw_queues || !set->ops->queue_rq || !set->ops->map_queue)
2090                 return -EINVAL;
2091
2092         if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
2093                 pr_info("blk-mq: reduced tag depth to %u\n",
2094                         BLK_MQ_MAX_DEPTH);
2095                 set->queue_depth = BLK_MQ_MAX_DEPTH;
2096         }
2097
2098         set->tags = kmalloc_node(set->nr_hw_queues *
2099                                  sizeof(struct blk_mq_tags *),
2100                                  GFP_KERNEL, set->numa_node);
2101         if (!set->tags)
2102                 return -ENOMEM;
2103
2104         if (blk_mq_alloc_rq_maps(set))
2105                 goto enomem;
2106
2107         mutex_init(&set->tag_list_lock);
2108         INIT_LIST_HEAD(&set->tag_list);
2109
2110         return 0;
2111 enomem:
2112         kfree(set->tags);
2113         set->tags = NULL;
2114         return -ENOMEM;
2115 }
2116 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2117
2118 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2119 {
2120         int i;
2121
2122         for (i = 0; i < set->nr_hw_queues; i++) {
2123                 if (set->tags[i])
2124                         blk_mq_free_rq_map(set, set->tags[i], i);
2125         }
2126
2127         kfree(set->tags);
2128         set->tags = NULL;
2129 }
2130 EXPORT_SYMBOL(blk_mq_free_tag_set);
2131
2132 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2133 {
2134         struct blk_mq_tag_set *set = q->tag_set;
2135         struct blk_mq_hw_ctx *hctx;
2136         int i, ret;
2137
2138         if (!set || nr > set->queue_depth)
2139                 return -EINVAL;
2140
2141         ret = 0;
2142         queue_for_each_hw_ctx(q, hctx, i) {
2143                 ret = blk_mq_tag_update_depth(hctx->tags, nr);
2144                 if (ret)
2145                         break;
2146         }
2147
2148         if (!ret)
2149                 q->nr_requests = nr;
2150
2151         return ret;
2152 }
2153
2154 void blk_mq_disable_hotplug(void)
2155 {
2156         mutex_lock(&all_q_mutex);
2157 }
2158
2159 void blk_mq_enable_hotplug(void)
2160 {
2161         mutex_unlock(&all_q_mutex);
2162 }
2163
2164 static int __init blk_mq_init(void)
2165 {
2166         blk_mq_cpu_init();
2167
2168         hotcpu_notifier(blk_mq_queue_reinit_notify, 0);
2169
2170         return 0;
2171 }
2172 subsys_initcall(blk_mq_init);