block: attempt direct issue of plug list

If we have just one queue type in the plug list, then we can extend our
direct issue to cover a full plug list as well. This allows sending a
batch of requests for direct issue, which is more efficient than doing
one-at-a-time kind of issue.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-core.c b/block/blk-core.c
index 14d2090..e6ad5b5 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1555,6 +1555,7 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios)
        plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);
        plug->rq_count = 0;
        plug->multiple_queues = false;
+       plug->has_elevator = false;
        plug->nowait = false;
        INIT_LIST_HEAD(&plug->cb_list);
 

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 7fa3027..71ab752 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2149,6 +2149,58 @@ void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
        spin_unlock(&ctx->lock);
 }
 
+static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int *queued,
+                             bool from_schedule)
+{
+       if (hctx->queue->mq_ops->commit_rqs) {
+               trace_block_unplug(hctx->queue, *queued, !from_schedule);
+               hctx->queue->mq_ops->commit_rqs(hctx);
+       }
+       *queued = 0;
+}
+
+static void blk_mq_plug_issue_direct(struct blk_plug *plug, bool from_schedule)
+{
+       struct blk_mq_hw_ctx *hctx = NULL;
+       struct request *rq;
+       int queued = 0;
+       int errors = 0;
+
+       while ((rq = rq_list_pop(&plug->mq_list))) {
+               bool last = rq_list_empty(plug->mq_list);
+               blk_status_t ret;
+
+               if (hctx != rq->mq_hctx) {
+                       if (hctx)
+                               blk_mq_commit_rqs(hctx, &queued, from_schedule);
+                       hctx = rq->mq_hctx;
+               }
+
+               ret = blk_mq_request_issue_directly(rq, last);
+               switch (ret) {
+               case BLK_STS_OK:
+                       queued++;
+                       break;
+               case BLK_STS_RESOURCE:
+               case BLK_STS_DEV_RESOURCE:
+                       blk_mq_request_bypass_insert(rq, false, last);
+                       blk_mq_commit_rqs(hctx, &queued, from_schedule);
+                       return;
+               default:
+                       blk_mq_end_request(rq, ret);
+                       errors++;
+                       break;
+               }
+       }
+
+       /*
+        * If we didn't flush the entire list, we could have told the driver
+        * there was more coming, but that turned out to be a lie.
+        */
+       if (errors)
+               blk_mq_commit_rqs(hctx, &queued, from_schedule);
+}
+
 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 {
        struct blk_mq_hw_ctx *this_hctx;
@@ -2160,6 +2212,12 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
                return;
        plug->rq_count = 0;
 
+       if (!plug->multiple_queues && !plug->has_elevator) {
+               blk_mq_plug_issue_direct(plug, from_schedule);
+               if (rq_list_empty(plug->mq_list))
+                       return;
+       }
+
        this_hctx = NULL;
        this_ctx = NULL;
        depth = 0;
@@ -2376,6 +2434,8 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
                if (nxt && nxt->q != rq->q)
                        plug->multiple_queues = true;
        }
+       if (!plug->has_elevator && (rq->rq_flags & RQF_ELV))
+               plug->has_elevator = true;
        rq->rq_next = NULL;
        rq_list_add(&plug->mq_list, rq);
        plug->rq_count++;

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 4027112..f13091d 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -737,6 +737,7 @@ struct blk_plug {
        unsigned short rq_count;
 
        bool multiple_queues;
+       bool has_elevator;
        bool nowait;
 
        struct list_head cb_list; /* md requires an unplug callback */