blk-mq: factor out a blk_mq_complete_need_ipi helper

Add a helper to decide if we can complete locally or need an IPI.

Reviewed-by: Daniel Wagner <dwagner@suse.de>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 95125bf..961635b 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -654,6 +654,24 @@ static void __blk_mq_complete_request_remote(void *data)
        __blk_mq_complete_request(data);
 }
 
+static inline bool blk_mq_complete_need_ipi(struct request *rq)
+{
+       int cpu = raw_smp_processor_id();
+
+       if (!IS_ENABLED(CONFIG_SMP) ||
+           !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
+               return false;
+
+       /* same CPU or cache domain?  Complete locally */
+       if (cpu == rq->mq_ctx->cpu ||
+           (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
+            cpus_share_cache(cpu, rq->mq_ctx->cpu)))
+               return false;
+
+       /* don't try to IPI to an offline CPU */
+       return cpu_online(rq->mq_ctx->cpu);
+}
+
 /**
  * blk_mq_complete_request - end I/O on a request
  * @rq:                the request being processed
@@ -663,11 +681,6 @@ static void __blk_mq_complete_request_remote(void *data)
  **/
 void blk_mq_complete_request(struct request *rq)
 {
-       struct blk_mq_ctx *ctx = rq->mq_ctx;
-       struct request_queue *q = rq->q;
-       bool shared = false;
-       int cpu;
-
        WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
 
        /*
@@ -675,25 +688,15 @@ void blk_mq_complete_request(struct request *rq)
         * to redirect the completion.
         */
        if (rq->cmd_flags & REQ_HIPRI) {
-               q->mq_ops->complete(rq);
-               return;
-       }
-
-       if (!IS_ENABLED(CONFIG_SMP) ||
-           !test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags)) {
-               __blk_mq_complete_request(rq);
+               rq->q->mq_ops->complete(rq);
                return;
        }
 
-       cpu = raw_smp_processor_id();
-       if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags))
-               shared = cpus_share_cache(cpu, ctx->cpu);
-
-       if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
+       if (blk_mq_complete_need_ipi(rq)) {
                rq->csd.func = __blk_mq_complete_request_remote;
                rq->csd.info = rq;
                rq->csd.flags = 0;
-               smp_call_function_single_async(ctx->cpu, &rq->csd);
+               smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd);
        } else {
                __blk_mq_complete_request(rq);
        }