blk-mq: enable checking two part inflight counts at the same time

Modify blk_mq_in_flight() to count both a partition and root at
the same time. Then we only have to call it once, instead of
potentially looping the tags twice.

Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-core.c b/block/blk-core.c
index 6ad2b86..d836c84 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1470,17 +1470,12 @@ static void add_acct_request(struct request_queue *q, struct request *rq,
 }
 
 static void part_round_stats_single(struct request_queue *q, int cpu,
-                                   struct hd_struct *part, unsigned long now)
+                                   struct hd_struct *part, unsigned long now,
+                                   unsigned int inflight)
 {
-       int inflight[2];
-
-       if (now == part->stamp)
-               return;
-
-       part_in_flight(q, part, inflight);
-       if (inflight[0]) {
+       if (inflight) {
                __part_stat_add(cpu, part, time_in_queue,
-                               inflight[0] * (now - part->stamp));
+                               inflight * (now - part->stamp));
                __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
        }
        part->stamp = now;
@@ -1505,12 +1500,29 @@ static void part_round_stats_single(struct request_queue *q, int cpu,
  */
 void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part)
 {
+       struct hd_struct *part2 = NULL;
        unsigned long now = jiffies;
+       unsigned int inflight[2];
+       int stats = 0;
+
+       if (part->stamp != now)
+               stats |= 1;
+
+       if (part->partno) {
+               part2 = &part_to_disk(part)->part0;
+               if (part2->stamp != now)
+                       stats |= 2;
+       }
+
+       if (!stats)
+               return;
+
+       part_in_flight(q, part, inflight);
 
-       if (part->partno)
-               part_round_stats_single(q, cpu, &part_to_disk(part)->part0,
-                                               now);
-       part_round_stats_single(q, cpu, part, now);
+       if (stats & 2)
+               part_round_stats_single(q, cpu, part2, now, inflight[1]);
+       if (stats & 1)
+               part_round_stats_single(q, cpu, part, now, inflight[0]);
 }
 EXPORT_SYMBOL_GPL(part_round_stats);
 

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 0dfc7a9..fe764ca 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -97,11 +97,15 @@ static void blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
        if (test_bit(REQ_ATOM_STARTED, &rq->atomic_flags) &&
            !test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) {
                /*
-                * Count as inflight if it either matches the partition we
-                * asked for, or if it's the root
+                * index[0] counts the specific partition that was asked
+                * for. index[1] counts the ones that are active on the
+                * whole device, so increment that if mi->part is indeed
+                * a partition, and not a whole device.
                 */
-               if (rq->part == mi->part || mi->part->partno)
+               if (rq->part == mi->part)
                        mi->inflight[0]++;
+               if (mi->part->partno)
+                       mi->inflight[1]++;
        }
 }
 
@@ -110,7 +114,7 @@ void blk_mq_in_flight(struct request_queue *q, struct hd_struct *part,
 {
        struct mq_inflight mi = { .part = part, .inflight = inflight, };
 
-       inflight[0] = 0;
+       inflight[0] = inflight[1] = 0;
        blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
 }