block: switch to per-cpu in-flight counters

[uclinux-h8/linux.git] / block / genhd.c

diff --git a/block/genhd.c b/block/genhd.c
index cdf174d..9827a2c 100644 (file)
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -50,9 +50,9 @@ void part_inc_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
        if (queue_is_mq(q))
                return;
 
-       atomic_inc(&part->in_flight[rw]);
+       part_stat_local_inc(part, in_flight[rw]);
        if (part->partno)
-               atomic_inc(&part_to_disk(part)->part0.in_flight[rw]);
+               part_stat_local_inc(&part_to_disk(part)->part0, in_flight[rw]);
 }
 
 void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
@@ -60,38 +60,61 @@ void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
        if (queue_is_mq(q))
                return;
 
-       atomic_dec(&part->in_flight[rw]);
+       part_stat_local_dec(part, in_flight[rw]);
        if (part->partno)
-               atomic_dec(&part_to_disk(part)->part0.in_flight[rw]);
+               part_stat_local_dec(&part_to_disk(part)->part0, in_flight[rw]);
 }
 
 void part_in_flight(struct request_queue *q, struct hd_struct *part,
                    unsigned int inflight[2])
 {
+       int cpu;
+
        if (queue_is_mq(q)) {
                blk_mq_in_flight(q, part, inflight);
                return;
        }
 
-       inflight[0] = atomic_read(&part->in_flight[0]) +
-                       atomic_read(&part->in_flight[1]);
+       inflight[0] = 0;
+       for_each_possible_cpu(cpu) {
+               inflight[0] += part_stat_local_read_cpu(part, in_flight[0], cpu) +
+                              part_stat_local_read_cpu(part, in_flight[1], cpu);
+       }
+       if ((int)inflight[0] < 0)
+               inflight[0] = 0;
+
        if (part->partno) {
                part = &part_to_disk(part)->part0;
-               inflight[1] = atomic_read(&part->in_flight[0]) +
-                               atomic_read(&part->in_flight[1]);
+               inflight[1] = 0;
+               for_each_possible_cpu(cpu) {
+                       inflight[1] += part_stat_local_read_cpu(part, in_flight[0], cpu) +
+                                      part_stat_local_read_cpu(part, in_flight[1], cpu);
+               }
+               if ((int)inflight[1] < 0)
+                       inflight[1] = 0;
        }
 }
 
 void part_in_flight_rw(struct request_queue *q, struct hd_struct *part,
                       unsigned int inflight[2])
 {
+       int cpu;
+
        if (queue_is_mq(q)) {
                blk_mq_in_flight_rw(q, part, inflight);
                return;
        }
 
-       inflight[0] = atomic_read(&part->in_flight[0]);
-       inflight[1] = atomic_read(&part->in_flight[1]);
+       inflight[0] = 0;
+       inflight[1] = 0;
+       for_each_possible_cpu(cpu) {
+               inflight[0] += part_stat_local_read_cpu(part, in_flight[0], cpu);
+               inflight[1] += part_stat_local_read_cpu(part, in_flight[1], cpu);
+       }
+       if ((int)inflight[0] < 0)
+               inflight[0] = 0;
+       if ((int)inflight[1] < 0)
+               inflight[1] = 0;
 }
 
 struct hd_struct *__disk_get_part(struct gendisk *disk, int partno)