Merge branch 'uaccess.csum' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs

[tomoyo/tomoyo-test1.git] / block / blk-core.c

diff --git a/block/blk-core.c b/block/blk-core.c
index d8df56e..03252af 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -20,6 +20,7 @@
 #include <linux/blk-mq.h>
 #include <linux/highmem.h>
 #include <linux/mm.h>
+#include <linux/pagemap.h>
 #include <linux/kernel_stat.h>
 #include <linux/string.h>
 #include <linux/init.h>
@@ -636,6 +637,16 @@ void blk_put_request(struct request *req)
 }
 EXPORT_SYMBOL(blk_put_request);
 
+static void blk_account_io_merge_bio(struct request *req)
+{
+       if (!blk_do_io_stat(req))
+               return;
+
+       part_stat_lock();
+       part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
+       part_stat_unlock();
+}
+
 bool bio_attempt_back_merge(struct request *req, struct bio *bio,
                unsigned int nr_segs)
 {
@@ -656,7 +667,7 @@ bool bio_attempt_back_merge(struct request *req, struct bio *bio,
 
        bio_crypt_free_ctx(bio);
 
-       blk_account_io_start(req, false);
+       blk_account_io_merge_bio(req);
        return true;
 }
 
@@ -682,7 +693,7 @@ bool bio_attempt_front_merge(struct request *req, struct bio *bio,
 
        bio_crypt_do_front_merge(req, bio);
 
-       blk_account_io_start(req, false);
+       blk_account_io_merge_bio(req);
        return true;
 }
 
@@ -704,7 +715,7 @@ bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
        req->__data_len += bio->bi_iter.bi_size;
        req->nr_phys_segments = segments + 1;
 
-       blk_account_io_start(req, false);
+       blk_account_io_merge_bio(req);
        return true;
 no_merge:
        req_set_nomerge(q, req);
@@ -961,14 +972,11 @@ generic_make_request_checks(struct bio *bio)
        }
 
        /*
-        * Non-mq queues do not honor REQ_NOWAIT, so complete a bio
-        * with BLK_STS_AGAIN status in order to catch -EAGAIN and
-        * to give a chance to the caller to repeat request gracefully.
+        * For a REQ_NOWAIT based request, return -EOPNOTSUPP
+        * if queue is not a request based queue.
         */
-       if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_mq(q)) {
-               status = BLK_STS_AGAIN;
-               goto end_io;
-       }
+       if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_mq(q))
+               goto not_supported;
 
        if (should_fail_bio(bio))
                goto end_io;
@@ -1329,7 +1337,7 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *
                return BLK_STS_IOERR;
 
        if (blk_queue_io_stat(q))
-               blk_account_io_start(rq, true);
+               blk_account_io_start(rq);
 
        /*
         * Since we have a scheduler attached on the top device,
@@ -1381,6 +1389,21 @@ unsigned int blk_rq_err_bytes(const struct request *rq)
 }
 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
 
+static void update_io_ticks(struct hd_struct *part, unsigned long now, bool end)
+{
+       unsigned long stamp;
+again:
+       stamp = READ_ONCE(part->stamp);
+       if (unlikely(stamp != now)) {
+               if (likely(cmpxchg(&part->stamp, stamp, now) == stamp))
+                       __part_stat_add(part, io_ticks, end ? now - stamp : 1);
+       }
+       if (part->partno) {
+               part = &part_to_disk(part)->part0;
+               goto again;
+       }
+}
+
 static void blk_account_io_completion(struct request *req, unsigned int bytes)
 {
        if (req->part && blk_do_io_stat(req)) {
@@ -1412,36 +1435,57 @@ void blk_account_io_done(struct request *req, u64 now)
                update_io_ticks(part, jiffies, true);
                part_stat_inc(part, ios[sgrp]);
                part_stat_add(part, nsecs[sgrp], now - req->start_time_ns);
-               part_dec_in_flight(req->q, part, rq_data_dir(req));
+               part_stat_unlock();
 
                hd_struct_put(part);
-               part_stat_unlock();
        }
 }
 
-void blk_account_io_start(struct request *rq, bool new_io)
+void blk_account_io_start(struct request *rq)
 {
-       struct hd_struct *part;
-       int rw = rq_data_dir(rq);
-
        if (!blk_do_io_stat(rq))
                return;
 
+       rq->part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
+
        part_stat_lock();
+       update_io_ticks(rq->part, jiffies, false);
+       part_stat_unlock();
+}
 
-       if (!new_io) {
-               part = rq->part;
-               part_stat_inc(part, merges[rw]);
-       } else {
-               part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
-               part_inc_in_flight(rq->q, part, rw);
-               rq->part = part;
-       }
+unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
+               unsigned int op)
+{
+       struct hd_struct *part = &disk->part0;
+       const int sgrp = op_stat_group(op);
+       unsigned long now = READ_ONCE(jiffies);
 
-       update_io_ticks(part, jiffies, false);
+       part_stat_lock();
+       update_io_ticks(part, now, false);
+       part_stat_inc(part, ios[sgrp]);
+       part_stat_add(part, sectors[sgrp], sectors);
+       part_stat_local_inc(part, in_flight[op_is_write(op)]);
+       part_stat_unlock();
+
+       return now;
+}
+EXPORT_SYMBOL(disk_start_io_acct);
 
+void disk_end_io_acct(struct gendisk *disk, unsigned int op,
+               unsigned long start_time)
+{
+       struct hd_struct *part = &disk->part0;
+       const int sgrp = op_stat_group(op);
+       unsigned long now = READ_ONCE(jiffies);
+       unsigned long duration = now - start_time;
+
+       part_stat_lock();
+       update_io_ticks(part, now, true);
+       part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration));
+       part_stat_local_dec(part, in_flight[op_is_write(op)]);
        part_stat_unlock();
 }
+EXPORT_SYMBOL(disk_end_io_acct);
 
 /*
  * Steal bios from a request and add them to a bio list.