block, bfq: make shared queues inherit wakers

Consider a bfq_queue bfqq that is about to be merged with another
bfq_queue new_bfqq. The processes associated with bfqq are cooperators
of the processes associated with new_bfqq. So, if bfqq has a waker,
then it is reasonable (and beneficial for throughput) to assume that
all these processes will be happy to let bfqq's waker freely inject
I/O when they have no I/O. So this commit makes new_bfqq inherit
bfqq's waker.

Tested-by: Jan Kara <jack@suse.cz>
Signed-off-by: Paolo Valente <paolo.valente@linaro.org>
Tested-by: Oleksandr Natalenko <oleksandr@natalenko.name>
Link: https://lore.kernel.org/r/20210304174627.161-4-paolo.valente@linaro.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index df840f3..b00dcc4 100644 (file)
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -2823,6 +2823,29 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
        bfq_clear_bfqq_IO_bound(bfqq);
 
        /*
+        * The processes associated with bfqq are cooperators of the
+        * processes associated with new_bfqq. So, if bfqq has a
+        * waker, then assume that all these processes will be happy
+        * to let bfqq's waker freely inject I/O when they have no
+        * I/O.
+        */
+       if (bfqq->waker_bfqq && !new_bfqq->waker_bfqq &&
+           bfqq->waker_bfqq != new_bfqq) {
+               new_bfqq->waker_bfqq = bfqq->waker_bfqq;
+               new_bfqq->tentative_waker_bfqq = NULL;
+
+               /*
+                * If the waker queue disappears, then
+                * new_bfqq->waker_bfqq must be reset. So insert
+                * new_bfqq into the woken_list of the waker. See
+                * bfq_check_waker for details.
+                */
+               hlist_add_head(&new_bfqq->woken_list_node,
+                              &new_bfqq->waker_bfqq->woken_list);
+
+       }
+
+       /*
         * If bfqq is weight-raised, then let new_bfqq inherit
         * weight-raising. To reduce false positives, neglect the case
         * where bfqq has just been created, but has not yet made it
@@ -6310,7 +6333,7 @@ static struct bfq_queue *bfq_init_rq(struct request *rq)
        if (likely(!new_queue)) {
                /* If the queue was seeky for too long, break it apart. */
                if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) {
-                       bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq");
+                       struct bfq_queue *old_bfqq = bfqq;
 
                        /* Update bic before losing reference to bfqq */
                        if (bfq_bfqq_in_large_burst(bfqq))
@@ -6319,11 +6342,24 @@ static struct bfq_queue *bfq_init_rq(struct request *rq)
                        bfqq = bfq_split_bfqq(bic, bfqq);
                        split = true;
 
-                       if (!bfqq)
+                       if (!bfqq) {
                                bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,
                                                                 true, is_sync,
                                                                 NULL);
-                       else
+                               bfqq->waker_bfqq = old_bfqq->waker_bfqq;
+                               bfqq->tentative_waker_bfqq = NULL;
+
+                               /*
+                                * If the waker queue disappears, then
+                                * new_bfqq->waker_bfqq must be
+                                * reset. So insert new_bfqq into the
+                                * woken_list of the waker. See
+                                * bfq_check_waker for details.
+                                */
+                               if (bfqq->waker_bfqq)
+                                       hlist_add_head(&bfqq->woken_list_node,
+                                                      &bfqq->waker_bfqq->woken_list);
+                       } else
                                bfqq_already_existing = true;
                }
        }