SUNRPC: Fix priority queue fairness

[sagit-ice-cold/kernel_xiaomi_msm8998.git] / net / sunrpc / sched.c

diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
index 1cb35c7..3eed71a 100644 (file)
--- a/net/sunrpc/sched.c
+++ b/net/sunrpc/sched.c
@@ -98,65 +98,79 @@ __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
        list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
 }
 
-static void rpc_rotate_queue_owner(struct rpc_wait_queue *queue)
-{
-       struct list_head *q = &queue->tasks[queue->priority];
-       struct rpc_task *task;
-
-       if (!list_empty(q)) {
-               task = list_first_entry(q, struct rpc_task, u.tk_wait.list);
-               if (task->tk_owner == queue->owner)
-                       list_move_tail(&task->u.tk_wait.list, q);
-       }
-}
-
 static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
 {
        if (queue->priority != priority) {
-               /* Fairness: rotate the list when changing priority */
-               rpc_rotate_queue_owner(queue);
                queue->priority = priority;
+               queue->nr = 1U << priority;
        }
 }
 
-static void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
-{
-       queue->owner = pid;
-       queue->nr = RPC_BATCH_COUNT;
-}
-
 static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
 {
        rpc_set_waitqueue_priority(queue, queue->maxpriority);
-       rpc_set_waitqueue_owner(queue, 0);
 }
 
 /*
- * Add new request to a priority queue.
+ * Add a request to a queue list
  */
-static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,
-               struct rpc_task *task,
-               unsigned char queue_priority)
+static void
+__rpc_list_enqueue_task(struct list_head *q, struct rpc_task *task)
 {
-       struct list_head *q;
        struct rpc_task *t;
 
-       INIT_LIST_HEAD(&task->u.tk_wait.links);
-       if (unlikely(queue_priority > queue->maxpriority))
-               queue_priority = queue->maxpriority;
-       if (queue_priority > queue->priority)
-               rpc_set_waitqueue_priority(queue, queue_priority);
-       q = &queue->tasks[queue_priority];
        list_for_each_entry(t, q, u.tk_wait.list) {
                if (t->tk_owner == task->tk_owner) {
-                       list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
+                       list_add_tail(&task->u.tk_wait.links,
+                                       &t->u.tk_wait.links);
+                       /* Cache the queue head in task->u.tk_wait.list */
+                       task->u.tk_wait.list.next = q;
+                       task->u.tk_wait.list.prev = NULL;
                        return;
                }
        }
+       INIT_LIST_HEAD(&task->u.tk_wait.links);
        list_add_tail(&task->u.tk_wait.list, q);
 }
 
 /*
+ * Remove request from a queue list
+ */
+static void
+__rpc_list_dequeue_task(struct rpc_task *task)
+{
+       struct list_head *q;
+       struct rpc_task *t;
+
+       if (task->u.tk_wait.list.prev == NULL) {
+               list_del(&task->u.tk_wait.links);
+               return;
+       }
+       if (!list_empty(&task->u.tk_wait.links)) {
+               t = list_first_entry(&task->u.tk_wait.links,
+                               struct rpc_task,
+                               u.tk_wait.links);
+               /* Assume __rpc_list_enqueue_task() cached the queue head */
+               q = t->u.tk_wait.list.next;
+               list_add_tail(&t->u.tk_wait.list, q);
+               list_del(&task->u.tk_wait.links);
+       }
+       list_del(&task->u.tk_wait.list);
+}
+
+/*
+ * Add new request to a priority queue.
+ */
+static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,
+               struct rpc_task *task,
+               unsigned char queue_priority)
+{
+       if (unlikely(queue_priority > queue->maxpriority))
+               queue_priority = queue->maxpriority;
+       __rpc_list_enqueue_task(&queue->tasks[queue_priority], task);
+}
+
+/*
  * Add new request to wait queue.
  *
  * Swapper tasks always get inserted at the head of the queue.
@@ -193,13 +207,7 @@ static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,
  */
 static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
 {
-       struct rpc_task *t;
-
-       if (!list_empty(&task->u.tk_wait.links)) {
-               t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
-               list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
-               list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
-       }
+       __rpc_list_dequeue_task(task);
 }
 
 /*
@@ -211,7 +219,8 @@ static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_tas
        __rpc_disable_timer(queue, task);
        if (RPC_IS_PRIORITY(queue))
                __rpc_remove_wait_queue_priority(task);
-       list_del(&task->u.tk_wait.list);
+       else
+               list_del(&task->u.tk_wait.list);
        queue->qlen--;
        dprintk("RPC: %5u removed from queue %p \"%s\"\n",
                        task->tk_pid, queue, rpc_qname(queue));
@@ -467,17 +476,9 @@ static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *q
         * Service a batch of tasks from a single owner.
         */
        q = &queue->tasks[queue->priority];
-       if (!list_empty(q)) {
-               task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
-               if (queue->owner == task->tk_owner) {
-                       if (--queue->nr)
-                               goto out;
-                       list_move_tail(&task->u.tk_wait.list, q);
-               }
-               /*
-                * Check if we need to switch queues.
-                */
-               goto new_owner;
+       if (!list_empty(q) && --queue->nr) {
+               task = list_first_entry(q, struct rpc_task, u.tk_wait.list);
+               goto out;
        }
 
        /*
@@ -489,7 +490,7 @@ static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *q
                else
                        q = q - 1;
                if (!list_empty(q)) {
-                       task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
+                       task = list_first_entry(q, struct rpc_task, u.tk_wait.list);
                        goto new_queue;
                }
        } while (q != &queue->tasks[queue->priority]);
@@ -499,8 +500,6 @@ static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *q
 
 new_queue:
        rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
-new_owner:
-       rpc_set_waitqueue_owner(queue, task->tk_owner);
 out:
        return task;
 }