io-wq: shuffle io_worker_handle_work() code

This is a preparation patch, it adds some helpers and makes
the next patches cleaner.

- extract io_impersonate_work() and io_assign_current_work()
- replace @next label with nested do-while
- move put_work() right after NULL'ing cur_work.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index 042c7e2..e438dc4 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -440,14 +440,43 @@ static void io_wq_switch_creds(struct io_worker *worker,
                worker->saved_creds = old_creds;
 }
 
+static void io_impersonate_work(struct io_worker *worker,
+                               struct io_wq_work *work)
+{
+       if (work->files && current->files != work->files) {
+               task_lock(current);
+               current->files = work->files;
+               task_unlock(current);
+       }
+       if (work->fs && current->fs != work->fs)
+               current->fs = work->fs;
+       if (work->mm != worker->mm)
+               io_wq_switch_mm(worker, work);
+       if (worker->cur_creds != work->creds)
+               io_wq_switch_creds(worker, work);
+}
+
+static void io_assign_current_work(struct io_worker *worker,
+                                  struct io_wq_work *work)
+{
+       /* flush pending signals before assigning new work */
+       if (signal_pending(current))
+               flush_signals(current);
+       cond_resched();
+
+       spin_lock_irq(&worker->lock);
+       worker->cur_work = work;
+       spin_unlock_irq(&worker->lock);
+}
+
 static void io_worker_handle_work(struct io_worker *worker)
        __releases(wqe->lock)
 {
-       struct io_wq_work *work, *old_work = NULL, *put_work = NULL;
        struct io_wqe *wqe = worker->wqe;
        struct io_wq *wq = wqe->wq;
 
        do {
+               struct io_wq_work *work, *old_work;
                unsigned hash = -1U;
 
                /*
@@ -464,69 +493,45 @@ static void io_worker_handle_work(struct io_worker *worker)
                        wqe->flags |= IO_WQE_FLAG_STALLED;
 
                spin_unlock_irq(&wqe->lock);
-               if (put_work && wq->put_work)
-                       wq->put_work(old_work);
                if (!work)
                        break;
-next:
-               /* flush any pending signals before assigning new work */
-               if (signal_pending(current))
-                       flush_signals(current);
-
-               cond_resched();
 
-               spin_lock_irq(&worker->lock);
-               worker->cur_work = work;
-               spin_unlock_irq(&worker->lock);
-
-               if (work->files && current->files != work->files) {
-                       task_lock(current);
-                       current->files = work->files;
-                       task_unlock(current);
-               }
-               if (work->fs && current->fs != work->fs)
-                       current->fs = work->fs;
-               if (work->mm != worker->mm)
-                       io_wq_switch_mm(worker, work);
-               if (worker->cur_creds != work->creds)
-                       io_wq_switch_creds(worker, work);
-               /*
-                * OK to set IO_WQ_WORK_CANCEL even for uncancellable work,
-                * the worker function will do the right thing.
-                */
-               if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))
-                       work->flags |= IO_WQ_WORK_CANCEL;
-
-               if (wq->get_work) {
-                       put_work = work;
-                       wq->get_work(work);
-               }
-
-               old_work = work;
-               work->func(&work);
-
-               spin_lock_irq(&worker->lock);
-               worker->cur_work = NULL;
-               spin_unlock_irq(&worker->lock);
-
-               spin_lock_irq(&wqe->lock);
-
-               if (hash != -1U) {
-                       wqe->hash_map &= ~BIT(hash);
-                       wqe->flags &= ~IO_WQE_FLAG_STALLED;
-               }
-               if (work && work != old_work) {
-                       spin_unlock_irq(&wqe->lock);
-
-                       if (put_work && wq->put_work) {
-                               wq->put_work(put_work);
-                               put_work = NULL;
+               /* handle a whole dependent link */
+               do {
+                       io_assign_current_work(worker, work);
+                       io_impersonate_work(worker, work);
+
+                       /*
+                        * OK to set IO_WQ_WORK_CANCEL even for uncancellable
+                        * work, the worker function will do the right thing.
+                        */
+                       if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))
+                               work->flags |= IO_WQ_WORK_CANCEL;
+
+                       if (wq->get_work)
+                               wq->get_work(work);
+
+                       old_work = work;
+                       work->func(&work);
+
+                       spin_lock_irq(&worker->lock);
+                       worker->cur_work = NULL;
+                       spin_unlock_irq(&worker->lock);
+
+                       if (wq->put_work)
+                               wq->put_work(old_work);
+
+                       if (hash != -1U) {
+                               spin_lock_irq(&wqe->lock);
+                               wqe->hash_map &= ~BIT_ULL(hash);
+                               wqe->flags &= ~IO_WQE_FLAG_STALLED;
+                               spin_unlock_irq(&wqe->lock);
+                               /* dependent work is not hashed */
+                               hash = -1U;
                        }
+               } while (work && work != old_work);
 
-                       /* dependent work not hashed */
-                       hash = -1U;
-                       goto next;
-               }
+               spin_lock_irq(&wqe->lock);
        } while (1);
 }