io_uring: clear IORING_SQ_NEED_WAKEUP after executing task works

In io_sq_thread(), if there are task works to handle, current codes
will skip schedule() and go on polling sq again, but forget to clear
IORING_SQ_NEED_WAKEUP flag, fix this issue. Also add two helpers to
set and clear IORING_SQ_NEED_WAKEUP flag,

Signed-off-by: Xiaoguang Wang <xiaoguang.wang@linux.alibaba.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 3a415d9..6f3f18a 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -6344,6 +6344,21 @@ fail_req:
        return submitted;
 }
 
+static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
+{
+       /* Tell userspace we may need a wakeup call */
+       spin_lock_irq(&ctx->completion_lock);
+       ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
+       spin_unlock_irq(&ctx->completion_lock);
+}
+
+static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
+{
+       spin_lock_irq(&ctx->completion_lock);
+       ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
+       spin_unlock_irq(&ctx->completion_lock);
+}
+
 static int io_sq_thread(void *data)
 {
        struct io_ring_ctx *ctx = data;
@@ -6417,10 +6432,7 @@ static int io_sq_thread(void *data)
                                continue;
                        }
 
-                       /* Tell userspace we may need a wakeup call */
-                       spin_lock_irq(&ctx->completion_lock);
-                       ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
-                       spin_unlock_irq(&ctx->completion_lock);
+                       io_ring_set_wakeup_flag(ctx);
 
                        to_submit = io_sqring_entries(ctx);
                        if (!to_submit || ret == -EBUSY) {
@@ -6430,6 +6442,7 @@ static int io_sq_thread(void *data)
                                }
                                if (io_run_task_work()) {
                                        finish_wait(&ctx->sqo_wait, &wait);
+                                       io_ring_clear_wakeup_flag(ctx);
                                        continue;
                                }
                                if (signal_pending(current))
@@ -6437,17 +6450,13 @@ static int io_sq_thread(void *data)
                                schedule();
                                finish_wait(&ctx->sqo_wait, &wait);
 
-                               spin_lock_irq(&ctx->completion_lock);
-                               ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
-                               spin_unlock_irq(&ctx->completion_lock);
+                               io_ring_clear_wakeup_flag(ctx);
                                ret = 0;
                                continue;
                        }
                        finish_wait(&ctx->sqo_wait, &wait);
 
-                       spin_lock_irq(&ctx->completion_lock);
-                       ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
-                       spin_unlock_irq(&ctx->completion_lock);
+                       io_ring_clear_wakeup_flag(ctx);
                }
 
                mutex_lock(&ctx->uring_lock);