refcount_t ref;
unsigned flags;
struct hlist_nulls_node nulls_node;
+ struct list_head all_list;
struct task_struct *task;
wait_queue_head_t wait;
struct io_wqe *wqe;
struct io_wq_nulls_list free_list;
struct io_wq_nulls_list busy_list;
+ struct list_head all_list;
struct io_wq *wq;
};
spin_lock_irq(&wqe->lock);
hlist_nulls_del_rcu(&worker->nulls_node);
+ list_del_rcu(&worker->all_list);
if (__io_worker_unuse(wqe, worker)) {
__release(&wqe->lock);
spin_lock_irq(&wqe->lock);
spin_lock_irq(&wqe->lock);
hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list.head);
+ list_add_tail_rcu(&worker->all_list, &wqe->all_list);
worker->flags |= IO_WORKER_F_FREE;
if (index == IO_WQ_ACCT_BOUND)
worker->flags |= IO_WORKER_F_BOUND;
* worker that isn't exiting
*/
static bool io_wq_for_each_worker(struct io_wqe *wqe,
- struct io_wq_nulls_list *list,
bool (*func)(struct io_worker *, void *),
void *data)
{
- struct hlist_nulls_node *n;
struct io_worker *worker;
bool ret = false;
-restart:
- hlist_nulls_for_each_entry_rcu(worker, n, &list->head, nulls_node) {
+ list_for_each_entry_rcu(worker, &wqe->all_list, all_list) {
if (io_worker_get(worker)) {
ret = func(worker, data);
io_worker_release(worker);
break;
}
}
- if (!ret && get_nulls_value(n) != list->nulls)
- goto restart;
+
return ret;
}
for (i = 0; i < wq->nr_wqes; i++) {
struct io_wqe *wqe = wq->wqes[i];
- io_wq_for_each_worker(wqe, &wqe->busy_list,
- io_wqe_worker_send_sig, NULL);
- io_wq_for_each_worker(wqe, &wqe->free_list,
- io_wqe_worker_send_sig, NULL);
+ io_wq_for_each_worker(wqe, io_wqe_worker_send_sig, NULL);
}
rcu_read_unlock();
}
}
rcu_read_lock();
- found = io_wq_for_each_worker(wqe, &wqe->free_list, io_work_cancel,
- &data);
- if (found)
- goto done;
-
- found = io_wq_for_each_worker(wqe, &wqe->busy_list, io_work_cancel,
- &data);
-done:
+ found = io_wq_for_each_worker(wqe, io_work_cancel, &data);
rcu_read_unlock();
return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
}
* completion will run normally in this case.
*/
rcu_read_lock();
- found = io_wq_for_each_worker(wqe, &wqe->free_list, io_wq_worker_cancel,
- cwork);
- if (found)
- goto done;
-
- found = io_wq_for_each_worker(wqe, &wqe->busy_list, io_wq_worker_cancel,
- cwork);
-done:
+ found = io_wq_for_each_worker(wqe, io_wq_worker_cancel, cwork);
rcu_read_unlock();
return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
}
wqe->free_list.nulls = 0;
INIT_HLIST_NULLS_HEAD(&wqe->busy_list.head, 1);
wqe->busy_list.nulls = 1;
+ INIT_LIST_HEAD(&wqe->all_list);
i++;
}
if (!wqe)
continue;
- io_wq_for_each_worker(wqe, &wqe->free_list, io_wq_worker_wake,
- NULL);
- io_wq_for_each_worker(wqe, &wqe->busy_list, io_wq_worker_wake,
- NULL);
+ io_wq_for_each_worker(wqe, io_wq_worker_wake, NULL);
}
rcu_read_unlock();