kevin/linux
1
Fork 0
Code

io_uring: add remote task_work execution helper

Browse Source 
All our task_work handling is targeted at the state in the io_kiocb
itself, which is what it is being used for. However, MSG_RING rolls its
own task_work handling, ignoring how that is usually done.

In preparation for switching MSG_RING to be able to use the normal
task_work handling, add io_req_task_work_add_remote() which allows the
caller to pass in the target io_ring_ctx.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2024-03-28 12:38:44 -06:00
parent d57afd8bb7
commit c3ac76f9ca
2 changed files with 18 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
io_uring/io_uring.c
View File

@ -1098,9 +1098,10 @@ void tctx_task_work(struct callback_head *cb)
WARN_ON_ONCE(ret); WARN_ON_ONCE(ret);
} }
static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags) static inline void io_req_local_work_add(struct io_kiocb *req,
struct io_ring_ctx *ctx,
unsigned flags)
{ {
struct io_ring_ctx *ctx = req->ctx;
unsigned nr_wait, nr_tw, nr_tw_prev; unsigned nr_wait, nr_tw, nr_tw_prev;
struct llist_node *head; struct llist_node *head;
@ -1114,6 +1115,8 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK))
flags &= ~IOU_F_TWQ_LAZY_WAKE; flags &= ~IOU_F_TWQ_LAZY_WAKE;
guard(rcu)();
head = READ_ONCE(ctx->work_llist.first); head = READ_ONCE(ctx->work_llist.first);
do { do {
nr_tw_prev = 0; nr_tw_prev = 0;
@ -1195,13 +1198,18 @@ static void io_req_normal_work_add(struct io_kiocb *req)
void __io_req_task_work_add(struct io_kiocb *req, unsigned flags) void __io_req_task_work_add(struct io_kiocb *req, unsigned flags)
{ {
if (req->ctx->flags & IORING_SETUP_DEFER_TASKRUN) { if (req->ctx->flags & IORING_SETUP_DEFER_TASKRUN)
rcu_read_lock(); io_req_local_work_add(req, req->ctx, flags);
io_req_local_work_add(req, flags); else
rcu_read_unlock();
} else {
io_req_normal_work_add(req); io_req_normal_work_add(req);
} }
void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx,
unsigned flags)
{
if (WARN_ON_ONCE(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)))
return;
io_req_local_work_add(req, ctx, flags);
} }
static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)

2
io_uring/io_uring.h
View File

@ -73,6 +73,8 @@ struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
unsigned issue_flags); unsigned issue_flags);
void __io_req_task_work_add(struct io_kiocb *req, unsigned flags); void __io_req_task_work_add(struct io_kiocb *req, unsigned flags);
void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx,
unsigned flags);
bool io_alloc_async_data(struct io_kiocb *req); bool io_alloc_async_data(struct io_kiocb *req);
void io_req_task_queue(struct io_kiocb *req); void io_req_task_queue(struct io_kiocb *req);
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts); void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts);