kevin/linux
1
Fork 0
Code

io_uring/napi: ensure napi polling is aborted when work is available

Browse Source 
While testing io_uring NAPI with DEFER_TASKRUN, I ran into slowdowns and
stalls in packet delivery. Turns out that while
io_napi_busy_loop_should_end() aborts appropriately on regular
task_work, it does not abort if we have local task_work pending.

Move io_has_work() into the private io_uring.h header, and gate whether
we should continue polling on that as well. This makes NAPI polling on
send/receive work as designed with IORING_SETUP_DEFER_TASKRUN as well.

Fixes: 8d0c12a80c ("io-uring: add napi busy poll support")
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2024-02-14 12:59:36 -07:00
parent 3fb1764c6b
commit 428f138268
3 changed files with 12 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
io_uring/io_uring.c
View File

@ -122,11 +122,6 @@
#define IO_COMPL_BATCH 32 #define IO_COMPL_BATCH 32
#define IO_REQ_ALLOC_BATCH 8 #define IO_REQ_ALLOC_BATCH 8
enum {
IO_CHECK_CQ_OVERFLOW_BIT,
IO_CHECK_CQ_DROPPED_BIT,
};
struct io_defer_entry { struct io_defer_entry {
struct list_head list; struct list_head list;
struct io_kiocb *req; struct io_kiocb *req;
@ -2479,12 +2474,6 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
return ret; return ret;
} }
static inline bool io_has_work(struct io_ring_ctx *ctx)
{
return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
!llist_empty(&ctx->work_llist);
}
static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
int wake_flags, void *key) int wake_flags, void *key)
{ {

11
io_uring/io_uring.h
View File

@ -448,4 +448,15 @@ static inline bool io_file_can_poll(struct io_kiocb *req)
} }
return false; return false;
} }
enum {
IO_CHECK_CQ_OVERFLOW_BIT,
IO_CHECK_CQ_DROPPED_BIT,
};
static inline bool io_has_work(struct io_ring_ctx *ctx)
{
return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
!llist_empty(&ctx->work_llist);
}
#endif #endif

2
io_uring/napi.c
View File

@ -122,7 +122,7 @@ static bool io_napi_busy_loop_should_end(void *data,
if (signal_pending(current)) if (signal_pending(current))
return true; return true;
if (io_should_wake(iowq)) if (io_should_wake(iowq) || io_has_work(iowq->ctx))
return true; return true;
if (io_napi_busy_loop_timeout(start_time, iowq->napi_busy_poll_to)) if (io_napi_busy_loop_timeout(start_time, iowq->napi_busy_poll_to))
return true; return true;