io_uring/eventfd: move eventfd handling to separate file
This is pretty nicely abstracted already, but let's move it to a separate file rather than have it in the main io_uring file. With that, we can also move the io_ev_fd struct and enum out of global scope. Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe
parent
60b6c075e8
commit
200f3abd14
@ -211,14 +211,6 @@ struct io_submit_state {
|
||||
struct blk_plug plug;
|
||||
};
|
||||
|
||||
struct io_ev_fd {
|
||||
struct eventfd_ctx *cq_ev_fd;
|
||||
unsigned int eventfd_async: 1;
|
||||
struct rcu_head rcu;
|
||||
atomic_t refs;
|
||||
atomic_t ops;
|
||||
};
|
||||
|
||||
struct io_alloc_cache {
|
||||
void **entries;
|
||||
unsigned int nr_cached;
|
||||
|
||||
@ -4,9 +4,9 @@
|
||||
|
||||
obj-$(CONFIG_IO_URING) += io_uring.o opdef.o kbuf.o rsrc.o notif.o \
|
||||
tctx.o filetable.o rw.o net.o poll.o \
|
||||
uring_cmd.o openclose.o sqpoll.o \
|
||||
xattr.o nop.o fs.o splice.o sync.o \
|
||||
msg_ring.o advise.o openclose.o \
|
||||
eventfd.o uring_cmd.o openclose.o \
|
||||
sqpoll.o xattr.o nop.o fs.o splice.o \
|
||||
sync.o msg_ring.o advise.o openclose.o \
|
||||
epoll.o statx.o timeout.o fdinfo.o \
|
||||
cancel.o waitid.o register.o \
|
||||
truncate.o memmap.o
|
||||
|
||||
160
io_uring/eventfd.c
Normal file
160
io_uring/eventfd.c
Normal file
@ -0,0 +1,160 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/errno.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/eventfd.h>
|
||||
#include <linux/eventpoll.h>
|
||||
#include <linux/io_uring.h>
|
||||
#include <linux/io_uring_types.h>
|
||||
|
||||
#include "io-wq.h"
|
||||
#include "eventfd.h"
|
||||
|
||||
struct io_ev_fd {
|
||||
struct eventfd_ctx *cq_ev_fd;
|
||||
unsigned int eventfd_async: 1;
|
||||
struct rcu_head rcu;
|
||||
atomic_t refs;
|
||||
atomic_t ops;
|
||||
};
|
||||
|
||||
enum {
|
||||
IO_EVENTFD_OP_SIGNAL_BIT,
|
||||
};
|
||||
|
||||
static void io_eventfd_free(struct rcu_head *rcu)
|
||||
{
|
||||
struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
|
||||
|
||||
eventfd_ctx_put(ev_fd->cq_ev_fd);
|
||||
kfree(ev_fd);
|
||||
}
|
||||
|
||||
static void io_eventfd_do_signal(struct rcu_head *rcu)
|
||||
{
|
||||
struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
|
||||
|
||||
eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
|
||||
|
||||
if (atomic_dec_and_test(&ev_fd->refs))
|
||||
io_eventfd_free(rcu);
|
||||
}
|
||||
|
||||
void io_eventfd_signal(struct io_ring_ctx *ctx)
|
||||
{
|
||||
struct io_ev_fd *ev_fd = NULL;
|
||||
|
||||
if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
|
||||
return;
|
||||
|
||||
guard(rcu)();
|
||||
|
||||
/*
|
||||
* rcu_dereference ctx->io_ev_fd once and use it for both for checking
|
||||
* and eventfd_signal
|
||||
*/
|
||||
ev_fd = rcu_dereference(ctx->io_ev_fd);
|
||||
|
||||
/*
|
||||
* Check again if ev_fd exists incase an io_eventfd_unregister call
|
||||
* completed between the NULL check of ctx->io_ev_fd at the start of
|
||||
* the function and rcu_read_lock.
|
||||
*/
|
||||
if (unlikely(!ev_fd))
|
||||
return;
|
||||
if (!atomic_inc_not_zero(&ev_fd->refs))
|
||||
return;
|
||||
if (ev_fd->eventfd_async && !io_wq_current_is_worker())
|
||||
goto out;
|
||||
|
||||
if (likely(eventfd_signal_allowed())) {
|
||||
eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
|
||||
} else {
|
||||
if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) {
|
||||
call_rcu_hurry(&ev_fd->rcu, io_eventfd_do_signal);
|
||||
return;
|
||||
}
|
||||
}
|
||||
out:
|
||||
if (atomic_dec_and_test(&ev_fd->refs))
|
||||
call_rcu(&ev_fd->rcu, io_eventfd_free);
|
||||
}
|
||||
|
||||
void io_eventfd_flush_signal(struct io_ring_ctx *ctx)
|
||||
{
|
||||
bool skip;
|
||||
|
||||
spin_lock(&ctx->completion_lock);
|
||||
|
||||
/*
|
||||
* Eventfd should only get triggered when at least one event has been
|
||||
* posted. Some applications rely on the eventfd notification count
|
||||
* only changing IFF a new CQE has been added to the CQ ring. There's
|
||||
* no depedency on 1:1 relationship between how many times this
|
||||
* function is called (and hence the eventfd count) and number of CQEs
|
||||
* posted to the CQ ring.
|
||||
*/
|
||||
skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail;
|
||||
ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
|
||||
spin_unlock(&ctx->completion_lock);
|
||||
if (skip)
|
||||
return;
|
||||
|
||||
io_eventfd_signal(ctx);
|
||||
}
|
||||
|
||||
int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
|
||||
unsigned int eventfd_async)
|
||||
{
|
||||
struct io_ev_fd *ev_fd;
|
||||
__s32 __user *fds = arg;
|
||||
int fd;
|
||||
|
||||
ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
|
||||
lockdep_is_held(&ctx->uring_lock));
|
||||
if (ev_fd)
|
||||
return -EBUSY;
|
||||
|
||||
if (copy_from_user(&fd, fds, sizeof(*fds)))
|
||||
return -EFAULT;
|
||||
|
||||
ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL);
|
||||
if (!ev_fd)
|
||||
return -ENOMEM;
|
||||
|
||||
ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd);
|
||||
if (IS_ERR(ev_fd->cq_ev_fd)) {
|
||||
int ret = PTR_ERR(ev_fd->cq_ev_fd);
|
||||
kfree(ev_fd);
|
||||
return ret;
|
||||
}
|
||||
|
||||
spin_lock(&ctx->completion_lock);
|
||||
ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
|
||||
spin_unlock(&ctx->completion_lock);
|
||||
|
||||
ev_fd->eventfd_async = eventfd_async;
|
||||
ctx->has_evfd = true;
|
||||
atomic_set(&ev_fd->refs, 1);
|
||||
atomic_set(&ev_fd->ops, 0);
|
||||
rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int io_eventfd_unregister(struct io_ring_ctx *ctx)
|
||||
{
|
||||
struct io_ev_fd *ev_fd;
|
||||
|
||||
ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
|
||||
lockdep_is_held(&ctx->uring_lock));
|
||||
if (ev_fd) {
|
||||
ctx->has_evfd = false;
|
||||
rcu_assign_pointer(ctx->io_ev_fd, NULL);
|
||||
if (atomic_dec_and_test(&ev_fd->refs))
|
||||
call_rcu(&ev_fd->rcu, io_eventfd_free);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return -ENXIO;
|
||||
}
|
||||
8
io_uring/eventfd.h
Normal file
8
io_uring/eventfd.h
Normal file
@ -0,0 +1,8 @@
|
||||
|
||||
struct io_ring_ctx;
|
||||
int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
|
||||
unsigned int eventfd_async);
|
||||
int io_eventfd_unregister(struct io_ring_ctx *ctx);
|
||||
|
||||
void io_eventfd_flush_signal(struct io_ring_ctx *ctx);
|
||||
void io_eventfd_signal(struct io_ring_ctx *ctx);
|
||||
@ -101,6 +101,7 @@
|
||||
#include "poll.h"
|
||||
#include "rw.h"
|
||||
#include "alloc_cache.h"
|
||||
#include "eventfd.h"
|
||||
|
||||
#define IORING_MAX_ENTRIES 32768
|
||||
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
|
||||
@ -541,87 +542,6 @@ static __cold void io_queue_deferred(struct io_ring_ctx *ctx)
|
||||
}
|
||||
}
|
||||
|
||||
void io_eventfd_free(struct rcu_head *rcu)
|
||||
{
|
||||
struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
|
||||
|
||||
eventfd_ctx_put(ev_fd->cq_ev_fd);
|
||||
kfree(ev_fd);
|
||||
}
|
||||
|
||||
void io_eventfd_do_signal(struct rcu_head *rcu)
|
||||
{
|
||||
struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
|
||||
|
||||
eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
|
||||
|
||||
if (atomic_dec_and_test(&ev_fd->refs))
|
||||
io_eventfd_free(rcu);
|
||||
}
|
||||
|
||||
static void io_eventfd_signal(struct io_ring_ctx *ctx)
|
||||
{
|
||||
struct io_ev_fd *ev_fd = NULL;
|
||||
|
||||
if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
|
||||
return;
|
||||
|
||||
guard(rcu)();
|
||||
|
||||
/*
|
||||
* rcu_dereference ctx->io_ev_fd once and use it for both for checking
|
||||
* and eventfd_signal
|
||||
*/
|
||||
ev_fd = rcu_dereference(ctx->io_ev_fd);
|
||||
|
||||
/*
|
||||
* Check again if ev_fd exists incase an io_eventfd_unregister call
|
||||
* completed between the NULL check of ctx->io_ev_fd at the start of
|
||||
* the function and rcu_read_lock.
|
||||
*/
|
||||
if (unlikely(!ev_fd))
|
||||
return;
|
||||
if (!atomic_inc_not_zero(&ev_fd->refs))
|
||||
return;
|
||||
if (ev_fd->eventfd_async && !io_wq_current_is_worker())
|
||||
goto out;
|
||||
|
||||
if (likely(eventfd_signal_allowed())) {
|
||||
eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
|
||||
} else {
|
||||
if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) {
|
||||
call_rcu_hurry(&ev_fd->rcu, io_eventfd_do_signal);
|
||||
return;
|
||||
}
|
||||
}
|
||||
out:
|
||||
if (atomic_dec_and_test(&ev_fd->refs))
|
||||
call_rcu(&ev_fd->rcu, io_eventfd_free);
|
||||
}
|
||||
|
||||
static void io_eventfd_flush_signal(struct io_ring_ctx *ctx)
|
||||
{
|
||||
bool skip;
|
||||
|
||||
spin_lock(&ctx->completion_lock);
|
||||
|
||||
/*
|
||||
* Eventfd should only get triggered when at least one event has been
|
||||
* posted. Some applications rely on the eventfd notification count
|
||||
* only changing IFF a new CQE has been added to the CQ ring. There's
|
||||
* no depedency on 1:1 relationship between how many times this
|
||||
* function is called (and hence the eventfd count) and number of CQEs
|
||||
* posted to the CQ ring.
|
||||
*/
|
||||
skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail;
|
||||
ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
|
||||
spin_unlock(&ctx->completion_lock);
|
||||
if (skip)
|
||||
return;
|
||||
|
||||
io_eventfd_signal(ctx);
|
||||
}
|
||||
|
||||
void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
|
||||
{
|
||||
if (ctx->poll_activated)
|
||||
|
||||
@ -104,12 +104,6 @@ bool __io_alloc_req_refill(struct io_ring_ctx *ctx);
|
||||
bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
|
||||
bool cancel_all);
|
||||
|
||||
enum {
|
||||
IO_EVENTFD_OP_SIGNAL_BIT,
|
||||
};
|
||||
|
||||
void io_eventfd_do_signal(struct rcu_head *rcu);
|
||||
void io_eventfd_free(struct rcu_head *rcu);
|
||||
void io_activate_pollwq(struct io_ring_ctx *ctx);
|
||||
|
||||
static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)
|
||||
|
||||
@ -27,65 +27,11 @@
|
||||
#include "cancel.h"
|
||||
#include "kbuf.h"
|
||||
#include "napi.h"
|
||||
#include "eventfd.h"
|
||||
|
||||
#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
|
||||
IORING_REGISTER_LAST + IORING_OP_LAST)
|
||||
|
||||
static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
|
||||
unsigned int eventfd_async)
|
||||
{
|
||||
struct io_ev_fd *ev_fd;
|
||||
__s32 __user *fds = arg;
|
||||
int fd;
|
||||
|
||||
ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
|
||||
lockdep_is_held(&ctx->uring_lock));
|
||||
if (ev_fd)
|
||||
return -EBUSY;
|
||||
|
||||
if (copy_from_user(&fd, fds, sizeof(*fds)))
|
||||
return -EFAULT;
|
||||
|
||||
ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL);
|
||||
if (!ev_fd)
|
||||
return -ENOMEM;
|
||||
|
||||
ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd);
|
||||
if (IS_ERR(ev_fd->cq_ev_fd)) {
|
||||
int ret = PTR_ERR(ev_fd->cq_ev_fd);
|
||||
kfree(ev_fd);
|
||||
return ret;
|
||||
}
|
||||
|
||||
spin_lock(&ctx->completion_lock);
|
||||
ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
|
||||
spin_unlock(&ctx->completion_lock);
|
||||
|
||||
ev_fd->eventfd_async = eventfd_async;
|
||||
ctx->has_evfd = true;
|
||||
atomic_set(&ev_fd->refs, 1);
|
||||
atomic_set(&ev_fd->ops, 0);
|
||||
rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int io_eventfd_unregister(struct io_ring_ctx *ctx)
|
||||
{
|
||||
struct io_ev_fd *ev_fd;
|
||||
|
||||
ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
|
||||
lockdep_is_held(&ctx->uring_lock));
|
||||
if (ev_fd) {
|
||||
ctx->has_evfd = false;
|
||||
rcu_assign_pointer(ctx->io_ev_fd, NULL);
|
||||
if (atomic_dec_and_test(&ev_fd->refs))
|
||||
call_rcu(&ev_fd->rcu, io_eventfd_free);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return -ENXIO;
|
||||
}
|
||||
|
||||
static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg,
|
||||
unsigned nr_args)
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user