From 4d868d18e569e1f74c2a59d70ee7f0f0f099f677 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 24 Nov 2022 23:52:28 -0500 Subject: [PATCH] bcachefs: More dio inlining Eliminate another function call in the O_DIRECT write path. Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index 8deb476a17c8..4dd5ebafe742 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -2218,7 +2218,7 @@ static __always_inline void bch2_dio_write_end(struct dio_write *dio) set_bit(EI_INODE_ERROR, &inode->ei_flags); } -static long bch2_dio_write_loop(struct dio_write *dio) +static __always_inline long bch2_dio_write_loop(struct dio_write *dio) { struct bch_fs *c = dio->op.c; struct kiocb *req = dio->req; @@ -2333,18 +2333,10 @@ err: goto out; } -static void bch2_dio_write_loop_async(struct bch_write_op *op) +static noinline __cold void bch2_dio_write_continue(struct dio_write *dio) { - struct dio_write *dio = container_of(op, struct dio_write, op); struct mm_struct *mm = dio->mm; - bch2_dio_write_end(dio); - - if (likely(!dio->iter.count) || dio->op.error) { - bch2_dio_write_done(dio); - return; - } - bio_reset(&dio->op.wbio.bio, NULL, REQ_OP_WRITE); if (mm) @@ -2354,6 +2346,18 @@ static void bch2_dio_write_loop_async(struct bch_write_op *op) kthread_unuse_mm(mm); } +static void bch2_dio_write_loop_async(struct bch_write_op *op) +{ + struct dio_write *dio = container_of(op, struct dio_write, op); + + bch2_dio_write_end(dio); + + if (likely(!dio->iter.count) || dio->op.error) + bch2_dio_write_done(dio); + else + bch2_dio_write_continue(dio); +} + static noinline ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) {