2022-02-17 06:30:38 -07:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
|
|
|
|
/* Object lifetime handling and tracing.
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2022 Red Hat, Inc. All Rights Reserved.
|
|
|
|
|
* Written by David Howells (dhowells@redhat.com)
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/slab.h>
|
2024-03-15 07:37:18 -07:00
|
|
|
#include <linux/mempool.h>
|
|
|
|
|
#include <linux/delay.h>
|
2022-02-17 06:30:38 -07:00
|
|
|
#include "internal.h"
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Allocate an I/O request and initialise it.
|
|
|
|
|
*/
|
2021-08-26 06:24:42 -07:00
|
|
|
struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
|
|
|
|
|
struct file *file,
|
|
|
|
|
loff_t start, size_t len,
|
|
|
|
|
enum netfs_io_origin origin)
|
2022-02-17 06:30:38 -07:00
|
|
|
{
|
|
|
|
|
static atomic_t debug_ids;
|
2021-06-29 14:37:05 -07:00
|
|
|
struct inode *inode = file ? file_inode(file) : mapping->host;
|
netfs: Fix gcc-12 warning by embedding vfs inode in netfs_i_context
While randstruct was satisfied with using an open-coded "void *" offset
cast for the netfs_i_context <-> inode casting, __builtin_object_size() as
used by FORTIFY_SOURCE was not as easily fooled. This was causing the
following complaint[1] from gcc v12:
In file included from include/linux/string.h:253,
from include/linux/ceph/ceph_debug.h:7,
from fs/ceph/inode.c:2:
In function 'fortify_memset_chk',
inlined from 'netfs_i_context_init' at include/linux/netfs.h:326:2,
inlined from 'ceph_alloc_inode' at fs/ceph/inode.c:463:2:
include/linux/fortify-string.h:242:25: warning: call to '__write_overflow_field' declared with attribute warning: detected write beyond size of field (1st parameter); maybe use struct_group()? [-Wattribute-warning]
242 | __write_overflow_field(p_size_field, size);
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Fix this by embedding a struct inode into struct netfs_i_context (which
should perhaps be renamed to struct netfs_inode). The struct inode
vfs_inode fields are then removed from the 9p, afs, ceph and cifs inode
structs and vfs_inode is then simply changed to "netfs.inode" in those
filesystems.
Further, rename netfs_i_context to netfs_inode, get rid of the
netfs_inode() function that converted a netfs_i_context pointer to an
inode pointer (that can now be done with &ctx->inode) and rename the
netfs_i_context() function to netfs_inode() (which is now a wrapper
around container_of()).
Most of the changes were done with:
perl -p -i -e 's/vfs_inode/netfs.inode/'g \
`git grep -l 'vfs_inode' -- fs/{9p,afs,ceph,cifs}/*.[ch]`
Kees suggested doing it with a pair structure[2] and a special
declarator to insert that into the network filesystem's inode
wrapper[3], but I think it's cleaner to embed it - and then it doesn't
matter if struct randomisation reorders things.
Dave Chinner suggested using a filesystem-specific VFS_I() function in
each filesystem to convert that filesystem's own inode wrapper struct
into the VFS inode struct[4].
Version #2:
- Fix a couple of missed name changes due to a disabled cifs option.
- Rename nfs_i_context to nfs_inode
- Use "netfs" instead of "nic" as the member name in per-fs inode wrapper
structs.
[ This also undoes commit 507160f46c55 ("netfs: gcc-12: temporarily
disable '-Wattribute-warning' for now") that is no longer needed ]
Fixes: bc899ee1c898 ("netfs: Add a netfs inode context")
Reported-by: Jeff Layton <jlayton@kernel.org>
Signed-off-by: David Howells <dhowells@redhat.com>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
Reviewed-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Xiubo Li <xiubli@redhat.com>
cc: Jonathan Corbet <corbet@lwn.net>
cc: Eric Van Hensbergen <ericvh@gmail.com>
cc: Latchesar Ionkov <lucho@ionkov.net>
cc: Dominique Martinet <asmadeus@codewreck.org>
cc: Christian Schoenebeck <linux_oss@crudebyte.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: Ilya Dryomov <idryomov@gmail.com>
cc: Steve French <smfrench@gmail.com>
cc: William Kucharski <william.kucharski@oracle.com>
cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>
cc: Dave Chinner <david@fromorbit.com>
cc: linux-doc@vger.kernel.org
cc: v9fs-developer@lists.sourceforge.net
cc: linux-afs@lists.infradead.org
cc: ceph-devel@vger.kernel.org
cc: linux-cifs@vger.kernel.org
cc: samba-technical@lists.samba.org
cc: linux-fsdevel@vger.kernel.org
cc: linux-hardening@vger.kernel.org
Link: https://lore.kernel.org/r/d2ad3a3d7bdd794c6efb562d2f2b655fb67756b9.camel@kernel.org/ [1]
Link: https://lore.kernel.org/r/20220517210230.864239-1-keescook@chromium.org/ [2]
Link: https://lore.kernel.org/r/20220518202212.2322058-1-keescook@chromium.org/ [3]
Link: https://lore.kernel.org/r/20220524101205.GI2306852@dread.disaster.area/ [4]
Link: https://lore.kernel.org/r/165296786831.3591209.12111293034669289733.stgit@warthog.procyon.org.uk/ # v1
Link: https://lore.kernel.org/r/165305805651.4094995.7763502506786714216.stgit@warthog.procyon.org.uk # v2
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2022-06-09 13:46:04 -07:00
|
|
|
struct netfs_inode *ctx = netfs_inode(inode);
|
2022-02-17 06:30:38 -07:00
|
|
|
struct netfs_io_request *rreq;
|
2024-03-15 07:37:18 -07:00
|
|
|
mempool_t *mempool = ctx->ops->request_pool ?: &netfs_request_pool;
|
|
|
|
|
struct kmem_cache *cache = mempool->pool_data;
|
2022-01-20 14:55:46 -07:00
|
|
|
int ret;
|
2022-02-17 06:30:38 -07:00
|
|
|
|
2024-03-15 07:37:18 -07:00
|
|
|
for (;;) {
|
|
|
|
|
rreq = mempool_alloc(mempool, GFP_KERNEL);
|
|
|
|
|
if (rreq)
|
|
|
|
|
break;
|
|
|
|
|
msleep(10);
|
|
|
|
|
}
|
2022-01-20 14:55:46 -07:00
|
|
|
|
2024-03-15 07:37:18 -07:00
|
|
|
memset(rreq, 0, kmem_cache_size(cache));
|
2022-01-20 14:55:46 -07:00
|
|
|
rreq->start = start;
|
|
|
|
|
rreq->len = len;
|
|
|
|
|
rreq->origin = origin;
|
2021-06-29 14:37:05 -07:00
|
|
|
rreq->netfs_ops = ctx->ops;
|
2022-01-20 14:55:46 -07:00
|
|
|
rreq->mapping = mapping;
|
2021-06-29 14:37:05 -07:00
|
|
|
rreq->inode = inode;
|
|
|
|
|
rreq->i_size = i_size_read(inode);
|
2022-01-20 14:55:46 -07:00
|
|
|
rreq->debug_id = atomic_inc_return(&debug_ids);
|
netfs: New writeback implementation
The current netfslib writeback implementation creates writeback requests of
contiguous folio data and then separately tiles subrequests over the space
twice, once for the server and once for the cache. This creates a few
issues:
(1) Every time there's a discontiguity or a change between writing to only
one destination or writing to both, it must create a new request.
This makes it harder to do vectored writes.
(2) The folios don't have the writeback mark removed until the end of the
request - and a request could be hundreds of megabytes.
(3) In future, I want to support a larger cache granularity, which will
require aggregation of some folios that contain unmodified data (which
only need to go to the cache) and some which contain modifications
(which need to be uploaded and stored to the cache) - but, currently,
these are treated as discontiguous.
There's also a move to get everyone to use writeback_iter() to extract
writable folios from the pagecache. That said, currently writeback_iter()
has some issues that make it less than ideal:
(1) there's no way to cancel the iteration, even if you find a "temporary"
error that means the current folio and all subsequent folios are going
to fail;
(2) there's no way to filter the folios being written back - something
that will impact Ceph with it's ordered snap system;
(3) and if you get a folio you can't immediately deal with (say you need
to flush the preceding writes), you are left with a folio hanging in
the locked state for the duration, when really we should unlock it and
relock it later.
In this new implementation, I use writeback_iter() to pump folios,
progressively creating two parallel, but separate streams and cleaning up
the finished folios as the subrequests complete. Either or both streams
can contain gaps, and the subrequests in each stream can be of variable
size, don't need to align with each other and don't need to align with the
folios.
Indeed, subrequests can cross folio boundaries, may cover several folios or
a folio may be spanned by multiple folios, e.g.:
+---+---+-----+-----+---+----------+
Folios: | | | | | | |
+---+---+-----+-----+---+----------+
+------+------+ +----+----+
Upload: | | |.....| | |
+------+------+ +----+----+
+------+------+------+------+------+
Cache: | | | | | |
+------+------+------+------+------+
The progressive subrequest construction permits the algorithm to be
preparing both the next upload to the server and the next write to the
cache whilst the previous ones are already in progress. Throttling can be
applied to control the rate of production of subrequests - and, in any
case, we probably want to write them to the server in ascending order,
particularly if the file will be extended.
Content crypto can also be prepared at the same time as the subrequests and
run asynchronously, with the prepped requests being stalled until the
crypto catches up with them. This might also be useful for transport
crypto, but that happens at a lower layer, so probably would be harder to
pull off.
The algorithm is split into three parts:
(1) The issuer. This walks through the data, packaging it up, encrypting
it and creating subrequests. The part of this that generates
subrequests only deals with file positions and spans and so is usable
for DIO/unbuffered writes as well as buffered writes.
(2) The collector. This asynchronously collects completed subrequests,
unlocks folios, frees crypto buffers and performs any retries. This
runs in a work queue so that the issuer can return to the caller for
writeback (so that the VM can have its kswapd thread back) or async
writes.
(3) The retryer. This pauses the issuer, waits for all outstanding
subrequests to complete and then goes through the failed subrequests
to reissue them. This may involve reprepping them (with cifs, the
credits must be renegotiated, and a subrequest may need splitting),
and doing RMW for content crypto if there's a conflicting change on
the server.
[!] Note that some of the functions are prefixed with "new_" to avoid
clashes with existing functions. These will be renamed in a later patch
that cuts over to the new algorithm.
Signed-off-by: David Howells <dhowells@redhat.com>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
cc: Eric Van Hensbergen <ericvh@kernel.org>
cc: Latchesar Ionkov <lucho@ionkov.net>
cc: Dominique Martinet <asmadeus@codewreck.org>
cc: Christian Schoenebeck <linux_oss@crudebyte.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: v9fs@lists.linux.dev
cc: linux-afs@lists.infradead.org
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
2024-03-18 09:52:05 -07:00
|
|
|
rreq->wsize = INT_MAX;
|
2024-07-01 16:40:22 -07:00
|
|
|
rreq->io_streams[0].sreq_max_len = ULONG_MAX;
|
|
|
|
|
rreq->io_streams[0].sreq_max_segs = 0;
|
netfs: New writeback implementation
The current netfslib writeback implementation creates writeback requests of
contiguous folio data and then separately tiles subrequests over the space
twice, once for the server and once for the cache. This creates a few
issues:
(1) Every time there's a discontiguity or a change between writing to only
one destination or writing to both, it must create a new request.
This makes it harder to do vectored writes.
(2) The folios don't have the writeback mark removed until the end of the
request - and a request could be hundreds of megabytes.
(3) In future, I want to support a larger cache granularity, which will
require aggregation of some folios that contain unmodified data (which
only need to go to the cache) and some which contain modifications
(which need to be uploaded and stored to the cache) - but, currently,
these are treated as discontiguous.
There's also a move to get everyone to use writeback_iter() to extract
writable folios from the pagecache. That said, currently writeback_iter()
has some issues that make it less than ideal:
(1) there's no way to cancel the iteration, even if you find a "temporary"
error that means the current folio and all subsequent folios are going
to fail;
(2) there's no way to filter the folios being written back - something
that will impact Ceph with it's ordered snap system;
(3) and if you get a folio you can't immediately deal with (say you need
to flush the preceding writes), you are left with a folio hanging in
the locked state for the duration, when really we should unlock it and
relock it later.
In this new implementation, I use writeback_iter() to pump folios,
progressively creating two parallel, but separate streams and cleaning up
the finished folios as the subrequests complete. Either or both streams
can contain gaps, and the subrequests in each stream can be of variable
size, don't need to align with each other and don't need to align with the
folios.
Indeed, subrequests can cross folio boundaries, may cover several folios or
a folio may be spanned by multiple folios, e.g.:
+---+---+-----+-----+---+----------+
Folios: | | | | | | |
+---+---+-----+-----+---+----------+
+------+------+ +----+----+
Upload: | | |.....| | |
+------+------+ +----+----+
+------+------+------+------+------+
Cache: | | | | | |
+------+------+------+------+------+
The progressive subrequest construction permits the algorithm to be
preparing both the next upload to the server and the next write to the
cache whilst the previous ones are already in progress. Throttling can be
applied to control the rate of production of subrequests - and, in any
case, we probably want to write them to the server in ascending order,
particularly if the file will be extended.
Content crypto can also be prepared at the same time as the subrequests and
run asynchronously, with the prepped requests being stalled until the
crypto catches up with them. This might also be useful for transport
crypto, but that happens at a lower layer, so probably would be harder to
pull off.
The algorithm is split into three parts:
(1) The issuer. This walks through the data, packaging it up, encrypting
it and creating subrequests. The part of this that generates
subrequests only deals with file positions and spans and so is usable
for DIO/unbuffered writes as well as buffered writes.
(2) The collector. This asynchronously collects completed subrequests,
unlocks folios, frees crypto buffers and performs any retries. This
runs in a work queue so that the issuer can return to the caller for
writeback (so that the VM can have its kswapd thread back) or async
writes.
(3) The retryer. This pauses the issuer, waits for all outstanding
subrequests to complete and then goes through the failed subrequests
to reissue them. This may involve reprepping them (with cifs, the
credits must be renegotiated, and a subrequest may need splitting),
and doing RMW for content crypto if there's a conflicting change on
the server.
[!] Note that some of the functions are prefixed with "new_" to avoid
clashes with existing functions. These will be renamed in a later patch
that cuts over to the new algorithm.
Signed-off-by: David Howells <dhowells@redhat.com>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
cc: Eric Van Hensbergen <ericvh@kernel.org>
cc: Latchesar Ionkov <lucho@ionkov.net>
cc: Dominique Martinet <asmadeus@codewreck.org>
cc: Christian Schoenebeck <linux_oss@crudebyte.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: v9fs@lists.linux.dev
cc: linux-afs@lists.infradead.org
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
2024-03-18 09:52:05 -07:00
|
|
|
spin_lock_init(&rreq->lock);
|
|
|
|
|
INIT_LIST_HEAD(&rreq->io_streams[0].subrequests);
|
|
|
|
|
INIT_LIST_HEAD(&rreq->io_streams[1].subrequests);
|
2022-01-20 14:55:46 -07:00
|
|
|
INIT_LIST_HEAD(&rreq->subrequests);
|
|
|
|
|
refcount_set(&rreq->ref, 1);
|
2022-02-09 12:52:13 -07:00
|
|
|
|
2024-07-09 01:47:17 -07:00
|
|
|
if (origin == NETFS_READAHEAD ||
|
|
|
|
|
origin == NETFS_READPAGE ||
|
2024-07-01 16:40:22 -07:00
|
|
|
origin == NETFS_READ_GAPS ||
|
2024-07-09 01:47:17 -07:00
|
|
|
origin == NETFS_READ_FOR_WRITE ||
|
|
|
|
|
origin == NETFS_DIO_READ)
|
2024-07-01 16:40:22 -07:00
|
|
|
INIT_WORK(&rreq->work, netfs_read_termination_worker);
|
2024-07-09 01:47:17 -07:00
|
|
|
else
|
|
|
|
|
INIT_WORK(&rreq->work, netfs_write_collection_worker);
|
|
|
|
|
|
2022-01-20 14:55:46 -07:00
|
|
|
__set_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
|
netfs: Implement unbuffered/DIO read support
Implement support for unbuffered and DIO reads in the netfs library,
utilising the existing read helper code to do block splitting and
individual queuing. The code also handles extraction of the destination
buffer from the supplied iterator, allowing async unbuffered reads to take
place.
The read will be split up according to the rsize setting and, if supplied,
the ->clamp_length() method. Note that the next subrequest will be issued
as soon as issue_op returns, without waiting for previous ones to finish.
The network filesystem needs to pause or handle queuing them if it doesn't
want to fire them all at the server simultaneously.
Once all the subrequests have finished, the state will be assessed and the
amount of data to be indicated as having being obtained will be
determined. As the subrequests may finish in any order, if an intermediate
subrequest is short, any further subrequests may be copied into the buffer
and then abandoned.
In the future, this will also take care of doing an unbuffered read from
encrypted content, with the decryption being done by the library.
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Jeff Layton <jlayton@kernel.org>
cc: linux-cachefs@redhat.com
cc: linux-fsdevel@vger.kernel.org
cc: linux-mm@kvack.org
2022-01-14 10:39:55 -07:00
|
|
|
if (file && file->f_flags & O_NONBLOCK)
|
|
|
|
|
__set_bit(NETFS_RREQ_NONBLOCK, &rreq->flags);
|
2022-01-20 14:55:46 -07:00
|
|
|
if (rreq->netfs_ops->init_request) {
|
|
|
|
|
ret = rreq->netfs_ops->init_request(rreq, file);
|
|
|
|
|
if (ret < 0) {
|
2024-03-15 07:37:18 -07:00
|
|
|
mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool);
|
2022-01-20 14:55:46 -07:00
|
|
|
return ERR_PTR(ret);
|
|
|
|
|
}
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
|
|
|
|
|
2024-05-24 07:26:11 -07:00
|
|
|
atomic_inc(&ctx->io_count);
|
2022-02-09 12:52:13 -07:00
|
|
|
trace_netfs_rreq_ref(rreq->debug_id, 1, netfs_rreq_trace_new);
|
2022-03-04 03:34:27 -07:00
|
|
|
netfs_proc_add_rreq(rreq);
|
2022-01-20 14:55:46 -07:00
|
|
|
netfs_stat(&netfs_n_rh_rreq);
|
2022-02-17 06:30:38 -07:00
|
|
|
return rreq;
|
|
|
|
|
}
|
|
|
|
|
|
2022-02-17 14:13:05 -07:00
|
|
|
void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what)
|
2022-02-17 06:30:38 -07:00
|
|
|
{
|
2022-02-17 14:13:05 -07:00
|
|
|
int r;
|
|
|
|
|
|
|
|
|
|
__refcount_inc(&rreq->ref, &r);
|
|
|
|
|
trace_netfs_rreq_ref(rreq->debug_id, r + 1, what);
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void netfs_clear_subrequests(struct netfs_io_request *rreq, bool was_async)
|
|
|
|
|
{
|
|
|
|
|
struct netfs_io_subrequest *subreq;
|
netfs: New writeback implementation
The current netfslib writeback implementation creates writeback requests of
contiguous folio data and then separately tiles subrequests over the space
twice, once for the server and once for the cache. This creates a few
issues:
(1) Every time there's a discontiguity or a change between writing to only
one destination or writing to both, it must create a new request.
This makes it harder to do vectored writes.
(2) The folios don't have the writeback mark removed until the end of the
request - and a request could be hundreds of megabytes.
(3) In future, I want to support a larger cache granularity, which will
require aggregation of some folios that contain unmodified data (which
only need to go to the cache) and some which contain modifications
(which need to be uploaded and stored to the cache) - but, currently,
these are treated as discontiguous.
There's also a move to get everyone to use writeback_iter() to extract
writable folios from the pagecache. That said, currently writeback_iter()
has some issues that make it less than ideal:
(1) there's no way to cancel the iteration, even if you find a "temporary"
error that means the current folio and all subsequent folios are going
to fail;
(2) there's no way to filter the folios being written back - something
that will impact Ceph with it's ordered snap system;
(3) and if you get a folio you can't immediately deal with (say you need
to flush the preceding writes), you are left with a folio hanging in
the locked state for the duration, when really we should unlock it and
relock it later.
In this new implementation, I use writeback_iter() to pump folios,
progressively creating two parallel, but separate streams and cleaning up
the finished folios as the subrequests complete. Either or both streams
can contain gaps, and the subrequests in each stream can be of variable
size, don't need to align with each other and don't need to align with the
folios.
Indeed, subrequests can cross folio boundaries, may cover several folios or
a folio may be spanned by multiple folios, e.g.:
+---+---+-----+-----+---+----------+
Folios: | | | | | | |
+---+---+-----+-----+---+----------+
+------+------+ +----+----+
Upload: | | |.....| | |
+------+------+ +----+----+
+------+------+------+------+------+
Cache: | | | | | |
+------+------+------+------+------+
The progressive subrequest construction permits the algorithm to be
preparing both the next upload to the server and the next write to the
cache whilst the previous ones are already in progress. Throttling can be
applied to control the rate of production of subrequests - and, in any
case, we probably want to write them to the server in ascending order,
particularly if the file will be extended.
Content crypto can also be prepared at the same time as the subrequests and
run asynchronously, with the prepped requests being stalled until the
crypto catches up with them. This might also be useful for transport
crypto, but that happens at a lower layer, so probably would be harder to
pull off.
The algorithm is split into three parts:
(1) The issuer. This walks through the data, packaging it up, encrypting
it and creating subrequests. The part of this that generates
subrequests only deals with file positions and spans and so is usable
for DIO/unbuffered writes as well as buffered writes.
(2) The collector. This asynchronously collects completed subrequests,
unlocks folios, frees crypto buffers and performs any retries. This
runs in a work queue so that the issuer can return to the caller for
writeback (so that the VM can have its kswapd thread back) or async
writes.
(3) The retryer. This pauses the issuer, waits for all outstanding
subrequests to complete and then goes through the failed subrequests
to reissue them. This may involve reprepping them (with cifs, the
credits must be renegotiated, and a subrequest may need splitting),
and doing RMW for content crypto if there's a conflicting change on
the server.
[!] Note that some of the functions are prefixed with "new_" to avoid
clashes with existing functions. These will be renamed in a later patch
that cuts over to the new algorithm.
Signed-off-by: David Howells <dhowells@redhat.com>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
cc: Eric Van Hensbergen <ericvh@kernel.org>
cc: Latchesar Ionkov <lucho@ionkov.net>
cc: Dominique Martinet <asmadeus@codewreck.org>
cc: Christian Schoenebeck <linux_oss@crudebyte.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: v9fs@lists.linux.dev
cc: linux-afs@lists.infradead.org
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
2024-03-18 09:52:05 -07:00
|
|
|
struct netfs_io_stream *stream;
|
|
|
|
|
int s;
|
2022-02-17 06:30:38 -07:00
|
|
|
|
|
|
|
|
while (!list_empty(&rreq->subrequests)) {
|
|
|
|
|
subreq = list_first_entry(&rreq->subrequests,
|
|
|
|
|
struct netfs_io_subrequest, rreq_link);
|
|
|
|
|
list_del(&subreq->rreq_link);
|
2022-02-17 08:01:24 -07:00
|
|
|
netfs_put_subrequest(subreq, was_async,
|
|
|
|
|
netfs_sreq_trace_put_clear);
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
netfs: New writeback implementation
The current netfslib writeback implementation creates writeback requests of
contiguous folio data and then separately tiles subrequests over the space
twice, once for the server and once for the cache. This creates a few
issues:
(1) Every time there's a discontiguity or a change between writing to only
one destination or writing to both, it must create a new request.
This makes it harder to do vectored writes.
(2) The folios don't have the writeback mark removed until the end of the
request - and a request could be hundreds of megabytes.
(3) In future, I want to support a larger cache granularity, which will
require aggregation of some folios that contain unmodified data (which
only need to go to the cache) and some which contain modifications
(which need to be uploaded and stored to the cache) - but, currently,
these are treated as discontiguous.
There's also a move to get everyone to use writeback_iter() to extract
writable folios from the pagecache. That said, currently writeback_iter()
has some issues that make it less than ideal:
(1) there's no way to cancel the iteration, even if you find a "temporary"
error that means the current folio and all subsequent folios are going
to fail;
(2) there's no way to filter the folios being written back - something
that will impact Ceph with it's ordered snap system;
(3) and if you get a folio you can't immediately deal with (say you need
to flush the preceding writes), you are left with a folio hanging in
the locked state for the duration, when really we should unlock it and
relock it later.
In this new implementation, I use writeback_iter() to pump folios,
progressively creating two parallel, but separate streams and cleaning up
the finished folios as the subrequests complete. Either or both streams
can contain gaps, and the subrequests in each stream can be of variable
size, don't need to align with each other and don't need to align with the
folios.
Indeed, subrequests can cross folio boundaries, may cover several folios or
a folio may be spanned by multiple folios, e.g.:
+---+---+-----+-----+---+----------+
Folios: | | | | | | |
+---+---+-----+-----+---+----------+
+------+------+ +----+----+
Upload: | | |.....| | |
+------+------+ +----+----+
+------+------+------+------+------+
Cache: | | | | | |
+------+------+------+------+------+
The progressive subrequest construction permits the algorithm to be
preparing both the next upload to the server and the next write to the
cache whilst the previous ones are already in progress. Throttling can be
applied to control the rate of production of subrequests - and, in any
case, we probably want to write them to the server in ascending order,
particularly if the file will be extended.
Content crypto can also be prepared at the same time as the subrequests and
run asynchronously, with the prepped requests being stalled until the
crypto catches up with them. This might also be useful for transport
crypto, but that happens at a lower layer, so probably would be harder to
pull off.
The algorithm is split into three parts:
(1) The issuer. This walks through the data, packaging it up, encrypting
it and creating subrequests. The part of this that generates
subrequests only deals with file positions and spans and so is usable
for DIO/unbuffered writes as well as buffered writes.
(2) The collector. This asynchronously collects completed subrequests,
unlocks folios, frees crypto buffers and performs any retries. This
runs in a work queue so that the issuer can return to the caller for
writeback (so that the VM can have its kswapd thread back) or async
writes.
(3) The retryer. This pauses the issuer, waits for all outstanding
subrequests to complete and then goes through the failed subrequests
to reissue them. This may involve reprepping them (with cifs, the
credits must be renegotiated, and a subrequest may need splitting),
and doing RMW for content crypto if there's a conflicting change on
the server.
[!] Note that some of the functions are prefixed with "new_" to avoid
clashes with existing functions. These will be renamed in a later patch
that cuts over to the new algorithm.
Signed-off-by: David Howells <dhowells@redhat.com>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
cc: Eric Van Hensbergen <ericvh@kernel.org>
cc: Latchesar Ionkov <lucho@ionkov.net>
cc: Dominique Martinet <asmadeus@codewreck.org>
cc: Christian Schoenebeck <linux_oss@crudebyte.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: v9fs@lists.linux.dev
cc: linux-afs@lists.infradead.org
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
2024-03-18 09:52:05 -07:00
|
|
|
|
|
|
|
|
for (s = 0; s < ARRAY_SIZE(rreq->io_streams); s++) {
|
|
|
|
|
stream = &rreq->io_streams[s];
|
|
|
|
|
while (!list_empty(&stream->subrequests)) {
|
|
|
|
|
subreq = list_first_entry(&stream->subrequests,
|
|
|
|
|
struct netfs_io_subrequest, rreq_link);
|
|
|
|
|
list_del(&subreq->rreq_link);
|
|
|
|
|
netfs_put_subrequest(subreq, was_async,
|
|
|
|
|
netfs_sreq_trace_put_clear);
|
|
|
|
|
}
|
|
|
|
|
}
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
|
|
|
|
|
2024-03-15 07:37:18 -07:00
|
|
|
static void netfs_free_request_rcu(struct rcu_head *rcu)
|
|
|
|
|
{
|
|
|
|
|
struct netfs_io_request *rreq = container_of(rcu, struct netfs_io_request, rcu);
|
|
|
|
|
|
|
|
|
|
mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool);
|
|
|
|
|
netfs_stat_d(&netfs_n_rh_rreq);
|
|
|
|
|
}
|
|
|
|
|
|
2022-02-17 06:30:38 -07:00
|
|
|
static void netfs_free_request(struct work_struct *work)
|
|
|
|
|
{
|
|
|
|
|
struct netfs_io_request *rreq =
|
|
|
|
|
container_of(work, struct netfs_io_request, work);
|
2024-05-24 07:26:11 -07:00
|
|
|
struct netfs_inode *ictx = netfs_inode(rreq->inode);
|
2021-07-09 00:41:17 -07:00
|
|
|
unsigned int i;
|
2021-06-29 14:37:05 -07:00
|
|
|
|
2022-02-17 06:30:38 -07:00
|
|
|
trace_netfs_rreq(rreq, netfs_rreq_trace_free);
|
2022-03-04 03:34:27 -07:00
|
|
|
netfs_proc_del_rreq(rreq);
|
2022-02-25 04:19:14 -07:00
|
|
|
netfs_clear_subrequests(rreq, false);
|
|
|
|
|
if (rreq->netfs_ops->free_request)
|
|
|
|
|
rreq->netfs_ops->free_request(rreq);
|
2022-02-17 06:30:38 -07:00
|
|
|
if (rreq->cache_resources.ops)
|
|
|
|
|
rreq->cache_resources.ops->end_operation(&rreq->cache_resources);
|
2021-07-09 00:41:17 -07:00
|
|
|
if (rreq->direct_bv) {
|
|
|
|
|
for (i = 0; i < rreq->direct_bv_count; i++) {
|
|
|
|
|
if (rreq->direct_bv[i].bv_page) {
|
|
|
|
|
if (rreq->direct_bv_unpin)
|
|
|
|
|
unpin_user_page(rreq->direct_bv[i].bv_page);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
kvfree(rreq->direct_bv);
|
|
|
|
|
}
|
2024-05-29 13:47:07 -07:00
|
|
|
netfs_clear_buffer(rreq);
|
2024-05-24 07:26:11 -07:00
|
|
|
|
|
|
|
|
if (atomic_dec_and_test(&ictx->io_count))
|
|
|
|
|
wake_up_var(&ictx->io_count);
|
2024-03-15 07:37:18 -07:00
|
|
|
call_rcu(&rreq->rcu, netfs_free_request_rcu);
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
|
|
|
|
|
2022-02-17 14:13:05 -07:00
|
|
|
void netfs_put_request(struct netfs_io_request *rreq, bool was_async,
|
|
|
|
|
enum netfs_rreq_ref_trace what)
|
2022-02-17 06:30:38 -07:00
|
|
|
{
|
2023-09-29 06:35:17 -07:00
|
|
|
unsigned int debug_id;
|
2022-02-17 14:13:05 -07:00
|
|
|
bool dead;
|
|
|
|
|
int r;
|
|
|
|
|
|
2023-09-29 06:35:17 -07:00
|
|
|
if (rreq) {
|
|
|
|
|
debug_id = rreq->debug_id;
|
|
|
|
|
dead = __refcount_dec_and_test(&rreq->ref, &r);
|
|
|
|
|
trace_netfs_rreq_ref(debug_id, r - 1, what);
|
|
|
|
|
if (dead) {
|
|
|
|
|
if (was_async) {
|
|
|
|
|
rreq->work.func = netfs_free_request;
|
|
|
|
|
if (!queue_work(system_unbound_wq, &rreq->work))
|
2024-07-01 16:40:22 -07:00
|
|
|
WARN_ON(1);
|
2023-09-29 06:35:17 -07:00
|
|
|
} else {
|
|
|
|
|
netfs_free_request(&rreq->work);
|
|
|
|
|
}
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Allocate and partially initialise an I/O request structure.
|
|
|
|
|
*/
|
|
|
|
|
struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq)
|
|
|
|
|
{
|
|
|
|
|
struct netfs_io_subrequest *subreq;
|
2024-03-15 07:37:18 -07:00
|
|
|
mempool_t *mempool = rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool;
|
|
|
|
|
struct kmem_cache *cache = mempool->pool_data;
|
|
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
|
subreq = mempool_alloc(rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool,
|
|
|
|
|
GFP_KERNEL);
|
|
|
|
|
if (subreq)
|
|
|
|
|
break;
|
|
|
|
|
msleep(10);
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
|
|
|
|
|
2024-03-15 07:37:18 -07:00
|
|
|
memset(subreq, 0, kmem_cache_size(cache));
|
|
|
|
|
INIT_WORK(&subreq->work, NULL);
|
|
|
|
|
INIT_LIST_HEAD(&subreq->rreq_link);
|
|
|
|
|
refcount_set(&subreq->ref, 2);
|
|
|
|
|
subreq->rreq = rreq;
|
|
|
|
|
subreq->debug_index = atomic_inc_return(&rreq->subreq_counter);
|
|
|
|
|
netfs_get_request(rreq, netfs_rreq_trace_get_subreq);
|
|
|
|
|
netfs_stat(&netfs_n_rh_sreq);
|
2022-02-17 06:30:38 -07:00
|
|
|
return subreq;
|
|
|
|
|
}
|
|
|
|
|
|
2022-02-17 08:01:24 -07:00
|
|
|
void netfs_get_subrequest(struct netfs_io_subrequest *subreq,
|
|
|
|
|
enum netfs_sreq_ref_trace what)
|
2022-02-17 06:30:38 -07:00
|
|
|
{
|
2022-02-17 08:01:24 -07:00
|
|
|
int r;
|
|
|
|
|
|
|
|
|
|
__refcount_inc(&subreq->ref, &r);
|
|
|
|
|
trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index, r + 1,
|
|
|
|
|
what);
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
|
|
|
|
|
2022-02-17 08:01:24 -07:00
|
|
|
static void netfs_free_subrequest(struct netfs_io_subrequest *subreq,
|
|
|
|
|
bool was_async)
|
2022-02-17 06:30:38 -07:00
|
|
|
{
|
|
|
|
|
struct netfs_io_request *rreq = subreq->rreq;
|
|
|
|
|
|
|
|
|
|
trace_netfs_sreq(subreq, netfs_sreq_trace_free);
|
2022-02-25 04:19:14 -07:00
|
|
|
if (rreq->netfs_ops->free_subrequest)
|
|
|
|
|
rreq->netfs_ops->free_subrequest(subreq);
|
2024-03-15 07:37:18 -07:00
|
|
|
mempool_free(subreq, rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool);
|
2022-02-17 06:30:38 -07:00
|
|
|
netfs_stat_d(&netfs_n_rh_sreq);
|
2022-02-17 14:13:05 -07:00
|
|
|
netfs_put_request(rreq, was_async, netfs_rreq_trace_put_subreq);
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|
|
|
|
|
|
2022-02-17 08:01:24 -07:00
|
|
|
void netfs_put_subrequest(struct netfs_io_subrequest *subreq, bool was_async,
|
|
|
|
|
enum netfs_sreq_ref_trace what)
|
2022-02-17 06:30:38 -07:00
|
|
|
{
|
2022-02-17 08:01:24 -07:00
|
|
|
unsigned int debug_index = subreq->debug_index;
|
|
|
|
|
unsigned int debug_id = subreq->rreq->debug_id;
|
|
|
|
|
bool dead;
|
|
|
|
|
int r;
|
|
|
|
|
|
|
|
|
|
dead = __refcount_dec_and_test(&subreq->ref, &r);
|
|
|
|
|
trace_netfs_sreq_ref(debug_id, debug_index, r - 1, what);
|
|
|
|
|
if (dead)
|
|
|
|
|
netfs_free_subrequest(subreq, was_async);
|
2022-02-17 06:30:38 -07:00
|
|
|
}
|