2018-06-05 19:42:14 -07:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
2005-04-16 15:20:36 -07:00
|
|
|
/*
|
2005-11-01 20:58:39 -07:00
|
|
|
* Copyright (c) 2000-2005 Silicon Graphics, Inc.
|
|
|
|
* All Rights Reserved.
|
2005-04-16 15:20:36 -07:00
|
|
|
*/
|
|
|
|
#include "xfs.h"
|
2010-02-15 02:44:46 -07:00
|
|
|
#include "xfs_fs.h"
|
2013-10-22 16:36:05 -07:00
|
|
|
#include "xfs_shared.h"
|
2013-10-22 16:51:50 -07:00
|
|
|
#include "xfs_format.h"
|
2013-10-22 16:50:10 -07:00
|
|
|
#include "xfs_log_format.h"
|
|
|
|
#include "xfs_trans_resv.h"
|
2005-04-16 15:20:36 -07:00
|
|
|
#include "xfs_mount.h"
|
|
|
|
#include "xfs_inode.h"
|
2013-10-22 16:50:10 -07:00
|
|
|
#include "xfs_trans.h"
|
2010-02-15 02:44:48 -07:00
|
|
|
#include "xfs_inode_item.h"
|
2010-02-15 02:44:46 -07:00
|
|
|
#include "xfs_bmap.h"
|
2013-08-12 03:49:45 -07:00
|
|
|
#include "xfs_bmap_util.h"
|
2013-08-12 03:49:37 -07:00
|
|
|
#include "xfs_dir2.h"
|
2013-08-12 03:49:45 -07:00
|
|
|
#include "xfs_dir2_priv.h"
|
2008-12-03 05:55:34 -07:00
|
|
|
#include "xfs_ioctl.h"
|
2010-02-15 02:44:46 -07:00
|
|
|
#include "xfs_trace.h"
|
2013-10-22 16:50:10 -07:00
|
|
|
#include "xfs_log.h"
|
2014-07-24 02:49:28 -07:00
|
|
|
#include "xfs_icache.h"
|
2015-02-15 17:59:50 -07:00
|
|
|
#include "xfs_pnfs.h"
|
2016-06-20 16:53:44 -07:00
|
|
|
#include "xfs_iomap.h"
|
2016-10-03 09:11:37 -07:00
|
|
|
#include "xfs_reflink.h"
|
2024-04-15 14:54:09 -07:00
|
|
|
#include "xfs_file.h"
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2022-06-02 22:37:37 -07:00
|
|
|
#include <linux/dax.h>
|
2011-01-14 05:07:43 -07:00
|
|
|
#include <linux/falloc.h>
|
2015-05-22 14:13:32 -07:00
|
|
|
#include <linux/backing-dev.h>
|
2017-11-01 08:36:47 -07:00
|
|
|
#include <linux/mman.h>
|
2019-08-29 09:04:12 -07:00
|
|
|
#include <linux/fadvise.h>
|
2021-01-21 06:19:58 -07:00
|
|
|
#include <linux/mount.h>
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-09-27 11:29:37 -07:00
|
|
|
static const struct vm_operations_struct xfs_file_vm_ops;
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2020-10-09 16:42:59 -07:00
|
|
|
/*
|
|
|
|
* Decide if the given file range is aligned to the size of the fundamental
|
|
|
|
* allocation unit for the file.
|
|
|
|
*/
|
2024-04-15 14:54:10 -07:00
|
|
|
bool
|
2020-10-09 16:42:59 -07:00
|
|
|
xfs_is_falloc_aligned(
|
|
|
|
struct xfs_inode *ip,
|
|
|
|
loff_t pos,
|
|
|
|
long long int len)
|
|
|
|
{
|
2024-04-15 14:54:10 -07:00
|
|
|
unsigned int alloc_unit = xfs_inode_alloc_unitsize(ip);
|
2020-10-09 16:42:59 -07:00
|
|
|
|
2024-04-15 14:54:12 -07:00
|
|
|
if (!is_power_of_2(alloc_unit))
|
|
|
|
return isaligned_64(pos, alloc_unit) &&
|
|
|
|
isaligned_64(len, alloc_unit);
|
2020-10-09 16:42:59 -07:00
|
|
|
|
2024-04-15 14:54:10 -07:00
|
|
|
return !((pos | len) & (alloc_unit - 1));
|
2020-10-09 16:42:59 -07:00
|
|
|
}
|
|
|
|
|
2011-10-02 07:25:16 -07:00
|
|
|
/*
|
|
|
|
* Fsync operations on directories are much simpler than on regular files,
|
|
|
|
* as there is no file data to flush, and thus also no need for explicit
|
|
|
|
* cache flush operations, and there are no non-transaction metadata updates
|
|
|
|
* on directories either.
|
|
|
|
*/
|
|
|
|
STATIC int
|
|
|
|
xfs_dir_fsync(
|
|
|
|
struct file *file,
|
|
|
|
loff_t start,
|
|
|
|
loff_t end,
|
|
|
|
int datasync)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(file->f_mapping->host);
|
|
|
|
|
|
|
|
trace_xfs_dir_fsync(ip);
|
2020-04-03 11:45:37 -07:00
|
|
|
return xfs_log_force_inode(ip);
|
2011-10-02 07:25:16 -07:00
|
|
|
}
|
|
|
|
|
2021-06-18 08:21:52 -07:00
|
|
|
static xfs_csn_t
|
|
|
|
xfs_fsync_seq(
|
2021-01-22 17:48:25 -07:00
|
|
|
struct xfs_inode *ip,
|
|
|
|
bool datasync)
|
|
|
|
{
|
|
|
|
if (!xfs_ipincount(ip))
|
|
|
|
return 0;
|
|
|
|
if (datasync && !(ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
|
|
|
|
return 0;
|
2021-06-18 08:21:52 -07:00
|
|
|
return ip->i_itemp->ili_commit_seq;
|
2021-01-22 17:48:25 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* All metadata updates are logged, which means that we just have to flush the
|
|
|
|
* log up to the latest LSN that touched the inode.
|
|
|
|
*
|
|
|
|
* If we have concurrent fsync/fdatasync() calls, we need them to all block on
|
|
|
|
* the log force before we clear the ili_fsync_fields field. This ensures that
|
|
|
|
* we don't get a racing sync operation that does not wait for the metadata to
|
|
|
|
* hit the journal before returning. If we race with clearing ili_fsync_fields,
|
|
|
|
* then all that will happen is the log force will do nothing as the lsn will
|
|
|
|
* already be on disk. We can't race with setting ili_fsync_fields because that
|
|
|
|
* is done under XFS_ILOCK_EXCL, and that can't happen because we hold the lock
|
|
|
|
* shared until after the ili_fsync_fields is cleared.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
xfs_fsync_flush_log(
|
|
|
|
struct xfs_inode *ip,
|
|
|
|
bool datasync,
|
|
|
|
int *log_flushed)
|
|
|
|
{
|
|
|
|
int error = 0;
|
2021-06-18 08:21:52 -07:00
|
|
|
xfs_csn_t seq;
|
2021-01-22 17:48:25 -07:00
|
|
|
|
|
|
|
xfs_ilock(ip, XFS_ILOCK_SHARED);
|
2021-06-18 08:21:52 -07:00
|
|
|
seq = xfs_fsync_seq(ip, datasync);
|
|
|
|
if (seq) {
|
|
|
|
error = xfs_log_force_seq(ip->i_mount, seq, XFS_LOG_SYNC,
|
2021-01-22 17:48:25 -07:00
|
|
|
log_flushed);
|
|
|
|
|
|
|
|
spin_lock(&ip->i_itemp->ili_lock);
|
|
|
|
ip->i_itemp->ili_fsync_fields = 0;
|
|
|
|
spin_unlock(&ip->i_itemp->ili_lock);
|
|
|
|
}
|
|
|
|
xfs_iunlock(ip, XFS_ILOCK_SHARED);
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
2010-02-15 02:44:48 -07:00
|
|
|
STATIC int
|
|
|
|
xfs_file_fsync(
|
|
|
|
struct file *file,
|
2011-07-16 17:44:56 -07:00
|
|
|
loff_t start,
|
|
|
|
loff_t end,
|
2010-02-15 02:44:48 -07:00
|
|
|
int datasync)
|
|
|
|
{
|
2021-01-22 17:48:25 -07:00
|
|
|
struct xfs_inode *ip = XFS_I(file->f_mapping->host);
|
2011-06-16 05:02:23 -07:00
|
|
|
struct xfs_mount *mp = ip->i_mount;
|
2022-07-28 13:35:31 -07:00
|
|
|
int error, err2;
|
2010-02-15 02:44:48 -07:00
|
|
|
int log_flushed = 0;
|
|
|
|
|
2010-06-23 18:57:09 -07:00
|
|
|
trace_xfs_file_fsync(ip);
|
2010-02-15 02:44:48 -07:00
|
|
|
|
2017-07-06 04:02:30 -07:00
|
|
|
error = file_write_and_wait_range(file, start, end);
|
2011-07-16 17:44:56 -07:00
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
2021-08-18 18:46:53 -07:00
|
|
|
if (xfs_is_shutdown(mp))
|
2014-06-21 22:04:54 -07:00
|
|
|
return -EIO;
|
2010-02-15 02:44:48 -07:00
|
|
|
|
|
|
|
xfs_iflags_clear(ip, XFS_ITRUNCATED);
|
|
|
|
|
2016-12-08 22:49:54 -07:00
|
|
|
/*
|
|
|
|
* If we have an RT and/or log subvolume we need to make sure to flush
|
|
|
|
* the write cache the device used for file data first. This is to
|
|
|
|
* ensure newly written file data make it to disk before logging the new
|
|
|
|
* inode size in case of an extending write.
|
|
|
|
*/
|
|
|
|
if (XFS_IS_REALTIME_INODE(ip))
|
2022-07-28 13:35:31 -07:00
|
|
|
error = blkdev_issue_flush(mp->m_rtdev_targp->bt_bdev);
|
2016-12-08 22:49:54 -07:00
|
|
|
else if (mp->m_logdev_targp != mp->m_ddev_targp)
|
2022-07-28 13:35:31 -07:00
|
|
|
error = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
|
2011-06-16 05:02:23 -07:00
|
|
|
|
2010-02-15 02:44:48 -07:00
|
|
|
/*
|
2021-01-22 17:48:25 -07:00
|
|
|
* Any inode that has dirty modifications in the log is pinned. The
|
2022-07-28 13:35:31 -07:00
|
|
|
* racy check here for a pinned inode will not catch modifications
|
2021-01-22 17:48:25 -07:00
|
|
|
* that happen concurrently to the fsync call, but fsync semantics
|
|
|
|
* only require to sync previously completed I/O.
|
2010-02-15 02:44:48 -07:00
|
|
|
*/
|
2022-07-28 13:35:31 -07:00
|
|
|
if (xfs_ipincount(ip)) {
|
|
|
|
err2 = xfs_fsync_flush_log(ip, datasync, &log_flushed);
|
|
|
|
if (err2 && !error)
|
|
|
|
error = err2;
|
|
|
|
}
|
2011-09-19 07:55:51 -07:00
|
|
|
|
2011-06-16 05:02:23 -07:00
|
|
|
/*
|
|
|
|
* If we only have a single device, and the log force about was
|
|
|
|
* a no-op we might have to flush the data device cache here.
|
|
|
|
* This can only happen for fdatasync/O_DSYNC if we were overwriting
|
|
|
|
* an already allocated file and thus do not have any metadata to
|
|
|
|
* commit.
|
|
|
|
*/
|
2016-12-08 22:49:54 -07:00
|
|
|
if (!log_flushed && !XFS_IS_REALTIME_INODE(ip) &&
|
2022-07-28 13:35:31 -07:00
|
|
|
mp->m_logdev_targp == mp->m_ddev_targp) {
|
|
|
|
err2 = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
|
|
|
|
if (err2 && !error)
|
|
|
|
error = err2;
|
|
|
|
}
|
2010-02-15 02:44:48 -07:00
|
|
|
|
2014-06-24 21:58:08 -07:00
|
|
|
return error;
|
2010-02-15 02:44:48 -07:00
|
|
|
}
|
|
|
|
|
2021-01-23 11:06:27 -07:00
|
|
|
static int
|
|
|
|
xfs_ilock_iocb(
|
|
|
|
struct kiocb *iocb,
|
|
|
|
unsigned int lock_mode)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
|
|
|
|
|
|
|
|
if (iocb->ki_flags & IOCB_NOWAIT) {
|
|
|
|
if (!xfs_ilock_nowait(ip, lock_mode))
|
|
|
|
return -EAGAIN;
|
|
|
|
} else {
|
|
|
|
xfs_ilock(ip, lock_mode);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2023-10-17 13:12:08 -07:00
|
|
|
static int
|
|
|
|
xfs_ilock_iocb_for_write(
|
|
|
|
struct kiocb *iocb,
|
|
|
|
unsigned int *lock_mode)
|
|
|
|
{
|
|
|
|
ssize_t ret;
|
|
|
|
struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
|
|
|
|
|
|
|
|
ret = xfs_ilock_iocb(iocb, *lock_mode);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2024-06-22 22:44:27 -07:00
|
|
|
/*
|
|
|
|
* If a reflink remap is in progress we always need to take the iolock
|
|
|
|
* exclusively to wait for it to finish.
|
|
|
|
*/
|
|
|
|
if (*lock_mode == XFS_IOLOCK_SHARED &&
|
|
|
|
xfs_iflags_test(ip, XFS_IREMAPPING)) {
|
|
|
|
xfs_iunlock(ip, *lock_mode);
|
|
|
|
*lock_mode = XFS_IOLOCK_EXCL;
|
|
|
|
return xfs_ilock_iocb(iocb, *lock_mode);
|
|
|
|
}
|
2023-10-17 13:12:08 -07:00
|
|
|
|
2024-06-22 22:44:27 -07:00
|
|
|
return 0;
|
2023-10-17 13:12:08 -07:00
|
|
|
}
|
|
|
|
|
2010-02-15 02:44:47 -07:00
|
|
|
STATIC ssize_t
|
2021-01-23 11:06:28 -07:00
|
|
|
xfs_file_dio_read(
|
2010-02-15 02:44:46 -07:00
|
|
|
struct kiocb *iocb,
|
2014-04-02 11:37:59 -07:00
|
|
|
struct iov_iter *to)
|
2010-02-15 02:44:46 -07:00
|
|
|
{
|
2016-11-29 20:37:15 -07:00
|
|
|
struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
|
|
|
|
ssize_t ret;
|
2010-02-15 02:44:46 -07:00
|
|
|
|
2021-01-23 11:06:29 -07:00
|
|
|
trace_xfs_file_direct_read(iocb, to);
|
2010-02-15 02:44:46 -07:00
|
|
|
|
2021-01-23 11:06:29 -07:00
|
|
|
if (!iov_iter_count(to))
|
2016-07-19 18:36:57 -07:00
|
|
|
return 0; /* skip atime */
|
2010-02-15 02:44:46 -07:00
|
|
|
|
2016-10-02 15:47:34 -07:00
|
|
|
file_accessed(iocb->ki_filp);
|
|
|
|
|
2021-01-23 11:06:27 -07:00
|
|
|
ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2022-05-05 13:11:11 -07:00
|
|
|
ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0, NULL, 0);
|
2016-11-29 20:33:25 -07:00
|
|
|
xfs_iunlock(ip, XFS_IOLOCK_SHARED);
|
2016-11-29 20:37:15 -07:00
|
|
|
|
2016-07-19 18:38:55 -07:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-07-21 16:50:55 -07:00
|
|
|
static noinline ssize_t
|
2016-07-19 18:38:55 -07:00
|
|
|
xfs_file_dax_read(
|
|
|
|
struct kiocb *iocb,
|
|
|
|
struct iov_iter *to)
|
|
|
|
{
|
2016-09-18 18:28:38 -07:00
|
|
|
struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host);
|
2016-07-19 18:38:55 -07:00
|
|
|
ssize_t ret = 0;
|
|
|
|
|
2021-01-23 11:06:29 -07:00
|
|
|
trace_xfs_file_dax_read(iocb, to);
|
2016-07-19 18:38:55 -07:00
|
|
|
|
2021-01-23 11:06:29 -07:00
|
|
|
if (!iov_iter_count(to))
|
2016-07-19 18:38:55 -07:00
|
|
|
return 0; /* skip atime */
|
|
|
|
|
2021-01-23 11:06:27 -07:00
|
|
|
ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2019-10-19 09:09:45 -07:00
|
|
|
ret = dax_iomap_rw(iocb, to, &xfs_read_iomap_ops);
|
2016-11-29 20:33:25 -07:00
|
|
|
xfs_iunlock(ip, XFS_IOLOCK_SHARED);
|
2016-07-19 18:35:42 -07:00
|
|
|
|
2016-07-19 18:36:57 -07:00
|
|
|
file_accessed(iocb->ki_filp);
|
2016-07-19 18:35:42 -07:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
STATIC ssize_t
|
2021-01-23 11:06:28 -07:00
|
|
|
xfs_file_buffered_read(
|
2016-07-19 18:35:42 -07:00
|
|
|
struct kiocb *iocb,
|
|
|
|
struct iov_iter *to)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
|
|
|
|
ssize_t ret;
|
|
|
|
|
2021-01-23 11:06:29 -07:00
|
|
|
trace_xfs_file_buffered_read(iocb, to);
|
2010-02-15 02:44:46 -07:00
|
|
|
|
2021-01-23 11:06:27 -07:00
|
|
|
ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2014-04-02 11:37:59 -07:00
|
|
|
ret = generic_file_read_iter(iocb, to);
|
2016-11-29 20:33:25 -07:00
|
|
|
xfs_iunlock(ip, XFS_IOLOCK_SHARED);
|
2016-07-19 18:35:42 -07:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
STATIC ssize_t
|
|
|
|
xfs_file_read_iter(
|
|
|
|
struct kiocb *iocb,
|
|
|
|
struct iov_iter *to)
|
|
|
|
{
|
2016-07-19 18:38:55 -07:00
|
|
|
struct inode *inode = file_inode(iocb->ki_filp);
|
|
|
|
struct xfs_mount *mp = XFS_I(inode)->i_mount;
|
2016-07-19 18:35:42 -07:00
|
|
|
ssize_t ret = 0;
|
|
|
|
|
|
|
|
XFS_STATS_INC(mp, xs_read_calls);
|
|
|
|
|
2021-08-18 18:46:53 -07:00
|
|
|
if (xfs_is_shutdown(mp))
|
2016-07-19 18:35:42 -07:00
|
|
|
return -EIO;
|
|
|
|
|
2016-07-19 18:38:55 -07:00
|
|
|
if (IS_DAX(inode))
|
|
|
|
ret = xfs_file_dax_read(iocb, to);
|
|
|
|
else if (iocb->ki_flags & IOCB_DIRECT)
|
2021-01-23 11:06:28 -07:00
|
|
|
ret = xfs_file_dio_read(iocb, to);
|
2016-07-19 18:31:42 -07:00
|
|
|
else
|
2021-01-23 11:06:28 -07:00
|
|
|
ret = xfs_file_buffered_read(iocb, to);
|
2010-02-15 02:44:46 -07:00
|
|
|
|
|
|
|
if (ret > 0)
|
2015-10-12 00:21:22 -07:00
|
|
|
XFS_STATS_ADD(mp, xs_read_bytes, ret);
|
2010-02-15 02:44:46 -07:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2023-05-22 06:50:11 -07:00
|
|
|
STATIC ssize_t
|
|
|
|
xfs_file_splice_read(
|
|
|
|
struct file *in,
|
|
|
|
loff_t *ppos,
|
|
|
|
struct pipe_inode_info *pipe,
|
|
|
|
size_t len,
|
|
|
|
unsigned int flags)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(in);
|
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
|
|
|
struct xfs_mount *mp = ip->i_mount;
|
|
|
|
ssize_t ret = 0;
|
|
|
|
|
|
|
|
XFS_STATS_INC(mp, xs_read_calls);
|
|
|
|
|
|
|
|
if (xfs_is_shutdown(mp))
|
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
trace_xfs_file_splice_read(ip, *ppos, len);
|
|
|
|
|
|
|
|
xfs_ilock(ip, XFS_IOLOCK_SHARED);
|
|
|
|
ret = filemap_splice_read(in, ppos, pipe, len, flags);
|
|
|
|
xfs_iunlock(ip, XFS_IOLOCK_SHARED);
|
|
|
|
if (ret > 0)
|
|
|
|
XFS_STATS_ADD(mp, xs_read_bytes, ret);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2024-10-08 01:59:15 -07:00
|
|
|
/*
|
|
|
|
* Take care of zeroing post-EOF blocks when they might exist.
|
|
|
|
*
|
|
|
|
* Returns 0 if successfully, a negative error for a failure, or 1 if this
|
|
|
|
* function dropped the iolock and reacquired it exclusively and the caller
|
|
|
|
* needs to restart the write sanity checks.
|
|
|
|
*/
|
|
|
|
static ssize_t
|
|
|
|
xfs_file_write_zero_eof(
|
|
|
|
struct kiocb *iocb,
|
|
|
|
struct iov_iter *from,
|
|
|
|
unsigned int *iolock,
|
|
|
|
size_t count,
|
|
|
|
bool *drained_dio)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host);
|
|
|
|
loff_t isize;
|
2024-10-08 01:59:16 -07:00
|
|
|
int error;
|
2024-10-08 01:59:15 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We need to serialise against EOF updates that occur in IO completions
|
|
|
|
* here. We want to make sure that nobody is changing the size while
|
|
|
|
* we do this check until we have placed an IO barrier (i.e. hold
|
|
|
|
* XFS_IOLOCK_EXCL) that prevents new IO from being dispatched. The
|
|
|
|
* spinlock effectively forms a memory barrier once we have
|
|
|
|
* XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value and
|
|
|
|
* hence be able to correctly determine if we need to run zeroing.
|
|
|
|
*/
|
|
|
|
spin_lock(&ip->i_flags_lock);
|
|
|
|
isize = i_size_read(VFS_I(ip));
|
|
|
|
if (iocb->ki_pos <= isize) {
|
|
|
|
spin_unlock(&ip->i_flags_lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
spin_unlock(&ip->i_flags_lock);
|
|
|
|
|
|
|
|
if (iocb->ki_flags & IOCB_NOWAIT)
|
|
|
|
return -EAGAIN;
|
|
|
|
|
|
|
|
if (!*drained_dio) {
|
|
|
|
/*
|
|
|
|
* If zeroing is needed and we are currently holding the iolock
|
|
|
|
* shared, we need to update it to exclusive which implies
|
|
|
|
* having to redo all checks before.
|
|
|
|
*/
|
|
|
|
if (*iolock == XFS_IOLOCK_SHARED) {
|
|
|
|
xfs_iunlock(ip, *iolock);
|
|
|
|
*iolock = XFS_IOLOCK_EXCL;
|
|
|
|
xfs_ilock(ip, *iolock);
|
|
|
|
iov_iter_reexpand(from, count);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We now have an IO submission barrier in place, but AIO can do
|
|
|
|
* EOF updates during IO completion and hence we now need to
|
|
|
|
* wait for all of them to drain. Non-AIO DIO will have drained
|
|
|
|
* before we are given the XFS_IOLOCK_EXCL, and so for most
|
|
|
|
* cases this wait is a no-op.
|
|
|
|
*/
|
|
|
|
inode_dio_wait(VFS_I(ip));
|
|
|
|
*drained_dio = true;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
trace_xfs_zero_eof(ip, isize, iocb->ki_pos - isize);
|
2024-10-08 01:59:16 -07:00
|
|
|
|
|
|
|
xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
|
|
|
|
error = xfs_zero_range(ip, isize, iocb->ki_pos - isize, NULL);
|
|
|
|
xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
|
|
|
|
|
|
|
|
return error;
|
2024-10-08 01:59:15 -07:00
|
|
|
}
|
|
|
|
|
2011-01-10 16:23:42 -07:00
|
|
|
/*
|
|
|
|
* Common pre-write limit and setup checks.
|
|
|
|
*
|
2024-10-08 01:59:15 -07:00
|
|
|
* Called with the iolock held either shared and exclusive according to
|
2011-12-18 13:00:13 -07:00
|
|
|
* @iolock, and returns with it held. Might upgrade the iolock to exclusive
|
|
|
|
* if called for a direct write beyond i_size.
|
2011-01-10 16:23:42 -07:00
|
|
|
*/
|
|
|
|
STATIC ssize_t
|
2021-01-23 11:06:28 -07:00
|
|
|
xfs_file_write_checks(
|
2015-04-07 11:25:18 -07:00
|
|
|
struct kiocb *iocb,
|
|
|
|
struct iov_iter *from,
|
2022-04-20 17:47:16 -07:00
|
|
|
unsigned int *iolock)
|
2011-01-10 16:23:42 -07:00
|
|
|
{
|
2024-10-08 01:59:15 -07:00
|
|
|
struct inode *inode = iocb->ki_filp->f_mapping->host;
|
2015-04-07 11:25:18 -07:00
|
|
|
size_t count = iov_iter_count(from);
|
2015-10-11 22:02:05 -07:00
|
|
|
bool drained_dio = false;
|
2024-10-08 01:59:15 -07:00
|
|
|
ssize_t error;
|
2011-01-10 16:23:42 -07:00
|
|
|
|
xfs: don't serialise adjacent concurrent direct IO appending writes
For append write workloads, extending the file requires a certain
amount of exclusive locking to be done up front to ensure sanity in
things like ensuring that we've zeroed any allocated regions
between the old EOF and the start of the new IO.
For single threads, this typically isn't a problem, and for large
IOs we don't serialise enough for it to be a problem for two
threads on really fast block devices. However for smaller IO and
larger thread counts we have a problem.
Take 4 concurrent sequential, single block sized and aligned IOs.
After the first IO is submitted but before it completes, we end up
with this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
And the IO is done without exclusive locking because offset <=
ip->i_size. When we submit IO 2, we see offset > ip->i_size, and
grab the IO lock exclusive, because there is a chance we need to do
EOF zeroing. However, there is already an IO in progress that avoids
the need for IO zeroing because offset <= ip->i_new_size. hence we
could avoid holding the IO lock exlcusive for this. Hence after
submission of the second IO, we'd end up this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
There is no need to grab the i_mutex of the IO lock in exclusive
mode if we don't need to invalidate the page cache. Taking these
locks on every direct IO effective serialises them as taking the IO
lock in exclusive mode has to wait for all shared holders to drop
the lock. That only happens when IO is complete, so effective it
prevents dispatch of concurrent direct IO writes to the same inode.
And so you can see that for the third concurrent IO, we'd avoid
exclusive locking for the same reason we avoided the exclusive lock
for the second IO.
Fixing this is a bit more complex than that, because we need to hold
a write-submission local value of ip->i_new_size to that clearing
the value is only done if no other thread has updated it before our
IO completes.....
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Alex Elder <aelder@sgi.com>
2011-08-25 00:17:02 -07:00
|
|
|
restart:
|
2015-04-09 09:55:47 -07:00
|
|
|
error = generic_write_checks(iocb, from);
|
|
|
|
if (error <= 0)
|
2011-01-10 16:23:42 -07:00
|
|
|
return error;
|
|
|
|
|
2021-01-23 11:06:28 -07:00
|
|
|
if (iocb->ki_flags & IOCB_NOWAIT) {
|
|
|
|
error = break_layout(inode, false);
|
|
|
|
if (error == -EWOULDBLOCK)
|
|
|
|
error = -EAGAIN;
|
|
|
|
} else {
|
|
|
|
error = xfs_break_layouts(inode, iolock, BREAK_WRITE);
|
|
|
|
}
|
|
|
|
|
2015-02-15 17:59:50 -07:00
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
2016-11-29 20:33:25 -07:00
|
|
|
/*
|
|
|
|
* For changing security info in file_remove_privs() we need i_rwsem
|
|
|
|
* exclusively.
|
|
|
|
*/
|
2015-05-21 07:05:56 -07:00
|
|
|
if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
|
2024-10-08 01:59:15 -07:00
|
|
|
xfs_iunlock(XFS_I(inode), *iolock);
|
2015-05-21 07:05:56 -07:00
|
|
|
*iolock = XFS_IOLOCK_EXCL;
|
2021-01-23 11:06:28 -07:00
|
|
|
error = xfs_ilock_iocb(iocb, *iolock);
|
|
|
|
if (error) {
|
|
|
|
*iolock = 0;
|
|
|
|
return error;
|
|
|
|
}
|
2015-05-21 07:05:56 -07:00
|
|
|
goto restart;
|
|
|
|
}
|
2021-06-02 15:00:38 -07:00
|
|
|
|
2011-01-10 16:23:42 -07:00
|
|
|
/*
|
2024-10-08 01:59:15 -07:00
|
|
|
* If the offset is beyond the size of the file, we need to zero all
|
2011-01-10 16:23:42 -07:00
|
|
|
* blocks that fall between the existing EOF and the start of this
|
2024-10-08 01:59:15 -07:00
|
|
|
* write.
|
2015-04-16 05:03:07 -07:00
|
|
|
*
|
2024-10-08 01:59:15 -07:00
|
|
|
* We can do an unlocked check for i_size here safely as I/O completion
|
|
|
|
* can only extend EOF. Truncate is locked out at this point, so the
|
|
|
|
* EOF can not move backwards, only forwards. Hence we only need to take
|
|
|
|
* the slow path when we are at or beyond the current EOF.
|
2011-01-10 16:23:42 -07:00
|
|
|
*/
|
2024-10-08 01:59:15 -07:00
|
|
|
if (iocb->ki_pos > i_size_read(inode)) {
|
|
|
|
error = xfs_file_write_zero_eof(iocb, from, iolock, count,
|
|
|
|
&drained_dio);
|
|
|
|
if (error == 1)
|
xfs: don't serialise adjacent concurrent direct IO appending writes
For append write workloads, extending the file requires a certain
amount of exclusive locking to be done up front to ensure sanity in
things like ensuring that we've zeroed any allocated regions
between the old EOF and the start of the new IO.
For single threads, this typically isn't a problem, and for large
IOs we don't serialise enough for it to be a problem for two
threads on really fast block devices. However for smaller IO and
larger thread counts we have a problem.
Take 4 concurrent sequential, single block sized and aligned IOs.
After the first IO is submitted but before it completes, we end up
with this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
And the IO is done without exclusive locking because offset <=
ip->i_size. When we submit IO 2, we see offset > ip->i_size, and
grab the IO lock exclusive, because there is a chance we need to do
EOF zeroing. However, there is already an IO in progress that avoids
the need for IO zeroing because offset <= ip->i_new_size. hence we
could avoid holding the IO lock exlcusive for this. Hence after
submission of the second IO, we'd end up this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
There is no need to grab the i_mutex of the IO lock in exclusive
mode if we don't need to invalidate the page cache. Taking these
locks on every direct IO effective serialises them as taking the IO
lock in exclusive mode has to wait for all shared holders to drop
the lock. That only happens when IO is complete, so effective it
prevents dispatch of concurrent direct IO writes to the same inode.
And so you can see that for the third concurrent IO, we'd avoid
exclusive locking for the same reason we avoided the exclusive lock
for the second IO.
Fixing this is a bit more complex than that, because we need to hold
a write-submission local value of ip->i_new_size to that clearing
the value is only done if no other thread has updated it before our
IO completes.....
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Alex Elder <aelder@sgi.com>
2011-08-25 00:17:02 -07:00
|
|
|
goto restart;
|
2012-03-27 07:34:47 -07:00
|
|
|
if (error)
|
|
|
|
return error;
|
2024-10-08 01:59:15 -07:00
|
|
|
}
|
2011-01-10 16:23:42 -07:00
|
|
|
|
2022-06-23 10:51:57 -07:00
|
|
|
return kiocb_modified(iocb);
|
2011-01-10 16:23:42 -07:00
|
|
|
}
|
|
|
|
|
2016-11-29 20:37:15 -07:00
|
|
|
static int
|
|
|
|
xfs_dio_write_end_io(
|
|
|
|
struct kiocb *iocb,
|
|
|
|
ssize_t size,
|
2019-09-19 15:32:44 -07:00
|
|
|
int error,
|
2016-11-29 20:37:15 -07:00
|
|
|
unsigned flags)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(iocb->ki_filp);
|
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
|
|
|
loff_t offset = iocb->ki_pos;
|
2019-06-28 19:31:38 -07:00
|
|
|
unsigned int nofs_flag;
|
2016-11-29 20:37:15 -07:00
|
|
|
|
|
|
|
trace_xfs_end_io_direct_write(ip, offset, size);
|
|
|
|
|
2021-08-18 18:46:53 -07:00
|
|
|
if (xfs_is_shutdown(ip->i_mount))
|
2016-11-29 20:37:15 -07:00
|
|
|
return -EIO;
|
|
|
|
|
2019-09-19 15:32:44 -07:00
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
if (!size)
|
|
|
|
return 0;
|
2016-11-29 20:37:15 -07:00
|
|
|
|
2018-05-02 12:54:52 -07:00
|
|
|
/*
|
|
|
|
* Capture amount written on completion as we can't reliably account
|
|
|
|
* for it on submission.
|
|
|
|
*/
|
|
|
|
XFS_STATS_ADD(ip->i_mount, xs_write_bytes, size);
|
|
|
|
|
2019-06-28 19:31:38 -07:00
|
|
|
/*
|
|
|
|
* We can allocate memory here while doing writeback on behalf of
|
|
|
|
* memory reclaim. To avoid memory allocation deadlocks set the
|
|
|
|
* task-wide nofs context for the following operations.
|
|
|
|
*/
|
|
|
|
nofs_flag = memalloc_nofs_save();
|
|
|
|
|
2017-09-21 11:26:18 -07:00
|
|
|
if (flags & IOMAP_DIO_COW) {
|
|
|
|
error = xfs_reflink_end_cow(ip, offset, size);
|
|
|
|
if (error)
|
2019-06-28 19:31:38 -07:00
|
|
|
goto out;
|
2017-09-21 11:26:18 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Unwritten conversion updates the in-core isize after extent
|
|
|
|
* conversion but before updating the on-disk size. Updating isize any
|
|
|
|
* earlier allows a racing dio read to find unwritten extents before
|
|
|
|
* they are converted.
|
|
|
|
*/
|
2019-06-28 19:31:38 -07:00
|
|
|
if (flags & IOMAP_DIO_UNWRITTEN) {
|
|
|
|
error = xfs_iomap_write_unwritten(ip, offset, size, true);
|
|
|
|
goto out;
|
|
|
|
}
|
2017-09-21 11:26:18 -07:00
|
|
|
|
2016-11-29 20:37:15 -07:00
|
|
|
/*
|
|
|
|
* We need to update the in-core inode size here so that we don't end up
|
|
|
|
* with the on-disk inode size being outside the in-core inode size. We
|
|
|
|
* have no other method of updating EOF for AIO, so always do it here
|
|
|
|
* if necessary.
|
|
|
|
*
|
|
|
|
* We need to lock the test/set EOF update as we can be racing with
|
|
|
|
* other IO completions here to update the EOF. Failing to serialise
|
|
|
|
* here can result in EOF moving backwards and Bad Things Happen when
|
|
|
|
* that occurs.
|
2021-06-02 15:00:38 -07:00
|
|
|
*
|
|
|
|
* As IO completion only ever extends EOF, we can do an unlocked check
|
|
|
|
* here to avoid taking the spinlock. If we land within the current EOF,
|
|
|
|
* then we do not need to do an extending update at all, and we don't
|
|
|
|
* need to take the lock to check this. If we race with an update moving
|
|
|
|
* EOF, then we'll either still be beyond EOF and need to take the lock,
|
|
|
|
* or we'll be within EOF and we don't need to take it at all.
|
2016-11-29 20:37:15 -07:00
|
|
|
*/
|
2021-06-02 15:00:38 -07:00
|
|
|
if (offset + size <= i_size_read(inode))
|
|
|
|
goto out;
|
|
|
|
|
2016-11-29 20:37:15 -07:00
|
|
|
spin_lock(&ip->i_flags_lock);
|
|
|
|
if (offset + size > i_size_read(inode)) {
|
|
|
|
i_size_write(inode, offset + size);
|
2017-09-21 11:26:18 -07:00
|
|
|
spin_unlock(&ip->i_flags_lock);
|
2016-11-29 20:37:15 -07:00
|
|
|
error = xfs_setfilesize(ip, offset, size);
|
2017-09-21 11:26:18 -07:00
|
|
|
} else {
|
|
|
|
spin_unlock(&ip->i_flags_lock);
|
|
|
|
}
|
2016-11-29 20:37:15 -07:00
|
|
|
|
2019-06-28 19:31:38 -07:00
|
|
|
out:
|
|
|
|
memalloc_nofs_restore(nofs_flag);
|
2016-11-29 20:37:15 -07:00
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
2019-09-19 15:32:45 -07:00
|
|
|
static const struct iomap_dio_ops xfs_dio_write_ops = {
|
|
|
|
.end_io = xfs_dio_write_end_io,
|
|
|
|
};
|
|
|
|
|
2011-01-10 16:15:36 -07:00
|
|
|
/*
|
2021-01-23 11:06:30 -07:00
|
|
|
* Handle block aligned direct I/O writes
|
2011-01-10 16:15:36 -07:00
|
|
|
*/
|
2021-01-23 11:06:30 -07:00
|
|
|
static noinline ssize_t
|
|
|
|
xfs_file_dio_write_aligned(
|
|
|
|
struct xfs_inode *ip,
|
2011-01-10 16:15:36 -07:00
|
|
|
struct kiocb *iocb,
|
2014-04-02 04:06:30 -07:00
|
|
|
struct iov_iter *from)
|
2011-01-10 16:15:36 -07:00
|
|
|
{
|
2022-04-20 17:47:16 -07:00
|
|
|
unsigned int iolock = XFS_IOLOCK_SHARED;
|
2021-01-23 11:06:30 -07:00
|
|
|
ssize_t ret;
|
2011-01-10 16:15:36 -07:00
|
|
|
|
2023-10-17 13:12:08 -07:00
|
|
|
ret = xfs_ilock_iocb_for_write(iocb, &iolock);
|
2021-01-23 11:06:30 -07:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
ret = xfs_file_write_checks(iocb, from, &iolock);
|
|
|
|
if (ret)
|
|
|
|
goto out_unlock;
|
2011-01-10 16:15:36 -07:00
|
|
|
|
xfs: don't serialise adjacent concurrent direct IO appending writes
For append write workloads, extending the file requires a certain
amount of exclusive locking to be done up front to ensure sanity in
things like ensuring that we've zeroed any allocated regions
between the old EOF and the start of the new IO.
For single threads, this typically isn't a problem, and for large
IOs we don't serialise enough for it to be a problem for two
threads on really fast block devices. However for smaller IO and
larger thread counts we have a problem.
Take 4 concurrent sequential, single block sized and aligned IOs.
After the first IO is submitted but before it completes, we end up
with this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
And the IO is done without exclusive locking because offset <=
ip->i_size. When we submit IO 2, we see offset > ip->i_size, and
grab the IO lock exclusive, because there is a chance we need to do
EOF zeroing. However, there is already an IO in progress that avoids
the need for IO zeroing because offset <= ip->i_new_size. hence we
could avoid holding the IO lock exlcusive for this. Hence after
submission of the second IO, we'd end up this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
There is no need to grab the i_mutex of the IO lock in exclusive
mode if we don't need to invalidate the page cache. Taking these
locks on every direct IO effective serialises them as taking the IO
lock in exclusive mode has to wait for all shared holders to drop
the lock. That only happens when IO is complete, so effective it
prevents dispatch of concurrent direct IO writes to the same inode.
And so you can see that for the third concurrent IO, we'd avoid
exclusive locking for the same reason we avoided the exclusive lock
for the second IO.
Fixing this is a bit more complex than that, because we need to hold
a write-submission local value of ip->i_new_size to that clearing
the value is only done if no other thread has updated it before our
IO completes.....
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Alex Elder <aelder@sgi.com>
2011-08-25 00:17:02 -07:00
|
|
|
/*
|
2021-01-23 11:06:30 -07:00
|
|
|
* We don't need to hold the IOLOCK exclusively across the IO, so demote
|
|
|
|
* the iolock back to shared if we had to take the exclusive lock in
|
|
|
|
* xfs_file_write_checks() for other reasons.
|
xfs: don't serialise adjacent concurrent direct IO appending writes
For append write workloads, extending the file requires a certain
amount of exclusive locking to be done up front to ensure sanity in
things like ensuring that we've zeroed any allocated regions
between the old EOF and the start of the new IO.
For single threads, this typically isn't a problem, and for large
IOs we don't serialise enough for it to be a problem for two
threads on really fast block devices. However for smaller IO and
larger thread counts we have a problem.
Take 4 concurrent sequential, single block sized and aligned IOs.
After the first IO is submitted but before it completes, we end up
with this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
And the IO is done without exclusive locking because offset <=
ip->i_size. When we submit IO 2, we see offset > ip->i_size, and
grab the IO lock exclusive, because there is a chance we need to do
EOF zeroing. However, there is already an IO in progress that avoids
the need for IO zeroing because offset <= ip->i_new_size. hence we
could avoid holding the IO lock exlcusive for this. Hence after
submission of the second IO, we'd end up this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
There is no need to grab the i_mutex of the IO lock in exclusive
mode if we don't need to invalidate the page cache. Taking these
locks on every direct IO effective serialises them as taking the IO
lock in exclusive mode has to wait for all shared holders to drop
the lock. That only happens when IO is complete, so effective it
prevents dispatch of concurrent direct IO writes to the same inode.
And so you can see that for the third concurrent IO, we'd avoid
exclusive locking for the same reason we avoided the exclusive lock
for the second IO.
Fixing this is a bit more complex than that, because we need to hold
a write-submission local value of ip->i_new_size to that clearing
the value is only done if no other thread has updated it before our
IO completes.....
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Alex Elder <aelder@sgi.com>
2011-08-25 00:17:02 -07:00
|
|
|
*/
|
2021-01-23 11:06:30 -07:00
|
|
|
if (iolock == XFS_IOLOCK_EXCL) {
|
|
|
|
xfs_ilock_demote(ip, XFS_IOLOCK_EXCL);
|
2011-12-18 13:00:14 -07:00
|
|
|
iolock = XFS_IOLOCK_SHARED;
|
2011-08-27 07:42:53 -07:00
|
|
|
}
|
2021-01-23 11:06:30 -07:00
|
|
|
trace_xfs_file_direct_write(iocb, from);
|
|
|
|
ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
|
2022-05-05 13:11:11 -07:00
|
|
|
&xfs_dio_write_ops, 0, NULL, 0);
|
2021-01-23 11:06:30 -07:00
|
|
|
out_unlock:
|
|
|
|
if (iolock)
|
|
|
|
xfs_iunlock(ip, iolock);
|
|
|
|
return ret;
|
|
|
|
}
|
2011-01-10 16:15:36 -07:00
|
|
|
|
2021-01-23 11:06:30 -07:00
|
|
|
/*
|
|
|
|
* Handle block unaligned direct I/O writes
|
|
|
|
*
|
|
|
|
* In most cases direct I/O writes will be done holding IOLOCK_SHARED, allowing
|
|
|
|
* them to be done in parallel with reads and other direct I/O writes. However,
|
|
|
|
* if the I/O is not aligned to filesystem blocks, the direct I/O layer may need
|
|
|
|
* to do sub-block zeroing and that requires serialisation against other direct
|
|
|
|
* I/O to the same block. In this case we need to serialise the submission of
|
|
|
|
* the unaligned I/O so that we don't get racing block zeroing in the dio layer.
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
* In the case where sub-block zeroing is not required, we can do concurrent
|
|
|
|
* sub-block dios to the same block successfully.
|
2021-01-23 11:06:30 -07:00
|
|
|
*
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
* Optimistically submit the I/O using the shared lock first, but use the
|
|
|
|
* IOMAP_DIO_OVERWRITE_ONLY flag to tell the lower layers to return -EAGAIN
|
|
|
|
* if block allocation or partial block zeroing would be required. In that case
|
|
|
|
* we try again with the exclusive lock.
|
2021-01-23 11:06:30 -07:00
|
|
|
*/
|
|
|
|
static noinline ssize_t
|
|
|
|
xfs_file_dio_write_unaligned(
|
|
|
|
struct xfs_inode *ip,
|
|
|
|
struct kiocb *iocb,
|
|
|
|
struct iov_iter *from)
|
|
|
|
{
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
size_t isize = i_size_read(VFS_I(ip));
|
|
|
|
size_t count = iov_iter_count(from);
|
2022-04-20 17:47:16 -07:00
|
|
|
unsigned int iolock = XFS_IOLOCK_SHARED;
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
unsigned int flags = IOMAP_DIO_OVERWRITE_ONLY;
|
2021-01-23 11:06:30 -07:00
|
|
|
ssize_t ret;
|
|
|
|
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
/*
|
|
|
|
* Extending writes need exclusivity because of the sub-block zeroing
|
|
|
|
* that the DIO code always does for partial tail blocks beyond EOF, so
|
|
|
|
* don't even bother trying the fast path in this case.
|
|
|
|
*/
|
|
|
|
if (iocb->ki_pos > isize || iocb->ki_pos + count >= isize) {
|
|
|
|
if (iocb->ki_flags & IOCB_NOWAIT)
|
|
|
|
return -EAGAIN;
|
2022-05-21 23:47:11 -07:00
|
|
|
retry_exclusive:
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
iolock = XFS_IOLOCK_EXCL;
|
|
|
|
flags = IOMAP_DIO_FORCE_WAIT;
|
|
|
|
}
|
|
|
|
|
2023-10-17 13:12:08 -07:00
|
|
|
ret = xfs_ilock_iocb_for_write(iocb, &iolock);
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
2021-01-23 11:06:30 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We can't properly handle unaligned direct I/O to reflink files yet,
|
|
|
|
* as we can't unshare a partial block.
|
|
|
|
*/
|
|
|
|
if (xfs_is_cow_inode(ip)) {
|
|
|
|
trace_xfs_reflink_bounce_dio_write(iocb, from);
|
|
|
|
ret = -ENOTBLK;
|
|
|
|
goto out_unlock;
|
2017-06-20 05:05:48 -07:00
|
|
|
}
|
2016-10-19 21:44:14 -07:00
|
|
|
|
2021-01-23 11:06:28 -07:00
|
|
|
ret = xfs_file_write_checks(iocb, from, &iolock);
|
2011-01-10 16:23:42 -07:00
|
|
|
if (ret)
|
2021-01-23 11:06:30 -07:00
|
|
|
goto out_unlock;
|
2011-01-10 16:15:36 -07:00
|
|
|
|
2011-01-10 16:22:40 -07:00
|
|
|
/*
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
* If we are doing exclusive unaligned I/O, this must be the only I/O
|
|
|
|
* in-flight. Otherwise we risk data corruption due to unwritten extent
|
|
|
|
* conversions from the AIO end_io handler. Wait for all other I/O to
|
|
|
|
* drain first.
|
2011-01-10 16:22:40 -07:00
|
|
|
*/
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
if (flags & IOMAP_DIO_FORCE_WAIT)
|
|
|
|
inode_dio_wait(VFS_I(ip));
|
2011-01-10 16:15:36 -07:00
|
|
|
|
2021-01-23 11:06:29 -07:00
|
|
|
trace_xfs_file_direct_write(iocb, from);
|
2019-10-19 09:09:46 -07:00
|
|
|
ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
|
2022-05-05 13:11:11 -07:00
|
|
|
&xfs_dio_write_ops, flags, NULL, 0);
|
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 11:06:31 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Retry unaligned I/O with exclusive blocking semantics if the DIO
|
|
|
|
* layer rejected it for mapping or locking reasons. If we are doing
|
|
|
|
* nonblocking user I/O, propagate the error.
|
|
|
|
*/
|
|
|
|
if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT)) {
|
|
|
|
ASSERT(flags & IOMAP_DIO_OVERWRITE_ONLY);
|
|
|
|
xfs_iunlock(ip, iolock);
|
|
|
|
goto retry_exclusive;
|
|
|
|
}
|
|
|
|
|
2021-01-23 11:06:30 -07:00
|
|
|
out_unlock:
|
2021-01-23 11:06:28 -07:00
|
|
|
if (iolock)
|
|
|
|
xfs_iunlock(ip, iolock);
|
2016-07-19 18:38:55 -07:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2021-01-23 11:06:30 -07:00
|
|
|
static ssize_t
|
|
|
|
xfs_file_dio_write(
|
|
|
|
struct kiocb *iocb,
|
|
|
|
struct iov_iter *from)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
|
|
|
|
struct xfs_buftarg *target = xfs_inode_buftarg(ip);
|
|
|
|
size_t count = iov_iter_count(from);
|
|
|
|
|
|
|
|
/* direct I/O must be aligned to device logical sector size */
|
|
|
|
if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
|
|
|
|
return -EINVAL;
|
|
|
|
if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask)
|
|
|
|
return xfs_file_dio_write_unaligned(ip, iocb, from);
|
|
|
|
return xfs_file_dio_write_aligned(ip, iocb, from);
|
|
|
|
}
|
|
|
|
|
2016-07-21 16:50:55 -07:00
|
|
|
static noinline ssize_t
|
2016-07-19 18:38:55 -07:00
|
|
|
xfs_file_dax_write(
|
|
|
|
struct kiocb *iocb,
|
|
|
|
struct iov_iter *from)
|
|
|
|
{
|
2016-09-18 18:28:38 -07:00
|
|
|
struct inode *inode = iocb->ki_filp->f_mapping->host;
|
2016-07-19 18:38:55 -07:00
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
2022-04-20 17:47:16 -07:00
|
|
|
unsigned int iolock = XFS_IOLOCK_EXCL;
|
2016-09-18 18:28:38 -07:00
|
|
|
ssize_t ret, error = 0;
|
|
|
|
loff_t pos;
|
2016-07-19 18:38:55 -07:00
|
|
|
|
2021-01-23 11:06:27 -07:00
|
|
|
ret = xfs_ilock_iocb(iocb, iolock);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2021-01-23 11:06:28 -07:00
|
|
|
ret = xfs_file_write_checks(iocb, from, &iolock);
|
2016-07-19 18:38:55 -07:00
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
|
2016-09-18 18:28:38 -07:00
|
|
|
pos = iocb->ki_pos;
|
2016-08-16 15:31:33 -07:00
|
|
|
|
2021-01-23 11:06:29 -07:00
|
|
|
trace_xfs_file_dax_write(iocb, from);
|
2022-06-02 22:37:37 -07:00
|
|
|
ret = dax_iomap_rw(iocb, from, &xfs_dax_write_iomap_ops);
|
2016-09-18 18:28:38 -07:00
|
|
|
if (ret > 0 && iocb->ki_pos > i_size_read(inode)) {
|
|
|
|
i_size_write(inode, iocb->ki_pos);
|
|
|
|
error = xfs_setfilesize(ip, pos, ret);
|
2016-07-19 18:38:55 -07:00
|
|
|
}
|
|
|
|
out:
|
2021-01-23 11:06:28 -07:00
|
|
|
if (iolock)
|
|
|
|
xfs_iunlock(ip, iolock);
|
2018-05-02 12:54:52 -07:00
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
|
|
|
if (ret > 0) {
|
|
|
|
XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
|
|
|
|
|
|
|
|
/* Handle various SYNC-type writes */
|
|
|
|
ret = generic_write_sync(iocb, ret);
|
|
|
|
}
|
|
|
|
return ret;
|
2011-01-10 16:15:36 -07:00
|
|
|
}
|
|
|
|
|
2010-02-15 02:44:47 -07:00
|
|
|
STATIC ssize_t
|
2021-01-23 11:06:28 -07:00
|
|
|
xfs_file_buffered_write(
|
2010-02-15 02:44:46 -07:00
|
|
|
struct kiocb *iocb,
|
2014-04-02 04:06:30 -07:00
|
|
|
struct iov_iter *from)
|
2010-02-15 02:44:46 -07:00
|
|
|
{
|
2022-04-20 15:47:54 -07:00
|
|
|
struct inode *inode = iocb->ki_filp->f_mapping->host;
|
2010-02-15 02:44:47 -07:00
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
2011-01-10 16:17:30 -07:00
|
|
|
ssize_t ret;
|
2021-01-22 17:48:34 -07:00
|
|
|
bool cleared_space = false;
|
2022-04-20 17:47:16 -07:00
|
|
|
unsigned int iolock;
|
2010-02-15 02:44:46 -07:00
|
|
|
|
2017-01-28 00:22:56 -07:00
|
|
|
write_retry:
|
|
|
|
iolock = XFS_IOLOCK_EXCL;
|
2022-06-23 10:51:57 -07:00
|
|
|
ret = xfs_ilock_iocb(iocb, iolock);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2010-02-15 02:44:46 -07:00
|
|
|
|
2021-01-23 11:06:28 -07:00
|
|
|
ret = xfs_file_write_checks(iocb, from, &iolock);
|
2011-01-10 16:23:42 -07:00
|
|
|
if (ret)
|
2011-12-18 13:00:14 -07:00
|
|
|
goto out;
|
2010-02-15 02:44:46 -07:00
|
|
|
|
2021-01-23 11:06:29 -07:00
|
|
|
trace_xfs_file_buffered_write(iocb, from);
|
2019-10-19 09:09:46 -07:00
|
|
|
ret = iomap_file_buffered_write(iocb, from,
|
2024-08-27 03:51:36 -07:00
|
|
|
&xfs_buffered_write_iomap_ops, NULL);
|
2014-07-24 02:49:28 -07:00
|
|
|
|
2011-01-10 16:17:30 -07:00
|
|
|
/*
|
2014-07-24 02:49:28 -07:00
|
|
|
* If we hit a space limit, try to free up some lingering preallocated
|
|
|
|
* space before returning an error. In the case of ENOSPC, first try to
|
|
|
|
* write back all dirty inodes to free up some of the excess reserved
|
|
|
|
* metadata space. This reduces the chances that the eofblocks scan
|
|
|
|
* waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
|
|
|
|
* also behaves as a filter to prevent too many eofblocks scans from
|
2021-01-22 17:48:36 -07:00
|
|
|
* running at the same time. Use a synchronous scan to increase the
|
|
|
|
* effectiveness of the scan.
|
2011-01-10 16:17:30 -07:00
|
|
|
*/
|
2021-01-22 17:48:34 -07:00
|
|
|
if (ret == -EDQUOT && !cleared_space) {
|
2017-01-28 00:22:56 -07:00
|
|
|
xfs_iunlock(ip, iolock);
|
2021-06-07 09:34:51 -07:00
|
|
|
xfs_blockgc_free_quota(ip, XFS_ICWALK_FLAG_SYNC);
|
2021-01-22 17:48:36 -07:00
|
|
|
cleared_space = true;
|
|
|
|
goto write_retry;
|
2021-01-22 17:48:34 -07:00
|
|
|
} else if (ret == -ENOSPC && !cleared_space) {
|
2021-06-07 09:34:51 -07:00
|
|
|
struct xfs_icwalk icw = {0};
|
2014-07-24 02:49:28 -07:00
|
|
|
|
2021-01-22 17:48:34 -07:00
|
|
|
cleared_space = true;
|
2012-10-08 03:56:04 -07:00
|
|
|
xfs_flush_inodes(ip->i_mount);
|
2017-01-28 00:22:56 -07:00
|
|
|
|
|
|
|
xfs_iunlock(ip, iolock);
|
2021-06-07 09:34:51 -07:00
|
|
|
icw.icw_flags = XFS_ICWALK_FLAG_SYNC;
|
|
|
|
xfs_blockgc_free_space(ip->i_mount, &icw);
|
2012-10-08 03:56:04 -07:00
|
|
|
goto write_retry;
|
2010-02-15 02:44:46 -07:00
|
|
|
}
|
2011-12-18 13:00:14 -07:00
|
|
|
|
|
|
|
out:
|
2017-01-28 00:22:56 -07:00
|
|
|
if (iolock)
|
|
|
|
xfs_iunlock(ip, iolock);
|
2018-05-02 12:54:52 -07:00
|
|
|
|
|
|
|
if (ret > 0) {
|
|
|
|
XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
|
|
|
|
/* Handle various SYNC-type writes */
|
|
|
|
ret = generic_write_sync(iocb, ret);
|
|
|
|
}
|
2011-01-10 16:17:30 -07:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
STATIC ssize_t
|
2014-04-03 11:20:23 -07:00
|
|
|
xfs_file_write_iter(
|
2011-01-10 16:17:30 -07:00
|
|
|
struct kiocb *iocb,
|
2014-04-03 11:20:23 -07:00
|
|
|
struct iov_iter *from)
|
2011-01-10 16:17:30 -07:00
|
|
|
{
|
2022-04-20 15:47:54 -07:00
|
|
|
struct inode *inode = iocb->ki_filp->f_mapping->host;
|
2011-01-10 16:17:30 -07:00
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
|
|
|
ssize_t ret;
|
2014-04-03 11:20:23 -07:00
|
|
|
size_t ocount = iov_iter_count(from);
|
2011-01-10 16:17:30 -07:00
|
|
|
|
2015-10-12 00:21:22 -07:00
|
|
|
XFS_STATS_INC(ip->i_mount, xs_write_calls);
|
2011-01-10 16:17:30 -07:00
|
|
|
|
|
|
|
if (ocount == 0)
|
|
|
|
return 0;
|
|
|
|
|
2021-08-18 18:46:53 -07:00
|
|
|
if (xfs_is_shutdown(ip->i_mount))
|
2014-04-03 11:20:23 -07:00
|
|
|
return -EIO;
|
2011-01-10 16:17:30 -07:00
|
|
|
|
2016-07-19 18:38:55 -07:00
|
|
|
if (IS_DAX(inode))
|
2018-05-02 12:54:52 -07:00
|
|
|
return xfs_file_dax_write(iocb, from);
|
|
|
|
|
|
|
|
if (iocb->ki_flags & IOCB_DIRECT) {
|
2016-10-03 09:11:37 -07:00
|
|
|
/*
|
|
|
|
* Allow a directio write to fall back to a buffered
|
|
|
|
* write *only* in the case that we're doing a reflink
|
|
|
|
* CoW. In all other directio scenarios we do not
|
|
|
|
* allow an operation to fall back to buffered mode.
|
|
|
|
*/
|
2021-01-23 11:06:28 -07:00
|
|
|
ret = xfs_file_dio_write(iocb, from);
|
2020-07-23 22:45:58 -07:00
|
|
|
if (ret != -ENOTBLK)
|
2018-05-02 12:54:52 -07:00
|
|
|
return ret;
|
2016-10-03 09:11:37 -07:00
|
|
|
}
|
2010-02-15 02:44:46 -07:00
|
|
|
|
2021-01-23 11:06:28 -07:00
|
|
|
return xfs_file_buffered_write(iocb, from);
|
2010-02-15 02:44:46 -07:00
|
|
|
}
|
|
|
|
|
2022-01-31 14:20:10 -07:00
|
|
|
/* Does this file, inode, or mount want synchronous writes? */
|
|
|
|
static inline bool xfs_file_sync_writes(struct file *filp)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(file_inode(filp));
|
|
|
|
|
|
|
|
if (xfs_has_wsync(ip->i_mount))
|
|
|
|
return true;
|
|
|
|
if (filp->f_flags & (__O_SYNC | O_DSYNC))
|
|
|
|
return true;
|
|
|
|
if (IS_SYNC(file_inode(filp)))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2024-08-26 23:50:50 -07:00
|
|
|
static int
|
|
|
|
xfs_falloc_newsize(
|
|
|
|
struct file *file,
|
|
|
|
int mode,
|
|
|
|
loff_t offset,
|
|
|
|
loff_t len,
|
|
|
|
loff_t *new_size)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
|
|
|
|
if ((mode & FALLOC_FL_KEEP_SIZE) || offset + len <= i_size_read(inode))
|
|
|
|
return 0;
|
|
|
|
*new_size = offset + len;
|
|
|
|
return inode_newsize_ok(inode, *new_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
xfs_falloc_setsize(
|
|
|
|
struct file *file,
|
|
|
|
loff_t new_size)
|
|
|
|
{
|
|
|
|
struct iattr iattr = {
|
|
|
|
.ia_valid = ATTR_SIZE,
|
|
|
|
.ia_size = new_size,
|
|
|
|
};
|
|
|
|
|
|
|
|
if (!new_size)
|
|
|
|
return 0;
|
|
|
|
return xfs_vn_setattr_size(file_mnt_idmap(file), file_dentry(file),
|
|
|
|
&iattr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
xfs_falloc_collapse_range(
|
|
|
|
struct file *file,
|
|
|
|
loff_t offset,
|
|
|
|
loff_t len)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
loff_t new_size = i_size_read(inode) - len;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
if (!xfs_is_falloc_aligned(XFS_I(inode), offset, len))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* There is no need to overlap collapse range with EOF, in which case it
|
|
|
|
* is effectively a truncate operation
|
|
|
|
*/
|
|
|
|
if (offset + len >= i_size_read(inode))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
error = xfs_collapse_file_space(XFS_I(inode), offset, len);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
return xfs_falloc_setsize(file, new_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
xfs_falloc_insert_range(
|
|
|
|
struct file *file,
|
|
|
|
loff_t offset,
|
|
|
|
loff_t len)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
loff_t isize = i_size_read(inode);
|
|
|
|
int error;
|
|
|
|
|
|
|
|
if (!xfs_is_falloc_aligned(XFS_I(inode), offset, len))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* New inode size must not exceed ->s_maxbytes, accounting for
|
|
|
|
* possible signed overflow.
|
|
|
|
*/
|
|
|
|
if (inode->i_sb->s_maxbytes - isize < len)
|
|
|
|
return -EFBIG;
|
|
|
|
|
|
|
|
/* Offset should be less than i_size */
|
|
|
|
if (offset >= isize)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
error = xfs_falloc_setsize(file, isize + len);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform hole insertion now that the file size has been updated so
|
|
|
|
* that if we crash during the operation we don't leave shifted extents
|
|
|
|
* past EOF and hence losing access to the data that is contained within
|
|
|
|
* them.
|
|
|
|
*/
|
|
|
|
return xfs_insert_file_space(XFS_I(inode), offset, len);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Punch a hole and prealloc the range. We use a hole punch rather than
|
|
|
|
* unwritten extent conversion for two reasons:
|
|
|
|
*
|
|
|
|
* 1.) Hole punch handles partial block zeroing for us.
|
|
|
|
* 2.) If prealloc returns ENOSPC, the file range is still zero-valued by
|
|
|
|
* virtue of the hole punch.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
xfs_falloc_zero_range(
|
|
|
|
struct file *file,
|
|
|
|
int mode,
|
|
|
|
loff_t offset,
|
|
|
|
loff_t len)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
unsigned int blksize = i_blocksize(inode);
|
|
|
|
loff_t new_size = 0;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
trace_xfs_zero_file_space(XFS_I(inode));
|
|
|
|
|
|
|
|
error = xfs_falloc_newsize(file, mode, offset, len, &new_size);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
|
|
|
error = xfs_free_file_space(XFS_I(inode), offset, len);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
|
|
|
len = round_up(offset + len, blksize) - round_down(offset, blksize);
|
|
|
|
offset = round_down(offset, blksize);
|
|
|
|
error = xfs_alloc_file_space(XFS_I(inode), offset, len);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
return xfs_falloc_setsize(file, new_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
xfs_falloc_unshare_range(
|
|
|
|
struct file *file,
|
|
|
|
int mode,
|
|
|
|
loff_t offset,
|
|
|
|
loff_t len)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
loff_t new_size = 0;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
error = xfs_falloc_newsize(file, mode, offset, len, &new_size);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
|
|
|
error = xfs_reflink_unshare(XFS_I(inode), offset, len);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
|
|
|
error = xfs_alloc_file_space(XFS_I(inode), offset, len);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
return xfs_falloc_setsize(file, new_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
xfs_falloc_allocate_range(
|
|
|
|
struct file *file,
|
|
|
|
int mode,
|
|
|
|
loff_t offset,
|
|
|
|
loff_t len)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
loff_t new_size = 0;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If always_cow mode we can't use preallocations and thus should not
|
|
|
|
* create them.
|
|
|
|
*/
|
|
|
|
if (xfs_is_always_cow_inode(XFS_I(inode)))
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
|
|
|
error = xfs_falloc_newsize(file, mode, offset, len, &new_size);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
|
|
|
error = xfs_alloc_file_space(XFS_I(inode), offset, len);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
return xfs_falloc_setsize(file, new_size);
|
|
|
|
}
|
|
|
|
|
2015-03-24 21:08:56 -07:00
|
|
|
#define XFS_FALLOC_FL_SUPPORTED \
|
|
|
|
(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
|
|
|
|
FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | \
|
2016-10-03 09:11:43 -07:00
|
|
|
FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE)
|
2015-03-24 21:08:56 -07:00
|
|
|
|
2011-01-14 05:07:43 -07:00
|
|
|
STATIC long
|
|
|
|
xfs_file_fallocate(
|
2013-10-12 00:55:07 -07:00
|
|
|
struct file *file,
|
|
|
|
int mode,
|
|
|
|
loff_t offset,
|
|
|
|
loff_t len)
|
2011-01-14 05:07:43 -07:00
|
|
|
{
|
2013-10-12 00:55:07 -07:00
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
|
|
|
long error;
|
2018-03-12 14:12:29 -07:00
|
|
|
uint iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
|
2011-01-14 05:07:43 -07:00
|
|
|
|
2013-10-12 00:55:07 -07:00
|
|
|
if (!S_ISREG(inode->i_mode))
|
|
|
|
return -EINVAL;
|
2015-03-24 21:08:56 -07:00
|
|
|
if (mode & ~XFS_FALLOC_FL_SUPPORTED)
|
2011-01-14 05:07:43 -07:00
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
2015-02-15 17:59:50 -07:00
|
|
|
xfs_ilock(ip, iolock);
|
2018-03-20 14:42:38 -07:00
|
|
|
error = xfs_break_layouts(inode, &iolock, BREAK_UNMAP);
|
2015-02-15 17:59:50 -07:00
|
|
|
if (error)
|
|
|
|
goto out_unlock;
|
|
|
|
|
xfs: properly serialise fallocate against AIO+DIO
AIO+DIO can extend the file size on IO completion, and it holds
no inode locks while the IO is in flight. Therefore, a race
condition exists in file size updates if we do something like this:
aio-thread fallocate-thread
lock inode
submit IO beyond inode->i_size
unlock inode
.....
lock inode
break layouts
if (off + len > inode->i_size)
new_size = off + len
.....
inode_dio_wait()
<blocks>
.....
completes
inode->i_size updated
inode_dio_done()
....
<wakes>
<does stuff no long beyond EOF>
if (new_size)
xfs_vn_setattr(inode, new_size)
Yup, that attempt to extend the file size in the fallocate code
turns into a truncate - it removes the whatever the aio write
allocated and put to disk, and reduced the inode size back down to
where the fallocate operation ends.
Fundamentally, xfs_file_fallocate() not compatible with racing
AIO+DIO completions, so we need to move the inode_dio_wait() call
up to where the lock the inode and break the layouts.
Secondly, storing the inode size and then using it unchecked without
holding the ILOCK is not safe; we can only do such a thing if we've
locked out and drained all IO and other modification operations,
which we don't do initially in xfs_file_fallocate.
It should be noted that some of the fallocate operations are
compound operations - they are made up of multiple manipulations
that may zero data, and so we may need to flush and invalidate the
file multiple times during an operation. However, we only need to
lock out IO and other space manipulation operations once, as that
lockout is maintained until the entire fallocate operation has been
completed.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-10-29 13:04:32 -07:00
|
|
|
/*
|
|
|
|
* Must wait for all AIO to complete before we continue as AIO can
|
|
|
|
* change the file size on completion without holding any locks we
|
|
|
|
* currently hold. We must do this first because AIO can update both
|
|
|
|
* the on disk and in memory inode sizes, and the operations that follow
|
|
|
|
* require the in-memory size to be fully up-to-date.
|
|
|
|
*/
|
|
|
|
inode_dio_wait(inode);
|
|
|
|
|
2022-01-31 14:20:09 -07:00
|
|
|
error = file_modified(file);
|
|
|
|
if (error)
|
|
|
|
goto out_unlock;
|
|
|
|
|
2024-08-26 23:50:50 -07:00
|
|
|
switch (mode & FALLOC_FL_MODE_MASK) {
|
|
|
|
case FALLOC_FL_PUNCH_HOLE:
|
2013-10-12 00:55:07 -07:00
|
|
|
error = xfs_free_file_space(ip, offset, len);
|
2024-08-26 23:50:50 -07:00
|
|
|
break;
|
|
|
|
case FALLOC_FL_COLLAPSE_RANGE:
|
|
|
|
error = xfs_falloc_collapse_range(file, offset, len);
|
|
|
|
break;
|
|
|
|
case FALLOC_FL_INSERT_RANGE:
|
|
|
|
error = xfs_falloc_insert_range(file, offset, len);
|
|
|
|
break;
|
|
|
|
case FALLOC_FL_ZERO_RANGE:
|
|
|
|
error = xfs_falloc_zero_range(file, mode, offset, len);
|
|
|
|
break;
|
|
|
|
case FALLOC_FL_UNSHARE_RANGE:
|
|
|
|
error = xfs_falloc_unshare_range(file, mode, offset, len);
|
|
|
|
break;
|
|
|
|
case FALLOC_FL_ALLOCATE_RANGE:
|
|
|
|
error = xfs_falloc_allocate_range(file, mode, offset, len);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
error = -EOPNOTSUPP;
|
|
|
|
break;
|
2022-01-31 14:20:08 -07:00
|
|
|
}
|
|
|
|
|
2024-08-26 23:50:50 -07:00
|
|
|
if (!error && xfs_file_sync_writes(file))
|
2022-01-31 14:20:08 -07:00
|
|
|
error = xfs_log_force_inode(ip);
|
2015-03-24 21:08:56 -07:00
|
|
|
|
2011-01-14 05:07:43 -07:00
|
|
|
out_unlock:
|
2015-02-15 17:59:50 -07:00
|
|
|
xfs_iunlock(ip, iolock);
|
2014-06-24 21:58:08 -07:00
|
|
|
return error;
|
2011-01-14 05:07:43 -07:00
|
|
|
}
|
|
|
|
|
2019-08-29 09:04:12 -07:00
|
|
|
STATIC int
|
|
|
|
xfs_file_fadvise(
|
|
|
|
struct file *file,
|
|
|
|
loff_t start,
|
|
|
|
loff_t end,
|
|
|
|
int advice)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(file_inode(file));
|
|
|
|
int ret;
|
|
|
|
int lockflags = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Operations creating pages in page cache need protection from hole
|
|
|
|
* punching and similar ops
|
|
|
|
*/
|
|
|
|
if (advice == POSIX_FADV_WILLNEED) {
|
|
|
|
lockflags = XFS_IOLOCK_SHARED;
|
|
|
|
xfs_ilock(ip, lockflags);
|
|
|
|
}
|
|
|
|
ret = generic_fadvise(file, start, end, advice);
|
|
|
|
if (lockflags)
|
|
|
|
xfs_iunlock(ip, lockflags);
|
|
|
|
return ret;
|
|
|
|
}
|
2018-10-29 16:47:26 -07:00
|
|
|
|
2018-11-14 22:48:18 -07:00
|
|
|
STATIC loff_t
|
2018-10-29 16:41:21 -07:00
|
|
|
xfs_file_remap_range(
|
2018-10-29 16:47:26 -07:00
|
|
|
struct file *file_in,
|
|
|
|
loff_t pos_in,
|
|
|
|
struct file *file_out,
|
|
|
|
loff_t pos_out,
|
|
|
|
loff_t len,
|
|
|
|
unsigned int remap_flags)
|
2016-10-03 09:11:40 -07:00
|
|
|
{
|
2018-10-29 16:47:26 -07:00
|
|
|
struct inode *inode_in = file_inode(file_in);
|
|
|
|
struct xfs_inode *src = XFS_I(inode_in);
|
|
|
|
struct inode *inode_out = file_inode(file_out);
|
|
|
|
struct xfs_inode *dest = XFS_I(inode_out);
|
|
|
|
struct xfs_mount *mp = src->i_mount;
|
|
|
|
loff_t remapped = 0;
|
|
|
|
xfs_extlen_t cowextsize;
|
|
|
|
int ret;
|
|
|
|
|
2018-10-29 16:41:21 -07:00
|
|
|
if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY))
|
|
|
|
return -EINVAL;
|
2016-10-03 09:11:41 -07:00
|
|
|
|
2021-08-18 18:46:37 -07:00
|
|
|
if (!xfs_has_reflink(mp))
|
2018-10-29 16:47:26 -07:00
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
2021-08-18 18:46:53 -07:00
|
|
|
if (xfs_is_shutdown(mp))
|
2018-10-29 16:47:26 -07:00
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
/* Prepare and then clone file data. */
|
|
|
|
ret = xfs_reflink_remap_prep(file_in, pos_in, file_out, pos_out,
|
|
|
|
&len, remap_flags);
|
2020-06-29 14:47:20 -07:00
|
|
|
if (ret || len == 0)
|
2018-10-29 16:47:26 -07:00
|
|
|
return ret;
|
|
|
|
|
|
|
|
trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out);
|
|
|
|
|
|
|
|
ret = xfs_reflink_remap_blocks(src, pos_in, dest, pos_out, len,
|
|
|
|
&remapped);
|
|
|
|
if (ret)
|
|
|
|
goto out_unlock;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Carry the cowextsize hint from src to dest if we're sharing the
|
|
|
|
* entire source file to the entire destination file, the source file
|
|
|
|
* has a cowextsize hint, and the destination file does not.
|
|
|
|
*/
|
|
|
|
cowextsize = 0;
|
|
|
|
if (pos_in == 0 && len == i_size_read(inode_in) &&
|
2021-03-29 11:11:45 -07:00
|
|
|
(src->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) &&
|
2018-10-29 16:47:26 -07:00
|
|
|
pos_out == 0 && len >= i_size_read(inode_out) &&
|
2021-03-29 11:11:45 -07:00
|
|
|
!(dest->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE))
|
2021-03-29 11:11:42 -07:00
|
|
|
cowextsize = src->i_cowextsize;
|
2018-10-29 16:47:26 -07:00
|
|
|
|
|
|
|
ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize,
|
|
|
|
remap_flags);
|
2020-04-03 11:45:37 -07:00
|
|
|
if (ret)
|
|
|
|
goto out_unlock;
|
2018-10-29 16:47:26 -07:00
|
|
|
|
2020-09-04 10:20:16 -07:00
|
|
|
if (xfs_file_sync_writes(file_in) || xfs_file_sync_writes(file_out))
|
2020-04-03 11:45:37 -07:00
|
|
|
xfs_log_force_inode(dest);
|
2018-10-29 16:47:26 -07:00
|
|
|
out_unlock:
|
2023-10-17 13:12:08 -07:00
|
|
|
xfs_iunlock2_remapping(src, dest);
|
2018-10-29 16:47:26 -07:00
|
|
|
if (ret)
|
|
|
|
trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_);
|
|
|
|
return remapped > 0 ? remapped : ret;
|
2016-10-03 09:11:40 -07:00
|
|
|
}
|
2011-01-14 05:07:43 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
STATIC int
|
2006-03-13 20:00:35 -07:00
|
|
|
xfs_file_open(
|
2005-04-16 15:20:36 -07:00
|
|
|
struct inode *inode,
|
2008-11-27 20:23:32 -07:00
|
|
|
struct file *file)
|
2005-04-16 15:20:36 -07:00
|
|
|
{
|
2021-08-18 18:46:53 -07:00
|
|
|
if (xfs_is_shutdown(XFS_M(inode->i_sb)))
|
2008-11-27 20:23:32 -07:00
|
|
|
return -EIO;
|
2024-03-28 05:27:24 -07:00
|
|
|
file->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT;
|
2022-04-11 13:49:40 -07:00
|
|
|
return generic_file_open(inode, file);
|
2008-11-27 20:23:32 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
STATIC int
|
|
|
|
xfs_dir_open(
|
|
|
|
struct inode *inode,
|
|
|
|
struct file *file)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
2022-04-20 17:47:16 -07:00
|
|
|
unsigned int mode;
|
2008-11-27 20:23:32 -07:00
|
|
|
int error;
|
|
|
|
|
2024-04-23 05:46:08 -07:00
|
|
|
if (xfs_is_shutdown(ip->i_mount))
|
|
|
|
return -EIO;
|
|
|
|
error = generic_file_open(inode, file);
|
2008-11-27 20:23:32 -07:00
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there are any blocks, read-ahead block 0 as we're almost
|
|
|
|
* certain to have the next operation be a read there.
|
|
|
|
*/
|
2013-12-06 13:30:09 -07:00
|
|
|
mode = xfs_ilock_data_map_shared(ip);
|
2020-05-18 10:27:22 -07:00
|
|
|
if (ip->i_df.if_nextents > 0)
|
2019-11-20 10:46:02 -07:00
|
|
|
error = xfs_dir3_data_readahead(ip, 0, 0);
|
2008-11-27 20:23:32 -07:00
|
|
|
xfs_iunlock(ip, mode);
|
2017-02-02 16:13:58 -07:00
|
|
|
return error;
|
2005-04-16 15:20:36 -07:00
|
|
|
}
|
|
|
|
|
2024-08-13 00:39:36 -07:00
|
|
|
/*
|
|
|
|
* Don't bother propagating errors. We're just doing cleanup, and the caller
|
|
|
|
* ignores the return value anyway.
|
|
|
|
*/
|
2005-04-16 15:20:36 -07:00
|
|
|
STATIC int
|
2006-03-13 20:00:35 -07:00
|
|
|
xfs_file_release(
|
2024-08-13 00:39:35 -07:00
|
|
|
struct inode *inode,
|
|
|
|
struct file *file)
|
2005-04-16 15:20:36 -07:00
|
|
|
{
|
2024-08-13 00:39:35 -07:00
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
|
|
|
struct xfs_mount *mp = ip->i_mount;
|
|
|
|
|
2024-08-13 00:39:37 -07:00
|
|
|
/*
|
|
|
|
* If this is a read-only mount or the file system has been shut down,
|
|
|
|
* don't generate I/O.
|
|
|
|
*/
|
|
|
|
if (xfs_is_readonly(mp) || xfs_is_shutdown(mp))
|
2024-08-13 00:39:35 -07:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we previously truncated this file and removed old data in the
|
|
|
|
* process, we want to initiate "early" writeout on the last close.
|
|
|
|
* This is an attempt to combat the notorious NULL files problem which
|
|
|
|
* is particularly noticeable from a truncate down, buffered (re-)write
|
|
|
|
* (delalloc), followed by a crash. What we are effectively doing here
|
|
|
|
* is significantly reducing the time window where we'd otherwise be
|
|
|
|
* exposed to that problem.
|
|
|
|
*/
|
2024-08-13 00:39:37 -07:00
|
|
|
if (xfs_iflags_test_and_clear(ip, XFS_ITRUNCATED)) {
|
2024-08-13 00:39:39 -07:00
|
|
|
xfs_iflags_clear(ip, XFS_EOFBLOCKS_RELEASED);
|
2024-08-13 00:39:36 -07:00
|
|
|
if (ip->i_delayed_blks > 0)
|
|
|
|
filemap_flush(inode->i_mapping);
|
2024-08-13 00:39:35 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* XFS aggressively preallocates post-EOF space to generate contiguous
|
2024-08-13 00:39:39 -07:00
|
|
|
* allocations for writers that append to the end of the file.
|
|
|
|
*
|
|
|
|
* To support workloads that close and reopen the file frequently, these
|
|
|
|
* preallocations usually persist after a close unless it is the first
|
|
|
|
* close for the inode. This is a tradeoff to generate tightly packed
|
|
|
|
* data layouts for unpacking tarballs or similar archives that write
|
|
|
|
* one file after another without going back to it while keeping the
|
|
|
|
* preallocation for files that have recurring open/write/close cycles.
|
2024-08-13 00:39:35 -07:00
|
|
|
*
|
2024-08-13 00:39:42 -07:00
|
|
|
* This heuristic is skipped for inodes with the append-only flag as
|
|
|
|
* that flag is rather pointless for inodes written only once.
|
|
|
|
*
|
2024-08-13 00:39:35 -07:00
|
|
|
* There is no point in freeing blocks here for open but unlinked files
|
|
|
|
* as they will be taken care of by the inactivation path soon.
|
|
|
|
*
|
2024-08-13 00:39:38 -07:00
|
|
|
* When releasing a read-only context, don't flush data or trim post-EOF
|
|
|
|
* blocks. This avoids open/read/close workloads from removing EOF
|
|
|
|
* blocks that other writers depend upon to reduce fragmentation.
|
|
|
|
*
|
2024-08-13 00:39:35 -07:00
|
|
|
* If we can't get the iolock just skip truncating the blocks past EOF
|
|
|
|
* because we could deadlock with the mmap_lock otherwise. We'll get
|
|
|
|
* another chance to drop them once the last reference to the inode is
|
|
|
|
* dropped, so we'll never leak blocks permanently.
|
|
|
|
*/
|
2024-08-13 00:39:38 -07:00
|
|
|
if (inode->i_nlink &&
|
|
|
|
(file->f_mode & FMODE_WRITE) &&
|
2024-08-13 00:39:42 -07:00
|
|
|
!(ip->i_diflags & XFS_DIFLAG_APPEND) &&
|
2024-08-13 00:39:40 -07:00
|
|
|
!xfs_iflags_test(ip, XFS_EOFBLOCKS_RELEASED) &&
|
2024-08-13 00:39:38 -07:00
|
|
|
xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
|
2024-08-13 00:39:35 -07:00
|
|
|
if (xfs_can_free_eofblocks(ip) &&
|
2024-08-13 00:39:40 -07:00
|
|
|
!xfs_iflags_test_and_set(ip, XFS_EOFBLOCKS_RELEASED))
|
2024-08-13 00:39:36 -07:00
|
|
|
xfs_free_eofblocks(ip);
|
2024-08-13 00:39:35 -07:00
|
|
|
xfs_iunlock(ip, XFS_IOLOCK_EXCL);
|
|
|
|
}
|
|
|
|
|
2024-08-13 00:39:36 -07:00
|
|
|
return 0;
|
2005-04-16 15:20:36 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
STATIC int
|
2006-03-13 20:00:35 -07:00
|
|
|
xfs_file_readdir(
|
2013-05-22 14:07:56 -07:00
|
|
|
struct file *file,
|
|
|
|
struct dir_context *ctx)
|
2005-04-16 15:20:36 -07:00
|
|
|
{
|
2013-05-22 14:07:56 -07:00
|
|
|
struct inode *inode = file_inode(file);
|
2007-08-28 17:58:01 -07:00
|
|
|
xfs_inode_t *ip = XFS_I(inode);
|
2007-08-27 20:58:24 -07:00
|
|
|
size_t bufsize;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The Linux API doesn't pass down the total size of the buffer
|
|
|
|
* we read into down to the filesystem. With the filldir concept
|
|
|
|
* it's not needed for correct information, but the XFS dir2 leaf
|
|
|
|
* code wants an estimate of the buffer size to calculate it's
|
|
|
|
* readahead window and size the buffers used for mapping to
|
|
|
|
* physical blocks.
|
|
|
|
*
|
|
|
|
* Try to give it an estimate that's good enough, maybe at some
|
|
|
|
* point we can change the ->readdir prototype to include the
|
2010-02-03 10:50:13 -07:00
|
|
|
* buffer size. For now we use the current glibc buffer size.
|
2007-08-27 20:58:24 -07:00
|
|
|
*/
|
2021-03-29 11:11:40 -07:00
|
|
|
bufsize = (size_t)min_t(loff_t, XFS_READDIR_BUFSIZE, ip->i_disk_size);
|
2007-08-27 20:58:24 -07:00
|
|
|
|
2017-06-16 11:00:14 -07:00
|
|
|
return xfs_readdir(NULL, ip, ctx, bufsize);
|
2012-05-10 06:29:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
STATIC loff_t
|
|
|
|
xfs_file_llseek(
|
|
|
|
struct file *file,
|
|
|
|
loff_t offset,
|
2014-09-08 18:57:10 -07:00
|
|
|
int whence)
|
2012-05-10 06:29:17 -07:00
|
|
|
{
|
2017-06-29 11:43:21 -07:00
|
|
|
struct inode *inode = file->f_mapping->host;
|
|
|
|
|
2021-08-18 18:46:53 -07:00
|
|
|
if (xfs_is_shutdown(XFS_I(inode)->i_mount))
|
2017-06-29 11:43:21 -07:00
|
|
|
return -EIO;
|
|
|
|
|
2014-09-08 18:57:10 -07:00
|
|
|
switch (whence) {
|
2017-06-29 11:43:21 -07:00
|
|
|
default:
|
2014-09-08 18:57:10 -07:00
|
|
|
return generic_file_llseek(file, offset, whence);
|
2012-05-10 06:29:17 -07:00
|
|
|
case SEEK_HOLE:
|
2019-02-18 10:38:46 -07:00
|
|
|
offset = iomap_seek_hole(inode, offset, &xfs_seek_iomap_ops);
|
2017-06-29 11:43:21 -07:00
|
|
|
break;
|
2014-09-08 18:56:48 -07:00
|
|
|
case SEEK_DATA:
|
2019-02-18 10:38:46 -07:00
|
|
|
offset = iomap_seek_data(inode, offset, &xfs_seek_iomap_ops);
|
2017-06-29 11:43:21 -07:00
|
|
|
break;
|
2012-05-10 06:29:17 -07:00
|
|
|
}
|
2017-06-29 11:43:21 -07:00
|
|
|
|
|
|
|
if (offset < 0)
|
|
|
|
return offset;
|
|
|
|
return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
|
2012-05-10 06:29:17 -07:00
|
|
|
}
|
|
|
|
|
2022-10-24 10:11:02 -07:00
|
|
|
static inline vm_fault_t
|
2024-06-22 22:44:29 -07:00
|
|
|
xfs_dax_fault_locked(
|
2022-06-02 22:37:37 -07:00
|
|
|
struct vm_fault *vmf,
|
2023-08-18 13:23:35 -07:00
|
|
|
unsigned int order,
|
2024-06-22 22:44:28 -07:00
|
|
|
bool write_fault)
|
2022-06-02 22:37:37 -07:00
|
|
|
{
|
2024-06-22 22:44:28 -07:00
|
|
|
vm_fault_t ret;
|
|
|
|
pfn_t pfn;
|
|
|
|
|
|
|
|
if (!IS_ENABLED(CONFIG_FS_DAX)) {
|
|
|
|
ASSERT(0);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
ret = dax_iomap_fault(vmf, order, &pfn, NULL,
|
2022-06-02 22:37:37 -07:00
|
|
|
(write_fault && !vmf->cow_page) ?
|
|
|
|
&xfs_dax_write_iomap_ops :
|
|
|
|
&xfs_read_iomap_ops);
|
2024-06-22 22:44:28 -07:00
|
|
|
if (ret & VM_FAULT_NEEDDSYNC)
|
|
|
|
ret = dax_finish_sync_fault(vmf, order, pfn);
|
|
|
|
return ret;
|
2022-06-02 22:37:37 -07:00
|
|
|
}
|
|
|
|
|
2024-06-22 22:44:29 -07:00
|
|
|
static vm_fault_t
|
|
|
|
xfs_dax_read_fault(
|
|
|
|
struct vm_fault *vmf,
|
|
|
|
unsigned int order)
|
|
|
|
{
|
|
|
|
struct xfs_inode *ip = XFS_I(file_inode(vmf->vma->vm_file));
|
|
|
|
vm_fault_t ret;
|
|
|
|
|
2024-06-22 22:44:30 -07:00
|
|
|
xfs_ilock(ip, XFS_MMAPLOCK_SHARED);
|
2024-06-22 22:44:29 -07:00
|
|
|
ret = xfs_dax_fault_locked(vmf, order, false);
|
2024-06-22 22:44:30 -07:00
|
|
|
xfs_iunlock(ip, XFS_MMAPLOCK_SHARED);
|
2024-06-22 22:44:29 -07:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static vm_fault_t
|
|
|
|
xfs_write_fault(
|
|
|
|
struct vm_fault *vmf,
|
|
|
|
unsigned int order)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(vmf->vma->vm_file);
|
2024-06-22 22:44:31 -07:00
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
|
|
|
unsigned int lock_mode = XFS_MMAPLOCK_SHARED;
|
2024-06-22 22:44:29 -07:00
|
|
|
vm_fault_t ret;
|
|
|
|
|
|
|
|
sb_start_pagefault(inode->i_sb);
|
|
|
|
file_update_time(vmf->vma->vm_file);
|
|
|
|
|
2024-06-22 22:44:31 -07:00
|
|
|
/*
|
|
|
|
* Normally we only need the shared mmaplock, but if a reflink remap is
|
|
|
|
* in progress we take the exclusive lock to wait for the remap to
|
|
|
|
* finish before taking a write fault.
|
|
|
|
*/
|
|
|
|
xfs_ilock(ip, XFS_MMAPLOCK_SHARED);
|
|
|
|
if (xfs_iflags_test(ip, XFS_IREMAPPING)) {
|
|
|
|
xfs_iunlock(ip, XFS_MMAPLOCK_SHARED);
|
|
|
|
xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
|
|
|
|
lock_mode = XFS_MMAPLOCK_EXCL;
|
|
|
|
}
|
|
|
|
|
2024-06-22 22:44:29 -07:00
|
|
|
if (IS_DAX(inode))
|
|
|
|
ret = xfs_dax_fault_locked(vmf, order, true);
|
|
|
|
else
|
|
|
|
ret = iomap_page_mkwrite(vmf, &xfs_page_mkwrite_iomap_ops);
|
2024-06-22 22:44:31 -07:00
|
|
|
xfs_iunlock(ip, lock_mode);
|
2024-06-22 22:44:29 -07:00
|
|
|
|
|
|
|
sb_end_pagefault(inode->i_sb);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-02-23 03:44:19 -07:00
|
|
|
/*
|
|
|
|
* Locking for serialisation of IO during page faults. This results in a lock
|
|
|
|
* ordering of:
|
|
|
|
*
|
2020-06-08 21:33:54 -07:00
|
|
|
* mmap_lock (MM)
|
2015-06-03 16:18:53 -07:00
|
|
|
* sb_start_pagefault(vfs, freeze)
|
2021-04-12 09:56:24 -07:00
|
|
|
* invalidate_lock (vfs/XFS_MMAPLOCK - truncate serialisation)
|
2015-06-03 16:18:53 -07:00
|
|
|
* page_lock (MM)
|
|
|
|
* i_lock (XFS - extent map serialisation)
|
2015-02-23 03:44:19 -07:00
|
|
|
*/
|
2018-05-29 10:39:03 -07:00
|
|
|
static vm_fault_t
|
2017-08-29 10:08:41 -07:00
|
|
|
__xfs_filemap_fault(
|
|
|
|
struct vm_fault *vmf,
|
2023-08-18 13:23:35 -07:00
|
|
|
unsigned int order,
|
2017-08-29 10:08:41 -07:00
|
|
|
bool write_fault)
|
2015-02-23 03:44:19 -07:00
|
|
|
{
|
2017-02-24 15:56:41 -07:00
|
|
|
struct inode *inode = file_inode(vmf->vma->vm_file);
|
2015-02-23 03:44:19 -07:00
|
|
|
|
2024-06-22 22:44:29 -07:00
|
|
|
trace_xfs_filemap_fault(XFS_I(inode), order, write_fault);
|
2023-10-17 13:12:08 -07:00
|
|
|
|
2017-08-29 10:08:41 -07:00
|
|
|
if (write_fault)
|
2024-06-22 22:44:29 -07:00
|
|
|
return xfs_write_fault(vmf, order);
|
|
|
|
if (IS_DAX(inode))
|
|
|
|
return xfs_dax_read_fault(vmf, order);
|
|
|
|
return filemap_fault(vmf);
|
2015-02-23 03:44:19 -07:00
|
|
|
}
|
|
|
|
|
2020-09-05 05:13:02 -07:00
|
|
|
static inline bool
|
|
|
|
xfs_is_write_fault(
|
|
|
|
struct vm_fault *vmf)
|
|
|
|
{
|
|
|
|
return (vmf->flags & FAULT_FLAG_WRITE) &&
|
|
|
|
(vmf->vma->vm_flags & VM_SHARED);
|
|
|
|
}
|
|
|
|
|
2018-05-29 10:39:03 -07:00
|
|
|
static vm_fault_t
|
2015-06-03 16:18:53 -07:00
|
|
|
xfs_filemap_fault(
|
2015-02-23 03:44:54 -07:00
|
|
|
struct vm_fault *vmf)
|
|
|
|
{
|
2015-06-03 16:18:53 -07:00
|
|
|
/* DAX can shortcut the normal fault path on write faults! */
|
2023-08-18 13:23:35 -07:00
|
|
|
return __xfs_filemap_fault(vmf, 0,
|
2017-08-29 10:08:41 -07:00
|
|
|
IS_DAX(file_inode(vmf->vma->vm_file)) &&
|
2020-09-05 05:13:02 -07:00
|
|
|
xfs_is_write_fault(vmf));
|
2015-06-03 16:18:53 -07:00
|
|
|
}
|
|
|
|
|
2018-05-29 10:39:03 -07:00
|
|
|
static vm_fault_t
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-24 15:56:59 -07:00
|
|
|
xfs_filemap_huge_fault(
|
2017-02-24 15:57:08 -07:00
|
|
|
struct vm_fault *vmf,
|
2023-08-18 13:23:35 -07:00
|
|
|
unsigned int order)
|
2015-09-08 14:59:06 -07:00
|
|
|
{
|
2017-08-29 10:08:41 -07:00
|
|
|
if (!IS_DAX(file_inode(vmf->vma->vm_file)))
|
2015-09-08 14:59:06 -07:00
|
|
|
return VM_FAULT_FALLBACK;
|
|
|
|
|
2017-08-29 10:08:41 -07:00
|
|
|
/* DAX can shortcut the normal fault path on write faults! */
|
2023-08-18 13:23:35 -07:00
|
|
|
return __xfs_filemap_fault(vmf, order,
|
2020-09-05 05:13:02 -07:00
|
|
|
xfs_is_write_fault(vmf));
|
2017-08-29 10:08:41 -07:00
|
|
|
}
|
2015-09-08 14:59:06 -07:00
|
|
|
|
2018-05-29 10:39:03 -07:00
|
|
|
static vm_fault_t
|
2017-08-29 10:08:41 -07:00
|
|
|
xfs_filemap_page_mkwrite(
|
|
|
|
struct vm_fault *vmf)
|
|
|
|
{
|
2023-08-18 13:23:35 -07:00
|
|
|
return __xfs_filemap_fault(vmf, 0, true);
|
2015-09-08 14:59:06 -07:00
|
|
|
}
|
|
|
|
|
2015-11-02 18:37:02 -07:00
|
|
|
/*
|
2017-11-01 08:36:46 -07:00
|
|
|
* pfn_mkwrite was originally intended to ensure we capture time stamp updates
|
|
|
|
* on write faults. In reality, it needs to serialise against truncate and
|
|
|
|
* prepare memory for writing so handle is as standard write fault.
|
2015-11-02 18:37:02 -07:00
|
|
|
*/
|
2018-05-29 10:39:03 -07:00
|
|
|
static vm_fault_t
|
2015-11-02 18:37:02 -07:00
|
|
|
xfs_filemap_pfn_mkwrite(
|
|
|
|
struct vm_fault *vmf)
|
|
|
|
{
|
|
|
|
|
2023-08-18 13:23:35 -07:00
|
|
|
return __xfs_filemap_fault(vmf, 0, true);
|
2015-09-08 14:59:06 -07:00
|
|
|
}
|
|
|
|
|
2015-06-03 16:18:53 -07:00
|
|
|
static const struct vm_operations_struct xfs_file_vm_ops = {
|
|
|
|
.fault = xfs_filemap_fault,
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-24 15:56:59 -07:00
|
|
|
.huge_fault = xfs_filemap_huge_fault,
|
2023-03-27 10:45:13 -07:00
|
|
|
.map_pages = filemap_map_pages,
|
2015-06-03 16:18:53 -07:00
|
|
|
.page_mkwrite = xfs_filemap_page_mkwrite,
|
2015-11-02 18:37:02 -07:00
|
|
|
.pfn_mkwrite = xfs_filemap_pfn_mkwrite,
|
2015-06-03 16:18:53 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
STATIC int
|
|
|
|
xfs_file_mmap(
|
2019-10-24 22:25:38 -07:00
|
|
|
struct file *file,
|
|
|
|
struct vm_area_struct *vma)
|
2015-06-03 16:18:53 -07:00
|
|
|
{
|
2019-10-24 22:25:38 -07:00
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
struct xfs_buftarg *target = xfs_inode_buftarg(XFS_I(inode));
|
2019-07-05 07:03:28 -07:00
|
|
|
|
2017-11-01 08:36:47 -07:00
|
|
|
/*
|
2019-07-05 07:03:28 -07:00
|
|
|
* We don't support synchronous mappings for non-DAX files and
|
|
|
|
* for DAX files if underneath dax_device is not synchronous.
|
2017-11-01 08:36:47 -07:00
|
|
|
*/
|
2019-10-24 22:25:38 -07:00
|
|
|
if (!daxdev_mapping_supported(vma, target->bt_daxdev))
|
2017-11-01 08:36:47 -07:00
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
2019-10-24 22:25:38 -07:00
|
|
|
file_accessed(file);
|
2015-06-03 16:18:53 -07:00
|
|
|
vma->vm_ops = &xfs_file_vm_ops;
|
2019-10-24 22:25:38 -07:00
|
|
|
if (IS_DAX(inode))
|
2023-01-26 12:37:49 -07:00
|
|
|
vm_flags_set(vma, VM_HUGEPAGE);
|
2015-06-03 16:18:53 -07:00
|
|
|
return 0;
|
2015-02-23 03:44:54 -07:00
|
|
|
}
|
|
|
|
|
2006-03-28 02:56:42 -07:00
|
|
|
const struct file_operations xfs_file_operations = {
|
2012-05-10 06:29:17 -07:00
|
|
|
.llseek = xfs_file_llseek,
|
2014-04-02 11:37:59 -07:00
|
|
|
.read_iter = xfs_file_read_iter,
|
2014-04-03 11:20:23 -07:00
|
|
|
.write_iter = xfs_file_write_iter,
|
2023-05-22 06:50:11 -07:00
|
|
|
.splice_read = xfs_file_splice_read,
|
2014-04-05 01:27:08 -07:00
|
|
|
.splice_write = iter_file_splice_write,
|
2021-10-12 04:12:24 -07:00
|
|
|
.iopoll = iocb_bio_iopoll,
|
2006-03-13 20:00:35 -07:00
|
|
|
.unlocked_ioctl = xfs_file_ioctl,
|
2005-04-16 15:20:36 -07:00
|
|
|
#ifdef CONFIG_COMPAT
|
2006-03-13 20:00:35 -07:00
|
|
|
.compat_ioctl = xfs_file_compat_ioctl,
|
2005-04-16 15:20:36 -07:00
|
|
|
#endif
|
2006-03-13 20:00:35 -07:00
|
|
|
.mmap = xfs_file_mmap,
|
|
|
|
.open = xfs_file_open,
|
|
|
|
.release = xfs_file_release,
|
|
|
|
.fsync = xfs_file_fsync,
|
2016-10-07 16:59:59 -07:00
|
|
|
.get_unmapped_area = thp_get_unmapped_area,
|
2011-01-14 05:07:43 -07:00
|
|
|
.fallocate = xfs_file_fallocate,
|
2019-08-29 09:04:12 -07:00
|
|
|
.fadvise = xfs_file_fadvise,
|
2018-10-29 16:41:21 -07:00
|
|
|
.remap_file_range = xfs_file_remap_range,
|
2024-04-23 05:46:06 -07:00
|
|
|
.fop_flags = FOP_MMAP_SYNC | FOP_BUFFER_RASYNC |
|
|
|
|
FOP_BUFFER_WASYNC | FOP_DIO_PARALLEL_WRITE,
|
2005-04-16 15:20:36 -07:00
|
|
|
};
|
|
|
|
|
2006-03-28 02:56:42 -07:00
|
|
|
const struct file_operations xfs_dir_file_operations = {
|
2008-11-27 20:23:32 -07:00
|
|
|
.open = xfs_dir_open,
|
2005-04-16 15:20:36 -07:00
|
|
|
.read = generic_read_dir,
|
2016-04-20 20:42:46 -07:00
|
|
|
.iterate_shared = xfs_file_readdir,
|
2008-08-24 04:24:41 -07:00
|
|
|
.llseek = generic_file_llseek,
|
2006-03-13 20:00:35 -07:00
|
|
|
.unlocked_ioctl = xfs_file_ioctl,
|
2005-05-06 06:44:46 -07:00
|
|
|
#ifdef CONFIG_COMPAT
|
2006-03-13 20:00:35 -07:00
|
|
|
.compat_ioctl = xfs_file_compat_ioctl,
|
2005-05-06 06:44:46 -07:00
|
|
|
#endif
|
2011-10-02 07:25:16 -07:00
|
|
|
.fsync = xfs_dir_fsync,
|
2005-04-16 15:20:36 -07:00
|
|
|
};
|