mm: move internal core VMA manipulation functions to own file
This patch introduces vma.c and moves internal core VMA manipulation functions to this file from mmap.c. This allows us to isolate VMA functionality in a single place such that we can create userspace testing code that invokes this functionality in an environment where we can implement simple unit tests of core functionality. This patch ensures that core VMA functionality is explicitly marked as such by its presence in mm/vma.h. It also places the header includes required by vma.c in vma_internal.h, which is simply imported by vma.c. This makes the VMA functionality testable, as userland testing code can simply stub out functionality as required. Link: https://lkml.kernel.org/r/c77a6aafb4c42aaadb8e7271a853658cbdca2e22.1722251717.git.lorenzo.stoakes@oracle.com Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com> Reviewed-by: Vlastimil Babka <vbabka@suse.cz> Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Brendan Higgins <brendanhiggins@google.com> Cc: Christian Brauner <brauner@kernel.org> Cc: David Gow <davidgow@google.com> Cc: Eric W. Biederman <ebiederm@xmission.com> Cc: Jan Kara <jack@suse.cz> Cc: Kees Cook <kees@kernel.org> Cc: Matthew Wilcox (Oracle) <willy@infradead.org> Cc: Rae Moar <rmoar@google.com> Cc: SeongJae Park <sj@kernel.org> Cc: Shuah Khan <shuah@kernel.org> Cc: Suren Baghdasaryan <surenb@google.com> Cc: Pengfei Xu <pengfei.xu@intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Lorenzo Stoakes
Andrew Morton
parent
d61f0d5968
commit
49b1b8d6f6
@ -1005,21 +1005,6 @@ static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi)
|
||||
return mas_prev(&vmi->mas, 0);
|
||||
}
|
||||
|
||||
static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
|
||||
{
|
||||
return vmi->mas.index;
|
||||
}
|
||||
|
||||
static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
|
||||
{
|
||||
return vmi->mas.last + 1;
|
||||
}
|
||||
static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
|
||||
unsigned long count)
|
||||
{
|
||||
return mas_expected_entries(&vmi->mas, count);
|
||||
}
|
||||
|
||||
static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
|
||||
unsigned long start, unsigned long end, gfp_t gfp)
|
||||
{
|
||||
@ -2534,21 +2519,6 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
|
||||
#define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \
|
||||
MM_CP_UFFD_WP_RESOLVE)
|
||||
|
||||
bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
|
||||
bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
|
||||
static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
|
||||
{
|
||||
/*
|
||||
* We want to check manually if we can change individual PTEs writable
|
||||
* if we can't do that automatically for all PTEs in a mapping. For
|
||||
* private mappings, that's always the case when we have write
|
||||
* permissions as we properly have to handle COW.
|
||||
*/
|
||||
if (vma->vm_flags & VM_SHARED)
|
||||
return vma_wants_writenotify(vma, vma->vm_page_prot);
|
||||
return !!(vma->vm_flags & VM_WRITE);
|
||||
|
||||
}
|
||||
bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
|
||||
pte_t pte);
|
||||
extern long change_protection(struct mmu_gather *tlb,
|
||||
@ -3256,12 +3226,7 @@ void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
|
||||
|
||||
/* mmap.c */
|
||||
extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
|
||||
extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
|
||||
extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
|
||||
extern void unlink_file_vma(struct vm_area_struct *);
|
||||
extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
|
||||
unsigned long addr, unsigned long len, pgoff_t pgoff,
|
||||
bool *need_rmap_locks);
|
||||
extern void exit_mmap(struct mm_struct *);
|
||||
int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift);
|
||||
|
||||
|
||||
@ -37,7 +37,7 @@ mmu-y := nommu.o
|
||||
mmu-$(CONFIG_MMU) := highmem.o memory.o mincore.o \
|
||||
mlock.o mmap.o mmu_gather.o mprotect.o mremap.o \
|
||||
msync.o page_vma_mapped.o pagewalk.o \
|
||||
pgtable-generic.o rmap.o vmalloc.o
|
||||
pgtable-generic.o rmap.o vmalloc.o vma.o
|
||||
|
||||
|
||||
ifdef CONFIG_CROSS_MEMORY_ATTACH
|
||||
|
||||
236
mm/internal.h
236
mm/internal.h
@ -8,13 +8,18 @@
|
||||
#define __MM_INTERNAL_H
|
||||
|
||||
#include <linux/fs.h>
|
||||
#include <linux/khugepaged.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/mm_inline.h>
|
||||
#include <linux/pagemap.h>
|
||||
#include <linux/rmap.h>
|
||||
#include <linux/swap.h>
|
||||
#include <linux/swapops.h>
|
||||
#include <linux/tracepoint-defs.h>
|
||||
|
||||
/* Internal core VMA manipulation functions. */
|
||||
#include "vma.h"
|
||||
|
||||
struct folio_batch;
|
||||
|
||||
/*
|
||||
@ -778,37 +783,6 @@ static inline bool free_area_empty(struct free_area *area, int migratetype)
|
||||
return list_empty(&area->free_list[migratetype]);
|
||||
}
|
||||
|
||||
/*
|
||||
* These three helpers classifies VMAs for virtual memory accounting.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Executable code area - executable, not writable, not stack
|
||||
*/
|
||||
static inline bool is_exec_mapping(vm_flags_t flags)
|
||||
{
|
||||
return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
|
||||
}
|
||||
|
||||
/*
|
||||
* Stack area (including shadow stacks)
|
||||
*
|
||||
* VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
|
||||
* do_mmap() forbids all other combinations.
|
||||
*/
|
||||
static inline bool is_stack_mapping(vm_flags_t flags)
|
||||
{
|
||||
return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
|
||||
}
|
||||
|
||||
/*
|
||||
* Data area - private, writable, not stack
|
||||
*/
|
||||
static inline bool is_data_mapping(vm_flags_t flags)
|
||||
{
|
||||
return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
|
||||
}
|
||||
|
||||
/* mm/util.c */
|
||||
struct anon_vma *folio_anon_vma(struct folio *folio);
|
||||
|
||||
@ -1237,80 +1211,6 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr,
|
||||
void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
|
||||
pmd_t *pmd, bool write);
|
||||
|
||||
/*
|
||||
* mm/mmap.c
|
||||
*/
|
||||
struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long delta);
|
||||
|
||||
struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end,
|
||||
unsigned long vm_flags,
|
||||
struct mempolicy *policy,
|
||||
struct vm_userfaultfd_ctx uffd_ctx,
|
||||
struct anon_vma_name *anon_name);
|
||||
|
||||
/* We are about to modify the VMA's flags. */
|
||||
static inline struct vm_area_struct
|
||||
*vma_modify_flags(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end,
|
||||
unsigned long new_flags)
|
||||
{
|
||||
return vma_modify(vmi, prev, vma, start, end, new_flags,
|
||||
vma_policy(vma), vma->vm_userfaultfd_ctx,
|
||||
anon_vma_name(vma));
|
||||
}
|
||||
|
||||
/* We are about to modify the VMA's flags and/or anon_name. */
|
||||
static inline struct vm_area_struct
|
||||
*vma_modify_flags_name(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start,
|
||||
unsigned long end,
|
||||
unsigned long new_flags,
|
||||
struct anon_vma_name *new_name)
|
||||
{
|
||||
return vma_modify(vmi, prev, vma, start, end, new_flags,
|
||||
vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
|
||||
}
|
||||
|
||||
/* We are about to modify the VMA's memory policy. */
|
||||
static inline struct vm_area_struct
|
||||
*vma_modify_policy(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end,
|
||||
struct mempolicy *new_pol)
|
||||
{
|
||||
return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
|
||||
new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
|
||||
}
|
||||
|
||||
/* We are about to modify the VMA's flags and/or uffd context. */
|
||||
static inline struct vm_area_struct
|
||||
*vma_modify_flags_uffd(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end,
|
||||
unsigned long new_flags,
|
||||
struct vm_userfaultfd_ctx new_ctx)
|
||||
{
|
||||
return vma_modify(vmi, prev, vma, start, end, new_flags,
|
||||
vma_policy(vma), new_ctx, anon_vma_name(vma));
|
||||
}
|
||||
|
||||
int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end, pgoff_t pgoff,
|
||||
struct vm_area_struct *next);
|
||||
int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end, pgoff_t pgoff);
|
||||
|
||||
enum {
|
||||
/* mark page accessed */
|
||||
FOLL_TOUCH = 1 << 16,
|
||||
@ -1437,123 +1337,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte
|
||||
return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte);
|
||||
}
|
||||
|
||||
static inline void vma_iter_config(struct vma_iterator *vmi,
|
||||
unsigned long index, unsigned long last)
|
||||
{
|
||||
__mas_set_range(&vmi->mas, index, last - 1);
|
||||
}
|
||||
|
||||
static inline void vma_iter_reset(struct vma_iterator *vmi)
|
||||
{
|
||||
mas_reset(&vmi->mas);
|
||||
}
|
||||
|
||||
static inline
|
||||
struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
|
||||
{
|
||||
return mas_prev_range(&vmi->mas, min);
|
||||
}
|
||||
|
||||
static inline
|
||||
struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
|
||||
{
|
||||
return mas_next_range(&vmi->mas, max);
|
||||
}
|
||||
|
||||
static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
|
||||
unsigned long max, unsigned long size)
|
||||
{
|
||||
return mas_empty_area(&vmi->mas, min, max - 1, size);
|
||||
}
|
||||
|
||||
static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
|
||||
unsigned long max, unsigned long size)
|
||||
{
|
||||
return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
|
||||
}
|
||||
|
||||
/*
|
||||
* VMA Iterator functions shared between nommu and mmap
|
||||
*/
|
||||
static inline int vma_iter_prealloc(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *vma)
|
||||
{
|
||||
return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
|
||||
}
|
||||
|
||||
static inline void vma_iter_clear(struct vma_iterator *vmi)
|
||||
{
|
||||
mas_store_prealloc(&vmi->mas, NULL);
|
||||
}
|
||||
|
||||
static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
|
||||
{
|
||||
return mas_walk(&vmi->mas);
|
||||
}
|
||||
|
||||
/* Store a VMA with preallocated memory */
|
||||
static inline void vma_iter_store(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *vma)
|
||||
{
|
||||
|
||||
#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
|
||||
if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
|
||||
vmi->mas.index > vma->vm_start)) {
|
||||
pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
|
||||
vmi->mas.index, vma->vm_start, vma->vm_start,
|
||||
vma->vm_end, vmi->mas.index, vmi->mas.last);
|
||||
}
|
||||
if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
|
||||
vmi->mas.last < vma->vm_start)) {
|
||||
pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
|
||||
vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
|
||||
vmi->mas.index, vmi->mas.last);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (vmi->mas.status != ma_start &&
|
||||
((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
|
||||
vma_iter_invalidate(vmi);
|
||||
|
||||
__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
|
||||
mas_store_prealloc(&vmi->mas, vma);
|
||||
}
|
||||
|
||||
static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *vma, gfp_t gfp)
|
||||
{
|
||||
if (vmi->mas.status != ma_start &&
|
||||
((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
|
||||
vma_iter_invalidate(vmi);
|
||||
|
||||
__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
|
||||
mas_store_gfp(&vmi->mas, vma, gfp);
|
||||
if (unlikely(mas_is_err(&vmi->mas)))
|
||||
return -ENOMEM;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline
|
||||
struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
|
||||
{
|
||||
return mas_prev_range(&vmi->mas, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* VMA lock generalization
|
||||
*/
|
||||
struct vma_prepare {
|
||||
struct vm_area_struct *vma;
|
||||
struct vm_area_struct *adj_next;
|
||||
struct file *file;
|
||||
struct address_space *mapping;
|
||||
struct anon_vma *anon_vma;
|
||||
struct vm_area_struct *insert;
|
||||
struct vm_area_struct *remove;
|
||||
struct vm_area_struct *remove2;
|
||||
};
|
||||
|
||||
void __meminit __init_single_page(struct page *page, unsigned long pfn,
|
||||
unsigned long zone, int nid);
|
||||
|
||||
@ -1642,15 +1425,6 @@ static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry,
|
||||
void workingset_update_node(struct xa_node *node);
|
||||
extern struct list_lru shadow_nodes;
|
||||
|
||||
struct unlink_vma_file_batch {
|
||||
int count;
|
||||
struct vm_area_struct *vmas[8];
|
||||
};
|
||||
|
||||
void unlink_file_vma_batch_init(struct unlink_vma_file_batch *);
|
||||
void unlink_file_vma_batch_add(struct unlink_vma_file_batch *, struct vm_area_struct *);
|
||||
void unlink_file_vma_batch_final(struct unlink_vma_file_batch *);
|
||||
|
||||
/* mremap.c */
|
||||
unsigned long move_page_tables(struct vm_area_struct *vma,
|
||||
unsigned long old_addr, struct vm_area_struct *new_vma,
|
||||
|
||||
@ -19,6 +19,8 @@
|
||||
#include <linux/sched/mm.h>
|
||||
#include <linux/slab.h>
|
||||
|
||||
#include "vma.h"
|
||||
|
||||
/* global SRCU for all MMs */
|
||||
DEFINE_STATIC_SRCU(srcu);
|
||||
|
||||
|
||||
364
mm/vma.h
Normal file
364
mm/vma.h
Normal file
@ -0,0 +1,364 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* vma.h
|
||||
*
|
||||
* Core VMA manipulation API implemented in vma.c.
|
||||
*/
|
||||
#ifndef __MM_VMA_H
|
||||
#define __MM_VMA_H
|
||||
|
||||
/*
|
||||
* VMA lock generalization
|
||||
*/
|
||||
struct vma_prepare {
|
||||
struct vm_area_struct *vma;
|
||||
struct vm_area_struct *adj_next;
|
||||
struct file *file;
|
||||
struct address_space *mapping;
|
||||
struct anon_vma *anon_vma;
|
||||
struct vm_area_struct *insert;
|
||||
struct vm_area_struct *remove;
|
||||
struct vm_area_struct *remove2;
|
||||
};
|
||||
|
||||
struct unlink_vma_file_batch {
|
||||
int count;
|
||||
struct vm_area_struct *vmas[8];
|
||||
};
|
||||
|
||||
#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
|
||||
void validate_mm(struct mm_struct *mm);
|
||||
#else
|
||||
#define validate_mm(mm) do { } while (0)
|
||||
#endif
|
||||
|
||||
/* Required for expand_downwards(). */
|
||||
void anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma);
|
||||
|
||||
/* Required for expand_downwards(). */
|
||||
void anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma);
|
||||
|
||||
/* Required for do_brk_flags(). */
|
||||
void vma_prepare(struct vma_prepare *vp);
|
||||
|
||||
/* Required for do_brk_flags(). */
|
||||
void init_vma_prep(struct vma_prepare *vp,
|
||||
struct vm_area_struct *vma);
|
||||
|
||||
/* Required for do_brk_flags(). */
|
||||
void vma_complete(struct vma_prepare *vp,
|
||||
struct vma_iterator *vmi, struct mm_struct *mm);
|
||||
|
||||
int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end, pgoff_t pgoff,
|
||||
struct vm_area_struct *next);
|
||||
|
||||
int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end, pgoff_t pgoff);
|
||||
|
||||
int
|
||||
do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
|
||||
struct mm_struct *mm, unsigned long start,
|
||||
unsigned long end, struct list_head *uf, bool unlock);
|
||||
|
||||
int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
|
||||
unsigned long start, size_t len, struct list_head *uf,
|
||||
bool unlock);
|
||||
|
||||
void remove_vma(struct vm_area_struct *vma, bool unreachable);
|
||||
|
||||
void unmap_region(struct mm_struct *mm, struct ma_state *mas,
|
||||
struct vm_area_struct *vma, struct vm_area_struct *prev,
|
||||
struct vm_area_struct *next, unsigned long start,
|
||||
unsigned long end, unsigned long tree_end, bool mm_wr_locked);
|
||||
|
||||
/* Required by mmap_region(). */
|
||||
bool
|
||||
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
|
||||
struct anon_vma *anon_vma, struct file *file,
|
||||
pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
|
||||
struct anon_vma_name *anon_name);
|
||||
|
||||
/* Required by mmap_region() and do_brk_flags(). */
|
||||
bool
|
||||
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
|
||||
struct anon_vma *anon_vma, struct file *file,
|
||||
pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
|
||||
struct anon_vma_name *anon_name);
|
||||
|
||||
struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end,
|
||||
unsigned long vm_flags,
|
||||
struct mempolicy *policy,
|
||||
struct vm_userfaultfd_ctx uffd_ctx,
|
||||
struct anon_vma_name *anon_name);
|
||||
|
||||
/* We are about to modify the VMA's flags. */
|
||||
static inline struct vm_area_struct
|
||||
*vma_modify_flags(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end,
|
||||
unsigned long new_flags)
|
||||
{
|
||||
return vma_modify(vmi, prev, vma, start, end, new_flags,
|
||||
vma_policy(vma), vma->vm_userfaultfd_ctx,
|
||||
anon_vma_name(vma));
|
||||
}
|
||||
|
||||
/* We are about to modify the VMA's flags and/or anon_name. */
|
||||
static inline struct vm_area_struct
|
||||
*vma_modify_flags_name(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start,
|
||||
unsigned long end,
|
||||
unsigned long new_flags,
|
||||
struct anon_vma_name *new_name)
|
||||
{
|
||||
return vma_modify(vmi, prev, vma, start, end, new_flags,
|
||||
vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
|
||||
}
|
||||
|
||||
/* We are about to modify the VMA's memory policy. */
|
||||
static inline struct vm_area_struct
|
||||
*vma_modify_policy(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end,
|
||||
struct mempolicy *new_pol)
|
||||
{
|
||||
return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
|
||||
new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
|
||||
}
|
||||
|
||||
/* We are about to modify the VMA's flags and/or uffd context. */
|
||||
static inline struct vm_area_struct
|
||||
*vma_modify_flags_uffd(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end,
|
||||
unsigned long new_flags,
|
||||
struct vm_userfaultfd_ctx new_ctx)
|
||||
{
|
||||
return vma_modify(vmi, prev, vma, start, end, new_flags,
|
||||
vma_policy(vma), new_ctx, anon_vma_name(vma));
|
||||
}
|
||||
|
||||
struct vm_area_struct
|
||||
*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma, unsigned long start,
|
||||
unsigned long end, pgoff_t pgoff);
|
||||
|
||||
struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long delta);
|
||||
|
||||
void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb);
|
||||
|
||||
void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb);
|
||||
|
||||
void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
|
||||
struct vm_area_struct *vma);
|
||||
|
||||
void unlink_file_vma(struct vm_area_struct *vma);
|
||||
|
||||
void vma_link_file(struct vm_area_struct *vma);
|
||||
|
||||
int vma_link(struct mm_struct *mm, struct vm_area_struct *vma);
|
||||
|
||||
struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
|
||||
unsigned long addr, unsigned long len, pgoff_t pgoff,
|
||||
bool *need_rmap_locks);
|
||||
|
||||
struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);
|
||||
|
||||
bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
|
||||
bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
|
||||
|
||||
int mm_take_all_locks(struct mm_struct *mm);
|
||||
void mm_drop_all_locks(struct mm_struct *mm);
|
||||
unsigned long count_vma_pages_range(struct mm_struct *mm,
|
||||
unsigned long addr, unsigned long end);
|
||||
|
||||
static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
|
||||
{
|
||||
/*
|
||||
* We want to check manually if we can change individual PTEs writable
|
||||
* if we can't do that automatically for all PTEs in a mapping. For
|
||||
* private mappings, that's always the case when we have write
|
||||
* permissions as we properly have to handle COW.
|
||||
*/
|
||||
if (vma->vm_flags & VM_SHARED)
|
||||
return vma_wants_writenotify(vma, vma->vm_page_prot);
|
||||
return !!(vma->vm_flags & VM_WRITE);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MMU
|
||||
static inline pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
|
||||
{
|
||||
return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
|
||||
unsigned long min)
|
||||
{
|
||||
return mas_prev(&vmi->mas, min);
|
||||
}
|
||||
|
||||
static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *vma, gfp_t gfp)
|
||||
{
|
||||
if (vmi->mas.status != ma_start &&
|
||||
((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
|
||||
vma_iter_invalidate(vmi);
|
||||
|
||||
__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
|
||||
mas_store_gfp(&vmi->mas, vma, gfp);
|
||||
if (unlikely(mas_is_err(&vmi->mas)))
|
||||
return -ENOMEM;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* These three helpers classifies VMAs for virtual memory accounting.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Executable code area - executable, not writable, not stack
|
||||
*/
|
||||
static inline bool is_exec_mapping(vm_flags_t flags)
|
||||
{
|
||||
return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
|
||||
}
|
||||
|
||||
/*
|
||||
* Stack area (including shadow stacks)
|
||||
*
|
||||
* VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
|
||||
* do_mmap() forbids all other combinations.
|
||||
*/
|
||||
static inline bool is_stack_mapping(vm_flags_t flags)
|
||||
{
|
||||
return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
|
||||
}
|
||||
|
||||
/*
|
||||
* Data area - private, writable, not stack
|
||||
*/
|
||||
static inline bool is_data_mapping(vm_flags_t flags)
|
||||
{
|
||||
return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
|
||||
}
|
||||
|
||||
|
||||
static inline void vma_iter_config(struct vma_iterator *vmi,
|
||||
unsigned long index, unsigned long last)
|
||||
{
|
||||
__mas_set_range(&vmi->mas, index, last - 1);
|
||||
}
|
||||
|
||||
static inline void vma_iter_reset(struct vma_iterator *vmi)
|
||||
{
|
||||
mas_reset(&vmi->mas);
|
||||
}
|
||||
|
||||
static inline
|
||||
struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
|
||||
{
|
||||
return mas_prev_range(&vmi->mas, min);
|
||||
}
|
||||
|
||||
static inline
|
||||
struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
|
||||
{
|
||||
return mas_next_range(&vmi->mas, max);
|
||||
}
|
||||
|
||||
static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
|
||||
unsigned long max, unsigned long size)
|
||||
{
|
||||
return mas_empty_area(&vmi->mas, min, max - 1, size);
|
||||
}
|
||||
|
||||
static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
|
||||
unsigned long max, unsigned long size)
|
||||
{
|
||||
return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
|
||||
}
|
||||
|
||||
/*
|
||||
* VMA Iterator functions shared between nommu and mmap
|
||||
*/
|
||||
static inline int vma_iter_prealloc(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *vma)
|
||||
{
|
||||
return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
|
||||
}
|
||||
|
||||
static inline void vma_iter_clear(struct vma_iterator *vmi)
|
||||
{
|
||||
mas_store_prealloc(&vmi->mas, NULL);
|
||||
}
|
||||
|
||||
static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
|
||||
{
|
||||
return mas_walk(&vmi->mas);
|
||||
}
|
||||
|
||||
/* Store a VMA with preallocated memory */
|
||||
static inline void vma_iter_store(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *vma)
|
||||
{
|
||||
|
||||
#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
|
||||
if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
|
||||
vmi->mas.index > vma->vm_start)) {
|
||||
pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
|
||||
vmi->mas.index, vma->vm_start, vma->vm_start,
|
||||
vma->vm_end, vmi->mas.index, vmi->mas.last);
|
||||
}
|
||||
if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
|
||||
vmi->mas.last < vma->vm_start)) {
|
||||
pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
|
||||
vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
|
||||
vmi->mas.index, vmi->mas.last);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (vmi->mas.status != ma_start &&
|
||||
((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
|
||||
vma_iter_invalidate(vmi);
|
||||
|
||||
__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
|
||||
mas_store_prealloc(&vmi->mas, vma);
|
||||
}
|
||||
|
||||
static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
|
||||
{
|
||||
return vmi->mas.index;
|
||||
}
|
||||
|
||||
static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
|
||||
{
|
||||
return vmi->mas.last + 1;
|
||||
}
|
||||
|
||||
static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
|
||||
unsigned long count)
|
||||
{
|
||||
return mas_expected_entries(&vmi->mas, count);
|
||||
}
|
||||
|
||||
static inline
|
||||
struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
|
||||
{
|
||||
return mas_prev_range(&vmi->mas, 0);
|
||||
}
|
||||
|
||||
#endif /* __MM_VMA_H */
|
||||
50
mm/vma_internal.h
Normal file
50
mm/vma_internal.h
Normal file
@ -0,0 +1,50 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* vma_internal.h
|
||||
*
|
||||
* Headers required by vma.c, which can be substituted accordingly when testing
|
||||
* VMA functionality.
|
||||
*/
|
||||
|
||||
#ifndef __MM_VMA_INTERNAL_H
|
||||
#define __MM_VMA_INTERNAL_H
|
||||
|
||||
#include <linux/backing-dev.h>
|
||||
#include <linux/bitops.h>
|
||||
#include <linux/bug.h>
|
||||
#include <linux/bug.h>
|
||||
#include <linux/cacheflush.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/file.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/huge_mm.h>
|
||||
#include <linux/hugetlb_inline.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/khugepaged.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/maple_tree.h>
|
||||
#include <linux/mempolicy.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/mm_inline.h>
|
||||
#include <linux/mm_types.h>
|
||||
#include <linux/mman.h>
|
||||
#include <linux/mmap_lock.h>
|
||||
#include <linux/mmdebug.h>
|
||||
#include <linux/mmu_context.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/pagemap.h>
|
||||
#include <linux/pfn.h>
|
||||
#include <linux/rcupdate.h>
|
||||
#include <linux/rmap.h>
|
||||
#include <linux/rwsem.h>
|
||||
#include <linux/sched/signal.h>
|
||||
#include <linux/swap.h>
|
||||
#include <linux/uprobes.h>
|
||||
#include <linux/userfaultfd_k.h>
|
||||
|
||||
#include <asm/current.h>
|
||||
#include <asm/tlb.h>
|
||||
|
||||
#include "internal.h"
|
||||
|
||||
#endif /* __MM_VMA_INTERNAL_H */
|
||||
Loading…
Reference in New Issue
Block a user