0c66c6f4e2
Since commita4d5613c4d
("arm: extend pfn_valid to take into account freed memory map alignment") changes the semantics of pfn_valid() to check presence of the memory map for a PFN. A valid page for an address which is reserved but not mapped by the kernel[1], the system crashed during some uio test with the following memory layout: node 0: [mem 0x00000000c0a00000-0x00000000cc8fffff] node 0: [mem 0x00000000d0000000-0x00000000da1fffff] the uio layout is:0xc0900000, 0x100000 the crash backtrace like: Unable to handle kernel paging request at virtual address bff00000 [...] CPU: 1 PID: 465 Comm: startapp.bin Tainted: G O 5.10.0 #1 Hardware name: Generic DT based system PC is at b15_flush_kern_dcache_area+0x24/0x3c LR is at __sync_icache_dcache+0x6c/0x98 [...] (b15_flush_kern_dcache_area) from (__sync_icache_dcache+0x6c/0x98) (__sync_icache_dcache) from (set_pte_at+0x28/0x54) (set_pte_at) from (remap_pfn_range+0x1a0/0x274) (remap_pfn_range) from (uio_mmap+0x184/0x1b8 [uio]) (uio_mmap [uio]) from (__mmap_region+0x264/0x5f4) (__mmap_region) from (__do_mmap_mm+0x3ec/0x440) (__do_mmap_mm) from (do_mmap+0x50/0x58) (do_mmap) from (vm_mmap_pgoff+0xfc/0x188) (vm_mmap_pgoff) from (ksys_mmap_pgoff+0xac/0xc4) (ksys_mmap_pgoff) from (ret_fast_syscall+0x0/0x5c) Code: e0801001 e2423001 e1c00003 f57ff04f (ee070f3e) ---[ end trace 09cf0734c3805d52 ]--- Kernel panic - not syncing: Fatal exception So check if PG_reserved was set to solve this issue. [1]: https://lore.kernel.org/lkml/Zbtdue57RO0QScJM@linux.ibm.com/ Fixes:a4d5613c4d
("arm: extend pfn_valid to take into account freed memory map alignment") Suggested-by: Mike Rapoport <rppt@linux.ibm.com> Signed-off-by: Yongqiang Liu <liuyongqiang13@huawei.com> Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
412 lines
10 KiB
C
412 lines
10 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* linux/arch/arm/mm/flush.c
|
|
*
|
|
* Copyright (C) 1995-2002 Russell King
|
|
*/
|
|
#include <linux/module.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/highmem.h>
|
|
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/cachetype.h>
|
|
#include <asm/highmem.h>
|
|
#include <asm/smp_plat.h>
|
|
#include <asm/tlbflush.h>
|
|
#include <linux/hugetlb.h>
|
|
|
|
#include "mm.h"
|
|
|
|
#ifdef CONFIG_ARM_HEAVY_MB
|
|
void (*soc_mb)(void);
|
|
|
|
void arm_heavy_mb(void)
|
|
{
|
|
#ifdef CONFIG_OUTER_CACHE_SYNC
|
|
if (outer_cache.sync)
|
|
outer_cache.sync();
|
|
#endif
|
|
if (soc_mb)
|
|
soc_mb();
|
|
}
|
|
EXPORT_SYMBOL(arm_heavy_mb);
|
|
#endif
|
|
|
|
#ifdef CONFIG_CPU_CACHE_VIPT
|
|
|
|
static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
|
|
{
|
|
unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
|
|
const int zero = 0;
|
|
|
|
set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL));
|
|
|
|
asm( "mcrr p15, 0, %1, %0, c14\n"
|
|
" mcr p15, 0, %2, c7, c10, 4"
|
|
:
|
|
: "r" (to), "r" (to + PAGE_SIZE - 1), "r" (zero)
|
|
: "cc");
|
|
}
|
|
|
|
static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
|
|
{
|
|
unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
|
|
unsigned long offset = vaddr & (PAGE_SIZE - 1);
|
|
unsigned long to;
|
|
|
|
set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL));
|
|
to = va + offset;
|
|
flush_icache_range(to, to + len);
|
|
}
|
|
|
|
void flush_cache_mm(struct mm_struct *mm)
|
|
{
|
|
if (cache_is_vivt()) {
|
|
vivt_flush_cache_mm(mm);
|
|
return;
|
|
}
|
|
|
|
if (cache_is_vipt_aliasing()) {
|
|
asm( "mcr p15, 0, %0, c7, c14, 0\n"
|
|
" mcr p15, 0, %0, c7, c10, 4"
|
|
:
|
|
: "r" (0)
|
|
: "cc");
|
|
}
|
|
}
|
|
|
|
void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
|
|
{
|
|
if (cache_is_vivt()) {
|
|
vivt_flush_cache_range(vma, start, end);
|
|
return;
|
|
}
|
|
|
|
if (cache_is_vipt_aliasing()) {
|
|
asm( "mcr p15, 0, %0, c7, c14, 0\n"
|
|
" mcr p15, 0, %0, c7, c10, 4"
|
|
:
|
|
: "r" (0)
|
|
: "cc");
|
|
}
|
|
|
|
if (vma->vm_flags & VM_EXEC)
|
|
__flush_icache_all();
|
|
}
|
|
|
|
void flush_cache_pages(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn, unsigned int nr)
|
|
{
|
|
if (cache_is_vivt()) {
|
|
vivt_flush_cache_pages(vma, user_addr, pfn, nr);
|
|
return;
|
|
}
|
|
|
|
if (cache_is_vipt_aliasing()) {
|
|
flush_pfn_alias(pfn, user_addr);
|
|
__flush_icache_all();
|
|
}
|
|
|
|
if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
|
|
__flush_icache_all();
|
|
}
|
|
|
|
#else
|
|
#define flush_pfn_alias(pfn,vaddr) do { } while (0)
|
|
#define flush_icache_alias(pfn,vaddr,len) do { } while (0)
|
|
#endif
|
|
|
|
#define FLAG_PA_IS_EXEC 1
|
|
#define FLAG_PA_CORE_IN_MM 2
|
|
|
|
static void flush_ptrace_access_other(void *args)
|
|
{
|
|
__flush_icache_all();
|
|
}
|
|
|
|
static inline
|
|
void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr,
|
|
unsigned long len, unsigned int flags)
|
|
{
|
|
if (cache_is_vivt()) {
|
|
if (flags & FLAG_PA_CORE_IN_MM) {
|
|
unsigned long addr = (unsigned long)kaddr;
|
|
__cpuc_coherent_kern_range(addr, addr + len);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (cache_is_vipt_aliasing()) {
|
|
flush_pfn_alias(page_to_pfn(page), uaddr);
|
|
__flush_icache_all();
|
|
return;
|
|
}
|
|
|
|
/* VIPT non-aliasing D-cache */
|
|
if (flags & FLAG_PA_IS_EXEC) {
|
|
unsigned long addr = (unsigned long)kaddr;
|
|
if (icache_is_vipt_aliasing())
|
|
flush_icache_alias(page_to_pfn(page), uaddr, len);
|
|
else
|
|
__cpuc_coherent_kern_range(addr, addr + len);
|
|
if (cache_ops_need_broadcast())
|
|
smp_call_function(flush_ptrace_access_other,
|
|
NULL, 1);
|
|
}
|
|
}
|
|
|
|
static
|
|
void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
|
|
unsigned long uaddr, void *kaddr, unsigned long len)
|
|
{
|
|
unsigned int flags = 0;
|
|
if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
|
|
flags |= FLAG_PA_CORE_IN_MM;
|
|
if (vma->vm_flags & VM_EXEC)
|
|
flags |= FLAG_PA_IS_EXEC;
|
|
__flush_ptrace_access(page, uaddr, kaddr, len, flags);
|
|
}
|
|
|
|
void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
|
|
void *kaddr, unsigned long len)
|
|
{
|
|
unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC;
|
|
|
|
__flush_ptrace_access(page, uaddr, kaddr, len, flags);
|
|
}
|
|
|
|
/*
|
|
* Copy user data from/to a page which is mapped into a different
|
|
* processes address space. Really, we want to allow our "user
|
|
* space" model to handle this.
|
|
*
|
|
* Note that this code needs to run on the current CPU.
|
|
*/
|
|
void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
|
|
unsigned long uaddr, void *dst, const void *src,
|
|
unsigned long len)
|
|
{
|
|
#ifdef CONFIG_SMP
|
|
preempt_disable();
|
|
#endif
|
|
memcpy(dst, src, len);
|
|
flush_ptrace_access(vma, page, uaddr, dst, len);
|
|
#ifdef CONFIG_SMP
|
|
preempt_enable();
|
|
#endif
|
|
}
|
|
|
|
void __flush_dcache_folio(struct address_space *mapping, struct folio *folio)
|
|
{
|
|
/*
|
|
* Writeback any data associated with the kernel mapping of this
|
|
* page. This ensures that data in the physical page is mutually
|
|
* coherent with the kernels mapping.
|
|
*/
|
|
if (!folio_test_highmem(folio)) {
|
|
__cpuc_flush_dcache_area(folio_address(folio),
|
|
folio_size(folio));
|
|
} else {
|
|
unsigned long i;
|
|
if (cache_is_vipt_nonaliasing()) {
|
|
for (i = 0; i < folio_nr_pages(folio); i++) {
|
|
void *addr = kmap_local_folio(folio,
|
|
i * PAGE_SIZE);
|
|
__cpuc_flush_dcache_area(addr, PAGE_SIZE);
|
|
kunmap_local(addr);
|
|
}
|
|
} else {
|
|
for (i = 0; i < folio_nr_pages(folio); i++) {
|
|
void *addr = kmap_high_get(folio_page(folio, i));
|
|
if (addr) {
|
|
__cpuc_flush_dcache_area(addr, PAGE_SIZE);
|
|
kunmap_high(folio_page(folio, i));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* If this is a page cache page, and we have an aliasing VIPT cache,
|
|
* we only need to do one flush - which would be at the relevant
|
|
* userspace colour, which is congruent with page->index.
|
|
*/
|
|
if (mapping && cache_is_vipt_aliasing())
|
|
flush_pfn_alias(folio_pfn(folio), folio_pos(folio));
|
|
}
|
|
|
|
static void __flush_dcache_aliases(struct address_space *mapping, struct folio *folio)
|
|
{
|
|
struct mm_struct *mm = current->active_mm;
|
|
struct vm_area_struct *vma;
|
|
pgoff_t pgoff, pgoff_end;
|
|
|
|
/*
|
|
* There are possible user space mappings of this page:
|
|
* - VIVT cache: we need to also write back and invalidate all user
|
|
* data in the current VM view associated with this page.
|
|
* - aliasing VIPT: we only need to find one mapping of this page.
|
|
*/
|
|
pgoff = folio->index;
|
|
pgoff_end = pgoff + folio_nr_pages(folio) - 1;
|
|
|
|
flush_dcache_mmap_lock(mapping);
|
|
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff_end) {
|
|
unsigned long start, offset, pfn;
|
|
unsigned int nr;
|
|
|
|
/*
|
|
* If this VMA is not in our MM, we can ignore it.
|
|
*/
|
|
if (vma->vm_mm != mm)
|
|
continue;
|
|
if (!(vma->vm_flags & VM_MAYSHARE))
|
|
continue;
|
|
|
|
start = vma->vm_start;
|
|
pfn = folio_pfn(folio);
|
|
nr = folio_nr_pages(folio);
|
|
offset = pgoff - vma->vm_pgoff;
|
|
if (offset > -nr) {
|
|
pfn -= offset;
|
|
nr += offset;
|
|
} else {
|
|
start += offset * PAGE_SIZE;
|
|
}
|
|
if (start + nr * PAGE_SIZE > vma->vm_end)
|
|
nr = (vma->vm_end - start) / PAGE_SIZE;
|
|
|
|
flush_cache_pages(vma, start, pfn, nr);
|
|
}
|
|
flush_dcache_mmap_unlock(mapping);
|
|
}
|
|
|
|
#if __LINUX_ARM_ARCH__ >= 6
|
|
void __sync_icache_dcache(pte_t pteval)
|
|
{
|
|
unsigned long pfn;
|
|
struct folio *folio;
|
|
struct address_space *mapping;
|
|
|
|
if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
|
|
/* only flush non-aliasing VIPT caches for exec mappings */
|
|
return;
|
|
pfn = pte_pfn(pteval);
|
|
if (!pfn_valid(pfn))
|
|
return;
|
|
|
|
folio = page_folio(pfn_to_page(pfn));
|
|
if (folio_test_reserved(folio))
|
|
return;
|
|
|
|
if (cache_is_vipt_aliasing())
|
|
mapping = folio_flush_mapping(folio);
|
|
else
|
|
mapping = NULL;
|
|
|
|
if (!test_and_set_bit(PG_dcache_clean, &folio->flags))
|
|
__flush_dcache_folio(mapping, folio);
|
|
|
|
if (pte_exec(pteval))
|
|
__flush_icache_all();
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* Ensure cache coherency between kernel mapping and userspace mapping
|
|
* of this page.
|
|
*
|
|
* We have three cases to consider:
|
|
* - VIPT non-aliasing cache: fully coherent so nothing required.
|
|
* - VIVT: fully aliasing, so we need to handle every alias in our
|
|
* current VM view.
|
|
* - VIPT aliasing: need to handle one alias in our current VM view.
|
|
*
|
|
* If we need to handle aliasing:
|
|
* If the page only exists in the page cache and there are no user
|
|
* space mappings, we can be lazy and remember that we may have dirty
|
|
* kernel cache lines for later. Otherwise, we assume we have
|
|
* aliasing mappings.
|
|
*
|
|
* Note that we disable the lazy flush for SMP configurations where
|
|
* the cache maintenance operations are not automatically broadcasted.
|
|
*/
|
|
void flush_dcache_folio(struct folio *folio)
|
|
{
|
|
struct address_space *mapping;
|
|
|
|
/*
|
|
* The zero page is never written to, so never has any dirty
|
|
* cache lines, and therefore never needs to be flushed.
|
|
*/
|
|
if (is_zero_pfn(folio_pfn(folio)))
|
|
return;
|
|
|
|
if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) {
|
|
if (test_bit(PG_dcache_clean, &folio->flags))
|
|
clear_bit(PG_dcache_clean, &folio->flags);
|
|
return;
|
|
}
|
|
|
|
mapping = folio_flush_mapping(folio);
|
|
|
|
if (!cache_ops_need_broadcast() &&
|
|
mapping && !folio_mapped(folio))
|
|
clear_bit(PG_dcache_clean, &folio->flags);
|
|
else {
|
|
__flush_dcache_folio(mapping, folio);
|
|
if (mapping && cache_is_vivt())
|
|
__flush_dcache_aliases(mapping, folio);
|
|
else if (mapping)
|
|
__flush_icache_all();
|
|
set_bit(PG_dcache_clean, &folio->flags);
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(flush_dcache_folio);
|
|
|
|
void flush_dcache_page(struct page *page)
|
|
{
|
|
flush_dcache_folio(page_folio(page));
|
|
}
|
|
EXPORT_SYMBOL(flush_dcache_page);
|
|
/*
|
|
* Flush an anonymous page so that users of get_user_pages()
|
|
* can safely access the data. The expected sequence is:
|
|
*
|
|
* get_user_pages()
|
|
* -> flush_anon_page
|
|
* memcpy() to/from page
|
|
* if written to page, flush_dcache_page()
|
|
*/
|
|
void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr);
|
|
void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
|
|
{
|
|
unsigned long pfn;
|
|
|
|
/* VIPT non-aliasing caches need do nothing */
|
|
if (cache_is_vipt_nonaliasing())
|
|
return;
|
|
|
|
/*
|
|
* Write back and invalidate userspace mapping.
|
|
*/
|
|
pfn = page_to_pfn(page);
|
|
if (cache_is_vivt()) {
|
|
flush_cache_page(vma, vmaddr, pfn);
|
|
} else {
|
|
/*
|
|
* For aliasing VIPT, we can flush an alias of the
|
|
* userspace address only.
|
|
*/
|
|
flush_pfn_alias(pfn, vmaddr);
|
|
__flush_icache_all();
|
|
}
|
|
|
|
/*
|
|
* Invalidate kernel mapping. No data should be contained
|
|
* in this mapping of the page. FIXME: this is overkill
|
|
* since we actually ask for a write-back and invalidate.
|
|
*/
|
|
__cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
|
|
}
|