2019-05-23 02:14:57 -07:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2017-07-10 18:06:09 -07:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2009 Sunplus Core Technology Co., Ltd.
|
|
|
|
* Lennox Wu <lennox.wu@sunplusct.com>
|
|
|
|
* Chen Liqin <liqin.chen@sunplusct.com>
|
|
|
|
* Copyright (C) 2012 Regents of the University of California
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/perf_event.h>
|
|
|
|
#include <linux/signal.h>
|
|
|
|
#include <linux/uaccess.h>
|
2020-12-17 09:01:42 -07:00
|
|
|
#include <linux/kprobes.h>
|
2021-06-14 20:07:34 -07:00
|
|
|
#include <linux/kfence.h>
|
2023-02-21 20:30:18 -07:00
|
|
|
#include <linux/entry-common.h>
|
2017-07-10 18:06:09 -07:00
|
|
|
|
|
|
|
#include <asm/ptrace.h>
|
2019-06-16 21:26:17 -07:00
|
|
|
#include <asm/tlbflush.h>
|
2017-07-10 18:06:09 -07:00
|
|
|
|
2019-10-17 15:00:17 -07:00
|
|
|
#include "../kernel/head.h"
|
|
|
|
|
2020-12-03 22:42:58 -07:00
|
|
|
static void die_kernel_fault(const char *msg, unsigned long addr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
bust_spinlocks(1);
|
|
|
|
|
|
|
|
pr_alert("Unable to handle kernel %s at virtual address " REG_FMT "\n", msg,
|
|
|
|
addr);
|
|
|
|
|
|
|
|
bust_spinlocks(0);
|
|
|
|
die(regs, "Oops");
|
2021-06-28 12:52:01 -07:00
|
|
|
make_task_dead(SIGKILL);
|
2020-12-03 22:42:58 -07:00
|
|
|
}
|
|
|
|
|
2020-08-25 08:38:58 -07:00
|
|
|
static inline void no_context(struct pt_regs *regs, unsigned long addr)
|
|
|
|
{
|
2020-12-03 22:42:58 -07:00
|
|
|
const char *msg;
|
|
|
|
|
2020-08-25 08:38:58 -07:00
|
|
|
/* Are we prepared to handle this kernel fault? */
|
|
|
|
if (fixup_exception(regs))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Oops. The kernel tried to access some bad page. We'll have to
|
|
|
|
* terminate things with extreme prejudice.
|
|
|
|
*/
|
2021-06-14 20:07:34 -07:00
|
|
|
if (addr < PAGE_SIZE)
|
|
|
|
msg = "NULL pointer dereference";
|
|
|
|
else {
|
|
|
|
if (kfence_handle_page_fault(addr, regs->cause == EXC_STORE_PAGE_FAULT, regs))
|
|
|
|
return;
|
|
|
|
|
|
|
|
msg = "paging request";
|
|
|
|
}
|
|
|
|
|
2020-12-03 22:42:58 -07:00
|
|
|
die_kernel_fault(msg, addr, regs);
|
2020-08-25 08:38:58 -07:00
|
|
|
}
|
|
|
|
|
2020-08-25 09:04:09 -07:00
|
|
|
static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault)
|
|
|
|
{
|
2024-07-31 01:45:47 -07:00
|
|
|
if (!user_mode(regs)) {
|
|
|
|
no_context(regs, addr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-08-25 09:05:44 -07:00
|
|
|
if (fault & VM_FAULT_OOM) {
|
|
|
|
/*
|
|
|
|
* We ran out of memory, call the OOM killer, and return the userspace
|
|
|
|
* (which will retry the fault, or kill us if we got oom-killed).
|
|
|
|
*/
|
|
|
|
pagefault_out_of_memory();
|
2020-08-25 09:04:09 -07:00
|
|
|
return;
|
2023-09-28 08:18:45 -07:00
|
|
|
} else if (fault & (VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) {
|
2020-08-25 09:05:44 -07:00
|
|
|
/* Kernel mode? Handle exceptions or die */
|
|
|
|
do_trap(regs, SIGBUS, BUS_ADRERR, addr);
|
2020-08-25 09:04:09 -07:00
|
|
|
return;
|
2024-07-31 01:45:47 -07:00
|
|
|
} else if (fault & VM_FAULT_SIGSEGV) {
|
|
|
|
do_trap(regs, SIGSEGV, SEGV_MAPERR, addr);
|
|
|
|
return;
|
2020-08-25 09:04:09 -07:00
|
|
|
}
|
2024-07-31 01:45:47 -07:00
|
|
|
|
2020-08-25 09:05:44 -07:00
|
|
|
BUG();
|
2020-08-25 09:04:09 -07:00
|
|
|
}
|
|
|
|
|
2023-06-22 11:18:18 -07:00
|
|
|
static inline void
|
|
|
|
bad_area_nosemaphore(struct pt_regs *regs, int code, unsigned long addr)
|
2020-08-25 08:48:01 -07:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Something tried to access memory that isn't in our memory map.
|
|
|
|
* Fix it, but check if it's kernel or user first.
|
|
|
|
*/
|
|
|
|
/* User mode accesses just cause a SIGSEGV */
|
|
|
|
if (user_mode(regs)) {
|
|
|
|
do_trap(regs, SIGSEGV, code, addr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
no_context(regs, addr);
|
|
|
|
}
|
|
|
|
|
2023-06-22 11:18:18 -07:00
|
|
|
static inline void
|
|
|
|
bad_area(struct pt_regs *regs, struct mm_struct *mm, int code,
|
|
|
|
unsigned long addr)
|
|
|
|
{
|
|
|
|
mmap_read_unlock(mm);
|
|
|
|
|
|
|
|
bad_area_nosemaphore(regs, code, addr);
|
|
|
|
}
|
|
|
|
|
2020-09-04 22:52:52 -07:00
|
|
|
static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long addr)
|
2020-08-25 08:54:26 -07:00
|
|
|
{
|
|
|
|
pgd_t *pgd, *pgd_k;
|
2022-03-31 19:43:20 -07:00
|
|
|
pud_t *pud_k;
|
|
|
|
p4d_t *p4d_k;
|
|
|
|
pmd_t *pmd_k;
|
2020-08-25 08:54:26 -07:00
|
|
|
pte_t *pte_k;
|
|
|
|
int index;
|
2020-10-26 05:26:54 -07:00
|
|
|
unsigned long pfn;
|
2020-08-25 08:54:26 -07:00
|
|
|
|
|
|
|
/* User mode accesses just cause a SIGSEGV */
|
|
|
|
if (user_mode(regs))
|
|
|
|
return do_trap(regs, SIGSEGV, code, addr);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Synchronize this task's top level page-table
|
|
|
|
* with the 'reference' page table.
|
|
|
|
*
|
|
|
|
* Do _not_ use "tsk->active_mm->pgd" here.
|
|
|
|
* We might be inside an interrupt in the middle
|
|
|
|
* of a task switch.
|
|
|
|
*/
|
|
|
|
index = pgd_index(addr);
|
2020-10-26 05:26:54 -07:00
|
|
|
pfn = csr_read(CSR_SATP) & SATP_PPN;
|
|
|
|
pgd = (pgd_t *)pfn_to_virt(pfn) + index;
|
2020-08-25 08:54:26 -07:00
|
|
|
pgd_k = init_mm.pgd + index;
|
|
|
|
|
2023-12-13 13:30:01 -07:00
|
|
|
if (!pgd_present(pgdp_get(pgd_k))) {
|
2020-08-25 08:54:26 -07:00
|
|
|
no_context(regs, addr);
|
|
|
|
return;
|
|
|
|
}
|
2023-12-13 13:30:01 -07:00
|
|
|
set_pgd(pgd, pgdp_get(pgd_k));
|
2020-08-25 08:54:26 -07:00
|
|
|
|
|
|
|
p4d_k = p4d_offset(pgd_k, addr);
|
2023-12-13 13:30:01 -07:00
|
|
|
if (!p4d_present(p4dp_get(p4d_k))) {
|
2020-08-25 08:54:26 -07:00
|
|
|
no_context(regs, addr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
pud_k = pud_offset(p4d_k, addr);
|
2023-12-13 13:30:01 -07:00
|
|
|
if (!pud_present(pudp_get(pud_k))) {
|
2020-08-25 08:54:26 -07:00
|
|
|
no_context(regs, addr);
|
|
|
|
return;
|
|
|
|
}
|
2023-12-13 13:30:01 -07:00
|
|
|
if (pud_leaf(pudp_get(pud_k)))
|
2023-03-10 00:50:21 -07:00
|
|
|
goto flush_tlb;
|
2020-08-25 08:54:26 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Since the vmalloc area is global, it is unnecessary
|
|
|
|
* to copy individual PTEs
|
|
|
|
*/
|
|
|
|
pmd_k = pmd_offset(pud_k, addr);
|
2023-12-13 13:30:01 -07:00
|
|
|
if (!pmd_present(pmdp_get(pmd_k))) {
|
2020-08-25 08:54:26 -07:00
|
|
|
no_context(regs, addr);
|
|
|
|
return;
|
|
|
|
}
|
2023-12-13 13:30:01 -07:00
|
|
|
if (pmd_leaf(pmdp_get(pmd_k)))
|
2023-03-10 00:50:21 -07:00
|
|
|
goto flush_tlb;
|
2020-08-25 08:54:26 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure the actual PTE exists as well to
|
|
|
|
* catch kernel vmalloc-area accesses to non-mapped
|
|
|
|
* addresses. If we don't do this, this will just
|
|
|
|
* silently loop forever.
|
|
|
|
*/
|
|
|
|
pte_k = pte_offset_kernel(pmd_k, addr);
|
2023-12-13 13:30:01 -07:00
|
|
|
if (!pte_present(ptep_get(pte_k))) {
|
2020-08-25 08:54:26 -07:00
|
|
|
no_context(regs, addr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The kernel assumes that TLBs don't cache invalid
|
|
|
|
* entries, but in RISC-V, SFENCE.VMA specifies an
|
|
|
|
* ordering constraint, not a cache flush; it is
|
|
|
|
* necessary even after writing invalid entries.
|
|
|
|
*/
|
2023-03-10 00:50:21 -07:00
|
|
|
flush_tlb:
|
2020-08-25 08:54:26 -07:00
|
|
|
local_flush_tlb_page(addr);
|
|
|
|
}
|
|
|
|
|
2020-08-25 09:41:17 -07:00
|
|
|
static inline bool access_error(unsigned long cause, struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
switch (cause) {
|
|
|
|
case EXC_INST_PAGE_FAULT:
|
|
|
|
if (!(vma->vm_flags & VM_EXEC)) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case EXC_LOAD_PAGE_FAULT:
|
2022-09-15 12:37:01 -07:00
|
|
|
/* Write implies read */
|
|
|
|
if (!(vma->vm_flags & (VM_READ | VM_WRITE))) {
|
2020-08-25 09:41:17 -07:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case EXC_STORE_PAGE_FAULT:
|
|
|
|
if (!(vma->vm_flags & VM_WRITE)) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
panic("%s: unhandled cause %lu", __func__, cause);
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2017-07-10 18:06:09 -07:00
|
|
|
/*
|
|
|
|
* This routine handles page faults. It determines the address and the
|
|
|
|
* problem, and then passes it off to one of the appropriate routines.
|
|
|
|
*/
|
2023-02-21 20:30:18 -07:00
|
|
|
void handle_page_fault(struct pt_regs *regs)
|
2017-07-10 18:06:09 -07:00
|
|
|
{
|
|
|
|
struct task_struct *tsk;
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
struct mm_struct *mm;
|
|
|
|
unsigned long addr, cause;
|
2020-04-01 21:08:37 -07:00
|
|
|
unsigned int flags = FAULT_FLAG_DEFAULT;
|
2018-08-17 15:44:47 -07:00
|
|
|
int code = SEGV_MAPERR;
|
|
|
|
vm_fault_t fault;
|
2017-07-10 18:06:09 -07:00
|
|
|
|
2019-10-28 05:10:32 -07:00
|
|
|
cause = regs->cause;
|
|
|
|
addr = regs->badaddr;
|
2017-07-10 18:06:09 -07:00
|
|
|
|
|
|
|
tsk = current;
|
|
|
|
mm = tsk->mm;
|
|
|
|
|
2020-12-17 09:01:42 -07:00
|
|
|
if (kprobe_page_fault(regs, cause))
|
|
|
|
return;
|
|
|
|
|
2017-07-10 18:06:09 -07:00
|
|
|
/*
|
|
|
|
* Fault-in kernel-space virtual memory on-demand.
|
|
|
|
* The 'reference' page table is init_mm.pgd.
|
|
|
|
*
|
|
|
|
* NOTE! We MUST NOT take any locks for this case. We may
|
|
|
|
* be in an interrupt or a critical region, and should
|
|
|
|
* only copy the information from the master page table,
|
|
|
|
* nothing more.
|
|
|
|
*/
|
2023-05-31 02:38:17 -07:00
|
|
|
if ((!IS_ENABLED(CONFIG_MMU) || !IS_ENABLED(CONFIG_64BIT)) &&
|
|
|
|
unlikely(addr >= VMALLOC_START && addr < VMALLOC_END)) {
|
2020-08-25 08:54:26 -07:00
|
|
|
vmalloc_fault(regs, code, addr);
|
|
|
|
return;
|
|
|
|
}
|
2017-07-10 18:06:09 -07:00
|
|
|
|
|
|
|
/* Enable interrupts if they were enabled in the parent context. */
|
2023-02-21 20:30:18 -07:00
|
|
|
if (!regs_irqs_disabled(regs))
|
2017-07-10 18:06:09 -07:00
|
|
|
local_irq_enable();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we're in an interrupt, have no user context, or are running
|
|
|
|
* in an atomic region, then we must not take the fault.
|
|
|
|
*/
|
2020-08-25 08:38:58 -07:00
|
|
|
if (unlikely(faulthandler_disabled() || !mm)) {
|
2020-12-17 09:01:44 -07:00
|
|
|
tsk->thread.bad_cause = cause;
|
2020-08-25 08:38:58 -07:00
|
|
|
no_context(regs, addr);
|
|
|
|
return;
|
|
|
|
}
|
2017-07-10 18:06:09 -07:00
|
|
|
|
|
|
|
if (user_mode(regs))
|
|
|
|
flags |= FAULT_FLAG_USER;
|
|
|
|
|
2023-02-14 09:25:15 -07:00
|
|
|
if (!user_mode(regs) && addr < TASK_SIZE && unlikely(!(regs->status & SR_SUM))) {
|
|
|
|
if (fixup_exception(regs))
|
|
|
|
return;
|
|
|
|
|
|
|
|
die_kernel_fault("access to user memory without uaccess routines", addr, regs);
|
|
|
|
}
|
2020-12-03 22:42:59 -07:00
|
|
|
|
2017-07-10 18:06:09 -07:00
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
|
|
|
|
|
2020-08-25 09:42:54 -07:00
|
|
|
if (cause == EXC_STORE_PAGE_FAULT)
|
|
|
|
flags |= FAULT_FLAG_WRITE;
|
2020-09-04 23:07:26 -07:00
|
|
|
else if (cause == EXC_INST_PAGE_FAULT)
|
|
|
|
flags |= FAULT_FLAG_INSTRUCTION;
|
2023-05-23 09:59:42 -07:00
|
|
|
if (!(flags & FAULT_FLAG_USER))
|
|
|
|
goto lock_mmap;
|
|
|
|
|
|
|
|
vma = lock_vma_under_rcu(mm, addr);
|
|
|
|
if (!vma)
|
|
|
|
goto lock_mmap;
|
|
|
|
|
|
|
|
if (unlikely(access_error(cause, vma))) {
|
|
|
|
vma_end_read(vma);
|
2024-04-03 01:38:03 -07:00
|
|
|
count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
|
2024-05-30 09:44:51 -07:00
|
|
|
tsk->thread.bad_cause = cause;
|
|
|
|
bad_area_nosemaphore(regs, SEGV_ACCERR, addr);
|
2024-04-03 01:38:03 -07:00
|
|
|
return;
|
2023-05-23 09:59:42 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs);
|
2023-06-30 14:19:54 -07:00
|
|
|
if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
|
|
|
|
vma_end_read(vma);
|
2023-05-23 09:59:42 -07:00
|
|
|
|
|
|
|
if (!(fault & VM_FAULT_RETRY)) {
|
|
|
|
count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
count_vm_vma_lock_event(VMA_LOCK_RETRY);
|
2023-12-26 14:46:10 -07:00
|
|
|
if (fault & VM_FAULT_MAJOR)
|
|
|
|
flags |= FAULT_FLAG_TRIED;
|
2023-05-23 09:59:42 -07:00
|
|
|
|
|
|
|
if (fault_signal_pending(fault, regs)) {
|
|
|
|
if (!user_mode(regs))
|
|
|
|
no_context(regs, addr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
lock_mmap:
|
|
|
|
|
2017-07-10 18:06:09 -07:00
|
|
|
retry:
|
2023-06-22 11:18:18 -07:00
|
|
|
vma = lock_mm_and_find_vma(mm, addr, regs);
|
2020-08-25 08:48:01 -07:00
|
|
|
if (unlikely(!vma)) {
|
2020-12-17 09:01:44 -07:00
|
|
|
tsk->thread.bad_cause = cause;
|
2023-06-22 11:18:18 -07:00
|
|
|
bad_area_nosemaphore(regs, code, addr);
|
2020-08-25 08:48:01 -07:00
|
|
|
return;
|
|
|
|
}
|
2017-07-10 18:06:09 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Ok, we have a good vm_area for this memory access, so
|
|
|
|
* we can handle it.
|
|
|
|
*/
|
|
|
|
code = SEGV_ACCERR;
|
|
|
|
|
2020-08-25 09:41:17 -07:00
|
|
|
if (unlikely(access_error(cause, vma))) {
|
2020-12-17 09:01:44 -07:00
|
|
|
tsk->thread.bad_cause = cause;
|
2020-08-25 09:41:17 -07:00
|
|
|
bad_area(regs, mm, code, addr);
|
|
|
|
return;
|
2017-07-10 18:06:09 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If for any reason at all we could not handle the fault,
|
|
|
|
* make sure we exit gracefully rather than endlessly redo
|
|
|
|
* the fault.
|
|
|
|
*/
|
2020-08-11 18:38:34 -07:00
|
|
|
fault = handle_mm_fault(vma, addr, flags, regs);
|
2017-07-10 18:06:09 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we need to retry but a fatal signal is pending, handle the
|
2020-06-08 21:33:54 -07:00
|
|
|
* signal first. We do not need to release the mmap_lock because it
|
2017-07-10 18:06:09 -07:00
|
|
|
* would already be released in __lock_page_or_retry in mm/filemap.c.
|
|
|
|
*/
|
2023-01-30 21:55:52 -07:00
|
|
|
if (fault_signal_pending(fault, regs)) {
|
|
|
|
if (!user_mode(regs))
|
|
|
|
no_context(regs, addr);
|
2017-07-10 18:06:09 -07:00
|
|
|
return;
|
2023-01-30 21:55:52 -07:00
|
|
|
}
|
2017-07-10 18:06:09 -07:00
|
|
|
|
mm: avoid unnecessary page fault retires on shared memory types
I observed that for each of the shared file-backed page faults, we're very
likely to retry one more time for the 1st write fault upon no page. It's
because we'll need to release the mmap lock for dirty rate limit purpose
with balance_dirty_pages_ratelimited() (in fault_dirty_shared_page()).
Then after that throttling we return VM_FAULT_RETRY.
We did that probably because VM_FAULT_RETRY is the only way we can return
to the fault handler at that time telling it we've released the mmap lock.
However that's not ideal because it's very likely the fault does not need
to be retried at all since the pgtable was well installed before the
throttling, so the next continuous fault (including taking mmap read lock,
walk the pgtable, etc.) could be in most cases unnecessary.
It's not only slowing down page faults for shared file-backed, but also add
more mmap lock contention which is in most cases not needed at all.
To observe this, one could try to write to some shmem page and look at
"pgfault" value in /proc/vmstat, then we should expect 2 counts for each
shmem write simply because we retried, and vm event "pgfault" will capture
that.
To make it more efficient, add a new VM_FAULT_COMPLETED return code just to
show that we've completed the whole fault and released the lock. It's also
a hint that we should very possibly not need another fault immediately on
this page because we've just completed it.
This patch provides a ~12% perf boost on my aarch64 test VM with a simple
program sequentially dirtying 400MB shmem file being mmap()ed and these are
the time it needs:
Before: 650.980 ms (+-1.94%)
After: 569.396 ms (+-1.38%)
I believe it could help more than that.
We need some special care on GUP and the s390 pgfault handler (for gmap
code before returning from pgfault), the rest changes in the page fault
handlers should be relatively straightforward.
Another thing to mention is that mm_account_fault() does take this new
fault as a generic fault to be accounted, unlike VM_FAULT_RETRY.
I explicitly didn't touch hmm_vma_fault() and break_ksm() because they do
not handle VM_FAULT_RETRY even with existing code, so I'm literally keeping
them as-is.
Link: https://lkml.kernel.org/r/20220530183450.42886-1-peterx@redhat.com
Signed-off-by: Peter Xu <peterx@redhat.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vineet Gupta <vgupta@kernel.org>
Acked-by: Guo Ren <guoren@kernel.org>
Acked-by: Max Filippov <jcmvbkbc@gmail.com>
Acked-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc)
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Alistair Popple <apopple@nvidia.com>
Reviewed-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk> [arm part]
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Stafford Horne <shorne@gmail.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: Brian Cain <bcain@quicinc.com>
Cc: Richard Henderson <rth@twiddle.net>
Cc: Richard Weinberger <richard@nod.at>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Janosch Frank <frankja@linux.ibm.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: James Bottomley <James.Bottomley@HansenPartnership.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Will Deacon <will@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: Chris Zankel <chris@zankel.net>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Rich Felker <dalias@libc.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Helge Deller <deller@gmx.de>
Cc: Yoshinori Sato <ysato@users.osdn.me>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2022-05-30 11:34:50 -07:00
|
|
|
/* The fault is fully completed (including releasing mmap lock) */
|
|
|
|
if (fault & VM_FAULT_COMPLETED)
|
|
|
|
return;
|
|
|
|
|
2022-01-14 15:05:51 -07:00
|
|
|
if (unlikely(fault & VM_FAULT_RETRY)) {
|
2020-08-19 07:10:11 -07:00
|
|
|
flags |= FAULT_FLAG_TRIED;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No need to mmap_read_unlock(mm) as we would
|
|
|
|
* have already released it in __lock_page_or_retry
|
|
|
|
* in mm/filemap.c.
|
|
|
|
*/
|
|
|
|
goto retry;
|
2017-07-10 18:06:09 -07:00
|
|
|
}
|
|
|
|
|
2020-06-08 21:33:25 -07:00
|
|
|
mmap_read_unlock(mm);
|
2020-08-25 09:01:43 -07:00
|
|
|
|
2023-05-23 09:59:42 -07:00
|
|
|
done:
|
2020-08-25 09:01:43 -07:00
|
|
|
if (unlikely(fault & VM_FAULT_ERROR)) {
|
2020-12-17 09:01:44 -07:00
|
|
|
tsk->thread.bad_cause = cause;
|
2020-08-25 09:04:09 -07:00
|
|
|
mm_fault_error(regs, addr, fault);
|
2020-08-25 08:38:58 -07:00
|
|
|
return;
|
|
|
|
}
|
2017-07-10 18:06:09 -07:00
|
|
|
return;
|
|
|
|
}
|