From ebfc726eae3f31bdb5fae1bbd74ef235d71046ca Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 8 May 2024 09:13:57 +0100 Subject: [PATCH 1/9] arm64: barrier: Restore spec_bar() macro Upcoming errata workarounds will need to use SB from C code. Restore the spec_bar() macro so that we can use SB. This is effectively a revert of commit: 4f30ba1cce36d413 ("arm64: barrier: Remove spec_bar() macro") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: James Morse Cc: Will Deacon Link: https://lore.kernel.org/r/20240508081400.235362-2-mark.rutland@arm.com Signed-off-by: Will Deacon --- arch/arm64/include/asm/barrier.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index cf2987464c18..1ca947d5c939 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -40,6 +40,10 @@ */ #define dgh() asm volatile("hint #6" : : : "memory") +#define spec_bar() asm volatile(ALTERNATIVE("dsb nsh\nisb\n", \ + SB_BARRIER_INSN"nop\n", \ + ARM64_HAS_SB)) + #ifdef CONFIG_ARM64_PSEUDO_NMI #define pmr_sync() \ do { \ From 02a0a04676fa7796d9cbc9eb5ca120aaa194d2dd Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 8 May 2024 09:13:58 +0100 Subject: [PATCH 2/9] arm64: cputype: Add Cortex-X4 definitions Add cputype definitions for Cortex-X4. These will be used for errata detection in subsequent patches. These values can be found in Table B-249 ("MIDR_EL1 bit descriptions") in issue 0002-05 of the Cortex-X4 TRM, which can be found at: https://developer.arm.com/documentation/102484/0002/?lang=en Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: James Morse Cc: Will Deacon Link: https://lore.kernel.org/r/20240508081400.235362-3-mark.rutland@arm.com Signed-off-by: Will Deacon --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 52f076afeb96..2989c023c6f3 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -86,6 +86,7 @@ #define ARM_CPU_PART_CORTEX_X2 0xD48 #define ARM_CPU_PART_NEOVERSE_N2 0xD49 #define ARM_CPU_PART_CORTEX_A78C 0xD4B +#define ARM_CPU_PART_CORTEX_X4 0xD82 #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -159,6 +160,7 @@ #define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2) #define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2) #define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C) +#define MIDR_CORTEX_X4 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X4) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) From 0ce85db6c2141b7ffb95709d76fc55a27ff3cdc1 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 8 May 2024 09:13:59 +0100 Subject: [PATCH 3/9] arm64: cputype: Add Neoverse-V3 definitions Add cputype definitions for Neoverse-V3. These will be used for errata detection in subsequent patches. These values can be found in Table B-249 ("MIDR_EL1 bit descriptions") in issue 0001-04 of the Neoverse-V3 TRM, which can be found at: https://developer.arm.com/documentation/107734/0001/?lang=en Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: James Morse Cc: Will Deacon Link: https://lore.kernel.org/r/20240508081400.235362-4-mark.rutland@arm.com Signed-off-by: Will Deacon --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 2989c023c6f3..67a86926ae16 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -87,6 +87,7 @@ #define ARM_CPU_PART_NEOVERSE_N2 0xD49 #define ARM_CPU_PART_CORTEX_A78C 0xD4B #define ARM_CPU_PART_CORTEX_X4 0xD82 +#define ARM_CPU_PART_NEOVERSE_V3 0xD84 #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -161,6 +162,7 @@ #define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2) #define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C) #define MIDR_CORTEX_X4 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X4) +#define MIDR_NEOVERSE_V3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V3) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) From 7187bb7d0b5c7dfa18ca82e9e5c75e13861b1d88 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 8 May 2024 09:14:00 +0100 Subject: [PATCH 4/9] arm64: errata: Add workaround for Arm errata 3194386 and 3312417 Cortex-X4 and Neoverse-V3 suffer from errata whereby an MSR to the SSBS special-purpose register does not affect subsequent speculative instructions, permitting speculative store bypassing for a window of time. This is described in their Software Developer Errata Notice (SDEN) documents: * Cortex-X4 SDEN v8.0, erratum 3194386: https://developer.arm.com/documentation/SDEN-2432808/0800/ * Neoverse-V3 SDEN v6.0, erratum 3312417: https://developer.arm.com/documentation/SDEN-2891958/0600/ To workaround these errata, it is necessary to place a speculation barrier (SB) after MSR to the SSBS special-purpose register. This patch adds the requisite SB after writes to SSBS within the kernel, and hides the presence of SSBS from EL0 such that userspace software which cares about SSBS will manipulate this via prctl(PR_GET_SPECULATION_CTRL, ...). Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: James Morse Cc: Will Deacon Link: https://lore.kernel.org/r/20240508081400.235362-5-mark.rutland@arm.com Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 4 ++ arch/arm64/Kconfig | 42 +++++++++++++++++++++ arch/arm64/include/asm/cpucaps.h | 2 + arch/arm64/kernel/cpu_errata.c | 19 ++++++++++ arch/arm64/kernel/cpufeature.c | 8 ++++ arch/arm64/kernel/proton-pack.c | 12 ++++++ arch/arm64/tools/cpucaps | 1 + 7 files changed, 88 insertions(+) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index d33e27c5ce61..eb8af8032c31 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -140,6 +140,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X4 | #3194386 | ARM64_ERRATUM_3194386 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1349291 | N/A | @@ -156,6 +158,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #1619801 | N/A | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3312417 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-600 | #1076982,1209401| N/A | diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 7b11c98b3e84..89410e16d390 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1064,6 +1064,48 @@ config ARM64_ERRATUM_3117295 If unsure, say Y. +config ARM64_WORKAROUND_SPECULATIVE_SSBS + bool + +config ARM64_ERRATUM_3194386 + bool "Cortex-X4: 3194386: workaround for MSR SSBS not self-synchronizing" + select ARM64_WORKAROUND_SPECULATIVE_SSBS + default y + help + This option adds the workaround for ARM Cortex-X4 erratum 3194386. + + On affected cores "MSR SSBS, #0" instructions may not affect + subsequent speculative instructions, which may permit unexepected + speculative store bypassing. + + Work around this problem by placing a speculation barrier after + kernel changes to SSBS. The presence of the SSBS special-purpose + register is hidden from hwcaps and EL0 reads of ID_AA64PFR1_EL1, such + that userspace will use the PR_SPEC_STORE_BYPASS prctl to change + SSBS. + + If unsure, say Y. + +config ARM64_ERRATUM_3312417 + bool "Neoverse-V3: 3312417: workaround for MSR SSBS not self-synchronizing" + select ARM64_WORKAROUND_SPECULATIVE_SSBS + default y + help + This option adds the workaround for ARM Neoverse-V3 erratum 3312417. + + On affected cores "MSR SSBS, #0" instructions may not affect + subsequent speculative instructions, which may permit unexepected + speculative store bypassing. + + Work around this problem by placing a speculation barrier after + kernel changes to SSBS. The presence of the SSBS special-purpose + register is hidden from hwcaps and EL0 reads of ID_AA64PFR1_EL1, such + that userspace will use the PR_SPEC_STORE_BYPASS prctl to change + SSBS. + + If unsure, say Y. + + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 270680e2b5c4..7529c0263933 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -58,6 +58,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_NVIDIA_CARMEL_CNP_ERRATUM); case ARM64_WORKAROUND_REPEAT_TLBI: return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); + case ARM64_WORKAROUND_SPECULATIVE_SSBS: + return IS_ENABLED(CONFIG_ARM64_WORKAROUND_SPECULATIVE_SSBS); } return true; diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 76b8dd37092a..828be635e7e1 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -432,6 +432,18 @@ static const struct midr_range erratum_spec_unpriv_load_list[] = { }; #endif +#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_SSBS +static const struct midr_range erratum_spec_ssbs_list[] = { +#ifdef CONFIG_ARM64_ERRATUM_3194386 + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), +#endif +#ifdef CONFIG_ARM64_ERRATUM_3312417 + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), +#endif + {} +}; +#endif + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -729,6 +741,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { MIDR_FIXED(MIDR_CPU_VAR_REV(1,1), BIT(25)), }, #endif +#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_SSBS + { + .desc = "ARM errata 3194386, 3312417", + .capability = ARM64_WORKAROUND_SPECULATIVE_SSBS, + ERRATA_MIDR_RANGE_LIST(erratum_spec_ssbs_list), + }, +#endif #ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD { .desc = "ARM errata 2966298, 3117295", diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 56583677c1f2..48e7029f1054 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2307,6 +2307,14 @@ static void user_feature_fixup(void) if (regp) regp->user_mask &= ~ID_AA64ISAR1_EL1_BF16_MASK; } + + if (cpus_have_cap(ARM64_WORKAROUND_SPECULATIVE_SSBS)) { + struct arm64_ftr_reg *regp; + + regp = get_arm64_ftr_reg(SYS_ID_AA64PFR1_EL1); + if (regp) + regp->user_mask &= ~ID_AA64PFR1_EL1_SSBS_MASK; + } } static void elf_hwcap_fixup(void) diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 6268a13a1d58..baca47bd443c 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -558,6 +558,18 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void) /* SCTLR_EL1.DSSBS was initialised to 0 during boot */ set_pstate_ssbs(0); + + /* + * SSBS is self-synchronizing and is intended to affect subsequent + * speculative instructions, but some CPUs can speculate with a stale + * value of SSBS. + * + * Mitigate this with an unconditional speculation barrier, as CPUs + * could mis-speculate branches and bypass a conditional barrier. + */ + if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_SPECULATIVE_SSBS)) + spec_bar(); + return SPECTRE_MITIGATED; } diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 62b2838a231a..ac3429d892b9 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -102,4 +102,5 @@ WORKAROUND_NVIDIA_CARMEL_CNP WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_REPEAT_TLBI WORKAROUND_SPECULATIVE_AT +WORKAROUND_SPECULATIVE_SSBS WORKAROUND_SPECULATIVE_UNPRIV_LOAD From b8995a18417088bb53f87c49d200ec72a9dd4ec1 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 17 May 2024 12:55:55 +0100 Subject: [PATCH 5/9] Revert "arm64: fpsimd: Implement lazy restore for kernel mode FPSIMD" This reverts commit 2632e25217696712681dd1f3ecc0d71624ea3b23. Johannes (and others) report data corruption with dm-crypt on Apple M1 which has been bisected to this change. Revert the offending commit while we figure out what's going on. Cc: stable@vger.kernel.org Reported-by: Johannes Nixdorf Link: https://lore.kernel.org/all/D1B7GPIR9K1E.5JFV37G0YTIF@shadowice.org/ Signed-off-by: Will Deacon --- arch/arm64/include/asm/processor.h | 1 - arch/arm64/kernel/fpsimd.c | 18 ------------------ 2 files changed, 19 deletions(-) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index f77371232d8c..4f122b932cca 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -171,7 +171,6 @@ struct thread_struct { struct debug_info debug; /* debugging */ struct user_fpsimd_state kernel_fpsimd_state; - unsigned int kernel_fpsimd_cpu; #ifdef CONFIG_ARM64_PTR_AUTH struct ptrauth_keys_user keys_user; #ifdef CONFIG_ARM64_PTR_AUTH_KERNEL diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index ebb0158997ca..e9d70dc91511 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1509,30 +1509,12 @@ void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs) static void fpsimd_load_kernel_state(struct task_struct *task) { - struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state); - - /* - * Elide the load if this CPU holds the most recent kernel mode - * FPSIMD context of the current task. - */ - if (last->st == &task->thread.kernel_fpsimd_state && - task->thread.kernel_fpsimd_cpu == smp_processor_id()) - return; - fpsimd_load_state(&task->thread.kernel_fpsimd_state); } static void fpsimd_save_kernel_state(struct task_struct *task) { - struct cpu_fp_state cpu_fp_state = { - .st = &task->thread.kernel_fpsimd_state, - .to_save = FP_STATE_FPSIMD, - }; - fpsimd_save_state(&task->thread.kernel_fpsimd_state); - fpsimd_bind_state_to_cpu(&cpu_fp_state); - - task->thread.kernel_fpsimd_cpu = smp_processor_id(); } void fpsimd_thread_switch(struct task_struct *next) From a4c5a457c6107dfe9dc65a104af1634811396bac Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 14 May 2024 11:00:50 -0700 Subject: [PATCH 6/9] perf/arm-dmc620: Fix lockdep assert in ->event_init() for_each_sibling_event() checks leader's ctx but it doesn't have the ctx yet if it's the leader. Like in perf_event_validate_size(), we should skip checking siblings in that case. Acked-by: Mark Rutland Fixes: f3c0eba28704 ("perf: Add a few assertions") Reported-by: Greg Thelen Cc: Robin Murphy Cc: Tuan Phan Signed-off-by: Namhyung Kim Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20240514180050.182454-1-namhyung@kernel.org Signed-off-by: Will Deacon --- drivers/perf/arm_dmc620_pmu.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/perf/arm_dmc620_pmu.c b/drivers/perf/arm_dmc620_pmu.c index 2ec96e204c40..7e5f1d4fca0f 100644 --- a/drivers/perf/arm_dmc620_pmu.c +++ b/drivers/perf/arm_dmc620_pmu.c @@ -542,12 +542,16 @@ static int dmc620_pmu_event_init(struct perf_event *event) if (event->cpu < 0) return -EINVAL; + hwc->idx = -1; + + if (event->group_leader == event) + return 0; + /* * We can't atomically disable all HW counters so only one event allowed, * although software events are acceptable. */ - if (event->group_leader != event && - !is_software_event(event->group_leader)) + if (!is_software_event(event->group_leader)) return -EINVAL; for_each_sibling_event(sibling, event->group_leader) { @@ -556,7 +560,6 @@ static int dmc620_pmu_event_init(struct perf_event *event) return -EINVAL; } - hwc->idx = -1; return 0; } From ffbf4fb9b5c12ff878a10ea17997147ea4ebea6f Mon Sep 17 00:00:00 2001 From: Jiangfeng Xiao Date: Mon, 20 May 2024 21:34:37 +0800 Subject: [PATCH 7/9] arm64: asm-bug: Add .align 2 to the end of __BUG_ENTRY When CONFIG_DEBUG_BUGVERBOSE=n, we fail to add necessary padding bytes to bug_table entries, and as a result the last entry in a bug table will be ignored, potentially leading to an unexpected panic(). All prior entries in the table will be handled correctly. The arm64 ABI requires that struct fields of up to 8 bytes are naturally-aligned, with padding added within a struct such that struct are suitably aligned within arrays. When CONFIG_DEBUG_BUGVERPOSE=y, the layout of a bug_entry is: struct bug_entry { signed int bug_addr_disp; // 4 bytes signed int file_disp; // 4 bytes unsigned short line; // 2 bytes unsigned short flags; // 2 bytes } ... with 12 bytes total, requiring 4-byte alignment. When CONFIG_DEBUG_BUGVERBOSE=n, the layout of a bug_entry is: struct bug_entry { signed int bug_addr_disp; // 4 bytes unsigned short flags; // 2 bytes < implicit padding > // 2 bytes } ... with 8 bytes total, with 6 bytes of data and 2 bytes of trailing padding, requiring 4-byte alginment. When we create a bug_entry in assembly, we align the start of the entry to 4 bytes, which implicitly handles padding for any prior entries. However, we do not align the end of the entry, and so when CONFIG_DEBUG_BUGVERBOSE=n, the final entry lacks the trailing padding bytes. For the main kernel image this is not a problem as find_bug() doesn't depend on the trailing padding bytes when searching for entries: for (bug = __start___bug_table; bug < __stop___bug_table; ++bug) if (bugaddr == bug_addr(bug)) return bug; However for modules, module_bug_finalize() depends on the trailing bytes when calculating the number of entries: mod->num_bugs = sechdrs[i].sh_size / sizeof(struct bug_entry); ... and as the last bug_entry lacks the necessary padding bytes, this entry will not be counted, e.g. in the case of a single entry: sechdrs[i].sh_size == 6 sizeof(struct bug_entry) == 8; sechdrs[i].sh_size / sizeof(struct bug_entry) == 0; Consequently module_find_bug() will miss the last bug_entry when it does: for (i = 0; i < mod->num_bugs; ++i, ++bug) if (bugaddr == bug_addr(bug)) goto out; ... which can lead to a kenrel panic due to an unhandled bug. This can be demonstrated with the following module: static int __init buginit(void) { WARN(1, "hello\n"); return 0; } static void __exit bugexit(void) { } module_init(buginit); module_exit(bugexit); MODULE_LICENSE("GPL"); ... which will trigger a kernel panic when loaded: ------------[ cut here ]------------ hello Unexpected kernel BRK exception at EL1 Internal error: BRK handler: 00000000f2000800 [#1] PREEMPT SMP Modules linked in: hello(O+) CPU: 0 PID: 50 Comm: insmod Tainted: G O 6.9.1 #8 Hardware name: linux,dummy-virt (DT) pstate: 60400005 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : buginit+0x18/0x1000 [hello] lr : buginit+0x18/0x1000 [hello] sp : ffff800080533ae0 x29: ffff800080533ae0 x28: 0000000000000000 x27: 0000000000000000 x26: ffffaba8c4e70510 x25: ffff800080533c30 x24: ffffaba8c4a28a58 x23: 0000000000000000 x22: 0000000000000000 x21: ffff3947c0eab3c0 x20: ffffaba8c4e3f000 x19: ffffaba846464000 x18: 0000000000000006 x17: 0000000000000000 x16: ffffaba8c2492834 x15: 0720072007200720 x14: 0720072007200720 x13: ffffaba8c49b27c8 x12: 0000000000000312 x11: 0000000000000106 x10: ffffaba8c4a0a7c8 x9 : ffffaba8c49b27c8 x8 : 00000000ffffefff x7 : ffffaba8c4a0a7c8 x6 : 80000000fffff000 x5 : 0000000000000107 x4 : 0000000000000000 x3 : 0000000000000000 x2 : 0000000000000000 x1 : 0000000000000000 x0 : ffff3947c0eab3c0 Call trace: buginit+0x18/0x1000 [hello] do_one_initcall+0x80/0x1c8 do_init_module+0x60/0x218 load_module+0x1ba4/0x1d70 __do_sys_init_module+0x198/0x1d0 __arm64_sys_init_module+0x1c/0x28 invoke_syscall+0x48/0x114 el0_svc_common.constprop.0+0x40/0xe0 do_el0_svc+0x1c/0x28 el0_svc+0x34/0xd8 el0t_64_sync_handler+0x120/0x12c el0t_64_sync+0x190/0x194 Code: d0ffffe0 910003fd 91000000 9400000b (d4210000) ---[ end trace 0000000000000000 ]--- Kernel panic - not syncing: BRK handler: Fatal exception Fix this by always aligning the end of a bug_entry to 4 bytes, which is correct regardless of CONFIG_DEBUG_BUGVERBOSE. Fixes: 9fb7410f955f ("arm64/BUG: Use BRK instruction for generic BUG traps") Signed-off-by: Yuanbin Xie Signed-off-by: Jiangfeng Xiao Reviewed-by: Mark Rutland Link: https://lore.kernel.org/r/1716212077-43826-1-git-send-email-xiaojiangfeng@huawei.com Signed-off-by: Will Deacon --- arch/arm64/include/asm/asm-bug.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/include/asm/asm-bug.h b/arch/arm64/include/asm/asm-bug.h index c762038ba400..6e73809f6492 100644 --- a/arch/arm64/include/asm/asm-bug.h +++ b/arch/arm64/include/asm/asm-bug.h @@ -28,6 +28,7 @@ 14470: .long 14471f - .; \ _BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ .short flags; \ + .align 2; \ .popsection; \ 14471: #else From f481bb32d60e45fb3d19ea68ce79c5629f3fc3a0 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 22 May 2024 11:53:05 +0100 Subject: [PATCH 8/9] Reapply "arm64: fpsimd: Implement lazy restore for kernel mode FPSIMD" This reverts commit b8995a18417088bb53f87c49d200ec72a9dd4ec1. Ard managed to reproduce the dm-crypt corruption problem and got to the bottom of it, so re-apply the problematic patch in preparation for fixing things properly. Cc: stable@vger.kernel.org Signed-off-by: Will Deacon --- arch/arm64/include/asm/processor.h | 1 + arch/arm64/kernel/fpsimd.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 4f122b932cca..f77371232d8c 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -171,6 +171,7 @@ struct thread_struct { struct debug_info debug; /* debugging */ struct user_fpsimd_state kernel_fpsimd_state; + unsigned int kernel_fpsimd_cpu; #ifdef CONFIG_ARM64_PTR_AUTH struct ptrauth_keys_user keys_user; #ifdef CONFIG_ARM64_PTR_AUTH_KERNEL diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index e9d70dc91511..ebb0158997ca 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1509,12 +1509,30 @@ void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs) static void fpsimd_load_kernel_state(struct task_struct *task) { + struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state); + + /* + * Elide the load if this CPU holds the most recent kernel mode + * FPSIMD context of the current task. + */ + if (last->st == &task->thread.kernel_fpsimd_state && + task->thread.kernel_fpsimd_cpu == smp_processor_id()) + return; + fpsimd_load_state(&task->thread.kernel_fpsimd_state); } static void fpsimd_save_kernel_state(struct task_struct *task) { + struct cpu_fp_state cpu_fp_state = { + .st = &task->thread.kernel_fpsimd_state, + .to_save = FP_STATE_FPSIMD, + }; + fpsimd_save_state(&task->thread.kernel_fpsimd_state); + fpsimd_bind_state_to_cpu(&cpu_fp_state); + + task->thread.kernel_fpsimd_cpu = smp_processor_id(); } void fpsimd_thread_switch(struct task_struct *next) From e92bee9f861b466c676f0200be3e46af7bc4ac6b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 22 May 2024 11:13:36 +0200 Subject: [PATCH 9/9] arm64/fpsimd: Avoid erroneous elide of user state reload TIF_FOREIGN_FPSTATE is a 'convenience' flag that should reflect whether the current CPU holds the most recent user mode FP/SIMD state of the current task. It combines two conditions: - whether the current CPU's FP/SIMD state belongs to the task; - whether that state is the most recent associated with the task (as a task may have executed on other CPUs as well). When a task is scheduled in and TIF_KERNEL_FPSTATE is set, it means the task was in a kernel mode NEON section when it was scheduled out, and so the kernel mode FP/SIMD state is restored. Since this implies that the current CPU is *not* holding the most recent user mode FP/SIMD state of the current task, the TIF_FOREIGN_FPSTATE flag is set too, so that the user mode FP/SIMD state is reloaded from memory when returning to userland. However, the task may be scheduled out after completing the kernel mode NEON section, but before returning to userland. When this happens, the TIF_FOREIGN_FPSTATE flag will not be preserved, but will be set as usual the next time the task is scheduled in, and will be based on the above conditions. This means that, rather than setting TIF_FOREIGN_FPSTATE when scheduling in a task with TIF_KERNEL_FPSTATE set, the underlying state should be updated so that TIF_FOREIGN_FPSTATE will assume the expected value as a result. So instead, call fpsimd_flush_cpu_state(), which takes care of this. Closes: https://lore.kernel.org/all/cb8822182231850108fa43e0446a4c7f@kernel.org Reported-by: Johannes Nixdorf Fixes: aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch") Cc: Mark Brown Cc: Dave Martin Cc: Janne Grunau Cc: stable@vger.kernel.org Signed-off-by: Ard Biesheuvel Tested-by: Janne Grunau Tested-by: Johannes Nixdorf Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20240522091335.335346-2-ardb+git@google.com Signed-off-by: Will Deacon --- arch/arm64/kernel/fpsimd.c | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index ebb0158997ca..82e8a6017382 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1535,6 +1535,27 @@ static void fpsimd_save_kernel_state(struct task_struct *task) task->thread.kernel_fpsimd_cpu = smp_processor_id(); } +/* + * Invalidate any task's FPSIMD state that is present on this cpu. + * The FPSIMD context should be acquired with get_cpu_fpsimd_context() + * before calling this function. + */ +static void fpsimd_flush_cpu_state(void) +{ + WARN_ON(!system_supports_fpsimd()); + __this_cpu_write(fpsimd_last_state.st, NULL); + + /* + * Leaving streaming mode enabled will cause issues for any kernel + * NEON and leaving streaming mode or ZA enabled may increase power + * consumption. + */ + if (system_supports_sme()) + sme_smstop(); + + set_thread_flag(TIF_FOREIGN_FPSTATE); +} + void fpsimd_thread_switch(struct task_struct *next) { bool wrong_task, wrong_cpu; @@ -1552,7 +1573,7 @@ void fpsimd_thread_switch(struct task_struct *next) if (test_tsk_thread_flag(next, TIF_KERNEL_FPSTATE)) { fpsimd_load_kernel_state(next); - set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE); + fpsimd_flush_cpu_state(); } else { /* * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's @@ -1842,27 +1863,6 @@ void fpsimd_flush_task_state(struct task_struct *t) barrier(); } -/* - * Invalidate any task's FPSIMD state that is present on this cpu. - * The FPSIMD context should be acquired with get_cpu_fpsimd_context() - * before calling this function. - */ -static void fpsimd_flush_cpu_state(void) -{ - WARN_ON(!system_supports_fpsimd()); - __this_cpu_write(fpsimd_last_state.st, NULL); - - /* - * Leaving streaming mode enabled will cause issues for any kernel - * NEON and leaving streaming mode or ZA enabled may increase power - * consumption. - */ - if (system_supports_sme()) - sme_smstop(); - - set_thread_flag(TIF_FOREIGN_FPSTATE); -} - /* * Save the FPSIMD state to memory and invalidate cpu view. * This function must be called with preemption disabled.