From 1e8a0be5ed3aaa26dd77927df32bbd0df3449a1e Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:35 -0800 Subject: [PATCH 01/52] alpha/cpu: Expose arch_cpu_idle_dead()'s prototype declaration Include to make sure arch_cpu_idle_dead() matches its prototype going forward. Link: https://lore.kernel.org/r/b0405c2ac5686303b6026e1ac27cfd769b21a7d0.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/alpha/kernel/process.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index e9cf7193eb81..3251102a9a47 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -9,6 +9,7 @@ * This file handles the architecture-dependent parts of process handling. */ +#include #include #include #include From 550db6795e9bb0b1b40f9c52bbe3d0ee735dbdb0 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:36 -0800 Subject: [PATCH 02/52] alpha/cpu: Make sure arch_cpu_idle_dead() doesn't return arch_cpu_idle_dead() doesn't return. Make that more explicit with a BUG(). BUG() is preferable to unreachable() because BUG() is a more explicit failure mode and avoids undefined behavior like falling off the edge of the function into whatever code happens to be next. Link: https://lore.kernel.org/r/19ffef09a175fecb783abcd44d6bcfeade2857eb.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/alpha/kernel/process.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index 3251102a9a47..a82fefa31bdb 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -63,6 +63,7 @@ void arch_cpu_idle(void) void arch_cpu_idle_dead(void) { wtint(INT_MAX); + BUG(); } #endif /* ALPHA_WTINT */ From b40c7d6d31ac2f6b78371cdc08bc1b6b62f01375 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Thu, 16 Feb 2023 10:38:55 -0800 Subject: [PATCH 03/52] arm/cpu: Add unreachable() to arch_cpu_idle_dead() arch_cpu_idle_dead() doesn't return. Make that visible to the compiler with an unreachable() code annotation. Link: https://lkml.kernel.org/r/20230216183851.s5bnvniomq44rytu@treble Signed-off-by: Josh Poimboeuf --- arch/arm/kernel/smp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 0b8c25763adc..441ea5cff390 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -382,6 +382,8 @@ void arch_cpu_idle_dead(void) : "r" (task_stack_page(current) + THREAD_SIZE - 8), "r" (current) : "r0"); + + unreachable(); } #endif /* CONFIG_HOTPLUG_CPU */ From 9bdc61ef27db5eaac48f1cc5deb8224603e79c89 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Thu, 16 Feb 2023 10:42:01 -0800 Subject: [PATCH 04/52] arm64/cpu: Mark cpu_die() __noreturn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpu_die() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Acked-by: Mark Rutland Reviewed-by: Philippe Mathieu-Daudé Link: https://lkml.kernel.org/r/20230216184157.4hup6y6mmspr2kll@treble Signed-off-by: Josh Poimboeuf --- arch/arm64/include/asm/smp.h | 2 +- arch/arm64/kernel/smp.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index fc55f5a57a06..5733a31bab08 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -100,7 +100,7 @@ static inline void arch_send_wakeup_ipi_mask(const struct cpumask *mask) extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); -extern void cpu_die(void); +extern void __noreturn cpu_die(void); extern void cpu_die_early(void); static inline void cpu_park_loop(void) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 4e8327264255..d5d09a18b4f8 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -361,7 +361,7 @@ void __cpu_die(unsigned int cpu) * Called from the idle thread for the CPU which has been shutdown. * */ -void cpu_die(void) +void __noreturn cpu_die(void) { unsigned int cpu = smp_processor_id(); const struct cpu_operations *ops = get_cpu_ops(cpu); From 1b2442a835a0acc7d76ae2f48ca7a94f962b29c8 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:39 -0800 Subject: [PATCH 05/52] csky/cpu: Make sure arch_cpu_idle_dead() doesn't return arch_cpu_idle_dead() doesn't return. Make that more explicit with a BUG(). BUG() is preferable to unreachable() because BUG() is a more explicit failure mode and avoids undefined behavior like falling off the edge of the function into whatever code happens to be next. Acked-by: Guo Ren Link: https://lore.kernel.org/r/1e9ecc3d248e82973e80bc336fc9f97e3ba2708d.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/csky/kernel/smp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index b45d1073307f..0ec20efaf5fd 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -317,5 +317,7 @@ void arch_cpu_idle_dead(void) "jmpi csky_start_secondary" : : "r" (secondary_stack)); + + BUG(); } #endif From 93c0edffbc92abe1efb8c7081df0cc1577a79ace Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:40 -0800 Subject: [PATCH 06/52] ia64/cpu: Mark play_dead() __noreturn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit play_dead() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Reviewed-by: Philippe Mathieu-Daudé Link: https://lore.kernel.org/r/7575bb38417bd8bcb5be980443f99cab29319342.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/ia64/kernel/process.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index f6195a0a00ae..78f5794b2dde 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -201,7 +201,7 @@ __setup("nohalt", nohalt_setup); #ifdef CONFIG_HOTPLUG_CPU /* We don't actually take CPU down, just spin without interrupts. */ -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { unsigned int this_cpu = smp_processor_id(); @@ -219,7 +219,7 @@ static inline void play_dead(void) BUG(); } #else -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { BUG(); } From 13bf7923a4dd34f3d2681768a148b10ddbdb95ed Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:41 -0800 Subject: [PATCH 07/52] loongarch/cpu: Make sure play_dead() doesn't return play_dead() doesn't return. Make that more explicit with a BUG(). BUG() is preferable to unreachable() because BUG() is a more explicit failure mode and avoids undefined behavior like falling off the edge of the function into whatever code happens to be next. Link: https://lore.kernel.org/r/21245d687ffeda34dbcf04961a2df3724f04f7c8.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/loongarch/kernel/smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 8c6e227cb29d..51f328169a7b 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -336,7 +336,7 @@ void play_dead(void) iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); init_fn(); - unreachable(); + BUG(); } #endif From 6c0f2d071eded22cb72413e0abfaa5b3373b4d96 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:42 -0800 Subject: [PATCH 08/52] loongarch/cpu: Mark play_dead() __noreturn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit play_dead() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Reviewed-by: Philippe Mathieu-Daudé Link: https://lore.kernel.org/r/4da55acfdec8a9132c4e21ffb7edb1f846841193.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/loongarch/include/asm/smp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h index d82687390b4a..416b653bccb4 100644 --- a/arch/loongarch/include/asm/smp.h +++ b/arch/loongarch/include/asm/smp.h @@ -99,7 +99,7 @@ static inline void __cpu_die(unsigned int cpu) loongson_cpu_die(cpu); } -extern void play_dead(void); +extern void __noreturn play_dead(void); #endif #endif /* __ASM_SMP_H */ From 142dbcf3b6a93ecfe8ccbee60c611175f5459c23 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Thu, 16 Feb 2023 10:42:52 -0800 Subject: [PATCH 09/52] mips/cpu: Expose play_dead()'s prototype definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include to make sure play_dead() matches its prototype going forward. Acked-by: Florian Fainelli Reviewed-by: Philippe Mathieu-Daudé Link: https://lkml.kernel.org/r/20230216184249.ogaqsaykottpxtcb@treble Signed-off-by: Josh Poimboeuf --- arch/mips/cavium-octeon/smp.c | 1 + arch/mips/kernel/smp-bmips.c | 2 ++ arch/mips/kernel/smp-cps.c | 1 + arch/mips/loongson64/smp.c | 1 + 4 files changed, 5 insertions(+) diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c index 89954f5f87fb..4212584e6efa 100644 --- a/arch/mips/cavium-octeon/smp.c +++ b/arch/mips/cavium-octeon/smp.c @@ -20,6 +20,7 @@ #include #include #include +#include #include diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c index f5d7bfa3472a..51d5dae1d2f0 100644 --- a/arch/mips/kernel/smp-bmips.c +++ b/arch/mips/kernel/smp-bmips.c @@ -54,6 +54,8 @@ static void bmips_set_reset_vec(int cpu, u32 val); #ifdef CONFIG_SMP +#include + /* initial $sp, $gp - used by arch/mips/kernel/bmips_vec.S */ unsigned long bmips_smp_boot_sp; unsigned long bmips_smp_boot_gp; diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c index 4fc288bb85b9..00a0e2571768 100644 --- a/arch/mips/kernel/smp-cps.c +++ b/arch/mips/kernel/smp-cps.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/loongson64/smp.c b/arch/mips/loongson64/smp.c index 660e1de4412a..4e24b317e7cb 100644 --- a/arch/mips/loongson64/smp.c +++ b/arch/mips/loongson64/smp.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include From a80ceed6c9c24088d7a2e3c414eae1fe88a8ccbe Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:44 -0800 Subject: [PATCH 10/52] mips/cpu: Make sure play_dead() doesn't return MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit play_dead() doesn't return. Make that more explicit with a BUG(). BUG() is preferable to unreachable() because BUG() is a more explicit failure mode and avoids undefined behavior like falling off the edge of the function into whatever code happens to be next. Acked-by: Florian Fainelli Reviewed-by: Philippe Mathieu-Daudé Link: https://lore.kernel.org/r/b195e4da190bb06b7d4af15d66ce6129e2347630.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/mips/kernel/smp-bmips.c | 2 ++ arch/mips/loongson64/smp.c | 1 + 2 files changed, 3 insertions(+) diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c index 51d5dae1d2f0..15466d4cf4a0 100644 --- a/arch/mips/kernel/smp-bmips.c +++ b/arch/mips/kernel/smp-bmips.c @@ -415,6 +415,8 @@ void __ref play_dead(void) " wait\n" " j bmips_secondary_reentry\n" : : : "memory"); + + BUG(); } #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/mips/loongson64/smp.c b/arch/mips/loongson64/smp.c index 4e24b317e7cb..df8d789ede3c 100644 --- a/arch/mips/loongson64/smp.c +++ b/arch/mips/loongson64/smp.c @@ -823,6 +823,7 @@ out: state_addr = &per_cpu(cpu_state, cpu); mb(); play_dead_at_ckseg1(state_addr); + BUG(); } static int loongson3_disable_clock(unsigned int cpu) From 9e57f049d1571db1dc77c69c87403defc9d3e5ae Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:45 -0800 Subject: [PATCH 11/52] mips/cpu: Mark play_dead() __noreturn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit play_dead() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Acked-by: Florian Fainelli Reviewed-by: Philippe Mathieu-Daudé Link: https://lore.kernel.org/r/2897b51a9b8beb5b594fe66fb1d3a479ddd2a0e2.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/mips/include/asm/smp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h index 5d9ff61004ca..4eee29b7845c 100644 --- a/arch/mips/include/asm/smp.h +++ b/arch/mips/include/asm/smp.h @@ -88,7 +88,7 @@ static inline void __cpu_die(unsigned int cpu) mp_ops->cpu_die(cpu); } -extern void play_dead(void); +extern void __noreturn play_dead(void); #endif #ifdef CONFIG_KEXEC From 5e00d69cdef4cd1c737e6286a39f83ce20a91034 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:46 -0800 Subject: [PATCH 12/52] powerpc/cpu: Mark start_secondary_resume() __noreturn start_secondary_resume() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Acked-by: Michael Ellerman (powerpc) Reviewed-by: Christophe Leroy Link: https://lore.kernel.org/r/b6b2141f832d8cd8ade65f190d04b011cda5f9bb.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/powerpc/include/asm/smp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index 6c6cb53d7045..aaaa576d0e15 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -67,7 +67,7 @@ void start_secondary(void *unused); extern int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us); extern int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us); extern void smp_send_debugger_break(void); -extern void start_secondary_resume(void); +extern void __noreturn start_secondary_resume(void); extern void smp_generic_give_timebase(void); extern void smp_generic_take_timebase(void); From 243971885418fcf772f18019eb3fabadcf0205d1 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:47 -0800 Subject: [PATCH 13/52] sh/cpu: Make sure play_dead() doesn't return play_dead() doesn't return. Make that more explicit with a BUG(). BUG() is preferable to unreachable() because BUG() is a more explicit failure mode and avoids undefined behavior like falling off the edge of the function into whatever code happens to be next. Link: https://lore.kernel.org/r/d0c3ff5349adfe8fd227acc236ae2c278a05eb4c.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/sh/include/asm/smp-ops.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sh/include/asm/smp-ops.h b/arch/sh/include/asm/smp-ops.h index e27702130eb6..63866b1595a0 100644 --- a/arch/sh/include/asm/smp-ops.h +++ b/arch/sh/include/asm/smp-ops.h @@ -27,6 +27,7 @@ static inline void plat_smp_setup(void) static inline void play_dead(void) { mp_ops->play_dead(); + BUG(); } extern void register_smp_ops(struct plat_smp_ops *ops); From 1644b74192265875cdf37dadfa33f34e0ea4fcc8 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:48 -0800 Subject: [PATCH 14/52] sh/cpu: Mark play_dead() __noreturn play_dead() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Link: https://lore.kernel.org/r/03549a74fad9f73576d57e6fc0b5102322f9cff4.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/sh/include/asm/smp-ops.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sh/include/asm/smp-ops.h b/arch/sh/include/asm/smp-ops.h index 63866b1595a0..97331fcb7b85 100644 --- a/arch/sh/include/asm/smp-ops.h +++ b/arch/sh/include/asm/smp-ops.h @@ -24,7 +24,7 @@ static inline void plat_smp_setup(void) mp_ops->smp_setup(); } -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { mp_ops->play_dead(); BUG(); @@ -43,7 +43,7 @@ static inline void register_smp_ops(struct plat_smp_ops *ops) { } -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { BUG(); } From fd49efb3c75475b65d7541ee40603498807ff110 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:49 -0800 Subject: [PATCH 15/52] sh/cpu: Expose arch_cpu_idle_dead()'s prototype definition Include to make sure arch_cpu_idle_dead() matches its prototype going forward. Link: https://lore.kernel.org/r/3d9661e97828fb464a48d4becf18f12604831903.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/sh/kernel/idle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c index 3418c40f0099..114f0c4abeac 100644 --- a/arch/sh/kernel/idle.c +++ b/arch/sh/kernel/idle.c @@ -4,6 +4,7 @@ * * Copyright (C) 2002 - 2009 Paul Mundt */ +#include #include #include #include From b9952d5009440b01dedd793a57abbe6cfe10f995 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:50 -0800 Subject: [PATCH 16/52] sparc/cpu: Mark cpu_play_dead() __noreturn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpu_play_dead() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Reviewed-by: Philippe Mathieu-Daudé Link: https://lore.kernel.org/r/847fdb53cc7124bb7c94e3e104e443a29be85184.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/sparc/include/asm/smp_64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sparc/include/asm/smp_64.h b/arch/sparc/include/asm/smp_64.h index e75783b6abc4..505b6700805d 100644 --- a/arch/sparc/include/asm/smp_64.h +++ b/arch/sparc/include/asm/smp_64.h @@ -49,7 +49,7 @@ int hard_smp_processor_id(void); void smp_fill_in_cpu_possible_map(void); void smp_fill_in_sib_core_maps(void); -void cpu_play_dead(void); +void __noreturn cpu_play_dead(void); void smp_fetch_global_regs(void); void smp_fetch_global_pmu(void); From a02f50b573b3c90a3633a5ab67435fe4c0de144d Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:51 -0800 Subject: [PATCH 17/52] x86/cpu: Make sure play_dead() doesn't return After commit 076cbf5d2163 ("x86/xen: don't let xen_pv_play_dead() return"), play_dead() never returns. Make that more explicit with a BUG(). BUG() is preferable to unreachable() because BUG() is a more explicit failure mode and avoids undefined behavior like falling off the edge of the function into whatever code happens to be next. Link: https://lore.kernel.org/r/11e6ac1cf10f92967882926e3ac16287b50642f2.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/x86/include/asm/smp.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index b4dbb20dab1a..8f628e08b25a 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -96,6 +96,7 @@ static inline void __cpu_die(unsigned int cpu) static inline void play_dead(void) { smp_ops.play_dead(); + BUG(); } static inline void smp_send_reschedule(int cpu) From eab89405b6b59947ee29cc21fb39ead66142c9b5 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:52 -0800 Subject: [PATCH 18/52] x86/cpu: Mark play_dead() __noreturn play_dead() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Link: https://lore.kernel.org/r/f3a069e6869c51ccfdda656b76882363bc9fcfa4.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/x86/include/asm/smp.h | 2 +- arch/x86/kernel/process.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 8f628e08b25a..e6d1d2810e38 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -93,7 +93,7 @@ static inline void __cpu_die(unsigned int cpu) smp_ops.cpu_die(cpu); } -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { smp_ops.play_dead(); BUG(); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index b650cde3f64d..f1ec36caf1d8 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -715,7 +715,7 @@ static bool x86_idle_set(void) } #ifndef CONFIG_SMP -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { BUG(); } From d08e12e8126ec88d6cb1a7eb160d8d7b52c8699f Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:53 -0800 Subject: [PATCH 19/52] xtensa/cpu: Make sure cpu_die() doesn't return cpu_die() doesn't return. Make that more explicit with a BUG(). BUG() is preferable to unreachable() because BUG() is a more explicit failure mode and avoids undefined behavior like falling off the edge of the function into whatever code happens to be next. Link: https://lore.kernel.org/r/cca346b5c87693499e630291d78fb0bf12c24290.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/xtensa/kernel/smp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/xtensa/kernel/smp.c b/arch/xtensa/kernel/smp.c index 4dc109dd6214..7bad78495536 100644 --- a/arch/xtensa/kernel/smp.c +++ b/arch/xtensa/kernel/smp.c @@ -341,6 +341,8 @@ void __ref cpu_die(void) __asm__ __volatile__( " movi a2, cpu_restart\n" " jx a2\n"); + + BUG(); } #endif /* CONFIG_HOTPLUG_CPU */ From 69dee6f0338a20c76a7bc61c0a6b59f25e1b25c8 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:54 -0800 Subject: [PATCH 20/52] xtensa/cpu: Mark cpu_die() __noreturn cpu_die() doesn't return. Annotate it as such. By extension this also makes arch_cpu_idle_dead() noreturn. Acked-by: Max Filippov Link: https://lore.kernel.org/r/ad801544cab7c26a0f3bbf7cfefb67303f4cd866.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/xtensa/include/asm/smp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/xtensa/include/asm/smp.h b/arch/xtensa/include/asm/smp.h index 4e43f5643891..5dc5bf8cdd77 100644 --- a/arch/xtensa/include/asm/smp.h +++ b/arch/xtensa/include/asm/smp.h @@ -33,7 +33,7 @@ void show_ipi_list(struct seq_file *p, int prec); void __cpu_die(unsigned int cpu); int __cpu_disable(void); -void cpu_die(void); +void __noreturn cpu_die(void); void cpu_restart(void); #endif /* CONFIG_HOTPLUG_CPU */ From dfb0f170cadb03da4f408ae53cc2908120e1f90e Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:55 -0800 Subject: [PATCH 21/52] sched/idle: Make sure weak version of arch_cpu_idle_dead() doesn't return arch_cpu_idle_dead() should never return. Make it so. Link: https://lore.kernel.org/r/cf5ad95eef50f7704bb30e7770c59bfe23372af7.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- kernel/sched/idle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index e9ef66be2870..56e152f06d0f 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -75,7 +75,7 @@ static noinline int __cpuidle cpu_idle_poll(void) void __weak arch_cpu_idle_prepare(void) { } void __weak arch_cpu_idle_enter(void) { } void __weak arch_cpu_idle_exit(void) { } -void __weak arch_cpu_idle_dead(void) { } +void __weak arch_cpu_idle_dead(void) { while (1); } void __weak arch_cpu_idle(void) { cpu_idle_force_poll = 1; From 071c44e4278156f18a6a56958617223b6bffa6ab Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:58 -0800 Subject: [PATCH 22/52] sched/idle: Mark arch_cpu_idle_dead() __noreturn Before commit 076cbf5d2163 ("x86/xen: don't let xen_pv_play_dead() return"), in Xen, when a previously offlined CPU was brought back online, it unexpectedly resumed execution where it left off in the middle of the idle loop. There were some hacks to make that work, but the behavior was surprising as do_idle() doesn't expect an offlined CPU to return from the dead (in arch_cpu_idle_dead()). Now that Xen has been fixed, and the arch-specific implementations of arch_cpu_idle_dead() also don't return, give it a __noreturn attribute. This will cause the compiler to complain if an arch-specific implementation might return. It also improves code generation for both caller and callee. Also fixes the following warning: vmlinux.o: warning: objtool: do_idle+0x25f: unreachable instruction Reported-by: Paul E. McKenney Tested-by: Paul E. McKenney Link: https://lore.kernel.org/r/60d527353da8c99d4cf13b6473131d46719ed16d.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/alpha/kernel/process.c | 2 +- arch/arm/kernel/smp.c | 2 +- arch/arm64/kernel/process.c | 2 +- arch/csky/kernel/smp.c | 2 +- arch/ia64/kernel/process.c | 2 +- arch/loongarch/kernel/process.c | 2 +- arch/mips/kernel/process.c | 2 +- arch/parisc/kernel/process.c | 2 +- arch/powerpc/kernel/smp.c | 2 +- arch/riscv/kernel/cpu-hotplug.c | 2 +- arch/s390/kernel/idle.c | 2 +- arch/sh/kernel/idle.c | 2 +- arch/sparc/kernel/process_64.c | 2 +- arch/x86/kernel/process.c | 2 +- arch/xtensa/kernel/smp.c | 2 +- include/linux/cpu.h | 2 +- kernel/sched/idle.c | 2 +- tools/objtool/check.c | 1 + 18 files changed, 18 insertions(+), 17 deletions(-) diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index a82fefa31bdb..582d96548385 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -60,7 +60,7 @@ void arch_cpu_idle(void) wtint(0); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { wtint(INT_MAX); BUG(); diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 441ea5cff390..d6be4507d22d 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -320,7 +320,7 @@ void __cpu_die(unsigned int cpu) * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { unsigned int cpu = smp_processor_id(); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 71d59b5abede..089ced6d6bd6 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -69,7 +69,7 @@ void (*pm_power_off)(void); EXPORT_SYMBOL_GPL(pm_power_off); #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index 0ec20efaf5fd..9c7a20b73ac6 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -300,7 +300,7 @@ void __cpu_die(unsigned int cpu) pr_notice("CPU%u: shutdown\n", cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { idle_task_exit(); diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index 78f5794b2dde..9a5cd9fad3a9 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -225,7 +225,7 @@ static inline void __noreturn play_dead(void) } #endif /* CONFIG_HOTPLUG_CPU */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c index fa2443c7afb2..b71e17c1cc0c 100644 --- a/arch/loongarch/kernel/process.c +++ b/arch/loongarch/kernel/process.c @@ -62,7 +62,7 @@ unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE; EXPORT_SYMBOL(boot_option_idle_override); #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 093dbbd6b843..a3225912c862 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -40,7 +40,7 @@ #include #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index c064719b49b0..97c6f875bd0e 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -159,7 +159,7 @@ EXPORT_SYMBOL(running_on_qemu); /* * Called from the idle thread for the CPU which has been shutdown. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { #ifdef CONFIG_HOTPLUG_CPU idle_task_exit(); diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 6b90f10a6c81..f62e5e651bcd 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -1752,7 +1752,7 @@ void __cpu_die(unsigned int cpu) smp_ops->cpu_die(cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { /* * Disable on the down path. This will be re-enabled by diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index f7a832e3a1d1..59b80211c25f 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -71,7 +71,7 @@ void __cpu_die(unsigned int cpu) /* * Called from the idle thread for the CPU which has been shutdown. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { idle_task_exit(); diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 38e267c7bff7..e7239aaf428b 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -88,7 +88,7 @@ void arch_cpu_idle_exit(void) { } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c index 114f0c4abeac..d662503b0665 100644 --- a/arch/sh/kernel/idle.c +++ b/arch/sh/kernel/idle.c @@ -30,7 +30,7 @@ void default_idle(void) clear_bl_bit(); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index 91c2b8124527..b51d8fb0ecdc 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -95,7 +95,7 @@ void arch_cpu_idle(void) } #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { sched_preempt_enable_no_resched(); cpu_play_dead(); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index f1ec36caf1d8..3e30147a537e 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -727,7 +727,7 @@ void arch_cpu_idle_enter(void) local_touch_nmi(); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/xtensa/kernel/smp.c b/arch/xtensa/kernel/smp.c index 7bad78495536..054bd64eab19 100644 --- a/arch/xtensa/kernel/smp.c +++ b/arch/xtensa/kernel/smp.c @@ -322,7 +322,7 @@ void __cpu_die(unsigned int cpu) pr_err("CPU%u: unable to kill\n", cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } diff --git a/include/linux/cpu.h b/include/linux/cpu.h index f83e4519c5f0..8582a7142623 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -182,7 +182,7 @@ void arch_cpu_idle(void); void arch_cpu_idle_prepare(void); void arch_cpu_idle_enter(void); void arch_cpu_idle_exit(void); -void arch_cpu_idle_dead(void); +void __noreturn arch_cpu_idle_dead(void); int cpu_report_state(int cpu); int cpu_check_up_prepare(int cpu); diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 56e152f06d0f..342f58a329f5 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -75,7 +75,7 @@ static noinline int __cpuidle cpu_idle_poll(void) void __weak arch_cpu_idle_prepare(void) { } void __weak arch_cpu_idle_enter(void) { } void __weak arch_cpu_idle_exit(void) { } -void __weak arch_cpu_idle_dead(void) { while (1); } +void __weak __noreturn arch_cpu_idle_dead(void) { while (1); } void __weak arch_cpu_idle(void) { cpu_idle_force_poll = 1; diff --git a/tools/objtool/check.c b/tools/objtool/check.c index f937be1afe65..37c36dc1d868 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -202,6 +202,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "__reiserfs_panic", "__stack_chk_fail", "__ubsan_handle_builtin_unreachable", + "arch_cpu_idle_dead", "cpu_bringup_and_idle", "cpu_startup_entry", "do_exit", From b4c108d7daf1039cf19388baff30a94563de3f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 14 Feb 2023 09:38:57 +0100 Subject: [PATCH 23/52] x86/cpu: Expose arch_cpu_idle_dead()'s prototype definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include to make sure arch_cpu_idle_dead() matches its prototype going forward. Inspired-by: Josh Poimboeuf Signed-off-by: Philippe Mathieu-Daudé Link: https://lore.kernel.org/r/20230214083857.50163-1-philmd@linaro.org Signed-off-by: Josh Poimboeuf --- arch/x86/kernel/process.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 3e30147a537e..d9ecaa618430 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include From f7515d9fe8fc4b80754cd4d98a5fcaee84adeebb Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:07 -0800 Subject: [PATCH 24/52] objtool: Add objtool_types.h Reduce the amount of header sync churn by splitting the shared objtool.h types into a new file. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/dec622720851210ceafa12d4f4c5f9e73c832152.1677683419.git.jpoimboe@kernel.org --- MAINTAINERS | 2 +- include/linux/objtool.h | 42 +----- include/linux/objtool_types.h | 48 +++++++ tools/include/linux/objtool.h | 200 ---------------------------- tools/include/linux/objtool_types.h | 48 +++++++ tools/objtool/check.c | 2 +- tools/objtool/orc_dump.c | 2 +- tools/objtool/orc_gen.c | 2 +- tools/objtool/sync-check.sh | 2 +- 9 files changed, 102 insertions(+), 246 deletions(-) create mode 100644 include/linux/objtool_types.h delete mode 100644 tools/include/linux/objtool.h create mode 100644 tools/include/linux/objtool_types.h diff --git a/MAINTAINERS b/MAINTAINERS index 8d5bc223f305..36e9e642c05e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15090,8 +15090,8 @@ OBJTOOL M: Josh Poimboeuf M: Peter Zijlstra S: Supported +F: include/linux/objtool*.h F: tools/objtool/ -F: include/linux/objtool.h OCELOT ETHERNET SWITCH DRIVER M: Vladimir Oltean diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 9ac3df3fccf0..8375792acfc0 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -2,47 +2,7 @@ #ifndef _LINUX_OBJTOOL_H #define _LINUX_OBJTOOL_H -#ifndef __ASSEMBLY__ - -#include - -/* - * This struct is used by asm and inline asm code to manually annotate the - * location of registers on the stack. - */ -struct unwind_hint { - u32 ip; - s16 sp_offset; - u8 sp_reg; - u8 type; - u8 signal; - u8 end; -}; -#endif - -/* - * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP - * (the caller's SP right before it made the call). Used for all callable - * functions, i.e. all C code and all callable asm functions. - * - * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset - * points to a fully populated pt_regs from a syscall, interrupt, or exception. - * - * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that - * sp_reg+sp_offset points to the iret return frame. - * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. - * Useful for code which doesn't have an ELF function annotation. - * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. - */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 +#include #ifdef CONFIG_OBJTOOL diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h new file mode 100644 index 000000000000..8513537a30ed --- /dev/null +++ b/include/linux/objtool_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_OBJTOOL_TYPES_H +#define _LINUX_OBJTOOL_TYPES_H + +#ifndef __ASSEMBLY__ + +#include + +/* + * This struct is used by asm and inline asm code to manually annotate the + * location of registers on the stack. + */ +struct unwind_hint { + u32 ip; + s16 sp_offset; + u8 sp_reg; + u8 type; + u8 signal; + u8 end; +}; + +#endif /* __ASSEMBLY__ */ + +/* + * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP + * (the caller's SP right before it made the call). Used for all callable + * functions, i.e. all C code and all callable asm functions. + * + * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset + * points to a fully populated pt_regs from a syscall, interrupt, or exception. + * + * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that + * sp_reg+sp_offset points to the iret return frame. + * + * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. + * Useful for code which doesn't have an ELF function annotation. + * + * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. + */ +#define UNWIND_HINT_TYPE_CALL 0 +#define UNWIND_HINT_TYPE_REGS 1 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_FUNC 3 +#define UNWIND_HINT_TYPE_ENTRY 4 +#define UNWIND_HINT_TYPE_SAVE 5 +#define UNWIND_HINT_TYPE_RESTORE 6 + +#endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h deleted file mode 100644 index 9ac3df3fccf0..000000000000 --- a/tools/include/linux/objtool.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_OBJTOOL_H -#define _LINUX_OBJTOOL_H - -#ifndef __ASSEMBLY__ - -#include - -/* - * This struct is used by asm and inline asm code to manually annotate the - * location of registers on the stack. - */ -struct unwind_hint { - u32 ip; - s16 sp_offset; - u8 sp_reg; - u8 type; - u8 signal; - u8 end; -}; -#endif - -/* - * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP - * (the caller's SP right before it made the call). Used for all callable - * functions, i.e. all C code and all callable asm functions. - * - * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset - * points to a fully populated pt_regs from a syscall, interrupt, or exception. - * - * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that - * sp_reg+sp_offset points to the iret return frame. - * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. - * Useful for code which doesn't have an ELF function annotation. - * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. - */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 - -#ifdef CONFIG_OBJTOOL - -#include - -#ifndef __ASSEMBLY__ - -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "987: \n\t" \ - ".pushsection .discard.unwind_hints\n\t" \ - /* struct unwind_hint */ \ - ".long 987b - .\n\t" \ - ".short " __stringify(sp_offset) "\n\t" \ - ".byte " __stringify(sp_reg) "\n\t" \ - ".byte " __stringify(type) "\n\t" \ - ".byte " __stringify(signal) "\n\t" \ - ".byte " __stringify(end) "\n\t" \ - ".balign 4 \n\t" \ - ".popsection\n\t" - -/* - * This macro marks the given function's stack frame as "non-standard", which - * tells objtool to ignore the function when doing stack metadata validation. - * It should only be used in special cases where you're 100% sure it won't - * affect the reliability of frame pointers and kernel stack traces. - * - * For more information, see tools/objtool/Documentation/objtool.txt. - */ -#define STACK_FRAME_NON_STANDARD(func) \ - static void __used __section(".discard.func_stack_frame_non_standard") \ - *__func_stack_frame_non_standard_##func = func - -/* - * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore - * for the case where a function is intentionally missing frame pointer setup, - * but otherwise needs objtool/ORC coverage when frame pointers are disabled. - */ -#ifdef CONFIG_FRAME_POINTER -#define STACK_FRAME_NON_STANDARD_FP(func) STACK_FRAME_NON_STANDARD(func) -#else -#define STACK_FRAME_NON_STANDARD_FP(func) -#endif - -#define ANNOTATE_NOENDBR \ - "986: \n\t" \ - ".pushsection .discard.noendbr\n\t" \ - _ASM_PTR " 986b\n\t" \ - ".popsection\n\t" - -#define ASM_REACHABLE \ - "998:\n\t" \ - ".pushsection .discard.reachable\n\t" \ - ".long 998b - .\n\t" \ - ".popsection\n\t" - -#else /* __ASSEMBLY__ */ - -/* - * This macro indicates that the following intra-function call is valid. - * Any non-annotated intra-function call will cause objtool to issue a warning. - */ -#define ANNOTATE_INTRA_FUNCTION_CALL \ - 999: \ - .pushsection .discard.intra_function_calls; \ - .long 999b; \ - .popsection; - -/* - * In asm, there are two kinds of code: normal C-type callable functions and - * the rest. The normal callable functions can be called by other code, and - * don't do anything unusual with the stack. Such normal callable functions - * are annotated with the ENTRY/ENDPROC macros. Most asm code falls in this - * category. In this case, no special debugging annotations are needed because - * objtool can automatically generate the ORC data for the ORC unwinder to read - * at runtime. - * - * Anything which doesn't fall into the above category, such as syscall and - * interrupt handlers, tends to not be called directly by other functions, and - * often does unusual non-C-function-type things with the stack pointer. Such - * code needs to be annotated such that objtool can understand it. The - * following CFI hint macros are for this type of code. - * - * These macros provide hints to objtool about the state of the stack at each - * instruction. Objtool starts from the hints and follows the code flow, - * making automatic CFI adjustments when it sees pushes and pops, filling out - * the debuginfo as necessary. It will also warn if it sees any - * inconsistencies. - */ -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.Lunwind_hint_ip_\@: - .pushsection .discard.unwind_hints - /* struct unwind_hint */ - .long .Lunwind_hint_ip_\@ - . - .short \sp_offset - .byte \sp_reg - .byte \type - .byte \signal - .byte \end - .balign 4 - .popsection -.endm - -.macro STACK_FRAME_NON_STANDARD func:req - .pushsection .discard.func_stack_frame_non_standard, "aw" - _ASM_PTR \func - .popsection -.endm - -.macro STACK_FRAME_NON_STANDARD_FP func:req -#ifdef CONFIG_FRAME_POINTER - STACK_FRAME_NON_STANDARD \func -#endif -.endm - -.macro ANNOTATE_NOENDBR -.Lhere_\@: - .pushsection .discard.noendbr - .quad .Lhere_\@ - .popsection -.endm - -.macro REACHABLE -.Lhere_\@: - .pushsection .discard.reachable - .long .Lhere_\@ - . - .popsection -.endm - -#endif /* __ASSEMBLY__ */ - -#else /* !CONFIG_OBJTOOL */ - -#ifndef __ASSEMBLY__ - -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "\n\t" -#define STACK_FRAME_NON_STANDARD(func) -#define STACK_FRAME_NON_STANDARD_FP(func) -#define ANNOTATE_NOENDBR -#define ASM_REACHABLE -#else -#define ANNOTATE_INTRA_FUNCTION_CALL -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.endm -.macro STACK_FRAME_NON_STANDARD func:req -.endm -.macro ANNOTATE_NOENDBR -.endm -.macro REACHABLE -.endm -#endif - -#endif /* CONFIG_OBJTOOL */ - -#endif /* _LINUX_OBJTOOL_H */ diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h new file mode 100644 index 000000000000..8513537a30ed --- /dev/null +++ b/tools/include/linux/objtool_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_OBJTOOL_TYPES_H +#define _LINUX_OBJTOOL_TYPES_H + +#ifndef __ASSEMBLY__ + +#include + +/* + * This struct is used by asm and inline asm code to manually annotate the + * location of registers on the stack. + */ +struct unwind_hint { + u32 ip; + s16 sp_offset; + u8 sp_reg; + u8 type; + u8 signal; + u8 end; +}; + +#endif /* __ASSEMBLY__ */ + +/* + * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP + * (the caller's SP right before it made the call). Used for all callable + * functions, i.e. all C code and all callable asm functions. + * + * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset + * points to a fully populated pt_regs from a syscall, interrupt, or exception. + * + * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that + * sp_reg+sp_offset points to the iret return frame. + * + * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. + * Useful for code which doesn't have an ELF function annotation. + * + * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. + */ +#define UNWIND_HINT_TYPE_CALL 0 +#define UNWIND_HINT_TYPE_REGS 1 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_FUNC 3 +#define UNWIND_HINT_TYPE_ENTRY 4 +#define UNWIND_HINT_TYPE_SAVE 5 +#define UNWIND_HINT_TYPE_RESTORE 6 + +#endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 37c36dc1d868..efc2baa02883 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include #include diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index 2d8ebdcd1db3..9f6c528c26f2 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -4,7 +4,7 @@ */ #include -#include +#include #include #include #include diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index 57a4527d5988..f49630a21e0f 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include diff --git a/tools/objtool/sync-check.sh b/tools/objtool/sync-check.sh index 105a291ff8e7..81d120d05442 100755 --- a/tools/objtool/sync-check.sh +++ b/tools/objtool/sync-check.sh @@ -6,7 +6,7 @@ if [ -z "$SRCARCH" ]; then exit 1 fi -FILES="include/linux/objtool.h" +FILES="include/linux/objtool_types.h" if [ "$SRCARCH" = "x86" ]; then FILES="$FILES From 1c0c1faf5692c18c127d044ecc0cc92c7bab3477 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:08 -0800 Subject: [PATCH 25/52] objtool: Use relative pointers for annotations They produce the needed relocations while using half the space. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/bed05c64e28200220c9b1754a2f3ce71f73076ea.1677683419.git.jpoimboe@kernel.org --- arch/x86/include/asm/nospec-branch.h | 6 +++--- include/linux/objtool.h | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 3ef70e54a858..78ed1546b775 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -194,9 +194,9 @@ * builds. */ .macro ANNOTATE_RETPOLINE_SAFE - .Lannotate_\@: +.Lhere_\@: .pushsection .discard.retpoline_safe - _ASM_PTR .Lannotate_\@ + .long .Lhere_\@ - . .popsection .endm @@ -318,7 +318,7 @@ #define ANNOTATE_RETPOLINE_SAFE \ "999:\n\t" \ ".pushsection .discard.retpoline_safe\n\t" \ - _ASM_PTR " 999b\n\t" \ + ".long 999b - .\n\t" \ ".popsection\n\t" typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 8375792acfc0..2b0258d273fc 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -49,7 +49,7 @@ #define ANNOTATE_NOENDBR \ "986: \n\t" \ ".pushsection .discard.noendbr\n\t" \ - _ASM_PTR " 986b\n\t" \ + ".long 986b - .\n\t" \ ".popsection\n\t" #define ASM_REACHABLE \ @@ -67,7 +67,7 @@ #define ANNOTATE_INTRA_FUNCTION_CALL \ 999: \ .pushsection .discard.intra_function_calls; \ - .long 999b; \ + .long 999b - .; \ .popsection; /* @@ -92,10 +92,10 @@ * inconsistencies. */ .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.Lunwind_hint_ip_\@: +.Lhere_\@: .pushsection .discard.unwind_hints /* struct unwind_hint */ - .long .Lunwind_hint_ip_\@ - . + .long .Lhere_\@ - . .short \sp_offset .byte \sp_reg .byte \type @@ -107,7 +107,7 @@ .macro STACK_FRAME_NON_STANDARD func:req .pushsection .discard.func_stack_frame_non_standard, "aw" - _ASM_PTR \func + .long \func - . .popsection .endm @@ -120,7 +120,7 @@ .macro ANNOTATE_NOENDBR .Lhere_\@: .pushsection .discard.noendbr - .quad .Lhere_\@ + .long .Lhere_\@ - . .popsection .endm From d88ebba45dfe67114a6ac8c6514f2c65b6ed64c7 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:09 -0800 Subject: [PATCH 26/52] objtool: Change UNWIND_HINT() argument order The most important argument is 'type', make that one first. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/d994f8c29376c5618c75698df28fc03b52d3a868.1677683419.git.jpoimboe@kernel.org --- arch/x86/include/asm/unwind_hints.h | 2 +- include/linux/objtool.h | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index e7c71750b309..97b392217c0f 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -67,7 +67,7 @@ #else #define UNWIND_HINT_FUNC \ - UNWIND_HINT(ORC_REG_SP, 8, UNWIND_HINT_TYPE_FUNC, 0, 0) + UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0, 0) #endif /* __ASSEMBLY__ */ diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 2b0258d273fc..725d7f0b6748 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -10,7 +10,7 @@ #ifndef __ASSEMBLY__ -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ +#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) \ "987: \n\t" \ ".pushsection .discard.unwind_hints\n\t" \ /* struct unwind_hint */ \ @@ -137,8 +137,7 @@ #ifndef __ASSEMBLY__ -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "\n\t" +#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) "\n\t" #define STACK_FRAME_NON_STANDARD(func) #define STACK_FRAME_NON_STANDARD_FP(func) #define ANNOTATE_NOENDBR From f902cfdd46aedd2afb3e8033223312dbf5fbb675 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:10 -0800 Subject: [PATCH 27/52] x86,objtool: Introduce ORC_TYPE_* Unwind hints and ORC entry types are two distinct things. Separate them out more explicitly. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/cc879d38fff8a43f8f7beb2fd56e35a5a384d7cd.1677683419.git.jpoimboe@kernel.org --- arch/x86/include/asm/orc_types.h | 4 ++++ arch/x86/kernel/unwind_orc.c | 12 ++++++------ include/linux/objtool_types.h | 1 + tools/arch/x86/include/asm/orc_types.h | 4 ++++ tools/include/linux/objtool_types.h | 1 + tools/objtool/orc_dump.c | 7 +++---- tools/objtool/orc_gen.c | 19 +++++++++++++++++-- 7 files changed, 36 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index 1343a62106de..b4d4ec78589e 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -39,6 +39,10 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 +#define ORC_TYPE_CALL 0 +#define ORC_TYPE_REGS 1 +#define ORC_TYPE_REGS_PARTIAL 2 + #ifndef __ASSEMBLY__ #include diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index 37307b40f8da..8348ac581de3 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -133,7 +133,7 @@ static struct orc_entry null_orc_entry = { .sp_offset = sizeof(long), .sp_reg = ORC_REG_SP, .bp_reg = ORC_REG_UNDEFINED, - .type = UNWIND_HINT_TYPE_CALL + .type = ORC_TYPE_CALL }; #ifdef CONFIG_CALL_THUNKS @@ -153,7 +153,7 @@ static struct orc_entry *orc_callthunk_find(unsigned long ip) /* Fake frame pointer entry -- used as a fallback for generated code */ static struct orc_entry orc_fp_entry = { - .type = UNWIND_HINT_TYPE_CALL, + .type = ORC_TYPE_CALL, .sp_reg = ORC_REG_BP, .sp_offset = 16, .bp_reg = ORC_REG_PREV_SP, @@ -554,7 +554,7 @@ bool unwind_next_frame(struct unwind_state *state) /* Find IP, SP and possibly regs: */ switch (orc->type) { - case UNWIND_HINT_TYPE_CALL: + case ORC_TYPE_CALL: ip_p = sp - sizeof(long); if (!deref_stack_reg(state, ip_p, &state->ip)) @@ -567,7 +567,7 @@ bool unwind_next_frame(struct unwind_state *state) state->prev_regs = NULL; break; - case UNWIND_HINT_TYPE_REGS: + case ORC_TYPE_REGS: if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access registers at %pB\n", (void *)orig_ip); @@ -590,13 +590,13 @@ bool unwind_next_frame(struct unwind_state *state) state->full_regs = true; break; - case UNWIND_HINT_TYPE_REGS_PARTIAL: + case ORC_TYPE_REGS_PARTIAL: if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access iret registers at %pB\n", (void *)orig_ip); goto err; } - /* See UNWIND_HINT_TYPE_REGS case comment. */ + /* See ORC_TYPE_REGS case comment. */ state->ip = unwind_recover_rethook(state, state->ip, (unsigned long *)(state->sp - sizeof(long))); diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h index 8513537a30ed..9a83468c0039 100644 --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -40,6 +40,7 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_CALL 0 #define UNWIND_HINT_TYPE_REGS 1 #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +/* The below hint types don't have corresponding ORC types */ #define UNWIND_HINT_TYPE_FUNC 3 #define UNWIND_HINT_TYPE_ENTRY 4 #define UNWIND_HINT_TYPE_SAVE 5 diff --git a/tools/arch/x86/include/asm/orc_types.h b/tools/arch/x86/include/asm/orc_types.h index 1343a62106de..b4d4ec78589e 100644 --- a/tools/arch/x86/include/asm/orc_types.h +++ b/tools/arch/x86/include/asm/orc_types.h @@ -39,6 +39,10 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 +#define ORC_TYPE_CALL 0 +#define ORC_TYPE_REGS 1 +#define ORC_TYPE_REGS_PARTIAL 2 + #ifndef __ASSEMBLY__ #include diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h index 8513537a30ed..9a83468c0039 100644 --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -40,6 +40,7 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_CALL 0 #define UNWIND_HINT_TYPE_REGS 1 #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +/* The below hint types don't have corresponding ORC types */ #define UNWIND_HINT_TYPE_FUNC 3 #define UNWIND_HINT_TYPE_ENTRY 4 #define UNWIND_HINT_TYPE_SAVE 5 diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index 9f6c528c26f2..97ecbb8b9034 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -4,7 +4,6 @@ */ #include -#include #include #include #include @@ -39,11 +38,11 @@ static const char *reg_name(unsigned int reg) static const char *orc_type_name(unsigned int type) { switch (type) { - case UNWIND_HINT_TYPE_CALL: + case ORC_TYPE_CALL: return "call"; - case UNWIND_HINT_TYPE_REGS: + case ORC_TYPE_REGS: return "regs"; - case UNWIND_HINT_TYPE_REGS_PARTIAL: + case ORC_TYPE_REGS_PARTIAL: return "regs (partial)"; default: return "?"; diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index f49630a21e0f..e85bbb996f6c 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -26,6 +26,22 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, return 0; } + switch (cfi->type) { + case UNWIND_HINT_TYPE_CALL: + orc->type = ORC_TYPE_CALL; + break; + case UNWIND_HINT_TYPE_REGS: + orc->type = ORC_TYPE_REGS; + break; + case UNWIND_HINT_TYPE_REGS_PARTIAL: + orc->type = ORC_TYPE_REGS_PARTIAL; + break; + default: + WARN_FUNC("unknown unwind hint type %d", + insn->sec, insn->offset, cfi->type); + return -1; + } + orc->end = cfi->end; orc->signal = cfi->signal; @@ -83,7 +99,6 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, orc->sp_offset = cfi->cfa.offset; orc->bp_offset = bp->offset; - orc->type = cfi->type; return 0; } @@ -151,7 +166,7 @@ int orc_create(struct objtool_file *file) struct orc_entry null = { .sp_reg = ORC_REG_UNDEFINED, .bp_reg = ORC_REG_UNDEFINED, - .type = UNWIND_HINT_TYPE_CALL, + .type = ORC_TYPE_CALL, }; /* Build a deduplicated list of ORC entries: */ From 4708ea14bef314fc901857eefd65678236a9f2d9 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:11 -0800 Subject: [PATCH 28/52] x86,objtool: Separate unret validation from unwind hints The ENTRY unwind hint type is serving double duty as both an empty unwind hint and an unret validation annotation. Unret validation is unrelated to unwinding. Separate it out into its own annotation. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/ff7448d492ea21b86d8a90264b105fbd0d751077.1677683419.git.jpoimboe@kernel.org --- arch/x86/entry/entry_64.S | 14 +++--- arch/x86/include/asm/nospec-branch.h | 8 ++-- arch/x86/include/asm/unwind_hints.h | 8 +++- arch/x86/kernel/head_64.S | 5 --- include/linux/objtool.h | 16 +++++++ include/linux/objtool_types.h | 5 +-- tools/include/linux/objtool_types.h | 5 +-- tools/objtool/check.c | 65 ++++++++++++++++++--------- tools/objtool/include/objtool/check.h | 4 +- 9 files changed, 85 insertions(+), 45 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index eccc3431e515..5b93eb7db0ab 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -388,9 +388,9 @@ SYM_CODE_START(\asmsym) .if \vector == X86_TRAP_BP /* #BP advances %rip to the next instruction */ - UNWIND_HINT_IRET_REGS offset=\has_error_code*8 signal=0 + UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 signal=0 .else - UNWIND_HINT_IRET_REGS offset=\has_error_code*8 + UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 .endif ENDBR @@ -461,7 +461,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_mce_db vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR ASM_CLAC cld @@ -518,7 +518,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_vc vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR ASM_CLAC cld @@ -582,7 +582,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_df vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS offset=8 + UNWIND_HINT_IRET_ENTRY offset=8 ENDBR ASM_CLAC cld @@ -1107,7 +1107,7 @@ SYM_CODE_START(error_entry) FENCE_SWAPGS_KERNEL_ENTRY CALL_DEPTH_ACCOUNT leaq 8(%rsp), %rax /* return pt_regs pointer */ - ANNOTATE_UNRET_END + VALIDATE_UNRET_END RET .Lbstep_iret: @@ -1153,7 +1153,7 @@ SYM_CODE_END(error_return) * when PAGE_TABLE_ISOLATION is in use. Do not clobber. */ SYM_CODE_START(asm_exc_nmi) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR /* diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 78ed1546b775..edb2b0cb8efe 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -210,8 +210,8 @@ * Abuse ANNOTATE_RETPOLINE_SAFE on a NOP to indicate UNRET_END, should * eventually turn into it's own annotation. */ -.macro ANNOTATE_UNRET_END -#ifdef CONFIG_DEBUG_ENTRY +.macro VALIDATE_UNRET_END +#if defined(CONFIG_NOINSTR_VALIDATION) && defined(CONFIG_CPU_UNRET_ENTRY) ANNOTATE_RETPOLINE_SAFE nop #endif @@ -286,7 +286,7 @@ .macro UNTRAIN_RET #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ defined(CONFIG_CALL_DEPTH_TRACKING) - ANNOTATE_UNRET_END + VALIDATE_UNRET_END ALTERNATIVE_3 "", \ CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ @@ -297,7 +297,7 @@ .macro UNTRAIN_RET_FROM_CALL #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ defined(CONFIG_CALL_DEPTH_TRACKING) - ANNOTATE_UNRET_END + VALIDATE_UNRET_END ALTERNATIVE_3 "", \ CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index 97b392217c0f..4c0f28d665eb 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -12,7 +12,8 @@ .endm .macro UNWIND_HINT_ENTRY - UNWIND_HINT type=UNWIND_HINT_TYPE_ENTRY end=1 + VALIDATE_UNRET_BEGIN + UNWIND_HINT_EMPTY .endm .macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 signal=1 @@ -52,6 +53,11 @@ UNWIND_HINT_REGS base=\base offset=\offset partial=1 signal=\signal .endm +.macro UNWIND_HINT_IRET_ENTRY base=%rsp offset=0 signal=1 + VALIDATE_UNRET_BEGIN + UNWIND_HINT_IRET_REGS base=\base offset=\offset signal=\signal +.endm + .macro UNWIND_HINT_FUNC UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=8 type=UNWIND_HINT_TYPE_FUNC .endm diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 222efd4a09bc..ee3ed15ee1f8 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -390,8 +390,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) UNWIND_HINT_IRET_REGS offset=8 ENDBR - ANNOTATE_UNRET_END - /* Build pt_regs */ PUSH_AND_CLEAR_REGS @@ -451,7 +449,6 @@ SYM_CODE_END(early_idt_handler_array) SYM_CODE_START_LOCAL(early_idt_handler_common) UNWIND_HINT_IRET_REGS offset=16 - ANNOTATE_UNRET_END /* * The stack is the hardware frame, an error code or zero, and the * vector number. @@ -501,8 +498,6 @@ SYM_CODE_START_NOALIGN(vc_no_ghcb) UNWIND_HINT_IRET_REGS offset=8 ENDBR - ANNOTATE_UNRET_END - /* Build pt_regs */ PUSH_AND_CLEAR_REGS diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 725d7f0b6748..5aa475118820 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -124,6 +124,22 @@ .popsection .endm +/* + * Use objtool to validate the entry requirement that all code paths do + * VALIDATE_UNRET_END before RET. + * + * NOTE: The macro must be used at the beginning of a global symbol, otherwise + * it will be ignored. + */ +.macro VALIDATE_UNRET_BEGIN +#if defined(CONFIG_NOINSTR_VALIDATION) && defined(CONFIG_CPU_UNRET_ENTRY) +.Lhere_\@: + .pushsection .discard.validate_unret + .long .Lhere_\@ - . + .popsection +#endif +.endm + .macro REACHABLE .Lhere_\@: .pushsection .discard.reachable diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h index 9a83468c0039..9787ad0f2ef4 100644 --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -42,8 +42,7 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 /* The below hint types don't have corresponding ORC types */ #define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 +#define UNWIND_HINT_TYPE_SAVE 4 +#define UNWIND_HINT_TYPE_RESTORE 5 #endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h index 9a83468c0039..9787ad0f2ef4 100644 --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -42,8 +42,7 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 /* The below hint types don't have corresponding ORC types */ #define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 +#define UNWIND_HINT_TYPE_SAVE 4 +#define UNWIND_HINT_TYPE_RESTORE 5 #endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index efc2baa02883..10be80b3fe67 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2307,16 +2307,9 @@ static int read_unwind_hints(struct objtool_file *file) WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", insn->sec, insn->offset); } - - insn->entry = 1; } } - if (hint->type == UNWIND_HINT_TYPE_ENTRY) { - hint->type = UNWIND_HINT_TYPE_CALL; - insn->entry = 1; - } - if (hint->type == UNWIND_HINT_TYPE_FUNC) { insn->cfi = &func_cfi; continue; @@ -2449,6 +2442,34 @@ static int read_instr_hints(struct objtool_file *file) return 0; } +static int read_validate_unret_hints(struct objtool_file *file) +{ + struct section *sec; + struct instruction *insn; + struct reloc *reloc; + + sec = find_section_by_name(file->elf, ".rela.discard.validate_unret"); + if (!sec) + return 0; + + list_for_each_entry(reloc, &sec->reloc_list, list) { + if (reloc->sym->type != STT_SECTION) { + WARN("unexpected relocation symbol type in %s", sec->name); + return -1; + } + + insn = find_insn(file, reloc->sym->sec, reloc->addend); + if (!insn) { + WARN("bad .discard.instr_end entry"); + return -1; + } + insn->unret = 1; + } + + return 0; +} + + static int read_intra_function_calls(struct objtool_file *file) { struct instruction *insn; @@ -2667,6 +2688,10 @@ static int decode_sections(struct objtool_file *file) if (ret) return ret; + ret = read_validate_unret_hints(file); + if (ret) + return ret; + return 0; } @@ -3863,10 +3888,10 @@ static int validate_unwind_hints(struct objtool_file *file, struct section *sec) /* * Validate rethunk entry constraint: must untrain RET before the first RET. * - * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes + * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes * before an actual RET instruction. */ -static int validate_entry(struct objtool_file *file, struct instruction *insn) +static int validate_unret(struct objtool_file *file, struct instruction *insn) { struct instruction *next, *dest; int ret, warnings = 0; @@ -3874,10 +3899,10 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) for (;;) { next = next_insn_to_validate(file, insn); - if (insn->visited & VISITED_ENTRY) + if (insn->visited & VISITED_UNRET) return 0; - insn->visited |= VISITED_ENTRY; + insn->visited |= VISITED_UNRET; if (!insn->ignore_alts && insn->alts) { struct alternative *alt; @@ -3887,7 +3912,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) if (alt->skip_orig) skip_orig = true; - ret = validate_entry(file, alt->insn); + ret = validate_unret(file, alt->insn); if (ret) { if (opts.backtrace) BT_FUNC("(alt)", insn); @@ -3915,7 +3940,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) insn->sec, insn->offset); return -1; } - ret = validate_entry(file, insn->jump_dest); + ret = validate_unret(file, insn->jump_dest); if (ret) { if (opts.backtrace) { BT_FUNC("(branch%s)", insn, @@ -3940,7 +3965,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) return -1; } - ret = validate_entry(file, dest); + ret = validate_unret(file, dest); if (ret) { if (opts.backtrace) BT_FUNC("(call)", insn); @@ -3976,19 +4001,19 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) } /* - * Validate that all branches starting at 'insn->entry' encounter UNRET_END - * before RET. + * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter + * VALIDATE_UNRET_END before RET. */ -static int validate_unret(struct objtool_file *file) +static int validate_unrets(struct objtool_file *file) { struct instruction *insn; int ret, warnings = 0; for_each_insn(file, insn) { - if (!insn->entry) + if (!insn->unret) continue; - ret = validate_entry(file, insn); + ret = validate_unret(file, insn); if (ret < 0) { WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset); return ret; @@ -4607,7 +4632,7 @@ int check(struct objtool_file *file) * Must be after validate_branch() and friends, it plays * further games with insn->visited. */ - ret = validate_unret(file); + ret = validate_unrets(file); if (ret < 0) return ret; warnings += ret; diff --git a/tools/objtool/include/objtool/check.h b/tools/objtool/include/objtool/check.h index 3e7c7004f7df..daa46f1f0965 100644 --- a/tools/objtool/include/objtool/check.h +++ b/tools/objtool/include/objtool/check.h @@ -61,7 +61,7 @@ struct instruction { restore : 1, retpoline_safe : 1, noendbr : 1, - entry : 1, + unret : 1, visited : 4, no_reloc : 1; /* 10 bit hole */ @@ -92,7 +92,7 @@ static inline struct symbol *insn_func(struct instruction *insn) #define VISITED_BRANCH 0x01 #define VISITED_BRANCH_UACCESS 0x02 #define VISITED_BRANCH_MASK 0x03 -#define VISITED_ENTRY 0x04 +#define VISITED_UNRET 0x04 static inline bool is_static_jump(struct instruction *insn) { From fb799447ae2974a07907906dff5bd4b9e47b7123 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:12 -0800 Subject: [PATCH 29/52] x86,objtool: Split UNWIND_HINT_EMPTY in two Mark reported that the ORC unwinder incorrectly marks an unwind as reliable when the unwind terminates prematurely in the dark corners of return_to_handler() due to lack of information about the next frame. The problem is UNWIND_HINT_EMPTY is used in two different situations: 1) The end of the kernel stack unwind before hitting user entry, boot code, or fork entry 2) A blind spot in ORC coverage where the unwinder has to bail due to lack of information about the next frame The ORC unwinder has no way to tell the difference between the two. When it encounters an undefined stack state with 'end=1', it blindly marks the stack reliable, which can break the livepatch consistency model. Fix it by splitting UNWIND_HINT_EMPTY into UNWIND_HINT_UNDEFINED and UNWIND_HINT_END_OF_STACK. Reported-by: Mark Rutland Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Steven Rostedt (Google) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/fd6212c8b450d3564b855e1cb48404d6277b4d9f.1677683419.git.jpoimboe@kernel.org --- .../livepatch/reliable-stacktrace.rst | 2 +- arch/x86/entry/entry_64.S | 12 ++++----- arch/x86/include/asm/orc_types.h | 14 +++++----- arch/x86/include/asm/unwind_hints.h | 12 ++++++--- arch/x86/kernel/ftrace_64.S | 2 +- arch/x86/kernel/head_64.S | 12 ++++----- arch/x86/kernel/relocate_kernel_64.S | 10 +++---- arch/x86/kernel/unwind_orc.c | 15 +++++------ arch/x86/lib/retpoline.S | 6 ++--- arch/x86/platform/pvh/head.S | 2 +- arch/x86/xen/xen-asm.S | 4 +-- arch/x86/xen/xen-head.S | 4 +-- include/linux/objtool.h | 10 +++---- include/linux/objtool_types.h | 27 ++++++++++++------- scripts/sorttable.h | 2 +- tools/arch/x86/include/asm/orc_types.h | 14 +++++----- tools/include/linux/objtool_types.h | 27 ++++++++++++------- tools/objtool/check.c | 2 +- tools/objtool/orc_dump.c | 8 ++++-- tools/objtool/orc_gen.c | 26 +++++++++--------- 20 files changed, 116 insertions(+), 95 deletions(-) diff --git a/Documentation/livepatch/reliable-stacktrace.rst b/Documentation/livepatch/reliable-stacktrace.rst index 67459d2ca2af..d56bb706172f 100644 --- a/Documentation/livepatch/reliable-stacktrace.rst +++ b/Documentation/livepatch/reliable-stacktrace.rst @@ -183,7 +183,7 @@ trampoline or return trampoline. For example, considering the x86_64 .. code-block:: none SYM_CODE_START(return_to_handler) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED subq $24, %rsp /* Save the return values */ diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 5b93eb7db0ab..45e135be2a9f 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -205,7 +205,7 @@ syscall_return_via_sysret: */ movq %rsp, %rdi movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK pushq RSP-RDI(%rdi) /* RSP */ pushq (%rdi) /* RDI */ @@ -286,7 +286,7 @@ SYM_FUNC_END(__switch_to_asm) .pushsection .text, "ax" __FUNC_ALIGN SYM_CODE_START_NOALIGN(ret_from_fork) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // copy_thread CALL_DEPTH_ACCOUNT movq %rax, %rdi @@ -303,7 +303,7 @@ SYM_CODE_START_NOALIGN(ret_from_fork) 1: /* kernel thread */ - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK movq %r12, %rdi CALL_NOSPEC rbx /* @@ -643,7 +643,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) */ movq %rsp, %rdi movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* Copy the IRET frame to the trampoline stack. */ pushq 6*8(%rdi) /* SS */ @@ -869,7 +869,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback) */ __FUNC_ALIGN SYM_CODE_START_NOALIGN(xen_failsafe_callback) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ENDBR movl %ds, %ecx cmpw %cx, 0x10(%rsp) @@ -1520,7 +1520,7 @@ SYM_CODE_END(asm_exc_nmi) * MSRs to fully disable 32-bit SYSCALL. */ SYM_CODE_START(ignore_sysret) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ENDBR mov $-ENOSYS, %eax sysretl diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index b4d4ec78589e..46d7e06763c9 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -39,9 +39,11 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 -#define ORC_TYPE_CALL 0 -#define ORC_TYPE_REGS 1 -#define ORC_TYPE_REGS_PARTIAL 2 +#define ORC_TYPE_UNDEFINED 0 +#define ORC_TYPE_END_OF_STACK 1 +#define ORC_TYPE_CALL 2 +#define ORC_TYPE_REGS 3 +#define ORC_TYPE_REGS_PARTIAL 4 #ifndef __ASSEMBLY__ #include @@ -60,16 +62,14 @@ struct orc_entry { #if defined(__LITTLE_ENDIAN_BITFIELD) unsigned sp_reg:4; unsigned bp_reg:4; - unsigned type:2; + unsigned type:3; unsigned signal:1; - unsigned end:1; #elif defined(__BIG_ENDIAN_BITFIELD) unsigned bp_reg:4; unsigned sp_reg:4; unsigned unused:4; - unsigned end:1; unsigned signal:1; - unsigned type:2; + unsigned type:3; #endif } __packed; diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index 4c0f28d665eb..01cb9692b160 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -7,13 +7,17 @@ #ifdef __ASSEMBLY__ -.macro UNWIND_HINT_EMPTY - UNWIND_HINT type=UNWIND_HINT_TYPE_CALL end=1 +.macro UNWIND_HINT_END_OF_STACK + UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK +.endm + +.macro UNWIND_HINT_UNDEFINED + UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED .endm .macro UNWIND_HINT_ENTRY VALIDATE_UNRET_BEGIN - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK .endm .macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 signal=1 @@ -73,7 +77,7 @@ #else #define UNWIND_HINT_FUNC \ - UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0, 0) + UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0) #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 1265ad519249..0387732e9c3f 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -340,7 +340,7 @@ STACK_FRAME_NON_STANDARD_FP(__fentry__) #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_CODE_START(return_to_handler) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR subq $16, %rsp diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index ee3ed15ee1f8..8d8d25c64827 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -42,7 +42,7 @@ L3_START_KERNEL = pud_index(__START_KERNEL_map) __HEAD .code64 SYM_CODE_START_NOALIGN(startup_64) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * and someone has loaded an identity mapped page table @@ -105,7 +105,7 @@ SYM_CODE_START_NOALIGN(startup_64) lretq .Lon_kernel_cs: - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* Sanitize CPU configuration */ call verify_cpu @@ -127,7 +127,7 @@ SYM_CODE_START_NOALIGN(startup_64) SYM_CODE_END(startup_64) SYM_CODE_START(secondary_startup_64) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, @@ -156,7 +156,7 @@ SYM_CODE_START(secondary_startup_64) * verify_cpu() above to make sure NX is enabled. */ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* @@ -238,7 +238,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) ANNOTATE_RETPOLINE_SAFE jmp *%rax 1: - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // above /* @@ -371,7 +371,7 @@ SYM_CODE_END(secondary_startup_64) */ SYM_CODE_START(start_cpu0) ANNOTATE_NOENDBR - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK movq initial_stack(%rip), %rsp jmp .Ljump_to_C_code SYM_CODE_END(start_cpu0) diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 4a73351f87f8..56cab1bb25f5 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -43,7 +43,7 @@ .code64 SYM_CODE_START_NOALIGN(relocate_range) SYM_CODE_START_NOALIGN(relocate_kernel) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* * %rdi indirection_page @@ -113,7 +113,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* set return address to 0 if not preserving context */ pushq $0 /* store the start address on the stack */ @@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // RET target, above movq RSP(%r8), %rsp movq CR4(%r8), %rax @@ -256,8 +256,8 @@ SYM_CODE_END(virtual_mapped) /* Do the copies */ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) - UNWIND_HINT_EMPTY - movq %rdi, %rcx /* Put the page_list in %rcx */ + UNWIND_HINT_END_OF_STACK + movq %rdi, %rcx /* Put the page_list in %rcx */ xorl %edi, %edi xorl %esi, %esi jmp 1f diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index 8348ac581de3..3ac50b7298d1 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -158,7 +158,6 @@ static struct orc_entry orc_fp_entry = { .sp_offset = 16, .bp_reg = ORC_REG_PREV_SP, .bp_offset = -16, - .end = 0, }; static struct orc_entry *orc_find(unsigned long ip) @@ -250,13 +249,13 @@ static int orc_sort_cmp(const void *_a, const void *_b) return -1; /* - * The "weak" section terminator entries need to always be on the left + * The "weak" section terminator entries need to always be first * to ensure the lookup code skips them in favor of real entries. * These terminator entries exist to handle any gaps created by * whitelisted .o files which didn't get objtool generation. */ orc_a = cur_orc_table + (a - cur_orc_ip_table); - return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; } void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, @@ -474,14 +473,12 @@ bool unwind_next_frame(struct unwind_state *state) */ orc = &orc_fp_entry; state->error = true; - } - - /* End-of-stack check for kernel threads: */ - if (orc->sp_reg == ORC_REG_UNDEFINED) { - if (!orc->end) + } else { + if (orc->type == ORC_TYPE_UNDEFINED) goto err; - goto the_end; + if (orc->type == ORC_TYPE_END_OF_STACK) + goto the_end; } state->signal = orc->signal; diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 5f61c65322be..27ef53fab6bd 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -33,7 +33,7 @@ .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ @@ -75,7 +75,7 @@ SYM_CODE_END(__x86_indirect_thunk_array) .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT @@ -103,7 +103,7 @@ SYM_CODE_END(__x86_indirect_call_thunk_array) .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR POLINE \reg ANNOTATE_UNRET_SAFE diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 7fe564eaf228..c4365a05ab83 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -50,7 +50,7 @@ #define PVH_DS_SEL (PVH_GDT_ENTRY_DS * 8) SYM_CODE_START_LOCAL(pvh_start_xen) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK cld lgdt (_pa(gdt)) diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S index 4a184f6e4e4d..08f1ceb9eb81 100644 --- a/arch/x86/xen/xen-asm.S +++ b/arch/x86/xen/xen-asm.S @@ -165,7 +165,7 @@ xen_pv_trap asm_exc_xen_hypervisor_callback SYM_CODE_START(xen_early_idt_handler_array) i = 0 .rept NUM_EXCEPTION_VECTORS - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ENDBR pop %rcx pop %r11 @@ -193,7 +193,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 * rsp->rax } */ SYM_CODE_START(xen_iret) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR pushq $0 jmp hypercall_iret diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index e36ea4268bd2..6eafdd17c242 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S @@ -45,7 +45,7 @@ SYM_CODE_END(hypercall_page) #ifdef CONFIG_XEN_PV __INIT SYM_CODE_START(startup_xen) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR cld @@ -71,7 +71,7 @@ SYM_CODE_END(startup_xen) #ifdef CONFIG_XEN_PV_SMP .pushsection .text SYM_CODE_START(asm_cpu_bringup_and_idle) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ENDBR call cpu_bringup_and_idle diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 5aa475118820..03f82c2c2ebf 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -10,7 +10,7 @@ #ifndef __ASSEMBLY__ -#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) \ +#define UNWIND_HINT(type, sp_reg, sp_offset, signal) \ "987: \n\t" \ ".pushsection .discard.unwind_hints\n\t" \ /* struct unwind_hint */ \ @@ -19,7 +19,6 @@ ".byte " __stringify(sp_reg) "\n\t" \ ".byte " __stringify(type) "\n\t" \ ".byte " __stringify(signal) "\n\t" \ - ".byte " __stringify(end) "\n\t" \ ".balign 4 \n\t" \ ".popsection\n\t" @@ -91,7 +90,7 @@ * the debuginfo as necessary. It will also warn if it sees any * inconsistencies. */ -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 +.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 .Lhere_\@: .pushsection .discard.unwind_hints /* struct unwind_hint */ @@ -100,7 +99,6 @@ .byte \sp_reg .byte \type .byte \signal - .byte \end .balign 4 .popsection .endm @@ -153,14 +151,14 @@ #ifndef __ASSEMBLY__ -#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) "\n\t" +#define UNWIND_HINT(type, sp_reg, sp_offset, signal) "\n\t" #define STACK_FRAME_NON_STANDARD(func) #define STACK_FRAME_NON_STANDARD_FP(func) #define ANNOTATE_NOENDBR #define ASM_REACHABLE #else #define ANNOTATE_INTRA_FUNCTION_CALL -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 +.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h index 9787ad0f2ef4..453a4f4ef39d 100644 --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -16,12 +16,18 @@ struct unwind_hint { u8 sp_reg; u8 type; u8 signal; - u8 end; }; #endif /* __ASSEMBLY__ */ /* + * UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in + * a truncated and unreliable stack unwind. + * + * UNWIND_HINT_TYPE_END_OF_STACK: The end of the kernel stack unwind before + * hitting user entry, boot code, or fork entry (when there are no pt_regs + * available). + * * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP * (the caller's SP right before it made the call). Used for all callable * functions, i.e. all C code and all callable asm functions. @@ -32,17 +38,20 @@ struct unwind_hint { * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that * sp_reg+sp_offset points to the iret return frame. * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. + * UNWIND_HINT_TYPE_FUNC: Generate the unwind metadata of a callable function. * Useful for code which doesn't have an ELF function annotation. * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. + * UNWIND_HINT_TYPE_{SAVE,RESTORE}: Save the unwind metadata at a certain + * location so that it can be restored later. */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_UNDEFINED 0 +#define UNWIND_HINT_TYPE_END_OF_STACK 1 +#define UNWIND_HINT_TYPE_CALL 2 +#define UNWIND_HINT_TYPE_REGS 3 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 4 /* The below hint types don't have corresponding ORC types */ -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_SAVE 4 -#define UNWIND_HINT_TYPE_RESTORE 5 +#define UNWIND_HINT_TYPE_FUNC 5 +#define UNWIND_HINT_TYPE_SAVE 6 +#define UNWIND_HINT_TYPE_RESTORE 7 #endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/scripts/sorttable.h b/scripts/sorttable.h index deb7c1d3e979..7bd0184380d3 100644 --- a/scripts/sorttable.h +++ b/scripts/sorttable.h @@ -128,7 +128,7 @@ static int orc_sort_cmp(const void *_a, const void *_b) * whitelisted .o files which didn't get objtool generation. */ orc_a = g_orc_table + (a - g_orc_ip_table); - return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; } static void *sort_orctable(void *arg) diff --git a/tools/arch/x86/include/asm/orc_types.h b/tools/arch/x86/include/asm/orc_types.h index b4d4ec78589e..46d7e06763c9 100644 --- a/tools/arch/x86/include/asm/orc_types.h +++ b/tools/arch/x86/include/asm/orc_types.h @@ -39,9 +39,11 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 -#define ORC_TYPE_CALL 0 -#define ORC_TYPE_REGS 1 -#define ORC_TYPE_REGS_PARTIAL 2 +#define ORC_TYPE_UNDEFINED 0 +#define ORC_TYPE_END_OF_STACK 1 +#define ORC_TYPE_CALL 2 +#define ORC_TYPE_REGS 3 +#define ORC_TYPE_REGS_PARTIAL 4 #ifndef __ASSEMBLY__ #include @@ -60,16 +62,14 @@ struct orc_entry { #if defined(__LITTLE_ENDIAN_BITFIELD) unsigned sp_reg:4; unsigned bp_reg:4; - unsigned type:2; + unsigned type:3; unsigned signal:1; - unsigned end:1; #elif defined(__BIG_ENDIAN_BITFIELD) unsigned bp_reg:4; unsigned sp_reg:4; unsigned unused:4; - unsigned end:1; unsigned signal:1; - unsigned type:2; + unsigned type:3; #endif } __packed; diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h index 9787ad0f2ef4..453a4f4ef39d 100644 --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -16,12 +16,18 @@ struct unwind_hint { u8 sp_reg; u8 type; u8 signal; - u8 end; }; #endif /* __ASSEMBLY__ */ /* + * UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in + * a truncated and unreliable stack unwind. + * + * UNWIND_HINT_TYPE_END_OF_STACK: The end of the kernel stack unwind before + * hitting user entry, boot code, or fork entry (when there are no pt_regs + * available). + * * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP * (the caller's SP right before it made the call). Used for all callable * functions, i.e. all C code and all callable asm functions. @@ -32,17 +38,20 @@ struct unwind_hint { * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that * sp_reg+sp_offset points to the iret return frame. * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. + * UNWIND_HINT_TYPE_FUNC: Generate the unwind metadata of a callable function. * Useful for code which doesn't have an ELF function annotation. * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. + * UNWIND_HINT_TYPE_{SAVE,RESTORE}: Save the unwind metadata at a certain + * location so that it can be restored later. */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_UNDEFINED 0 +#define UNWIND_HINT_TYPE_END_OF_STACK 1 +#define UNWIND_HINT_TYPE_CALL 2 +#define UNWIND_HINT_TYPE_REGS 3 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 4 /* The below hint types don't have corresponding ORC types */ -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_SAVE 4 -#define UNWIND_HINT_TYPE_RESTORE 5 +#define UNWIND_HINT_TYPE_FUNC 5 +#define UNWIND_HINT_TYPE_SAVE 6 +#define UNWIND_HINT_TYPE_RESTORE 7 #endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 10be80b3fe67..cae6ac6ff246 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2243,6 +2243,7 @@ static void set_func_state(struct cfi_state *state) memcpy(&state->regs, &initial_func_cfi.regs, CFI_NUM_REGS * sizeof(struct cfi_reg)); state->stack_size = initial_func_cfi.cfa.offset; + state->type = UNWIND_HINT_TYPE_CALL; } static int read_unwind_hints(struct objtool_file *file) @@ -2327,7 +2328,6 @@ static int read_unwind_hints(struct objtool_file *file) cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); cfi.type = hint->type; cfi.signal = hint->signal; - cfi.end = hint->end; insn->cfi = cfi_hash_find_or_add(&cfi); } diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index 97ecbb8b9034..0e183bb1c720 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -38,6 +38,10 @@ static const char *reg_name(unsigned int reg) static const char *orc_type_name(unsigned int type) { switch (type) { + case ORC_TYPE_UNDEFINED: + return "(und)"; + case ORC_TYPE_END_OF_STACK: + return "end"; case ORC_TYPE_CALL: return "call"; case ORC_TYPE_REGS: @@ -201,6 +205,7 @@ int orc_dump(const char *_objname) printf("%llx:", (unsigned long long)(orc_ip_addr + (i * sizeof(int)) + orc_ip[i])); } + printf("type:%s", orc_type_name(orc[i].type)); printf(" sp:"); @@ -210,8 +215,7 @@ int orc_dump(const char *_objname) print_reg(orc[i].bp_reg, bswap_if_needed(&dummy_elf, orc[i].bp_offset)); - printf(" type:%s signal:%d end:%d\n", - orc_type_name(orc[i].type), orc[i].signal, orc[i].end); + printf(" signal:%d\n", orc[i].signal); } elf_end(elf); diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index e85bbb996f6c..b327f9ccfe73 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -21,12 +21,22 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, memset(orc, 0, sizeof(*orc)); if (!cfi) { - orc->end = 0; - orc->sp_reg = ORC_REG_UNDEFINED; + /* + * This is usually either unreachable nops/traps (which don't + * trigger unreachable instruction warnings), or + * STACK_FRAME_NON_STANDARD functions. + */ + orc->type = ORC_TYPE_UNDEFINED; return 0; } switch (cfi->type) { + case UNWIND_HINT_TYPE_UNDEFINED: + orc->type = ORC_TYPE_UNDEFINED; + return 0; + case UNWIND_HINT_TYPE_END_OF_STACK: + orc->type = ORC_TYPE_END_OF_STACK; + return 0; case UNWIND_HINT_TYPE_CALL: orc->type = ORC_TYPE_CALL; break; @@ -42,14 +52,8 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, return -1; } - orc->end = cfi->end; orc->signal = cfi->signal; - if (cfi->cfa.base == CFI_UNDEFINED) { - orc->sp_reg = ORC_REG_UNDEFINED; - return 0; - } - switch (cfi->cfa.base) { case CFI_SP: orc->sp_reg = ORC_REG_SP; @@ -163,11 +167,7 @@ int orc_create(struct objtool_file *file) struct orc_list_entry *entry; struct list_head orc_list; - struct orc_entry null = { - .sp_reg = ORC_REG_UNDEFINED, - .bp_reg = ORC_REG_UNDEFINED, - .type = ORC_TYPE_CALL, - }; + struct orc_entry null = { .type = ORC_TYPE_UNDEFINED }; /* Build a deduplicated list of ORC entries: */ INIT_LIST_HEAD(&orc_list); From e0b081d17a9f4e5c0cbb0e5fbeb1abe3de0f7e4e Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 10:24:07 -0700 Subject: [PATCH 30/52] sched: Fix KCSAN noinstr violation With KCSAN enabled, end_of_stack() can get out-of-lined. Force it inline. Fixes the following warnings: vmlinux.o: warning: objtool: check_stackleak_irqoff+0x2b: call to end_of_stack() leaves .noinstr.text section Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/cc1b4d73d3a428a00d206242a68fdf99a934ca7b.1681320026.git.jpoimboe@kernel.org --- include/linux/sched/task_stack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/sched/task_stack.h b/include/linux/sched/task_stack.h index 5e799a47431e..f158b025c175 100644 --- a/include/linux/sched/task_stack.h +++ b/include/linux/sched/task_stack.h @@ -23,7 +23,7 @@ static __always_inline void *task_stack_page(const struct task_struct *task) #define setup_thread_stack(new,old) do { } while(0) -static inline unsigned long *end_of_stack(const struct task_struct *task) +static __always_inline unsigned long *end_of_stack(const struct task_struct *task) { #ifdef CONFIG_STACK_GROWSUP return (unsigned long *)((unsigned long)task->stack + THREAD_SIZE) - 1; From f571da059f86fd9d432aea32c9c7e5aaa53245d8 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 10:24:08 -0700 Subject: [PATCH 31/52] lkdtm/stackleak: Fix noinstr violation Fixes the following warning: vmlinux.o: warning: objtool: check_stackleak_irqoff+0x2b6: call to _printk() leaves .noinstr.text section Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/ee5209f53aa0a62aea58be18f2b78b17606779a6.1681320026.git.jpoimboe@kernel.org --- drivers/misc/lkdtm/stackleak.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/misc/lkdtm/stackleak.c b/drivers/misc/lkdtm/stackleak.c index 025b133297a6..f1d022160913 100644 --- a/drivers/misc/lkdtm/stackleak.c +++ b/drivers/misc/lkdtm/stackleak.c @@ -43,12 +43,14 @@ static void noinstr check_stackleak_irqoff(void) * STACK_END_MAGIC, and in either casee something is seriously wrong. */ if (current_sp < task_stack_low || current_sp >= task_stack_high) { + instrumentation_begin(); pr_err("FAIL: current_stack_pointer (0x%lx) outside of task stack bounds [0x%lx..0x%lx]\n", current_sp, task_stack_low, task_stack_high - 1); test_failed = true; goto out; } if (lowest_sp < task_stack_low || lowest_sp >= task_stack_high) { + instrumentation_begin(); pr_err("FAIL: current->lowest_stack (0x%lx) outside of task stack bounds [0x%lx..0x%lx]\n", lowest_sp, task_stack_low, task_stack_high - 1); test_failed = true; @@ -86,11 +88,14 @@ static void noinstr check_stackleak_irqoff(void) if (*(unsigned long *)poison_low == STACKLEAK_POISON) continue; + instrumentation_begin(); pr_err("FAIL: non-poison value %lu bytes below poison boundary: 0x%lx\n", poison_high - poison_low, *(unsigned long *)poison_low); test_failed = true; + goto out; } + instrumentation_begin(); pr_info("stackleak stack usage:\n" " high offset: %lu bytes\n" " current: %lu bytes\n" @@ -113,6 +118,7 @@ out: } else { pr_info("OK: the rest of the thread stack is properly erased\n"); } + instrumentation_end(); } static void lkdtm_STACKLEAK_ERASING(void) From e18398e80c73e3cc7d9c3d2e0bc06a4af8f4f1cb Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 10:29:01 -0700 Subject: [PATCH 32/52] Revert "objtool: Support addition to set CFA base" Commit 468af56a7bba ("objtool: Support addition to set CFA base") was added as a preparatory patch for arm64 support, but that support never came. It triggers a false positive warning on x86, so just revert it for now. Fixes the following warning: vmlinux.o: warning: objtool: cdce925_regmap_i2c_write+0xdb: stack state mismatch: cfa1=4+120 cfa2=5+40 Fixes: 468af56a7bba ("objtool: Support addition to set CFA base") Reported-by: kernel test robot Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/oe-kbuild-all/202304080538.j5G6h1AB-lkp@intel.com/ --- tools/objtool/check.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index cae6ac6ff246..9440b07cd3b6 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -3002,17 +3002,6 @@ static int update_cfi_state(struct instruction *insn, break; } - if (!cfi->drap && op->src.reg == CFI_SP && - op->dest.reg == CFI_BP && cfa->base == CFI_SP && - check_reg_frame_pos(®s[CFI_BP], -cfa->offset + op->src.offset)) { - - /* lea disp(%rsp), %rbp */ - cfa->base = CFI_BP; - cfa->offset -= op->src.offset; - cfi->bp_scratch = false; - break; - } - if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { /* drap: lea disp(%rsp), %drap */ From 7f530fba1123edcad00d59e1a73019814935f0c1 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 10:29:35 -0700 Subject: [PATCH 33/52] objtool: Add stackleak instrumentation to uaccess safe list If a function has a large stack frame, the stackleak plugin adds a call to stackleak_track_stack() after the prologue. This function may be called in uaccess-enabled code. Add it to the uaccess safe list. Fixes the following warning: vmlinux.o: warning: objtool: kasan_report+0x12: call to stackleak_track_stack() with UACCESS enabled Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/42e9b487ef89e9b237fd5220ad1c7cf1a2ad7eb8.1681320562.git.jpoimboe@kernel.org --- tools/objtool/check.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 9440b07cd3b6..4c8ef8173a79 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -1279,6 +1279,8 @@ static const char *uaccess_safe_builtin[] = { "__ubsan_handle_type_mismatch_v1", "__ubsan_handle_shift_out_of_bounds", "__ubsan_handle_load_invalid_value", + /* STACKLEAK */ + "stackleak_track_stack", /* misc */ "csum_partial_copy_generic", "copy_mc_fragile", From e8deb00c0c4808654d1bf96a8f79cf1deb59b631 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 10:24:06 -0700 Subject: [PATCH 34/52] context_tracking: Fix KCSAN noinstr violation With KCSAN enabled, even empty inline stubs can be out-of-lined. Force the context_tracking_guest_exit() stub inline. Fixes the following warnings: vmlinux.o: warning: objtool: vmx_vcpu_enter_exit+0x1be: call to context_tracking_guest_exit() leaves .noinstr.text section vmlinux.o: warning: objtool: svm_vcpu_enter_exit+0x85: call to context_tracking_guest_exit() leaves .noinstr.text section Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/dc93f45abdec90c171108b4b590b7fff5790963c.1681320026.git.jpoimboe@kernel.org --- include/linux/context_tracking.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index d4afa8508a80..5ae3abd767b4 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -97,7 +97,7 @@ static inline int exception_enter(void) { return 0; } static inline void exception_exit(enum ctx_state prev_ctx) { } static inline int ct_state(void) { return -1; } static __always_inline bool context_tracking_guest_enter(void) { return false; } -static inline void context_tracking_guest_exit(void) { } +static __always_inline void context_tracking_guest_exit(void) { } #define CT_WARN_ON(cond) do { } while (0) #endif /* !CONFIG_CONTEXT_TRACKING_USER */ From 27d000d635ce48b579988e9b3240352a2a0306e0 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 12:03:16 -0700 Subject: [PATCH 35/52] scripts/objdump-func: Support multiple functions Allow specifying multiple functions on the cmdline. Note this removes the secret EXTRA_ARGS feature. While at it, spread out the awk to make it more readable. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/0bf5f4f5978660985037b24c6db49b114374eb4d.1681325924.git.jpoimboe@kernel.org --- scripts/objdump-func | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/scripts/objdump-func b/scripts/objdump-func index 4eb463dd9f52..7b15b873d0e2 100755 --- a/scripts/objdump-func +++ b/scripts/objdump-func @@ -3,7 +3,7 @@ # # Disassemble a single function. # -# usage: objdump-func +# usage: objdump-func [ ...] set -o errexit set -o nounset @@ -13,17 +13,33 @@ OBJDUMP="${CROSS_COMPILE:-}objdump" command -v gawk >/dev/null 2>&1 || die "gawk isn't installed" usage() { - echo "usage: objdump-func " >&2 + echo "usage: objdump-func [ ...]" >&2 exit 1 } [[ $# -lt 2 ]] && usage OBJ=$1; shift -FUNC=$1; shift +FUNCS=("$@") -# Secret feature to allow adding extra objdump args at the end -EXTRA_ARGS=$@ - -# Note this also matches compiler-added suffixes like ".cold", etc -${OBJDUMP} -wdr $EXTRA_ARGS $OBJ | gawk -M -v f=$FUNC '/^$/ { P=0; } $0 ~ "<" f "(\\..*)?>:" { P=1; O=strtonum("0x" $1); } { if (P) { o=strtonum("0x" $1); printf("%04x ", o-O); print $0; } }' +${OBJDUMP} -wdr $OBJ | gawk -M -v _funcs="${FUNCS[*]}" ' + BEGIN { split(_funcs, funcs); } + /^$/ { func_match=0; } + /<.*>:/ { + f = gensub(/.*<(.*)>:/, "\\1", 1); + for (i in funcs) { + # match compiler-added suffixes like ".cold", etc + if (f ~ "^" funcs[i] "(\\..*)?") { + func_match = 1; + base = strtonum("0x" $1); + break; + } + } + } + { + if (func_match) { + addr = strtonum("0x" $1); + printf("%04x ", addr - base); + print; + } + }' From 246b2c85487a7bc5f6a09098e18a96506b1b55df Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 12:03:17 -0700 Subject: [PATCH 36/52] objtool: Add WARN_INSN() It's easier to use and also gives easy access to the instruction's containing function, which is useful for printing that function's symbol. It will also be useful in the future for rate-limiting and disassembly of warned functions. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/2eaa3155c90fba683d8723599f279c46025b75f3.1681325924.git.jpoimboe@kernel.org --- tools/objtool/check.c | 171 ++++++++++----------------- tools/objtool/include/objtool/warn.h | 5 + tools/objtool/orc_gen.c | 9 +- 3 files changed, 70 insertions(+), 115 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 4c8ef8173a79..7d1a42b31f60 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -1446,7 +1446,7 @@ static void annotate_call_site(struct objtool_file *file, if (opts.mcount && sym->fentry) { if (sibling) - WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset); + WARN_INSN(insn, "tail call to __fentry__ !?!?"); if (opts.mnop) { if (reloc) { reloc->type = R_NONE; @@ -1648,9 +1648,8 @@ static int add_jump_destinations(struct objtool_file *file) continue; } - WARN_FUNC("can't find jump dest instruction at %s+0x%lx", - insn->sec, insn->offset, dest_sec->name, - dest_off); + WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx", + dest_sec->name, dest_off); return -1; } @@ -1733,13 +1732,12 @@ static int add_call_destinations(struct objtool_file *file) continue; if (!insn_call_dest(insn)) { - WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset); + WARN_INSN(insn, "unannotated intra-function call"); return -1; } if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) { - WARN_FUNC("unsupported call to non-function", - insn->sec, insn->offset); + WARN_INSN(insn, "unsupported call to non-function"); return -1; } @@ -1747,10 +1745,8 @@ static int add_call_destinations(struct objtool_file *file) dest_off = arch_dest_reloc_offset(reloc->addend); dest = find_call_destination(reloc->sym->sec, dest_off); if (!dest) { - WARN_FUNC("can't find call dest symbol at %s+0x%lx", - insn->sec, insn->offset, - reloc->sym->sec->name, - dest_off); + WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx", + reloc->sym->sec->name, dest_off); return -1; } @@ -1810,8 +1806,7 @@ static int handle_group_alt(struct objtool_file *file, } else { if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - orig_alt_group->first_insn->offset != special_alt->orig_len) { - WARN_FUNC("weirdly overlapping alternative! %ld != %d", - orig_insn->sec, orig_insn->offset, + WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - orig_alt_group->first_insn->offset, @@ -1880,8 +1875,7 @@ static int handle_group_alt(struct objtool_file *file, if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { - WARN_FUNC("unsupported relocation in alternatives section", - insn->sec, insn->offset); + WARN_INSN(insn, "unsupported relocation in alternatives section"); return -1; } @@ -1895,8 +1889,7 @@ static int handle_group_alt(struct objtool_file *file, if (dest_off == special_alt->new_off + special_alt->new_len) { insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); if (!insn->jump_dest) { - WARN_FUNC("can't find alternative jump destination", - insn->sec, insn->offset); + WARN_INSN(insn, "can't find alternative jump destination"); return -1; } } @@ -1930,8 +1923,7 @@ static int handle_jump_alt(struct objtool_file *file, if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && orig_insn->type != INSN_NOP) { - WARN_FUNC("unsupported instruction at jump label", - orig_insn->sec, orig_insn->offset); + WARN_INSN(orig_insn, "unsupported instruction at jump label"); return -1; } @@ -2010,8 +2002,7 @@ static int add_special_section_alts(struct objtool_file *file) if (special_alt->group) { if (!special_alt->orig_len) { - WARN_FUNC("empty alternative entry", - orig_insn->sec, orig_insn->offset); + WARN_INSN(orig_insn, "empty alternative entry"); continue; } @@ -2102,8 +2093,7 @@ static int add_jump_table(struct objtool_file *file, struct instruction *insn, } if (!prev_offset) { - WARN_FUNC("can't find switch jump table", - insn->sec, insn->offset); + WARN_INSN(insn, "can't find switch jump table"); return -1; } @@ -2307,8 +2297,7 @@ static int read_unwind_hints(struct objtool_file *file) if (sym && sym->bind == STB_GLOBAL) { if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { - WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", - insn->sec, insn->offset); + WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); } } } @@ -2322,8 +2311,7 @@ static int read_unwind_hints(struct objtool_file *file) cfi = *(insn->cfi); if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { - WARN_FUNC("unsupported unwind_hint sp base reg %d", - insn->sec, insn->offset, hint->sp_reg); + WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); return -1; } @@ -2386,8 +2374,7 @@ static int read_retpoline_hints(struct objtool_file *file) insn->type != INSN_CALL_DYNAMIC && insn->type != INSN_RETURN && insn->type != INSN_NOP) { - WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop", - insn->sec, insn->offset); + WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); return -1; } @@ -2498,8 +2485,7 @@ static int read_intra_function_calls(struct objtool_file *file) } if (insn->type != INSN_CALL) { - WARN_FUNC("intra_function_call not a direct call", - insn->sec, insn->offset); + WARN_INSN(insn, "intra_function_call not a direct call"); return -1; } @@ -2513,8 +2499,7 @@ static int read_intra_function_calls(struct objtool_file *file) dest_off = arch_jump_destination(insn); insn->jump_dest = find_insn(file, insn->sec, dest_off); if (!insn->jump_dest) { - WARN_FUNC("can't find call dest at %s+0x%lx", - insn->sec, insn->offset, + WARN_INSN(insn, "can't find call dest at %s+0x%lx", insn->sec->name, dest_off); return -1; } @@ -2855,7 +2840,7 @@ static int update_cfi_state(struct instruction *insn, /* stack operations don't make sense with an undefined CFA */ if (cfa->base == CFI_UNDEFINED) { if (insn_func(insn)) { - WARN_FUNC("undefined stack state", insn->sec, insn->offset); + WARN_INSN(insn, "undefined stack state"); return -1; } return 0; @@ -3038,8 +3023,7 @@ static int update_cfi_state(struct instruction *insn, } if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { - WARN_FUNC("unsupported stack register modification", - insn->sec, insn->offset); + WARN_INSN(insn, "unsupported stack register modification"); return -1; } @@ -3049,8 +3033,7 @@ static int update_cfi_state(struct instruction *insn, if (op->dest.reg != CFI_SP || (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { - WARN_FUNC("unsupported stack pointer realignment", - insn->sec, insn->offset); + WARN_INSN(insn, "unsupported stack pointer realignment"); return -1; } @@ -3145,8 +3128,7 @@ static int update_cfi_state(struct instruction *insn, break; default: - WARN_FUNC("unknown stack-related instruction", - insn->sec, insn->offset); + WARN_INSN(insn, "unknown stack-related instruction"); return -1; } @@ -3235,8 +3217,7 @@ static int update_cfi_state(struct instruction *insn, case OP_DEST_MEM: if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { - WARN_FUNC("unknown stack-related memory operation", - insn->sec, insn->offset); + WARN_INSN(insn, "unknown stack-related memory operation"); return -1; } @@ -3248,8 +3229,7 @@ static int update_cfi_state(struct instruction *insn, break; default: - WARN_FUNC("unknown stack-related instruction", - insn->sec, insn->offset); + WARN_INSN(insn, "unknown stack-related instruction"); return -1; } @@ -3288,8 +3268,7 @@ static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; struct instruction *orig = orig_group->first_insn; char *where = offstr(insn->sec, insn->offset); - WARN_FUNC("stack layout conflict in alternatives: %s", - orig->sec, orig->offset, where); + WARN_INSN(orig, "stack layout conflict in alternatives: %s", where); free(where); return -1; } @@ -3316,8 +3295,7 @@ static int handle_insn_ops(struct instruction *insn, if (!state->uaccess_stack) { state->uaccess_stack = 1; } else if (state->uaccess_stack >> 31) { - WARN_FUNC("PUSHF stack exhausted", - insn->sec, insn->offset); + WARN_INSN(insn, "PUSHF stack exhausted"); return 1; } state->uaccess_stack <<= 1; @@ -3349,8 +3327,7 @@ static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { - WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d", - insn->sec, insn->offset, + WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", cfi1->cfa.base, cfi1->cfa.offset, cfi2->cfa.base, cfi2->cfa.offset); @@ -3360,8 +3337,7 @@ static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) sizeof(struct cfi_reg))) continue; - WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", - insn->sec, insn->offset, + WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", i, cfi1->regs[i].base, cfi1->regs[i].offset, i, cfi2->regs[i].base, cfi2->regs[i].offset); break; @@ -3369,15 +3345,14 @@ static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) } else if (cfi1->type != cfi2->type) { - WARN_FUNC("stack state mismatch: type1=%d type2=%d", - insn->sec, insn->offset, cfi1->type, cfi2->type); + WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", + cfi1->type, cfi2->type); } else if (cfi1->drap != cfi2->drap || (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { - WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", - insn->sec, insn->offset, + WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); @@ -3485,20 +3460,17 @@ static int validate_call(struct objtool_file *file, { if (state->noinstr && state->instr <= 0 && !noinstr_call_dest(file, insn, insn_call_dest(insn))) { - WARN_FUNC("call to %s() leaves .noinstr.text section", - insn->sec, insn->offset, call_dest_name(insn)); + WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); return 1; } if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { - WARN_FUNC("call to %s() with UACCESS enabled", - insn->sec, insn->offset, call_dest_name(insn)); + WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); return 1; } if (state->df) { - WARN_FUNC("call to %s() with DF set", - insn->sec, insn->offset, call_dest_name(insn)); + WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); return 1; } @@ -3510,8 +3482,7 @@ static int validate_sibling_call(struct objtool_file *file, struct insn_state *state) { if (insn_func(insn) && has_modified_stack_frame(insn, state)) { - WARN_FUNC("sibling call from callable instruction with modified stack frame", - insn->sec, insn->offset); + WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); return 1; } @@ -3521,38 +3492,32 @@ static int validate_sibling_call(struct objtool_file *file, static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) { if (state->noinstr && state->instr > 0) { - WARN_FUNC("return with instrumentation enabled", - insn->sec, insn->offset); + WARN_INSN(insn, "return with instrumentation enabled"); return 1; } if (state->uaccess && !func_uaccess_safe(func)) { - WARN_FUNC("return with UACCESS enabled", - insn->sec, insn->offset); + WARN_INSN(insn, "return with UACCESS enabled"); return 1; } if (!state->uaccess && func_uaccess_safe(func)) { - WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function", - insn->sec, insn->offset); + WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); return 1; } if (state->df) { - WARN_FUNC("return with DF set", - insn->sec, insn->offset); + WARN_INSN(insn, "return with DF set"); return 1; } if (func && has_modified_stack_frame(insn, state)) { - WARN_FUNC("return with modified stack frame", - insn->sec, insn->offset); + WARN_INSN(insn, "return with modified stack frame"); return 1; } if (state->cfi.bp_scratch) { - WARN_FUNC("BP used as a scratch register", - insn->sec, insn->offset); + WARN_INSN(insn, "BP used as a scratch register"); return 1; } @@ -3624,8 +3589,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, } if (func && insn->ignore) { - WARN_FUNC("BUG: why am I validating an ignored function?", - sec, insn->offset); + WARN_INSN(insn, "BUG: why am I validating an ignored function?"); return 1; } @@ -3658,14 +3622,12 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, } if (!save_insn) { - WARN_FUNC("no corresponding CFI save for CFI restore", - sec, insn->offset); + WARN_INSN(insn, "no corresponding CFI save for CFI restore"); return 1; } if (!save_insn->visited) { - WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo", - sec, insn->offset); + WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); return 1; } @@ -3725,8 +3687,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, if (opts.stackval && func && !is_fentry_call(insn) && !has_valid_stack_frame(&state)) { - WARN_FUNC("call without frame pointer save/setup", - sec, insn->offset); + WARN_INSN(insn, "call without frame pointer save/setup"); return 1; } @@ -3772,15 +3733,14 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, case INSN_CONTEXT_SWITCH: if (func && (!next_insn || !next_insn->hint)) { - WARN_FUNC("unsupported instruction in callable function", - sec, insn->offset); + WARN_INSN(insn, "unsupported instruction in callable function"); return 1; } return 0; case INSN_STAC: if (state.uaccess) { - WARN_FUNC("recursive UACCESS enable", sec, insn->offset); + WARN_INSN(insn, "recursive UACCESS enable"); return 1; } @@ -3789,12 +3749,12 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, case INSN_CLAC: if (!state.uaccess && func) { - WARN_FUNC("redundant UACCESS disable", sec, insn->offset); + WARN_INSN(insn, "redundant UACCESS disable"); return 1; } if (func_uaccess_safe(func) && !state.uaccess_stack) { - WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset); + WARN_INSN(insn, "UACCESS-safe disables UACCESS"); return 1; } @@ -3803,7 +3763,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, case INSN_STD: if (state.df) { - WARN_FUNC("recursive STD", sec, insn->offset); + WARN_INSN(insn, "recursive STD"); return 1; } @@ -3812,7 +3772,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, case INSN_CLD: if (!state.df && func) { - WARN_FUNC("redundant CLD", sec, insn->offset); + WARN_INSN(insn, "redundant CLD"); return 1; } @@ -3920,15 +3880,14 @@ static int validate_unret(struct objtool_file *file, struct instruction *insn) case INSN_CALL_DYNAMIC: case INSN_JUMP_DYNAMIC: case INSN_JUMP_DYNAMIC_CONDITIONAL: - WARN_FUNC("early indirect call", insn->sec, insn->offset); + WARN_INSN(insn, "early indirect call"); return 1; case INSN_JUMP_UNCONDITIONAL: case INSN_JUMP_CONDITIONAL: if (!is_sibling_call(insn)) { if (!insn->jump_dest) { - WARN_FUNC("unresolved jump target after linking?!?", - insn->sec, insn->offset); + WARN_INSN(insn, "unresolved jump target after linking?!?"); return -1; } ret = validate_unret(file, insn->jump_dest); @@ -3969,7 +3928,7 @@ static int validate_unret(struct objtool_file *file, struct instruction *insn) return 0; case INSN_RETURN: - WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset); + WARN_INSN(insn, "RET before UNTRAIN"); return 1; case INSN_NOP: @@ -3982,7 +3941,7 @@ static int validate_unret(struct objtool_file *file, struct instruction *insn) } if (!next) { - WARN_FUNC("teh end!", insn->sec, insn->offset); + WARN_INSN(insn, "teh end!"); return -1; } insn = next; @@ -4006,7 +3965,7 @@ static int validate_unrets(struct objtool_file *file) ret = validate_unret(file, insn); if (ret < 0) { - WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset); + WARN_INSN(insn, "Failed UNRET validation"); return ret; } warnings += ret; @@ -4034,13 +3993,11 @@ static int validate_retpoline(struct objtool_file *file) if (insn->type == INSN_RETURN) { if (opts.rethunk) { - WARN_FUNC("'naked' return found in RETHUNK build", - insn->sec, insn->offset); + WARN_INSN(insn, "'naked' return found in RETHUNK build"); } else continue; } else { - WARN_FUNC("indirect %s found in RETPOLINE build", - insn->sec, insn->offset, + WARN_INSN(insn, "indirect %s found in RETPOLINE build", insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); } @@ -4419,9 +4376,7 @@ static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn if (noendbr_range(file, dest)) continue; - WARN_FUNC("relocation to !ENDBR: %s", - insn->sec, insn->offset, - offstr(dest->sec, dest->offset)); + WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); warnings++; } @@ -4523,16 +4478,14 @@ static int validate_sls(struct objtool_file *file) switch (insn->type) { case INSN_RETURN: if (!next_insn || next_insn->type != INSN_TRAP) { - WARN_FUNC("missing int3 after ret", - insn->sec, insn->offset); + WARN_INSN(insn, "missing int3 after ret"); warnings++; } break; case INSN_JUMP_DYNAMIC: if (!next_insn || next_insn->type != INSN_TRAP) { - WARN_FUNC("missing int3 after indirect jump", - insn->sec, insn->offset); + WARN_INSN(insn, "missing int3 after indirect jump"); warnings++; } break; @@ -4555,7 +4508,7 @@ static int validate_reachable_instructions(struct objtool_file *file) if (insn->visited || ignore_unreachable_insn(file, insn)) continue; - WARN_FUNC("unreachable instruction", insn->sec, insn->offset); + WARN_INSN(insn, "unreachable instruction"); return 1; } diff --git a/tools/objtool/include/objtool/warn.h b/tools/objtool/include/objtool/warn.h index a3e79ae75f2e..b1c920dc9516 100644 --- a/tools/objtool/include/objtool/warn.h +++ b/tools/objtool/include/objtool/warn.h @@ -53,6 +53,11 @@ static inline char *offstr(struct section *sec, unsigned long offset) free(_str); \ }) +#define WARN_INSN(insn, format, ...) \ +({ \ + WARN_FUNC(format, insn->sec, insn->offset, ##__VA_ARGS__); \ +}) + #define BT_FUNC(format, insn, ...) \ ({ \ struct instruction *_insn = (insn); \ diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index b327f9ccfe73..48efd1e2f00d 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -47,8 +47,7 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, orc->type = ORC_TYPE_REGS_PARTIAL; break; default: - WARN_FUNC("unknown unwind hint type %d", - insn->sec, insn->offset, cfi->type); + WARN_INSN(insn, "unknown unwind hint type %d", cfi->type); return -1; } @@ -80,8 +79,7 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, orc->sp_reg = ORC_REG_DX; break; default: - WARN_FUNC("unknown CFA base reg %d", - insn->sec, insn->offset, cfi->cfa.base); + WARN_INSN(insn, "unknown CFA base reg %d", cfi->cfa.base); return -1; } @@ -96,8 +94,7 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, orc->bp_reg = ORC_REG_BP; break; default: - WARN_FUNC("unknown BP base reg %d", - insn->sec, insn->offset, bp->base); + WARN_INSN(insn, "unknown BP base reg %d", bp->base); return -1; } From 9290e772baccecec324ae9f2e0b470f870c097de Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 12:03:19 -0700 Subject: [PATCH 37/52] objtool: Add symbol iteration helpers Add [sec_]for_each_sym() and use them. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/59023e5886ab125aa30702e633be7732b1acaa7e.1681325924.git.jpoimboe@kernel.org --- tools/objtool/check.c | 98 ++++++++++++----------------- tools/objtool/elf.c | 2 +- tools/objtool/include/objtool/elf.h | 9 +++ 3 files changed, 51 insertions(+), 58 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 7d1a42b31f60..9de3972a9b1c 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -470,7 +470,7 @@ static int decode_instructions(struct objtool_file *file) // printf("%s: last chunk used: %d\n", sec->name, (int)idx); - list_for_each_entry(func, &sec->symbol_list, list) { + sec_for_each_sym(sec, func) { if (func->type != STT_NOTYPE && func->type != STT_FUNC) continue; @@ -924,7 +924,7 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file) static int create_cfi_sections(struct objtool_file *file) { - struct section *sec, *s; + struct section *sec; struct symbol *sym; unsigned int *loc; int idx; @@ -937,19 +937,14 @@ static int create_cfi_sections(struct objtool_file *file) } idx = 0; - for_each_sec(file, s) { - if (!s->text) + for_each_sym(file, sym) { + if (sym->type != STT_FUNC) continue; - list_for_each_entry(sym, &s->symbol_list, list) { - if (sym->type != STT_FUNC) - continue; + if (strncmp(sym->name, "__cfi_", 6)) + continue; - if (strncmp(sym->name, "__cfi_", 6)) - continue; - - idx++; - } + idx++; } sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx); @@ -957,28 +952,23 @@ static int create_cfi_sections(struct objtool_file *file) return -1; idx = 0; - for_each_sec(file, s) { - if (!s->text) + for_each_sym(file, sym) { + if (sym->type != STT_FUNC) continue; - list_for_each_entry(sym, &s->symbol_list, list) { - if (sym->type != STT_FUNC) - continue; + if (strncmp(sym->name, "__cfi_", 6)) + continue; - if (strncmp(sym->name, "__cfi_", 6)) - continue; + loc = (unsigned int *)sec->data->d_buf + idx; + memset(loc, 0, sizeof(unsigned int)); - loc = (unsigned int *)sec->data->d_buf + idx; - memset(loc, 0, sizeof(unsigned int)); + if (elf_add_reloc_to_insn(file->elf, sec, + idx * sizeof(unsigned int), + R_X86_64_PC32, + sym->sec, sym->offset)) + return -1; - if (elf_add_reloc_to_insn(file->elf, sec, - idx * sizeof(unsigned int), - R_X86_64_PC32, - s, sym->offset)) - return -1; - - idx++; - } + idx++; } return 0; @@ -2207,23 +2197,20 @@ static int add_func_jump_tables(struct objtool_file *file, */ static int add_jump_table_alts(struct objtool_file *file) { - struct section *sec; struct symbol *func; int ret; if (!file->rodata) return 0; - for_each_sec(file, sec) { - list_for_each_entry(func, &sec->symbol_list, list) { - if (func->type != STT_FUNC) - continue; + for_each_sym(file, func) { + if (func->type != STT_FUNC) + continue; - mark_func_jump_tables(file, func); - ret = add_func_jump_tables(file, func); - if (ret) - return ret; - } + mark_func_jump_tables(file, func); + ret = add_func_jump_tables(file, func); + if (ret) + return ret; } return 0; @@ -2535,30 +2522,27 @@ static bool is_profiling_func(const char *name) static int classify_symbols(struct objtool_file *file) { - struct section *sec; struct symbol *func; - for_each_sec(file, sec) { - list_for_each_entry(func, &sec->symbol_list, list) { - if (func->bind != STB_GLOBAL) - continue; + for_each_sym(file, func) { + if (func->bind != STB_GLOBAL) + continue; - if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, - strlen(STATIC_CALL_TRAMP_PREFIX_STR))) - func->static_call_tramp = true; + if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, + strlen(STATIC_CALL_TRAMP_PREFIX_STR))) + func->static_call_tramp = true; - if (arch_is_retpoline(func)) - func->retpoline_thunk = true; + if (arch_is_retpoline(func)) + func->retpoline_thunk = true; - if (arch_is_rethunk(func)) - func->return_thunk = true; + if (arch_is_rethunk(func)) + func->return_thunk = true; - if (arch_ftrace_match(func->name)) - func->fentry = true; + if (arch_ftrace_match(func->name)) + func->fentry = true; - if (is_profiling_func(func->name)) - func->profiling_func = true; - } + if (is_profiling_func(func->name)) + func->profiling_func = true; } return 0; @@ -4213,7 +4197,7 @@ static int validate_section(struct objtool_file *file, struct section *sec) struct symbol *func; int warnings = 0; - list_for_each_entry(func, &sec->symbol_list, list) { + sec_for_each_sym(sec, func) { if (func->type != STT_FUNC) continue; diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index 6806ce01d933..500e92979a31 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -474,7 +474,7 @@ static int read_symbols(struct elf *elf) /* Create parent/child links for any cold subfunctions */ list_for_each_entry(sec, &elf->sections, list) { - list_for_each_entry(sym, &sec->symbol_list, list) { + sec_for_each_sym(sec, sym) { char pname[MAX_NAME_LEN + 1]; size_t pnamelen; if (sym->type != STT_FUNC) diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index ad0024da262b..e1ca588eb69d 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -188,4 +188,13 @@ struct symbol *find_func_containing(struct section *sec, unsigned long offset); #define for_each_sec(file, sec) \ list_for_each_entry(sec, &file->elf->sections, list) +#define sec_for_each_sym(sec, sym) \ + list_for_each_entry(sym, &sec->symbol_list, list) + +#define for_each_sym(file, sym) \ + for (struct section *__sec, *__fake = (struct section *)1; \ + __fake; __fake = NULL) \ + for_each_sec(file, __sec) \ + sec_for_each_sym(__sec, sym) + #endif /* _OBJTOOL_ELF_H */ From 6126ed5dfbc656374e851bfdfb128f3aa9e1263a Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 12:03:22 -0700 Subject: [PATCH 38/52] objtool: Remove superfluous dead_end_function() check annotate_call_site() already sets 'insn->dead_end' for calls to dead end functions. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/5d603a301e9a8b1036b61503385907e154867ace.1681325924.git.jpoimboe@kernel.org --- tools/objtool/check.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 9de3972a9b1c..1cf2e2841c47 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -4078,8 +4078,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio * It may also insert a UD2 after calling a __noreturn function. */ prev_insn = prev_insn_same_sec(file, insn); - if ((prev_insn->dead_end || - dead_end_function(file, insn_call_dest(prev_insn))) && + if (prev_insn->dead_end && (insn->type == INSN_BUG || (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && insn->jump_dest->type == INSN_BUG))) From bd456a1bedd20cebd37493f8cb0291294a7356ea Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 13:26:13 -0700 Subject: [PATCH 39/52] objtool: Separate prefix code from stack validation code Simplify the prefix code by moving it after validate_reachable_instructions(). Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/d7f31ac2de462d0cd7b1db01b7ecb525c057c8f6.1681331135.git.jpoimboe@kernel.org --- tools/objtool/check.c | 88 ++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 1cf2e2841c47..8ee4d51de6fa 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -4114,54 +4114,61 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio return false; } -static int add_prefix_symbol(struct objtool_file *file, struct symbol *func, - struct instruction *insn) +static int add_prefix_symbol(struct objtool_file *file, struct symbol *func) { - if (!opts.prefix) - return 0; + struct instruction *insn, *prev; - for (;;) { - struct instruction *prev = prev_insn_same_sec(file, insn); + insn = find_insn(file, func->sec, func->offset); + if (!insn) + return -1; + + for (prev = prev_insn_same_sec(file, insn); + prev; + prev = prev_insn_same_sec(file, prev)) { u64 offset; - if (!prev) - break; - if (prev->type != INSN_NOP) - break; + return -1; offset = func->offset - prev->offset; - if (offset >= opts.prefix) { - if (offset == opts.prefix) { - /* - * Since the sec->symbol_list is ordered by - * offset (see elf_add_symbol()) the added - * symbol will not be seen by the iteration in - * validate_section(). - * - * Hence the lack of list_for_each_entry_safe() - * there. - * - * The direct concequence is that prefix symbols - * don't get visited (because pointless), except - * for the logic in ignore_unreachable_insn() - * that needs the terminating insn to be visited - * otherwise it will report the hole. - * - * Hence mark the first instruction of the - * prefix symbol as visisted. - */ - prev->visited |= VISITED_BRANCH; - elf_create_prefix_symbol(file->elf, func, opts.prefix); - } - break; - } - insn = prev; + + if (offset > opts.prefix) + return -1; + + if (offset < opts.prefix) + continue; + + elf_create_prefix_symbol(file->elf, func, opts.prefix); + break; } + if (!prev) + return -1; + return 0; } +static int add_prefix_symbols(struct objtool_file *file) +{ + struct section *sec; + struct symbol *func; + int warnings = 0; + + for_each_sec(file, sec) { + if (!(sec->sh.sh_flags & SHF_EXECINSTR)) + continue; + + sec_for_each_sym(sec, func) { + if (func->type != STT_FUNC) + continue; + + add_prefix_symbol(file, func); + } + } + + return warnings; +} + static int validate_symbol(struct objtool_file *file, struct section *sec, struct symbol *sym, struct insn_state *state) { @@ -4180,8 +4187,6 @@ static int validate_symbol(struct objtool_file *file, struct section *sec, if (!insn || insn->ignore || insn->visited) return 0; - add_prefix_symbol(file, sym, insn); - state->uaccess = sym->uaccess_safe; ret = validate_branch(file, insn_func(insn), insn, *state); @@ -4621,6 +4626,13 @@ int check(struct objtool_file *file) warnings += ret; } + if (opts.prefix) { + ret = add_prefix_symbols(file); + if (ret < 0) + return ret; + warnings += ret; + } + if (opts.ibt) { ret = create_ibt_endbr_seal_sections(file); if (ret < 0) From 4a2c3448ed3d362431c249ec0eb0f90281804ea8 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 13:26:14 -0700 Subject: [PATCH 40/52] x86/linkage: Fix padding for typed functions CFI typed functions are failing to get padded properly for CONFIG_CALL_PADDING. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/721f0da48d2a49fe907225711b8b76a2b787f9a8.1681331135.git.jpoimboe@kernel.org --- arch/x86/include/asm/linkage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index dd9b8118f784..0953aa32a324 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -99,7 +99,7 @@ /* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */ #define SYM_TYPED_FUNC_START(name) \ - SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_F_ALIGN) \ ENDBR /* SYM_FUNC_START -- use for global functions */ From 5743654f5e2ebd56df99f56fca5ba4b23fe3c815 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 13:26:15 -0700 Subject: [PATCH 41/52] objtool: Generate ORC data for __pfx code Allow unwinding from prefix code by copying the CFI from the starting instruction of the corresponding function. Even when the NOPs are replaced, they're still stack-invariant instructions so the same ORC entry can be reused everywhere. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/bc3344e51f3e87102f1301a0be0f72a7689ea4a4.1681331135.git.jpoimboe@kernel.org --- tools/objtool/check.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 8ee4d51de6fa..df634dafefc4 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -4117,6 +4117,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio static int add_prefix_symbol(struct objtool_file *file, struct symbol *func) { struct instruction *insn, *prev; + struct cfi_state *cfi; insn = find_insn(file, func->sec, func->offset); if (!insn) @@ -4145,6 +4146,19 @@ static int add_prefix_symbol(struct objtool_file *file, struct symbol *func) if (!prev) return -1; + if (!insn->cfi) { + /* + * This can happen if stack validation isn't enabled or the + * function is annotated with STACK_FRAME_NON_STANDARD. + */ + return 0; + } + + /* Propagate insn->cfi to the prefix code */ + cfi = cfi_hash_find_or_add(insn->cfi); + for (; prev != insn; prev = next_insn_same_sec(file, prev)) + prev->cfi = cfi; + return 0; } From 9ea7e6b62c2bd2f7bbfc3f10099df803002dd33b Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:31 -0700 Subject: [PATCH 42/52] init: Mark [arch_call_]rest_init() __noreturn In preparation for improving objtool's handling of weak noreturn functions, mark start_kernel(), arch_call_rest_init(), and rest_init() __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Nick Desaulniers Link: https://lore.kernel.org/r/7194ed8a989a85b98d92e62df660f4a90435a723.1681342859.git.jpoimboe@kernel.org --- arch/s390/kernel/setup.c | 2 +- include/linux/start_kernel.h | 4 ++-- init/main.c | 4 ++-- tools/objtool/check.c | 2 ++ 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 8ec5cdf9dadc..4259b6c50516 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -396,7 +396,7 @@ int __init arch_early_irq_init(void) return 0; } -void __init arch_call_rest_init(void) +void __init __noreturn arch_call_rest_init(void) { unsigned long stack; diff --git a/include/linux/start_kernel.h b/include/linux/start_kernel.h index 8b369a41c03c..864921e54c92 100644 --- a/include/linux/start_kernel.h +++ b/include/linux/start_kernel.h @@ -9,7 +9,7 @@ up something else. */ extern asmlinkage void __init start_kernel(void); -extern void __init arch_call_rest_init(void); -extern void __ref rest_init(void); +extern void __init __noreturn arch_call_rest_init(void); +extern void __ref __noreturn rest_init(void); #endif /* _LINUX_START_KERNEL_H */ diff --git a/init/main.c b/init/main.c index 4425d1783d5c..161ed956d738 100644 --- a/init/main.c +++ b/init/main.c @@ -683,7 +683,7 @@ static void __init setup_command_line(char *command_line) static __initdata DECLARE_COMPLETION(kthreadd_done); -noinline void __ref rest_init(void) +noinline void __ref __noreturn rest_init(void) { struct task_struct *tsk; int pid; @@ -889,7 +889,7 @@ static int __init early_randomize_kstack_offset(char *buf) early_param("randomize_kstack_offset", early_randomize_kstack_offset); #endif -void __init __weak arch_call_rest_init(void) +void __init __weak __noreturn arch_call_rest_init(void) { rest_init(); } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index df634dafefc4..3d7227f0ea2a 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -202,6 +202,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "__reiserfs_panic", "__stack_chk_fail", "__ubsan_handle_builtin_unreachable", + "arch_call_rest_init", "arch_cpu_idle_dead", "cpu_bringup_and_idle", "cpu_startup_entry", @@ -217,6 +218,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "machine_real_restart", "make_task_dead", "panic", + "rest_init", "rewind_stack_and_make_dead", "sev_es_terminate", "snp_abort", From 25a6917ca63ad4470bf88535c56f0dec72b570fe Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:32 -0700 Subject: [PATCH 43/52] init: Mark start_kernel() __noreturn Now that arch_call_rest_init() is __noreturn, mark its caller start_kernel() __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/7069acf026a195f26a88061227fba5a3b0337b9a.1681342859.git.jpoimboe@kernel.org --- include/linux/start_kernel.h | 2 +- init/main.c | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/linux/start_kernel.h b/include/linux/start_kernel.h index 864921e54c92..a9806a44a605 100644 --- a/include/linux/start_kernel.h +++ b/include/linux/start_kernel.h @@ -8,7 +8,7 @@ /* Define the prototype for start_kernel here, rather than cluttering up something else. */ -extern asmlinkage void __init start_kernel(void); +extern asmlinkage void __init __noreturn start_kernel(void); extern void __init __noreturn arch_call_rest_init(void); extern void __ref __noreturn rest_init(void); diff --git a/init/main.c b/init/main.c index 161ed956d738..65aab4e9bb49 100644 --- a/init/main.c +++ b/init/main.c @@ -937,7 +937,7 @@ static void __init print_unknown_bootoptions(void) memblock_free(unknown_options, len); } -asmlinkage __visible void __init __no_sanitize_address start_kernel(void) +asmlinkage __visible void __init __no_sanitize_address __noreturn start_kernel(void) { char *command_line; char *after_dashes; diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 3d7227f0ea2a..9aaad9dc6c20 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -222,6 +222,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "rewind_stack_and_make_dead", "sev_es_terminate", "snp_abort", + "start_kernel", "stop_this_cpu", "usercopy_abort", "xen_cpu_bringup_again", From 4208d2d79837ef70f260d6170e3ac7fd6fde7788 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:33 -0700 Subject: [PATCH 44/52] x86/head: Mark *_start_kernel() __noreturn Now that start_kernel() is __noreturn, mark its chain of callers __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/c2525f96b88be98ee027ee0291d58003036d4120.1681342859.git.jpoimboe@kernel.org --- arch/x86/include/asm/setup.h | 6 +++--- arch/x86/kernel/head32.c | 2 +- arch/x86/kernel/head64.c | 4 ++-- tools/objtool/check.c | 2 ++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index f37cbff7354c..f3495623ac99 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -125,11 +125,11 @@ void clear_bss(void); #ifdef __i386__ -asmlinkage void __init i386_start_kernel(void); +asmlinkage void __init __noreturn i386_start_kernel(void); #else -asmlinkage void __init x86_64_start_kernel(char *real_mode); -asmlinkage void __init x86_64_start_reservations(char *real_mode_data); +asmlinkage void __init __noreturn x86_64_start_kernel(char *real_mode); +asmlinkage void __init __noreturn x86_64_start_reservations(char *real_mode_data); #endif /* __i386__ */ #endif /* _SETUP */ diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c index ec6fefbfd3c0..10c27b4261eb 100644 --- a/arch/x86/kernel/head32.c +++ b/arch/x86/kernel/head32.c @@ -29,7 +29,7 @@ static void __init i386_default_early_setup(void) x86_init.mpparse.setup_ioapic_ids = setup_ioapic_ids_from_mpc; } -asmlinkage __visible void __init i386_start_kernel(void) +asmlinkage __visible void __init __noreturn i386_start_kernel(void) { /* Make sure IDT is set up before any exception happens */ idt_setup_early_handler(); diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 387e4b12e823..49f7629b17f7 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -471,7 +471,7 @@ static void __init copy_bootdata(char *real_mode_data) sme_unmap_bootdata(real_mode_data); } -asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) +asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode_data) { /* * Build-time sanity checks on the kernel image and module @@ -537,7 +537,7 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) x86_64_start_reservations(real_mode_data); } -void __init x86_64_start_reservations(char *real_mode_data) +void __init __noreturn x86_64_start_reservations(char *real_mode_data) { /* version is always not zero if it is copied */ if (!boot_params.hdr.version) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 9aaad9dc6c20..a436bc1e63d9 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -225,6 +225,8 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "start_kernel", "stop_this_cpu", "usercopy_abort", + "x86_64_start_kernel", + "x86_64_start_reservations", "xen_cpu_bringup_again", "xen_start_kernel", }; From 5ab6876c7843db5fe8bef691c5fdb92518b12070 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:34 -0700 Subject: [PATCH 45/52] arm64/cpu: Mark cpu_park_loop() and friends __noreturn In preparation for marking panic_smp_self_stop() __noreturn across the kernel, first mark the arm64 implementation of cpu_park_loop() and related functions __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/55787d3193ea3e295ccbb097abfab0a10ae49d45.1681342859.git.jpoimboe@kernel.org --- arch/arm64/include/asm/exception.h | 4 ++-- arch/arm64/include/asm/smp.h | 6 +++--- arch/arm64/kernel/entry-common.c | 2 +- arch/arm64/kernel/smp.c | 8 +++++--- arch/arm64/kernel/traps.c | 3 +-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index 92963f98afec..e73af709cb7a 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -31,7 +31,7 @@ static inline unsigned long disr_to_esr(u64 disr) return esr; } -asmlinkage void handle_bad_stack(struct pt_regs *regs); +asmlinkage void __noreturn handle_bad_stack(struct pt_regs *regs); asmlinkage void el1t_64_sync_handler(struct pt_regs *regs); asmlinkage void el1t_64_irq_handler(struct pt_regs *regs); @@ -80,5 +80,5 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr); void do_serror(struct pt_regs *regs, unsigned long esr); void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags); -void panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far); +void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far); #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 5733a31bab08..07f4ea1490f4 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -101,9 +101,9 @@ extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); extern void __noreturn cpu_die(void); -extern void cpu_die_early(void); +extern void __noreturn cpu_die_early(void); -static inline void cpu_park_loop(void) +static inline void __noreturn cpu_park_loop(void) { for (;;) { wfe(); @@ -123,7 +123,7 @@ static inline void update_cpu_boot_status(int val) * which calls for a kernel panic. Update the boot status and park the calling * CPU. */ -static inline void cpu_panic_kernel(void) +static inline void __noreturn cpu_panic_kernel(void) { update_cpu_boot_status(CPU_PANIC_KERNEL); cpu_park_loop(); diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index cce1167199e3..3af3c01c93a6 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -840,7 +840,7 @@ UNHANDLED(el0t, 32, error) #endif /* CONFIG_COMPAT */ #ifdef CONFIG_VMAP_STACK -asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs) +asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) { unsigned long esr = read_sysreg(esr_el1); unsigned long far = read_sysreg(far_el1); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index d5d09a18b4f8..07d156fddb5f 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -398,7 +398,7 @@ static void __cpu_try_die(int cpu) * Kill the calling secondary CPU, early in bringup before it is turned * online. */ -void cpu_die_early(void) +void __noreturn cpu_die_early(void) { int cpu = smp_processor_id(); @@ -816,7 +816,7 @@ void arch_irq_work_raise(void) } #endif -static void local_cpu_stop(void) +static void __noreturn local_cpu_stop(void) { set_cpu_online(smp_processor_id(), false); @@ -839,7 +839,7 @@ void panic_smp_self_stop(void) static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0); #endif -static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) +static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) { #ifdef CONFIG_KEXEC_CORE crash_save_cpu(regs, cpu); @@ -854,6 +854,8 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) /* just in case */ cpu_park_loop(); +#else + BUG(); #endif } diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 4a79ba100799..4bb1b8f47298 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -863,7 +863,7 @@ void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr) DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) __aligned(16); -void panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far) +void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far) { unsigned long tsk_stk = (unsigned long)current->stack; unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); @@ -905,7 +905,6 @@ void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr) nmi_panic(regs, "Asynchronous SError Interrupt"); cpu_park_loop(); - unreachable(); } bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned long esr) From 7412a60decec2a6744cf773280ff17a0f89e8395 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:35 -0700 Subject: [PATCH 46/52] cpu: Mark panic_smp_self_stop() __noreturn In preparation for improving objtool's handling of weak noreturn functions, mark panic_smp_self_stop() __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/92d76ab5c8bf660f04fdcd3da1084519212de248.1681342859.git.jpoimboe@kernel.org --- arch/arm/kernel/smp.c | 2 +- arch/arm64/include/asm/smp.h | 1 - arch/arm64/kernel/smp.c | 2 +- arch/powerpc/kernel/setup_64.c | 2 +- include/linux/smp.h | 2 +- kernel/panic.c | 2 +- tools/objtool/check.c | 1 + 7 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index d6be4507d22d..f4a4ac028b6b 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -779,7 +779,7 @@ void smp_send_stop(void) * kdump fails. So split out the panic_smp_self_stop() and add * set_cpu_online(smp_processor_id(), false). */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { pr_debug("CPU %u will stop doing anything useful since another CPU has paniced\n", smp_processor_id()); diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 07f4ea1490f4..f2d26235bfb4 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -143,7 +143,6 @@ bool cpus_are_stuck_in_kernel(void); extern void crash_smp_send_stop(void); extern bool smp_crash_stop_failed(void); -extern void panic_smp_self_stop(void); #endif /* ifndef __ASSEMBLY__ */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 07d156fddb5f..05fe797e4203 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -830,7 +830,7 @@ static void __noreturn local_cpu_stop(void) * that cpu_online_mask gets correctly updated and smp_send_stop() can skip * CPUs that have already stopped themselves. */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { local_cpu_stop(); } diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index b2e0d3ce4261..246201d0d879 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -480,7 +480,7 @@ void early_setup_secondary(void) #endif /* CONFIG_SMP */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { hard_irq_disable(); spin_begin(); diff --git a/include/linux/smp.h b/include/linux/smp.h index a80ab58ae3f1..2a737b39cf0a 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -59,7 +59,7 @@ int smp_call_function_single_async(int cpu, struct __call_single_data *csd); * Cpus stopping functions in panic. All have default weak definitions. * Architecture-dependent code may override them. */ -void panic_smp_self_stop(void); +void __noreturn panic_smp_self_stop(void); void nmi_panic_self_stop(struct pt_regs *regs); void crash_smp_send_stop(void); diff --git a/kernel/panic.c b/kernel/panic.c index 5cfea8302d23..5e4982db8dc9 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -141,7 +141,7 @@ EXPORT_SYMBOL(panic_blink); /* * Stop ourself in panic -- architecture code may override this */ -void __weak panic_smp_self_stop(void) +void __weak __noreturn panic_smp_self_stop(void) { while (1) cpu_relax(); diff --git a/tools/objtool/check.c b/tools/objtool/check.c index a436bc1e63d9..4b52ed6bff66 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -218,6 +218,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "machine_real_restart", "make_task_dead", "panic", + "panic_smp_self_stop", "rest_init", "rewind_stack_and_make_dead", "sev_es_terminate", From 27dea14c7f05106f39850a9239874cd38703b405 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:36 -0700 Subject: [PATCH 47/52] cpu: Mark nmi_panic_self_stop() __noreturn In preparation for improving objtool's handling of weak noreturn functions, mark nmi_panic_self_stop() __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/316fc6dfab5a8c4e024c7185484a1ee5fb0afb79.1681342859.git.jpoimboe@kernel.org --- arch/x86/include/asm/reboot.h | 1 - arch/x86/kernel/reboot.c | 2 +- include/linux/smp.h | 2 +- kernel/panic.c | 2 +- tools/objtool/check.c | 1 + 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/reboot.h b/arch/x86/include/asm/reboot.h index bc5b4d788c08..9177b4354c3f 100644 --- a/arch/x86/include/asm/reboot.h +++ b/arch/x86/include/asm/reboot.h @@ -28,7 +28,6 @@ void __noreturn machine_real_restart(unsigned int type); void cpu_emergency_disable_virtualization(void); typedef void (*nmi_shootdown_cb)(int, struct pt_regs*); -void nmi_panic_self_stop(struct pt_regs *regs); void nmi_shootdown_cpus(nmi_shootdown_cb callback); void run_crash_ipi_callback(struct pt_regs *regs); diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index d03c551defcc..3adbe97015c1 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -920,7 +920,7 @@ void run_crash_ipi_callback(struct pt_regs *regs) } /* Override the weak function in kernel/panic.c */ -void nmi_panic_self_stop(struct pt_regs *regs) +void __noreturn nmi_panic_self_stop(struct pt_regs *regs) { while (1) { /* If no CPU is preparing crash dump, we simply loop here. */ diff --git a/include/linux/smp.h b/include/linux/smp.h index 2a737b39cf0a..7b93504eed26 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -60,7 +60,7 @@ int smp_call_function_single_async(int cpu, struct __call_single_data *csd); * Architecture-dependent code may override them. */ void __noreturn panic_smp_self_stop(void); -void nmi_panic_self_stop(struct pt_regs *regs); +void __noreturn nmi_panic_self_stop(struct pt_regs *regs); void crash_smp_send_stop(void); /* diff --git a/kernel/panic.c b/kernel/panic.c index 5e4982db8dc9..886d2ebd0a0d 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -151,7 +151,7 @@ void __weak __noreturn panic_smp_self_stop(void) * Stop ourselves in NMI context if another CPU has already panicked. Arch code * may override this to prepare for crash dumping, e.g. save regs info. */ -void __weak nmi_panic_self_stop(struct pt_regs *regs) +void __weak __noreturn nmi_panic_self_stop(struct pt_regs *regs) { panic_smp_self_stop(); } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 4b52ed6bff66..8d073bfd7d7f 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -217,6 +217,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "lbug_with_loc", "machine_real_restart", "make_task_dead", + "nmi_panic_self_stop", "panic", "panic_smp_self_stop", "rest_init", From 1c47c8758a11345ac643fa68cb70b708a6668883 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:37 -0700 Subject: [PATCH 48/52] objtool: Include weak functions in global_noreturns check If a global function doesn't return, and its prototype has the __noreturn attribute, its weak counterpart must also not return so that it matches the prototype and meets call site expectations. To properly follow the compiled control flow at the call sites, change the global_noreturns check to include both global and weak functions. On the other hand, if a weak function isn't in global_noreturns, assume the prototype doesn't have __noreturn. Even if the weak function doesn't return, call sites treat it like a returnable function. Fixes the following warning: kernel/sched/build_policy.o: warning: objtool: do_idle() falls through to next function play_idle_precise() Reported-by: kernel test robot Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Miroslav Benes Link: https://lore.kernel.org/r/ede3460d63f4a65d282c86f1175bd2662c2286ba.1681342859.git.jpoimboe@kernel.org --- tools/objtool/check.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 8d073bfd7d7f..ae0c94285124 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -236,14 +236,14 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, if (!func) return false; - if (func->bind == STB_WEAK) - return false; - - if (func->bind == STB_GLOBAL) + if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) if (!strcmp(func->name, global_noreturns[i])) return true; + if (func->bind == STB_WEAK) + return false; + if (!func->len) return false; From 09c5ae30d007514a1be870fa5873ad55c3319f3a Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:38 -0700 Subject: [PATCH 49/52] btrfs: Mark btrfs_assertfail() __noreturn Fixes a bunch of warnings including: vmlinux.o: warning: objtool: select_reloc_root+0x314: unreachable instruction vmlinux.o: warning: objtool: finish_inode_if_needed+0x15b1: unreachable instruction vmlinux.o: warning: objtool: get_bio_sector_nr+0x259: unreachable instruction vmlinux.o: warning: objtool: raid_wait_read_end_io+0xc26: unreachable instruction vmlinux.o: warning: objtool: raid56_parity_alloc_scrub_rbio+0x37b: unreachable instruction ... Reported-by: kernel test robot Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/960bd9c0c9e3cfc409ba9c35a17644b11b832956.1681342859.git.jpoimboe@kernel.org --- fs/btrfs/messages.c | 2 +- fs/btrfs/messages.h | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/messages.c b/fs/btrfs/messages.c index fde5aaa6e7c9..310a05cf95ef 100644 --- a/fs/btrfs/messages.c +++ b/fs/btrfs/messages.c @@ -253,7 +253,7 @@ void __cold _btrfs_printk(const struct btrfs_fs_info *fs_info, const char *fmt, #endif #ifdef CONFIG_BTRFS_ASSERT -void __cold btrfs_assertfail(const char *expr, const char *file, int line) +void __cold __noreturn btrfs_assertfail(const char *expr, const char *file, int line) { pr_err("assertion failed: %s, in %s:%d\n", expr, file, line); BUG(); diff --git a/fs/btrfs/messages.h b/fs/btrfs/messages.h index 8c516ee58ff9..ac2d1982ba3d 100644 --- a/fs/btrfs/messages.h +++ b/fs/btrfs/messages.h @@ -160,7 +160,7 @@ do { \ } while (0) #ifdef CONFIG_BTRFS_ASSERT -void __cold btrfs_assertfail(const char *expr, const char *file, int line); +void __cold __noreturn btrfs_assertfail(const char *expr, const char *file, int line); #define ASSERT(expr) \ (likely(expr) ? (void)0 : btrfs_assertfail(#expr, __FILE__, __LINE__)) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index ae0c94285124..ceb98489d718 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -204,6 +204,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "__ubsan_handle_builtin_unreachable", "arch_call_rest_init", "arch_cpu_idle_dead", + "btrfs_assertfail", "cpu_bringup_and_idle", "cpu_startup_entry", "do_exit", From 52668badd34b4b346f32c33a9bcba069a06c3caa Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:39 -0700 Subject: [PATCH 50/52] x86/cpu: Mark {hlt,resume}_play_dead() __noreturn Fixes the following warning: vmlinux.o: warning: objtool: resume_play_dead+0x21: unreachable instruction Reported-by: kernel test robot Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/ce1407c4bf88b1334fe40413126343792a77ca50.1681342859.git.jpoimboe@kernel.org --- arch/x86/include/asm/smp.h | 2 +- arch/x86/kernel/smpboot.c | 2 +- arch/x86/power/cpu.c | 2 +- tools/objtool/check.c | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index e6d1d2810e38..47ce4c79a3b0 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -125,7 +125,7 @@ int native_cpu_up(unsigned int cpunum, struct task_struct *tidle); int native_cpu_disable(void); int common_cpu_die(unsigned int cpu); void native_cpu_die(unsigned int cpu); -void hlt_play_dead(void); +void __noreturn hlt_play_dead(void); void native_play_dead(void); void play_dead_common(void); void wbinvd_on_cpu(int cpu); diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 9013bb28255a..a6da3f94b7b6 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1816,7 +1816,7 @@ static inline void mwait_play_dead(void) } } -void hlt_play_dead(void) +void __noreturn hlt_play_dead(void) { if (__this_cpu_read(cpu_info.x86) >= 4) wbinvd(); diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 236447ee9beb..7a4d5e911415 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -288,7 +288,7 @@ EXPORT_SYMBOL(restore_processor_state); #endif #if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU) -static void resume_play_dead(void) +static void __noreturn resume_play_dead(void) { play_dead_common(); tboot_shutdown(TB_SHUTDOWN_WFS); diff --git a/tools/objtool/check.c b/tools/objtool/check.c index ceb98489d718..724a63bdc412 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -212,6 +212,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "do_task_dead", "ex_handler_msr_mce", "fortify_panic", + "hlt_play_dead", "kthread_complete_and_exit", "kthread_exit", "kunit_try_catch_throw", @@ -222,6 +223,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "panic", "panic_smp_self_stop", "rest_init", + "resume_play_dead", "rewind_stack_and_make_dead", "sev_es_terminate", "snp_abort", From 6e36a56a5f617262c0e8ae7e961487361c720b9e Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:40 -0700 Subject: [PATCH 51/52] scsi: message: fusion: Mark mpt_halt_firmware() __noreturn mpt_halt_firmware() doesn't return. Mark it as such. Fixes the following warnings: vmlinux.o: warning: objtool: mptscsih_abort+0x7f4: unreachable instruction vmlinux.o: warning: objtool: mptctl_timeout_expired+0x310: unreachable instruction Reported-by: kernel test robot Reported-by: Mark Rutland Debugged-by: Peter Zijlstra Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/d8129817423422355bf30e90dadc6764261b53e0.1681342859.git.jpoimboe@kernel.org --- drivers/message/fusion/mptbase.c | 2 +- drivers/message/fusion/mptbase.h | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c index 9b3ba2df71c7..4f0afce8428d 100644 --- a/drivers/message/fusion/mptbase.c +++ b/drivers/message/fusion/mptbase.c @@ -6935,7 +6935,7 @@ EXPORT_SYMBOL(mpt_clear_taskmgmt_in_progress_flag); * @ioc: Pointer to MPT_ADAPTER structure * **/ -void +void __noreturn mpt_halt_firmware(MPT_ADAPTER *ioc) { u32 ioc_raw_state; diff --git a/drivers/message/fusion/mptbase.h b/drivers/message/fusion/mptbase.h index 4bd0682c65d3..0f226cdad64f 100644 --- a/drivers/message/fusion/mptbase.h +++ b/drivers/message/fusion/mptbase.h @@ -945,7 +945,7 @@ extern int mpt_raid_phys_disk_get_num_paths(MPT_ADAPTER *ioc, u8 phys_disk_num); extern int mpt_set_taskmgmt_in_progress_flag(MPT_ADAPTER *ioc); extern void mpt_clear_taskmgmt_in_progress_flag(MPT_ADAPTER *ioc); -extern void mpt_halt_firmware(MPT_ADAPTER *ioc); +extern void __noreturn mpt_halt_firmware(MPT_ADAPTER *ioc); /* diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 724a63bdc412..e1b01ea6ad1b 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -219,6 +219,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "lbug_with_loc", "machine_real_restart", "make_task_dead", + "mpt_halt_firmware", "nmi_panic_self_stop", "panic", "panic_smp_self_stop", From 611d4c716db0141cfc436994dc5aff1d69c924ad Mon Sep 17 00:00:00 2001 From: "Guilherme G. Piccoli" Date: Wed, 12 Apr 2023 16:49:41 -0700 Subject: [PATCH 52/52] x86/hyperv: Mark hv_ghcb_terminate() as noreturn Annotate the function prototype and definition as noreturn to prevent objtool warnings like: vmlinux.o: warning: objtool: hyperv_init+0x55c: unreachable instruction Also, as per Josh's suggestion, add it to the global_noreturns list. As a comparison, an objdump output without the annotation: [...] 1b63: mov $0x1,%esi 1b68: xor %edi,%edi 1b6a: callq ffffffff8102f680 1b6f: jmpq ffffffff82f217ec # unreachable 1b74: cmpq $0xffffffffffffffff,-0x702a24(%rip) [...] Now, after adding the __noreturn to the function prototype: [...] 17df: callq ffffffff8102f6d0 17e4: test %al,%al 17e6: je ffffffff82f21bb9 [...] 1bb9: mov $0x1,%esi 1bbe: xor %edi,%edi 1bc0: callq ffffffff8102f680 1bc5: nopw %cs:0x0(%rax,%rax,1) # end of function Reported-by: Arnd Bergmann Signed-off-by: Guilherme G. Piccoli Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/32453a703dfcf0d007b473c9acbf70718222b74b.1681342859.git.jpoimboe@kernel.org --- arch/x86/hyperv/ivm.c | 2 +- arch/x86/include/asm/mshyperv.h | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c index 1dbcbd9da74d..4f79dc76042d 100644 --- a/arch/x86/hyperv/ivm.c +++ b/arch/x86/hyperv/ivm.c @@ -127,7 +127,7 @@ static enum es_result hv_ghcb_hv_call(struct ghcb *ghcb, u64 exit_code, return ES_OK; } -void hv_ghcb_terminate(unsigned int set, unsigned int reason) +void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason) { u64 val = GHCB_MSR_TERM_REQ; diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index 4c4c0ec3b62e..09c26e658bcc 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -212,7 +212,7 @@ int hv_set_mem_host_visibility(unsigned long addr, int numpages, bool visible); void hv_ghcb_msr_write(u64 msr, u64 value); void hv_ghcb_msr_read(u64 msr, u64 *value); bool hv_ghcb_negotiate_protocol(void); -void hv_ghcb_terminate(unsigned int set, unsigned int reason); +void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason); #else static inline void hv_ghcb_msr_write(u64 msr, u64 value) {} static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {} diff --git a/tools/objtool/check.c b/tools/objtool/check.c index e1b01ea6ad1b..5b600bbf2389 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -213,6 +213,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "ex_handler_msr_mce", "fortify_panic", "hlt_play_dead", + "hv_ghcb_terminate", "kthread_complete_and_exit", "kthread_exit", "kunit_try_catch_throw",