diff --git a/Documentation/features/core/jump-labels/arch-support.txt b/Documentation/features/core/jump-labels/arch-support.txt index 632a1c7aefa2..760243d18ed7 100644 --- a/Documentation/features/core/jump-labels/arch-support.txt +++ b/Documentation/features/core/jump-labels/arch-support.txt @@ -23,7 +23,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | TODO | | sparc: | ok | diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index b55be0980b2b..bf6a9feee803 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -47,6 +47,8 @@ config RISCV select GENERIC_TIME_VSYSCALL if MMU && 64BIT select HANDLE_DOMAIN_IRQ select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if MMU && 64BIT select HAVE_ARCH_KGDB select HAVE_ARCH_KGDB_QXFER_PKT diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 4da4886246a4..d58c93efb603 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_SMP=y +CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_NET=y diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index b48138e329ea..cd1df62b13c7 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -33,6 +33,7 @@ CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0" CONFIG_CMDLINE_FORCE=y +CONFIG_JUMP_LABEL=y # CONFIG_BLOCK is not set CONFIG_BINFMT_FLAT=y # CONFIG_COREDUMP is not set diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig index cf74e179bf90..f27596e9663e 100644 --- a/arch/riscv/configs/nommu_virt_defconfig +++ b/arch/riscv/configs/nommu_virt_defconfig @@ -30,6 +30,7 @@ CONFIG_MAXPHYSMEM_2GB=y CONFIG_SMP=y CONFIG_CMDLINE="root=/dev/vda rw earlycon=uart8250,mmio,0x10000000,115200n8 console=ttyS0" CONFIG_CMDLINE_FORCE=y +CONFIG_JUMP_LABEL=y # CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y # CONFIG_MSDOS_PARTITION is not set diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 05bbf5240569..3a55f0e00d6c 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y +CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_NET=y diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h new file mode 100644 index 000000000000..38af2ec7b9bf --- /dev/null +++ b/arch/riscv/include/asm/jump_label.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Emil Renner Berthing + * + * Based on arch/arm64/include/asm/jump_label.h + */ +#ifndef __ASM_JUMP_LABEL_H +#define __ASM_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + +#include +#include + +#define JUMP_LABEL_NOP_SIZE 4 + +static __always_inline bool arch_static_branch(struct static_key *key, + bool branch) +{ + asm_volatile_goto( + " .option push \n\t" + " .option norelax \n\t" + " .option norvc \n\t" + "1: nop \n\t" + " .option pop \n\t" + " .pushsection __jump_table, \"aw\" \n\t" + " .align " RISCV_LGPTR " \n\t" + " .long 1b - ., %l[label] - . \n\t" + " " RISCV_PTR " %0 - . \n\t" + " .popsection \n\t" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, + bool branch) +{ + asm_volatile_goto( + " .option push \n\t" + " .option norelax \n\t" + " .option norvc \n\t" + "1: jal zero, %l[label] \n\t" + " .option pop \n\t" + " .pushsection __jump_table, \"aw\" \n\t" + " .align " RISCV_LGPTR " \n\t" + " .long 1b - ., %l[label] - . \n\t" + " " RISCV_PTR " %0 - . \n\t" + " .popsection \n\t" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index b355cf485671..a5287ab9f7f2 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -53,4 +53,6 @@ endif obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o obj-$(CONFIG_KGDB) += kgdb.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o + clean: diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c new file mode 100644 index 000000000000..20e09056d141 --- /dev/null +++ b/arch/riscv/kernel/jump_label.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Emil Renner Berthing + * + * Based on arch/arm64/kernel/jump_label.c + */ +#include +#include +#include +#include +#include +#include + +#define RISCV_INSN_NOP 0x00000013U +#define RISCV_INSN_JAL 0x0000006fU + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + void *addr = (void *)jump_entry_code(entry); + u32 insn; + + if (type == JUMP_LABEL_JMP) { + long offset = jump_entry_target(entry) - jump_entry_code(entry); + + if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288)) + return; + + insn = RISCV_INSN_JAL | + (((u32)offset & GENMASK(19, 12)) << (12 - 12)) | + (((u32)offset & GENMASK(11, 11)) << (20 - 11)) | + (((u32)offset & GENMASK(10, 1)) << (21 - 1)) | + (((u32)offset & GENMASK(20, 20)) << (31 - 20)); + } else { + insn = RISCV_INSN_NOP; + } + + mutex_lock(&text_mutex); + patch_text_nosync(addr, &insn, sizeof(insn)); + mutex_unlock(&text_mutex); +} + +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + /* + * We use the same instructions in the arch_static_branch and + * arch_static_branch_jump inline functions, so there's no + * need to patch them up here. + * The core will call arch_jump_label_transform when those + * instructions need to be replaced. + */ +}