From da5b2ad1c2f18834cb1ce429e2e5a5cf5cbdf21b Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 17 Sep 2024 22:23:09 +0800 Subject: objtool: Handle frame pointer related instructions After commit a0f7085f6a63 ("LoongArch: Add RANDOMIZE_KSTACK_OFFSET support"), there are three new instructions "addi.d $fp, $sp, 32", "sub.d $sp, $sp, $t0" and "addi.d $sp, $fp, -32" for the secondary stack in do_syscall(), then there is a objtool warning "return with modified stack frame" and no handle_syscall() which is the previous frame of do_syscall() in the call trace when executing the command "echo l > /proc/sysrq-trigger". objdump shows something like this: 0000000000000000 : 0: 02ff8063 addi.d $sp, $sp, -32 4: 29c04076 st.d $fp, $sp, 16 8: 29c02077 st.d $s0, $sp, 8 c: 29c06061 st.d $ra, $sp, 24 10: 02c08076 addi.d $fp, $sp, 32 ... 74: 0011b063 sub.d $sp, $sp, $t0 ... a8: 4c000181 jirl $ra, $t0, 0 ... dc: 02ff82c3 addi.d $sp, $fp, -32 e0: 28c06061 ld.d $ra, $sp, 24 e4: 28c04076 ld.d $fp, $sp, 16 e8: 28c02077 ld.d $s0, $sp, 8 ec: 02c08063 addi.d $sp, $sp, 32 f0: 4c000020 jirl $zero, $ra, 0 The instruction "sub.d $sp, $sp, $t0" changes the stack bottom and the new stack size is a random value, in order to find the return address of do_syscall() which is stored in the original stack frame after executing "jirl $ra, $t0, 0", it should use fp which points to the original stack top. At the beginning, the thought is tended to decode the secondary stack instruction "sub.d $sp, $sp, $t0" and set it as a label, then check this label for the two frame pointer instructions to change the cfa base and cfa offset during the period of secondary stack in update_cfi_state(). This is valid for GCC but invalid for Clang due to there are different secondary stack instructions for ClangBuiltLinux on LoongArch, something like this: 0000000000000000 : ... 88: 00119064 sub.d $a0, $sp, $a0 8c: 00150083 or $sp, $a0, $zero ... Actually, it equals to a single instruction "sub.d $sp, $sp, $a0", but there is no proper condition to check it as a label like GCC, and so the beginning thought is not a good way. Essentially, there are two special frame pointer instructions which are "addi.d $fp, $sp, imm" and "addi.d $sp, $fp, imm", the first one points fp to the original stack top and the second one restores the original stack bottom from fp. Based on the above analysis, in order to avoid adding an arch-specific update_cfi_state(), we just add a member "frame_pointer" in the "struct symbol" as a label to avoid affecting the current normal case, then set it as true only if there is "addi.d $sp, $fp, imm". The last is to check this label for the two frame pointer instructions to change the cfa base and cfa offset in update_cfi_state(). Tested with the following two configs: (1) CONFIG_RANDOMIZE_KSTACK_OFFSET=y && CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT=n (2) CONFIG_RANDOMIZE_KSTACK_OFFSET=y && CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT=y By the way, there is no effect for x86 with this patch, tested on the x86 machine with Fedora 40 system. Cc: stable@vger.kernel.org # 6.9+ Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- tools/objtool/arch/loongarch/decode.c | 11 ++++++++++- tools/objtool/check.c | 23 ++++++++++++++++++++--- tools/objtool/include/objtool/elf.h | 1 + 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/tools/objtool/arch/loongarch/decode.c b/tools/objtool/arch/loongarch/decode.c index aee479d2191c..69b66994f2a1 100644 --- a/tools/objtool/arch/loongarch/decode.c +++ b/tools/objtool/arch/loongarch/decode.c @@ -122,7 +122,7 @@ static bool decode_insn_reg2i12_fomat(union loongarch_instruction inst, switch (inst.reg2i12_format.opcode) { case addid_op: if ((inst.reg2i12_format.rd == CFI_SP) || (inst.reg2i12_format.rj == CFI_SP)) { - /* addi.d sp,sp,si12 or addi.d fp,sp,si12 */ + /* addi.d sp,sp,si12 or addi.d fp,sp,si12 or addi.d sp,fp,si12 */ insn->immediate = sign_extend64(inst.reg2i12_format.immediate, 11); ADD_OP(op) { op->src.type = OP_SRC_ADD; @@ -132,6 +132,15 @@ static bool decode_insn_reg2i12_fomat(union loongarch_instruction inst, op->dest.reg = inst.reg2i12_format.rd; } } + if ((inst.reg2i12_format.rd == CFI_SP) && (inst.reg2i12_format.rj == CFI_FP)) { + /* addi.d sp,fp,si12 */ + struct symbol *func = find_func_containing(insn->sec, insn->offset); + + if (!func) + return false; + + func->frame_pointer = true; + } break; case ldd_op: if (inst.reg2i12_format.rj == CFI_SP) { diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 01237d167223..af9cfed7f4ec 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2993,10 +2993,27 @@ static int update_cfi_state(struct instruction *insn, break; } - if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { + if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && + insn->sym->frame_pointer) { + /* addi.d fp,sp,imm on LoongArch */ + if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { + cfa->base = CFI_BP; + cfa->offset = 0; + } + break; + } - /* lea disp(%rbp), %rsp */ - cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); + if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { + /* addi.d sp,fp,imm on LoongArch */ + if (cfa->base == CFI_BP && cfa->offset == 0) { + if (insn->sym->frame_pointer) { + cfa->base = CFI_SP; + cfa->offset = -op->src.offset; + } + } else { + /* lea disp(%rbp), %rsp */ + cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); + } break; } diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index 2b8a69de4db8..d7e815c2fd15 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -68,6 +68,7 @@ struct symbol { u8 warned : 1; u8 embedded_insn : 1; u8 local_label : 1; + u8 frame_pointer : 1; struct list_head pv_target; struct reloc *relocs; }; -- cgit From b8468bd92ae19939d4844899fa05147888732519 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 17 Sep 2024 22:23:09 +0800 Subject: LoongArch: Enable objtool for Clang For now, it can enable objtool for Clang, just remove !CC_IS_CLANG for HAVE_OBJTOOL in arch/loongarch/Kconfig. Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 0e3abf7b0bd3..8dff6ddaf9c5 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -147,7 +147,7 @@ config LOONGARCH select HAVE_LIVEPATCH select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI - select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS && AS_HAS_THIN_ADD_SUB && !CC_IS_CLANG + select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS && AS_HAS_THIN_ADD_SUB select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS -- cgit From a7e0837724562ea8c1d869dd1a5cb1119ef651c3 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 17 Sep 2024 22:23:09 +0800 Subject: LoongArch: Set AS_HAS_THIN_ADD_SUB as y if AS_IS_LLVM When building kernel with "make CC=clang defconfig", LLVM Assembler is used due to LLVM_IAS=0 is not specified, then AS_HAS_THIN_ADD_SUB is not set, thus objtool can not be built after enable it for Clang. config AS_HAS_THIN_ADD_SUB is to check whether -mthin-add-sub option is available to know R_LARCH_{32,64}_PCREL are supported for GNU Assembler, there is no such an option for LLVM Assembler. The minimal version of Clang is 18 for building LoongArch kernel, and Clang >= 17 has already supported R_LARCH_{32,64}_PCREL, that is to say, there is no need to depend on AS_HAS_THIN_ADD_SUB for Clang, so just set AS_HAS_THIN_ADD_SUB as y if AS_IS_LLVM. Fixes: 120dd4118e58 ("LoongArch: Only allow OBJTOOL & ORC unwinder if toolchain supports -mthin-add-sub") Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 8dff6ddaf9c5..34531dc8a61f 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -266,7 +266,7 @@ config AS_HAS_FCSR_CLASS def_bool $(as-instr,movfcsr2gr \$t0$(comma)\$fcsr0) config AS_HAS_THIN_ADD_SUB - def_bool $(cc-option,-Wa$(comma)-mthin-add-sub) + def_bool $(cc-option,-Wa$(comma)-mthin-add-sub) || AS_IS_LLVM config AS_HAS_LSX_EXTENSION def_bool $(as-instr,vld \$vr0$(comma)\$a0$(comma)0) -- cgit From 0eb0bd21e8382d10be8108952a0bb819915e1e2d Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 17 Sep 2024 22:23:10 +0800 Subject: LoongArch: Remove STACK_FRAME_NON_STANDARD(do_syscall) For now, we can remove STACK_FRAME_NON_STANDARD(do_syscall) because there is no objtool warning "do_syscall+0x11c: return with modified stack frame", then there is handle_syscall() which is the previous frame of do_syscall() in the call trace when executing the command "echo l > /proc/sysrq-trigger". Fixes: a0f7085f6a63 ("LoongArch: Add RANDOMIZE_KSTACK_OFFSET support") Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/kernel/syscall.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/loongarch/kernel/syscall.c b/arch/loongarch/kernel/syscall.c index ba5d0930a74f..168bd97540f8 100644 --- a/arch/loongarch/kernel/syscall.c +++ b/arch/loongarch/kernel/syscall.c @@ -79,7 +79,3 @@ void noinstr __no_stack_protector do_syscall(struct pt_regs *regs) syscall_exit_to_user_mode(regs); } - -#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET -STACK_FRAME_NON_STANDARD(do_syscall); -#endif -- cgit From e8dd556c74325f29597665c2c557a8ea699a0686 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 24 Sep 2024 15:31:51 +0800 Subject: LoongArch: Enable generic CPU vulnerabilites support Currently, many architectures support generic CPU vulnerabilites, such as x86, arm64 and riscv: commit 61dc0f555b5c ("x86/cpu: Implement CPU vulnerabilites sysfs functions") commit 61ae1321f06c ("arm64: enable generic CPU vulnerabilites support") commit 0e3f3649d44b ("riscv: Enable generic CPU vulnerabilites support") All LoongArch CPUs (since Loongson-3A5000) implement a special mechanism in the processor core to prevent "Meltdown" and "Spectre" attacks, so it can enable generic CPU vulnerabilites support for LoongArch too. Without this patch, there are no user interfaces of vulnerabilities to check on LoongArch. The output of those files reflects the state of the CPUs in the system, the output value "Not affected" means "CPU is not affected by the vulnerability". Before: # cat /sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow cat: /sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow: No such file or directory # cat /sys/devices/system/cpu/vulnerabilities/spec_store_bypass cat: /sys/devices/system/cpu/vulnerabilities/spec_store_bypass: No such file or directory # cat /sys/devices/system/cpu/vulnerabilities/meltdown cat: /sys/devices/system/cpu/vulnerabilities/meltdown: No such file or directory After: # cat /sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow Not affected # cat /sys/devices/system/cpu/vulnerabilities/spec_store_bypass Not affected # cat /sys/devices/system/cpu/vulnerabilities/meltdown Not affected Link: https://www.loongson.cn/EN/news/show?id=633 Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 34531dc8a61f..b9d3c98efe18 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -82,6 +82,7 @@ config LOONGARCH select GENERIC_CMOS_UPDATE select GENERIC_CPU_AUTOPROBE select GENERIC_CPU_DEVICES + select GENERIC_CPU_VULNERABILITIES select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY select GENERIC_IOREMAP if !ARCH_IOREMAP -- cgit From d0bb0b600081bc7c246b97b2901abbb6c357ff7b Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 24 Sep 2024 15:32:06 +0800 Subject: LoongArch: Enable ACPI BGRT handling Add ACPI BGRT support on LoongArch so it can display image provied by acpi table at boot stage and switch to graphical UI smoothly. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/kernel/acpi.c | 4 ++++ drivers/acpi/Kconfig | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 929a497c987e..f1a74b80f22c 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -212,6 +213,9 @@ void __init acpi_boot_table_init(void) /* Do not enable ACPI SPCR console by default */ acpi_parse_spcr(earlycon_acpi_spcr_enable, false); + if (IS_ENABLED(CONFIG_ACPI_BGRT)) + acpi_table_parse(ACPI_SIG_BGRT, acpi_parse_bgrt); + return; fdt_earlycon: diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index e3a7c2aedd5f..d67f63d93b2a 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -451,7 +451,7 @@ config ACPI_HED config ACPI_BGRT bool "Boottime Graphics Resource Table support" - depends on EFI && (X86 || ARM64) + depends on EFI && (X86 || ARM64 || LOONGARCH) help This driver adds support for exposing the ACPI Boottime Graphics Resource Table, which allows the operating system to obtain -- cgit From 34e3c4500cdc06094b37a41b622598098308ba8f Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: LoongArch: Rework CPU feature probe from CPUCFG/IOCSR Probe ISA level, TLB, IOCSR information from CPUCFG to improve kernel resilience to different core implementations. BTW, IOCSR register definition appears to be a platform-specific spec instead of an architecture spec, even for the Loongson CPUs there is no guarantee that IOCSR will always present. Thus it's dangerous to perform IOCSR probing without checking CPU type and instruction availability. Signed-off-by: Jiaxun Yang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/cpu-features.h | 2 + arch/loongarch/include/asm/cpu.h | 30 ++++---- arch/loongarch/include/asm/loongarch.h | 1 + arch/loongarch/include/uapi/asm/hwcap.h | 1 + arch/loongarch/kernel/cpu-probe.c | 120 ++++++++++++++++++------------ arch/loongarch/kernel/proc.c | 10 ++- 6 files changed, 100 insertions(+), 64 deletions(-) diff --git a/arch/loongarch/include/asm/cpu-features.h b/arch/loongarch/include/asm/cpu-features.h index 16a716f88a5c..fc83bb32f9f0 100644 --- a/arch/loongarch/include/asm/cpu-features.h +++ b/arch/loongarch/include/asm/cpu-features.h @@ -51,6 +51,7 @@ #define cpu_has_lbt_mips cpu_opt(LOONGARCH_CPU_LBT_MIPS) #define cpu_has_lbt (cpu_has_lbt_x86|cpu_has_lbt_arm|cpu_has_lbt_mips) #define cpu_has_csr cpu_opt(LOONGARCH_CPU_CSR) +#define cpu_has_iocsr cpu_opt(LOONGARCH_CPU_IOCSR) #define cpu_has_tlb cpu_opt(LOONGARCH_CPU_TLB) #define cpu_has_watch cpu_opt(LOONGARCH_CPU_WATCH) #define cpu_has_vint cpu_opt(LOONGARCH_CPU_VINT) @@ -65,6 +66,7 @@ #define cpu_has_guestid cpu_opt(LOONGARCH_CPU_GUESTID) #define cpu_has_hypervisor cpu_opt(LOONGARCH_CPU_HYPERVISOR) #define cpu_has_ptw cpu_opt(LOONGARCH_CPU_PTW) +#define cpu_has_lspw cpu_opt(LOONGARCH_CPU_LSPW) #define cpu_has_avecint cpu_opt(LOONGARCH_CPU_AVECINT) #endif /* __ASM_CPU_FEATURES_H */ diff --git a/arch/loongarch/include/asm/cpu.h b/arch/loongarch/include/asm/cpu.h index 843f9c4ec980..98cf4d7b4b0a 100644 --- a/arch/loongarch/include/asm/cpu.h +++ b/arch/loongarch/include/asm/cpu.h @@ -87,19 +87,21 @@ enum cpu_type_enum { #define CPU_FEATURE_LBT_MIPS 12 /* CPU has MIPS Binary Translation */ #define CPU_FEATURE_TLB 13 /* CPU has TLB */ #define CPU_FEATURE_CSR 14 /* CPU has CSR */ -#define CPU_FEATURE_WATCH 15 /* CPU has watchpoint registers */ -#define CPU_FEATURE_VINT 16 /* CPU has vectored interrupts */ -#define CPU_FEATURE_CSRIPI 17 /* CPU has CSR-IPI */ -#define CPU_FEATURE_EXTIOI 18 /* CPU has EXT-IOI */ -#define CPU_FEATURE_PREFETCH 19 /* CPU has prefetch instructions */ -#define CPU_FEATURE_PMP 20 /* CPU has perfermance counter */ -#define CPU_FEATURE_SCALEFREQ 21 /* CPU supports cpufreq scaling */ -#define CPU_FEATURE_FLATMODE 22 /* CPU has flat mode */ -#define CPU_FEATURE_EIODECODE 23 /* CPU has EXTIOI interrupt pin decode mode */ -#define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */ -#define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */ -#define CPU_FEATURE_PTW 26 /* CPU has hardware page table walker */ -#define CPU_FEATURE_AVECINT 27 /* CPU has avec interrupt */ +#define CPU_FEATURE_IOCSR 15 /* CPU has IOCSR */ +#define CPU_FEATURE_WATCH 16 /* CPU has watchpoint registers */ +#define CPU_FEATURE_VINT 17 /* CPU has vectored interrupts */ +#define CPU_FEATURE_CSRIPI 18 /* CPU has CSR-IPI */ +#define CPU_FEATURE_EXTIOI 19 /* CPU has EXT-IOI */ +#define CPU_FEATURE_PREFETCH 20 /* CPU has prefetch instructions */ +#define CPU_FEATURE_PMP 21 /* CPU has perfermance counter */ +#define CPU_FEATURE_SCALEFREQ 22 /* CPU supports cpufreq scaling */ +#define CPU_FEATURE_FLATMODE 23 /* CPU has flat mode */ +#define CPU_FEATURE_EIODECODE 24 /* CPU has EXTIOI interrupt pin decode mode */ +#define CPU_FEATURE_GUESTID 25 /* CPU has GuestID feature */ +#define CPU_FEATURE_HYPERVISOR 26 /* CPU has hypervisor (running in VM) */ +#define CPU_FEATURE_PTW 27 /* CPU has hardware page table walker */ +#define CPU_FEATURE_LSPW 28 /* CPU has LSPW (lddir/ldpte instructions) */ +#define CPU_FEATURE_AVECINT 29 /* CPU has AVEC interrupt */ #define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG) #define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM) @@ -115,6 +117,7 @@ enum cpu_type_enum { #define LOONGARCH_CPU_LBT_ARM BIT_ULL(CPU_FEATURE_LBT_ARM) #define LOONGARCH_CPU_LBT_MIPS BIT_ULL(CPU_FEATURE_LBT_MIPS) #define LOONGARCH_CPU_TLB BIT_ULL(CPU_FEATURE_TLB) +#define LOONGARCH_CPU_IOCSR BIT_ULL(CPU_FEATURE_IOCSR) #define LOONGARCH_CPU_CSR BIT_ULL(CPU_FEATURE_CSR) #define LOONGARCH_CPU_WATCH BIT_ULL(CPU_FEATURE_WATCH) #define LOONGARCH_CPU_VINT BIT_ULL(CPU_FEATURE_VINT) @@ -128,6 +131,7 @@ enum cpu_type_enum { #define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID) #define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR) #define LOONGARCH_CPU_PTW BIT_ULL(CPU_FEATURE_PTW) +#define LOONGARCH_CPU_LSPW BIT_ULL(CPU_FEATURE_LSPW) #define LOONGARCH_CPU_AVECINT BIT_ULL(CPU_FEATURE_AVECINT) #endif /* _ASM_CPU_H */ diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 631d249b3ef2..4084cb64ba06 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -62,6 +62,7 @@ #define LOONGARCH_CPUCFG1 0x1 #define CPUCFG1_ISGR32 BIT(0) #define CPUCFG1_ISGR64 BIT(1) +#define CPUCFG1_ISA GENMASK(1, 0) #define CPUCFG1_PAGING BIT(2) #define CPUCFG1_IOCSR BIT(3) #define CPUCFG1_PABITS GENMASK(11, 4) diff --git a/arch/loongarch/include/uapi/asm/hwcap.h b/arch/loongarch/include/uapi/asm/hwcap.h index 6955a7cb2c65..2b34e56cfa9e 100644 --- a/arch/loongarch/include/uapi/asm/hwcap.h +++ b/arch/loongarch/include/uapi/asm/hwcap.h @@ -17,5 +17,6 @@ #define HWCAP_LOONGARCH_LBT_ARM (1 << 11) #define HWCAP_LOONGARCH_LBT_MIPS (1 << 12) #define HWCAP_LOONGARCH_PTW (1 << 13) +#define HWCAP_LOONGARCH_LSPW (1 << 14) #endif /* _UAPI_ASM_HWCAP_H */ diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 14f0449f5452..cbce099037b2 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c @@ -91,12 +91,30 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) unsigned int config; unsigned long asid_mask; - c->options = LOONGARCH_CPU_CPUCFG | LOONGARCH_CPU_CSR | - LOONGARCH_CPU_TLB | LOONGARCH_CPU_VINT | LOONGARCH_CPU_WATCH; + c->options = LOONGARCH_CPU_CPUCFG | LOONGARCH_CPU_CSR | LOONGARCH_CPU_VINT; elf_hwcap = HWCAP_LOONGARCH_CPUCFG; config = read_cpucfg(LOONGARCH_CPUCFG1); + + switch (config & CPUCFG1_ISA) { + case 0: + set_isa(c, LOONGARCH_CPU_ISA_LA32R); + break; + case 1: + set_isa(c, LOONGARCH_CPU_ISA_LA32S); + break; + case 2: + set_isa(c, LOONGARCH_CPU_ISA_LA64); + break; + default: + pr_warn("Warning: unknown ISA level\n"); + } + + if (config & CPUCFG1_PAGING) + c->options |= LOONGARCH_CPU_TLB; + if (config & CPUCFG1_IOCSR) + c->options |= LOONGARCH_CPU_IOCSR; if (config & CPUCFG1_UAL) { c->options |= LOONGARCH_CPU_UAL; elf_hwcap |= HWCAP_LOONGARCH_UAL; @@ -139,6 +157,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) c->options |= LOONGARCH_CPU_PTW; elf_hwcap |= HWCAP_LOONGARCH_PTW; } + if (config & CPUCFG2_LSPW) { + c->options |= LOONGARCH_CPU_LSPW; + elf_hwcap |= HWCAP_LOONGARCH_LSPW; + } if (config & CPUCFG2_LVZP) { c->options |= LOONGARCH_CPU_LVZ; elf_hwcap |= HWCAP_LOONGARCH_LVZ; @@ -162,22 +184,6 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) if (config & CPUCFG6_PMP) c->options |= LOONGARCH_CPU_PMP; - config = iocsr_read32(LOONGARCH_IOCSR_FEATURES); - if (config & IOCSRF_CSRIPI) - c->options |= LOONGARCH_CPU_CSRIPI; - if (config & IOCSRF_EXTIOI) - c->options |= LOONGARCH_CPU_EXTIOI; - if (config & IOCSRF_FREQSCALE) - c->options |= LOONGARCH_CPU_SCALEFREQ; - if (config & IOCSRF_FLATMODE) - c->options |= LOONGARCH_CPU_FLATMODE; - if (config & IOCSRF_EIODECODE) - c->options |= LOONGARCH_CPU_EIODECODE; - if (config & IOCSRF_AVEC) - c->options |= LOONGARCH_CPU_AVECINT; - if (config & IOCSRF_VM) - c->options |= LOONGARCH_CPU_HYPERVISOR; - config = csr_read32(LOONGARCH_CSR_ASID); config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT; asid_mask = GENMASK(config - 1, 0); @@ -210,6 +216,9 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) default: pr_warn("Warning: unknown TLB type\n"); } + + if (get_num_brps() + get_num_wrps()) + c->options |= LOONGARCH_CPU_WATCH; } #define MAX_NAME_LEN 32 @@ -220,52 +229,67 @@ static char cpu_full_name[MAX_NAME_LEN] = " - "; static inline void cpu_probe_loongson(struct cpuinfo_loongarch *c, unsigned int cpu) { + uint32_t config; uint64_t *vendor = (void *)(&cpu_full_name[VENDOR_OFFSET]); uint64_t *cpuname = (void *)(&cpu_full_name[CPUNAME_OFFSET]); + const char *core_name = "Unknown"; - if (!__cpu_full_name[cpu]) - __cpu_full_name[cpu] = cpu_full_name; - - *vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR); - *cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME); - - switch (c->processor_id & PRID_SERIES_MASK) { - case PRID_SERIES_LA132: + switch (BIT(fls(c->isa_level) - 1)) { + case LOONGARCH_CPU_ISA_LA32R: + case LOONGARCH_CPU_ISA_LA32S: c->cputype = CPU_LOONGSON32; - set_isa(c, LOONGARCH_CPU_ISA_LA32S); __cpu_family[cpu] = "Loongson-32bit"; - pr_info("32-bit Loongson Processor probed (LA132 Core)\n"); break; - case PRID_SERIES_LA264: + case LOONGARCH_CPU_ISA_LA64: c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (LA264 Core)\n"); + break; + } + + switch (c->processor_id & PRID_SERIES_MASK) { + case PRID_SERIES_LA132: + core_name = "LA132"; + break; + case PRID_SERIES_LA264: + core_name = "LA264"; break; case PRID_SERIES_LA364: - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (LA364 Core)\n"); + core_name = "LA364"; break; case PRID_SERIES_LA464: - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (LA464 Core)\n"); + core_name = "LA464"; break; case PRID_SERIES_LA664: - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (LA664 Core)\n"); + core_name = "LA664"; break; - default: /* Default to 64 bit */ - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (Unknown Core)\n"); } + + pr_info("%s Processor probed (%s Core)\n", __cpu_family[cpu], core_name); + + if (!cpu_has_iocsr) + return; + + if (!__cpu_full_name[cpu]) + __cpu_full_name[cpu] = cpu_full_name; + + *vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR); + *cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME); + + config = iocsr_read32(LOONGARCH_IOCSR_FEATURES); + if (config & IOCSRF_CSRIPI) + c->options |= LOONGARCH_CPU_CSRIPI; + if (config & IOCSRF_EXTIOI) + c->options |= LOONGARCH_CPU_EXTIOI; + if (config & IOCSRF_FREQSCALE) + c->options |= LOONGARCH_CPU_SCALEFREQ; + if (config & IOCSRF_FLATMODE) + c->options |= LOONGARCH_CPU_FLATMODE; + if (config & IOCSRF_EIODECODE) + c->options |= LOONGARCH_CPU_EIODECODE; + if (config & IOCSRF_AVEC) + c->options |= LOONGARCH_CPU_AVECINT; + if (config & IOCSRF_VM) + c->options |= LOONGARCH_CPU_HYPERVISOR; } #ifdef CONFIG_64BIT diff --git a/arch/loongarch/kernel/proc.c b/arch/loongarch/kernel/proc.c index 0d33cbc47e51..6ce46d92f1f1 100644 --- a/arch/loongarch/kernel/proc.c +++ b/arch/loongarch/kernel/proc.c @@ -31,6 +31,7 @@ int proc_cpuinfo_notifier_call_chain(unsigned long val, void *v) static int show_cpuinfo(struct seq_file *m, void *v) { unsigned long n = (unsigned long) v - 1; + unsigned int isa = cpu_data[n].isa_level; unsigned int version = cpu_data[n].processor_id & 0xff; unsigned int fp_version = cpu_data[n].fpu_vers; struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args; @@ -64,9 +65,11 @@ static int show_cpuinfo(struct seq_file *m, void *v) cpu_pabits + 1, cpu_vabits + 1); seq_printf(m, "ISA\t\t\t:"); - if (cpu_has_loongarch32) - seq_printf(m, " loongarch32"); - if (cpu_has_loongarch64) + if (isa & LOONGARCH_CPU_ISA_LA32R) + seq_printf(m, " loongarch32r"); + if (isa & LOONGARCH_CPU_ISA_LA32S) + seq_printf(m, " loongarch32s"); + if (isa & LOONGARCH_CPU_ISA_LA64) seq_printf(m, " loongarch64"); seq_printf(m, "\n"); @@ -81,6 +84,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) if (cpu_has_complex) seq_printf(m, " complex"); if (cpu_has_crypto) seq_printf(m, " crypto"); if (cpu_has_ptw) seq_printf(m, " ptw"); + if (cpu_has_lspw) seq_printf(m, " lspw"); if (cpu_has_lvz) seq_printf(m, " lvz"); if (cpu_has_lbt_x86) seq_printf(m, " lbt_x86"); if (cpu_has_lbt_arm) seq_printf(m, " lbt_arm"); -- cgit From e86935f705fa732d8c7c3ecf0c50ea461ffab76f Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: LoongArch: Add ARCH_HAS_SET_MEMORY support Add set_memory_ro/rw/x/nx architecture hooks to change the page attribution. Use own set_memory.h rather than generic set_memory.h (i.e. include/asm-generic/set_memory.h), because we want to add other function prototypes here. Note: We can only set attributes for KVRANGE/XKVRANGE kernel addresses. Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/set_memory.h | 17 ++++ arch/loongarch/mm/Makefile | 3 +- arch/loongarch/mm/pageattr.c | 158 ++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 arch/loongarch/include/asm/set_memory.h create mode 100644 arch/loongarch/mm/pageattr.c diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index b9d3c98efe18..46fa58bde685 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -25,6 +25,7 @@ config LOONGARCH select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE select ARCH_HAS_PTE_DEVMAP select ARCH_HAS_PTE_SPECIAL + select ARCH_HAS_SET_MEMORY select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_INLINE_READ_LOCK if !PREEMPTION select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h new file mode 100644 index 000000000000..64c7f942e8ec --- /dev/null +++ b/arch/loongarch/include/asm/set_memory.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_LOONGARCH_SET_MEMORY_H +#define _ASM_LOONGARCH_SET_MEMORY_H + +/* + * Functions to change memory attributes. + */ +int set_memory_x(unsigned long addr, int numpages); +int set_memory_nx(unsigned long addr, int numpages); +int set_memory_ro(unsigned long addr, int numpages); +int set_memory_rw(unsigned long addr, int numpages); + +#endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/Makefile b/arch/loongarch/mm/Makefile index e4d1e581dbae..278be2c8fc36 100644 --- a/arch/loongarch/mm/Makefile +++ b/arch/loongarch/mm/Makefile @@ -4,7 +4,8 @@ # obj-y += init.o cache.o tlb.o tlbex.o extable.o \ - fault.o ioremap.o maccess.o mmap.o pgtable.o page.o + fault.o ioremap.o maccess.o mmap.o pgtable.o \ + page.o pageattr.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c new file mode 100644 index 000000000000..e3db1c38b5fe --- /dev/null +++ b/arch/loongarch/mm/pageattr.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include + +struct pageattr_masks { + pgprot_t set_mask; + pgprot_t clear_mask; +}; + +static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk) +{ + unsigned long new_val = val; + struct pageattr_masks *masks = walk->private; + + new_val &= ~(pgprot_val(masks->clear_mask)); + new_val |= (pgprot_val(masks->set_mask)); + + return new_val; +} + +static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pgd_t val = pgdp_get(pgd); + + if (pgd_leaf(val)) { + val = __pgd(set_pageattr_masks(pgd_val(val), walk)); + set_pgd(pgd, val); + } + + return 0; +} + +static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + p4d_t val = p4dp_get(p4d); + + if (p4d_leaf(val)) { + val = __p4d(set_pageattr_masks(p4d_val(val), walk)); + set_p4d(p4d, val); + } + + return 0; +} + +static int pageattr_pud_entry(pud_t *pud, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pud_t val = pudp_get(pud); + + if (pud_leaf(val)) { + val = __pud(set_pageattr_masks(pud_val(val), walk)); + set_pud(pud, val); + } + + return 0; +} + +static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pmd_t val = pmdp_get(pmd); + + if (pmd_leaf(val)) { + val = __pmd(set_pageattr_masks(pmd_val(val), walk)); + set_pmd(pmd, val); + } + + return 0; +} + +static int pageattr_pte_entry(pte_t *pte, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t val = ptep_get(pte); + + val = __pte(set_pageattr_masks(pte_val(val), walk)); + set_pte(pte, val); + + return 0; +} + +static int pageattr_pte_hole(unsigned long addr, unsigned long next, + int depth, struct mm_walk *walk) +{ + return 0; +} + +static const struct mm_walk_ops pageattr_ops = { + .pgd_entry = pageattr_pgd_entry, + .p4d_entry = pageattr_p4d_entry, + .pud_entry = pageattr_pud_entry, + .pmd_entry = pageattr_pmd_entry, + .pte_entry = pageattr_pte_entry, + .pte_hole = pageattr_pte_hole, + .walk_lock = PGWALK_RDLOCK, +}; + +static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, pgprot_t clear_mask) +{ + int ret; + unsigned long start = addr; + unsigned long end = start + PAGE_SIZE * numpages; + struct pageattr_masks masks = { + .set_mask = set_mask, + .clear_mask = clear_mask + }; + + if (!numpages) + return 0; + + mmap_write_lock(&init_mm); + ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, &masks); + mmap_write_unlock(&init_mm); + + flush_tlb_kernel_range(start, end); + + return ret; +} + +int set_memory_x(unsigned long addr, int numpages) +{ + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_NO_EXEC)); +} + +int set_memory_nx(unsigned long addr, int numpages) +{ + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, numpages, __pgprot(_PAGE_NO_EXEC), __pgprot(0)); +} + +int set_memory_ro(unsigned long addr, int numpages) +{ + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_WRITE | _PAGE_DIRTY)); +} + +int set_memory_rw(unsigned long addr, int numpages) +{ + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), __pgprot(0)); +} -- cgit From f04de6d8f252ec6434b846895474cc205527b8b8 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: LoongArch: Add ARCH_HAS_SET_DIRECT_MAP support Add set_direct_map_*() functions for setting the direct map alias for the page to its default permissions and to an invalid state that cannot be cached in a TLB. (See d253ca0c3 ("x86/mm/cpa: Add set_direct_map_*() functions")) Add a similar implementation for LoongArch. This fixes the KFENCE warnings during hibernation: ================================================================== BUG: KFENCE: invalid read in swsusp_save+0x368/0x4d8 Invalid read at 0x00000000f7b89a3c: swsusp_save+0x368/0x4d8 hibernation_snapshot+0x3f0/0x4e0 hibernate+0x20c/0x440 state_store+0x128/0x140 kernfs_fop_write_iter+0x160/0x260 vfs_write+0x2c0/0x520 ksys_write+0x74/0x160 do_syscall+0xb0/0x160 CPU: 0 UID: 0 PID: 812 Comm: bash Tainted: G B 6.11.0-rc1+ #1566 Tainted: [B]=BAD_PAGE Hardware name: Loongson-LS3A5000-7A1000-1w-CRB, BIOS vUDK2018-LoongArch-V2.0.0 10/21/2022 ================================================================== Note: We can only set permissions for KVRANGE/XKVRANGE kernel addresses. Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/set_memory.h | 4 +++ arch/loongarch/mm/pageattr.c | 60 +++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 46fa58bde685..4b02b0492c3b 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -26,6 +26,7 @@ config LOONGARCH select ARCH_HAS_PTE_DEVMAP select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_MEMORY + select ARCH_HAS_SET_DIRECT_MAP select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_INLINE_READ_LOCK if !PREEMPTION select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h index 64c7f942e8ec..d70505b6676c 100644 --- a/arch/loongarch/include/asm/set_memory.h +++ b/arch/loongarch/include/asm/set_memory.h @@ -14,4 +14,8 @@ int set_memory_nx(unsigned long addr, int numpages); int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); +bool kernel_page_present(struct page *page); +int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page); + #endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index e3db1c38b5fe..ffd8d76021d4 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -156,3 +156,63 @@ int set_memory_rw(unsigned long addr, int numpages) return __set_memory(addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), __pgprot(0)); } + +bool kernel_page_present(struct page *page) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + unsigned long addr = (unsigned long)page_address(page); + + if (addr < vm_map_base) + return true; + + pgd = pgd_offset_k(addr); + if (pgd_none(pgdp_get(pgd))) + return false; + if (pgd_leaf(pgdp_get(pgd))) + return true; + + p4d = p4d_offset(pgd, addr); + if (p4d_none(p4dp_get(p4d))) + return false; + if (p4d_leaf(p4dp_get(p4d))) + return true; + + pud = pud_offset(p4d, addr); + if (pud_none(pudp_get(pud))) + return false; + if (pud_leaf(pudp_get(pud))) + return true; + + pmd = pmd_offset(pud, addr); + if (pmd_none(pmdp_get(pmd))) + return false; + if (pmd_leaf(pmdp_get(pmd))) + return true; + + pte = pte_offset_kernel(pmd, addr); + return pte_present(ptep_get(pte)); +} + +int set_direct_map_default_noflush(struct page *page) +{ + unsigned long addr = (unsigned long)page_address(page); + + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0)); +} + +int set_direct_map_invalid_noflush(struct page *page) +{ + unsigned long addr = (unsigned long)page_address(page); + + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); +} -- cgit From f93f67d06b1023313ef1662eac490e29c025c030 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: LoongArch: Improve hardware page table walker LoongArch has similar problems explained in commit 7f0b1bf04511348995d6 ("arm64: Fix barriers used for page table modifications"), when hardware page table walker (PTW) enabled, speculative accesses may cause spurious page fault in kernel space. Theoretically, in order to completely avoid spurious page fault we need a "dbar + ibar" pair between the page table modifications and the subsequent memory accesses using the corresponding virtual address. But "ibar" is too heavy for performace, so we only use a "dbar 0b11000" in set_pte(). And let spurious_fault() filter the rest rare spurious page faults which should be avoided by "ibar". Besides, we replace the llsc loop with amo in set_pte() which has better performace, and refactor mmu_context.h to 1) avoid any load/store/branch instructions between the writing of CSR.ASID & CSR.PGDL, 2) ensure flush tlb operation is after updating ASID. Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/atomic.h | 2 ++ arch/loongarch/include/asm/mmu_context.h | 35 ++++++++++++++++++++------- arch/loongarch/include/asm/pgtable.h | 32 ++++++++++--------------- arch/loongarch/mm/fault.c | 41 ++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 27 deletions(-) diff --git a/arch/loongarch/include/asm/atomic.h b/arch/loongarch/include/asm/atomic.h index 99af8b3160a8..c86f0ab922ec 100644 --- a/arch/loongarch/include/asm/atomic.h +++ b/arch/loongarch/include/asm/atomic.h @@ -15,6 +15,7 @@ #define __LL "ll.w " #define __SC "sc.w " #define __AMADD "amadd.w " +#define __AMOR "amor.w " #define __AMAND_DB "amand_db.w " #define __AMOR_DB "amor_db.w " #define __AMXOR_DB "amxor_db.w " @@ -22,6 +23,7 @@ #define __LL "ll.d " #define __SC "sc.d " #define __AMADD "amadd.d " +#define __AMOR "amor.d " #define __AMAND_DB "amand_db.d " #define __AMOR_DB "amor_db.d " #define __AMXOR_DB "amxor_db.d " diff --git a/arch/loongarch/include/asm/mmu_context.h b/arch/loongarch/include/asm/mmu_context.h index 9f97c3453b9c..304363bd3935 100644 --- a/arch/loongarch/include/asm/mmu_context.h +++ b/arch/loongarch/include/asm/mmu_context.h @@ -49,12 +49,12 @@ static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) /* Normal, classic get_new_mmu_context */ static inline void -get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) +get_new_mmu_context(struct mm_struct *mm, unsigned long cpu, bool *need_flush) { u64 asid = asid_cache(cpu); if (!((++asid) & cpu_asid_mask(&cpu_data[cpu]))) - local_flush_tlb_user(); /* start new asid cycle */ + *need_flush = true; /* start new asid cycle */ cpu_context(cpu, mm) = asid_cache(cpu) = asid; } @@ -74,21 +74,34 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) return 0; } +static inline void atomic_update_pgd_asid(unsigned long asid, unsigned long pgdl) +{ + __asm__ __volatile__( + "csrwr %[pgdl_val], %[pgdl_reg] \n\t" + "csrwr %[asid_val], %[asid_reg] \n\t" + : [asid_val] "+r" (asid), [pgdl_val] "+r" (pgdl) + : [asid_reg] "i" (LOONGARCH_CSR_ASID), [pgdl_reg] "i" (LOONGARCH_CSR_PGDL) + : "memory" + ); +} + static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { + bool need_flush = false; unsigned int cpu = smp_processor_id(); /* Check if our ASID is of an older version and thus invalid */ if (!asid_valid(next, cpu)) - get_new_mmu_context(next, cpu); - - write_csr_asid(cpu_asid(cpu, next)); + get_new_mmu_context(next, cpu, &need_flush); if (next != &init_mm) - csr_write64((unsigned long)next->pgd, LOONGARCH_CSR_PGDL); + atomic_update_pgd_asid(cpu_asid(cpu, next), (unsigned long)next->pgd); else - csr_write64((unsigned long)invalid_pg_dir, LOONGARCH_CSR_PGDL); + atomic_update_pgd_asid(cpu_asid(cpu, next), (unsigned long)invalid_pg_dir); + + if (need_flush) + local_flush_tlb_user(); /* Flush tlb after update ASID */ /* * Mark current->active_mm as not "active" anymore. @@ -135,9 +148,15 @@ drop_mmu_context(struct mm_struct *mm, unsigned int cpu) asid = read_csr_asid() & cpu_asid_mask(¤t_cpu_data); if (asid == cpu_asid(cpu, mm)) { + bool need_flush = false; + if (!current->mm || (current->mm == mm)) { - get_new_mmu_context(mm, cpu); + get_new_mmu_context(mm, cpu, &need_flush); + write_csr_asid(cpu_asid(cpu, mm)); + if (need_flush) + local_flush_tlb_user(); /* Flush tlb after update ASID */ + goto out; } } diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 85431f20a14d..9965f52ef65b 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -331,29 +331,23 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) * Make sure the buddy is global too (if it's !none, * it better already be global) */ + if (pte_none(ptep_get(buddy))) { #ifdef CONFIG_SMP - /* - * For SMP, multiple CPUs can race, so we need to do - * this atomically. - */ - unsigned long page_global = _PAGE_GLOBAL; - unsigned long tmp; - - __asm__ __volatile__ ( - "1:" __LL "%[tmp], %[buddy] \n" - " bnez %[tmp], 2f \n" - " or %[tmp], %[tmp], %[global] \n" - __SC "%[tmp], %[buddy] \n" - " beqz %[tmp], 1b \n" - " nop \n" - "2: \n" - __WEAK_LLSC_MB - : [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp) - : [global] "r" (page_global)); + /* + * For SMP, multiple CPUs can race, so we need + * to do this atomically. + */ + __asm__ __volatile__( + __AMOR "$zero, %[global], %[buddy] \n" + : [buddy] "+ZB" (buddy->pte) + : [global] "r" (_PAGE_GLOBAL) + : "memory"); + + DBAR(0b11000); /* o_wrw = 0b11000 */ #else /* !CONFIG_SMP */ - if (pte_none(ptep_get(buddy))) WRITE_ONCE(*buddy, __pte(pte_val(ptep_get(buddy)) | _PAGE_GLOBAL)); #endif /* CONFIG_SMP */ + } } } diff --git a/arch/loongarch/mm/fault.c b/arch/loongarch/mm/fault.c index 97b40defde06..deefd9617d00 100644 --- a/arch/loongarch/mm/fault.c +++ b/arch/loongarch/mm/fault.c @@ -31,11 +31,52 @@ int show_unhandled_signals = 1; +static int __kprobes spurious_fault(unsigned long write, unsigned long address) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + if (!(address & __UA_LIMIT)) + return 0; + + pgd = pgd_offset_k(address); + if (!pgd_present(pgdp_get(pgd))) + return 0; + + p4d = p4d_offset(pgd, address); + if (!p4d_present(p4dp_get(p4d))) + return 0; + + pud = pud_offset(p4d, address); + if (!pud_present(pudp_get(pud))) + return 0; + + pmd = pmd_offset(pud, address); + if (!pmd_present(pmdp_get(pmd))) + return 0; + + if (pmd_leaf(*pmd)) { + return write ? pmd_write(pmdp_get(pmd)) : 1; + } else { + pte = pte_offset_kernel(pmd, address); + if (!pte_present(ptep_get(pte))) + return 0; + + return write ? pte_write(ptep_get(pte)) : 1; + } +} + static void __kprobes no_context(struct pt_regs *regs, unsigned long write, unsigned long address) { const int field = sizeof(unsigned long) * 2; + if (spurious_fault(write, address)) + return; + /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; -- cgit From d4f31acf1302088a5b16d1e4de890729acfa9638 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: LoongArch: Simplify _percpu_read() and _percpu_write() Now _percpu_read() and _percpu_write() macros call __percpu_read() and __percpu_write() static inline functions that result in a single assembly instruction. However, percpu infrastructure expects its leaf definitions to encode the size of their percpu variable, so the patch merges all the asm clauses from the static inline function into the corresponding leaf macros. The secondary effect of this change is to avoid explicit __percpu annotations for function arguments. Currently, __percpu macro is defined in include/linux/compiler_types.h, but with proposed patch [1], __percpu definition will need macros from include/asm-generic/percpu.h, creating forward dependency loop. The proposed solution is the same as x86 architecture uses. [1] https://lore.kernel.org/lkml/20240812115945.484051-4-ubizjak@gmail.com/ Tested-by: Xi Ruoyao Signed-off-by: Uros Bizjak Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/percpu.h | 124 ++++++++++-------------------------- 1 file changed, 35 insertions(+), 89 deletions(-) diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h index 8f290e5546cf..87be9b14e9da 100644 --- a/arch/loongarch/include/asm/percpu.h +++ b/arch/loongarch/include/asm/percpu.h @@ -68,75 +68,6 @@ PERCPU_OP(and, and, &) PERCPU_OP(or, or, |) #undef PERCPU_OP -static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size) -{ - unsigned long ret; - - switch (size) { - case 1: - __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n" - : [ret] "=&r"(ret) - : [ptr] "r"(ptr) - : "memory"); - break; - case 2: - __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n" - : [ret] "=&r"(ret) - : [ptr] "r"(ptr) - : "memory"); - break; - case 4: - __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n" - : [ret] "=&r"(ret) - : [ptr] "r"(ptr) - : "memory"); - break; - case 8: - __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n" - : [ret] "=&r"(ret) - : [ptr] "r"(ptr) - : "memory"); - break; - default: - ret = 0; - BUILD_BUG(); - } - - return ret; -} - -static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size) -{ - switch (size) { - case 1: - __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n" - : - : [val] "r" (val), [ptr] "r" (ptr) - : "memory"); - break; - case 2: - __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n" - : - : [val] "r" (val), [ptr] "r" (ptr) - : "memory"); - break; - case 4: - __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n" - : - : [val] "r" (val), [ptr] "r" (ptr) - : "memory"); - break; - case 8: - __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n" - : - : [val] "r" (val), [ptr] "r" (ptr) - : "memory"); - break; - default: - BUILD_BUG(); - } -} - static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size) { switch (size) { @@ -157,6 +88,33 @@ static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, return 0; } +#define __pcpu_op_1(op) op ".b " +#define __pcpu_op_2(op) op ".h " +#define __pcpu_op_4(op) op ".w " +#define __pcpu_op_8(op) op ".d " + +#define _percpu_read(size, _pcp) \ +({ \ + typeof(_pcp) __pcp_ret; \ + \ + __asm__ __volatile__( \ + __pcpu_op_##size("ldx") "%[ret], $r21, %[ptr] \n" \ + : [ret] "=&r"(__pcp_ret) \ + : [ptr] "r"(&(_pcp)) \ + : "memory"); \ + \ + __pcp_ret; \ +}) + +#define _percpu_write(size, _pcp, _val) \ +do { \ + __asm__ __volatile__( \ + __pcpu_op_##size("stx") "%[val], $r21, %[ptr] \n" \ + : \ + : [val] "r"(_val), [ptr] "r"(&(_pcp)) \ + : "memory"); \ +} while (0) + /* this_cpu_cmpxchg */ #define _protect_cmpxchg_local(pcp, o, n) \ ({ \ @@ -167,18 +125,6 @@ static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, __ret; \ }) -#define _percpu_read(pcp) \ -({ \ - typeof(pcp) __retval; \ - __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \ - __retval; \ -}) - -#define _percpu_write(pcp, val) \ -do { \ - __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \ -} while (0) \ - #define _pcp_protect(operation, pcp, val) \ ({ \ typeof(pcp) __retval; \ @@ -215,15 +161,15 @@ do { \ #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) -#define this_cpu_read_1(pcp) _percpu_read(pcp) -#define this_cpu_read_2(pcp) _percpu_read(pcp) -#define this_cpu_read_4(pcp) _percpu_read(pcp) -#define this_cpu_read_8(pcp) _percpu_read(pcp) +#define this_cpu_read_1(pcp) _percpu_read(1, pcp) +#define this_cpu_read_2(pcp) _percpu_read(2, pcp) +#define this_cpu_read_4(pcp) _percpu_read(4, pcp) +#define this_cpu_read_8(pcp) _percpu_read(8, pcp) -#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) -#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) -#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) -#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_1(pcp, val) _percpu_write(1, pcp, val) +#define this_cpu_write_2(pcp, val) _percpu_write(2, pcp, val) +#define this_cpu_write_4(pcp, val) _percpu_write(4, pcp, val) +#define this_cpu_write_8(pcp, val) _percpu_write(8, pcp, val) #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) -- cgit From 5016c3a31a6d74eaf2fdfdec673eae8fcf90379e Mon Sep 17 00:00:00 2001 From: Wentao Guan Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: LoongArch: Fix memleak in pci_acpi_scan_root() Add kfree(root_ops) in this case to avoid memleak of root_ops, leaks when pci_find_bus() != 0. Signed-off-by: Yuli Wang Signed-off-by: Wentao Guan Signed-off-by: Huacai Chen --- arch/loongarch/pci/acpi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/loongarch/pci/acpi.c b/arch/loongarch/pci/acpi.c index 365f7de771cb..1da4dc46df43 100644 --- a/arch/loongarch/pci/acpi.c +++ b/arch/loongarch/pci/acpi.c @@ -225,6 +225,7 @@ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) if (bus) { memcpy(bus->sysdata, info->cfg, sizeof(struct pci_config_window)); kfree(info); + kfree(root_ops); } else { struct pci_bus *child; -- cgit From 64c35d6c0ff95e9507f5fb3cce4936c7c62f3d3a Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: LoongArch: Remove posix_types.h include from sigcontext.h Nothing in sigcontext.h seems to require anything from linux/posix_types.h. This include seems a MIPS relic originated from an error in Linux 2.6.11-rc2 (in 2005). The unneeded include was found debugging some vDSO self test build failure (it's not the root cause though). Link: https://lore.kernel.org/linux-mips/20240828030413.143930-2-xry111@xry111.site/ Link: https://lore.kernel.org/loongarch/0b540679ec8cfccec75aeb3463810924f6ff71e6.camel@xry111.site/ Signed-off-by: Xi Ruoyao Signed-off-by: Huacai Chen --- arch/loongarch/include/uapi/asm/sigcontext.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/loongarch/include/uapi/asm/sigcontext.h b/arch/loongarch/include/uapi/asm/sigcontext.h index 6c22f616b8f1..5cd121275bac 100644 --- a/arch/loongarch/include/uapi/asm/sigcontext.h +++ b/arch/loongarch/include/uapi/asm/sigcontext.h @@ -9,7 +9,6 @@ #define _UAPI_ASM_SIGCONTEXT_H #include -#include /* FP context was used */ #define SC_USED_FP (1 << 0) -- cgit From f339bd3b51dac675fbbc08b861d2371ae3df0c0b Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: Docs/LoongArch: Add advanced extended IRQ model description Introduce the advanced extended interrupt controllers (AVECINTC). This feature will allow each core to have 256 independent interrupt vectors and MSI interrupts can be independently routed to any vector on any CPU. The whole topology of irqchips in LoongArch machines looks like this if AVECINTC is supported: +-----+ +-----------------------+ +-------+ | IPI | --> | CPUINTC | <-- | Timer | +-----+ +-----------------------+ +-------+ ^ ^ ^ | | | +---------+ +----------+ +---------+ +-------+ | EIOINTC | | AVECINTC | | LIOINTC | <-- | UARTs | +---------+ +----------+ +---------+ +-------+ ^ ^ | | +---------+ +---------+ | PCH-PIC | | PCH-MSI | +---------+ +---------+ ^ ^ ^ | | | +---------+ +---------+ +---------+ | Devices | | PCH-LPC | | Devices | +---------+ +---------+ +---------+ ^ | +---------+ | Devices | +---------+ Signed-off-by: Huacai Chen Signed-off-by: Tianyang Zhang --- Documentation/arch/loongarch/irq-chip-model.rst | 32 ++++++++++++++++++++++ .../zh_CN/arch/loongarch/irq-chip-model.rst | 32 ++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/Documentation/arch/loongarch/irq-chip-model.rst b/Documentation/arch/loongarch/irq-chip-model.rst index 7988f4192363..6dd48256e39f 100644 --- a/Documentation/arch/loongarch/irq-chip-model.rst +++ b/Documentation/arch/loongarch/irq-chip-model.rst @@ -85,6 +85,38 @@ to CPUINTC directly:: | Devices | +---------+ +Advanced Extended IRQ model +=========================== + +In this model, IPI (Inter-Processor Interrupt) and CPU Local Timer interrupt go +to CPUINTC directly, CPU UARTS interrupts go to LIOINTC, PCH-MSI interrupts go +to AVECINTC, and then go to CPUINTC directly, while all other devices interrupts +go to PCH-PIC/PCH-LPC and gathered by EIOINTC, and then go to CPUINTC directly:: + + +-----+ +-----------------------+ +-------+ + | IPI | --> | CPUINTC | <-- | Timer | + +-----+ +-----------------------+ +-------+ + ^ ^ ^ + | | | + +---------+ +----------+ +---------+ +-------+ + | EIOINTC | | AVECINTC | | LIOINTC | <-- | UARTs | + +---------+ +----------+ +---------+ +-------+ + ^ ^ + | | + +---------+ +---------+ + | PCH-PIC | | PCH-MSI | + +---------+ +---------+ + ^ ^ ^ + | | | + +---------+ +---------+ +---------+ + | Devices | | PCH-LPC | | Devices | + +---------+ +---------+ +---------+ + ^ + | + +---------+ + | Devices | + +---------+ + ACPI-related definitions ======================== diff --git a/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst b/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst index f1e9ab18206c..472761938682 100644 --- a/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst +++ b/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst @@ -87,6 +87,38 @@ PCH-LPC/PCH-MSI,然后被EIOINTC统一收集,再直接到达CPUINTC:: | Devices | +---------+ +高级扩展IRQ模型 +=============== + +在这种模型里面,IPI(Inter-Processor Interrupt)和CPU本地时钟中断直接发送到CPUINTC, +CPU串口(UARTs)中断发送到LIOINTC,PCH-MSI中断发送到AVECINTC,而后通过AVECINTC直接 +送达CPUINTC,而其他所有设备的中断则分别发送到所连接的PCH-PIC/PCH-LPC,然后由EIOINTC +统一收集,再直接到达CPUINTC:: + + +-----+ +-----------------------+ +-------+ + | IPI | --> | CPUINTC | <-- | Timer | + +-----+ +-----------------------+ +-------+ + ^ ^ ^ + | | | + +---------+ +----------+ +---------+ +-------+ + | EIOINTC | | AVECINTC | | LIOINTC | <-- | UARTs | + +---------+ +----------+ +---------+ +-------+ + ^ ^ + | | + +---------+ +---------+ + | PCH-PIC | | PCH-MSI | + +---------+ +---------+ + ^ ^ ^ + | | | + +---------+ +---------+ +---------+ + | Devices | | PCH-LPC | | Devices | + +---------+ +---------+ +---------+ + ^ + | + +---------+ + | Devices | + +---------+ + ACPI相关的定义 ============== -- cgit