From ecc7e37d4dadd16f6be125ca496feccd05454da4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 11 Nov 2019 23:03:20 +0100 Subject: x86/io: Speedup schedule out of I/O bitmap user There is no requirement to update the TSS I/O bitmap when a thread using it is scheduled out and the incoming thread does not use it. For the permission check based on the TSS I/O bitmap the CPU calculates the memory location of the I/O bitmap by the address of the TSS and the io_bitmap_base member of the tss_struct. The easiest way to invalidate the I/O bitmap is to switch the offset to an address outside of the TSS limit. If an I/O instruction is issued from user space the TSS limit causes #GP to be raised in the same was as valid I/O bitmap with all bits set to 1 would do. This removes the extra work when an I/O bitmap using task is scheduled out and puts the burden on the rare I/O bitmap users when they are scheduled in. Signed-off-by: Thomas Gleixner --- arch/x86/kernel/doublefault.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/kernel/doublefault.c') diff --git a/arch/x86/kernel/doublefault.c b/arch/x86/kernel/doublefault.c index 0b8cedb20d6d..cedb07d0909c 100644 --- a/arch/x86/kernel/doublefault.c +++ b/arch/x86/kernel/doublefault.c @@ -54,7 +54,7 @@ struct x86_hw_tss doublefault_tss __cacheline_aligned = { .sp0 = STACK_START, .ss0 = __KERNEL_DS, .ldt = 0, - .io_bitmap_base = INVALID_IO_BITMAP_OFFSET, + .io_bitmap_base = IO_BITMAP_OFFSET_INVALID, .ip = (unsigned long) doublefault_fn, /* 0x2 bit is always set */ -- cgit From 3580d0b29cab08483f84a16ce6a1151a1013695f Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Thu, 21 Nov 2019 11:50:12 +0100 Subject: x86/doublefault/32: Fix stack canaries in the double fault handler The double fault TSS was missing GS setup, which is needed for stack canaries to work. Signed-off-by: Andy Lutomirski Signed-off-by: Peter Zijlstra (Intel) Cc: stable@kernel.org --- arch/x86/kernel/doublefault.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/x86/kernel/doublefault.c') diff --git a/arch/x86/kernel/doublefault.c b/arch/x86/kernel/doublefault.c index 0b8cedb20d6d..d5c9b13bafdf 100644 --- a/arch/x86/kernel/doublefault.c +++ b/arch/x86/kernel/doublefault.c @@ -65,6 +65,9 @@ struct x86_hw_tss doublefault_tss __cacheline_aligned = { .ss = __KERNEL_DS, .ds = __USER_DS, .fs = __KERNEL_PERCPU, +#ifndef CONFIG_X86_32_LAZY_GS + .gs = __KERNEL_STACK_CANARY, +#endif .__cr3 = __pa_nodebug(swapper_pg_dir), }; -- cgit From 93efbde2c331004d8053f04b4bf0ca3e630b474a Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 20 Nov 2019 22:12:38 -0800 Subject: x86/traps: Disentangle the 32-bit and 64-bit doublefault code The 64-bit doublefault handler is much nicer than the 32-bit one. As a first step toward unifying them, make the 64-bit handler self-contained. This should have no effect no functional effect except in the odd case of x86_64 with CONFIG_DOUBLEFAULT=n in which case it will change the logging a bit. This also gets rid of CONFIG_DOUBLEFAULT configurability on 64-bit kernels. It didn't do anything useful -- CONFIG_DOUBLEFAULT=n didn't actually disable doublefault handling on x86_64. Signed-off-by: Andy Lutomirski Cc: Borislav Petkov Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Linus Torvalds Signed-off-by: Ingo Molnar --- arch/x86/Kconfig.debug | 2 +- arch/x86/include/asm/processor.h | 1 - arch/x86/kernel/doublefault.c | 11 ----------- arch/x86/kernel/traps.c | 12 +++--------- 4 files changed, 4 insertions(+), 22 deletions(-) (limited to 'arch/x86/kernel/doublefault.c') diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index 409c00f74e60..c4eab8ed33a3 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug @@ -117,7 +117,7 @@ config DEBUG_WX config DOUBLEFAULT default y - bool "Enable doublefault exception handler" if EXPERT + bool "Enable doublefault exception handler" if EXPERT && X86_32 ---help--- This option allows trapping of rare doublefault exceptions that would otherwise cause a system to silently reboot. Disabling this diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index e51afbb0cbfb..f6c630097d9f 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -997,7 +997,6 @@ bool xen_set_default_idle(void); #endif void stop_this_cpu(void *dummy); -void df_debug(struct pt_regs *regs, long error_code); void microcode_check(void); enum l1tf_mitigations { diff --git a/arch/x86/kernel/doublefault.c b/arch/x86/kernel/doublefault.c index 0d6c657593f8..0b3c616b61a3 100644 --- a/arch/x86/kernel/doublefault.c +++ b/arch/x86/kernel/doublefault.c @@ -72,15 +72,4 @@ struct x86_hw_tss doublefault_tss __cacheline_aligned = { .__cr3 = __pa_nodebug(swapper_pg_dir), }; -/* dummy for do_double_fault() call */ -void df_debug(struct pt_regs *regs, long error_code) {} - -#else /* !CONFIG_X86_32 */ - -void df_debug(struct pt_regs *regs, long error_code) -{ - pr_emerg("PANIC: double fault, error_code: 0x%lx\n", error_code); - show_regs(regs); - panic("Machine halted."); -} #endif diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index c90312146da0..76381b04dc93 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -411,15 +411,9 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsign handle_stack_overflow("kernel stack overflow (double-fault)", regs, cr2); #endif -#ifdef CONFIG_DOUBLEFAULT - df_debug(regs, error_code); -#endif - /* - * This is always a kernel trap and never fixable (and thus must - * never return). - */ - for (;;) - die(str, regs, error_code); + pr_emerg("PANIC: double fault, error_code: 0x%lx\n", error_code); + show_regs(regs); + panic("Machine halted."); } #endif -- cgit From e99b6f46ee5c127d39d2f3a2682fdeef10386316 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Thu, 21 Nov 2019 09:42:30 -0800 Subject: x86/doublefault/32: Rename doublefault.c to doublefault_32.c doublefault.c now only contains 32-bit code. Rename it to doublefault_32.c. Signed-off-by: Andy Lutomirski Cc: Borislav Petkov Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Linus Torvalds Signed-off-by: Ingo Molnar --- arch/x86/kernel/Makefile | 4 ++- arch/x86/kernel/doublefault.c | 75 ---------------------------------------- arch/x86/kernel/doublefault_32.c | 71 +++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 76 deletions(-) delete mode 100644 arch/x86/kernel/doublefault.c create mode 100644 arch/x86/kernel/doublefault_32.c (limited to 'arch/x86/kernel/doublefault.c') diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 32acb970f416..6175e370ee4a 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -100,7 +100,9 @@ obj-$(CONFIG_KEXEC_FILE) += kexec-bzimage64.o obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o obj-y += kprobes/ obj-$(CONFIG_MODULES) += module.o -obj-$(CONFIG_DOUBLEFAULT) += doublefault.o +ifeq ($(CONFIG_X86_32),y) +obj-$(CONFIG_DOUBLEFAULT) += doublefault_32.o +endif obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_VM86) += vm86_32.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o diff --git a/arch/x86/kernel/doublefault.c b/arch/x86/kernel/doublefault.c deleted file mode 100644 index 0b3c616b61a3..000000000000 --- a/arch/x86/kernel/doublefault.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#ifdef CONFIG_X86_32 - -#define DOUBLEFAULT_STACKSIZE (1024) -static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE]; -#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE) - -#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM) - -static void doublefault_fn(void) -{ - struct desc_ptr gdt_desc = {0, 0}; - unsigned long gdt, tss; - - native_store_gdt(&gdt_desc); - gdt = gdt_desc.address; - - printk(KERN_EMERG "PANIC: double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size); - - if (ptr_ok(gdt)) { - gdt += GDT_ENTRY_TSS << 3; - tss = get_desc_base((struct desc_struct *)gdt); - printk(KERN_EMERG "double fault, tss at %08lx\n", tss); - - if (ptr_ok(tss)) { - struct x86_hw_tss *t = (struct x86_hw_tss *)tss; - - printk(KERN_EMERG "eip = %08lx, esp = %08lx\n", - t->ip, t->sp); - - printk(KERN_EMERG "eax = %08lx, ebx = %08lx, ecx = %08lx, edx = %08lx\n", - t->ax, t->bx, t->cx, t->dx); - printk(KERN_EMERG "esi = %08lx, edi = %08lx\n", - t->si, t->di); - } - } - - for (;;) - cpu_relax(); -} - -struct x86_hw_tss doublefault_tss __cacheline_aligned = { - .sp0 = STACK_START, - .ss0 = __KERNEL_DS, - .ldt = 0, - .io_bitmap_base = IO_BITMAP_OFFSET_INVALID, - - .ip = (unsigned long) doublefault_fn, - /* 0x2 bit is always set */ - .flags = X86_EFLAGS_SF | 0x2, - .sp = STACK_START, - .es = __USER_DS, - .cs = __KERNEL_CS, - .ss = __KERNEL_DS, - .ds = __USER_DS, - .fs = __KERNEL_PERCPU, -#ifndef CONFIG_X86_32_LAZY_GS - .gs = __KERNEL_STACK_CANARY, -#endif - - .__cr3 = __pa_nodebug(swapper_pg_dir), -}; - -#endif diff --git a/arch/x86/kernel/doublefault_32.c b/arch/x86/kernel/doublefault_32.c new file mode 100644 index 000000000000..61c707ca8a09 --- /dev/null +++ b/arch/x86/kernel/doublefault_32.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define DOUBLEFAULT_STACKSIZE (1024) +static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE]; +#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE) + +#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM) + +static void doublefault_fn(void) +{ + struct desc_ptr gdt_desc = {0, 0}; + unsigned long gdt, tss; + + native_store_gdt(&gdt_desc); + gdt = gdt_desc.address; + + printk(KERN_EMERG "PANIC: double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size); + + if (ptr_ok(gdt)) { + gdt += GDT_ENTRY_TSS << 3; + tss = get_desc_base((struct desc_struct *)gdt); + printk(KERN_EMERG "double fault, tss at %08lx\n", tss); + + if (ptr_ok(tss)) { + struct x86_hw_tss *t = (struct x86_hw_tss *)tss; + + printk(KERN_EMERG "eip = %08lx, esp = %08lx\n", + t->ip, t->sp); + + printk(KERN_EMERG "eax = %08lx, ebx = %08lx, ecx = %08lx, edx = %08lx\n", + t->ax, t->bx, t->cx, t->dx); + printk(KERN_EMERG "esi = %08lx, edi = %08lx\n", + t->si, t->di); + } + } + + for (;;) + cpu_relax(); +} + +struct x86_hw_tss doublefault_tss __cacheline_aligned = { + .sp0 = STACK_START, + .ss0 = __KERNEL_DS, + .ldt = 0, + .io_bitmap_base = IO_BITMAP_OFFSET_INVALID, + + .ip = (unsigned long) doublefault_fn, + /* 0x2 bit is always set */ + .flags = X86_EFLAGS_SF | 0x2, + .sp = STACK_START, + .es = __USER_DS, + .cs = __KERNEL_CS, + .ss = __KERNEL_DS, + .ds = __USER_DS, + .fs = __KERNEL_PERCPU, +#ifndef CONFIG_X86_32_LAZY_GS + .gs = __KERNEL_STACK_CANARY, +#endif + + .__cr3 = __pa_nodebug(swapper_pg_dir), +}; -- cgit