diff options
Diffstat (limited to 'arch/loongarch/include/asm')
| -rw-r--r-- | arch/loongarch/include/asm/asm-prototypes.h | 1 | ||||
| -rw-r--r-- | arch/loongarch/include/asm/asmmacro.h | 51 | ||||
| -rw-r--r-- | arch/loongarch/include/asm/lbt.h | 109 | ||||
| -rw-r--r-- | arch/loongarch/include/asm/loongarch.h | 4 | ||||
| -rw-r--r-- | arch/loongarch/include/asm/processor.h | 26 | ||||
| -rw-r--r-- | arch/loongarch/include/asm/switch_to.h | 2 | ||||
| -rw-r--r-- | arch/loongarch/include/asm/thread_info.h | 4 |
7 files changed, 180 insertions, 17 deletions
diff --git a/arch/loongarch/include/asm/asm-prototypes.h b/arch/loongarch/include/asm/asm-prototypes.h index ed06d3997420..cf8e1a4e7c19 100644 --- a/arch/loongarch/include/asm/asm-prototypes.h +++ b/arch/loongarch/include/asm/asm-prototypes.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/uaccess.h> #include <asm/fpu.h> +#include <asm/lbt.h> #include <asm/mmu_context.h> #include <asm/page.h> #include <asm/ftrace.h> diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h index af542a8d847f..c9544f358c33 100644 --- a/arch/loongarch/include/asm/asmmacro.h +++ b/arch/loongarch/include/asm/asmmacro.h @@ -41,12 +41,51 @@ .macro fpu_save_csr thread tmp movfcsr2gr \tmp, fcsr0 - stptr.w \tmp, \thread, THREAD_FCSR + stptr.w \tmp, \thread, THREAD_FCSR +#ifdef CONFIG_CPU_HAS_LBT + /* TM bit is always 0 if LBT not supported */ + andi \tmp, \tmp, FPU_CSR_TM + beqz \tmp, 1f + /* Save FTOP */ + x86mftop \tmp + stptr.w \tmp, \thread, THREAD_FTOP + /* Turn off TM to ensure the order of FPR in memory independent of TM */ + x86clrtm +1: +#endif .endm - .macro fpu_restore_csr thread tmp - ldptr.w \tmp, \thread, THREAD_FCSR - movgr2fcsr fcsr0, \tmp + .macro fpu_restore_csr thread tmp0 tmp1 + ldptr.w \tmp0, \thread, THREAD_FCSR + movgr2fcsr fcsr0, \tmp0 +#ifdef CONFIG_CPU_HAS_LBT + /* TM bit is always 0 if LBT not supported */ + andi \tmp0, \tmp0, FPU_CSR_TM + beqz \tmp0, 2f + /* Restore FTOP */ + ldptr.w \tmp0, \thread, THREAD_FTOP + andi \tmp0, \tmp0, 0x7 + la.pcrel \tmp1, 1f + alsl.d \tmp1, \tmp0, \tmp1, 3 + jr \tmp1 +1: + x86mttop 0 + b 2f + x86mttop 1 + b 2f + x86mttop 2 + b 2f + x86mttop 3 + b 2f + x86mttop 4 + b 2f + x86mttop 5 + b 2f + x86mttop 6 + b 2f + x86mttop 7 +2: +#endif .endm .macro fpu_save_cc thread tmp0 tmp1 @@ -246,7 +285,7 @@ .macro lsx_restore_all thread tmp0 tmp1 lsx_restore_data \thread, \tmp0 fpu_restore_cc \thread, \tmp0, \tmp1 - fpu_restore_csr \thread, \tmp0 + fpu_restore_csr \thread, \tmp0, \tmp1 .endm .macro lsx_save_upper vd base tmp off @@ -456,7 +495,7 @@ .macro lasx_restore_all thread tmp0 tmp1 lasx_restore_data \thread, \tmp0 fpu_restore_cc \thread, \tmp0, \tmp1 - fpu_restore_csr \thread, \tmp0 + fpu_restore_csr \thread, \tmp0, \tmp1 .endm .macro lasx_save_upper xd base tmp off diff --git a/arch/loongarch/include/asm/lbt.h b/arch/loongarch/include/asm/lbt.h new file mode 100644 index 000000000000..e671978bf552 --- /dev/null +++ b/arch/loongarch/include/asm/lbt.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Qi Hu <huqi@loongson.cn> + * Huacai Chen <chenhuacai@loongson.cn> + * Copyright (C) 2020-2023 Loongson Technology Corporation Limited + */ +#ifndef _ASM_LBT_H +#define _ASM_LBT_H + +#include <asm/cpu.h> +#include <asm/current.h> +#include <asm/loongarch.h> +#include <asm/processor.h> + +extern void _init_lbt(void); +extern void _save_lbt(struct loongarch_lbt *); +extern void _restore_lbt(struct loongarch_lbt *); + +static inline int is_lbt_enabled(void) +{ + if (!cpu_has_lbt) + return 0; + + return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LBTEN) ? + 1 : 0; +} + +static inline int is_lbt_owner(void) +{ + return test_thread_flag(TIF_USEDLBT); +} + +#ifdef CONFIG_CPU_HAS_LBT + +static inline void enable_lbt(void) +{ + if (cpu_has_lbt) + csr_xchg32(CSR_EUEN_LBTEN, CSR_EUEN_LBTEN, LOONGARCH_CSR_EUEN); +} + +static inline void disable_lbt(void) +{ + if (cpu_has_lbt) + csr_xchg32(0, CSR_EUEN_LBTEN, LOONGARCH_CSR_EUEN); +} + +static inline void __own_lbt(void) +{ + enable_lbt(); + set_thread_flag(TIF_USEDLBT); + KSTK_EUEN(current) |= CSR_EUEN_LBTEN; +} + +static inline void own_lbt_inatomic(int restore) +{ + if (cpu_has_lbt && !is_lbt_owner()) { + __own_lbt(); + if (restore) + _restore_lbt(¤t->thread.lbt); + } +} + +static inline void own_lbt(int restore) +{ + preempt_disable(); + own_lbt_inatomic(restore); + preempt_enable(); +} + +static inline void lose_lbt_inatomic(int save, struct task_struct *tsk) +{ + if (cpu_has_lbt && is_lbt_owner()) { + if (save) + _save_lbt(&tsk->thread.lbt); + + disable_lbt(); + clear_tsk_thread_flag(tsk, TIF_USEDLBT); + } + KSTK_EUEN(tsk) &= ~(CSR_EUEN_LBTEN); +} + +static inline void lose_lbt(int save) +{ + preempt_disable(); + lose_lbt_inatomic(save, current); + preempt_enable(); +} + +static inline void init_lbt(void) +{ + __own_lbt(); + _init_lbt(); +} +#else +static inline void own_lbt_inatomic(int restore) {} +static inline void lose_lbt_inatomic(int save, struct task_struct *tsk) {} +static inline void init_lbt(void) {} +static inline void lose_lbt(int save) {} +#endif + +static inline int thread_lbt_context_live(void) +{ + if (!cpu_has_lbt) + return 0; + + return test_thread_flag(TIF_LBT_CTX_LIVE); +} + +#endif /* _ASM_LBT_H */ diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index a500efe0fd92..33531d432b49 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -1410,6 +1410,10 @@ __BUILD_CSR_OP(tlbidx) #define FPU_CSR_RU 0x200 /* towards +Infinity */ #define FPU_CSR_RD 0x300 /* towards -Infinity */ +/* Bit 6 of FPU Status Register specify the LBT TOP simulation mode */ +#define FPU_CSR_TM_SHIFT 0x6 +#define FPU_CSR_TM (_ULCAST_(1) << FPU_CSR_TM_SHIFT) + #define read_fcsr(source) \ ({ \ unsigned int __res; \ diff --git a/arch/loongarch/include/asm/processor.h b/arch/loongarch/include/asm/processor.h index 636e1c66398c..c3bc44b5f5b3 100644 --- a/arch/loongarch/include/asm/processor.h +++ b/arch/loongarch/include/asm/processor.h @@ -80,11 +80,22 @@ BUILD_FPR_ACCESS(32) BUILD_FPR_ACCESS(64) struct loongarch_fpu { - unsigned int fcsr; uint64_t fcc; /* 8x8 */ + uint32_t fcsr; + uint32_t ftop; union fpureg fpr[NUM_FPU_REGS]; }; +struct loongarch_lbt { + /* Scratch registers */ + unsigned long scr0; + unsigned long scr1; + unsigned long scr2; + unsigned long scr3; + /* Eflags register */ + unsigned long eflags; +}; + #define INIT_CPUMASK { \ {0,} \ } @@ -113,15 +124,6 @@ struct thread_struct { unsigned long csr_ecfg; unsigned long csr_badvaddr; /* Last user fault */ - /* Scratch registers */ - unsigned long scr0; - unsigned long scr1; - unsigned long scr2; - unsigned long scr3; - - /* Eflags register */ - unsigned long eflags; - /* Other stuff associated with the thread. */ unsigned long trap_nr; unsigned long error_code; @@ -133,6 +135,7 @@ struct thread_struct { * context because they are conditionally copied at fork(). */ struct loongarch_fpu fpu FPU_ALIGN; + struct loongarch_lbt lbt; /* Also conditionally copied */ /* Hardware breakpoints pinned to this task. */ struct perf_event *hbp_break[LOONGARCH_MAX_BRP]; @@ -174,8 +177,9 @@ struct thread_struct { * FPU & vector registers \ */ \ .fpu = { \ - .fcsr = 0, \ .fcc = 0, \ + .fcsr = 0, \ + .ftop = 0, \ .fpr = {{{0,},},}, \ }, \ .hbp_break = {0}, \ diff --git a/arch/loongarch/include/asm/switch_to.h b/arch/loongarch/include/asm/switch_to.h index 24e3094bebab..5b225aff3ba2 100644 --- a/arch/loongarch/include/asm/switch_to.h +++ b/arch/loongarch/include/asm/switch_to.h @@ -7,6 +7,7 @@ #include <asm/cpu-features.h> #include <asm/fpu.h> +#include <asm/lbt.h> struct task_struct; @@ -34,6 +35,7 @@ extern asmlinkage struct task_struct *__switch_to(struct task_struct *prev, #define switch_to(prev, next, last) \ do { \ lose_fpu_inatomic(1, prev); \ + lose_lbt_inatomic(1, prev); \ hw_breakpoint_thread_switch(next); \ (last) = __switch_to(prev, next, task_thread_info(next), \ __builtin_return_address(0), __builtin_frame_address(0)); \ diff --git a/arch/loongarch/include/asm/thread_info.h b/arch/loongarch/include/asm/thread_info.h index 1a3354ca056e..8cb653d49a54 100644 --- a/arch/loongarch/include/asm/thread_info.h +++ b/arch/loongarch/include/asm/thread_info.h @@ -84,6 +84,8 @@ register unsigned long current_stack_pointer __asm__("$sp"); #define TIF_SINGLESTEP 16 /* Single Step */ #define TIF_LSX_CTX_LIVE 17 /* LSX context must be preserved */ #define TIF_LASX_CTX_LIVE 18 /* LASX context must be preserved */ +#define TIF_USEDLBT 19 /* LBT was used by this task this quantum (SMP) */ +#define TIF_LBT_CTX_LIVE 20 /* LBT context must be preserved */ #define _TIF_SIGPENDING (1<<TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) @@ -101,6 +103,8 @@ register unsigned long current_stack_pointer __asm__("$sp"); #define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP) #define _TIF_LSX_CTX_LIVE (1<<TIF_LSX_CTX_LIVE) #define _TIF_LASX_CTX_LIVE (1<<TIF_LASX_CTX_LIVE) +#define _TIF_USEDLBT (1<<TIF_USEDLBT) +#define _TIF_LBT_CTX_LIVE (1<<TIF_LBT_CTX_LIVE) #endif /* __KERNEL__ */ #endif /* _ASM_THREAD_INFO_H */ |