diff options
| -rw-r--r-- | arch/s390/include/asm/fpu-insn.h | 70 | ||||
| -rw-r--r-- | arch/s390/include/asm/fpu.h | 21 | ||||
| -rw-r--r-- | arch/s390/kernel/fpu.c | 24 |
3 files changed, 88 insertions, 27 deletions
diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index df2cad95b598..538201864a47 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -108,5 +108,75 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) : "memory"); } +#ifdef CONFIG_CC_IS_CLANG + +#define fpu_vlm(_v1, _v3, _vxrs) do { \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_read(_v, size); \ + asm volatile("\n" \ + " la 1,%[vxrs]\n" \ + " VLM %[v1],%[v3],0,1\n" \ + : \ + : [vxrs] "R" (*_v), \ + [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory", "1"); \ +} while (0) + +#else /* CONFIG_CC_IS_CLANG */ + +#define fpu_vlm(_v1, _v3, _vxrs) do { \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_read(_v, size); \ + asm volatile("VLM %[v1],%[v3],%O[vxrs],%R[vxrs]\n" \ + : \ + : [vxrs] "Q" (*_v), \ + [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory"); \ +} while (0) + +#endif /* CONFIG_CC_IS_CLANG */ + +#ifdef CONFIG_CC_IS_CLANG + +#define fpu_vstm(_v1, _v3, _vxrs) do { \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_write(_v, size); \ + asm volatile("\n" \ + " la 1,%[vxrs]\n" \ + " VSTM %[v1],%[v3],0,1\n" \ + : [vxrs] "=R" (*_v) \ + : [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory", "1"); \ +} while (0) + +#else /* CONFIG_CC_IS_CLANG */ + +#define fpu_vstm(_v1, _v3, _vxrs) do { \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_write(_v, size); \ + asm volatile("VSTM %[v1],%[v3],%O[vxrs],%R[vxrs]\n" \ + : [vxrs] "=Q" (*_v) \ + : [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory"); \ +} while (0) + +#endif /* CONFIG_CC_IS_CLANG */ + #endif /* __ASSEMBLY__ */ #endif /* __ASM_S390_FPU_INSN_H */ diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index 626695de6085..6a0a23a28ce8 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -84,6 +84,18 @@ void __load_fpu_regs(void); void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags); void __kernel_fpu_end(struct kernel_fpu *state, u32 flags); +static __always_inline void save_vx_regs(__vector128 *vxrs) +{ + fpu_vstm(0, 15, &vxrs[0]); + fpu_vstm(16, 31, &vxrs[16]); +} + +static __always_inline void load_vx_regs(__vector128 *vxrs) +{ + fpu_vlm(0, 15, &vxrs[0]); + fpu_vlm(16, 31, &vxrs[16]); +} + static __always_inline void save_fp_regs(freg_t *fprs) { fpu_std(0, &fprs[0]); @@ -148,15 +160,6 @@ static inline void kernel_fpu_end(struct kernel_fpu *state, u32 flags) preempt_enable(); } -static inline void save_vx_regs(__vector128 *vxrs) -{ - asm volatile("\n" - " la 1,%0\n" - " .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */ - " .word 0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */ - : "=Q" (*(struct vx_array *)vxrs) : : "1"); -} - static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) { int i; diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 6bfd4d0f33e1..092a4bdf88ed 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -137,16 +137,10 @@ void __load_fpu_regs(void) void *regs = current->thread.fpu.regs; fpu_lfpc_safe(&state->fpc); - if (likely(cpu_has_vx())) { - asm volatile("lgr 1,%0\n" - "VLM 0,15,0,1\n" - "VLM 16,31,256,1\n" - : - : "d" (regs) - : "1", "cc", "memory"); - } else { + if (likely(cpu_has_vx())) + load_vx_regs(regs); + else load_fp_regs(regs); - } clear_cpu_flag(CIF_FPU); } @@ -173,16 +167,10 @@ void save_fpu_regs(void) regs = current->thread.fpu.regs; fpu_stfpc(&state->fpc); - if (likely(cpu_has_vx())) { - asm volatile("lgr 1,%0\n" - "VSTM 0,15,0,1\n" - "VSTM 16,31,256,1\n" - : - : "d" (regs) - : "1", "cc", "memory"); - } else { + if (likely(cpu_has_vx())) + save_vx_regs(regs); + else save_fp_regs(regs); - } set_cpu_flag(CIF_FPU); out: local_irq_restore(flags); |