Merge branch kvm-arm64/nv-sve into kvmarm/next
* kvm-arm64/nv-sve: : CPTR_EL2, FPSIMD/SVE support for nested : : This series brings support for honoring the guest hypervisor's CPTR_EL2 : trap configuration when running a nested guest, along with support for : FPSIMD/SVE usage at L1 and L2. KVM: arm64: Allow the use of SVE+NV KVM: arm64: nv: Add additional trap setup for CPTR_EL2 KVM: arm64: nv: Add trap description for CPTR_EL2 KVM: arm64: nv: Add TCPAC/TTA to CPTR->CPACR conversion helper KVM: arm64: nv: Honor guest hypervisor's FP/SVE traps in CPTR_EL2 KVM: arm64: nv: Load guest FP state for ZCR_EL2 trap KVM: arm64: nv: Handle CPACR_EL1 traps KVM: arm64: Spin off helper for programming CPTR traps KVM: arm64: nv: Ensure correct VL is loaded before saving SVE state KVM: arm64: nv: Use guest hypervisor's max VL when running nested guest KVM: arm64: nv: Save guest's ZCR_EL2 when in hyp context KVM: arm64: nv: Load guest hyp's ZCR into EL1 state KVM: arm64: nv: Handle ZCR_EL2 traps KVM: arm64: nv: Forward SVE traps to guest hypervisor KVM: arm64: nv: Forward FP/ASIMD traps to guest hypervisor Signed-off-by: Oliver Upton <oliver.upton@linux.dev>
This commit is contained in:
Oliver Upton
commit
8c2899e770
11 changed files with 370 additions and 42 deletions
|
|
@ -11,6 +11,7 @@
|
|||
#ifndef __ARM64_KVM_EMULATE_H__
|
||||
#define __ARM64_KVM_EMULATE_H__
|
||||
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/kvm_host.h>
|
||||
|
||||
#include <asm/debug-monitors.h>
|
||||
|
|
@ -55,6 +56,14 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);
|
|||
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);
|
||||
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);
|
||||
|
||||
static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 esr = FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SVE) |
|
||||
ESR_ELx_IL;
|
||||
|
||||
kvm_inject_nested_sync(vcpu, esr);
|
||||
}
|
||||
|
||||
#if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__)
|
||||
static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
|
|
@ -638,4 +647,50 @@ static __always_inline void kvm_reset_cptr_el2(struct kvm_vcpu *vcpu)
|
|||
|
||||
kvm_write_cptr_el2(val);
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE
|
||||
* format if E2H isn't set.
|
||||
*/
|
||||
static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2);
|
||||
|
||||
if (!vcpu_el2_e2h_is_set(vcpu))
|
||||
cptr = translate_cptr_el2_to_cpacr_el1(cptr);
|
||||
|
||||
return cptr;
|
||||
}
|
||||
|
||||
static inline bool ____cptr_xen_trap_enabled(const struct kvm_vcpu *vcpu,
|
||||
unsigned int xen)
|
||||
{
|
||||
switch (xen) {
|
||||
case 0b00:
|
||||
case 0b10:
|
||||
return true;
|
||||
case 0b01:
|
||||
return vcpu_el2_tge_is_set(vcpu) && !vcpu_is_el2(vcpu);
|
||||
case 0b11:
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#define __guest_hyp_cptr_xen_trap_enabled(vcpu, xen) \
|
||||
(!vcpu_has_nv(vcpu) ? false : \
|
||||
____cptr_xen_trap_enabled(vcpu, \
|
||||
SYS_FIELD_GET(CPACR_ELx, xen, \
|
||||
vcpu_sanitised_cptr_el2(vcpu))))
|
||||
|
||||
static inline bool guest_hyp_fpsimd_traps_enabled(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return __guest_hyp_cptr_xen_trap_enabled(vcpu, FPEN);
|
||||
}
|
||||
|
||||
static inline bool guest_hyp_sve_traps_enabled(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return __guest_hyp_cptr_xen_trap_enabled(vcpu, ZEN);
|
||||
}
|
||||
|
||||
#endif /* __ARM64_KVM_EMULATE_H__ */
|
||||
|
|
|
|||
|
|
@ -458,6 +458,7 @@ enum vcpu_sysreg {
|
|||
MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */
|
||||
CPTR_EL2, /* Architectural Feature Trap Register (EL2) */
|
||||
HACR_EL2, /* Hypervisor Auxiliary Control Register */
|
||||
ZCR_EL2, /* SVE Control Register (EL2) */
|
||||
TTBR0_EL2, /* Translation Table Base Register 0 (EL2) */
|
||||
TTBR1_EL2, /* Translation Table Base Register 1 (EL2) */
|
||||
TCR_EL2, /* Translation Control Register (EL2) */
|
||||
|
|
@ -902,6 +903,9 @@ struct kvm_vcpu_arch {
|
|||
|
||||
#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl)
|
||||
|
||||
#define vcpu_sve_zcr_elx(vcpu) \
|
||||
(unlikely(is_hyp_ctxt(vcpu)) ? ZCR_EL2 : ZCR_EL1)
|
||||
|
||||
#define vcpu_sve_state_size(vcpu) ({ \
|
||||
size_t __size_ret; \
|
||||
unsigned int __vcpu_vq; \
|
||||
|
|
@ -1026,6 +1030,7 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
|
|||
case DACR32_EL2: *val = read_sysreg_s(SYS_DACR32_EL2); break;
|
||||
case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break;
|
||||
case DBGVCR32_EL2: *val = read_sysreg_s(SYS_DBGVCR32_EL2); break;
|
||||
case ZCR_EL1: *val = read_sysreg_s(SYS_ZCR_EL12); break;
|
||||
default: return false;
|
||||
}
|
||||
|
||||
|
|
@ -1071,6 +1076,7 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg)
|
|||
case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break;
|
||||
case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break;
|
||||
case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break;
|
||||
case ZCR_EL1: write_sysreg_s(val, SYS_ZCR_EL12); break;
|
||||
default: return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
|
|||
|
||||
static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)
|
||||
{
|
||||
u64 cpacr_el1 = 0;
|
||||
u64 cpacr_el1 = CPACR_ELx_RES1;
|
||||
|
||||
if (cptr_el2 & CPTR_EL2_TTA)
|
||||
cpacr_el1 |= CPACR_ELx_TTA;
|
||||
|
|
@ -42,6 +42,8 @@ static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)
|
|||
if (!(cptr_el2 & CPTR_EL2_TZ))
|
||||
cpacr_el1 |= CPACR_ELx_ZEN;
|
||||
|
||||
cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM);
|
||||
|
||||
return cpacr_el1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1458,11 +1458,6 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
|
|||
test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
|
||||
return -EINVAL;
|
||||
|
||||
/* Disallow NV+SVE for the time being */
|
||||
if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features) &&
|
||||
test_bit(KVM_ARM_VCPU_SVE, &features))
|
||||
return -EINVAL;
|
||||
|
||||
if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
|
||||
return 0;
|
||||
|
||||
|
|
|
|||
|
|
@ -79,6 +79,10 @@ enum cgt_group_id {
|
|||
CGT_MDCR_E2TB,
|
||||
CGT_MDCR_TDCC,
|
||||
|
||||
CGT_CPACR_E0POE,
|
||||
CGT_CPTR_TAM,
|
||||
CGT_CPTR_TCPAC,
|
||||
|
||||
/*
|
||||
* Anything after this point is a combination of coarse trap
|
||||
* controls, which must all be evaluated to decide what to do.
|
||||
|
|
@ -106,6 +110,8 @@ enum cgt_group_id {
|
|||
CGT_CNTHCTL_EL1PCTEN = __COMPLEX_CONDITIONS__,
|
||||
CGT_CNTHCTL_EL1PTEN,
|
||||
|
||||
CGT_CPTR_TTA,
|
||||
|
||||
/* Must be last */
|
||||
__NR_CGT_GROUP_IDS__
|
||||
};
|
||||
|
|
@ -345,6 +351,24 @@ static const struct trap_bits coarse_trap_bits[] = {
|
|||
.mask = MDCR_EL2_TDCC,
|
||||
.behaviour = BEHAVE_FORWARD_ANY,
|
||||
},
|
||||
[CGT_CPACR_E0POE] = {
|
||||
.index = CPTR_EL2,
|
||||
.value = CPACR_ELx_E0POE,
|
||||
.mask = CPACR_ELx_E0POE,
|
||||
.behaviour = BEHAVE_FORWARD_ANY,
|
||||
},
|
||||
[CGT_CPTR_TAM] = {
|
||||
.index = CPTR_EL2,
|
||||
.value = CPTR_EL2_TAM,
|
||||
.mask = CPTR_EL2_TAM,
|
||||
.behaviour = BEHAVE_FORWARD_ANY,
|
||||
},
|
||||
[CGT_CPTR_TCPAC] = {
|
||||
.index = CPTR_EL2,
|
||||
.value = CPTR_EL2_TCPAC,
|
||||
.mask = CPTR_EL2_TCPAC,
|
||||
.behaviour = BEHAVE_FORWARD_ANY,
|
||||
},
|
||||
};
|
||||
|
||||
#define MCB(id, ...) \
|
||||
|
|
@ -410,12 +434,26 @@ static enum trap_behaviour check_cnthctl_el1pten(struct kvm_vcpu *vcpu)
|
|||
return BEHAVE_FORWARD_ANY;
|
||||
}
|
||||
|
||||
static enum trap_behaviour check_cptr_tta(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 val = __vcpu_sys_reg(vcpu, CPTR_EL2);
|
||||
|
||||
if (!vcpu_el2_e2h_is_set(vcpu))
|
||||
val = translate_cptr_el2_to_cpacr_el1(val);
|
||||
|
||||
if (val & CPACR_ELx_TTA)
|
||||
return BEHAVE_FORWARD_ANY;
|
||||
|
||||
return BEHAVE_HANDLE_LOCALLY;
|
||||
}
|
||||
|
||||
#define CCC(id, fn) \
|
||||
[id - __COMPLEX_CONDITIONS__] = fn
|
||||
|
||||
static const complex_condition_check ccc[] = {
|
||||
CCC(CGT_CNTHCTL_EL1PCTEN, check_cnthctl_el1pcten),
|
||||
CCC(CGT_CNTHCTL_EL1PTEN, check_cnthctl_el1pten),
|
||||
CCC(CGT_CPTR_TTA, check_cptr_tta),
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
@ -1000,6 +1038,59 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = {
|
|||
SR_TRAP(SYS_TRBPTR_EL1, CGT_MDCR_E2TB),
|
||||
SR_TRAP(SYS_TRBSR_EL1, CGT_MDCR_E2TB),
|
||||
SR_TRAP(SYS_TRBTRG_EL1, CGT_MDCR_E2TB),
|
||||
SR_TRAP(SYS_CPACR_EL1, CGT_CPTR_TCPAC),
|
||||
SR_TRAP(SYS_AMUSERENR_EL0, CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMCFGR_EL0, CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMCGCR_EL0, CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMCNTENCLR0_EL0, CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMCNTENCLR1_EL0, CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMCNTENSET0_EL0, CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMCNTENSET1_EL0, CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMCR_EL0, CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR0_EL0(0), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR0_EL0(1), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR0_EL0(2), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR0_EL0(3), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(0), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(1), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(2), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(3), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(4), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(5), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(6), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(7), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(8), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(9), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(10), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(11), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(12), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(13), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(14), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVCNTR1_EL0(15), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER0_EL0(0), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER0_EL0(1), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER0_EL0(2), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER0_EL0(3), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(0), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(1), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(2), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(3), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(4), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(5), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(6), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(7), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(8), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(9), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(10), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(11), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(12), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(13), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(14), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_AMEVTYPER1_EL0(15), CGT_CPTR_TAM),
|
||||
SR_TRAP(SYS_POR_EL0, CGT_CPACR_E0POE),
|
||||
/* op0=2, op1=1, and CRn<0b1000 */
|
||||
SR_RANGE_TRAP(sys_reg(2, 1, 0, 0, 0),
|
||||
sys_reg(2, 1, 7, 15, 7), CGT_CPTR_TTA),
|
||||
SR_TRAP(SYS_CNTP_TVAL_EL0, CGT_CNTHCTL_EL1PTEN),
|
||||
SR_TRAP(SYS_CNTP_CVAL_EL0, CGT_CNTHCTL_EL1PTEN),
|
||||
SR_TRAP(SYS_CNTP_CTL_EL0, CGT_CNTHCTL_EL1PTEN),
|
||||
|
|
|
|||
|
|
@ -178,7 +178,13 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
|
|||
|
||||
if (guest_owns_fp_regs()) {
|
||||
if (vcpu_has_sve(vcpu)) {
|
||||
__vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
|
||||
u64 zcr = read_sysreg_el1(SYS_ZCR);
|
||||
|
||||
/*
|
||||
* If the vCPU is in the hyp context then ZCR_EL1 is
|
||||
* loaded with its vEL2 counterpart.
|
||||
*/
|
||||
__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)) = zcr;
|
||||
|
||||
/*
|
||||
* Restore the VL that was saved when bound to the CPU,
|
||||
|
|
@ -189,11 +195,14 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
|
|||
* Note that this means that at guest exit ZCR_EL1 is
|
||||
* not necessarily the same as on guest entry.
|
||||
*
|
||||
* Restoring the VL isn't needed in VHE mode since
|
||||
* ZCR_EL2 (accessed via ZCR_EL1) would fulfill the same
|
||||
* role when doing the save from EL2.
|
||||
* ZCR_EL2 holds the guest hypervisor's VL when running
|
||||
* a nested guest, which could be smaller than the
|
||||
* max for the vCPU. Similar to above, we first need to
|
||||
* switch to a VL consistent with the layout of the
|
||||
* vCPU's SVE state. KVM support for NV implies VHE, so
|
||||
* using the ZCR_EL1 alias is safe.
|
||||
*/
|
||||
if (!has_vhe())
|
||||
if (!has_vhe() || (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)))
|
||||
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1,
|
||||
SYS_ZCR_EL1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -94,11 +94,19 @@ static int handle_smc(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
|
||||
/*
|
||||
* Guest access to FP/ASIMD registers are routed to this handler only
|
||||
* when the system doesn't support FP/ASIMD.
|
||||
* This handles the cases where the system does not support FP/ASIMD or when
|
||||
* we are running nested virtualization and the guest hypervisor is trapping
|
||||
* FP/ASIMD accesses by its guest guest.
|
||||
*
|
||||
* All other handling of guest vs. host FP/ASIMD register state is handled in
|
||||
* fixup_guest_exit().
|
||||
*/
|
||||
static int handle_no_fpsimd(struct kvm_vcpu *vcpu)
|
||||
static int kvm_handle_fpasimd(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (guest_hyp_fpsimd_traps_enabled(vcpu))
|
||||
return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
|
||||
|
||||
/* This is the case when the system doesn't support FP/ASIMD. */
|
||||
kvm_inject_undefined(vcpu);
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -209,6 +217,9 @@ static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu)
|
|||
*/
|
||||
static int handle_sve(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (guest_hyp_sve_traps_enabled(vcpu))
|
||||
return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
|
||||
|
||||
kvm_inject_undefined(vcpu);
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -304,7 +315,7 @@ static exit_handle_fn arm_exit_handlers[] = {
|
|||
[ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
|
||||
[ESR_ELx_EC_BKPT32] = kvm_handle_guest_debug,
|
||||
[ESR_ELx_EC_BRK64] = kvm_handle_guest_debug,
|
||||
[ESR_ELx_EC_FP_ASIMD] = handle_no_fpsimd,
|
||||
[ESR_ELx_EC_FP_ASIMD] = kvm_handle_fpasimd,
|
||||
[ESR_ELx_EC_PAC] = kvm_handle_ptrauth,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -314,11 +314,24 @@ static bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
|
||||
static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
/*
|
||||
* The vCPU's saved SVE state layout always matches the max VL of the
|
||||
* vCPU. Start off with the max VL so we can load the SVE state.
|
||||
*/
|
||||
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
|
||||
__sve_restore_state(vcpu_sve_pffr(vcpu),
|
||||
&vcpu->arch.ctxt.fp_regs.fpsr,
|
||||
true);
|
||||
write_sysreg_el1(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR);
|
||||
|
||||
/*
|
||||
* The effective VL for a VM could differ from the max VL when running a
|
||||
* nested guest, as the guest hypervisor could select a smaller VL. Slap
|
||||
* that into hardware before wrapping up.
|
||||
*/
|
||||
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
|
||||
sve_cond_update_zcr_vq(__vcpu_sys_reg(vcpu, ZCR_EL2), SYS_ZCR_EL2);
|
||||
|
||||
write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR);
|
||||
}
|
||||
|
||||
static inline void __hyp_sve_save_host(void)
|
||||
|
|
@ -354,10 +367,19 @@ static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
/* Only handle traps the vCPU can support here: */
|
||||
switch (esr_ec) {
|
||||
case ESR_ELx_EC_FP_ASIMD:
|
||||
/* Forward traps to the guest hypervisor as required */
|
||||
if (guest_hyp_fpsimd_traps_enabled(vcpu))
|
||||
return false;
|
||||
break;
|
||||
case ESR_ELx_EC_SYS64:
|
||||
if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu)))
|
||||
return false;
|
||||
fallthrough;
|
||||
case ESR_ELx_EC_SVE:
|
||||
if (!sve_guest)
|
||||
return false;
|
||||
if (guest_hyp_sve_traps_enabled(vcpu))
|
||||
return false;
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -65,6 +65,77 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu)
|
|||
return hcr | (__vcpu_sys_reg(vcpu, HCR_EL2) & ~NV_HCR_GUEST_EXCLUDE);
|
||||
}
|
||||
|
||||
static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 cptr;
|
||||
|
||||
/*
|
||||
* With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to
|
||||
* CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2,
|
||||
* except for some missing controls, such as TAM.
|
||||
* In this case, CPTR_EL2.TAM has the same position with or without
|
||||
* VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM
|
||||
* shift value for trapping the AMU accesses.
|
||||
*/
|
||||
u64 val = CPACR_ELx_TTA | CPTR_EL2_TAM;
|
||||
|
||||
if (guest_owns_fp_regs()) {
|
||||
val |= CPACR_ELx_FPEN;
|
||||
if (vcpu_has_sve(vcpu))
|
||||
val |= CPACR_ELx_ZEN;
|
||||
} else {
|
||||
__activate_traps_fpsimd32(vcpu);
|
||||
}
|
||||
|
||||
if (!vcpu_has_nv(vcpu))
|
||||
goto write;
|
||||
|
||||
/*
|
||||
* The architecture is a bit crap (what a surprise): an EL2 guest
|
||||
* writing to CPTR_EL2 via CPACR_EL1 can't set any of TCPAC or TTA,
|
||||
* as they are RES0 in the guest's view. To work around it, trap the
|
||||
* sucker using the very same bit it can't set...
|
||||
*/
|
||||
if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu))
|
||||
val |= CPTR_EL2_TCPAC;
|
||||
|
||||
/*
|
||||
* Layer the guest hypervisor's trap configuration on top of our own if
|
||||
* we're in a nested context.
|
||||
*/
|
||||
if (is_hyp_ctxt(vcpu))
|
||||
goto write;
|
||||
|
||||
cptr = vcpu_sanitised_cptr_el2(vcpu);
|
||||
|
||||
/*
|
||||
* Pay attention, there's some interesting detail here.
|
||||
*
|
||||
* The CPTR_EL2.xEN fields are 2 bits wide, although there are only two
|
||||
* meaningful trap states when HCR_EL2.TGE = 0 (running a nested guest):
|
||||
*
|
||||
* - CPTR_EL2.xEN = x0, traps are enabled
|
||||
* - CPTR_EL2.xEN = x1, traps are disabled
|
||||
*
|
||||
* In other words, bit[0] determines if guest accesses trap or not. In
|
||||
* the interest of simplicity, clear the entire field if the guest
|
||||
* hypervisor has traps enabled to dispel any illusion of something more
|
||||
* complicated taking place.
|
||||
*/
|
||||
if (!(SYS_FIELD_GET(CPACR_ELx, FPEN, cptr) & BIT(0)))
|
||||
val &= ~CPACR_ELx_FPEN;
|
||||
if (!(SYS_FIELD_GET(CPACR_ELx, ZEN, cptr) & BIT(0)))
|
||||
val &= ~CPACR_ELx_ZEN;
|
||||
|
||||
if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S2POE, IMP))
|
||||
val |= cptr & CPACR_ELx_E0POE;
|
||||
|
||||
val |= cptr & CPTR_EL2_TCPAC;
|
||||
|
||||
write:
|
||||
write_sysreg(val, cpacr_el1);
|
||||
}
|
||||
|
||||
static void __activate_traps(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 val;
|
||||
|
|
@ -91,30 +162,7 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
}
|
||||
|
||||
val = read_sysreg(cpacr_el1);
|
||||
val |= CPACR_ELx_TTA;
|
||||
val &= ~(CPACR_ELx_ZEN | CPACR_ELx_SMEN);
|
||||
|
||||
/*
|
||||
* With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to
|
||||
* CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2,
|
||||
* except for some missing controls, such as TAM.
|
||||
* In this case, CPTR_EL2.TAM has the same position with or without
|
||||
* VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM
|
||||
* shift value for trapping the AMU accesses.
|
||||
*/
|
||||
|
||||
val |= CPTR_EL2_TAM;
|
||||
|
||||
if (guest_owns_fp_regs()) {
|
||||
if (vcpu_has_sve(vcpu))
|
||||
val |= CPACR_ELx_ZEN;
|
||||
} else {
|
||||
val &= ~CPACR_ELx_FPEN;
|
||||
__activate_traps_fpsimd32(vcpu);
|
||||
}
|
||||
|
||||
write_sysreg(val, cpacr_el1);
|
||||
__activate_cptr_traps(vcpu);
|
||||
|
||||
write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el1);
|
||||
}
|
||||
|
|
@ -307,11 +355,63 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
return true;
|
||||
}
|
||||
|
||||
static bool kvm_hyp_handle_cpacr_el1(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
u64 esr = kvm_vcpu_get_esr(vcpu);
|
||||
int rt;
|
||||
|
||||
if (!is_hyp_ctxt(vcpu) || esr_sys64_to_sysreg(esr) != SYS_CPACR_EL1)
|
||||
return false;
|
||||
|
||||
rt = kvm_vcpu_sys_get_rt(vcpu);
|
||||
|
||||
if ((esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ) {
|
||||
vcpu_set_reg(vcpu, rt, __vcpu_sys_reg(vcpu, CPTR_EL2));
|
||||
} else {
|
||||
vcpu_write_sys_reg(vcpu, vcpu_get_reg(vcpu, rt), CPTR_EL2);
|
||||
__activate_cptr_traps(vcpu);
|
||||
}
|
||||
|
||||
__kvm_skip_instr(vcpu);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool kvm_hyp_handle_zcr_el2(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu));
|
||||
|
||||
if (!vcpu_has_nv(vcpu))
|
||||
return false;
|
||||
|
||||
if (sysreg != SYS_ZCR_EL2)
|
||||
return false;
|
||||
|
||||
if (guest_owns_fp_regs())
|
||||
return false;
|
||||
|
||||
/*
|
||||
* ZCR_EL2 traps are handled in the slow path, with the expectation
|
||||
* that the guest's FP context has already been loaded onto the CPU.
|
||||
*
|
||||
* Load the guest's FP context and unconditionally forward to the
|
||||
* slow path for handling (i.e. return false).
|
||||
*/
|
||||
kvm_hyp_handle_fpsimd(vcpu, exit_code);
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool kvm_hyp_handle_sysreg_vhe(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
if (kvm_hyp_handle_tlbi_el2(vcpu, exit_code))
|
||||
return true;
|
||||
|
||||
if (kvm_hyp_handle_cpacr_el1(vcpu, exit_code))
|
||||
return true;
|
||||
|
||||
if (kvm_hyp_handle_zcr_el2(vcpu, exit_code))
|
||||
return true;
|
||||
|
||||
return kvm_hyp_handle_sysreg(vcpu, exit_code);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -814,14 +814,13 @@ static void limit_nv_id_regs(struct kvm *kvm)
|
|||
NV_FTR(ISAR1, SPECRES));
|
||||
kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1, val);
|
||||
|
||||
/* No AMU, MPAM, S-EL2, RAS or SVE */
|
||||
/* No AMU, MPAM, S-EL2, or RAS */
|
||||
val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1);
|
||||
val &= ~(GENMASK_ULL(55, 52) |
|
||||
NV_FTR(PFR0, AMU) |
|
||||
NV_FTR(PFR0, MPAM) |
|
||||
NV_FTR(PFR0, SEL2) |
|
||||
NV_FTR(PFR0, RAS) |
|
||||
NV_FTR(PFR0, SVE) |
|
||||
NV_FTR(PFR0, EL3) |
|
||||
NV_FTR(PFR0, EL2) |
|
||||
NV_FTR(PFR0, EL1));
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ static bool get_el2_to_el1_mapping(unsigned int reg,
|
|||
MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );
|
||||
MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );
|
||||
MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
|
||||
MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1, NULL );
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
|
@ -2214,6 +2215,40 @@ static u64 reset_hcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
|
|||
return __vcpu_sys_reg(vcpu, r->reg) = val;
|
||||
}
|
||||
|
||||
static unsigned int sve_el2_visibility(const struct kvm_vcpu *vcpu,
|
||||
const struct sys_reg_desc *rd)
|
||||
{
|
||||
unsigned int r;
|
||||
|
||||
r = el2_visibility(vcpu, rd);
|
||||
if (r)
|
||||
return r;
|
||||
|
||||
return sve_visibility(vcpu, rd);
|
||||
}
|
||||
|
||||
static bool access_zcr_el2(struct kvm_vcpu *vcpu,
|
||||
struct sys_reg_params *p,
|
||||
const struct sys_reg_desc *r)
|
||||
{
|
||||
unsigned int vq;
|
||||
|
||||
if (guest_hyp_sve_traps_enabled(vcpu)) {
|
||||
kvm_inject_nested_sve_trap(vcpu);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!p->is_write) {
|
||||
p->regval = vcpu_read_sys_reg(vcpu, ZCR_EL2);
|
||||
return true;
|
||||
}
|
||||
|
||||
vq = SYS_FIELD_GET(ZCR_ELx, LEN, p->regval) + 1;
|
||||
vq = min(vq, vcpu_sve_max_vq(vcpu));
|
||||
vcpu_write_sys_reg(vcpu, vq - 1, ZCR_EL2);
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Architected system registers.
|
||||
* Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
|
||||
|
|
@ -2706,6 +2741,9 @@ static const struct sys_reg_desc sys_reg_descs[] = {
|
|||
EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
|
||||
EL2_REG_VNCR(HACR_EL2, reset_val, 0),
|
||||
|
||||
{ SYS_DESC(SYS_ZCR_EL2), .access = access_zcr_el2, .reset = reset_val,
|
||||
.visibility = sve_el2_visibility, .reg = ZCR_EL2 },
|
||||
|
||||
EL2_REG_VNCR(HCRX_EL2, reset_val, 0),
|
||||
|
||||
EL2_REG(TTBR0_EL2, access_rw, reset_val, 0),
|
||||
|
|
|
|||
Loading…
Reference in a new issue