From 216306346959aa21c20291457999a9413d86deb5 Mon Sep 17 00:00:00 2001 From: LiQian Date: Mon, 14 Jul 2025 09:55:52 +0800 Subject: [PATCH] phytium:support live migration on the phytium platform This patch emulate the ID registers. Support live migration on the phytium platform. Signed-off-by: LiQian --- arch/arm64/include/asm/cpufeature.h | 4 + arch/arm64/include/asm/kvm_host.h | 21 ++ arch/arm64/include/asm/sysreg.h | 5 + arch/arm64/kernel/cpufeature.c | 237 ++++++++++++ arch/arm64/kvm/sys_regs.c | 555 +++++++++++++++++++++++++++- virt/kvm/arm/arm.c | 4 + 6 files changed, 822 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index d4fd3b29e31b..425f0e208155 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -503,6 +503,10 @@ void check_local_cpu_capabilities(void); u64 read_sanitised_ftr_reg(u32 id); +#ifdef CONFIG_ARCH_PHYTIUM +int arm64_check_features_kvm(u32 sys_reg, u64 val, u64 limit); +#endif + static inline bool cpu_supports_mixed_endian_el0(void) { return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 01886b83d120..850ef4a6d389 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -63,6 +63,15 @@ struct kvm_vmid { u32 vmid; }; +#ifdef CONFIG_ARCH_PHYTIUM +/* + * (Op0, Op1, CRn, CRm, Op2) of ID registers is (3, 0, 0, crm, op2), + * where 0<=crm<8, 0<=op2<8. + */ +#define KVM_ARM_ID_REG_MAX_NUM 64 +#define IDREG_IDX(id) ((sys_reg_CRm(id) << 3) | sys_reg_Op2(id)) +#endif + struct kvm_arch { struct kvm_vmid vmid; @@ -84,6 +93,13 @@ struct kvm_arch { /* Mandated version of PSCI */ u32 psci_version; + +#ifdef CONFIG_ARCH_PHYTIUM + bool ran_once; + + /* ID registers for the guest. */ + u64 id_regs[KVM_ARM_ID_REG_MAX_NUM]; +#endif }; #define KVM_NR_MEM_OBJS 40 @@ -566,6 +582,11 @@ int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, static inline void __cpu_init_stage2(void) {} +#ifdef CONFIG_ARCH_PHYTIUM +void set_default_id_regs(struct kvm *kvm); +int kvm_set_id_reg_feature(struct kvm *kvm, u32 id, u8 field_shift, u8 fval); +#endif + /* Guest/host FPSIMD coordination helpers */ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 60c6affb9135..8f9c4e4ff68d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -776,6 +776,11 @@ /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */ #define SYS_MPIDR_SAFE_VAL (BIT(31)) +#ifdef CONFIG_ARCH_PHYTIUM +#define ARM64_FEATURE_FIELD_BITS 4 +#define ARM64_FEATURE_FIELD_MASK ((1ull << ARM64_FEATURE_FIELD_BITS) - 1) +#endif + #ifdef __ASSEMBLY__ .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6c0cdac84fb6..13cdaa65ce2a 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -587,6 +587,12 @@ static void __init init_cpu_hwcaps_indirect_list(void) static void __init setup_boot_cpu_capabilities(void); +#ifdef CONFIG_ARCH_PHYTIUM +#ifdef CONFIG_KVM +static void init_arm64_ftr_bits_kvm(void); +#endif +#endif + void __init init_cpu_features(struct cpuinfo_arm64 *info) { /* Before we start using the tables, make sure it is sorted */ @@ -642,6 +648,16 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) * after we have initialised the CPU feature infrastructure. */ setup_boot_cpu_capabilities(); + +#ifdef CONFIG_ARCH_PHYTIUM +#ifdef CONFIG_KVM + /* + * Initialize arm64_ftr_bits_kvm, which will be used to provide + * KVM with general feature checking for its guests. + */ + init_arm64_ftr_bits_kvm(); +#endif +#endif } static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new) @@ -2301,3 +2317,224 @@ ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, return sprintf(buf, "Vulnerable\n"); } + +#ifdef CONFIG_ARCH_PHYTIUM +#ifdef CONFIG_KVM +/* + * arm64_ftr_bits_kvm[] is used for KVM to check if features that are + * indicated in an ID register value for the guest are available on the host. + * arm64_ftr_bits_kvm[] is created based on arm64_ftr_regs[]. But, for + * registers for which arm64_ftr_bits_kvm_override[] has a corresponding + * entry, replace arm64_ftr_bits entries in arm64_ftr_bits_kvm[] with the + * ones in arm64_ftr_bits_kvm_override[]. + * + * What to add to arm64_ftr_bits_kvm_override[] shouldn't be decided according + * to KVM's implementation, but according to schemes of ID register fields. + * (e.g. For ID_AA64DFR0_EL1.PMUVER, a higher value on the field indicates + * more features. So, the arm64_ftr_bits' type for the field can be + * FTR_LOWER_SAFE instead of FTR_EXACT unlike arm64_ftr_regs) + */ + +/* + * The number of __ftr_reg_bits_entry entries in arm64_ftr_bits_kvm must be + * the same as the number of __ftr_reg_entry entries in arm64_ftr_regs. + */ +static struct __ftr_reg_bits_entry { + u32 sys_id; + struct arm64_ftr_bits *ftr_bits; +} arm64_ftr_bits_kvm[ARRAY_SIZE(arm64_ftr_regs)]; + +/* + * Number of arm64_ftr_bits entries for each register. + * (Number of 4 bits fields in 64 bit register + 1 entry for ARM64_FTR_END) + */ +#define MAX_FTR_BITS_LEN 17 + +/* Use FTR_LOWER_SAFE for AA64DFR0_EL1.PMUVER and AA64DFR0_EL1.DEBUGVER. */ +static struct arm64_ftr_bits ftr_id_aa64dfr0_kvm[MAX_FTR_BITS_LEN] = { + S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, + FTR_LOWER_SAFE, ID_AA64DFR0_PMUVER_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, + FTR_LOWER_SAFE, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6), + ARM64_FTR_END, +}; + +#define ARM64_FTR_REG_BITS(id, table) { \ + .sys_id = id, \ + .ftr_bits = &((table)[0]), \ +} + +/* + * All entries in arm64_ftr_bits_kvm_override[] are used to override + * the corresponding entries in arm64_ftr_bits_kvm[]. + */ +static struct __ftr_reg_bits_entry arm64_ftr_bits_kvm_override[] = { +ARM64_FTR_REG_BITS(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0_kvm), +}; + +/* + * Override entries in @orig_ftrp with the ones in @new_ftrp when their shift + * fields match. The last entry of @orig_ftrp and @new_ftrp must be + * ARM64_FTR_END (.width == 0). + */ +static void arm64_ftr_reg_bits_override(struct arm64_ftr_bits *orig_ftrp, + const struct arm64_ftr_bits *new_ftrp) +{ + const struct arm64_ftr_bits *n_ftrp; + struct arm64_ftr_bits *o_ftrp; + + for (n_ftrp = new_ftrp; n_ftrp->width; n_ftrp++) { + for (o_ftrp = orig_ftrp; o_ftrp->width; o_ftrp++) { + if (o_ftrp->shift == n_ftrp->shift) { + *o_ftrp = *n_ftrp; + break; + } + } + } +} + +/* + * Copy arm64_ftr_bits entries from @src_ftrp to @dst_ftrp. The last entries + * of @dst_ftrp and @src_ftrp must be ARM64_FTR_END (.width == 0). + */ +static void copy_arm64_ftr_bits(struct arm64_ftr_bits *dst_ftrp, + const struct arm64_ftr_bits *src_ftrp) +{ + int i = 0; + + for (; src_ftrp[i].width; i++) { + if (WARN_ON_ONCE(i >= (MAX_FTR_BITS_LEN - 1))) + break; + + dst_ftrp[i] = src_ftrp[i]; + } + + dst_ftrp[i].width = 0; +} + +/* + * Initialize arm64_ftr_bits_kvm. Copy arm64_ftr_bits for each ID register + * from arm64_ftr_regs to arm64_ftr_bits_kvm, and then override entries in + * arm64_ftr_bits_kvm with ones in arm64_ftr_bits_kvm_override. + */ +static void init_arm64_ftr_bits_kvm(void) +{ + struct arm64_ftr_bits ftr_temp[MAX_FTR_BITS_LEN]; + static struct __ftr_reg_bits_entry *bits, *o_bits; + int i, j; + + /* Copy entries from arm64_ftr_regs to arm64_ftr_bits_kvm */ + for (i = 0; i < ARRAY_SIZE(arm64_ftr_bits_kvm); i++) { + bits = &arm64_ftr_bits_kvm[i]; + bits->sys_id = arm64_ftr_regs[i].sys_id; + bits->ftr_bits = + (struct arm64_ftr_bits *) + arm64_ftr_regs[i].reg->ftr_bits; + }; + /* + * Override the entries in arm64_ftr_bits_kvm with the ones in + * arm64_ftr_bits_kvm_override. + */ + for (i = 0; i < ARRAY_SIZE(arm64_ftr_bits_kvm_override); i++) { + o_bits = &arm64_ftr_bits_kvm_override[i]; + for (j = 0; j < ARRAY_SIZE(arm64_ftr_bits_kvm); j++) { + bits = &arm64_ftr_bits_kvm[j]; + if (bits->sys_id != o_bits->sys_id) + continue; + + /* The code below tries to sustain the ordering of + * entries in bits even in o_bits, just in case + * arm64_ftr_bits entries in arm64_ftr_regs have + * any ordering requirements in the future (so that + * the ones in arm64_ftr_bits_kvm_override doesn't + * have to care). + * So, rather than directly copying them to empty + * slots in o_bits, the code simply copies entries + * from bits to o_bits first, then overrides them with + * original entries in o_bits. + */ + memset(ftr_temp, 0, sizeof(ftr_temp)); + /* + * Temporary save all entries in o_bits->ftr_bits + * to ftr_temp. + */ + copy_arm64_ftr_bits(ftr_temp, o_bits->ftr_bits); + + /* + * Copy entries from bits->ftr_bits to o_bits->ftr_bits. + */ + copy_arm64_ftr_bits(o_bits->ftr_bits, bits->ftr_bits); + + /* + * Override entries in o_bits->ftr_bits with the + * saved ones, and update bits->ftr_bits with + * o_bits->ftr_bits. + */ + arm64_ftr_reg_bits_override(o_bits->ftr_bits, ftr_temp); + bits->ftr_bits = o_bits->ftr_bits; + break; + } + } +} + +static int search_cmp_ftr_reg_bits(const void *id, const void *regp) +{ + return ((int)(unsigned long)id - + (int)((const struct __ftr_reg_bits_entry *) + regp)->sys_id); +} + +static const struct arm64_ftr_bits *get_arm64_ftr_bits_kvm(u32 sys_id) +{ + const struct __ftr_reg_bits_entry *ret; + + ret = bsearch((const void *)(unsigned long)sys_id, + arm64_ftr_bits_kvm, + ARRAY_SIZE(arm64_ftr_bits_kvm), + sizeof(arm64_ftr_bits_kvm[0]), + search_cmp_ftr_reg_bits); + if (ret) + return ret->ftr_bits; + + return NULL; +} + +/* + * Check if features (or levels of features) that are indicated in the ID + * register value @val are also indicated in @limit. + * This function is for KVM to check if features that are indicated in @val, + * which will be used as the ID register value for its guest, are supported + * on the host. + * For AA64MMFR0_EL1.TGranX_2 fields, which don't follow the standard ID + * scheme, the function checks if values of the fields in @val are the same + * as the ones in @limit. + */ +int arm64_check_features_kvm(u32 sys_reg, u64 val, u64 limit) +{ + const struct arm64_ftr_bits *ftrp = get_arm64_ftr_bits_kvm(sys_reg); + u64 exposed_mask = 0; + + if (!ftrp) + return -ENOENT; + + for (; ftrp->width; ftrp++) { + s64 ftr_val = arm64_ftr_value(ftrp, val); + s64 ftr_lim = arm64_ftr_value(ftrp, limit); + + exposed_mask |= arm64_ftr_mask(ftrp); + + if (ftr_val == ftr_lim) + continue; + + if (ftr_val != arm64_ftr_safe_value(ftrp, ftr_val, ftr_lim)) + return -E2BIG; + } + + /* Make sure that no unrecognized fields are set in @val. */ + if (val & ~exposed_mask) + return -E2BIG; + + return 0; +} +#endif /* CONFIG_KVM */ +#endif /* CONFIG_ARCH_PHYTIUM */ diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 605222ed2943..3ccd1e7b7628 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -35,6 +35,10 @@ #include "trace.h" +#ifdef CONFIG_ARCH_PHYTIUM +static u64 __read_id_reg(const struct kvm_vcpu *vcpu, u32 id); +#endif + /* * All of this file is extremly similar to the ARM coproc.c, but the * types are different. My gut feeling is that it should be pretty @@ -309,6 +313,300 @@ static bool trap_raz_wi(struct kvm_vcpu *vcpu, return read_zero(vcpu, p); } +#ifdef CONFIG_ARCH_PHYTIUM +struct id_reg_info { + /* Register ID */ + u32 sys_reg; + + /* + * Limit value of the register for a vcpu. The value is the sanitized + * system value with bits set/cleared for unsupported features for the + * guest. + */ + u64 vcpu_limit_val; + + /* Fields that are not validated by arm64_check_features_kvm. */ + u64 ignore_mask; + + /* An optional initialization function of the id_reg_info */ + void (*init)(struct id_reg_info *id_reg); + + /* + * This is an optional ID register specific validation function. When + * userspace tries to set the ID register, arm64_check_features_kvm() + * will check if the requested value indicates any features that cannot + * be supported by KVM on the host. But, some ID register fields need + * a special checking, and this function can be used for such fields. + * e.g. When SVE is configured for a vCPU by KVM_ARM_VCPU_INIT, + * ID_AA64PFR0_EL1.SVE shouldn't be set to 0 for the vCPU. + * The validation function for ID_AA64PFR0_EL1 could be used to check + * the field is consistent with SVE configuration. + */ + int (*validate)(const struct kvm_vcpu *vcpu, + const struct id_reg_info *id_reg, + u64 val); + + /* + * Return a bitmask of the vCPU's ID register fields that are not + * synced with saved (per VM) ID register value, which usually + * indicates opt-in CPU features that are not configured for the vCPU. + * ID registers are saved per VM, but some opt-in CPU features can + * be configured per vCPU. The saved (per VM) values for such + * features are for vCPUs with the features (and zero for + * vCPUs without the features). + * Return value of this function is used to handle such fields + * for per vCPU ID register read/write request with saved per VM + * ID register. See the __write_id_reg's comment for more detail. + */ + u64 (*vcpu_mask)(const struct kvm_vcpu *vcpu, + const struct id_reg_info *id_reg); +}; + +static void id_reg_info_init(struct id_reg_info *id_reg) +{ + u64 val = read_sanitised_ftr_reg(id_reg->sys_reg); + + id_reg->vcpu_limit_val = val; + if (id_reg->init) + id_reg->init(id_reg); + + /* + * id_reg->init() might update id_reg->vcpu_limit_val. + * Make sure that id_reg->vcpu_limit_val, which will be the default + * register value for guests, is a safe value to use for guests + * on the host. + */ + WARN_ON_ONCE(arm64_check_features_kvm(id_reg->sys_reg, + id_reg->vcpu_limit_val, val)); +} + +static int validate_id_aa64isar0_el1(const struct kvm_vcpu *vcpu, + const struct id_reg_info *id_reg, u64 val) +{ + unsigned int sm3, sm4, sha1, sha2, sha3; + + /* Run consistency checkings according to Arm ARM */ + sm3 = cpuid_feature_extract_unsigned_field(val, ID_AA64ISAR0_SM3_SHIFT); + sm4 = cpuid_feature_extract_unsigned_field(val, ID_AA64ISAR0_SM4_SHIFT); + if (sm3 != sm4) + return -EINVAL; + + sha1 = cpuid_feature_extract_unsigned_field + (val, ID_AA64ISAR0_SHA1_SHIFT); + sha2 = cpuid_feature_extract_unsigned_field + (val, ID_AA64ISAR0_SHA2_SHIFT); + if ((sha1 == 0) ^ (sha2 == 0)) + return -EINVAL; + + sha3 = cpuid_feature_extract_unsigned_field + (val, ID_AA64ISAR0_SHA3_SHIFT); + if (((sha2 == 2) ^ (sha3 == 1)) || (!sha1 && sha3)) + return -EINVAL; + + return 0; +} + +static int validate_id(const struct kvm_vcpu *vcpu, + const struct id_reg_info *id_reg, u64 val) +{ + return 0; +} + +static struct id_reg_info id_pfr0_el1_info = { + .sys_reg = SYS_ID_PFR0_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_pfr1_el1_info = { + .sys_reg = SYS_ID_PFR1_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_dfr0_el1_info = { + .sys_reg = SYS_ID_DFR0_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_mmfr0_el1_info = { + .sys_reg = SYS_ID_MMFR0_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_mmfr1_el1_info = { + .sys_reg = SYS_ID_MMFR1_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_mmfr2_el1_info = { + .sys_reg = SYS_ID_MMFR2_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_mmfr3_el1_info = { + .sys_reg = SYS_ID_MMFR3_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_mmfr4_el1_info = { + .sys_reg = SYS_ID_MMFR4_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_isar0_el1_info = { + .sys_reg = SYS_ID_ISAR0_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_isar1_el1_info = { + .sys_reg = SYS_ID_ISAR1_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_isar2_el1_info = { + .sys_reg = SYS_ID_ISAR2_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_isar3_el1_info = { + .sys_reg = SYS_ID_ISAR3_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_isar4_el1_info = { + .sys_reg = SYS_ID_ISAR4_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_isar5_el1_info = { + .sys_reg = SYS_ID_ISAR5_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_mvfr0_el1_info = { + .sys_reg = SYS_MVFR0_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_mvfr1_el1_info = { + .sys_reg = SYS_MVFR1_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_mvfr2_el1_info = { + .sys_reg = SYS_MVFR2_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_aa64pfr0_el1_info = { + .sys_reg = SYS_ID_AA64PFR0_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_aa64pfr1_el1_info = { + .sys_reg = SYS_ID_AA64PFR1_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_aa64dfr0_el1_info = { + .sys_reg = SYS_ID_AA64DFR0_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_aa64isar0_el1_info = { + .sys_reg = SYS_ID_AA64ISAR0_EL1, + .validate = validate_id_aa64isar0_el1, +}; + +static struct id_reg_info id_aa64isar1_el1_info = { + .sys_reg = SYS_ID_AA64ISAR1_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_aa64mmfr0_el1_info = { + .sys_reg = SYS_ID_AA64MMFR0_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_aa64mmfr1_el1_info = { + .sys_reg = SYS_ID_AA64MMFR1_EL1, + .validate = validate_id, +}; + +static struct id_reg_info id_aa64mmfr2_el1_info = { + .sys_reg = SYS_ID_AA64MMFR2_EL1, + .validate = validate_id, +}; + +/* + * An ID register that needs special handling to control the value for the + * guest must have its own id_reg_info in id_reg_info_table. + * (i.e. the reset value is different from the host's sanitized value, + * the value is affected by opt-in features, some fields need specific + * validation, etc.) + */ +#define GET_ID_REG_INFO(id) (id_reg_info_table[IDREG_IDX(id)]) + +static struct id_reg_info *id_reg_info_table[KVM_ARM_ID_REG_MAX_NUM] = { + [IDREG_IDX(SYS_ID_PFR0_EL1)] = &id_pfr0_el1_info, + [IDREG_IDX(SYS_ID_PFR1_EL1)] = &id_pfr1_el1_info, + [IDREG_IDX(SYS_ID_DFR0_EL1)] = &id_dfr0_el1_info, + [IDREG_IDX(SYS_ID_MMFR0_EL1)] = &id_mmfr0_el1_info, + [IDREG_IDX(SYS_ID_MMFR1_EL1)] = &id_mmfr1_el1_info, + [IDREG_IDX(SYS_ID_MMFR2_EL1)] = &id_mmfr2_el1_info, + [IDREG_IDX(SYS_ID_MMFR3_EL1)] = &id_mmfr3_el1_info, + [IDREG_IDX(SYS_ID_MMFR4_EL1)] = &id_mmfr4_el1_info, + [IDREG_IDX(SYS_ID_ISAR0_EL1)] = &id_isar0_el1_info, + [IDREG_IDX(SYS_ID_ISAR1_EL1)] = &id_isar1_el1_info, + [IDREG_IDX(SYS_ID_ISAR2_EL1)] = &id_isar2_el1_info, + [IDREG_IDX(SYS_ID_ISAR3_EL1)] = &id_isar3_el1_info, + [IDREG_IDX(SYS_ID_ISAR4_EL1)] = &id_isar4_el1_info, + [IDREG_IDX(SYS_ID_ISAR5_EL1)] = &id_isar5_el1_info, + [IDREG_IDX(SYS_MVFR0_EL1)] = &id_mvfr0_el1_info, + [IDREG_IDX(SYS_MVFR1_EL1)] = &id_mvfr1_el1_info, + [IDREG_IDX(SYS_MVFR2_EL1)] = &id_mvfr2_el1_info, + [IDREG_IDX(SYS_ID_AA64PFR0_EL1)] = &id_aa64pfr0_el1_info, + [IDREG_IDX(SYS_ID_AA64PFR1_EL1)] = &id_aa64pfr1_el1_info, + [IDREG_IDX(SYS_ID_AA64DFR0_EL1)] = &id_aa64dfr0_el1_info, + [IDREG_IDX(SYS_ID_AA64ISAR0_EL1)] = &id_aa64isar0_el1_info, + [IDREG_IDX(SYS_ID_AA64ISAR1_EL1)] = &id_aa64isar1_el1_info, + [IDREG_IDX(SYS_ID_AA64MMFR0_EL1)] = &id_aa64mmfr0_el1_info, + [IDREG_IDX(SYS_ID_AA64MMFR1_EL1)] = &id_aa64mmfr1_el1_info, + [IDREG_IDX(SYS_ID_AA64MMFR2_EL1)] = &id_aa64mmfr2_el1_info +}; + +static int validate_id_reg(const struct kvm_vcpu *vcpu, u32 id, u64 val) +{ + const struct id_reg_info *id_reg = GET_ID_REG_INFO(id); + u64 limit, tmp_val; + int err; + + if (id_reg) { + limit = id_reg->vcpu_limit_val; + /* + * Replace the fields that are indicated in ignore_mask with + * the value in the limit to not have arm64_check_features_kvm() + * check the field in @val. + */ + tmp_val = val & ~id_reg->ignore_mask; + tmp_val |= (limit & id_reg->ignore_mask); + } else { + limit = read_sanitised_ftr_reg(id); + tmp_val = val; + } + + /* Check if the value indicates any feature that is not in the limit. */ + err = arm64_check_features_kvm(id, tmp_val, limit); + if (err) + return err; + + if (id_reg && id_reg->validate) + /* Run the ID register specific validity check. */ + err = id_reg->validate(vcpu, id_reg, val); + + return err; +} +#endif + /* * ARMv8.1 mandates at least a trivial LORegion implementation, where all the * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0 @@ -319,7 +617,11 @@ static bool trap_loregion(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) { +#ifdef CONFIG_ARCH_PHYTIUM + u64 val = __read_id_reg(vcpu, SYS_ID_AA64MMFR1_EL1); +#else u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); +#endif u32 sr = sys_reg((u32)r->Op0, (u32)r->Op1, (u32)r->CRn, (u32)r->CRm, (u32)r->Op2); @@ -1080,13 +1382,110 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, return true; } +#ifdef CONFIG_ARCH_PHYTIUM +static bool is_id_reg(u32 id) +{ + return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 && + sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 0 && + sys_reg_CRm(id) < 8); +} + +static u64 read_kvm_id_reg(struct kvm *kvm, u32 id) +{ + return kvm->arch.id_regs[IDREG_IDX(id)]; +} + +static int __modify_kvm_id_reg(struct kvm *kvm, u32 id, u64 val, + u64 preserve_mask) +{ + u64 old, new; + + lockdep_assert_held(&kvm->lock); + + old = kvm->arch.id_regs[IDREG_IDX(id)]; + + /* Preserve the value at the bit position set in preserve_mask */ + new = old & preserve_mask; + new |= (val & ~preserve_mask); + + /* Don't allow to modify ID register value after KVM_RUN on any vCPUs */ + if (kvm->arch.ran_once && new != old) + return -EBUSY; + + WRITE_ONCE(kvm->arch.id_regs[IDREG_IDX(id)], new); + + return 0; +} + +static int modify_kvm_id_reg(struct kvm *kvm, u32 id, u64 val, + u64 preserve_mask) +{ + int ret; + + mutex_lock(&kvm->lock); + ret = __modify_kvm_id_reg(kvm, id, val, preserve_mask); + mutex_unlock(&kvm->lock); + + return ret; +} + +static int write_kvm_id_reg(struct kvm *kvm, u32 id, u64 val) +{ + return modify_kvm_id_reg(kvm, id, val, 0); +} + +/* + * KVM basically forces all vCPUs of the guest to have a uniform value for + * each ID register (it means KVM_SET_ONE_REG for a vCPU affects all + * the vCPUs of the guest), and the id_regs[] of kvm_arch holds values + * of ID registers for the guest. However, there is an exception for + * ID register fields corresponding to CPU features that can be + * configured per vCPU by KVM_ARM_VCPU_INIT, or etc (e.g. PMUv3, SVE, etc). + * For such fields, all vCPUs that have the feature will have a non-zero + * uniform value, which can be updated by userspace, but the vCPUs that + * don't have the feature will have zero for the fields. + * Values that @id_regs holds are for vCPUs that have such features. So, + * to get the ID register value for a vCPU that doesn't have those features, + * the corresponding fields in id_regs[] needs to be cleared. + * A bitmask of the fields are provided by id_reg_info's vcpu_mask(), and + * __write_id_reg() and __read_id_reg() take care of those fields using + * the bitmask. + */ +static int __write_id_reg(const struct kvm_vcpu *vcpu, u32 id, u64 val) +{ + const struct id_reg_info *id_reg = GET_ID_REG_INFO(id); + u64 mask = 0; + + if (id_reg && id_reg->vcpu_mask) + mask = id_reg->vcpu_mask(vcpu, id_reg); + + /* + * Update the ID register for the guest with @val, except for fields + * that are set in the mask, which indicates fields for opt-in + * features that are not configured for the vCPU. + */ + return modify_kvm_id_reg(vcpu->kvm, id, val, mask); +} +#endif + /* Read a sanitised cpufeature ID register by sys_reg_desc */ +#ifdef CONFIG_ARCH_PHYTIUM +static u64 __read_id_reg(const struct kvm_vcpu *vcpu, u32 id) +{ + const struct id_reg_info *id_reg = GET_ID_REG_INFO(id); + u64 val = read_kvm_id_reg(vcpu->kvm, id); + + if (id_reg && id_reg->vcpu_mask) + /* Clear fields for opt-in features that are not configured. */ + val &= ~(id_reg->vcpu_mask(vcpu, id_reg)); +#else static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r, bool raz) { u32 id = sys_reg((u32)r->Op0, (u32)r->Op1, - (u32)r->CRn, (u32)r->CRm, (u32)r->Op2); + (u32)r->CRn, (u32)r->CRm, (u32)r->Op2); u64 val = raz ? 0 : read_sanitised_ftr_reg(id); +#endif if (id == SYS_ID_AA64PFR0_EL1 && !vcpu_has_sve(vcpu)) { val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT); @@ -1100,6 +1499,16 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, return val; } +#ifdef CONFIG_ARCH_PHYTIUM +static u64 read_id_reg(const struct kvm_vcpu *vcpu, + struct sys_reg_desc const *r, bool raz) +{ + u32 id = reg_to_encoding(r); + + return raz ? 0 : __read_id_reg(vcpu, id); +} +#endif + /* cpufeature ID register access trap handlers */ static bool __access_id_reg(struct kvm_vcpu *vcpu, @@ -1172,6 +1581,7 @@ static int get_id_aa64zfr0_el1(struct kvm_vcpu *vcpu, return reg_to_user(uaddr, &val, reg->id); } +#ifndef CONFIG_ARCH_PHYTIUM static int set_id_aa64zfr0_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, const struct kvm_one_reg *reg, void __user *uaddr) @@ -1190,7 +1600,7 @@ static int set_id_aa64zfr0_el1(struct kvm_vcpu *vcpu, return 0; } - +#endif /* * cpufeature ID register user accessors * @@ -1208,23 +1618,89 @@ static int __get_id_reg(const struct kvm_vcpu *vcpu, return reg_to_user(uaddr, &val, id); } +#ifdef CONFIG_ARCH_PHYTIUM +/* + * Check if the given id indicates AArch32 ID register encoding. + */ +static bool is_aarch32_id_reg(u32 id) +{ + u32 crm, op2; + + if (!is_id_reg(id)) + return false; + + crm = sys_reg_CRm(id); + op2 = sys_reg_Op2(id); + if (crm == 1 || crm == 2 || (crm == 3 && (op2 != 3 && op2 != 7))) + /* AArch32 ID register */ + return true; + + return false; +} +#endif + static int __set_id_reg(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, void __user *uaddr, bool raz) { - const u64 id = sys_reg_to_index(rd); int err; u64 val; + const u64 id = sys_reg_to_index(rd); +#ifdef CONFIG_ARCH_PHYTIUM + u8 csv2; + u32 encoding = reg_to_encoding(rd); +#endif err = reg_from_user(&val, uaddr, id); if (err) return err; - /* This is what we mean by invariant: you can't change it. */ +#ifdef CONFIG_ARCH_PHYTIUM + if (val == read_id_reg(vcpu, rd, raz)) + /* The value is same as the current value. Nothing to do. */ + return 0; + + /* + * Don't allow to modify the register's value if the register is raz, + * or the reg doesn't have the id_reg_info. + */ + if (raz || !GET_ID_REG_INFO(encoding)) + return -EINVAL; + + /* + * Skip the validation of AArch32 ID registers if the system doesn't + * 32bit EL0 (their value are UNKNOWN). + */ + if (system_supports_32bit_el0() || !is_aarch32_id_reg(encoding)) { + err = validate_id_reg(vcpu, encoding, val); + if (err) + return err; + } + + /* + * Allow AA64PFR0_EL1.CSV2 to be set from userspace as long as + * it doesn't promise more than what is actually provided (the + * guest could otherwise be covered in ectoplasmic residue). + */ + if (encoding == SYS_ID_AA64ZFR0_EL1) { + if (WARN_ON(!vcpu_has_sve(vcpu))) + return -ENOENT; + + err = reg_from_user(&val, uaddr, id); + if (err) + return err; + + if (val != guest_id_aa64zfr0_el1(vcpu)) + return -EINVAL; + } + + return __write_id_reg(vcpu, encoding, val); +#else if (val != read_id_reg(vcpu, rd, raz)) return -EINVAL; return 0; +#endif } static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, @@ -1442,7 +1918,11 @@ static const struct sys_reg_desc sys_reg_descs[] = { ID_SANITISED(ID_AA64PFR1_EL1), ID_UNALLOCATED(4,2), ID_UNALLOCATED(4,3), +#ifdef CONFIG_ARCH_PHYTIUM + ID_SANITISED(ID_AA64ZFR0_EL1), +#else { SYS_DESC(SYS_ID_AA64ZFR0_EL1), access_id_aa64zfr0_el1, .get_user = get_id_aa64zfr0_el1, .set_user = set_id_aa64zfr0_el1, }, +#endif ID_UNALLOCATED(4,5), ID_UNALLOCATED(4,6), ID_UNALLOCATED(4,7), @@ -1651,8 +2131,13 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu, if (p->is_write) { return ignore_write(vcpu, p); } else { +#ifdef CONFIG_ARCH_PHYTIUM + u64 dfr = __read_id_reg(vcpu, SYS_ID_AA64DFR0_EL1); + u64 pfr = __read_id_reg(vcpu, SYS_ID_AA64PFR0_EL1); +#else u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1); u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); +#endif u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT); p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) | @@ -2741,6 +3226,21 @@ static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n) return 0; } +#ifdef CONFIG_ARCH_PHYTIUM +static void id_reg_info_init_all(void) +{ + int i; + struct id_reg_info *id_reg; + + for (i = 0; i < ARRAY_SIZE(id_reg_info_table); i++) { + id_reg = (struct id_reg_info *)id_reg_info_table[i]; + if (!id_reg) + continue; + id_reg_info_init(id_reg); + } +} +#endif + void kvm_sys_reg_table_init(void) { unsigned int i; @@ -2775,7 +3275,54 @@ void kvm_sys_reg_table_init(void) break; /* Clear all higher bits. */ cache_levels &= (1 << (i*3))-1; + +#ifdef CONFIG_ARCH_PHYTIUM + id_reg_info_init_all(); +#endif +} + +#ifdef CONFIG_ARCH_PHYTIUM +/* + * Set the guest's ID registers that are defined in sys_reg_descs[] + * with ID_SANITISED() to the host's sanitized value. + */ +void set_default_id_regs(struct kvm *kvm) +{ + int i; + u32 id; + const struct sys_reg_desc *rd; + u64 val; + struct id_reg_info *idr; + + for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) { + rd = &sys_reg_descs[i]; + if (rd->access != access_id_reg) + /* Not ID register or hidden/reserved ID register */ + continue; + id = reg_to_encoding(rd); + if (WARN_ON_ONCE(!is_id_reg(id))) + /* Shouldn't happen */ + continue; + + idr = GET_ID_REG_INFO(id); + val = idr ? idr->vcpu_limit_val : read_sanitised_ftr_reg(id); + WARN_ON_ONCE(write_kvm_id_reg(kvm, id, val)); + } +} + +/* + * Update the ID register's field with @fval for the guest. + * The caller is expected to hold the kvm->lock. + * This will not fail unless any vCPUs in the guest have started. + */ +int kvm_set_id_reg_feature(struct kvm *kvm, u32 id, u8 field_shift, u8 fval) +{ + u64 val = ((u64)fval & ARM64_FEATURE_FIELD_MASK) << field_shift; + u64 preserve_mask = ~(ARM64_FEATURE_FIELD_MASK << field_shift); + + return __modify_kvm_id_reg(kvm, id, val, preserve_mask); } +#endif /** * kvm_reset_sys_regs - sets system registers to reset value diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index 28cdd2f4d1ef..254537f5ed00 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -135,6 +135,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.max_vcpus = vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; +#ifdef CONFIG_ARCH_PHYTIUM + set_default_id_regs(kvm); +#endif + return ret; out_free_stage2_pgd: kvm_free_stage2_pgd(kvm); -- Gitee