diff -r 187abacabbe9 sys/dev/nvmm/x86/nvmm_x86.c --- a/sys/dev/nvmm/x86/nvmm_x86.c Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86.c Tue Jul 14 18:14:28 2026 +0000 @@ -258,7 +258,7 @@ const struct nvmm_x86_cpuid_mask nvmm_cp CPUID2_AESNI | CPUID2_XSAVE | CPUID2_OSXSAVE | - /* CPUID2_AVX excluded */ + CPUID2_AVX | CPUID2_F16C | CPUID2_RDRAND, /* CPUID2_RAZ excluded */ @@ -302,7 +302,7 @@ const struct nvmm_x86_cpuid_mask nvmm_cp /* CPUID_SEF_SGX excluded */ CPUID_SEF_BMI1 | /* CPUID_SEF_HLE excluded */ - /* CPUID_SEF_AVX2 excluded */ + CPUID_SEF_AVX2 | CPUID_SEF_FDPEXONLY | CPUID_SEF_SMEP | CPUID_SEF_BMI2 | @@ -466,3 +466,58 @@ nvmm_x86_pat_validate(uint64_t val) return true; } + +/* + * nvmm_x86_xsave_size(xcr0) + * + * Returns the maximum XSAVE area size in bytes needed to + * represent all user state components corresponding to bits set + * in xcr0. Bit 63, which is reserved for future architecture + * extension at the time of writing, MUST NOT be set. + */ +uint32_t +nvmm_x86_xsave_size(uint64_t xcr0) +{ + uint32_t totalsize = sizeof(struct xsave_header); + unsigned i; + + /* + * Caller must not pass bit 63 until the architectural + * extension mechanism it is reserved for has been defined. + * Caller must also not pass any bits that the CPU has not + * advertised support for. + */ + KASSERTMSG((xcr0 & ~__BITS(62, 0)) == 0, "xcr0=0x%"PRIx64, xcr0); + KASSERTMSG((xcr0 & ~x86_xsave_features) == 0, + "xcr0=0x%"PRIx64" x86_xsave_features=0x%"PRIx64, + xcr0, x86_xsave_features); + + CTASSERT(sizeof(struct xsave_header) == 512 + 64); + for (i = 2; i < 63; i++) { + uint32_t descs[4]; + uint32_t size, offset; + + /* + * Skip state components that are not of interest to + * the caller. + */ + if ((xcr0 & __BIT(i)) == 0) + continue; + + /* + * Can't use x86_xsave_offsets[i] + x86_xsave_sizes[i] + * because the NetBSD kernel only queries those for the + * user state components it knows about, but in + * principle this should support any user state + * component the guest wants to use even if NetBSD + * doesn't know how. + */ + x86_cpuid2(0x0d, i, descs); + size = descs[0]; /* CPUID[EAX=0x0d,ECX=i].EAX */ + offset = descs[1]; /* CPUID[EAX=0x0d,ECX=i].EBX */ + KASSERT(size <= UINT32_MAX - offset); + totalsize = MAX(totalsize, offset + size); + } + + return totalsize; +} diff -r 187abacabbe9 sys/dev/nvmm/x86/nvmm_x86.h --- a/sys/dev/nvmm/x86/nvmm_x86.h Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86.h Tue Jul 14 18:14:28 2026 +0000 @@ -132,6 +132,7 @@ struct nvmm_cap_md { uint64_t vcpu_conf_support; #define NVMM_CAP_ARCH_VCPU_CONF_CPUID __BIT(0) #define NVMM_CAP_ARCH_VCPU_CONF_TPR __BIT(1) +#define NVMM_CAP_ARCH_VCPU_CONF_XCR0_MASK __BIT(2) uint64_t xcr0_mask; uint32_t mxcsr_mask; @@ -269,6 +270,7 @@ struct nvmm_x64_state { #define NVMM_VCPU_CONF_CPUID NVMM_VCPU_CONF_MD_BEGIN #define NVMM_VCPU_CONF_TPR (NVMM_VCPU_CONF_MD_BEGIN + 1) +#define NVMM_VCPU_CONF_XCR0_MASK (NVMM_VCPU_CONF_MD_BEGIN + 2) struct nvmm_vcpu_conf_cpuid { /* The options. */ @@ -309,7 +311,7 @@ struct nvmm_vcpu_conf_tpr { #ifdef _KERNEL #define NVMM_X86_MACH_NCONF 0 -#define NVMM_X86_VCPU_NCONF 2 +#define NVMM_X86_VCPU_NCONF 3 struct nvmm_x86_cpuid_mask { uint32_t eax; uint32_t ebx; @@ -323,6 +325,7 @@ extern const struct nvmm_x86_cpuid_mask extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000007; extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000008; bool nvmm_x86_pat_validate(uint64_t); +uint32_t nvmm_x86_xsave_size(uint64_t); #endif #endif /* ASM_NVMM */ diff -r 187abacabbe9 sys/dev/nvmm/x86/nvmm_x86_svm.c --- a/sys/dev/nvmm/x86/nvmm_x86_svm.c Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86_svm.c Tue Jul 14 18:14:28 2026 +0000 @@ -499,7 +499,6 @@ static kmutex_t svm_asidlock __cacheline static bool svm_decode_assist __read_mostly; static uint32_t svm_ctrl_tlb_flush __read_mostly; -#define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) static uint64_t svm_xcr0_mask __read_mostly; #define SVM_NCPUIDS 32 @@ -560,7 +559,9 @@ static const size_t svm_vcpu_conf_sizes[ [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = sizeof(struct nvmm_vcpu_conf_cpuid), [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = - sizeof(struct nvmm_vcpu_conf_tpr) + sizeof(struct nvmm_vcpu_conf_tpr), + [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK)] = + sizeof(uint64_t), }; struct svm_cpudata { @@ -601,11 +602,17 @@ struct svm_cpudata { uint64_t gprs[NVMM_X64_NGPR]; uint64_t drs[NVMM_X64_NDR]; uint64_t gtsc; - struct xsave_header gfpu __aligned(64); /* VCPU configuration. */ bool cpuidpresent[SVM_NCPUIDS]; struct nvmm_vcpu_conf_cpuid cpuid[SVM_NCPUIDS]; + uint64_t xcr0_mask; + + /* + * Guest XSAVE state. Must be the last member because it may + * be extended variably by whatever CPU we're running on. + */ + struct xsave_header gfpu __aligned(64); }; static void @@ -960,20 +967,19 @@ svm_inkernel_handle_cpuid(struct nvmm_cp cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; case 0x0000000D: /* Processor Extended State Enumeration */ - if (svm_xcr0_mask == 0) { + if (cpudata->xcr0_mask == 0) { break; } switch (ecx) { case 0: - cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; - if (cpudata->gxcr0 & XCR0_SSE) { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); - } else { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); - } - cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ - cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; - cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; + cpudata->vmcb->state.rax = + cpudata->xcr0_mask & 0xFFFFFFFF; + cpudata->gprs[NVMM_X64_GPR_RBX] = + nvmm_x86_xsave_size(cpudata->gxcr0); + cpudata->gprs[NVMM_X64_GPR_RCX] = + nvmm_x86_xsave_size(cpudata->xcr0_mask); + cpudata->gprs[NVMM_X64_GPR_RDX] = + cpudata->xcr0_mask >> 32; break; case 1: cpudata->vmcb->state.rax &= @@ -983,6 +989,21 @@ svm_inkernel_handle_cpuid(struct nvmm_cp cpudata->gprs[NVMM_X64_GPR_RCX] = 0; cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; + case 2 ... 62: + /* + * CPUID[EAX=0x0d,ECX=n], 2 <= n <= 62: size + * and offset of nth component in XSAVE area. + * If the nth bit of XCR0 is disabled in the + * vCPU configuration, we return all-zero + * instead. + */ + if ((cpudata->xcr0_mask & __BIT(ecx)) == 0) { + cpudata->vmcb->state.rax = 0; + cpudata->gprs[NVMM_X64_GPR_RBX] = 0; + cpudata->gprs[NVMM_X64_GPR_RCX] = 0; + cpudata->gprs[NVMM_X64_GPR_RDX] = 0; + } + break; default: cpudata->vmcb->state.rax = 0; cpudata->gprs[NVMM_X64_GPR_RBX] = 0; @@ -1377,7 +1398,7 @@ svm_exit_xsetbv(struct nvmm_machine *mac goto error; } else if (__predict_false(vmcb->state.cpl != 0)) { goto error; - } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { + } else if (__predict_false((val & ~cpudata->xcr0_mask) != 0)) { goto error; } else if (__predict_false((val & XCR0_X87) == 0)) { goto error; @@ -1406,14 +1427,65 @@ svm_vcpu_guest_fpu_enter(struct nvmm_cpu { struct svm_cpudata *cpudata = vcpu->cpudata; + /* + * The guest's XCR0 had better not have any bits that aren't + * allowed in the vCPU configuration, and the current XSAVE + * area had better not store any either according to + * cpudata->gfpu.xsh_xstate_bv. + * + * Note that XRSTOR will trap if XSTATE_BV has any bits that + * are not set in XCR0. + */ + KASSERT((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0); + KASSERT((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0); + + /* + * Save anything in the FPU registers that this thread might + * have been using to memory, and raise the IPL to IPL_VM to + * block interrupt handlers that might use the FPU. + * + * After this point, we are free to use the FPU registers. + */ fpu_kern_enter(); + + /* + * If the guest is allowed to use any extended CPU state + * (including FPU registers), set XCR0 to allow access to + * anything the guest has previously used and is saved to + * memory, _and_ to anything the guest has asked to use in + * cpudata->gxcr0. + * + * The guest may have used some extended CPU state like the + * zmmN registers, and then later disabled them in XCR0; in + * that case, the state must be preserved in case the guest + * later enables it in XCR0, but we can only load while all + * bits in cpudata->gfpu.xsh_xstate_bv are set in XCR0. + * + * Similarly, the guest may _not_ have used some extended CPU + * state since reset, but may have since enabled it in XCR0. + * Such state will be clear in cpudata->gfpu.xsh_xstate_bv and + * must be initialized afresh by the CPU, which requires the + * bits be set in XCR0 to allow that. + */ + cpudata->hxcr0 = rdxcr(0); + wrxcr(0, cpudata->xcr0_mask & + (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)); + + /* + * Load the guest's saved extended CPU state from memory into + * the CPU. + */ /* TODO: should we use *XSAVE64 here? */ - fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask, false); - - if (svm_xcr0_mask != 0) { - cpudata->hxcr0 = rdxcr(0); - wrxcr(0, cpudata->gxcr0); - } + fpu_area_restore(&cpudata->gfpu, cpudata->xcr0_mask, false); + + /* + * If we temporarily set XCR0 beyond what the guest asked for + * in order to restore state that is currently disabled, reduce + * it down to what the guest asked for. + */ + if (__predict_false(cpudata->gxcr0 != (cpudata->xcr0_mask & + (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)))) + wrxcr(0, cpudata->xcr0_mask & cpudata->gxcr0); } static void @@ -1421,14 +1493,57 @@ svm_vcpu_guest_fpu_leave(struct nvmm_cpu { struct svm_cpudata *cpudata = vcpu->cpudata; - if (svm_xcr0_mask != 0) { - cpudata->gxcr0 = rdxcr(0); + /* + * In case the guest has cleared some XCR0 bits but used the + * corresponding extended CPU state, increase XCR0 to the + * maximum supported for this guest before we XSAVE. + * + * Note that XRSTOR will trap if XSTATE_BV has any bits that + * are not set in XCR0. + */ + cpudata->gxcr0 = rdxcr(0); + wrxcr(0, cpudata->xcr0_mask); + + /* + * Paranoia: Ensure the guest's XCR0 has no forbidden bits. + * Should not be possible because we filter them on XSETBV + * exits. + */ + KASSERT((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0); + cpudata->gxcr0 &= cpudata->xcr0_mask; + + /* + * Save any extended CPU state that could be in use by the + * guest. + */ + /* TODO: should we use *XSAVE64 here? */ + fpu_area_save(&cpudata->gfpu, cpudata->xcr0_mask, false); + + /* + * If the host XCR0 is different from the maximum guest XCR0, + * switch back to the host XCR0 so we can restore NetBSD's FPU + * state. + */ + if (cpudata->xcr0_mask != cpudata->hxcr0) wrxcr(0, cpudata->hxcr0); - } - - /* TODO: should we use *XSAVE64 here? */ - fpu_area_save(&cpudata->gfpu, svm_xcr0_mask, false); + + /* + * Restore any FPU registers that we might have saved in + * svm_vcpu_guest_fpu_enter for this thread, and restore the + * IPL from IPL_VM. + * + * After this point, we must not touch the FPU registers. + */ fpu_kern_leave(); + + /* + * The guest's XCR0 had better not have any bits that aren't + * allowed in the vCPU configuration, and the current XSAVE + * area had better not store any either according to + * cpudata->gfpu.xsh_xstate_bv. + */ + KASSERT((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0); + KASSERT((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0); } static void @@ -1915,10 +2030,10 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu) vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], VMCB_CTRL_V_TPR); - if (svm_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { /* Clear illegal XCR0 bits, set mandatory X87 bit. */ cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; - cpudata->gxcr0 &= svm_xcr0_mask; + cpudata->gxcr0 &= cpudata->xcr0_mask; cpudata->gxcr0 |= XCR0_X87; } } @@ -1983,11 +2098,15 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu) fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; - if (svm_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { /* Reset XSTATE_BV, to force a reload. */ - cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; } } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ svm_vmcb_cache_update(vmcb, flags); @@ -2093,6 +2212,10 @@ svm_vcpu_getstate(struct nvmm_cpu *vcpu) memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, sizeof(state->fpu)); } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ comm->state_wanted = 0; comm->state_cached |= flags; @@ -2282,7 +2405,8 @@ svm_vcpu_init(struct nvmm_machine *mach, vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; + cpudata->xcr0_mask = svm_xcr0_mask; + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; cpudata->gfpu.xsh_xcomp_bv = 0; /* These MSRs are static. */ @@ -2302,12 +2426,27 @@ svm_vcpu_init(struct nvmm_machine *mach, static int svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) { + size_t xsave_size, cpudata_size; struct svm_cpudata *cpudata; int error; + /* + * Compute the size of the SVM cpudata. We put the + * variable-length XSAVE area at the end so if it's small + * enough, it stays within a single page. We size the XSAVE + * area for the maximum set of features supported by the CPU + * which a guest can enable (which may be more than the NetBSD + * host enables for itself -- hence we don't use + * x86_fpu_save_size here!). + */ + xsave_size = nvmm_x86_xsave_size(svm_xcr0_mask); + KASSERT(xsave_size < SIZE_MAX - offsetof(struct svm_cpudata, gfpu)); + cpudata_size = MAX(sizeof(*cpudata), + offsetof(struct svm_cpudata, gfpu) + xsave_size); + /* Allocate the SVM cpudata. */ cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, + roundup(cpudata_size, PAGE_SIZE), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); vcpu->cpudata = cpudata; @@ -2425,6 +2564,39 @@ svm_vcpu_configure_cpuid(struct svm_cpud } static int +svm_vcpu_configure_xcr0_mask(struct svm_cpudata *cpudata, void *data) +{ + const uint64_t *xcr0_maskp = data; + + /* + * Refuse to enable XCR0 bits (extended CPU state components) + * not supported by this system. + */ + if (*xcr0_maskp & ~svm_xcr0_mask) + return EINVAL; + + /* + * Out of paranoia, clear any existing extended CPU state. + * This operation is unlikely to be used before the guest has + * begun execution at all, so the extended CPU state is + * probably all zero. But in case some weird hypervisor + * software tries to change the XCR0 mask dynamically, let's + * avoid accidentally leaking things through any extended CPU + * state. + */ + memset(&cpudata->gfpu, 0, nvmm_x86_xsave_size(svm_xcr0_mask)); + + /* + * Set the XCR0 mask, and limit the guest's XCR0 to this mask. + * Any extended CPU state the guest had previously been using + * will be wiped out. + */ + cpudata->xcr0_mask = *xcr0_maskp; + cpudata->gxcr0 &= cpudata->xcr0_mask; + return 0; +} + +static int svm_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) { struct svm_cpudata *cpudata = vcpu->cpudata; @@ -2432,6 +2604,8 @@ svm_vcpu_configure(struct nvmm_cpu *vcpu switch (op) { case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID): return svm_vcpu_configure_cpuid(cpudata, data); + case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK): + return svm_vcpu_configure_xcr0_mask(cpudata, data); default: return EINVAL; } @@ -2602,8 +2776,22 @@ svm_init(void) /* Init the ASID. */ svm_init_asid(descs[1]); - /* Init the XCR0 mask. */ - svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; + /* + * Init the XCR0 mask. + * + * x86_xsave_features is the cached result of + * CPUID[EAX=0x0000000d,ECX=0].EDX:EAX, the set of all + * supported XCR0 bits for user XSAVE state components on the + * physical CPU. Hypervisor software can use + * nvmm_vcpu_configure(NVMM_VCPU_CONF_XCR0_MASK) to restrict + * the available features on a per-vCPU basis, e.g. in order to + * limit guests to compatible features for migration. + * + * Out of paranoia, we mask off bit 63 which is reserved for + * future extension which we don't understand because it's not + * yet defined. + */ + svm_xcr0_mask = x86_xsave_features & __BITS(62, 0); /* Init the max basic CPUID leaf. */ svm_cpuid_max_basic = uimin(cpuid_level, SVM_CPUID_MAX_BASIC); diff -r 187abacabbe9 sys/dev/nvmm/x86/nvmm_x86_vmx.c --- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c Tue Jul 14 18:14:28 2026 +0000 @@ -717,7 +717,6 @@ static uint8_t *vmx_asidmap __read_mostl static uint32_t vmx_maxasid __read_mostly; static kmutex_t vmx_asidlock __cacheline_aligned; -#define VMX_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) static uint64_t vmx_xcr0_mask __read_mostly; #define VMX_NCPUIDS 32 @@ -775,7 +774,9 @@ static const size_t vmx_vcpu_conf_sizes[ [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = sizeof(struct nvmm_vcpu_conf_cpuid), [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = - sizeof(struct nvmm_vcpu_conf_tpr) + sizeof(struct nvmm_vcpu_conf_tpr), + [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK)] = + sizeof(uint64_t), }; struct vmx_cpudata { @@ -820,12 +821,18 @@ struct vmx_cpudata { uint64_t gprs[NVMM_X64_NGPR]; uint64_t drs[NVMM_X64_NDR]; uint64_t gtsc; - struct xsave_header gfpu __aligned(64); /* VCPU configuration. */ bool cpuidpresent[VMX_NCPUIDS]; struct nvmm_vcpu_conf_cpuid cpuid[VMX_NCPUIDS]; struct nvmm_vcpu_conf_tpr tpr; + uint64_t xcr0_mask; + + /* + * Guest XSAVE state. Must be the last member because it may + * be extended variably by whatever CPU we're running on. + */ + struct xsave_header gfpu __aligned(64); }; static const struct { @@ -1407,20 +1414,19 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; case 0x0000000D: /* Processor Extended State Enumeration */ - if (vmx_xcr0_mask == 0) { + if (cpudata->xcr0_mask == 0) { break; } switch (ecx) { case 0: - cpudata->gprs[NVMM_X64_GPR_RAX] = vmx_xcr0_mask & 0xFFFFFFFF; - if (cpudata->gxcr0 & XCR0_SSE) { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); - } else { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); - } - cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ - cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; - cpudata->gprs[NVMM_X64_GPR_RDX] = vmx_xcr0_mask >> 32; + cpudata->gprs[NVMM_X64_GPR_RAX] = + cpudata->xcr0_mask & 0xFFFFFFFF; + cpudata->gprs[NVMM_X64_GPR_RBX] = + nvmm_x86_xsave_size(cpudata->gxcr0); + cpudata->gprs[NVMM_X64_GPR_RCX] = + nvmm_x86_xsave_size(cpudata->xcr0_mask); + cpudata->gprs[NVMM_X64_GPR_RDX] = + cpudata->xcr0_mask >> 32; break; case 1: cpudata->gprs[NVMM_X64_GPR_RAX] &= @@ -1430,6 +1436,21 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma cpudata->gprs[NVMM_X64_GPR_RCX] = 0; cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; + case 2 ... 62: + /* + * CPUID[EAX=0x0d,ECX=n], 2 <= n <= 62: size + * and offset of nth component in XSAVE area. + * If the nth bit of XCR0 is disabled in the + * vCPU configuration, we return all-zero + * instead. + */ + if ((cpudata->xcr0_mask & __BIT(ecx)) == 0) { + cpudata->gprs[NVMM_X64_GPR_RAX] = 0; + cpudata->gprs[NVMM_X64_GPR_RBX] = 0; + cpudata->gprs[NVMM_X64_GPR_RCX] = 0; + cpudata->gprs[NVMM_X64_GPR_RDX] = 0; + } + break; default: cpudata->gprs[NVMM_X64_GPR_RAX] = 0; cpudata->gprs[NVMM_X64_GPR_RBX] = 0; @@ -2004,7 +2025,7 @@ vmx_exit_xsetbv(struct nvmm_machine *mac if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { goto error; - } else if (__predict_false((val & ~vmx_xcr0_mask) != 0)) { + } else if (__predict_false((val & ~cpudata->xcr0_mask) != 0)) { goto error; } else if (__predict_false((val & XCR0_X87) == 0)) { goto error; @@ -2055,14 +2076,65 @@ vmx_vcpu_guest_fpu_enter(struct nvmm_cpu { struct vmx_cpudata *cpudata = vcpu->cpudata; + /* + * The guest's XCR0 had better not have any bits that aren't + * allowed in the vCPU configuration, and the current XSAVE + * area had better not store any either according to + * cpudata->gfpu.xsh_xstate_bv. + * + * Note that XRSTOR will trap if XSTATE_BV has any bits that + * are not set in XCR0. + */ + KASSERT((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0); + KASSERT((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0); + + /* + * Save anything in the FPU registers that this thread might + * have been using to memory, and raise the IPL to IPL_VM to + * block interrupt handlers that might use the FPU. + * + * After this point, we are free to use the FPU registers. + */ fpu_kern_enter(); + + /* + * If the guest is allowed to use any extended CPU state + * (including FPU registers), set XCR0 to allow access to + * anything the guest has previously used and is saved to + * memory, _and_ to anything the guest has asked to use in + * cpudata->gxcr0. + * + * The guest may have used some extended CPU state like the + * zmmN registers, and then later disabled them in XCR0; in + * that case, the state must be preserved in case the guest + * later enables it in XCR0, but we can only load while all + * bits in cpudata->gfpu.xsh_xstate_bv are set in XCR0. + * + * Similarly, the guest may _not_ have used some extended CPU + * state since reset, but may have since enabled it in XCR0. + * Such state will be clear in cpudata->gfpu.xsh_xstate_bv and + * must be initialized afresh by the CPU, which requires the + * bits be set in XCR0 to allow that. + */ + cpudata->hxcr0 = rdxcr(0); + wrxcr(0, cpudata->xcr0_mask & + (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)); + + /* + * Load the guest's saved extended CPU state from memory into + * the CPU. + */ /* TODO: should we use *XSAVE64 here? */ - fpu_area_restore(&cpudata->gfpu, vmx_xcr0_mask, false); - - if (vmx_xcr0_mask != 0) { - cpudata->hxcr0 = rdxcr(0); - wrxcr(0, cpudata->gxcr0); - } + fpu_area_restore(&cpudata->gfpu, cpudata->xcr0_mask, false); + + /* + * If we temporarily set XCR0 beyond what the guest asked for + * in order to restore state that is currently disabled, reduce + * it down to what the guest asked for. + */ + if (__predict_false(cpudata->gxcr0 != (cpudata->xcr0_mask & + (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)))) + wrxcr(0, cpudata->xcr0_mask & cpudata->gxcr0); } static void @@ -2070,14 +2142,57 @@ vmx_vcpu_guest_fpu_leave(struct nvmm_cpu { struct vmx_cpudata *cpudata = vcpu->cpudata; - if (vmx_xcr0_mask != 0) { - cpudata->gxcr0 = rdxcr(0); + /* + * In case the guest has cleared some XCR0 bits but used the + * corresponding extended CPU state, increase XCR0 to the + * maximum supported for this guest before we XSAVE. + * + * Note that XRSTOR will trap if XSTATE_BV has any bits that + * are not set in XCR0. + */ + cpudata->gxcr0 = rdxcr(0); + wrxcr(0, cpudata->xcr0_mask); + + /* + * Paranoia: Ensure the guest's XCR0 has no forbidden bits. + * Should not be possible because we filter them on XSETBV + * exits. + */ + KASSERT((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0); + cpudata->gxcr0 &= cpudata->xcr0_mask; + + /* + * Save any extended CPU state that could be in use by the + * guest. + */ + /* TODO: should we use *XSAVE64 here? */ + fpu_area_save(&cpudata->gfpu, cpudata->xcr0_mask, false); + + /* + * If the host XCR0 is different from the maximum guest XCR0, + * switch back to the host XCR0 so we can restore NetBSD's FPU + * state. + */ + if (cpudata->xcr0_mask != cpudata->hxcr0) wrxcr(0, cpudata->hxcr0); - } - - /* TODO: should we use *XSAVE64 here? */ - fpu_area_save(&cpudata->gfpu, vmx_xcr0_mask, false); + + /* + * Restore any FPU registers that we might have saved in + * vmx_vcpu_guest_fpu_enter for this thread, and restore the + * IPL from IPL_VM. + * + * After this point, we must not touch the FPU registers. + */ fpu_kern_leave(); + + /* + * The guest's XCR0 had better not have any bits that aren't + * allowed in the vCPU configuration, and the current XSAVE + * area had better not store any either according to + * cpudata->gfpu.xsh_xstate_bv. + */ + KASSERT((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0); + KASSERT((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0); } static void @@ -2649,10 +2764,10 @@ vmx_vcpu_setstate(struct nvmm_cpu *vcpu) cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8]; - if (vmx_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { /* Clear illegal XCR0 bits, set mandatory X87 bit. */ cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; - cpudata->gxcr0 &= vmx_xcr0_mask; + cpudata->gxcr0 &= cpudata->xcr0_mask; cpudata->gxcr0 |= XCR0_X87; } } @@ -2731,11 +2846,15 @@ vmx_vcpu_setstate(struct nvmm_cpu *vcpu) fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; - if (vmx_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { /* Reset XSTATE_BV, to force a reload. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; } } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ vmx_vmcs_leave(vcpu); @@ -2836,6 +2955,10 @@ vmx_vcpu_getstate(struct nvmm_cpu *vcpu) memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, sizeof(state->fpu)); } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ vmx_vmcs_leave(vcpu); @@ -3010,7 +3133,8 @@ vmx_vcpu_init(struct nvmm_machine *mach, (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL); /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + cpudata->xcr0_mask = vmx_xcr0_mask; + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; cpudata->gfpu.xsh_xcomp_bv = 0; /* These MSRs are static. */ @@ -3032,12 +3156,27 @@ vmx_vcpu_init(struct nvmm_machine *mach, static int vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) { + size_t xsave_size, cpudata_size; struct vmx_cpudata *cpudata; int error; + /* + * Compute the size of the VMX cpudata. We put the + * variable-length XSAVE area at the end so if it's small + * enough, it stays within a single page. We size the XSAVE + * area for the maximum set of features supported by the CPU + * which a guest can enable (which may be more than the NetBSD + * host enables for itself -- hence we don't use + * x86_fpu_save_size here!). + */ + xsave_size = nvmm_x86_xsave_size(vmx_xcr0_mask); + KASSERT(xsave_size < SIZE_MAX - offsetof(struct vmx_cpudata, gfpu)); + cpudata_size = MAX(sizeof(*cpudata), + offsetof(struct vmx_cpudata, gfpu) + xsave_size); + /* Allocate the VMX cpudata. */ cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, + roundup(cpudata_size, PAGE_SIZE), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); vcpu->cpudata = cpudata; @@ -3167,6 +3306,39 @@ vmx_vcpu_configure_tpr(struct vmx_cpudat } static int +vmx_vcpu_configure_xcr0_mask(struct vmx_cpudata *cpudata, void *data) +{ + const uint64_t *xcr0_maskp = data; + + /* + * Refuse to enable XCR0 bits (extended CPU state components) + * not supported by this system. + */ + if (*xcr0_maskp & ~vmx_xcr0_mask) + return EINVAL; + + /* + * Out of paranoia, clear any existing extended CPU state. + * This operation is unlikely to be used before the guest has + * begun execution at all, so the extended CPU state is + * probably all zero. But in case some weird hypervisor + * software tries to change the XCR0 mask dynamically, let's + * avoid accidentally leaking things through any extended CPU + * state. + */ + memset(&cpudata->gfpu, 0, nvmm_x86_xsave_size(vmx_xcr0_mask)); + + /* + * Set the XCR0 mask, and limit the guest's XCR0 to this mask. + * Any extended CPU state the guest had previously been using + * will be wiped out. + */ + cpudata->xcr0_mask = *xcr0_maskp; + cpudata->gxcr0 &= cpudata->xcr0_mask; + return 0; +} + +static int vmx_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) { struct vmx_cpudata *cpudata = vcpu->cpudata; @@ -3176,6 +3348,8 @@ vmx_vcpu_configure(struct nvmm_cpu *vcpu return vmx_vcpu_configure_cpuid(cpudata, data); case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR): return vmx_vcpu_configure_tpr(cpudata, data); + case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK): + return vmx_vcpu_configure_xcr0_mask(cpudata, data); default: return EINVAL; } @@ -3589,8 +3763,22 @@ vmx_init(void) /* Init the ASID bitmap (VPID). */ vmx_init_asid(VPID_MAX); - /* Init the XCR0 mask. */ - vmx_xcr0_mask = VMX_XCR0_MASK_DEFAULT & x86_xsave_features; + /* + * Init the XCR0 mask. + * + * x86_xsave_features is the cached result of + * CPUID[EAX=0x0000000d,ECX=0].EDX:EAX, the set of all + * supported XCR0 bits for user XSAVE state components on the + * physical CPU. Hypervisor software can use + * nvmm_vcpu_configure(NVMM_VCPU_CONF_XCR0_MASK) to restrict + * the available features on a per-vCPU basis, e.g. in order to + * limit guests to compatible features for migration. + * + * Out of paranoia, we mask off bit 63 which is reserved for + * future extension which we don't understand because it's not + * yet defined. + */ + vmx_xcr0_mask = x86_xsave_features & __BITS(62, 0); /* Init the max basic CPUID leaf. */ vmx_cpuid_max_basic = uimin(cpuid_level, VMX_CPUID_MAX_BASIC);