# HG changeset patch # User Taylor R Campbell # Date 1784052724 0 # Tue Jul 14 18:12:04 2026 +0000 # Branch trunk # Node ID 67c17711ba4c7c71787656928263db99528b5246 # Parent 187abacabbe955f8145e7c12ca1a0909553a72a3 # EXP-Topic riastradh-pr59860-nvmmxsave WIP: nvmm: Add support for extended CPU state (XSAVE) beyond x87/SSE. New machine-dependent x86 vCPU configuration command NVMM_VCPU_CONF_XCR0_MASK sets the vCPU's XCR0 mask, that is, the set of XSAVE features that the guest sees as supported in the vCPU. This command is advertised by the new machine capability NVMM_CAP_ARCH_VCPU_CONF_XCR0_MASK. (Changing the XCR0 mask after the guest has begun execution will clear all extended CPU state for now; it is unlikely that hypervisor software will do this anyway.) This change doesn't expose the XSAVE area to userland in the comm page -- there are already machine-independent members in struct nvmm_comm_page at fixed offsets past the machine-dependent state so we can't just extend struct nvmm_vcpu_state without breaking the ABI. And the XSAVE area may exceed a page, but the address space for the virtual machine uvm object packs comm areas in consecutive pages, so we can't put it in the comm page at all without breaking the ABI another way. Instead, we can expose the XSAVE area in another location in the virtual machine uvm object, and create an ioctl for querying its object address and size so userland can mmap it. TBD. PR port-amd64/59860: nvmm: support AVX2, AVX512, &c. diff -r 187abacabbe9 -r 67c17711ba4c sys/dev/nvmm/x86/nvmm_x86.c --- a/sys/dev/nvmm/x86/nvmm_x86.c Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86.c Tue Jul 14 18:12:04 2026 +0000 @@ -258,7 +258,7 @@ const struct nvmm_x86_cpuid_mask nvmm_cp CPUID2_AESNI | CPUID2_XSAVE | CPUID2_OSXSAVE | - /* CPUID2_AVX excluded */ + CPUID2_AVX | CPUID2_F16C | CPUID2_RDRAND, /* CPUID2_RAZ excluded */ @@ -302,7 +302,7 @@ const struct nvmm_x86_cpuid_mask nvmm_cp /* CPUID_SEF_SGX excluded */ CPUID_SEF_BMI1 | /* CPUID_SEF_HLE excluded */ - /* CPUID_SEF_AVX2 excluded */ + CPUID_SEF_AVX2 | CPUID_SEF_FDPEXONLY | CPUID_SEF_SMEP | CPUID_SEF_BMI2 | @@ -466,3 +466,187 @@ nvmm_x86_pat_validate(uint64_t val) return true; } + +/* + * nvmm_x86_xsave_size(xcr0) + * + * Returns the maximum XSAVE area size in bytes needed to + * represent all user state components corresponding to bits set + * in xcr0. Bit 63, which is reserved for future architecture + * extension at the time of writing, MUST NOT be set. Bits that + * were not previously advertised in CPUID[EAX=0x0d,ECX=0].EDX:EAX + * MUST NOT be set. + */ +uint32_t +nvmm_x86_xsave_size(uint64_t xcr0) +{ + uint32_t totalsize = sizeof(struct xsave_header); + unsigned i; + + /* + * Caller must not pass bit 63 until the architectural + * extension mechanism it is reserved for has been defined. + * Caller must also not pass any bits that the CPU has not + * advertised support for. + */ + KASSERTMSG((xcr0 & ~__BITS(62, 0)) == 0, "xcr0=0x%"PRIx64, xcr0); + KASSERTMSG((xcr0 & ~x86_xsave_features) == 0, + "xcr0=0x%"PRIx64" x86_xsave_features=0x%"PRIx64, + xcr0, x86_xsave_features); + + /* + * Bit 0 is for x87 state, and bit 1 is for SSE state, for + * which (a) space is already included in the 512-byte legacy + * area of XSAVE (which is included in `struct xsave_header'), + * and (b) we can't query the CPUID[EAX=0x0d,ECX=i] leaf + * because for i=0 and i=1 the leaf serves a different purpose. + * + * We stop before bit 63 because it is reserved for future + * architectural extension which is not yet defined. + */ + CTASSERT(sizeof(struct xsave_header) == 512 + 64); + for (i = 2; i < 63; i++) { + uint32_t descs[4]; + uint32_t size, offset; + + /* + * Skip state components that are not of interest to + * the caller. + */ + if ((xcr0 & __BIT(i)) == 0) + continue; + + /* + * Can't use x86_xsave_offsets[i] + x86_xsave_sizes[i] + * because the NetBSD kernel only queries those for the + * user state components it knows about, but in + * principle this should support any user state + * component the guest wants to use even if NetBSD + * doesn't know how. + */ + x86_cpuid2(0x0d, i, descs); + size = descs[0]; /* CPUID[EAX=0x0d,ECX=i].EAX */ + offset = descs[1]; /* CPUID[EAX=0x0d,ECX=i].EBX */ + KASSERT(size <= UINT32_MAX - offset); + totalsize = MAX(totalsize, offset + size); + } + + return totalsize; +} + +/* + * nvmm_x86_xcr0_valid(xcr0, xcr0_mask) + * + * True if xcr0 is a valid content for the XCR0 register, false if + * not. + * + * - Bits outside xcr0_mask must not be set. + * - Bit for x87 state must be set. + * - The bit for the YMM_Hi128 state (high 128-bit halves of the + * ymm registers) can only be set if the SSE state (xmm + * registers, a.k.a. low 128-bit halves of the ymm registers) is + * also enabled. + * + * What about ZMM_Hi256? According to even recent architecture + * manuals from Intel and AMD, this does _not_ seem to require + * that SSE or YMM_Hi128 be enabled for ZMM_Hi256 to be enabled. + * Not requiring this is sensible because it enables XSAVE + * extensions to work safely even if software doesn't know such + * details. But I have not tested on real hardware whether + * ZMM_Hi256 actually does require YMM_Hi128 or SSE to be enabled + * in XCR0. + * + * References: + * + * - Intel 64 and IA-32 Architectures Software Developer's Manual, + * Volume 2 (2A, 2B, 2C, & 2D): Instruction Set Reference, A-Z, + * Intel, Order Number: 325383-092US, June 2026, Sec. 6.1 + * `Instructions (W-Z)', `XSETBV---Set Extended Control + * Register', pp. 6-70 -- 6-71. + * https://cdrdv2.intel.com/v1/dl/getContent/671110 + * https://web.archive.org/web/20260711031355/https://cdrdv2-public.intel.com/922478/325383-092-sdm-vol-2abcd.pdf + * + * - AMD64 Architecture Programmer's Manual, Volume 4: 128-bit, + * 256-bit, and 512-bit Media Instructions, Advanced Micro + * Devices, Publication No. 26568, Rev. 3.26, January 2026, + * pp. 1576--1577. + * https://docs.amd.com/v/u/en-US/26568_3.26_APM_Vol4 + * https://web.archive.org/web/20260714233827/https://docs.amd.com/api/khub/documents/ioQiNhSqxlMkRaU~IyGQYQ/content?Ft-Calling-App=ft%2Fturnkey-portal&Ft-Calling-App-Version=5.3.24 + */ +bool +nvmm_x86_xcr0_valid(uint64_t xcr0, uint64_t xcr0_mask) +{ + + /* + * Refuse setting any bits in XCR0 that are disabled in the + * vCPU configuration. + */ + if (__predict_false(xcr0 & ~xcr0_mask)) + return false; + + /* + * Refuse clearing the x87 state component in XCR0, as the + * physical CPU would. + */ + if (__predict_false((xcr0 & XCR0_X87) == 0)) + return false; + + /* + * Refuse setting the YMM_Hi128 state component in XCR0 if the + * SSE/XMM state component is not also enabled, like the + * physical CPU would. + */ + if (__predict_false((xcr0 & (XCR0_YMM_Hi128|XCR0_SSE)) == + XCR0_YMM_Hi128)) + return false; + + /* Looks good! */ + return true; +} + +/* + * nvmm_x86_munge_xcr0(xcr0, xcr0_mask) + * + * Map an arbitrary 64-bit word into a plausible value for XCR0 + * under the given mask (which must itself be a plausible value + * for XCR0). This function MUST NOT be called on CPUs without + * XSAVE at all. + */ +uint64_t +nvmm_x86_munge_xcr0(uint64_t xcr0, uint64_t xcr0_mask) +{ + uint64_t origxcr0 __diagused = xcr0; + + KASSERT(xcr0_mask != 0); + KASSERTMSG(xcr0_mask & XCR0_X87, "xcr0_mask=0x%"PRIx64, xcr0_mask); + KASSERTMSG((xcr0_mask & (XCR0_YMM_Hi128|XCR0_SSE)) != XCR0_YMM_Hi128, + "xcr0_mask=0x%"PRIx64, xcr0_mask); + + /* + * Clear any bits not in xcr0_mask. + */ + xcr0 &= xcr0_mask; + + /* + * Set the mandatory x87 bit. + */ + xcr0 |= XCR0_X87; + + /* + * If YMM_Hi128 (high 128-bit halves of ymmN) is enabled, then + * SSE (xmmN, a.k.a. low 128-bit halves of ymmN) must also be + * enabled. + * + * Note: The same may not apply to zmmN; see above about + * nvmm_x86_xcr0_valid. + */ + if (__predict_false((xcr0 & (XCR0_YMM_Hi128|XCR0_SSE)) == + XCR0_YMM_Hi128)) + xcr0 |= XCR0_SSE; + + KASSERTMSG(nvmm_x86_xcr0_valid(xcr0, xcr0_mask), + "origxcr0=0x%"PRIx64" xcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + origxcr0, xcr0, xcr0_mask); + + return xcr0; +} diff -r 187abacabbe9 -r 67c17711ba4c sys/dev/nvmm/x86/nvmm_x86.h --- a/sys/dev/nvmm/x86/nvmm_x86.h Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86.h Tue Jul 14 18:12:04 2026 +0000 @@ -132,6 +132,7 @@ struct nvmm_cap_md { uint64_t vcpu_conf_support; #define NVMM_CAP_ARCH_VCPU_CONF_CPUID __BIT(0) #define NVMM_CAP_ARCH_VCPU_CONF_TPR __BIT(1) +#define NVMM_CAP_ARCH_VCPU_CONF_XCR0_MASK __BIT(2) uint64_t xcr0_mask; uint32_t mxcsr_mask; @@ -269,6 +270,7 @@ struct nvmm_x64_state { #define NVMM_VCPU_CONF_CPUID NVMM_VCPU_CONF_MD_BEGIN #define NVMM_VCPU_CONF_TPR (NVMM_VCPU_CONF_MD_BEGIN + 1) +#define NVMM_VCPU_CONF_XCR0_MASK (NVMM_VCPU_CONF_MD_BEGIN + 2) struct nvmm_vcpu_conf_cpuid { /* The options. */ @@ -309,7 +311,7 @@ struct nvmm_vcpu_conf_tpr { #ifdef _KERNEL #define NVMM_X86_MACH_NCONF 0 -#define NVMM_X86_VCPU_NCONF 2 +#define NVMM_X86_VCPU_NCONF 3 struct nvmm_x86_cpuid_mask { uint32_t eax; uint32_t ebx; @@ -323,6 +325,9 @@ extern const struct nvmm_x86_cpuid_mask extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000007; extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000008; bool nvmm_x86_pat_validate(uint64_t); +uint32_t nvmm_x86_xsave_size(uint64_t); +bool nvmm_x86_xcr0_valid(uint64_t, uint64_t); +uint64_t nvmm_x86_munge_xcr0(uint64_t, uint64_t); #endif #endif /* ASM_NVMM */ diff -r 187abacabbe9 -r 67c17711ba4c sys/dev/nvmm/x86/nvmm_x86_svm.c --- a/sys/dev/nvmm/x86/nvmm_x86_svm.c Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86_svm.c Tue Jul 14 18:12:04 2026 +0000 @@ -499,7 +499,6 @@ static kmutex_t svm_asidlock __cacheline static bool svm_decode_assist __read_mostly; static uint32_t svm_ctrl_tlb_flush __read_mostly; -#define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) static uint64_t svm_xcr0_mask __read_mostly; #define SVM_NCPUIDS 32 @@ -560,7 +559,9 @@ static const size_t svm_vcpu_conf_sizes[ [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = sizeof(struct nvmm_vcpu_conf_cpuid), [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = - sizeof(struct nvmm_vcpu_conf_tpr) + sizeof(struct nvmm_vcpu_conf_tpr), + [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK)] = + sizeof(uint64_t), }; struct svm_cpudata { @@ -601,11 +602,17 @@ struct svm_cpudata { uint64_t gprs[NVMM_X64_NGPR]; uint64_t drs[NVMM_X64_NDR]; uint64_t gtsc; - struct xsave_header gfpu __aligned(64); /* VCPU configuration. */ bool cpuidpresent[SVM_NCPUIDS]; struct nvmm_vcpu_conf_cpuid cpuid[SVM_NCPUIDS]; + uint64_t xcr0_mask; + + /* + * Guest XSAVE state. Must be the last member because it may + * be extended variably by whatever CPU we're running on. + */ + struct xsave_header gfpu __aligned(64); }; static void @@ -960,20 +967,23 @@ svm_inkernel_handle_cpuid(struct nvmm_cp cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; case 0x0000000D: /* Processor Extended State Enumeration */ - if (svm_xcr0_mask == 0) { + if (cpudata->xcr0_mask == 0) { + cpudata->vmcb->state.rax = 0; + cpudata->gprs[NVMM_X64_GPR_RBX] = 0; + cpudata->gprs[NVMM_X64_GPR_RCX] = 0; + cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; } switch (ecx) { case 0: - cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; - if (cpudata->gxcr0 & XCR0_SSE) { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); - } else { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); - } - cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ - cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; - cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; + cpudata->vmcb->state.rax = + cpudata->xcr0_mask & 0xFFFFFFFF; + cpudata->gprs[NVMM_X64_GPR_RBX] = + nvmm_x86_xsave_size(cpudata->gxcr0); + cpudata->gprs[NVMM_X64_GPR_RCX] = + nvmm_x86_xsave_size(cpudata->xcr0_mask); + cpudata->gprs[NVMM_X64_GPR_RDX] = + cpudata->xcr0_mask >> 32; break; case 1: cpudata->vmcb->state.rax &= @@ -983,6 +993,21 @@ svm_inkernel_handle_cpuid(struct nvmm_cp cpudata->gprs[NVMM_X64_GPR_RCX] = 0; cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; + case 2 ... 62: + /* + * CPUID[EAX=0x0d,ECX=n], 2 <= n <= 62: size + * and offset of nth component in XSAVE area. + * If the nth bit of XCR0 is disabled in the + * vCPU configuration, we return all-zero + * instead. + */ + if ((cpudata->xcr0_mask & __BIT(ecx)) == 0) { + cpudata->vmcb->state.rax = 0; + cpudata->gprs[NVMM_X64_GPR_RBX] = 0; + cpudata->gprs[NVMM_X64_GPR_RCX] = 0; + cpudata->gprs[NVMM_X64_GPR_RDX] = 0; + } + break; default: cpudata->vmcb->state.rax = 0; cpudata->gprs[NVMM_X64_GPR_RBX] = 0; @@ -1377,12 +1402,16 @@ svm_exit_xsetbv(struct nvmm_machine *mac goto error; } else if (__predict_false(vmcb->state.cpl != 0)) { goto error; - } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { + } else if (__predict_false(cpudata->xcr0_mask == 0)) { goto error; - } else if (__predict_false((val & XCR0_X87) == 0)) { + } else if (__predict_false(!nvmm_x86_xcr0_valid(val, + cpudata->xcr0_mask))) { goto error; } + KASSERTMSG(nvmm_x86_xcr0_valid(val, cpudata->xcr0_mask), + "val=0x%"PRIx64" xcr0_mask=0x%"PRIx64" (gxcr0=0x%"PRIx64")", + val, cpudata->xcr0_mask, cpudata->gxcr0); cpudata->gxcr0 = val; svm_inkernel_advance(cpudata->vmcb); @@ -1406,14 +1435,80 @@ svm_vcpu_guest_fpu_enter(struct nvmm_cpu { struct svm_cpudata *cpudata = vcpu->cpudata; + /* + * The guest's XCR0 had better not have any bits that aren't + * allowed in the vCPU configuration, and the current XSAVE + * area had better not store any either according to + * cpudata->gfpu.xsh_xstate_bv. + * + * Note that XRSTOR will trap if XSTATE_BV has any bits that + * are not set in XCR0. + */ + KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, + "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gxcr0, cpudata->xcr0_mask); + KASSERTMSG((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0, + "XSTATE_BV=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gfpu.xsh_xstate_bv, cpudata->xcr0_mask); + + /* + * Save anything in the FPU registers that this thread might + * have been using to memory, and raise the IPL to IPL_VM to + * block interrupt handlers that might use the FPU. This also + * zeroes any FPU registers that the NetBSD host uses. + * + * After this point, we are free to use the FPU registers. + */ fpu_kern_enter(); + + /* + * If the host CPU doesn't support XSAVE or we're simulating a + * vCPU without it, just restore the x87 and SSE state. The + * host should already have both x87 and SSE enabled in XCR0, + * if the host uses XSAVE. + */ + if (cpudata->xcr0_mask == 0) { + /* TODO: should we use *XSAVE64 here? */ + fpu_area_restore(&cpudata->gfpu, XCR0_X87|XCR0_SSE, false); + return; + } + + /* + * Set XCR0 to allow access to anything the guest has + * previously used and is saved to memory, _and_ to anything + * the guest has asked to use in cpudata->gxcr0. + * + * The guest may have used some extended CPU state like the + * zmmN registers, and then later disabled them in XCR0; in + * that case, the state must be preserved in case the guest + * later enables it in XCR0, but we can only load while all + * bits in cpudata->gfpu.xsh_xstate_bv are set in XCR0. + * + * Similarly, the guest may _not_ have used some extended CPU + * state since reset, but may have since enabled it in XCR0. + * Such state will be clear in cpudata->gfpu.xsh_xstate_bv and + * must be initialized afresh by the CPU, which requires the + * bits be set in XCR0 to allow that. + */ + cpudata->hxcr0 = rdxcr(0); + wrxcr(0, cpudata->xcr0_mask & + (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)); + + /* + * Load the guest's saved extended CPU state from memory into + * the CPU. + */ /* TODO: should we use *XSAVE64 here? */ - fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask, false); - - if (svm_xcr0_mask != 0) { - cpudata->hxcr0 = rdxcr(0); - wrxcr(0, cpudata->gxcr0); - } + fpu_area_restore(&cpudata->gfpu, cpudata->xcr0_mask, false); + + /* + * If we temporarily set XCR0 beyond what the guest asked for + * in order to restore state that is currently disabled, reduce + * it down to what the guest asked for. + */ + if (__predict_false(cpudata->gxcr0 != (cpudata->xcr0_mask & + (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)))) + wrxcr(0, cpudata->xcr0_mask & cpudata->gxcr0); } static void @@ -1421,14 +1516,76 @@ svm_vcpu_guest_fpu_leave(struct nvmm_cpu { struct svm_cpudata *cpudata = vcpu->cpudata; - if (svm_xcr0_mask != 0) { - cpudata->gxcr0 = rdxcr(0); - wrxcr(0, cpudata->hxcr0); + /* + * If the host CPU doesn't support XSAVE or we're simulating a + * vCPU without it, just save the x87 and SSE state. If the + * host uses XSAVE, it should still have both x87 and SSE + * enabled in XCR0; if the host doesn't use XSAVE, doesn't + * matter. + */ + if (cpudata->xcr0_mask == 0) { + /* TODO: should we use *XSAVE64 here? */ + fpu_area_save(&cpudata->gfpu, XCR0_X87|XCR0_SSE, false); + goto leave; } + /* + * In case the guest has cleared some XCR0 bits but used the + * corresponding extended CPU state, increase XCR0 to the + * maximum supported for this guest before we XSAVE. + * + * Note that XRSTOR will trap if XSTATE_BV has any bits that + * are not set in XCR0. + */ + cpudata->gxcr0 = rdxcr(0); + wrxcr(0, cpudata->xcr0_mask); + + /* + * Paranoia: Ensure the guest's XCR0 has no forbidden bits. + * Should not be possible because we filter them on XSETBV + * exits. + */ + KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, + "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gxcr0, cpudata->xcr0_mask); + cpudata->gxcr0 &= cpudata->xcr0_mask; + + /* + * Save any extended CPU state that could be in use by the + * guest. + */ /* TODO: should we use *XSAVE64 here? */ - fpu_area_save(&cpudata->gfpu, svm_xcr0_mask, false); + fpu_area_save(&cpudata->gfpu, cpudata->xcr0_mask, false); + + /* + * If the host XCR0 is different from the maximum guest XCR0, + * switch back to the host XCR0 so we can restore NetBSD's FPU + * state. + */ + if (cpudata->xcr0_mask != cpudata->hxcr0) + wrxcr(0, cpudata->hxcr0); + +leave: /* + * Restore any FPU registers that we might have saved in + * svm_vcpu_guest_fpu_enter for this thread, and restore the + * IPL from IPL_VM. + * + * After this point, we must not touch the FPU registers. + */ fpu_kern_leave(); + + /* + * The guest's XCR0 had better not have any bits that aren't + * allowed in the vCPU configuration, and the current XSAVE + * area had better not store any either according to + * cpudata->gfpu.xsh_xstate_bv. + */ + KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, + "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gxcr0, cpudata->xcr0_mask); + KASSERTMSG((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0, + "XSTATE_BV=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gfpu.xsh_xstate_bv, cpudata->xcr0_mask); } static void @@ -1915,11 +2072,11 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu) vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], VMCB_CTRL_V_TPR); - if (svm_xcr0_mask != 0) { - /* Clear illegal XCR0 bits, set mandatory X87 bit. */ - cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; - cpudata->gxcr0 &= svm_xcr0_mask; - cpudata->gxcr0 |= XCR0_X87; + if (cpudata->xcr0_mask != 0) { + const uint64_t xcr0 = state->crs[NVMM_X64_CR_XCR0]; + + cpudata->gxcr0 = nvmm_x86_munge_xcr0(xcr0, + cpudata->xcr0_mask); } } @@ -1983,11 +2140,15 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu) fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; - if (svm_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { /* Reset XSTATE_BV, to force a reload. */ - cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; } } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ svm_vmcb_cache_update(vmcb, flags); @@ -2093,6 +2254,10 @@ svm_vcpu_getstate(struct nvmm_cpu *vcpu) memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, sizeof(state->fpu)); } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ comm->state_wanted = 0; comm->state_cached |= flags; @@ -2282,7 +2447,11 @@ svm_vcpu_init(struct nvmm_machine *mach, vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; + cpudata->xcr0_mask = svm_xcr0_mask; + KASSERTMSG(nvmm_x86_xcr0_valid(cpudata->xcr0_mask, svm_xcr0_mask), + "cpudata->xcr0_mask=0x%"PRIx64" svm_xcr0_mask=0x%"PRIx64, + cpudata->xcr0_mask, svm_xcr0_mask); + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; cpudata->gfpu.xsh_xcomp_bv = 0; /* These MSRs are static. */ @@ -2302,12 +2471,27 @@ svm_vcpu_init(struct nvmm_machine *mach, static int svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) { + size_t xsave_size, cpudata_size; struct svm_cpudata *cpudata; int error; + /* + * Compute the size of the SVM cpudata. We put the + * variable-length XSAVE area at the end so if it's small + * enough, it stays within a single page. We size the XSAVE + * area for the maximum set of features supported by the CPU + * which a guest can enable (which may be more than the NetBSD + * host enables for itself -- hence we don't use + * x86_fpu_save_size here!). + */ + xsave_size = nvmm_x86_xsave_size(svm_xcr0_mask); + KASSERT(xsave_size < SIZE_MAX - offsetof(struct svm_cpudata, gfpu)); + cpudata_size = MAX(sizeof(*cpudata), + offsetof(struct svm_cpudata, gfpu) + xsave_size); + /* Allocate the SVM cpudata. */ cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, + roundup(cpudata_size, PAGE_SIZE), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); vcpu->cpudata = cpudata; @@ -2425,6 +2609,54 @@ svm_vcpu_configure_cpuid(struct svm_cpud } static int +svm_vcpu_configure_xcr0_mask(struct svm_cpudata *cpudata, void *data) +{ + const uint64_t *xcr0_maskp = data; + + /* + * Refuse to enable XCR0 bits (extended CPU state components) + * not supported by this system, or to set up otherwise + * nonsensical masks like AVX (YMM_Hi128) but not SSE (XMM) + * registers. Exception: The mask can be all-zero to disable + * all XSAVE state components. + */ + if (*xcr0_maskp != 0 && + !nvmm_x86_xcr0_valid(*xcr0_maskp, svm_xcr0_mask)) + return EINVAL; + + /* + * Out of paranoia, clear any existing extended CPU state. + * This operation is unlikely to be used before the guest has + * begun execution at all, so the extended CPU state is + * probably all zero. But in case some weird hypervisor + * software tries to change the XCR0 mask dynamically, let's + * avoid accidentally leaking things through any extended CPU + * state. + * + * We could zero only the components that are getting disabled. + * But if the saved state is compated (XSAVEC), we wouldd also + * have to move the remaining components around in order to + * avoid zeroing them. Since no software is likely to try this + * anyway, we'll just zero everything to keep it simple and + * avoid having to test the difficult-and-unused paths. + */ + memset(&cpudata->gfpu, 0, nvmm_x86_xsave_size(svm_xcr0_mask)); + + /* + * Set the XCR0 mask, and limit the guest's XCR0 to this mask. + * Any extended CPU state the guest had previously been using + * will be wiped out. + */ + cpudata->xcr0_mask = *xcr0_maskp; + cpudata->gxcr0 &= cpudata->xcr0_mask; + KASSERTMSG((cpudata->xcr0_mask == 0 || + nvmm_x86_xcr0_valid(cpudata->gxcr0, cpudata->xcr0_mask)), + "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gxcr0, cpudata->xcr0_mask); + return 0; +} + +static int svm_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) { struct svm_cpudata *cpudata = vcpu->cpudata; @@ -2432,6 +2664,8 @@ svm_vcpu_configure(struct nvmm_cpu *vcpu switch (op) { case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID): return svm_vcpu_configure_cpuid(cpudata, data); + case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK): + return svm_vcpu_configure_xcr0_mask(cpudata, data); default: return EINVAL; } @@ -2602,8 +2836,25 @@ svm_init(void) /* Init the ASID. */ svm_init_asid(descs[1]); - /* Init the XCR0 mask. */ - svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; + /* + * Init the XCR0 mask. + * + * x86_xsave_features is the cached result of + * CPUID[EAX=0x0000000d,ECX=0].EDX:EAX, the set of all + * supported XCR0 bits for user XSAVE state components on the + * physical CPU. Hypervisor software can use + * nvmm_vcpu_configure(NVMM_VCPU_CONF_XCR0_MASK) to restrict + * the available features on a per-vCPU basis, e.g. in order to + * limit guests to compatible features for migration. + * + * Out of paranoia, we mask off bit 63 which is reserved for + * future extension which we don't understand because it's not + * yet defined. + */ + svm_xcr0_mask = x86_xsave_features & __BITS(62, 0); + KASSERTMSG((svm_xcr0_mask == 0 || + nvmm_x86_xcr0_valid(svm_xcr0_mask, svm_xcr0_mask)), + "svm_xcr0_mask=0x%"PRIx64, svm_xcr0_mask); /* Init the max basic CPUID leaf. */ svm_cpuid_max_basic = uimin(cpuid_level, SVM_CPUID_MAX_BASIC); diff -r 187abacabbe9 -r 67c17711ba4c sys/dev/nvmm/x86/nvmm_x86_vmx.c --- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c Tue Jul 14 18:12:04 2026 +0000 @@ -717,7 +717,6 @@ static uint8_t *vmx_asidmap __read_mostl static uint32_t vmx_maxasid __read_mostly; static kmutex_t vmx_asidlock __cacheline_aligned; -#define VMX_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) static uint64_t vmx_xcr0_mask __read_mostly; #define VMX_NCPUIDS 32 @@ -775,7 +774,9 @@ static const size_t vmx_vcpu_conf_sizes[ [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = sizeof(struct nvmm_vcpu_conf_cpuid), [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = - sizeof(struct nvmm_vcpu_conf_tpr) + sizeof(struct nvmm_vcpu_conf_tpr), + [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK)] = + sizeof(uint64_t), }; struct vmx_cpudata { @@ -820,12 +821,18 @@ struct vmx_cpudata { uint64_t gprs[NVMM_X64_NGPR]; uint64_t drs[NVMM_X64_NDR]; uint64_t gtsc; - struct xsave_header gfpu __aligned(64); /* VCPU configuration. */ bool cpuidpresent[VMX_NCPUIDS]; struct nvmm_vcpu_conf_cpuid cpuid[VMX_NCPUIDS]; struct nvmm_vcpu_conf_tpr tpr; + uint64_t xcr0_mask; + + /* + * Guest XSAVE state. Must be the last member because it may + * be extended variably by whatever CPU we're running on. + */ + struct xsave_header gfpu __aligned(64); }; static const struct { @@ -1407,20 +1414,23 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; case 0x0000000D: /* Processor Extended State Enumeration */ - if (vmx_xcr0_mask == 0) { + if (cpudata->xcr0_mask == 0) { + cpudata->gprs[NVMM_X64_GPR_RAX] = 0; + cpudata->gprs[NVMM_X64_GPR_RBX] = 0; + cpudata->gprs[NVMM_X64_GPR_RCX] = 0; + cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; } switch (ecx) { case 0: - cpudata->gprs[NVMM_X64_GPR_RAX] = vmx_xcr0_mask & 0xFFFFFFFF; - if (cpudata->gxcr0 & XCR0_SSE) { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); - } else { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); - } - cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ - cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; - cpudata->gprs[NVMM_X64_GPR_RDX] = vmx_xcr0_mask >> 32; + cpudata->gprs[NVMM_X64_GPR_RAX] = + cpudata->xcr0_mask & 0xFFFFFFFF; + cpudata->gprs[NVMM_X64_GPR_RBX] = + nvmm_x86_xsave_size(cpudata->gxcr0); + cpudata->gprs[NVMM_X64_GPR_RCX] = + nvmm_x86_xsave_size(cpudata->xcr0_mask); + cpudata->gprs[NVMM_X64_GPR_RDX] = + cpudata->xcr0_mask >> 32; break; case 1: cpudata->gprs[NVMM_X64_GPR_RAX] &= @@ -1430,6 +1440,21 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma cpudata->gprs[NVMM_X64_GPR_RCX] = 0; cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; + case 2 ... 62: + /* + * CPUID[EAX=0x0d,ECX=n], 2 <= n <= 62: size + * and offset of nth component in XSAVE area. + * If the nth bit of XCR0 is disabled in the + * vCPU configuration, we return all-zero + * instead. + */ + if ((cpudata->xcr0_mask & __BIT(ecx)) == 0) { + cpudata->gprs[NVMM_X64_GPR_RAX] = 0; + cpudata->gprs[NVMM_X64_GPR_RBX] = 0; + cpudata->gprs[NVMM_X64_GPR_RCX] = 0; + cpudata->gprs[NVMM_X64_GPR_RDX] = 0; + } + break; default: cpudata->gprs[NVMM_X64_GPR_RAX] = 0; cpudata->gprs[NVMM_X64_GPR_RBX] = 0; @@ -2004,12 +2029,16 @@ vmx_exit_xsetbv(struct nvmm_machine *mac if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { goto error; - } else if (__predict_false((val & ~vmx_xcr0_mask) != 0)) { + } else if (__predict_false(cpudata->xcr0_mask == 0)) { goto error; - } else if (__predict_false((val & XCR0_X87) == 0)) { + } else if (__predict_false(!nvmm_x86_xcr0_valid(val, + cpudata->xcr0_mask))) { goto error; } + KASSERTMSG(nvmm_x86_xcr0_valid(val, cpudata->xcr0_mask), + "val=0x%"PRIx64" xcr0_mask=0x%"PRIx64" (gxcr0=0x%"PRIx64")", + val, cpudata->xcr0_mask, cpudata->gxcr0); cpudata->gxcr0 = val; vmx_inkernel_advance(); @@ -2055,14 +2084,80 @@ vmx_vcpu_guest_fpu_enter(struct nvmm_cpu { struct vmx_cpudata *cpudata = vcpu->cpudata; + /* + * The guest's XCR0 had better not have any bits that aren't + * allowed in the vCPU configuration, and the current XSAVE + * area had better not store any either according to + * cpudata->gfpu.xsh_xstate_bv. + * + * Note that XRSTOR will trap if XSTATE_BV has any bits that + * are not set in XCR0. + */ + KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, + "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gxcr0, cpudata->xcr0_mask); + KASSERTMSG((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0, + "XSTATE_BV=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gfpu.xsh_xstate_bv, cpudata->xcr0_mask); + + /* + * Save anything in the FPU registers that this thread might + * have been using to memory, and raise the IPL to IPL_VM to + * block interrupt handlers that might use the FPU. This also + * zeroes any FPU registers that the NetBSD host uses. + * + * After this point, we are free to use the FPU registers. + */ fpu_kern_enter(); + + /* + * If the host CPU doesn't support XSAVE or we're simulating a + * vCPU without it, just restore the x87 and SSE state. The + * host should already have both x87 and SSE enabled in XCR0, + * if the host uses XSAVE. + */ + if (cpudata->xcr0_mask == 0) { + /* TODO: should we use *XSAVE64 here? */ + fpu_area_restore(&cpudata->gfpu, XCR0_X87|XCR0_SSE, false); + return; + } + + /* + * Set XCR0 to allow access to anything the guest has + * previously used and is saved to memory, _and_ to anything + * the guest has asked to use in cpudata->gxcr0. + * + * The guest may have used some extended CPU state like the + * zmmN registers, and then later disabled them in XCR0; in + * that case, the state must be preserved in case the guest + * later enables it in XCR0, but we can only load while all + * bits in cpudata->gfpu.xsh_xstate_bv are set in XCR0. + * + * Similarly, the guest may _not_ have used some extended CPU + * state since reset, but may have since enabled it in XCR0. + * Such state will be clear in cpudata->gfpu.xsh_xstate_bv and + * must be initialized afresh by the CPU, which requires the + * bits be set in XCR0 to allow that. + */ + cpudata->hxcr0 = rdxcr(0); + wrxcr(0, cpudata->xcr0_mask & + (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)); + + /* + * Load the guest's saved extended CPU state from memory into + * the CPU. + */ /* TODO: should we use *XSAVE64 here? */ - fpu_area_restore(&cpudata->gfpu, vmx_xcr0_mask, false); - - if (vmx_xcr0_mask != 0) { - cpudata->hxcr0 = rdxcr(0); - wrxcr(0, cpudata->gxcr0); - } + fpu_area_restore(&cpudata->gfpu, cpudata->xcr0_mask, true); + + /* + * If we temporarily set XCR0 beyond what the guest asked for + * in order to restore state that is currently disabled, reduce + * it down to what the guest asked for. + */ + if (__predict_false(cpudata->gxcr0 != (cpudata->xcr0_mask & + (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)))) + wrxcr(0, cpudata->xcr0_mask & cpudata->gxcr0); } static void @@ -2070,14 +2165,76 @@ vmx_vcpu_guest_fpu_leave(struct nvmm_cpu { struct vmx_cpudata *cpudata = vcpu->cpudata; - if (vmx_xcr0_mask != 0) { - cpudata->gxcr0 = rdxcr(0); - wrxcr(0, cpudata->hxcr0); + /* + * If the host CPU doesn't support XSAVE or we're simulating a + * vCPU without it, just save the x87 and SSE state. If the + * host uses XSAVE, it should still have both x87 and SSE + * enabled in XCR0; if the host doesn't use XSAVE, doesn't + * matter. + */ + if (cpudata->xcr0_mask == 0) { + /* TODO: should we use *XSAVE64 here? */ + fpu_area_save(&cpudata->gfpu, XCR0_X87|XCR0_SSE, false); + goto leave; } + /* + * In case the guest has cleared some XCR0 bits but used the + * corresponding extended CPU state, increase XCR0 to the + * maximum supported for this guest before we XSAVE. + * + * Note that XRSTOR will trap if XSTATE_BV has any bits that + * are not set in XCR0. + */ + cpudata->gxcr0 = rdxcr(0); + wrxcr(0, cpudata->xcr0_mask); + + /* + * Paranoia: Ensure the guest's XCR0 has no forbidden bits. + * Should not be possible because we filter them on XSETBV + * exits. + */ + KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, + "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gxcr0, cpudata->xcr0_mask); + cpudata->gxcr0 &= cpudata->xcr0_mask; + + /* + * Save any extended CPU state that could be in use by the + * guest. + */ /* TODO: should we use *XSAVE64 here? */ - fpu_area_save(&cpudata->gfpu, vmx_xcr0_mask, false); + fpu_area_save(&cpudata->gfpu, cpudata->xcr0_mask, false); + + /* + * If the host XCR0 is different from the maximum guest XCR0, + * switch back to the host XCR0 so we can restore NetBSD's FPU + * state. + */ + if (cpudata->xcr0_mask != cpudata->hxcr0) + wrxcr(0, cpudata->hxcr0); + +leave: /* + * Restore any FPU registers that we might have saved in + * vmx_vcpu_guest_fpu_enter for this thread, and restore the + * IPL from IPL_VM. + * + * After this point, we must not touch the FPU registers. + */ fpu_kern_leave(); + + /* + * The guest's XCR0 had better not have any bits that aren't + * allowed in the vCPU configuration, and the current XSAVE + * area had better not store any either according to + * cpudata->gfpu.xsh_xstate_bv. + */ + KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, + "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gxcr0, cpudata->xcr0_mask); + KASSERTMSG((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0, + "XSTATE_BV=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gfpu.xsh_xstate_bv, cpudata->xcr0_mask); } static void @@ -2649,11 +2806,11 @@ vmx_vcpu_setstate(struct nvmm_cpu *vcpu) cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8]; - if (vmx_xcr0_mask != 0) { - /* Clear illegal XCR0 bits, set mandatory X87 bit. */ - cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; - cpudata->gxcr0 &= vmx_xcr0_mask; - cpudata->gxcr0 |= XCR0_X87; + if (cpudata->xcr0_mask != 0) { + const uint64_t xcr0 = state->crs[NVMM_X64_CR_XCR0]; + + cpudata->gxcr0 = nvmm_x86_munge_xcr0(xcr0, + cpudata->xcr0_mask); } } @@ -2731,11 +2888,15 @@ vmx_vcpu_setstate(struct nvmm_cpu *vcpu) fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; - if (vmx_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { /* Reset XSTATE_BV, to force a reload. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; } } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ vmx_vmcs_leave(vcpu); @@ -2836,6 +2997,10 @@ vmx_vcpu_getstate(struct nvmm_cpu *vcpu) memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, sizeof(state->fpu)); } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ vmx_vmcs_leave(vcpu); @@ -3010,7 +3175,11 @@ vmx_vcpu_init(struct nvmm_machine *mach, (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL); /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + cpudata->xcr0_mask = vmx_xcr0_mask; + KASSERTMSG(nvmm_x86_xcr0_valid(cpudata->xcr0_mask, vmx_xcr0_mask), + "cpudata->xcr0_mask=0x%"PRIx64" vmx_xcr0_mask=0x%"PRIx64, + cpudata->xcr0_mask, vmx_xcr0_mask); + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; cpudata->gfpu.xsh_xcomp_bv = 0; /* These MSRs are static. */ @@ -3032,12 +3201,27 @@ vmx_vcpu_init(struct nvmm_machine *mach, static int vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) { + size_t xsave_size, cpudata_size; struct vmx_cpudata *cpudata; int error; + /* + * Compute the size of the VMX cpudata. We put the + * variable-length XSAVE area at the end so if it's small + * enough, it stays within a single page. We size the XSAVE + * area for the maximum set of features supported by the CPU + * which a guest can enable (which may be more than the NetBSD + * host enables for itself -- hence we don't use + * x86_fpu_save_size here!). + */ + xsave_size = nvmm_x86_xsave_size(vmx_xcr0_mask); + KASSERT(xsave_size < SIZE_MAX - offsetof(struct vmx_cpudata, gfpu)); + cpudata_size = MAX(sizeof(*cpudata), + offsetof(struct vmx_cpudata, gfpu) + xsave_size); + /* Allocate the VMX cpudata. */ cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, + roundup(cpudata_size, PAGE_SIZE), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); vcpu->cpudata = cpudata; @@ -3167,6 +3351,54 @@ vmx_vcpu_configure_tpr(struct vmx_cpudat } static int +vmx_vcpu_configure_xcr0_mask(struct vmx_cpudata *cpudata, void *data) +{ + const uint64_t *xcr0_maskp = data; + + /* + * Refuse to enable XCR0 bits (extended CPU state components) + * not supported by this system, or to set up otherwise + * nonsensical masks like AVX (YMM_Hi128) but not SSE (XMM) + * registers. Exception: The mask can be all-zero to disable + * all XSAVE state components. + */ + if (*xcr0_maskp != 0 && + !nvmm_x86_xcr0_valid(*xcr0_maskp, vmx_xcr0_mask)) + return EINVAL; + + /* + * Out of paranoia, clear any existing extended CPU state. + * This operation is unlikely to be used before the guest has + * begun execution at all, so the extended CPU state is + * probably all zero. But in case some weird hypervisor + * software tries to change the XCR0 mask dynamically, let's + * avoid accidentally leaking things through any extended CPU + * state. + * + * We could zero only the components that are getting disabled. + * But if the saved state is compated (XSAVEC), we wouldd also + * have to move the remaining components around in order to + * avoid zeroing them. Since no software is likely to try this + * anyway, we'll just zero everything to keep it simple and + * avoid having to test the difficult-and-unused paths. + */ + memset(&cpudata->gfpu, 0, nvmm_x86_xsave_size(vmx_xcr0_mask)); + + /* + * Set the XCR0 mask, and limit the guest's XCR0 to this mask. + * Any extended CPU state the guest had previously been using + * will be wiped out. + */ + cpudata->xcr0_mask = *xcr0_maskp; + cpudata->gxcr0 &= cpudata->xcr0_mask; + KASSERTMSG((cpudata->xcr0_mask == 0 || + nvmm_x86_xcr0_valid(cpudata->gxcr0, cpudata->xcr0_mask)), + "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, + cpudata->gxcr0, cpudata->xcr0_mask); + return 0; +} + +static int vmx_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) { struct vmx_cpudata *cpudata = vcpu->cpudata; @@ -3176,6 +3408,8 @@ vmx_vcpu_configure(struct nvmm_cpu *vcpu return vmx_vcpu_configure_cpuid(cpudata, data); case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR): return vmx_vcpu_configure_tpr(cpudata, data); + case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK): + return vmx_vcpu_configure_xcr0_mask(cpudata, data); default: return EINVAL; } @@ -3589,8 +3823,25 @@ vmx_init(void) /* Init the ASID bitmap (VPID). */ vmx_init_asid(VPID_MAX); - /* Init the XCR0 mask. */ - vmx_xcr0_mask = VMX_XCR0_MASK_DEFAULT & x86_xsave_features; + /* + * Init the XCR0 mask. + * + * x86_xsave_features is the cached result of + * CPUID[EAX=0x0000000d,ECX=0].EDX:EAX, the set of all + * supported XCR0 bits for user XSAVE state components on the + * physical CPU. Hypervisor software can use + * nvmm_vcpu_configure(NVMM_VCPU_CONF_XCR0_MASK) to restrict + * the available features on a per-vCPU basis, e.g. in order to + * limit guests to compatible features for migration. + * + * Out of paranoia, we mask off bit 63 which is reserved for + * future extension which we don't understand because it's not + * yet defined. + */ + vmx_xcr0_mask = x86_xsave_features & __BITS(62, 0); + KASSERTMSG((vmx_xcr0_mask == 0 || + nvmm_x86_xcr0_valid(vmx_xcr0_mask, vmx_xcr0_mask)), + "vmx_xcr0_mask=0x%"PRIx64, vmx_xcr0_mask); /* Init the max basic CPUID leaf. */ vmx_cpuid_max_basic = uimin(cpuid_level, VMX_CPUID_MAX_BASIC);