# HG changeset patch # User Taylor R Campbell # Date 1783811482 0 # Sat Jul 11 23:11:22 2026 +0000 # Branch trunk # Node ID 69e5242c35ee9c681e6fe98c101e083a330eeb8b # Parent 187abacabbe955f8145e7c12ca1a0909553a72a3 # EXP-Topic riastradh-pr60426-x86xsavesignal WIP: nvmm: Add support for extended CPU state (XSAVE) beyond x87/SSE. XXX Currently limited to Intel VMX since I have no AMD SVM to test. Changes for nvmm_x86_svm.c should be very similar to those for nvmm_x86_vmx.c, though. New machine-dependent x86 vCPU configuration command NVMM_VCPU_CONF_XCR0_MASK sets the vCPU's XCR0 mask, that is, the set of XSAVE features that the guest sees as supported in the vCPU. This command is advertised by the new machine capability NVMM_CAP_ARCH_VCPU_CONF_XCR0_MASK. This doesn't expose the XSAVE area to userland in the comm page -- there are machine-independent members in struct nvmm_comm_page at fixed offsets past the machine-dependent state so we can't just extend struct nvmm_vcpu_state without breaking the ABI, and the XSAVE area may exceed a page but the address space for the virtual machine uvm object packs comm areas in consecutive pages so we can't put it in the comm page at all without breaking the ABI. Instead, we can expose the XSAVE area in another location in the virtual machine uvm object, and create an ioctl for querying its object address and size so userland can mmap it. TBD. PR port-amd64/59860: nvmm: support AVX2, AVX512, &c. diff -r 187abacabbe9 -r 69e5242c35ee sys/dev/nvmm/x86/nvmm_x86.c --- a/sys/dev/nvmm/x86/nvmm_x86.c Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86.c Sat Jul 11 23:11:22 2026 +0000 @@ -258,7 +258,7 @@ const struct nvmm_x86_cpuid_mask nvmm_cp CPUID2_AESNI | CPUID2_XSAVE | CPUID2_OSXSAVE | - /* CPUID2_AVX excluded */ + CPUID2_AVX | CPUID2_F16C | CPUID2_RDRAND, /* CPUID2_RAZ excluded */ @@ -302,7 +302,7 @@ const struct nvmm_x86_cpuid_mask nvmm_cp /* CPUID_SEF_SGX excluded */ CPUID_SEF_BMI1 | /* CPUID_SEF_HLE excluded */ - /* CPUID_SEF_AVX2 excluded */ + CPUID_SEF_AVX2 | CPUID_SEF_FDPEXONLY | CPUID_SEF_SMEP | CPUID_SEF_BMI2 | @@ -466,3 +466,24 @@ nvmm_x86_pat_validate(uint64_t val) return true; } + +uint32_t +nvmm_x86_xsave_size(uint64_t xcr0) +{ + uint32_t size = sizeof(struct xsave_header); + unsigned i; + + KASSERT((xcr0 & ~__BITS(XSAVE_MAX_COMPONENT, 0)) == 0); + + CTASSERT(sizeof(struct xsave_header) == 512 + 64); + for (i = 0; i < XSAVE_MAX_COMPONENT; i++) { + if ((xcr0 & __BIT(i)) == 0) + continue; + KASSERT(x86_xsave_sizes[i] <= + UINT32_MAX - x86_xsave_offsets[i]); + if (size < x86_xsave_offsets[i] + x86_xsave_sizes[i]) + size = x86_xsave_offsets[i] + x86_xsave_sizes[i]; + } + + return size; +} diff -r 187abacabbe9 -r 69e5242c35ee sys/dev/nvmm/x86/nvmm_x86.h --- a/sys/dev/nvmm/x86/nvmm_x86.h Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86.h Sat Jul 11 23:11:22 2026 +0000 @@ -132,6 +132,7 @@ struct nvmm_cap_md { uint64_t vcpu_conf_support; #define NVMM_CAP_ARCH_VCPU_CONF_CPUID __BIT(0) #define NVMM_CAP_ARCH_VCPU_CONF_TPR __BIT(1) +#define NVMM_CAP_ARCH_VCPU_CONF_XCR0_MASK __BIT(2) uint64_t xcr0_mask; uint32_t mxcsr_mask; @@ -269,6 +270,7 @@ struct nvmm_x64_state { #define NVMM_VCPU_CONF_CPUID NVMM_VCPU_CONF_MD_BEGIN #define NVMM_VCPU_CONF_TPR (NVMM_VCPU_CONF_MD_BEGIN + 1) +#define NVMM_VCPU_CONF_XCR0_MASK (NVMM_VCPU_CONF_MD_BEGIN + 2) struct nvmm_vcpu_conf_cpuid { /* The options. */ @@ -309,7 +311,7 @@ struct nvmm_vcpu_conf_tpr { #ifdef _KERNEL #define NVMM_X86_MACH_NCONF 0 -#define NVMM_X86_VCPU_NCONF 2 +#define NVMM_X86_VCPU_NCONF 3 struct nvmm_x86_cpuid_mask { uint32_t eax; uint32_t ebx; @@ -323,6 +325,7 @@ extern const struct nvmm_x86_cpuid_mask extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000007; extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000008; bool nvmm_x86_pat_validate(uint64_t); +uint32_t nvmm_x86_xsave_size(uint64_t); #endif #endif /* ASM_NVMM */ diff -r 187abacabbe9 -r 69e5242c35ee sys/dev/nvmm/x86/nvmm_x86_vmx.c --- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c Sat Jul 11 14:02:58 2026 +0000 +++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c Sat Jul 11 23:11:22 2026 +0000 @@ -717,7 +717,6 @@ static uint8_t *vmx_asidmap __read_mostl static uint32_t vmx_maxasid __read_mostly; static kmutex_t vmx_asidlock __cacheline_aligned; -#define VMX_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) static uint64_t vmx_xcr0_mask __read_mostly; #define VMX_NCPUIDS 32 @@ -775,7 +774,9 @@ static const size_t vmx_vcpu_conf_sizes[ [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = sizeof(struct nvmm_vcpu_conf_cpuid), [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = - sizeof(struct nvmm_vcpu_conf_tpr) + sizeof(struct nvmm_vcpu_conf_tpr), + [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK)] = + sizeof(uint64_t), }; struct vmx_cpudata { @@ -820,12 +821,15 @@ struct vmx_cpudata { uint64_t gprs[NVMM_X64_NGPR]; uint64_t drs[NVMM_X64_NDR]; uint64_t gtsc; - struct xsave_header gfpu __aligned(64); /* VCPU configuration. */ bool cpuidpresent[VMX_NCPUIDS]; struct nvmm_vcpu_conf_cpuid cpuid[VMX_NCPUIDS]; struct nvmm_vcpu_conf_tpr tpr; + uint64_t xcr0_mask; + + /* Guest XSAVE state. */ + struct xsave_header gfpu __aligned(64); }; static const struct { @@ -1318,6 +1322,14 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx; + /* + * CPUID2_AVX depends on XSAVE support for the YMM high + * halves. + */ + if ((cpudata->xcr0_mask & XCR0_YMM_Hi128) == 0) { + cpudata->gprs[NVMM_X64_GPR_RAX] &= ~CPUID2_AVX; + } + /* CPUID2_OSXSAVE depends on CR4. */ cr4 = vmx_vmread(VMCS_GUEST_CR4); if (!(cr4 & CR4_OSXSAVE)) { @@ -1351,6 +1363,16 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma if (vmx_procbased_ctls2 & PROC_CTLS2_INVPCID_ENABLE) { cpudata->gprs[NVMM_X64_GPR_RBX] |= CPUID_SEF_INVPCID; } + + /* + * CPUID_SEF_AVX2 depends on XSAVE support for + * the YMM high halves. + */ + if ((cpudata->xcr0_mask & XCR0_YMM_Hi128) == 0) { + cpudata->gprs[NVMM_X64_GPR_RAX] &= + ~CPUID_SEF_AVX2; + } + break; default: cpudata->gprs[NVMM_X64_GPR_RAX] = 0; @@ -1407,20 +1429,19 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; case 0x0000000D: /* Processor Extended State Enumeration */ - if (vmx_xcr0_mask == 0) { + if (cpudata->xcr0_mask == 0) { break; } switch (ecx) { case 0: - cpudata->gprs[NVMM_X64_GPR_RAX] = vmx_xcr0_mask & 0xFFFFFFFF; - if (cpudata->gxcr0 & XCR0_SSE) { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); - } else { - cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); - } - cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ - cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; - cpudata->gprs[NVMM_X64_GPR_RDX] = vmx_xcr0_mask >> 32; + cpudata->gprs[NVMM_X64_GPR_RAX] = + cpudata->xcr0_mask & 0xFFFFFFFF; + cpudata->gprs[NVMM_X64_GPR_RBX] = + nvmm_x86_xsave_size(cpudata->gxcr0); + cpudata->gprs[NVMM_X64_GPR_RCX] = + nvmm_x86_xsave_size(cpudata->xcr0_mask); + cpudata->gprs[NVMM_X64_GPR_RDX] = + cpudata->xcr0_mask >> 32; break; case 1: cpudata->gprs[NVMM_X64_GPR_RAX] &= @@ -1430,6 +1451,21 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma cpudata->gprs[NVMM_X64_GPR_RCX] = 0; cpudata->gprs[NVMM_X64_GPR_RDX] = 0; break; + case 2 ... 62: + /* + * CPUID[EAX=0x0d,ECX=n], 2 <= n <= 62: size + * and offset of nth component in XSAVE area. + * If the nth bit of XCR0 is disabled in the + * vCPU configuration, we return all-zero + * instead. + */ + if ((cpudata->xcr0_mask & __BIT(ecx)) == 0) { + cpudata->gprs[NVMM_X64_GPR_RAX] = 0; + cpudata->gprs[NVMM_X64_GPR_RBX] = 0; + cpudata->gprs[NVMM_X64_GPR_RCX] = 0; + cpudata->gprs[NVMM_X64_GPR_RDX] = 0; + } + break; default: cpudata->gprs[NVMM_X64_GPR_RAX] = 0; cpudata->gprs[NVMM_X64_GPR_RBX] = 0; @@ -2004,7 +2040,7 @@ vmx_exit_xsetbv(struct nvmm_machine *mac if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { goto error; - } else if (__predict_false((val & ~vmx_xcr0_mask) != 0)) { + } else if (__predict_false((val & ~cpudata->xcr0_mask) != 0)) { goto error; } else if (__predict_false((val & XCR0_X87) == 0)) { goto error; @@ -2057,9 +2093,9 @@ vmx_vcpu_guest_fpu_enter(struct nvmm_cpu fpu_kern_enter(); /* TODO: should we use *XSAVE64 here? */ - fpu_area_restore(&cpudata->gfpu, vmx_xcr0_mask, false); - - if (vmx_xcr0_mask != 0) { + fpu_area_restore(&cpudata->gfpu, cpudata->xcr0_mask, false); + + if (cpudata->xcr0_mask != 0) { cpudata->hxcr0 = rdxcr(0); wrxcr(0, cpudata->gxcr0); } @@ -2070,13 +2106,13 @@ vmx_vcpu_guest_fpu_leave(struct nvmm_cpu { struct vmx_cpudata *cpudata = vcpu->cpudata; - if (vmx_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { cpudata->gxcr0 = rdxcr(0); wrxcr(0, cpudata->hxcr0); } /* TODO: should we use *XSAVE64 here? */ - fpu_area_save(&cpudata->gfpu, vmx_xcr0_mask, false); + fpu_area_save(&cpudata->gfpu, cpudata->xcr0_mask, false); fpu_kern_leave(); } @@ -2649,10 +2685,10 @@ vmx_vcpu_setstate(struct nvmm_cpu *vcpu) cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8]; - if (vmx_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { /* Clear illegal XCR0 bits, set mandatory X87 bit. */ cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; - cpudata->gxcr0 &= vmx_xcr0_mask; + cpudata->gxcr0 &= cpudata->xcr0_mask; cpudata->gxcr0 |= XCR0_X87; } } @@ -2731,11 +2767,15 @@ vmx_vcpu_setstate(struct nvmm_cpu *vcpu) fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; - if (vmx_xcr0_mask != 0) { + if (cpudata->xcr0_mask != 0) { /* Reset XSTATE_BV, to force a reload. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; } } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ vmx_vmcs_leave(vcpu); @@ -2836,6 +2876,10 @@ vmx_vcpu_getstate(struct nvmm_cpu *vcpu) memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, sizeof(state->fpu)); } + /* + * XXX XSAVE area -- need to allocate and map it separately + * since it may exceed the comm page size + */ vmx_vmcs_leave(vcpu); @@ -3010,7 +3054,8 @@ vmx_vcpu_init(struct nvmm_machine *mach, (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL); /* Init XSAVE header. */ - cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask; + cpudata->xcr0_mask = vmx_xcr0_mask; + cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; cpudata->gfpu.xsh_xcomp_bv = 0; /* These MSRs are static. */ @@ -3032,12 +3077,27 @@ vmx_vcpu_init(struct nvmm_machine *mach, static int vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) { + size_t xsave_size, cpudata_size; struct vmx_cpudata *cpudata; int error; + /* + * Compute the size of the VMX cpudata. We put the + * variable-length XSAVE area at the end so if it's small + * enough, it stays within a single page. We size the XSAVE + * area for the maximum set of features supported by the CPU + * which a guest can enable (which may be more than the NetBSD + * host enables for itself -- hence we don't use + * x86_fpu_save_size here!) + */ + xsave_size = nvmm_x86_xsave_size(vmx_xcr0_mask); + KASSERT(xsave_size < SIZE_MAX - offsetof(struct vmx_cpudata, gfpu)); + cpudata_size = MAX(sizeof(*cpudata), + offsetof(struct vmx_cpudata, gfpu) + xsave_size); + /* Allocate the VMX cpudata. */ cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map, - roundup(sizeof(*cpudata), PAGE_SIZE), 0, + roundup(cpudata_size, PAGE_SIZE), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); vcpu->cpudata = cpudata; @@ -3167,6 +3227,18 @@ vmx_vcpu_configure_tpr(struct vmx_cpudat } static int +vmx_vcpu_configure_xcr0_mask(struct vmx_cpudata *cpudata, void *data) +{ + const uint64_t *xcr0_maskp = data; + + if (*xcr0_maskp & ~vmx_xcr0_mask) + return EINVAL; + cpudata->xcr0_mask = *xcr0_maskp; + return 0; +} + + +static int vmx_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) { struct vmx_cpudata *cpudata = vcpu->cpudata; @@ -3176,6 +3248,8 @@ vmx_vcpu_configure(struct nvmm_cpu *vcpu return vmx_vcpu_configure_cpuid(cpudata, data); case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR): return vmx_vcpu_configure_tpr(cpudata, data); + case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK): + return vmx_vcpu_configure_xcr0_mask(cpudata, data); default: return EINVAL; } @@ -3589,8 +3663,16 @@ vmx_init(void) /* Init the ASID bitmap (VPID). */ vmx_init_asid(VPID_MAX); - /* Init the XCR0 mask. */ - vmx_xcr0_mask = VMX_XCR0_MASK_DEFAULT & x86_xsave_features; + /* + * Init the XCR0 mask. x86_xsave_features is the cached result + * of CPUID[EAX=0x0000000d,ECX=0].EDX:EAX, the set of all + * supported XCR0 bits for user XSAVE state components on the + * physical CPU. Hypervisor software can use + * nvmm_vcpu_configure(NVMM_VCPU_CONF_XCR0_MASK) to restrict + * the available features on a per-vCPU basis, e.g. in order to + * limit guests to compatible features for migration. + */ + vmx_xcr0_mask = x86_xsave_features; /* Init the max basic CPUID leaf. */ vmx_cpuid_max_basic = uimin(cpuid_level, VMX_CPUID_MAX_BASIC);