diff --git a/sys/arch/amd64/amd64/db_interface.c b/sys/arch/amd64/amd64/db_interface.c index 9acf7dd497c..eb693c151a2 100644 --- a/sys/arch/amd64/amd64/db_interface.c +++ b/sys/arch/amd64/amd64/db_interface.c @@ -111,12 +111,14 @@ db_machine_init(void) #ifdef MULTIPROCESSOR #ifndef XENPV + struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); vector *handler = &Xintr_ddbipi; + idt_descriptor_t *idt = iv->iv_idt; #if NLAPIC > 0 if (lapic_is_x2apic()) handler = &Xintr_x2apic_ddbipi; #endif - ddb_vec = idt_vec_alloc(0xf0, 0xff); + ddb_vec = idt_vec_alloc(iv, 0xf0, 0xff); set_idtgate(&idt[ddb_vec], handler, 1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #else diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c index 01c8dec8b50..67387d49bdc 100644 --- a/sys/arch/amd64/amd64/machdep.c +++ b/sys/arch/amd64/amd64/machdep.c @@ -1392,11 +1392,15 @@ char *ldtstore; char *gdtstore; void -setgate(struct gate_descriptor *gd, void *func, int ist, int type, int dpl, int sel) +setgate(struct gate_descriptor *gd, void *func, + int ist, int type, int dpl, int sel) { + vaddr_t vaddr; + + vaddr = ((vaddr_t)gd) & ~PAGE_MASK; kpreempt_disable(); - pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE); + pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); gd->gd_looffset = (uint64_t)func & 0xffff; gd->gd_selector = sel; @@ -1410,20 +1414,23 @@ setgate(struct gate_descriptor *gd, void *func, int ist, int type, int dpl, int gd->gd_xx2 = 0; gd->gd_xx3 = 0; - pmap_changeprot_local(idt_vaddr, VM_PROT_READ); + pmap_changeprot_local(vaddr, VM_PROT_READ); kpreempt_enable(); } void unsetgate(struct gate_descriptor *gd) { + vaddr_t vaddr; + + vaddr = ((vaddr_t)gd) & ~PAGE_MASK; kpreempt_disable(); - pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE); + pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); memset(gd, 0, sizeof (*gd)); - pmap_changeprot_local(idt_vaddr, VM_PROT_READ); + pmap_changeprot_local(vaddr, VM_PROT_READ); kpreempt_enable(); } @@ -1470,10 +1477,12 @@ set_sys_segment(struct sys_segment_descriptor *sd, void *base, size_t limit, } void -cpu_init_idt(void) +cpu_init_idt(struct cpu_info *ci) { struct region_descriptor region; + idt_descriptor_t *idt; + idt = ci->ci_idtvec.iv_idt; setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1); lidt(®ion); } @@ -1677,6 +1686,8 @@ init_x86_64(paddr_t first_avail) extern void consinit(void); struct region_descriptor region; struct mem_segment_descriptor *ldt_segp; + struct idt_vec *iv; + idt_descriptor_t *idt; int x; struct pcb *pcb; extern vaddr_t lwp0uarea; @@ -1807,7 +1818,9 @@ init_x86_64(paddr_t first_avail) pmap_update(pmap_kernel()); - idt = (idt_descriptor_t *)idt_vaddr; + iv = &(cpu_info_primary.ci_idtvec); + idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary)); + idt = iv->iv_idt; gdtstore = (char *)gdt_vaddr; ldtstore = (char *)ldt_vaddr; @@ -1872,7 +1885,7 @@ init_x86_64(paddr_t first_avail) sel = SEL_KPL; ist = 0; - idt_vec_reserve(x); + idt_vec_reserve(iv, x); switch (x) { case 1: /* DB */ @@ -1902,7 +1915,7 @@ init_x86_64(paddr_t first_avail) } /* new-style interrupt gate for syscalls */ - idt_vec_reserve(128); + idt_vec_reserve(iv, 128); set_idtgate(&idt[128], &IDTVEC(osyscall), 0, SDT_SYS386IGT, SEL_UPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -1920,7 +1933,7 @@ init_x86_64(paddr_t first_avail) panic("HYPERVISOR_set_callbacks() failed"); #endif /* XENPV */ - cpu_init_idt(); + cpu_init_idt(&cpu_info_primary); #ifdef XENPV xen_init_ksyms(); @@ -1961,6 +1974,14 @@ init_x86_64(paddr_t first_avail) void cpu_reset(void) { +#ifndef XENPV + idt_descriptor_t *idt; + vaddr_t vaddr; + + idt = cpu_info_primary.ci_idtvec.iv_idt; + vaddr = (vaddr_t)idt; +#endif + x86_disable_intr(); #ifdef XENPV @@ -1974,7 +1995,7 @@ cpu_reset(void) * invalid and causing a fault. */ kpreempt_disable(); - pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE); + pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); memset((void *)idt, 0, NIDT * sizeof(idt[0])); kpreempt_enable(); breakpoint(); @@ -2321,3 +2342,56 @@ mm_md_direct_mapped_phys(paddr_t paddr, vaddr_t *vaddr) return false; #endif } + +static void +idt_vec_copy(struct idt_vec *dst, struct idt_vec *src) +{ + idt_descriptor_t *idt_dst; + + idt_dst = dst->iv_idt; + + kpreempt_disable(); + pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ|VM_PROT_WRITE); + + memcpy(idt_dst, src->iv_idt, PAGE_SIZE); + memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap)); + + pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ); + kpreempt_enable(); +} + +void +idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid) +{ + vaddr_t va; + + if (cid != cpu_index(&cpu_info_primary) && + idt_vec_is_pcpu()) { +#ifdef __HAVE_PCPU_AREA + va = (vaddr_t)&pcpuarea->ent[cid].idt; +#else + struct vm_page *pg; + + va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, + UVM_KMF_VAONLY); + pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); + if (pg == NULL) { + panic("failed to allocate a page for IDT"); + } + pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), + VM_PROT_READ|VM_PROT_WRITE, 0); + pmap_update(pmap_kernel()); +#endif + + memset((void *)va, 0, PAGE_SIZE); +#ifndef XENPV + pmap_changeprot_local(va, VM_PROT_READ); +#endif + pmap_update(pmap_kernel()); + + iv->iv_idt = (void *)va; + idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec)); + } else { + iv->iv_idt = (void *)idt_vaddr; + } +} diff --git a/sys/arch/amd64/conf/ALL b/sys/arch/amd64/conf/ALL index a26e456f6f9..e5634c1bb15 100644 --- a/sys/arch/amd64/conf/ALL +++ b/sys/arch/amd64/conf/ALL @@ -29,6 +29,7 @@ options X86EMU # 386 Real Mode emulator #options PAE # PAE mode (36 bits physical addressing) makeoptions SPECTRE_V2_GCC_MITIGATION=1 # GCC Spectre variant 2 # migitation +options PCPU_IDT # Per CPU IDT # CPU features acpicpu* at cpu? # ACPI CPU (including frequency scaling) diff --git a/sys/arch/amd64/conf/GENERIC b/sys/arch/amd64/conf/GENERIC index 50346814eca..d8dcd800cf4 100644 --- a/sys/arch/amd64/conf/GENERIC +++ b/sys/arch/amd64/conf/GENERIC @@ -79,6 +79,7 @@ options SVS # Separate Virtual Space makeoptions SPECTRE_V2_GCC_MITIGATION=1 # GCC Spectre variant 2 # migitation options SPECTRE_V2_GCC_MITIGATION +options PCPU_IDT # Per CPU IDT # CPU features acpicpu* at cpu? # ACPI CPU (including frequency scaling) diff --git a/sys/arch/amd64/include/segments.h b/sys/arch/amd64/include/segments.h index 27bf69d4496..fc88bdbccac 100644 --- a/sys/arch/amd64/include/segments.h +++ b/sys/arch/amd64/include/segments.h @@ -238,7 +238,6 @@ typedef struct trap_info idt_descriptor_t; #else typedef struct gate_descriptor idt_descriptor_t; #endif /* XENPV */ -extern idt_descriptor_t *idt; extern char *gdtstore; extern char *ldtstore; @@ -251,14 +250,16 @@ void set_sys_segment(struct sys_segment_descriptor *, void *, size_t, int, int, int); void set_mem_segment(struct mem_segment_descriptor *, void *, size_t, int, int, int, int, int); -void cpu_init_idt(void); void update_descriptor(void *, void *); - -void idt_vec_reserve(int); -int idt_vec_alloc(int, int); -void idt_vec_set(int, void (*)(void)); -void idt_vec_free(int); +struct idt_vec; +void idt_vec_reserve(struct idt_vec *, int); +int idt_vec_alloc(struct idt_vec *, int, int); +void idt_vec_set(struct idt_vec *, int, void (*)(void)); +void idt_vec_free(struct idt_vec *, int); +void idt_vec_init_cpu_md(struct idt_vec *, cpuid_t); +bool idt_vec_is_pcpu(void); +struct idt_vec * idt_vec_ref(struct idt_vec *); struct lwp; diff --git a/sys/arch/i386/conf/ALL b/sys/arch/i386/conf/ALL index d37e479b6f7..fbb1462c0a0 100644 --- a/sys/arch/i386/conf/ALL +++ b/sys/arch/i386/conf/ALL @@ -29,6 +29,7 @@ options X86EMU # 386 Real Mode emulator options PAE # PAE mode (36 bits physical addressing) makeoptions SPECTRE_V2_GCC_MITIGATION=1 # GCC Spectre variant 2 # migitation +options PCPU_IDT # Per CPU IDT # CPU features acpicpu* at cpu? # ACPI CPU (including frequency scaling) diff --git a/sys/arch/i386/i386/db_interface.c b/sys/arch/i386/i386/db_interface.c index c44ec273100..03241c64769 100644 --- a/sys/arch/i386/i386/db_interface.c +++ b/sys/arch/i386/i386/db_interface.c @@ -114,12 +114,15 @@ db_machine_init(void) #ifdef MULTIPROCESSOR #ifndef XENPV vector *handler = &Xintr_ddbipi; + struct idt_vec *iv; + + iv = &(cpu_info_primary.ci_idtvec); #if NLAPIC > 0 if (lapic_is_x2apic()) handler = &Xintr_x2apic_ddbipi; #endif - ddb_vec = idt_vec_alloc(0xf0, 0xff); - idt_vec_set(ddb_vec, handler); + ddb_vec = idt_vec_alloc(iv, 0xf0, 0xff); + idt_vec_set(iv, ddb_vec, handler); #else /* Initialised as part of xen_ipi_init() */ #endif /* XENPV */ diff --git a/sys/arch/i386/i386/machdep.c b/sys/arch/i386/i386/machdep.c index f4f803d7ba5..b2e70bb15ce 100644 --- a/sys/arch/i386/i386/machdep.c +++ b/sys/arch/i386/i386/machdep.c @@ -554,6 +554,7 @@ cpu_set_tss_gates(struct cpu_info *ci) { struct segment_descriptor sd; void *doubleflt_stack; + idt_descriptor_t *idt; doubleflt_stack = (void *)uvm_km_alloc(kernel_map, USPACE, 0, UVM_KMF_WIRED); @@ -563,6 +564,7 @@ cpu_set_tss_gates(struct cpu_info *ci) SDT_SYS386TSS, SEL_KPL, 0, 0); ci->ci_gdt[GTRAPTSS_SEL].sd = sd; + idt = cpu_info_primary.ci_idtvec.iv_idt; set_idtgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, GSEL(GTRAPTSS_SEL, SEL_KPL)); @@ -906,6 +908,7 @@ setgate(struct gate_descriptor *gd, void *func, int args, int type, int dpl, void unsetgate(struct gate_descriptor *gd) { + gd->gd_p = 0; gd->gd_hioffset = 0; gd->gd_looffset = 0; @@ -949,10 +952,15 @@ extern union descriptor tmpgdt[]; #endif void -cpu_init_idt(void) +cpu_init_idt(struct cpu_info *ci) { struct region_descriptor region; - setregion(®ion, pentium_idt, NIDT * sizeof(idt[0]) - 1); + struct idt_vec *iv; + idt_descriptor_t *idt; + + iv = &ci->ci_idtvec; + idt = iv->iv_idt_pentium; + setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1); lidt(®ion); } @@ -1134,6 +1142,8 @@ init386(paddr_t first_avail) #endif #endif /* !XENPV */ struct pcb *pcb; + struct idt_vec *iv; + idt_descriptor_t *idt; KASSERT(first_avail % PAGE_SIZE == 0); @@ -1305,8 +1315,9 @@ init386(paddr_t first_avail) pmap_kenter_pa(pentium_idt_vaddr, idt_paddr, VM_PROT_READ, 0); pmap_update(pmap_kernel()); - pentium_idt = (union descriptor *)pentium_idt_vaddr; - idt = (idt_descriptor_t *)idt_vaddr; + iv = &(cpu_info_primary.ci_idtvec); + idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary)); + idt = (idt_descriptor_t *)iv->iv_idt; #ifndef XENPV tgdt = gdtstore; @@ -1340,7 +1351,7 @@ init386(paddr_t first_avail) sel = SEL_KPL; #endif /* XENPV */ - idt_vec_reserve(x); + idt_vec_reserve(iv, x); switch (x) { #ifdef XENPV @@ -1361,7 +1372,7 @@ init386(paddr_t first_avail) } /* new-style interrupt gate for syscalls */ - idt_vec_reserve(128); + idt_vec_reserve(iv, 128); set_idtgate(&idt[128], &IDTVEC(syscall), 0, SDT_SYS386IGT, SEL_UPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -1371,7 +1382,7 @@ init386(paddr_t first_avail) #endif lldt(GSEL(GLDT_SEL, SEL_KPL)); - cpu_init_idt(); + cpu_init_idt(&cpu_info_primary); #ifdef XENPV xen_init_ksyms(); @@ -1441,7 +1452,9 @@ cpu_reset(void) for (;;); #else /* XENPV */ struct region_descriptor region; + idt_descriptor_t *idt; + idt = (idt_descriptor_t *)cpu_info_primary.ci_idtvec.iv_idt; x86_disable_intr(); /* @@ -1698,3 +1711,50 @@ cpu_alloc_l3_page(struct cpu_info *ci) pmap_update(pmap_kernel()); } #endif /* PAE */ + +static void +idt_vec_copy(struct idt_vec *dst, struct idt_vec *src) +{ + idt_descriptor_t *idt_dst; + + idt_dst = dst->iv_idt; + memcpy(idt_dst, src->iv_idt, PAGE_SIZE); + memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap)); +} + +void +idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid) +{ + vaddr_t va_idt, va_pentium_idt; + struct vm_page *pg; + + if (idt_vec_is_pcpu() && + cid != cpu_index(&cpu_info_primary)) { + va_idt = uvm_km_alloc(kernel_map, PAGE_SIZE, + 0, UVM_KMF_VAONLY); + pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); + if (pg == NULL) { + panic("failed to allocate pcpu idt PA"); + } + pmap_kenter_pa(va_idt, VM_PAGE_TO_PHYS(pg), + VM_PROT_READ|VM_PROT_WRITE, 0); + pmap_update(pmap_kernel()); + + memset((void *)va_idt, 0, PAGE_SIZE); + + /* pentium f00f bug stuff */ + va_pentium_idt = uvm_km_alloc(kernel_map, PAGE_SIZE, + 0, UVM_KMF_VAONLY); + pmap_kenter_pa(va_pentium_idt, VM_PAGE_TO_PHYS(pg), + VM_PROT_READ, 0); + pmap_update(pmap_kernel()); + + iv->iv_idt = (void *)va_idt; + iv->iv_idt_pentium = (void *)va_pentium_idt; + + idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec)); + } else { + iv->iv_idt = (void *)idt_vaddr; + iv->iv_idt_pentium = (void *)pentium_idt_vaddr; + } +} diff --git a/sys/arch/i386/i386/trap.c b/sys/arch/i386/i386/trap.c index a1a314d50b2..91451ed2413 100644 --- a/sys/arch/i386/i386/trap.c +++ b/sys/arch/i386/i386/trap.c @@ -245,10 +245,16 @@ int ss_shadow(struct trapframe *tf) { struct gate_descriptor *gd; + struct cpu_info *ci; + struct idt_vec *iv; + idt_descriptor_t *idt; uintptr_t eip, func; size_t i; eip = tf->tf_eip; + ci = curcpu(); + iv = idt_vec_ref(&ci->ci_idtvec); + idt = iv->iv_idt; for (i = 0; i < 256; i++) { gd = &idt[i]; diff --git a/sys/arch/i386/include/segments.h b/sys/arch/i386/include/segments.h index 77a1b85794f..7388aa3befb 100644 --- a/sys/arch/i386/include/segments.h +++ b/sys/arch/i386/include/segments.h @@ -196,7 +196,6 @@ typedef struct trap_info idt_descriptor_t; #else typedef struct gate_descriptor idt_descriptor_t; #endif /* XENPV */ -extern idt_descriptor_t *idt; extern union descriptor *gdtstore, *ldtstore; void setgate(struct gate_descriptor *, void *, int, int, int, int); @@ -206,15 +205,16 @@ void setregion(struct region_descriptor *, void *, size_t); void setsegment(struct segment_descriptor *, const void *, size_t, int, int, int, int); void unsetgate(struct gate_descriptor *); - -void cpu_init_idt(void); void update_descriptor(union descriptor *, union descriptor *); - -void idt_vec_reserve(int); -int idt_vec_alloc(int, int); -void idt_vec_set(int, void (*)(void)); -void idt_vec_free(int); +struct idt_vec; +void idt_vec_reserve(struct idt_vec *, int); +int idt_vec_alloc(struct idt_vec *, int, int); +void idt_vec_set(struct idt_vec *, int, void (*)(void)); +void idt_vec_free(struct idt_vec *, int); +void idt_vec_init_cpu_md(struct idt_vec *, cpuid_t); +bool idt_vec_is_pcpu(void); +struct idt_vec* idt_vec_ref(struct idt_vec *); #endif /* _KERNEL */ diff --git a/sys/arch/x86/conf/files.x86 b/sys/arch/x86/conf/files.x86 index b6486164093..918505da0d2 100644 --- a/sys/arch/x86/conf/files.x86 +++ b/sys/arch/x86/conf/files.x86 @@ -22,6 +22,8 @@ defflag opt_kaslr.h NO_X86_ASLR defflag SVS +defflag PCPU_IDT + define cpubus { [apid = -1] } define cpufeaturebus {} define ioapicbus { [apid = -1] } diff --git a/sys/arch/x86/include/cpu.h b/sys/arch/x86/include/cpu.h index 5750540138c..372e4c66663 100644 --- a/sys/arch/x86/include/cpu.h +++ b/sys/arch/x86/include/cpu.h @@ -103,6 +103,12 @@ struct clockframe { struct intrframe cf_if; }; +struct idt_vec { + void *iv_idt; + void *iv_idt_pentium; + char iv_allocmap[NIDT]; +}; + /* * a bunch of this belongs in cpuvar.h; move it later.. */ @@ -126,6 +132,7 @@ struct cpu_info { uint64_t ci_scratch; uintptr_t ci_pmap_data[128 / sizeof(uintptr_t)]; struct kcpuset *ci_tlb_cpuset; + struct idt_vec ci_idtvec; int ci_kfpu_spl; diff --git a/sys/arch/x86/include/cpuvar.h b/sys/arch/x86/include/cpuvar.h index 66bd3f60018..a86f76edbeb 100644 --- a/sys/arch/x86/include/cpuvar.h +++ b/sys/arch/x86/include/cpuvar.h @@ -112,6 +112,7 @@ void identifycpu_cpuids(struct cpu_info *); void cpu_init(struct cpu_info *); void cpu_init_tss(struct cpu_info *); void cpu_init_first(void); +void cpu_init_idt(struct cpu_info *); void x86_cpu_idle_init(void); void x86_cpu_idle_halt(void); diff --git a/sys/arch/x86/include/pmap.h b/sys/arch/x86/include/pmap.h index c2db99cd83a..b4a6c53e67f 100644 --- a/sys/arch/x86/include/pmap.h +++ b/sys/arch/x86/include/pmap.h @@ -198,6 +198,7 @@ extern struct slotspace slotspace; struct pcpu_entry { uint8_t gdt[MAXGDTSIZ]; uint8_t ldt[MAX_USERLDT_SIZE]; + uint8_t idt[PAGE_SIZE]; uint8_t tss[PAGE_SIZE]; uint8_t ist0[PAGE_SIZE]; uint8_t ist1[PAGE_SIZE]; @@ -210,7 +211,6 @@ struct pcpu_area { #ifdef SVS uint8_t utls[PAGE_SIZE]; #endif - uint8_t idt[PAGE_SIZE]; uint8_t ldt[PAGE_SIZE]; struct pcpu_entry ent[MAXCPUS]; } __packed; diff --git a/sys/arch/x86/x86/cpu.c b/sys/arch/x86/x86/cpu.c index 734dfe4d0f4..9b6f32873e5 100644 --- a/sys/arch/x86/x86/cpu.c +++ b/sys/arch/x86/x86/cpu.c @@ -492,6 +492,7 @@ cpu_attach(device_t parent, device_t self, void *aux) * report on an AP */ cpu_intr_init(ci); + idt_vec_init_cpu_md(&ci->ci_idtvec, cpu_index(ci)); gdt_alloc_cpu(ci); #ifdef i386 cpu_set_tss_gates(ci); @@ -980,7 +981,7 @@ cpu_hatch(void *v) pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); lcr0(pcb->pcb_cr0); - cpu_init_idt(); + cpu_init_idt(ci); gdt_init_cpu(ci); #if NLAPIC > 0 lapic_enable(); diff --git a/sys/arch/x86/x86/hyperv.c b/sys/arch/x86/x86/hyperv.c index 0fbf52175f3..e865b9610a5 100644 --- a/sys/arch/x86/x86/hyperv.c +++ b/sys/arch/x86/x86/hyperv.c @@ -95,6 +95,10 @@ struct hyperv_hypercall_ctx { paddr_t hc_paddr; }; +struct hyperv_percpu_data { + int pd_idtvec; +}; + static struct hyperv_hypercall_ctx hyperv_hypercall_ctx; static char hyperv_hypercall_page[PAGE_SIZE] @@ -115,8 +119,6 @@ static char hyperv_features_str[256]; static char hyperv_pm_features_str[256]; static char hyperv_features3_str[256]; -static int hyperv_idtvec; - uint32_t hyperv_vcpuid[MAXCPUS]; static struct timecounter hyperv_timecounter = { @@ -778,37 +780,91 @@ void vmbus_init_interrupts_md(struct vmbus_softc *sc) { extern void Xintr_hyperv_hypercall(void); + struct vmbus_percpu_data *pd; + struct hyperv_percpu_data *hv_pd; + struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); + cpuid_t cid; + if (idt_vec_is_pcpu()) + return; /* * All Hyper-V ISR required resources are setup, now let's find a * free IDT vector for Hyper-V ISR and set it up. */ + iv = &(cpu_info_primary.ci_idtvec); + cid = cpu_index(&cpu_info_primary); + pd = &sc->sc_percpu[cid]; + + hv_pd = kmem_zalloc(sizeof(*hv_pd), KM_SLEEP); mutex_enter(&cpu_lock); - hyperv_idtvec = idt_vec_alloc(APIC_LEVEL(NIPL), IDT_INTR_HIGH); + hv_pd->pd_idtvec = idt_vec_alloc(iv, + APIC_LEVEL(NIPL), IDT_INTR_HIGH); mutex_exit(&cpu_lock); - KASSERT(hyperv_idtvec > 0); - idt_vec_set(hyperv_idtvec, Xintr_hyperv_hypercall); + KASSERT(hv_pd->pd_idtvec > 0); + idt_vec_set(iv, hv_pd->pd_idtvec, Xintr_hyperv_hypercall); + pd->md_cookie = (void *)hv_pd; } void vmbus_deinit_interrupts_md(struct vmbus_softc *sc) { + struct vmbus_percpu_data *pd; + struct hyperv_percpu_data *hv_pd; + struct idt_vec *iv; + cpuid_t cid; - if (hyperv_idtvec > 0) { - idt_vec_free(hyperv_idtvec); - hyperv_idtvec = 0; - } + if (idt_vec_is_pcpu()) + return; + + iv = &(cpu_info_primary.ci_idtvec); + cid = cpu_index(&cpu_info_primary); + pd = &sc->sc_percpu[cid]; + hv_pd = pd->md_cookie; + + if (hv_pd->pd_idtvec > 0) + idt_vec_free(iv, hv_pd->pd_idtvec); + + pd->md_cookie = NULL; + kmem_free(hv_pd, sizeof(*hv_pd)); } void vmbus_init_synic_md(struct vmbus_softc *sc, cpuid_t cpu) { - struct vmbus_percpu_data *pd; + extern void Xintr_hyperv_hypercall(void); + struct vmbus_percpu_data *pd, *pd0; + struct hyperv_percpu_data *hv_pd; + struct cpu_info *ci; + struct idt_vec *iv; uint64_t val, orig; uint32_t sint; + int hyperv_idtvec; pd = &sc->sc_percpu[cpu]; + hv_pd = kmem_alloc(sizeof(*hv_pd), KM_SLEEP); + pd->md_cookie = (void *)hv_pd; + + /* Allocate IDT vector for ISR and set it up. */ + if (idt_vec_is_pcpu()) { + ci = curcpu(); + iv = &ci->ci_idtvec; + + mutex_enter(&cpu_lock); + hyperv_idtvec = idt_vec_alloc(iv, APIC_LEVEL(NIPL), IDT_INTR_HIGH); + mutex_exit(&cpu_lock); + KASSERT(hyperv_idtvec > 0); + idt_vec_set(iv, hyperv_idtvec, Xintr_hyperv_hypercall); + + hv_pd = kmem_alloc(sizeof(*hv_pd), KM_SLEEP); + hv_pd->pd_idtvec = hyperv_idtvec; + pd->md_cookie = hv_pd; + } else { + pd0 = &sc->sc_percpu[cpu_index(&cpu_info_primary)]; + hv_pd = pd0->md_cookie; + hyperv_idtvec = hv_pd->pd_idtvec; + } + /* * Setup the SynIC message. */ @@ -854,6 +910,10 @@ vmbus_init_synic_md(struct vmbus_softc *sc, cpuid_t cpu) void vmbus_deinit_synic_md(struct vmbus_softc *sc, cpuid_t cpu) { + struct vmbus_percpu_data *pd; + struct hyperv_percpu_data *hv_pd; + struct cpu_info *ci; + struct idt_vec *iv; uint64_t orig; uint32_t sint; @@ -888,6 +948,22 @@ vmbus_deinit_synic_md(struct vmbus_softc *sc, cpuid_t cpu) */ orig = rdmsr(MSR_HV_SIEFP); wrmsr(MSR_HV_SIEFP, (orig & MSR_HV_SIEFP_RSVD_MASK)); + + /* + * Free IDT vector + */ + if (idt_vec_is_pcpu()) { + ci = curcpu(); + iv = &ci->ci_idtvec; + pd = &sc->sc_percpu[cpu_index(ci)]; + hv_pd = pd->md_cookie; + + if (hv_pd->pd_idtvec > 0) + idt_vec_free(iv, hv_pd->pd_idtvec); + + pd->md_cookie = NULL; + kmem_free(hv_pd, sizeof(*hv_pd)); + } } static int diff --git a/sys/arch/x86/x86/idt.c b/sys/arch/x86/x86/idt.c index 5e91d2521b6..c9be9963f9c 100644 --- a/sys/arch/x86/x86/idt.c +++ b/sys/arch/x86/x86/idt.c @@ -67,6 +67,8 @@ #include __KERNEL_RCSID(0, "$NetBSD: idt.c,v 1.11 2019/06/17 06:38:30 msaitoh Exp $"); +#include "opt_pcpu_idt.h" + #include #include #include @@ -81,9 +83,6 @@ __KERNEL_RCSID(0, "$NetBSD: idt.c,v 1.11 2019/06/17 06:38:30 msaitoh Exp $"); * XEN PV and native have a different idea of what idt entries should * look like. */ -idt_descriptor_t *idt; - -static char idt_allocmap[NIDT]; /* Normalise across XEN PV and native */ #if defined(XENPV) @@ -164,9 +163,10 @@ unset_idtgate(struct gate_descriptor *idd) * cpu_lock will be held unless single threaded during early boot. */ int -idt_vec_alloc(int low, int high) +idt_vec_alloc(struct idt_vec *iv, int low, int high) { int vec; + char *idt_allocmap = iv->iv_allocmap; KASSERT(mutex_owned(&cpu_lock) || !mp_online); @@ -182,23 +182,26 @@ idt_vec_alloc(int low, int high) } void -idt_vec_reserve(int vec) +idt_vec_reserve(struct idt_vec *iv, int vec) { int result; KASSERT(mutex_owned(&cpu_lock) || !mp_online); - result = idt_vec_alloc(vec, vec); + result = idt_vec_alloc(iv, vec, vec); if (result != vec) { panic("%s: failed to reserve vec %d", __func__, vec); } } void -idt_vec_set(int vec, void (*function)(void)) +idt_vec_set(struct idt_vec *iv, int vec, void (*function)(void)) { + idt_descriptor_t *idt; + char *idt_allocmap = iv->iv_allocmap; KASSERT(idt_allocmap[vec] == 1); + idt = iv->iv_idt; set_idtgate(&idt[vec], function, 0, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); } @@ -207,10 +210,32 @@ idt_vec_set(int vec, void (*function)(void)) * Free IDT vector. No locking required as release is atomic. */ void -idt_vec_free(int vec) +idt_vec_free(struct idt_vec *iv, int vec) { + idt_descriptor_t *idt; + char *idt_allocmap = iv->iv_allocmap; + idt = iv->iv_idt; unset_idtgate(&idt[vec]); idt_allocmap[vec] = 0; } +bool +idt_vec_is_pcpu(void) +{ + +#ifdef PCPU_IDT + return true; +#else + return false; +#endif +} + +struct idt_vec * +idt_vec_ref(struct idt_vec *iv) +{ + if (idt_vec_is_pcpu()) + return iv; + + return &(cpu_info_primary.ci_idtvec); +} diff --git a/sys/arch/x86/x86/intr.c b/sys/arch/x86/x86/intr.c index 6fb1956e785..488aa3e631d 100644 --- a/sys/arch/x86/x86/intr.c +++ b/sys/arch/x86/x86/intr.c @@ -252,12 +252,13 @@ static int intr_set_affinity(struct intrsource *, const kcpuset_t *); void intr_default_setup(void) { + struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); int i; /* icu vectors */ for (i = 0; i < NUM_LEGACY_IRQS; i++) { - idt_vec_reserve(ICU_OFFSET + i); - idt_vec_set(ICU_OFFSET + i, legacy_stubs[i].ist_entry); + idt_vec_reserve(iv, ICU_OFFSET + i); + idt_vec_set(iv, ICU_OFFSET + i, legacy_stubs[i].ist_entry); } /* @@ -503,6 +504,7 @@ intr_allocate_slot(struct pic *pic, int pin, int level, struct cpu_info *ci, *lci; struct intrsource *isp; int slot = 0, idtvec, error; + struct idt_vec *iv; KASSERT(mutex_owned(&cpu_lock)); @@ -600,7 +602,8 @@ intr_allocate_slot(struct pic *pic, int pin, int level, * are used by a device using MSI multiple vectors must be * continuous. */ - idtvec = idt_vec_alloc(APIC_LEVEL(level), IDT_INTR_HIGH); + iv = idt_vec_ref(&ci->ci_idtvec); + idtvec = idt_vec_alloc(iv, APIC_LEVEL(level), IDT_INTR_HIGH); } if (idtvec == 0) { evcnt_detach(&ci->ci_isources[slot]->is_evcnt); @@ -619,14 +622,16 @@ static void intr_source_free(struct cpu_info *ci, int slot, struct pic *pic, int idtvec) { struct intrsource *isp; + struct idt_vec *iv; isp = ci->ci_isources[slot]; + iv = idt_vec_ref(&ci->ci_idtvec); if (isp->is_handlers != NULL) return; ci->ci_isources[slot] = NULL; if (pic != &i8259_pic) - idt_vec_free(idtvec); + idt_vec_free(iv, idtvec); } #ifdef MULTIPROCESSOR @@ -703,6 +708,7 @@ intr_hwunmask_xcall(void *arg1, void *arg2) static void intr_establish_xcall(void *arg1, void *arg2) { + struct idt_vec *iv; struct intrsource *source; struct intrstub *stubp; struct intrhand *ih; @@ -717,6 +723,7 @@ intr_establish_xcall(void *arg1, void *arg2) ci = ih->ih_cpu; source = ci->ci_isources[ih->ih_slot]; idt_vec = (int)(intptr_t)arg2; + iv = idt_vec_ref(&ci->ci_idtvec); /* Disable interrupts locally. */ psl = x86_read_psl(); @@ -729,7 +736,7 @@ intr_establish_xcall(void *arg1, void *arg2) /* Hook in new IDT vector and SPL state. */ if (source->is_resume == NULL || source->is_idtvec != idt_vec) { if (source->is_idtvec != 0 && source->is_idtvec != idt_vec) - idt_vec_free(source->is_idtvec); + idt_vec_free(iv, source->is_idtvec); source->is_idtvec = idt_vec; if (source->is_type == IST_LEVEL) { stubp = &source->is_pic->pic_level_stubs[ih->ih_slot]; @@ -738,7 +745,7 @@ intr_establish_xcall(void *arg1, void *arg2) } source->is_resume = stubp->ist_resume; source->is_recurse = stubp->ist_recurse; - idt_vec_set(idt_vec, stubp->ist_entry); + idt_vec_set(iv, idt_vec, stubp->ist_entry); } /* Re-enable interrupts locally. */ @@ -1713,6 +1720,7 @@ intr_activate_xcall(void *arg1, void *arg2) struct intrsource *source; struct intrstub *stubp; struct intrhand *ih; + struct idt_vec *iv; u_long psl; int idt_vec; int slot; @@ -1727,6 +1735,7 @@ intr_activate_xcall(void *arg1, void *arg2) slot = ih->ih_slot; source = ci->ci_isources[slot]; idt_vec = source->is_idtvec; + iv = idt_vec_ref(&ci->ci_idtvec); psl = x86_read_psl(); x86_disable_intr(); @@ -1738,9 +1747,10 @@ intr_activate_xcall(void *arg1, void *arg2) } else { stubp = &source->is_pic->pic_edge_stubs[slot]; } + source->is_resume = stubp->ist_resume; source->is_recurse = stubp->ist_recurse; - idt_vec_set(idt_vec, stubp->ist_entry); + idt_vec_set(iv, idt_vec, stubp->ist_entry); x86_write_psl(psl); @@ -1755,7 +1765,9 @@ intr_deactivate_xcall(void *arg1, void *arg2) { struct cpu_info *ci; struct intrhand *ih, *lih; + struct intrsource *isp; u_long psl; + int idt_vec; int slot; ih = arg1; @@ -1766,6 +1778,8 @@ intr_deactivate_xcall(void *arg1, void *arg2) ci = ih->ih_cpu; slot = ih->ih_slot; + isp = ci->ci_isources[slot]; + idt_vec = isp->is_idtvec; psl = x86_read_psl(); x86_disable_intr(); @@ -1778,10 +1792,14 @@ intr_deactivate_xcall(void *arg1, void *arg2) x86_intr_calculatemasks(ci); - /* - * Skip unsetgate(), because the same itd[] entry is overwritten in - * intr_activate_xcall(). - */ + if (idt_vec > 0 && idt_vec_is_pcpu()) { + idt_vec_free(&ci->ci_idtvec, idt_vec); + } else { + /* + * Skip unsetgate(), because the same idt[] entry is + * overwritten in intr_activate_xcall(). + */ + } x86_write_psl(psl); @@ -1820,7 +1838,7 @@ intr_set_affinity(struct intrsource *isp, const kcpuset_t *cpuset) struct intrhand *ih, *lih; struct pic *pic; u_int cpu_idx; - int idt_vec; + int old_idtvec, new_idtvec; int oldslot, newslot; int err; int pin; @@ -1863,7 +1881,6 @@ intr_set_affinity(struct intrsource *isp, const kcpuset_t *cpuset) return 0; oldslot = ih->ih_slot; - idt_vec = isp->is_idtvec; err = intr_find_unused_slot(newci, &newslot); if (err) { @@ -1872,6 +1889,19 @@ intr_set_affinity(struct intrsource *isp, const kcpuset_t *cpuset) return err; } + old_idtvec = isp->is_idtvec; + + if (isp->is_idtvec > 0 && idt_vec_is_pcpu()) { + new_idtvec = idt_vec_alloc(&newci->ci_idtvec, + APIC_LEVEL(ih->ih_level), IDT_INTR_HIGH); + if (new_idtvec == 0) + return EBUSY; + DPRINTF(("interrupt from cpu%d vec %d to cpu%d vec %d\n", + cpu_index(oldci), old_idtvec, cpu_index(newci), new_idtvec)); + } else { + new_idtvec = isp->is_idtvec; + } + /* Prevent intr_unmask() from reenabling the source at the hw. */ isp->is_distribute_pending = true; @@ -1893,9 +1923,10 @@ intr_set_affinity(struct intrsource *isp, const kcpuset_t *cpuset) xc_wait(where); } intr_save_evcnt(isp, oldci->ci_cpuid); - (*pic->pic_delroute)(pic, oldci, pin, idt_vec, isp->is_type); + (*pic->pic_delroute)(pic, oldci, pin, old_idtvec, isp->is_type); /* activate new interrupt setting */ + isp->is_idtvec = new_idtvec; newci->ci_isources[newslot] = isp; for (lih = ih; lih != NULL; lih = lih->ih_next) { newci->ci_nintrhand++; @@ -1912,7 +1943,7 @@ intr_set_affinity(struct intrsource *isp, const kcpuset_t *cpuset) } intr_restore_evcnt(isp, newci->ci_cpuid); isp->is_active_cpu = newci->ci_cpuid; - (*pic->pic_addroute)(pic, newci, pin, idt_vec, isp->is_type); + (*pic->pic_addroute)(pic, newci, pin, new_idtvec, isp->is_type); isp->is_distribute_pending = false; if (newci == curcpu() || !mp_online) { diff --git a/sys/arch/x86/x86/lapic.c b/sys/arch/x86/x86/lapic.c index 8a74c038987..623bde72d42 100644 --- a/sys/arch/x86/x86/lapic.c +++ b/sys/arch/x86/x86/lapic.c @@ -338,6 +338,8 @@ lapic_setup_bsp(paddr_t lapic_base) #endif #if defined(DDB) && defined(MULTIPROCESSOR) #ifdef __x86_64__ + struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); + idt_descriptor_t *idt = iv->iv_idt; set_idtgate(&idt[ddb_vec], &Xintr_x2apic_ddbipi, 1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #else @@ -475,23 +477,24 @@ lapic_set_lvt(void) void lapic_boot_init(paddr_t lapic_base) { + struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); lapic_setup_bsp(lapic_base); #ifdef MULTIPROCESSOR - idt_vec_reserve(LAPIC_IPI_VECTOR); - idt_vec_set(LAPIC_IPI_VECTOR, + idt_vec_reserve(iv, LAPIC_IPI_VECTOR); + idt_vec_set(iv, LAPIC_IPI_VECTOR, x2apic_mode ? Xintr_x2apic_ipi : Xintr_lapic_ipi); - idt_vec_reserve(LAPIC_TLB_VECTOR); - idt_vec_set(LAPIC_TLB_VECTOR, + idt_vec_reserve(iv, LAPIC_TLB_VECTOR); + idt_vec_set(iv, LAPIC_TLB_VECTOR, x2apic_mode ? Xintr_x2apic_tlb : Xintr_lapic_tlb); #endif - idt_vec_reserve(LAPIC_SPURIOUS_VECTOR); - idt_vec_set(LAPIC_SPURIOUS_VECTOR, Xintrspurious); + idt_vec_reserve(iv, LAPIC_SPURIOUS_VECTOR); + idt_vec_set(iv, LAPIC_SPURIOUS_VECTOR, Xintrspurious); - idt_vec_reserve(LAPIC_TIMER_VECTOR); - idt_vec_set(LAPIC_TIMER_VECTOR, + idt_vec_reserve(iv, LAPIC_TIMER_VECTOR); + idt_vec_set(iv, LAPIC_TIMER_VECTOR, x2apic_mode ? Xintr_x2apic_ltimer : Xintr_lapic_ltimer); } diff --git a/sys/arch/x86/x86/pmap.c b/sys/arch/x86/x86/pmap.c index 65caafa86f3..610ea926a64 100644 --- a/sys/arch/x86/x86/pmap.c +++ b/sys/arch/x86/x86/pmap.c @@ -1309,11 +1309,7 @@ pmap_bootstrap(vaddr_t kva_start) /* * Allocate space for the IDT, GDT and LDT. */ -#ifdef __HAVE_PCPU_AREA - idt_vaddr = (vaddr_t)&pcpuarea->idt; -#else idt_vaddr = pmap_bootstrap_valloc(1); -#endif idt_paddr = pmap_bootstrap_palloc(1); gdt_vaddr = pmap_bootstrap_valloc(1); diff --git a/sys/arch/x86/x86/svs.c b/sys/arch/x86/x86/svs.c index e0f5e4ac15e..5ca3db3e8a1 100644 --- a/sys/arch/x86/x86/svs.c +++ b/sys/arch/x86/x86/svs.c @@ -519,6 +519,7 @@ cpu_svs_init(struct cpu_info *ci) { extern char __text_user_start; extern char __text_user_end; + extern vaddr_t idt_vaddr; const cpuid_t cid = cpu_index(ci); struct vm_page *pg; @@ -543,7 +544,8 @@ cpu_svs_init(struct cpu_info *ci) mutex_init(&ci->ci_svs_mtx, MUTEX_DEFAULT, IPL_VM); - svs_page_add(ci, (vaddr_t)&pcpuarea->idt, true); + if (cid == cpu_index(&cpu_info_primary) || !idt_vec_is_pcpu()) + svs_page_add(ci, idt_vaddr, true); svs_page_add(ci, (vaddr_t)&pcpuarea->ldt, true); svs_range_add(ci, (vaddr_t)&pcpuarea->ent[cid], offsetof(struct pcpu_entry, rsp0), true); diff --git a/sys/arch/xen/x86/cpu.c b/sys/arch/xen/x86/cpu.c index 7ddb53d74e7..38013a4d225 100644 --- a/sys/arch/xen/x86/cpu.c +++ b/sys/arch/xen/x86/cpu.c @@ -475,6 +475,9 @@ cpu_attach_common(device_t parent, device_t self, void *aux) /* interrupt handler stack */ cpu_intr_init(ci); + /* Setup per-cpu memory for idt */ + idt_vec_init_cpu_md(&ci->ci_idtvec, cpu_index(ci)); + /* Setup per-cpu memory for gdt */ gdt_alloc_cpu(ci); @@ -698,7 +701,7 @@ cpu_hatch(void *v) /* Setup TLS and kernel GS/FS */ cpu_init_msrs(ci, true); - cpu_init_idt(); + cpu_init_idt(ci); gdt_init_cpu(ci); cpu_probe(ci); diff --git a/sys/arch/xen/xen/hypervisor.c b/sys/arch/xen/xen/hypervisor.c index c3241238f03..95b846e4142 100644 --- a/sys/arch/xen/xen/hypervisor.c +++ b/sys/arch/xen/xen/hypervisor.c @@ -295,6 +295,8 @@ xen_check_hypervisordev(void) static int xen_hvm_init_late(void) { + struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); + if (HYPERVISOR_xen_version(XENVER_version, NULL) < 0) { aprint_error("Xen HVM: hypercall page not working\n"); return 0; @@ -350,8 +352,8 @@ xen_hvm_init_late(void) * prepare vector. * We don't really care where it is, as long as it's free */ - xen_hvm_vec = idt_vec_alloc(129, 255); - idt_vec_set(xen_hvm_vec, &IDTVEC(hypervisor_pvhvm_callback)); + xen_hvm_vec = idt_vec_alloc(iv, 129, 255); + idt_vec_set(iv, xen_hvm_vec, &IDTVEC(hypervisor_pvhvm_callback)); events_default_setup(); return 1; diff --git a/sys/dev/hyperv/vmbusvar.h b/sys/dev/hyperv/vmbusvar.h index c023551a0ab..7c62ee1ba7e 100644 --- a/sys/dev/hyperv/vmbusvar.h +++ b/sys/dev/hyperv/vmbusvar.h @@ -159,6 +159,8 @@ struct vmbus_percpu_data { /* Rarely used fields */ struct hyperv_dma simp_dma; struct hyperv_dma siep_dma; + + void *md_cookie; } __aligned(CACHE_LINE_SIZE); struct vmbus_softc { diff --git a/sys/dev/nvmm/x86/nvmm_x86_vmx.c b/sys/dev/nvmm/x86/nvmm_x86_vmx.c index 755a04310d1..3281f2e2e15 100644 --- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c +++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c @@ -2698,7 +2698,8 @@ vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) struct vmcs *vmcs = cpudata->vmcs; struct msr_entry *gmsr = cpudata->gmsr; extern uint8_t vmx_resume_rip; - uint64_t rev, eptp; + uint64_t rev, eptp, idt; + struct cpu_info *ci; rev = vmx_get_revision(); @@ -2765,6 +2766,9 @@ vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE); /* Set the Host state for resuming. */ + ci = curcpu(); + idt = (uint64_t)ci->ci_idtvec.iv_idt; + vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip); vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL)); vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL)); @@ -2775,7 +2779,7 @@ vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0); vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0); vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0); - vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt); + vmx_vmwrite(VMCS_HOST_IDTR_BASE, idt); vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT)); vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER)); vmx_vmwrite(VMCS_HOST_CR0, rcr0() & ~CR0_TS);