Index: x86/include/cpu.h =================================================================== RCS file: /cvsroot/src/sys/arch/x86/include/cpu.h,v retrieving revision 1.100 diff -u -p -r1.100 cpu.h --- x86/include/cpu.h 18 Nov 2018 23:50:48 -0000 1.100 +++ x86/include/cpu.h 14 Dec 2018 19:05:55 -0000 @@ -142,17 +142,18 @@ struct cpu_info { volatile int ci_mtx_oldspl; /* Old SPL at this ci_idepth */ /* The following must be aligned for cmpxchg8b. */ - struct { - uint32_t ipending; - int ilevel; - } ci_istate __aligned(8); -#define ci_ipending ci_istate.ipending -#define ci_ilevel ci_istate.ilevel + union { + uint64_t ci_istate; + struct { + uint64_t ci_ilevel:8; + uint64_t ci_ipending:56; + }; + } __aligned(8); int ci_idepth; void * ci_intrstack; - uint32_t ci_imask[NIPL]; - uint32_t ci_iunmask[NIPL]; + uint64_t ci_imask[NIPL]; + uint64_t ci_iunmask[NIPL]; uint32_t ci_flags; /* flags; see below */ uint32_t ci_ipis; /* interprocessor interrupts pending */ Index: x86/include/intrdefs.h =================================================================== RCS file: /cvsroot/src/sys/arch/x86/include/intrdefs.h,v retrieving revision 1.20 diff -u -p -r1.20 intrdefs.h --- x86/include/intrdefs.h 19 May 2014 22:47:54 -0000 1.20 +++ x86/include/intrdefs.h 14 Dec 2018 19:05:55 -0000 @@ -41,12 +41,12 @@ #define SIR_PREEMPT 25 /* - * Maximum # of interrupt sources per CPU. 32 to fit in one word. + * Maximum # of interrupt sources per CPU. Bitmask must still fit in one quad. * ioapics can theoretically produce more, but it's not likely to * happen. For multiple ioapics, things can be routed to different * CPUs. */ -#define MAX_INTR_SOURCES 32 +#define MAX_INTR_SOURCES 56 #define NUM_LEGACY_IRQS 16 /* Index: x86/x86/intr.c =================================================================== RCS file: /cvsroot/src/sys/arch/x86/x86/intr.c,v retrieving revision 1.137 diff -u -p -r1.137 intr.c --- x86/x86/intr.c 4 Dec 2018 19:27:22 -0000 1.137 +++ x86/x86/intr.c 14 Dec 2018 19:05:56 -0000 @@ -149,7 +149,6 @@ __KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.1 #include #include #include -#include #include #include #include /* for AB_VERBOSE */ @@ -321,7 +320,8 @@ x86_nmi(void) static void intr_calculatemasks(struct cpu_info *ci) { - int irq, level, unusedirqs, intrlevel[MAX_INTR_SOURCES]; + uint64_t irq, unusedirqs; + int level, intrlevel[MAX_INTR_SOURCES]; struct intrhand *q; /* First, figure out which levels each IRQ uses. */ @@ -342,10 +342,10 @@ intr_calculatemasks(struct cpu_info *ci) /* Then figure out which IRQs use each level. */ for (level = 0; level < NIPL; level++) { - int irqs = 0; + uint64_t irqs = 0; for (irq = 0; irq < MAX_INTR_SOURCES; irq++) if (intrlevel[irq] & (1U << level)) - irqs |= 1U << irq; + irqs |= 1ULL << irq; ci->ci_imask[level] = irqs | unusedirqs; } @@ -1153,7 +1153,9 @@ intr_disestablish_xcall(void *arg1, void idtvec = source->is_idtvec; (*pic->pic_hwmask)(pic, ih->ih_pin); - atomic_and_32(&ci->ci_ipending, ~(1 << ih->ih_slot)); + membar_sync(); + ci->ci_ipending &= ~(1ULL << ih->ih_slot); + membar_sync(); /* * Remove the handler from the chain. @@ -2127,8 +2129,11 @@ intr_set_affinity(struct intrsource *isp pin = isp->is_pin; (*pic->pic_hwmask)(pic, pin); /* for ci_ipending check */ - while(oldci->ci_ipending & (1 << oldslot)) + membar_sync(); + while (oldci->ci_ipending & (1ULL << oldslot)) { (void)kpause("intrdist", false, 1, &cpu_lock); + membar_sync(); + } kpreempt_disable(); Index: x86/x86/lapic.c =================================================================== RCS file: /cvsroot/src/sys/arch/x86/x86/lapic.c,v retrieving revision 1.67 diff -u -p -r1.67 lapic.c --- x86/x86/lapic.c 23 Sep 2018 00:59:59 -0000 1.67 +++ x86/x86/lapic.c 14 Dec 2018 19:05:56 -0000 @@ -535,7 +535,7 @@ lapic_get_timecount(struct timecounter * if (lapic_readreg(reg) & (1 << (LAPIC_TIMER_VECTOR % 32))) { cur_timer -= lapic_tval; } - } else if (ci->ci_istate.ipending & (1 << LIR_TIMER)) + } else if (ci->ci_ipending & (1 << LIR_TIMER)) cur_timer = lapic_gettick() - lapic_tval; cur_timer = ci->ci_lapic_counter - cur_timer; splx(s); Index: amd64/amd64/amd64_trap.S =================================================================== RCS file: /cvsroot/src/sys/arch/amd64/amd64/amd64_trap.S,v retrieving revision 1.45 diff -u -p -r1.45 amd64_trap.S --- amd64/amd64/amd64_trap.S 12 Aug 2018 06:11:47 -0000 1.45 +++ amd64/amd64/amd64_trap.S 14 Dec 2018 19:05:56 -0000 @@ -334,7 +334,7 @@ IDTVEC(trap07) ZTRAP_NJ(T_DNA) INTRENTRY #ifdef DIAGNOSTIC - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl #endif movq %rsp,%rdi call _C_LABEL(fpudna) @@ -422,7 +422,7 @@ IDTVEC(trap15) ZTRAP_NJ(T_ASTFLT) INTRENTRY #ifdef DIAGNOSTIC - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl #endif jmp .Lalltraps_checkusr IDTVEC_END(trap15) @@ -432,7 +432,7 @@ IDTVEC(trap16) .Ldo_fputrap: INTRENTRY #ifdef DIAGNOSTIC - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl #endif movq %rsp,%rdi call _C_LABEL(fputrap) @@ -483,7 +483,7 @@ IDTVEC(intrspurious) ZTRAP_NJ(T_ASTFLT) INTRENTRY #ifdef DIAGNOSTIC - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl #endif jmp .Lalltraps_checkusr IDTVEC_END(intrspurious) @@ -633,7 +633,7 @@ ENTRY(alltraps) calltrap: #ifdef DIAGNOSTIC - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl #endif movq %rsp,%rdi incq CPUVAR(NTRAP) @@ -661,7 +661,7 @@ calltrap: 6: #ifdef DIAGNOSTIC - cmpl CPUVAR(ILEVEL),%ebx + cmpb CPUVAR(ISTATE),%bl jne .Lspl_error #endif INTRFASTEXIT @@ -674,7 +674,7 @@ calltrap: .Lspl_error: STI(si) movabsq $4f,%rdi - movl CPUVAR(ILEVEL),%esi + movb CPUVAR(ISTATE),%sil /* -> %esi */ movl %ebx,%edx xorq %rax,%rax call _C_LABEL(printf) Index: amd64/amd64/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/amd64/amd64/genassym.cf,v retrieving revision 1.70 diff -u -p -r1.70 genassym.cf --- amd64/amd64/genassym.cf 12 Aug 2018 15:31:01 -0000 1.70 +++ amd64/amd64/genassym.cf 14 Dec 2018 19:05:56 -0000 @@ -248,16 +248,14 @@ define CPU_INFO_CURPRIORITY offsetof(str define CPU_INFO_FPCURLWP offsetof(struct cpu_info, ci_fpcurlwp) define CPU_INFO_GDT offsetof(struct cpu_info, ci_gdt) -define CPU_INFO_IPENDING offsetof(struct cpu_info, ci_ipending) +define CPU_INFO_ISTATE offsetof(struct cpu_info, ci_istate) define CPU_INFO_IMASK offsetof(struct cpu_info, ci_imask) define CPU_INFO_IUNMASK offsetof(struct cpu_info, ci_iunmask) -define CPU_INFO_ILEVEL offsetof(struct cpu_info, ci_ilevel) define CPU_INFO_IDEPTH offsetof(struct cpu_info, ci_idepth) define CPU_INFO_ISOURCES offsetof(struct cpu_info, ci_isources) define CPU_INFO_MTX_COUNT offsetof(struct cpu_info, ci_mtx_count) define CPU_INFO_MTX_OLDSPL offsetof(struct cpu_info, ci_mtx_oldspl) define CPU_INFO_CPUID offsetof(struct cpu_info, ci_cpuid) -define CPU_INFO_ISTATE offsetof(struct cpu_info, ci_istate) define CPU_INFO_CC_SKEW offsetof(struct cpu_info, ci_data.cpu_cc_skew) define ACPI_SUSPEND_GDT offsetof(struct cpu_info, ci_suspend_gdt) Index: amd64/amd64/locore.S =================================================================== RCS file: /cvsroot/src/sys/arch/amd64/amd64/locore.S,v retrieving revision 1.174 diff -u -p -r1.174 locore.S --- amd64/amd64/locore.S 12 Aug 2018 15:31:01 -0000 1.174 +++ amd64/amd64/locore.S 14 Dec 2018 19:05:56 -0000 @@ -1174,7 +1174,7 @@ ENTRY(cpu_switchto) * IPL_SCHED prevents from FPU interrupt altering the LWP's saved cr0. */ #ifndef XEN - movl $IPL_HIGH,CPUVAR(ILEVEL) + movb $IPL_HIGH,CPUVAR(ISTATE) movl PCB_CR0(%r14),%ecx /* has CR0_TS clear */ movq %cr0,%rdx @@ -1300,7 +1300,7 @@ ENTRY(handle_syscall) jnz 9f #ifdef DIAGNOSTIC - cmpl $IPL_NONE,CPUVAR(ILEVEL) + cmpb $IPL_NONE,CPUVAR(ISTATE) jne .Lspl_error #endif @@ -1324,7 +1324,7 @@ ENTRY(handle_syscall) movl TF_RAX(%rsp),%esi movl TF_RDI(%rsp),%edx movl %ebx,%ecx - movl CPUVAR(ILEVEL),%r8d + movb CPUVAR(ISTATE),%r8b xorq %rax,%rax call _C_LABEL(printf) movl $IPL_NONE,%edi Index: amd64/amd64/spl.S =================================================================== RCS file: /cvsroot/src/sys/arch/amd64/amd64/spl.S,v retrieving revision 1.36 diff -u -p -r1.36 spl.S --- amd64/amd64/spl.S 22 Aug 2018 17:04:36 -0000 1.36 +++ amd64/amd64/spl.S 14 Dec 2018 19:05:56 -0000 @@ -99,7 +99,7 @@ IDTVEC(softintr) pushq %r14 pushq %r15 - movl $IPL_HIGH,CPUVAR(ILEVEL) + movb $IPL_HIGH,CPUVAR(ISTATE) movq CPUVAR(CURLWP),%r15 movq IS_LWP(%rax),%rdi /* switch to handler LWP */ movq L_PCB(%rdi),%rdx @@ -162,7 +162,7 @@ END(softintr_ret) * Software interrupt registration. */ ENTRY(softint_trigger) - orl %edi,CPUVAR(IPENDING) /* atomic on local cpu */ + orq %rdi,CPUVAR(ISTATE) /* atomic on local cpu */ ret END(softint_trigger) @@ -173,7 +173,7 @@ END(softint_trigger) * Handles preemption interrupts via Xspllower(). */ IDTVEC(recurse_preempt) - movl $IPL_PREEMPT,CPUVAR(ILEVEL) + movb $IPL_PREEMPT,CPUVAR(ISTATE) sti xorq %rdi,%rdi call _C_LABEL(kpreempt) @@ -187,7 +187,7 @@ IDTVEC_END(recurse_preempt) * Handles preemption interrupts via Xdoreti(). */ IDTVEC(resume_preempt) - movl $IPL_PREEMPT,CPUVAR(ILEVEL) + movb $IPL_PREEMPT,CPUVAR(ISTATE) sti testq $SEL_RPL,TF_CS(%rsp) jnz 1f @@ -205,10 +205,10 @@ IDTVEC_END(resume_preempt) * int splraise(int s); */ ENTRY(splraise) - movl CPUVAR(ILEVEL),%eax + movb CPUVAR(ISTATE),%al cmpl %edi,%eax cmoval %eax,%edi - movl %edi,CPUVAR(ILEVEL) + movb %sil,CPUVAR(ISTATE) ret END(splraise) @@ -220,14 +220,14 @@ END(splraise) * are disabled via eflags/IE. */ ENTRY(spllower) - cmpl CPUVAR(ILEVEL),%edi + cmpb CPUVAR(ISTATE),%sil jae 1f - movl CPUVAR(IUNMASK)(,%rdi,4),%edx + movq CPUVAR(IUNMASK)(,%rdi,8),%rdx pushf cli - testl CPUVAR(IPENDING),%edx + testl CPUVAR(ISTATE),%edx # XXX jnz 2f - movl %edi,CPUVAR(ILEVEL) + movl %edi,CPUVAR(ISTATE) popf 1: ret @@ -247,24 +247,32 @@ LABEL(spllower_end) * * For cmpxchg8b, edx/ecx are the high words and eax/ebx the low. * - * edx : eax = old level / old ipending - * ecx : ebx = new level / old ipending + * edi = s + * edx : eax = old level + high 24 bit old ipending / low 32 bit old ipending + * ecx : ebx = new level + high 24 bit old ipending / low 32 bit old ipending */ ENTRY(cx8_spllower) - movl CPUVAR(ILEVEL),%edx - movq %rbx,%r8 + xorq %r9,%r9 # prepare ipending mask + notq %r9 + shrq $8,%r9 + + movq %rbx,%r8 # save %rbx + + movb CPUVAR(ISTATE),%dl /* %edx now old level */ cmpl %edx,%edi /* new level is lower? */ jae 1f 0: - movl CPUVAR(IPENDING),%eax - movl %edi,%ecx - testl %eax,CPUVAR(IUNMASK)(,%rcx,4)/* deferred interrupts? */ - movl %eax,%ebx - /* - * On the P4 this jump is cheaper than patching in junk - * using cmov. Is cmpxchg expensive if it fails? - */ + movq CPUVAR(ISTATE),%rax + andq %r9,%rax /* %rax now old ipending */ + testq %rax,CPUVAR(IUNMASK)(,%edi,8) /* deferred interrupts? */ jnz 2f + + movq %rax,%r11 + shrq $32,%r11 /* %r11 high dword of %rax */ + movl %eax,%ebx /* %ebx low dword of %rax */ + movl %edi,%ecx /* %ecx now new level */ + shll $24,%ecx + orq %r11,%rcx # %ecx now ilevel << 24 | %r11 cmpxchg8b CPUVAR(ISTATE) /* swap in new ilevel */ jnz 0b 1: @@ -308,16 +316,17 @@ IDTVEC(spllower) movl %edi,%ebx leaq 1f(%rip),%r13 /* address to resume loop at */ 1: movl %ebx,%eax /* get cpl */ - movl CPUVAR(IUNMASK)(,%rax,4),%eax + movq CPUVAR(IUNMASK)(,%rax,8),%rax CLI(si) - andl CPUVAR(IPENDING),%eax /* any non-masked bits left? */ + andq CPUVAR(ISTATE),%rax /* any non-masked bits left? */ jz 2f - bsrl %eax,%eax - btrl %eax,CPUVAR(IPENDING) + + bsrq %rax,%rax /* %rax now highest bit set */ + btrq %rax,CPUVAR(ISTATE) movq CPUVAR(ISOURCES)(,%rax,8),%rax jmp *IS_RECURSE(%rax) 2: - movl %ebx,CPUVAR(ILEVEL) + movb %bl,CPUVAR(ISTATE) STI(si) popq %r12 popq %r13 @@ -339,16 +348,16 @@ IDTVEC(doreti) decl CPUVAR(IDEPTH) leaq 1f(%rip),%r13 1: movl %ebx,%eax - movl CPUVAR(IUNMASK)(,%rax,4),%eax + movq CPUVAR(IUNMASK)(,%rax,8),%rax CLI(si) - andl CPUVAR(IPENDING),%eax + andq CPUVAR(ISTATE),%rax jz 2f - bsrl %eax,%eax /* slow, but not worth optimizing */ - btrl %eax,CPUVAR(IPENDING) + bsrq %rax,%rax /* slow, but not worth optimizing */ + btrq %rax,CPUVAR(ISTATE) movq CPUVAR(ISOURCES)(,%rax,8),%rax jmp *IS_RESUME(%rax) 2: /* Check for ASTs on exit to user mode. */ - movl %ebx,CPUVAR(ILEVEL) + mov %bx,CPUVAR(ISTATE) 5: testb $SEL_RPL,TF_CS(%rsp) jz 6f Index: amd64/amd64/vector.S =================================================================== RCS file: /cvsroot/src/sys/arch/amd64/amd64/vector.S,v retrieving revision 1.64 diff -u -p -r1.64 vector.S --- amd64/amd64/vector.S 14 Jul 2018 14:29:40 -0000 1.64 +++ amd64/amd64/vector.S 14 Dec 2018 19:05:56 -0000 @@ -119,7 +119,7 @@ IDTVEC(handle_x2apic_ipi) xorl %eax,%eax xorl %edx,%edx wrmsr - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl cmpl $IPL_HIGH,%ebx jae 2f jmp 1f @@ -127,7 +127,7 @@ IDTVEC_END(handle_x2apic_ipi) IDTVEC(handle_lapic_ipi) movq _C_LABEL(local_apic_va),%rbx movl $0,LAPIC_EOI(%rbx) - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl cmpl $IPL_HIGH,%ebx jae 2f jmp 1f @@ -135,13 +135,13 @@ IDTVEC_END(handle_lapic_ipi) IDTVEC(resume_lapic_ipi) 1: incl CPUVAR(IDEPTH) - movl $IPL_HIGH,CPUVAR(ILEVEL) + movb $IPL_HIGH,CPUVAR(ISTATE) sti pushq %rbx call _C_LABEL(x86_ipi_handler) jmp _C_LABEL(Xdoreti) 2: - orl $(1 << LIR_IPI),CPUVAR(IPENDING) + btsq $(LIR_IPI),CPUVAR(ISTATE) INTRFASTEXIT IDTVEC_END(resume_lapic_ipi) @@ -219,7 +219,7 @@ IDTVEC(handle_x2apic_ltimer) xorl %eax,%eax xorl %edx,%edx wrmsr - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl cmpl $IPL_CLOCK,%ebx jae 2f jmp 1f @@ -227,7 +227,7 @@ IDTVEC_END(handle_x2apic_ltimer) IDTVEC(handle_lapic_ltimer) movq _C_LABEL(local_apic_va),%rbx movl $0,LAPIC_EOI(%rbx) - movl CPUVAR(ILEVEL),%ebx + movb CPUVAR(ISTATE),%bl cmpl $IPL_CLOCK,%ebx jae 2f jmp 1f @@ -235,7 +235,7 @@ IDTVEC_END(handle_lapic_ltimer) IDTVEC(resume_lapic_ltimer) 1: incl CPUVAR(IDEPTH) - movl $IPL_CLOCK,CPUVAR(ILEVEL) + movb $IPL_CLOCK,CPUVAR(ISTATE) sti pushq %rbx movq %rsp,%rsi @@ -243,7 +243,7 @@ IDTVEC(resume_lapic_ltimer) call _C_LABEL(lapic_clockintr) jmp _C_LABEL(Xdoreti) 2: - orl $(1 << LIR_TIMER),CPUVAR(IPENDING) + btsq $(LIR_TIMER),CPUVAR(ISTATE) INTRFASTEXIT IDTVEC_END(resume_lapic_ltimer) @@ -331,14 +331,14 @@ IDTVEC(handle_ ## name ## num) ;\ testq %r14,%r14 ;\ jz 9f /* stray */ ;\ movl IS_MAXLEVEL(%r14),%ebx ;\ - movl CPUVAR(ILEVEL),%r13d ;\ + movb CPUVAR(ISTATE),%r13b ;\ cmpl %ebx,%r13d ;\ jae 10f /* currently masked; hold it */ ;\ incq CPUVAR(NINTR) /* statistical info */ ;\ incq IS_EVCNT(%r14) ;\ 1: \ pushq %r13 /* save for Xdoreti */ ;\ - movl %ebx,CPUVAR(ILEVEL) ;\ + movb %bl,CPUVAR(ISTATE) ;\ sti ;\ incl CPUVAR(IDEPTH) ;\ movq IS_HANDLERS(%r14),%rbx ;\ @@ -348,7 +348,7 @@ IDTVEC(handle_ ## name ## num) ;\ jle 7f ;\ movq %rsp,%rsi ;\ movq IH_ARG(%rbx),%rdi ;\ - movl %r12d,CPUVAR(ILEVEL) ;\ + movb %r12b,CPUVAR(ISTATE) ;\ call *IH_FUN(%rbx) /* call it */ ;\ movq IH_NEXT(%rbx),%rbx /* next handler in chain */ ;\ testq %rbx,%rbx ;\ @@ -361,14 +361,14 @@ IDTVEC(handle_ ## name ## num) ;\ jmp _C_LABEL(Xdoreti) /* lower spl and do ASTs */ ;\ 7: \ cli ;\ - orl $(1 << num),CPUVAR(IPENDING) ;\ + btsq $(num),CPUVAR(ISTATE) ;\ level_mask(num) ;\ late_ack(num) ;\ sti ;\ jmp _C_LABEL(Xdoreti) /* lower spl and do ASTs */ ;\ 10: \ cli ;\ - orl $(1 << num),CPUVAR(IPENDING) ;\ + btsq $(num),CPUVAR(ISTATE) ;\ level_mask(num) ;\ late_ack(num) ;\ INTRFASTEXIT ;\ @@ -659,7 +659,7 @@ IDTVEC(resume_ ## name ## num) \ movq CPUVAR(ISOURCES) + (num) * 8,%r14 ;\ 1: \ pushq %r13 ;\ - movl $num,CPUVAR(ILEVEL) ;\ + movb $num,CPUVAR(ISTATE) ;\ STI(si) ;\ incl CPUVAR(IDEPTH) ;\ movq IS_HANDLERS(%r14),%rbx ;\