Index: arch/x86/acpi/acpi_wakeup.c =================================================================== RCS file: /cvsroot/src/sys/arch/x86/acpi/acpi_wakeup.c,v retrieving revision 1.52 diff -u -p -r1.52 acpi_wakeup.c --- arch/x86/acpi/acpi_wakeup.c 22 Feb 2020 19:49:11 -0000 1.52 +++ arch/x86/acpi/acpi_wakeup.c 19 May 2020 22:37:12 -0000 @@ -277,7 +277,7 @@ acpi_cpu_sleep(struct cpu_info *ci) #if NLAPIC > 0 lapic_enable(); lapic_set_lvt(); - lapic_initclocks(); + lapic_reset(); #endif atomic_or_32(&ci->ci_flags, CPUF_RUNNING); @@ -351,7 +351,7 @@ acpi_md_sleep(int state) #if NLAPIC > 0 lapic_enable(); lapic_set_lvt(); - lapic_initclocks(); + lapic_reset(); #endif #if NIOAPIC > 0 ioapic_reenable(); Index: arch/x86/include/i82489var.h =================================================================== RCS file: /cvsroot/src/sys/arch/x86/include/i82489var.h,v retrieving revision 1.20 diff -u -p -r1.20 i82489var.h --- arch/x86/include/i82489var.h 1 Dec 2019 08:23:09 -0000 1.20 +++ arch/x86/include/i82489var.h 19 May 2020 22:37:13 -0000 @@ -88,8 +88,8 @@ struct cpu_info; extern void lapic_boot_init(paddr_t); extern void lapic_set_lvt(void); extern void lapic_enable(void); -extern void lapic_calibrate_timer(struct cpu_info *ci); -extern void lapic_initclocks(void); +extern void lapic_calibrate_timer(bool); +extern void lapic_reset(void); extern uint32_t lapic_readreg(u_int); extern void lapic_writereg(u_int, uint32_t); Index: arch/x86/x86/cpu.c =================================================================== RCS file: /cvsroot/src/sys/arch/x86/x86/cpu.c,v retrieving revision 1.191 diff -u -p -r1.191 cpu.c --- arch/x86/x86/cpu.c 12 May 2020 06:32:05 -0000 1.191 +++ arch/x86/x86/cpu.c 19 May 2020 22:37:13 -0000 @@ -456,7 +456,7 @@ cpu_attach(device_t parent, device_t sel lapic_enable(); lapic_set_lvt(); if (!vm_guest_is_xenpvh_or_pvhvm()) - lapic_calibrate_timer(ci); + lapic_calibrate_timer(false); } #endif kcsan_cpu_init(ci); @@ -471,7 +471,6 @@ cpu_attach(device_t parent, device_t sel cpu_identify(ci); x86_errata(); x86_cpu_idle_init(); - (*x86_cpu_initclock_func)(); #ifdef XENPVHVM xen_hvm_init_cpu(ci); #endif @@ -485,7 +484,6 @@ cpu_attach(device_t parent, device_t sel #ifdef XENPVHVM xen_hvm_init_cpu(ci); #endif - (*x86_cpu_initclock_func)(); break; #ifdef MULTIPROCESSOR @@ -739,14 +737,6 @@ cpu_boot_secondary_processors(void) kcpuset_t *cpus; u_long i; -#if NHPET > 0 - /* Use HPET delay, and re-calibrate TSC on boot CPU using HPET. */ - if (hpet_delay_p() && x86_delay == i8254_delay) { - delay_func = x86_delay = hpet_delay; - cpu_get_tsc_freq(curcpu()); - } -#endif - /* Now that we know the number of CPUs, patch the text segment. */ x86_patch(false); @@ -1010,7 +1000,7 @@ cpu_hatch(void *v) #ifdef XENPVHVM xen_hvm_init_cpu(ci); #endif - (*x86_cpu_initclock_func)(); + (*x86_initclock_func)(); cpu_get_tsc_freq(ci); s = splhigh(); @@ -1346,7 +1336,7 @@ cpu_get_tsc_freq(struct cpu_info *ci) overhead = 0; for (int i = 0; i <= 8; i++) { t0 = cpu_counter(); - x86_delay(0); + delay_func(0); t1 = cpu_counter(); if (i > 0) { overhead += (t1 - t0); @@ -1356,7 +1346,7 @@ cpu_get_tsc_freq(struct cpu_info *ci) /* Now do the calibration. */ t0 = cpu_counter(); - x86_delay(100000); + delay_func(100000); t1 = cpu_counter(); freq = (t1 - t0 - overhead) * 10; } Index: arch/x86/x86/lapic.c =================================================================== RCS file: /cvsroot/src/sys/arch/x86/x86/lapic.c,v retrieving revision 1.79 diff -u -p -r1.79 lapic.c --- arch/x86/x86/lapic.c 19 May 2020 21:39:11 -0000 1.79 +++ arch/x86/x86/lapic.c 19 May 2020 22:37:13 -0000 @@ -580,11 +580,10 @@ lapic_clockintr(void *arg, struct intrfr } void -lapic_initclocks(void) +lapic_reset(void) { + /* - * Start local apic countdown timer running, in repeated mode. - * * Mask the clock interrupt and set mode, * then set divisor, * then unmask and set the vector. @@ -598,6 +597,29 @@ lapic_initclocks(void) lapic_writereg(LAPIC_EOI, 0); } +static void +lapic_initclock(void) +{ + + if (curcpu() == &cpu_info_primary) { + /* + * Recalibrate the timer using the cycle counter, now that + * the cycle counter itself has been recalibrated. + */ + lapic_calibrate_timer(true); + + /* + * Hook up time counter. This assume that all LAPICs have + * the same frequency. + */ + lapic_timecounter.tc_frequency = lapic_per_second; + tc_init(&lapic_timecounter); + } + + /* Start local apic countdown timer running, in repeated mode. */ + lapic_reset(); +} + /* * Calibrate the local apic count-down timer (which is running at * bus-clock speed) vs. the i8254 counter/timer (which is running at @@ -610,50 +632,71 @@ lapic_initclocks(void) * We're actually using the IRQ0 timer. Hmm. */ void -lapic_calibrate_timer(struct cpu_info *ci) +lapic_calibrate_timer(bool secondpass) { - unsigned int seen, delta, initial_i8254, initial_lapic; - unsigned int cur_i8254, cur_lapic; + struct cpu_info *ci = curcpu(); uint64_t tmp; int i; char tbuf[9]; - if (lapic_per_second != 0) - goto calibrate_done; + KASSERT(ci == &cpu_info_primary); - aprint_debug_dev(ci->ci_dev, "calibrating local timer\n"); + aprint_debug_dev(ci->ci_dev, "[re]calibrating local timer\n"); /* * Configure timer to one-shot, interrupt masked, * large positive number. */ + x86_disable_intr(); lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_MASKED); lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); lapic_writereg(LAPIC_ICR_TIMER, 0x80000000); + (void)lapic_gettick(); - x86_disable_intr(); + if (secondpass && cpu_hascounter()) { + /* + * Second pass calibration, using the TSC which has ideally + * been calibrated using the HPET or information gleaned + * from MSRs by this point. + */ + uint64_t l0, l1, t0, t1; - initial_lapic = lapic_gettick(); - initial_i8254 = gettick(); + (void)cpu_counter(); + t0 = cpu_counter(); + l0 = lapic_gettick(); + t0 += cpu_counter(); + DELAY(50000); + t1 = cpu_counter(); + l1 = lapic_gettick(); + t1 += cpu_counter(); + + tmp = (l0 - l1) * cpu_frequency(ci) / ((t1 - t0 + 1) / 2); + lapic_per_second = rounddown(tmp + 500, 1000); + } else if (lapic_per_second == 0) { + /* + * Inaccurate first pass calibration using the i8254. + */ + unsigned int seen, delta, initial_i8254, initial_lapic; + unsigned int cur_i8254, cur_lapic; - for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) { - cur_i8254 = gettick(); - if (cur_i8254 > initial_i8254) - delta = x86_rtclock_tval - (cur_i8254 - initial_i8254); - else - delta = initial_i8254 - cur_i8254; - initial_i8254 = cur_i8254; + (void)gettick(); + initial_lapic = lapic_gettick(); + initial_i8254 = gettick(); + for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) { + cur_i8254 = gettick(); + if (cur_i8254 > initial_i8254) + delta = x86_rtclock_tval - (cur_i8254 - initial_i8254); + else + delta = initial_i8254 - cur_i8254; + initial_i8254 = cur_i8254; + } + cur_lapic = lapic_gettick(); + tmp = initial_lapic - cur_lapic; + lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen; } - cur_lapic = lapic_gettick(); - x86_enable_intr(); - tmp = initial_lapic - cur_lapic; - lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen; - -calibrate_done: humanize_number(tbuf, sizeof(tbuf), lapic_per_second, "Hz", 1000); - aprint_debug_dev(ci->ci_dev, "apic clock running at %s\n", tbuf); if (lapic_per_second != 0) { @@ -703,18 +746,10 @@ calibrate_done: * Now that the timer's calibrated, use the apic timer routines * for all our timing needs.. */ - delay_func = lapic_delay; - x86_cpu_initclock_func = lapic_initclocks; - x86_initclock_func = x86_dummy_initclock; - initrtclock(0); - - if (lapic_timecounter.tc_frequency == 0) { - /* - * Hook up time counter. - * This assume that all LAPICs have the same frequency. - */ - lapic_timecounter.tc_frequency = lapic_per_second; - tc_init(&lapic_timecounter); + if (!secondpass) { + delay_func = lapic_delay; + x86_initclock_func = lapic_initclock; + initrtclock(0); } } } Index: arch/x86/x86/x86_machdep.c =================================================================== RCS file: /cvsroot/src/sys/arch/x86/x86/x86_machdep.c,v retrieving revision 1.142 diff -u -p -r1.142 x86_machdep.c --- arch/x86/x86/x86_machdep.c 3 May 2020 17:22:03 -0000 1.142 +++ arch/x86/x86/x86_machdep.c 19 May 2020 22:37:13 -0000 @@ -108,11 +108,9 @@ char module_machine_i386pae_xen[] = "i38 #ifndef XENPV void (*delay_func)(unsigned int) = i8254_delay; void (*x86_initclock_func)(void) = i8254_initclocks; -void (*x86_cpu_initclock_func)(void) = x86_dummy_initclock; #else /* XENPV */ void (*delay_func)(unsigned int) = xen_delay; void (*x86_initclock_func)(void) = xen_initclocks; -void (*x86_cpu_initclock_func)(void) = xen_cpu_initclocks; #endif @@ -1500,10 +1498,12 @@ void cpu_initclocks(void) { - (*x86_initclock_func)(); -} + /* + * Re-calibrate TSC on boot CPU using most accurate time source, + * thus making accurate TSC available for x86_initclock_func(). + */ + cpu_get_tsc_freq(curcpu()); -void -x86_dummy_initclock(void) -{ + /* Now start the clocks on this CPU (the boot CPU). */ + (*x86_initclock_func)(); } Index: arch/xen/x86/cpu.c =================================================================== RCS file: /cvsroot/src/sys/arch/xen/x86/cpu.c,v retrieving revision 1.135 diff -u -p -r1.135 cpu.c --- arch/xen/x86/cpu.c 25 Apr 2020 15:26:17 -0000 1.135 +++ arch/xen/x86/cpu.c 19 May 2020 22:37:13 -0000 @@ -456,14 +456,12 @@ cpu_attach_common(device_t parent, devic atomic_or_32(&ci->ci_flags, CPUF_SP); cpu_identify(ci); x86_cpu_idle_init(); - xen_cpu_initclocks(); break; case CPU_ROLE_BP: atomic_or_32(&ci->ci_flags, CPUF_BSP); cpu_identify(ci); x86_cpu_idle_init(); - xen_cpu_initclocks(); break; case CPU_ROLE_AP: @@ -726,7 +724,7 @@ cpu_hatch(void *v) xen_ipi_init(); - xen_cpu_initclocks(); + xen_initclocks(); #ifdef __x86_64__ fpuinit(ci); Index: arch/xen/xen/hypervisor.c =================================================================== RCS file: /cvsroot/src/sys/arch/xen/xen/hypervisor.c,v retrieving revision 1.84 diff -u -p -r1.84 hypervisor.c --- arch/xen/xen/hypervisor.c 9 May 2020 08:01:38 -0000 1.84 +++ arch/xen/xen/hypervisor.c 19 May 2020 22:37:13 -0000 @@ -260,7 +260,6 @@ init_xen_early(void) } delay_func = x86_delay = xen_delay; x86_initclock_func = xen_initclocks; - x86_cpu_initclock_func = xen_cpu_initclocks; if (hvm_start_info->cmdline_paddr != 0) { cmd_line = (void *)((uintptr_t)hvm_start_info->cmdline_paddr + KERNBASE); @@ -436,7 +435,6 @@ xen_hvm_init(void) delay_func = x86_delay = xen_delay; x86_initclock_func = xen_initclocks; - x86_cpu_initclock_func = xen_cpu_initclocks; vm_guest = VM_GUEST_XENPVHVM; /* Be more specific */ return 1; Index: arch/xen/xen/xen_clock.c =================================================================== RCS file: /cvsroot/src/sys/arch/xen/xen/xen_clock.c,v retrieving revision 1.5 diff -u -p -r1.5 xen_clock.c --- arch/xen/xen/xen_clock.c 7 May 2020 19:48:57 -0000 1.5 +++ arch/xen/xen/xen_clock.c 19 May 2020 22:37:13 -0000 @@ -729,12 +729,12 @@ again: } /* - * xen_cpu_initclocks() + * xen_initclocks() * * Initialize the Xen clocks on the current CPU. */ void -xen_cpu_initclocks(void) +xen_initclocks(void) { struct cpu_info *ci = curcpu(); @@ -766,22 +766,13 @@ xen_cpu_initclocks(void) /* Fire up the clocks. */ xen_resumeclocks(ci); -} -/* - * xen_initclocks() - * - * Initialize the Xen global clock - */ -void -xen_initclocks(void) -{ #ifdef DOM0OPS /* * If this is a privileged dom0, start pushing the wall * clock time back to the Xen hypervisor. */ - if (xendomain_is_privileged()) + if (ci == &cpu_info_primary && xendomain_is_privileged()) xen_timepush_init(); #endif }