# HG changeset patch # User Taylor R Campbell # Date 1747618103 0 # Mon May 19 01:28:23 2025 +0000 # Branch trunk # Node ID a09802de9029d936f992a7ef52b44c22896ca0a3 # Parent e747126f13a71d2047753b266ff1c59dc9a800b7 # EXP-Topic riastradh-highrestimer WIP: draft high-resolution timer interrupt infrastructure with lapic diff -r e747126f13a7 -r a09802de9029 sys/arch/x86/x86/lapic.c --- a/sys/arch/x86/x86/lapic.c Sat May 17 20:59:27 2025 +0000 +++ b/sys/arch/x86/x86/lapic.c Mon May 19 01:28:23 2025 +0000 @@ -94,6 +94,283 @@ extern int ddb_vec; #include /* for vmt_hvcall() */ #include /* for vmt_hvcall() */ +static void lapic_schedule(const struct timespec *, const struct timespec *, + const struct timespec *); + +/* + * XXX BEGIN SECTION THAT SHOULD BE MOVED TO MI kern/ XXX + */ + +/* XXX replace me by callout(9) adapted with high-resolution prioq */ + +struct clockevent { /* XXX opaque */ + struct timespec clkev_deadline; /* XXX */ + uint64_t clkev_ticket; /* XXX */ + void (*clkev_func)(struct clockevent *, + struct clockframe *); + int clkev_flags; + struct rb_node clkev_rb_node; + struct cpu_info *clkev_owner; +}; + +#define CLOCKEVENT_BOUND __BIT(0) + +void clockevent_init(struct clockevent *, + void (*)(struct clockevent *, struct clockframe *), + int); +void clockevent_schedule(struct clockevent *, const struct timespec *); +void clockevent_cancel(struct clockevent *); +void clockevent_destroy(struct clockevent *); + +#include +#include /* XXX move to struct cpu_data */ + +struct clockevent_cpu { + struct rb_tree cec_queue; + uint64_t cec_ticket; + struct timespec cec_next; + struct clockevent cec_hardclock; + struct evcnt cec_early; +}; + +static int +clockevent_compare_nodes(void *cookie, const void *va, const void *vb) +{ + const struct clockevent *a = va, *b = vb; + + if (timespeccmp(&a->clkev_deadline, &b->clkev_deadline, <)) + return -1; + if (timespeccmp(&a->clkev_deadline, &b->clkev_deadline, >)) + return +1; + if (a->clkev_ticket < b->clkev_ticket) + return -1; + if (a->clkev_ticket > b->clkev_ticket) + return +1; + return 0; +} + +static struct percpu *clockevent_percpu; /* struct clockevent_cpu * */ +static const rb_tree_ops_t clockevent_rb_ops = { + .rbto_compare_nodes = clockevent_compare_nodes, + .rbto_node_offset = offsetof(struct clockevent, clkev_rb_node), +}; + +static void +clockevents_init_cpu(void *ptr, void *cookie, struct cpu_info *ci) +{ + struct clockevent_cpu **cecp = ptr, *cec; + + cec = kmem_zalloc(sizeof(*cec), KM_SLEEP); + rb_tree_init(&cec->cec_queue, &clockevent_rb_ops); + evcnt_attach_dynamic(&cec->cec_early, EVCNT_TYPE_INTR, NULL, + device_xname(ci->ci_dev), "timearly"); + *cecp = cec; +} + +static void +clockevents_fini_cpu(void *ptr, void *cookie, struct cpu_info *ci) +{ + struct clockevent_cpu **cecp = ptr, *cec = *cecp; + + KASSERT(RB_TREE_MIN(&cec->cec_queue) == NULL); +} + +static unsigned nsec_per_tick; /* XXX */ + +static void +clockevents_init(void) +{ + + nsec_per_tick = 1000000000/hz; + clockevent_percpu = percpu_create(sizeof(struct clockevent_cpu), + clockevents_init_cpu, clockevents_fini_cpu, NULL); +} + +void +clockevent_init(struct clockevent *clkev, + void (*func)(struct clockevent *, struct clockframe *), + int flags) +{ + + memset(clkev, 0, sizeof(*clkev)); + clkev->clkev_func = func; + clkev->clkev_owner = NULL; +} + +void +clockevent_schedule(struct clockevent *clkev, const struct timespec *deadline) +{ + struct clockevent_cpu **cecp, *cec; + struct clockevent *collision __diagused; + struct cpu_info *owner __diagused; + int s; + + clkev->clkev_deadline = *deadline; + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + s = splsched(); + KASSERTMSG(clkev == &cec->cec_hardclock, /* XXX */ + "[%s] clkev=%p &cec->cec_hardclock=%p", + device_xname(curcpu()->ci_dev), clkev, &cec->cec_hardclock); + KASSERTMSG((owner = + atomic_cas_ptr(&clkev->clkev_owner, NULL, curcpu())) == NULL, + "clkev %p owned by %s", + clkev, device_xname(owner->ci_dev)); + clkev->clkev_ticket = cec->cec_ticket++; + collision = rb_tree_insert_node(&cec->cec_queue, clkev); + KASSERT(collision == clkev); + if (timespeccmp(&clkev->clkev_deadline, &cec->cec_next, <) || + __predict_false(!timespecisset(&cec->cec_next))) { + struct timespec now, delta; + + nanouptime(&now); + if (timespeccmp(&clkev->clkev_deadline, &now, <=)) { + timespecclear(&delta); + } else { + timespecsub(&now, &clkev->clkev_deadline, &delta); + } + lapic_schedule(&now, &delta, &clkev->clkev_deadline); + cec->cec_next = clkev->clkev_deadline; + } + splx(s); + percpu_putref(clockevent_percpu); +} + +static void +ipl_assert_atleast(int expected) +{ + int actual = curcpu()->ci_ilevel; + + KASSERT(kpreempt_disabled()); + KASSERTMSG(actual >= expected, "ipl is %d, expected >=%d", + actual, expected); +} + +static void +ipl_assert_exactly(int expected) +{ + int actual = curcpu()->ci_ilevel; + + KASSERT(kpreempt_disabled()); + KASSERTMSG(actual == expected, "ipl is %d, expected %d", + actual, expected); +} + +static void +clockevent_intr(void *frame) +{ + struct cpu_info *ci = curcpu(), *owner __diagused; + struct clockevent_cpu **cecp, *cec; + struct clockevent *clkev; + struct timespec now, delta; + + ipl_assert_exactly(IPL_SCHED); + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + + nanouptime(&now); + /* XXX record tardiness relative to cec->cec_next */ + if (__predict_false(timespeccmp(&now, &cec->cec_next, <))) { + timespecsub(&cec->cec_next, &now, &delta); + device_printf(ci->ci_dev, "%s: early by %lld.%09ld sec\n", + __func__, (long long)delta.tv_sec, delta.tv_nsec); + cec->cec_early.ev_count++; + if ((clkev = RB_TREE_MIN(&cec->cec_queue)) == NULL) { + device_printf(ci->ci_dev, "WARNING 1:" + " timer queue empty\n"); + goto out; + } + KASSERT(timespeccmp(&clkev->clkev_deadline, &cec->cec_next, + ==)); + goto reschedule; + } +restart: + while ((clkev = RB_TREE_MIN(&cec->cec_queue)) != NULL) { + KASSERTMSG(clkev == &cec->cec_hardclock, /* XXX */ + "[%s] clkev=%p &cec->cec_hardclock=%p", + device_xname(ci->ci_dev), clkev, &cec->cec_hardclock); + KASSERTMSG((owner = clkev->clkev_owner) == ci, + "clkev %p owned by %s, expected %s", + clkev, + device_xname(owner->ci_dev), + device_xname(ci->ci_dev)); + if (timespeccmp(&now, &clkev->clkev_deadline, <)) + goto reschedule; + rb_tree_remove_node(&cec->cec_queue, clkev); + KASSERTMSG((owner = atomic_cas_ptr(&clkev->clkev_owner, ci, + NULL)) == ci, + "clkev %p owned by %s, expected %s", + clkev, + device_xname(owner->ci_dev), + device_xname(ci->ci_dev)); + (*clkev->clkev_func)(clkev, frame); + ipl_assert_atleast(IPL_SCHED); + KASSERT(ci == curcpu()); + } + device_printf(ci->ci_dev, "WARNING 2: timer queue empty\n"); + goto out; + +reschedule: + /* + * Re-read the clock so that for relative wakeups, we don't + * drift by the time spent processing the queue. + */ + nanouptime(&now); + if (__predict_false(timespeccmp(&now, &clkev->clkev_deadline, >=))) + goto restart; + + timespecsub(&clkev->clkev_deadline, &now, &delta); + lapic_schedule(&now, &delta, &clkev->clkev_deadline); + cec->cec_next = clkev->clkev_deadline; + +out: percpu_putref(clockevent_percpu); +} + +static void +hardclock_intr(struct clockevent *clkev, struct clockframe *frame) +{ + struct clockevent_cpu **cecp, *cec; + struct timespec delta, next; + + ipl_assert_atleast(IPL_SCHED); + + delta.tv_sec = 0; + delta.tv_nsec = nsec_per_tick; + timespecadd(&clkev->clkev_deadline, &delta, &next); + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + KASSERTMSG(clkev == &cec->cec_hardclock, + "clkev=%p &cec->cec_hardclock=%p", + clkev, &cec->cec_hardclock); + clockevent_schedule(&cec->cec_hardclock, &next); + percpu_putref(clockevent_percpu); + + hardclock(frame); +} + +static void +hardclock_init(void) +{ + struct clockevent_cpu **cecp, *cec; + struct timespec now; + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + device_printf(curcpu()->ci_dev, "initialize hardclock %p\n", + &cec->cec_hardclock); + clockevent_init(&cec->cec_hardclock, &hardclock_intr, 0); + nanouptime(&now); + clockevent_schedule(&cec->cec_hardclock, &now); + percpu_putref(clockevent_percpu); +} + +/* + * XXX END SECTION THAT SHOULD BE MOVED TO MI kern/ XXX + */ + /* Referenced from vector.S */ void lapic_clockintr(void *, struct intrframe *); @@ -576,9 +853,32 @@ lapic_clockintr(void *arg, struct intrfr { struct cpu_info *ci = curcpu(); +// lapic_writereg(LAPIC_ICR_TIMER, lapic_tval); ci->ci_lapic_counter += lapic_tval; ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++; - hardclock((struct clockframe *)frame); + clockevent_intr((struct clockframe *)frame); +} + +static uint64_t lapic_max_nanosecond; /* XXX move me */ +static uint64_t lapic_max_second; /* XXX move me */ +static uint64_t lapic_frac_cycle_per_nsec; + +static void +lapic_schedule(const struct timespec *now, const struct timespec *delta, + const struct timespec *deadline) +{ + uint64_t delta_ns, delta_cyc; + + ipl_assert_atleast(IPL_SCHED); + + if (__predict_false(delta->tv_sec >= lapic_max_second) || + __predict_false((delta_ns = 1000000000*delta->tv_sec + + delta->tv_nsec) > lapic_max_nanosecond)) { + delta_cyc = UINT32_MAX; + } else { + delta_cyc = (lapic_frac_cycle_per_nsec * delta_ns) >> 32; + } + lapic_writereg(LAPIC_ICR_TIMER, MAX(1, delta_cyc)); } void @@ -591,11 +891,11 @@ lapic_reset(void) * then unmask and set the vector. */ lapic_writereg(LAPIC_LVT_TIMER, - LAPIC_LVT_TMM_PERIODIC | LAPIC_LVT_MASKED); + LAPIC_LVT_TMM_ONESHOT | LAPIC_LVT_MASKED); lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); lapic_writereg(LAPIC_ICR_TIMER, lapic_tval); lapic_writereg(LAPIC_LVT_TIMER, - LAPIC_LVT_TMM_PERIODIC | LAPIC_TIMER_VECTOR); + LAPIC_LVT_TMM_ONESHOT | LAPIC_TIMER_VECTOR); lapic_writereg(LAPIC_EOI, 0); } @@ -603,7 +903,11 @@ static void lapic_initclock(void) { + device_printf(curcpu()->ci_dev, "%s\n", __func__); + if (curcpu() == &cpu_info_primary) { + clockevents_init(); /* XXX */ + /* * Recalibrate the timer using the cycle counter, now that * the cycle counter itself has been recalibrated. @@ -620,8 +924,17 @@ lapic_initclock(void) tc_init(&lapic_timecounter); } - /* Start local apic countdown timer running, in repeated mode. */ - lapic_reset(); + /* XXX explain what's going on here */ + lapic_writereg(LAPIC_LVT_TIMER, + LAPIC_LVT_TMM_ONESHOT | LAPIC_LVT_MASKED); + lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); + lapic_writereg(LAPIC_ICR_TIMER, 0); + lapic_writereg(LAPIC_LVT_TIMER, + LAPIC_LVT_TMM_ONESHOT | LAPIC_TIMER_VECTOR); + lapic_writereg(LAPIC_EOI, 0); + hardclock_init(); + + device_printf(curcpu()->ci_dev, "%s done\n", __func__); } /* @@ -645,6 +958,8 @@ lapic_calibrate_timer(bool secondpass) KASSERT(ci == &cpu_info_primary); + device_printf(ci->ci_dev, "%scalibrating local timer\n", + secondpass ? "re" : ""); aprint_debug_dev(ci->ci_dev, "[re]calibrating local timer\n"); /* @@ -726,8 +1041,20 @@ lapic_calibrate_timer(bool secondpass) lapic_frac_usec_per_cycle = tmp; tmp = (lapic_per_second * (uint64_t)1 << 32) / 1000000; + lapic_frac_cycle_per_usec = tmp; - lapic_frac_cycle_per_usec = tmp; + tmp = (lapic_per_second * (uint64_t)1 << 32) / 1000000000; + lapic_frac_cycle_per_nsec = tmp; + + lapic_max_nanosecond = UINT64_MAX/lapic_frac_cycle_per_nsec; + lapic_max_second = + (lapic_max_nanosecond - 999999999)/1000000000; + printf("lapic_frac_cycle_per_nsec=%llu\n", + (long long)lapic_frac_cycle_per_nsec); + printf("lapic_max_second=%llu\n", + (long long)lapic_max_second); + printf("lapic_max_nanosecond=%llu\n", + (long long)lapic_max_nanosecond); /* * Compute delay in cycles for likely short delays in usec. @@ -751,11 +1078,14 @@ lapic_calibrate_timer(bool secondpass) * for all our timing needs.. */ if (!secondpass) { - delay_func = lapic_delay; + if (0) delay_func = lapic_delay; /* XXX */ x86_initclock_func = lapic_initclock; initrtclock(0); } } + + device_printf(ci->ci_dev, "%scalibrating local timer done\n", + secondpass ? "re" : ""); } /* # HG changeset patch # User Taylor R Campbell # Date 1747620305 0 # Mon May 19 02:05:05 2025 +0000 # Branch trunk # Node ID a3967838f695a137109a514675b4e3389584af43 # Parent a09802de9029d936f992a7ef52b44c22896ca0a3 # EXP-Topic riastradh-highrestimer sys/conf/debugsyms.c: Sort includes. No functional change intended. diff -r a09802de9029 -r a3967838f695 sys/conf/debugsyms.c --- a/sys/conf/debugsyms.c Mon May 19 01:28:23 2025 +0000 +++ b/sys/conf/debugsyms.c Mon May 19 02:05:05 2025 +0000 @@ -11,20 +11,21 @@ #define __KAUTH_PRIVATE #include + +#include #include #include #include #include #include #include +#include #include #include #include #include #include #include -#include -#include /* * Without a dummy function referencing some of the types, gcc will # HG changeset patch # User Taylor R Campbell # Date 1747620452 0 # Mon May 19 02:07:32 2025 +0000 # Branch trunk # Node ID bc92d187f9d859f47cd85ce659606c0ace2fe306 # Parent a3967838f695a137109a514675b4e3389584af43 # EXP-Topic riastradh-highrestimer kern_timeout.c: Sort includes. No functional change intended. diff -r a3967838f695 -r bc92d187f9d8 sys/kern/kern_timeout.c --- a/sys/kern/kern_timeout.c Mon May 19 02:05:05 2025 +0000 +++ b/sys/kern/kern_timeout.c Mon May 19 02:07:32 2025 +0000 @@ -85,27 +85,28 @@ #define _CALLOUT_PRIVATE #include -#include + +#include +#include +#include +#include #include -#include +#include #include #include #include +#include #include #include -#include -#include -#include -#include -#include +#include #ifdef DDB #include -#include #include #include +#include +#include #include -#include #endif #define BUCKETS 1024 # HG changeset patch # User Taylor R Campbell # Date 1747620939 0 # Mon May 19 02:15:39 2025 +0000 # Branch trunk # Node ID 6f198220806f2f760d06a74ffa5be043ce012ac5 # Parent bc92d187f9d859f47cd85ce659606c0ace2fe306 # EXP-Topic riastradh-highrestimer callout.h: Split _CALLOUT_PRIVATE stuff into new callout_impl.h. Saves me the trouble of rebuilding ~everything while tweaking what goes in the private guts of a callout. diff -r bc92d187f9d8 -r 6f198220806f distrib/sets/lists/comp/mi --- a/distrib/sets/lists/comp/mi Mon May 19 02:07:32 2025 +0000 +++ b/distrib/sets/lists/comp/mi Mon May 19 02:15:39 2025 +0000 @@ -3404,6 +3404,7 @@ ./usr/include/sys/bus.h comp-obsolete obsolete ./usr/include/sys/callback.h comp-c-include ./usr/include/sys/callout.h comp-c-include +./usr/include/sys/callout_impl.h comp-c-include ./usr/include/sys/cc_microtime.h comp-obsolete obsolete ./usr/include/sys/cdbr.h comp-c-include ./usr/include/sys/cdefs.h comp-c-include diff -r bc92d187f9d8 -r 6f198220806f sys/conf/debugsyms.c --- a/sys/conf/debugsyms.c Mon May 19 02:07:32 2025 +0000 +++ b/sys/conf/debugsyms.c Mon May 19 02:15:39 2025 +0000 @@ -6,12 +6,12 @@ #include __KERNEL_RCSID(0, "$NetBSD: debugsyms.c,v 1.4 2015/10/06 22:13:39 christos Exp $"); -#define _CALLOUT_PRIVATE #define __MUTEX_PRIVATE #define __KAUTH_PRIVATE #include +#include #include #include #include diff -r bc92d187f9d8 -r 6f198220806f sys/kern/kern_timeout.c --- a/sys/kern/kern_timeout.c Mon May 19 02:07:32 2025 +0000 +++ b/sys/kern/kern_timeout.c Mon May 19 02:15:39 2025 +0000 @@ -82,11 +82,10 @@ * timeouts and 0 or negative for due timeouts. */ -#define _CALLOUT_PRIVATE - #include #include +#include #include #include #include diff -r bc92d187f9d8 -r 6f198220806f sys/sys/Makefile --- a/sys/sys/Makefile Mon May 19 02:07:32 2025 +0000 +++ b/sys/sys/Makefile Mon May 19 02:15:39 2025 +0000 @@ -7,7 +7,7 @@ INCSDIR= /usr/include/sys INCS= acct.h acl.h agpio.h aio.h ansi.h aout_mids.h ataio.h atomic.h \ audioio.h \ bitops.h bootblock.h bswap.h buf.h \ - callback.h callout.h cdbr.h cdefs.h cdefs_aout.h \ + callback.h callout.h callout_impl.h cdbr.h cdefs.h cdefs_aout.h \ cdefs_elf.h cdio.h chio.h clock.h clockctl.h \ common_ansi.h common_int_const.h common_int_fmtio.h \ common_int_limits.h common_int_mwgwtypes.h common_int_types.h \ diff -r bc92d187f9d8 -r 6f198220806f sys/sys/callout.h --- a/sys/sys/callout.h Mon May 19 02:07:32 2025 +0000 +++ b/sys/sys/callout.h Mon May 19 02:15:39 2025 +0000 @@ -66,41 +66,6 @@ b\02FIRED\0\ b\03INVOKING\0\ b\10MPSAFE\0" -#ifdef _CALLOUT_PRIVATE - -/* The following funkyness is to appease gcc3's strict aliasing. */ -struct callout_circq { - /* next element */ - union { - struct callout_impl *elem; - struct callout_circq *list; - } cq_next; - /* previous element */ - union { - struct callout_impl *elem; - struct callout_circq *list; - } cq_prev; -}; -#define cq_next_e cq_next.elem -#define cq_prev_e cq_prev.elem -#define cq_next_l cq_next.list -#define cq_prev_l cq_prev.list - -struct callout_cpu; - -typedef struct callout_impl { - struct callout_circq c_list; /* linkage on queue */ - void (*c_func)(void *); /* function to call */ - void *c_arg; /* function argument */ - struct callout_cpu * volatile c_cpu; /* associated CPU */ - int c_time; /* when callout fires */ - u_int c_flags; /* state of this entry */ - u_int c_magic; /* magic number */ -} callout_impl_t; -#define CALLOUT_MAGIC 0x11deeba1 - -#endif /* _CALLOUT_PRIVATE */ - #ifdef _KERNEL struct cpu_info; diff -r bc92d187f9d8 -r 6f198220806f sys/sys/callout_impl.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/sys/callout_impl.h Mon May 19 02:15:39 2025 +0000 @@ -0,0 +1,76 @@ +/* $NetBSD: callout.h,v 1.32 2015/02/07 19:36:42 christos Exp $ */ + +/*- + * Copyright (c) 2000, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, + * NASA Ames Research Center, and by Andrew Doran. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_CALLOUT_IMPL_H_ +#define _SYS_CALLOUT_IMPL_H_ + +#include + +#include + +/* + * This is a private kernel API that is not stable across versions. + * Userland tools that reach into it may get what they deserve. + */ + +/* The following funkyness is to appease gcc3's strict aliasing. */ +struct callout_circq { + /* next element */ + union { + struct callout_impl *elem; + struct callout_circq *list; + } cq_next; + /* previous element */ + union { + struct callout_impl *elem; + struct callout_circq *list; + } cq_prev; +}; +#define cq_next_e cq_next.elem +#define cq_prev_e cq_prev.elem +#define cq_next_l cq_next.list +#define cq_prev_l cq_prev.list + +struct callout_cpu; + +typedef struct callout_impl { + struct callout_circq c_list; /* linkage on queue */ + void (*c_func)(void *); /* function to call */ + void *c_arg; /* function argument */ + struct callout_cpu * volatile c_cpu; /* associated CPU */ + int c_time; /* when callout fires */ + u_int c_flags; /* state of this entry */ + u_int c_magic; /* magic number */ +} callout_impl_t; +#define CALLOUT_MAGIC 0x11deeba1 + +#endif /* !_SYS_CALLOUT_IMPL_H_ */ diff -r bc92d187f9d8 -r 6f198220806f usr.bin/netstat/inet.c --- a/usr.bin/netstat/inet.c Mon May 19 02:07:32 2025 +0000 +++ b/usr.bin/netstat/inet.c Mon May 19 02:15:39 2025 +0000 @@ -38,9 +38,9 @@ static char sccsid[] = "from: @(#)inet.c #endif #endif /* not lint */ -#define _CALLOUT_PRIVATE /* for defs in sys/callout.h */ +#include -#include +#include #include #include #include diff -r bc92d187f9d8 -r 6f198220806f usr.bin/netstat/inet6.c --- a/usr.bin/netstat/inet6.c Mon May 19 02:07:32 2025 +0000 +++ b/usr.bin/netstat/inet6.c Mon May 19 02:15:39 2025 +0000 @@ -68,9 +68,9 @@ static char sccsid[] = "@(#)inet.c 8.4 ( #endif #endif /* not lint */ -#define _CALLOUT_PRIVATE +#include -#include +#include #include #include #include diff -r bc92d187f9d8 -r 6f198220806f usr.bin/netstat/pfsync.c --- a/usr.bin/netstat/pfsync.c Mon May 19 02:07:32 2025 +0000 +++ b/usr.bin/netstat/pfsync.c Mon May 19 02:15:39 2025 +0000 @@ -34,9 +34,9 @@ __RCSID("$NetBSD: pfsync.c,v 1.5 2022/09/02 06:25:43 msaitoh Exp $"); #endif /* not lint */ -#define _CALLOUT_PRIVATE /* for defs in sys/callout.h */ +#include -#include +#include #include #include #include diff -r bc92d187f9d8 -r 6f198220806f usr.bin/netstat/vtw.c --- a/usr.bin/netstat/vtw.c Mon May 19 02:07:32 2025 +0000 +++ b/usr.bin/netstat/vtw.c Mon May 19 02:15:39 2025 +0000 @@ -66,9 +66,9 @@ static char sccsid[] = "from: @(#)inet.c #endif #endif /* not lint */ -#define _CALLOUT_PRIVATE /* for defs in sys/callout.h */ +#include -#include +#include #include #include #include diff -r bc92d187f9d8 -r 6f198220806f usr.sbin/trpt/trpt.c --- a/usr.sbin/trpt/trpt.c Mon May 19 02:07:32 2025 +0000 +++ b/usr.sbin/trpt/trpt.c Mon May 19 02:15:39 2025 +0000 @@ -73,9 +73,9 @@ static char sccsid[] = "@(#)trpt.c 8.1 ( #endif #endif /* not lint */ -#define _CALLOUT_PRIVATE /* for defs in sys/callout.h */ +#include -#include +#include #include #include #include # HG changeset patch # User Taylor R Campbell # Date 1747968040 0 # Fri May 23 02:40:40 2025 +0000 # Branch trunk # Node ID d58aa4faa8da37b33d2316138c517ec5855121ee # Parent 6f198220806f2f760d06a74ffa5be043ce012ac5 # EXP-Topic riastradh-highrestimer WIP: Count pending callouts on each CPU. XXX evcnt(9) is not really meant for this. diff -r 6f198220806f -r d58aa4faa8da sys/kern/kern_timeout.c --- a/sys/kern/kern_timeout.c Mon May 19 02:15:39 2025 +0000 +++ b/sys/kern/kern_timeout.c Fri May 23 02:40:40 2025 +0000 @@ -177,10 +177,12 @@ struct callout_cpu { callout_impl_t *cc_active; struct evcnt cc_ev_late; struct evcnt cc_ev_block; + struct evcnt cc_ev_pend; struct callout_circq cc_todo; /* Worklist */ struct callout_circq cc_wheel[BUCKETS]; /* Queues of timeouts */ char cc_name1[12]; char cc_name2[12]; + char cc_name3[12]; struct cpu_info *cc_cpu; }; @@ -344,6 +346,11 @@ callout_init_cpu(struct cpu_info *ci) evcnt_attach_dynamic(&cc->cc_ev_block, EVCNT_TYPE_MISC, NULL, "callout", cc->cc_name2); + snprintf(cc->cc_name3, sizeof(cc->cc_name3), "pend/%u", + cpu_index(ci)); + evcnt_attach_dynamic(&cc->cc_ev_pend, EVCNT_TYPE_MISC, + NULL, "callout", cc->cc_name3); + cc->cc_cpu = ci; ci->ci_data.cpu_callout = cc; } @@ -452,12 +459,14 @@ callout_schedule_locked(callout_impl_t * c->c_time = to_ticks + occ->cc_ticks; c->c_flags |= CALLOUT_PENDING; CIRCQ_INSERT(&c->c_list, &occ->cc_todo); + occ->cc_ev_pend.ev_count++; } else { /* Move to this CPU. */ c->c_cpu = cc; c->c_time = to_ticks + cc->cc_ticks; c->c_flags |= CALLOUT_PENDING; CIRCQ_INSERT(&c->c_list, &cc->cc_todo); + cc->cc_ev_pend.ev_count++; mutex_spin_exit(cc->cc_lock); SDT_PROBE6(sdt, kernel, callout, migrate, c, c->c_func, c->c_arg, c->c_flags, @@ -524,8 +533,10 @@ callout_stop(callout_t *cs) lock = callout_lock(c); - if ((c->c_flags & CALLOUT_PENDING) != 0) + if ((c->c_flags & CALLOUT_PENDING) != 0) { CIRCQ_REMOVE(&c->c_list); + c->c_cpu->cc_ev_pend.ev_count--; + } expired = ((c->c_flags & CALLOUT_FIRED) != 0); c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED); @@ -561,8 +572,10 @@ callout_halt(callout_t *cs, void *interl lock = callout_lock(c); SDT_PROBE4(sdt, kernel, callout, halt, c, c->c_func, c->c_arg, c->c_flags); - if ((c->c_flags & CALLOUT_PENDING) != 0) + if ((c->c_flags & CALLOUT_PENDING) != 0) { CIRCQ_REMOVE(&c->c_list); + c->c_cpu->cc_ev_pend.ev_count--; + } c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED); if (__predict_false(callout_running_somewhere_else(c, c->c_cpu))) { callout_wait(c, interlock, lock); @@ -632,8 +645,10 @@ callout_wait(callout_impl_t *c, void *in * themselves so put a stop to it again if needed. */ lock = callout_lock(c); - if ((c->c_flags & CALLOUT_PENDING) != 0) + if ((c->c_flags & CALLOUT_PENDING) != 0) { CIRCQ_REMOVE(&c->c_list); + c->c_cpu->cc_ev_pend.ev_count--; + } c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED); } @@ -853,6 +868,7 @@ callout_softclock(void *v) if (delta < 0) cc->cc_ev_late.ev_count++; + cc->cc_ev_pend.ev_count--; c->c_flags = (c->c_flags & ~CALLOUT_PENDING) | (CALLOUT_FIRED | CALLOUT_INVOKING); mpsafe = (c->c_flags & CALLOUT_MPSAFE); # HG changeset patch # User Taylor R Campbell # Date 1747748859 0 # Tue May 20 13:47:39 2025 +0000 # Branch trunk # Node ID 0362df1f713562f3e98b5d6a4428a09371426401 # Parent 421608551bbf6105a331f80b23afd9b4604bf6f9 # EXP-Topic riastradh-highrestimer xen: Avoid playing catchup with the hardclock timer. It's not clear this is worthwhile for anything except possibly the emulated schedclock, which really shouldn't be running via hardclock anyway -- should be done with separate high-resolution timer logic, since Xen gives us a high-resolution timer. Also: 1. Rearm the timer immediately, rather than after calling hardclock, so less time has elapsed since it fired -- this should reduce the chance that rearming the timer fails. 2. Add a dtrace probe for when we retry rearming the timer. PR port-xen/59437: Xen time stability PR kern/57920: hardclock(9) contract is unclear about missed ticks diff -r 421608551bbf -r 0362df1f7135 sys/arch/xen/xen/xen_clock.c --- a/sys/arch/xen/xen/xen_clock.c Tue May 20 13:43:03 2025 +0000 +++ b/sys/arch/xen/xen/xen_clock.c Tue May 20 13:47:39 2025 +0000 @@ -127,6 +127,10 @@ SDT_PROBE_DEFINE3(sdt, xen, hardclock, j "uint64_t"/*last_systime_ns*/, "uint64_t"/*this_systime_ns*/, "uint64_t"/*nticks*/); +SDT_PROBE_DEFINE3(sdt, xen, hardclock, rearm__failed, + "uint64_t"/*last_systime_ns*/, + "uint64_t"/*this_systime_ns*/, + "uint64_t"/*next_systime_ns*/); SDT_PROBE_DEFINE3(sdt, xen, hardclock, missed, "uint64_t"/*last_systime_ns*/, "uint64_t"/*this_systime_ns*/, @@ -786,6 +790,7 @@ xen_timer_handler(void *cookie, struct c #if defined(XENPV) frame = NULL; /* We use values cached in curcpu() */ #endif + /* * Find how many nanoseconds of Xen system time has elapsed * since the last hardclock tick. @@ -808,12 +813,18 @@ xen_timer_handler(void *cookie, struct c */ ci->ci_xen_hardclock_systime_ns = last = now - ns_per_tick; } + + /* + * Compute the time since the last tick. Ideally, this should + * lie somewhere in [ns_per_tick, 2*ns_per_tick), but if the + * timer fired early it might lie in [0, ns_per_tick) and if + * the timer fired excessively late it might lie in + * [2*ns_per_tick, \infty). + */ delta = now - last; /* - * Play hardclock catchup: run the hardclock timer as many - * times as appears necessary based on how much time has - * passed. + * Check to see if we ran late by a tick or more. */ if (__predict_false(delta >= 2*ns_per_tick)) { SDT_PROBE3(sdt, xen, hardclock, jump, @@ -834,36 +845,44 @@ xen_timer_handler(void *cookie, struct c xen_timecounter.tc_counter_mask); ci->ci_xen_timecounter_jump_evcnt.ev_count++; } - /* don't try to catch up more than one second at once */ - if (delta > 1000000000UL) - delta = 1000000000UL; + + /* + * If delta \in [k*ns_per_tick, (k + 1)*ns_per_tick), + * set next := (k + 1)*ns_per_tick. Note that + * rounddown(delta, ns_per_tick) = k*ns_per_tick, so we + * add one more to get (k + 1)*ns_per_tick. + */ + next = last + rounddown(delta, ns_per_tick) + ns_per_tick; + } else { + next = last + 2*ns_per_tick; } - while (delta >= ns_per_tick) { - ci->ci_xen_hardclock_systime_ns += ns_per_tick; - delta -= ns_per_tick; - hardclock(frame); - if (__predict_false(delta >= ns_per_tick)) { - SDT_PROBE3(sdt, xen, hardclock, missed, - last, now, delta); - ci->ci_xen_missed_hardclock_evcnt.ev_count++; + + /* + * Re-arm the timer. If it fails, the time may have already + * passed, so try one more tick later. If that fails, panic. + */ + ci->ci_xen_hardclock_systime_ns = next - ns_per_tick; + error = HYPERVISOR_set_timer_op(next); + if (error) { + SDT_PROBE3(sdt, xen, hardclock, rearm__failed, + last, now, next); + last = next; + next += roundup(xen_vcputime_systime_ns() - last, + ns_per_tick); + next += ns_per_tick; + ci->ci_xen_hardclock_systime_ns = next - ns_per_tick; + error = HYPERVISOR_set_timer_op(next); + if (error) { + panic("failed to rearm Xen timer" + " at %"PRIu64" ns or %"PRIu64" ns: %d", + last, next, error); } } /* - * Re-arm the timer. If it fails, it's probably because the - * time is in the past, possibly because we're in the - * process of catching up missed hardclock calls. - * In this case schedule a tick in the near future. + * Call hardclock once. */ - next = ci->ci_xen_hardclock_systime_ns + ns_per_tick; - error = HYPERVISOR_set_timer_op(next); - if (error) { - next = xen_vcputime_systime_ns() + ns_per_tick / 2; - error = HYPERVISOR_set_timer_op(next); - if (error) { - panic("failed to re-arm Xen timer %d", error); - } - } + hardclock(frame); /* Success! */ return 0; # HG changeset patch # User Taylor R Campbell # Date 1748143883 0 # Sun May 25 03:31:23 2025 +0000 # Branch trunk # Node ID 9bf92a837cd52d6388f759d614f2f0ee8b5082e5 # Parent 0362df1f713562f3e98b5d6a4428a09371426401 # EXP-Topic riastradh-highrestimer WIP: Draft a separate statclock. diff -r 0362df1f7135 -r 9bf92a837cd5 sys/arch/x86/x86/lapic.c --- a/sys/arch/x86/x86/lapic.c Tue May 20 13:47:39 2025 +0000 +++ b/sys/arch/x86/x86/lapic.c Sun May 25 03:31:23 2025 +0000 @@ -124,12 +124,14 @@ void clockevent_destroy(struct clockeven #include #include /* XXX move to struct cpu_data */ +#include struct clockevent_cpu { struct rb_tree cec_queue; uint64_t cec_ticket; struct timespec cec_next; struct clockevent cec_hardclock; + struct clockevent cec_statclock; struct evcnt cec_early; }; @@ -176,12 +178,14 @@ clockevents_fini_cpu(void *ptr, void *co } static unsigned nsec_per_tick; /* XXX */ +static unsigned nsec_per_stat; /* XXX */ static void clockevents_init(void) { nsec_per_tick = 1000000000/hz; + nsec_per_stat = 1000000000/stathz; clockevent_percpu = percpu_create(sizeof(struct clockevent_cpu), clockevents_init_cpu, clockevents_fini_cpu, NULL); } @@ -210,9 +214,6 @@ clockevent_schedule(struct clockevent *c cecp = percpu_getref(clockevent_percpu); cec = *cecp; s = splsched(); - KASSERTMSG(clkev == &cec->cec_hardclock, /* XXX */ - "[%s] clkev=%p &cec->cec_hardclock=%p", - device_xname(curcpu()->ci_dev), clkev, &cec->cec_hardclock); KASSERTMSG((owner = atomic_cas_ptr(&clkev->clkev_owner, NULL, curcpu())) == NULL, "clkev %p owned by %s", @@ -288,9 +289,6 @@ clockevent_intr(void *frame) } restart: while ((clkev = RB_TREE_MIN(&cec->cec_queue)) != NULL) { - KASSERTMSG(clkev == &cec->cec_hardclock, /* XXX */ - "[%s] clkev=%p &cec->cec_hardclock=%p", - device_xname(ci->ci_dev), clkev, &cec->cec_hardclock); KASSERTMSG((owner = clkev->clkev_owner) == ci, "clkev %p owned by %s, expected %s", clkev, @@ -367,6 +365,45 @@ hardclock_init(void) percpu_putref(clockevent_percpu); } +static void +statclock_intr(struct clockevent *clkev, struct clockframe *frame) +{ + struct clockevent_cpu **cecp, *cec; + struct timespec delta, next; + + ipl_assert_atleast(IPL_SCHED); + + delta.tv_sec = 0; + delta.tv_nsec = nsec_per_stat; + timespecadd(&clkev->clkev_deadline, &delta, &next); + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + KASSERTMSG(clkev == &cec->cec_statclock, + "clkev=%p &cec->cec_statclock=%p", + clkev, &cec->cec_statclock); + clockevent_schedule(&cec->cec_statclock, &next); + percpu_putref(clockevent_percpu); + + statclock(frame); +} + +static void +statclock_init(void) +{ + struct clockevent_cpu **cecp, *cec; + struct timespec now; + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + device_printf(curcpu()->ci_dev, "initialize statclock %p\n", + &cec->cec_statclock); + clockevent_init(&cec->cec_statclock, &statclock_intr, 0); + nanouptime(&now); + clockevent_schedule(&cec->cec_statclock, &now); + percpu_putref(clockevent_percpu); +} + /* * XXX END SECTION THAT SHOULD BE MOVED TO MI kern/ XXX */ @@ -906,6 +943,7 @@ lapic_initclock(void) device_printf(curcpu()->ci_dev, "%s\n", __func__); if (curcpu() == &cpu_info_primary) { + stathz = 16; /* XXX */ clockevents_init(); /* XXX */ /* @@ -933,6 +971,7 @@ lapic_initclock(void) LAPIC_LVT_TMM_ONESHOT | LAPIC_TIMER_VECTOR); lapic_writereg(LAPIC_EOI, 0); hardclock_init(); + statclock_init(); device_printf(curcpu()->ci_dev, "%s done\n", __func__); } # HG changeset patch # User Taylor R Campbell # Date 1748209633 0 # Sun May 25 21:47:13 2025 +0000 # Branch trunk # Node ID 6fae1301093d1d57217069883d62e199f2b16ad1 # Parent 9bf92a837cd52d6388f759d614f2f0ee8b5082e5 # EXP-Topic riastradh-highrestimer WIP: callout(9): New callout_nanoschedule(). diff -r 9bf92a837cd5 -r 6fae1301093d share/man/man9/callout.9 --- a/share/man/man9/callout.9 Sun May 25 03:31:23 2025 +0000 +++ b/share/man/man9/callout.9 Sun May 25 21:47:13 2025 +0000 @@ -36,6 +36,7 @@ .Nm callout_halt , .Nm callout_reset , .Nm callout_schedule , +.Nm callout_nanoschedule , .Nm callout_setfunc , .Nm callout_stop , .Nm callout_pending , @@ -55,6 +56,9 @@ .Ft void .Fn "callout_schedule" "callout_t *c" "int ticks" .Ft void +.Fn "callout_nanoschedule" "callout_t *c" "const struct timespec *deadline" \ + "const struct timespec *slop" +.Ft void .Fn "callout_setfunc" "callout_t *c" "void (*func)(void *)" "void *arg" .Ft bool .Fn "callout_stop" "callout_t *c" @@ -165,6 +169,15 @@ is slightly more efficient than using .Fn callout_reset . .Pp The +.Fn callout_nanoschedule +function schedules a callout to run after a given +.Dv CLOCK_MONOTONIC +deadline, plus an optional slop, plus unavoidable scheduling latency. +The optional slop may be used to select between costlier +high-resolution clocks and cheaper low-resolution clocks, and may be +used to coalesce callouts that are nearby in time. +.Pp +The .Fn callout_stop function requests that the timer associated with the callout handle .Fa c diff -r 9bf92a837cd5 -r 6fae1301093d sys/conf/param.c --- a/sys/conf/param.c Sun May 25 03:31:23 2025 +0000 +++ b/sys/conf/param.c Sun May 25 21:47:13 2025 +0000 @@ -111,7 +111,8 @@ #endif int hz = HZ; -int tick = 1000000 / HZ; +int tick = 1000000 / HZ; /* microseconds per tick */ +int tick_nsec = 1000000000 / HZ; /* nanoseconds per tick */ /* can adjust 240ms in 60s */ int tickadj = (240000 / (60 * HZ)) ? (240000 / (60 * HZ)) : 1; int rtc_offset = RTC_OFFSET; diff -r 9bf92a837cd5 -r 6fae1301093d sys/kern/kern_clock.c --- a/sys/kern/kern_clock.c Sun May 25 03:31:23 2025 +0000 +++ b/sys/kern/kern_clock.c Sun May 25 21:47:13 2025 +0000 @@ -245,6 +245,7 @@ initclocks(void) cpu_initclocks(); if (old_hz != hz) { tick = 1000000 / hz; + tick_nsec = 1000000000 / hz; tickadj = (240000 / (60 * hz)) ? (240000 / (60 * hz)) : 1; } diff -r 9bf92a837cd5 -r 6fae1301093d sys/kern/kern_timeout.c --- a/sys/kern/kern_timeout.c Sun May 25 03:31:23 2025 +0000 +++ b/sys/kern/kern_timeout.c Sun May 25 21:47:13 2025 +0000 @@ -512,6 +512,39 @@ callout_schedule(callout_t *cs, int to_t KASSERT(c->c_magic == CALLOUT_MAGIC); lock = callout_lock(c); + c->c_flags &= ~CALLOUT_DEADLINE; + callout_schedule_locked(c, lock, to_ticks); +} + +/* + * callout_nanoschedule: + * + * Schedule a callout to run at a deadline on CLOCK_MONOTONIC. + */ +void +callout_nanoschedule(callout_t *cs, const struct timespec *deadline, + const struct timespec *slop) +{ + callout_impl_t *c = (callout_impl_t *)cs; + kmutex_t *lock; + struct timespec now, delta; + int to_ticks; + + KASSERT(c->c_magic == CALLOUT_MAGIC); + + lock = callout_lock(c); + c->c_flags |= CALLOUT_DEADLINE; + c->c_deadline_sec = deadline->tv_sec; + c->c_deadline_nsec = deadline->tv_nsec; + c->c_deadline_slop_nsec = + (slop->tv_sec > 0 ? tick_nsec : MIN(tick_nsec, slop->tv_nsec)); + getnanouptime(&now); + if (timespeccmp(deadline, &now, <)) { + timespecsub(&now, deadline, &delta); + to_ticks = tstohz(&delta); + } else { + to_ticks = 0; + } callout_schedule_locked(c, lock, to_ticks); } @@ -839,6 +872,7 @@ callout_softclock(void *v) struct callout_cpu *cc; void (*func)(void *); void *arg; + struct timespec now; int mpsafe, count, ticks, delta; u_int flags __unused; lwp_t *l; @@ -847,6 +881,8 @@ callout_softclock(void *v) KASSERT(l->l_cpu == curcpu()); cc = l->l_cpu->ci_data.cpu_callout; + getnanouptime(&now); + mutex_spin_enter(cc->cc_lock); cc->cc_lwp = l; while (!CIRCQ_EMPTY(&cc->cc_todo)) { @@ -868,9 +904,34 @@ callout_softclock(void *v) if (delta < 0) cc->cc_ev_late.ev_count++; + /* + * If it has an absolute deadline, check whether the + * deadline has passed. + */ + if (c->c_flags & CALLOUT_DEADLINE) { + const struct timespec deadline = { + .tv_sec = c->c_deadline_sec, + .tv_nsec = c->c_deadline_nsec, + }; + + if (timespeccmp(&deadline, &now, <)) { + struct timespec nanodelta; + + /* XXX count event of pre-deadline fire */ + timespecsub(&now, &deadline, &nanodelta); + c->c_time = tstohz(&nanodelta); + delta = (int)((unsigned)c->c_time - + (unsigned)ticks); + KASSERT(delta > 0); + CIRCQ_INSERT(&c->c_list, + BUCKET(cc, delta, c->c_time)); + continue; + } + } + cc->cc_ev_pend.ev_count--; - c->c_flags = (c->c_flags & ~CALLOUT_PENDING) | - (CALLOUT_FIRED | CALLOUT_INVOKING); + c->c_flags &= ~(CALLOUT_PENDING | CALLOUT_DEADLINE); + c->c_flags |= (CALLOUT_FIRED | CALLOUT_INVOKING); mpsafe = (c->c_flags & CALLOUT_MPSAFE); func = c->c_func; arg = c->c_arg; diff -r 9bf92a837cd5 -r 6fae1301093d sys/sys/callout.h --- a/sys/sys/callout.h Sun May 25 03:31:23 2025 +0000 +++ b/sys/sys/callout.h Sun May 25 21:47:13 2025 +0000 @@ -35,6 +35,8 @@ #include +struct timespec; + /* * The callout implementation is private to kern_timeout.c yet uses * caller-supplied storage, as lightweight callout operations are @@ -54,6 +56,7 @@ typedef struct callout { #define CALLOUT_PENDING 0x0002 /* callout is on the queue */ #define CALLOUT_FIRED 0x0004 /* callout has fired */ #define CALLOUT_INVOKING 0x0008 /* callout function is being invoked */ +#define CALLOUT_DEADLINE 0x0010 /* scheduled for absolute deadline */ /* End-user flags. */ #define CALLOUT_MPSAFE 0x0100 /* does not need kernel_lock */ @@ -78,6 +81,8 @@ void callout_destroy(callout_t *); void callout_setfunc(callout_t *, void (*)(void *), void *); void callout_reset(callout_t *, int, void (*)(void *), void *); void callout_schedule(callout_t *, int); +void callout_nanoschedule(callout_t *, const struct timespec *, + const struct timespec *); bool callout_stop(callout_t *); bool callout_halt(callout_t *, void *); bool callout_pending(callout_t *); diff -r 9bf92a837cd5 -r 6fae1301093d sys/sys/callout_impl.h --- a/sys/sys/callout_impl.h Sun May 25 03:31:23 2025 +0000 +++ b/sys/sys/callout_impl.h Sun May 25 21:47:13 2025 +0000 @@ -70,6 +70,9 @@ typedef struct callout_impl { int c_time; /* when callout fires */ u_int c_flags; /* state of this entry */ u_int c_magic; /* magic number */ + uint32_t c_deadline_slop_nsec; + uint64_t c_deadline_sec; + uint32_t c_deadline_nsec; } callout_impl_t; #define CALLOUT_MAGIC 0x11deeba1 diff -r 9bf92a837cd5 -r 6fae1301093d sys/sys/kernel.h --- a/sys/sys/kernel.h Sun May 25 03:31:23 2025 +0000 +++ b/sys/sys/kernel.h Sun May 25 21:47:13 2025 +0000 @@ -56,6 +56,7 @@ extern int cold; /* still working on st extern int start_init_exec; /* init(8) may have started */ extern int shutting_down; /* system is shutting down */ extern int tick; /* usec per tick (1000000 / hz) */ +extern int tick_nsec; /* nsec per tick (1000000000 / hz) */ extern int tickadj; /* "standard" clock skew, us./tick */ extern int hz; /* system clock's frequency */ extern int stathz; /* statistics clock's frequency */ # HG changeset patch # User Taylor R Campbell # Date 1753152343 0 # Tue Jul 22 02:45:43 2025 +0000 # Branch trunk # Node ID 830a9bcff0347a232ea1eb252923360426a4d59c # Parent 6fae1301093d1d57217069883d62e199f2b16ad1 # EXP-Topic riastradh-highrestimer WIP: hack hack hack clockintr diff -r 6fae1301093d -r 830a9bcff034 share/man/man9/clockintr.9 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/share/man/man9/clockintr.9 Tue Jul 22 02:45:43 2025 +0000 @@ -0,0 +1,242 @@ +.\" $NetBSD$ +.\" +.\" Copyright (c) 2025 The NetBSD Foundation, Inc. +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS +.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS +.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGE. +.\" +.Dd May 28, 2025 +.Dt CLOCKINTR 9 +.Os +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh NAME +.Nm clockintr +.Nd per-CPU high-resolution clock interrupt configuration +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh SYNOPSIS +.In sys/clockintr.h +.\" +.\"""""""""""""""""""""""""""""""""""""" +.Fd "/* MI subroutines called from MD logic */" +.\" +.Ft void +.Fn clockintr_init +.\" +.Ft void +.Fn clockintr_suspend +.\" +.Ft void +.Fn clockintr_resume +.\" +.Ft void +.Fn clockintr "struct clockframe *frame" "uint32_t hwtime" +.\" +.\"""""""""""""""""""""""""""""""""""""" +.Fd "/* MD subroutines called from MI logic */" +.Ft uint32_t +.Fn cpu_clockintr_ns2hwtime "uint32_t ns" +.\" +.Vt uint32_t cpu_clockintr_maxns ; +.\" +.Ft uint32_t +.Fn cpu_clockintr_now +.\" +.Ft void +.Fn cpu_clockintr_enable "uint32_t hwtime" +.\" +.Ft void +.Fn cpu_clockintr_disable +.\" +.Ft void +.Fn cpu_clockintr_arm "uint32_t hwtime" +.\"""""""""""""""""""""""""""""""""""""" +.Fd "/* MI subroutines called from MI logic */" +.Ft void +.Fn clockintr_schedule_callout "uint32_t hwtime" +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh DESCRIPTION +The +.Nm +subsystem for per-CPU clock interrupts is the glue that lies between: +.Bl -bullet +.It +the general-purpose +.Xr callout 9 +for scheduling functions to be called at certain times, and +.It +machine-dependent logic to configure clocks that deliver +high-resolution timed interrupts. +.El +.Pp +The +.Nm +functions are not meant for general-purpose use in the kernel: they are +meant to be used only by +.Xr callout 9 +itself and machine-dependent logic that elects to use the same +high-resolution timer for +.Xr hardclock 9 . +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh FUNCTIONS +.Ss MI subroutines called from MD logic +.Bl -tag -width abcd +.It Fn clockintr_init +Initialize use of the +.Nm +subsystem on the current CPU. +Must be called during +.Xr cpu_initclocks 9 +by machine-dependent logic that elects to use +.Nm , +on every CPU in the system. +XXX Maybe also on CPU hot-plug, if we ever supported that. +.It Fn clockintr_suspend , Fn clockintr_resume +Suspend and resume +.Nm +on the current CPU. +Must be called for each CPU around offlining/onlining. +.Pp +XXX Should these be called by the MI +.Xr cpu_setstate 9 +function? +.Pp +XXX What happens to callouts that have passed their deadlines while +suspended? +.It Fn clockintr frame hwtime +Called once for every clock interrupt on the CPU. +.Pp +The argument +.Fa frame +is fit for use with the macros +.Xr CLKF_USERMODE 9 , +.Xr CLKF_INTR 9 , +and +.Xr CLKF_PC 9 . +.Pp +The argument +.Fa hwtime +gives a readout of the current time in hardware units. +.\"""""""""""""""" +.It Fn __cpu_simple_lock lock +Acquire +.Fa lock , +waiting until it is released if currently held. +.Pp +Any memory operations preceding the previous +.Fn __cpu_simple_unlock +call that released the lock happen before any memory operations after +the next +.Fn __cpu_simple_lock +call that acquires it. +.\"""""""""""""""" +.It Fn __cpu_simple_lock_try lock +Try to acquire +.Fa lock , +without waiting if it is currently held. +Return 1 if successful, 0 if not. +.Pp +Any memory operations preceding the previous +.Fn __cpu_simple_unlock +call that released the lock happen before any memory operations after +the next +.Em successful +.Fn __cpu_simple_lock_try +call that acquires it. +.\"""""""""""""""" +.It Fn __cpu_simple_unlock lock +Release +.Fa lock . +.Pp +Any memory operations preceding +.Fn __cpu_simple_unlock +happen before the next call to +.Fn __cpu_simple_lock , +or the next successful call to +.Fn __cpu_simple_lock_try , +that acquires +.Fa lock . +.\"""""""""""""""" +.It Fn __SIMPLELOCK_LOCKED_P lock +True if +.Fa lock +is currently locked, by anyone. +.Pp +This is normally only used for diagnostic assertions, or for loops +around +.Fn __cpu_simple_lock_try +that also have higher-level functions like blocking interrupts and +performing exponential backoff. +.Pp +No memory ordering is implied. +.El +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh OBSOLETE FUNCTIONS +The following functions abuse the +.Vt __cpu_simple_lock_t +type to store a boolean. +They are used inside the +.Xr pthread 3 +library, and were included in the library ABI, so they can't be removed +without breaking the +.Xr pthread 3 +ABI. +Do not use these in new code +.Po +except +.Fn __SIMPLELOCK_LOCKED_P +.Pc . +.Bl -tag -width ".Fn __SIMPLELOCK_UNLOCKED_P lock" +.It Fn __cpu_simple_lock_set lock +Set +.Fa lock +to true. +.It Fn __cpu_simple_lock_clear lock +Set +.Fa lock +to false. +.It Fn __SIMPLELOCK_LOCKED_P lock +True iff +.Fa lock +is true. +.It Fn __SIMPLELOCK_UNLOCKED_P lock +True iff +.Fa lock +is false. +.El +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh CODE REFERENCES +The +.Nm +functions are implemented in +.Pa sys/arch/$ARCH/include/lock.h . +.Pp +A machine-independent implementation, using compiler support for +atomic and memory barrier builtins, is available in +.Pa sys/sys/common_lock.h . +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh SEE ALSO +.Xr locking 9 , +.Xr mutex 9 +.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.Sh HISTORY +.Nm +appeared a long time ago. diff -r 6fae1301093d -r 830a9bcff034 sys/dev/ic/hpet.c --- a/sys/dev/ic/hpet.c Sun May 25 21:47:13 2025 +0000 +++ b/sys/dev/ic/hpet.c Tue Jul 22 02:45:43 2025 +0000 @@ -92,6 +92,17 @@ hpet_attach_subr(device_t dv) tc->tc_counter_mask = 0xffffffff; + aprint_normal_dev(dv, "info=0x%x\n", + bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_INFO)); + aprint_normal_dev(dv, "period=0x%x\n", + bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_PERIOD)); + aprint_normal_dev(dv, "config=0x%x\n", + bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_CONFIG)); + aprint_normal_dev(dv, "config[32:63]=0x%x\n", + bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_CONFIG + 4)); + aprint_normal_dev(dv, "int_status=0x%x\n", + bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_INT_STATUS)); + /* Get frequency */ sc->sc_period = bus_space_read_4(sc->sc_memt, sc->sc_memh, HPET_PERIOD); if (sc->sc_period == 0 || sc->sc_period > HPET_PERIOD_MAX) { diff -r 6fae1301093d -r 830a9bcff034 sys/kern/files.kern --- a/sys/kern/files.kern Sun May 25 21:47:13 2025 +0000 +++ b/sys/kern/files.kern Tue Jul 22 02:45:43 2025 +0000 @@ -36,6 +36,7 @@ file kern/kern_acct.c kern file kern/kern_auth.c kern file kern/kern_cfglock.c kern file kern/kern_clock.c kern +file kern/kern_clockintr.c kern & clockintr file kern/kern_condvar.c kern file kern/kern_core.c coredump file kern/kern_cpu.c kern diff -r 6fae1301093d -r 830a9bcff034 sys/kern/kern_clockintr.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/kern/kern_clockintr.c Tue Jul 22 02:45:43 2025 +0000 @@ -0,0 +1,439 @@ +/* $NetBSD$ */ + +/*- + * Copyright (c) 2025 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__KERNEL_RCSID(0, "$NetBSD$"); + +#include +#include + +#include +#include +#include +#include +#include /* XXX move to struct cpu_data */ +#include +#include + +/* XXX move me */ +static void +ipl_assert_atleast(int expected) +{ +#if defined __i386__ || defined __x86_64__ + int actual = curcpu()->ci_ilevel; + + KASSERT(kpreempt_disabled()); + KASSERTMSG(actual >= expected, "ipl is %d, expected >=%d", + actual, expected); +#endif +} + +static void sched_clockintr(struct clockframe *); +static void callout_clockintr(struct clockframe *); + +enum clockintr { + CLOCKINTR_HARDCLOCK, + CLOCKINTR_SCHEDCLOCK, + CLOCKINTR_STATCLOCK, + CLOCKINTR_CALLOUT, + NCLOCKINTR +}; + +static void (*const clockintrfn[NCLOCKINTR])(struct clockframe *) = { + [CLOCKINTR_HARDCLOCK] = &hardclock, + [CLOCKINTR_SCHEDCLOCK] = &sched_clockintr, + [CLOCKINTR_STATCLOCK] = &statclock, + [CLOCKINTR_CALLOUT] = &callout_clockintr, +}; + +struct clockintr_cpu { + uint32_t cic_hwepoch; + uint32_t cic_hwnext; + uint32_t cic_scheduled; + uint32_t cic_hwdeadline[NCLOCKINTR]; + uint32_t cic_periodic; + uint32_t cic_hwperiod[NCLOCKINTR]; + bool cic_running; +}; + +static struct percpu *clockintr_percpu; /* struct clockintr_cpu */ + +/* + * clockintr_soonenough(t, epoch) + * + * True if t is less than 2^31 hardware cycles after epoch. + */ +static bool +clockintr_soonenough(uint32_t t, uint32_t epoch) +{ + + return ((t - epoch) <= INT32_MAX); +} + +/* + * clockintr_notafter(t0, t1, epoch) + * + * True if t0 <= t1, relative to epoch. + */ +static bool +clockintr_notafter(uint32_t t0, uint32_t t1, uint32_t epoch) +{ + + KDASSERT(clockintr_soonenough(t0, epoch)); + KDASSERT(clockintr_soonenough(t1, epoch)); + return ((t1 - epoch) - (t0 - epoch) <= INT32_MAX); +} + +/* + * clockintr_before(t0, t1, epoch) + * + * True if t0 < t1, relative to epoch. + */ +static bool +clockintr_before(uint32_t t0, uint32_t t1, uint32_t epoch) +{ + + return (t0 != t1 && clockintr_notafter(t0, t1, epoch)); +} + +/* + * clockintr_min(t0, t1, epoch) + * + * Return the sooner of t0 or t1 after epoch. + */ +static uint32_t +clockintr_min(uint32_t t0, uint32_t t1, uint32_t epoch) +{ + + return (clockintr_notafter(t0, t1, epoch) ? t0 : t1); +} + +/* + * clockintr_compute_next(cic) + * + * Compute the next scheduled deadline. + */ +static uint32_t +clockintr_compute_next(const struct clockintr_cpu *cic) +{ + uint32_t epoch = cic->cic_hwepoch; + uint32_t t; + enum clockintr cintr; + + KASSERT(cic->cic_scheduled & (__BIT(NCLOCKINTR) - 1)); + for (cintr = 0; cintr < NCLOCKINTR; cintr++) { + if (cic->cic_scheduled & __BIT(cintr)) { + t = cic->cic_hwdeadline[cintr]; + for (; cintr < NCLOCKINTR; cintr++) { + t = clockintr_min(t, + cic->cic_hwdeadline[cintr], + epoch); + } + return t; + } + } + panic("clockintr_compute_next screwup"); +} + +/* + * clockintr_schedule_oneshot(cintr, hwtime) + * + * Schedule cintr to fire at hwtime. It may fire early. + */ +static void +clockintr_schedule_oneshot(enum clockintr cintr, uint32_t hwtime) +{ + struct clockintr_cpu *cic; + uint32_t epoch; + int s; + + /* + * Get the per-CPU state at IPL_SCHED to protect access from + * the interrupt handler. + */ + s = splsched(); + cic = percpu_getref(clockintr_percpu); + + /* + * If the clockintr is not currently scheduled, or we are + * scheduling it earlier than it is currently scheduled, update + * its deadline. + * + * If we are scheduling it later than it is currently + * scheduled, do nothing -- it'll fire early and the caller can + * reschedule. + */ + epoch = cic->cic_hwepoch; + if (__predict_false((cic->cic_scheduled & __BIT(cintr)) == 0) || + __predict_true(clockintr_before(hwtime, cic->cic_hwdeadline[cintr], + epoch))) { + cic->cic_scheduled |= __BIT(cintr); + cic->cic_hwdeadline[cintr] = hwtime; + + /* + * If clockintr is currently running, it will pick up + * the updated deadline when it's done. Otherwise, if + * this is sooner than the current next deadline, + * schedule a sooner interrupt. + */ + if (!cic->cic_running && + clockintr_before(hwtime, cic->cic_hwnext, epoch)) { + cic->cic_hwnext = hwtime; + cpu_clockintr_arm(hwtime); + } + } + + percpu_putref(clockintr_percpu); + splx(s); +} + +/* + * clockintr_schedule_periodic(cintr, hwtime, hwperiod) + * + * Schedule cintr to fire at hwtime + k*hwperiod for any k >= 0. + * Must not currently be scheduled. The period must be at most + * 2^31 - 1 cycles. + */ +static void +clockintr_schedule_periodic(enum clockintr cintr, uint32_t hwtime, + uint32_t hwperiod) +{ + struct clockintr_cpu *cic; + int s; + + KASSERT(clockintr_soonenough(hwperiod, 0)); + + /* + * Get the per-CPU state at IPL_SCHED to protect access from + * the interrupt handler. + */ + s = splsched(); + cic = percpu_getref(clockintr_percpu); + + KASSERT((cic->cic_scheduled & __BIT(cintr)) == 0); + + cic->cic_periodic |= __BIT(cintr); + cic->cic_hwperiod[cintr] = hwperiod; + + cic->cic_scheduled |= __BIT(cintr); + cic->cic_hwdeadline[cintr] = hwtime; + + /* + * If clockintr is currently running, it will pick up the + * updated deadline when it's done. Otherwise, if this is + * sooner than the current next deadline, schedule a sooner + * interrupt. + */ + if (!cic->cic_running && + clockintr_before(hwtime, cic->cic_hwnext, cic->cic_hwepoch)) { + cic->cic_hwnext = hwtime; + cpu_clockintr_arm(hwtime); + } + + percpu_putref(clockintr_percpu); + splx(s); +} + +static void +clockintr_nextperiod(struct clockintr_cpu *cic, enum clockintr cintr, + uint32_t now) +{ + uint32_t delta; + + cic->cic_hwdeadline[cintr] += cic->cic_hwperiod[cintr]; + if (__predict_true(clockintr_before(cic->cic_hwdeadline[cintr], now, + cic->cic_hwepoch))) + return; + delta = now - cic->cic_hwdeadline[cintr]; + cic->cic_hwdeadline[cintr] += rounddown(delta, + cic->cic_hwperiod[cintr]); + cic->cic_hwdeadline[cintr] += cic->cic_hwperiod[cintr]; +} + +void +clockintr(struct clockframe *frame, uint32_t now) +{ + struct clockintr_cpu *cic; + uint32_t epoch; + bool handled __diagused; + enum clockintr cintr; + + ipl_assert_atleast(IPL_SCHED); + + cic = percpu_getref(clockintr_percpu); + + /* + * If there are no clockintrs scheduled, this is a spurious + * interrupt -- stop here. May happen when suspended. + */ + if (__predict_false(cic->cic_scheduled == 0)) { + /* XXX record spurious interrupt */ + goto out; + } + + /* + * If we haven't yet passed the next deadline, stop and + * reschedule for the next one. + */ + epoch = cic->cic_hwepoch; + if (__predict_false(clockintr_before(now, cic->cic_hwnext, epoch))) { + /* XXX record early interrupt */ + cpu_clockintr_arm(cic->cic_hwnext); + goto out; + } + + /* + * Call the clockintrfn of each scheduled clockintr that has + * expired. (We could use ffs to iterate over only those that + * are scheduled, but there's only four of them of which at + * least two are generally always scheduled.) + * + * Note that we are running the clockintr functions while we do + * this to skip recomputing the next deadline in + * clockintr_schedule -- we'll do that once we're done + * processing all the expired clockintrs. + */ + handled = false; + for (cintr = 0; cintr < NCLOCKINTR; cintr++) { + if ((cic->cic_scheduled & __BIT(cintr)) != 0 && + clockintr_before(cic->cic_hwdeadline[cintr], now, epoch)) { + if (cic->cic_periodic & __BIT(cintr)) { + clockintr_nextperiod(cic, cintr, now); + } else { + cic->cic_scheduled &= ~__BIT(cintr); + } + (*clockintrfn[cintr])(frame); + handled = true; + } + } + KASSERT(handled); + + /* + * Any updated deadlines had all better be soon enough after + * now. + */ + for (cintr = 0; cintr < NCLOCKINTR; cintr++) { + KASSERT((cic->cic_scheduled & __BIT(cintr)) == 0 || + clockintr_soonenough(cic->cic_hwdeadline[cintr], now)); + } + + /* + * If there is another interrupt to schedule, schedule it. + */ + if (cic->cic_scheduled) { + cic->cic_hwnext = clockintr_compute_next(cic); + KASSERT(clockintr_soonenough(cic->cic_hwnext, now)); + cpu_clockintr_arm(cic->cic_hwnext); + } + + /* + * Advance the epoch. None of the times involved -- the + * scheduled deadlines or the current time -- can precede `now' + * by this point. + */ + cic->cic_hwepoch = now; + +out: percpu_putref(clockintr_percpu); +} + +void +clockintr_schedule_callout(uint32_t hwtime) +{ + + clockintr_schedule_oneshot(CLOCKINTR_CALLOUT, hwtime); +} + +static void +sched_clockintr(struct clockframe *frame) +{ + + schedclock(curcpu()->ci_onproc); +} + +static void +callout_clockintr(struct clockframe *frame) +{ + +#if 0 /* not yet */ + callout_highresclock(); +#endif +} + +void +clockintr_init(void) +{ + struct clockintr_cpu *cic; + int s; + + s = splsched(); + cic = percpu_getref(clockintr_percpu); + + /* + * Schedule the initial clocks. Pretend we're running the + * interrupt handler so we defer arming the next timer, or even + * examining cic->cic_hwnext; then enable the interrupt with + * the first expiry time. + */ + cic->cic_running = true; + cic->cic_hwepoch = cpu_clockintr_now(); + clockintr_schedule_periodic(CLOCKINTR_HARDCLOCK, cic->cic_hwepoch, + cpu_clockintr_ns2hwtime(tick_nsec)); + /* XXX statclock, schedclock */ + KASSERT(cic->cic_running); + KASSERT(cic->cic_scheduled); + cic->cic_hwnext = clockintr_compute_next(cic); + KASSERT(clockintr_soonenough(cic->cic_hwnext, cic->cic_hwepoch)); + cpu_clockintr_enable(cic->cic_hwnext); + + percpu_putref(clockintr_percpu); + splx(s); +} + +void +clockintr_suspend(void) +{ + struct clockintr_cpu *cic; + int s; + + s = splsched(); + cic = percpu_getref(clockintr_percpu); + cpu_clockintr_disable(); + cic->cic_scheduled = 0; + percpu_putref(clockintr_percpu); + splx(s); +} + +void +clockintr_resume(void) +{ + + clockintr_init(); +#if 0 /* not yet */ + callout_highresclock(); +#endif +} diff -r 6fae1301093d -r 830a9bcff034 sys/sys/clockintr.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/sys/clockintr.h Tue Jul 22 02:45:43 2025 +0000 @@ -0,0 +1,68 @@ +/* $NetBSD$ */ + +/*- + * Copyright (c) 2025 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_CLOCKINTR_H_ +#define _SYS_CLOCKINTR_H_ + +#include + +struct clockframe; + +/* + * Machine-dependent code must implement these functions. + */ +uint32_t cpu_clockintr_maxns; +uint32_t cpu_clockintr_ns2hwtime(uint32_t); +uint32_t cpu_clockintr_now(void); +void cpu_clockintr_enable(uint32_t); +void cpu_clockintr_disable(void); +void cpu_clockintr_arm(uint32_t); + +/* + * Machine-dependent timer interrupt handler must call this. + */ +void clockintr(struct clockframe *, uint32_t); + +/* + * Machine-dependent initialization must call this in order to opt into + * the clockintr(9) framework. + */ +void clockintr_init(void); + +/* + * Machine-dependent suspend/resume logic must call these on each CPU. + */ +void clockintr_suspend(void); +void clockintr_resume(void); + +/* + * Called by callout(9) to arm the next callout timer. + */ +void clockintr_schedule_callout(uint32_t); + +#endif /* _SYS_CLOCKINTR_H_ */ # HG changeset patch # User Taylor R Campbell # Date 1753152387 0 # Tue Jul 22 02:46:27 2025 +0000 # Branch trunk # Node ID fe2957b3647dae019f0a43e43fb1674381b8f96f # Parent 830a9bcff0347a232ea1eb252923360426a4d59c # EXP-Topic riastradh-highrestimer WIP: hack hack hack clockevent diff -r 830a9bcff034 -r fe2957b3647d sys/kern/kern_clockevent.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/kern/kern_clockevent.c Tue Jul 22 02:46:27 2025 +0000 @@ -0,0 +1,317 @@ +/* $NetBSD$ */ + +/*- + * Copyright (c) 2025 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__KERNEL_RCSID(0, "$NetBSD$"); + +#include +#include +#include +#include /* XXX move to struct cpu_data */ +#include +#include + +struct clockevent_cpu { + struct rb_tree cec_queue; + uint64_t cec_ticket; + struct bintime cec_last, cec_next; + uint32_t cec_hwlast, cec_hwnext; + struct clockevent cec_hardclock; + struct clockevent cec_profclock; + struct clockevent cec_statclock; + struct evcnt cec_early; +}; + +static int +timespecorder(const struct timespec *a, const struct timespec *b) +{ + + if (a->tv_sec < b->tv_sec) + return -1; + if (a->tv_sec > b->tv_sec) + return +1; + if (a->tv_nsec < b->tv_nsec) + return -1; + if (a->tv_nsec > b->tv_nsec) + return +1; + return 0; +} + +static int +clockevent_compare_nodes(void *cookie, const void *va, const void *vb) +{ + const struct clockevent *a = va, *b = vb; + int order; + + order = timespecorder(&a->clkev_deadline, &b->clkev_deadline); + if (__predict_true(order != 0)) + return order; + if (a->clkev_ticket < b->clkev_ticket) + return -1; + if (a->clkev_ticket > b->clkev_ticket) + return +1; + return 0; +} + +static struct percpu *clockevent_percpu; /* struct clockevent_cpu * */ +static const rb_tree_ops_t clockevent_rb_ops = { + .rbto_compare_nodes = clockevent_compare_nodes, + .rbto_node_offset = offsetof(struct clockevent, clkev_rb_node), +}; + +static void +clockevents_init_cpu(void *ptr, void *cookie, struct cpu_info *ci) +{ + struct clockevent_cpu **cecp = ptr, *cec; + + cec = kmem_zalloc(sizeof(*cec), KM_SLEEP); + rb_tree_init(&cec->cec_queue, &clockevent_rb_ops); + evcnt_attach_dynamic(&cec->cec_early, EVCNT_TYPE_INTR, NULL, + device_xname(ci->ci_dev), "timearly"); + *cecp = cec; +} + +static void +clockevents_fini_cpu(void *ptr, void *cookie, struct cpu_info *ci) +{ + struct clockevent_cpu **cecp = ptr, *cec = *cecp; + + KASSERT(RB_TREE_MIN(&cec->cec_queue) == NULL); +} + +static unsigned nsec_per_tick; /* XXX move me */ + +static void +clockevents_init(void) +{ + + nsec_per_tick = 1000000000/hz; + clockevent_percpu = percpu_create(sizeof(struct clockevent_cpu), + clockevents_init_cpu, clockevents_fini_cpu, NULL); +} + +void +clockevent_init(struct clockevent *clkev, + void (*func)(struct clockevent *, struct clockframe *), + int flags) +{ + + memset(clkev, 0, sizeof(*clkev)); + clkev->clkev_func = func; +} + +#if 0 +void +clockintr_rearm(struct clockevent_cpu *cec, const struct bintime *deadline) +{ + struct bintime now, delta0, delta1, delta; + uint32_t hwdelta0, hwdelta1, hwdelta, hwnext; + + /* + * We have two time scales, and three relevant times plus a + * delta: + * + * last now deadline + * |----+-------+----------+-------> (bintime) + * |----+-------+----------+-------> (hwtime) + * \ / \ / + * delta0 delta1 + * + * We have last as bintime, last as hwtime, and now as bintime, + * but not now as hwtime, and we want delta1 as hwtime. + */ + + binuptime(&now); + delta0 = now; + bintime_sub(&delta0, &cec->cec_last); + delta1 = deadline; + bintime_sub(&delta1, &now); + hwdelta0 = clockintr_bintime2hwtime(&delta0); + hwdelta1 = clockintr_bintime2hwtime(&delta1); + hwnext = cec->cec_hwlast + hwdelta0 + hwdelta1; + + cec->cec_next = deadline; + cec->cec_hwnext = hwnext; + clockintr_arm(hwdelta); +} +#endif + +void +clockevent_schedule(struct clockevent *clkev, const struct timespec *deadline) +{ + struct clockevent_cpu **cecp, *cec; + struct clockevent *collision __diagused; + int s; + + clkev->clkev_deadline = *deadline; + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + s = splsched(); + KASSERTMSG(clkev == &cec->cec_hardclock, /* XXX */ + "[%s] clkev=%p &cec->cec_hardclock=%p", + device_xname(curcpu()->ci_dev), clkev, &cec->cec_hardclock); + clkev->clkev_ticket = cec->cec_ticket++; + collision = rb_tree_insert_node(&cec->cec_queue, clkev); + KASSERT(collision == clkev); + if (timespeccmp(&clkev->clkev_deadline, &cec->cec_next, <) || + __predict_false(!timespecisset(&cec->cec_next))) { + struct timespec now, delta; + + nanouptime(&now); + if (timespeccmp(&clkev->clkev_deadline, &now, <=)) { + timespecclear(&delta); + } else { + timespecsub(&now, &clkev->clkev_deadline, &delta); + } + lapic_schedule(&now, &delta, &clkev->clkev_deadline); + cec->cec_next = clkev->clkev_deadline; + } + splx(s); + percpu_putref(clockevent_percpu); +} + +static void +ipl_assert_atleast(int expected) +{ +#if defined __i386__ || defined __x86_64__ + int actual = curcpu()->ci_ilevel; + + KASSERT(kpreempt_disabled()); + KASSERTMSG(actual >= expected, "ipl is %d, expected >=%d", + actual, expected); +#endif +} + +static void +ipl_assert_exactly(int expected) +{ +#if defined __i386__ || defined __x86_64__ + int actual = curcpu()->ci_ilevel; + + KASSERT(kpreempt_disabled()); + KASSERTMSG(actual == expected, "ipl is %d, expected %d", + actual, expected); +#endif +} + +void +clockevent_intr(struct clockframe *frame) +{ + struct cpu_info *ci = curcpu(); + struct clockevent_cpu **cecp, *cec; + struct clockevent *clkev; + struct timespec now, delta; + + ipl_assert_exactly(IPL_SCHED); + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + + nanouptime(&now); + /* XXX DTRACE_PROBE: record tardiness relative to cec->cec_next */ + if (__predict_false(timespeccmp(&now, &cec->cec_next, <))) { + timespecsub(&cec->cec_next, &now, &delta); + /* XXX DTRACE_PROBE */ + cec->cec_early.ev_count++; + if ((clkev = RB_TREE_MIN(&cec->cec_queue)) == NULL) + goto out; + KASSERT(timespeccmp(&clkev->clkev_deadline, &cec->cec_next, + ==)); + goto reschedule; + } +restart: + while ((clkev = RB_TREE_MIN(&cec->cec_queue)) != NULL) { + KASSERTMSG(clkev == &cec->cec_hardclock, /* XXX */ + "[%s] clkev=%p &cec->cec_hardclock=%p", + device_xname(ci->ci_dev), clkev, &cec->cec_hardclock); + if (timespeccmp(&now, &clkev->clkev_deadline, <)) + goto reschedule; + rb_tree_remove_node(&cec->cec_queue, clkev); + (*clkev->clkev_func)(clkev, frame); + ipl_assert_atleast(IPL_SCHED); + KASSERT(ci == curcpu()); + } + device_printf(ci->ci_dev, "WARNING 2: timer queue empty\n"); + goto out; + +reschedule: + /* + * Re-read the clock so that for relative wakeups, we don't + * drift by the time spent processing the queue. + */ + nanouptime(&now); + if (__predict_false(timespeccmp(&now, &clkev->clkev_deadline, >=))) + goto restart; + + timespecsub(&clkev->clkev_deadline, &now, &delta); + lapic_schedule(&now, &delta, &clkev->clkev_deadline); + cec->cec_next = clkev->clkev_deadline; + +out: percpu_putref(clockevent_percpu); +} + +static void +hardclock_intr(struct clockevent *clkev, struct clockframe *frame) +{ + struct clockevent_cpu **cecp, *cec; + struct timespec delta, next; + + ipl_assert_atleast(IPL_SCHED); + + delta.tv_sec = 0; + delta.tv_nsec = nsec_per_tick; + timespecadd(&clkev->clkev_deadline, &delta, &next); + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + KASSERTMSG(clkev == &cec->cec_hardclock, + "clkev=%p &cec->cec_hardclock=%p", + clkev, &cec->cec_hardclock); + clockevent_schedule(&cec->cec_hardclock, &next); + percpu_putref(clockevent_percpu); + + hardclock(frame); +} + +void +hardstatprofclock_init(void) +{ + struct clockevent_cpu **cecp, *cec; + struct timespec now; + + cecp = percpu_getref(clockevent_percpu); + cec = *cecp; + device_printf(curcpu()->ci_dev, "initialize hardclock %p\n", + &cec->cec_hardclock); + clockevent_init(&cec->cec_hardclock, &hardclock_intr, + CLOCKEVENT_BOUND); + nanouptime(&now); + clockevent_schedule(&cec->cec_hardclock, &now); + percpu_putref(clockevent_percpu); +} diff -r 830a9bcff034 -r fe2957b3647d sys/sys/clockevent.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/sys/clockevent.h Tue Jul 22 02:46:27 2025 +0000 @@ -0,0 +1,59 @@ +/* $NetBSD$ */ + +/*- + * Copyright (c) 2025 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_CLOCKEVENT_H_ +#define _SYS_CLOCKEVENT_H_ + +#include +#include +#include + +struct clockframe; +struct clockevent_cpu; + +struct clockevent { /* XXX opaque */ + void (*clkev_func)(struct clockevent *, + struct clockframe *); + struct clockevent_cpu *volatile clkev_cpu; + struct rb_node clkev_rb_node; + struct timespec clkev_deadline; + uint64_t clkev_ticket; + uint32_t clkev_hwtime; + int clkev_flags; +}; + +#define CLOCKEVENT_BOUND __BIT(0) + +void clockevent_init(struct clockevent *, + void (*)(struct clockevent *, struct clockframe *), + int); +void clockevent_schedule(struct clockevent *, const struct timespec *); +void clockevent_cancel(struct clockevent *); +void clockevent_destroy(struct clockevent *); + +#endif /* _SYS_CLOCKEVENT_H_ */