Index: kern/kern_lwp.c =================================================================== RCS file: /cvsroot/src/sys/kern/kern_lwp.c,v retrieving revision 1.83.2.4 diff -u -r1.83.2.4 kern_lwp.c --- kern/kern_lwp.c 26 Dec 2007 23:05:53 -0000 1.83.2.4 +++ kern/kern_lwp.c 2 Jan 2008 09:24:59 -0000 @@ -228,6 +228,7 @@ #include #include #include +#include #include #include @@ -663,6 +664,11 @@ membar_exit(); prev->l_ctxswtch = 0; } + + /* Note trip through cpu_switchto(). */ + xc_sync_point(); + + /* Set up the new LWP's execution environment. */ spl0(); pmap_activate(new); LOCKDEBUG_BARRIER(NULL, 0); Index: kern/kern_synch.c =================================================================== RCS file: /cvsroot/src/sys/kern/kern_synch.c,v retrieving revision 1.211.2.4 diff -u -r1.211.2.4 kern_synch.c --- kern/kern_synch.c 28 Dec 2007 21:40:48 -0000 1.211.2.4 +++ kern/kern_synch.c 2 Jan 2008 09:24:59 -0000 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_synch.c,v 1.211.2.4 2007/12/28 21:40:48 ad Exp $ */ + /*- * Copyright (c) 1999, 2000, 2004, 2006, 2007 The NetBSD Foundation, Inc. @@ -101,6 +101,7 @@ #include #include #include +#include #include @@ -562,6 +563,9 @@ l->l_lwpctl->lc_curcpu = (int)cpu_index(ci); retval = 1; + + /* Note trip through cpu_switchto(). */ + xc_sync_point(); } else { /* Nothing to do - just unlock and return. */ mutex_spin_exit(spc->spc_mutex); Index: kern/subr_xcall.c =================================================================== RCS file: /cvsroot/src/sys/kern/subr_xcall.c,v retrieving revision 1.5 diff -u -r1.5 subr_xcall.c --- kern/subr_xcall.c 6 Nov 2007 00:42:44 -0000 1.5 +++ kern/subr_xcall.c 2 Jan 2008 09:25:00 -0000 @@ -106,6 +106,9 @@ static uint64_t xc_tailp; static uint64_t xc_donep; +static kmutex_t xc_sync_lock; +static TAILQ_HEAD(,xcsync) xc_sync_queue; + /* * xc_init_cpu: * @@ -122,6 +125,7 @@ /* Autoconfiguration will prevent re-entry. */ again = true; mutex_init(&xc_lock, MUTEX_DEFAULT, IPL_NONE); + mutex_init(&xc_lock, MUTEX_DEFAULT, IPL_SCHED); cv_init(&xc_busy, "xcallbsy"); evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, "crosscall", "unicast"); @@ -137,7 +141,7 @@ } /* - * xc_unicast: + * xc_broadcast: * * Trigger a call on all CPUs in the system. */ @@ -262,3 +266,135 @@ } /* NOTREACHED */ } + +/* + * xc_sync_init: + * + * Initialize a passive sync object. + */ +void +xc_sync_init(xcsync_t *xs) +{ + + xs->xs_owner = NULL; + xs->xs_target = 0; + cv_init(&xs->xs_notifier, "xcsync"); +} + +/* + * xc_sync_destroy: + * + * Destroy a passive sync object. + */ +void +xc_sync_destroy(xcsync_t *xs) +{ + + KASSERT(xs->xs_owner == NULL); + KASSERT(xs->xs_target == 0); + + cv_destroy(&xs->xs_notifier); +} + +/* + * xc_sync: + * + * Perform the write side of passive synchronization. The calling + * thread holds an exclusive lock on the data object(s) being updated. + * We wait until every processor in the system has made at least two + * passes through cpu_swichto(). The wait is made with the caller's + * update lock held, but is short term. + */ +void +xc_sync(xcsync_t *xs) +{ + + if (panicstr != NULL) + return; + + KASSERT(xs->xs_owner == NULL); + KASSERT(xs->xs_target == 0); + KASSERT(ncpu > 0); + + /* + * Set up the object and put it onto the queue. The lock + * activity here provides the necessary memory barrier to + * make the caller's data update completly visible to + * other processors. + */ + xs->xs_owner = curlwp; + xs->xs_target = (1 << ncpu) - 1; + xs->xs_pass1 = 0; + xs->xs_pass2 = 0; + mutex_spin_enter(&xc_sync_lock); + TAILQ_INSERT_TAIL(&xc_sync_queue, xs, xs_chain); + mutex_spin_exit(&xc_sync_lock); + + /* + * Force some context switch activity on every CPU, as + * the system may not be busy. + */ + xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL); + + /* + * Wait for all CPUs to cycle through cpu_switchto() twice. + * The last one through will remove our update from the + * queue and awaken us. + */ + mutex_spin_enter(&xc_sync_lock); + while (xs->xs_target != 0) { + cv_wait(&xs->xs_notifier, &xc_sync_lock); + } + mutex_spin_exit(&xc_sync_lock); + + xs->xs_owner = NULL; +} + +/* + * xc_sync_point: + * + * Monitor system context switch activity. Called from machine + * independent code after cpu_switchto() returns. + */ +#if MAXCPUS > 32 +#error fixme +#endif + +void +xc_sync_point(void) +{ + xcsync_t *xs, *next; + u_int mask; + + /* + * If no updates pending, bail out. We don't need to lock in + * order to test xc_sync_queue; the only ill effect of missing + * an update would be to delay LWPs waiting in xc_sync(). That + * won't happen because updates are on the queue before an xcall + * is generated to tickle every CPU. + */ + if (TAILQ_EMPTY(&xc_sync_queue) || panicstr != NULL) + return; + + /* + * Scan through the queue and update each request. If an entry + * has been seen twice on every processor, remove from the queue + * and notify the updating thread. + */ + mask = (1 << curcpu()->ci_index); + mutex_spin_enter(&xc_sync_lock); + for (xs = TAILQ_FIRST(&xc_sync_queue); xs != NULL; xs = next) { + next = TAILQ_NEXT(xs, xs_chain); + if ((xs->xs_pass1 & xs->xs_target) != xs->xs_target) { + xs->xs_pass1 |= mask; + continue; + } + xs->xs_pass2 |= mask; + if ((xs->xs_pass2 & xs->xs_target) == xs->xs_target) { + TAILQ_REMOVE(&xc_sync_queue, xs, xs_chain); + cv_signal(&xs->xs_notifier); + xs->xs_target = 0; + } + } + mutex_spin_exit(&xc_sync_lock); +} Index: sys/xcall.h =================================================================== RCS file: /cvsroot/src/sys/sys/xcall.h,v retrieving revision 1.2 diff -u -r1.2 xcall.h --- sys/xcall.h 8 Oct 2007 15:12:11 -0000 1.2 +++ sys/xcall.h 2 Jan 2008 09:25:03 -0000 @@ -1,7 +1,7 @@ /* $NetBSD: xcall.h,v 1.2 2007/10/08 15:12:11 ad Exp $ */ /*- - * Copyright (c) 2007 The NetBSD Foundation, Inc. + * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -44,13 +44,31 @@ typedef void (*xcfunc_t)(void *, void *); +typedef struct xcsync { + TAILQ_ENTRY(xcsync) xs_chain; + struct lwp *xs_owner; + kcondvar_t xs_notifier; + u_int xs_target; + u_int xs_pass1; + u_int xs_pass2; +} xcsync_t; + struct cpu_info; +/* Hooks into the kernel. */ void xc_init_cpu(struct cpu_info *); +void xc_sync_point(void); + +/* Cross-calls. */ uint64_t xc_broadcast(u_int, xcfunc_t, void *, void *); uint64_t xc_unicast(u_int, xcfunc_t, void *, void *, struct cpu_info *); void xc_wait(uint64_t); +/* Cross-call based synchronization. */ +void xc_sync_init(xcsync_t *); +void xc_sync_destroy(xcsync_t *); +void xc_sync(xcsync_t *); + #endif /* _KERNEL */ #endif /* _SYS_XCALL_H_ */