diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index 5f4d0a4183a..226e330cb27 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -43,6 +43,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.159 2017/09/16 23:55:33 christos Exp #include #include #include +#include #include @@ -88,6 +89,9 @@ assert_sleepable(void) if (cpu_softintr_p()) { reason = "softint"; } + if (!pserialize_not_in_read_section()) { + reason = "pserialize"; + } if (reason) { panic("%s: %s caller=%p", __func__, reason, diff --git a/sys/kern/subr_pserialize.c b/sys/kern/subr_pserialize.c index 650f81f3092..f4e94842cf0 100644 --- a/sys/kern/subr_pserialize.c +++ b/sys/kern/subr_pserialize.c @@ -73,6 +73,12 @@ static TAILQ_HEAD(, pserialize) psz_queue0 __cacheline_aligned; static TAILQ_HEAD(, pserialize) psz_queue1 __cacheline_aligned; static TAILQ_HEAD(, pserialize) psz_queue2 __cacheline_aligned; +#ifdef LOCKDEBUG +#include + +static percpu_t *psz_debug_nreads __cacheline_aligned; +#endif + /* * pserialize_init: * @@ -89,6 +95,9 @@ pserialize_init(void) mutex_init(&psz_lock, MUTEX_DEFAULT, IPL_SCHED); evcnt_attach_dynamic(&psz_ev_excl, EVCNT_TYPE_MISC, NULL, "pserialize", "exclusive access"); +#ifdef LOCKDEBUG + psz_debug_nreads = percpu_alloc(sizeof(int)); +#endif } /* @@ -185,15 +194,35 @@ pserialize_perform(pserialize_t psz) int pserialize_read_enter(void) { + int s; KASSERT(!cpu_intr_p()); - return splsoftserial(); + s = splsoftserial(); +#ifdef LOCKDEBUG + { + int *nreads; + nreads = percpu_getref(psz_debug_nreads); + (*nreads)++; + percpu_putref(psz_debug_nreads); + } +#endif + return s; } void pserialize_read_exit(int s) { +#ifdef LOCKDEBUG + { + int *nreads; + nreads = percpu_getref(psz_debug_nreads); + (*nreads)--; + if (*nreads < 0) + panic("nreads went negative: %d", *nreads); + percpu_putref(psz_debug_nreads); + } +#endif splx(s); } @@ -209,6 +238,9 @@ pserialize_switchpoint(void) pserialize_t psz, next; cpuid_t cid; + /* We must to ensure not to come here from inside a read section. */ + KASSERT(pserialize_not_in_read_section()); + /* * If no updates pending, bail out. No need to lock in order to * test psz_work_todo; the only ill effect of missing an update @@ -261,3 +293,61 @@ pserialize_switchpoint(void) } mutex_spin_exit(&psz_lock); } + +/* + * pserialize_in_read_section: + * + * True if the caller is in a pserialize read section. To be used only + * for diagnostic assertions where we want to guarantee the condition like: + * + * KASSERT(pserialize_in_read_section()); + */ +bool +pserialize_in_read_section(void) +{ +#ifdef LOCKDEBUG + int *nreads; + bool in; + + /* Not initialized yet */ + if (__predict_false(psz_debug_nreads == NULL)) + return true; + + nreads = percpu_getref(psz_debug_nreads); + in = *nreads != 0; + percpu_putref(psz_debug_nreads); + + return in; +#else + return true; +#endif +} + +/* + * pserialize_not_in_read_section: + * + * True if the caller is not in a pserialize read section. To be used only + * for diagnostic assertions where we want to guarantee the condition like: + * + * KASSERT(pserialize_not_in_read_section()); + */ +bool +pserialize_not_in_read_section(void) +{ +#ifdef LOCKDEBUG + int *nreads; + bool notin; + + /* Not initialized yet */ + if (__predict_false(psz_debug_nreads == NULL)) + return true; + + nreads = percpu_getref(psz_debug_nreads); + notin = *nreads == 0; + percpu_putref(psz_debug_nreads); + + return notin; +#else + return true; +#endif +} diff --git a/sys/rump/librump/rumpkern/emul.c b/sys/rump/librump/rumpkern/emul.c index a8ecd3886cd..60c8bf14082 100644 --- a/sys/rump/librump/rumpkern/emul.c +++ b/sys/rump/librump/rumpkern/emul.c @@ -165,11 +165,23 @@ calc_cache_size(vsize_t vasz, int pct, int va_pct) return t; } +#define RETURN_ADDRESS (uintptr_t)__builtin_return_address(0) + void assert_sleepable(void) { + const char *reason = NULL; /* always sleepable, although we should improve this */ + + if (!pserialize_not_in_read_section()) { + reason = "pserialize"; + } + + if (reason) { + panic("%s: %s caller=%p", __func__, reason, + (void *)RETURN_ADDRESS); + } } void diff --git a/sys/rump/librump/rumpkern/rump.c b/sys/rump/librump/rumpkern/rump.c index fe85ae888f7..7719f7abb1e 100644 --- a/sys/rump/librump/rumpkern/rump.c +++ b/sys/rump/librump/rumpkern/rump.c @@ -301,6 +301,7 @@ rump_init(void) callout_startup(); kprintf_init(); + percpu_init(); pserialize_init(); kauth_init(); @@ -351,7 +352,6 @@ rump_init(void) rump_schedule(); bootlwp = curlwp; - percpu_init(); inittimecounter(); ntp_init(); diff --git a/sys/sys/pserialize.h b/sys/sys/pserialize.h index a1b36515e52..53b512834f6 100644 --- a/sys/sys/pserialize.h +++ b/sys/sys/pserialize.h @@ -44,6 +44,8 @@ void pserialize_perform(pserialize_t); int pserialize_read_enter(void); void pserialize_read_exit(int); +bool pserialize_in_read_section(void); +bool pserialize_not_in_read_section(void); #endif /* _KERNEL */ #endif /* _SYS_PSERIALIZE_H_ */