From 318ceb94af25b053441e5c11788e6f497cbefb09 Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Thu, 24 Feb 2022 15:49:40 +0000
Subject: [PATCH] sys: Membar audit around reference count releases.

---
 sys/dev/hyperv/vmbus.c                         |  2 ++
 sys/dev/scsipi/atapiconf.c                     |  5 ++++-
 sys/dev/scsipi/scsipi_base.c                   |  2 ++
 .../bsd/drm2/linux/linux_stop_machine.c        |  7 ++++---
 sys/kern/kern_auth.c                           |  2 ++
 sys/kern/kern_mutex_obj.c                      |  2 ++
 sys/kern/kern_rwlock_obj.c                     |  2 ++
 sys/kern/kern_sig.c                            |  2 ++
 sys/kern/subr_kcpuset.c                        |  2 ++
 sys/kern/uipc_mbuf.c                           |  2 ++
 sys/kern/vfs_cwd.c                             |  2 ++
 sys/kern/vfs_mount.c                           |  2 ++
 sys/kern/vfs_vnode.c                           | 18 +++++++++++++-----
 sys/kern/vfs_wapbl.c                           |  2 ++
 sys/net/if.c                                   |  8 +++++---
 sys/net/npf/npf_nat.c                          |  2 ++
 sys/net/npf/npf_rproc.c                        |  2 ++
 sys/net/npf/npf_tableset.c                     | 10 +++++++---
 sys/uvm/pmap/pmap.c                            |  2 ++
 sys/uvm/uvm_aobj.c                             |  2 ++
 sys/uvm/uvm_map.c                              |  3 +++
 21 files changed, 66 insertions(+), 15 deletions(-)

diff --git a/sys/dev/hyperv/vmbus.c b/sys/dev/hyperv/vmbus.c
index 6c7e79131ed1..a5dfe14efe46 100644
--- a/sys/dev/hyperv/vmbus.c
+++ b/sys/dev/hyperv/vmbus.c
@@ -1452,8 +1452,10 @@ vmbus_channel_detach(struct vmbus_channel *ch)
 	KASSERTMSG(ch->ch_refs > 0, "channel%u: invalid refcnt %d",
 	    ch->ch_id, ch->ch_refs);
 
+	membar_exit();
 	refs = atomic_dec_uint_nv(&ch->ch_refs);
 	if (refs == 0) {
+		membar_enter();
 		/* Detach the target channel. */
 		vmbus_devq_enqueue(ch->ch_sc, VMBUS_DEV_TYPE_DETACH, ch);
 	}
diff --git a/sys/dev/scsipi/atapiconf.c b/sys/dev/scsipi/atapiconf.c
index 80ce1996016d..bd9e784a130b 100644
--- a/sys/dev/scsipi/atapiconf.c
+++ b/sys/dev/scsipi/atapiconf.c
@@ -219,8 +219,11 @@ atapibusdetach(device_t self, int flags)
 	cv_destroy(&chan->chan_cv_comp);
 	cv_destroy(&chan->chan_cv_thr);
 
-	if (atomic_dec_uint_nv(&chan_running(chan)) == 0)
+	membar_exit();
+	if (atomic_dec_uint_nv(&chan_running(chan)) == 0) {
+		membar_enter();
 		mutex_destroy(chan_mtx(chan));
+	}
 
 	return 0;
 }
diff --git a/sys/dev/scsipi/scsipi_base.c b/sys/dev/scsipi/scsipi_base.c
index 423f97a09050..654cd693929f 100644
--- a/sys/dev/scsipi/scsipi_base.c
+++ b/sys/dev/scsipi/scsipi_base.c
@@ -2733,8 +2733,10 @@ void
 scsipi_adapter_delref(struct scsipi_adapter *adapt)
 {
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&adapt->adapt_refcnt) == 0
 	    && adapt->adapt_enable != NULL) {
+		membar_enter();
 		scsipi_adapter_lock(adapt);
 		(void) scsipi_adapter_enable(adapt, 0);
 		scsipi_adapter_unlock(adapt);
diff --git a/sys/external/bsd/drm2/linux/linux_stop_machine.c b/sys/external/bsd/drm2/linux/linux_stop_machine.c
index 532f10898cee..178a43ce5dee 100644
--- a/sys/external/bsd/drm2/linux/linux_stop_machine.c
+++ b/sys/external/bsd/drm2/linux/linux_stop_machine.c
@@ -59,18 +59,19 @@ stop_machine_xcall(void *a, void *b)
 	s = splhigh();
 
 	/* Note that we're ready, and see whether we're the chosen one.  */
+	membar_exit();
 	if (atomic_dec_uint_nv(&S->ncpu) != 0) {
-		/* Not the chosen one.  Wait until done.  */
-		while (!S->done)
+		while (!atomic_load_acquire(&S->done))
 			SPINLOCK_BACKOFF_HOOK;
 		goto out;
 	}
+	membar_enter();
 
 	/* It's time.  Call the callback.  */
 	S->callback(S->cookie);
 
 	/* Notify everyone else that we're done.  */
-	S->done = true;
+	atomic_store_release(&S->done, true);
 
 	/* Allow activity again.  */
 out:	splx(s);
diff --git a/sys/kern/kern_auth.c b/sys/kern/kern_auth.c
index 0b49deb3b872..aefa9209a03c 100644
--- a/sys/kern/kern_auth.c
+++ b/sys/kern/kern_auth.c
@@ -144,8 +144,10 @@ kauth_cred_free(kauth_cred_t cred)
 	KASSERT(cred->cr_refcnt > 0);
 	ASSERT_SLEEPABLE();
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&cred->cr_refcnt) > 0)
 		return;
+	membar_enter();
 
 	kauth_cred_hook(cred, KAUTH_CRED_FREE, NULL, NULL);
 	specificdata_fini(kauth_domain, &cred->cr_sd);
diff --git a/sys/kern/kern_mutex_obj.c b/sys/kern/kern_mutex_obj.c
index b11cf2c95414..95bab126edf2 100644
--- a/sys/kern/kern_mutex_obj.c
+++ b/sys/kern/kern_mutex_obj.c
@@ -157,9 +157,11 @@ mutex_obj_free(kmutex_t *lock)
 	    "%s: lock %p: mo->mo_refcnt (%#x) == 0",
 	     __func__, mo, mo->mo_refcnt);
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&mo->mo_refcnt) > 0) {
 		return false;
 	}
+	membar_enter();
 	mutex_destroy(&mo->mo_lock);
 	pool_cache_put(mutex_obj_cache, mo);
 	return true;
diff --git a/sys/kern/kern_rwlock_obj.c b/sys/kern/kern_rwlock_obj.c
index 9a26023585a3..4d475aadf17c 100644
--- a/sys/kern/kern_rwlock_obj.c
+++ b/sys/kern/kern_rwlock_obj.c
@@ -147,9 +147,11 @@ rw_obj_free(krwlock_t *lock)
 	KASSERT(ro->ro_magic == RW_OBJ_MAGIC);
 	KASSERT(ro->ro_refcnt > 0);
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&ro->ro_refcnt) > 0) {
 		return false;
 	}
+	membar_enter();
 	rw_destroy(&ro->ro_lock);
 	pool_cache_put(rw_obj_cache, ro);
 	return true;
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 133347805952..3cde6a99330b 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -311,7 +311,9 @@ void
 sigactsfree(struct sigacts *ps)
 {
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&ps->sa_refcnt) == 0) {
+		membar_enter();
 		mutex_destroy(&ps->sa_mutex);
 		pool_cache_put(sigacts_cache, ps);
 	}
diff --git a/sys/kern/subr_kcpuset.c b/sys/kern/subr_kcpuset.c
index 995313578c0f..b3aaba2e2f80 100644
--- a/sys/kern/subr_kcpuset.c
+++ b/sys/kern/subr_kcpuset.c
@@ -262,9 +262,11 @@ kcpuset_unuse(kcpuset_t *kcp, kcpuset_t **lst)
 	KASSERT(kc_initialised);
 	KASSERT(kc->kc_refcnt > 0);
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&kc->kc_refcnt) != 0) {
 		return;
 	}
+	membar_enter();
 	KASSERT(kc->kc_next == NULL);
 	if (lst == NULL) {
 		kcpuset_destroy(kcp);
diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index 836aa621c43b..f01cf59120bd 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -1914,6 +1914,7 @@ m_ext_free(struct mbuf *m)
 	if (__predict_true(m->m_ext.ext_refcnt == 1)) {
 		refcnt = m->m_ext.ext_refcnt = 0;
 	} else {
+		membar_exit();
 		refcnt = atomic_dec_uint_nv(&m->m_ext.ext_refcnt);
 	}
 
@@ -1930,6 +1931,7 @@ m_ext_free(struct mbuf *m)
 		/*
 		 * dropping the last reference
 		 */
+		membar_enter();
 		if (!embedded) {
 			m->m_ext.ext_refcnt++; /* XXX */
 			m_ext_free(m->m_ext_ref);
diff --git a/sys/kern/vfs_cwd.c b/sys/kern/vfs_cwd.c
index 390f03ba4229..58496dcc98d3 100644
--- a/sys/kern/vfs_cwd.c
+++ b/sys/kern/vfs_cwd.c
@@ -138,8 +138,10 @@ void
 cwdfree(struct cwdinfo *cwdi)
 {
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&cwdi->cwdi_refcnt) > 0)
 		return;
+	membar_enter();
 
 	vrele(cwdi->cwdi_cdir);
 	if (cwdi->cwdi_rdir)
diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index 7ee616d58a81..8bcfab64f930 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -283,9 +283,11 @@ void
 vfs_rele(struct mount *mp)
 {
 
+	membar_exit();
 	if (__predict_true((int)atomic_dec_uint_nv(&mp->mnt_refcnt) > 0)) {
 		return;
 	}
+	membar_enter();
 
 	/*
 	 * Nothing else has visibility of the mount: we can now
diff --git a/sys/kern/vfs_vnode.c b/sys/kern/vfs_vnode.c
index e959da7aa0c9..ba96beb50140 100644
--- a/sys/kern/vfs_vnode.c
+++ b/sys/kern/vfs_vnode.c
@@ -352,11 +352,13 @@ vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to,
 		vnpanic(vp, "state is %s, gate %d does not match at %s:%d\n",
 		    vstate_name(vip->vi_state), gated, func, line);
 
-	/* Open/close the gate for vcache_tryvget(). */	
-	if (to == VS_LOADED)
+	/* Open/close the gate for vcache_tryvget(). */
+	if (to == VS_LOADED) {
+		membar_exit();
 		atomic_or_uint(&vp->v_usecount, VUSECOUNT_GATE);
-	else
+	} else {
 		atomic_and_uint(&vp->v_usecount, ~VUSECOUNT_GATE);
+	}
 
 	vip->vi_state = to;
 	if (from == VS_LOADING)
@@ -395,10 +397,12 @@ vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to)
 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
 
 	/* Open/close the gate for vcache_tryvget(). */	
-	if (to == VS_LOADED)
+	if (to == VS_LOADED) {
+		membar_exit();
 		atomic_or_uint(&vp->v_usecount, VUSECOUNT_GATE);
-	else
+	} else {
 		atomic_and_uint(&vp->v_usecount, ~VUSECOUNT_GATE);
+	}
 
 	vip->vi_state = to;
 	if (from == VS_LOADING)
@@ -727,6 +731,7 @@ vtryrele(vnode_t *vp)
 {
 	u_int use, next;
 
+	membar_exit();
 	for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) {
 		if (__predict_false((use & VUSECOUNT_MASK) == 1)) {
 			return false;
@@ -827,6 +832,7 @@ retry:
 			break;
 		}
 	}
+	membar_enter();
 	if (vrefcnt(vp) <= 0 || vp->v_writecount != 0) {
 		vnpanic(vp, "%s: bad ref count", __func__);
 	}
@@ -990,6 +996,7 @@ out:
 			break;
 		}
 	}
+	membar_enter();
 
 	if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
 		/*
@@ -1460,6 +1467,7 @@ vcache_tryvget(vnode_t *vp)
 		next = atomic_cas_uint(&vp->v_usecount,
 		    use, (use + 1) | VUSECOUNT_VGET);
 		if (__predict_true(next == use)) {
+			membar_enter();
 			return 0;
 		}
 	}
diff --git a/sys/kern/vfs_wapbl.c b/sys/kern/vfs_wapbl.c
index 082ea2f802ed..b7723b497c67 100644
--- a/sys/kern/vfs_wapbl.c
+++ b/sys/kern/vfs_wapbl.c
@@ -2273,7 +2273,9 @@ wapbl_inodetrk_free(struct wapbl *wl)
 	/* XXX this KASSERT needs locking/mutex analysis */
 	KASSERT(wl->wl_inohashcnt == 0);
 	hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask);
+	membar_exit();
 	if (atomic_dec_uint_nv(&wapbl_ino_pool_refcount) == 0) {
+		membar_enter();
 		pool_destroy(&wapbl_ino_pool);
 	}
 }
diff --git a/sys/net/if.c b/sys/net/if.c
index a8cf207ffcb1..42639ed99132 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1819,9 +1819,11 @@ ifafree(struct ifaddr *ifa)
 	KASSERT(ifa != NULL);
 	KASSERTMSG(ifa->ifa_refcnt > 0, "ifa_refcnt=%d", ifa->ifa_refcnt);
 
-	if (atomic_dec_uint_nv(&ifa->ifa_refcnt) == 0) {
-		free(ifa, M_IFADDR);
-	}
+	membar_exit();
+	if (atomic_dec_uint_nv(&ifa->ifa_refcnt) != 0)
+		return;
+	membar_enter();
+	free(ifa, M_IFADDR);
 }
 
 bool
diff --git a/sys/net/npf/npf_nat.c b/sys/net/npf/npf_nat.c
index d29ceb75b841..47d52a3549a0 100644
--- a/sys/net/npf/npf_nat.c
+++ b/sys/net/npf/npf_nat.c
@@ -279,9 +279,11 @@ npf_natpolicy_release(npf_natpolicy_t *np)
 {
 	KASSERT(atomic_load_relaxed(&np->n_refcnt) > 0);
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&np->n_refcnt) != 0) {
 		return;
 	}
+	membar_enter();
 	KASSERT(LIST_EMPTY(&np->n_nat_list));
 	mutex_destroy(&np->n_lock);
 	kmem_free(np, sizeof(npf_natpolicy_t));
diff --git a/sys/net/npf/npf_rproc.c b/sys/net/npf/npf_rproc.c
index e72c4522662b..928290c78ec1 100644
--- a/sys/net/npf/npf_rproc.c
+++ b/sys/net/npf/npf_rproc.c
@@ -330,9 +330,11 @@ npf_rproc_release(npf_rproc_t *rp)
 {
 	KASSERT(atomic_load_relaxed(&rp->rp_refcnt) > 0);
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&rp->rp_refcnt) != 0) {
 		return;
 	}
+	membar_enter();
 	/* XXXintr */
 	for (unsigned i = 0; i < rp->rp_ext_count; i++) {
 		npf_ext_t *ext = rp->rp_ext[i];
diff --git a/sys/net/npf/npf_tableset.c b/sys/net/npf/npf_tableset.c
index 5b05c2bbfb64..e034b83ef87f 100644
--- a/sys/net/npf/npf_tableset.c
+++ b/sys/net/npf/npf_tableset.c
@@ -158,9 +158,13 @@ npf_tableset_destroy(npf_tableset_t *ts)
 	for (u_int tid = 0; tid < ts->ts_nitems; tid++) {
 		npf_table_t *t = ts->ts_map[tid];
 
-		if (t && atomic_dec_uint_nv(&t->t_refcnt) == 0) {
-			npf_table_destroy(t);
-		}
+		if (t == NULL)
+			continue;
+		membar_exit();
+		if (atomic_dec_uint_nv(&t->t_refcnt) > 0)
+			continue;
+		membar_enter();
+		npf_table_destroy(t);
 	}
 	kmem_free(ts, NPF_TABLESET_SIZE(ts->ts_nitems));
 }
diff --git a/sys/uvm/pmap/pmap.c b/sys/uvm/pmap/pmap.c
index 8009afd67919..af0050b0079c 100644
--- a/sys/uvm/pmap/pmap.c
+++ b/sys/uvm/pmap/pmap.c
@@ -690,11 +690,13 @@ pmap_destroy(pmap_t pmap)
 	UVMHIST_FUNC(__func__);
 	UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0);
 
+	membar_exit();
 	if (atomic_dec_uint_nv(&pmap->pm_count) > 0) {
 		PMAP_COUNT(dereference);
 		UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0);
 		return;
 	}
+	membar_enter();
 
 	PMAP_COUNT(destroy);
 	KASSERT(pmap->pm_count == 0);
diff --git a/sys/uvm/uvm_aobj.c b/sys/uvm/uvm_aobj.c
index 5f8759b99987..90808f50e010 100644
--- a/sys/uvm/uvm_aobj.c
+++ b/sys/uvm/uvm_aobj.c
@@ -604,10 +604,12 @@ uao_detach(struct uvm_object *uobj)
 	KASSERT(uobj->uo_refs > 0);
 	UVMHIST_LOG(maphist,"  (uobj=%#jx)  ref=%jd",
 	    (uintptr_t)uobj, uobj->uo_refs, 0, 0);
+	membar_exit();
 	if (atomic_dec_uint_nv(&uobj->uo_refs) > 0) {
 		UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
 		return;
 	}
+	membar_enter();
 
 	/*
 	 * Remove the aobj from the global list.
diff --git a/sys/uvm/uvm_map.c b/sys/uvm/uvm_map.c
index 008d390a1f84..79161589261f 100644
--- a/sys/uvm/uvm_map.c
+++ b/sys/uvm/uvm_map.c
@@ -4234,8 +4234,11 @@ uvmspace_free(struct vmspace *vm)
 	UVMHIST_FUNC(__func__);
 	UVMHIST_CALLARGS(maphist,"(vm=%#jx) ref=%jd", (uintptr_t)vm,
 	    vm->vm_refcnt, 0, 0);
+
+	membar_exit();
 	if (atomic_dec_uint_nv(&vm->vm_refcnt) > 0)
 		return;
+	membar_enter();
 
 	/*
 	 * at this point, there should be no other references to the map.