# HG changeset patch
# User Taylor R Campbell <riastradh@NetBSD.org>
# Date 1786987598 0
#      Mon Aug 17 17:26:38 2026 +0000
# Branch trunk
# Node ID e8f7aecf524ee7faaa0ec00abbdad0a5d8aac963
# Parent  38311baef8db7eac79b292feba87621d933fa3f1
# EXP-Topic riastradh-pr60029-uvmpdlockup
WIP: uvm: Attempt to bypass uvmpd_lock in several hot paths.

Add Dekker-synchronized paths to the following routines so they avoid
taking uvmpd_lock under contention:

- uvm_wait
- uvm_kick_pdaemon
- uvm_pageout_done

Additionally, move one call to uvm_availmem out from uvmpd_lock to
match the other calls to it -- this should help reduce the time
anyone spends holding uvmpd_lock, and thus the time everyone else
spends waiting for it.

Empirically, this seems to enable a heavily loaded bulk build to
complete when before the softbio completions would starve softclk
leading to heartbeat panics.  Even without the heartbeat panics, we
measured over 8sec running time for softint_dispatch for softbio, and
extremely heavy contention on uvmpd_lock in uvm_pageout_done and
uvm_wait.

How is sprinkling the costliest membar_sync Dekker barriers around,
for sequential consistency (or, more specifically, store-before-load
ordering) supposed to help?

1. If the page daemon is working hard, and some pageouts complete but
   not enough, uvm_pageout_done doesn't need to take uvmpd_lock to
   tell the page daemon to keep working -- and hence logic at softbio
   doesn't need to spend time spinning while a thundering herd of
   uvm_wait calls competes over it.

2. If the page daemon is working hard and an allocation fails leading
   (e.g.) uvm_pagealloc to do uvm_kick_pdaemon, it need not put
   contention on uvmpd_lock to ask the page daemon to go to work --
   and hence it contends less with logic at softbio trying to run
   uvm_pageout_done.

Note: Before, all access to uvm_pagedaemon_waiters was serialized by
uvmpd_lock.  Now, all _writes_ are serialized by uvmpd_lock, but
_reads_ are allowed unlocked.  So the writes have to be issued with
atomic_store_*, but there is no need for atomic_r/m/w_*; hence we can
increment it with

	atomic_store_relaxed(&uvm_pagedaemon_waiters,
	    atomic_load_relaxed(&uvm_pagedaemon_waiters) + 1);

rather than the often costlier

	atomic_inc_uint(&uvm_pagedaemon_waiters);

PR kern/60029: panic: cpu0: softints stuck for 16 seconds

diff -r 38311baef8db -r e8f7aecf524e sys/uvm/uvm_pdaemon.c
--- a/sys/uvm/uvm_pdaemon.c	Sun Aug 16 17:26:26 2026 +0000
+++ b/sys/uvm/uvm_pdaemon.c	Mon Aug 17 17:26:38 2026 +0000
@@ -114,13 +114,17 @@ static void	uvmpd_tune(void);
 static void	uvmpd_pool_drain_thread(void *);
 static void	uvmpd_pool_drain_wakeup(void);
 
-static unsigned int uvm_pagedaemon_waiters;
+static volatile unsigned uvm_pagedaemon_waiters __cacheline_aligned;
+static volatile unsigned uvm_waiter_wakeup __cacheline_aligned;
 
 /* State for the pool drainer thread */
 static kmutex_t uvmpd_lock __cacheline_aligned;
 static kcondvar_t uvmpd_pool_drain_cv;
 static bool uvmpd_pool_drain_run = false;
 
+static volatile unsigned uvmpd_sleeping __cacheline_aligned;
+static volatile unsigned uvmpd_wakeup __cacheline_aligned;
+
 /*
  * XXX hack to avoid hangs when large processes fork.
  */
@@ -173,7 +177,15 @@ uvm_wait(const char *wmsg)
 #endif
 	}
 
-	uvm_pagedaemon_waiters++;
+	atomic_store_relaxed(&uvm_pagedaemon_waiters,
+	    atomic_load_relaxed(&uvm_pagedaemon_waiters) + 1);
+	membar_sync();
+	if (atomic_swap_uint(&uvm_waiter_wakeup, 0)) {
+		atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
+		wakeup(&uvmexp.free);
+		mutex_spin_exit(&uvmpd_lock);
+		return;
+	}
 	wakeup(&uvm.pagedaemon);		/* wake the daemon! */
 	UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_lock, false, wmsg, timo);
 }
@@ -192,6 +204,10 @@ uvm_kick_pdaemon(void)
 	    (fpages + uvmexp.paging < uvmexp.freetarg &&
 	     uvmpdpol_needsscan_p()) ||
 	     uvm_km_va_starved_p()) {
+		atomic_store_relaxed(&uvmpd_wakeup, 1);
+		membar_sync();
+		if (!atomic_load_relaxed(&uvmpd_sleeping))
+			return;
 	     	mutex_spin_enter(&uvmpd_lock);
 		wakeup(&uvm.pagedaemon);
 	     	mutex_spin_exit(&uvmpd_lock);
@@ -288,11 +304,19 @@ uvm_pageout(void *arg)
 		kmem_va_starved = uvm_km_va_starved_p();
 
 		mutex_spin_enter(&uvmpd_lock);
-		if ((uvm_pagedaemon_waiters == 0 || uvmexp.paging > 0) &&
+		if ((atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0 ||
+			uvmexp.paging > 0) &&
 		    !kmem_va_starved) {
 			UVMHIST_LOG(pdhist,"  <<SLEEPING>>",0,0,0,0);
-			UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon,
-			    &uvmpd_lock, false, "pgdaemon", 0);
+			atomic_store_relaxed(&uvmpd_sleeping, 1);
+			membar_sync();
+			if (atomic_swap_uint(&uvmpd_wakeup, 0)) {
+				mutex_spin_exit(&uvmpd_lock);
+			} else {
+				UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon,
+				    &uvmpd_lock, false, "pgdaemon", 0);
+			}
+			atomic_store_relaxed(&uvmpd_sleeping, 0);
 			uvmexp.pdwoke++;
 			UVMHIST_LOG(pdhist,"  <<WOKE UP>>",0,0,0,0);
 		} else {
@@ -337,7 +361,7 @@ uvm_pageout(void *arg)
 		    uvmexp.paging == 0) {
 			mutex_spin_enter(&uvmpd_lock);
 			wakeup(&uvmexp.free);
-			uvm_pagedaemon_waiters = 0;
+			atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
 			mutex_spin_exit(&uvmpd_lock);
 		}
 
@@ -379,15 +403,26 @@ uvm_pageout_done(int npages)
 	/*
 	 * wake up either of pagedaemon or LWPs waiting for it.
 	 */
-
-	mutex_spin_enter(&uvmpd_lock);
 	if (uvm_availmem(false) <= uvmexp.reserve_kernel) {
+		atomic_store_relaxed(&uvmpd_wakeup, 1);
+		membar_sync();
+		if (!atomic_load_relaxed(&uvmpd_sleeping))
+			return;
+		mutex_spin_enter(&uvmpd_lock);
 		wakeup(&uvm.pagedaemon);
-	} else if (uvm_pagedaemon_waiters != 0) {
-		wakeup(&uvmexp.free);
-		uvm_pagedaemon_waiters = 0;
+		mutex_spin_exit(&uvmpd_lock);
+	} else {
+		atomic_store_relaxed(&uvm_waiter_wakeup, 1);
+		membar_sync();
+		if (atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0)
+			return;
+		mutex_spin_enter(&uvmpd_lock);
+		if (atomic_load_relaxed(&uvm_pagedaemon_waiters) != 0) {
+			wakeup(&uvmexp.free);
+			atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
+		}
+		mutex_spin_exit(&uvmpd_lock);
 	}
-	mutex_spin_exit(&uvmpd_lock);
 }
 
 static krwlock_t *
# HG changeset patch
# User Taylor R Campbell <riastradh@NetBSD.org>
# Date 1787069600 0
#      Tue Aug 18 16:13:20 2026 +0000
# Branch trunk
# Node ID 45c035f4cc0605040bf45290d66d625194c2bedf
# Parent  e8f7aecf524ee7faaa0ec00abbdad0a5d8aac963
# EXP-Topic riastradh-pr58964-uvmwaitwakeup
WIP: uvm: Avoid missed wakeups on uvm_wait.

New function uvm_wait_prepare returns a cookie _before_ whatever
allocation might require uvm_wait.  Callers must then pass it into
uvm_wait, which will return without sleeping if anything has changed
between uvm_wait_prepare and uvm_wait.

Usage model:

	again:	waitcookie = uvm_wait_prepare();
		if ((pg = uvm_pagealloc(...)) == NULL) {
			uvm_wait("moarpgsplz", waitcookie);
			goto again;
		}

Internally, the cookie is a 64-bit generation number, so it can't
possibly overflow, that is advanced on any wakeup(&uvmexp.free).  For
LP32 platforms, we manage it with a seqlock/Lamport-type algorithm,
so that reading it is cheap and unlocked.

This should help to avoid missing wakeups from the page daemon:

PR kern/58964: uvm: missing wakeup on uvmexp.free

This may also help reduce contention on uvmpd_lock under heavy paging
load by having uvm_wait skip the lock if there has been concurrent
paging since uvm_wait_prepare, so it may help with:

PR kern/60029: panic: cpu0: softints stuck for 16 seconds

diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/aarch64/aarch64/pmap.c
--- a/sys/arch/aarch64/aarch64/pmap.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/aarch64/aarch64/pmap.c	Tue Aug 18 16:13:20 2026 +0000
@@ -552,6 +552,7 @@ static paddr_t
 pmap_alloc_pdp(struct pmap *pm, struct vm_page **pgp, int flags, bool waitok)
 {
 	paddr_t pa;
+	uint64_t ticket;
 	struct vm_page *pg;
 
 	UVMHIST_FUNC(__func__);
@@ -562,10 +563,11 @@ pmap_alloc_pdp(struct pmap *pm, struct v
 		int aflags = ((flags & PMAP_CANFAIL) ? 0 : UVM_PGA_USERESERVE) |
 		    UVM_PGA_ZERO;
  retry:
+		ticket = uvm_wait_prepare();
 		pg = uvm_pagealloc(NULL, 0, NULL, aflags);
 		if (pg == NULL) {
 			if (waitok) {
-				uvm_wait("pmap_alloc_pdp");
+				uvm_wait("pmap_alloc_pdp", ticket);
 				goto retry;
 			}
 			return POOL_PADDR_INVALID;
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/aarch64/include/asan.h
--- a/sys/arch/aarch64/include/asan.h	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/aarch64/include/asan.h	Tue Aug 18 16:13:20 2026 +0000
@@ -79,11 +79,13 @@ static paddr_t
 		va = uvm_pageboot_alloc(PAGE_SIZE);
 		pa = AARCH64_KVA_TO_PA(va);
 	} else {
+		uint64_t ticket;
 		struct vm_page *pg;
 retry:
+		ticket = uvm_wait_prepare();
 		pg = uvm_pagealloc(NULL, 0, NULL, 0);
 		if (pg == NULL) {
-			uvm_wait(__func__);
+			uvm_wait(__func__, ticket);
 			goto retry;
 		}
 
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/amd64/amd64/gdt.c
--- a/sys/arch/amd64/amd64/gdt.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/amd64/amd64/gdt.c	Tue Aug 18 16:13:20 2026 +0000
@@ -184,6 +184,7 @@ gdt_alloc_cpu(struct cpu_info *ci)
 #ifdef __HAVE_PCPU_AREA
 	ci->ci_gdt = (union descriptor *)&pcpuarea->ent[cpu_index(ci)].gdt;
 #else
+	uint64_t ticket;
 	struct vm_page *pg;
 	vaddr_t va;
 
@@ -191,9 +192,10 @@ gdt_alloc_cpu(struct cpu_info *ci)
 	    0, UVM_KMF_VAONLY);
 	for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + gdt_size;
 	    va += PAGE_SIZE) {
-		while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO))
+		while (ticket = uvm_wait_prepare(),
+		    (pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO))
 		    == NULL) {
-			uvm_wait("gdt_alloc_cpu");
+			uvm_wait("gdt_alloc_cpu", ticket);
 		}
 		pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
 		    VM_PROT_READ | VM_PROT_WRITE, 0);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/arm/include/asan.h
--- a/sys/arch/arm/include/asan.h	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/arm/include/asan.h	Tue Aug 18 16:13:20 2026 +0000
@@ -111,11 +111,13 @@ static vaddr_t
 		return pa;
 	}
 
+	uint64_t ticket;
 	struct vm_page *pg;
 retry:
+	ticket = uvm_wait_prepare();
 	pg = uvm_pagealloc(NULL, 0, NULL, 0);
 	if (pg == NULL) {
-		uvm_wait(__func__);
+		uvm_wait(__func__, ticket);
 		goto retry;
 	}
 	pa = VM_PAGE_TO_PHYS(pg);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/i386/i386/gdt.c
--- a/sys/arch/i386/i386/gdt.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/i386/i386/gdt.c	Tue Aug 18 16:13:20 2026 +0000
@@ -184,6 +184,7 @@ gdt_init(void)
 void
 gdt_alloc_cpu(struct cpu_info *ci)
 {
+	uint64_t ticket;
 	struct vm_page *pg;
 	vaddr_t va;
 
@@ -191,9 +192,10 @@ gdt_alloc_cpu(struct cpu_info *ci)
 	    0, UVM_KMF_VAONLY);
 	for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + gdt_size;
 	    va += PAGE_SIZE) {
-		while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO))
+		while (ticket = uvm_wait_prepare(),
+		    (pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO))
 		    == NULL) {
-			uvm_wait("gdt_alloc_cpu");
+			uvm_wait("gdt_alloc_cpu", ticket);
 		}
 		pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
 		    VM_PROT_READ | VM_PROT_WRITE, 0);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/m68k/m68k/pmap_68k.c
--- a/sys/arch/m68k/m68k/pmap_68k.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/m68k/m68k/pmap_68k.c	Tue Aug 18 16:13:20 2026 +0000
@@ -643,14 +643,16 @@ pmap_ptpage_init(void)
 static struct vm_page *
 pmap_page_alloc(bool nowait)
 {
+	uint64_t ticket;
 	struct vm_page *pg;
 	const int flags = nowait ? UVM_PGA_USERESERVE : 0;
 
-	while ((pg = uvm_pagealloc(NULL, 0, NULL, flags)) == NULL) {
+	while (ticket = uvm_wait_prepare(),
+	    (pg = uvm_pagealloc(NULL, 0, NULL, flags)) == NULL) {
 		if (nowait) {
 			return NULL;
 		}
-		uvm_wait("pmappg");
+		uvm_wait("pmappg", ticket);
 	}
 	pg->flags &= ~PG_BUSY;	/* never busy */
 
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/m68k/m68k/pmap_motorola.c
--- a/sys/arch/m68k/m68k/pmap_motorola.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/m68k/m68k/pmap_motorola.c	Tue Aug 18 16:13:20 2026 +0000
@@ -2551,6 +2551,7 @@ int
 pmap_enter_ptpage(pmap_t pmap, vaddr_t va, bool can_fail)
 {
 	paddr_t ptpa;
+	uint64_t ticket;
 	struct vm_page *pg;
 	struct pv_header *pvh;
 	struct pv_entry *pv;
@@ -2683,7 +2684,6 @@ pmap_enter_ptpage(pmap_t pmap, vaddr_t v
 #endif
 		splx(s);
 	} else {
-
 		/*
 		 * For user processes we just allocate a page from the
 		 * VM system.  Note that we set the page "wired" count to 1,
@@ -2698,15 +2698,16 @@ pmap_enter_ptpage(pmap_t pmap, vaddr_t v
 		PMAP_DPRINTF(PDB_ENTER|PDB_PTPAGE,
 		    ("enter: about to alloc UPT pg at %lx\n", va));
 		rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER);
-		while ((pg = uvm_pagealloc(uvm_kernel_object,
-					   va - vm_map_min(kernel_map),
-					   NULL, UVM_PGA_ZERO)) == NULL) {
+		while (ticket = uvm_wait_prepare(),
+		    (pg = uvm_pagealloc(uvm_kernel_object,
+			va - vm_map_min(kernel_map),
+			NULL, UVM_PGA_ZERO)) == NULL) {
 			rw_exit(uvm_kernel_object->vmobjlock);
 			if (can_fail) {
 				pmap->pm_sref--;
 				return ENOMEM;
 			}
-			uvm_wait("ptpage");
+			uvm_wait("ptpage", ticket);
 			rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER);
 		}
 		rw_exit(uvm_kernel_object->vmobjlock);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/powerpc/oea/pmap.c
--- a/sys/arch/powerpc/oea/pmap.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/powerpc/oea/pmap.c	Tue Aug 18 16:13:20 2026 +0000
@@ -2885,6 +2885,7 @@ pmap_free_unmanaged(struct pup *pup)
 void *
 pmap_pool_alloc(struct pool *pp, int flags)
 {
+	uint64_t ticket;
 	struct vm_page *pg;
 	paddr_t pa;
 
@@ -2892,6 +2893,7 @@ pmap_pool_alloc(struct pool *pp, int fla
 		return (void *)uvm_pageboot_alloc(PAGE_SIZE);
 
  retry:
+	ticket = uvm_wait_prepare();
 	pg = uvm_pagealloc_strat(NULL /*obj*/, 0 /*off*/, NULL /*anon*/,
 	    UVM_PGA_USERESERVE /*flags*/, UVM_PGA_STRAT_ONLY /*strat*/,
 	    VM_FREELIST_DIRECT_MAPPED /*free_list*/);
@@ -2902,7 +2904,7 @@ pmap_pool_alloc(struct pool *pp, int fla
 
 		if ((flags & PR_WAITOK) == 0)
 			return NULL;
-		uvm_wait("plpg");
+		uvm_wait("plpg", ticket);
 		goto retry;
 	}
 	KDASSERT(VM_PAGE_TO_PHYS(pg) == (uintptr_t)VM_PAGE_TO_PHYS(pg));
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/sparc/sparc/pmap.c
--- a/sys/arch/sparc/sparc/pmap.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/sparc/sparc/pmap.c	Tue Aug 18 16:13:20 2026 +0000
@@ -907,14 +907,16 @@ void *
 pgt_page_alloc(struct pool *pp, int flags)
 {
 	int cacheit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0;
+	uint64_t ticket;
 	struct vm_page *pg;
 	vaddr_t va;
 	paddr_t pa;
 
 	/* Allocate a page of physical memory */
-	while ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL &&
-	       (flags & PR_WAITOK) != 0) {
-		uvm_wait("pgtpg");
+	while (ticket = uvm_wait_prepare(),
+	    (pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL &&
+	    (flags & PR_WAITOK) != 0) {
+		uvm_wait("pgtpg", ticket);
 	}
 	if (pg == NULL) {
 		KASSERT((flags & PR_WAITOK) == 0);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/arch/sparc64/sparc64/pmap.c
--- a/sys/arch/sparc64/sparc64/pmap.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/arch/sparc64/sparc64/pmap.c	Tue Aug 18 16:13:20 2026 +0000
@@ -1475,8 +1475,11 @@ pmap_create(void)
 	pm->pm_refs = 1;
 	TAILQ_INIT(&pm->pm_ptps);
 	if (pm != pmap_kernel()) {
-		while (!pmap_get_page(&pm->pm_physaddr)) {
-			uvm_wait("pmap_create");
+		uint64_t ticket;
+
+		while (ticket = uvm_wait_prepare(),
+		    !pmap_get_page(&pm->pm_physaddr)) {
+			uvm_wait("pmap_create", ticket);
 		}
 		pm->pm_segs = (paddr_t *)(u_long)pm->pm_physaddr;
 	}
diff -r e8f7aecf524e -r 45c035f4cc06 sys/rump/librump/rumpkern/vm.c
--- a/sys/rump/librump/rumpkern/vm.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/rump/librump/rumpkern/vm.c	Tue Aug 18 16:13:20 2026 +0000
@@ -106,6 +106,7 @@ struct pmap *const kernel_pmap_ptr = &pm
 vmem_t *kmem_arena;
 vmem_t *kmem_va_arena;
 
+static uint64_t uvm_pdaemon_ticket;
 static unsigned int pdaemon_waiters;
 static kmutex_t pdaemonmtx;
 static kcondvar_t pdaemoncv, oomwait;
@@ -1101,8 +1102,20 @@ uvm_uarea_free(vaddr_t uarea)
  * Routines related to the Page Baroness.
  */
 
+uint64_t
+uvm_wait_prepare(void)
+{
+	uint64_t ticket;
+
+	mutex_enter(&pdaemonmtx);
+	ticket = uvm_pdaemon_ticket;
+	mutex_exit(&pdaemonmtx);
+
+	return ticket;
+}
+
 void
-uvm_wait(const char *msg)
+uvm_wait(const char *msg, uint64_t ticket)
 {
 
 	if (__predict_false(rump_threads == 0))
@@ -1115,10 +1128,12 @@ uvm_wait(const char *msg)
 	}
 
 	mutex_enter(&pdaemonmtx);
+	if (ticket != uvm_pdaemon_ticket)
+		goto out;
 	pdaemon_waiters++;
 	cv_signal(&pdaemoncv);
 	cv_wait(&oomwait, &pdaemonmtx);
-	mutex_exit(&pdaemonmtx);
+out:	mutex_exit(&pdaemonmtx);
 }
 
 void
@@ -1141,6 +1156,7 @@ uvm_pageout_done(int npages)
 	KASSERT(uvmexp.paging >= npages);
 	uvmexp.paging -= npages;
 
+	uvm_pdaemon_ticket++;
 	if (pdaemon_waiters) {
 		pdaemon_waiters = 0;
 		cv_broadcast(&oomwait);
@@ -1185,6 +1201,7 @@ uvm_pageout(void *arg)
 
 	mutex_enter(&pdaemonmtx);
 	for (;;) {
+		uvm_pdaemon_ticket++;
 		if (pdaemon_waiters) {
 			pdaemon_waiters = 0;
 			cv_broadcast(&oomwait);
@@ -1309,6 +1326,7 @@ rump_hypermalloc(size_t howmuch, int ali
 {
 	const unsigned long thelimit =
 	    uvm_lwp_is_pagedaemon(curlwp) ? pdlimit : rump_physmemlimit;
+	uint64_t ticket;
 	unsigned long newmem;
 	void *rv;
 	int error;
@@ -1318,22 +1336,24 @@ rump_hypermalloc(size_t howmuch, int ali
 	/* first we must be within the limit */
  limitagain:
 	if (thelimit != RUMPMEM_UNLIMITED) {
+		ticket = uvm_wait_prepare();
 		newmem = atomic_add_long_nv(&curphysmem, howmuch);
 		if (newmem > thelimit) {
 			newmem = atomic_add_long_nv(&curphysmem, -howmuch);
 			if (!waitok) {
 				return NULL;
 			}
-			uvm_wait(wmsg);
+			uvm_wait(wmsg, ticket);
 			goto limitagain;
 		}
 	}
 
 	/* second, we must get something from the backend */
  again:
+	ticket = uvm_wait_prepare();
 	error = rumpuser_malloc(howmuch, alignment, &rv);
 	if (__predict_false(error && waitok)) {
-		uvm_wait(wmsg);
+		uvm_wait(wmsg, ticket);
 		goto again;
 	}
 
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/pmap/pmap_segtab.c
--- a/sys/uvm/pmap/pmap_segtab.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/pmap/pmap_segtab.c	Tue Aug 18 16:13:20 2026 +0000
@@ -584,6 +584,8 @@ pmap_pdetab_alloc(struct pmap *pmap)
 
 	struct vm_page *ptb_pg = NULL;
 	if (__predict_false(ptb == NULL)) {
+		uint64_t ticket = uvm_wait_prepare();
+
 		ptb_pg = pmap_pte_pagealloc();
 
 		UVMHIST_LOG(pmapxtabhist, "ptb_pg=%#jx",
@@ -592,7 +594,7 @@ pmap_pdetab_alloc(struct pmap *pmap)
 			/*
 			 * XXX What else can we do?  Could we deadlock here?
 			 */
-			uvm_wait("pdetab");
+			uvm_wait("pdetab", ticket);
 			goto again;
 		}
 
@@ -658,13 +660,15 @@ pmap_segtab_alloc(struct pmap *pmap)
 
 	struct vm_page *stb_pg = NULL;
 	if (__predict_false(stb == NULL)) {
+		uint64_t ticket = uvm_wait_prepare();
+
 		stb_pg = pmap_pte_pagealloc();
 
 		if (__predict_false(stb_pg == NULL)) {
 			/*
 			 * XXX What else can we do?  Could we deadlock here?
 			 */
-			uvm_wait("segtab");
+			uvm_wait("segtab", ticket);
 			goto again;
 		}
 		SEGTAB_ADD(npage, 1);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_amap.c
--- a/sys/uvm/uvm_amap.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_amap.c	Tue Aug 18 16:13:20 2026 +0000
@@ -1026,6 +1026,7 @@ void
 amap_cow_now(struct vm_map *map, struct vm_map_entry *entry)
 {
 	struct vm_amap *amap = entry->aref.ar_amap;
+	uint64_t ticket;
 	struct vm_anon *anon, *nanon;
 	struct vm_page *pg, *npg;
 	u_int lcv, slot;
@@ -1089,6 +1090,7 @@ ReStart:
 		 * First - get a new anon and a page.
 		 */
 
+		ticket = uvm_wait_prepare();
 		nanon = uvm_analloc();
 		if (nanon) {
 			nanon->an_lock = amap->am_lock;
@@ -1104,7 +1106,7 @@ ReStart:
 				KASSERT(nanon->an_ref == 0);
 				uvm_anfree(nanon);
 			}
-			uvm_wait("cownowpage");
+			uvm_wait("cownowpage", ticket);
 			goto ReStart;
 		}
 
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_aobj.c
--- a/sys/uvm/uvm_aobj.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_aobj.c	Tue Aug 18 16:13:20 2026 +0000
@@ -806,6 +806,7 @@ uao_get(struct uvm_object *uobj, voff_t 
     int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags)
 {
 	voff_t current_offset;
+	uint64_t ticket;
 	struct vm_page *ptmp;
 	int lcv, gotpages, maxpages, swslot, pageidx;
 	bool overwrite = ((flags & PGO_OVERWRITE) != 0);
@@ -958,6 +959,7 @@ uao_get(struct uvm_object *uobj, voff_t 
 
 		pageidx = current_offset >> PAGE_SHIFT;
 		swslot = uao_find_swslot(uobj, pageidx);
+		ticket = uvm_wait_prepare();
 		ptmp = uao_pagealloc(uobj, current_offset,
 		    swslot != 0 || overwrite ? 0 : UVM_PGA_ZERO);
 
@@ -965,7 +967,7 @@ uao_get(struct uvm_object *uobj, voff_t 
 		if (ptmp == NULL) {
 			rw_exit(uobj->vmobjlock);
 			UVMHIST_LOG(pdhist, "sleeping, ptmp == NULL",0,0,0,0);
-			uvm_wait("uao_getpage");
+			uvm_wait("uao_getpage", ticket);
 			rw_enter(uobj->vmobjlock, RW_WRITER);
 			uvm_page_array_clear(&a);
 			continue;
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_bio.c
--- a/sys/uvm/uvm_bio.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_bio.c	Tue Aug 18 16:13:20 2026 +0000
@@ -423,6 +423,7 @@ again:
 	 */
 	rw_enter(uobj->vmobjlock, RW_WRITER);
 	for (i = 0; va < eva; i++, va += PAGE_SIZE) {
+		uint64_t ticket;
 		struct vm_page *pg;
 
 		UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i, (uintptr_t)pgs[i],
@@ -432,6 +433,7 @@ again:
 		if (pg == NULL || pg == PGO_DONTCARE) {
 			continue;
 		}
+		ticket = uvm_wait_prepare();
 		KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock);
 		error = ubc_fault_page(ufi, umap, pg, prot, access_type, va);
 		if (error) {
@@ -441,7 +443,7 @@ again:
 			 */
 			pmap_update(ufi->orig_map->pmap);
 			rw_exit(uobj->vmobjlock);
-			uvm_wait("ubc_fault");
+			uvm_wait("ubc_fault", ticket);
 			rw_enter(uobj->vmobjlock, RW_WRITER);
 		}
 	}
@@ -601,11 +603,13 @@ again_faultbusy:
 		}
 		for (i = 0; i < npages; i++) {
 			struct vm_page *pg = pgs[i];
+			uint64_t ticket = 0;	/* XXX spurious init */
 
 			KASSERT(pg->uobject == uobj);
 			if (pg->loan_count != 0) {
 				rw_enter(uobj->vmobjlock, RW_WRITER);
 				if (pg->loan_count != 0) {
+					ticket = uvm_wait_prepare();
 					pg = uvm_loanbreak(pg);
 				}
 				if (pg == NULL) {
@@ -613,7 +617,7 @@ again_faultbusy:
 					pmap_update(pmap_kernel());
 					uvm_page_unbusy(pgs, npages);
 					rw_exit(uobj->vmobjlock);
-					uvm_wait("ubc_alloc");
+					uvm_wait("ubc_alloc", ticket);
 					goto again_faultbusy;
 				}
 				rw_exit(uobj->vmobjlock);
@@ -901,11 +905,13 @@ again:
 
 		/* Avoid breaking loan if possible, only do it on write */
 		if ((flags & UBC_WRITE) && pg->loan_count != 0) {
+			uint64_t ticket = uvm_wait_prepare();
+
 			pg = uvm_loanbreak(pg);
 			if (pg == NULL) {
 				uvm_page_unbusy(pgs, *npages);
 				rw_exit(uobj->vmobjlock);
-				uvm_wait("ubc_alloc_directl");
+				uvm_wait("ubc_alloc_directl", ticket);
 				goto again;
 			}
 			pgs[i] = pg;
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_fault.c
--- a/sys/uvm/uvm_fault.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_fault.c	Tue Aug 18 16:13:20 2026 +0000
@@ -216,6 +216,7 @@ static void
 uvmfault_amapcopy(struct uvm_faultinfo *ufi)
 {
 	for (;;) {
+		uint64_t ticket;
 
 		/*
 		 * no mapping?  give up.
@@ -228,9 +229,11 @@ uvmfault_amapcopy(struct uvm_faultinfo *
 		 * copy if needed.
 		 */
 
-		if (UVM_ET_ISNEEDSCOPY(ufi->entry))
+		if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
+			ticket = uvm_wait_prepare();
 			amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
 				ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
+		}
 
 		/*
 		 * didn't work?  must be out of RAM.   unlock and sleep.
@@ -238,7 +241,7 @@ uvmfault_amapcopy(struct uvm_faultinfo *
 
 		if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
 			uvmfault_unlockmaps(ufi, true);
-			uvm_wait("fltamapcopy");
+			uvm_wait("fltamapcopy", ticket);
 			continue;
 		}
 
@@ -354,10 +357,12 @@ uvmfault_anonget(struct uvm_faultinfo *u
 			 * required for this.  If the caller didn't supply
 			 * one, fail now and have them retry.
 			 */
+			uint64_t ticket;
 
 			if (lock_type == RW_READER) {
 				return ENOLCK;
 			}
+			ticket = uvm_wait_prepare();
 			pg = uvm_pagealloc(NULL,
 			    ufi != NULL ? ufi->orig_rvaddr : 0,
 			    anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0);
@@ -370,7 +375,7 @@ uvmfault_anonget(struct uvm_faultinfo *u
 				if (!uvm_reclaimable()) {
 					return ENOMEM;
 				}
-				uvm_wait("flt_noram1");
+				uvm_wait("flt_noram1", ticket);
 			} else {
 				/* PG_BUSY bit is set. */
 				we_own = true;
@@ -551,6 +556,7 @@ uvmfault_promote(struct uvm_faultinfo *u
 {
 	struct vm_amap *amap = ufi->entry->aref.ar_amap;
 	struct uvm_object *uobj;
+	uint64_t ticket;
 	struct vm_anon *anon;
 	struct vm_page *pg;
 	struct vm_page *opg;
@@ -582,6 +588,7 @@ uvmfault_promote(struct uvm_faultinfo *u
 	KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock);
 	KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock));
 
+	ticket = uvm_wait_prepare();
 	if (*spare != NULL) {
 		anon = *spare;
 		*spare = NULL;
@@ -629,7 +636,7 @@ uvmfault_promote(struct uvm_faultinfo *u
 
 		UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
 		cpu_count(CPU_COUNT_FLTNORAM, 1);
-		uvm_wait("flt_noram5");
+		uvm_wait("flt_noram5", ticket);
 		error = ERESTART;
 		goto done;
 	}
@@ -913,10 +920,13 @@ uvm_fault_internal(struct vm_map *orig_m
 			    ufi.entry->object.uvm_obj;
 
 			if (uobj && uobj->pgops->pgo_fault != NULL) {
+				uint64_t ticket;
+
 				/*
 				 * invoke "special" fault routine.
 				 */
 				rw_enter(uobj->vmobjlock, RW_WRITER);
+				ticket = uvm_wait_prepare();
 				/* locked: maps(read), amap(if there), uobj */
 				error = uobj->pgops->pgo_fault(&ufi,
 				    flt.startva, pages, flt.npages,
@@ -939,7 +949,7 @@ uvm_fault_internal(struct vm_map *orig_m
 				 * reclaimed.
 				 */
 				if (error == ENOMEM && uvm_reclaimable()) {
-					uvm_wait("pgo_fault");
+					uvm_wait("pgo_fault", ticket);
 					error = ERESTART;
 				}
 			} else {
@@ -1563,6 +1573,8 @@ uvm_fault_upper_loan(
 
 		/* >1 case is already ok */
 		if (anon->an_ref == 1) {
+			uint64_t ticket;
+
 			/* breaking loan requires a write lock. */
 			error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL);
 			if (error != 0) {
@@ -1570,10 +1582,11 @@ uvm_fault_upper_loan(
 			}
 			KASSERT(rw_write_held(amap->am_lock));
 
+			ticket = uvm_wait_prepare();
 			error = uvm_loanbreak_anon(anon, *ruobj);
 			if (error != 0) {
 				uvmfault_unlockall(ufi, amap, *ruobj);
-				uvm_wait("flt_noram2");
+				uvm_wait("flt_noram2", ticket);
 				return ERESTART;
 			}
 			/* if we were a loan receiver uobj is gone */
@@ -1677,6 +1690,7 @@ uvm_fault_upper_enter(
 	struct pmap *pmap = ufi->orig_map->pmap;
 	vaddr_t va = ufi->orig_rvaddr;
 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
+	uint64_t ticket;
 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
 
 	/* locked: maps(read), amap, oanon, anon(if different from oanon) */
@@ -1694,6 +1708,7 @@ uvm_fault_upper_enter(
 	UVMHIST_LOG(maphist,
 	    "  MAPPING: anon: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd",
 	    (uintptr_t)pmap, va, (uintptr_t)pg, flt->promote);
+	ticket = uvm_wait_prepare();
 	if (pmap_enter(pmap, va, VM_PAGE_TO_PHYS(pg),
 	    flt->enter_prot, flt->access_type | PMAP_CANFAIL |
 	    (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
@@ -1741,7 +1756,7 @@ uvm_fault_upper_enter(
 			return ENOMEM;
 		}
 		/* XXX instrumentation */
-		uvm_wait("flt_pmfail1");
+		uvm_wait("flt_pmfail1", ticket);
 		return ERESTART;
 	}
 
@@ -2330,6 +2345,7 @@ uvm_fault_lower_direct_loan(
 		 * write fault: must break the loan here.  to do this
 		 * we need a write lock on the object.
 		 */
+		uint64_t ticket;
 
 		error = uvm_fault_lower_upgrade(ufi, flt, amap, uobj, uobjpage);
 		if (error != 0) {
@@ -2337,6 +2353,7 @@ uvm_fault_lower_direct_loan(
 		}
 		KASSERT(rw_write_held(uobj->vmobjlock));
 
+		ticket = uvm_wait_prepare();
 		pg = uvm_loanbreak(uobjpage);
 		if (pg == NULL) {
 
@@ -2345,7 +2362,7 @@ uvm_fault_lower_direct_loan(
 			  "  out of RAM breaking loan, waiting",
 			  0,0,0,0);
 			cpu_count(CPU_COUNT_FLTNORAM, 1);
-			uvm_wait("flt_noram4");
+			uvm_wait("flt_noram4", ticket);
 			return ERESTART;
 		}
 		*rpg = pg;
@@ -2462,6 +2479,7 @@ uvm_fault_lower_enter(
 {
 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
 	const bool readonly = uvm_pagereadonly_p(pg);
+	uint64_t ticket;
 	int error;
 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
 
@@ -2506,6 +2524,7 @@ uvm_fault_lower_enter(
 	    UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map,
 	    (void *)ufi->orig_rvaddr, pg);
 	KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly);
+	ticket = uvm_wait_prepare();
 	if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
 	    VM_PAGE_TO_PHYS(pg),
 	    readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
@@ -2541,7 +2560,7 @@ uvm_fault_lower_enter(
 			return error;
 		}
 		/* XXX instrumentation */
-		uvm_wait("flt_pmfail2");
+		uvm_wait("flt_pmfail2", ticket);
 		return ERESTART;
 	}
 
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_glue.c
--- a/sys/uvm/uvm_glue.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_glue.c	Tue Aug 18 16:13:20 2026 +0000
@@ -257,15 +257,18 @@ uarea_poolpage_alloc(struct pool *pp, in
 #if defined(PMAP_MAP_POOLPAGE)
 	while (USPACE == PAGE_SIZE &&
 	    (USPACE_ALIGN == 0 || USPACE_ALIGN == PAGE_SIZE)) {
+		uint64_t ticket;
 		struct vm_page *pg;
 		vaddr_t va;
+
+		ticket = uvm_wait_prepare();
 #if defined(PMAP_ALLOC_POOLPAGE)
 		pg = PMAP_ALLOC_POOLPAGE(0);
 #else
 		pg = uvm_pagealloc(NULL, 0, NULL, 0);
 #endif
 		if (pg == NULL) {
-			uvm_wait("uarea");
+			uvm_wait("uarea", ticket);
 			continue;
 		}
 		va = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg));
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_km.c
--- a/sys/uvm/uvm_km.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_km.c	Tue Aug 18 16:13:20 2026 +0000
@@ -614,6 +614,7 @@ uvm_km_alloc(struct vm_map *map, vsize_t
 	vaddr_t kva, loopva;
 	vaddr_t offset;
 	vsize_t loopsize;
+	uint64_t ticket;
 	struct vm_page *pg;
 	struct uvm_object *obj;
 	int pgaflags;
@@ -687,6 +688,7 @@ uvm_km_alloc(struct vm_map *map, vsize_t
 		KASSERTMSG(!pmap_extract(pmap_kernel(), loopva, NULL),
 		    "loopva=%#"PRIxVADDR, loopva);
 
+		ticket = uvm_wait_prepare();
 		pg = uvm_pagealloc_strat(NULL, offset, NULL, pgaflags,
 #ifdef UVM_KM_VMFREELIST
 		   UVM_PGA_STRAT_ONLY, UVM_KM_VMFREELIST
@@ -707,7 +709,8 @@ uvm_km_alloc(struct vm_map *map, vsize_t
 				    flags & UVM_KMF_TYPEMASK);
 				return (0);
 			} else {
-				uvm_wait("km_getwait2");	/* sleep here */
+				/* sleep here */
+				uvm_wait("km_getwait2", ticket);
 				continue;
 			}
 		}
@@ -798,6 +801,7 @@ int
 uvm_km_kmem_alloc(vmem_t *vm, vmem_size_t size, vm_flag_t flags,
     vmem_addr_t *addr)
 {
+	uint64_t ticket;
 	struct vm_page *pg;
 	vmem_addr_t va;
 	int rc;
@@ -809,6 +813,7 @@ uvm_km_kmem_alloc(vmem_t *vm, vmem_size_
 #if defined(PMAP_MAP_POOLPAGE)
 	if (size == PAGE_SIZE) {
 again:
+		ticket = uvm_wait_prepare();
 #ifdef PMAP_ALLOC_POOLPAGE
 		pg = PMAP_ALLOC_POOLPAGE((flags & VM_SLEEP) ?
 		   0 : UVM_PGA_USERESERVE);
@@ -818,7 +823,7 @@ again:
 #endif /* PMAP_ALLOC_POOLPAGE */
 		if (__predict_false(pg == NULL)) {
 			if (flags & VM_SLEEP) {
-				uvm_wait("plpg");
+				uvm_wait("plpg", ticket);
 				goto again;
 			}
 			return ENOMEM;
@@ -854,12 +859,13 @@ again:
 		    " pa=%#"PRIxPADDR" vmem=%p",
 		    loopva, loopsize, pa, vm);
 
+		ticket = uvm_wait_prepare();
 		pg = uvm_pagealloc(NULL, loopva, NULL,
 		    UVM_FLAG_COLORMATCH
 		    | ((flags & VM_SLEEP) ? 0 : UVM_PGA_USERESERVE));
 		if (__predict_false(pg == NULL)) {
 			if (flags & VM_SLEEP) {
-				uvm_wait("plpg");
+				uvm_wait("plpg", ticket);
 				continue;
 			} else {
 				uvm_km_pgremove_intrsafe(kernel_map, va,
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_loan.c
--- a/sys/uvm/uvm_loan.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_loan.c	Tue Aug 18 16:13:20 2026 +0000
@@ -834,11 +834,14 @@ again:
 
 	pg = uvm_pagelookup(&uvm_loanzero_object, 0);
 	if (__predict_false(pg == NULL)) {
-		while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
-					   UVM_PGA_ZERO)) == NULL) {
+		uint64_t ticket;
+
+		while (ticket = uvm_wait_prepare(),
+		    (pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
+			UVM_PGA_ZERO)) == NULL) {
 			rw_exit(uvm_loanzero_object.vmobjlock);
 			uvmfault_unlockall(ufi, amap, NULL);
-			uvm_wait("loanzero");
+			uvm_wait("loanzero", ticket);
 			if (!uvmfault_relock(ufi)) {
 				return (0);
 			}
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_object.c
--- a/sys/uvm/uvm_object.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_object.c	Tue Aug 18 16:13:20 2026 +0000
@@ -162,11 +162,15 @@ uvm_obj_wirepages(struct uvm_object *uob
 			 */
 			if (pgs[i]->loan_count) {
 				while (pgs[i]->loan_count) {
+					uint64_t ticket =
+					    uvm_wait_prepare();
+
 					pg = uvm_loanbreak(pgs[i]);
 					if (!pg) {
 						rw_exit(uobj->vmobjlock);
-						uvm_wait("uobjwirepg");
-						rw_enter(uobj->vmobjlock, RW_WRITER);
+						uvm_wait("uobjwirepg", ticket);
+						rw_enter(uobj->vmobjlock,
+						    RW_WRITER);
 						continue;
 					}
 				}
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_pdaemon.c
--- a/sys/uvm/uvm_pdaemon.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_pdaemon.c	Tue Aug 18 16:13:20 2026 +0000
@@ -125,28 +125,117 @@ static bool uvmpd_pool_drain_run = false
 static volatile unsigned uvmpd_sleeping __cacheline_aligned;
 static volatile unsigned uvmpd_wakeup __cacheline_aligned;
 
+#if __HAVE_ATOMIC64_LOADSTORE
+
+static volatile uint64_t uvmpd_gen;
+
+static uint64_t
+uvmpd_read_gen(void)
+{
+
+	return atomic_load_relaxed(&uvmpd_gen);
+}
+
+static void
+uvmpd_advance_gen(void)
+{
+
+	KASSERT(mutex_owned(&uvmpd_lock));
+	atomic_store_relaxed(&uvmpd_gen, atomic_load_relaxed(&uvmpd_gen) + 1);
+}
+
+#else  /* !__HAVE_ATOMIC64_LOADSTORE */
+
+static volatile struct {
+#if _BYTE_ORDER == _BIG_ENDIAN
+	uint32_t	hi, lo;
+#else
+	uint32_t	lo, hi;
+#endif
+} uvmpd_gen;
+
+static uint64_t
+uvmpd_read_gen(void)
+{
+	uint32_t hi, lo;
+
+top:	while (__predict_false((hi = atomic_load_relaxed(&uvmpd_gen.hi)) ==
+		0xffffffff))
+		SPINLOCK_BACKOFF_HOOK;
+	membar_consumer();
+	lo = atomic_load_relaxed(&uvmpd_gen.lo);
+	membar_consumer();
+	if (__predict_false(hi != atomic_load_relaxed(&uvmpd_gen.hi)))
+		goto top;
+
+	return (uint64_t)hi << 32 | lo;
+}
+
+static void
+uvmpd_advance_gen(void)
+{
+	uint32_t hi, lo;
+
+	KASSERT(mutex_owned(&uvmpd_lock));
+
+	lo = atomic_load_relaxed(&uvmpd_gen.lo);
+	if (__predict_false(++lo == 0)) {
+		hi = atomic_load_relaxed(&uvmpd_gen.hi) + 1; /* carry */
+		atomic_store_relaxed(&uvmpd_gen.hi, 0xffffffff);
+		membar_producer();
+		atomic_store_relaxed(&uvmpd_gen.lo, lo);
+		membar_producer();
+		atomic_store_relaxed(&uvmpd_gen.hi, hi);
+	} else {
+		atomic_store_relaxed(&uvmpd_gen.lo, lo);
+	}
+}
+
+#endif
+
 /*
  * XXX hack to avoid hangs when large processes fork.
  */
 u_int uvm_extrapages;
 
+uint64_t
+uvm_wait_prepare(void)
+{
+	uint64_t ticket;
+
+	ticket = uvmpd_read_gen();
+
+	return ticket;
+}
+
 /*
  * uvm_wait: wait (sleep) for the page daemon to free some pages
  *
  * => should be called with all locks released
  * => should _not_ be called by the page daemon (to avoid deadlock)
+ * => caller must have acquired a ticket with uvm_wait_prepare,
+ *    before trying an allocation that we now wait for
  */
 
 void
-uvm_wait(const char *wmsg)
+uvm_wait(const char *wmsg, uint64_t ticket)
 {
+	uint64_t gen = ticket;
 	int timo = 0;
 
 	if (uvm.pagedaemon_lwp == NULL)
 		panic("out of memory before the pagedaemon thread exists");
 
+	if (gen != uvmpd_read_gen())
+		return;
+
 	mutex_spin_enter(&uvmpd_lock);
 
+	if (gen != uvmpd_read_gen()) {
+		mutex_spin_exit(&uvmpd_lock);
+		return;
+	}
+
 	/*
 	 * check for page daemon going to sleep (waiting for itself)
 	 */
@@ -181,6 +270,7 @@ uvm_wait(const char *wmsg)
 	    atomic_load_relaxed(&uvm_pagedaemon_waiters) + 1);
 	membar_sync();
 	if (atomic_swap_uint(&uvm_waiter_wakeup, 0)) {
+		uvmpd_advance_gen();
 		atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
 		wakeup(&uvmexp.free);
 		mutex_spin_exit(&uvmpd_lock);
@@ -360,6 +450,7 @@ uvm_pageout(void *arg)
 		if (uvm_availmem(false) > uvmexp.reserve_kernel ||
 		    uvmexp.paging == 0) {
 			mutex_spin_enter(&uvmpd_lock);
+			uvmpd_advance_gen();
 			wakeup(&uvmexp.free);
 			atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
 			mutex_spin_exit(&uvmpd_lock);
@@ -417,6 +508,7 @@ uvm_pageout_done(int npages)
 		if (atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0)
 			return;
 		mutex_spin_enter(&uvmpd_lock);
+		uvmpd_advance_gen();
 		if (atomic_load_relaxed(&uvm_pagedaemon_waiters) != 0) {
 			wakeup(&uvmexp.free);
 			atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_pdaemon.h
--- a/sys/uvm/uvm_pdaemon.h	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_pdaemon.h	Tue Aug 18 16:13:20 2026 +0000
@@ -83,7 +83,8 @@ struct krwlock;
  * prototypes
  */
 
-void uvm_wait(const char *);
+uint64_t uvm_wait_prepare(void);
+void uvm_wait(const char *, uint64_t);
 bool uvm_reclaimable(void);
 
 struct krwlock *uvmpd_trylockowner(struct vm_page *);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_pglist.c
--- a/sys/uvm/uvm_pglist.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_pglist.c	Tue Aug 18 16:13:20 2026 +0000
@@ -300,6 +300,7 @@ static int
 uvm_pglistalloc_contig_aggressive(int num, paddr_t low, paddr_t high,
     paddr_t alignment, paddr_t boundary, struct pglist *rlist)
 {
+	uint64_t ticket;
 	struct vm_page *pg;
 	struct pglist tmp;
 	paddr_t pa, off, spa, amask, bmask, rlo, rhi;
@@ -321,9 +322,10 @@ uvm_pglistalloc_contig_aggressive(int nu
 	KASSERT(bmask <= amask);
 	mutex_enter(&uvm_pglistalloc_contig_lock);
 	while (uvm_reclaimable()) {
+		ticket = uvm_wait_prepare();
 		pg = uvm_pagealloc(NULL, 0, NULL, 0);
 		if (pg == NULL) {
-			uvm_wait("pglac2");
+			uvm_wait("pglac2", ticket);
 			continue;
 		}
 		pg->flags |= PG_PGLCA;
@@ -615,6 +617,7 @@ uvm_pglistalloc_simple(int num, paddr_t 
     struct pglist *rlist, int waitok)
 {
 	int fl, error;
+	uint64_t ticket;
 	uvm_physseg_t psi;
 	int count = 0;
 
@@ -623,6 +626,8 @@ uvm_pglistalloc_simple(int num, paddr_t 
 	bool valid = false;
 
 again:
+	ticket = uvm_wait_prepare();
+
 	/*
 	 * Block all memory allocation and lock the free list.
 	 */
@@ -673,7 +678,7 @@ out:
 
 	if (error) {
 		if (waitok) {
-			uvm_wait("pglalloc");
+			uvm_wait("pglalloc", ticket);
 			goto again;
 		} else
 			uvm_pglistfree(rlist);
diff -r e8f7aecf524e -r 45c035f4cc06 sys/uvm/uvm_vnode.c
--- a/sys/uvm/uvm_vnode.c	Mon Aug 17 17:26:38 2026 +0000
+++ b/sys/uvm/uvm_vnode.c	Tue Aug 18 16:13:20 2026 +0000
@@ -287,6 +287,7 @@ static int
 uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
     unsigned int flags, struct uvm_page_array *a, unsigned int nleft)
 {
+	uint64_t ticket;
 	struct vm_page *pg;
 	UVMHIST_FUNC(__func__);
 	UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)uobj, offset,
@@ -332,6 +333,7 @@ uvn_findpage(struct uvm_object *uobj, vo
 				UVMHIST_LOG(ubchist, "noalloc", 0,0,0,0);
 				return 0;
 			}
+			ticket = uvm_wait_prepare();
 			pg = uvm_pagealloc(uobj, offset, NULL,
 			    UVM_FLAG_COLORMATCH);
 			if (pg == NULL) {
@@ -340,7 +342,7 @@ uvn_findpage(struct uvm_object *uobj, vo
 					return 0;
 				}
 				rw_exit(uobj->vmobjlock);
-				uvm_wait("uvnfp1");
+				uvm_wait("uvnfp1", ticket);
 				uvm_page_array_clear(a);
 				rw_enter(uobj->vmobjlock, RW_WRITER);
 				continue;
# HG changeset patch
# User Taylor R Campbell <riastradh@NetBSD.org>
# Date 1787078155 0
#      Tue Aug 18 18:35:55 2026 +0000
# Branch trunk
# Node ID 603563208b3515ff18e03c86e0d41b2bab589940
# Parent  45c035f4cc0605040bf45290d66d625194c2bedf
# EXP-Topic riastradh-pr58964-uvmwaitwakeup
WIP: uvm_pdaemon: Sprinkle comments explaining wait/wakeup protocol.

PR kern/58964: uvm: missing wakeup on uvmexp.free
PR kern/60029: panic: cpu0: softints stuck for 16 seconds

diff -r 45c035f4cc06 -r 603563208b35 sys/uvm/uvm_pdaemon.c
--- a/sys/uvm/uvm_pdaemon.c	Tue Aug 18 16:13:20 2026 +0000
+++ b/sys/uvm/uvm_pdaemon.c	Tue Aug 18 18:35:55 2026 +0000
@@ -198,6 +198,15 @@ uvmpd_advance_gen(void)
  */
 u_int uvm_extrapages;
 
+/*
+ * uvm_wait_prepare: return a ticket for uvm_wait
+ *
+ * => caller must:
+ *    1. get a ticket _first_,
+ *    2. _then_ try allocation, and
+ *    3. uvm_wait with the ticket if it fails
+ */
+
 uint64_t
 uvm_wait_prepare(void)
 {
@@ -226,11 +235,22 @@ uvm_wait(const char *wmsg, uint64_t tick
 	if (uvm.pagedaemon_lwp == NULL)
 		panic("out of memory before the pagedaemon thread exists");
 
+	/*
+	 * Fast path: if the page daemon wakeup generation has already
+	 * advanced since the caller began uvm_wait_prepare, don't add
+	 * to contention on uvmpd_lock; we already know the caller can
+	 * retry its allocation.
+	 */
 	if (gen != uvmpd_read_gen())
 		return;
 
 	mutex_spin_enter(&uvmpd_lock);
 
+	/*
+	 * If the page daemon wakeup generation has advanced since the
+	 * caller began uvm_wait_prepare, don't sleep; the caller can
+	 * already retry its allocation.
+	 */
 	if (gen != uvmpd_read_gen()) {
 		mutex_spin_exit(&uvmpd_lock);
 		return;
@@ -239,7 +259,6 @@ uvm_wait(const char *wmsg, uint64_t tick
 	/*
 	 * check for page daemon going to sleep (waiting for itself)
 	 */
-
 	if (uvm_lwp_is_pagedaemon(curlwp) && uvmexp.paging == 0) {
 		/*
 		 * now we have a problem: the pagedaemon wants to go to
@@ -266,6 +285,30 @@ uvm_wait(const char *wmsg, uint64_t tick
 #endif
 	}
 
+	/*
+	 * Notify uvm_pageout (the page daemon) or uvm_pageout_done
+	 * (the paging completion handler) that there are waiters
+	 * before we check whether there's a pending unlocked wakeup.
+	 *
+	 * membar_sync here matches membar_sync in uvm_pageout_done
+	 * between storing uvm_waiter_wakeup = 1 and loading
+	 * uvm_pagedaemon_waiters.
+	 *
+	 * If there's pending unlocked wakeup, consume it by advancing
+	 * the wakeup generation and waking any other waiters -- that
+	 * can be done only with the lock.  Generally there should be
+	 * no other waiters at this point -- if there is a pending
+	 * wakeup, any other call to uvm_wait would have consumed it
+	 * and so can't be waiting; if another call to uvm_wait didn't
+	 * find a pending wakeup, then any subsequent uvm_pageout or
+	 * uvm_pageout_done wakeup should have noticed that call and
+	 * issued wakeup(&uvmexp.free) -- but it takes a more effort to
+	 * prove that it is safe to elide wakeup(&uvmexp.free) than it
+	 * takes to just call it anyway here.
+	 *
+	 * The wakeup might be spurious by now, but the caller has to
+	 * cope with spurious wakeups anyway.
+	 */
 	atomic_store_relaxed(&uvm_pagedaemon_waiters,
 	    atomic_load_relaxed(&uvm_pagedaemon_waiters) + 1);
 	membar_sync();
@@ -276,6 +319,7 @@ uvm_wait(const char *wmsg, uint64_t tick
 		mutex_spin_exit(&uvmpd_lock);
 		return;
 	}
+
 	wakeup(&uvm.pagedaemon);		/* wake the daemon! */
 	UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_lock, false, wmsg, timo);
 }
@@ -294,10 +338,31 @@ uvm_kick_pdaemon(void)
 	    (fpages + uvmexp.paging < uvmexp.freetarg &&
 	     uvmpdpol_needsscan_p()) ||
 	     uvm_km_va_starved_p()) {
+		/*
+		 * First make sure the page daemon won't go to sleep if
+		 * it's still working by setting uvmpd_wakeup := 1;
+		 * then check whether it is about to go to sleep, by
+		 * querying uvmpd_sleeping.
+		 *
+		 * If _after_ we have told it to wake up, it is not
+		 * sleeping, we can avoid adding to contention on
+		 * uvmpd_lock -- the page daemon is guaranteed to try
+		 * making the progress we want.
+		 *
+		 * membar_sync here matches membar_sync in uvm_pageout
+		 * between storing uvmpd_sleeping = 1 and loading
+		 * uvmpd_wakeup.
+		 */
 		atomic_store_relaxed(&uvmpd_wakeup, 1);
 		membar_sync();
 		if (!atomic_load_relaxed(&uvmpd_sleeping))
 			return;
+
+		/*
+		 * The page daemon has already decided to go to sleep,
+		 * and it may be too late for us to stop it.  Wake it
+		 * up.
+		 */
 	     	mutex_spin_enter(&uvmpd_lock);
 		wakeup(&uvm.pagedaemon);
 	     	mutex_spin_exit(&uvmpd_lock);
@@ -398,6 +463,29 @@ uvm_pageout(void *arg)
 			uvmexp.paging > 0) &&
 		    !kmem_va_starved) {
 			UVMHIST_LOG(pdhist,"  <<SLEEPING>>",0,0,0,0);
+
+			/*
+			 * Notify uvm_kick_pdaemon that we're about to
+			 * go to sleep.  Then check whether it has
+			 * already asked us to wake up again before it
+			 * puts contention on uvmpd_lock.  If, after we
+			 * have announced our intent to sleep, there's
+			 * still no pending wakeup, go to sleep and
+			 * release uvm_pdlock.
+			 *
+			 * membar_sync here matches membar_sync in
+			 * uvm_pageout_done between storing
+			 * uvmpd_wakeup := 1 and loading
+			 * uvmpd_sleeping.
+			 *
+			 * Once we're not asleep -- either because
+			 * uvmpd_wakeup = 1 or because of a concurrent
+			 * wakeup(&uvm.pagedaemon) -- set
+			 * uvmpd_sleeping back to 0 again so that while
+			 * we work, uvm_kick_pdaemon need not take
+			 * uvmpd_lock in order to make sure we wake up
+			 * next time.
+			 */
 			atomic_store_relaxed(&uvmpd_sleeping, 1);
 			membar_sync();
 			if (atomic_swap_uint(&uvmpd_wakeup, 0)) {
@@ -408,6 +496,7 @@ uvm_pageout(void *arg)
 			}
 			atomic_store_relaxed(&uvmpd_sleeping, 0);
 			uvmexp.pdwoke++;
+
 			UVMHIST_LOG(pdhist,"  <<WOKE UP>>",0,0,0,0);
 		} else {
 			mutex_spin_exit(&uvmpd_lock);
@@ -495,18 +584,73 @@ uvm_pageout_done(int npages)
 	 * wake up either of pagedaemon or LWPs waiting for it.
 	 */
 	if (uvm_availmem(false) <= uvmexp.reserve_kernel) {
+		/*
+		 * If the page daemon is still working, ask it not to
+		 * go to sleep when it's done with this iteration.  If,
+		 * after we have asked that by setting uvmpd_wakeup :=
+		 * 1, the page daemon still hasn't announced its intent
+		 * to sleep, we can stop here -- once it reaches the
+		 * next iteration, it will notice uvmpd_wakeup = 1 and
+		 * avoid sleeping.  This way we avoid putting
+		 * contention on uvmpd_lock when the page daemon is
+		 * busy working.
+		 *
+		 * membar_sync here matches membar_sync in uvm_pageout
+		 * between storing uvmpd_sleeping := 1 and swapping
+		 * uvmpd_wakeup with 0.
+		 */
 		atomic_store_relaxed(&uvmpd_wakeup, 1);
 		membar_sync();
 		if (!atomic_load_relaxed(&uvmpd_sleeping))
 			return;
+
+		/*
+		 * Page daemon is asleep or about to go to sleep, and
+		 * it may be too late for us to stop it, so we have to
+		 * take the lock to wake it up.
+		 */
 		mutex_spin_enter(&uvmpd_lock);
 		wakeup(&uvm.pagedaemon);
 		mutex_spin_exit(&uvmpd_lock);
 	} else {
+		/*
+		 * Make sure the next call to uvm_wait will wake up
+		 * even if we don't take uvmpd_lock, and then check --
+		 * again without taking uvmpd_lock -- whether there are
+		 * any waiters that might be sleeping.  If not, we're
+		 * done and we don't need to put contention on
+		 * uvmpd_lock.
+		 *
+		 * membar_sync here matches membar_sync in uvm_wait.
+		 *
+		 * Note that the next call to uvm_wait will necessarily
+		 * notice uvm_waiter_wakeup = 1 and act on it by
+		 * advancing the page daemon wakeup generation (under
+		 * the lock), so we don't need to do that here.
+		 */
 		atomic_store_relaxed(&uvm_waiter_wakeup, 1);
 		membar_sync();
 		if (atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0)
 			return;
+
+		/*
+		 * uvm_wait waiters are either asleep or about to go to
+		 * sleep, and it may be too late to stop them, so we
+		 * have to take the lock to wake them up.
+		 *
+		 * Make sure to advance the generation so that logic
+		 * still in the ... part of
+		 *
+		 * top:	ticket = uvm_wait_prepare();
+		 *	...
+		 *	if (allocfailed) {
+		 *		uvm_wait("wmesg", ticket);
+		 *		goto top;
+		 *	}
+		 *
+		 * will wake up and not sleep forever when it hits
+		 * uvm_wait.
+		 */
 		mutex_spin_enter(&uvmpd_lock);
 		uvmpd_advance_gen();
 		if (atomic_load_relaxed(&uvm_pagedaemon_waiters) != 0) {
# HG changeset patch
# User Taylor R Campbell <riastradh@NetBSD.org>
# Date 1787143885 0
#      Wed Aug 19 12:51:25 2026 +0000
# Branch trunk
# Node ID 2f7d94cea6633d48b951ce99732e5383b697e925
# Parent  603563208b3515ff18e03c86e0d41b2bab589940
# EXP-Topic riastradh-pr58964-uvmwaitwakeup
WIP: uvm: Split uvmpd_lock up.

Into:

1. uvmpd_waiter_lock at IPL_SOFTBIO, for coordinating uvm_wait with
   either uvm_pageout (page daemon) or uvm_pageout_done (paging
   completion handler called from softbio)

2. uvmpd_kick_lock at IPL_VM, for coordinating uvm_kick_pdaemon (and
   the page daemon wakeup part of uvm_wait) with page daemon

3. uvmpd_pool_drain_lock at IPL_NONE, for coordinating the page
   daemon thread with the pool draining thread

PR kern/60029: panic: cpu0: softints stuck for 16 seconds

diff -r 603563208b35 -r 2f7d94cea663 sys/uvm/uvm_pdaemon.c
--- a/sys/uvm/uvm_pdaemon.c	Tue Aug 18 18:35:55 2026 +0000
+++ b/sys/uvm/uvm_pdaemon.c	Wed Aug 19 12:51:25 2026 +0000
@@ -114,14 +114,16 @@ static void	uvmpd_tune(void);
 static void	uvmpd_pool_drain_thread(void *);
 static void	uvmpd_pool_drain_wakeup(void);
 
+static kmutex_t uvmpd_waiter_lock __cacheline_aligned;
 static volatile unsigned uvm_pagedaemon_waiters __cacheline_aligned;
 static volatile unsigned uvm_waiter_wakeup __cacheline_aligned;
 
 /* State for the pool drainer thread */
-static kmutex_t uvmpd_lock __cacheline_aligned;
+static kmutex_t uvmpd_pool_drain_lock;
 static kcondvar_t uvmpd_pool_drain_cv;
 static bool uvmpd_pool_drain_run = false;
 
+static kmutex_t uvmpd_kick_lock __cacheline_aligned;
 static volatile unsigned uvmpd_sleeping __cacheline_aligned;
 static volatile unsigned uvmpd_wakeup __cacheline_aligned;
 
@@ -140,7 +142,7 @@ static void
 uvmpd_advance_gen(void)
 {
 
-	KASSERT(mutex_owned(&uvmpd_lock));
+	KASSERT(mutex_owned(&uvmpd_waiter_lock));
 	atomic_store_relaxed(&uvmpd_gen, atomic_load_relaxed(&uvmpd_gen) + 1);
 }
 
@@ -176,7 +178,7 @@ uvmpd_advance_gen(void)
 {
 	uint32_t hi, lo;
 
-	KASSERT(mutex_owned(&uvmpd_lock));
+	KASSERT(mutex_owned(&uvmpd_waiter_lock));
 
 	lo = atomic_load_relaxed(&uvmpd_gen.lo);
 	if (__predict_false(++lo == 0)) {
@@ -238,13 +240,13 @@ uvm_wait(const char *wmsg, uint64_t tick
 	/*
 	 * Fast path: if the page daemon wakeup generation has already
 	 * advanced since the caller began uvm_wait_prepare, don't add
-	 * to contention on uvmpd_lock; we already know the caller can
-	 * retry its allocation.
+	 * to contention on uvmpd_waiter_lock; we already know the
+	 * caller can retry its allocation.
 	 */
 	if (gen != uvmpd_read_gen())
 		return;
 
-	mutex_spin_enter(&uvmpd_lock);
+	mutex_enter(&uvmpd_waiter_lock);
 
 	/*
 	 * If the page daemon wakeup generation has advanced since the
@@ -252,7 +254,7 @@ uvm_wait(const char *wmsg, uint64_t tick
 	 * already retry its allocation.
 	 */
 	if (gen != uvmpd_read_gen()) {
-		mutex_spin_exit(&uvmpd_lock);
+		mutex_exit(&uvmpd_waiter_lock);
 		return;
 	}
 
@@ -316,12 +318,20 @@ uvm_wait(const char *wmsg, uint64_t tick
 		uvmpd_advance_gen();
 		atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
 		wakeup(&uvmexp.free);
-		mutex_spin_exit(&uvmpd_lock);
+		mutex_exit(&uvmpd_waiter_lock);
 		return;
 	}
 
-	wakeup(&uvm.pagedaemon);		/* wake the daemon! */
-	UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_lock, false, wmsg, timo);
+	atomic_store_relaxed(&uvmpd_wakeup, 1);
+	membar_sync();
+	if (atomic_load_relaxed(&uvmpd_sleeping)) {
+		mutex_spin_enter(&uvmpd_kick_lock);
+		wakeup(&uvm.pagedaemon);		/* wake the daemon! */
+		mutex_spin_exit(&uvmpd_kick_lock);
+	}
+
+	UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_waiter_lock, false, wmsg,
+	    timo);
 }
 
 /*
@@ -346,8 +356,8 @@ uvm_kick_pdaemon(void)
 		 *
 		 * If _after_ we have told it to wake up, it is not
 		 * sleeping, we can avoid adding to contention on
-		 * uvmpd_lock -- the page daemon is guaranteed to try
-		 * making the progress we want.
+		 * uvmpd_kick_lock -- the page daemon is guaranteed to
+		 * try making the progress we want.
 		 *
 		 * membar_sync here matches membar_sync in uvm_pageout
 		 * between storing uvmpd_sleeping = 1 and loading
@@ -363,9 +373,9 @@ uvm_kick_pdaemon(void)
 		 * and it may be too late for us to stop it.  Wake it
 		 * up.
 		 */
-	     	mutex_spin_enter(&uvmpd_lock);
+	     	mutex_spin_enter(&uvmpd_kick_lock);
 		wakeup(&uvm.pagedaemon);
-	     	mutex_spin_exit(&uvmpd_lock);
+	     	mutex_spin_exit(&uvmpd_kick_lock);
 	}
 }
 
@@ -433,7 +443,9 @@ uvm_pageout(void *arg)
 
 	UVMHIST_LOG(pdhist,"<starting uvm pagedaemon>", 0, 0, 0, 0);
 
-	mutex_init(&uvmpd_lock, MUTEX_DEFAULT, IPL_VM);
+	mutex_init(&uvmpd_kick_lock, MUTEX_DEFAULT, IPL_VM);
+	mutex_init(&uvmpd_waiter_lock, MUTEX_DEFAULT, IPL_SOFTBIO);
+	mutex_init(&uvmpd_pool_drain_lock, MUTEX_DEFAULT, IPL_NONE);
 	cv_init(&uvmpd_pool_drain_cv, "pooldrain");
 
 	/* Create the pool drainer kernel thread. */
@@ -458,7 +470,7 @@ uvm_pageout(void *arg)
 
 		kmem_va_starved = uvm_km_va_starved_p();
 
-		mutex_spin_enter(&uvmpd_lock);
+		mutex_spin_enter(&uvmpd_kick_lock);
 		if ((atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0 ||
 			uvmexp.paging > 0) &&
 		    !kmem_va_starved) {
@@ -468,10 +480,10 @@ uvm_pageout(void *arg)
 			 * Notify uvm_kick_pdaemon that we're about to
 			 * go to sleep.  Then check whether it has
 			 * already asked us to wake up again before it
-			 * puts contention on uvmpd_lock.  If, after we
-			 * have announced our intent to sleep, there's
-			 * still no pending wakeup, go to sleep and
-			 * release uvm_pdlock.
+			 * puts contention on uvmpd_kick_lock.  If,
+			 * after we have announced our intent to sleep,
+			 * there's still no pending wakeup, go to sleep
+			 * and release uvm_pdlock.
 			 *
 			 * membar_sync here matches membar_sync in
 			 * uvm_pageout_done between storing
@@ -483,23 +495,23 @@ uvm_pageout(void *arg)
 			 * wakeup(&uvm.pagedaemon) -- set
 			 * uvmpd_sleeping back to 0 again so that while
 			 * we work, uvm_kick_pdaemon need not take
-			 * uvmpd_lock in order to make sure we wake up
-			 * next time.
+			 * uvmpd_kick_lock in order to make sure we
+			 * wake up next time.
 			 */
 			atomic_store_relaxed(&uvmpd_sleeping, 1);
 			membar_sync();
 			if (atomic_swap_uint(&uvmpd_wakeup, 0)) {
-				mutex_spin_exit(&uvmpd_lock);
+				mutex_spin_exit(&uvmpd_kick_lock);
 			} else {
 				UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon,
-				    &uvmpd_lock, false, "pgdaemon", 0);
+				    &uvmpd_kick_lock, false, "pgdaemon", 0);
 			}
 			atomic_store_relaxed(&uvmpd_sleeping, 0);
 			uvmexp.pdwoke++;
 
 			UVMHIST_LOG(pdhist,"  <<WOKE UP>>",0,0,0,0);
 		} else {
-			mutex_spin_exit(&uvmpd_lock);
+			mutex_spin_exit(&uvmpd_kick_lock);
 		}
 
 		/*
@@ -538,11 +550,11 @@ uvm_pageout(void *arg)
 		 */
 		if (uvm_availmem(false) > uvmexp.reserve_kernel ||
 		    uvmexp.paging == 0) {
-			mutex_spin_enter(&uvmpd_lock);
+			mutex_enter(&uvmpd_waiter_lock);
 			uvmpd_advance_gen();
 			wakeup(&uvmexp.free);
 			atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
-			mutex_spin_exit(&uvmpd_lock);
+			mutex_exit(&uvmpd_waiter_lock);
 		}
 
 		/*
@@ -592,8 +604,8 @@ uvm_pageout_done(int npages)
 		 * to sleep, we can stop here -- once it reaches the
 		 * next iteration, it will notice uvmpd_wakeup = 1 and
 		 * avoid sleeping.  This way we avoid putting
-		 * contention on uvmpd_lock when the page daemon is
-		 * busy working.
+		 * contention on uvmpd_kick_lock when the page daemon
+		 * is busy working.
 		 *
 		 * membar_sync here matches membar_sync in uvm_pageout
 		 * between storing uvmpd_sleeping := 1 and swapping
@@ -609,17 +621,17 @@ uvm_pageout_done(int npages)
 		 * it may be too late for us to stop it, so we have to
 		 * take the lock to wake it up.
 		 */
-		mutex_spin_enter(&uvmpd_lock);
+		mutex_spin_enter(&uvmpd_kick_lock);
 		wakeup(&uvm.pagedaemon);
-		mutex_spin_exit(&uvmpd_lock);
+		mutex_spin_exit(&uvmpd_kick_lock);
 	} else {
 		/*
 		 * Make sure the next call to uvm_wait will wake up
-		 * even if we don't take uvmpd_lock, and then check --
-		 * again without taking uvmpd_lock -- whether there are
-		 * any waiters that might be sleeping.  If not, we're
-		 * done and we don't need to put contention on
-		 * uvmpd_lock.
+		 * even if we don't take uvmpd_waiter_lock, and then
+		 * check -- again without taking uvmpd_waiter_lock --
+		 * whether there are any waiters that might be
+		 * sleeping.  If not, we're done and we don't need to
+		 * put contention on uvmpd_waiter_lock.
 		 *
 		 * membar_sync here matches membar_sync in uvm_wait.
 		 *
@@ -651,13 +663,13 @@ uvm_pageout_done(int npages)
 		 * will wake up and not sleep forever when it hits
 		 * uvm_wait.
 		 */
-		mutex_spin_enter(&uvmpd_lock);
+		mutex_enter(&uvmpd_waiter_lock);
 		uvmpd_advance_gen();
 		if (atomic_load_relaxed(&uvm_pagedaemon_waiters) != 0) {
 			wakeup(&uvmexp.free);
 			atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
 		}
-		mutex_spin_exit(&uvmpd_lock);
+		mutex_exit(&uvmpd_waiter_lock);
 	}
 }
 
@@ -1295,17 +1307,17 @@ uvmpd_pool_drain_thread(void *arg)
 		/*
 		 * sleep until awoken by the pagedaemon.
 		 */
-		mutex_enter(&uvmpd_lock);
+		mutex_enter(&uvmpd_pool_drain_lock);
 		if (!uvmpd_pool_drain_run) {
 			lastslept = getticks();
-			cv_wait(&uvmpd_pool_drain_cv, &uvmpd_lock);
+			cv_wait(&uvmpd_pool_drain_cv, &uvmpd_pool_drain_lock);
 			if (getticks() != lastslept) {
 				cycled = false;
 				firstpool = NULL;
 			}
 		}
 		uvmpd_pool_drain_run = false;
-		mutex_exit(&uvmpd_lock);
+		mutex_exit(&uvmpd_pool_drain_lock);
 
 		/*
 		 * rate limit draining, otherwise in desperate circumstances
@@ -1352,8 +1364,8 @@ static void
 uvmpd_pool_drain_wakeup(void)
 {
 
-	mutex_enter(&uvmpd_lock);
+	mutex_enter(&uvmpd_pool_drain_lock);
 	uvmpd_pool_drain_run = true;
 	cv_signal(&uvmpd_pool_drain_cv);
-	mutex_exit(&uvmpd_lock);
+	mutex_exit(&uvmpd_pool_drain_lock);
 }
# HG changeset patch
# User Taylor R Campbell <riastradh@NetBSD.org>
# Date 1787144449 0
#      Wed Aug 19 13:00:49 2026 +0000
# Branch trunk
# Node ID 79c345a21435b7915d410db30881fd49678bd62c
# Parent  2f7d94cea6633d48b951ce99732e5383b697e925
# EXP-Topic riastradh-pr58964-uvmwaitwakeup
WIP: uvm: Back off if a wakeup is already pending.

PR kern/60029: panic: cpu0: softints stuck for 16 seconds

diff -r 2f7d94cea663 -r 79c345a21435 sys/uvm/uvm_pdaemon.c
--- a/sys/uvm/uvm_pdaemon.c	Wed Aug 19 12:51:25 2026 +0000
+++ b/sys/uvm/uvm_pdaemon.c	Wed Aug 19 13:00:49 2026 +0000
@@ -322,12 +322,13 @@ uvm_wait(const char *wmsg, uint64_t tick
 		return;
 	}
 
-	atomic_store_relaxed(&uvmpd_wakeup, 1);
-	membar_sync();
-	if (atomic_load_relaxed(&uvmpd_sleeping)) {
-		mutex_spin_enter(&uvmpd_kick_lock);
-		wakeup(&uvm.pagedaemon);		/* wake the daemon! */
-		mutex_spin_exit(&uvmpd_kick_lock);
+	if (!atomic_swap_uint(&uvmpd_wakeup, 1)) {
+		membar_sync();
+		if (atomic_load_relaxed(&uvmpd_sleeping)) {
+			mutex_spin_enter(&uvmpd_kick_lock);
+			wakeup(&uvm.pagedaemon); /* wake the daemon! */
+			mutex_spin_exit(&uvmpd_kick_lock);
+		}
 	}
 
 	UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_waiter_lock, false, wmsg,
@@ -363,7 +364,8 @@ uvm_kick_pdaemon(void)
 		 * between storing uvmpd_sleeping = 1 and loading
 		 * uvmpd_wakeup.
 		 */
-		atomic_store_relaxed(&uvmpd_wakeup, 1);
+		if (atomic_swap_uint(&uvmpd_wakeup, 1))
+			return;
 		membar_sync();
 		if (!atomic_load_relaxed(&uvmpd_sleeping))
 			return;
@@ -611,7 +613,8 @@ uvm_pageout_done(int npages)
 		 * between storing uvmpd_sleeping := 1 and swapping
 		 * uvmpd_wakeup with 0.
 		 */
-		atomic_store_relaxed(&uvmpd_wakeup, 1);
+		if (atomic_swap_uint(&uvmpd_wakeup, 1))
+			return;
 		membar_sync();
 		if (!atomic_load_relaxed(&uvmpd_sleeping))
 			return;
@@ -640,7 +643,8 @@ uvm_pageout_done(int npages)
 		 * advancing the page daemon wakeup generation (under
 		 * the lock), so we don't need to do that here.
 		 */
-		atomic_store_relaxed(&uvm_waiter_wakeup, 1);
+		if (atomic_swap_uint(&uvm_waiter_wakeup, 1))
+			return;
 		membar_sync();
 		if (atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0)
 			return;