diff -r 38311baef8db -r 79c345a21435 sys/arch/aarch64/aarch64/pmap.c --- a/sys/arch/aarch64/aarch64/pmap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/aarch64/aarch64/pmap.c Wed Aug 19 13:00:49 2026 +0000 @@ -552,6 +552,7 @@ static paddr_t pmap_alloc_pdp(struct pmap *pm, struct vm_page **pgp, int flags, bool waitok) { paddr_t pa; + uint64_t ticket; struct vm_page *pg; UVMHIST_FUNC(__func__); @@ -562,10 +563,11 @@ pmap_alloc_pdp(struct pmap *pm, struct v int aflags = ((flags & PMAP_CANFAIL) ? 0 : UVM_PGA_USERESERVE) | UVM_PGA_ZERO; retry: + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, 0, NULL, aflags); if (pg == NULL) { if (waitok) { - uvm_wait("pmap_alloc_pdp"); + uvm_wait("pmap_alloc_pdp", ticket); goto retry; } return POOL_PADDR_INVALID; diff -r 38311baef8db -r 79c345a21435 sys/arch/aarch64/include/asan.h --- a/sys/arch/aarch64/include/asan.h Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/aarch64/include/asan.h Wed Aug 19 13:00:49 2026 +0000 @@ -79,11 +79,13 @@ static paddr_t va = uvm_pageboot_alloc(PAGE_SIZE); pa = AARCH64_KVA_TO_PA(va); } else { + uint64_t ticket; struct vm_page *pg; retry: + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, 0, NULL, 0); if (pg == NULL) { - uvm_wait(__func__); + uvm_wait(__func__, ticket); goto retry; } diff -r 38311baef8db -r 79c345a21435 sys/arch/amd64/amd64/gdt.c --- a/sys/arch/amd64/amd64/gdt.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/amd64/amd64/gdt.c Wed Aug 19 13:00:49 2026 +0000 @@ -184,6 +184,7 @@ gdt_alloc_cpu(struct cpu_info *ci) #ifdef __HAVE_PCPU_AREA ci->ci_gdt = (union descriptor *)&pcpuarea->ent[cpu_index(ci)].gdt; #else + uint64_t ticket; struct vm_page *pg; vaddr_t va; @@ -191,9 +192,10 @@ gdt_alloc_cpu(struct cpu_info *ci) 0, UVM_KMF_VAONLY); for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + gdt_size; va += PAGE_SIZE) { - while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) == NULL) { - uvm_wait("gdt_alloc_cpu"); + uvm_wait("gdt_alloc_cpu", ticket); } pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE, 0); diff -r 38311baef8db -r 79c345a21435 sys/arch/arm/include/asan.h --- a/sys/arch/arm/include/asan.h Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/arm/include/asan.h Wed Aug 19 13:00:49 2026 +0000 @@ -111,11 +111,13 @@ static vaddr_t return pa; } + uint64_t ticket; struct vm_page *pg; retry: + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, 0, NULL, 0); if (pg == NULL) { - uvm_wait(__func__); + uvm_wait(__func__, ticket); goto retry; } pa = VM_PAGE_TO_PHYS(pg); diff -r 38311baef8db -r 79c345a21435 sys/arch/i386/i386/gdt.c --- a/sys/arch/i386/i386/gdt.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/i386/i386/gdt.c Wed Aug 19 13:00:49 2026 +0000 @@ -184,6 +184,7 @@ gdt_init(void) void gdt_alloc_cpu(struct cpu_info *ci) { + uint64_t ticket; struct vm_page *pg; vaddr_t va; @@ -191,9 +192,10 @@ gdt_alloc_cpu(struct cpu_info *ci) 0, UVM_KMF_VAONLY); for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + gdt_size; va += PAGE_SIZE) { - while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) == NULL) { - uvm_wait("gdt_alloc_cpu"); + uvm_wait("gdt_alloc_cpu", ticket); } pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE, 0); diff -r 38311baef8db -r 79c345a21435 sys/arch/m68k/m68k/pmap_68k.c --- a/sys/arch/m68k/m68k/pmap_68k.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/m68k/m68k/pmap_68k.c Wed Aug 19 13:00:49 2026 +0000 @@ -643,14 +643,16 @@ pmap_ptpage_init(void) static struct vm_page * pmap_page_alloc(bool nowait) { + uint64_t ticket; struct vm_page *pg; const int flags = nowait ? UVM_PGA_USERESERVE : 0; - while ((pg = uvm_pagealloc(NULL, 0, NULL, flags)) == NULL) { + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(NULL, 0, NULL, flags)) == NULL) { if (nowait) { return NULL; } - uvm_wait("pmappg"); + uvm_wait("pmappg", ticket); } pg->flags &= ~PG_BUSY; /* never busy */ diff -r 38311baef8db -r 79c345a21435 sys/arch/m68k/m68k/pmap_motorola.c --- a/sys/arch/m68k/m68k/pmap_motorola.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/m68k/m68k/pmap_motorola.c Wed Aug 19 13:00:49 2026 +0000 @@ -2551,6 +2551,7 @@ int pmap_enter_ptpage(pmap_t pmap, vaddr_t va, bool can_fail) { paddr_t ptpa; + uint64_t ticket; struct vm_page *pg; struct pv_header *pvh; struct pv_entry *pv; @@ -2683,7 +2684,6 @@ pmap_enter_ptpage(pmap_t pmap, vaddr_t v #endif splx(s); } else { - /* * For user processes we just allocate a page from the * VM system. Note that we set the page "wired" count to 1, @@ -2698,15 +2698,16 @@ pmap_enter_ptpage(pmap_t pmap, vaddr_t v PMAP_DPRINTF(PDB_ENTER|PDB_PTPAGE, ("enter: about to alloc UPT pg at %lx\n", va)); rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); - while ((pg = uvm_pagealloc(uvm_kernel_object, - va - vm_map_min(kernel_map), - NULL, UVM_PGA_ZERO)) == NULL) { + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(uvm_kernel_object, + va - vm_map_min(kernel_map), + NULL, UVM_PGA_ZERO)) == NULL) { rw_exit(uvm_kernel_object->vmobjlock); if (can_fail) { pmap->pm_sref--; return ENOMEM; } - uvm_wait("ptpage"); + uvm_wait("ptpage", ticket); rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); } rw_exit(uvm_kernel_object->vmobjlock); diff -r 38311baef8db -r 79c345a21435 sys/arch/powerpc/oea/pmap.c --- a/sys/arch/powerpc/oea/pmap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/powerpc/oea/pmap.c Wed Aug 19 13:00:49 2026 +0000 @@ -2885,6 +2885,7 @@ pmap_free_unmanaged(struct pup *pup) void * pmap_pool_alloc(struct pool *pp, int flags) { + uint64_t ticket; struct vm_page *pg; paddr_t pa; @@ -2892,6 +2893,7 @@ pmap_pool_alloc(struct pool *pp, int fla return (void *)uvm_pageboot_alloc(PAGE_SIZE); retry: + ticket = uvm_wait_prepare(); pg = uvm_pagealloc_strat(NULL /*obj*/, 0 /*off*/, NULL /*anon*/, UVM_PGA_USERESERVE /*flags*/, UVM_PGA_STRAT_ONLY /*strat*/, VM_FREELIST_DIRECT_MAPPED /*free_list*/); @@ -2902,7 +2904,7 @@ pmap_pool_alloc(struct pool *pp, int fla if ((flags & PR_WAITOK) == 0) return NULL; - uvm_wait("plpg"); + uvm_wait("plpg", ticket); goto retry; } KDASSERT(VM_PAGE_TO_PHYS(pg) == (uintptr_t)VM_PAGE_TO_PHYS(pg)); diff -r 38311baef8db -r 79c345a21435 sys/arch/sparc/sparc/pmap.c --- a/sys/arch/sparc/sparc/pmap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/sparc/sparc/pmap.c Wed Aug 19 13:00:49 2026 +0000 @@ -907,14 +907,16 @@ void * pgt_page_alloc(struct pool *pp, int flags) { int cacheit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0; + uint64_t ticket; struct vm_page *pg; vaddr_t va; paddr_t pa; /* Allocate a page of physical memory */ - while ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL && - (flags & PR_WAITOK) != 0) { - uvm_wait("pgtpg"); + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL && + (flags & PR_WAITOK) != 0) { + uvm_wait("pgtpg", ticket); } if (pg == NULL) { KASSERT((flags & PR_WAITOK) == 0); diff -r 38311baef8db -r 79c345a21435 sys/arch/sparc64/sparc64/pmap.c --- a/sys/arch/sparc64/sparc64/pmap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/sparc64/sparc64/pmap.c Wed Aug 19 13:00:49 2026 +0000 @@ -1475,8 +1475,11 @@ pmap_create(void) pm->pm_refs = 1; TAILQ_INIT(&pm->pm_ptps); if (pm != pmap_kernel()) { - while (!pmap_get_page(&pm->pm_physaddr)) { - uvm_wait("pmap_create"); + uint64_t ticket; + + while (ticket = uvm_wait_prepare(), + !pmap_get_page(&pm->pm_physaddr)) { + uvm_wait("pmap_create", ticket); } pm->pm_segs = (paddr_t *)(u_long)pm->pm_physaddr; } diff -r 38311baef8db -r 79c345a21435 sys/rump/librump/rumpkern/vm.c --- a/sys/rump/librump/rumpkern/vm.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/rump/librump/rumpkern/vm.c Wed Aug 19 13:00:49 2026 +0000 @@ -106,6 +106,7 @@ struct pmap *const kernel_pmap_ptr = &pm vmem_t *kmem_arena; vmem_t *kmem_va_arena; +static uint64_t uvm_pdaemon_ticket; static unsigned int pdaemon_waiters; static kmutex_t pdaemonmtx; static kcondvar_t pdaemoncv, oomwait; @@ -1101,8 +1102,20 @@ uvm_uarea_free(vaddr_t uarea) * Routines related to the Page Baroness. */ +uint64_t +uvm_wait_prepare(void) +{ + uint64_t ticket; + + mutex_enter(&pdaemonmtx); + ticket = uvm_pdaemon_ticket; + mutex_exit(&pdaemonmtx); + + return ticket; +} + void -uvm_wait(const char *msg) +uvm_wait(const char *msg, uint64_t ticket) { if (__predict_false(rump_threads == 0)) @@ -1115,10 +1128,12 @@ uvm_wait(const char *msg) } mutex_enter(&pdaemonmtx); + if (ticket != uvm_pdaemon_ticket) + goto out; pdaemon_waiters++; cv_signal(&pdaemoncv); cv_wait(&oomwait, &pdaemonmtx); - mutex_exit(&pdaemonmtx); +out: mutex_exit(&pdaemonmtx); } void @@ -1141,6 +1156,7 @@ uvm_pageout_done(int npages) KASSERT(uvmexp.paging >= npages); uvmexp.paging -= npages; + uvm_pdaemon_ticket++; if (pdaemon_waiters) { pdaemon_waiters = 0; cv_broadcast(&oomwait); @@ -1185,6 +1201,7 @@ uvm_pageout(void *arg) mutex_enter(&pdaemonmtx); for (;;) { + uvm_pdaemon_ticket++; if (pdaemon_waiters) { pdaemon_waiters = 0; cv_broadcast(&oomwait); @@ -1309,6 +1326,7 @@ rump_hypermalloc(size_t howmuch, int ali { const unsigned long thelimit = uvm_lwp_is_pagedaemon(curlwp) ? pdlimit : rump_physmemlimit; + uint64_t ticket; unsigned long newmem; void *rv; int error; @@ -1318,22 +1336,24 @@ rump_hypermalloc(size_t howmuch, int ali /* first we must be within the limit */ limitagain: if (thelimit != RUMPMEM_UNLIMITED) { + ticket = uvm_wait_prepare(); newmem = atomic_add_long_nv(&curphysmem, howmuch); if (newmem > thelimit) { newmem = atomic_add_long_nv(&curphysmem, -howmuch); if (!waitok) { return NULL; } - uvm_wait(wmsg); + uvm_wait(wmsg, ticket); goto limitagain; } } /* second, we must get something from the backend */ again: + ticket = uvm_wait_prepare(); error = rumpuser_malloc(howmuch, alignment, &rv); if (__predict_false(error && waitok)) { - uvm_wait(wmsg); + uvm_wait(wmsg, ticket); goto again; } diff -r 38311baef8db -r 79c345a21435 sys/uvm/pmap/pmap_segtab.c --- a/sys/uvm/pmap/pmap_segtab.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/pmap/pmap_segtab.c Wed Aug 19 13:00:49 2026 +0000 @@ -584,6 +584,8 @@ pmap_pdetab_alloc(struct pmap *pmap) struct vm_page *ptb_pg = NULL; if (__predict_false(ptb == NULL)) { + uint64_t ticket = uvm_wait_prepare(); + ptb_pg = pmap_pte_pagealloc(); UVMHIST_LOG(pmapxtabhist, "ptb_pg=%#jx", @@ -592,7 +594,7 @@ pmap_pdetab_alloc(struct pmap *pmap) /* * XXX What else can we do? Could we deadlock here? */ - uvm_wait("pdetab"); + uvm_wait("pdetab", ticket); goto again; } @@ -658,13 +660,15 @@ pmap_segtab_alloc(struct pmap *pmap) struct vm_page *stb_pg = NULL; if (__predict_false(stb == NULL)) { + uint64_t ticket = uvm_wait_prepare(); + stb_pg = pmap_pte_pagealloc(); if (__predict_false(stb_pg == NULL)) { /* * XXX What else can we do? Could we deadlock here? */ - uvm_wait("segtab"); + uvm_wait("segtab", ticket); goto again; } SEGTAB_ADD(npage, 1); diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_amap.c --- a/sys/uvm/uvm_amap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_amap.c Wed Aug 19 13:00:49 2026 +0000 @@ -1026,6 +1026,7 @@ void amap_cow_now(struct vm_map *map, struct vm_map_entry *entry) { struct vm_amap *amap = entry->aref.ar_amap; + uint64_t ticket; struct vm_anon *anon, *nanon; struct vm_page *pg, *npg; u_int lcv, slot; @@ -1089,6 +1090,7 @@ ReStart: * First - get a new anon and a page. */ + ticket = uvm_wait_prepare(); nanon = uvm_analloc(); if (nanon) { nanon->an_lock = amap->am_lock; @@ -1104,7 +1106,7 @@ ReStart: KASSERT(nanon->an_ref == 0); uvm_anfree(nanon); } - uvm_wait("cownowpage"); + uvm_wait("cownowpage", ticket); goto ReStart; } diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_aobj.c --- a/sys/uvm/uvm_aobj.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_aobj.c Wed Aug 19 13:00:49 2026 +0000 @@ -806,6 +806,7 @@ uao_get(struct uvm_object *uobj, voff_t int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags) { voff_t current_offset; + uint64_t ticket; struct vm_page *ptmp; int lcv, gotpages, maxpages, swslot, pageidx; bool overwrite = ((flags & PGO_OVERWRITE) != 0); @@ -958,6 +959,7 @@ uao_get(struct uvm_object *uobj, voff_t pageidx = current_offset >> PAGE_SHIFT; swslot = uao_find_swslot(uobj, pageidx); + ticket = uvm_wait_prepare(); ptmp = uao_pagealloc(uobj, current_offset, swslot != 0 || overwrite ? 0 : UVM_PGA_ZERO); @@ -965,7 +967,7 @@ uao_get(struct uvm_object *uobj, voff_t if (ptmp == NULL) { rw_exit(uobj->vmobjlock); UVMHIST_LOG(pdhist, "sleeping, ptmp == NULL",0,0,0,0); - uvm_wait("uao_getpage"); + uvm_wait("uao_getpage", ticket); rw_enter(uobj->vmobjlock, RW_WRITER); uvm_page_array_clear(&a); continue; diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_bio.c --- a/sys/uvm/uvm_bio.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_bio.c Wed Aug 19 13:00:49 2026 +0000 @@ -423,6 +423,7 @@ again: */ rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; va < eva; i++, va += PAGE_SIZE) { + uint64_t ticket; struct vm_page *pg; UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i, (uintptr_t)pgs[i], @@ -432,6 +433,7 @@ again: if (pg == NULL || pg == PGO_DONTCARE) { continue; } + ticket = uvm_wait_prepare(); KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock); error = ubc_fault_page(ufi, umap, pg, prot, access_type, va); if (error) { @@ -441,7 +443,7 @@ again: */ pmap_update(ufi->orig_map->pmap); rw_exit(uobj->vmobjlock); - uvm_wait("ubc_fault"); + uvm_wait("ubc_fault", ticket); rw_enter(uobj->vmobjlock, RW_WRITER); } } @@ -601,11 +603,13 @@ again_faultbusy: } for (i = 0; i < npages; i++) { struct vm_page *pg = pgs[i]; + uint64_t ticket = 0; /* XXX spurious init */ KASSERT(pg->uobject == uobj); if (pg->loan_count != 0) { rw_enter(uobj->vmobjlock, RW_WRITER); if (pg->loan_count != 0) { + ticket = uvm_wait_prepare(); pg = uvm_loanbreak(pg); } if (pg == NULL) { @@ -613,7 +617,7 @@ again_faultbusy: pmap_update(pmap_kernel()); uvm_page_unbusy(pgs, npages); rw_exit(uobj->vmobjlock); - uvm_wait("ubc_alloc"); + uvm_wait("ubc_alloc", ticket); goto again_faultbusy; } rw_exit(uobj->vmobjlock); @@ -901,11 +905,13 @@ again: /* Avoid breaking loan if possible, only do it on write */ if ((flags & UBC_WRITE) && pg->loan_count != 0) { + uint64_t ticket = uvm_wait_prepare(); + pg = uvm_loanbreak(pg); if (pg == NULL) { uvm_page_unbusy(pgs, *npages); rw_exit(uobj->vmobjlock); - uvm_wait("ubc_alloc_directl"); + uvm_wait("ubc_alloc_directl", ticket); goto again; } pgs[i] = pg; diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_fault.c --- a/sys/uvm/uvm_fault.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_fault.c Wed Aug 19 13:00:49 2026 +0000 @@ -216,6 +216,7 @@ static void uvmfault_amapcopy(struct uvm_faultinfo *ufi) { for (;;) { + uint64_t ticket; /* * no mapping? give up. @@ -228,9 +229,11 @@ uvmfault_amapcopy(struct uvm_faultinfo * * copy if needed. */ - if (UVM_ET_ISNEEDSCOPY(ufi->entry)) + if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { + ticket = uvm_wait_prepare(); amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT, ufi->orig_rvaddr, ufi->orig_rvaddr + 1); + } /* * didn't work? must be out of RAM. unlock and sleep. @@ -238,7 +241,7 @@ uvmfault_amapcopy(struct uvm_faultinfo * if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { uvmfault_unlockmaps(ufi, true); - uvm_wait("fltamapcopy"); + uvm_wait("fltamapcopy", ticket); continue; } @@ -354,10 +357,12 @@ uvmfault_anonget(struct uvm_faultinfo *u * required for this. If the caller didn't supply * one, fail now and have them retry. */ + uint64_t ticket; if (lock_type == RW_READER) { return ENOLCK; } + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, ufi != NULL ? ufi->orig_rvaddr : 0, anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0); @@ -370,7 +375,7 @@ uvmfault_anonget(struct uvm_faultinfo *u if (!uvm_reclaimable()) { return ENOMEM; } - uvm_wait("flt_noram1"); + uvm_wait("flt_noram1", ticket); } else { /* PG_BUSY bit is set. */ we_own = true; @@ -551,6 +556,7 @@ uvmfault_promote(struct uvm_faultinfo *u { struct vm_amap *amap = ufi->entry->aref.ar_amap; struct uvm_object *uobj; + uint64_t ticket; struct vm_anon *anon; struct vm_page *pg; struct vm_page *opg; @@ -582,6 +588,7 @@ uvmfault_promote(struct uvm_faultinfo *u KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock); KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock)); + ticket = uvm_wait_prepare(); if (*spare != NULL) { anon = *spare; *spare = NULL; @@ -629,7 +636,7 @@ uvmfault_promote(struct uvm_faultinfo *u UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0); cpu_count(CPU_COUNT_FLTNORAM, 1); - uvm_wait("flt_noram5"); + uvm_wait("flt_noram5", ticket); error = ERESTART; goto done; } @@ -913,10 +920,13 @@ uvm_fault_internal(struct vm_map *orig_m ufi.entry->object.uvm_obj; if (uobj && uobj->pgops->pgo_fault != NULL) { + uint64_t ticket; + /* * invoke "special" fault routine. */ rw_enter(uobj->vmobjlock, RW_WRITER); + ticket = uvm_wait_prepare(); /* locked: maps(read), amap(if there), uobj */ error = uobj->pgops->pgo_fault(&ufi, flt.startva, pages, flt.npages, @@ -939,7 +949,7 @@ uvm_fault_internal(struct vm_map *orig_m * reclaimed. */ if (error == ENOMEM && uvm_reclaimable()) { - uvm_wait("pgo_fault"); + uvm_wait("pgo_fault", ticket); error = ERESTART; } } else { @@ -1563,6 +1573,8 @@ uvm_fault_upper_loan( /* >1 case is already ok */ if (anon->an_ref == 1) { + uint64_t ticket; + /* breaking loan requires a write lock. */ error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL); if (error != 0) { @@ -1570,10 +1582,11 @@ uvm_fault_upper_loan( } KASSERT(rw_write_held(amap->am_lock)); + ticket = uvm_wait_prepare(); error = uvm_loanbreak_anon(anon, *ruobj); if (error != 0) { uvmfault_unlockall(ufi, amap, *ruobj); - uvm_wait("flt_noram2"); + uvm_wait("flt_noram2", ticket); return ERESTART; } /* if we were a loan receiver uobj is gone */ @@ -1677,6 +1690,7 @@ uvm_fault_upper_enter( struct pmap *pmap = ufi->orig_map->pmap; vaddr_t va = ufi->orig_rvaddr; struct vm_amap * const amap = ufi->entry->aref.ar_amap; + uint64_t ticket; UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); /* locked: maps(read), amap, oanon, anon(if different from oanon) */ @@ -1694,6 +1708,7 @@ uvm_fault_upper_enter( UVMHIST_LOG(maphist, " MAPPING: anon: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd", (uintptr_t)pmap, va, (uintptr_t)pg, flt->promote); + ticket = uvm_wait_prepare(); if (pmap_enter(pmap, va, VM_PAGE_TO_PHYS(pg), flt->enter_prot, flt->access_type | PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) { @@ -1741,7 +1756,7 @@ uvm_fault_upper_enter( return ENOMEM; } /* XXX instrumentation */ - uvm_wait("flt_pmfail1"); + uvm_wait("flt_pmfail1", ticket); return ERESTART; } @@ -2330,6 +2345,7 @@ uvm_fault_lower_direct_loan( * write fault: must break the loan here. to do this * we need a write lock on the object. */ + uint64_t ticket; error = uvm_fault_lower_upgrade(ufi, flt, amap, uobj, uobjpage); if (error != 0) { @@ -2337,6 +2353,7 @@ uvm_fault_lower_direct_loan( } KASSERT(rw_write_held(uobj->vmobjlock)); + ticket = uvm_wait_prepare(); pg = uvm_loanbreak(uobjpage); if (pg == NULL) { @@ -2345,7 +2362,7 @@ uvm_fault_lower_direct_loan( " out of RAM breaking loan, waiting", 0,0,0,0); cpu_count(CPU_COUNT_FLTNORAM, 1); - uvm_wait("flt_noram4"); + uvm_wait("flt_noram4", ticket); return ERESTART; } *rpg = pg; @@ -2462,6 +2479,7 @@ uvm_fault_lower_enter( { struct vm_amap * const amap = ufi->entry->aref.ar_amap; const bool readonly = uvm_pagereadonly_p(pg); + uint64_t ticket; int error; UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); @@ -2506,6 +2524,7 @@ uvm_fault_lower_enter( UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map, (void *)ufi->orig_rvaddr, pg); KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly); + ticket = uvm_wait_prepare(); if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, VM_PAGE_TO_PHYS(pg), readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot, @@ -2541,7 +2560,7 @@ uvm_fault_lower_enter( return error; } /* XXX instrumentation */ - uvm_wait("flt_pmfail2"); + uvm_wait("flt_pmfail2", ticket); return ERESTART; } diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_glue.c --- a/sys/uvm/uvm_glue.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_glue.c Wed Aug 19 13:00:49 2026 +0000 @@ -257,15 +257,18 @@ uarea_poolpage_alloc(struct pool *pp, in #if defined(PMAP_MAP_POOLPAGE) while (USPACE == PAGE_SIZE && (USPACE_ALIGN == 0 || USPACE_ALIGN == PAGE_SIZE)) { + uint64_t ticket; struct vm_page *pg; vaddr_t va; + + ticket = uvm_wait_prepare(); #if defined(PMAP_ALLOC_POOLPAGE) pg = PMAP_ALLOC_POOLPAGE(0); #else pg = uvm_pagealloc(NULL, 0, NULL, 0); #endif if (pg == NULL) { - uvm_wait("uarea"); + uvm_wait("uarea", ticket); continue; } va = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg)); diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_km.c --- a/sys/uvm/uvm_km.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_km.c Wed Aug 19 13:00:49 2026 +0000 @@ -614,6 +614,7 @@ uvm_km_alloc(struct vm_map *map, vsize_t vaddr_t kva, loopva; vaddr_t offset; vsize_t loopsize; + uint64_t ticket; struct vm_page *pg; struct uvm_object *obj; int pgaflags; @@ -687,6 +688,7 @@ uvm_km_alloc(struct vm_map *map, vsize_t KASSERTMSG(!pmap_extract(pmap_kernel(), loopva, NULL), "loopva=%#"PRIxVADDR, loopva); + ticket = uvm_wait_prepare(); pg = uvm_pagealloc_strat(NULL, offset, NULL, pgaflags, #ifdef UVM_KM_VMFREELIST UVM_PGA_STRAT_ONLY, UVM_KM_VMFREELIST @@ -707,7 +709,8 @@ uvm_km_alloc(struct vm_map *map, vsize_t flags & UVM_KMF_TYPEMASK); return (0); } else { - uvm_wait("km_getwait2"); /* sleep here */ + /* sleep here */ + uvm_wait("km_getwait2", ticket); continue; } } @@ -798,6 +801,7 @@ int uvm_km_kmem_alloc(vmem_t *vm, vmem_size_t size, vm_flag_t flags, vmem_addr_t *addr) { + uint64_t ticket; struct vm_page *pg; vmem_addr_t va; int rc; @@ -809,6 +813,7 @@ uvm_km_kmem_alloc(vmem_t *vm, vmem_size_ #if defined(PMAP_MAP_POOLPAGE) if (size == PAGE_SIZE) { again: + ticket = uvm_wait_prepare(); #ifdef PMAP_ALLOC_POOLPAGE pg = PMAP_ALLOC_POOLPAGE((flags & VM_SLEEP) ? 0 : UVM_PGA_USERESERVE); @@ -818,7 +823,7 @@ again: #endif /* PMAP_ALLOC_POOLPAGE */ if (__predict_false(pg == NULL)) { if (flags & VM_SLEEP) { - uvm_wait("plpg"); + uvm_wait("plpg", ticket); goto again; } return ENOMEM; @@ -854,12 +859,13 @@ again: " pa=%#"PRIxPADDR" vmem=%p", loopva, loopsize, pa, vm); + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, loopva, NULL, UVM_FLAG_COLORMATCH | ((flags & VM_SLEEP) ? 0 : UVM_PGA_USERESERVE)); if (__predict_false(pg == NULL)) { if (flags & VM_SLEEP) { - uvm_wait("plpg"); + uvm_wait("plpg", ticket); continue; } else { uvm_km_pgremove_intrsafe(kernel_map, va, diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_loan.c --- a/sys/uvm/uvm_loan.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_loan.c Wed Aug 19 13:00:49 2026 +0000 @@ -834,11 +834,14 @@ again: pg = uvm_pagelookup(&uvm_loanzero_object, 0); if (__predict_false(pg == NULL)) { - while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL, - UVM_PGA_ZERO)) == NULL) { + uint64_t ticket; + + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL, + UVM_PGA_ZERO)) == NULL) { rw_exit(uvm_loanzero_object.vmobjlock); uvmfault_unlockall(ufi, amap, NULL); - uvm_wait("loanzero"); + uvm_wait("loanzero", ticket); if (!uvmfault_relock(ufi)) { return (0); } diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_object.c --- a/sys/uvm/uvm_object.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_object.c Wed Aug 19 13:00:49 2026 +0000 @@ -162,11 +162,15 @@ uvm_obj_wirepages(struct uvm_object *uob */ if (pgs[i]->loan_count) { while (pgs[i]->loan_count) { + uint64_t ticket = + uvm_wait_prepare(); + pg = uvm_loanbreak(pgs[i]); if (!pg) { rw_exit(uobj->vmobjlock); - uvm_wait("uobjwirepg"); - rw_enter(uobj->vmobjlock, RW_WRITER); + uvm_wait("uobjwirepg", ticket); + rw_enter(uobj->vmobjlock, + RW_WRITER); continue; } } diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_pdaemon.c --- a/sys/uvm/uvm_pdaemon.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_pdaemon.c Wed Aug 19 13:00:49 2026 +0000 @@ -114,39 +114,153 @@ static void uvmpd_tune(void); static void uvmpd_pool_drain_thread(void *); static void uvmpd_pool_drain_wakeup(void); -static unsigned int uvm_pagedaemon_waiters; +static kmutex_t uvmpd_waiter_lock __cacheline_aligned; +static volatile unsigned uvm_pagedaemon_waiters __cacheline_aligned; +static volatile unsigned uvm_waiter_wakeup __cacheline_aligned; /* State for the pool drainer thread */ -static kmutex_t uvmpd_lock __cacheline_aligned; +static kmutex_t uvmpd_pool_drain_lock; static kcondvar_t uvmpd_pool_drain_cv; static bool uvmpd_pool_drain_run = false; +static kmutex_t uvmpd_kick_lock __cacheline_aligned; +static volatile unsigned uvmpd_sleeping __cacheline_aligned; +static volatile unsigned uvmpd_wakeup __cacheline_aligned; + +#if __HAVE_ATOMIC64_LOADSTORE + +static volatile uint64_t uvmpd_gen; + +static uint64_t +uvmpd_read_gen(void) +{ + + return atomic_load_relaxed(&uvmpd_gen); +} + +static void +uvmpd_advance_gen(void) +{ + + KASSERT(mutex_owned(&uvmpd_waiter_lock)); + atomic_store_relaxed(&uvmpd_gen, atomic_load_relaxed(&uvmpd_gen) + 1); +} + +#else /* !__HAVE_ATOMIC64_LOADSTORE */ + +static volatile struct { +#if _BYTE_ORDER == _BIG_ENDIAN + uint32_t hi, lo; +#else + uint32_t lo, hi; +#endif +} uvmpd_gen; + +static uint64_t +uvmpd_read_gen(void) +{ + uint32_t hi, lo; + +top: while (__predict_false((hi = atomic_load_relaxed(&uvmpd_gen.hi)) == + 0xffffffff)) + SPINLOCK_BACKOFF_HOOK; + membar_consumer(); + lo = atomic_load_relaxed(&uvmpd_gen.lo); + membar_consumer(); + if (__predict_false(hi != atomic_load_relaxed(&uvmpd_gen.hi))) + goto top; + + return (uint64_t)hi << 32 | lo; +} + +static void +uvmpd_advance_gen(void) +{ + uint32_t hi, lo; + + KASSERT(mutex_owned(&uvmpd_waiter_lock)); + + lo = atomic_load_relaxed(&uvmpd_gen.lo); + if (__predict_false(++lo == 0)) { + hi = atomic_load_relaxed(&uvmpd_gen.hi) + 1; /* carry */ + atomic_store_relaxed(&uvmpd_gen.hi, 0xffffffff); + membar_producer(); + atomic_store_relaxed(&uvmpd_gen.lo, lo); + membar_producer(); + atomic_store_relaxed(&uvmpd_gen.hi, hi); + } else { + atomic_store_relaxed(&uvmpd_gen.lo, lo); + } +} + +#endif + /* * XXX hack to avoid hangs when large processes fork. */ u_int uvm_extrapages; /* + * uvm_wait_prepare: return a ticket for uvm_wait + * + * => caller must: + * 1. get a ticket _first_, + * 2. _then_ try allocation, and + * 3. uvm_wait with the ticket if it fails + */ + +uint64_t +uvm_wait_prepare(void) +{ + uint64_t ticket; + + ticket = uvmpd_read_gen(); + + return ticket; +} + +/* * uvm_wait: wait (sleep) for the page daemon to free some pages * * => should be called with all locks released * => should _not_ be called by the page daemon (to avoid deadlock) + * => caller must have acquired a ticket with uvm_wait_prepare, + * before trying an allocation that we now wait for */ void -uvm_wait(const char *wmsg) +uvm_wait(const char *wmsg, uint64_t ticket) { + uint64_t gen = ticket; int timo = 0; if (uvm.pagedaemon_lwp == NULL) panic("out of memory before the pagedaemon thread exists"); - mutex_spin_enter(&uvmpd_lock); + /* + * Fast path: if the page daemon wakeup generation has already + * advanced since the caller began uvm_wait_prepare, don't add + * to contention on uvmpd_waiter_lock; we already know the + * caller can retry its allocation. + */ + if (gen != uvmpd_read_gen()) + return; + + mutex_enter(&uvmpd_waiter_lock); + + /* + * If the page daemon wakeup generation has advanced since the + * caller began uvm_wait_prepare, don't sleep; the caller can + * already retry its allocation. + */ + if (gen != uvmpd_read_gen()) { + mutex_exit(&uvmpd_waiter_lock); + return; + } /* * check for page daemon going to sleep (waiting for itself) */ - if (uvm_lwp_is_pagedaemon(curlwp) && uvmexp.paging == 0) { /* * now we have a problem: the pagedaemon wants to go to @@ -173,9 +287,52 @@ uvm_wait(const char *wmsg) #endif } - uvm_pagedaemon_waiters++; - wakeup(&uvm.pagedaemon); /* wake the daemon! */ - UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_lock, false, wmsg, timo); + /* + * Notify uvm_pageout (the page daemon) or uvm_pageout_done + * (the paging completion handler) that there are waiters + * before we check whether there's a pending unlocked wakeup. + * + * membar_sync here matches membar_sync in uvm_pageout_done + * between storing uvm_waiter_wakeup = 1 and loading + * uvm_pagedaemon_waiters. + * + * If there's pending unlocked wakeup, consume it by advancing + * the wakeup generation and waking any other waiters -- that + * can be done only with the lock. Generally there should be + * no other waiters at this point -- if there is a pending + * wakeup, any other call to uvm_wait would have consumed it + * and so can't be waiting; if another call to uvm_wait didn't + * find a pending wakeup, then any subsequent uvm_pageout or + * uvm_pageout_done wakeup should have noticed that call and + * issued wakeup(&uvmexp.free) -- but it takes a more effort to + * prove that it is safe to elide wakeup(&uvmexp.free) than it + * takes to just call it anyway here. + * + * The wakeup might be spurious by now, but the caller has to + * cope with spurious wakeups anyway. + */ + atomic_store_relaxed(&uvm_pagedaemon_waiters, + atomic_load_relaxed(&uvm_pagedaemon_waiters) + 1); + membar_sync(); + if (atomic_swap_uint(&uvm_waiter_wakeup, 0)) { + uvmpd_advance_gen(); + atomic_store_relaxed(&uvm_pagedaemon_waiters, 0); + wakeup(&uvmexp.free); + mutex_exit(&uvmpd_waiter_lock); + return; + } + + if (!atomic_swap_uint(&uvmpd_wakeup, 1)) { + membar_sync(); + if (atomic_load_relaxed(&uvmpd_sleeping)) { + mutex_spin_enter(&uvmpd_kick_lock); + wakeup(&uvm.pagedaemon); /* wake the daemon! */ + mutex_spin_exit(&uvmpd_kick_lock); + } + } + + UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_waiter_lock, false, wmsg, + timo); } /* @@ -192,9 +349,35 @@ uvm_kick_pdaemon(void) (fpages + uvmexp.paging < uvmexp.freetarg && uvmpdpol_needsscan_p()) || uvm_km_va_starved_p()) { - mutex_spin_enter(&uvmpd_lock); + /* + * First make sure the page daemon won't go to sleep if + * it's still working by setting uvmpd_wakeup := 1; + * then check whether it is about to go to sleep, by + * querying uvmpd_sleeping. + * + * If _after_ we have told it to wake up, it is not + * sleeping, we can avoid adding to contention on + * uvmpd_kick_lock -- the page daemon is guaranteed to + * try making the progress we want. + * + * membar_sync here matches membar_sync in uvm_pageout + * between storing uvmpd_sleeping = 1 and loading + * uvmpd_wakeup. + */ + if (atomic_swap_uint(&uvmpd_wakeup, 1)) + return; + membar_sync(); + if (!atomic_load_relaxed(&uvmpd_sleeping)) + return; + + /* + * The page daemon has already decided to go to sleep, + * and it may be too late for us to stop it. Wake it + * up. + */ + mutex_spin_enter(&uvmpd_kick_lock); wakeup(&uvm.pagedaemon); - mutex_spin_exit(&uvmpd_lock); + mutex_spin_exit(&uvmpd_kick_lock); } } @@ -262,7 +445,9 @@ uvm_pageout(void *arg) UVMHIST_LOG(pdhist,"", 0, 0, 0, 0); - mutex_init(&uvmpd_lock, MUTEX_DEFAULT, IPL_VM); + mutex_init(&uvmpd_kick_lock, MUTEX_DEFAULT, IPL_VM); + mutex_init(&uvmpd_waiter_lock, MUTEX_DEFAULT, IPL_SOFTBIO); + mutex_init(&uvmpd_pool_drain_lock, MUTEX_DEFAULT, IPL_NONE); cv_init(&uvmpd_pool_drain_cv, "pooldrain"); /* Create the pool drainer kernel thread. */ @@ -287,16 +472,48 @@ uvm_pageout(void *arg) kmem_va_starved = uvm_km_va_starved_p(); - mutex_spin_enter(&uvmpd_lock); - if ((uvm_pagedaemon_waiters == 0 || uvmexp.paging > 0) && + mutex_spin_enter(&uvmpd_kick_lock); + if ((atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0 || + uvmexp.paging > 0) && !kmem_va_starved) { UVMHIST_LOG(pdhist," <>",0,0,0,0); - UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon, - &uvmpd_lock, false, "pgdaemon", 0); + + /* + * Notify uvm_kick_pdaemon that we're about to + * go to sleep. Then check whether it has + * already asked us to wake up again before it + * puts contention on uvmpd_kick_lock. If, + * after we have announced our intent to sleep, + * there's still no pending wakeup, go to sleep + * and release uvm_pdlock. + * + * membar_sync here matches membar_sync in + * uvm_pageout_done between storing + * uvmpd_wakeup := 1 and loading + * uvmpd_sleeping. + * + * Once we're not asleep -- either because + * uvmpd_wakeup = 1 or because of a concurrent + * wakeup(&uvm.pagedaemon) -- set + * uvmpd_sleeping back to 0 again so that while + * we work, uvm_kick_pdaemon need not take + * uvmpd_kick_lock in order to make sure we + * wake up next time. + */ + atomic_store_relaxed(&uvmpd_sleeping, 1); + membar_sync(); + if (atomic_swap_uint(&uvmpd_wakeup, 0)) { + mutex_spin_exit(&uvmpd_kick_lock); + } else { + UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon, + &uvmpd_kick_lock, false, "pgdaemon", 0); + } + atomic_store_relaxed(&uvmpd_sleeping, 0); uvmexp.pdwoke++; + UVMHIST_LOG(pdhist," <>",0,0,0,0); } else { - mutex_spin_exit(&uvmpd_lock); + mutex_spin_exit(&uvmpd_kick_lock); } /* @@ -335,10 +552,11 @@ uvm_pageout(void *arg) */ if (uvm_availmem(false) > uvmexp.reserve_kernel || uvmexp.paging == 0) { - mutex_spin_enter(&uvmpd_lock); + mutex_enter(&uvmpd_waiter_lock); + uvmpd_advance_gen(); wakeup(&uvmexp.free); - uvm_pagedaemon_waiters = 0; - mutex_spin_exit(&uvmpd_lock); + atomic_store_relaxed(&uvm_pagedaemon_waiters, 0); + mutex_exit(&uvmpd_waiter_lock); } /* @@ -379,15 +597,84 @@ uvm_pageout_done(int npages) /* * wake up either of pagedaemon or LWPs waiting for it. */ + if (uvm_availmem(false) <= uvmexp.reserve_kernel) { + /* + * If the page daemon is still working, ask it not to + * go to sleep when it's done with this iteration. If, + * after we have asked that by setting uvmpd_wakeup := + * 1, the page daemon still hasn't announced its intent + * to sleep, we can stop here -- once it reaches the + * next iteration, it will notice uvmpd_wakeup = 1 and + * avoid sleeping. This way we avoid putting + * contention on uvmpd_kick_lock when the page daemon + * is busy working. + * + * membar_sync here matches membar_sync in uvm_pageout + * between storing uvmpd_sleeping := 1 and swapping + * uvmpd_wakeup with 0. + */ + if (atomic_swap_uint(&uvmpd_wakeup, 1)) + return; + membar_sync(); + if (!atomic_load_relaxed(&uvmpd_sleeping)) + return; - mutex_spin_enter(&uvmpd_lock); - if (uvm_availmem(false) <= uvmexp.reserve_kernel) { + /* + * Page daemon is asleep or about to go to sleep, and + * it may be too late for us to stop it, so we have to + * take the lock to wake it up. + */ + mutex_spin_enter(&uvmpd_kick_lock); wakeup(&uvm.pagedaemon); - } else if (uvm_pagedaemon_waiters != 0) { - wakeup(&uvmexp.free); - uvm_pagedaemon_waiters = 0; + mutex_spin_exit(&uvmpd_kick_lock); + } else { + /* + * Make sure the next call to uvm_wait will wake up + * even if we don't take uvmpd_waiter_lock, and then + * check -- again without taking uvmpd_waiter_lock -- + * whether there are any waiters that might be + * sleeping. If not, we're done and we don't need to + * put contention on uvmpd_waiter_lock. + * + * membar_sync here matches membar_sync in uvm_wait. + * + * Note that the next call to uvm_wait will necessarily + * notice uvm_waiter_wakeup = 1 and act on it by + * advancing the page daemon wakeup generation (under + * the lock), so we don't need to do that here. + */ + if (atomic_swap_uint(&uvm_waiter_wakeup, 1)) + return; + membar_sync(); + if (atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0) + return; + + /* + * uvm_wait waiters are either asleep or about to go to + * sleep, and it may be too late to stop them, so we + * have to take the lock to wake them up. + * + * Make sure to advance the generation so that logic + * still in the ... part of + * + * top: ticket = uvm_wait_prepare(); + * ... + * if (allocfailed) { + * uvm_wait("wmesg", ticket); + * goto top; + * } + * + * will wake up and not sleep forever when it hits + * uvm_wait. + */ + mutex_enter(&uvmpd_waiter_lock); + uvmpd_advance_gen(); + if (atomic_load_relaxed(&uvm_pagedaemon_waiters) != 0) { + wakeup(&uvmexp.free); + atomic_store_relaxed(&uvm_pagedaemon_waiters, 0); + } + mutex_exit(&uvmpd_waiter_lock); } - mutex_spin_exit(&uvmpd_lock); } static krwlock_t * @@ -1024,17 +1311,17 @@ uvmpd_pool_drain_thread(void *arg) /* * sleep until awoken by the pagedaemon. */ - mutex_enter(&uvmpd_lock); + mutex_enter(&uvmpd_pool_drain_lock); if (!uvmpd_pool_drain_run) { lastslept = getticks(); - cv_wait(&uvmpd_pool_drain_cv, &uvmpd_lock); + cv_wait(&uvmpd_pool_drain_cv, &uvmpd_pool_drain_lock); if (getticks() != lastslept) { cycled = false; firstpool = NULL; } } uvmpd_pool_drain_run = false; - mutex_exit(&uvmpd_lock); + mutex_exit(&uvmpd_pool_drain_lock); /* * rate limit draining, otherwise in desperate circumstances @@ -1081,8 +1368,8 @@ static void uvmpd_pool_drain_wakeup(void) { - mutex_enter(&uvmpd_lock); + mutex_enter(&uvmpd_pool_drain_lock); uvmpd_pool_drain_run = true; cv_signal(&uvmpd_pool_drain_cv); - mutex_exit(&uvmpd_lock); + mutex_exit(&uvmpd_pool_drain_lock); } diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_pdaemon.h --- a/sys/uvm/uvm_pdaemon.h Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_pdaemon.h Wed Aug 19 13:00:49 2026 +0000 @@ -83,7 +83,8 @@ struct krwlock; * prototypes */ -void uvm_wait(const char *); +uint64_t uvm_wait_prepare(void); +void uvm_wait(const char *, uint64_t); bool uvm_reclaimable(void); struct krwlock *uvmpd_trylockowner(struct vm_page *); diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_pglist.c --- a/sys/uvm/uvm_pglist.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_pglist.c Wed Aug 19 13:00:49 2026 +0000 @@ -300,6 +300,7 @@ static int uvm_pglistalloc_contig_aggressive(int num, paddr_t low, paddr_t high, paddr_t alignment, paddr_t boundary, struct pglist *rlist) { + uint64_t ticket; struct vm_page *pg; struct pglist tmp; paddr_t pa, off, spa, amask, bmask, rlo, rhi; @@ -321,9 +322,10 @@ uvm_pglistalloc_contig_aggressive(int nu KASSERT(bmask <= amask); mutex_enter(&uvm_pglistalloc_contig_lock); while (uvm_reclaimable()) { + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, 0, NULL, 0); if (pg == NULL) { - uvm_wait("pglac2"); + uvm_wait("pglac2", ticket); continue; } pg->flags |= PG_PGLCA; @@ -615,6 +617,7 @@ uvm_pglistalloc_simple(int num, paddr_t struct pglist *rlist, int waitok) { int fl, error; + uint64_t ticket; uvm_physseg_t psi; int count = 0; @@ -623,6 +626,8 @@ uvm_pglistalloc_simple(int num, paddr_t bool valid = false; again: + ticket = uvm_wait_prepare(); + /* * Block all memory allocation and lock the free list. */ @@ -673,7 +678,7 @@ out: if (error) { if (waitok) { - uvm_wait("pglalloc"); + uvm_wait("pglalloc", ticket); goto again; } else uvm_pglistfree(rlist); diff -r 38311baef8db -r 79c345a21435 sys/uvm/uvm_vnode.c --- a/sys/uvm/uvm_vnode.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_vnode.c Wed Aug 19 13:00:49 2026 +0000 @@ -287,6 +287,7 @@ static int uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp, unsigned int flags, struct uvm_page_array *a, unsigned int nleft) { + uint64_t ticket; struct vm_page *pg; UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)uobj, offset, @@ -332,6 +333,7 @@ uvn_findpage(struct uvm_object *uobj, vo UVMHIST_LOG(ubchist, "noalloc", 0,0,0,0); return 0; } + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(uobj, offset, NULL, UVM_FLAG_COLORMATCH); if (pg == NULL) { @@ -340,7 +342,7 @@ uvn_findpage(struct uvm_object *uobj, vo return 0; } rw_exit(uobj->vmobjlock); - uvm_wait("uvnfp1"); + uvm_wait("uvnfp1", ticket); uvm_page_array_clear(a); rw_enter(uobj->vmobjlock, RW_WRITER); continue;