diff -r 38311baef8db -r 801b63d6bff9 sys/arch/aarch64/aarch64/pmap.c --- a/sys/arch/aarch64/aarch64/pmap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/aarch64/aarch64/pmap.c Tue Aug 18 16:13:20 2026 +0000 @@ -552,6 +552,7 @@ static paddr_t pmap_alloc_pdp(struct pmap *pm, struct vm_page **pgp, int flags, bool waitok) { paddr_t pa; + uint64_t ticket; struct vm_page *pg; UVMHIST_FUNC(__func__); @@ -562,10 +563,11 @@ pmap_alloc_pdp(struct pmap *pm, struct v int aflags = ((flags & PMAP_CANFAIL) ? 0 : UVM_PGA_USERESERVE) | UVM_PGA_ZERO; retry: + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, 0, NULL, aflags); if (pg == NULL) { if (waitok) { - uvm_wait("pmap_alloc_pdp"); + uvm_wait("pmap_alloc_pdp", ticket); goto retry; } return POOL_PADDR_INVALID; diff -r 38311baef8db -r 801b63d6bff9 sys/arch/aarch64/include/asan.h --- a/sys/arch/aarch64/include/asan.h Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/aarch64/include/asan.h Tue Aug 18 16:13:20 2026 +0000 @@ -79,11 +79,13 @@ static paddr_t va = uvm_pageboot_alloc(PAGE_SIZE); pa = AARCH64_KVA_TO_PA(va); } else { + uint64_t ticket; struct vm_page *pg; retry: + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, 0, NULL, 0); if (pg == NULL) { - uvm_wait(__func__); + uvm_wait(__func__, ticket); goto retry; } diff -r 38311baef8db -r 801b63d6bff9 sys/arch/amd64/amd64/gdt.c --- a/sys/arch/amd64/amd64/gdt.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/amd64/amd64/gdt.c Tue Aug 18 16:13:20 2026 +0000 @@ -184,6 +184,7 @@ gdt_alloc_cpu(struct cpu_info *ci) #ifdef __HAVE_PCPU_AREA ci->ci_gdt = (union descriptor *)&pcpuarea->ent[cpu_index(ci)].gdt; #else + uint64_t ticket; struct vm_page *pg; vaddr_t va; @@ -191,9 +192,10 @@ gdt_alloc_cpu(struct cpu_info *ci) 0, UVM_KMF_VAONLY); for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + gdt_size; va += PAGE_SIZE) { - while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) == NULL) { - uvm_wait("gdt_alloc_cpu"); + uvm_wait("gdt_alloc_cpu", ticket); } pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE, 0); diff -r 38311baef8db -r 801b63d6bff9 sys/arch/arm/include/asan.h --- a/sys/arch/arm/include/asan.h Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/arm/include/asan.h Tue Aug 18 16:13:20 2026 +0000 @@ -111,11 +111,13 @@ static vaddr_t return pa; } + uint64_t ticket; struct vm_page *pg; retry: + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, 0, NULL, 0); if (pg == NULL) { - uvm_wait(__func__); + uvm_wait(__func__, ticket); goto retry; } pa = VM_PAGE_TO_PHYS(pg); diff -r 38311baef8db -r 801b63d6bff9 sys/arch/i386/i386/gdt.c --- a/sys/arch/i386/i386/gdt.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/i386/i386/gdt.c Tue Aug 18 16:13:20 2026 +0000 @@ -184,6 +184,7 @@ gdt_init(void) void gdt_alloc_cpu(struct cpu_info *ci) { + uint64_t ticket; struct vm_page *pg; vaddr_t va; @@ -191,9 +192,10 @@ gdt_alloc_cpu(struct cpu_info *ci) 0, UVM_KMF_VAONLY); for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + gdt_size; va += PAGE_SIZE) { - while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) == NULL) { - uvm_wait("gdt_alloc_cpu"); + uvm_wait("gdt_alloc_cpu", ticket); } pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE, 0); diff -r 38311baef8db -r 801b63d6bff9 sys/arch/m68k/m68k/pmap_68k.c --- a/sys/arch/m68k/m68k/pmap_68k.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/m68k/m68k/pmap_68k.c Tue Aug 18 16:13:20 2026 +0000 @@ -643,14 +643,16 @@ pmap_ptpage_init(void) static struct vm_page * pmap_page_alloc(bool nowait) { + uint64_t ticket; struct vm_page *pg; const int flags = nowait ? UVM_PGA_USERESERVE : 0; - while ((pg = uvm_pagealloc(NULL, 0, NULL, flags)) == NULL) { + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(NULL, 0, NULL, flags)) == NULL) { if (nowait) { return NULL; } - uvm_wait("pmappg"); + uvm_wait("pmappg", ticket); } pg->flags &= ~PG_BUSY; /* never busy */ diff -r 38311baef8db -r 801b63d6bff9 sys/arch/m68k/m68k/pmap_motorola.c --- a/sys/arch/m68k/m68k/pmap_motorola.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/m68k/m68k/pmap_motorola.c Tue Aug 18 16:13:20 2026 +0000 @@ -2551,6 +2551,7 @@ int pmap_enter_ptpage(pmap_t pmap, vaddr_t va, bool can_fail) { paddr_t ptpa; + uint64_t ticket; struct vm_page *pg; struct pv_header *pvh; struct pv_entry *pv; @@ -2683,7 +2684,6 @@ pmap_enter_ptpage(pmap_t pmap, vaddr_t v #endif splx(s); } else { - /* * For user processes we just allocate a page from the * VM system. Note that we set the page "wired" count to 1, @@ -2698,15 +2698,16 @@ pmap_enter_ptpage(pmap_t pmap, vaddr_t v PMAP_DPRINTF(PDB_ENTER|PDB_PTPAGE, ("enter: about to alloc UPT pg at %lx\n", va)); rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); - while ((pg = uvm_pagealloc(uvm_kernel_object, - va - vm_map_min(kernel_map), - NULL, UVM_PGA_ZERO)) == NULL) { + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(uvm_kernel_object, + va - vm_map_min(kernel_map), + NULL, UVM_PGA_ZERO)) == NULL) { rw_exit(uvm_kernel_object->vmobjlock); if (can_fail) { pmap->pm_sref--; return ENOMEM; } - uvm_wait("ptpage"); + uvm_wait("ptpage", ticket); rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); } rw_exit(uvm_kernel_object->vmobjlock); diff -r 38311baef8db -r 801b63d6bff9 sys/arch/powerpc/oea/pmap.c --- a/sys/arch/powerpc/oea/pmap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/powerpc/oea/pmap.c Tue Aug 18 16:13:20 2026 +0000 @@ -2885,6 +2885,7 @@ pmap_free_unmanaged(struct pup *pup) void * pmap_pool_alloc(struct pool *pp, int flags) { + uint64_t ticket; struct vm_page *pg; paddr_t pa; @@ -2892,6 +2893,7 @@ pmap_pool_alloc(struct pool *pp, int fla return (void *)uvm_pageboot_alloc(PAGE_SIZE); retry: + ticket = uvm_wait_prepare(); pg = uvm_pagealloc_strat(NULL /*obj*/, 0 /*off*/, NULL /*anon*/, UVM_PGA_USERESERVE /*flags*/, UVM_PGA_STRAT_ONLY /*strat*/, VM_FREELIST_DIRECT_MAPPED /*free_list*/); @@ -2902,7 +2904,7 @@ pmap_pool_alloc(struct pool *pp, int fla if ((flags & PR_WAITOK) == 0) return NULL; - uvm_wait("plpg"); + uvm_wait("plpg", ticket); goto retry; } KDASSERT(VM_PAGE_TO_PHYS(pg) == (uintptr_t)VM_PAGE_TO_PHYS(pg)); diff -r 38311baef8db -r 801b63d6bff9 sys/arch/sparc/sparc/pmap.c --- a/sys/arch/sparc/sparc/pmap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/sparc/sparc/pmap.c Tue Aug 18 16:13:20 2026 +0000 @@ -907,14 +907,16 @@ void * pgt_page_alloc(struct pool *pp, int flags) { int cacheit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0; + uint64_t ticket; struct vm_page *pg; vaddr_t va; paddr_t pa; /* Allocate a page of physical memory */ - while ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL && - (flags & PR_WAITOK) != 0) { - uvm_wait("pgtpg"); + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL && + (flags & PR_WAITOK) != 0) { + uvm_wait("pgtpg", ticket); } if (pg == NULL) { KASSERT((flags & PR_WAITOK) == 0); diff -r 38311baef8db -r 801b63d6bff9 sys/arch/sparc64/sparc64/pmap.c --- a/sys/arch/sparc64/sparc64/pmap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/arch/sparc64/sparc64/pmap.c Tue Aug 18 16:13:20 2026 +0000 @@ -1475,8 +1475,11 @@ pmap_create(void) pm->pm_refs = 1; TAILQ_INIT(&pm->pm_ptps); if (pm != pmap_kernel()) { - while (!pmap_get_page(&pm->pm_physaddr)) { - uvm_wait("pmap_create"); + uint64_t ticket; + + while (ticket = uvm_wait_prepare(), + !pmap_get_page(&pm->pm_physaddr)) { + uvm_wait("pmap_create", ticket); } pm->pm_segs = (paddr_t *)(u_long)pm->pm_physaddr; } diff -r 38311baef8db -r 801b63d6bff9 sys/rump/librump/rumpkern/vm.c --- a/sys/rump/librump/rumpkern/vm.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/rump/librump/rumpkern/vm.c Tue Aug 18 16:13:20 2026 +0000 @@ -106,6 +106,7 @@ struct pmap *const kernel_pmap_ptr = &pm vmem_t *kmem_arena; vmem_t *kmem_va_arena; +static uint64_t uvm_pdaemon_ticket; static unsigned int pdaemon_waiters; static kmutex_t pdaemonmtx; static kcondvar_t pdaemoncv, oomwait; @@ -1101,8 +1102,20 @@ uvm_uarea_free(vaddr_t uarea) * Routines related to the Page Baroness. */ +uint64_t +uvm_wait_prepare(void) +{ + uint64_t ticket; + + mutex_enter(&pdaemonmtx); + ticket = uvm_pdaemon_ticket; + mutex_exit(&pdaemonmtx); + + return ticket; +} + void -uvm_wait(const char *msg) +uvm_wait(const char *msg, uint64_t ticket) { if (__predict_false(rump_threads == 0)) @@ -1115,10 +1128,12 @@ uvm_wait(const char *msg) } mutex_enter(&pdaemonmtx); + if (ticket != uvm_pdaemon_ticket) + goto out; pdaemon_waiters++; cv_signal(&pdaemoncv); cv_wait(&oomwait, &pdaemonmtx); - mutex_exit(&pdaemonmtx); +out: mutex_exit(&pdaemonmtx); } void @@ -1141,6 +1156,7 @@ uvm_pageout_done(int npages) KASSERT(uvmexp.paging >= npages); uvmexp.paging -= npages; + uvm_pdaemon_ticket++; if (pdaemon_waiters) { pdaemon_waiters = 0; cv_broadcast(&oomwait); @@ -1185,6 +1201,7 @@ uvm_pageout(void *arg) mutex_enter(&pdaemonmtx); for (;;) { + uvm_pdaemon_ticket++; if (pdaemon_waiters) { pdaemon_waiters = 0; cv_broadcast(&oomwait); @@ -1309,6 +1326,7 @@ rump_hypermalloc(size_t howmuch, int ali { const unsigned long thelimit = uvm_lwp_is_pagedaemon(curlwp) ? pdlimit : rump_physmemlimit; + uint64_t ticket; unsigned long newmem; void *rv; int error; @@ -1318,22 +1336,24 @@ rump_hypermalloc(size_t howmuch, int ali /* first we must be within the limit */ limitagain: if (thelimit != RUMPMEM_UNLIMITED) { + ticket = uvm_wait_prepare(); newmem = atomic_add_long_nv(&curphysmem, howmuch); if (newmem > thelimit) { newmem = atomic_add_long_nv(&curphysmem, -howmuch); if (!waitok) { return NULL; } - uvm_wait(wmsg); + uvm_wait(wmsg, ticket); goto limitagain; } } /* second, we must get something from the backend */ again: + ticket = uvm_wait_prepare(); error = rumpuser_malloc(howmuch, alignment, &rv); if (__predict_false(error && waitok)) { - uvm_wait(wmsg); + uvm_wait(wmsg, ticket); goto again; } diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/pmap/pmap_segtab.c --- a/sys/uvm/pmap/pmap_segtab.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/pmap/pmap_segtab.c Tue Aug 18 16:13:20 2026 +0000 @@ -584,6 +584,8 @@ pmap_pdetab_alloc(struct pmap *pmap) struct vm_page *ptb_pg = NULL; if (__predict_false(ptb == NULL)) { + uint64_t ticket = uvm_wait_prepare(); + ptb_pg = pmap_pte_pagealloc(); UVMHIST_LOG(pmapxtabhist, "ptb_pg=%#jx", @@ -592,7 +594,7 @@ pmap_pdetab_alloc(struct pmap *pmap) /* * XXX What else can we do? Could we deadlock here? */ - uvm_wait("pdetab"); + uvm_wait("pdetab", ticket); goto again; } @@ -658,13 +660,15 @@ pmap_segtab_alloc(struct pmap *pmap) struct vm_page *stb_pg = NULL; if (__predict_false(stb == NULL)) { + uint64_t ticket = uvm_wait_prepare(); + stb_pg = pmap_pte_pagealloc(); if (__predict_false(stb_pg == NULL)) { /* * XXX What else can we do? Could we deadlock here? */ - uvm_wait("segtab"); + uvm_wait("segtab", ticket); goto again; } SEGTAB_ADD(npage, 1); diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_amap.c --- a/sys/uvm/uvm_amap.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_amap.c Tue Aug 18 16:13:20 2026 +0000 @@ -1026,6 +1026,7 @@ void amap_cow_now(struct vm_map *map, struct vm_map_entry *entry) { struct vm_amap *amap = entry->aref.ar_amap; + uint64_t ticket; struct vm_anon *anon, *nanon; struct vm_page *pg, *npg; u_int lcv, slot; @@ -1089,6 +1090,7 @@ ReStart: * First - get a new anon and a page. */ + ticket = uvm_wait_prepare(); nanon = uvm_analloc(); if (nanon) { nanon->an_lock = amap->am_lock; @@ -1104,7 +1106,7 @@ ReStart: KASSERT(nanon->an_ref == 0); uvm_anfree(nanon); } - uvm_wait("cownowpage"); + uvm_wait("cownowpage", ticket); goto ReStart; } diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_aobj.c --- a/sys/uvm/uvm_aobj.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_aobj.c Tue Aug 18 16:13:20 2026 +0000 @@ -806,6 +806,7 @@ uao_get(struct uvm_object *uobj, voff_t int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags) { voff_t current_offset; + uint64_t ticket; struct vm_page *ptmp; int lcv, gotpages, maxpages, swslot, pageidx; bool overwrite = ((flags & PGO_OVERWRITE) != 0); @@ -958,6 +959,7 @@ uao_get(struct uvm_object *uobj, voff_t pageidx = current_offset >> PAGE_SHIFT; swslot = uao_find_swslot(uobj, pageidx); + ticket = uvm_wait_prepare(); ptmp = uao_pagealloc(uobj, current_offset, swslot != 0 || overwrite ? 0 : UVM_PGA_ZERO); @@ -965,7 +967,7 @@ uao_get(struct uvm_object *uobj, voff_t if (ptmp == NULL) { rw_exit(uobj->vmobjlock); UVMHIST_LOG(pdhist, "sleeping, ptmp == NULL",0,0,0,0); - uvm_wait("uao_getpage"); + uvm_wait("uao_getpage", ticket); rw_enter(uobj->vmobjlock, RW_WRITER); uvm_page_array_clear(&a); continue; diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_bio.c --- a/sys/uvm/uvm_bio.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_bio.c Tue Aug 18 16:13:20 2026 +0000 @@ -423,6 +423,7 @@ again: */ rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; va < eva; i++, va += PAGE_SIZE) { + uint64_t ticket; struct vm_page *pg; UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i, (uintptr_t)pgs[i], @@ -432,6 +433,7 @@ again: if (pg == NULL || pg == PGO_DONTCARE) { continue; } + ticket = uvm_wait_prepare(); KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock); error = ubc_fault_page(ufi, umap, pg, prot, access_type, va); if (error) { @@ -441,7 +443,7 @@ again: */ pmap_update(ufi->orig_map->pmap); rw_exit(uobj->vmobjlock); - uvm_wait("ubc_fault"); + uvm_wait("ubc_fault", ticket); rw_enter(uobj->vmobjlock, RW_WRITER); } } @@ -601,11 +603,13 @@ again_faultbusy: } for (i = 0; i < npages; i++) { struct vm_page *pg = pgs[i]; + uint64_t ticket = 0; /* XXX spurious init */ KASSERT(pg->uobject == uobj); if (pg->loan_count != 0) { rw_enter(uobj->vmobjlock, RW_WRITER); if (pg->loan_count != 0) { + ticket = uvm_wait_prepare(); pg = uvm_loanbreak(pg); } if (pg == NULL) { @@ -613,7 +617,7 @@ again_faultbusy: pmap_update(pmap_kernel()); uvm_page_unbusy(pgs, npages); rw_exit(uobj->vmobjlock); - uvm_wait("ubc_alloc"); + uvm_wait("ubc_alloc", ticket); goto again_faultbusy; } rw_exit(uobj->vmobjlock); @@ -901,11 +905,13 @@ again: /* Avoid breaking loan if possible, only do it on write */ if ((flags & UBC_WRITE) && pg->loan_count != 0) { + uint64_t ticket = uvm_wait_prepare(); + pg = uvm_loanbreak(pg); if (pg == NULL) { uvm_page_unbusy(pgs, *npages); rw_exit(uobj->vmobjlock); - uvm_wait("ubc_alloc_directl"); + uvm_wait("ubc_alloc_directl", ticket); goto again; } pgs[i] = pg; diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_fault.c --- a/sys/uvm/uvm_fault.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_fault.c Tue Aug 18 16:13:20 2026 +0000 @@ -216,6 +216,7 @@ static void uvmfault_amapcopy(struct uvm_faultinfo *ufi) { for (;;) { + uint64_t ticket; /* * no mapping? give up. @@ -228,9 +229,11 @@ uvmfault_amapcopy(struct uvm_faultinfo * * copy if needed. */ - if (UVM_ET_ISNEEDSCOPY(ufi->entry)) + if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { + ticket = uvm_wait_prepare(); amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT, ufi->orig_rvaddr, ufi->orig_rvaddr + 1); + } /* * didn't work? must be out of RAM. unlock and sleep. @@ -238,7 +241,7 @@ uvmfault_amapcopy(struct uvm_faultinfo * if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { uvmfault_unlockmaps(ufi, true); - uvm_wait("fltamapcopy"); + uvm_wait("fltamapcopy", ticket); continue; } @@ -354,10 +357,12 @@ uvmfault_anonget(struct uvm_faultinfo *u * required for this. If the caller didn't supply * one, fail now and have them retry. */ + uint64_t ticket; if (lock_type == RW_READER) { return ENOLCK; } + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, ufi != NULL ? ufi->orig_rvaddr : 0, anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0); @@ -370,7 +375,7 @@ uvmfault_anonget(struct uvm_faultinfo *u if (!uvm_reclaimable()) { return ENOMEM; } - uvm_wait("flt_noram1"); + uvm_wait("flt_noram1", ticket); } else { /* PG_BUSY bit is set. */ we_own = true; @@ -551,6 +556,7 @@ uvmfault_promote(struct uvm_faultinfo *u { struct vm_amap *amap = ufi->entry->aref.ar_amap; struct uvm_object *uobj; + uint64_t ticket; struct vm_anon *anon; struct vm_page *pg; struct vm_page *opg; @@ -582,6 +588,7 @@ uvmfault_promote(struct uvm_faultinfo *u KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock); KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock)); + ticket = uvm_wait_prepare(); if (*spare != NULL) { anon = *spare; *spare = NULL; @@ -629,7 +636,7 @@ uvmfault_promote(struct uvm_faultinfo *u UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0); cpu_count(CPU_COUNT_FLTNORAM, 1); - uvm_wait("flt_noram5"); + uvm_wait("flt_noram5", ticket); error = ERESTART; goto done; } @@ -913,10 +920,13 @@ uvm_fault_internal(struct vm_map *orig_m ufi.entry->object.uvm_obj; if (uobj && uobj->pgops->pgo_fault != NULL) { + uint64_t ticket; + /* * invoke "special" fault routine. */ rw_enter(uobj->vmobjlock, RW_WRITER); + ticket = uvm_wait_prepare(); /* locked: maps(read), amap(if there), uobj */ error = uobj->pgops->pgo_fault(&ufi, flt.startva, pages, flt.npages, @@ -939,7 +949,7 @@ uvm_fault_internal(struct vm_map *orig_m * reclaimed. */ if (error == ENOMEM && uvm_reclaimable()) { - uvm_wait("pgo_fault"); + uvm_wait("pgo_fault", ticket); error = ERESTART; } } else { @@ -1563,6 +1573,8 @@ uvm_fault_upper_loan( /* >1 case is already ok */ if (anon->an_ref == 1) { + uint64_t ticket; + /* breaking loan requires a write lock. */ error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL); if (error != 0) { @@ -1570,10 +1582,11 @@ uvm_fault_upper_loan( } KASSERT(rw_write_held(amap->am_lock)); + ticket = uvm_wait_prepare(); error = uvm_loanbreak_anon(anon, *ruobj); if (error != 0) { uvmfault_unlockall(ufi, amap, *ruobj); - uvm_wait("flt_noram2"); + uvm_wait("flt_noram2", ticket); return ERESTART; } /* if we were a loan receiver uobj is gone */ @@ -1677,6 +1690,7 @@ uvm_fault_upper_enter( struct pmap *pmap = ufi->orig_map->pmap; vaddr_t va = ufi->orig_rvaddr; struct vm_amap * const amap = ufi->entry->aref.ar_amap; + uint64_t ticket; UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); /* locked: maps(read), amap, oanon, anon(if different from oanon) */ @@ -1694,6 +1708,7 @@ uvm_fault_upper_enter( UVMHIST_LOG(maphist, " MAPPING: anon: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd", (uintptr_t)pmap, va, (uintptr_t)pg, flt->promote); + ticket = uvm_wait_prepare(); if (pmap_enter(pmap, va, VM_PAGE_TO_PHYS(pg), flt->enter_prot, flt->access_type | PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) { @@ -1741,7 +1756,7 @@ uvm_fault_upper_enter( return ENOMEM; } /* XXX instrumentation */ - uvm_wait("flt_pmfail1"); + uvm_wait("flt_pmfail1", ticket); return ERESTART; } @@ -2330,6 +2345,7 @@ uvm_fault_lower_direct_loan( * write fault: must break the loan here. to do this * we need a write lock on the object. */ + uint64_t ticket; error = uvm_fault_lower_upgrade(ufi, flt, amap, uobj, uobjpage); if (error != 0) { @@ -2337,6 +2353,7 @@ uvm_fault_lower_direct_loan( } KASSERT(rw_write_held(uobj->vmobjlock)); + ticket = uvm_wait_prepare(); pg = uvm_loanbreak(uobjpage); if (pg == NULL) { @@ -2345,7 +2362,7 @@ uvm_fault_lower_direct_loan( " out of RAM breaking loan, waiting", 0,0,0,0); cpu_count(CPU_COUNT_FLTNORAM, 1); - uvm_wait("flt_noram4"); + uvm_wait("flt_noram4", ticket); return ERESTART; } *rpg = pg; @@ -2462,6 +2479,7 @@ uvm_fault_lower_enter( { struct vm_amap * const amap = ufi->entry->aref.ar_amap; const bool readonly = uvm_pagereadonly_p(pg); + uint64_t ticket; int error; UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); @@ -2506,6 +2524,7 @@ uvm_fault_lower_enter( UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map, (void *)ufi->orig_rvaddr, pg); KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly); + ticket = uvm_wait_prepare(); if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, VM_PAGE_TO_PHYS(pg), readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot, @@ -2541,7 +2560,7 @@ uvm_fault_lower_enter( return error; } /* XXX instrumentation */ - uvm_wait("flt_pmfail2"); + uvm_wait("flt_pmfail2", ticket); return ERESTART; } diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_glue.c --- a/sys/uvm/uvm_glue.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_glue.c Tue Aug 18 16:13:20 2026 +0000 @@ -257,15 +257,18 @@ uarea_poolpage_alloc(struct pool *pp, in #if defined(PMAP_MAP_POOLPAGE) while (USPACE == PAGE_SIZE && (USPACE_ALIGN == 0 || USPACE_ALIGN == PAGE_SIZE)) { + uint64_t ticket; struct vm_page *pg; vaddr_t va; + + ticket = uvm_wait_prepare(); #if defined(PMAP_ALLOC_POOLPAGE) pg = PMAP_ALLOC_POOLPAGE(0); #else pg = uvm_pagealloc(NULL, 0, NULL, 0); #endif if (pg == NULL) { - uvm_wait("uarea"); + uvm_wait("uarea", ticket); continue; } va = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg)); diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_km.c --- a/sys/uvm/uvm_km.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_km.c Tue Aug 18 16:13:20 2026 +0000 @@ -614,6 +614,7 @@ uvm_km_alloc(struct vm_map *map, vsize_t vaddr_t kva, loopva; vaddr_t offset; vsize_t loopsize; + uint64_t ticket; struct vm_page *pg; struct uvm_object *obj; int pgaflags; @@ -687,6 +688,7 @@ uvm_km_alloc(struct vm_map *map, vsize_t KASSERTMSG(!pmap_extract(pmap_kernel(), loopva, NULL), "loopva=%#"PRIxVADDR, loopva); + ticket = uvm_wait_prepare(); pg = uvm_pagealloc_strat(NULL, offset, NULL, pgaflags, #ifdef UVM_KM_VMFREELIST UVM_PGA_STRAT_ONLY, UVM_KM_VMFREELIST @@ -707,7 +709,8 @@ uvm_km_alloc(struct vm_map *map, vsize_t flags & UVM_KMF_TYPEMASK); return (0); } else { - uvm_wait("km_getwait2"); /* sleep here */ + /* sleep here */ + uvm_wait("km_getwait2", ticket); continue; } } @@ -798,6 +801,7 @@ int uvm_km_kmem_alloc(vmem_t *vm, vmem_size_t size, vm_flag_t flags, vmem_addr_t *addr) { + uint64_t ticket; struct vm_page *pg; vmem_addr_t va; int rc; @@ -809,6 +813,7 @@ uvm_km_kmem_alloc(vmem_t *vm, vmem_size_ #if defined(PMAP_MAP_POOLPAGE) if (size == PAGE_SIZE) { again: + ticket = uvm_wait_prepare(); #ifdef PMAP_ALLOC_POOLPAGE pg = PMAP_ALLOC_POOLPAGE((flags & VM_SLEEP) ? 0 : UVM_PGA_USERESERVE); @@ -818,7 +823,7 @@ again: #endif /* PMAP_ALLOC_POOLPAGE */ if (__predict_false(pg == NULL)) { if (flags & VM_SLEEP) { - uvm_wait("plpg"); + uvm_wait("plpg", ticket); goto again; } return ENOMEM; @@ -854,12 +859,13 @@ again: " pa=%#"PRIxPADDR" vmem=%p", loopva, loopsize, pa, vm); + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, loopva, NULL, UVM_FLAG_COLORMATCH | ((flags & VM_SLEEP) ? 0 : UVM_PGA_USERESERVE)); if (__predict_false(pg == NULL)) { if (flags & VM_SLEEP) { - uvm_wait("plpg"); + uvm_wait("plpg", ticket); continue; } else { uvm_km_pgremove_intrsafe(kernel_map, va, diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_loan.c --- a/sys/uvm/uvm_loan.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_loan.c Tue Aug 18 16:13:20 2026 +0000 @@ -834,11 +834,14 @@ again: pg = uvm_pagelookup(&uvm_loanzero_object, 0); if (__predict_false(pg == NULL)) { - while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL, - UVM_PGA_ZERO)) == NULL) { + uint64_t ticket; + + while (ticket = uvm_wait_prepare(), + (pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL, + UVM_PGA_ZERO)) == NULL) { rw_exit(uvm_loanzero_object.vmobjlock); uvmfault_unlockall(ufi, amap, NULL); - uvm_wait("loanzero"); + uvm_wait("loanzero", ticket); if (!uvmfault_relock(ufi)) { return (0); } diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_object.c --- a/sys/uvm/uvm_object.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_object.c Tue Aug 18 16:13:20 2026 +0000 @@ -162,11 +162,15 @@ uvm_obj_wirepages(struct uvm_object *uob */ if (pgs[i]->loan_count) { while (pgs[i]->loan_count) { + uint64_t ticket = + uvm_wait_prepare(); + pg = uvm_loanbreak(pgs[i]); if (!pg) { rw_exit(uobj->vmobjlock); - uvm_wait("uobjwirepg"); - rw_enter(uobj->vmobjlock, RW_WRITER); + uvm_wait("uobjwirepg", ticket); + rw_enter(uobj->vmobjlock, + RW_WRITER); continue; } } diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_pdaemon.c --- a/sys/uvm/uvm_pdaemon.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_pdaemon.c Tue Aug 18 16:13:20 2026 +0000 @@ -114,35 +114,129 @@ static void uvmpd_tune(void); static void uvmpd_pool_drain_thread(void *); static void uvmpd_pool_drain_wakeup(void); -static unsigned int uvm_pagedaemon_waiters; +static volatile unsigned uvm_pagedaemon_waiters __cacheline_aligned; +static volatile unsigned uvm_waiter_wakeup __cacheline_aligned; /* State for the pool drainer thread */ static kmutex_t uvmpd_lock __cacheline_aligned; static kcondvar_t uvmpd_pool_drain_cv; static bool uvmpd_pool_drain_run = false; +static volatile unsigned uvmpd_sleeping __cacheline_aligned; +static volatile unsigned uvmpd_wakeup __cacheline_aligned; + +#if __HAVE_ATOMIC64_LOADSTORE + +static volatile uint64_t uvmpd_gen; + +static uint64_t +uvmpd_read_gen(void) +{ + + return atomic_load_relaxed(&uvmpd_gen); +} + +static void +uvmpd_advance_gen(void) +{ + + KASSERT(mutex_owned(&uvmpd_lock)); + atomic_store_relaxed(&uvmpd_gen, atomic_load_relaxed(&uvmpd_gen) + 1); +} + +#else + +static volatile struct { +#if _BYTE_ORDER == _BIG_ENDIAN + uint32_t hi, lo; +#else + uint32_t lo, hi; +#endif +} uvmpd_gen; + +static uint64_t +uvmpd_read_gen(void) +{ + uint32_t hi, lo; + +top: while (__predict_false((hi = atomic_load_relaxed(&uvmpd_gen.hi)) == + 0xffffffff)) + SPINLOCK_BACKOFF_HOOK; + membar_consumer(); + lo = atomic_load_relaxed(&uvmpd_gen.lo); + membar_consumer(); + if (hi != atomic_load_relaxed(&uvmpd_gen.hi)) + goto top; + + return (uint64_t)hi << 32 | lo; +} + +static void +uvmpd_advance_gen(void) +{ + uint32_t hi, lo; + + KASSERT(mutex_owned(&uvmpd_lock)); + + hi = atomic_load_relaxed(&uvmpd_gen.hi); + lo = atomic_load_relaxed(&uvmpd_gen.lo); + if (++lo == 0) { + hi++; + atomic_store_relaxed(&uvmpd_gen.hi, 0xffffffff); + membar_producer(); + atomic_store_relaxed(&uvmpd_gen.lo, lo); + membar_producer(); + atomic_store_relaxed(&uvmpd_gen.hi, hi); + } else { + atomic_store_relaxed(&uvmpd_gen.lo, lo); + } +} + +#endif + /* * XXX hack to avoid hangs when large processes fork. */ u_int uvm_extrapages; +uint64_t +uvm_wait_prepare(void) +{ + uint64_t ticket; + + ticket = uvmpd_read_gen(); + + return ticket; +} + /* * uvm_wait: wait (sleep) for the page daemon to free some pages * * => should be called with all locks released * => should _not_ be called by the page daemon (to avoid deadlock) + * => caller must have acquired a ticket with uvm_wait_prepare, + * before trying an allocation that we now wait for */ void -uvm_wait(const char *wmsg) +uvm_wait(const char *wmsg, uint64_t ticket) { + uint64_t gen = ticket; int timo = 0; if (uvm.pagedaemon_lwp == NULL) panic("out of memory before the pagedaemon thread exists"); + if (gen != uvmpd_read_gen()) + return; + mutex_spin_enter(&uvmpd_lock); + if (gen != uvmpd_read_gen()) { + mutex_spin_exit(&uvmpd_lock); + return; + } + /* * check for page daemon going to sleep (waiting for itself) */ @@ -173,7 +267,15 @@ uvm_wait(const char *wmsg) #endif } - uvm_pagedaemon_waiters++; + atomic_store_relaxed(&uvm_pagedaemon_waiters, + atomic_load_relaxed(&uvm_pagedaemon_waiters) + 1); + membar_sync(); + if (atomic_swap_uint(&uvm_waiter_wakeup, 0)) { + atomic_store_relaxed(&uvm_pagedaemon_waiters, 0); + wakeup(&uvmexp.free); + mutex_spin_exit(&uvmpd_lock); + return; + } wakeup(&uvm.pagedaemon); /* wake the daemon! */ UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_lock, false, wmsg, timo); } @@ -192,6 +294,10 @@ uvm_kick_pdaemon(void) (fpages + uvmexp.paging < uvmexp.freetarg && uvmpdpol_needsscan_p()) || uvm_km_va_starved_p()) { + atomic_store_relaxed(&uvmpd_wakeup, 1); + membar_sync(); + if (!atomic_load_relaxed(&uvmpd_sleeping)) + return; mutex_spin_enter(&uvmpd_lock); wakeup(&uvm.pagedaemon); mutex_spin_exit(&uvmpd_lock); @@ -288,11 +394,19 @@ uvm_pageout(void *arg) kmem_va_starved = uvm_km_va_starved_p(); mutex_spin_enter(&uvmpd_lock); - if ((uvm_pagedaemon_waiters == 0 || uvmexp.paging > 0) && + if ((atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0 || + uvmexp.paging > 0) && !kmem_va_starved) { UVMHIST_LOG(pdhist," <>",0,0,0,0); - UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon, - &uvmpd_lock, false, "pgdaemon", 0); + atomic_store_relaxed(&uvmpd_sleeping, 1); + membar_sync(); + if (atomic_swap_uint(&uvmpd_wakeup, 0)) { + mutex_spin_exit(&uvmpd_lock); + } else { + UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon, + &uvmpd_lock, false, "pgdaemon", 0); + } + atomic_store_relaxed(&uvmpd_sleeping, 0); uvmexp.pdwoke++; UVMHIST_LOG(pdhist," <>",0,0,0,0); } else { @@ -336,8 +450,9 @@ uvm_pageout(void *arg) if (uvm_availmem(false) > uvmexp.reserve_kernel || uvmexp.paging == 0) { mutex_spin_enter(&uvmpd_lock); + uvmpd_advance_gen(); wakeup(&uvmexp.free); - uvm_pagedaemon_waiters = 0; + atomic_store_relaxed(&uvm_pagedaemon_waiters, 0); mutex_spin_exit(&uvmpd_lock); } @@ -379,15 +494,27 @@ uvm_pageout_done(int npages) /* * wake up either of pagedaemon or LWPs waiting for it. */ - - mutex_spin_enter(&uvmpd_lock); if (uvm_availmem(false) <= uvmexp.reserve_kernel) { + atomic_store_relaxed(&uvmpd_wakeup, 1); + membar_sync(); + if (!atomic_load_relaxed(&uvmpd_sleeping)) + return; + mutex_spin_enter(&uvmpd_lock); wakeup(&uvm.pagedaemon); - } else if (uvm_pagedaemon_waiters != 0) { - wakeup(&uvmexp.free); - uvm_pagedaemon_waiters = 0; + mutex_spin_exit(&uvmpd_lock); + } else { + atomic_store_relaxed(&uvm_waiter_wakeup, 1); + membar_sync(); + if (atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0) + return; + mutex_spin_enter(&uvmpd_lock); + uvmpd_advance_gen(); + if (atomic_load_relaxed(&uvm_pagedaemon_waiters) != 0) { + wakeup(&uvmexp.free); + atomic_store_relaxed(&uvm_pagedaemon_waiters, 0); + } + mutex_spin_exit(&uvmpd_lock); } - mutex_spin_exit(&uvmpd_lock); } static krwlock_t * diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_pdaemon.h --- a/sys/uvm/uvm_pdaemon.h Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_pdaemon.h Tue Aug 18 16:13:20 2026 +0000 @@ -83,7 +83,8 @@ struct krwlock; * prototypes */ -void uvm_wait(const char *); +uint64_t uvm_wait_prepare(void); +void uvm_wait(const char *, uint64_t); bool uvm_reclaimable(void); struct krwlock *uvmpd_trylockowner(struct vm_page *); diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_pglist.c --- a/sys/uvm/uvm_pglist.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_pglist.c Tue Aug 18 16:13:20 2026 +0000 @@ -300,6 +300,7 @@ static int uvm_pglistalloc_contig_aggressive(int num, paddr_t low, paddr_t high, paddr_t alignment, paddr_t boundary, struct pglist *rlist) { + uint64_t ticket; struct vm_page *pg; struct pglist tmp; paddr_t pa, off, spa, amask, bmask, rlo, rhi; @@ -321,9 +322,10 @@ uvm_pglistalloc_contig_aggressive(int nu KASSERT(bmask <= amask); mutex_enter(&uvm_pglistalloc_contig_lock); while (uvm_reclaimable()) { + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(NULL, 0, NULL, 0); if (pg == NULL) { - uvm_wait("pglac2"); + uvm_wait("pglac2", ticket); continue; } pg->flags |= PG_PGLCA; @@ -615,6 +617,7 @@ uvm_pglistalloc_simple(int num, paddr_t struct pglist *rlist, int waitok) { int fl, error; + uint64_t ticket; uvm_physseg_t psi; int count = 0; @@ -623,6 +626,8 @@ uvm_pglistalloc_simple(int num, paddr_t bool valid = false; again: + ticket = uvm_wait_prepare(); + /* * Block all memory allocation and lock the free list. */ @@ -673,7 +678,7 @@ out: if (error) { if (waitok) { - uvm_wait("pglalloc"); + uvm_wait("pglalloc", ticket); goto again; } else uvm_pglistfree(rlist); diff -r 38311baef8db -r 801b63d6bff9 sys/uvm/uvm_vnode.c --- a/sys/uvm/uvm_vnode.c Sun Aug 16 17:26:26 2026 +0000 +++ b/sys/uvm/uvm_vnode.c Tue Aug 18 16:13:20 2026 +0000 @@ -287,6 +287,7 @@ static int uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp, unsigned int flags, struct uvm_page_array *a, unsigned int nleft) { + uint64_t ticket; struct vm_page *pg; UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)uobj, offset, @@ -332,6 +333,7 @@ uvn_findpage(struct uvm_object *uobj, vo UVMHIST_LOG(ubchist, "noalloc", 0,0,0,0); return 0; } + ticket = uvm_wait_prepare(); pg = uvm_pagealloc(uobj, offset, NULL, UVM_FLAG_COLORMATCH); if (pg == NULL) { @@ -340,7 +342,7 @@ uvn_findpage(struct uvm_object *uobj, vo return 0; } rw_exit(uobj->vmobjlock); - uvm_wait("uvnfp1"); + uvm_wait("uvnfp1", ticket); uvm_page_array_clear(a); rw_enter(uobj->vmobjlock, RW_WRITER); continue;