Index: uvm/uvm_pager.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pager.c,v retrieving revision 1.111.8.1 diff -u -p -r1.111.8.1 uvm_pager.c --- uvm/uvm_pager.c 27 Dec 2019 06:58:56 -0000 1.111.8.1 +++ uvm/uvm_pager.c 15 Jan 2020 14:35:42 -0000 @@ -42,6 +42,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_pager.c, #include #include #include +#include #include @@ -478,7 +479,7 @@ uvm_aio_aiodone_pages(struct vm_page **p else uvm_swap_free(swslot, npages); } - uvmexp.pdpending--; + atomic_dec_uint(&uvmexp.pdpending); #endif /* defined(VMSWAP) */ } } Index: uvm/uvm_pdaemon.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdaemon.c,v retrieving revision 1.110 diff -u -p -r1.110 uvm_pdaemon.c --- uvm/uvm_pdaemon.c 21 Apr 2019 15:32:18 -0000 1.110 +++ uvm/uvm_pdaemon.c 15 Jan 2020 14:35:42 -0000 @@ -80,6 +80,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon. #include #include #include +#include #include #include @@ -97,7 +98,7 @@ UVMHIST_DEFINE(pdhist); #define UVMPD_NUMDIRTYREACTS 16 -#define UVMPD_NUMTRYLOCKOWNER 16 +#define UVMPD_NUMTRYLOCKOWNER 128 /* * local prototypes @@ -413,6 +414,7 @@ uvmpd_trylockowner(struct vm_page *pg) { struct uvm_object *uobj = pg->uobject; kmutex_t *slock; + u_int count, i; KASSERT(mutex_owned(&uvm_pageqlock)); @@ -425,7 +427,15 @@ uvmpd_trylockowner(struct vm_page *pg) slock = anon->an_lock; } - if (!mutex_tryenter(slock)) { + count = SPINLOCK_BACKOFF_MAX; + for (i = UVMPD_NUMTRYLOCKOWNER; i > 0; i--) { + if (mutex_tryenter(slock)) { + break; + } + SPINLOCK_BACKOFF(count); + } + if (i == 0) { + kpause("pdtrylck", false, 1, &uvm_pageqlock); return NULL; } @@ -666,7 +676,6 @@ uvmpd_scan_queue(void) struct swapcluster swc; #endif /* defined(VMSWAP) */ int dirtyreacts; - int lockownerfail; kmutex_t *slock; UVMHIST_FUNC("uvmpd_scan_queue"); UVMHIST_CALLED(pdhist); @@ -681,7 +690,6 @@ uvmpd_scan_queue(void) #endif /* defined(VMSWAP) */ dirtyreacts = 0; - lockownerfail = 0; uvmpdpol_scaninit(); while (/* CONSTCOND */ 1) { @@ -733,20 +741,6 @@ uvmpd_scan_queue(void) slock = uvmpd_trylockowner(p); if (slock == NULL) { - /* - * yield cpu to make a chance for an LWP holding - * the lock run. otherwise we can busy-loop too long - * if the page queue is filled with a lot of pages - * from few objects. - */ - lockownerfail++; - if (lockownerfail > UVMPD_NUMTRYLOCKOWNER) { - mutex_exit(&uvm_pageqlock); - /* XXX Better than yielding but inadequate. */ - kpause("livelock", false, 1, NULL); - mutex_enter(&uvm_pageqlock); - lockownerfail = 0; - } continue; } if (p->flags & PG_BUSY) { @@ -907,7 +901,7 @@ uvmpd_scan_queue(void) * for the next loop. */ - uvmexp.pdpending++; + atomic_inc_uint(&uvmexp.pdpending); #else /* defined(VMSWAP) */ uvm_pageactivate(p); @@ -915,6 +909,8 @@ uvmpd_scan_queue(void) #endif /* defined(VMSWAP) */ } + uvmpdpol_scanfini(); + #if defined(VMSWAP) mutex_exit(&uvm_pageqlock); swapcluster_flush(&swc, true); @@ -1038,17 +1034,39 @@ uvm_estimatepageable(int *active, int *i static void uvmpd_pool_drain_thread(void *arg) { - int bufcnt; + struct pool *firstpool, *curpool; + int bufcnt, lastslept; + bool cycled; + firstpool = NULL; + cycled = true; for (;;) { + /* + * sleep until awoken by the pagedaemon. + */ mutex_enter(&uvmpd_pool_drain_lock); if (!uvmpd_pool_drain_run) { + lastslept = hardclock_ticks; cv_wait(&uvmpd_pool_drain_cv, &uvmpd_pool_drain_lock); + if (hardclock_ticks != lastslept) { + cycled = false; + firstpool = NULL; + } } uvmpd_pool_drain_run = false; mutex_exit(&uvmpd_pool_drain_lock); /* + * rate limit draining, otherwise in desperate circumstances + * this can totally saturate the system with xcall activity. + */ + if (cycled) { + kpause("uvmpdlmt", false, 1, NULL); + cycled = false; + firstpool = NULL; + } + + /* * kill unused metadata buffers. */ mutex_spin_enter(&uvm_fpageqlock); @@ -1064,7 +1082,13 @@ uvmpd_pool_drain_thread(void *arg) /* * drain a pool. */ - pool_drain(NULL); + (void)pool_drain(&curpool); + KASSERT(curpool != NULL); + if (firstpool == NULL) { + firstpool = curpool; + } else if (firstpool == curpool) { + cycled = true; + } } /*NOTREACHED*/ } Index: uvm/uvm_pdpolicy.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdpolicy.h,v retrieving revision 1.3 diff -u -p -r1.3 uvm_pdpolicy.h --- uvm/uvm_pdpolicy.h 21 Feb 2007 23:00:14 -0000 1.3 +++ uvm/uvm_pdpolicy.h 15 Jan 2020 14:35:42 -0000 @@ -51,6 +51,7 @@ void uvmpdpol_anfree(struct vm_anon *); void uvmpdpol_tune(void); void uvmpdpol_scaninit(void); +void uvmpdpol_scanfini(void); struct vm_page *uvmpdpol_selectvictim(void); void uvmpdpol_balancequeue(int); Index: uvm/uvm_pdpolicy_clock.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdpolicy_clock.c,v retrieving revision 1.17 diff -u -p -r1.17 uvm_pdpolicy_clock.c --- uvm/uvm_pdpolicy_clock.c 30 Jan 2012 17:21:52 -0000 1.17 +++ uvm/uvm_pdpolicy_clock.c 15 Jan 2020 14:35:43 -0000 @@ -105,9 +105,8 @@ struct uvmpdpol_globalstate { }; struct uvmpdpol_scanstate { - bool ss_first; bool ss_anonreact, ss_filereact, ss_execreact; - struct vm_page *ss_nextpg; + struct vm_page ss_marker; }; static struct uvmpdpol_globalstate pdpol_state; @@ -160,8 +159,18 @@ uvmpdpol_scaninit(void) ss->ss_anonreact = anonreact; ss->ss_filereact = filereact; ss->ss_execreact = execreact; + memset(&ss->ss_marker, 0, sizeof(ss->ss_marker)); + ss->ss_marker.flags = PG_MARKER; + TAILQ_INSERT_HEAD(&pdpol_state.s_inactiveq, &ss->ss_marker, + pageq.queue); +} + +void +uvmpdpol_scanfini(void) +{ + struct uvmpdpol_scanstate *ss = &pdpol_scanstate; - ss->ss_first = true; + TAILQ_REMOVE(&pdpol_state.s_inactiveq, &ss->ss_marker, pageq.queue); } struct vm_page * @@ -177,20 +186,11 @@ uvmpdpol_selectvictim(void) struct vm_anon *anon; struct uvm_object *uobj; - if (ss->ss_first) { - pg = TAILQ_FIRST(&pdpol_state.s_inactiveq); - ss->ss_first = false; - } else { - pg = ss->ss_nextpg; - if (pg != NULL && (pg->pqflags & PQ_INACTIVE) == 0) { - pg = TAILQ_FIRST(&pdpol_state.s_inactiveq); - } - } + pg = TAILQ_NEXT(&ss->ss_marker, pageq.queue); if (pg == NULL) { break; } - ss->ss_nextpg = TAILQ_NEXT(pg, pageq.queue); - + KASSERT((pg->flags & PG_MARKER) == 0); uvmexp.pdscans++; /* @@ -213,6 +213,15 @@ uvmpdpol_selectvictim(void) uobj = pg->uobject; /* + * now prepare to move on to the next page. + */ + + TAILQ_REMOVE(&pdpol_state.s_inactiveq, &ss->ss_marker, + pageq.queue); + TAILQ_INSERT_AFTER(&pdpol_state.s_inactiveq, pg, + &ss->ss_marker, pageq.queue); + + /* * enforce the minimum thresholds on different * types of memory usage. if reusing the current * page would reduce that type of usage below its @@ -247,7 +256,7 @@ void uvmpdpol_balancequeue(int swap_shortage) { int inactive_shortage; - struct vm_page *p, *nextpg; + struct vm_page *p, marker; kmutex_t *lock; /* @@ -255,11 +264,28 @@ uvmpdpol_balancequeue(int swap_shortage) * our inactive target. */ - inactive_shortage = pdpol_state.s_inactarg - pdpol_state.s_inactive; - for (p = TAILQ_FIRST(&pdpol_state.s_activeq); - p != NULL && (inactive_shortage > 0 || swap_shortage > 0); - p = nextpg) { - nextpg = TAILQ_NEXT(p, pageq.queue); + memset(&marker, 0, sizeof(marker)); + marker.flags = PG_MARKER; + + TAILQ_INSERT_HEAD(&pdpol_state.s_activeq, &marker, pageq.queue); + for (;;) { + inactive_shortage = + pdpol_state.s_inactarg - pdpol_state.s_inactive; + if (inactive_shortage <= 0 && swap_shortage <= 0) { + break; + } + p = TAILQ_NEXT(&marker, pageq.queue); + if (p == NULL) { + break; + } + KASSERT((p->flags & PG_MARKER) == 0); + + /* + * now prepare to move on to the next page. + */ + TAILQ_REMOVE(&pdpol_state.s_activeq, &marker, pageq.queue); + TAILQ_INSERT_AFTER(&pdpol_state.s_activeq, p, &marker, + pageq.queue); /* * if there's a shortage of swap slots, try to free it. @@ -288,6 +314,7 @@ uvmpdpol_balancequeue(int swap_shortage) mutex_exit(lock); } } + TAILQ_REMOVE(&pdpol_state.s_activeq, &marker, pageq.queue); } void Index: uvm/uvm_pdpolicy_clockpro.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdpolicy_clockpro.c,v retrieving revision 1.17 diff -u -p -r1.17 uvm_pdpolicy_clockpro.c --- uvm/uvm_pdpolicy_clockpro.c 20 Jun 2011 23:18:58 -0000 1.17 +++ uvm/uvm_pdpolicy_clockpro.c 15 Jan 2020 14:35:43 -0000 @@ -1190,6 +1190,12 @@ uvmpdpol_scaninit(void) ss->ss_nscanned = 0; } +void +uvmpdpol_scanfini(void) +{ + +} + struct vm_page * uvmpdpol_selectvictim(void) {