Index: miscfs/genfs/genfs_io.c
===================================================================
RCS file: /cvsroot/src/sys/miscfs/genfs/genfs_io.c,v
retrieving revision 1.76
diff -u -p -r1.76 genfs_io.c
--- miscfs/genfs/genfs_io.c 6 Oct 2019 05:48:00 -0000 1.76
+++ miscfs/genfs/genfs_io.c 6 Dec 2019 13:11:12 -0000
@@ -79,9 +79,7 @@ genfs_rel_pages(struct vm_page **pgs, un
pg->flags |= PG_RELEASED;
}
}
- mutex_enter(&uvm_pageqlock);
uvm_page_unbusy(pgs, npages);
- mutex_exit(&uvm_pageqlock);
}
static void
@@ -466,7 +464,6 @@ startover:
out:
UVMHIST_LOG(ubchist, "succeeding, npages %jd", npages,0,0,0);
error = 0;
- mutex_enter(&uvm_pageqlock);
for (i = 0; i < npages; i++) {
struct vm_page *pg = pgs[i];
if (pg == NULL) {
@@ -490,7 +487,10 @@ out:
uvm_pagezero(pg);
}
if (pg->flags & PG_RELEASED) {
- uvm_pagefree(pg);
+ mutex_enter(&uvm_pageqlock);
+ uvm_pagefree1(pg);
+ mutex_exit(&uvm_pageqlock);
+ uvm_pagefree2(pg);
continue;
}
uvm_pageenqueue(pg);
@@ -498,7 +498,6 @@ out:
UVM_PAGE_OWN(pg, NULL);
}
}
- mutex_exit(&uvm_pageqlock);
if (memwrite) {
genfs_markdirty(vp);
}
@@ -1201,9 +1200,6 @@ retry:
* apply FREE or DEACTIVATE options if requested.
*/
- if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
- mutex_enter(&uvm_pageqlock);
- }
for (i = 0; i < npages; i++) {
tpg = pgs[i];
KASSERT(tpg->uobject == uobj);
@@ -1219,7 +1215,9 @@ retry:
tpg->flags |= freeflag;
if (pagedaemon) {
uvm_pageout_start(1);
+ mutex_enter(&uvm_pageqlock);
uvm_pagedequeue(tpg);
+ mutex_exit(&uvm_pageqlock);
}
} else {
@@ -1230,15 +1228,15 @@ retry:
*/
nextpg = TAILQ_NEXT(tpg, listq.queue);
- uvm_pagefree(tpg);
+ mutex_enter(&uvm_pageqlock);
+ uvm_pagefree1(tpg);
+ mutex_exit(&uvm_pageqlock);
+ uvm_pagefree2(tpg);
if (pagedaemon)
uvmexp.pdfreed++;
}
}
}
- if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
- mutex_exit(&uvm_pageqlock);
- }
if (needs_clean) {
modified = true;
@@ -1646,7 +1644,6 @@ genfs_compat_getpages(void *v)
}
uvm_pagermapout(kva, npages);
mutex_enter(uobj->vmobjlock);
- mutex_enter(&uvm_pageqlock);
for (i = 0; i < npages; i++) {
pg = pgs[i];
if (error && (pg->flags & PG_FAKE) != 0) {
@@ -1659,7 +1656,6 @@ genfs_compat_getpages(void *v)
if (error) {
uvm_page_unbusy(pgs, npages);
}
- mutex_exit(&uvm_pageqlock);
if (error == 0 && memwrite) {
genfs_markdirty(vp);
}
Index: nfs/nfs_bio.c
===================================================================
RCS file: /cvsroot/src/sys/nfs/nfs_bio.c,v
retrieving revision 1.191
diff -u -p -r1.191 nfs_bio.c
--- nfs/nfs_bio.c 15 Jul 2015 03:28:55 -0000 1.191
+++ nfs/nfs_bio.c 6 Dec 2019 13:11:13 -0000
@@ -1340,9 +1340,7 @@ nfs_getpages(void *v)
* available and put back original pgs array.
*/
- mutex_enter(&uvm_pageqlock);
uvm_page_unbusy(pgs, npages);
- mutex_exit(&uvm_pageqlock);
*ap->a_count = 0;
memcpy(pgs, opgs,
npages * sizeof(struct vm_pages *));
Index: ufs/lfs/lfs_pages.c
===================================================================
RCS file: /cvsroot/src/sys/ufs/lfs/lfs_pages.c,v
retrieving revision 1.15
diff -u -p -r1.15 lfs_pages.c
--- ufs/lfs/lfs_pages.c 19 Aug 2017 14:22:49 -0000 1.15
+++ ufs/lfs/lfs_pages.c 6 Dec 2019 13:11:13 -0000
@@ -539,9 +539,7 @@ retry:
"lfsput2", 0);
mutex_enter(vp->v_interlock);
}
- mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
- mutex_exit(&uvm_pageqlock);
}
ap->a_offlo = blkeof;
if (ap->a_offhi > 0 && ap->a_offhi <= ap->a_offlo) {
Index: ufs/lfs/lfs_vfsops.c
===================================================================
RCS file: /cvsroot/src/sys/ufs/lfs/lfs_vfsops.c,v
retrieving revision 1.365
diff -u -p -r1.365 lfs_vfsops.c
--- ufs/lfs/lfs_vfsops.c 28 May 2019 08:59:35 -0000 1.365
+++ ufs/lfs/lfs_vfsops.c 6 Dec 2019 13:11:14 -0000
@@ -2238,14 +2238,15 @@ lfs_gop_write(struct vnode *vp, struct v
pgs[0]->offset, eof, npages));
}
- mutex_enter(&uvm_pageqlock);
for (i = 0; i < npages; i++) {
pg = pgs[i];
if (pg->flags & PG_PAGEOUT)
uvm_pageout_done(1);
if (pg->flags & PG_DELWRI) {
+ mutex_enter(&uvm_pageqlock);
uvm_pageunwire(pg);
+ mutex_exit(&uvm_pageqlock);
}
uvm_pageactivate(pg);
pg->flags &= ~(PG_CLEAN|PG_DELWRI|PG_PAGEOUT|PG_RELEASED);
@@ -2262,7 +2263,6 @@ lfs_gop_write(struct vnode *vp, struct v
}
/* uvm_pageunbusy takes care of PG_BUSY, PG_WANTED */
uvm_page_unbusy(pgs, npages);
- mutex_exit(&uvm_pageqlock);
mutex_exit(vp->v_interlock);
return EAGAIN;
}
Index: ufs/lfs/ulfs_inode.c
===================================================================
RCS file: /cvsroot/src/sys/ufs/lfs/ulfs_inode.c,v
retrieving revision 1.21
diff -u -p -r1.21 ulfs_inode.c
--- ufs/lfs/ulfs_inode.c 28 Oct 2017 00:37:13 -0000 1.21
+++ ufs/lfs/ulfs_inode.c 6 Dec 2019 13:11:14 -0000
@@ -234,7 +234,6 @@ ulfs_balloc_range(struct vnode *vp, off_
GOP_SIZE(vp, off + len, &eob, 0);
mutex_enter(uobj->vmobjlock);
- mutex_enter(&uvm_pageqlock);
for (i = 0; i < npages; i++) {
KASSERT((pgs[i]->flags & PG_RELEASED) == 0);
if (!error) {
@@ -246,7 +245,6 @@ ulfs_balloc_range(struct vnode *vp, off_
}
uvm_pageactivate(pgs[i]);
}
- mutex_exit(&uvm_pageqlock);
uvm_page_unbusy(pgs, npages);
mutex_exit(uobj->vmobjlock);
Index: ufs/ufs/ufs_inode.c
===================================================================
RCS file: /cvsroot/src/sys/ufs/ufs/ufs_inode.c,v
retrieving revision 1.105
diff -u -p -r1.105 ufs_inode.c
--- ufs/ufs/ufs_inode.c 10 Dec 2018 20:48:34 -0000 1.105
+++ ufs/ufs/ufs_inode.c 6 Dec 2019 13:11:14 -0000
@@ -270,7 +270,6 @@ ufs_balloc_range(struct vnode *vp, off_t
GOP_SIZE(vp, off + len, &eob, 0);
mutex_enter(uobj->vmobjlock);
- mutex_enter(&uvm_pageqlock);
for (i = 0; i < npages; i++) {
KASSERT((pgs[i]->flags & PG_RELEASED) == 0);
if (!error) {
@@ -282,7 +281,6 @@ ufs_balloc_range(struct vnode *vp, off_t
}
uvm_pageactivate(pgs[i]);
}
- mutex_exit(&uvm_pageqlock);
uvm_page_unbusy(pgs, npages);
mutex_exit(uobj->vmobjlock);
Index: uvm/uvm_amap.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_amap.c,v
retrieving revision 1.110
diff -u -p -r1.110 uvm_amap.c
--- uvm/uvm_amap.c 1 Dec 2019 14:24:43 -0000 1.110
+++ uvm/uvm_amap.c 6 Dec 2019 13:11:14 -0000
@@ -1074,7 +1074,6 @@ ReStart:
UVM_PAGE_OWN(npg, NULL);
}
/* Activate all pages. Some may be missing because of retry above. */
- mutex_enter(&uvm_pageqlock);
for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
anon = amap->am_anon[amap->am_slots[lcv]];
KASSERT(anon->an_lock == amap->am_lock);
@@ -1082,7 +1081,6 @@ ReStart:
uvm_pageactivate(anon->an_page);
}
}
- mutex_exit(&uvm_pageqlock);
amap_unlock(amap);
}
Index: uvm/uvm_anon.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_anon.c,v
retrieving revision 1.68
diff -u -p -r1.68 uvm_anon.c
--- uvm/uvm_anon.c 2 Dec 2019 20:02:02 -0000 1.68
+++ uvm/uvm_anon.c 6 Dec 2019 13:11:14 -0000
@@ -230,16 +236,24 @@ uvm_anon_freelst(struct vm_amap *amap, s
}
}
- /* Free pages and leave a page replacement hint. */
+ /* Strip pages of their identities and leave a page replacement hint. */
mutex_enter(&uvm_pageqlock);
for (anon = anonlst; anon != NULL; anon = anon->an_link) {
UVMHIST_LOG(maphist, "anon 0x%#jx, page 0x%#jx: "
"releasing now!", (uintptr_t)anon,
(uintptr_t)anon->an_page, 0, 0);
+ /*
+ * Messing with an_page behind uvm_pagefree1()'s back is
+ * ugly, but means we can take pressure off uvm_pageqlock
+ * by sending the pages to the free list later, and avoid
+ * the overhead of constructing and traversing yet another
+ * list to do so.
+ */
if ((pg = anon->an_page) != NULL) {
- uvm_pagefree(pg);
+ uvm_pagefree1(pg);
}
uvmpdpol_anfree(anon);
+ anon->an_page = pg;
}
mutex_exit(&uvm_pageqlock);
amap_unlock(amap);
@@ -247,9 +261,13 @@ uvm_anon_freelst(struct vm_amap *amap, s
/* Free swap space, pages and vm_anon. */
while (anonlst) {
anon = anonlst->an_link;
+ if ((pg = anonlst->an_page) != NULL) {
+ uvm_pagefree2(pg);
+ }
/* Note: clears an_ref as well. */
anonlst->an_link = NULL;
anonlst->an_lock = NULL;
+ anonlst->an_page = NULL;
uvm_anon_free(anonlst);
anonlst = anon;
}
@@ -396,11 +414,9 @@ uvm_anon_pagein(struct vm_amap *amap, st
* Deactivate the page (to put it on a page queue).
*/
- mutex_enter(&uvm_pageqlock);
if (pg->wire_count == 0) {
uvm_pagedeactivate(pg);
}
- mutex_exit(&uvm_pageqlock);
if (pg->flags & PG_WANTED) {
pg->flags &= ~PG_WANTED;
@@ -458,13 +474,14 @@ uvm_anon_release(struct vm_anon *anon)
KASSERT(anon->an_ref == 0);
mutex_enter(&uvm_pageqlock);
- uvm_pagefree(pg);
+ uvm_pagefree1(pg);
mutex_exit(&uvm_pageqlock);
KASSERT(anon->an_page == NULL);
/* dispose should succeed as no one can reach this anon anymore. */
success = uvm_anon_dispose(anon);
KASSERT(success);
mutex_exit(anon->an_lock);
+ uvm_pagefree2(pg);
/* Note: extra reference is held for PG_RELEASED case. */
mutex_obj_free(anon->an_lock);
anon->an_lock = NULL;
Index: uvm/uvm_aobj.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_aobj.c,v
retrieving revision 1.130
diff -u -p -r1.130 uvm_aobj.c
--- uvm/uvm_aobj.c 1 Dec 2019 20:31:40 -0000 1.130
+++ uvm/uvm_aobj.c 6 Dec 2019 13:11:14 -0000
@@ -803,12 +803,10 @@ uao_put(struct uvm_object *uobj, voff_t
case PGO_CLEANIT|PGO_DEACTIVATE:
case PGO_DEACTIVATE:
deactivate_it:
- mutex_enter(&uvm_pageqlock);
/* skip the page if it's wired */
if (pg->wire_count == 0) {
uvm_pagedeactivate(pg);
}
- mutex_exit(&uvm_pageqlock);
break;
case PGO_FREE:
@@ -834,8 +832,9 @@ uao_put(struct uvm_object *uobj, voff_t
uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
mutex_enter(&uvm_pageqlock);
- uvm_pagefree(pg);
+ uvm_pagefree1(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pagefree2(pg);
break;
default:
@@ -1380,10 +1379,7 @@ uao_pagein_page(struct uvm_aobj *aobj, i
/*
* make sure it's on a page queue.
*/
- mutex_enter(&uvm_pageqlock);
- if (pg->wire_count == 0)
- uvm_pageenqueue(pg);
- mutex_exit(&uvm_pageqlock);
+ uvm_pageenqueue(pg);
if (pg->flags & PG_WANTED) {
wakeup(pg);
Index: uvm/uvm_bio.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_bio.c,v
retrieving revision 1.100
diff -u -p -r1.100 uvm_bio.c
--- uvm/uvm_bio.c 7 Nov 2019 07:45:14 -0000 1.100
+++ uvm/uvm_bio.c 6 Dec 2019 13:11:14 -0000
@@ -244,8 +244,9 @@ ubc_fault_page(const struct uvm_faultinf
KASSERT((pg->flags & PG_FAKE) == 0);
if (pg->flags & PG_RELEASED) {
mutex_enter(&uvm_pageqlock);
- uvm_pagefree(pg);
+ uvm_pagefree1(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pagefree2(pg);
return 0;
}
if (pg->loan_count != 0) {
@@ -287,9 +288,7 @@ ubc_fault_page(const struct uvm_faultinf
error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg),
prot & mask, PMAP_CANFAIL | (access_type & mask));
- mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
- mutex_exit(&uvm_pageqlock);
pg->flags &= ~(PG_BUSY|PG_WANTED);
UVM_PAGE_OWN(pg, NULL);
@@ -659,7 +658,6 @@ ubc_release(void *va, int flags)
}
umap->flags &= ~UMAP_PAGES_LOCKED;
mutex_enter(uobj->vmobjlock);
- mutex_enter(&uvm_pageqlock);
for (u_int i = 0; i < npages; i++) {
paddr_t pa;
bool rv __diagused;
@@ -672,7 +670,6 @@ ubc_release(void *va, int flags)
KASSERT(pgs[i]->loan_count == 0);
uvm_pageactivate(pgs[i]);
}
- mutex_exit(&uvm_pageqlock);
pmap_kremove(umapva, ubc_winsize);
pmap_update(pmap_kernel());
uvm_page_unbusy(pgs, npages);
@@ -891,7 +888,6 @@ ubc_direct_release(struct uvm_object *uo
int flags, struct vm_page **pgs, int npages)
{
mutex_enter(uobj->vmobjlock);
- mutex_enter(&uvm_pageqlock);
for (int i = 0; i < npages; i++) {
struct vm_page *pg = pgs[i];
@@ -901,8 +897,6 @@ ubc_direct_release(struct uvm_object *uo
if (flags & UBC_WRITE)
pg->flags &= ~(PG_FAKE|PG_CLEAN);
}
- mutex_exit(&uvm_pageqlock);
-
uvm_page_unbusy(pgs, npages);
mutex_exit(uobj->vmobjlock);
}
Index: uvm/uvm_fault.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_fault.c,v
retrieving revision 1.211
diff -u -p -r1.211 uvm_fault.c
--- uvm/uvm_fault.c 1 Dec 2019 14:30:01 -0000 1.211
+++ uvm/uvm_fault.c 6 Dec 2019 13:11:15 -0000
@@ -191,7 +192,6 @@ uvmfault_anonflush(struct vm_anon **anon
int lcv;
struct vm_page *pg;
- mutex_enter(&uvm_pageqlock);
for (lcv = 0; lcv < n; lcv++) {
if (anons[lcv] == NULL)
continue;
@@ -203,7 +203,6 @@ uvmfault_anonflush(struct vm_anon **anon
}
}
}
- mutex_exit(&uvm_pageqlock);
}
/*
@@ -492,9 +487,7 @@ released:
* We have successfully read the page, activate it.
*/
- mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
- mutex_exit(&uvm_pageqlock);
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
#else
@@ -1272,14 +1261,11 @@ uvm_fault_upper_neighbor(
/* locked: amap, anon */
- mutex_enter(&uvm_pageqlock);
uvm_pageenqueue(pg);
- mutex_exit(&uvm_pageqlock);
UVMHIST_LOG(maphist,
" MAPPING: n anon: pm=%#jx, va=%#jx, pg=%#jx",
(uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0);
/*
* Since this page isn't the page that's actually faulting,
@@ -1492,9 +1477,8 @@ uvm_fault_upper_promote(
KASSERT(anon == NULL || anon->an_lock == oanon->an_lock);
pg = anon->an_page;
- mutex_enter(&uvm_pageqlock);
- uvm_pageenqueue(pg); /* uvm_fault_upper_done will activate the page */
- mutex_exit(&uvm_pageqlock);
+ /* uvm_fault_upper_done will activate the page */
+ uvm_pageenqueue(pg);
pg->flags &= ~(PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
@@ -1627,9 +1610,10 @@ uvm_fault_upper_done(
* ... update the page queues.
*/
- mutex_enter(&uvm_pageqlock);
if (wire_paging) {
+ mutex_enter(&uvm_pageqlock);
uvm_pagewire(pg);
+ mutex_exit(&uvm_pageqlock);
/*
* since the now-wired page cannot be paged out,
@@ -1639,11 +1623,9 @@ uvm_fault_upper_done(
*/
pg->flags &= ~(PG_CLEAN);
-
} else {
uvm_pageactivate(pg);
}
- mutex_exit(&uvm_pageqlock);
if (wire_paging) {
uvm_anon_dropswap(anon);
@@ -1863,14 +1844,11 @@ uvm_fault_lower_neighbor(
* for this. we can just directly enter the pages.
*/
- mutex_enter(&uvm_pageqlock);
uvm_pageenqueue(pg);
- mutex_exit(&uvm_pageqlock);
UVMHIST_LOG(maphist,
" MAPPING: n obj: pm=%#jx, va=%#jx, pg=%#jx",
(uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0);
@@ -1984,9 +1961,7 @@ uvm_fault_lower_io(
mutex_enter(uobj->vmobjlock);
KASSERT((pg->flags & PG_BUSY) != 0);
- mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
- mutex_exit(&uvm_pageqlock);
/* locked(locked): maps(read), amap(if !null), uobj, pg */
/* locked(!locked): uobj, pg */
@@ -2288,9 +2260,7 @@ uvm_fault_lower_enter(
* we just promoted the page.
*/
- mutex_enter(&uvm_pageqlock);
uvm_pageenqueue(pg);
- mutex_exit(&uvm_pageqlock);
if (pg->flags & PG_WANTED)
wakeup(pg);
@@ -2349,8 +2319,8 @@ uvm_fault_lower_done(
UVMHIST_FUNC("uvm_fault_lower_done"); UVMHIST_CALLED(maphist);
- mutex_enter(&uvm_pageqlock);
if (flt->wire_paging) {
+ mutex_enter(&uvm_pageqlock);
uvm_pagewire(pg);
if (pg->pqflags & PQ_AOBJ) {
@@ -2365,10 +2335,10 @@ uvm_fault_lower_done(
pg->flags &= ~(PG_CLEAN);
dropswap = true;
}
+ mutex_exit(&uvm_pageqlock);
} else {
uvm_pageactivate(pg);
}
- mutex_exit(&uvm_pageqlock);
if (dropswap) {
uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
Index: uvm/uvm_km.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_km.c,v
retrieving revision 1.150
diff -u -p -r1.150 uvm_km.c
--- uvm/uvm_km.c 1 Dec 2019 23:14:47 -0000 1.150
+++ uvm/uvm_km.c 6 Dec 2019 13:11:15 -0000
@@ -475,8 +475,9 @@ uvm_km_pgremove(vaddr_t startva, vaddr_t
uao_dropswap(uobj, curoff >> PAGE_SHIFT);
if (pg != NULL) {
mutex_enter(&uvm_pageqlock);
- uvm_pagefree(pg);
+ uvm_pagefree1(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pagefree2(pg);
}
}
mutex_exit(uobj->vmobjlock);
Index: uvm/uvm_loan.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_loan.c,v
retrieving revision 1.88
diff -u -p -r1.88 uvm_loan.c
--- uvm/uvm_loan.c 1 Dec 2019 14:40:31 -0000 1.88
+++ uvm/uvm_loan.c 6 Dec 2019 13:11:15 -0000
@@ -412,7 +412,6 @@ uvm_loananon(struct uvm_faultinfo *ufi,
*/
pg = anon->an_page;
- mutex_enter(&uvm_pageqlock);
if (pg->wire_count > 0) {
mutex_exit(&uvm_pageqlock);
UVMHIST_LOG(loanhist, "->K wired %#jx", (uintptr_t)pg, 0, 0, 0);
@@ -423,10 +422,11 @@ uvm_loananon(struct uvm_faultinfo *ufi,
if (pg->loan_count == 0) {
pmap_page_protect(pg, VM_PROT_READ);
}
+ mutex_enter(&uvm_pageqlock);
pg->loan_count++;
KASSERT(pg->loan_count > 0); /* detect wrap-around */
- uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pageactivate(pg);
**output = pg;
(*output)++;
@@ -463,9 +463,7 @@ uvm_loanpage(struct vm_page **pgpp, int
KASSERT(mutex_owned(pg->uobject->vmobjlock));
KASSERT(pg->flags & PG_BUSY);
- mutex_enter(&uvm_pageqlock);
if (pg->wire_count > 0) {
- mutex_exit(&uvm_pageqlock);
UVMHIST_LOG(loanhist, "wired %#jx", (uintptr_t)pg,
0, 0, 0);
error = EBUSY;
@@ -474,10 +472,11 @@ uvm_loanpage(struct vm_page **pgpp, int
if (pg->loan_count == 0) {
pmap_page_protect(pg, VM_PROT_READ);
}
+ mutex_enter(&uvm_pageqlock);
pg->loan_count++;
KASSERT(pg->loan_count > 0); /* detect wrap-around */
- uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pageactivate(pg);
}
uvm_page_unbusy(pgpp, npages);
@@ -576,9 +575,7 @@ reget:
slock = pg->uobject->vmobjlock;
mutex_enter(slock);
- mutex_enter(&uvm_pageqlock);
uvm_page_unbusy(&pg, 1);
- mutex_exit(&uvm_pageqlock);
mutex_exit(slock);
}
goto reget;
@@ -714,14 +711,13 @@ uvm_loanuobj(struct uvm_faultinfo *ufi,
}
if (pg->flags & PG_RELEASED) {
mutex_enter(&uvm_pageqlock);
- uvm_pagefree(pg);
+ uvm_pagefree1(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pagefree2(pg);
mutex_exit(uobj->vmobjlock);
return (0);
}
- mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
- mutex_exit(&uvm_pageqlock);
pg->flags &= ~(PG_BUSY|PG_WANTED);
UVM_PAGE_OWN(pg, NULL);
mutex_exit(uobj->vmobjlock);
@@ -779,7 +775,6 @@ uvm_loanuobj(struct uvm_faultinfo *ufi,
if (anon == NULL) {
goto fail;
}
- mutex_enter(&uvm_pageqlock);
if (pg->wire_count > 0) {
mutex_exit(&uvm_pageqlock);
UVMHIST_LOG(loanhist, "wired %#jx", (uintptr_t)pg, 0, 0, 0);
@@ -788,13 +783,14 @@ uvm_loanuobj(struct uvm_faultinfo *ufi,
if (pg->loan_count == 0) {
pmap_page_protect(pg, VM_PROT_READ);
}
+ mutex_enter(&uvm_pageqlock);
pg->loan_count++;
KASSERT(pg->loan_count > 0); /* detect wrap-around */
pg->uanon = anon;
anon->an_page = pg;
anon->an_lock = /* TODO: share amap lock */
- uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pageactivate(pg);
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
@@ -872,9 +868,7 @@ again:
/* got a zero'd page. */
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
pg->flags |= PG_RDONLY;
- mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
- mutex_exit(&uvm_pageqlock);
UVM_PAGE_OWN(pg, NULL);
}
@@ -923,8 +917,8 @@ again:
mutex_enter(&uvm_pageqlock);
pg->loan_count++;
KASSERT(pg->loan_count > 0); /* detect wrap-around */
- uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pageactivate(pg);
mutex_exit(&anon->an_lock);
mutex_exit(uvm_loanzero_object.vmobjlock);
**output = anon;
@@ -1075,12 +1069,13 @@ ulz_put(struct uvm_object *uobj, voff_t
KASSERT(pg != NULL);
KASSERT(TAILQ_NEXT(pg, listq.queue) == NULL);
- mutex_enter(&uvm_pageqlock);
if (pg->uanon)
uvm_pageactivate(pg);
- else
+ else {
+ mutex_enter(&uvm_pageqlock);
uvm_pagedequeue(pg);
- mutex_exit(&uvm_pageqlock);
+ mutex_exit(&uvm_pageqlock);
+ }
mutex_exit(uobj->vmobjlock);
return 0;
@@ -1177,8 +1172,8 @@ uvm_loanbreak(struct vm_page *uobjpage)
*/
/* install new page */
- uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pageactivate(pg);
/*
* done! loan is broken and "pg" is
@@ -1224,18 +1219,19 @@ uvm_loanbreak_anon(struct vm_anon *anon,
uvm_pagedequeue(anon->an_page);
}
- if (uobj) {
- mutex_exit(uobj->vmobjlock);
- }
-
/* install new page in anon */
anon->an_page = pg;
pg->uanon = anon;
pg->pqflags |= PQ_ANON;
- uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
+ uvm_pageactivate(pg);
+
+ if (uobj) {
+ mutex_exit(uobj->vmobjlock);
+ }
+
pg->flags &= ~(PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
Index: uvm/uvm_map.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_map.c,v
retrieving revision 1.366
diff -u -p -r1.366 uvm_map.c
--- uvm/uvm_map.c 1 Nov 2019 13:04:22 -0000 1.366
+++ uvm/uvm_map.c 6 Dec 2019 13:11:16 -0000
@@ -3944,15 +3945,12 @@ uvm_map_clean(struct vm_map *map, vaddr_
* at all in these cases.
*/
- mutex_enter(&uvm_pageqlock);
if (pg->loan_count != 0 ||
pg->wire_count != 0) {
- mutex_exit(&uvm_pageqlock);
continue;
}
KASSERT(pg->uanon == anon);
uvm_pagedeactivate(pg);
- mutex_exit(&uvm_pageqlock);
continue;
case PGO_FREE:
Index: uvm/uvm_page.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_page.c,v
retrieving revision 1.200
diff -u -p -r1.200 uvm_page.c
--- uvm/uvm_page.c 20 Sep 2019 11:09:43 -0000 1.200
+++ uvm/uvm_page.c 6 Dec 2019 13:11:17 -0000
@@ -1348,7 +1453,7 @@ uvm_pagefree(struct vm_page *pg)
* => pages must either all belong to the same object, or all belong to anons.
* => if pages are object-owned, object must be locked.
* => if pages are anon-owned, anons must be locked.
- * => caller must lock page queues if pages may be released.
+ * => caller must NOT have locked page queues
* => caller must make sure that anon-owned pages are not PG_RELEASED.
*/
@@ -1369,6 +1474,7 @@ uvm_page_unbusy(struct vm_page **pgs, in
KASSERT(pg->flags & PG_BUSY);
KASSERT((pg->flags & PG_PAGEOUT) == 0);
if (pg->flags & PG_WANTED) {
+ /* XXXAD thundering herd problem. */
wakeup(pg);
}
if (pg->flags & PG_RELEASED) {
@@ -1377,7 +1483,10 @@ uvm_page_unbusy(struct vm_page **pgs, in
KASSERT(pg->uobject != NULL ||
(pg->uanon != NULL && pg->uanon->an_ref > 0));
pg->flags &= ~PG_RELEASED;
- uvm_pagefree(pg);
+ mutex_enter(&uvm_pageqlock);
+ uvm_pagefree1(pg);
+ mutex_exit(&uvm_pageqlock);
+ uvm_pagefree2(pg);
} else {
UVMHIST_LOG(ubchist, "unbusying pg %#jx",
(uintptr_t)pg, 0, 0, 0);
@@ -1598,20 +1604,21 @@ uvm_pagewire(struct vm_page *pg)
void
uvm_pageunwire(struct vm_page *pg)
{
+
KASSERT(mutex_owned(&uvm_pageqlock));
KASSERT(pg->wire_count != 0);
pg->wire_count--;
if (pg->wire_count == 0) {
uvm_pageactivate(pg);
- KASSERT(uvmexp.wired != 0);
- uvmexp.wired--;
+ /* KASSERT(uvmexp.wired != 0); */
+ uvm_stat_dec(UVM_STAT_WIRED);
}
}
/*
* uvm_pagedeactivate: deactivate page
*
- * => caller must lock page queues
+ * => caller must lock objects
* => caller must check to make sure page is not wired
* => object that page belongs to must be locked (so we can adjust pg->flags)
* => caller must clear the reference on the page before calling
@@ -1621,7 +1628,6 @@ void
uvm_pagedeactivate(struct vm_page *pg)
{
- KASSERT(mutex_owned(&uvm_pageqlock));
KASSERT(uvm_page_locked_p(pg));
KASSERT(pg->wire_count != 0 || uvmpdpol_pageisqueued_p(pg));
uvmpdpol_pagedeactivate(pg);
@@ -1630,17 +1636,18 @@ uvm_pagedeactivate(struct vm_page *pg)
/*
* uvm_pageactivate: activate page
*
- * => caller must lock page queues
+ * => caller must lock objects
*/
void
uvm_pageactivate(struct vm_page *pg)
{
- KASSERT(mutex_owned(&uvm_pageqlock));
KASSERT(uvm_page_locked_p(pg));
#if defined(READAHEAD_STATS)
if ((pg->pqflags & PQ_READAHEAD) != 0) {
uvm_ra_hit.ev_count++;
pg->pqflags &= ~PQ_READAHEAD;
}
@@ -1653,33 +1660,35 @@ uvm_pageactivate(struct vm_page *pg)
/*
* uvm_pagedequeue: remove a page from any paging queue
+ *
+ * => caller must objects and uvm_pageqlock
*/
-
void
uvm_pagedequeue(struct vm_page *pg)
{
+ KASSERT(uvm_page_locked_p(pg));
+ KASSERT(mutex_owned(&uvm_pageqlock));
+
if (uvmpdpol_pageisqueued_p(pg)) {
- KASSERT(mutex_owned(&uvm_pageqlock));
+ uvmpdpol_pagedequeue(pg);
}
-
- uvmpdpol_pagedequeue(pg);
}
/*
* uvm_pageenqueue: add a page to a paging queue without activating.
* used where a page is not really demanded (yet). eg. read-ahead
*/
-
void
uvm_pageenqueue(struct vm_page *pg)
{
- KASSERT(mutex_owned(&uvm_pageqlock));
- if (pg->wire_count != 0) {
- return;
+ KASSERT(uvm_page_locked_p(pg));
+
+ if (pg->wire_count == 0) {
+ uvmpdpol_pageenqueue(pg);
}
- uvmpdpol_pageenqueue(pg);
}
/*
Index: uvm/uvm_page.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_page.h,v
retrieving revision 1.84
diff -u -p -r1.84 uvm_page.h
--- uvm/uvm_page.h 7 Jan 2019 22:48:01 -0000 1.84
+++ uvm/uvm_page.h 6 Dec 2019 13:11:17 -0000
@@ -323,6 +337,8 @@ void uvm_pagedeactivate(struct vm_page *
void uvm_pagedequeue(struct vm_page *);
void uvm_pageenqueue(struct vm_page *);
void uvm_pagefree(struct vm_page *);
+void uvm_pagefree1(struct vm_page *);
+void uvm_pagefree2(struct vm_page *);
void uvm_page_unbusy(struct vm_page **, int);
struct vm_page *uvm_pagelookup(struct uvm_object *, voff_t);
void uvm_pageunwire(struct vm_page *);
Index: uvm/uvm_pager.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_pager.c,v
retrieving revision 1.113
diff -u -p -r1.113 uvm_pager.c
--- uvm/uvm_pager.c 1 Dec 2019 23:14:47 -0000 1.113
+++ uvm/uvm_pager.c 6 Dec 2019 13:11:17 -0000
@@ -323,7 +323,6 @@ uvm_aio_aiodone_pages(struct vm_page **p
uobj = pg->uobject;
slock = uobj->vmobjlock;
mutex_enter(slock);
- mutex_enter(&uvm_pageqlock);
} else {
#if defined(VMSWAP)
if (error) {
@@ -362,7 +361,6 @@ uvm_aio_aiodone_pages(struct vm_page **p
slock = pg->uanon->an_lock;
}
mutex_enter(slock);
- mutex_enter(&uvm_pageqlock);
anon_disposed = (pg->flags & PG_RELEASED) != 0;
KASSERT(!anon_disposed || pg->uobject != NULL ||
pg->uanon->an_ref == 0);
@@ -421,8 +419,10 @@ uvm_aio_aiodone_pages(struct vm_page **p
KASSERT(!write);
pg->flags &= ~PG_FAKE;
#if defined(READAHEAD_STATS)
+ mutex_enter(&uvm_pageqlock);
pg->pqflags |= PQ_READAHEAD;
uvm_ra_total.ev_count++;
+ mutex_exit(&uvm_pageqlock);
#endif /* defined(READAHEAD_STATS) */
KASSERT((pg->flags & PG_CLEAN) != 0);
uvm_pageenqueue(pg);
@@ -437,7 +437,7 @@ uvm_aio_aiodone_pages(struct vm_page **p
if (pg->flags & PG_PAGEOUT) {
pg->flags &= ~PG_PAGEOUT;
pageout_done++;
- uvmexp.pdfreed++;
+ atomic_inc_uint(&uvmexp.pdfreed);
pg->flags |= PG_RELEASED;
}
@@ -448,11 +448,9 @@ uvm_aio_aiodone_pages(struct vm_page **p
if (swap) {
if (pg->uobject == NULL && anon_disposed) {
- mutex_exit(&uvm_pageqlock);
uvm_anon_release(pg->uanon);
} else {
uvm_page_unbusy(&pg, 1);
- mutex_exit(&uvm_pageqlock);
mutex_exit(slock);
}
}
@@ -461,7 +459,6 @@ uvm_aio_aiodone_pages(struct vm_page **p
uvm_pageout_done(pageout_done);
if (!swap) {
uvm_page_unbusy(pgs, npages);
- mutex_exit(&uvm_pageqlock);
mutex_exit(slock);
} else {
#if defined(VMSWAP)
@@ -478,7 +475,7 @@ uvm_aio_aiodone_pages(struct vm_page **p
else
uvm_swap_free(swslot, npages);
}
- uvmexp.pdpending--;
+ atomic_dec_uint(&uvmexp.pdpending);
#endif /* defined(VMSWAP) */
}
}
Index: uvm/uvm_pdaemon.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_pdaemon.c,v
retrieving revision 1.112
diff -u -p -r1.112 uvm_pdaemon.c
--- uvm/uvm_pdaemon.c 1 Dec 2019 14:40:31 -0000 1.112
+++ uvm/uvm_pdaemon.c 6 Dec 2019 13:11:17 -0000
@@ -261,8 +277,10 @@ uvm_pageout(void *arg)
uvm.pagedaemon_lwp = curlwp;
mutex_enter(&uvm_pageqlock);
+ uvmpdpol_lock();
npages = uvmexp.npages;
uvmpd_tune();
+ uvmpdpol_unlock();
mutex_exit(&uvm_pageqlock);
/*
@@ -291,6 +309,7 @@ uvm_pageout(void *arg)
*/
mutex_enter(&uvm_pageqlock);
+ uvmpdpol_lock();
if (npages != uvmexp.npages || extrapages != uvm_extrapages) {
npages = uvmexp.npages;
extrapages = uvm_extrapages;
@@ -334,8 +354,9 @@ uvm_pageout(void *arg)
mutex_spin_exit(&uvm_fpageqlock);
/*
- * scan done. unlock page queues (the only lock we are holding)
+ * scan done. unlock page queues.
*/
+ uvmpdpol_unlock();
mutex_exit(&uvm_pageqlock);
/*
@@ -673,6 +695,9 @@ uvmpd_scan_queue(void)
kmutex_t *slock;
UVMHIST_FUNC("uvmpd_scan_queue"); UVMHIST_CALLED(pdhist);
+ KASSERT(mutex_owned(&uvm_pageqlock));
+ KASSERT(uvmpdpol_locked_p());
+
/*
* swslot is non-zero if we are building a swap cluster. we want
* to stay in the loop while we have a page to scan or we have
@@ -744,10 +768,10 @@ uvmpd_scan_queue(void)
*/
lockownerfail++;
if (lockownerfail > UVMPD_NUMTRYLOCKOWNER) {
- mutex_exit(&uvm_pageqlock);
+ uvmpdpol_unlock();
/* XXX Better than yielding but inadequate. */
- kpause("livelock", false, 1, NULL);
- mutex_enter(&uvm_pageqlock);
+ kpause("livelock", false, 1, &uvm_pageqlock);
+ uvmpdpol_lock();
lockownerfail = 0;
}
continue;
@@ -786,10 +810,12 @@ uvmpd_scan_queue(void)
if ((p->pqflags & PQ_SWAPBACKED) == 0) {
KASSERT(uobj != NULL);
+ uvmpdpol_unlock();
mutex_exit(&uvm_pageqlock);
(void) (uobj->pgops->pgo_put)(uobj, p->offset,
p->offset + PAGE_SIZE, PGO_CLEANIT|PGO_FREE);
mutex_enter(&uvm_pageqlock);
+ uvmpdpol_lock();
continue;
}
@@ -862,7 +889,9 @@ uvmpd_scan_queue(void)
if (swapcluster_allocslots(&swc)) {
dirtyreacts++;
+ uvmpdpol_unlock();
uvm_pageactivate(p);
+ uvmpdpol_lock();
mutex_exit(slock);
continue;
}
@@ -877,11 +906,11 @@ uvmpd_scan_queue(void)
p->flags |= PG_BUSY;
UVM_PAGE_OWN(p, "scan_queue");
-
p->flags |= PG_PAGEOUT;
- uvm_pagedequeue(p);
-
uvmexp.pgswapout++;
+
+ uvmpdpol_unlock();
+ uvm_pagedequeue(p);
mutex_exit(&uvm_pageqlock);
/*
@@ -895,30 +924,36 @@ uvmpd_scan_queue(void)
dirtyreacts++;
uvm_pageactivate(p);
mutex_exit(slock);
+ uvmpdpol_lock();
continue;
}
mutex_exit(slock);
swapcluster_flush(&swc, false);
mutex_enter(&uvm_pageqlock);
+ uvmpdpol_lock();
/*
* the pageout is in progress. bump counters and set up
* for the next loop.
*/
- uvmexp.pdpending++;
+ atomic_inc_uint(&uvmexp.pdpending);
#else /* defined(VMSWAP) */
+ uvmpdpol_unlock();
uvm_pageactivate(p);
mutex_exit(slock);
+ uvmpdpol_lock();
#endif /* defined(VMSWAP) */
}
#if defined(VMSWAP)
+ uvmpdpol_unlock();
mutex_exit(&uvm_pageqlock);
swapcluster_flush(&swc, true);
mutex_enter(&uvm_pageqlock);
+ uvmpdpol_lock();
#endif /* defined(VMSWAP) */
}
@@ -931,9 +966,12 @@ uvmpd_scan_queue(void)
static void
uvmpd_scan(void)
{
- int swap_shortage, pages_freed;
+ int swap_shortage, pages_freed, tmp;
UVMHIST_FUNC("uvmpd_scan"); UVMHIST_CALLED(pdhist);
+ KASSERT(mutex_owned(&uvm_pageqlock));
+ KASSERT(uvmpdpol_locked_p());
+
uvmexp.pdrevs++;
/*
Index: uvm/uvm_pdpolicy.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_pdpolicy.h,v
retrieving revision 1.3
diff -u -p -r1.3 uvm_pdpolicy.h
--- uvm/uvm_pdpolicy.h 21 Feb 2007 23:00:14 -0000 1.3
+++ uvm/uvm_pdpolicy.h 6 Dec 2019 13:11:18 -0000
@@ -41,6 +41,9 @@ void uvmpdpol_init(void);
void uvmpdpol_reinit(void);
void uvmpdpol_estimatepageable(int *, int *);
bool uvmpdpol_needsscan_p(void);
+void uvmpdpol_lock(void);
+void uvmpdpol_unlock(void);
+bool uvmpdpol_locked_p(void);
void uvmpdpol_pageactivate(struct vm_page *);
void uvmpdpol_pagedeactivate(struct vm_page *);
Index: uvm/uvm_pdpolicy_clock.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_pdpolicy_clock.c,v
retrieving revision 1.17
diff -u -p -r1.17 uvm_pdpolicy_clock.c
--- uvm/uvm_pdpolicy_clock.c 30 Jan 2012 17:21:52 -0000 1.17
+++ uvm/uvm_pdpolicy_clock.c 6 Dec 2019 13:11:18 -0000
@@ -110,8 +111,13 @@ struct uvmpdpol_scanstate {
struct vm_page *ss_nextpg;
};
+static void uvmpdpol_pageactivate_locked(struct vm_page *);
+static void uvmpdpol_pagedeactivate_locked(struct vm_page *);
+static void uvmpdpol_pagedequeue_locked(struct vm_page *);
+
static struct uvmpdpol_globalstate pdpol_state;
static struct uvmpdpol_scanstate pdpol_scanstate;
+static kmutex_t uvm_pdpol_lock __cacheline_aligned;
PDPOL_EVCNT_DEFINE(reactexec)
PDPOL_EVCNT_DEFINE(reactfile)
@@ -171,7 +178,8 @@ uvmpdpol_selectvictim(void)
struct vm_page *pg;
kmutex_t *lock;
- KASSERT(mutex_owned(&uvm_pageqlock));
+ KASSERT(mutex_owned(&uvm_pageqlock)); /* for uvmpd_trylockowner() */
+ KASSERT(mutex_owned(&uvm_pdpol_lock));
while (/* CONSTCOND */ 1) {
struct vm_anon *anon;
@@ -201,7 +209,7 @@ uvmpdpol_selectvictim(void)
lock = uvmpd_trylockowner(pg);
if (lock != NULL) {
if (pmap_is_referenced(pg)) {
- uvmpdpol_pageactivate(pg);
+ uvmpdpol_pageactivate_locked(pg);
uvmexp.pdreact++;
mutex_exit(lock);
continue;
@@ -221,18 +229,18 @@ uvmpdpol_selectvictim(void)
*/
if (uobj && UVM_OBJ_IS_VTEXT(uobj) && ss->ss_execreact) {
- uvmpdpol_pageactivate(pg);
+ uvmpdpol_pageactivate_locked(pg);
PDPOL_EVCNT_INCR(reactexec);
continue;
}
if (uobj && UVM_OBJ_IS_VNODE(uobj) &&
!UVM_OBJ_IS_VTEXT(uobj) && ss->ss_filereact) {
- uvmpdpol_pageactivate(pg);
+ uvmpdpol_pageactivate_locked(pg);
PDPOL_EVCNT_INCR(reactfile);
continue;
}
if ((anon || UVM_OBJ_IS_AOBJ(uobj)) && ss->ss_anonreact) {
- uvmpdpol_pageactivate(pg);
+ uvmpdpol_pageactivate_locked(pg);
PDPOL_EVCNT_INCR(reactanon);
continue;
}
@@ -282,7 +290,7 @@ uvmpdpol_balancequeue(int swap_shortage)
/* no need to check wire_count as pg is "active" */
lock = uvmpd_trylockowner(p);
if (lock != NULL) {
- uvmpdpol_pagedeactivate(p);
+ uvmpdpol_pagedeactivate_locked(p);
uvmexp.pddeact++;
inactive_shortage--;
mutex_exit(lock);
@@ -290,12 +298,12 @@ uvmpdpol_balancequeue(int swap_shortage)
}
}
-void
-uvmpdpol_pagedeactivate(struct vm_page *pg)
+static void
+uvmpdpol_pagedeactivate_locked(struct vm_page *pg)
{
KASSERT(uvm_page_locked_p(pg));
- KASSERT(mutex_owned(&uvm_pageqlock));
+ KASSERT(mutex_owned(&uvm_pdpol_lock));
if (pg->pqflags & PQ_ACTIVE) {
TAILQ_REMOVE(&pdpol_state.s_activeq, pg, pageq.queue);
@@ -313,27 +321,50 @@ uvmpdpol_pagedeactivate(struct vm_page *
}
void
-uvmpdpol_pageactivate(struct vm_page *pg)
+uvmpdpol_pagedeactivate(struct vm_page *pg)
+{
+
+ mutex_enter(&uvm_pdpol_lock);
+ uvmpdpol_pagedeactivate_locked(pg);
+ mutex_exit(&uvm_pdpol_lock);
+}
+
+static void
+uvmpdpol_pageactivate_locked(struct vm_page *pg)
{
- uvmpdpol_pagedequeue(pg);
+ uvmpdpol_pagedequeue_locked(pg);
TAILQ_INSERT_TAIL(&pdpol_state.s_activeq, pg, pageq.queue);
pg->pqflags |= PQ_ACTIVE;
+ pg->pdpol = hardclock_ticks;
pdpol_state.s_active++;
}
void
-uvmpdpol_pagedequeue(struct vm_page *pg)
+uvmpdpol_pageactivate(struct vm_page *pg)
+{
+
+ /* Safety: PQ_ACTIVE clear also tells us if it is not enqueued. */
+ if ((pg->pqflags & PQ_ACTIVE) == 0 ||
+ (hardclock_ticks - pg->pdpol) > hz) {
+ mutex_enter(&uvm_pdpol_lock);
+ uvmpdpol_pageactivate_locked(pg);
+ mutex_exit(&uvm_pdpol_lock);
+ }
+}
+
+static void
+uvmpdpol_pagedequeue_locked(struct vm_page *pg)
{
+ KASSERT(mutex_owned(&uvm_pdpol_lock));
+
if (pg->pqflags & PQ_ACTIVE) {
- KASSERT(mutex_owned(&uvm_pageqlock));
TAILQ_REMOVE(&pdpol_state.s_activeq, pg, pageq.queue);
pg->pqflags &= ~PQ_ACTIVE;
KASSERT(pdpol_state.s_active > 0);
pdpol_state.s_active--;
} else if (pg->pqflags & PQ_INACTIVE) {
- KASSERT(mutex_owned(&uvm_pageqlock));
TAILQ_REMOVE(&pdpol_state.s_inactiveq, pg, pageq.queue);
pg->pqflags &= ~PQ_INACTIVE;
KASSERT(pdpol_state.s_inactive > 0);
@@ -342,10 +373,45 @@ uvmpdpol_pagedequeue(struct vm_page *pg)
}
void
+uvmpdpol_pagedequeue(struct vm_page *pg)
+{
+
+ mutex_enter(&uvm_pdpol_lock);
+ uvmpdpol_pagedequeue_locked(pg);
+ mutex_exit(&uvm_pdpol_lock);
+}
+
+void
uvmpdpol_pageenqueue(struct vm_page *pg)
{
- uvmpdpol_pageactivate(pg);
+ /* Safe to test unlocked due to page life-cycle. */
+ if (!uvmpdpol_pageisqueued_p(pg)) {
+ mutex_enter(&uvm_pdpol_lock);
+ uvmpdpol_pageactivate_locked(pg);
+ mutex_exit(&uvm_pdpol_lock);
+ }
+}
+
+void
+uvmpdpol_lock(void)
+{
+
+ mutex_enter(&uvm_pdpol_lock);
+}
+
+void
+uvmpdpol_unlock(void)
+{
+
+ mutex_exit(&uvm_pdpol_lock);
+}
+
+bool
+uvmpdpol_locked_p(void)
+{
+
+ return mutex_owned(&uvm_pdpol_lock);
}
void
@@ -400,6 +466,7 @@ uvmpdpol_init(void)
{
struct uvmpdpol_globalstate *s = &pdpol_state;
+ mutex_init(&uvm_pdpol_lock, MUTEX_DEFAULT, IPL_NONE);
TAILQ_INIT(&s->s_activeq);
TAILQ_INIT(&s->s_inactiveq);
uvm_pctparam_init(&s->s_inactivepct, CLOCK_INACTIVEPCT, NULL);