Index: miscfs/genfs/genfs_io.c =================================================================== RCS file: /cvsroot/src/sys/miscfs/genfs/genfs_io.c,v retrieving revision 1.16 diff -u -p -r1.16 genfs_io.c --- miscfs/genfs/genfs_io.c 1 Dec 2008 11:22:12 -0000 1.16 +++ miscfs/genfs/genfs_io.c 30 Dec 2008 13:40:33 -0000 @@ -1731,11 +1731,13 @@ genfs_do_directio(struct vmspace *vs, va kva = uvm_km_alloc(kernel_map, klen, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA); puva = trunc_page(uva); + mutex_enter(&vp->v_interlock); for (poff = 0; poff < klen; poff += PAGE_SIZE) { rv = pmap_extract(upm, puva + poff, &pa); KASSERT(rv); pmap_enter(kpm, kva + poff, pa, prot, prot | PMAP_WIRED); } + mutex_exit(&vp->v_interlock); pmap_update(kpm); /* @@ -1750,7 +1752,9 @@ genfs_do_directio(struct vmspace *vs, va * Tear down the kernel mapping. */ + mutex_enter(&vp->v_interlock); pmap_remove(kpm, kva, kva + klen); + mutex_exit(&vp->v_interlock); pmap_update(kpm); uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); Index: uvm/uvm_amap.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_amap.c,v retrieving revision 1.85 diff -u -p -r1.85 uvm_amap.c --- uvm/uvm_amap.c 3 Dec 2008 11:43:51 -0000 1.85 +++ uvm/uvm_amap.c 30 Dec 2008 13:40:34 -0000 @@ -171,7 +171,7 @@ pp_setreflen(int *ppref, int offset, int * amap_alloc1: internal function that allocates an amap, but does not * init the overlay. * - * => lock on returned amap is init'd + * => lock is not initialized */ static inline struct vm_amap * amap_alloc1(int slots, int padslots, int waitf) @@ -187,7 +187,7 @@ amap_alloc1(int slots, int padslots, int kmflags = ((waitf & UVM_FLAG_NOWAIT) != 0) ? KM_NOSLEEP : KM_SLEEP; totalslots = amap_roundup_slots(slots + padslots); - mutex_init(&amap->am_l, MUTEX_DEFAULT, IPL_NONE); + amap->am_lock = NULL; amap->am_ref = 1; amap->am_flags = 0; #ifdef UVM_AMAP_PPREF @@ -217,7 +217,6 @@ fail3: fail2: kmem_free(amap->am_slots, totalslots * sizeof(int)); fail1: - mutex_destroy(&amap->am_l); pool_cache_put(&uvm_amap_cache, amap); /* @@ -255,6 +254,7 @@ amap_alloc(vaddr_t sz, vaddr_t padsz, in if (amap) { memset(amap->am_anon, 0, amap->am_maxslot * sizeof(struct vm_anon *)); + amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); amap_list_insert(amap); } @@ -290,7 +290,10 @@ amap_free(struct vm_amap *amap) KASSERT(amap->am_ref == 0 && amap->am_nused == 0); KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0); - KASSERT(!mutex_owned(&amap->am_l)); + if (amap->am_lock != NULL) { + KASSERT(!mutex_owned(amap->am_lock)); + mutex_obj_free(amap->am_lock); + } slots = amap->am_maxslot; kmem_free(amap->am_slots, slots * sizeof(*amap->am_slots)); kmem_free(amap->am_bckptr, slots * sizeof(*amap->am_bckptr)); @@ -299,7 +302,6 @@ amap_free(struct vm_amap *amap) if (amap->am_ppref && amap->am_ppref != PPREF_NONE) kmem_free(amap->am_ppref, slots * sizeof(*amap->am_ppref)); #endif - mutex_destroy(&amap->am_l); pool_cache_put(&uvm_amap_cache, amap); UVMHIST_LOG(maphist,"<- done, freed amap = 0x%x", amap, 0, 0, 0); } @@ -325,7 +327,7 @@ amap_extend(struct vm_map_entry *entry, int *newppref, *oldppref; #endif int i, *newsl, *newbck, *oldsl, *oldbck; - struct vm_anon **newover, **oldover; + struct vm_anon **newover, **oldover, *tofree; const km_flag_t kmflags = (flags & AMAP_EXTEND_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; @@ -354,6 +356,7 @@ amap_extend(struct vm_map_entry *entry, slotadj = slotadd - slotoff; slotspace = amap->am_maxslot - slotmapped; } + tofree = NULL; /* * case 1: we already have enough slots in the map and thus @@ -366,8 +369,9 @@ amap_extend(struct vm_map_entry *entry, #ifdef UVM_AMAP_PPREF if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { amap_pp_adjref(amap, slotoff + slotmapped, - slotadd, 1); + slotadd, 1, &tofree); } + uvm_anfree(tofree); #endif amap_unlock(amap); UVMHIST_LOG(maphist, @@ -381,8 +385,10 @@ amap_extend(struct vm_map_entry *entry, entry->aref.ar_pageoff = slotoff; #ifdef UVM_AMAP_PPREF if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { - amap_pp_adjref(amap, slotoff, slotadd, 1); + amap_pp_adjref(amap, slotoff, slotadd, 1, + &tofree); } + uvm_anfree(tofree); #endif amap_unlock(amap); UVMHIST_LOG(maphist, @@ -405,12 +411,14 @@ amap_extend(struct vm_map_entry *entry, amap_pp_adjref(amap, slotoff + slotmapped, (amap->am_nslot - - (slotoff + slotmapped)), 1); + (slotoff + slotmapped)), 1, + &tofree); pp_setreflen(amap->am_ppref, amap->am_nslot, 1, slotneed - amap->am_nslot); } #endif amap->am_nslot = slotneed; + uvm_anfree(tofree); amap_unlock(amap); /* @@ -583,7 +591,8 @@ amap_extend(struct vm_map_entry *entry, if ((flags & AMAP_EXTEND_FORWARDS) && (slotoff + slotmapped) < amap->am_nslot) amap_pp_adjref(amap, slotoff + slotmapped, - (amap->am_nslot - (slotoff + slotmapped)), 1); + (amap->am_nslot - (slotoff + slotmapped)), 1, + &tofree); if (flags & AMAP_EXTEND_FORWARDS) pp_setreflen(newppref, amap->am_nslot, 1, slotneed - amap->am_nslot); @@ -609,6 +618,7 @@ amap_extend(struct vm_map_entry *entry, oldnslots = amap->am_maxslot; amap->am_maxslot = slotalloc; + uvm_anfree(tofree); amap_unlock(amap); kmem_free(oldsl, oldnslots * sizeof(*oldsl)); kmem_free(oldbck, oldnslots * sizeof(*oldbck)); @@ -641,7 +651,7 @@ amap_share_protect(struct vm_map_entry * struct vm_amap *amap = entry->aref.ar_amap; int slots, lcv, slot, stop; - KASSERT(mutex_owned(&amap->am_l)); + KASSERT(mutex_owned(amap->am_lock)); AMAP_B2SLOT(slots, (entry->end - entry->start)); stop = entry->aref.ar_pageoff + slots; @@ -694,7 +704,6 @@ amap_wipeout(struct vm_amap *amap) return; } amap_list_remove(amap); - amap_unlock(amap); for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { int refs; @@ -705,11 +714,10 @@ amap_wipeout(struct vm_amap *amap) if (anon == NULL || anon->an_ref == 0) panic("amap_wipeout: corrupt amap"); - mutex_enter(&anon->an_lock); + KASSERT(anon->an_lock == amap->am_lock); UVMHIST_LOG(maphist," processing anon 0x%x, ref=%d", anon, anon->an_ref, 0, 0); refs = --anon->an_ref; - mutex_exit(&anon->an_lock); if (refs == 0) { /* @@ -728,6 +736,7 @@ amap_wipeout(struct vm_amap *amap) */ amap->am_nused = 0; + amap_unlock(amap); amap_free(amap); /* will unlock and free amap */ UVMHIST_LOG(maphist,"<- done!", 0,0,0,0); } @@ -751,10 +760,12 @@ amap_copy(struct vm_map *map, struct vm_ vaddr_t startva, vaddr_t endva) { struct vm_amap *amap, *srcamap; + struct vm_anon *tofree; int slots, lcv; vaddr_t chunksize; const int waitf = (flags & AMAP_COPY_NOWAIT) ? UVM_FLAG_NOWAIT : 0; const bool canchunk = (flags & AMAP_COPY_NOCHUNK) == 0; + kmutex_t *lock; UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist); UVMHIST_LOG(maphist, " (map=%p, entry=%p, flags=%d)", map, entry, flags, 0); @@ -861,9 +872,8 @@ amap_copy(struct vm_map *map, struct vm_ srcamap->am_anon[entry->aref.ar_pageoff + lcv]; if (amap->am_anon[lcv] == NULL) continue; - mutex_enter(&amap->am_anon[lcv]->an_lock); + KASSERT(amap->am_anon[lcv]->an_lock == srcamap->am_lock); amap->am_anon[lcv]->an_ref++; - mutex_exit(&amap->am_anon[lcv]->an_lock); amap->am_bckptr[lcv] = amap->am_nused; amap->am_slots[amap->am_nused] = lcv; amap->am_nused++; @@ -881,15 +891,28 @@ amap_copy(struct vm_map *map, struct vm_ srcamap->am_ref--; if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0) srcamap->am_flags &= ~AMAP_SHARED; /* clear shared flag */ + tofree = NULL; #ifdef UVM_AMAP_PPREF if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) { amap_pp_adjref(srcamap, entry->aref.ar_pageoff, - (entry->end - entry->start) >> PAGE_SHIFT, -1); + (entry->end - entry->start) >> PAGE_SHIFT, -1, &tofree); } #endif - + uvm_anfree(tofree); amap_unlock(srcamap); + /* + * if we referenced any anons then share the source amap's lock. + * otherwise we have nothing in common, so allocate a new one. + */ + + if (amap->am_nused != 0) { + lock = srcamap->am_lock; + mutex_obj_hold(lock); + } else { + lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); + } + amap->am_lock = lock; amap_list_insert(amap); /* @@ -948,7 +971,7 @@ ReStart: slot = amap->am_slots[lcv]; anon = amap->am_anon[slot]; - mutex_enter(&anon->an_lock); + KASSERT(anon->an_lock == amap->am_lock); /* * If the anon has only one ref, we must have already copied it. @@ -959,7 +982,6 @@ ReStart: if (anon->an_ref == 1) { KASSERT(anon->an_page != NULL || anon->an_swslot != 0); - mutex_exit(&anon->an_lock); continue; } @@ -977,7 +999,6 @@ ReStart: */ if (pg->loan_count != 0) { - mutex_exit(&anon->an_lock); continue; } KASSERT(pg->uanon == anon && pg->uobject == NULL); @@ -989,8 +1010,7 @@ ReStart: if (pg->flags & PG_BUSY) { pg->flags |= PG_WANTED; - amap_unlock(amap); - UVM_UNLOCK_AND_WAIT(pg, &anon->an_lock, false, + UVM_UNLOCK_AND_WAIT(pg, amap->am_lock, false, "cownow", 0); goto ReStart; } @@ -1012,10 +1032,9 @@ ReStart: if (nanon) { nanon->an_ref--; - mutex_exit(&nanon->an_lock); + KASSERT(nanon->an_ref == 0); uvm_anfree(nanon); } - mutex_exit(&anon->an_lock); amap_unlock(amap); uvm_wait("cownowpage"); goto ReStart; @@ -1026,6 +1045,8 @@ ReStart: * with our new one... */ + nanon->an_lock = amap->am_lock; + mutex_obj_hold(nanon->an_lock); uvm_pagecopy(pg, npg); /* old -> new */ anon->an_ref--; /* can't drop to zero */ amap->am_anon[slot] = nanon; /* replace */ @@ -1040,8 +1061,6 @@ ReStart: mutex_exit(&uvm_pageqlock); npg->flags &= ~(PG_BUSY|PG_FAKE); UVM_PAGE_OWN(npg, NULL); - mutex_exit(&nanon->an_lock); - mutex_exit(&anon->an_lock); } amap_unlock(amap); } @@ -1125,7 +1144,8 @@ amap_pp_establish(struct vm_amap *amap, * => caller must check that ppref != PPREF_NONE before calling */ void -amap_pp_adjref(struct vm_amap *amap, int curslot, vsize_t slotlen, int adjval) +amap_pp_adjref(struct vm_amap *amap, int curslot, vsize_t slotlen, int adjval, + struct vm_anon **tofree) { int stopslot, *ppref, lcv, prevlcv; int ref, len, prevref, prevlen; @@ -1185,7 +1205,7 @@ amap_pp_adjref(struct vm_amap *amap, int pp_setreflen(ppref, lcv, ref, len); } if (ref == 0) - amap_wiperange(amap, lcv, len); + amap_wiperange(amap, lcv, len, tofree); } } @@ -1197,11 +1217,14 @@ amap_pp_adjref(struct vm_amap *amap, int * => both map and amap must be locked by caller. */ void -amap_wiperange(struct vm_amap *amap, int slotoff, int slots) +amap_wiperange(struct vm_amap *amap, int slotoff, int slots, + struct vm_anon **tofree) { int byanon, lcv, stop, curslot, ptr, slotend; struct vm_anon *anon; + KASSERT(mutex_owned(amap->am_lock)); + /* * we can either traverse the amap by am_anon or by am_slots depending * on which is cheaper. decide now. @@ -1254,17 +1277,18 @@ amap_wiperange(struct vm_amap *amap, int * drop anon reference count */ - mutex_enter(&anon->an_lock); + KASSERT(anon->an_lock == amap->am_lock); refs = --anon->an_ref; - mutex_exit(&anon->an_lock); if (refs == 0) { /* * we just eliminated the last reference to an anon. - * free it. + * defer freeing it as uvm_anfree() can unlock the + * amap. */ - uvm_anfree(anon); + anon->an_link = *tofree; + *tofree = anon; } } } @@ -1335,20 +1359,18 @@ amap_swap_off(int startslot, int endslot slot = am->am_slots[i]; anon = am->am_anon[slot]; - mutex_enter(&anon->an_lock); + KASSERT(anon->an_lock == am->am_lock); swslot = anon->an_swslot; if (swslot < startslot || endslot <= swslot) { - mutex_exit(&anon->an_lock); continue; } am->am_flags |= AMAP_SWAPOFF; - amap_unlock(am); rv = uvm_anon_pagein(anon); - amap_lock(am); + am->am_flags &= ~AMAP_SWAPOFF; if (amap_refs(am) == 0) { amap_wipeout(am); @@ -1393,7 +1415,7 @@ amap_lookup(struct vm_aref *aref, vaddr_ int slot; struct vm_amap *amap = aref->ar_amap; UVMHIST_FUNC("amap_lookup"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(&amap->am_l)); + KASSERT(mutex_owned(amap->am_lock)); AMAP_B2SLOT(slot, offset); slot += aref->ar_pageoff; @@ -1419,7 +1441,7 @@ amap_lookups(struct vm_aref *aref, vaddr int slot; struct vm_amap *amap = aref->ar_amap; UVMHIST_FUNC("amap_lookups"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(&amap->am_l)); + KASSERT(mutex_owned(amap->am_lock)); AMAP_B2SLOT(slot, offset); slot += aref->ar_pageoff; @@ -1440,8 +1462,6 @@ amap_lookups(struct vm_aref *aref, vaddr * amap_add: add (or replace) a page to an amap * * => caller must lock amap. - * => if (replace) caller must lock anon because we might have to call - * pmap_page_protect on the anon's page. */ void amap_add(struct vm_aref *aref, vaddr_t offset, struct vm_anon *anon, @@ -1450,7 +1470,7 @@ amap_add(struct vm_aref *aref, vaddr_t o int slot; struct vm_amap *amap = aref->ar_amap; UVMHIST_FUNC("amap_add"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(&amap->am_l)); + KASSERT(mutex_owned(amap->am_lock)); AMAP_B2SLOT(slot, offset); slot += aref->ar_pageoff; @@ -1495,7 +1515,7 @@ amap_unadd(struct vm_aref *aref, vaddr_t int ptr, slot; struct vm_amap *amap = aref->ar_amap; UVMHIST_FUNC("amap_unadd"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(&amap->am_l)); + KASSERT(mutex_owned(amap->am_lock)); AMAP_B2SLOT(slot, offset); slot += aref->ar_pageoff; @@ -1527,6 +1547,7 @@ amap_unadd(struct vm_aref *aref, vaddr_t void amap_ref(struct vm_amap *amap, vaddr_t offset, vsize_t len, int flags) { + UVMHIST_FUNC("amap_ref"); UVMHIST_CALLED(maphist); amap_lock(amap); @@ -1540,10 +1561,12 @@ amap_ref(struct vm_amap *amap, vaddr_t o amap->am_ref++; #ifdef UVM_AMAP_PPREF if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { + struct vm_anon *tofree = NULL; if (flags & AMAP_REFALL) - amap_pp_adjref(amap, 0, amap->am_nslot, 1); + amap_pp_adjref(amap, 0, amap->am_nslot, 1, &tofree); else - amap_pp_adjref(amap, offset, len, 1); + amap_pp_adjref(amap, offset, len, 1, &tofree); + uvm_anfree(tofree); /* must be last action before unlock */ } #endif amap_unlock(amap); @@ -1556,12 +1579,14 @@ amap_ref(struct vm_amap *amap, vaddr_t o * => caller must remove all pmap-level references to this amap before * dropping the reference * => called from uvm_unmap_detach [only] ... note that entry is no - * longer part of a map and thus has no need for locking + * longer part of a map * => amap must be unlocked (we will lock it). */ void amap_unref(struct vm_amap *amap, vaddr_t offset, vsize_t len, bool all) { + struct vm_anon *tofree; + UVMHIST_FUNC("amap_unref"); UVMHIST_CALLED(maphist); /* @@ -1591,16 +1616,18 @@ amap_unref(struct vm_amap *amap, vaddr_t if (amap_refs(amap) == 1 && (amap->am_flags & AMAP_SHARED) != 0) amap->am_flags &= ~AMAP_SHARED; /* clear shared flag */ + tofree = NULL; #ifdef UVM_AMAP_PPREF if (amap->am_ppref == NULL && all == 0 && len != amap->am_nslot) amap_pp_establish(amap, offset); if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { if (all) - amap_pp_adjref(amap, 0, amap->am_nslot, -1); + amap_pp_adjref(amap, 0, amap->am_nslot, -1, &tofree); else - amap_pp_adjref(amap, offset, len, -1); + amap_pp_adjref(amap, offset, len, -1, &tofree); } #endif + uvm_anfree(tofree); amap_unlock(amap); UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); Index: uvm/uvm_amap.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_amap.h,v retrieving revision 1.34 diff -u -p -r1.34 uvm_amap.h --- uvm/uvm_amap.h 26 Oct 2008 08:32:02 -0000 1.34 +++ uvm/uvm_amap.h 30 Dec 2008 13:40:34 -0000 @@ -158,7 +158,7 @@ bool amap_swap_off */ struct vm_amap { - kmutex_t am_l; /* lock [locks all vm_amap fields] */ + kmutex_t *am_lock; /* lock [locks all vm_amap fields] */ int am_ref; /* reference count */ int am_flags; /* flags */ int am_maxslot; /* max # of slots allocated */ @@ -258,10 +258,10 @@ struct vm_amap { */ #define amap_flags(AMAP) ((AMAP)->am_flags) -#define amap_lock(AMAP) mutex_enter(&(AMAP)->am_l) -#define amap_lock_try(AMAP) mutex_tryenter(&(AMAP)->am_l) +#define amap_lock(AMAP) mutex_enter((AMAP)->am_lock) +#define amap_lock_try(AMAP) mutex_tryenter((AMAP)->am_lock) #define amap_refs(AMAP) ((AMAP)->am_ref) -#define amap_unlock(AMAP) mutex_exit(&(AMAP)->am_l) +#define amap_unlock(AMAP) mutex_exit((AMAP)->am_lock) /* * if we enable PPREF, then we have a couple of extra functions that @@ -273,11 +273,12 @@ struct vm_amap { #define PPREF_NONE ((int *) -1) /* not using ppref */ void amap_pp_adjref /* adjust references */ - (struct vm_amap *, int, vsize_t, int); + (struct vm_amap *, int, vsize_t, int, + struct vm_anon **); void amap_pp_establish /* establish ppref */ (struct vm_amap *, vaddr_t); void amap_wiperange /* wipe part of an amap */ - (struct vm_amap *, int, int); + (struct vm_amap *, int, int, struct vm_anon **); #endif /* UVM_AMAP_PPREF */ #endif /* _KERNEL */ Index: uvm/uvm_anon.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_anon.c,v retrieving revision 1.51 diff -u -p -r1.51 uvm_anon.c --- uvm/uvm_anon.c 18 Jan 2008 10:48:23 -0000 1.51 +++ uvm/uvm_anon.c 30 Dec 2008 13:40:34 -0000 @@ -75,7 +75,6 @@ uvm_anon_ctor(void *arg, void *object, i struct vm_anon *anon = object; anon->an_ref = 0; - mutex_init(&anon->an_lock, MUTEX_DEFAULT, IPL_NONE); anon->an_page = NULL; #if defined(VMSWAP) anon->an_swslot = 0; @@ -87,9 +86,7 @@ uvm_anon_ctor(void *arg, void *object, i static void uvm_anon_dtor(void *arg, void *object) { - struct vm_anon *anon = object; - mutex_destroy(&anon->an_lock); } /* @@ -110,29 +107,30 @@ uvm_analloc(void) KASSERT(anon->an_swslot == 0); #endif /* defined(VMSWAP) */ anon->an_ref = 1; - mutex_enter(&anon->an_lock); + anon->an_lock = NULL; } return anon; } /* - * uvm_anfree: free a single anon structure + * uvm_anfree: free a linked list of anon structures * * => caller must remove anon from its amap before calling (if it was in * an amap). - * => anon must be unlocked and have a zero reference count. + * => amap must be locked, or anon must not be associated with a lock + * or any other objects. * => we may lock the pageq's. + * => we may drop and re-acquire amap lock */ -void -uvm_anfree(struct vm_anon *anon) +static void +uvm_anfree1(struct vm_anon *anon) { struct vm_page *pg; UVMHIST_FUNC("uvm_anfree"); UVMHIST_CALLED(maphist); UVMHIST_LOG(maphist,"(anon=0x%x)", anon, 0,0,0); - KASSERT(anon->an_ref == 0); - KASSERT(!mutex_owned(&anon->an_lock)); + KASSERT(anon->an_lock == NULL || mutex_owned(anon->an_lock)); /* * get page @@ -147,9 +145,8 @@ uvm_anfree(struct vm_anon *anon) */ if (pg && pg->loan_count) { - mutex_enter(&anon->an_lock); + KASSERT(anon->an_lock != NULL); pg = uvm_anon_lockloanpg(anon); - mutex_exit(&anon->an_lock); } /* @@ -158,6 +155,7 @@ uvm_anfree(struct vm_anon *anon) */ if (pg) { + KASSERT(anon->an_lock != NULL); /* * if the page is owned by a uobject (now locked), then we must @@ -178,7 +176,6 @@ uvm_anfree(struct vm_anon *anon) */ KASSERT((pg->flags & PG_RELEASED) == 0); - mutex_enter(&anon->an_lock); pmap_page_protect(pg, VM_PROT_NONE); /* @@ -188,13 +185,11 @@ uvm_anfree(struct vm_anon *anon) if (pg->flags & PG_BUSY) { pg->flags |= PG_RELEASED; - mutex_exit(&anon->an_lock); return; } mutex_enter(&uvm_pageqlock); uvm_pagefree(pg); mutex_exit(&uvm_pageqlock); - mutex_exit(&anon->an_lock); UVMHIST_LOG(maphist, "anon 0x%x, page 0x%x: " "freed now!", anon, pg, 0, 0); } @@ -231,10 +226,25 @@ uvm_anfree(struct vm_anon *anon) KASSERT(anon->an_swslot == 0); #endif /* defined(VMSWAP) */ + if (anon->an_lock != NULL) { + mutex_obj_free(anon->an_lock); + } pool_cache_put(&uvm_anon_cache, anon); UVMHIST_LOG(maphist,"<- done!",0,0,0,0); } +void +uvm_anfree(struct vm_anon *anon) +{ + struct vm_anon *next; + + for (; anon != NULL; anon = next) { + next = anon->an_link; + anon->an_link = NULL; /* also clears reference count */ + uvm_anfree1(anon); + } +} + #if defined(VMSWAP) /* @@ -281,7 +291,7 @@ uvm_anon_lockloanpg(struct vm_anon *anon struct vm_page *pg; bool locked = false; - KASSERT(mutex_owned(&anon->an_lock)); + KASSERT(mutex_owned(anon->an_lock)); /* * loop while we have a resident page that has a non-zero loan count. @@ -318,16 +328,13 @@ uvm_anon_lockloanpg(struct vm_anon *anon */ if (!locked) { - mutex_exit(&anon->an_lock); - /* * someone locking the object has a chance to * lock us right now + * + * XXX Better than yielding but inadequate. */ - /* XXX Better than yielding but inadequate. */ - kpause("livelock", false, 1, NULL); - - mutex_enter(&anon->an_lock); + kpause("livelock", false, 1, anon->an_lock); continue; } } @@ -365,7 +372,7 @@ uvm_anon_pagein(struct vm_anon *anon) int rv; /* locked: anon */ - KASSERT(mutex_owned(&anon->an_lock)); + KASSERT(mutex_owned(anon->an_lock)); rv = uvmfault_anonget(NULL, NULL, anon); @@ -423,7 +430,7 @@ uvm_anon_pagein(struct vm_anon *anon) * unlock the anon and we're done. */ - mutex_exit(&anon->an_lock); + mutex_exit(anon->an_lock); if (uobj) { mutex_exit(&uobj->vmobjlock); } @@ -443,7 +450,7 @@ uvm_anon_release(struct vm_anon *anon) { struct vm_page *pg = anon->an_page; - KASSERT(mutex_owned(&anon->an_lock)); + KASSERT(mutex_owned(anon->an_lock)); KASSERT(pg != NULL); KASSERT((pg->flags & PG_RELEASED) != 0); KASSERT((pg->flags & PG_BUSY) != 0); @@ -455,7 +462,7 @@ uvm_anon_release(struct vm_anon *anon) mutex_enter(&uvm_pageqlock); uvm_pagefree(pg); mutex_exit(&uvm_pageqlock); - mutex_exit(&anon->an_lock); + mutex_exit(anon->an_lock); KASSERT(anon->an_page == NULL); Index: uvm/uvm_anon.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_anon.h,v retrieving revision 1.25 diff -u -p -r1.25 uvm_anon.h --- uvm/uvm_anon.h 2 Jan 2008 11:49:15 -0000 1.25 +++ uvm/uvm_anon.h 30 Dec 2008 13:40:34 -0000 @@ -52,8 +52,13 @@ */ struct vm_anon { - int an_ref; /* reference count [an_lock] */ - kmutex_t an_lock; /* lock for an_ref */ + union { + uintptr_t au_ref; /* reference count [an_lock] */ + struct vm_anon *au_link;/* link for deferred free */ + } an_u; +#define an_ref an_u.au_ref +#define an_link an_u.au_link + kmutex_t *an_lock; /* lock for an_ref */ struct vm_page *an_page;/* if in RAM [an_lock] */ #if defined(VMSWAP) || 1 /* XXX libkvm */ int an_swslot; /* drum swap slot # (if != 0) Index: uvm/uvm_bio.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_bio.c,v retrieving revision 1.66 diff -u -p -r1.66 uvm_bio.c --- uvm/uvm_bio.c 27 Nov 2008 08:46:09 -0000 1.66 +++ uvm/uvm_bio.c 30 Dec 2008 13:40:35 -0000 @@ -1,6 +1,35 @@ /* $NetBSD: uvm_bio.c,v 1.66 2008/11/27 08:46:09 pooka Exp $ */ /* + * Copyright (c) 2008 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Andrew Doran. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* * Copyright (c) 1998 Chuck Silvers. * All rights reserved. * @@ -90,6 +119,7 @@ struct ubc_map LIST_ENTRY(ubc_map) hash; /* hash table */ TAILQ_ENTRY(ubc_map) inactive; /* inactive queue */ + LIST_ENTRY(ubc_map) list; /* per-object list */ }; static struct ubc_object @@ -237,7 +267,7 @@ ubc_fault(struct uvm_faultinfo *ufi, vad */ if (flags & PGO_LOCKED) { - uvmfault_unlockall(ufi, NULL, &ubc_object.uobj, NULL); + uvmfault_unlockall(ufi, NULL, &ubc_object.uobj); flags &= ~PGO_LOCKED; } @@ -432,6 +462,7 @@ void * ubc_alloc(struct uvm_object *uobj, voff_t offset, vsize_t *lenp, int advice, int flags) { + struct uvm_object *oobj; vaddr_t slot_offset, va; struct ubc_map *umap; voff_t umap_offset; @@ -447,18 +478,18 @@ ubc_alloc(struct uvm_object *uobj, voff_ *lenp = MIN(*lenp, ubc_winsize - slot_offset); /* - * the object is always locked here, so we don't need to add a ref. + * the object is already referenced, so we don't need to add a ref. */ -again: mutex_enter(&ubc_object.uobj.vmobjlock); +again: umap = ubc_find_mapping(uobj, umap_offset); if (umap == NULL) { UBC_EVCNT_INCR(wincachemiss); umap = TAILQ_FIRST(UBC_QUEUE(offset)); if (umap == NULL) { - mutex_exit(&ubc_object.uobj.vmobjlock); - kpause("ubc_alloc", false, hz, NULL); + kpause("ubc_alloc", false, hz, + &ubc_object.uobj.vmobjlock); goto again; } @@ -466,19 +497,27 @@ again: * remove from old hash (if any), add to new hash. */ - if (umap->uobj != NULL) { + va = UBC_UMAP_ADDR(umap); + oobj = umap->uobj; + if (oobj != NULL) { LIST_REMOVE(umap, hash); + LIST_REMOVE(umap, list); + if (umap->flags & UMAP_MAPPING_CACHED) { + umap->flags &= ~UMAP_MAPPING_CACHED; + mutex_enter(&oobj->vmobjlock); + pmap_remove(pmap_kernel(), va, + va + ubc_winsize); + mutex_exit(&oobj->vmobjlock); + pmap_update(pmap_kernel()); + } + } else { + KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0); } umap->uobj = uobj; umap->offset = umap_offset; LIST_INSERT_HEAD(&ubc_object.hash[UBC_HASH(uobj, umap_offset)], umap, hash); - va = UBC_UMAP_ADDR(umap); - if (umap->flags & UMAP_MAPPING_CACHED) { - umap->flags &= ~UMAP_MAPPING_CACHED; - pmap_remove(pmap_kernel(), va, va + ubc_winsize); - pmap_update(pmap_kernel()); - } + LIST_INSERT_HEAD(&uobj->uo_ubc, umap, list); } else { UBC_EVCNT_INCR(wincachehit); va = UBC_UMAP_ADDR(umap); @@ -515,13 +554,13 @@ again: KASSERT(umap->refcount == 1); UBC_EVCNT_INCR(faultbusy); +again_faultbusy: + mutex_enter(&uobj->vmobjlock); if (umap->flags & UMAP_MAPPING_CACHED) { umap->flags &= ~UMAP_MAPPING_CACHED; pmap_remove(pmap_kernel(), va, va + ubc_winsize); } -again_faultbusy: memset(pgs, 0, sizeof(pgs)); - mutex_enter(&uobj->vmobjlock); error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs, &npages, 0, VM_PROT_READ | VM_PROT_WRITE, advice, gpflags); UVMHIST_LOG(ubchist, "faultbusy getpages %d", error, 0, 0, 0); @@ -537,16 +576,15 @@ again_faultbusy: if (pg->loan_count != 0) { pg = uvm_loanbreak(pg); } - mutex_exit(&uobj->vmobjlock); if (pg == NULL) { pmap_kremove(va, ubc_winsize); pmap_update(pmap_kernel()); - mutex_enter(&uobj->vmobjlock); uvm_page_unbusy(pgs, npages); mutex_exit(&uobj->vmobjlock); uvm_wait("ubc_alloc"); goto again_faultbusy; } + mutex_exit(&uobj->vmobjlock); pgs[i] = pg; } pmap_kenter_pa(va + slot_offset + (i << PAGE_SHIFT), @@ -597,6 +635,7 @@ ubc_release(void *va, int flags) memset((char *)umapva + endoff, 0, zerolen); } umap->flags &= ~UMAP_PAGES_LOCKED; + mutex_enter(&uobj->vmobjlock); mutex_enter(&uvm_pageqlock); for (i = 0; i < npages; i++) { rv = pmap_extract(pmap_kernel(), @@ -610,7 +649,6 @@ ubc_release(void *va, int flags) mutex_exit(&uvm_pageqlock); pmap_kremove(umapva, ubc_winsize); pmap_update(pmap_kernel()); - mutex_enter(&uobj->vmobjlock); uvm_page_unbusy(pgs, npages); mutex_exit(&uobj->vmobjlock); unmapped = true; @@ -631,11 +669,14 @@ ubc_release(void *va, int flags) * incompatible cache aliases around indefinitely. */ + mutex_enter(&uobj->vmobjlock); pmap_remove(pmap_kernel(), umapva, umapva + ubc_winsize); + mutex_exit(&uobj->vmobjlock); umap->flags &= ~UMAP_MAPPING_CACHED; pmap_update(pmap_kernel()); LIST_REMOVE(umap, hash); + LIST_REMOVE(umap, list); umap->uobj = NULL; TAILQ_INSERT_HEAD(UBC_QUEUE(umap->offset), umap, inactive); @@ -696,3 +737,30 @@ ubc_uiomove(struct uvm_object *uobj, str return error; } + +/* + * ubc_purge: disassociate ubc_map structures from an empty uvm_object. + */ + +void +ubc_purge(struct uvm_object *uobj) +{ + struct ubc_map *umap; + vaddr_t va; + + KASSERT(uobj->uo_npages == 0); + + mutex_enter(&ubc_object.uobj.vmobjlock); + while ((umap = LIST_FIRST(&uobj->uo_ubc)) != NULL) { + KASSERT(umap->refcount == 0); + for (va = 0; va < ubc_winsize; va += PAGE_SIZE) { + KASSERT(!pmap_extract(pmap_kernel(), + va + UBC_UMAP_ADDR(umap), NULL)); + } + LIST_REMOVE(umap, list); + LIST_REMOVE(umap, hash); + umap->flags &= ~UMAP_MAPPING_CACHED; + umap->uobj = NULL; + } + mutex_exit(&ubc_object.uobj.vmobjlock); +} Index: uvm/uvm_device.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_device.c,v retrieving revision 1.55 diff -u -p -r1.55 uvm_device.c --- uvm/uvm_device.c 17 Dec 2008 20:51:39 -0000 1.55 +++ uvm/uvm_device.c 30 Dec 2008 13:40:35 -0000 @@ -376,7 +376,7 @@ udv_fault(struct uvm_faultinfo *ufi, vad if (UVM_ET_ISCOPYONWRITE(entry)) { UVMHIST_LOG(maphist, "<- failed -- COW entry (etype=0x%x)", entry->etype, 0,0,0); - uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL); + uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj); return(EIO); } @@ -387,7 +387,7 @@ udv_fault(struct uvm_faultinfo *ufi, vad device = udv->u_device; if (cdevsw_lookup(device) == NULL) { /* XXX This should not happen */ - uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL); + uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj); return (EIO); } @@ -440,13 +440,13 @@ udv_fault(struct uvm_faultinfo *ufi, vad */ pmap_update(ufi->orig_map->pmap); /* sync what we have so far */ uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, - uobj, NULL); + uobj); uvm_wait("udv_fault"); return (ERESTART); } } pmap_update(ufi->orig_map->pmap); - uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL); + uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj); return (retval); } Index: uvm/uvm_fault.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_fault.c,v retrieving revision 1.126 diff -u -p -r1.126 uvm_fault.c --- uvm/uvm_fault.c 20 Dec 2008 11:33:38 -0000 1.126 +++ uvm/uvm_fault.c 30 Dec 2008 13:40:35 -0000 @@ -167,9 +167,9 @@ struct uvm_advice { */ static const struct uvm_advice uvmadvice[] = { - { MADV_NORMAL, 3, 4 }, - { MADV_RANDOM, 0, 0 }, - { MADV_SEQUENTIAL, 8, 7}, + { UVM_ADV_NORMAL, 3, 4 }, + { UVM_ADV_RANDOM, 0, 0 }, + { UVM_ADV_SEQUENTIAL, 8, 7}, }; #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ @@ -197,7 +197,7 @@ uvmfault_anonflush(struct vm_anon **anon for (lcv = 0 ; lcv < n ; lcv++) { if (anons[lcv] == NULL) continue; - mutex_enter(&anons[lcv]->an_lock); + KASSERT(mutex_owned(anons[lcv]->an_lock)); pg = anons[lcv]->an_page; if (pg && (pg->flags & PG_BUSY) == 0) { mutex_enter(&uvm_pageqlock); @@ -206,7 +206,6 @@ uvmfault_anonflush(struct vm_anon **anon } mutex_exit(&uvm_pageqlock); } - mutex_exit(&anons[lcv]->an_lock); } } @@ -288,7 +287,8 @@ uvmfault_anonget(struct uvm_faultinfo *u int error; UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(&anon->an_lock)); + KASSERT(mutex_owned(anon->an_lock)); + KASSERT(amap == NULL || anon->an_lock == amap->am_lock); error = 0; uvmexp.fltanget++; @@ -341,7 +341,7 @@ uvmfault_anonget(struct uvm_faultinfo *u */ if (pg->uobject) { /* owner is uobject ? */ - uvmfault_unlockall(ufi, amap, NULL, anon); + uvmfault_unlockall(ufi, amap, NULL); UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 0,0,0); UVM_UNLOCK_AND_WAIT(pg, @@ -349,10 +349,10 @@ uvmfault_anonget(struct uvm_faultinfo *u false, "anonget1",0); } else { /* anon owns page */ - uvmfault_unlockall(ufi, amap, NULL, NULL); + uvmfault_unlockall(ufi, NULL, NULL); UVMHIST_LOG(maphist, " unlock+wait on anon",0, 0,0,0); - UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, + UVM_UNLOCK_AND_WAIT(pg, anon->an_lock, 0, "anonget2",0); } } else { @@ -364,7 +364,7 @@ uvmfault_anonget(struct uvm_faultinfo *u pg = uvm_pagealloc(NULL, 0, anon, 0); if (pg == NULL) { /* out of RAM. */ - uvmfault_unlockall(ufi, amap, NULL, anon); + uvmfault_unlockall(ufi, amap, NULL); uvmexp.fltnoram++; UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 0,0,0); @@ -375,7 +375,7 @@ uvmfault_anonget(struct uvm_faultinfo *u } else { /* we set the PG_BUSY bit */ we_own = true; - uvmfault_unlockall(ufi, amap, NULL, anon); + uvmfault_unlockall(ufi, amap, NULL); /* * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN @@ -403,11 +403,9 @@ uvmfault_anonget(struct uvm_faultinfo *u */ locked = uvmfault_relock(ufi); - if (locked && amap != NULL) { - amap_lock(amap); + if (locked || we_own) { + mutex_enter(anon->an_lock); } - if (locked || we_own) - mutex_enter(&anon->an_lock); /* * if we own the page (i.e. we set PG_BUSY), then we need @@ -452,10 +450,8 @@ uvmfault_anonget(struct uvm_faultinfo *u mutex_exit(&uvm_pageqlock); if (locked) - uvmfault_unlockall(ufi, amap, NULL, - anon); - else - mutex_exit(&anon->an_lock); + uvmfault_unlockall(ufi, NULL, NULL); + mutex_exit(anon->an_lock); UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); return error; } @@ -469,8 +465,7 @@ released: */ if (locked) - uvmfault_unlockall(ufi, amap, NULL, - NULL); + uvmfault_unlockall(ufi, NULL, NULL); uvm_anon_release(anon); @@ -493,8 +488,6 @@ released: mutex_exit(&uvm_pageqlock); pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); UVM_PAGE_OWN(pg, NULL); - if (!locked) - mutex_exit(&anon->an_lock); #else /* defined(VMSWAP) */ panic("%s: we_own", __func__); #endif /* defined(VMSWAP) */ @@ -505,6 +498,9 @@ released: */ if (!locked) { + if (we_own) { + mutex_exit(anon->an_lock); + } UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); return (ERESTART); } @@ -513,11 +509,10 @@ released: * verify no one has touched the amap and moved the anon on us. */ - if (ufi != NULL && - amap_lookup(&ufi->entry->aref, - ufi->orig_rvaddr - ufi->entry->start) != anon) { + if (ufi != NULL && amap_lookup(&ufi->entry->aref, + ufi->orig_rvaddr - ufi->entry->start) != anon) { - uvmfault_unlockall(ufi, amap, NULL, anon); + uvmfault_unlockall(ufi, amap, NULL); UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); return (ERESTART); } @@ -580,17 +575,13 @@ uvmfault_promote(struct uvm_faultinfo *u KASSERT(amap != NULL); KASSERT(uobjpage != NULL); KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0); - KASSERT(mutex_owned(&amap->am_l)); - KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock)); + KASSERT(mutex_owned(amap->am_lock)); + KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock); KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); -#if 0 - KASSERT(*spare == NULL || !mutex_owned(&(*spare)->an_lock)); -#endif if (*spare != NULL) { anon = *spare; *spare = NULL; - mutex_enter(&anon->an_lock); } else if (ufi->map != kernel_map) { anon = uvm_analloc(); } else { @@ -601,13 +592,13 @@ uvmfault_promote(struct uvm_faultinfo *u */ uvm_page_unbusy(&uobjpage, 1); - uvmfault_unlockall(ufi, amap, uobj, oanon); + uvmfault_unlockall(ufi, amap, uobj); *spare = uvm_analloc(); if (*spare == NULL) { goto nomem; } - mutex_exit(&(*spare)->an_lock); + KASSERT((*spare)->an_lock == NULL); error = ERESTART; goto done; } @@ -620,8 +611,15 @@ uvmfault_promote(struct uvm_faultinfo *u * so have uvm_pagealloc() do that for us. */ + KASSERT(anon->an_lock == NULL); + anon->an_lock = amap->am_lock; + mutex_obj_hold(anon->an_lock); pg = uvm_pagealloc(NULL, 0, anon, (opg == NULL) ? UVM_PGA_ZERO : 0); + if (pg == NULL) { + mutex_obj_free(anon->an_lock); + anon->an_lock = NULL; + } } else { pg = NULL; } @@ -633,13 +631,12 @@ uvmfault_promote(struct uvm_faultinfo *u if (pg == NULL) { /* save anon for the next try. */ if (anon != NULL) { - mutex_exit(&anon->an_lock); *spare = anon; } /* unlock and fail ... */ uvm_page_unbusy(&uobjpage, 1); - uvmfault_unlockall(ufi, amap, uobj, oanon); + uvmfault_unlockall(ufi, amap, uobj); nomem: if (!uvm_reclaimable()) { UVMHIST_LOG(maphist, "out of VM", 0,0,0,0); @@ -894,7 +891,7 @@ ReFault: } /* locked: maps(read), amap(if there) */ - KASSERT(amap == NULL || mutex_owned(&amap->am_l)); + KASSERT(amap == NULL || mutex_owned(amap->am_lock)); /* * for MADV_SEQUENTIAL mappings we want to deactivate the back pages @@ -926,7 +923,7 @@ ReFault: } /* locked: maps(read), amap(if there) */ - KASSERT(amap == NULL || mutex_owned(&amap->am_l)); + KASSERT(amap == NULL || mutex_owned(amap->am_lock)); /* * map in the backpages and frontpages we found in the amap in hopes @@ -966,7 +963,7 @@ ReFault: continue; } anon = anons[lcv]; - mutex_enter(&anon->an_lock); + KASSERT(anon->an_lock == amap->am_lock); /* ignore loaned pages */ if (anon->an_page && anon->an_page->loan_count == 0 && (anon->an_page->flags & PG_BUSY) == 0) { @@ -992,11 +989,10 @@ ReFault: (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); } pmap_update(ufi.orig_map->pmap); - mutex_exit(&anon->an_lock); } /* locked: maps(read), amap(if there) */ - KASSERT(amap == NULL || mutex_owned(&amap->am_l)); + KASSERT(amap == NULL || mutex_owned(amap->am_lock)); /* (shadowed == true) if there is an anon at the faulting address */ UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, (uobj && shadowed == false),0,0); @@ -1120,6 +1116,7 @@ ReFault: || (curpg->loan_count > 0) || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject); + KASSERT(mutex_owned(&uobj->vmobjlock)); (void) pmap_enter(ufi.orig_map->pmap, currva, VM_PAGE_TO_PHYS(curpg), readonly ? @@ -1149,9 +1146,9 @@ ReFault: /* locked (!shadowed): maps(read), amap(if there), uobj(if !null), uobjpage(if !null) */ if (shadowed) { - KASSERT(mutex_owned(&amap->am_l)); + KASSERT(mutex_owned(amap->am_lock)); } else { - KASSERT(amap == NULL || mutex_owned(&amap->am_l)); + KASSERT(amap == NULL || mutex_owned(amap->am_lock)); KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); } @@ -1186,11 +1183,10 @@ ReFault: anon = anons[centeridx]; UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); - mutex_enter(&anon->an_lock); /* locked: maps(read), amap, anon */ - KASSERT(mutex_owned(&amap->am_l)); - KASSERT(mutex_owned(&anon->an_lock)); + KASSERT(mutex_owned(amap->am_lock)); + KASSERT(anon->an_lock == amap->am_lock); /* * no matter if we have case 1A or case 1B we are going to need to @@ -1229,8 +1225,8 @@ ReFault: uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ /* locked: maps(read), amap, anon, uobj(if one) */ - KASSERT(mutex_owned(&amap->am_l)); - KASSERT(mutex_owned(&anon->an_lock)); + KASSERT(mutex_owned(amap->am_lock)); + KASSERT(anon->an_lock == amap->am_lock); KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); /* @@ -1268,8 +1264,7 @@ ReFault: /* get new un-owned replacement page */ pg = uvm_pagealloc(NULL, 0, NULL, 0); if (pg == NULL) { - uvmfault_unlockall(&ufi, amap, uobj, - anon); + uvmfault_unlockall(&ufi, amap, uobj); uvm_wait("flt_noram2"); goto ReFault; } @@ -1351,6 +1346,8 @@ ReFault: goto done; } + KASSERT(anon == NULL || anon->an_lock == oanon->an_lock); + pg = anon->an_page; mutex_enter(&uvm_pageqlock); uvm_pageactivate(pg); @@ -1378,9 +1375,10 @@ ReFault: } /* locked: maps(read), amap, oanon, anon (if different from oanon) */ - KASSERT(mutex_owned(&amap->am_l)); - KASSERT(mutex_owned(&anon->an_lock)); - KASSERT(mutex_owned(&oanon->an_lock)); + KASSERT(mutex_owned(amap->am_lock)); + KASSERT(anon->an_lock == amap->am_lock); + KASSERT(oanon->an_lock == amap->am_lock); + KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); /* * now map the page in. @@ -1400,9 +1398,7 @@ ReFault: * as the map may change while we're asleep. */ - if (anon != oanon) - mutex_exit(&anon->an_lock); - uvmfault_unlockall(&ufi, amap, uobj, oanon); + uvmfault_unlockall(&ufi, amap, uobj); if (!uvm_reclaimable()) { UVMHIST_LOG(maphist, "<- failed. out of VM",0,0,0,0); @@ -1441,9 +1437,7 @@ ReFault: * done case 1! finish up by unlocking everything and returning success */ - if (anon != oanon) - mutex_exit(&anon->an_lock); - uvmfault_unlockall(&ufi, amap, uobj, oanon); + uvmfault_unlockall(&ufi, amap, uobj); pmap_update(ufi.orig_map->pmap); error = 0; goto done; @@ -1457,7 +1451,7 @@ Case2: * locked: * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) */ - KASSERT(amap == NULL || mutex_owned(&amap->am_l)); + KASSERT(amap == NULL || mutex_owned(amap->am_lock)); KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); @@ -1495,7 +1489,7 @@ Case2: curlwp->l_ru.ru_majflt++; /* locked: maps(read), amap(if there), uobj */ - uvmfault_unlockall(&ufi, amap, NULL, NULL); + uvmfault_unlockall(&ufi, amap, NULL); /* locked: uobj */ uvmexp.fltget++; @@ -1524,12 +1518,6 @@ Case2: goto done; } - /* locked: uobjpage */ - - mutex_enter(&uvm_pageqlock); - uvm_pageactivate(uobjpage); - mutex_exit(&uvm_pageqlock); - /* * re-verify the state of the world by first trying to relock * the maps. always relock the object. @@ -1541,6 +1529,12 @@ Case2: uobj = uobjpage->uobject; mutex_enter(&uobj->vmobjlock); + /* locked: uobjpage */ + + mutex_enter(&uvm_pageqlock); + uvm_pageactivate(uobjpage); + mutex_exit(&uvm_pageqlock); + /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ /* locked(!locked): uobj, uobjpage */ @@ -1555,7 +1549,7 @@ Case2: amap_lookup(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start))) { if (locked) - uvmfault_unlockall(&ufi, amap, NULL, NULL); + uvmfault_unlockall(&ufi, amap, NULL); locked = false; } @@ -1593,7 +1587,7 @@ Case2: * locked: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) */ - KASSERT(amap == NULL || mutex_owned(&amap->am_l)); + KASSERT(amap == NULL || mutex_owned(amap->am_lock)); KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); @@ -1656,8 +1650,7 @@ Case2: uobjpage->flags &= ~(PG_BUSY|PG_WANTED); UVM_PAGE_OWN(uobjpage, NULL); - uvmfault_unlockall(&ufi, amap, uobj, - NULL); + uvmfault_unlockall(&ufi, amap, uobj); UVMHIST_LOG(maphist, " out of RAM breaking loan, waiting", 0,0,0,0); @@ -1713,7 +1706,6 @@ Case2: /* * dispose of uobjpage. it can't be PG_RELEASED * since we still hold the object lock. - * drop handle to uobj as well. */ if (uobjpage->flags & PG_WANTED) @@ -1721,8 +1713,6 @@ Case2: wakeup(uobjpage); uobjpage->flags &= ~(PG_BUSY|PG_WANTED); UVM_PAGE_OWN(uobjpage, NULL); - mutex_exit(&uobj->vmobjlock); - uobj = NULL; UVMHIST_LOG(maphist, " promote uobjpage 0x%x to anon/page 0x%x/0x%x", @@ -1743,15 +1733,14 @@ Case2: /* * locked: - * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj), - * anon(if !null), pg(if anon) + * maps(read), amap(if !null), uobj(if !null), + * anon(if !null), pg(if anon), unlock_uobj(if !null) * * note: pg is either the uobjpage or the new page in the new anon */ - KASSERT(amap == NULL || mutex_owned(&amap->am_l)); + KASSERT(amap == NULL || mutex_owned(amap->am_lock)); KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); - KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); - KASSERT(anon == NULL || mutex_owned(&anon->an_lock)); + KASSERT(anon == NULL || anon->an_lock == amap->am_lock); KASSERT((pg->flags & PG_BUSY) != 0); /* @@ -1787,7 +1776,7 @@ Case2: pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); UVM_PAGE_OWN(pg, NULL); - uvmfault_unlockall(&ufi, amap, uobj, anon); + uvmfault_unlockall(&ufi, amap, uobj); if (!uvm_reclaimable()) { UVMHIST_LOG(maphist, "<- failed. out of VM",0,0,0,0); @@ -1831,13 +1820,15 @@ Case2: pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); UVM_PAGE_OWN(pg, NULL); - uvmfault_unlockall(&ufi, amap, uobj, anon); + uvmfault_unlockall(&ufi, amap, uobj); pmap_update(ufi.orig_map->pmap); UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); error = 0; done: if (anon_spare != NULL) { anon_spare->an_ref--; + KASSERT(anon_spare->an_ref == 0); + KASSERT(anon_spare->an_lock == NULL); uvm_anfree(anon_spare); } return error; @@ -1908,7 +1899,7 @@ uvm_fault_unwire(struct vm_map *map, vad void uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end) { - struct vm_map_entry *entry; + struct vm_map_entry *entry, *oentry; pmap_t pmap = vm_map_pmap(map); vaddr_t va; paddr_t pa; @@ -1923,7 +1914,6 @@ uvm_fault_unwire_locked(struct vm_map *m * we can call uvm_pageunwire. */ - mutex_enter(&uvm_pageqlock); /* * find the beginning map entry for the region. @@ -1933,6 +1923,7 @@ uvm_fault_unwire_locked(struct vm_map *m if (uvm_map_lookup_entry(map, start, &entry) == false) panic("uvm_fault_unwire_locked: address not in map"); + oentry = NULL; for (va = start; va < end; va += PAGE_SIZE) { if (pmap_extract(pmap, va, &pa) == false) continue; @@ -1949,6 +1940,20 @@ uvm_fault_unwire_locked(struct vm_map *m } /* + * lock it. + */ + + if (entry != oentry) { + if (oentry != NULL) { + mutex_exit(&uvm_pageqlock); + uvm_map_unlock_entry(oentry); + } + uvm_map_lock_entry(entry); + mutex_enter(&uvm_pageqlock); + oentry = entry; + } + + /* * if the entry is no longer wired, tell the pmap. */ @@ -1960,5 +1965,8 @@ uvm_fault_unwire_locked(struct vm_map *m uvm_pageunwire(pg); } - mutex_exit(&uvm_pageqlock); + if (oentry != NULL) { + mutex_exit(&uvm_pageqlock); + uvm_map_unlock_entry(entry); + } } Index: uvm/uvm_fault_i.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_fault_i.h,v retrieving revision 1.24 diff -u -p -r1.24 uvm_fault_i.h --- uvm/uvm_fault_i.h 2 Jan 2008 11:49:16 -0000 1.24 +++ uvm/uvm_fault_i.h 30 Dec 2008 13:40:35 -0000 @@ -72,11 +72,9 @@ uvmfault_unlockmaps(struct uvm_faultinfo static __inline void uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, - struct uvm_object *uobj, struct vm_anon *anon) + struct uvm_object *uobj) { - if (anon) - mutex_exit(&anon->an_lock); if (uobj) mutex_exit(&uobj->vmobjlock); if (amap) @@ -139,7 +137,7 @@ uvmfault_lookup(struct uvm_faultinfo *uf * lookup */ if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, - &ufi->entry)) { + &ufi->entry)) { uvmfault_unlockmaps(ufi, write_lock); return(false); } Index: uvm/uvm_km.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_km.c,v retrieving revision 1.103 diff -u -p -r1.103 uvm_km.c --- uvm/uvm_km.c 13 Dec 2008 11:34:43 -0000 1.103 +++ uvm/uvm_km.c 30 Dec 2008 13:40:35 -0000 @@ -654,12 +654,19 @@ uvm_km_free(struct vm_map *map, vaddr_t size = round_page(size); + if (flags & UVM_KMF_PAGEABLE) { - uvm_km_pgremove(addr, addr + size); + /* + * no need to lock for pmap, as the kernel is + * self-consistent. the pages cannot be in + * use elsewhere. + */ + pmap_remove(pmap_kernel(), addr, addr + size); + uvm_km_pgremove(addr, addr + size); } else if (flags & UVM_KMF_WIRED) { - uvm_km_pgremove_intrsafe(map, addr, addr + size); pmap_kremove(addr, size); + uvm_km_pgremove_intrsafe(map, addr, addr + size); } /* Index: uvm/uvm_loan.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_loan.c,v retrieving revision 1.73 diff -u -p -r1.73 uvm_loan.c --- uvm/uvm_loan.c 3 Dec 2008 14:46:24 -0000 1.73 +++ uvm/uvm_loan.c 30 Dec 2008 13:40:35 -0000 @@ -153,8 +153,9 @@ uvm_loanentry(struct uvm_faultinfo *ufi, /* * lock us the rest of the way down (we unlock before return) */ - if (aref->ar_amap) + if (aref->ar_amap) { amap_lock(aref->ar_amap); + } /* * loop until done @@ -179,12 +180,12 @@ uvm_loanentry(struct uvm_faultinfo *ufi, } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) { rv = uvm_loanzero(ufi, output, flags); } else { - uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL); + uvmfault_unlockall(ufi, aref->ar_amap, uobj); rv = -1; } /* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */ KASSERT(rv > 0 || aref->ar_amap == NULL || - !mutex_owned(&aref->ar_amap->am_l)); + !mutex_owned(aref->ar_amap->am_lock)); KASSERT(rv > 0 || uobj == NULL || !mutex_owned(&uobj->vmobjlock)); @@ -214,8 +215,9 @@ uvm_loanentry(struct uvm_faultinfo *ufi, * unlock what we locked, unlock the maps and return */ - if (aref->ar_amap) + if (aref->ar_amap) { amap_unlock(aref->ar_amap); + } uvmfault_unlockmaps(ufi, false); UVMHIST_LOG(loanhist, "done %d", result, 0,0,0); return (result); @@ -359,14 +361,14 @@ uvm_loananon(struct uvm_faultinfo *ufi, */ if (flags & UVM_LOAN_TOANON) { - mutex_enter(&anon->an_lock); + KASSERT(mutex_owned(anon->an_lock)); pg = anon->an_page; if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) { if (pg->wire_count > 0) { UVMHIST_LOG(loanhist, "->A wired %p", pg,0,0,0); uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, - ufi->entry->object.uvm_obj, anon); + ufi->entry->object.uvm_obj); return (-1); } pmap_page_protect(pg, VM_PROT_READ); @@ -374,7 +376,6 @@ uvm_loananon(struct uvm_faultinfo *ufi, anon->an_ref++; **output = anon; (*output)++; - mutex_exit(&anon->an_lock); UVMHIST_LOG(loanhist, "->A done", 0,0,0,0); return (1); } @@ -385,7 +386,7 @@ uvm_loananon(struct uvm_faultinfo *ufi, * this for us. */ - mutex_enter(&anon->an_lock); + KASSERT(mutex_owned(anon->an_lock)); error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon); /* @@ -403,7 +404,7 @@ uvm_loananon(struct uvm_faultinfo *ufi, /* "try again"? sleep a bit and retry ... */ if (error == EAGAIN) { - tsleep(&lbolt, PVM, "loanagain", 0); + kpause("loanagain", false, 1, NULL); return (0); } @@ -421,8 +422,7 @@ uvm_loananon(struct uvm_faultinfo *ufi, mutex_exit(&uvm_pageqlock); UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0); KASSERT(pg->uobject == NULL); - uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, - NULL, anon); + uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL); return (-1); } if (pg->loan_count == 0) { @@ -434,10 +434,9 @@ uvm_loananon(struct uvm_faultinfo *ufi, **output = pg; (*output)++; - /* unlock anon and return success */ + /* unlock and return success */ if (pg->uobject) mutex_exit(&pg->uobject->vmobjlock); - mutex_exit(&anon->an_lock); UVMHIST_LOG(loanhist, "->K done", 0,0,0,0); return (1); } @@ -535,7 +534,7 @@ reget: pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0, VM_PROT_READ, 0, PGO_SYNCIO); if (error == EAGAIN) { - tsleep(&lbolt, PVM, "loanuopg", 0); + kpause("loanuopg", false, 1, NULL); continue; } if (error) @@ -626,7 +625,6 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, struct vm_amap *amap = ufi->entry->aref.ar_amap; struct uvm_object *uobj = ufi->entry->object.uvm_obj; struct vm_page *pg; - struct vm_anon *anon; int error, npages; bool locked; @@ -655,7 +653,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, */ if (error && error != EBUSY) { - uvmfault_unlockall(ufi, amap, uobj, NULL); + uvmfault_unlockall(ufi, amap, uobj); return (-1); } @@ -664,7 +662,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, */ if (error == EBUSY) { - uvmfault_unlockall(ufi, amap, NULL, NULL); + uvmfault_unlockall(ufi, amap, NULL); /* locked: uobj */ npages = 1; @@ -675,7 +673,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, if (error) { if (error == EAGAIN) { - tsleep(&lbolt, PVM, "fltagain2", 0); + kpause("fltagain2", false, 1, NULL); return (0); } return (-1); @@ -701,7 +699,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, (locked && amap && amap_lookup(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start))) { if (locked) - uvmfault_unlockall(ufi, amap, NULL, NULL); + uvmfault_unlockall(ufi, amap, NULL); locked = false; } @@ -740,7 +738,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, if ((flags & UVM_LOAN_TOANON) == 0) { if (uvm_loanpage(&pg, 1)) { - uvmfault_unlockall(ufi, amap, uobj, NULL); + uvmfault_unlockall(ufi, amap, uobj); return (-1); } mutex_exit(&uobj->vmobjlock); @@ -749,6 +747,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, return (1); } +#ifdef notdef /* * must be a loan to an anon. check to see if there is already * an anon associated with this page. if so, then just return @@ -758,9 +757,9 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, if (pg->uanon) { anon = pg->uanon; - mutex_enter(&anon->an_lock); + mutex_enter(anon->an_lock); anon->an_ref++; - mutex_exit(&anon->an_lock); + mutex_exit(anon->an_lock); if (pg->flags & PG_WANTED) { wakeup(pg); } @@ -789,7 +788,6 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, pg->uanon = NULL; anon->an_page = NULL; anon->an_ref--; - mutex_exit(&anon->an_lock); uvm_anfree(anon); goto fail; } @@ -797,6 +795,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, pmap_page_protect(pg, VM_PROT_READ); } pg->loan_count++; + anon->an_lock = uvm_pageactivate(pg); mutex_exit(&uvm_pageqlock); if (pg->flags & PG_WANTED) { @@ -821,6 +820,7 @@ fail: pg->flags &= ~(PG_WANTED|PG_BUSY); UVM_PAGE_OWN(pg, NULL); uvmfault_unlockall(ufi, amap, uobj, NULL); +#endif /* notdef */ return (-1); } @@ -840,7 +840,6 @@ static struct uvm_object uvm_loanzero_ob static int uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags) { - struct vm_anon *anon; struct vm_page *pg; struct vm_amap *amap = ufi->entry->aref.ar_amap; @@ -857,7 +856,7 @@ again: while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL, UVM_PGA_ZERO)) == NULL) { mutex_exit(&uvm_loanzero_object.vmobjlock); - uvmfault_unlockall(ufi, amap, NULL, NULL); + uvmfault_unlockall(ufi, amap, NULL); uvm_wait("loanzero"); if (!uvmfault_relock(ufi)) { return (0); @@ -887,6 +886,7 @@ again: return (1); } +#ifdef notdef /* * loaning to an anon. check to see if there is already an anon * associated with this page. if so, then just return a reference @@ -926,6 +926,9 @@ again: **output = anon; (*output)++; return (1); +#else + return (-1); +#endif } @@ -938,20 +941,22 @@ again: static void uvm_unloananon(struct vm_anon **aloans, int nanons) { +#ifdef notdef struct vm_anon *anon; while (nanons-- > 0) { int refs; anon = *aloans++; - mutex_enter(&anon->an_lock); + mutex_enter(anon->an_lock); refs = --anon->an_ref; - mutex_exit(&anon->an_lock); + mutex_exit(anon->an_lock); if (refs == 0) { uvm_anfree(anon); } } +#endif /* notdef */ } /* @@ -981,15 +986,13 @@ uvm_unloanpage(struct vm_page **ploans, if (pg->uobject != NULL) { slock = &pg->uobject->vmobjlock; } else { - slock = &pg->uanon->an_lock; + slock = pg->uanon->an_lock; } if (mutex_tryenter(slock)) { break; } - mutex_exit(&uvm_pageqlock); /* XXX Better than yielding but inadequate. */ - kpause("livelock", false, 1, NULL); - mutex_enter(&uvm_pageqlock); + kpause("livelock", false, 1, &uvm_pageqlock); slock = NULL; } Index: uvm/uvm_map.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_map.c,v retrieving revision 1.268 diff -u -p -r1.268 uvm_map.c --- uvm/uvm_map.c 20 Dec 2008 11:33:38 -0000 1.268 +++ uvm/uvm_map.c 30 Dec 2008 13:40:36 -0000 @@ -2366,6 +2366,9 @@ uvm_unmap_remove(struct vm_map *map, vad * remove mappings from pmap and drop the pages * from the object. offsets are always relative * to vm_map_min(kernel_map). + * + * don't need to lock object as the kernel is + * always self-consistent. */ pmap_remove(pmap_kernel(), entry->start, @@ -2380,12 +2383,15 @@ uvm_unmap_remove(struct vm_map *map, vad entry->etype &= ~UVM_ET_OBJ; entry->object.uvm_obj = NULL; } else if (UVM_ET_ISOBJ(entry) || entry->aref.ar_amap) { - /* - * remove mappings the standard way. + * remove mappings the standard way. lock object + * and/or amap to ensure vm_page state does not + * change while in pmap_remove(). */ + uvm_map_lock_entry(entry); pmap_remove(map->pmap, entry->start, entry->end); + uvm_map_unlock_entry(entry); } #if defined(DEBUG) @@ -2931,8 +2937,10 @@ uvm_map_extract(struct vm_map *srcmap, v /* we advance "entry" in the following if statement */ if (flags & UVM_EXTRACT_REMOVE) { + uvm_map_lock_entry(entry); pmap_remove(srcmap->pmap, entry->start, entry->end); + uvm_map_unlock_entry(entry); oldentry = entry; /* save entry */ entry = entry->next; /* advance */ uvm_map_entry_unlink(srcmap, oldentry); @@ -3172,8 +3180,10 @@ uvm_map_protect(struct vm_map *map, vadd if (current->protection != old_prot) { /* update pmap! */ + uvm_map_lock_entry(current); pmap_protect(map->pmap, current->start, current->end, current->protection & MASK(entry)); + uvm_map_unlock_entry(current); /* * If this entry points at a vnode, and the @@ -3830,7 +3840,7 @@ uvm_map_clean(struct vm_map *map, vaddr_ struct vm_map_entry *current, *entry; struct uvm_object *uobj; struct vm_amap *amap; - struct vm_anon *anon; + struct vm_anon *anon, *anon_tofree; struct vm_page *pg; vaddr_t offset; vsize_t size; @@ -3890,6 +3900,7 @@ uvm_map_clean(struct vm_map *map, vaddr_ goto flush_object; amap_lock(amap); + anon_tofree = NULL; offset = start - current->start; size = MIN(end, current->end) - start; for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) { @@ -3897,10 +3908,9 @@ uvm_map_clean(struct vm_map *map, vaddr_ if (anon == NULL) continue; - mutex_enter(&anon->an_lock); + KASSERT(anon->an_lock == amap->am_lock); pg = anon->an_page; if (pg == NULL) { - mutex_exit(&anon->an_lock); continue; } @@ -3924,13 +3934,11 @@ uvm_map_clean(struct vm_map *map, vaddr_ if (pg->loan_count != 0 || pg->wire_count != 0) { mutex_exit(&uvm_pageqlock); - mutex_exit(&anon->an_lock); continue; } KASSERT(pg->uanon == anon); uvm_pagedeactivate(pg); mutex_exit(&uvm_pageqlock); - mutex_exit(&anon->an_lock); continue; case PGO_FREE: @@ -3945,17 +3953,18 @@ uvm_map_clean(struct vm_map *map, vaddr_ /* skip the page if it's wired */ if (pg->wire_count != 0) { - mutex_exit(&anon->an_lock); continue; } amap_unadd(¤t->aref, offset); refs = --anon->an_ref; - mutex_exit(&anon->an_lock); - if (refs == 0) - uvm_anfree(anon); + if (refs == 0) { + anon->an_link = anon_tofree; + anon_tofree = anon; + } continue; } } + uvm_anfree(anon_tofree); amap_unlock(amap); flush_object: @@ -5254,6 +5263,30 @@ vm_map_starved_p(struct vm_map *map) return false; } +void +uvm_map_lock_entry(struct vm_map_entry *entry) +{ + + if (UVM_ET_ISOBJ(entry)) { + mutex_enter(&entry->object.uvm_obj->vmobjlock); + } + if (entry->aref.ar_amap != NULL) { + amap_lock(entry->aref.ar_amap); + } +} + +void +uvm_map_unlock_entry(struct vm_map_entry *entry) +{ + + if (entry->aref.ar_amap != NULL) { + amap_unlock(entry->aref.ar_amap); + } + if (UVM_ET_ISOBJ(entry)) { + mutex_exit(&entry->object.uvm_obj->vmobjlock); + } +} + #if defined(DDB) void uvm_whatis(uintptr_t addr, void (*pr)(const char *, ...)) Index: uvm/uvm_map.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_map.h,v retrieving revision 1.62 diff -u -p -r1.62 uvm_map.h --- uvm/uvm_map.h 29 Jul 2008 00:03:06 -0000 1.62 +++ uvm/uvm_map.h 30 Dec 2008 13:40:36 -0000 @@ -365,6 +365,9 @@ void vm_map_unlock_read(struct vm_map * void vm_map_busy(struct vm_map *); bool vm_map_locked_p(struct vm_map *); +void uvm_map_lock_entry(struct vm_map_entry *); +void uvm_map_unlock_entry(struct vm_map_entry *); + #endif /* _KERNEL */ /* Index: uvm/uvm_object.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_object.h,v retrieving revision 1.26 diff -u -p -r1.26 uvm_object.h --- uvm/uvm_object.h 4 Jun 2008 15:06:04 -0000 1.26 +++ uvm/uvm_object.h 30 Dec 2008 13:40:36 -0000 @@ -41,6 +41,7 @@ * uvm_object.h */ +#include #include /* @@ -54,6 +55,7 @@ struct uvm_object { int uo_npages; /* # of pages in memq */ unsigned uo_refs; /* reference count */ struct rb_tree rb_tree; /* tree of pages */ + LIST_HEAD(,ubc_map) uo_ubc; /* ubc mappings */ }; /* @@ -107,16 +109,26 @@ extern const struct uvm_pagerops aobj_pa extern const struct rb_tree_ops uvm_page_tree_ops; +void ubc_purge(struct uvm_object *); + #define UVM_OBJ_INIT(uobj, ops, refs) \ do { \ mutex_init(&(uobj)->vmobjlock, MUTEX_DEFAULT, IPL_NONE);\ (uobj)->pgops = (ops); \ TAILQ_INIT(&(uobj)->memq); \ + LIST_INIT(&(uobj)->uo_ubc); \ (uobj)->uo_npages = 0; \ (uobj)->uo_refs = (refs); \ rb_tree_init(&(uobj)->rb_tree, &uvm_page_tree_ops); \ } while (/* CONSTCOND */ 0) +#define UVM_OBJ_REINIT(uobj) \ + do { \ + TAILQ_INIT(&(uobj)->memq); \ + (uobj)->uo_npages = 0; \ + rb_tree_init(&(uobj)->rb_tree, &uvm_page_tree_ops); \ + } while (/* CONSTCOND */ 0) + #ifdef DIAGNOSTIC #define UVM_OBJ_DESTROY(uobj) \ do { \ @@ -125,11 +137,15 @@ extern const struct rb_tree_ops uvm_page mutex_destroy(&(uobj)->vmobjlock); \ _xn = rb_tree_find_node_geq(&(uobj)->rb_tree, &_xo); \ KASSERT(_xn == NULL); \ + if (__predict_false(!LIST_EMPTY(&(uobj)->uo_ubc))) \ + ubc_purge(uobj); \ } while (/* CONSTCOND */ 0) #else #define UVM_OBJ_DESTROY(uobj) \ do { \ mutex_destroy(&(uobj)->vmobjlock); \ + if (__predict_false(!LIST_EMPTY(&(uobj)->uo_ubc))) \ + ubc_purge(uobj); \ } while (/* CONSTCOND */ 0) #endif /* DIAGNOSTIC */ Index: uvm/uvm_page.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_page.c,v retrieving revision 1.141 diff -u -p -r1.141 uvm_page.c --- uvm/uvm_page.c 13 Dec 2008 11:34:43 -0000 1.141 +++ uvm/uvm_page.c 30 Dec 2008 13:40:37 -0000 @@ -1077,7 +1077,7 @@ uvm_pagealloc_strat(struct uvm_object *o KASSERT(anon == NULL || off == 0); KASSERT(off == trunc_page(off)); KASSERT(obj == NULL || mutex_owned(&obj->vmobjlock)); - KASSERT(anon == NULL || mutex_owned(&anon->an_lock)); + KASSERT(anon == NULL || mutex_owned(anon->an_lock)); mutex_spin_enter(&uvm_fpageqlock); @@ -1354,7 +1354,7 @@ uvm_pagefree(struct vm_page *pg) KASSERT(mutex_owned(&uvm_pageqlock) || !uvmpdpol_pageisqueued_p(pg)); KASSERT(pg->uobject == NULL || mutex_owned(&pg->uobject->vmobjlock)); KASSERT(pg->uobject != NULL || pg->uanon == NULL || - mutex_owned(&pg->uanon->an_lock)); + mutex_owned(pg->uanon->an_lock)); /* * if the page is loaned, resolve the loan instead of freeing. @@ -1499,7 +1499,7 @@ uvm_page_unbusy(struct vm_page **pgs, in KASSERT(pg->uobject == NULL || mutex_owned(&pg->uobject->vmobjlock)); KASSERT(pg->uobject != NULL || - (pg->uanon != NULL && mutex_owned(&pg->uanon->an_lock))); + (pg->uanon != NULL && mutex_owned(pg->uanon->an_lock))); KASSERT(pg->flags & PG_BUSY); KASSERT((pg->flags & PG_PAGEOUT) == 0); @@ -1543,7 +1543,7 @@ uvm_page_own(struct vm_page *pg, const c if (uobj != NULL) { KASSERT(mutex_owned(&uobj->vmobjlock)); } else if (anon != NULL) { - KASSERT(mutex_owned(&anon->an_lock)); + KASSERT(mutex_owned(anon->an_lock)); } KASSERT((pg->flags & PG_WANTED) == 0); @@ -1748,6 +1748,7 @@ uvm_pagedeactivate(struct vm_page *pg) { KASSERT(mutex_owned(&uvm_pageqlock)); + KASSERT(uvm_page_locked_p(pg)); KASSERT(pg->wire_count != 0 || uvmpdpol_pageisqueued_p(pg)); uvmpdpol_pagedeactivate(pg); } @@ -1763,6 +1764,7 @@ uvm_pageactivate(struct vm_page *pg) { KASSERT(mutex_owned(&uvm_pageqlock)); + KASSERT(uvm_page_locked_p(pg)); #if defined(READAHEAD_STATS) if ((pg->pqflags & PQ_READAHEAD) != 0) { uvm_ra_hit.ev_count++; @@ -1848,3 +1850,21 @@ uvm_page_lookup_freelist(struct vm_page KASSERT(lcv != -1); return (vm_physmem[lcv].free_list); } + +/* + * uvm_page_locked_p: return true if object associated with page is + * locked. this is a weak check for runtime assertions only. + */ + +bool +uvm_page_locked_p(struct vm_page *pg) +{ + + if (pg->uobject != NULL) { + return mutex_owned(&pg->uobject->vmobjlock); + } + if (pg->uanon != NULL) { + return mutex_owned(pg->uanon->an_lock); + } + return true; +} Index: uvm/uvm_page.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_page.h,v retrieving revision 1.55 diff -u -p -r1.55 uvm_page.h --- uvm/uvm_page.h 4 Jun 2008 15:06:04 -0000 1.55 +++ uvm/uvm_page.h 30 Dec 2008 13:40:37 -0000 @@ -286,6 +286,7 @@ void uvm_pagewait(struct vm_page *, int) void uvm_pagewake(struct vm_page *); void uvm_pagewire(struct vm_page *); void uvm_pagezero(struct vm_page *); +bool uvm_page_locked_p(struct vm_page *); int uvm_page_lookup_freelist(struct vm_page *); Index: uvm/uvm_pager.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pager.c,v retrieving revision 1.93 diff -u -p -r1.93 uvm_pager.c --- uvm/uvm_pager.c 16 Nov 2008 19:34:29 -0000 1.93 +++ uvm/uvm_pager.c 30 Dec 2008 13:40:37 -0000 @@ -347,7 +347,7 @@ uvm_aio_aiodone_pages(struct vm_page **p if (pg->uobject != NULL) { slock = &pg->uobject->vmobjlock; } else { - slock = &pg->uanon->an_lock; + slock = pg->uanon->an_lock; } mutex_enter(slock); mutex_enter(&uvm_pageqlock); Index: uvm/uvm_pdaemon.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdaemon.c,v retrieving revision 1.97 diff -u -p -r1.97 uvm_pdaemon.c --- uvm/uvm_pdaemon.c 13 Dec 2008 11:26:57 -0000 1.97 +++ uvm/uvm_pdaemon.c 30 Dec 2008 13:40:37 -0000 @@ -419,7 +419,7 @@ uvmpd_trylockowner(struct vm_page *pg) struct vm_anon *anon = pg->uanon; KASSERT(anon != NULL); - slock = &anon->an_lock; + slock = anon->an_lock; } if (!mutex_tryenter(slock)) { @@ -496,7 +496,7 @@ swapcluster_add(struct swapcluster *swc, slot = swc->swc_slot + swc->swc_nused; uobj = pg->uobject; if (uobj == NULL) { - KASSERT(mutex_owned(&pg->uanon->an_lock)); + KASSERT(mutex_owned(pg->uanon->an_lock)); pg->uanon->an_swslot = slot; } else { int result; Index: uvm/uvm_pdpolicy_clock.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdpolicy_clock.c,v retrieving revision 1.12 diff -u -p -r1.12 uvm_pdpolicy_clock.c --- uvm/uvm_pdpolicy_clock.c 4 Jun 2008 12:41:40 -0000 1.12 +++ uvm/uvm_pdpolicy_clock.c 30 Dec 2008 13:40:37 -0000 @@ -174,6 +174,7 @@ uvmpdpol_selectvictim(void) { struct uvmpdpol_scanstate *ss = &pdpol_scanstate; struct vm_page *pg; + kmutex_t *lock; KASSERT(mutex_owned(&uvm_pageqlock)); @@ -202,10 +203,15 @@ uvmpdpol_selectvictim(void) * skip to next page. */ - if (pmap_is_referenced(pg)) { - uvmpdpol_pageactivate(pg); - uvmexp.pdreact++; - continue; + lock = uvmpd_trylockowner(pg); + if (lock != NULL) { + if (pmap_is_referenced(pg)) { + uvmpdpol_pageactivate(pg); + uvmexp.pdreact++; + mutex_exit(lock); + continue; + } + mutex_exit(lock); } anon = pg->uanon; @@ -247,6 +253,7 @@ uvmpdpol_balancequeue(int swap_shortage) { int inactive_shortage; struct vm_page *p, *nextpg; + kmutex_t *lock; /* * we have done the scan to get free pages. now we work on meeting @@ -273,11 +280,17 @@ uvmpdpol_balancequeue(int swap_shortage) * if there's a shortage of inactive pages, deactivate. */ - if (inactive_shortage > 0) { - /* no need to check wire_count as pg is "active" */ + if (inactive_shortage <= 0) { + continue; + } + + /* no need to check wire_count as pg is "active" */ + lock = uvmpd_trylockowner(p); + if (lock != NULL) { uvmpdpol_pagedeactivate(p); uvmexp.pddeact++; inactive_shortage--; + mutex_exit(lock); } } } @@ -286,7 +299,9 @@ void uvmpdpol_pagedeactivate(struct vm_page *pg) { + KASSERT(uvm_page_locked_p(pg)); KASSERT(mutex_owned(&uvm_pageqlock)); + if (pg->pqflags & PQ_ACTIVE) { TAILQ_REMOVE(&pdpol_state.s_activeq, pg, pageq.queue); pg->pqflags &= ~PQ_ACTIVE; Index: uvm/uvm_vnode.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_vnode.c,v retrieving revision 1.90 diff -u -p -r1.90 uvm_vnode.c --- uvm/uvm_vnode.c 2 Jan 2008 11:49:21 -0000 1.90 +++ uvm/uvm_vnode.c 30 Dec 2008 13:40:37 -0000 @@ -243,6 +243,8 @@ uvn_findpage(struct uvm_object *uobj, vo UVMHIST_FUNC("uvn_findpage"); UVMHIST_CALLED(ubchist); UVMHIST_LOG(ubchist, "vp %p off 0x%lx", uobj, offset,0,0); + KASSERT(mutex_owned(&uobj->vmobjlock)); + if (*pgp != NULL) { UVMHIST_LOG(ubchist, "dontcare", 0,0,0,0); return 0;