Index: external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c =================================================================== RCS file: /cvsroot/src/external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c,v retrieving revision 1.56 diff -u -p -r1.56 zfs_vnops.c --- external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c 18 Jan 2020 15:21:32 -0000 1.56 +++ external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c 21 Jan 2020 21:09:14 -0000 @@ -728,7 +728,7 @@ mappedread(vnode_t *vp, int nbytes, uio_ { znode_t *zp = VTOZ(vp); struct uvm_object *uobj = &vp->v_uobj; - kmutex_t *mtx = uobj->vmobjlock; + krwlock_t *rw = uobj->vmobjlock; int64_t start; caddr_t va; size_t len = nbytes; @@ -745,10 +745,10 @@ mappedread(vnode_t *vp, int nbytes, uio_ pp = NULL; npages = 1; - mutex_enter(mtx); + rw_enter(rw, RW_WRITER); found = uvn_findpages(uobj, start, &npages, &pp, NULL, UFP_NOALLOC); - mutex_exit(mtx); + rw_exit(rw); /* XXXNETBSD shouldn't access userspace with the page busy */ if (found) { @@ -760,9 +760,9 @@ mappedread(vnode_t *vp, int nbytes, uio_ uio, bytes); } - mutex_enter(mtx); + rw_enter(rw, RW_WRITER); uvm_page_unbusy(&pp, 1); - mutex_exit(mtx); + rw_exit(rw); len -= bytes; off = 0; @@ -777,13 +777,13 @@ update_pages(vnode_t *vp, int64_t start, int segflg, dmu_tx_t *tx) { struct uvm_object *uobj = &vp->v_uobj; - kmutex_t *mtx = uobj->vmobjlock; + krwlock_t *rw = uobj->vmobjlock; caddr_t va; int off, status; ASSERT(vp->v_mount != NULL); - mutex_enter(mtx); + rw_enter(rw, RW_WRITER); off = start & PAGEOFFSET; for (start &= PAGEMASK; len > 0; start += PAGESIZE) { @@ -816,20 +816,20 @@ update_pages(vnode_t *vp, int64_t start, /* Nothing to do. */ break; } - mutex_exit(mtx); + rw_exit(rw); va = zfs_map_page(pp, S_WRITE); (void) dmu_read(os, oid, start + off, nbytes, va + off, DMU_READ_PREFETCH); zfs_unmap_page(pp, va); - mutex_enter(mtx); + rw_enter(rw, RW_WRITER); uvm_page_unbusy(&pp, 1); } len -= nbytes; off = 0; } - mutex_exit(mtx); + rw_exit(rw); } #endif /* __NetBSD__ */ @@ -5958,7 +5958,7 @@ zfs_netbsd_getpages(void *v) const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; struct uvm_object * const uobj = &vp->v_uobj; - kmutex_t * const mtx = uobj->vmobjlock; + krwlock_t * const rw = uobj->vmobjlock; znode_t *zp = VTOZ(vp); zfsvfs_t *zfsvfs = zp->z_zfsvfs; vfs_t *mp; @@ -5971,7 +5971,7 @@ zfs_netbsd_getpages(void *v) ap->a_m[ap->a_centeridx] = NULL; return EBUSY; } - mutex_exit(mtx); + rw_exit(rw); if (async) { return 0; @@ -5989,9 +5989,9 @@ zfs_netbsd_getpages(void *v) ZFS_ENTER(zfsvfs); ZFS_VERIFY_ZP(zp); - mutex_enter(mtx); + rw_enter(rw, RW_WRITER); if (offset >= vp->v_size) { - mutex_exit(mtx); + rw_exit(rw); ZFS_EXIT(zfsvfs); fstrans_done(mp); return EINVAL; @@ -6001,14 +6001,14 @@ zfs_netbsd_getpages(void *v) uvn_findpages(uobj, offset, &npages, &pg, NULL, UFP_ALL); if (pg->flags & PG_FAKE) { - mutex_exit(mtx); + rw_exit(rw); va = zfs_map_page(pg, S_WRITE); err = dmu_read(zfsvfs->z_os, zp->z_id, offset, PAGE_SIZE, va, DMU_READ_PREFETCH); zfs_unmap_page(pg, va); - mutex_enter(mtx); + rw_enter(rw, RW_READER); pg->flags &= ~(PG_FAKE); } @@ -6024,7 +6024,7 @@ zfs_netbsd_getpages(void *v) vp->v_iflag |= VI_WRMAPDIRTY; } } - mutex_exit(mtx); + rw_exit(rw); ap->a_m[ap->a_centeridx] = pg; ZFS_EXIT(zfsvfs); @@ -6046,7 +6046,7 @@ zfs_putapage(vnode_t *vp, page_t **pp, i bool async = (flags & PGO_SYNCIO) == 0; bool *cleanedp; struct uvm_object *uobj = &vp->v_uobj; - kmutex_t *mtx = uobj->vmobjlock; + krwlock_t *rw = uobj->vmobjlock; if (zp->z_sa_hdl == NULL) { err = 0; @@ -6105,9 +6105,9 @@ zfs_putapage(vnode_t *vp, page_t **pp, i dmu_tx_commit(tx); out_unbusy: - mutex_enter(mtx); + rw_enter(rw, RW_WRITER); uvm_page_unbusy(pp, count); - mutex_exit(mtx); + rw_exit(rw); out: return (err); @@ -6169,7 +6169,7 @@ zfs_netbsd_putpages(void *v) len = UINT64_MAX; else len = offhi - offlo; - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); if (curlwp == uvm.pagedaemon_lwp) { error = fstrans_start_nowait(vp->v_mount); if (error) @@ -6190,7 +6190,7 @@ zfs_netbsd_putpages(void *v) rrm_enter(&zfsvfs->z_teardown_lock, RW_READER, FTAG); rl = zfs_range_lock(zp, offlo, len, RL_WRITER); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); tsd_set(zfs_putpage_key, &cleaned); } error = genfs_putpages(v); @@ -6228,7 +6228,7 @@ void zfs_netbsd_setsize(vnode_t *vp, off_t size) { struct uvm_object *uobj = &vp->v_uobj; - kmutex_t *mtx = uobj->vmobjlock; + krwlock_t *rw = uobj->vmobjlock; page_t *pg; int count, pgoff; caddr_t va; @@ -6246,7 +6246,7 @@ zfs_netbsd_setsize(vnode_t *vp, off_t si * If there's a partial page, we need to zero the tail. */ - mutex_enter(mtx); + rw_enter(rw, RW_WRITER); count = 1; pg = NULL; if (uvn_findpages(uobj, tsize, &count, &pg, NULL, UFP_NOALLOC)) { @@ -6257,7 +6257,7 @@ zfs_netbsd_setsize(vnode_t *vp, off_t si uvm_page_unbusy(&pg, 1); } - mutex_exit(mtx); + rw_exit(rw); } static int Index: external/cddl/osnet/dist/uts/common/fs/zfs/zfs_znode.c =================================================================== RCS file: /cvsroot/src/external/cddl/osnet/dist/uts/common/fs/zfs/zfs_znode.c,v retrieving revision 1.29 diff -u -p -r1.29 zfs_znode.c --- external/cddl/osnet/dist/uts/common/fs/zfs/zfs_znode.c 17 Jun 2019 08:08:50 -0000 1.29 +++ external/cddl/osnet/dist/uts/common/fs/zfs/zfs_znode.c 21 Jan 2020 21:09:14 -0000 @@ -1561,7 +1561,7 @@ zfs_rezget(znode_t *zp) zp->z_unlinked = (zp->z_links == 0); zp->z_blksz = doi.doi_data_block_size; #ifdef __NetBSD__ - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); (void)VOP_PUTPAGES(vp, 0, 0, PGO_ALLPAGES|PGO_FREE|PGO_SYNCIO); #else vn_pages_remove(vp, 0, 0); Index: sys/arch/hppa/hppa/pmap.c =================================================================== RCS file: /cvsroot/src/sys/arch/hppa/hppa/pmap.c,v retrieving revision 1.102 diff -u -p -r1.102 pmap.c --- sys/arch/hppa/hppa/pmap.c 31 Dec 2019 12:40:27 -0000 1.102 +++ sys/arch/hppa/hppa/pmap.c 21 Jan 2020 21:09:19 -0000 @@ -579,7 +579,7 @@ pmap_pv_enter(struct vm_page *pg, struct DPRINTF(PDB_FOLLOW|PDB_PV, ("%s(%p, %p, %p, 0x%lx, %p, 0x%x)\n", __func__, pg, pve, pm, va, pdep, flags)); - KASSERT(pm == pmap_kernel() || uvm_page_owner_locked_p(pg)); + KASSERT(pm == pmap_kernel() || uvm_page_owner_locked_p(pg, true)); pve->pv_pmap = pm; pve->pv_va = va | flags; @@ -594,7 +594,7 @@ pmap_pv_remove(struct vm_page *pg, pmap_ struct vm_page_md * const md = VM_PAGE_TO_MD(pg); struct pv_entry **pve, *pv; - KASSERT(pmap == pmap_kernel() || uvm_page_owner_locked_p(pg)); + KASSERT(pmap == pmap_kernel() || uvm_page_owner_locked_p(pg, true)); for (pv = *(pve = &md->pvh_list); pv; pv = *(pve = &(*pve)->pv_next)) { Index: sys/arch/m68k/m68k/pmap_motorola.c =================================================================== RCS file: /cvsroot/src/sys/arch/m68k/m68k/pmap_motorola.c,v retrieving revision 1.71 diff -u -p -r1.71 pmap_motorola.c --- sys/arch/m68k/m68k/pmap_motorola.c 3 Sep 2018 16:29:25 -0000 1.71 +++ sys/arch/m68k/m68k/pmap_motorola.c 21 Jan 2020 21:09:19 -0000 @@ -2105,9 +2105,9 @@ pmap_remove_mapping(pmap_t pmap, vaddr_t #endif pmap_remove_mapping(pmap_kernel(), ptpva, NULL, PRM_TFLUSH|PRM_CFLUSH, NULL); - mutex_enter(uvm_kernel_object->vmobjlock); + rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); uvm_pagefree(PHYS_TO_VM_PAGE(ptppa)); - mutex_exit(uvm_kernel_object->vmobjlock); + rw_exit(uvm_kernel_object->vmobjlock); PMAP_DPRINTF(PDB_REMOVE|PDB_PTPAGE, ("remove: PT page 0x%lx (0x%lx) freed\n", ptpva, ptppa)); @@ -2576,15 +2576,15 @@ pmap_enter_ptpage(pmap_t pmap, vaddr_t v pmap->pm_sref++; PMAP_DPRINTF(PDB_ENTER|PDB_PTPAGE, ("enter: about to alloc UPT pg at %lx\n", va)); - mutex_enter(uvm_kernel_object->vmobjlock); + rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); while ((pg = uvm_pagealloc(uvm_kernel_object, va - vm_map_min(kernel_map), NULL, UVM_PGA_ZERO)) == NULL) { - mutex_exit(uvm_kernel_object->vmobjlock); + rw_exit(uvm_kernel_object->vmobjlock); uvm_wait("ptpage"); - mutex_enter(uvm_kernel_object->vmobjlock); + rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); } - mutex_exit(uvm_kernel_object->vmobjlock); + rw_exit(uvm_kernel_object->vmobjlock); pg->flags &= ~(PG_BUSY|PG_FAKE); UVM_PAGE_OWN(pg, NULL); ptpa = VM_PAGE_TO_PHYS(pg); @@ -2695,13 +2695,13 @@ pmap_ptpage_addref(vaddr_t ptpva) { struct vm_page *pg; - mutex_enter(uvm_kernel_object->vmobjlock); + rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); pg = uvm_pagelookup(uvm_kernel_object, ptpva - vm_map_min(kernel_map)); pg->wire_count++; PMAP_DPRINTF(PDB_ENTER|PDB_PTPAGE|PDB_SEGTAB, ("ptpage addref: pg %p now %d\n", pg, pg->wire_count)); - mutex_exit(uvm_kernel_object->vmobjlock); + rw_exit(uvm_kernel_object->vmobjlock); } /* @@ -2715,13 +2715,13 @@ pmap_ptpage_delref(vaddr_t ptpva) struct vm_page *pg; int rv; - mutex_enter(uvm_kernel_object->vmobjlock); + rw_enter(uvm_kernel_object->vmobjlock, RW_WRITER); pg = uvm_pagelookup(uvm_kernel_object, ptpva - vm_map_min(kernel_map)); rv = --pg->wire_count; PMAP_DPRINTF(PDB_ENTER|PDB_PTPAGE|PDB_SEGTAB, ("ptpage delref: pg %p now %d\n", pg, pg->wire_count)); - mutex_exit(uvm_kernel_object->vmobjlock); + rw_exit(uvm_kernel_object->vmobjlock); return rv; } Index: sys/arch/x86/include/pmap.h =================================================================== RCS file: /cvsroot/src/sys/arch/x86/include/pmap.h,v retrieving revision 1.109 diff -u -p -r1.109 pmap.h --- sys/arch/x86/include/pmap.h 12 Jan 2020 13:01:11 -0000 1.109 +++ sys/arch/x86/include/pmap.h 21 Jan 2020 21:09:19 -0000 @@ -113,6 +113,7 @@ #if defined(_KERNEL) #include +#include #include #include @@ -291,6 +292,7 @@ struct pmap { kmutex_t pm_lock /* locks for pm_objs */ __aligned(64); /* give lock own cache line */ + krwlock_t pm_dummy_lock; /* ugly hack for abusing uvm_object */ }; /* macro to access pm_pdirpa slots */ Index: sys/arch/x86/include/pmap_pv.h =================================================================== RCS file: /cvsroot/src/sys/arch/x86/include/pmap_pv.h,v retrieving revision 1.10 diff -u -p -r1.10 pmap_pv.h --- sys/arch/x86/include/pmap_pv.h 12 Jan 2020 13:01:11 -0000 1.10 +++ sys/arch/x86/include/pmap_pv.h 21 Jan 2020 21:09:19 -0000 @@ -72,6 +72,7 @@ struct pmap_page { LIST_ENTRY(vm_page) u_link; } pp_u; LIST_HEAD(, pv_entry) pp_pvlist; + __cpu_simple_lock_t pp_lock; #define pp_pte pp_u.u_pte #define pp_link pp_u.u_link uint8_t pp_flags; @@ -85,6 +86,10 @@ struct pmap_page { #define PP_EMBEDDED 1 #define PP_FREEING 2 -#define PMAP_PAGE_INIT(pp) LIST_INIT(&(pp)->pp_pvlist) +#define PMAP_PAGE_INIT(pp) \ +do { \ + LIST_INIT(&(pp)->pp_pvlist); \ + __cpu_simple_lock_init(&(pp)->pp_lock); \ +} while (0); #endif /* !_X86_PMAP_PV_H_ */ Index: sys/arch/x86/x86/pmap.c =================================================================== RCS file: /cvsroot/src/sys/arch/x86/x86/pmap.c,v retrieving revision 1.355 diff -u -p -r1.355 pmap.c --- sys/arch/x86/x86/pmap.c 12 Jan 2020 13:01:11 -0000 1.355 +++ sys/arch/x86/x86/pmap.c 21 Jan 2020 21:09:19 -0000 @@ -229,8 +229,10 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.3 * * - pg->uobject->vmobjlock, pg->uanon->an_lock * These per-object locks are taken by the VM system before calling into - * the pmap module. Holding them prevents concurrent operations on the - * given page or set of pages. Asserted with uvm_page_owner_locked_p(). + * the pmap module. They are always held at least read locked, and held + * write locked when calling pmap_page_remove(). This prevents the PV + * lists changing while pmap_page_remove() is doing its difficult walk + * through them. Asserted with uvm_page_owner_locked_p(). * * - pmap->pm_lock (per pmap) * This lock protects the fields in the pmap structure including the @@ -241,8 +243,24 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.3 * - pmaps_lock * This lock protects the list of active pmaps (headed by "pmaps"). We * lock it when adding or removing pmaps from this list. + * + * - pp->pp_lock + * This per-page lock protects PV entry lists and the embedded PV + * entry in each vm_page. It's a __cpu_simple_lock_t, which is safe + * here because it's never taken from interrupt context, and is only + * taken for a short amount of time in code paths which have kernel + * preemption disabled. */ +/* uvm_object is abused here to index pmap_pages; make assertions happy. */ +#ifdef DIAGNOSTIC +#define PMAP_DUMMY_LOCK(pm) rw_enter(&(pm)->pm_dummy_lock, RW_WRITER) +#define PMAP_DUMMY_UNLOCK(pm) rw_exit(&(pm)->pm_dummy_lock) +#else +#define PMAP_DUMMY_LOCK(pm) +#define PMAP_DUMMY_UNLOCK(pm) +#endif + const vaddr_t ptp_masks[] = PTP_MASK_INITIALIZER; const vaddr_t ptp_frames[] = PTP_FRAME_INITIALIZER; const int ptp_shifts[] = PTP_SHIFT_INITIALIZER; @@ -1022,9 +1040,10 @@ pmap_bootstrap(vaddr_t kva_start) */ kpm = pmap_kernel(); mutex_init(&kpm->pm_lock, MUTEX_DEFAULT, IPL_NONE); + rw_init(&kpm->pm_dummy_lock); for (i = 0; i < PTP_LEVELS - 1; i++) { uvm_obj_init(&kpm->pm_obj[i], NULL, false, 1); - uvm_obj_setlock(&kpm->pm_obj[i], &kpm->pm_lock); + uvm_obj_setlock(&kpm->pm_obj[i], &kpm->pm_dummy_lock); kpm->pm_ptphint[i] = NULL; } memset(&kpm->pm_list, 0, sizeof(kpm->pm_list)); /* pm_list not used */ @@ -1769,29 +1788,6 @@ pmap_vpage_cpu_init(struct cpu_info *ci) * p v _ e n t r y f u n c t i o n s */ - -/* - * pmap_pp_needs_pve: return true if we need to allocate a pv entry and - * corresponding radix tree entry for the page. - */ -static bool -pmap_pp_needs_pve(struct pmap_page *pp, struct vm_page *ptp, vaddr_t va) -{ - - /* - * Adding a pv entry for this page only needs to allocate a pv_entry - * structure if the page already has at least one pv entry, since - * the first pv entry is stored in the pmap_page. However, because - * of subsequent removal(s), PP_EMBEDDED can be false and there can - * still be pv entries on the list. - */ - - if (pp == NULL || (pp->pp_flags & PP_EMBEDDED) == 0) { - return false; - } - return pp->pp_pte.pte_ptp != ptp || pp->pp_pte.pte_va != va; -} - /* * pmap_free_pvs: free a linked list of pv entries. the pv entries have * been removed from their respective pages, but are still entered into the @@ -1880,21 +1876,22 @@ pmap_enter_pv(struct pmap *pmap, struct KASSERT(ptp == NULL || ptp->wire_count >= 2); KASSERT(ptp == NULL || ptp->uobject != NULL); KASSERT(ptp == NULL || ptp_va2o(va, 1) == ptp->offset); + KASSERT(kpreempt_disabled()); /* for pp_lock */ + KASSERT(__SIMPLELOCK_LOCKED_P(&pp->pp_lock)); + KASSERT(pve != NULL); if ((pp->pp_flags & PP_EMBEDDED) == 0) { pp->pp_flags |= PP_EMBEDDED; pp->pp_pte.pte_ptp = ptp; pp->pp_pte.pte_va = va; - return pve; + } else { + pve->pve_pte.pte_ptp = ptp; + pve->pve_pte.pte_va = va; + LIST_INSERT_HEAD(&pp->pp_pvlist, pve, pve_list); + pve = NULL; } - KASSERT(pve != NULL); - pve->pve_pte.pte_ptp = ptp; - pve->pve_pte.pte_va = va; - KASSERT(pmap_pvmap_lookup(pmap, va) != NULL); - KASSERT(pmap_pvmap_lookup(pmap, va) == pve); - LIST_INSERT_HEAD(&pp->pp_pvlist, pve, pve_list); - return NULL; + return pve; } /* @@ -1905,7 +1902,7 @@ pmap_enter_pv(struct pmap *pmap, struct * => we return the removed pve * => caller can optionally supply pve, if looked up already */ -static struct pv_entry * +static void pmap_remove_pv(struct pmap *pmap, struct pmap_page *pp, struct vm_page *ptp, vaddr_t va, struct pv_entry *pve) { @@ -1914,27 +1911,21 @@ pmap_remove_pv(struct pmap *pmap, struct KASSERT(ptp_to_pmap(ptp) == pmap); KASSERT(ptp == NULL || ptp->uobject != NULL); KASSERT(ptp == NULL || ptp_va2o(va, 1) == ptp->offset); + KASSERT(kpreempt_disabled()); /* for pp_lock */ + KASSERT(__SIMPLELOCK_LOCKED_P(&pp->pp_lock)); if ((pp->pp_flags & PP_EMBEDDED) != 0 && pp->pp_pte.pte_ptp == ptp && pp->pp_pte.pte_va == va) { - KASSERT(pve == NULL); pp->pp_flags &= ~PP_EMBEDDED; pp->pp_pte.pte_ptp = NULL; pp->pp_pte.pte_va = 0; - return NULL; - } - - if (pve == NULL) { - pve = pmap_pvmap_lookup(pmap, va); - KASSERT(pve != NULL); } else { - KASSERT(pve == pmap_pvmap_lookup(pmap, va)); + KASSERT(pve != NULL); + KASSERT(pve->pve_pte.pte_ptp == ptp); + KASSERT(pve->pve_pte.pte_va == va); + LIST_REMOVE(pve, pve_list); } - KASSERT(pve->pve_pte.pte_ptp == ptp); - KASSERT(pve->pve_pte.pte_va == va); - LIST_REMOVE(pve, pve_list); - return pve; } /* @@ -1953,7 +1944,9 @@ pmap_find_ptp(struct pmap *pmap, vaddr_t pa == VM_PAGE_TO_PHYS(pmap->pm_ptphint[lidx])) { return (pmap->pm_ptphint[lidx]); } + PMAP_DUMMY_LOCK(pmap); pg = uvm_pagelookup(&pmap->pm_obj[lidx], ptp_va2o(va, level)); + PMAP_DUMMY_UNLOCK(pmap); if (pg != NULL) { if (__predict_false(pg->wire_count == 0)) { /* This page is queued to be freed - ignore. */ @@ -2078,7 +2071,9 @@ pmap_get_ptp(struct pmap *pmap, struct p obj = &pmap->pm_obj[i - 2]; off = ptp_va2o(va, i - 1); + PMAP_DUMMY_LOCK(pmap); pt->pg[i] = uvm_pagelookup(obj, off); + if (pt->pg[i] == NULL) { pt->pg[i] = uvm_pagealloc(obj, off, NULL, aflags); pt->alloced[i] = true; @@ -2092,6 +2087,7 @@ pmap_get_ptp(struct pmap *pmap, struct p KASSERT((VM_PAGE_TO_PP(pt->pg[i])->pp_flags & PP_FREEING) == 0); } + PMAP_DUMMY_UNLOCK(pmap); if (pt->pg[i] == NULL) { pmap_unget_ptp(pmap, pt); return ENOMEM; @@ -2200,7 +2196,9 @@ pmap_unget_ptp(struct pmap *pmap, struct KASSERT(pt->pg[i]->wire_count == 0); /* pmap zeros all pages before freeing. */ pt->pg[i]->flags |= PG_ZERO; + PMAP_DUMMY_LOCK(pmap); uvm_pagefree(pt->pg[i]); + PMAP_DUMMY_UNLOCK(pmap); pt->pg[i] = NULL; pmap->pm_ptphint[0] = NULL; } @@ -2385,6 +2383,7 @@ pmap_ctor(void *arg, void *obj, int flag KASSERT((flags & PR_WAITOK) != 0); mutex_init(&pmap->pm_lock, MUTEX_DEFAULT, IPL_NONE); + rw_init(&pmap->pm_dummy_lock); radix_tree_init_tree(&pmap->pm_pvtree); kcpuset_create(&pmap->pm_cpus, true); kcpuset_create(&pmap->pm_kernel_cpus, true); @@ -2433,6 +2432,7 @@ pmap_dtor(void *arg, void *obj) pool_put(&pmap_pdp_pool, pmap->pm_pdir); radix_tree_fini_tree(&pmap->pm_pvtree); mutex_destroy(&pmap->pm_lock); + rw_destroy(&pmap->pm_dummy_lock); kcpuset_destroy(pmap->pm_cpus); kcpuset_destroy(pmap->pm_kernel_cpus); #ifdef XENPV @@ -2454,7 +2454,7 @@ pmap_create(void) /* init uvm_object */ for (i = 0; i < PTP_LEVELS - 1; i++) { uvm_obj_init(&pmap->pm_obj[i], NULL, false, 1); - uvm_obj_setlock(&pmap->pm_obj[i], &pmap->pm_lock); + uvm_obj_setlock(&pmap->pm_obj[i], &pmap->pm_dummy_lock); pmap->pm_ptphint[i] = NULL; } pmap->pm_stats.wired_count = 0; @@ -3554,7 +3554,7 @@ pmap_remove_pte(struct pmap *pmap, struc } if ((pg = PHYS_TO_VM_PAGE(pmap_pte2pa(opte))) != NULL) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, false)); pp = VM_PAGE_TO_PP(pg); } else if ((pp = pmap_pv_tracked(pmap_pte2pa(opte))) == NULL) { paddr_t pa = pmap_pte2pa(opte); @@ -3564,8 +3564,11 @@ pmap_remove_pte(struct pmap *pmap, struc } /* Sync R/M bits. */ + pve = pmap_pvmap_lookup(pmap, va); + __cpu_simple_lock(&pp->pp_lock); pp->pp_attrs |= pmap_pte_to_pp_attrs(opte); - pve = pmap_remove_pv(pmap, pp, ptp, va, NULL); + pmap_remove_pv(pmap, pp, ptp, va, pve); + __cpu_simple_unlock(&pp->pp_lock); if (pve) { pve->pve_next = *pv_tofree; @@ -3829,9 +3832,13 @@ startover: pmap_reference(pmap); } mutex_enter(&pmap->pm_lock); + va = pvpte->pte_va; + pve = pmap_pvmap_lookup(pmap, va); + __cpu_simple_lock(&pp->pp_lock); error = pmap_sync_pv(pvpte, pa, ~0, &oattrs, &opte); if (error == EAGAIN) { int hold_count; + __cpu_simple_unlock(&pp->pp_lock); KERNEL_UNLOCK_ALL(curlwp, &hold_count); mutex_exit(&pmap->pm_lock); if (ptp != NULL) { @@ -3843,8 +3850,8 @@ startover: } pp->pp_attrs |= oattrs; - va = pvpte->pte_va; - pve = pmap_remove_pv(pmap, pp, ptp, va, NULL); + pmap_remove_pv(pmap, pp, ptp, va, pve); + __cpu_simple_unlock(&pp->pp_lock); /* Update the PTP reference count. Free if last reference. */ if (ptp != NULL) { @@ -3887,7 +3894,8 @@ pmap_page_remove(struct vm_page *pg) struct pmap_page *pp; paddr_t pa; - KASSERT(uvm_page_owner_locked_p(pg)); + /* Need an exclusive lock to prevent PV list changing behind us. */ + KASSERT(uvm_page_owner_locked_p(pg, true)); pp = VM_PAGE_TO_PP(pg); pa = VM_PAGE_TO_PHYS(pg); @@ -3928,7 +3936,7 @@ pmap_test_attrs(struct vm_page *pg, unsi u_int result; paddr_t pa; - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, false)); pp = VM_PAGE_TO_PP(pg); if ((pp->pp_attrs & testbits) != 0) { @@ -3936,6 +3944,7 @@ pmap_test_attrs(struct vm_page *pg, unsi } pa = VM_PAGE_TO_PHYS(pg); kpreempt_disable(); + __cpu_simple_lock(&pp->pp_lock); for (pvpte = pv_pte_first(pp); pvpte; pvpte = pv_pte_next(pp, pvpte)) { int error; @@ -3948,6 +3957,7 @@ pmap_test_attrs(struct vm_page *pg, unsi } } result = pp->pp_attrs & testbits; + __cpu_simple_unlock(&pp->pp_lock); kpreempt_enable(); /* @@ -3969,12 +3979,14 @@ pmap_pp_clear_attrs(struct pmap_page *pp count = SPINLOCK_BACKOFF_MIN; kpreempt_disable(); startover: + __cpu_simple_lock(&pp->pp_lock); for (pvpte = pv_pte_first(pp); pvpte; pvpte = pv_pte_next(pp, pvpte)) { int error; error = pmap_sync_pv(pvpte, pa, clearbits, &oattrs, NULL); if (error == EAGAIN) { int hold_count; + __cpu_simple_unlock(&pp->pp_lock); KERNEL_UNLOCK_ALL(curlwp, &hold_count); SPINLOCK_BACKOFF(count); KERNEL_LOCK(hold_count, curlwp); @@ -3984,6 +3996,7 @@ startover: } result = pp->pp_attrs & clearbits; pp->pp_attrs &= ~clearbits; + __cpu_simple_unlock(&pp->pp_lock); pmap_tlb_shootnow(); kpreempt_enable(); @@ -4001,7 +4014,7 @@ pmap_clear_attrs(struct vm_page *pg, uns struct pmap_page *pp; paddr_t pa; - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, false)); pp = VM_PAGE_TO_PP(pg); pa = VM_PAGE_TO_PHYS(pg); @@ -4238,6 +4251,7 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t bool wired = (flags & PMAP_WIRED) != 0; struct pmap *pmap2; struct pmap_ptparray pt; + bool alloced_pve; KASSERT(pmap_initialized); KASSERT(pmap->pm_remove_all == NULL); @@ -4306,11 +4320,17 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t * allocate and install in the radix tree. In any case look up the * pv entry in case the old mapping used it. */ + alloced_pve = false; pve = pmap_pvmap_lookup(pmap, va); - if (pve == NULL && pmap_pp_needs_pve(new_pp, ptp, va)) { + if (pve == NULL && new_pp != NULL) { pve = pool_cache_get(&pmap_pv_cache, PR_NOWAIT); if (pve == NULL) { if (flags & PMAP_CANFAIL) { + /* + * XXXAD may need to remove old mapping even + * in failure case, and forget about putting + * new one in place. investigate. + */ if (ptp != NULL) { pmap_unget_ptp(pmap, &pt); } @@ -4322,6 +4342,7 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t error = pmap_pvmap_insert(pmap, va, pve); if (error != 0) { if (flags & PMAP_CANFAIL) { + /* XXXAD same */ if (ptp != NULL) { pmap_unget_ptp(pmap, &pt); } @@ -4332,6 +4353,7 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t panic("%s: radixtree insert failed, error=%d", __func__, error); } + alloced_pve = true; } /* Map PTEs into address space. */ @@ -4394,14 +4416,13 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t /* * If the same page, we can skip pv_entry handling. + * If pv_entry is already installed, retain it. */ if (((opte ^ npte) & (PTE_FRAME | PTE_P)) == 0) { KASSERT(((opte ^ npte) & PTE_PVLIST) == 0); - if ((opte & PTE_PVLIST) != 0 && pve != NULL) { - KASSERT(pve->pve_pte.pte_ptp == ptp); - KASSERT(pve->pve_pte.pte_va == va); + if (alloced_pve == false) { + pve = NULL; } - pve = NULL; goto same_pa; } @@ -4410,7 +4431,7 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t */ if ((~opte & (PTE_P | PTE_PVLIST)) == 0) { if ((old_pg = PHYS_TO_VM_PAGE(oldpa)) != NULL) { - KASSERT(uvm_page_owner_locked_p(old_pg)); + KASSERT(uvm_page_owner_locked_p(old_pg, false)); old_pp = VM_PAGE_TO_PP(old_pg); } else if ((old_pp = pmap_pv_tracked(oldpa)) == NULL) { panic("%s: PTE_PVLIST with pv-untracked page" @@ -4419,15 +4440,19 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t __func__, va, oldpa, atop(pa)); } - (void)pmap_remove_pv(pmap, old_pp, ptp, va, pve); + __cpu_simple_lock(&old_pp->pp_lock); + pmap_remove_pv(pmap, old_pp, ptp, va, pve); old_pp->pp_attrs |= pmap_pte_to_pp_attrs(opte); + __cpu_simple_unlock(&old_pp->pp_lock); } /* * If new page is pv-tracked, insert pv_entry into its list. */ if (new_pp) { + __cpu_simple_lock(&new_pp->pp_lock); pve = pmap_enter_pv(pmap, new_pp, pve, ptp, va); + __cpu_simple_unlock(&new_pp->pp_lock); } same_pa: @@ -4804,7 +4829,9 @@ pmap_update(struct pmap *pmap) * assertions in uvm_pagefree(). Despite uobject * being set, this isn't a managed page. */ + PMAP_DUMMY_LOCK(pmap); uvm_pagerealloc(ptp, NULL, 0); + PMAP_DUMMY_UNLOCK(pmap); /* pmap zeros all pages before freeing */ ptp->flags |= PG_ZERO; @@ -5147,7 +5174,7 @@ pmap_ept_enter(struct pmap *pmap, vaddr_ struct pmap_page *new_pp, *old_pp; struct pv_entry *pve; bool wired = (flags & PMAP_WIRED) != 0; - bool accessed; + bool accessed, alloced_pve; struct pmap_ptparray pt; int error; @@ -5185,6 +5212,7 @@ pmap_ept_enter(struct pmap *pmap, vaddr_ if (pmap != pmap_kernel()) { error = pmap_get_ptp(pmap, &pt, va, flags, &ptp); if (error != 0) { + /* XXXAD as pmap_enter_ma() */ if (flags & PMAP_CANFAIL) { mutex_exit(&pmap->pm_lock); return error; @@ -5199,8 +5227,9 @@ pmap_ept_enter(struct pmap *pmap, vaddr_ * allocate and install in the radix tree. In any case look up the * pv entry in case the old mapping used it. */ + alloced_pve = false; pve = pmap_pvmap_lookup(pmap, va); - if (pve == NULL && pmap_pp_needs_pve(new_pp, ptp, va)) { + if (pve == NULL && new_pp != NULL) { pve = pool_cache_get(&pmap_pv_cache, PR_NOWAIT); if (pve == NULL) { if (flags & PMAP_CANFAIL) { @@ -5225,6 +5254,7 @@ pmap_ept_enter(struct pmap *pmap, vaddr_ panic("%s: radixtree insert failed, error=%d", __func__, error); } + alloced_pve = true; } /* Map PTEs into address space. */ @@ -5279,7 +5309,9 @@ pmap_ept_enter(struct pmap *pmap, vaddr_ KASSERT(pve->pve_pte.pte_ptp == ptp); KASSERT(pve->pve_pte.pte_va == va); } - pve = NULL; + if (alloced_pve == false) { + pve = NULL; + } goto same_pa; } @@ -5288,7 +5320,7 @@ pmap_ept_enter(struct pmap *pmap, vaddr_ */ if ((~opte & (EPT_R | EPT_PVLIST)) == 0) { if ((old_pg = PHYS_TO_VM_PAGE(oldpa)) != NULL) { - KASSERT(uvm_page_owner_locked_p(old_pg)); + KASSERT(uvm_page_owner_locked_p(old_pg, false)); old_pp = VM_PAGE_TO_PP(old_pg); } else if ((old_pp = pmap_pv_tracked(oldpa)) == NULL) { panic("%s: EPT_PVLIST with pv-untracked page" @@ -5297,15 +5329,19 @@ pmap_ept_enter(struct pmap *pmap, vaddr_ __func__, va, oldpa, atop(pa)); } - (void)pmap_remove_pv(pmap, old_pp, ptp, va, pve); + __cpu_simple_lock(&old_pp->pp_lock); + pmap_remove_pv(pmap, old_pp, ptp, va, pve); old_pp->pp_attrs |= pmap_ept_to_pp_attrs(opte); + __cpu_simple_unlock(&old_pp->pp_lock); } /* * If new page is pv-tracked, insert pv_entry into its list. */ if (new_pp) { + __cpu_simple_lock(&new_pp->pp_lock); pve = pmap_enter_pv(pmap, new_pp, pve, ptp, va); + __cpu_simple_unlock(&new_pp->pp_lock); } same_pa: @@ -5454,7 +5490,7 @@ pmap_ept_remove_pte(struct pmap *pmap, s } if ((pg = PHYS_TO_VM_PAGE(pmap_pte2pa(opte))) != NULL) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, false)); pp = VM_PAGE_TO_PP(pg); } else if ((pp = pmap_pv_tracked(pmap_pte2pa(opte))) == NULL) { paddr_t pa = pmap_pte2pa(opte); @@ -5464,8 +5500,11 @@ pmap_ept_remove_pte(struct pmap *pmap, s } /* Sync R/M bits. */ + pve = pmap_pvmap_lookup(pmap, va); + __cpu_simple_lock(&pp->pp_lock); pp->pp_attrs |= pmap_ept_to_pp_attrs(opte); - pve = pmap_remove_pv(pmap, pp, ptp, va, NULL); + pmap_remove_pv(pmap, pp, ptp, va, pve); + __cpu_simple_unlock(&pp->pp_lock); if (pve) { pve->pve_next = *pv_tofree; Index: sys/arch/xen/xen/privcmd.c =================================================================== RCS file: /cvsroot/src/sys/arch/xen/xen/privcmd.c,v retrieving revision 1.51 diff -u -p -r1.51 privcmd.c --- sys/arch/xen/xen/privcmd.c 22 Jun 2017 22:36:50 -0000 1.51 +++ sys/arch/xen/xen/privcmd.c 21 Jan 2020 21:09:19 -0000 @@ -450,9 +450,9 @@ static struct uvm_pagerops privpgops = { static void privpgop_reference(struct uvm_object *uobj) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); uobj->uo_refs++; - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } static void @@ -460,13 +460,13 @@ privpgop_detach(struct uvm_object *uobj) { struct privcmd_object *pobj = (struct privcmd_object *)uobj; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); if (uobj->uo_refs > 1) { uobj->uo_refs--; - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return; } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); kmem_free(pobj->maddr, sizeof(paddr_t) * pobj->npages); uvm_obj_destroy(uobj, true); kmem_free(pobj, sizeof(struct privcmd_object)); Index: sys/coda/coda_vnops.c =================================================================== RCS file: /cvsroot/src/sys/coda/coda_vnops.c,v retrieving revision 1.108 diff -u -p -r1.108 coda_vnops.c --- sys/coda/coda_vnops.c 17 Jan 2020 20:08:06 -0000 1.108 +++ sys/coda/coda_vnops.c 21 Jan 2020 21:09:19 -0000 @@ -1764,8 +1764,7 @@ coda_grab_vnode(vnode_t *uvp, dev_t dev, return(ENOENT); } /* share the underlying vnode lock with the coda vnode */ - mutex_obj_hold((*vpp)->v_interlock); - uvm_obj_setlock(&uvp->v_uobj, (*vpp)->v_interlock); + vshareilock(*vpp, uvp); KASSERT(VOP_ISLOCKED(*vpp)); return(0); } Index: sys/dev/ic/ssdfb.c =================================================================== RCS file: /cvsroot/src/sys/dev/ic/ssdfb.c,v retrieving revision 1.10 diff -u -p -r1.10 ssdfb.c --- sys/dev/ic/ssdfb.c 2 Nov 2019 17:13:20 -0000 1.10 +++ sys/dev/ic/ssdfb.c 21 Jan 2020 21:09:19 -0000 @@ -387,9 +387,9 @@ ssdfb_detach(struct ssdfb_softc *sc) kthread_join(sc->sc_thread); if (sc->sc_uobj != NULL) { - mutex_enter(sc->sc_uobj->vmobjlock); + rw_enter(sc->sc_uobj->vmobjlock, RW_WRITER); sc->sc_uobj->uo_refs--; - mutex_exit(sc->sc_uobj->vmobjlock); + rw_exit(sc->sc_uobj->vmobjlock); } config_detach(sc->sc_wsdisplay, DETACH_FORCE); Index: sys/external/bsd/drm2/dist/drm/drm_gem.c =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/dist/drm/drm_gem.c,v retrieving revision 1.12 diff -u -p -r1.12 drm_gem.c --- sys/external/bsd/drm2/dist/drm/drm_gem.c 17 Jan 2020 19:56:49 -0000 1.12 +++ sys/external/bsd/drm2/dist/drm/drm_gem.c 21 Jan 2020 21:09:20 -0000 @@ -171,7 +171,7 @@ int drm_gem_object_init(struct drm_devic * set the uao to have the main uvm object's lock. However, * uvm_obj_setlock is not safe on uvm_aobjs. */ - mutex_obj_hold(obj->filp->vmobjlock); + rw_obj_hold(obj->filp->vmobjlock); uvm_obj_setlock(&obj->gemo_uvmobj, obj->filp->vmobjlock); #else filp = shmem_file_setup("drm mm object", size, VM_NORESERVE); @@ -613,10 +613,10 @@ drm_gem_put_pages(struct drm_gem_object for (i = 0; i < (obj->size >> PAGE_SHIFT); i++) { if (dirty) { - mutex_enter(obj->filp->vmobjlock); + rw_enter(obj->filp->vmobjlock, RW_WRITER); uvm_pagemarkdirty(&pages[i]->p_vmp, UVM_PAGE_STATUS_DIRTY); - mutex_exit(obj->filp->vmobjlock); + rw_exit(obj->filp->vmobjlock); } } Index: sys/external/bsd/drm2/dist/drm/i915/i915_drv.h =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/dist/drm/i915/i915_drv.h,v retrieving revision 1.30 diff -u -p -r1.30 i915_drv.h --- sys/external/bsd/drm2/dist/drm/i915/i915_drv.h 5 Dec 2019 20:03:09 -0000 1.30 +++ sys/external/bsd/drm2/dist/drm/i915/i915_drv.h 21 Jan 2020 21:09:20 -0000 @@ -3021,9 +3021,9 @@ i915_gem_object_get_page(struct drm_i915 * lock to prevent them from disappearing. */ KASSERT(obj->pages != NULL); - mutex_enter(obj->base.filp->vmobjlock); + rw_enter(obj->base.filp->vmobjlock, RW_WRITER); page = uvm_pagelookup(obj->base.filp, ptoa(n)); - mutex_exit(obj->base.filp->vmobjlock); + rw_exit(obj->base.filp->vmobjlock); } KASSERT(page != NULL); return container_of(page, struct page, p_vmp); Index: sys/external/bsd/drm2/dist/drm/i915/i915_gem.c =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/dist/drm/i915/i915_gem.c,v retrieving revision 1.56 diff -u -p -r1.56 i915_gem.c --- sys/external/bsd/drm2/dist/drm/i915/i915_gem.c 17 Jan 2020 19:56:50 -0000 1.56 +++ sys/external/bsd/drm2/dist/drm/i915/i915_gem.c 21 Jan 2020 21:09:20 -0000 @@ -2026,7 +2026,7 @@ i915_gem_fault(struct uvm_faultinfo *ufi intel_runtime_pm_get(dev_priv); /* Thanks, uvm, but we don't need this lock. */ - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); ret = i915_mutex_lock_interruptible(dev); if (ret) @@ -2066,7 +2066,7 @@ unpin: unlock: mutex_unlock(&dev->struct_mutex); out: - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj); if (ret == -ERESTART) uvm_wait("i915flt"); @@ -2568,7 +2568,7 @@ i915_gem_object_truncate(struct drm_i915 if (uobj != NULL) { /* XXX Calling pgo_put like this is bogus. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); (*uobj->pgops->pgo_put)(uobj, 0, obj->base.size, (PGO_ALLPAGES | PGO_FREE)); } @@ -2606,7 +2606,7 @@ i915_gem_object_invalidate(struct drm_i9 #ifdef __NetBSD__ uobj = obj->base.filp; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); (*uobj->pgops->pgo_put)(uobj, 0, obj->base.size, PGO_ALLPAGES|PGO_DEACTIVATE|PGO_CLEANIT); #else @@ -2643,12 +2643,12 @@ i915_gem_object_put_pages_gtt(struct drm obj->dirty = 0; if (obj->dirty) { - mutex_enter(obj->base.filp->vmobjlock); + rw_enter(obj->base.filp->vmobjlock, RW_WRITER); TAILQ_FOREACH(page, &obj->pageq, pageq.queue) { uvm_pagemarkdirty(page, UVM_PAGE_STATUS_DIRTY); /* XXX mark page accessed */ } - mutex_exit(obj->base.filp->vmobjlock); + rw_exit(obj->base.filp->vmobjlock); } obj->dirty = 0; Index: sys/external/bsd/drm2/dist/drm/i915/i915_gem_fence.c =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/dist/drm/i915/i915_gem_fence.c,v retrieving revision 1.7 diff -u -p -r1.7 i915_gem_fence.c --- sys/external/bsd/drm2/dist/drm/i915/i915_gem_fence.c 17 Jan 2020 19:56:50 -0000 1.7 +++ sys/external/bsd/drm2/dist/drm/i915/i915_gem_fence.c 21 Jan 2020 21:09:20 -0000 @@ -769,9 +769,9 @@ i915_gem_object_do_bit_17_swizzle(struct (test_bit(i, obj->bit_17) != 0)) { i915_gem_swizzle_page(container_of(page, struct page, p_vmp)); - mutex_enter(obj->base.filp->vmobjlock); + rw_enter(obj->base.filp->vmobjlock, RW_WRITER); uvm_pagemarkdirty(page, UVM_PAGE_STATUS_DIRTY); - mutex_exit(obj->base.filp->vmobjlock); + rw_exit(obj->base.filp->vmobjlock); } i += 1; } Index: sys/external/bsd/drm2/dist/drm/ttm/ttm_bo.c =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/dist/drm/ttm/ttm_bo.c,v retrieving revision 1.15 diff -u -p -r1.15 ttm_bo.c --- sys/external/bsd/drm2/dist/drm/ttm/ttm_bo.c 2 Feb 2019 21:46:27 -0000 1.15 +++ sys/external/bsd/drm2/dist/drm/ttm/ttm_bo.c 21 Jan 2020 21:09:20 -0000 @@ -304,7 +304,7 @@ static int ttm_bo_add_ttm(struct ttm_buf * set the uao to have the main uvm object's lock. However, * uvm_obj_setlock is not safe on uvm_aobjs. */ - mutex_obj_hold(bo->ttm->swap_storage->vmobjlock); + rw_obj_hold(bo->ttm->swap_storage->vmobjlock); uvm_obj_setlock(&bo->uvmobj, bo->ttm->swap_storage->vmobjlock); return 0; #else @@ -1643,11 +1643,11 @@ void ttm_bo_unmap_virtual_locked(struct } else if (bo->ttm != NULL) { unsigned i; - mutex_enter(bo->uvmobj.vmobjlock); + rw_enter(bo->uvmobj.vmobjlock, RW_WRITER); for (i = 0; i < bo->ttm->num_pages; i++) pmap_page_protect(&bo->ttm->pages[i]->p_vmp, VM_PROT_NONE); - mutex_exit(bo->uvmobj.vmobjlock); + rw_exit(bo->uvmobj.vmobjlock); } #else struct ttm_bo_device *bdev = bo->bdev; Index: sys/external/bsd/drm2/dist/drm/ttm/ttm_bo_util.c =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/dist/drm/ttm/ttm_bo_util.c,v retrieving revision 1.14 diff -u -p -r1.14 ttm_bo_util.c --- sys/external/bsd/drm2/dist/drm/ttm/ttm_bo_util.c 30 Aug 2018 01:19:49 -0000 1.14 +++ sys/external/bsd/drm2/dist/drm/ttm/ttm_bo_util.c 21 Jan 2020 21:09:20 -0000 @@ -537,7 +537,7 @@ static int ttm_buffer_object_transfer(st linux_mutex_init(&fbo->wu_mutex); drm_vma_node_init(&fbo->vma_node); uvm_obj_init(&fbo->uvmobj, bo->bdev->driver->ttm_uvm_ops, true, 1); - mutex_obj_hold(bo->uvmobj.vmobjlock); + rw_obj_hold(bo->uvmobj.vmobjlock); uvm_obj_setlock(&fbo->uvmobj, bo->uvmobj.vmobjlock); #else mutex_init(&fbo->wu_mutex); Index: sys/external/bsd/drm2/include/linux/mm.h =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/include/linux/mm.h,v retrieving revision 1.12 diff -u -p -r1.12 mm.h --- sys/external/bsd/drm2/include/linux/mm.h 19 Jan 2020 15:31:56 -0000 1.12 +++ sys/external/bsd/drm2/include/linux/mm.h 21 Jan 2020 21:09:20 -0000 @@ -100,9 +100,9 @@ set_page_dirty(struct page *page) /* XXX */ if (pg->uobject != NULL) { - mutex_enter(pg->uobject->vmobjlock); + rw_enter(pg->uobject->vmobjlock, RW_WRITER); uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); - mutex_exit(pg->uobject->vmobjlock); + rw_exit(pg->uobject->vmobjlock); } else { uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); } Index: sys/external/bsd/drm2/ttm/ttm_bo_vm.c =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/ttm/ttm_bo_vm.c,v retrieving revision 1.12 diff -u -p -r1.12 ttm_bo_vm.c --- sys/external/bsd/drm2/ttm/ttm_bo_vm.c 9 Mar 2019 01:59:47 -0000 1.12 +++ sys/external/bsd/drm2/ttm/ttm_bo_vm.c 21 Jan 2020 21:09:20 -0000 @@ -93,7 +93,7 @@ ttm_bo_uvm_fault(struct uvm_faultinfo *u int ret; /* Thanks, uvm, but we don't need this lock. */ - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); /* Copy-on-write mappings make no sense for the graphics aperture. */ if (UVM_ET_ISCOPYONWRITE(ufi->entry)) { Index: sys/external/bsd/drm2/ttm/ttm_bus_dma.c =================================================================== RCS file: /cvsroot/src/sys/external/bsd/drm2/ttm/ttm_bus_dma.c,v retrieving revision 1.8 diff -u -p -r1.8 ttm_bus_dma.c --- sys/external/bsd/drm2/ttm/ttm_bus_dma.c 21 Apr 2019 15:49:50 -0000 1.8 +++ sys/external/bsd/drm2/ttm/ttm_bus_dma.c 21 Jan 2020 21:09:20 -0000 @@ -120,7 +120,7 @@ ttm_bus_dma_put(struct ttm_dma_tt *ttm_d KASSERT(uobj->pgops->pgo_put); /* Release or deactivate the pages. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); (void)(*uobj->pgops->pgo_put)(uobj, 0, size, flags); /* pgo_put unlocks uobj->vmobjlock. */ Index: sys/fs/puffs/puffs_msgif.c =================================================================== RCS file: /cvsroot/src/sys/fs/puffs/puffs_msgif.c,v retrieving revision 1.104 diff -u -p -r1.104 puffs_msgif.c --- sys/fs/puffs/puffs_msgif.c 1 Mar 2019 11:06:57 -0000 1.104 +++ sys/fs/puffs/puffs_msgif.c 21 Jan 2020 21:09:20 -0000 @@ -941,7 +941,7 @@ puffsop_flush(struct puffs_mount *pmp, s break; } - mutex_enter(vp->v_uobj.vmobjlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); rv = VOP_PUTPAGES(vp, offlo, offhi, flags); break; Index: sys/fs/puffs/puffs_vnops.c =================================================================== RCS file: /cvsroot/src/sys/fs/puffs/puffs_vnops.c,v retrieving revision 1.213 diff -u -p -r1.213 puffs_vnops.c --- sys/fs/puffs/puffs_vnops.c 6 Nov 2018 02:39:49 -0000 1.213 +++ sys/fs/puffs/puffs_vnops.c 21 Jan 2020 21:09:20 -0000 @@ -2528,7 +2528,7 @@ puffs_vnop_write(void *v) voff_t off_lo = trunc_page(origoff); voff_t off_hi = round_page(uio->uio_offset); - mutex_enter(vp->v_uobj.vmobjlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, off_lo, off_hi, PGO_FREE); } } @@ -2880,10 +2880,10 @@ puffs_vnop_strategy(void *v) DPRINTF(("puffs_strategy: write-protecting " "vp %p page %p, offset %" PRId64"\n", vp, vmp, vmp->offset)); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); vmp->flags |= PG_RDONLY; pmap_page_protect(vmp, VM_PROT_READ); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } } @@ -3116,7 +3116,7 @@ puffs_vnop_getpages(void *v) * when the page is actually write-faulted to. */ if (!locked) - mutex_enter(vp->v_uobj.vmobjlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); for (i = 0, si = 0, streakon = 0; i < npages; i++) { if (pgs[i] == NULL || pgs[i] == PGO_DONTCARE) { if (streakon && write) { @@ -3142,7 +3142,7 @@ puffs_vnop_getpages(void *v) si++; } if (!locked) - mutex_exit(vp->v_uobj.vmobjlock); + rw_exit(vp->v_uobj.vmobjlock); KASSERT(si <= (npages / 2) + 1); Index: sys/fs/tmpfs/tmpfs_subr.c =================================================================== RCS file: /cvsroot/src/sys/fs/tmpfs/tmpfs_subr.c,v retrieving revision 1.105 diff -u -p -r1.105 tmpfs_subr.c --- sys/fs/tmpfs/tmpfs_subr.c 18 Sep 2019 17:59:15 -0000 1.105 +++ sys/fs/tmpfs/tmpfs_subr.c 21 Jan 2020 21:09:20 -0000 @@ -106,14 +106,14 @@ static void tmpfs_dir_putseq(tmpfs_node_ static void tmpfs_init_vnode(struct vnode *vp, tmpfs_node_t *node) { - kmutex_t *slock; + krwlock_t *slock; KASSERT(node->tn_vnode == NULL); /* Share the interlock with the node. */ if (node->tn_type == VREG) { slock = node->tn_spec.tn_reg.tn_aobj->vmobjlock; - mutex_obj_hold(slock); + rw_obj_hold(slock); uvm_obj_setlock(&vp->v_uobj, slock); } @@ -934,11 +934,9 @@ tmpfs_reg_resize(struct vnode *vp, off_t * Free "backing store". */ if (newpages < oldpages) { - KASSERT(uobj->vmobjlock == vp->v_interlock); - - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); uao_dropswap_range(uobj, newpages, oldpages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); /* Decrease the used-memory counter. */ tmpfs_mem_decr(tmp, (oldpages - newpages) << PAGE_SHIFT); Index: sys/fs/tmpfs/tmpfs_vnops.c =================================================================== RCS file: /cvsroot/src/sys/fs/tmpfs/tmpfs_vnops.c,v retrieving revision 1.133 diff -u -p -r1.133 tmpfs_vnops.c --- sys/fs/tmpfs/tmpfs_vnops.c 26 May 2017 14:21:01 -0000 1.133 +++ sys/fs/tmpfs/tmpfs_vnops.c 21 Jan 2020 21:09:20 -0000 @@ -1164,14 +1164,14 @@ tmpfs_getpages(void *v) struct uvm_object *uobj; KASSERT(vp->v_type == VREG); - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_lock_held(vp->v_uobj.vmobjlock)); /* * Currently, PGO_PASTEOF is not supported. */ if (vp->v_size <= offset + (centeridx << PAGE_SHIFT)) { if ((flags & PGO_LOCKED) == 0) - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); return EINVAL; } @@ -1182,7 +1182,10 @@ tmpfs_getpages(void *v) if ((flags & PGO_LOCKED) != 0) return EBUSY; - if (vdead_check(vp, VDEAD_NOWAIT) != 0) + mutex_enter(vp->v_interlock); + error = vdead_check(vp, VDEAD_NOWAIT); + mutex_exit(vp->v_interlock); + if (error != 0) return ENOENT; node = VP_TO_TMPFS_NODE(vp); @@ -1211,7 +1214,7 @@ tmpfs_getpages(void *v) if (pgs) { memset(pgs, 0, sizeof(struct vm_pages *) * npages); } - KASSERT(vp->v_interlock == uobj->vmobjlock); + KASSERT(vp->v_uobj.vmobjlock == uobj->vmobjlock); error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, centeridx, access_type, advice, flags | PGO_ALLPAGES); @@ -1243,17 +1246,17 @@ tmpfs_putpages(void *v) struct uvm_object *uobj; int error; - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); if (vp->v_type != VREG) { - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); return 0; } node = VP_TO_TMPFS_NODE(vp); uobj = node->tn_spec.tn_reg.tn_aobj; - KASSERT(vp->v_interlock == uobj->vmobjlock); + KASSERT(vp->v_uobj.vmobjlock == uobj->vmobjlock); error = (*uobj->pgops->pgo_put)(uobj, offlo, offhi, flags); /* XXX mtime */ Index: sys/fs/union/union_subr.c =================================================================== RCS file: /cvsroot/src/sys/fs/union/union_subr.c,v retrieving revision 1.77 diff -u -p -r1.77 union_subr.c --- sys/fs/union/union_subr.c 28 Jan 2018 15:48:44 -0000 1.77 +++ sys/fs/union/union_subr.c 21 Jan 2020 21:09:20 -0000 @@ -232,9 +232,10 @@ union_newupper(struct union_node *un, st unlock_ap.a_desc = VDESC(vop_unlock); unlock_ap.a_vp = UNIONTOV(un); genfs_unlock(&unlock_ap); - /* Update union vnode interlock. */ - mutex_obj_hold(uppervp->v_interlock); - uvm_obj_setlock(&UNIONTOV(un)->v_uobj, uppervp->v_interlock); + /* Update union vnode interlock & vmobjlock. */ + vshareilock(UNIONTOV(un), uppervp); + rw_obj_hold(uppervp->v_uobj.vmobjlock); + uvm_obj_setlock(&UNIONTOV(un)->v_uobj, uppervp->v_uobj.vmobjlock); mutex_exit(&un->un_lock); if (ohash != nhash) { LIST_INSERT_HEAD(&uhashtbl[nhash], un, un_cache); @@ -572,8 +573,9 @@ union_loadvnode(struct mount *mp, struct if (svp->v_type == VCHR || svp->v_type == VBLK) spec_node_init(vp, svp->v_rdev); - mutex_obj_hold(svp->v_interlock); - uvm_obj_setlock(&vp->v_uobj, svp->v_interlock); + vshareilock(vp, svp); + rw_obj_hold(svp->v_uobj.vmobjlock); + uvm_obj_setlock(&vp->v_uobj, svp->v_uobj.vmobjlock); /* detect the root vnode (and aliases) */ if ((un->un_uppervp == um->um_uppervp) && Index: sys/kern/kern_synch.c =================================================================== RCS file: /cvsroot/src/sys/kern/kern_synch.c,v retrieving revision 1.336 diff -u -p -r1.336 kern_synch.c --- sys/kern/kern_synch.c 9 Jan 2020 16:35:03 -0000 1.336 +++ sys/kern/kern_synch.c 21 Jan 2020 21:09:20 -0000 @@ -1,7 +1,7 @@ /* $NetBSD: kern_synch.c,v 1.336 2020/01/09 16:35:03 ad Exp $ */ /*- - * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019 + * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019, 2020 * The NetBSD Foundation, Inc. * All rights reserved. * @@ -83,6 +83,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_synch.c #include #include #include +#include #include #include #include @@ -211,6 +212,45 @@ mtsleep(wchan_t ident, pri_t priority, c } /* + * XXXAD Temporary - for use of UVM only. Will go away once waits are + * interlocked by pg->interlock. To wake an LWP sleeping with this, you + * need to hold a write lock. + */ +int +rwtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, + krwlock_t *rw) +{ + struct lwp *l = curlwp; + sleepq_t *sq; + kmutex_t *mp; + int error; + krw_t op; + + KASSERT((l->l_pflag & LP_INTR) == 0); + KASSERT(ident != &lbolt); + + if (sleepq_dontsleep(l)) { + (void)sleepq_abort(NULL, (priority & PNORELOCK) != 0); + if ((priority & PNORELOCK) != 0) + rw_exit(rw); + return 0; + } + + l->l_kpriority = true; + sq = sleeptab_lookup(&sleeptab, ident, &mp); + sleepq_enter(sq, l, mp); + sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); + op = rw_write_held(rw) ? RW_WRITER : RW_READER; + rw_exit(rw); + error = sleepq_block(timo, priority & PCATCH); + + if ((priority & PNORELOCK) == 0) + rw_enter(rw, op); + + return error; +} + +/* * General sleep call for situations where a wake-up is not expected. */ int Index: sys/kern/sys_descrip.c =================================================================== RCS file: /cvsroot/src/sys/kern/sys_descrip.c,v retrieving revision 1.35 diff -u -p -r1.35 sys_descrip.c --- sys/kern/sys_descrip.c 15 Sep 2019 16:25:57 -0000 1.35 +++ sys/kern/sys_descrip.c 21 Jan 2020 21:09:20 -0000 @@ -740,7 +740,7 @@ do_posix_fadvise(int fd, off_t offset, o */ if (round_page(offset) < trunc_page(endoffset) && offset <= round_page(offset)) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, round_page(offset), trunc_page(endoffset), PGO_DEACTIVATE | PGO_CLEANIT); Index: sys/kern/vfs_subr.c =================================================================== RCS file: /cvsroot/src/sys/kern/vfs_subr.c,v retrieving revision 1.479 diff -u -p -r1.479 vfs_subr.c --- sys/kern/vfs_subr.c 17 Jan 2020 20:08:09 -0000 1.479 +++ sys/kern/vfs_subr.c 21 Jan 2020 21:09:20 -0000 @@ -155,7 +155,7 @@ vinvalbuf(struct vnode *vp, int flags, k (flags & V_SAVE ? PGO_CLEANIT | PGO_RECLAIM : 0); /* XXXUBC this doesn't look at flags or slp* */ - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, 0, 0, flushflags); if (error) { return error; @@ -234,7 +234,7 @@ vtruncbuf(struct vnode *vp, daddr_t lbn, voff_t off; off = round_page((voff_t)lbn << vp->v_mount->mnt_fs_bshift); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, off, 0, PGO_FREE | PGO_SYNCIO); if (error) { return error; @@ -292,7 +292,7 @@ vflushbuf(struct vnode *vp, int flags) pflags = PGO_CLEANIT | PGO_ALLPAGES | (sync ? PGO_SYNCIO : 0) | ((flags & FSYNC_LAZY) ? PGO_LAZY : 0); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); (void) VOP_PUTPAGES(vp, 0, 0, pflags); loop: Index: sys/kern/vfs_vnode.c =================================================================== RCS file: /cvsroot/src/sys/kern/vfs_vnode.c,v retrieving revision 1.107 diff -u -p -r1.107 vfs_vnode.c --- sys/kern/vfs_vnode.c 12 Jan 2020 17:49:17 -0000 1.107 +++ sys/kern/vfs_vnode.c 21 Jan 2020 21:09:20 -0000 @@ -403,9 +403,10 @@ vnalloc_marker(struct mount *mp) vip = pool_cache_get(vcache_pool, PR_WAITOK); memset(vip, 0, sizeof(*vip)); vp = VIMPL_TO_VNODE(vip); - uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0); + uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1); vp->v_mount = mp; vp->v_type = VBAD; + vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); vip->vi_state = VS_MARKER; return vp; @@ -421,6 +422,7 @@ vnfree_marker(vnode_t *vp) vip = VNODE_TO_VIMPL(vp); KASSERT(vip->vi_state == VS_MARKER); + mutex_obj_free(vp->v_interlock); uvm_obj_destroy(&vp->v_uobj, true); pool_cache_put(vcache_pool, vip); } @@ -808,7 +810,7 @@ vrelel(vnode_t *vp, int flags, int lktyp cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages); cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages); } - vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP); + vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP); /* XXXAD vmobjlock */ vp->v_vflag &= ~VV_MAPPED; /* @@ -1140,15 +1142,17 @@ vcache_alloc(void) vnode_t *vp; vip = pool_cache_get(vcache_pool, PR_WAITOK); + vp = VIMPL_TO_VNODE(vip); memset(vip, 0, sizeof(*vip)); vip->vi_lock = rw_obj_alloc(); + vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); + /* SLIST_INIT(&vip->vi_hash); */ TAILQ_INIT(&vip->vi_nclist); /* LIST_INIT(&vip->vi_dnclist); */ - vp = VIMPL_TO_VNODE(vip); - uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0); + uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1); cv_init(&vp->v_cv, "vnode"); vp->v_usecount = 1; @@ -1206,6 +1210,7 @@ vcache_free(vnode_impl_t *vip) if (vp->v_type == VBLK || vp->v_type == VCHR) spec_node_destroy(vp); + mutex_obj_free(vp->v_interlock); rw_obj_free(vip->vi_lock); uvm_obj_destroy(&vp->v_uobj, true); cv_destroy(&vp->v_cv); @@ -1576,7 +1581,7 @@ vcache_reclaim(vnode_t *vp) temp_key_len = vip->vi_key.vk_key_len; /* * Prevent the vnode from being recycled or brought into use - * while we clean it out. + * while we clean it out. XXXAD vmobjlock */ VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING); if ((vp->v_iflag & VI_EXECMAP) != 0 && vp->v_uobj.uo_npages != 0) { @@ -1826,3 +1831,17 @@ vnpanic(vnode_t *vp, const char *fmt, .. vpanic(fmt, ap); va_end(ap); } + +/* XXXAD crummy name, should do one for vnode locks too - layer */ +void +vshareilock(vnode_t *dvp, vnode_t *svp) +{ + kmutex_t *oldlock; + + oldlock = dvp->v_interlock; + mutex_obj_hold(svp->v_interlock); + dvp->v_interlock = svp->v_interlock; + mutex_obj_free(oldlock); +} + + Index: sys/kern/vfs_vnops.c =================================================================== RCS file: /cvsroot/src/sys/kern/vfs_vnops.c,v retrieving revision 1.205 diff -u -p -r1.205 vfs_vnops.c --- sys/kern/vfs_vnops.c 12 Jan 2020 18:37:10 -0000 1.205 +++ sys/kern/vfs_vnops.c 21 Jan 2020 21:09:20 -0000 @@ -1164,33 +1164,6 @@ vn_extattr_rm(struct vnode *vp, int iofl return (error); } -void -vn_ra_allocctx(struct vnode *vp) -{ - struct uvm_ractx *ra = NULL; - - KASSERT(mutex_owned(vp->v_interlock)); - - if (vp->v_type != VREG) { - return; - } - if (vp->v_ractx != NULL) { - return; - } - if (vp->v_ractx == NULL) { - mutex_exit(vp->v_interlock); - ra = uvm_ra_allocctx(); - mutex_enter(vp->v_interlock); - if (ra != NULL && vp->v_ractx == NULL) { - vp->v_ractx = ra; - ra = NULL; - } - } - if (ra != NULL) { - uvm_ra_freectx(ra); - } -} - int vn_fifo_bypass(void *v) { Index: sys/miscfs/genfs/genfs_io.c =================================================================== RCS file: /cvsroot/src/sys/miscfs/genfs/genfs_io.c,v retrieving revision 1.84 diff -u -p -r1.84 genfs_io.c --- sys/miscfs/genfs/genfs_io.c 15 Jan 2020 17:55:44 -0000 1.84 +++ sys/miscfs/genfs/genfs_io.c 21 Jan 2020 21:09:20 -0000 @@ -75,7 +75,7 @@ genfs_rel_pages(struct vm_page **pgs, un if (pg == NULL || pg == PGO_DONTCARE) continue; - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); if (pg->flags & PG_FAKE) { pg->flags |= PG_RELEASED; } @@ -87,13 +87,16 @@ static void genfs_markdirty(struct vnode *vp) { - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_lock_held(vp->v_uobj.vmobjlock)); + + mutex_enter(vp->v_interlock); if ((vp->v_iflag & VI_ONWORKLST) == 0) { vn_syncer_add_to_worklist(vp, filedelay); } if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { vp->v_iflag |= VI_WRMAPDIRTY; } + mutex_exit(vp->v_interlock); } /* @@ -144,10 +147,12 @@ genfs_getpages(void *v) WAPBL_JLOCK_ASSERT(vp->v_mount); #endif + mutex_enter(vp->v_interlock); error = vdead_check(vp, VDEAD_NOWAIT); + mutex_exit(vp->v_interlock); if (error) { if ((flags & PGO_LOCKED) == 0) - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return error; } @@ -187,7 +192,7 @@ startover: if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { if ((flags & PGO_LOCKED) == 0) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } UVMHIST_LOG(ubchist, "off 0x%jx count %jd goes past EOF 0x%jx", origoffset, *ap->a_count, memeof,0); @@ -277,7 +282,7 @@ startover: } goto out_err; } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); /* * find the requested pages and make some simple checks. @@ -356,7 +361,7 @@ startover: genfs_node_rdlock(vp); } } - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); if (vp->v_size < origvsize) { if (!glocked) { genfs_node_unlock(vp); @@ -373,7 +378,7 @@ startover: } KASSERT(async != 0); genfs_rel_pages(&pgs[ridx], orignmempages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); error = EBUSY; goto out_err_free; } @@ -463,13 +468,13 @@ startover: } KASSERT(async != 0); genfs_rel_pages(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); error = EBUSY; goto out_err_free; } } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); error = genfs_getpages_read(vp, pgs, npages, startoffset, diskeof, async, memwrite, blockalloc, glocked); if (!glocked) { @@ -477,7 +482,7 @@ startover: } if (error == 0 && async) goto out_err_free; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); /* * we're almost done! release the pages... @@ -488,7 +493,7 @@ startover: if (error) { genfs_rel_pages(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); UVMHIST_LOG(ubchist, "returning error %jd", error,0,0,0); goto out_err_free; } @@ -547,7 +552,7 @@ out: if (memwrite) { genfs_markdirty(vp); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); if (ap->a_m != NULL) { memcpy(ap->a_m, &pgs[ridx], orignmempages * sizeof(struct vm_page *)); @@ -737,11 +742,11 @@ genfs_getpages_read(struct vnode *vp, st skipbytes += iobytes; if (!blockalloc) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < holepages; i++) { pgs[pidx + i]->flags |= PG_RDONLY; } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } continue; } @@ -800,7 +805,7 @@ loopdone: UVMHIST_LOG(ubchist, "gop_alloc off 0x%jx/0x%jx -> %jd", startoffset, npages << PAGE_SHIFT, error,0); if (!error) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { struct vm_page *pg = pgs[i]; @@ -812,7 +817,7 @@ loopdone: UVMHIST_LOG(ubchist, "mark dirty pg %#jx", (uintptr_t)pg, 0, 0, 0); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } } @@ -867,7 +872,7 @@ genfs_do_putpages(struct vnode *vp, off_ int origflags, struct vm_page **busypg) { struct uvm_object * const uobj = &vp->v_uobj; - kmutex_t * const slock = uobj->vmobjlock; + krwlock_t * const slock = uobj->vmobjlock; off_t nextoff; int i, error, npages, nback; int freeflag; @@ -896,6 +901,7 @@ genfs_do_putpages(struct vnode *vp, off_ KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); KASSERT(startoff < endoff || endoff == 0); + KASSERT(rw_write_held(slock)); UVMHIST_LOG(ubchist, "vp %#jx pages %jd off 0x%jx len 0x%jx", (uintptr_t)vp, uobj->uo_npages, startoff, endoff - startoff); @@ -921,17 +927,19 @@ retry: if (uobj->uo_npages == 0 || (dirtyonly && radix_tree_empty_tagged_tree_p(&uobj->uo_pages, UVM_PAGE_DIRTY_TAG))) { + mutex_enter(vp->v_interlock); if (vp->v_iflag & VI_ONWORKLST) { vp->v_iflag &= ~VI_WRMAPDIRTY; if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) vn_syncer_remove_from_worklist(vp); } + mutex_exit(vp->v_interlock); if (trans_mp) { if (holds_wapbl) WAPBL_END(trans_mp); fstrans_done(trans_mp); } - mutex_exit(slock); + rw_exit(slock); return (0); } @@ -945,7 +953,7 @@ retry: trans_mp = vp->v_mount; error = fstrans_start_nowait(trans_mp); if (error) { - mutex_exit(slock); + rw_exit(slock); return error; } } else { @@ -954,7 +962,7 @@ retry: * usually gets used from VOP_RECLAIM(). Test for * change of v_mount instead and retry on change. */ - mutex_exit(slock); + rw_exit(slock); trans_mp = vp->v_mount; fstrans_start(trans_mp); if (vp->v_mount != trans_mp) { @@ -971,33 +979,19 @@ retry: } } } - mutex_enter(slock); + rw_enter(slock, RW_WRITER); goto retry; } } error = 0; - wasclean = (vp->v_numoutput == 0); + wasclean = (vp->v_numoutput == 0); /* XXXAD racy, think about it. */ nextoff = startoff; if (endoff == 0 || flags & PGO_ALLPAGES) { endoff = trunc_page(LLONG_MAX); } /* - * if this vnode is known not to have dirty pages, - * don't bother to clean it out. - */ - - if ((vp->v_iflag & VI_ONWORKLST) == 0) { -#if !defined(DEBUG) - if (dirtyonly) { - goto skip_scan; - } -#endif /* !defined(DEBUG) */ - flags &= ~PGO_CLEANIT; - } - - /* * start the loop to scan pages. */ @@ -1043,14 +1037,14 @@ retry: if ((l->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) != 0) { nextoff = pg->offset; /* visit this page again */ - mutex_exit(slock); + rw_exit(slock); preempt(); /* * as we dropped the object lock, our cached pages can * be stale. */ uvm_page_array_clear(&a); - mutex_enter(slock); + rw_enter(slock, RW_WRITER); continue; } @@ -1089,13 +1083,13 @@ retry: } nextoff = pg->offset; /* visit this page again */ pg->flags |= PG_WANTED; - UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); + UVM_UNLOCK_AND_WAIT_RW(pg, slock, 0, "genput", 0); /* * as we dropped the object lock, our cached pages can * be stale. */ uvm_page_array_clear(&a); - mutex_enter(slock); + rw_enter(slock, RW_WRITER); continue; } @@ -1292,14 +1286,14 @@ retry: /* * start the i/o. */ - mutex_exit(slock); + rw_exit(slock); error = GOP_WRITE(vp, pgs, npages, flags); /* * as we dropped the object lock, our cached pages can * be stale. */ uvm_page_array_clear(&a); - mutex_enter(slock); + rw_enter(slock, RW_WRITER); if (error) { break; } @@ -1317,6 +1311,7 @@ retry: * might not involve any page faults. */ + mutex_enter(vp->v_interlock); if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && (vp->v_type != VBLK || (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { @@ -1336,17 +1331,14 @@ retry: vn_syncer_remove_from_worklist(vp); } -#if !defined(DEBUG) -skip_scan: -#endif /* !defined(DEBUG) */ - /* Wait for output to complete. */ + rw_exit(slock); if (!wasclean && !async && vp->v_numoutput != 0) { while (vp->v_numoutput != 0) - cv_wait(&vp->v_cv, slock); + cv_wait(&vp->v_cv, vp->v_interlock); } onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; - mutex_exit(slock); + mutex_exit(vp->v_interlock); if ((flags & PGO_RECLAIM) != 0 && onworklst) { /* @@ -1354,7 +1346,7 @@ skip_scan: * retrying is not a big deal because, in many cases, * uobj->uo_npages is already 0 here. */ - mutex_enter(slock); + rw_enter(slock, RW_WRITER); goto retry; } @@ -1639,16 +1631,16 @@ genfs_compat_getpages(void *v) return error; } if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return EINVAL; } if ((ap->a_flags & PGO_SYNCIO) == 0) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return 0; } npages = orignpages; uvn_findpages(uobj, origoffset, &npages, pgs, NULL, UFP_ALL); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); kva = uvm_pagermapin(pgs, npages, UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); for (i = 0; i < npages; i++) { @@ -1674,7 +1666,7 @@ genfs_compat_getpages(void *v) } } uvm_pagermapout(kva, npages); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { pg = pgs[i]; if (error && (pg->flags & PG_FAKE) != 0) { @@ -1692,7 +1684,7 @@ genfs_compat_getpages(void *v) if (error == 0 && memwrite) { genfs_markdirty(vp); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return error; } Index: sys/miscfs/genfs/genfs_vnops.c =================================================================== RCS file: /cvsroot/src/sys/miscfs/genfs/genfs_vnops.c,v retrieving revision 1.200 diff -u -p -r1.200 genfs_vnops.c --- sys/miscfs/genfs/genfs_vnops.c 1 Dec 2019 13:56:29 -0000 1.200 +++ sys/miscfs/genfs/genfs_vnops.c 21 Jan 2020 21:09:20 -0000 @@ -453,7 +453,7 @@ genfs_null_putpages(void *v) struct vnode *vp = ap->a_vp; KASSERT(vp->v_uobj.uo_npages == 0); - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); return (0); } Index: sys/miscfs/genfs/layer_vfsops.c =================================================================== RCS file: /cvsroot/src/sys/miscfs/genfs/layer_vfsops.c,v retrieving revision 1.53 diff -u -p -r1.53 layer_vfsops.c --- sys/miscfs/genfs/layer_vfsops.c 17 Jan 2020 20:08:09 -0000 1.53 +++ sys/miscfs/genfs/layer_vfsops.c 21 Jan 2020 21:09:20 -0000 @@ -205,9 +205,10 @@ layerfs_loadvnode(struct mount *mp, stru xp = kmem_alloc(lmp->layerm_size, KM_SLEEP); - /* Share the interlock with the lower node. */ - mutex_obj_hold(lowervp->v_interlock); - uvm_obj_setlock(&vp->v_uobj, lowervp->v_interlock); + /* Share the interlock and vmobjlock with the lower node. */ + vshareilock(vp, lowervp); + rw_obj_hold(lowervp->v_uobj.vmobjlock); + uvm_obj_setlock(&vp->v_uobj, lowervp->v_uobj.vmobjlock); vp->v_tag = lmp->layerm_tag; vp->v_type = lowervp->v_type; Index: sys/nfs/nfs_bio.c =================================================================== RCS file: /cvsroot/src/sys/nfs/nfs_bio.c,v retrieving revision 1.193 diff -u -p -r1.193 nfs_bio.c --- sys/nfs/nfs_bio.c 15 Jan 2020 17:55:44 -0000 1.193 +++ sys/nfs/nfs_bio.c 21 Jan 2020 21:09:20 -0000 @@ -988,7 +988,7 @@ again: /* * this page belongs to our object. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); /* * write out the page stably if it's about to * be released because we can't resend it @@ -1005,19 +1005,19 @@ again: */ if ((pgs[i]->flags & PG_NEEDCOMMIT) == 0) needcommit = false; - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } else { iomode = NFSV3WRITE_FILESYNC; needcommit = false; } } if (!needcommit && iomode == NFSV3WRITE_UNSTABLE) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { pgs[i]->flags |= PG_NEEDCOMMIT | PG_RDONLY; pmap_page_protect(pgs[i], VM_PROT_READ); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); pageprotected = true; /* pages can't be modified during i/o. */ } else pageprotected = false; @@ -1069,11 +1069,11 @@ again: * pages are now on stable storage. */ uiop->uio_resid = 0; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { pgs[i]->flags &= ~(PG_NEEDCOMMIT | PG_RDONLY); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); goto out; } else if (error == NFSERR_STALEWRITEVERF) { nfs_clearcommit(vp->v_mount); @@ -1118,12 +1118,12 @@ again: * re-dirty pages so that they will be passed * to us later again. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { uvm_pagemarkdirty(pgs[i], UVM_PAGE_STATUS_DIRTY); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } mutex_exit(&np->n_commitlock); } else @@ -1135,11 +1135,11 @@ again: mutex_enter(&np->n_commitlock); nfs_del_committed_range(vp, off, cnt); mutex_exit(&np->n_commitlock); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { pgs[i]->flags &= ~(PG_NEEDCOMMIT | PG_RDONLY); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } else { /* * we got an error. @@ -1305,7 +1305,7 @@ nfs_getpages(void *v) if (!write && (np->n_flag & NMODIFIED) == 0 && pgs != NULL) { if (!locked) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); } for (i = 0; i < npages; i++) { pg = pgs[i]; @@ -1315,7 +1315,7 @@ nfs_getpages(void *v) pg->flags |= PG_RDONLY; } if (!locked) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } } if (!write) @@ -1354,7 +1354,7 @@ nfs_getpages(void *v) } np->n_flag |= NMODIFIED; if (!locked) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); } for (i = 0; i < npages; i++) { pg = pgs[i]; @@ -1364,7 +1364,7 @@ nfs_getpages(void *v) pg->flags &= ~(PG_NEEDCOMMIT | PG_RDONLY); } if (!locked) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } if (v3) { mutex_exit(&np->n_commitlock); Index: sys/nfs/nfs_clntsubs.c =================================================================== RCS file: /cvsroot/src/sys/nfs/nfs_clntsubs.c,v retrieving revision 1.3 diff -u -p -r1.3 nfs_clntsubs.c --- sys/nfs/nfs_clntsubs.c 3 Sep 2018 16:29:36 -0000 1.3 +++ sys/nfs/nfs_clntsubs.c 21 Jan 2020 21:09:20 -0000 @@ -285,7 +285,7 @@ nfs_loadattrcache(struct vnode **vpp, st np->n_flag |= NTRUNCDELAYED; } else { genfs_node_wrlock(vp); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); (void)VOP_PUTPAGES(vp, 0, 0, PGO_SYNCIO | PGO_CLEANIT | PGO_FREE | PGO_ALLPAGES); @@ -359,7 +359,7 @@ nfs_delayedtruncate(struct vnode *vp) if (np->n_flag & NTRUNCDELAYED) { np->n_flag &= ~NTRUNCDELAYED; genfs_node_wrlock(vp); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); (void)VOP_PUTPAGES(vp, 0, 0, PGO_SYNCIO | PGO_CLEANIT | PGO_FREE | PGO_ALLPAGES); uvm_vnp_setsize(vp, np->n_size); Index: sys/rump/librump/rumpkern/vm.c =================================================================== RCS file: /cvsroot/src/sys/rump/librump/rumpkern/vm.c,v retrieving revision 1.183 diff -u -p -r1.183 vm.c --- sys/rump/librump/rumpkern/vm.c 15 Jan 2020 17:55:44 -0000 1.183 +++ sys/rump/librump/rumpkern/vm.c 21 Jan 2020 21:09:20 -0000 @@ -160,7 +160,7 @@ uvm_pagealloc_strat(struct uvm_object *u { struct vm_page *pg; - KASSERT(uobj && mutex_owned(uobj->vmobjlock)); + KASSERT(uobj && rw_write_held(uobj->vmobjlock)); KASSERT(anon == NULL); pg = pool_cache_get(&pagecache, PR_NOWAIT); @@ -211,7 +211,7 @@ uvm_pagefree(struct vm_page *pg) struct uvm_object *uobj = pg->uobject; struct vm_page *pg2 __unused; - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); if (pg->flags & PG_WANTED) wakeup(pg); @@ -248,7 +248,7 @@ bool uvm_page_owner_locked_p(struct vm_page *pg) { - return mutex_owned(pg->uobject->vmobjlock); + return rw_write_held(pg->uobject->vmobjlock); } /* @@ -658,7 +658,7 @@ uvm_page_unbusy(struct vm_page **pgs, in int i; KASSERT(npgs > 0); - KASSERT(mutex_owned(pgs[0]->uobject->vmobjlock)); + KASSERT(rw_write_held(pgs[0]->uobject->vmobjlock)); for (i = 0; i < npgs; i++) { pg = pgs[i]; @@ -1063,16 +1063,16 @@ processpage(struct vm_page *pg, bool *lo struct uvm_object *uobj; uobj = pg->uobject; - if (mutex_tryenter(uobj->vmobjlock)) { + if (rw_tryenter(uobj->vmobjlock, RW_WRITER)) { if ((pg->flags & PG_BUSY) == 0) { mutex_exit(&vmpage_lruqueue_lock); uobj->pgops->pgo_put(uobj, pg->offset, pg->offset + PAGE_SIZE, PGO_CLEANIT|PGO_FREE); - KASSERT(!mutex_owned(uobj->vmobjlock)); + KASSERT(!rw_write_held(uobj->vmobjlock)); return true; } else { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } } else if (*lockrunning == false && ncpu > 1) { CPU_INFO_ITERATOR cii; Index: sys/rump/librump/rumpvfs/vm_vfs.c =================================================================== RCS file: /cvsroot/src/sys/rump/librump/rumpvfs/vm_vfs.c,v retrieving revision 1.36 diff -u -p -r1.36 vm_vfs.c --- sys/rump/librump/rumpvfs/vm_vfs.c 15 Jan 2020 17:55:44 -0000 1.36 +++ sys/rump/librump/rumpvfs/vm_vfs.c 21 Jan 2020 21:09:20 -0000 @@ -59,7 +59,7 @@ uvm_aio_aiodone(struct buf *bp) if (uobj == NULL) { uobj = pgs[i]->uobject; KASSERT(uobj != NULL); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); } else { KASSERT(uobj == pgs[i]->uobject); } @@ -71,18 +71,18 @@ uvm_aio_aiodone(struct buf *bp) pgs[i]->flags |= PG_RELEASED; } } - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); uvm_page_unbusy(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); uvm_pagermapout((vaddr_t)bp->b_data, npages); uvm_pageout_done(pageout); if (BUF_ISWRITE(bp) && (bp->b_cflags & BC_AGE) != 0) { - mutex_enter(bp->b_objlock); + rw_enter(bp->b_objlock, RW_WRITER); vwakeup(bp); - mutex_exit(bp->b_objlock); + rw_exit(bp->b_objlock); } putiobuf(bp); @@ -114,7 +114,7 @@ ubc_zerorange(struct uvm_object *uobj, o return; pgs = kmem_alloc(maxpages * sizeof(pgs), KM_SLEEP); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); while (len) { npages = MIN(maxpages, round_page(len) >> PAGE_SHIFT); memset(pgs, 0, npages * sizeof(struct vm_page *)); @@ -123,7 +123,7 @@ ubc_zerorange(struct uvm_object *uobj, o 0, PAGERFLAGS | PGO_PASTEOF); KASSERT(npages > 0); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { struct vm_page *pg; uint8_t *start; @@ -148,7 +148,7 @@ ubc_zerorange(struct uvm_object *uobj, o } uvm_page_unbusy(pgs, npages); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); kmem_free(pgs, maxpages * sizeof(pgs)); } @@ -178,7 +178,7 @@ ubc_uiomove(struct uvm_object *uobj, str if (flags & UBC_WRITE) prot |= VM_PROT_WRITE; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); do { npages = len2npages(uio->uio_offset, todo); memset(pgs, 0, pgalloc); @@ -187,7 +187,7 @@ ubc_uiomove(struct uvm_object *uobj, str if (rv) goto out; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { struct vm_page *pg; size_t xfersize; @@ -207,7 +207,7 @@ ubc_uiomove(struct uvm_object *uobj, str xfersize, uio); if (rv) { uvm_page_unbusy(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); goto out; } if (uio->uio_rw == UIO_WRITE) { @@ -218,7 +218,7 @@ ubc_uiomove(struct uvm_object *uobj, str } uvm_page_unbusy(pgs, npages); } while (todo); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); out: kmem_free(pgs, pgalloc); Index: sys/sys/proc.h =================================================================== RCS file: /cvsroot/src/sys/sys/proc.h,v retrieving revision 1.357 diff -u -p -r1.357 proc.h --- sys/sys/proc.h 12 Oct 2019 19:38:57 -0000 1.357 +++ sys/sys/proc.h 21 Jan 2020 21:09:20 -0000 @@ -505,6 +505,7 @@ void fixjobc(struct proc *, struct pgrp int tsleep(wchan_t, pri_t, const char *, int); int mtsleep(wchan_t, pri_t, const char *, int, kmutex_t *); +int rwtsleep(wchan_t, pri_t, const char *, int, krwlock_t *); void wakeup(wchan_t); int kpause(const char *, bool, int, kmutex_t *); void exit1(struct lwp *, int, int) __dead; Index: sys/sys/vnode.h =================================================================== RCS file: /cvsroot/src/sys/sys/vnode.h,v retrieving revision 1.286 diff -u -p -r1.286 vnode.h --- sys/sys/vnode.h 22 Dec 2019 19:47:34 -0000 1.286 +++ sys/sys/vnode.h 21 Jan 2020 21:09:20 -0000 @@ -123,45 +123,48 @@ LIST_HEAD(buflists, buf); * Reading or writing any of these items requires holding the appropriate * lock. Field markings and the corresponding locks: * - * : stable, reference to the vnode is required + * - stable, reference to the vnode is required + * b bufcache_lock * f vnode_free_list_lock, or vrele_lock for vrele_list * i v_interlock - * u locked by underlying filesystem + * i+b v_interlock + bufcache_lock to modify, either to inspect + * i+u v_interlock + v_uobj.vmobjlock to modify, either to inspect + * k locked by underlying filesystem (maybe kernel_lock) + * u v_uobj.vmobjlock * v vnode lock - * x v_interlock + bufcache_lock to modify, either to inspect * * Each underlying filesystem allocates its own private area and hangs * it from v_data. */ struct vnode { - struct uvm_object v_uobj; /* i: the VM object */ - kcondvar_t v_cv; /* i: synchronization */ - voff_t v_size; /* i: size of file */ - voff_t v_writesize; /* i: new size after write */ - int v_iflag; /* i: VI_* flags */ - int v_vflag; /* v: VV_* flags */ - int v_uflag; /* u: VU_* flags */ - int v_numoutput; /* i: # of pending writes */ - int v_writecount; /* i: ref count of writers */ - int v_holdcnt; /* i: page & buffer refs */ - struct mount *v_mount; /* v: ptr to vfs we are in */ - int (**v_op)(void *); /* :: vnode operations vector */ - struct buflists v_cleanblkhd; /* x: clean blocklist head */ - struct buflists v_dirtyblkhd; /* x: dirty blocklist head */ + struct uvm_object v_uobj; /* i the VM object */ + kmutex_t *v_interlock; /* - vnode interlock */ + kcondvar_t v_cv; /* i synchronization */ + voff_t v_size; /* i+u size of file */ + voff_t v_writesize; /* i+u new size after write */ + int v_iflag; /* i VI_* flags */ + int v_vflag; /* v VV_* flags */ + int v_uflag; /* k VU_* flags */ + int v_usecount; /* i reference count */ + int v_numoutput; /* i # of pending writes */ + int v_writecount; /* i ref count of writers */ + int v_holdcnt; /* i page & buffer refs */ + struct mount *v_mount; /* v ptr to vfs we are in */ + int (**v_op)(void *); /* - vnode operations vector */ + struct buflists v_cleanblkhd; /* i+b clean blocklist head */ + struct buflists v_dirtyblkhd; /* i+b dirty blocklist head */ union { - struct mount *vu_mountedhere;/* v: ptr to vfs (VDIR) */ - struct socket *vu_socket; /* v: unix ipc (VSOCK) */ - struct specnode *vu_specnode; /* v: device (VCHR, VBLK) */ - struct fifoinfo *vu_fifoinfo; /* v: fifo (VFIFO) */ - struct uvm_ractx *vu_ractx; /* i: read-ahead ctx (VREG) */ + struct mount *vu_mountedhere;/* v ptr to vfs (VDIR) */ + struct socket *vu_socket; /* v unix ipc (VSOCK) */ + struct specnode *vu_specnode; /* v device (VCHR, VBLK) */ + struct fifoinfo *vu_fifoinfo; /* v fifo (VFIFO) */ + struct uvm_ractx *vu_ractx; /* u read-ahead ctx (VREG) */ } v_un; - enum vtype v_type; /* :: vnode type */ - enum vtagtype v_tag; /* :: type of underlying data */ - void *v_data; /* :: private data for fs */ - struct klist v_klist; /* i: notes attached to vnode */ + enum vtype v_type; /* - vnode type */ + enum vtagtype v_tag; /* - type of underlying data */ + void *v_data; /* - private data for fs */ + struct klist v_klist; /* i notes attached to vnode */ }; -#define v_usecount v_uobj.uo_refs -#define v_interlock v_uobj.vmobjlock #define v_mountedhere v_un.vu_mountedhere #define v_socket v_un.vu_socket #define v_specnode v_un.vu_specnode @@ -185,6 +188,8 @@ typedef struct vnode vnode_t; /* * The second set are locked by vp->v_interlock. + * + * XXXAD tied up with vmobjlock. genfs ok. justify/fix use in reclaim/inactive path. */ #define VI_TEXT 0x00000100 /* vnode is a pure text prototype */ #define VI_EXECMAP 0x00000200 /* might have PROT_EXEC mappings */ @@ -518,6 +523,7 @@ void vwakeup(struct buf *); int vdead_check(struct vnode *, int); void vrevoke(struct vnode *); void vremfree(struct vnode *); +void vshareilock(struct vnode *, struct vnode *); int vcache_get(struct mount *, const void *, size_t, struct vnode **); int vcache_new(struct mount *, struct vnode *, struct vattr *, kauth_cred_t, void *, struct vnode **); @@ -547,7 +553,6 @@ int vn_extattr_get(struct vnode *, int, int vn_extattr_set(struct vnode *, int, int, const char *, size_t, const void *, struct lwp *); int vn_extattr_rm(struct vnode *, int, int, const char *, struct lwp *); -void vn_ra_allocctx(struct vnode *); int vn_fifo_bypass(void *); int vn_bdev_open(dev_t, struct vnode **, struct lwp *); int vn_bdev_openpath(struct pathbuf *pb, struct vnode **, struct lwp *); Index: sys/ufs/chfs/chfs_vnops.c =================================================================== RCS file: /cvsroot/src/sys/ufs/chfs/chfs_vnops.c,v retrieving revision 1.35 diff -u -p -r1.35 chfs_vnops.c --- sys/ufs/chfs/chfs_vnops.c 17 Jan 2020 20:08:10 -0000 1.35 +++ sys/ufs/chfs/chfs_vnops.c 21 Jan 2020 21:09:20 -0000 @@ -859,7 +859,7 @@ chfs_write(void *v) if (error) goto out; if (flags & B_SYNC) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_interlock, RW_WRITER); VOP_PUTPAGES(vp, trunc_page(osize & chmp->chm_fs_bmask), round_page(eob), @@ -954,7 +954,7 @@ chfs_write(void *v) */ if (!async && oldoff >> 16 != uio->uio_offset >> 16) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_interlock, RW_WRITER); error = VOP_PUTPAGES(vp, (oldoff >> 16) << 16, (uio->uio_offset >> 16) << 16, PGO_CLEANIT | PGO_JOURNALLOCKED); @@ -964,7 +964,7 @@ chfs_write(void *v) } out: if (error == 0 && ioflag & IO_SYNC) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_interlock, RW_WRITER); error = VOP_PUTPAGES(vp, trunc_page(origoff & chmp->chm_fs_bmask), round_page(chfs_blkroundup(chmp, uio->uio_offset)), Index: sys/ufs/ext2fs/ext2fs_readwrite.c =================================================================== RCS file: /cvsroot/src/sys/ufs/ext2fs/ext2fs_readwrite.c,v retrieving revision 1.75 diff -u -p -r1.75 ext2fs_readwrite.c --- sys/ufs/ext2fs/ext2fs_readwrite.c 13 Aug 2016 07:40:10 -0000 1.75 +++ sys/ufs/ext2fs/ext2fs_readwrite.c 21 Jan 2020 21:09:20 -0000 @@ -336,14 +336,14 @@ ext2fs_write(void *v) */ if (!async && oldoff >> 16 != uio->uio_offset >> 16) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, (oldoff >> 16) << 16, (uio->uio_offset >> 16) << 16, PGO_CLEANIT | PGO_LAZY); } } if (error == 0 && ioflag & IO_SYNC) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, trunc_page(oldoff), round_page(ext2_blkroundup(fs, uio->uio_offset)), PGO_CLEANIT | PGO_SYNCIO); Index: sys/ufs/ffs/ffs_alloc.c =================================================================== RCS file: /cvsroot/src/sys/ufs/ffs/ffs_alloc.c,v retrieving revision 1.164 diff -u -p -r1.164 ffs_alloc.c --- sys/ufs/ffs/ffs_alloc.c 14 Apr 2019 15:55:24 -0000 1.164 +++ sys/ufs/ffs/ffs_alloc.c 21 Jan 2020 21:09:20 -0000 @@ -213,7 +213,7 @@ ffs_alloc(struct inode *ip, daddr_t lbn, voff_t off = trunc_page(ffs_lblktosize(fs, lbn)); voff_t endoff = round_page(ffs_lblktosize(fs, lbn) + size); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); while (off < endoff) { pg = uvm_pagelookup(uobj, off); KASSERT((pg != NULL && pg->owner_tag != NULL && @@ -221,7 +221,7 @@ ffs_alloc(struct inode *ip, daddr_t lbn, pg->lowner == curlwp->l_lid)); off += PAGE_SIZE; } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } #endif @@ -333,14 +333,14 @@ ffs_realloccg(struct inode *ip, daddr_t voff_t off = trunc_page(ffs_lblktosize(fs, lbprev)); voff_t endoff = round_page(ffs_lblktosize(fs, lbprev) + osize); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); while (off < endoff) { pg = uvm_pagelookup(uobj, off); KASSERT(pg->owner == curproc->p_pid && pg->lowner == curlwp->l_lid); off += PAGE_SIZE; } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } #endif Index: sys/ufs/ffs/ffs_inode.c =================================================================== RCS file: /cvsroot/src/sys/ufs/ffs/ffs_inode.c,v retrieving revision 1.125 diff -u -p -r1.125 ffs_inode.c --- sys/ufs/ffs/ffs_inode.c 10 Dec 2018 19:29:41 -0000 1.125 +++ sys/ufs/ffs/ffs_inode.c 21 Jan 2020 21:09:20 -0000 @@ -279,7 +279,7 @@ ffs_truncate(struct vnode *ovp, off_t le return error; } if (ioflag & IO_SYNC) { - mutex_enter(ovp->v_interlock); + rw_enter(ovp->v_uobj.vmobjlock, RW_WRITER); VOP_PUTPAGES(ovp, trunc_page(osize & fs->fs_bmask), round_page(eob), PGO_CLEANIT | PGO_SYNCIO | @@ -331,7 +331,7 @@ ffs_truncate(struct vnode *ovp, off_t le ubc_zerorange(&ovp->v_uobj, length, eoz - length, UBC_UNMAP_FLAG(ovp)); if (round_page(eoz) > round_page(length)) { - mutex_enter(ovp->v_interlock); + rw_enter(ovp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(ovp, round_page(length), round_page(eoz), PGO_CLEANIT | PGO_DEACTIVATE | PGO_JOURNALLOCKED | Index: sys/ufs/ffs/ffs_snapshot.c =================================================================== RCS file: /cvsroot/src/sys/ufs/ffs/ffs_snapshot.c,v retrieving revision 1.150 diff -u -p -r1.150 ffs_snapshot.c --- sys/ufs/ffs/ffs_snapshot.c 17 Jan 2020 20:08:10 -0000 1.150 +++ sys/ufs/ffs/ffs_snapshot.c 21 Jan 2020 21:09:20 -0000 @@ -334,7 +334,7 @@ ffs_snapshot(struct mount *mp, struct vn * Invalidate and free all pages on the snapshot vnode. * We will read and write through the buffercache. */ - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, 0, 0, PGO_ALLPAGES | PGO_CLEANIT | PGO_SYNCIO | PGO_FREE); if (error) Index: sys/ufs/ffs/ffs_vfsops.c =================================================================== RCS file: /cvsroot/src/sys/ufs/ffs/ffs_vfsops.c,v retrieving revision 1.363 diff -u -p -r1.363 ffs_vfsops.c --- sys/ufs/ffs/ffs_vfsops.c 17 Jan 2020 20:08:10 -0000 1.363 +++ sys/ufs/ffs/ffs_vfsops.c 21 Jan 2020 21:09:20 -0000 @@ -2400,7 +2400,7 @@ ffs_vfs_fsync(vnode_t *vp, int flags) pflags = PGO_ALLPAGES | PGO_CLEANIT; if ((flags & FSYNC_WAIT) != 0) pflags |= PGO_SYNCIO; - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, 0, 0, pflags); if (error) return error; Index: sys/ufs/ffs/ffs_vnops.c =================================================================== RCS file: /cvsroot/src/sys/ufs/ffs/ffs_vnops.c,v retrieving revision 1.129 diff -u -p -r1.129 ffs_vnops.c --- sys/ufs/ffs/ffs_vnops.c 26 May 2017 14:21:02 -0000 1.129 +++ sys/ufs/ffs/ffs_vnops.c 21 Jan 2020 21:09:20 -0000 @@ -358,7 +358,7 @@ ffs_fsync(void *v) * First, flush all pages in range. */ - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, trunc_page(ap->a_offlo), round_page(ap->a_offhi), PGO_CLEANIT | ((ap->a_flags & FSYNC_WAIT) ? PGO_SYNCIO : 0)); @@ -473,7 +473,7 @@ ffs_full_fsync(struct vnode *vp, int fla pflags |= PGO_LAZY; if ((flags & FSYNC_WAIT)) pflags |= PGO_SYNCIO; - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, 0, 0, pflags); if (error) return error; Index: sys/ufs/lfs/lfs_inode.c =================================================================== RCS file: /cvsroot/src/sys/ufs/lfs/lfs_inode.c,v retrieving revision 1.157 diff -u -p -r1.157 lfs_inode.c --- sys/ufs/lfs/lfs_inode.c 10 Jun 2017 05:29:36 -0000 1.157 +++ sys/ufs/lfs/lfs_inode.c 21 Jan 2020 21:09:20 -0000 @@ -280,7 +280,7 @@ lfs_truncate(struct vnode *ovp, off_t le return error; } if (ioflag & IO_SYNC) { - mutex_enter(ovp->v_interlock); + rw_enter(ovp->v_uobj.vmobjlock, RW_WRITER); VOP_PUTPAGES(ovp, trunc_page(osize & lfs_sb_getbmask(fs)), round_page(eob), @@ -400,7 +400,7 @@ lfs_truncate(struct vnode *ovp, off_t le ubc_zerorange(&ovp->v_uobj, length, eoz - length, UBC_UNMAP_FLAG(ovp)); if (round_page(eoz) > round_page(length)) { - mutex_enter(ovp->v_interlock); + rw_enter(ovp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(ovp, round_page(length), round_page(eoz), PGO_CLEANIT | PGO_DEACTIVATE | @@ -861,7 +861,7 @@ lfs_vtruncbuf(struct vnode *vp, daddr_t voff_t off; off = round_page((voff_t)lbn << vp->v_mount->mnt_fs_bshift); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, off, 0, PGO_FREE | PGO_SYNCIO); if (error) return error; Index: sys/ufs/lfs/lfs_pages.c =================================================================== RCS file: /cvsroot/src/sys/ufs/lfs/lfs_pages.c,v retrieving revision 1.20 diff -u -p -r1.20 lfs_pages.c --- sys/ufs/lfs/lfs_pages.c 15 Jan 2020 17:55:44 -0000 1.20 +++ sys/ufs/lfs/lfs_pages.c 21 Jan 2020 21:09:20 -0000 @@ -142,12 +142,12 @@ lfs_getpages(void *v) * Wait for a page to become unbusy, possibly printing diagnostic messages * as well. * - * Called with vp->v_interlock held; return with it held. + * Called with vp->v_uobj.vmobjlock held; return with it held. */ static void wait_for_page(struct vnode *vp, struct vm_page *pg, const char *label) { - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); if ((pg->flags & PG_BUSY) == 0) return; /* Nothing to wait for! */ @@ -168,8 +168,8 @@ wait_for_page(struct vnode *vp, struct v #endif pg->flags |= PG_WANTED; - UVM_UNLOCK_AND_WAIT(pg, vp->v_interlock, 0, "lfsput", 0); - mutex_enter(vp->v_interlock); + UVM_UNLOCK_AND_WAIT_RW(pg, vp->v_uobj.vmobjlock, 0, "lfsput", 0); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); } /* @@ -181,14 +181,14 @@ wait_for_page(struct vnode *vp, struct v * disk). We don't need to give up the segment lock, but we might need * to call lfs_writeseg() to expedite the page's journey to disk. * - * Called with vp->v_interlock held; return with it held. + * Called with vp->v_uobj.vmobjlock held; return with it held. */ /* #define BUSYWAIT */ static void write_and_wait(struct lfs *fs, struct vnode *vp, struct vm_page *pg, int seglocked, const char *label) { - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); #ifndef BUSYWAIT struct inode *ip = VTOI(vp); struct segment *sp = fs->lfs_sp; @@ -199,7 +199,7 @@ write_and_wait(struct lfs *fs, struct vn while (pg->flags & PG_BUSY && pg->uobject == &vp->v_uobj) { - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); if (sp->cbpp - sp->bpp > 1) { /* Write gathered pages */ lfs_updatemeta(sp); @@ -214,7 +214,7 @@ write_and_wait(struct lfs *fs, struct vn ip->i_gen); } ++count; - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); wait_for_page(vp, pg, label); } if (label != NULL && count > 1) { @@ -225,7 +225,7 @@ write_and_wait(struct lfs *fs, struct vn #else preempt(1); #endif - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); } /* @@ -253,7 +253,7 @@ check_dirty(struct lfs *fs, struct vnode int pages_per_block = lfs_sb_getbsize(fs) >> PAGE_SHIFT; int pagedaemon = (curlwp == uvm.pagedaemon_lwp); - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); ASSERT_MAYBE_SEGLOCK(fs); top: any_dirty = 0; @@ -268,7 +268,7 @@ check_dirty(struct lfs *fs, struct vnode */ nonexistent = dirty = 0; for (i = 0; i == 0 || i < pages_per_block; i++) { - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); off = soff + (i << PAGE_SHIFT); pgs[i] = pg = uvm_pagelookup(&vp->v_uobj, off); if (pg == NULL) { @@ -290,16 +290,16 @@ check_dirty(struct lfs *fs, struct vnode DLOG((DLOG_PAGE, "lfs_putpages: avoiding 3-way or pagedaemon deadlock\n")); if (pgp) *pgp = pg; - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); return -1; } while (pg->flags & PG_BUSY) { wait_for_page(vp, pg, NULL); - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); if (i > 0) uvm_page_unbusy(pgs, i); - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); goto top; } pg->flags |= PG_BUSY; @@ -320,7 +320,7 @@ check_dirty(struct lfs *fs, struct vnode any_dirty += dirty; KASSERT(nonexistent == 0); - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); /* * If any are dirty make all dirty; unbusy them, @@ -329,7 +329,7 @@ check_dirty(struct lfs *fs, struct vnode * they're on their way to disk. */ for (i = 0; i == 0 || i < pages_per_block; i++) { - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); pg = pgs[i]; KASSERT(!(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_DIRTY && (pg->flags & PG_DELWRI))); @@ -361,7 +361,7 @@ check_dirty(struct lfs *fs, struct vnode soff += MAX(PAGE_SIZE, lfs_sb_getbsize(fs)); } - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); return any_dirty; } @@ -386,7 +386,7 @@ check_dirty(struct lfs *fs, struct vnode * there is a danger that when we expand the page range and busy the * pages we will deadlock. * - * (2) We are called with vp->v_interlock held; we must return with it + * (2) We are called with vp->v_uobj.vmobjlock held; we must return with it * released. * * (3) We don't absolutely have to free pages right away, provided that @@ -443,11 +443,11 @@ lfs_putpages(void *v) pagedaemon = (curlwp == uvm.pagedaemon_lwp); trans_mp = NULL; - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); /* Putpages does nothing for metadata. */ if (vp == fs->lfs_ivnode || vp->v_type != VREG) { - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); return 0; } @@ -463,7 +463,7 @@ retry: } if (trans_mp) fstrans_done(trans_mp); - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); /* Remove us from paging queue, if we were on it */ mutex_enter(&lfs_lock); @@ -473,7 +473,7 @@ retry: } mutex_exit(&lfs_lock); - KASSERT(!mutex_owned(vp->v_interlock)); + KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); return 0; } @@ -496,9 +496,9 @@ retry: KASSERT(pg != NULL); while (pg->flags & PG_BUSY) { pg->flags |= PG_WANTED; - UVM_UNLOCK_AND_WAIT(pg, vp->v_interlock, 0, + UVM_UNLOCK_AND_WAIT_RW(pg, vp->v_uobj.vmobjlock, 0, "lfsput2", 0); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); } uvm_pagelock(pg); uvm_pageactivate(pg); @@ -506,7 +506,7 @@ retry: } ap->a_offlo = blkeof; if (ap->a_offhi > 0 && ap->a_offhi <= ap->a_offlo) { - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); return 0; } } @@ -532,7 +532,7 @@ retry: KASSERT(startoffset > 0 || endoffset >= startoffset); if (startoffset == endoffset) { /* Nothing to do, why were we called? */ - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); DLOG((DLOG_PAGE, "lfs_putpages: startoffset = endoffset = %" PRId64 "\n", startoffset)); return 0; @@ -549,7 +549,7 @@ retry: DLOG((DLOG_PAGE, "lfs_putpages: no cleanit vn %p ino %d (flags %x)\n", vp, (int)ip->i_number, ap->a_flags)); int r = genfs_putpages(v); - KASSERT(!mutex_owned(vp->v_interlock)); + KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); return r; } @@ -559,7 +559,7 @@ retry: trans_mp = vp->v_mount; error = fstrans_start_nowait(trans_mp); if (error) { - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); return error; } } else { @@ -568,7 +568,7 @@ retry: * usually gets used from VOP_RECLAIM(). Test for * change of v_mount instead and retry on change. */ - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); trans_mp = vp->v_mount; fstrans_start(trans_mp); if (vp->v_mount != trans_mp) { @@ -576,7 +576,7 @@ retry: trans_mp = NULL; } } - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); goto retry; } @@ -592,14 +592,14 @@ retry: #endif do { int r; - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); /* Count the number of dirty pages */ r = check_dirty(fs, vp, startoffset, endoffset, blkeof, ap->a_flags, 1, NULL); if (r < 0) { /* Pages are busy with another process */ - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); error = EDEADLK; goto out; } @@ -617,13 +617,13 @@ retry: ap->a_flags & ~PGO_SYNCIO, &busypg); ip->i_lfs_iflags &= ~LFSI_NO_GOP_WRITE; if (r != EDEADLK) { - KASSERT(!mutex_owned(vp->v_interlock)); + KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); error = r; goto out; } /* One of the pages was busy. Start over. */ - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); wait_for_page(vp, busypg, "dirtyclean"); #ifdef DEBUG ++debug_n_dirtyclean; @@ -647,7 +647,7 @@ retry: * get a nasty deadlock with lfs_flush_pchain(). */ if (pagedaemon) { - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); mutex_enter(&lfs_lock); if (!(ip->i_state & IN_PAGING)) { ip->i_state |= IN_PAGING; @@ -656,7 +656,7 @@ retry: cv_broadcast(&lfs_writerd_cv); mutex_exit(&lfs_lock); preempt(); - KASSERT(!mutex_owned(vp->v_interlock)); + KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); error = EWOULDBLOCK; goto out; } @@ -680,13 +680,13 @@ retry: } else { DLOG((DLOG_PAGE, "lfs_putpages: dirop inode not locked\n")); } - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); mutex_enter(&lfs_lock); lfs_flush_fs(fs, sync ? SEGM_SYNC : 0); mutex_exit(&lfs_lock); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); lfs_writer_leave(fs); /* @@ -719,13 +719,13 @@ retry: */ seglocked = (ap->a_flags & PGO_LOCKED) != 0; if (!seglocked) { - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); error = lfs_seglock(fs, SEGM_PROT | (sync ? SEGM_SYNC : 0)); if (error != 0) { - KASSERT(!mutex_owned(vp->v_interlock)); + KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); goto out; } - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); lfs_acquire_finfo(fs, ip->i_number, ip->i_gen); } sp = fs->lfs_sp; @@ -760,15 +760,15 @@ retry: #endif do { busypg = NULL; - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); if (check_dirty(fs, vp, startoffset, endoffset, blkeof, ap->a_flags, 0, &busypg) < 0) { write_and_wait(fs, vp, busypg, seglocked, NULL); if (!seglocked) { - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); lfs_release_finfo(fs); lfs_segunlock(fs); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); } sp->vp = NULL; goto get_seglock; @@ -788,9 +788,9 @@ retry: lfs_dtosn(fs, lfs_sb_getoffset(fs)))); if (oreclaim) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); write_and_wait(fs, vp, busypg, seglocked, "again"); - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); } else { if ((sp->seg_flags & SEGM_SINGLE) && lfs_sb_getcurseg(fs) != fs->lfs_startseg) @@ -809,10 +809,10 @@ retry: if (oreclaim && error == EAGAIN) { DLOG((DLOG_PAGE, "vp %p ino %d vi_flags %x a_flags %x avoiding vclean panic\n", vp, (int)ip->i_number, vp->v_iflag, ap->a_flags)); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); } if (error == EDEADLK) - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); } while (error == EDEADLK || (oreclaim && error == EAGAIN)); #ifdef DEBUG if (debug_n_again > TOOMANY) @@ -845,7 +845,7 @@ retry: * the FIP or unlock the segment lock. We're done. */ if (seglocked) { - KASSERT(!mutex_owned(vp->v_interlock)); + KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); goto out; } @@ -891,7 +891,7 @@ retry: out:; if (trans_mp) fstrans_done(trans_mp); - KASSERT(!mutex_owned(vp->v_interlock)); + KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); return error; } Index: sys/ufs/lfs/lfs_segment.c =================================================================== RCS file: /cvsroot/src/sys/ufs/lfs/lfs_segment.c,v retrieving revision 1.281 diff -u -p -r1.281 lfs_segment.c --- sys/ufs/lfs/lfs_segment.c 15 Jan 2020 17:55:44 -0000 1.281 +++ sys/ufs/lfs/lfs_segment.c 21 Jan 2020 21:09:20 -0000 @@ -885,7 +885,7 @@ lfs_writefile(struct lfs *fs, struct seg * everything we've got. */ if (!IS_FLUSHING(fs, vp)) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, 0, 0, PGO_CLEANIT | PGO_ALLPAGES | PGO_LOCKED); } Index: sys/ufs/lfs/lfs_vfsops.c =================================================================== RCS file: /cvsroot/src/sys/ufs/lfs/lfs_vfsops.c,v retrieving revision 1.369 diff -u -p -r1.369 lfs_vfsops.c --- sys/ufs/lfs/lfs_vfsops.c 17 Jan 2020 20:08:10 -0000 1.369 +++ sys/ufs/lfs/lfs_vfsops.c 21 Jan 2020 21:09:20 -0000 @@ -2053,11 +2053,11 @@ lfs_gop_write(struct vnode *vp, struct v pgs[i]->flags &= ~PG_DELWRI; pgs[i]->flags |= PG_PAGEOUT; uvm_pageout_start(1); - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); uvm_pagelock(pgs[i]); uvm_pageunwire(pgs[i]); uvm_pageunlock(pgs[i]); - mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); } } Index: sys/ufs/lfs/lfs_vnops.c =================================================================== RCS file: /cvsroot/src/sys/ufs/lfs/lfs_vnops.c,v retrieving revision 1.325 diff -u -p -r1.325 lfs_vnops.c --- sys/ufs/lfs/lfs_vnops.c 18 Sep 2019 17:59:15 -0000 1.325 +++ sys/ufs/lfs/lfs_vnops.c 21 Jan 2020 21:09:20 -0000 @@ -482,7 +482,7 @@ lfs_fsync(void *v) wait = (ap->a_flags & FSYNC_WAIT); do { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, trunc_page(ap->a_offlo), round_page(ap->a_offhi), PGO_CLEANIT | (wait ? PGO_SYNCIO : 0)); Index: sys/ufs/lfs/ulfs_inode.c =================================================================== RCS file: /cvsroot/src/sys/ufs/lfs/ulfs_inode.c,v retrieving revision 1.24 diff -u -p -r1.24 ulfs_inode.c --- sys/ufs/lfs/ulfs_inode.c 15 Jan 2020 17:55:44 -0000 1.24 +++ sys/ufs/lfs/ulfs_inode.c 21 Jan 2020 21:09:20 -0000 @@ -206,7 +206,7 @@ ulfs_balloc_range(struct vnode *vp, off_ len += delta; genfs_node_wrlock(vp); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); error = VOP_GETPAGES(vp, pagestart, pgs, &npages, 0, VM_PROT_WRITE, 0, PGO_SYNCIO | PGO_PASTEOF | PGO_NOBLOCKALLOC | PGO_NOTIMESTAMP | PGO_GLOCKHELD); @@ -233,7 +233,7 @@ ulfs_balloc_range(struct vnode *vp, off_ */ GOP_SIZE(vp, off + len, &eob, 0); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { KASSERT((pgs[i]->flags & PG_RELEASED) == 0); if (!error) { @@ -248,7 +248,7 @@ ulfs_balloc_range(struct vnode *vp, off_ uvm_pageunlock(pgs[i]); } uvm_page_unbusy(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); out: kmem_free(pgs, pgssize); Index: sys/ufs/lfs/ulfs_readwrite.c =================================================================== RCS file: /cvsroot/src/sys/ufs/lfs/ulfs_readwrite.c,v retrieving revision 1.25 diff -u -p -r1.25 ulfs_readwrite.c --- sys/ufs/lfs/ulfs_readwrite.c 20 Jun 2019 00:49:11 -0000 1.25 +++ sys/ufs/lfs/ulfs_readwrite.c 21 Jan 2020 21:09:20 -0000 @@ -303,7 +303,7 @@ WRITE(void *v) if (error) goto out; if (flags & B_SYNC) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); VOP_PUTPAGES(vp, trunc_page(osize & lfs_sb_getbmask(fs)), round_page(eob), PGO_CLEANIT | PGO_SYNCIO); @@ -398,7 +398,7 @@ WRITE(void *v) __USE(async); } if (error == 0 && ioflag & IO_SYNC) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, trunc_page(origoff & lfs_sb_getbmask(fs)), round_page(lfs_blkroundup(fs, uio->uio_offset)), PGO_CLEANIT | PGO_SYNCIO); Index: sys/ufs/ufs/ufs_inode.c =================================================================== RCS file: /cvsroot/src/sys/ufs/ufs/ufs_inode.c,v retrieving revision 1.108 diff -u -p -r1.108 ufs_inode.c --- sys/ufs/ufs/ufs_inode.c 15 Jan 2020 17:55:44 -0000 1.108 +++ sys/ufs/ufs/ufs_inode.c 21 Jan 2020 21:09:20 -0000 @@ -242,7 +242,7 @@ ufs_balloc_range(struct vnode *vp, off_t len += delta; genfs_node_wrlock(vp); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); error = VOP_GETPAGES(vp, pagestart, pgs, &npages, 0, VM_PROT_WRITE, 0, PGO_SYNCIO | PGO_PASTEOF | PGO_NOBLOCKALLOC | PGO_NOTIMESTAMP | PGO_GLOCKHELD); @@ -269,7 +269,7 @@ ufs_balloc_range(struct vnode *vp, off_t */ GOP_SIZE(vp, off + len, &eob, 0); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { KASSERT((pgs[i]->flags & PG_RELEASED) == 0); if (!error) { @@ -284,7 +284,7 @@ ufs_balloc_range(struct vnode *vp, off_t uvm_pageunlock(pgs[i]); } uvm_page_unbusy(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); out: kmem_free(pgs, pgssize); Index: sys/ufs/ufs/ufs_readwrite.c =================================================================== RCS file: /cvsroot/src/sys/ufs/ufs/ufs_readwrite.c,v retrieving revision 1.124 diff -u -p -r1.124 ufs_readwrite.c --- sys/ufs/ufs/ufs_readwrite.c 20 Jun 2019 00:52:05 -0000 1.124 +++ sys/ufs/ufs/ufs_readwrite.c 21 Jan 2020 21:09:20 -0000 @@ -338,7 +338,7 @@ WRITE(void *v) if (error) goto out; if (flags & B_SYNC) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); VOP_PUTPAGES(vp, trunc_page(osize & fs->fs_bmask), round_page(eob), PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED); @@ -431,7 +431,7 @@ WRITE(void *v) */ if (!async && oldoff >> 16 != uio->uio_offset >> 16) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, (oldoff >> 16) << 16, (uio->uio_offset >> 16) << 16, PGO_CLEANIT | PGO_JOURNALLOCKED | PGO_LAZY); @@ -440,7 +440,7 @@ WRITE(void *v) } } if (error == 0 && ioflag & IO_SYNC) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); error = VOP_PUTPAGES(vp, trunc_page(origoff & fs->fs_bmask), round_page(ufs_blkroundup(fs, uio->uio_offset)), PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED); Index: sys/uvm/uvm.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm.h,v retrieving revision 1.73 diff -u -p -r1.73 uvm.h --- sys/uvm/uvm.h 31 Dec 2019 22:42:51 -0000 1.73 +++ sys/uvm/uvm.h 21 Jan 2020 21:09:20 -0000 @@ -180,6 +180,12 @@ do { \ msg, timo, slock); \ } while (/*CONSTCOND*/ 0) +#define UVM_UNLOCK_AND_WAIT_RW(event, slock, intr, msg, timo) \ +do { \ + (void) rwtsleep(event, PVM | PNORELOCK | (intr ? PCATCH : 0), \ + msg, timo, slock); \ +} while (/*CONSTCOND*/ 0) + void uvm_kick_pdaemon(void); /* Index: sys/uvm/uvm_amap.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_amap.c,v retrieving revision 1.114 diff -u -p -r1.114 uvm_amap.c --- sys/uvm/uvm_amap.c 2 Jan 2020 02:00:35 -0000 1.114 +++ sys/uvm/uvm_amap.c 21 Jan 2020 21:09:20 -0000 @@ -152,7 +152,7 @@ amap_alloc1(int slots, int padslots, int const bool nowait = (flags & UVM_FLAG_NOWAIT) != 0; const km_flag_t kmflags = nowait ? KM_NOSLEEP : KM_SLEEP; struct vm_amap *amap; - kmutex_t *newlock, *oldlock; + krwlock_t *newlock, *oldlock; int totalslots; amap = pool_cache_get(&uvm_amap_cache, nowait ? PR_NOWAIT : PR_WAITOK); @@ -163,14 +163,14 @@ amap_alloc1(int slots, int padslots, int KASSERT(amap->am_nused == 0); /* Try to privatize the lock if currently shared. */ - if (mutex_obj_refcnt(amap->am_lock) > 1) { - newlock = mutex_obj_tryalloc(MUTEX_DEFAULT, IPL_NONE); + if (rw_obj_refcnt(amap->am_lock) > 1) { + newlock = rw_obj_tryalloc(); if (newlock != NULL) { oldlock = amap->am_lock; mutex_enter(&amap_list_lock); amap->am_lock = newlock; mutex_exit(&amap_list_lock); - mutex_obj_free(oldlock); + rw_obj_free(oldlock); } } @@ -262,12 +262,12 @@ amap_ctor(void *arg, void *obj, int flag struct vm_amap *amap = obj; if ((flags & PR_NOWAIT) != 0) { - amap->am_lock = mutex_obj_tryalloc(MUTEX_DEFAULT, IPL_NONE); + amap->am_lock = rw_obj_tryalloc(); if (amap->am_lock == NULL) { return ENOMEM; } } else { - amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); + amap->am_lock = rw_obj_alloc(); } amap->am_nused = 0; amap->am_flags = 0; @@ -293,7 +293,7 @@ amap_dtor(void *arg, void *obj) mutex_enter(&amap_list_lock); LIST_REMOVE(amap, am_list); mutex_exit(&amap_list_lock); - mutex_obj_free(amap->am_lock); + rw_obj_free(amap->am_lock); } /* @@ -373,7 +373,7 @@ amap_extend(struct vm_map_entry *entry, * there are some unused slots before us in the amap. */ - amap_lock(amap); + amap_lock(amap, RW_WRITER); KASSERT(amap_refs(amap) == 1); /* amap can't be shared */ AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */ AMAP_B2SLOT(slotadd, addsize); /* slots to add */ @@ -559,7 +559,7 @@ amap_extend(struct vm_map_entry *entry, } return ENOMEM; } - amap_lock(amap); + amap_lock(amap, RW_WRITER); KASSERT(amap->am_maxslot < slotneed); /* @@ -681,7 +681,7 @@ amap_share_protect(struct vm_map_entry * u_int slots, lcv, slot, stop; struct vm_anon *anon; - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); AMAP_B2SLOT(slots, (entry->end - entry->start)); stop = entry->aref.ar_pageoff + slots; @@ -733,7 +733,7 @@ amap_wipeout(struct vm_amap *amap) UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist); UVMHIST_LOG(maphist,"(amap=0x%#jx)", (uintptr_t)amap, 0,0,0); - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); KASSERT(amap->am_ref == 0); if (__predict_false(amap->am_flags & AMAP_SWAPOFF)) { @@ -801,7 +801,7 @@ amap_copy(struct vm_map *map, struct vm_ struct vm_amap *amap, *srcamap; struct vm_anon *tofree; u_int slots, lcv; - kmutex_t *oldlock; + krwlock_t *oldlock; vsize_t len; UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist); @@ -906,10 +906,10 @@ amap_copy(struct vm_map *map, struct vm_ mutex_enter(&amap_list_lock); amap->am_lock = srcamap->am_lock; mutex_exit(&amap_list_lock); - mutex_obj_hold(amap->am_lock); - mutex_obj_free(oldlock); + rw_obj_hold(amap->am_lock); + rw_obj_free(oldlock); - amap_lock(srcamap); + amap_lock(srcamap, RW_WRITER); /* * Re-check the reference count with the lock held. If it has @@ -1015,7 +1015,7 @@ amap_cow_now(struct vm_map *map, struct */ ReStart: - amap_lock(amap); + amap_lock(amap, RW_WRITER); for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { slot = amap->am_slots[lcv]; anon = amap->am_anon[slot]; @@ -1058,7 +1058,7 @@ ReStart: if (pg->flags & PG_BUSY) { pg->flags |= PG_WANTED; - UVM_UNLOCK_AND_WAIT(pg, amap->am_lock, false, + UVM_UNLOCK_AND_WAIT_RW(pg, amap->am_lock, false, "cownow", 0); goto ReStart; } @@ -1127,7 +1127,7 @@ amap_splitref(struct vm_aref *origref, s AMAP_B2SLOT(leftslots, offset); KASSERT(leftslots != 0); - amap_lock(amap); + amap_lock(amap, RW_WRITER); KASSERT(amap->am_nslot - origref->ar_pageoff - leftslots > 0); #ifdef UVM_AMAP_PPREF @@ -1154,7 +1154,7 @@ amap_pp_establish(struct vm_amap *amap, { const size_t sz = amap->am_maxslot * sizeof(*amap->am_ppref); - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); amap->am_ppref = kmem_zalloc(sz, KM_NOSLEEP); if (amap->am_ppref == NULL) { @@ -1181,7 +1181,7 @@ amap_pp_adjref(struct vm_amap *amap, int int stopslot, *ppref, lcv, prevlcv; int ref, len, prevref, prevlen; - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); stopslot = curslot + slotlen; ppref = amap->am_ppref; @@ -1252,7 +1252,7 @@ amap_wiperange(struct vm_amap *amap, int u_int lcv, stop, slotend; bool byanon; - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); /* * We can either traverse the amap by am_anon or by am_slots. @@ -1354,7 +1354,7 @@ amap_swap_off(int startslot, int endslot LIST_INSERT_AFTER(am, &marker_next, am_list); /* amap_list_lock prevents the lock pointer from changing. */ - if (!amap_lock_try(am)) { + if (!amap_lock_try(am, RW_WRITER)) { (void)kpause("amapswpo", false, 1, &amap_list_lock); am_next = LIST_NEXT(&marker_prev, am_list); if (am_next == &marker_next) { @@ -1388,7 +1388,7 @@ amap_swap_off(int startslot, int endslot am->am_flags |= AMAP_SWAPOFF; rv = uvm_anon_pagein(am, anon); - amap_lock(am); + amap_lock(am, RW_WRITER); am->am_flags &= ~AMAP_SWAPOFF; if (amap_refs(am) == 0) { @@ -1434,7 +1434,7 @@ amap_lookup(struct vm_aref *aref, vaddr_ u_int slot; UVMHIST_FUNC("amap_lookup"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_lock_held(amap->am_lock)); AMAP_B2SLOT(slot, offset); slot += aref->ar_pageoff; @@ -1463,7 +1463,7 @@ amap_lookups(struct vm_aref *aref, vaddr u_int slot; UVMHIST_FUNC("amap_lookups"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_lock_held(amap->am_lock)); AMAP_B2SLOT(slot, offset); slot += aref->ar_pageoff; @@ -1501,7 +1501,7 @@ amap_add(struct vm_aref *aref, vaddr_t o u_int slot; UVMHIST_FUNC("amap_add"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); KASSERT(anon->an_lock == amap->am_lock); AMAP_B2SLOT(slot, offset); @@ -1543,7 +1543,7 @@ amap_unadd(struct vm_aref *aref, vaddr_t u_int slot, ptr, last; UVMHIST_FUNC("amap_unadd"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); AMAP_B2SLOT(slot, offset); slot += aref->ar_pageoff; @@ -1575,7 +1575,7 @@ amap_adjref_anons(struct vm_amap *amap, struct vm_anon *tofree = NULL; #ifdef UVM_AMAP_PPREF - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); /* * We must establish the ppref array before changing am_ref @@ -1613,7 +1613,7 @@ amap_ref(struct vm_amap *amap, vaddr_t o { UVMHIST_FUNC("amap_ref"); UVMHIST_CALLED(maphist); - amap_lock(amap); + amap_lock(amap, RW_WRITER); if (flags & AMAP_SHARED) { amap->am_flags |= AMAP_SHARED; } @@ -1634,7 +1634,7 @@ amap_unref(struct vm_amap *amap, vaddr_t { UVMHIST_FUNC("amap_unref"); UVMHIST_CALLED(maphist); - amap_lock(amap); + amap_lock(amap, RW_WRITER); UVMHIST_LOG(maphist," amap=0x%#jx refs=%d, nused=%d", (uintptr_t)amap, amap->am_ref, amap->am_nused, 0); Index: sys/uvm/uvm_amap.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_amap.h,v retrieving revision 1.39 diff -u -p -r1.39 uvm_amap.h --- sys/uvm/uvm_amap.h 2 Jan 2020 02:00:35 -0000 1.39 +++ sys/uvm/uvm_amap.h 21 Jan 2020 21:09:20 -0000 @@ -151,7 +151,7 @@ bool amap_swap_off */ struct vm_amap { - kmutex_t *am_lock; /* lock [locks all vm_amap fields] */ + krwlock_t *am_lock; /* lock [locks all vm_amap fields] */ int am_ref; /* reference count */ int am_flags; /* flags */ int am_maxslot; /* max # of slots allocated */ @@ -251,10 +251,10 @@ struct vm_amap { */ #define amap_flags(AMAP) ((AMAP)->am_flags) -#define amap_lock(AMAP) mutex_enter((AMAP)->am_lock) -#define amap_lock_try(AMAP) mutex_tryenter((AMAP)->am_lock) +#define amap_lock(AMAP, OP) rw_enter((AMAP)->am_lock, (OP)) +#define amap_lock_try(AMAP, OP) rw_tryenter((AMAP)->am_lock, (OP)) #define amap_refs(AMAP) ((AMAP)->am_ref) -#define amap_unlock(AMAP) mutex_exit((AMAP)->am_lock) +#define amap_unlock(AMAP) rw_exit((AMAP)->am_lock) /* * if we enable PPREF, then we have a couple of extra functions that Index: sys/uvm/uvm_anon.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_anon.c,v retrieving revision 1.71 diff -u -p -r1.71 uvm_anon.c --- sys/uvm/uvm_anon.c 15 Jan 2020 17:55:45 -0000 1.71 +++ sys/uvm/uvm_anon.c 21 Jan 2020 21:09:20 -0000 @@ -108,7 +108,7 @@ uvm_anon_dispose(struct vm_anon *anon) UVMHIST_FUNC("uvm_anon_dispose"); UVMHIST_CALLED(maphist); UVMHIST_LOG(maphist,"(anon=0x%#jx)", (uintptr_t)anon, 0,0,0); - KASSERT(mutex_owned(anon->an_lock)); + KASSERT(rw_write_held(anon->an_lock)); /* * Dispose the page, if it is resident. @@ -139,7 +139,7 @@ uvm_anon_dispose(struct vm_anon *anon) pg->loan_count--; pg->uanon = NULL; mutex_exit(&pg->interlock); - mutex_exit(pg->uobject->vmobjlock); + rw_exit(pg->uobject->vmobjlock); } else { /* @@ -157,7 +157,7 @@ uvm_anon_dispose(struct vm_anon *anon) if (pg->flags & PG_BUSY) { pg->flags |= PG_RELEASED; - mutex_obj_hold(anon->an_lock); + rw_obj_hold(anon->an_lock); return false; } uvm_pagefree(pg); @@ -215,7 +215,7 @@ uvm_anon_freelst(struct vm_amap *amap, s UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); for (; anonlst != NULL; anonlst = next) { next = anonlst->an_link; @@ -250,8 +250,9 @@ struct vm_page * uvm_anon_lockloanpg(struct vm_anon *anon) { struct vm_page *pg; + krw_t op; - KASSERT(mutex_owned(anon->an_lock)); + KASSERT(rw_lock_held(anon->an_lock)); /* * loop while we have a resident page that has a non-zero loan count. @@ -270,7 +271,7 @@ uvm_anon_lockloanpg(struct vm_anon *anon * toggle our anon lock and try again */ - if (!mutex_tryenter(pg->uobject->vmobjlock)) { + if (!rw_tryenter(pg->uobject->vmobjlock, RW_WRITER)) { /* * someone locking the object has a chance to * lock us right now @@ -278,7 +279,11 @@ uvm_anon_lockloanpg(struct vm_anon *anon * XXX Better than yielding but inadequate. */ mutex_exit(&pg->interlock); - kpause("livelock", false, 1, anon->an_lock); + op = rw_write_held(anon->an_lock) + ? RW_WRITER : RW_READER; + rw_exit(anon->an_lock); + kpause("lkloanpg", false, 1, NULL); + rw_enter(anon->an_lock, op); continue; } } @@ -313,7 +318,7 @@ uvm_anon_pagein(struct vm_amap *amap, st struct vm_page *pg; struct uvm_object *uobj; - KASSERT(mutex_owned(anon->an_lock)); + KASSERT(rw_write_held(anon->an_lock)); KASSERT(anon->an_lock == amap->am_lock); /* @@ -323,7 +328,7 @@ uvm_anon_pagein(struct vm_amap *amap, st switch (uvmfault_anonget(NULL, amap, anon)) { case 0: /* Success - we have the page. */ - KASSERT(mutex_owned(anon->an_lock)); + KASSERT(rw_write_held(anon->an_lock)); break; case EIO: case ERESTART: @@ -360,9 +365,9 @@ uvm_anon_pagein(struct vm_amap *amap, st wakeup(pg); } - mutex_exit(anon->an_lock); + rw_exit(anon->an_lock); if (uobj) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } return false; } @@ -401,7 +406,7 @@ uvm_anon_release(struct vm_anon *anon) struct vm_page *pg = anon->an_page; bool success __diagused; - KASSERT(mutex_owned(anon->an_lock)); + KASSERT(rw_write_held(anon->an_lock)); KASSERT(pg != NULL); KASSERT((pg->flags & PG_RELEASED) != 0); KASSERT((pg->flags & PG_BUSY) != 0); @@ -415,9 +420,9 @@ uvm_anon_release(struct vm_anon *anon) /* dispose should succeed as no one can reach this anon anymore. */ success = uvm_anon_dispose(anon); KASSERT(success); - mutex_exit(anon->an_lock); + rw_exit(anon->an_lock); /* Note: extra reference is held for PG_RELEASED case. */ - mutex_obj_free(anon->an_lock); + rw_obj_free(anon->an_lock); anon->an_lock = NULL; uvm_anon_free(anon); } Index: sys/uvm/uvm_anon.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_anon.h,v retrieving revision 1.30 diff -u -p -r1.30 uvm_anon.h --- sys/uvm/uvm_anon.h 6 Aug 2011 17:25:03 -0000 1.30 +++ sys/uvm/uvm_anon.h 21 Jan 2020 21:09:20 -0000 @@ -45,7 +45,7 @@ */ struct vm_anon { - kmutex_t *an_lock; /* Lock for an_ref */ + krwlock_t *an_lock; /* Lock for an_ref */ union { uintptr_t au_ref; /* Reference count [an_lock] */ struct vm_anon *au_link; /* Link for deferred free */ Index: sys/uvm/uvm_aobj.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_aobj.c,v retrieving revision 1.134 diff -u -p -r1.134 uvm_aobj.c --- sys/uvm/uvm_aobj.c 15 Jan 2020 17:55:45 -0000 1.134 +++ sys/uvm/uvm_aobj.c 21 Jan 2020 21:09:20 -0000 @@ -292,7 +292,7 @@ uao_set_swslot(struct uvm_object *uobj, UVMHIST_LOG(pdhist, "aobj %#jx pageidx %jd slot %jd", (uintptr_t)aobj, pageidx, slot, 0); - KASSERT(mutex_owned(uobj->vmobjlock) || uobj->uo_refs == 0); + KASSERT(rw_write_held(uobj->vmobjlock) || uobj->uo_refs == 0); /* * if noswap flag is set, then we can't set a non-zero slot. @@ -365,9 +365,9 @@ uao_free(struct uvm_aobj *aobj) { struct uvm_object *uobj = &aobj->u_obj; - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); uao_dropswap_range(uobj, 0, 0); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); #if defined(VMSWAP) if (UAO_USES_SWHASH(aobj)) { @@ -412,7 +412,7 @@ struct uvm_object * uao_create(voff_t size, int flags) { static struct uvm_aobj kernel_object_store; - static kmutex_t kernel_object_lock __cacheline_aligned; + static krwlock_t kernel_object_lock __cacheline_aligned; static int kobj_alloced __diagused = 0; pgoff_t pages = round_page((uint64_t)size) >> PAGE_SHIFT; struct uvm_aobj *aobj; @@ -487,7 +487,7 @@ uao_create(voff_t size, int flags) uvm_obj_init(&aobj->u_obj, &aobj_pager, !kernobj, refs); if (__predict_false(kernobj)) { /* Initialisation only once, for UAO_FLAG_KERNOBJ. */ - mutex_init(&kernel_object_lock, MUTEX_DEFAULT, IPL_NONE); + rw_init(&kernel_object_lock); uvm_obj_setlock(&aobj->u_obj, &kernel_object_lock); } @@ -615,17 +615,17 @@ uao_detach(struct uvm_object *uobj) * the page itself. */ uvm_page_array_init(&a); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); while ((pg = uvm_page_array_fill_and_peek(&a, uobj, 0, 0, 0)) != NULL) { uvm_page_array_advance(&a); pmap_page_protect(pg, VM_PROT_NONE); if (pg->flags & PG_BUSY) { pg->flags |= PG_WANTED; - UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, false, + UVM_UNLOCK_AND_WAIT_RW(pg, uobj->vmobjlock, false, "uao_det", 0); uvm_page_array_clear(&a); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); continue; } uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT); @@ -667,7 +667,7 @@ uao_put(struct uvm_object *uobj, voff_t voff_t curoff; UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); if (flags & PGO_ALLPAGES) { start = 0; @@ -697,7 +697,7 @@ uao_put(struct uvm_object *uobj, voff_t */ if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return 0; } @@ -716,10 +716,10 @@ uao_put(struct uvm_object *uobj, voff_t if (pg->flags & PG_BUSY) { pg->flags |= PG_WANTED; - UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, 0, + UVM_UNLOCK_AND_WAIT_RW(pg, uobj->vmobjlock, 0, "uao_put", 0); uvm_page_array_clear(&a); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); continue; } uvm_page_array_advance(&a); @@ -772,7 +772,7 @@ uao_put(struct uvm_object *uobj, voff_t panic("%s: impossible", __func__); } } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); uvm_page_array_fini(&a); return 0; } @@ -946,11 +946,11 @@ gotpage: /* out of RAM? */ if (ptmp == NULL) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); UVMHIST_LOG(pdhist, "sleeping, ptmp == NULL\n",0,0,0,0); uvm_wait("uao_getpage"); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); continue; } @@ -968,9 +968,9 @@ gotpage: UVMHIST_LOG(pdhist, "sleeping, ptmp->flags 0x%jx\n", ptmp->flags,0,0,0); - UVM_UNLOCK_AND_WAIT(ptmp, uobj->vmobjlock, + UVM_UNLOCK_AND_WAIT_RW(ptmp, uobj->vmobjlock, false, "uao_get", 0); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); continue; } @@ -1027,9 +1027,9 @@ gotpage: * unlock object for i/o, relock when done. */ - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); error = uvm_swap_get(ptmp, swslot, PGO_SYNCIO); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); /* * I/O done. check for errors. @@ -1055,7 +1055,7 @@ gotpage: } uvm_pagefree(ptmp); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return error; } #else /* defined(VMSWAP) */ @@ -1090,7 +1090,7 @@ gotpage: */ done: - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0); return 0; } @@ -1153,9 +1153,9 @@ uao_swap_off(int startslot, int endslot) /* * Page in all pages in the swap slot range. */ - mutex_enter(aobj->u_obj.vmobjlock); + rw_enter(aobj->u_obj.vmobjlock, RW_WRITER); rv = uao_pagein(aobj, startslot, endslot); - mutex_exit(aobj->u_obj.vmobjlock); + rw_exit(aobj->u_obj.vmobjlock); /* Drop the reference of the current object. */ uao_detach(&aobj->u_obj); @@ -1269,7 +1269,7 @@ uao_pagein_page(struct uvm_aobj *aobj, i pg = NULL; npages = 1; - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); rv = uao_get(uobj, (voff_t)pageidx << PAGE_SHIFT, &pg, &npages, 0, VM_PROT_READ | VM_PROT_WRITE, 0, PGO_SYNCIO); @@ -1277,7 +1277,7 @@ uao_pagein_page(struct uvm_aobj *aobj, i * relock and finish up. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); switch (rv) { case 0: break; @@ -1333,7 +1333,7 @@ uao_dropswap_range(struct uvm_object *uo struct uvm_aobj *aobj = (struct uvm_aobj *)uobj; int swpgonlydelta = 0; - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); if (end == 0) { end = INT64_MAX; Index: sys/uvm/uvm_bio.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_bio.c,v retrieving revision 1.103 diff -u -p -r1.103 uvm_bio.c --- sys/uvm/uvm_bio.c 15 Jan 2020 17:55:45 -0000 1.103 +++ sys/uvm/uvm_bio.c 21 Jan 2020 21:09:20 -0000 @@ -234,7 +234,7 @@ ubc_fault_page(const struct uvm_faultinf int error; bool rdonly; - KASSERT(mutex_owned(pg->uobject->vmobjlock)); + KASSERT(rw_write_held(pg->uobject->vmobjlock)); if (pg->flags & PG_WANTED) { wakeup(pg); @@ -369,7 +369,7 @@ ubc_fault(struct uvm_faultinfo *ufi, vad again: memset(pgs, 0, sizeof (pgs)); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); UVMHIST_LOG(ubchist, "slot_offset 0x%jx writeoff 0x%jx writelen 0x%jx ", slot_offset, umap->writeoff, umap->writelen, 0); @@ -414,7 +414,7 @@ again: * which belong to underlying UVM object. In such case, lock is * shared amongst the objects. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; va < eva; i++, va += PAGE_SIZE) { struct vm_page *pg; @@ -433,14 +433,14 @@ again: * and perform uvm_wait(). Note: page will re-fault. */ pmap_update(ufi->orig_map->pmap); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); uvm_wait("ubc_fault"); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); } } /* Must make VA visible before the unlock. */ pmap_update(ufi->orig_map->pmap); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return 0; } @@ -489,7 +489,7 @@ ubc_alloc(struct uvm_object *uobj, voff_ slot_offset = (vaddr_t)(offset & ((voff_t)ubc_winsize - 1)); *lenp = MIN(*lenp, ubc_winsize - slot_offset); - mutex_enter(ubc_object.uobj.vmobjlock); + rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER); again: /* * The UVM object is already referenced. @@ -502,8 +502,9 @@ again: UBC_EVCNT_INCR(wincachemiss); umap = TAILQ_FIRST(UBC_QUEUE(offset)); if (umap == NULL) { - kpause("ubc_alloc", false, hz >> 2, - ubc_object.uobj.vmobjlock); + rw_exit(ubc_object.uobj.vmobjlock); + kpause("ubc_alloc", false, hz >> 2, NULL); + rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER); goto again; } @@ -521,11 +522,11 @@ again: */ if (umap->flags & UMAP_MAPPING_CACHED) { umap->flags &= ~UMAP_MAPPING_CACHED; - mutex_enter(oobj->vmobjlock); + rw_enter(oobj->vmobjlock, RW_WRITER); pmap_remove(pmap_kernel(), va, va + ubc_winsize); pmap_update(pmap_kernel()); - mutex_exit(oobj->vmobjlock); + rw_exit(oobj->vmobjlock); } LIST_REMOVE(umap, hash); LIST_REMOVE(umap, list); @@ -555,7 +556,7 @@ again: umap->refcount++; umap->advice = advice; - mutex_exit(ubc_object.uobj.vmobjlock); + rw_exit(ubc_object.uobj.vmobjlock); UVMHIST_LOG(ubchist, "umap %#jx refs %jd va %#jx flags 0x%jx", (uintptr_t)umap, umap->refcount, (uintptr_t)va, flags); @@ -572,7 +573,7 @@ again: UBC_EVCNT_INCR(faultbusy); again_faultbusy: - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); if (umap->flags & UMAP_MAPPING_CACHED) { umap->flags &= ~UMAP_MAPPING_CACHED; pmap_remove(pmap_kernel(), va, va + ubc_winsize); @@ -594,7 +595,7 @@ again_faultbusy: KASSERT(pg->uobject == uobj); if (pg->loan_count != 0) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); if (pg->loan_count != 0) { pg = uvm_loanbreak(pg); } @@ -602,11 +603,11 @@ again_faultbusy: pmap_kremove(va, ubc_winsize); pmap_update(pmap_kernel()); uvm_page_unbusy(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); uvm_wait("ubc_alloc"); goto again_faultbusy; } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); pgs[i] = pg; } pmap_kenter_pa(va + slot_offset + (i << PAGE_SHIFT), @@ -655,7 +656,7 @@ ubc_release(void *va, int flags) memset((char *)umapva + endoff, 0, zerolen); } umap->flags &= ~UMAP_PAGES_LOCKED; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (u_int i = 0; i < npages; i++) { paddr_t pa; bool rv __diagused; @@ -676,13 +677,13 @@ ubc_release(void *va, int flags) pmap_kremove(umapva, ubc_winsize); pmap_update(pmap_kernel()); uvm_page_unbusy(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); unmapped = true; } else { unmapped = false; } - mutex_enter(ubc_object.uobj.vmobjlock); + rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER); umap->writeoff = 0; umap->writelen = 0; umap->refcount--; @@ -693,11 +694,11 @@ ubc_release(void *va, int flags) * This is typically used to avoid leaving * incompatible cache aliases around indefinitely. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); pmap_remove(pmap_kernel(), umapva, umapva + ubc_winsize); pmap_update(pmap_kernel()); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); umap->flags &= ~UMAP_MAPPING_CACHED; LIST_REMOVE(umap, hash); @@ -715,7 +716,7 @@ ubc_release(void *va, int flags) } UVMHIST_LOG(ubchist, "umap %#jx refs %jd", (uintptr_t)umap, umap->refcount, 0, 0); - mutex_exit(ubc_object.uobj.vmobjlock); + rw_exit(ubc_object.uobj.vmobjlock); } /* @@ -845,7 +846,7 @@ again: KASSERT(*lenp + pgoff <= ubc_winsize); memset(pgs, 0, *npages * sizeof(pgs[0])); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs, npages, 0, access_type, advice, gpflags); UVMHIST_LOG(ubchist, "alloc_direct getpages %jd", error, 0, 0, 0); @@ -857,7 +858,7 @@ again: return error; } - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (int i = 0; i < *npages; i++) { struct vm_page *pg = pgs[i]; @@ -871,7 +872,7 @@ again: pg = uvm_loanbreak(pg); if (pg == NULL) { uvm_page_unbusy(pgs, *npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); uvm_wait("ubc_alloc_directl"); goto again; } @@ -881,7 +882,7 @@ again: /* Page must be writable by now */ KASSERT((pg->flags & PG_RDONLY) == 0 || (flags & UBC_WRITE) == 0); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return 0; } @@ -890,7 +891,7 @@ static void __noinline ubc_direct_release(struct uvm_object *uobj, int flags, struct vm_page **pgs, int npages) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (int i = 0; i < npages; i++) { struct vm_page *pg = pgs[i]; @@ -912,7 +913,7 @@ ubc_direct_release(struct uvm_object *uo } } uvm_page_unbusy(pgs, npages); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } static int @@ -1033,7 +1034,7 @@ ubc_purge(struct uvm_object *uobj) if (__predict_true(LIST_EMPTY(&uobj->uo_ubc))) { return; } - mutex_enter(ubc_object.uobj.vmobjlock); + rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER); while ((umap = LIST_FIRST(&uobj->uo_ubc)) != NULL) { KASSERT(umap->refcount == 0); for (va = 0; va < ubc_winsize; va += PAGE_SIZE) { @@ -1045,5 +1046,5 @@ ubc_purge(struct uvm_object *uobj) umap->flags &= ~UMAP_MAPPING_CACHED; umap->uobj = NULL; } - mutex_exit(ubc_object.uobj.vmobjlock); + rw_exit(ubc_object.uobj.vmobjlock); } Index: sys/uvm/uvm_coredump.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_coredump.c,v retrieving revision 1.7 diff -u -p -r1.7 uvm_coredump.c --- sys/uvm/uvm_coredump.c 17 Feb 2019 13:46:03 -0000 1.7 +++ sys/uvm/uvm_coredump.c 21 Jan 2020 21:09:20 -0000 @@ -156,7 +156,7 @@ uvm_coredump_walkmap(struct proc *p, int if (entry->object.uvm_obj == NULL) { vaddr_t end; - amap_lock(entry->aref.ar_amap); + amap_lock(entry->aref.ar_amap, RW_READER); for (end = state.start; end < state.end; end += PAGE_SIZE) { struct vm_anon *anon; Index: sys/uvm/uvm_device.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_device.c,v retrieving revision 1.67 diff -u -p -r1.67 uvm_device.c --- sys/uvm/uvm_device.c 1 Dec 2019 16:44:11 -0000 1.67 +++ sys/uvm/uvm_device.c 21 Jan 2020 21:09:20 -0000 @@ -196,9 +196,9 @@ udv_attach(dev_t device, vm_prot_t acces * bump reference count, unhold, return. */ - mutex_enter(lcv->u_obj.vmobjlock); + rw_enter(lcv->u_obj.vmobjlock, RW_WRITER); lcv->u_obj.uo_refs++; - mutex_exit(lcv->u_obj.vmobjlock); + rw_exit(lcv->u_obj.vmobjlock); mutex_enter(&udv_lock); if (lcv->u_flags & UVM_DEVICE_WANTED) @@ -271,11 +271,11 @@ udv_reference(struct uvm_object *uobj) { UVMHIST_FUNC("udv_reference"); UVMHIST_CALLED(maphist); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); uobj->uo_refs++; UVMHIST_LOG(maphist, "<- done (uobj=0x%#jx, ref = %jd)", (uintptr_t)uobj, uobj->uo_refs,0,0); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } /* @@ -296,10 +296,10 @@ udv_detach(struct uvm_object *uobj) * loop until done */ again: - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); if (uobj->uo_refs > 1) { uobj->uo_refs--; - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); UVMHIST_LOG(maphist," <- done, uobj=0x%#jx, ref=%jd", (uintptr_t)uobj,uobj->uo_refs,0,0); return; @@ -312,7 +312,7 @@ again: mutex_enter(&udv_lock); if (udv->u_flags & UVM_DEVICE_HOLD) { udv->u_flags |= UVM_DEVICE_WANTED; - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); UVM_UNLOCK_AND_WAIT(udv, &udv_lock, false, "udv_detach",0); goto again; } @@ -325,7 +325,7 @@ again: if (udv->u_flags & UVM_DEVICE_WANTED) wakeup(udv); mutex_exit(&udv_lock); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); uvm_obj_destroy(uobj, true); kmem_free(udv, sizeof(*udv)); Index: sys/uvm/uvm_extern.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_extern.h,v retrieving revision 1.219 diff -u -p -r1.219 uvm_extern.h --- sys/uvm/uvm_extern.h 15 Jan 2020 17:55:45 -0000 1.219 +++ sys/uvm/uvm_extern.h 21 Jan 2020 21:09:20 -0000 @@ -736,7 +736,7 @@ int uvm_mremap(struct vm_map *, vaddr_ /* uvm_object.c */ void uvm_obj_init(struct uvm_object *, const struct uvm_pagerops *, bool, u_int); -void uvm_obj_setlock(struct uvm_object *, kmutex_t *); +void uvm_obj_setlock(struct uvm_object *, krwlock_t *); void uvm_obj_destroy(struct uvm_object *, bool); int uvm_obj_wirepages(struct uvm_object *, off_t, off_t, struct pglist *); Index: sys/uvm/uvm_fault.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_fault.c,v retrieving revision 1.215 diff -u -p -r1.215 uvm_fault.c --- sys/uvm/uvm_fault.c 15 Jan 2020 17:55:45 -0000 1.215 +++ sys/uvm/uvm_fault.c 21 Jan 2020 21:09:20 -0000 @@ -194,7 +194,7 @@ uvmfault_anonflush(struct vm_anon **anon for (lcv = 0; lcv < n; lcv++) { if (anons[lcv] == NULL) continue; - KASSERT(mutex_owned(anons[lcv]->an_lock)); + KASSERT(rw_write_held(anons[lcv]->an_lock)); pg = anons[lcv]->an_page; if (pg && (pg->flags & PG_BUSY) == 0) { uvm_pagelock(pg); @@ -279,7 +279,7 @@ uvmfault_anonget(struct uvm_faultinfo *u int error; UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); - KASSERT(mutex_owned(anon->an_lock)); + KASSERT(rw_write_held(anon->an_lock)); KASSERT(anon->an_lock == amap->am_lock); /* Increment the counters.*/ @@ -342,7 +342,7 @@ uvmfault_anonget(struct uvm_faultinfo *u uvmfault_unlockall(ufi, amap, NULL); UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 0,0,0); - UVM_UNLOCK_AND_WAIT(pg, + UVM_UNLOCK_AND_WAIT_RW(pg, pg->uobject->vmobjlock, false, "anonget1", 0); } else { @@ -350,7 +350,7 @@ uvmfault_anonget(struct uvm_faultinfo *u uvmfault_unlockall(ufi, NULL, NULL); UVMHIST_LOG(maphist, " unlock+wait on anon",0, 0,0,0); - UVM_UNLOCK_AND_WAIT(pg, anon->an_lock, + UVM_UNLOCK_AND_WAIT_RW(pg, anon->an_lock, false, "anonget2", 0); } } else { @@ -404,7 +404,7 @@ uvmfault_anonget(struct uvm_faultinfo *u locked = uvmfault_relock(ufi); if (locked || we_own) { - mutex_enter(anon->an_lock); + rw_enter(anon->an_lock, RW_WRITER); } /* @@ -452,7 +452,7 @@ uvmfault_anonget(struct uvm_faultinfo *u if (locked) { uvmfault_unlockall(ufi, NULL, NULL); } - mutex_exit(anon->an_lock); + rw_exit(anon->an_lock); UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); return error; } @@ -501,7 +501,7 @@ released: if (!locked) { if (we_own) { - mutex_exit(anon->an_lock); + rw_exit(anon->an_lock); } UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); return ERESTART; @@ -580,9 +580,9 @@ uvmfault_promote(struct uvm_faultinfo *u KASSERT(amap != NULL); KASSERT(uobjpage != NULL); KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0); - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock); - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); + KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); if (*spare != NULL) { anon = *spare; @@ -880,7 +880,7 @@ uvm_fault_internal(struct vm_map *orig_m /* * invoke "special" fault routine. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); /* locked: maps(read), amap(if there), uobj */ error = uobj->pgops->pgo_fault(&ufi, flt.startva, pages, flt.npages, @@ -1103,14 +1103,14 @@ uvm_fault_check( */ if (amap) { - amap_lock(amap); + amap_lock(amap, RW_WRITER); amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages); } else { *ranons = NULL; /* to be safe */ } /* locked: maps(read), amap(if there) */ - KASSERT(amap == NULL || mutex_owned(amap->am_lock)); + KASSERT(amap == NULL || rw_write_held(amap->am_lock)); /* * for MADV_SEQUENTIAL mappings we want to deactivate the back pages @@ -1130,7 +1130,7 @@ uvm_fault_check( voff_t uoff; uoff = ufi->entry->offset + eoff; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff + (nback << PAGE_SHIFT), PGO_DEACTIVATE); } @@ -1175,7 +1175,7 @@ uvm_fault_upper_lookup( UVMHIST_FUNC("uvm_fault_upper_lookup"); UVMHIST_CALLED(maphist); /* locked: maps(read), amap(if there) */ - KASSERT(amap == NULL || mutex_owned(amap->am_lock)); + KASSERT(amap == NULL || rw_write_held(amap->am_lock)); /* * map in the backpages and frontpages we found in the amap in hopes @@ -1227,7 +1227,7 @@ uvm_fault_upper_lookup( } /* locked: maps(read), amap(if there) */ - KASSERT(amap == NULL || mutex_owned(amap->am_lock)); + KASSERT(amap == NULL || rw_write_held(amap->am_lock)); /* (shadowed == true) if there is an anon at the faulting address */ UVMHIST_LOG(maphist, " shadowed=%jd, will_get=%jd", shadowed, (ufi->entry->object.uvm_obj && shadowed != false),0,0); @@ -1260,7 +1260,7 @@ uvm_fault_upper_neighbor( KASSERT(pg->uobject == NULL); KASSERT(pg->uanon != NULL); - KASSERT(mutex_owned(pg->uanon->an_lock)); + KASSERT(rw_write_held(pg->uanon->an_lock)); KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN); uvm_pagelock(pg); @@ -1307,7 +1307,7 @@ uvm_fault_upper( UVMHIST_FUNC("uvm_fault_upper"); UVMHIST_CALLED(maphist); /* locked: maps(read), amap, anon */ - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); KASSERT(anon->an_lock == amap->am_lock); /* @@ -1354,9 +1354,9 @@ uvm_fault_upper( uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ /* locked: maps(read), amap, anon, uobj(if one) */ - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); KASSERT(anon->an_lock == amap->am_lock); - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); + KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); /* * special handling for loaned pages @@ -1538,10 +1538,10 @@ uvm_fault_upper_enter( UVMHIST_FUNC("uvm_fault_upper_enter"); UVMHIST_CALLED(maphist); /* locked: maps(read), amap, oanon, anon(if different from oanon) */ - KASSERT(mutex_owned(amap->am_lock)); + KASSERT(rw_write_held(amap->am_lock)); KASSERT(anon->an_lock == amap->am_lock); KASSERT(oanon->an_lock == amap->am_lock); - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); + KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN); /* @@ -1693,8 +1693,8 @@ uvm_fault_lower( * locked: * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) */ - KASSERT(amap == NULL || mutex_owned(amap->am_lock)); - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); + KASSERT(amap == NULL || rw_write_held(amap->am_lock)); + KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); /* @@ -1736,8 +1736,8 @@ uvm_fault_lower( * locked: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) */ - KASSERT(amap == NULL || mutex_owned(amap->am_lock)); - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); + KASSERT(amap == NULL || rw_write_held(amap->am_lock)); + KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); /* @@ -1779,7 +1779,7 @@ uvm_fault_lower_lookup( vaddr_t currva; UVMHIST_FUNC("uvm_fault_lower_lookup"); UVMHIST_CALLED(maphist); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); /* Locked: maps(read), amap(if there), uobj */ cpu_count(CPU_COUNT_FLTLGET, 1); @@ -1789,7 +1789,7 @@ uvm_fault_lower_lookup( pages, &gotpages, flt->centeridx, flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED); - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); /* * check for pages to map, if we got any @@ -1869,7 +1869,7 @@ uvm_fault_lower_neighbor( pg->flags &= ~(PG_BUSY); UVM_PAGE_OWN(pg, NULL); - KASSERT(mutex_owned(pg->uobject->vmobjlock)); + KASSERT(rw_write_held(pg->uobject->vmobjlock)); const vm_prot_t mapprot = readonly ? (flt->enter_prot & ~VM_PROT_WRITE) : @@ -1917,7 +1917,7 @@ uvm_fault_lower_io( uvmfault_unlockall(ufi, amap, NULL); /* Locked: uobj */ - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); + KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); cpu_count(CPU_COUNT_FLTGET, 1); gotpages = 1; @@ -1958,12 +1958,12 @@ uvm_fault_lower_io( locked = uvmfault_relock(ufi); if (locked && amap) - amap_lock(amap); + amap_lock(amap, RW_WRITER); /* might be changed */ uobj = pg->uobject; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); KASSERT((pg->flags & PG_BUSY) != 0); uvm_pagelock(pg); @@ -2005,7 +2005,7 @@ uvm_fault_lower_io( cpu_count(CPU_COUNT_FLTPGRELE, 1); uvm_pagefree(pg); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return ERESTART; } @@ -2232,8 +2232,8 @@ uvm_fault_lower_enter( * * Note: pg is either the uobjpage or the new page in the new anon. */ - KASSERT(amap == NULL || mutex_owned(amap->am_lock)); - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); + KASSERT(amap == NULL || rw_write_held(amap->am_lock)); + KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); KASSERT(anon == NULL || anon->an_lock == amap->am_lock); KASSERT((pg->flags & PG_BUSY) != 0); @@ -2467,7 +2467,7 @@ uvm_fault_unwire_locked(struct vm_map *m if (oentry != NULL) { uvm_map_unlock_entry(oentry); } - uvm_map_lock_entry(entry); + uvm_map_lock_entry(entry, RW_WRITER); oentry = entry; } Index: sys/uvm/uvm_fault_i.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_fault_i.h,v retrieving revision 1.32 diff -u -p -r1.32 uvm_fault_i.h --- sys/uvm/uvm_fault_i.h 16 Dec 2019 22:47:55 -0000 1.32 +++ sys/uvm/uvm_fault_i.h 21 Jan 2020 21:09:20 -0000 @@ -74,7 +74,7 @@ uvmfault_unlockall(struct uvm_faultinfo { if (uobj) - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); if (amap) amap_unlock(amap); uvmfault_unlockmaps(ufi, false); Index: sys/uvm/uvm_km.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_km.c,v retrieving revision 1.153 diff -u -p -r1.153 uvm_km.c --- sys/uvm/uvm_km.c 20 Jan 2020 10:43:48 -0000 1.153 +++ sys/uvm/uvm_km.c 21 Jan 2020 21:09:20 -0000 @@ -450,16 +450,16 @@ uvm_km_pgremove(vaddr_t startva, vaddr_t KASSERT(startva < endva); KASSERT(endva <= VM_MAX_KERNEL_ADDRESS); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); pmap_remove(pmap_kernel(), startva, endva); for (curoff = start; curoff < end; curoff = nextoff) { nextoff = curoff + PAGE_SIZE; pg = uvm_pagelookup(uobj, curoff); if (pg != NULL && pg->flags & PG_BUSY) { pg->flags |= PG_WANTED; - UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, 0, + UVM_UNLOCK_AND_WAIT_RW(pg, uobj->vmobjlock, 0, "km_pgrm", 0); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); nextoff = curoff; continue; } @@ -477,7 +477,7 @@ uvm_km_pgremove(vaddr_t startva, vaddr_t uvm_pagefree(pg); } } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); if (swpgonlydelta > 0) { KASSERT(uvmexp.swpgonly >= swpgonlydelta); @@ -569,12 +569,12 @@ uvm_km_check_empty(struct vm_map *map, v * - we can recurse when allocating radix_node for * kernel_object. */ - if (mutex_tryenter(uvm_kernel_object->vmobjlock)) { + if (rw_tryenter(uvm_kernel_object->vmobjlock, RW_WRITER)) { struct vm_page *pg; pg = uvm_pagelookup(uvm_kernel_object, va - vm_map_min(kernel_map)); - mutex_exit(uvm_kernel_object->vmobjlock); + rw_exit(uvm_kernel_object->vmobjlock); if (pg) { panic("uvm_km_check_empty: " "has page hashed at %p", Index: sys/uvm/uvm_loan.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_loan.c,v retrieving revision 1.94 diff -u -p -r1.94 uvm_loan.c --- sys/uvm/uvm_loan.c 15 Jan 2020 17:55:45 -0000 1.94 +++ sys/uvm/uvm_loan.c 21 Jan 2020 21:09:20 -0000 @@ -149,7 +149,7 @@ uvm_loanentry(struct uvm_faultinfo *ufi, * lock us the rest of the way down (we unlock before return) */ if (aref->ar_amap) { - amap_lock(aref->ar_amap); + amap_lock(aref->ar_amap, RW_WRITER); } /* @@ -180,9 +180,9 @@ uvm_loanentry(struct uvm_faultinfo *ufi, } /* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */ KASSERT(rv > 0 || aref->ar_amap == NULL || - !mutex_owned(aref->ar_amap->am_lock)); + !rw_write_held(aref->ar_amap->am_lock)); KASSERT(rv > 0 || uobj == NULL || - !mutex_owned(uobj->vmobjlock)); + !rw_write_held(uobj->vmobjlock)); /* total failure */ if (rv < 0) { @@ -355,7 +355,7 @@ uvm_loananon(struct uvm_faultinfo *ufi, */ if (flags & UVM_LOAN_TOANON) { - KASSERT(mutex_owned(anon->an_lock)); + KASSERT(rw_write_held(anon->an_lock)); pg = anon->an_page; if (pg && (pg->flags & PG_ANON) != 0 && anon->an_ref == 1) { if (pg->wire_count > 0) { @@ -381,7 +381,7 @@ uvm_loananon(struct uvm_faultinfo *ufi, * this for us. */ - KASSERT(mutex_owned(anon->an_lock)); + KASSERT(rw_write_held(anon->an_lock)); error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon); /* @@ -431,7 +431,7 @@ uvm_loananon(struct uvm_faultinfo *ufi, /* unlock and return success */ if (pg->uobject) - mutex_exit(pg->uobject->vmobjlock); + rw_exit(pg->uobject->vmobjlock); UVMHIST_LOG(loanhist, "->K done", 0,0,0,0); return (1); } @@ -459,7 +459,7 @@ uvm_loanpage(struct vm_page **pgpp, int KASSERT(pg->uobject != NULL); KASSERT(pg->uobject == pgpp[0]->uobject); KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT))); - KASSERT(mutex_owned(pg->uobject->vmobjlock)); + KASSERT(rw_write_held(pg->uobject->vmobjlock)); KASSERT(pg->flags & PG_BUSY); if (pg->wire_count > 0) { @@ -484,11 +484,11 @@ uvm_loanpage(struct vm_page **pgpp, int /* * backout what we've done */ - kmutex_t *slock = pgpp[0]->uobject->vmobjlock; + krwlock_t *slock = pgpp[0]->uobject->vmobjlock; - mutex_exit(slock); + rw_exit(slock); uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE); - mutex_enter(slock); + rw_enter(slock, RW_WRITER); } UVMHIST_LOG(loanhist, "done %jd", error, 0, 0, 0); @@ -516,7 +516,7 @@ uvm_loanuobjpages(struct uvm_object *uob struct vm_page **pgpp; int error; int i; - kmutex_t *slock; + krwlock_t *slock; pgpp = origpgpp; for (ndone = 0; ndone < orignpages; ) { @@ -525,7 +525,7 @@ uvm_loanuobjpages(struct uvm_object *uob int npendloan = 0xdead; /* XXX gcc */ reget: npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); error = (*uobj->pgops->pgo_get)(uobj, pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0, VM_PROT_READ, 0, PGO_SYNCIO); @@ -541,7 +541,7 @@ reget: /* loan and unbusy pages */ slock = NULL; for (i = 0; i < npages; i++) { - kmutex_t *nextslock; /* slock for next page */ + krwlock_t *nextslock; /* slock for next page */ struct vm_page *pg = *pgpp; /* XXX assuming that the page is owned by uobj */ @@ -553,7 +553,7 @@ reget: KASSERT(npendloan > 0); error = uvm_loanpage(pgpp - npendloan, npendloan); - mutex_exit(slock); + rw_exit(slock); if (error) goto fail; ndone += npendloan; @@ -561,21 +561,21 @@ reget: } slock = nextslock; npendloan = 0; - mutex_enter(slock); + rw_enter(slock, RW_WRITER); } if ((pg->flags & PG_RELEASED) != 0) { /* * release pages and try again. */ - mutex_exit(slock); + rw_exit(slock); for (; i < npages; i++) { pg = pgpp[i]; slock = pg->uobject->vmobjlock; - mutex_enter(slock); + rw_enter(slock, RW_WRITER); uvm_page_unbusy(&pg, 1); - mutex_exit(slock); + rw_exit(slock); } goto reget; } @@ -587,7 +587,7 @@ reget: KASSERT(slock != NULL); KASSERT(npendloan > 0); error = uvm_loanpage(pgpp - npendloan, npendloan); - mutex_exit(slock); + rw_exit(slock); if (error) goto fail; ndone += npendloan; @@ -631,7 +631,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, */ /* locked: maps(read), amap(if there) */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); /* locked: maps(read), amap(if there), uobj */ if (uobj->pgops->pgo_get) { /* try locked pgo_get */ @@ -682,9 +682,9 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, locked = uvmfault_relock(ufi); if (locked && amap) - amap_lock(amap); + amap_lock(amap, RW_WRITER); uobj = pg->uobject; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); /* * verify that the page has not be released and re-verify @@ -710,7 +710,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, } if (pg->flags & PG_RELEASED) { uvm_pagefree(pg); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return (0); } uvm_pagelock(pg); @@ -718,7 +718,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, uvm_pageunlock(pg); pg->flags &= ~(PG_BUSY|PG_WANTED); UVM_PAGE_OWN(pg, NULL); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return (0); } } @@ -736,7 +736,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, uvmfault_unlockall(ufi, amap, uobj); return (-1); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); **output = pg; (*output)++; return (1); @@ -759,7 +759,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, } pg->flags &= ~(PG_WANTED|PG_BUSY); UVM_PAGE_OWN(pg, NULL); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); **output = anon; (*output)++; return (1); @@ -793,8 +793,8 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, } pg->flags &= ~(PG_WANTED|PG_BUSY); UVM_PAGE_OWN(pg, NULL); - mutex_exit(uobj->vmobjlock); - mutex_exit(&anon->an_lock); + rw_exit(uobj->vmobjlock); + rw_exit(&anon->an_lock); **output = anon; (*output)++; return (1); @@ -830,7 +830,7 @@ fail: */ static struct uvm_object uvm_loanzero_object; -static kmutex_t uvm_loanzero_lock __cacheline_aligned; +static krwlock_t uvm_loanzero_lock __cacheline_aligned; static int uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags) @@ -840,7 +840,7 @@ uvm_loanzero(struct uvm_faultinfo *ufi, UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist); again: - mutex_enter(uvm_loanzero_object.vmobjlock); + rw_enter(uvm_loanzero_object.vmobjlock, RW_WRITER); /* * first, get ahold of our single zero page. @@ -850,14 +850,14 @@ again: if (__predict_false(pg == NULL)) { while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL, UVM_PGA_ZERO)) == NULL) { - mutex_exit(uvm_loanzero_object.vmobjlock); + rw_exit(uvm_loanzero_object.vmobjlock); uvmfault_unlockall(ufi, amap, NULL); uvm_wait("loanzero"); if (!uvmfault_relock(ufi)) { return (0); } if (amap) { - amap_lock(amap); + amap_lock(amap, RW_WRITER); } goto again; } @@ -876,7 +876,7 @@ again: pg->loan_count++; KASSERT(pg->loan_count > 0); /* detect wrap-around */ mutex_exit(&pg->interlock); - mutex_exit(uvm_loanzero_object.vmobjlock); + rw_exit(uvm_loanzero_object.vmobjlock); **output = pg; (*output)++; return (1); @@ -891,10 +891,10 @@ again: if (pg->uanon) { anon = pg->uanon; - mutex_enter(&anon->an_lock); + rw_enter(&anon->an_lock, RW_WRITER); anon->an_ref++; - mutex_exit(&anon->an_lock); - mutex_exit(uvm_loanzero_object.vmobjlock); + rw_exit(&anon->an_lock); + rw_exit(uvm_loanzero_object.vmobjlock); **output = anon; (*output)++; return (1); @@ -907,7 +907,7 @@ again: anon = uvm_analloc(); if (anon == NULL) { /* out of swap causes us to fail */ - mutex_exit(uvm_loanzero_object.vmobjlock); + rw_exit(uvm_loanzero_object.vmobjlock); uvmfault_unlockall(ufi, amap, NULL, NULL); return (-1); } @@ -918,8 +918,8 @@ again: KASSERT(pg->loan_count > 0); /* detect wrap-around */ uvm_pageactivate(pg); uvm_pageunlock(pg); - mutex_exit(&anon->an_lock); - mutex_exit(uvm_loanzero_object.vmobjlock); + rw_exit(&anon->an_lock); + rw_exit(uvm_loanzero_object.vmobjlock); **output = anon; (*output)++; return (1); @@ -942,7 +942,7 @@ uvm_unloananon(struct vm_anon **aloans, struct vm_anon *anon, *to_free = NULL; /* TODO: locking */ - amap_lock(amap); + amap_lock(amap, RW_WRITER); while (nanons-- > 0) { anon = *aloans++; if (--anon->an_ref == 0) { @@ -964,7 +964,7 @@ static void uvm_unloanpage(struct vm_page **ploans, int npages) { struct vm_page *pg; - kmutex_t *slock; + krwlock_t *slock; while (npages-- > 0) { pg = *ploans++; @@ -983,12 +983,11 @@ uvm_unloanpage(struct vm_page **ploans, } else { slock = pg->uanon->an_lock; } - if (mutex_tryenter(slock)) { + if (rw_tryenter(slock, RW_WRITER)) { break; } /* XXX Better than yielding but inadequate. */ kpause("livelock", false, 1, &pg->interlock); - slock = NULL; } /* @@ -1016,9 +1015,7 @@ uvm_unloanpage(struct vm_page **ploans, KASSERT((pg->flags & PG_BUSY) == 0); uvm_pagefree(pg); } - if (slock != NULL) { - mutex_exit(slock); - } + rw_exit(slock); } } @@ -1055,7 +1052,7 @@ ulz_put(struct uvm_object *uobj, voff_t */ if ((flags & PGO_FREE) == 0) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return 0; } @@ -1075,7 +1072,7 @@ ulz_put(struct uvm_object *uobj, voff_t } uvm_pageunlock(pg); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return 0; } @@ -1091,7 +1088,7 @@ void uvm_loan_init(void) { - mutex_init(&uvm_loanzero_lock, MUTEX_DEFAULT, IPL_NONE); + rw_init(&uvm_loanzero_lock); uvm_obj_init(&uvm_loanzero_object, &ulz_pager, false, 0); uvm_obj_setlock(&uvm_loanzero_object, &uvm_loanzero_lock); @@ -1113,7 +1110,7 @@ uvm_loanbreak(struct vm_page *uobjpage) struct uvm_object *uobj __diagused = uobjpage->uobject; KASSERT(uobj != NULL); - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); KASSERT(uobjpage->flags & PG_BUSY); /* alloc new un-owned page */ @@ -1177,8 +1174,8 @@ uvm_loanbreak_anon(struct vm_anon *anon, struct vm_page *newpg, *oldpg; unsigned oldstatus; - KASSERT(mutex_owned(anon->an_lock)); - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(anon->an_lock)); + KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); KASSERT(anon->an_page->loan_count > 0); /* get new un-owned replacement page */ @@ -1228,7 +1225,7 @@ uvm_loanbreak_anon(struct vm_anon *anon, UVM_PAGE_OWN(newpg, NULL); if (uobj) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } /* done! */ Index: sys/uvm/uvm_map.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_map.c,v retrieving revision 1.371 diff -u -p -r1.371 uvm_map.c --- sys/uvm/uvm_map.c 12 Jan 2020 17:46:55 -0000 1.371 +++ sys/uvm/uvm_map.c 21 Jan 2020 21:09:20 -0000 @@ -2312,7 +2312,7 @@ uvm_unmap_remove(struct vm_map *map, vad * change while in pmap_remove(). */ - uvm_map_lock_entry(entry); + uvm_map_lock_entry(entry, RW_WRITER); pmap_remove(map->pmap, entry->start, entry->end); /* @@ -2887,7 +2887,7 @@ uvm_map_extract(struct vm_map *srcmap, v /* we advance "entry" in the following if statement */ if (flags & UVM_EXTRACT_REMOVE) { - uvm_map_lock_entry(entry); + uvm_map_lock_entry(entry, RW_WRITER); pmap_remove(srcmap->pmap, entry->start, entry->end); uvm_map_unlock_entry(entry); @@ -3119,7 +3119,7 @@ uvm_map_protect(struct vm_map *map, vadd if (current->protection != old_prot) { /* update pmap! */ - uvm_map_lock_entry(current); + uvm_map_lock_entry(current, RW_WRITER); pmap_protect(map->pmap, current->start, current->end, current->protection & MASK(current)); uvm_map_unlock_entry(current); @@ -3915,7 +3915,7 @@ uvm_map_clean(struct vm_map *map, vaddr_ size = MIN(end, current->end) - start; anon_tofree = NULL; - amap_lock(amap); + amap_lock(amap, RW_WRITER); for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) { anon = amap_lookup(¤t->aref, offset); if (anon == NULL) @@ -3992,7 +3992,7 @@ uvm_map_clean(struct vm_map *map, vaddr_ uoff = current->offset + (start - current->start); size = MIN(end, current->end) - start; if (uobj != NULL) { - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); if (uobj->pgops->pgo_put != NULL) error = (uobj->pgops->pgo_put)(uobj, uoff, uoff + size, flags | PGO_CLEANIT); @@ -4460,7 +4460,7 @@ uvm_mapent_forkcopy(struct vm_map *new_m if (old_entry->aref.ar_amap && !UVM_ET_ISNEEDSCOPY(old_entry)) { if (old_entry->max_protection & VM_PROT_WRITE) { - uvm_map_lock_entry(old_entry); + uvm_map_lock_entry(old_entry, RW_WRITER); pmap_protect(old_map->pmap, old_entry->start, old_entry->end, old_entry->protection & ~VM_PROT_WRITE); @@ -4782,14 +4782,14 @@ uvm_map_reference(struct vm_map *map) } void -uvm_map_lock_entry(struct vm_map_entry *entry) +uvm_map_lock_entry(struct vm_map_entry *entry, krw_t op) { if (entry->aref.ar_amap != NULL) { - amap_lock(entry->aref.ar_amap); + amap_lock(entry->aref.ar_amap, op); } if (UVM_ET_ISOBJ(entry)) { - mutex_enter(entry->object.uvm_obj->vmobjlock); + rw_enter(entry->object.uvm_obj->vmobjlock, op); } } @@ -4798,7 +4798,7 @@ uvm_map_unlock_entry(struct vm_map_entry { if (UVM_ET_ISOBJ(entry)) { - mutex_exit(entry->object.uvm_obj->vmobjlock); + rw_exit(entry->object.uvm_obj->vmobjlock); } if (entry->aref.ar_amap != NULL) { amap_unlock(entry->aref.ar_amap); Index: sys/uvm/uvm_map.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_map.h,v retrieving revision 1.77 diff -u -p -r1.77 uvm_map.h --- sys/uvm/uvm_map.h 12 Jan 2020 17:46:55 -0000 1.77 +++ sys/uvm/uvm_map.h 21 Jan 2020 21:09:20 -0000 @@ -329,7 +329,7 @@ void vm_map_unlock_read(struct vm_map * void vm_map_busy(struct vm_map *); bool vm_map_locked_p(struct vm_map *); -void uvm_map_lock_entry(struct vm_map_entry *); +void uvm_map_lock_entry(struct vm_map_entry *, krw_t); void uvm_map_unlock_entry(struct vm_map_entry *); #endif /* _KERNEL */ Index: sys/uvm/uvm_mmap.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_mmap.c,v retrieving revision 1.174 diff -u -p -r1.174 uvm_mmap.c --- sys/uvm/uvm_mmap.c 4 Oct 2019 22:48:45 -0000 1.174 +++ sys/uvm/uvm_mmap.c 21 Jan 2020 21:09:20 -0000 @@ -200,9 +200,9 @@ sys_mincore(struct lwp *l, const struct uobj = entry->object.uvm_obj; /* lower layer */ if (amap != NULL) - amap_lock(amap); + amap_lock(amap, RW_READER); if (uobj != NULL) - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_READER); for (/* nothing */; start < lim; start += PAGE_SIZE, vec++) { pgi = 0; @@ -238,7 +238,7 @@ sys_mincore(struct lwp *l, const struct (void) ustore_char(vec, pgi); } if (uobj != NULL) - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); if (amap != NULL) amap_unlock(amap); } Index: sys/uvm/uvm_mremap.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_mremap.c,v retrieving revision 1.19 diff -u -p -r1.19 uvm_mremap.c --- sys/uvm/uvm_mremap.c 6 May 2017 21:34:52 -0000 1.19 +++ sys/uvm/uvm_mremap.c 21 Jan 2020 21:09:20 -0000 @@ -80,10 +80,10 @@ uvm_mapent_extend(struct vm_map *map, va error = E2BIG; /* XXX */ goto done; } - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); KASSERT(uobj->uo_refs > 0); atomic_inc_uint(&uobj->uo_refs); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); reserved_entry->object.uvm_obj = uobj; reserved_entry->offset = newoffset; } Index: sys/uvm/uvm_object.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_object.c,v retrieving revision 1.20 diff -u -p -r1.20 uvm_object.c --- sys/uvm/uvm_object.c 15 Jan 2020 17:55:45 -0000 1.20 +++ sys/uvm/uvm_object.c 21 Jan 2020 21:09:20 -0000 @@ -44,7 +44,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_object.c #endif #include -#include +#include #include #include @@ -67,7 +67,7 @@ uvm_obj_init(struct uvm_object *uo, cons #endif if (alock) { /* Allocate and assign a lock. */ - uo->vmobjlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); + uo->vmobjlock = rw_obj_alloc(); } else { /* The lock will need to be set via uvm_obj_setlock(). */ uo->vmobjlock = NULL; @@ -93,7 +93,7 @@ uvm_obj_destroy(struct uvm_object *uo, b /* Destroy the lock, if requested. */ if (dlock) { - mutex_obj_free(uo->vmobjlock); + rw_obj_free(uo->vmobjlock); } radix_tree_fini_tree(&uo->uo_pages); } @@ -105,17 +105,17 @@ uvm_obj_destroy(struct uvm_object *uo, b * => Only dynamic lock may be previously set. We drop the reference then. */ void -uvm_obj_setlock(struct uvm_object *uo, kmutex_t *lockptr) +uvm_obj_setlock(struct uvm_object *uo, krwlock_t *lockptr) { - kmutex_t *olockptr = uo->vmobjlock; + krwlock_t *olockptr = uo->vmobjlock; if (olockptr) { /* Drop the reference on the old lock. */ - mutex_obj_free(olockptr); + rw_obj_free(olockptr); } if (lockptr == NULL) { /* If new lock is not passed - allocate default one. */ - lockptr = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); + lockptr = rw_obj_alloc(); } uo->vmobjlock = lockptr; } @@ -137,7 +137,7 @@ uvm_obj_wirepages(struct uvm_object *uob left = (end - start) >> PAGE_SHIFT; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); while (left) { npages = MIN(FETCH_PAGECOUNT, left); @@ -151,7 +151,7 @@ uvm_obj_wirepages(struct uvm_object *uob if (error) goto error; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; i < npages; i++) { KASSERT(pgs[i] != NULL); @@ -164,9 +164,9 @@ uvm_obj_wirepages(struct uvm_object *uob while (pgs[i]->loan_count) { pg = uvm_loanbreak(pgs[i]); if (!pg) { - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); uvm_wait("uobjwirepg"); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); continue; } } @@ -195,7 +195,7 @@ uvm_obj_wirepages(struct uvm_object *uob left -= npages; offset += npages << PAGE_SHIFT; } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); return 0; @@ -219,7 +219,7 @@ uvm_obj_unwirepages(struct uvm_object *u struct vm_page *pg; off_t offset; - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); for (offset = start; offset < end; offset += PAGE_SIZE) { pg = uvm_pagelookup(uobj, offset); @@ -230,7 +230,7 @@ uvm_obj_unwirepages(struct uvm_object *u uvm_pageunwire(pg); uvm_pageunlock(pg); } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); } #if defined(DDB) || defined(DEBUGPRINT) @@ -248,7 +248,7 @@ uvm_object_printit(struct uvm_object *uo voff_t off; (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ", - uobj, mutex_owned(uobj->vmobjlock), uobj->pgops, uobj->uo_npages); + uobj, rw_write_held(uobj->vmobjlock), uobj->pgops, uobj->uo_npages); if (UVM_OBJ_IS_KERN_OBJECT(uobj)) (*pr)("refs=\n"); else Index: sys/uvm/uvm_object.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_object.h,v retrieving revision 1.36 diff -u -p -r1.36 uvm_object.h --- sys/uvm/uvm_object.h 15 Jan 2020 17:55:45 -0000 1.36 +++ sys/uvm/uvm_object.h 21 Jan 2020 21:09:20 -0000 @@ -53,8 +53,9 @@ * For other objects, it is arbitrary (may use the lock or atomics). */ +struct krwlock; struct uvm_object { - kmutex_t * vmobjlock; /* lock on object */ + struct krwlock * vmobjlock; /* lock on object */ const struct uvm_pagerops *pgops; /* pager ops */ int uo_npages; /* # of pages in uo_pages */ unsigned uo_refs; /* reference count */ Index: sys/uvm/uvm_page.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_page.c,v retrieving revision 1.225 diff -u -p -r1.225 uvm_page.c --- sys/uvm/uvm_page.c 21 Jan 2020 20:37:06 -0000 1.225 +++ sys/uvm/uvm_page.c 21 Jan 2020 21:09:20 -0000 @@ -214,7 +214,7 @@ uvm_pageinsert_object(struct uvm_object { KASSERT(uobj == pg->uobject); - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); KASSERT((pg->flags & PG_TABLED) == 0); if ((pg->flags & PG_STAT) != 0) { @@ -225,9 +225,7 @@ uvm_pageinsert_object(struct uvm_object if (!isaobj) { KASSERT((pg->flags & PG_FILE) != 0); if (uobj->uo_npages == 0) { - struct vnode *vp = (struct vnode *)uobj; - - vholdl(vp); + vhold((struct vnode *)uobj); } kpreempt_disable(); if (UVM_OBJ_IS_VTEXT(uobj)) { @@ -276,7 +274,7 @@ uvm_pageremove_object(struct uvm_object { KASSERT(uobj == pg->uobject); - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); KASSERT(pg->flags & PG_TABLED); if ((pg->flags & PG_STAT) != 0) { @@ -287,9 +285,7 @@ uvm_pageremove_object(struct uvm_object if (!isaobj) { KASSERT((pg->flags & PG_FILE) != 0); if (uobj->uo_npages == 1) { - struct vnode *vp = (struct vnode *)uobj; - - holdrelel(vp); + holdrele((struct vnode *)uobj); } kpreempt_disable(); if (UVM_OBJ_IS_VTEXT(uobj)) { @@ -1189,9 +1185,9 @@ uvm_pagealloc_strat(struct uvm_object *o KASSERT(obj == NULL || anon == NULL); KASSERT(anon == NULL || (flags & UVM_FLAG_COLORMATCH) || off == 0); KASSERT(off == trunc_page(off)); - KASSERT(obj == NULL || mutex_owned(obj->vmobjlock)); + KASSERT(obj == NULL || rw_write_held(obj->vmobjlock)); KASSERT(anon == NULL || anon->an_lock == NULL || - mutex_owned(anon->an_lock)); + rw_write_held(anon->an_lock)); /* * This implements a global round-robin page coloring @@ -1333,7 +1329,7 @@ uvm_pagealloc_strat(struct uvm_object *o pg->offset = off; pg->uobject = obj; pg->uanon = anon; - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); pg->flags = PG_BUSY|PG_CLEAN|PG_FAKE; if (anon) { anon->an_page = pg; @@ -1399,7 +1395,7 @@ uvm_pagereplace(struct vm_page *oldpg, s KASSERT(uobj != NULL); KASSERT((newpg->flags & PG_TABLED) == 0); KASSERT(newpg->uobject == NULL); - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); KASSERT(mutex_owned(&oldpg->interlock)); KASSERT(mutex_owned(&newpg->interlock)); @@ -1521,9 +1517,9 @@ uvm_pagefree(struct vm_page *pg) KASSERT((pg->flags & PG_PAGEOUT) == 0); KASSERT(!(pg->flags & PG_FREE)); - KASSERT(pg->uobject == NULL || mutex_owned(pg->uobject->vmobjlock)); + KASSERT(pg->uobject == NULL || rw_write_held(pg->uobject->vmobjlock)); KASSERT(pg->uobject != NULL || pg->uanon == NULL || - mutex_owned(pg->uanon->an_lock)); + rw_write_held(pg->uanon->an_lock)); /* * remove the page from the object's tree beore acquiring any page @@ -1682,7 +1678,7 @@ uvm_page_unbusy(struct vm_page **pgs, in continue; } - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(pg->flags & PG_BUSY); KASSERT((pg->flags & PG_PAGEOUT) == 0); if (pg->flags & PG_WANTED) { @@ -1722,7 +1718,7 @@ uvm_page_own(struct vm_page *pg, const c KASSERT((pg->flags & (PG_PAGEOUT|PG_RELEASED)) == 0); KASSERT((pg->flags & PG_WANTED) == 0); - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); /* gain ownership? */ if (tag) { @@ -1776,7 +1772,7 @@ uvm_pagelookup(struct uvm_object *obj, v { struct vm_page *pg; - /* No - used from DDB. KASSERT(mutex_owned(obj->vmobjlock)); */ + /* No - used from DDB. KASSERT(rw_write_held(obj->vmobjlock)); */ pg = radix_tree_lookup_node(&obj->uo_pages, off >> PAGE_SHIFT); @@ -1797,7 +1793,7 @@ void uvm_pagewire(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); #if defined(READAHEAD_STATS) if ((pg->flags & PG_READAHEAD) != 0) { @@ -1825,7 +1821,7 @@ void uvm_pageunwire(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(pg->wire_count != 0); KASSERT(!uvmpdpol_pageisqueued_p(pg)); KASSERT(mutex_owned(&pg->interlock)); @@ -1851,7 +1847,7 @@ void uvm_pagedeactivate(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); if (pg->wire_count == 0) { KASSERT(uvmpdpol_pageisqueued_p(pg)); @@ -1870,7 +1866,7 @@ void uvm_pageactivate(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); #if defined(READAHEAD_STATS) if ((pg->flags & PG_READAHEAD) != 0) { @@ -1893,7 +1889,7 @@ void uvm_pagedequeue(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); if (uvmpdpol_pageisqueued_p(pg)) { uvmpdpol_pagedequeue(pg); @@ -1911,7 +1907,7 @@ void uvm_pageenqueue(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); if (pg->wire_count == 0 && !uvmpdpol_pageisqueued_p(pg)) { uvmpdpol_pageenqueue(pg); @@ -2062,14 +2058,18 @@ uvm_page_lookup_freelist(struct vm_page */ bool -uvm_page_owner_locked_p(struct vm_page *pg) +uvm_page_owner_locked_p(struct vm_page *pg, bool exclusive) { if (pg->uobject != NULL) { - return mutex_owned(pg->uobject->vmobjlock); + return exclusive + ? rw_write_held(pg->uobject->vmobjlock) + : rw_lock_held(pg->uobject->vmobjlock); } if (pg->uanon != NULL) { - return mutex_owned(pg->uanon->an_lock); + return exclusive + ? rw_write_held(pg->uanon->an_lock) + : rw_lock_held(pg->uanon->an_lock); } return true; } @@ -2083,8 +2083,8 @@ uvm_pagereadonly_p(struct vm_page *pg) { struct uvm_object * const uobj = pg->uobject; - KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); - KASSERT(uobj != NULL || mutex_owned(pg->uanon->an_lock)); + KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock)); + KASSERT(uobj != NULL || rw_lock_held(pg->uanon->an_lock)); if ((pg->flags & PG_RDONLY) != 0) { return true; } Index: sys/uvm/uvm_page.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_page.h,v retrieving revision 1.97 diff -u -p -r1.97 uvm_page.h --- sys/uvm/uvm_page.h 21 Jan 2020 20:37:06 -0000 1.97 +++ sys/uvm/uvm_page.h 21 Jan 2020 21:09:20 -0000 @@ -351,7 +351,7 @@ void uvm_pageunwire(struct vm_page *); void uvm_pagewire(struct vm_page *); void uvm_pagezero(struct vm_page *); bool uvm_pageismanaged(paddr_t); -bool uvm_page_owner_locked_p(struct vm_page *); +bool uvm_page_owner_locked_p(struct vm_page *, bool); void uvm_pgfl_lock(void); void uvm_pgfl_unlock(void); unsigned int uvm_pagegetdirty(struct vm_page *); Index: sys/uvm/uvm_page_array.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_page_array.c,v retrieving revision 1.3 diff -u -p -r1.3 uvm_page_array.c --- sys/uvm/uvm_page_array.c 15 Jan 2020 17:55:45 -0000 1.3 +++ sys/uvm/uvm_page_array.c 21 Jan 2020 21:09:20 -0000 @@ -139,7 +139,7 @@ uvm_page_array_fill(struct uvm_page_arra maxpages = nwant; } #if 0 /* called from DDB for "show obj/f" without lock */ - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); #endif KASSERT(uvm_page_array_peek(ar) == NULL); if ((flags & UVM_PAGE_ARRAY_FILL_DIRTY) != 0) { Index: sys/uvm/uvm_page_status.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_page_status.c,v retrieving revision 1.2 diff -u -p -r1.2 uvm_page_status.c --- sys/uvm/uvm_page_status.c 15 Jan 2020 17:55:45 -0000 1.2 +++ sys/uvm/uvm_page_status.c 21 Jan 2020 21:09:20 -0000 @@ -63,7 +63,7 @@ uvm_pagegetdirty(struct vm_page *pg) const uint64_t idx __diagused = pg->offset >> PAGE_SHIFT; KASSERT((~pg->flags & (PG_CLEAN|PG_DIRTY)) != 0); - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, false)); KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) == !!radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG)); return pg->flags & (PG_CLEAN|PG_DIRTY); @@ -91,7 +91,7 @@ uvm_pagemarkdirty(struct vm_page *pg, un KASSERT((~newstatus & (PG_CLEAN|PG_DIRTY)) != 0); KASSERT((newstatus & ~(PG_CLEAN|PG_DIRTY)) == 0); - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) == !!radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG)); @@ -153,7 +153,7 @@ uvm_pagecheckdirty(struct vm_page *pg, b const unsigned int oldstatus = uvm_pagegetdirty(pg); bool modified; - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); /* * if pgprotected is true, mark the page CLEAN. Index: sys/uvm/uvm_pager.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pager.c,v retrieving revision 1.120 diff -u -p -r1.120 uvm_pager.c --- sys/uvm/uvm_pager.c 15 Jan 2020 17:55:45 -0000 1.120 +++ sys/uvm/uvm_pager.c 21 Jan 2020 21:09:21 -0000 @@ -305,7 +305,7 @@ uvm_aio_aiodone_pages(struct vm_page **p { struct uvm_object *uobj; struct vm_page *pg; - kmutex_t *slock; + krwlock_t *slock; int pageout_done; /* number of PG_PAGEOUT pages processed */ int swslot; int i; @@ -322,7 +322,7 @@ uvm_aio_aiodone_pages(struct vm_page **p if (!swap) { uobj = pg->uobject; slock = uobj->vmobjlock; - mutex_enter(slock); + rw_enter(slock, RW_WRITER); } else { #if defined(VMSWAP) if (error) { @@ -360,7 +360,7 @@ uvm_aio_aiodone_pages(struct vm_page **p } else { slock = pg->uanon->an_lock; } - mutex_enter(slock); + rw_enter(slock, RW_WRITER); anon_disposed = (pg->flags & PG_RELEASED) != 0; KASSERT(!anon_disposed || pg->uobject != NULL || pg->uanon->an_ref == 0); @@ -457,7 +457,7 @@ uvm_aio_aiodone_pages(struct vm_page **p uvm_anon_release(pg->uanon); } else { uvm_page_unbusy(&pg, 1); - mutex_exit(slock); + rw_exit(slock); } } #endif /* defined(VMSWAP) */ @@ -465,7 +465,7 @@ uvm_aio_aiodone_pages(struct vm_page **p uvm_pageout_done(pageout_done); if (!swap) { uvm_page_unbusy(pgs, npages); - mutex_exit(slock); + rw_exit(slock); } else { #if defined(VMSWAP) KASSERT(write); Index: sys/uvm/uvm_pdaemon.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdaemon.c,v retrieving revision 1.123 diff -u -p -r1.123 uvm_pdaemon.c --- sys/uvm/uvm_pdaemon.c 15 Jan 2020 17:55:45 -0000 1.123 +++ sys/uvm/uvm_pdaemon.c 21 Jan 2020 21:09:21 -0000 @@ -71,6 +71,8 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon. #include "opt_uvmhist.h" #include "opt_readahead.h" +#define __RWLOCK_PRIVATE + #include #include #include @@ -397,14 +399,14 @@ uvm_pageout_done(int npages) * => return the locked mutex on success. otherwise, return NULL. */ -kmutex_t * +krwlock_t * uvmpd_trylockowner(struct vm_page *pg) { struct uvm_object *uobj = pg->uobject; struct vm_anon *anon = pg->uanon; int tries, count; bool running; - kmutex_t *slock; + krwlock_t *slock; KASSERT(mutex_owned(&pg->interlock)); @@ -426,7 +428,7 @@ uvmpd_trylockowner(struct vm_page *pg) */ tries = (curlwp == uvm.pagedaemon_lwp ? UVMPD_NUMTRYLOCKOWNER : 1); for (;;) { - if (mutex_tryenter(slock)) { + if (rw_tryenter(slock, RW_WRITER)) { if (uobj == NULL) { /* * set PG_ANON if it isn't set already. @@ -441,7 +443,7 @@ uvmpd_trylockowner(struct vm_page *pg) mutex_exit(&pg->interlock); return slock; } - running = mutex_owner_running(slock); + running = rw_owner_running(slock); if (!running || --tries <= 0) { break; } @@ -520,12 +522,12 @@ swapcluster_add(struct swapcluster *swc, slot = swc->swc_slot + swc->swc_nused; uobj = pg->uobject; if (uobj == NULL) { - KASSERT(mutex_owned(pg->uanon->an_lock)); + KASSERT(rw_write_held(pg->uanon->an_lock)); pg->uanon->an_swslot = slot; } else { int result; - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); result = uao_set_swslot(uobj, pg->offset >> PAGE_SHIFT, slot); if (result == -1) { return ENOMEM; @@ -645,7 +647,7 @@ uvmpd_scan_queue(void) struct swapcluster swc; #endif /* defined(VMSWAP) */ int dirtyreacts; - kmutex_t *slock; + krwlock_t *slock; UVMHIST_FUNC("uvmpd_scan_queue"); UVMHIST_CALLED(pdhist); /* @@ -700,7 +702,7 @@ uvmpd_scan_queue(void) break; } KASSERT(uvmpdpol_pageisqueued_p(p)); - KASSERT(uvm_page_owner_locked_p(p)); + KASSERT(uvm_page_owner_locked_p(p, true)); KASSERT(p->wire_count == 0); /* @@ -711,7 +713,7 @@ uvmpd_scan_queue(void) uobj = p->uobject; if (p->flags & PG_BUSY) { - mutex_exit(slock); + rw_exit(slock); uvmexp.pdbusy++; continue; } @@ -790,7 +792,7 @@ uvmpd_scan_queue(void) KASSERT(uvmexp.swpgonly < uvmexp.swpginuse); atomic_inc_uint(&uvmexp.swpgonly); } - mutex_exit(slock); + rw_exit(slock); continue; } @@ -801,7 +803,7 @@ uvmpd_scan_queue(void) */ if (uvm_availmem() + uvmexp.paging > uvmexp.freetarg << 2) { - mutex_exit(slock); + rw_exit(slock); continue; } @@ -825,7 +827,7 @@ uvmpd_scan_queue(void) uvm_pagelock(p); uvm_pageactivate(p); uvm_pageunlock(p); - mutex_exit(slock); + rw_exit(slock); continue; } @@ -857,10 +859,10 @@ uvmpd_scan_queue(void) uvm_pagelock(p); uvm_pageactivate(p); uvm_pageunlock(p); - mutex_exit(slock); + rw_exit(slock); continue; } - mutex_exit(slock); + rw_exit(slock); swapcluster_flush(&swc, false); @@ -875,7 +877,7 @@ uvmpd_scan_queue(void) uvm_pagelock(p); uvm_pageactivate(p); uvm_pageunlock(p); - mutex_exit(slock); + rw_exit(slock); #endif /* defined(VMSWAP) */ } Index: sys/uvm/uvm_pdaemon.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdaemon.h,v retrieving revision 1.18 diff -u -p -r1.18 uvm_pdaemon.h --- sys/uvm/uvm_pdaemon.h 30 Dec 2019 18:08:38 -0000 1.18 +++ sys/uvm/uvm_pdaemon.h 21 Jan 2020 21:09:21 -0000 @@ -77,7 +77,7 @@ void uvm_wait(const char *); bool uvm_reclaimable(void); -kmutex_t *uvmpd_trylockowner(struct vm_page *); +krwlock_t *uvmpd_trylockowner(struct vm_page *); #ifdef VMSWAP bool uvmpd_dropswap(struct vm_page *); #else Index: sys/uvm/uvm_pdpolicy.h =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdpolicy.h,v retrieving revision 1.6 diff -u -p -r1.6 uvm_pdpolicy.h --- sys/uvm/uvm_pdpolicy.h 31 Dec 2019 22:42:51 -0000 1.6 +++ sys/uvm/uvm_pdpolicy.h 21 Jan 2020 21:09:21 -0000 @@ -55,7 +55,7 @@ void uvmpdpol_anfree(struct vm_anon *); void uvmpdpol_tune(void); void uvmpdpol_scaninit(void); void uvmpdpol_scanfini(void); -struct vm_page *uvmpdpol_selectvictim(kmutex_t **lock); +struct vm_page *uvmpdpol_selectvictim(krwlock_t **lock); void uvmpdpol_balancequeue(int); void uvmpdpol_sysctlsetup(void); Index: sys/uvm/uvm_pdpolicy_clock.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_pdpolicy_clock.c,v retrieving revision 1.31 diff -u -p -r1.31 uvm_pdpolicy_clock.c --- sys/uvm/uvm_pdpolicy_clock.c 21 Jan 2020 20:37:06 -0000 1.31 +++ sys/uvm/uvm_pdpolicy_clock.c 21 Jan 2020 21:09:21 -0000 @@ -236,12 +236,12 @@ uvmpdpol_scanfini(void) } struct vm_page * -uvmpdpol_selectvictim(kmutex_t **plock) +uvmpdpol_selectvictim(krwlock_t **plock) { struct uvmpdpol_globalstate *s = &pdpol_state; struct uvmpdpol_scanstate *ss = &pdpol_scanstate; struct vm_page *pg; - kmutex_t *lock; + krwlock_t *lock; mutex_enter(&s->lock); while (/* CONSTCOND */ 1) { @@ -335,7 +335,7 @@ uvmpdpol_selectvictim(kmutex_t **plock) uvmpdpol_pageactivate_locked(pg); mutex_exit(&pg->interlock); uvmexp.pdreact++; - mutex_exit(lock); + rw_exit(lock); continue; } @@ -353,7 +353,7 @@ uvmpdpol_balancequeue(int swap_shortage) struct uvmpdpol_globalstate *s = &pdpol_state; int inactive_shortage; struct vm_page *p, marker; - kmutex_t *lock; + krwlock_t *lock; /* * we have done the scan to get free pages. now we work on meeting @@ -429,7 +429,7 @@ uvmpdpol_balancequeue(int swap_shortage) uvmexp.pddeact++; inactive_shortage--; } - mutex_exit(lock); + rw_exit(lock); } TAILQ_REMOVE(&pdpol_state.s_activeq, &marker, pdqueue); mutex_exit(&s->lock); @@ -462,7 +462,7 @@ void uvmpdpol_pagedeactivate(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); /* @@ -493,7 +493,7 @@ void uvmpdpol_pageactivate(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); uvmpdpol_set_intent(pg, PQ_INTENT_A); @@ -524,7 +524,7 @@ void uvmpdpol_pagedequeue(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); uvmpdpol_set_intent(pg, PQ_INTENT_D); @@ -534,7 +534,7 @@ void uvmpdpol_pageenqueue(struct vm_page *pg) { - KASSERT(uvm_page_owner_locked_p(pg)); + KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); uvmpdpol_set_intent(pg, PQ_INTENT_E); Index: sys/uvm/uvm_readahead.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_readahead.c,v retrieving revision 1.10 diff -u -p -r1.10 uvm_readahead.c --- sys/uvm/uvm_readahead.c 19 May 2018 15:18:02 -0000 1.10 +++ sys/uvm/uvm_readahead.c 21 Jan 2020 21:09:21 -0000 @@ -133,9 +133,9 @@ ra_startio(struct uvm_object *uobj, off_ * too. This speeds up I/O using cache, since it avoids lookups and temporary * allocations done by full pgo_get. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); struct vm_page *pg = uvm_pagelookup(uobj, trunc_page(endoff - 1)); - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); if (pg != NULL) { DPRINTF(("%s: off=%" PRIu64 ", sz=%zu already cached\n", __func__, off, sz)); @@ -162,7 +162,7 @@ ra_startio(struct uvm_object *uobj, off_ * use UVM_ADV_RANDOM to avoid recursion. */ - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); error = (*uobj->pgops->pgo_get)(uobj, off, NULL, &npages, 0, VM_PROT_READ, UVM_ADV_RANDOM, PGO_NOTIMESTAMP); DPRINTF(("%s: off=%" PRIu64 ", bytelen=%zu -> %d\n", @@ -224,7 +224,7 @@ uvm_ra_request(struct uvm_ractx *ra, int off_t reqoff, size_t reqsize) { - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); if (ra == NULL || advice == UVM_ADV_RANDOM) { return; @@ -332,9 +332,9 @@ do_readahead: if (rasize >= RA_MINSIZE) { off_t next; - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); next = ra_startio(uobj, raoff, rasize); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); ra->ra_next = next; } } Index: sys/uvm/uvm_vnode.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_vnode.c,v retrieving revision 1.105 diff -u -p -r1.105 uvm_vnode.c --- sys/uvm/uvm_vnode.c 15 Jan 2020 17:55:45 -0000 1.105 +++ sys/uvm/uvm_vnode.c 21 Jan 2020 21:09:21 -0000 @@ -76,6 +76,7 @@ UVMHIST_DEFINE(ubchist); * functions */ +static void uvn_alloc_ractx(struct uvm_object *); static void uvn_detach(struct uvm_object *); static int uvn_get(struct uvm_object *, voff_t, struct vm_page **, int *, int, vm_prot_t, int, int); @@ -146,7 +147,7 @@ uvn_put(struct uvm_object *uobj, voff_t struct vnode *vp = (struct vnode *)uobj; int error; - KASSERT(mutex_owned(vp->v_interlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); error = VOP_PUTPAGES(vp, offlo, offhi, flags); return error; @@ -180,7 +181,7 @@ uvn_get(struct uvm_object *uobj, voff_t if (vp->v_type == VREG && (access_type & VM_PROT_WRITE) == 0 && (flags & PGO_LOCKED) == 0) { - vn_ra_allocctx(vp); + uvn_alloc_ractx(uobj); uvm_ra_request(vp->v_ractx, advice, uobj, offset, *npagesp << PAGE_SHIFT); } @@ -188,7 +189,7 @@ uvn_get(struct uvm_object *uobj, voff_t error = VOP_GETPAGES(vp, offset, pps, npagesp, centeridx, access_type, advice, flags); - KASSERT(((flags & PGO_LOCKED) != 0 && mutex_owned(vp->v_interlock)) || + KASSERT(((flags & PGO_LOCKED) != 0 && rw_lock_held(uobj->vmobjlock)) || (flags & PGO_LOCKED) == 0); return error; } @@ -265,7 +266,7 @@ uvn_findpage(struct uvm_object *uobj, vo UVMHIST_LOG(ubchist, "vp %#jx off 0x%jx", (uintptr_t)uobj, offset, 0, 0); - KASSERT(mutex_owned(uobj->vmobjlock)); + KASSERT(rw_write_held(uobj->vmobjlock)); if (*pgp != NULL) { UVMHIST_LOG(ubchist, "dontcare", 0,0,0,0); @@ -312,10 +313,10 @@ uvn_findpage(struct uvm_object *uobj, vo UVMHIST_LOG(ubchist, "nowait",0,0,0,0); return 0; } - mutex_exit(uobj->vmobjlock); + rw_exit(uobj->vmobjlock); uvm_wait("uvn_fp1"); uvm_page_array_clear(a); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); continue; } UVMHIST_LOG(ubchist, "alloced %#jx (color %ju)", @@ -337,10 +338,10 @@ uvn_findpage(struct uvm_object *uobj, vo pg->flags |= PG_WANTED; UVMHIST_LOG(ubchist, "wait %#jx (color %ju)", (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0); - UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, 0, - "uvn_fp2", 0); + UVM_UNLOCK_AND_WAIT_RW(pg, uobj->vmobjlock, 0, + "uvn_fp2", 0); uvm_page_array_clear(a); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); continue; } @@ -412,7 +413,7 @@ uvm_vnp_setsize(struct vnode *vp, voff_t voff_t oldsize; UVMHIST_FUNC("uvm_vnp_setsize"); UVMHIST_CALLED(ubchist); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); UVMHIST_LOG(ubchist, "vp %#jx old 0x%jx new 0x%jx", (uintptr_t)vp, vp->v_size, newsize, 0); @@ -435,24 +436,28 @@ uvm_vnp_setsize(struct vnode *vp, voff_t */ if (oldsize > pgend && oldsize != VSIZENOTSET && pgend >= 0) { (void) uvn_put(uobj, pgend, 0, PGO_FREE | PGO_SYNCIO); - mutex_enter(uobj->vmobjlock); + rw_enter(uobj->vmobjlock, RW_WRITER); } + mutex_enter(vp->v_interlock); vp->v_size = vp->v_writesize = newsize; - mutex_exit(uobj->vmobjlock); + mutex_exit(vp->v_interlock); + rw_exit(uobj->vmobjlock); } void uvm_vnp_setwritesize(struct vnode *vp, voff_t newsize) { - mutex_enter(vp->v_interlock); + rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); KASSERT(newsize != VSIZENOTSET && newsize >= 0); KASSERT(vp->v_size != VSIZENOTSET); KASSERT(vp->v_writesize != VSIZENOTSET); KASSERT(vp->v_size <= vp->v_writesize); KASSERT(vp->v_size <= newsize); + mutex_enter(vp->v_interlock); vp->v_writesize = newsize; mutex_exit(vp->v_interlock); + rw_exit(vp->v_uobj.vmobjlock); } bool @@ -479,3 +484,31 @@ uvn_needs_writefault_p(struct uvm_object return uvn_clean_p(uobj) || (vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP; } + +static void +uvn_alloc_ractx(struct uvm_object *uobj) +{ + struct vnode *vp = (struct vnode *)uobj; + struct uvm_ractx *ra = NULL; + + KASSERT(rw_write_held(uobj->vmobjlock)); + + if (vp->v_type != VREG) { + return; + } + if (vp->v_ractx != NULL) { + return; + } + if (vp->v_ractx == NULL) { + rw_exit(uobj->vmobjlock); + ra = uvm_ra_allocctx(); + rw_enter(uobj->vmobjlock, RW_WRITER); + if (ra != NULL && vp->v_ractx == NULL) { + vp->v_ractx = ra; + ra = NULL; + } + } + if (ra != NULL) { + uvm_ra_freectx(ra); + } +}