Index: sys/sys/mount.h
===================================================================
RCS file: /cvsroot/src/sys/sys/mount.h,v
retrieving revision 1.211
diff -p -u -4 -r1.211 mount.h
--- sys/sys/mount.h	27 Feb 2014 13:00:06 -0000	1.211
+++ sys/sys/mount.h	3 Mar 2014 10:02:25 -0000
@@ -431,8 +431,11 @@ int	vfs_quotactl_cursorget(struct mount 
 int	vfs_quotactl_cursoratend(struct mount *, struct quotakcursor *, int *);
 int	vfs_quotactl_cursorrewind(struct mount *, struct quotakcursor *);
 int	vfs_quotactl_quotaon(struct mount *, int, const char *);
 int	vfs_quotactl_quotaoff(struct mount *, int);
+void	vfs_vnode_iterator_init(struct mount *, void **);
+void	vfs_vnode_iterator_destroy(void *);
+bool	vfs_vnode_iterator_next(void *, struct vnode **);
 
 extern	TAILQ_HEAD(mntlist, mount) mountlist;	/* mounted filesystem list */
 extern	struct vfsops *vfssw[];			/* filesystem type table */
 extern	int nvfssw;
Index: sys/kern/vfs_mount.c
===================================================================
RCS file: /cvsroot/src/sys/kern/vfs_mount.c,v
retrieving revision 1.26
diff -p -u -4 -r1.26 vfs_mount.c
--- sys/kern/vfs_mount.c	27 Feb 2014 13:00:06 -0000	1.26
+++ sys/kern/vfs_mount.c	3 Mar 2014 10:02:29 -0000
@@ -374,8 +374,68 @@ vunmark(vnode_t *mvp)
 
 	return vp;
 }
 
+void
+vfs_vnode_iterator_init(struct mount *mp, void **marker)
+{
+	struct vnode **mvpp = (struct vnode **)marker;
+
+	*mvpp = vnalloc(mp);
+
+	mutex_enter(&mntvnode_lock);
+	TAILQ_INSERT_HEAD(&mp->mnt_vnodelist, *mvpp, v_mntvnodes);
+	mutex_exit(&mntvnode_lock);
+}
+
+void
+vfs_vnode_iterator_destroy(void *marker)
+{
+	struct vnode *mvp = marker;
+
+	KASSERT((mvp->v_iflag & VI_MARKER) != 0);
+	vnfree(mvp);
+}
+
+bool
+vfs_vnode_iterator_next(void *marker, struct vnode **vpp)
+{
+	struct vnode *vp, *mvp = marker;
+	struct mount *mp = mvp->v_mount;
+	int error;
+
+	do {
+		mutex_enter(&mntvnode_lock);
+		vp = TAILQ_NEXT(mvp, v_mntvnodes);
+		TAILQ_REMOVE(&mp->mnt_vnodelist, mvp, v_mntvnodes);
+		if (vp == NULL) {
+	       		mutex_exit(&mntvnode_lock);
+			*vpp = NULL;
+	       		return false;
+		}
+
+		mutex_enter(vp->v_interlock);
+		while ((vp->v_iflag & VI_MARKER) != 0) {
+			mutex_exit(vp->v_interlock);
+			vp = TAILQ_NEXT(vp, v_mntvnodes);
+			if (vp == NULL) {
+				mutex_exit(&mntvnode_lock);
+				*vpp = NULL;
+				return false;
+			}
+			mutex_enter(vp->v_interlock);
+		}
+
+		TAILQ_INSERT_AFTER(&mp->mnt_vnodelist, vp, mvp, v_mntvnodes);
+		mutex_exit(&mntvnode_lock);
+		error = vget(vp, 0);
+		KASSERT(error == 0 || error == ENOENT);
+	} while (error != 0);
+
+	*vpp = vp;
+	return true;
+}
+
 /*
  * Move a vnode from one mount queue to another.
  */
 void
@@ -425,101 +485,80 @@ int busyprt = 0;	/* print out busy vnode
 struct ctldebug debug1 = { "busyprt", &busyprt };
 #endif
 
 static vnode_t *
-vflushnext(vnode_t *mvp, int *when)
+vflushnext(void *marker, int *when)
 {
+	struct vnode *vp;
 
 	if (hardclock_ticks > *when) {
-		mutex_exit(&mntvnode_lock);
 		yield();
-		mutex_enter(&mntvnode_lock);
 		*when = hardclock_ticks + hz / 10;
 	}
-	return vunmark(mvp);
+	if (vfs_vnode_iterator_next(marker, &vp))
+		return vp;
+	return NULL;
 }
 
 int
 vflush(struct mount *mp, vnode_t *skipvp, int flags)
 {
-	vnode_t *vp, *mvp;
+	vnode_t *vp;
+	void *marker;
 	int busy = 0, when = 0;
 
 	/* First, flush out any vnode references from vrele_list. */
 	vrele_flush();
 
-	/* Allocate a marker vnode. */
-	mvp = vnalloc(mp);
-
-	/*
-	 * NOTE: not using the TAILQ_FOREACH here since in this loop vgone()
-	 * and vclean() are called.
-	 */
-	mutex_enter(&mntvnode_lock);
-	for (vp = TAILQ_FIRST(&mp->mnt_vnodelist);
-	    vp != NULL;
-	    vp = vflushnext(mvp, &when)) {
-		vmark(mvp, vp);
-		if (vp->v_mount != mp || vismarker(vp))
-			continue;
+	vfs_vnode_iterator_init(mp, &marker);
+	while ((vp = vflushnext(marker, &when)) != NULL) {
 		/*
 		 * Skip over a selected vnode.
 		 */
-		if (vp == skipvp)
-			continue;
-		/*
-		 * First try to recycle the vnode.
-		 */
-		if (vrecycle(vp, &mntvnode_lock)) {
-			mutex_enter(&mntvnode_lock);
-			continue;
-		}
-		mutex_enter(vp->v_interlock);
-		/*
-		 * Ignore clean but still referenced vnodes.
-		 */
-		if ((vp->v_iflag & VI_CLEAN) != 0) {
-			mutex_exit(vp->v_interlock);
+		if (vp == skipvp) {
+			vrele(vp);
 			continue;
 		}
 		/*
 		 * Skip over a vnodes marked VSYSTEM.
 		 */
 		if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) {
-			mutex_exit(vp->v_interlock);
+			vrele(vp);
 			continue;
 		}
 		/*
 		 * If WRITECLOSE is set, only flush out regular file
 		 * vnodes open for writing.
 		 */
-		if ((flags & WRITECLOSE) &&
-		    (vp->v_writecount == 0 || vp->v_type != VREG)) {
+		if ((flags & WRITECLOSE) && vp->v_type == VREG) {
+			mutex_enter(vp->v_interlock);
+			if (vp->v_writecount == 0) {
+				mutex_exit(vp->v_interlock);
+				vrele(vp);
+				continue;
+			}
 			mutex_exit(vp->v_interlock);
-			continue;
 		}
 		/*
+		 * First try to recycle the vnode.
+		 */
+		if (vrecycle(vp))
+			continue;
+		/*
 		 * If FORCECLOSE is set, forcibly close the vnode.
-		 * For block or character devices, revert to an
-		 * anonymous device.  For all other files, just
-		 * kill them.
 		 */
 		if (flags & FORCECLOSE) {
-			mutex_exit(&mntvnode_lock);
-			if (vget(vp, 0) == 0)
-				vgone(vp);
-			mutex_enter(&mntvnode_lock);
+			vgone(vp);
 			continue;
 		}
 #ifdef DEBUG
 		if (busyprt)
 			vprint("vflush: busy vnode", vp);
 #endif
-		mutex_exit(vp->v_interlock);
+		vrele(vp);
 		busy++;
 	}
-	mutex_exit(&mntvnode_lock);
-	vnfree(mvp);
+	vfs_vnode_iterator_destroy(marker);
 	if (busy)
 		return (EBUSY);
 	return (0);
 }
Index: sys/sys/vnode.h
===================================================================
RCS file: /cvsroot/src/sys/sys/vnode.h,v
retrieving revision 1.243
diff -p -u -4 -r1.243 vnode.h
--- sys/sys/vnode.h	1 Dec 2013 17:29:40 -0000	1.243
+++ sys/sys/vnode.h	3 Mar 2014 10:02:32 -0000
@@ -544,9 +544,9 @@ int 	vget(struct vnode *, int);
 void 	vgone(struct vnode *);
 int	vinvalbuf(struct vnode *, int, kauth_cred_t, struct lwp *, bool, int);
 void	vprint(const char *, struct vnode *);
 void 	vput(struct vnode *);
-int	vrecycle(struct vnode *, kmutex_t *);
+bool	vrecycle(struct vnode *);
 void 	vrele(struct vnode *);
 void 	vrele_async(struct vnode *);
 void	vrele_flush(void);
 int	vtruncbuf(struct vnode *, daddr_t, bool, int);
Index: sys/kern/vfs_vnode.c
===================================================================
RCS file: /cvsroot/src/sys/kern/vfs_vnode.c,v
retrieving revision 1.32
diff -p -u -4 -r1.32 vfs_vnode.c
--- sys/kern/vfs_vnode.c	27 Feb 2014 16:51:38 -0000	1.32
+++ sys/kern/vfs_vnode.c	3 Mar 2014 10:02:36 -0000
@@ -1028,32 +1028,35 @@ vclean(vnode_t *vp)
 	KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
 }
 
 /*
- * Recycle an unused vnode to the front of the free list.
- * Release the passed interlock if the vnode will be recycled.
+ * Recycle an unused vnode if caller holds the last reference.
  */
-int
-vrecycle(vnode_t *vp, kmutex_t *inter_lkp)
+bool
+vrecycle(vnode_t *vp)
 {
 
+	mutex_enter(vp->v_interlock);
+
 	KASSERT((vp->v_iflag & VI_MARKER) == 0);
 
-	mutex_enter(vp->v_interlock);
-	if (vp->v_usecount != 0 || (vp->v_iflag & (VI_CLEAN|VI_XLOCK)) != 0) {
+	if (vp->v_usecount != 1) {
 		mutex_exit(vp->v_interlock);
-		return 0;
+		return false;
 	}
-	if (inter_lkp) {
-		mutex_exit(inter_lkp);
+	if ((vp->v_iflag & VI_CHANGING) != 0)
+		vwait(vp, VI_CHANGING);
+	if (vp->v_usecount != 1) {
+		mutex_exit(vp->v_interlock);
+		return false;
+	} else if ((vp->v_iflag & VI_CLEAN) != 0) {
+		mutex_exit(vp->v_interlock);
+		return true;
 	}
-	vremfree(vp);
-	vp->v_usecount = 1;
-	KASSERT((vp->v_iflag & VI_CHANGING) == 0);
 	vp->v_iflag |= VI_CHANGING;
 	vclean(vp);
 	vrelel(vp, VRELEL_CHANGING_SET);
-	return 1;
+	return true;
 }
 
 /*
  * Eliminate all activity associated with the requested vnode
Index: sys/ufs/ext2fs/ext2fs_vfsops.c
===================================================================
RCS file: /cvsroot/src/sys/ufs/ext2fs/ext2fs_vfsops.c,v
retrieving revision 1.176
diff -p -u -4 -r1.176 ext2fs_vfsops.c
--- sys/ufs/ext2fs/ext2fs_vfsops.c	25 Feb 2014 18:30:13 -0000	1.176
+++ sys/ufs/ext2fs/ext2fs_vfsops.c	3 Mar 2014 10:02:39 -0000
@@ -511,15 +511,15 @@ fail:
  */
 int
 ext2fs_reload(struct mount *mp, kauth_cred_t cred, struct lwp *l)
 {
-	struct vnode *vp, *mvp, *devvp;
+	struct vnode *vp, *devvp;
 	struct inode *ip;
 	struct buf *bp;
 	struct m_ext2fs *fs;
 	struct ext2fs *newfs;
 	int i, error;
-	void *cp;
+	void *cp, *marker;
 	struct ufsmount *ump;
 
 	if ((mp->mnt_flag & MNT_RDONLY) == 0)
 		return (EINVAL);
@@ -584,37 +584,21 @@ ext2fs_reload(struct mount *mp, kauth_cr
 		    fs->e2fs_bsize);
 		brelse(bp, 0);
 	}
 
-	/* Allocate a marker vnode. */
-	mvp = vnalloc(mp);
-	/*
-	 * NOTE: not using the TAILQ_FOREACH here since in this loop vgone()
-	 * and vclean() can be called indirectly
-	 */
-	mutex_enter(&mntvnode_lock);
-loop:
-	for (vp = TAILQ_FIRST(&mp->mnt_vnodelist); vp; vp = vunmark(mvp)) {
-		vmark(mvp, vp);
-		if (vp->v_mount != mp || vismarker(vp))
-			continue;
+	vfs_vnode_iterator_init(mp, &marker);
+	while (vfs_vnode_iterator_next(marker, &vp)) {
 		/*
 		 * Step 4: invalidate all inactive vnodes.
 		 */
-		if (vrecycle(vp, &mntvnode_lock)) {
-			mutex_enter(&mntvnode_lock);
-			(void)vunmark(mvp);
-			goto loop;
-		}
+		if (vrecycle(vp))
+			continue;
 		/*
 		 * Step 5: invalidate all cached file data.
 		 */
-		mutex_enter(vp->v_interlock);
-		mutex_exit(&mntvnode_lock);
-		if (vget(vp, LK_EXCLUSIVE)) {
-			mutex_enter(&mntvnode_lock);
-			(void)vunmark(mvp);
-			goto loop;
+		if (vn_lock(vp, LK_EXCLUSIVE)) {
+			vrele(vp);
+			continue;
 		}
 		if (vinvalbuf(vp, 0, cred, l, 0, 0))
 			panic("ext2fs_reload: dirty2");
 		/*
@@ -624,22 +608,18 @@ loop:
 		error = bread(devvp, EXT2_FSBTODB(fs, ino_to_fsba(fs, ip->i_number)),
 		    (int)fs->e2fs_bsize, NOCRED, 0, &bp);
 		if (error) {
 			vput(vp);
-			mutex_enter(&mntvnode_lock);
-			(void)vunmark(mvp);
 			break;
 		}
 		cp = (char *)bp->b_data +
 		    (ino_to_fsbo(fs, ip->i_number) * EXT2_DINODE_SIZE(fs));
 		e2fs_iload((struct ext2fs_dinode *)cp, ip->i_din.e2fs_din);
 		ext2fs_set_inode_guid(ip);
 		brelse(bp, 0);
 		vput(vp);
-		mutex_enter(&mntvnode_lock);
 	}
-	mutex_exit(&mntvnode_lock);
-	vnfree(mvp);
+	vfs_vnode_iterator_destroy(marker);
 	return (error);
 }
 
 /*
Index: sys/ufs/ffs/ffs_vfsops.c
===================================================================
RCS file: /cvsroot/src/sys/ufs/ffs/ffs_vfsops.c,v
retrieving revision 1.292
diff -p -u -4 -r1.292 ffs_vfsops.c
--- sys/ufs/ffs/ffs_vfsops.c	25 Feb 2014 18:30:13 -0000	1.292
+++ sys/ufs/ffs/ffs_vfsops.c	3 Mar 2014 10:02:43 -0000
@@ -635,11 +635,11 @@ fail:
  */
 int
 ffs_reload(struct mount *mp, kauth_cred_t cred, struct lwp *l)
 {
-	struct vnode *vp, *mvp, *devvp;
+	struct vnode *vp, *devvp;
 	struct inode *ip;
-	void *space;
+	void *space, *marker;
 	struct buf *bp;
 	struct fs *fs, *newfs;
 	struct dkwedge_info dkw;
 	int i, bsize, blks, error;
@@ -804,36 +804,21 @@ ffs_reload(struct mount *mp, kauth_cred_
 		for (i = 0; i < fs->fs_ncg; i++)
 			*lp++ = fs->fs_contigsumsize;
 	}
 
-	/* Allocate a marker vnode. */
-	mvp = vnalloc(mp);
-	/*
-	 * NOTE: not using the TAILQ_FOREACH here since in this loop vgone()
-	 * and vclean() can be called indirectly
-	 */
-	mutex_enter(&mntvnode_lock);
- loop:
-	for (vp = TAILQ_FIRST(&mp->mnt_vnodelist); vp; vp = vunmark(mvp)) {
-		vmark(mvp, vp);
-		if (vp->v_mount != mp || vismarker(vp))
-			continue;
+	vfs_vnode_iterator_init(mp, &marker);
+	while (vfs_vnode_iterator_next(marker, &vp)) {
 		/*
 		 * Step 4: invalidate all inactive vnodes.
 		 */
-		if (vrecycle(vp, &mntvnode_lock)) {
-			mutex_enter(&mntvnode_lock);
-			(void)vunmark(mvp);
-			goto loop;
-		}
+		if (vrecycle(vp))
+			continue;
 		/*
 		 * Step 5: invalidate all cached file data.
 		 */
-		mutex_enter(vp->v_interlock);
-		mutex_exit(&mntvnode_lock);
-		if (vget(vp, LK_EXCLUSIVE)) {
-			(void)vunmark(mvp);
-			goto loop;
+		if (vn_lock(vp, LK_EXCLUSIVE)) {
+			vrele(vp);
+			continue;
 		}
 		if (vinvalbuf(vp, 0, cred, l, 0, 0))
 			panic("ffs_reload: dirty2");
 		/*
@@ -843,18 +828,15 @@ ffs_reload(struct mount *mp, kauth_cred_
 		error = bread(devvp, FFS_FSBTODB(fs, ino_to_fsba(fs, ip->i_number)),
 			      (int)fs->fs_bsize, NOCRED, 0, &bp);
 		if (error) {
 			vput(vp);
-			(void)vunmark(mvp);
 			break;
 		}
 		ffs_load_inode(bp, ip, fs, ip->i_number);
 		brelse(bp, 0);
 		vput(vp);
-		mutex_enter(&mntvnode_lock);
 	}
-	mutex_exit(&mntvnode_lock);
-	vnfree(mvp);
+	vfs_vnode_iterator_destroy(marker);
 	return (error);
 }
 
 /*
Index: sys/ufs/lfs/lfs_syscalls.c
===================================================================
RCS file: /cvsroot/src/sys/ufs/lfs/lfs_syscalls.c,v
retrieving revision 1.150
diff -p -u -4 -r1.150 lfs_syscalls.c
--- sys/ufs/lfs/lfs_syscalls.c	29 Oct 2013 09:53:51 -0000	1.150
+++ sys/ufs/lfs/lfs_syscalls.c	3 Mar 2014 10:02:46 -0000
@@ -711,10 +711,15 @@ lfs_bmapv(struct proc *p, fsid_t *fsidp,
 			 * v_daddr.
 			 */
 			if (v_daddr != LFS_UNUSED_DADDR) {
 				lfs_vunref(vp);
-				if (VTOI(vp)->i_lfs_iflags & LFSI_BMAP)
-					vrecycle(vp, NULL);
+				if (VTOI(vp)->i_lfs_iflags & LFSI_BMAP) {
+					mutex_enter(vp->v_interlock);
+					if (vget(vp, LK_NOWAIT) == 0) {
+						if (! vrecycle(vp))
+							vrele(vp);
+					}
+				}
 				numrefed--;
 			}
 
 			/*
@@ -821,10 +826,15 @@ lfs_bmapv(struct proc *p, fsid_t *fsidp,
 	 */
 	if (v_daddr != LFS_UNUSED_DADDR) {
 		lfs_vunref(vp);
 		/* Recycle as above. */
-		if (ip->i_lfs_iflags & LFSI_BMAP)
-			vrecycle(vp, NULL);
+		if (ip->i_lfs_iflags & LFSI_BMAP) {
+			mutex_enter(vp->v_interlock);
+			if (vget(vp, LK_NOWAIT) == 0) {
+				if (! vrecycle(vp))
+					vrele(vp);
+			}
+		}
 		numrefed--;
 	}
 
 #ifdef DIAGNOSTIC