Index: uvm/uvm_bio.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_bio.c,v
retrieving revision 1.94
diff -u -p -r1.94 uvm_bio.c
--- uvm/uvm_bio.c	20 Apr 2018 18:58:10 -0000	1.94
+++ uvm/uvm_bio.c	11 May 2018 14:46:13 -0000
@@ -48,9 +48,9 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 
 
 #include <uvm/uvm.h>
 
-/*
- * global data structures
- */
+#ifdef __HAVE_PMAP_DIRECT
+#  define UBC_USE_PMAP_DIRECT
+#endif
 
 /*
  * local functions
@@ -59,6 +59,11 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 
 static int	ubc_fault(struct uvm_faultinfo *, vaddr_t, struct vm_page **,
 			  int, int, vm_prot_t, int);
 static struct ubc_map *ubc_find_mapping(struct uvm_object *, voff_t);
+#ifdef UBC_USE_PMAP_DIRECT
+static int __noinline ubc_uiomove_direct(struct uvm_object *, struct uio *, vsize_t,
+			  int, int);
+static void __noinline ubc_zerorange_direct(struct uvm_object *, off_t, size_t, int);
+#endif
 
 /*
  * local data structues
@@ -149,15 +154,12 @@ UBC_EVCNT_DEFINE(faultbusy)
 void
 ubc_init(void)
 {
-	struct ubc_map *umap;
-	vaddr_t va;
-	int i;
-
 	/*
 	 * Make sure ubc_winshift is sane.
 	 */
 	if (ubc_winshift < PAGE_SHIFT)
 		ubc_winshift = PAGE_SHIFT;
+	ubc_winsize = 1 << ubc_winshift;
 
 	/*
 	 * init ubc_object.
@@ -174,10 +176,7 @@ ubc_init(void)
 	if (ubc_object.umap == NULL)
 		panic("ubc_init: failed to allocate ubc_map");
 
-	if (ubc_winshift < PAGE_SHIFT) {
-		ubc_winshift = PAGE_SHIFT;
-	}
-	va = (vaddr_t)1L;
+	vaddr_t va = (vaddr_t)1L;
 #ifdef PMAP_PREFER
 	PMAP_PREFER(0, &va, 0, 0);	/* kernel is never topdown */
 	ubc_nqueues = va >> ubc_winshift;
@@ -185,13 +184,13 @@ ubc_init(void)
 		ubc_nqueues = 1;
 	}
 #endif
-	ubc_winsize = 1 << ubc_winshift;
 	ubc_object.inactive = kmem_alloc(UBC_NQUEUES *
 	    sizeof(struct ubc_inactive_head), KM_SLEEP);
-	for (i = 0; i < UBC_NQUEUES; i++) {
+	for (int i = 0; i < UBC_NQUEUES; i++) {
 		TAILQ_INIT(&ubc_object.inactive[i]);
 	}
-	for (i = 0; i < ubc_nwins; i++) {
+	for (int i = 0; i < ubc_nwins; i++) {
+		struct ubc_map *umap;
 		umap = &ubc_object.umap[i];
 		TAILQ_INSERT_TAIL(&ubc_object.inactive[i & (UBC_NQUEUES - 1)],
 				  umap, inactive);
@@ -199,7 +198,7 @@ ubc_init(void)
 
 	ubc_object.hash = hashinit(ubc_nwins, HASH_LIST, true,
 	    &ubc_object.hashmask);
-	for (i = 0; i <= ubc_object.hashmask; i++) {
+	for (int i = 0; i <= ubc_object.hashmask; i++) {
 		LIST_INIT(&ubc_object.hash[i]);
 	}
 
@@ -562,6 +561,7 @@ again:
 	    (uintptr_t)umap, umap->refcount, (uintptr_t)va, flags);
 
 	if (flags & UBC_FAULTBUSY) {
+		// XXX add offset from slot_offset?
 		int npages = (*lenp + PAGE_SIZE - 1) >> PAGE_SHIFT;
 		struct vm_page *pgs[npages];
 		int gpflags =
@@ -720,6 +720,7 @@ ubc_release(void *va, int flags)
  * ubc_uiomove: move data to/from an object.
  */
 
+int uiomove_direct = 1;
 int
 ubc_uiomove(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice,
     int flags)
@@ -732,6 +733,12 @@ ubc_uiomove(struct uvm_object *uobj, str
 	KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) ||
 	    ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ));
 
+#ifdef UBC_USE_PMAP_DIRECT
+	if (__predict_true(uiomove_direct)) {
+		return ubc_uiomove_direct(uobj, uio, todo, advice, flags);
+	}
+#endif
+
 	off = uio->uio_offset;
 	error = 0;
 	while (todo > 0) {
@@ -769,6 +776,9 @@ ubc_uiomove(struct uvm_object *uobj, str
 void
 ubc_zerorange(struct uvm_object *uobj, off_t off, size_t len, int flags)
 {
+
+	ubc_zerorange_direct(uobj, off, len, flags);
+#if 0
 	void *win;
 
 	/*
@@ -785,8 +795,198 @@ ubc_zerorange(struct uvm_object *uobj, o
 		off += bytelen;
 		len -= bytelen;
 	}
+#endif
+}
+
+#ifdef UBC_USE_PMAP_DIRECT
+/* Copy data using direct map */
+
+/*
+ * ubc_alloc_direct:  allocate a file mapping window using direct map
+ */
+static int __noinline
+ubc_alloc_direct(struct uvm_object *uobj, voff_t offset, vsize_t *lenp,
+    int advice, int flags, struct vm_page **pgs, int *npages)
+{
+	voff_t pgoff;
+	int error;
+	int gpflags = flags | PGO_NOTIMESTAMP | PGO_SYNCIO | PGO_ALLPAGES;
+	int access_type = VM_PROT_READ;
+
+	if (flags & UBC_WRITE) {
+		if (flags & UBC_FAULTBUSY)
+			gpflags |= PGO_OVERWRITE;
+#if 0
+		KASSERT(!UVM_OBJ_NEEDS_WRITEFAULT(uobj));
+#endif
+
+		gpflags |= PGO_PASTEOF;
+		access_type |= VM_PROT_WRITE;
+	}
+
+	pgoff = (offset & PAGE_MASK);
+	*lenp = MIN(*lenp, ubc_winsize - pgoff);
+
+again:
+	*npages = (*lenp + pgoff + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	KASSERT((*npages * PAGE_SIZE) <= ubc_winsize);
+	KASSERT(*lenp + pgoff <= ubc_winsize);
+	memset(pgs, 0, *npages * sizeof(pgs[0]));
+
+	mutex_enter(uobj->vmobjlock);
+	error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs,
+	    npages, 0, access_type, advice, gpflags);
+	UVMHIST_LOG(ubchist, "alloc_direct getpages %jd", error, 0, 0, 0);
+	if (error) {
+		if (error == EAGAIN) {
+			kpause("ubc_alloc_directg", false, hz >> 2, NULL);
+			goto again;
+		}
+		return error;
+	}
+
+	mutex_enter(uobj->vmobjlock);
+	for (int i = 0; i < *npages; i++) {
+		struct vm_page *pg = pgs[i];
+
+		KASSERT(pg != NULL);
+		KASSERT(pg != PGO_DONTCARE);
+		KASSERT((pg->flags & PG_FAKE) == 0 || (gpflags & PGO_OVERWRITE));
+		KASSERT(pg->uobject->vmobjlock == uobj->vmobjlock);
+
+		/* Avoid breaking loan if possible, only do it on write */
+		if ((flags & UBC_WRITE) && pg->loan_count != 0) {
+			pg = uvm_loanbreak(pg);
+			if (pg == NULL) {
+				uvm_page_unbusy(pgs, *npages);
+				mutex_exit(uobj->vmobjlock);
+				uvm_wait("ubc_alloc_directl");
+				goto again;
+			}
+			pgs[i] = pg;
+		}
+
+		/* Page must be writable by now */
+		KASSERT((pg->flags & PG_RDONLY) == 0 || (flags & UBC_WRITE) == 0);
+
+		mutex_enter(&uvm_pageqlock);
+		uvm_pageactivate(pg);
+		mutex_exit(&uvm_pageqlock);
+
+		/* Page will be changed, no longer clean */
+		if (flags & UBC_WRITE)
+			pg->flags &= ~(PG_FAKE|PG_CLEAN);
+	}
+	mutex_exit(uobj->vmobjlock);
+
+	return 0;
+}
+
+static int
+ubc_uiomove_process(void *win, size_t len, void *arg)
+{
+	struct uio *uio = (struct uio *)arg;
+
+	return uiomove(win, len, uio);
 }
 
+static int
+ubc_zerorange_process(void *win, size_t len, void *arg)
+{
+	memset(win, 0, len);
+	return 0;
+}
+
+static int __noinline
+ubc_uiomove_direct(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice,
+    int flags)
+{
+	const bool overwrite = (flags & UBC_FAULTBUSY) != 0;
+	voff_t off;
+	int error, npages;
+	struct vm_page *pgs[ubc_winsize >> PAGE_SHIFT];
+
+	KASSERT(todo <= uio->uio_resid);
+	KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) ||
+	    ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ));
+
+	off = uio->uio_offset;
+	error = 0;
+	while (todo > 0) {
+		vsize_t bytelen = todo;
+
+		error = ubc_alloc_direct(uobj, off, &bytelen, advice, flags,
+		    pgs, &npages);
+		if (error != 0) {
+			/* can't do anything, failed to get the pages */
+			break;
+		}
+
+		if (error == 0) {
+			error = pmap_direct_process(pgs, npages, off, bytelen,
+			    ubc_uiomove_process, uio);
+		}
+		if (error != 0 && overwrite) {
+			/*
+			 * if we haven't initialized the pages yet,
+			 * do it now.  it's safe to use memset here
+			 * because we just mapped the pages above.
+			 */
+			printf("%s: error=%d\n", __func__, error);
+			(void) pmap_direct_process(pgs, npages, off, bytelen,
+			    ubc_zerorange_process, NULL);
+		}
+
+		mutex_enter(uobj->vmobjlock);
+		uvm_page_unbusy(pgs, npages);
+		mutex_exit(uobj->vmobjlock);
+
+		off += bytelen;
+		todo -= bytelen;
+
+		if (error != 0 && ISSET(flags, UBC_PARTIALOK)) {
+			break;
+		}
+	}
+
+	return error;
+}
+
+static void __noinline
+ubc_zerorange_direct(struct uvm_object *uobj, off_t off, size_t todo, int flags)
+{
+	int error, npages;
+	struct vm_page *pgs[ubc_winsize >> PAGE_SHIFT];
+
+	error = 0;
+	while (todo > 0) {
+		vsize_t bytelen = todo;
+
+		error = ubc_alloc_direct(uobj, off, &bytelen, UVM_ADV_NORMAL,
+		    UBC_WRITE, pgs, &npages);
+		if (error != 0) {
+			/* can't do anything, failed to get the pages */
+			break;
+		}
+
+		error = pmap_direct_process(pgs, npages, off, bytelen,
+		    ubc_zerorange_process, NULL);
+
+		mutex_enter(uobj->vmobjlock);
+		uvm_page_unbusy(pgs, npages);
+		mutex_exit(uobj->vmobjlock);
+
+		off += bytelen;
+		todo -= bytelen;
+
+		if (error != 0 && ISSET(flags, UBC_PARTIALOK)) {
+			break;
+		}
+	}
+}
+
+#endif /* UBC_USE_PMAP_DIRECT */
+
 /*
  * ubc_purge: disassociate ubc_map structures from an empty uvm_object.
  */
Index: uvm/uvm_pmap.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_pmap.h,v
retrieving revision 1.38
diff -u -p -r1.38 uvm_pmap.h
--- uvm/uvm_pmap.h	2 Feb 2013 14:06:58 -0000	1.38
+++ uvm/uvm_pmap.h	11 May 2018 14:46:13 -0000
@@ -212,6 +212,13 @@ vaddr_t		pmap_steal_memory(vsize_t, vadd
 #if defined(PMAP_FORK)
 void		pmap_fork(pmap_t, pmap_t);
 #endif
+
+#ifdef __HAVE_PMAP_DIRECT
+int		pmap_direct_process(struct vm_page **, u_int, voff_t, vsize_t,
+		    int (*)(void *, size_t, void *),
+		    void *);
+#endif
+
 #endif	/* kernel*/
 #endif  /* PMAP_EXCLUDE_DECLS */
 
Index: arch/x86/x86/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/arch/x86/x86/pmap.c,v
retrieving revision 1.289
diff -u -p -r1.289 pmap.c
--- arch/x86/x86/pmap.c	4 Mar 2018 23:25:35 -0000	1.289
+++ arch/x86/x86/pmap.c	11 May 2018 14:46:13 -0000
@@ -4899,3 +4899,45 @@ x86_mmap_flags(paddr_t mdpgno)
 
 	return pflag;
 }
+
+#ifdef __HAVE_PMAP_DIRECT
+int
+pmap_direct_process(struct vm_page **pgs, u_int npages, voff_t off, vsize_t len,
+    int (*process)(void *, size_t, void *), void *arg)
+{
+	int error = 0;
+	paddr_t pa;
+	vaddr_t va;
+	size_t todo;
+	voff_t pgoff = (off & PAGE_MASK);
+	struct vm_page *pg;
+
+	KASSERT(npages > 0 && len > 0);
+
+	for (int i = 0; i < npages; i++) {
+		pg = pgs[i];
+
+		KASSERT(len > 0);
+
+		/*
+		 * Caller is responsible for ensuring all the pages are
+		 * available.
+		 */
+		KASSERT(pg != NULL && pg != PGO_DONTCARE);
+
+		pa = VM_PAGE_TO_PHYS(pg);
+		va = PMAP_DIRECT_MAP(pa);
+		todo = MIN(len, PAGE_SIZE - pgoff);
+
+		error = process((void *)(va + pgoff), todo, arg);
+		if (error)
+			break;
+
+		pgoff = 0;
+		len -= todo;
+	}
+
+	KASSERTMSG(error != 0 || len == 0, "len %lu != 0 for non-error", len);
+	return error;
+}
+#endif