Index: sys/pipe.h
===================================================================
RCS file: /cvsroot/src/sys/sys/pipe.h,v
retrieving revision 1.24
diff -u -p -r1.24 pipe.h
--- sys/pipe.h 29 Feb 2008 12:04:48 -0000 1.24
+++ sys/pipe.h 30 Dec 2008 13:40:34 -0000
@@ -115,7 +115,8 @@ struct pipe {
pid_t pipe_pgid; /* process group for sigio */
struct pipe *pipe_peer; /* link with other direction */
u_int pipe_state; /* pipe status info */
- int pipe_busy; /* busy flag, mostly to handle rundown sanely */
+ int pipe_busy; /* busy flag, to handle rundown */
+ vaddr_t pipe_kmem; /* preallocated PIPE_SIZE buffer */
};
/*
Index: kern/sys_pipe.c
===================================================================
RCS file: /cvsroot/src/sys/kern/sys_pipe.c,v
retrieving revision 1.103
diff -u -p -r1.103 sys_pipe.c
--- kern/sys_pipe.c 17 Sep 2008 14:00:41 -0000 1.103
+++ kern/sys_pipe.c 30 Dec 2008 13:40:31 -0000
@@ -46,21 +46,13 @@
* John S. Dyson.
* 4. Modifications may be freely made to this file if the above conditions
* are met.
- *
- * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.95 2002/03/09 22:06:31 alfred Exp $
*/
/*
* This file contains a high-performance replacement for the socket-based
- * pipes scheme originally used in FreeBSD/4.4Lite. It does not support
- * all features of sockets, but does do everything that pipes normally
- * do.
+ * pipes scheme originally used. It does not support all features of
+ * sockets, but does do everything that pipes normally do.
*
- * Adaption for NetBSD UVM, including uvm_loan() based direct write, was
- * written by Jaromir Dolecek.
- */
-
-/*
* This code has two modes of operation, a small write mode and a large
* write mode. The small write mode acts like conventional pipes with
* a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the
@@ -102,10 +94,7 @@ __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v
#include <uvm/uvm.h>
-/*
- * Use this define if you want to disable *fancy* VM things. Expect an
- * approx 30% decrease in transfer rate.
- */
+/* Use this define if you want to disable *fancy* VM things. */
/* #define PIPE_NODIRECT */
/*
@@ -127,15 +116,6 @@ static const struct fileops pipeops = {
};
/*
- * Single mutex shared between both ends of the pipe.
- */
-
-struct pipe_mutex {
- kmutex_t pm_mutex;
- u_int pm_refcnt;
-};
-
-/*
* Default pipe buffer size(s), this can be kind-of large now because pipe
* space is pageable. The pipe code will try to maintain locality of
* reference for performance reasons, so small amounts of outstanding I/O
@@ -172,7 +152,7 @@ static u_int amountpipekva = 0;
static void pipeclose(struct file *fp, struct pipe *pipe);
static void pipe_free_kmem(struct pipe *pipe);
-static int pipe_create(struct pipe **pipep, int allockva, struct pipe_mutex *);
+static int pipe_create(struct pipe **pipep, pool_cache_t, kmutex_t *);
static int pipelock(struct pipe *pipe, int catch);
static inline void pipeunlock(struct pipe *pipe);
static void pipeselwakeup(struct pipe *pipe, struct pipe *sigp, int code);
@@ -181,51 +161,77 @@ static int pipe_direct_write(struct file
struct uio *uio);
#endif
static int pipespace(struct pipe *pipe, int size);
+static int pipe_ctor(void *, void *, int);
+static void pipe_dtor(void *, void *);
#ifndef PIPE_NODIRECT
static int pipe_loan_alloc(struct pipe *, int);
static void pipe_loan_free(struct pipe *);
#endif /* PIPE_NODIRECT */
-static int pipe_mutex_ctor(void *, void *, int);
-static void pipe_mutex_dtor(void *, void *);
-
-static pool_cache_t pipe_cache;
-static pool_cache_t pipe_mutex_cache;
+static pool_cache_t pipe_wr_cache;
+static pool_cache_t pipe_rd_cache;
void
pipe_init(void)
{
- pipe_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipepl",
- NULL, IPL_NONE, NULL, NULL, NULL);
- KASSERT(pipe_cache != NULL);
-
- pipe_mutex_cache = pool_cache_init(sizeof(struct pipe_mutex),
- coherency_unit, 0, 0, "pipemtxpl", NULL, IPL_NONE, pipe_mutex_ctor,
- pipe_mutex_dtor, NULL);
- KASSERT(pipe_cache != NULL);
+ /* Writer side is not automatically allocated KVA. */
+ pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr",
+ NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL);
+ KASSERT(pipe_wr_cache != NULL);
+
+ /* Reader side gets preallocated KVA. */
+ pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd",
+ NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1);
+ KASSERT(pipe_rd_cache != NULL);
}
static int
-pipe_mutex_ctor(void *arg, void *obj, int flag)
+pipe_ctor(void *arg, void *obj, int flags)
{
- struct pipe_mutex *pm = obj;
+ struct pipe *pipe;
+ vaddr_t va;
- mutex_init(&pm->pm_mutex, MUTEX_DEFAULT, IPL_NONE);
- pm->pm_refcnt = 0;
+ pipe = obj;
+
+ memset(pipe, 0, sizeof(struct pipe));
+ if (arg != NULL) {
+ /* Preallocate space. */
+ va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, UVM_KMF_PAGEABLE);
+ if (va == 0) {
+ return ENOMEM;
+ }
+ pipe->pipe_kmem = va;
+ atomic_add_int(&amountpipekva, PIPE_SIZE);
+ }
+ cv_init(&pipe->pipe_rcv, "piperd");
+ cv_init(&pipe->pipe_wcv, "pipewr");
+ cv_init(&pipe->pipe_draincv, "pipedrain");
+ cv_init(&pipe->pipe_lkcv, "pipelk");
+ selinit(&pipe->pipe_sel);
+ pipe->pipe_state = PIPE_SIGNALR;
return 0;
}
static void
-pipe_mutex_dtor(void *arg, void *obj)
+pipe_dtor(void *arg, void *obj)
{
- struct pipe_mutex *pm = obj;
+ struct pipe *pipe;
- KASSERT(pm->pm_refcnt == 0);
+ pipe = obj;
- mutex_destroy(&pm->pm_mutex);
+ cv_destroy(&pipe->pipe_rcv);
+ cv_destroy(&pipe->pipe_wcv);
+ cv_destroy(&pipe->pipe_draincv);
+ cv_destroy(&pipe->pipe_lkcv);
+ seldestroy(&pipe->pipe_sel);
+ if (pipe->pipe_kmem != 0) {
+ uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE,
+ UVM_KMF_PAGEABLE);
+ atomic_add_int(&amountpipekva, -PIPE_SIZE);
+ }
}
/*
@@ -238,16 +244,18 @@ sys_pipe(struct lwp *l, const void *v, r
{
struct file *rf, *wf;
struct pipe *rpipe, *wpipe;
- struct pipe_mutex *mutex;
+ kmutex_t *mutex;
int fd, error;
proc_t *p;
p = curproc;
rpipe = wpipe = NULL;
- mutex = pool_cache_get(pipe_mutex_cache, PR_WAITOK);
+ mutex = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
if (mutex == NULL)
return (ENOMEM);
- if (pipe_create(&rpipe, 1, mutex) || pipe_create(&wpipe, 0, mutex)) {
+ mutex_obj_hold(mutex);
+ if (pipe_create(&rpipe, pipe_rd_cache, mutex) ||
+ pipe_create(&wpipe, pipe_wr_cache, mutex)) {
pipeclose(NULL, rpipe);
pipeclose(NULL, wpipe);
return (ENFILE);
@@ -292,18 +300,27 @@ free2:
* it will retain the old buffer.
* If it fails it will return ENOMEM.
*/
+int xxxad;
+
static int
pipespace(struct pipe *pipe, int size)
{
void *buffer;
+
/*
- * Allocate pageable virtual address space. Physical memory is
+ * Allocate pageable virtual address space. Physical memory is
* allocated on demand.
*/
- buffer = (void *) uvm_km_alloc(kernel_map, round_page(size), 0,
- UVM_KMF_PAGEABLE);
- if (buffer == NULL)
- return (ENOMEM);
+ if (size == PIPE_SIZE && pipe->pipe_kmem != 0) {
+ buffer = (void *)pipe->pipe_kmem;
+ } else {
+ buffer = (void *)uvm_km_alloc(kernel_map, round_page(size),
+ 0, UVM_KMF_PAGEABLE);
+ if (buffer == NULL)
+ return (ENOMEM);
+ atomic_add_int(&amountpipekva, size);
+ xxxad++; /* XXXAD */
+ }
/* free old resources if we're resizing */
pipe_free_kmem(pipe);
@@ -312,7 +329,6 @@ pipespace(struct pipe *pipe, int size)
pipe->pipe_buffer.in = 0;
pipe->pipe_buffer.out = 0;
pipe->pipe_buffer.cnt = 0;
- atomic_add_int(&amountpipekva, pipe->pipe_buffer.size);
return (0);
}
@@ -320,35 +336,24 @@ pipespace(struct pipe *pipe, int size)
* Initialize and allocate VM and memory for pipe.
*/
static int
-pipe_create(struct pipe **pipep, int allockva, struct pipe_mutex *mutex)
+pipe_create(struct pipe **pipep, pool_cache_t cache, kmutex_t *mutex)
{
struct pipe *pipe;
int error;
- pipe = *pipep = pool_cache_get(pipe_cache, PR_WAITOK);
- mutex->pm_refcnt++;
-
- /* Initialize */
- memset(pipe, 0, sizeof(struct pipe));
- pipe->pipe_state = PIPE_SIGNALR;
-
+ pipe = pool_cache_get(cache, PR_WAITOK);
+ *pipep = pipe;
+ error = 0;
getmicrotime(&pipe->pipe_ctime);
pipe->pipe_atime = pipe->pipe_ctime;
pipe->pipe_mtime = pipe->pipe_ctime;
- pipe->pipe_lock = &mutex->pm_mutex;
- cv_init(&pipe->pipe_rcv, "piperd");
- cv_init(&pipe->pipe_wcv, "pipewr");
- cv_init(&pipe->pipe_draincv, "pipedrain");
- cv_init(&pipe->pipe_lkcv, "pipelk");
- selinit(&pipe->pipe_sel);
-
- if (allockva && (error = pipespace(pipe, PIPE_SIZE)))
- return (error);
-
- return (0);
+ pipe->pipe_lock = mutex;
+ if (cache == pipe_rd_cache) {
+ error = pipespace(pipe, PIPE_SIZE);
+ }
+ return error;
}
-
/*
* Lock a pipe for I/O, blocking other access
* Called with pipe spin lock held.
@@ -1208,12 +1213,16 @@ pipe_free_kmem(struct pipe *pipe)
{
if (pipe->pipe_buffer.buffer != NULL) {
- if (pipe->pipe_buffer.size > PIPE_SIZE)
+ if (pipe->pipe_buffer.size > PIPE_SIZE) {
atomic_dec_uint(&nbigpipe);
- uvm_km_free(kernel_map,
- (vaddr_t)pipe->pipe_buffer.buffer,
- pipe->pipe_buffer.size, UVM_KMF_PAGEABLE);
- atomic_add_int(&amountpipekva, -pipe->pipe_buffer.size);
+ }
+ if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) {
+ uvm_km_free(kernel_map,
+ (vaddr_t)pipe->pipe_buffer.buffer,
+ pipe->pipe_buffer.size, UVM_KMF_PAGEABLE);
+ atomic_add_int(&amountpipekva,
+ -pipe->pipe_buffer.size);
+ }
pipe->pipe_buffer.buffer = NULL;
}
#ifndef PIPE_NODIRECT
@@ -1233,10 +1242,8 @@ pipe_free_kmem(struct pipe *pipe)
static void
pipeclose(struct file *fp, struct pipe *pipe)
{
- struct pipe_mutex *mutex;
kmutex_t *lock;
struct pipe *ppipe;
- u_int refcnt;
if (pipe == NULL)
return;
@@ -1273,24 +1280,20 @@ pipeclose(struct file *fp, struct pipe *
}
KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0);
-
- mutex = (struct pipe_mutex *)lock;
- refcnt = --(mutex->pm_refcnt);
- KASSERT(refcnt == 0 || refcnt == 1);
mutex_exit(lock);
/*
* free resources
*/
+ pipe->pipe_pgid = 0;
+ pipe->pipe_state = PIPE_SIGNALR;
pipe_free_kmem(pipe);
- cv_destroy(&pipe->pipe_rcv);
- cv_destroy(&pipe->pipe_wcv);
- cv_destroy(&pipe->pipe_draincv);
- cv_destroy(&pipe->pipe_lkcv);
- seldestroy(&pipe->pipe_sel);
- pool_cache_put(pipe_cache, pipe);
- if (refcnt == 0)
- pool_cache_put(pipe_mutex_cache, mutex);
+ if (pipe->pipe_kmem != 0) {
+ pool_cache_put(pipe_rd_cache, pipe);
+ } else {
+ pool_cache_put(pipe_wr_cache, pipe);
+ }
+ mutex_obj_free(lock);
}
static void