Index: kern/kern_exit.c =================================================================== RCS file: /cvsroot/src/sys/kern/kern_exit.c,v retrieving revision 1.287 diff -u -p -r1.287 kern_exit.c --- kern/kern_exit.c 4 Apr 2020 20:20:12 -0000 1.287 +++ kern/kern_exit.c 4 Apr 2020 21:02:36 -0000 @@ -426,7 +426,7 @@ exit1(struct lwp *l, int exitcode, int s fixjobc(p, p->p_pgrp, 0); /* Release fstrans private data. */ - fstrans_lwp_dtor(l); + fstrans_lwp_free(l); /* * Finalize the last LWP's specificdata, as well as the Index: kern/kern_lwp.c =================================================================== RCS file: /cvsroot/src/sys/kern/kern_lwp.c,v retrieving revision 1.233 diff -u -p -r1.233 kern_lwp.c --- kern/kern_lwp.c 4 Apr 2020 20:20:12 -0000 1.233 +++ kern/kern_lwp.c 4 Apr 2020 21:02:36 -0000 @@ -1205,7 +1205,7 @@ lwp_exit(struct lwp *l) fd_free(); /* Release fstrans private data. */ - fstrans_lwp_dtor(l); + fstrans_lwp_free(l); /* Delete the specificdata while it's still safe to sleep. */ lwp_finispecific(l); Index: kern/vfs_trans.c =================================================================== RCS file: /cvsroot/src/sys/kern/vfs_trans.c,v retrieving revision 1.61 diff -u -p -r1.61 vfs_trans.c --- kern/vfs_trans.c 17 Jun 2019 08:07:27 -0000 1.61 +++ kern/vfs_trans.c 4 Apr 2020 21:02:36 -0000 @@ -1,7 +1,7 @@ /* $NetBSD: vfs_trans.c,v 1.61 2019/06/17 08:07:27 hannken Exp $ */ /*- - * Copyright (c) 2007 The NetBSD Foundation, Inc. + * Copyright (c) 2007, 2020 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -50,6 +50,7 @@ __KERNEL_RCSID(0, "$NetBSD: vfs_trans.c, #include #include #include +#include #include @@ -85,21 +86,28 @@ struct fstrans_mount_info { struct lwp *fmi_owner; }; -static kmutex_t vfs_suspend_lock; /* Serialize suspensions. */ -static kmutex_t fstrans_lock; /* Fstrans big lock. */ -static kmutex_t fstrans_mount_lock; /* Fstrans mount big lock. */ +static kmutex_t vfs_suspend_lock /* Serialize suspensions. */ + __cacheline_aligned; +static kmutex_t fstrans_lock /* Fstrans big lock. */ + __cacheline_aligned; +static kmutex_t fstrans_mount_lock /* Fstrans mount big lock. */ + __cacheline_aligned; static kcondvar_t fstrans_state_cv; /* Fstrans or cow state changed. */ static kcondvar_t fstrans_count_cv; /* Fstrans or cow count changed. */ static pserialize_t fstrans_psz; /* Pserialize state. */ static LIST_HEAD(fstrans_lwp_head, fstrans_lwp_info) fstrans_fli_head; /* List of all fstrans_lwp_info. */ +static pool_cache_t fstrans_lwp_cache; /* Cache of fstrans_lwp_info. */ + static int fstrans_gone_count; /* Number of fstrans_mount_info gone. */ -static void fstrans_mount_dtor(struct fstrans_mount_info *); +static void fstrans_mount_free(struct fstrans_mount_info *); static void fstrans_clear_lwp_info(void); static inline struct fstrans_lwp_info * fstrans_get_lwp_info(struct mount *, bool); static struct fstrans_lwp_info *fstrans_alloc_lwp_info(struct mount *); +static int fstrans_lwp_ctor(void *, void *, int); +static void fstrans_lwp_dtor(void *, void *); static inline int _fstrans_start(struct mount *, enum fstrans_lock_type, int); static bool grant_lock(const struct fstrans_mount_info *, const enum fstrans_lock_type); @@ -188,13 +196,17 @@ fstrans_init(void) cv_init(&fstrans_count_cv, "fstcnt"); fstrans_psz = pserialize_create(); LIST_INIT(&fstrans_fli_head); + fstrans_lwp_cache = pool_cache_init(sizeof(struct fstrans_lwp_info), + coherency_unit, 0, 0, "fstlwp", NULL, IPL_NONE, fstrans_lwp_ctor, + fstrans_lwp_dtor, NULL); + KASSERT(fstrans_lwp_cache != NULL); } /* * Deallocate lwp state. */ void -fstrans_lwp_dtor(lwp_t *l) +fstrans_lwp_free(lwp_t *l) { struct fstrans_lwp_info *fli, *fli_next; @@ -203,7 +215,7 @@ fstrans_lwp_dtor(lwp_t *l) KASSERT(fli->fli_cow_cnt == 0); KASSERT(fli->fli_self == l); if (fli->fli_mount != NULL) - fstrans_mount_dtor(fli->fli_mountinfo); + fstrans_mount_free(fli->fli_mountinfo); fli_next = fli->fli_succ; fli->fli_alias_cnt = 0; fli->fli_mount = NULL; @@ -211,6 +223,7 @@ fstrans_lwp_dtor(lwp_t *l) fli->fli_mountinfo = NULL; membar_sync(); fli->fli_self = NULL; + pool_cache_put(fstrans_lwp_cache, fli); } l->l_fstrans = NULL; @@ -220,7 +233,7 @@ fstrans_lwp_dtor(lwp_t *l) * Dereference mount state. */ static void -fstrans_mount_dtor(struct fstrans_mount_info *fmi) +fstrans_mount_free(struct fstrans_mount_info *fmi) { mutex_enter(&fstrans_mount_lock); @@ -287,7 +300,7 @@ fstrans_unmount(struct mount *mp) fstrans_gone_count += 1; mutex_exit(&fstrans_mount_lock); - fstrans_mount_dtor(fmi); + fstrans_mount_free(fmi); } /* @@ -309,7 +322,7 @@ fstrans_clear_lwp_info(void) fli->fli_cow_cnt == 0 && fli->fli_alias_cnt == 0) { *p = (*p)->fli_succ; - fstrans_mount_dtor(fli->fli_mountinfo); + fstrans_mount_free(fli->fli_mountinfo); if (fli->fli_alias) { KASSERT(fli->fli_alias->fli_alias_cnt > 0); fli->fli_alias->fli_alias_cnt--; @@ -320,6 +333,7 @@ fstrans_clear_lwp_info(void) membar_sync(); fli->fli_self = NULL; p = &curlwp->l_fstrans; + pool_cache_put(fstrans_lwp_cache, fli); } else { p = &(*p)->fli_succ; } @@ -348,32 +362,16 @@ fstrans_alloc_lwp_info(struct mount *mp) /* * Try to reuse a cleared entry or allocate a new one. */ - mutex_enter(&fstrans_lock); - LIST_FOREACH(fli, &fstrans_fli_head, fli_list) { - membar_sync(); - if (fli->fli_self == NULL) { - KASSERT(fli->fli_mount == NULL); - KASSERT(fli->fli_trans_cnt == 0); - KASSERT(fli->fli_cow_cnt == 0); - KASSERT(fli->fli_alias_cnt == 0); - fli->fli_self = curlwp; - fli->fli_succ = curlwp->l_fstrans; - curlwp->l_fstrans = fli; - break; - } - } - mutex_exit(&fstrans_lock); - - if (fli == NULL) { - fli = kmem_alloc(sizeof(*fli), KM_SLEEP); - mutex_enter(&fstrans_lock); - memset(fli, 0, sizeof(*fli)); - fli->fli_self = curlwp; - LIST_INSERT_HEAD(&fstrans_fli_head, fli, fli_list); - mutex_exit(&fstrans_lock); - fli->fli_succ = curlwp->l_fstrans; - curlwp->l_fstrans = fli; - } + fli = pool_cache_get(fstrans_lwp_cache, PR_WAITOK); + KASSERT(fli->fli_self == NULL); + KASSERT(fli->fli_mountinfo == NULL); + KASSERT(fli->fli_trans_cnt == 0); + KASSERT(fli->fli_cow_cnt == 0); + KASSERT(fli->fli_alias_cnt == 0); + fli->fli_succ = curlwp->l_fstrans; + curlwp->l_fstrans = fli; + membar_sync(); + fli->fli_self = curlwp; /* * Attach the entry to the mount if its mnt_transinfo is valid. @@ -433,6 +431,41 @@ fstrans_get_lwp_info(struct mount *mp, b } /* + * Construct a new fstrans_lwp_info for the cache. + */ +static int +fstrans_lwp_ctor(void *arg, void *obj, int flags) +{ + struct fstrans_lwp_info *fli = obj; + + memset(fli, 0, sizeof(*fli)); + membar_sync(); + mutex_enter(&fstrans_lock); + LIST_INSERT_HEAD(&fstrans_fli_head, fli, fli_list); + mutex_exit(&fstrans_lock); + return 0; +} + +/* + * Destroy a fstrans_lwp_info before purge from the cache. + */ +static void +fstrans_lwp_dtor(void *arg, void *obj) +{ + struct fstrans_lwp_info *fli = obj; + + KASSERT(fli->fli_self == NULL); + KASSERT(fli->fli_mountinfo == NULL); + KASSERT(fli->fli_trans_cnt == 0); + KASSERT(fli->fli_cow_cnt == 0); + KASSERT(fli->fli_alias_cnt == 0); + + mutex_enter(&fstrans_lock); + LIST_REMOVE(fli, fli_list); + mutex_exit(&fstrans_lock); +} + +/* * Check if this lock type is granted at this state. */ static bool @@ -844,7 +877,7 @@ fscow_disestablish(struct mount *mp, int } cow_change_done(fmi); - fstrans_mount_dtor(fmi); + fstrans_mount_free(fmi); return hp ? 0 : EINVAL; } Index: rump/librump/rumpkern/emul.c =================================================================== RCS file: /cvsroot/src/sys/rump/librump/rumpkern/emul.c,v retrieving revision 1.195 diff -u -p -r1.195 emul.c --- rump/librump/rumpkern/emul.c 2 Jan 2020 15:42:27 -0000 1.195 +++ rump/librump/rumpkern/emul.c 4 Apr 2020 21:02:37 -0000 @@ -319,13 +319,13 @@ rump_fstrans_done(struct mount *mp) __weak_alias(fstrans_done,rump_fstrans_done); -void rump_fstrans_lwp_dtor(struct lwp *); +void rump_fstrans_lwp_free(struct lwp *); void -rump_fstrans_lwp_dtor(struct lwp *l) +rump_fstrans_lwp_free(struct lwp *l) { } -__weak_alias(fstrans_lwp_dtor,rump_fstrans_lwp_dtor); +__weak_alias(fstrans_lwp_free,rump_fstrans_lwp_free); /* * Provide weak aliases for tty routines used by printf. Index: rump/librump/rumpkern/lwproc.c =================================================================== RCS file: /cvsroot/src/sys/rump/librump/rumpkern/lwproc.c,v retrieving revision 1.44 diff -u -p -r1.44 lwproc.c --- rump/librump/rumpkern/lwproc.c 15 Feb 2020 18:12:15 -0000 1.44 +++ rump/librump/rumpkern/lwproc.c 4 Apr 2020 21:02:37 -0000 @@ -328,7 +328,7 @@ lwproc_freelwp(struct lwp *l) if (l->l_name) kmem_free(l->l_name, MAXCOMLEN); - fstrans_lwp_dtor(l); + fstrans_lwp_free(l); lwp_finispecific(l); lwproc_curlwpop(RUMPUSER_LWP_DESTROY, l); Index: sys/fstrans.h =================================================================== RCS file: /cvsroot/src/sys/sys/fstrans.h,v retrieving revision 1.13 diff -u -p -r1.13 fstrans.h --- sys/fstrans.h 1 Mar 2019 09:02:03 -0000 1.13 +++ sys/fstrans.h 4 Apr 2020 21:02:37 -0000 @@ -1,7 +1,7 @@ /* $NetBSD: fstrans.h,v 1.13 2019/03/01 09:02:03 hannken Exp $ */ /*- - * Copyright (c) 2007 The NetBSD Foundation, Inc. + * Copyright (c) 2007, 2020 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -48,7 +48,7 @@ enum fstrans_state { }; void fstrans_init(void); -void fstrans_lwp_dtor(lwp_t *); +void fstrans_lwp_free(lwp_t *); void fstrans_start(struct mount *); int fstrans_start_nowait(struct mount *); void fstrans_start_lazy(struct mount *);