From 818cf64bd686ec7f87bb31b78c56a7aebe2647dc Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Mon, 14 Apr 2014 03:01:04 +0000 Subject: [PATCH 11/14] Add vinotab(9) abstraction for inode number->vnode cache. --- sys/conf/files | 1 + sys/kern/init_sysctl.c | 2 + sys/kern/vfs_init.c | 6 + sys/kern/vfs_vinotab.c | 356 ++++++++++++++++++++++++++++++ sys/rump/librump/rumpvfs/Makefile.rumpvfs | 4 +- sys/sys/vinotab.h | 78 +++++++ 6 files changed, 445 insertions(+), 2 deletions(-) create mode 100644 sys/kern/vfs_vinotab.c create mode 100644 sys/sys/vinotab.h diff --git a/sys/conf/files b/sys/conf/files index 45cadb5..4f09cb4 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1672,6 +1672,7 @@ file kern/vfs_quotactl.c file kern/vfs_subr.c file kern/vfs_syscalls.c file kern/vfs_trans.c +file kern/vfs_vinotab.c file kern/vfs_vnode.c file kern/vfs_vnops.c file kern/vfs_wapbl.c wapbl diff --git a/sys/kern/init_sysctl.c b/sys/kern/init_sysctl.c index 6f4bc1a..bfd6c92 100644 --- a/sys/kern/init_sysctl.c +++ b/sys/kern/init_sysctl.c @@ -66,6 +66,7 @@ __KERNEL_RCSID(0, "$NetBSD: init_sysctl.c,v 1.202 2014/03/24 20:07:41 christos E #include #include #include +#include #include @@ -863,6 +864,7 @@ sysctl_kern_maxvnodes(SYSCTLFN_ARGS) } vfs_reinit(); nchreinit(); + vinotabs_reinit(); return (0); } diff --git a/sys/kern/vfs_init.c b/sys/kern/vfs_init.c index 47967c5..d53a561 100644 --- a/sys/kern/vfs_init.c +++ b/sys/kern/vfs_init.c @@ -84,6 +84,7 @@ __KERNEL_RCSID(0, "$NetBSD: vfs_init.c,v 1.47 2014/02/25 18:30:11 pooka Exp $"); #include #include #include +#include /* * Sigh, such primitive tools are these... @@ -417,6 +418,11 @@ vfsinit(void) */ nchinit(); + /* + * Initialize the vnode inode cache + */ + vinotabs_init(); + #ifdef DEBUG /* * Check the list of vnode operations. diff --git a/sys/kern/vfs_vinotab.c b/sys/kern/vfs_vinotab.c new file mode 100644 index 0000000..b1e2362 --- /dev/null +++ b/sys/kern/vfs_vinotab.c @@ -0,0 +1,356 @@ +/* $NetBSD$ */ + +/*- + * Copyright (c) 2014 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Taylor R Campbell. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__KERNEL_RCSID(0, "$NetBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct vinotab { + struct mount *vt_mount; + const struct vinotab_ops *vt_ops; + LIST_ENTRY(vinotab) vt_entry; +}; + +LIST_HEAD(vinotab_head, vinotab_entry); + +/* List of vinotabs, for rehashing. */ +static struct { + kmutex_t lock; + LIST_HEAD(, vinotab) head; +} vinotabs __cacheline_aligned; + +/* Global hash table. */ +static struct { + kmutex_t lock; + kcondvar_t cv; + struct vinohashtab { + struct vinotab_head *vht_buckets; + unsigned long vht_mask; + } *hashtab; +} vinohash __cacheline_aligned; + +/* Pool caches for struct vinotab and struct vinohashtab allocation. */ +static pool_cache_t vinohashtab_pc; +static pool_cache_t vinotab_pc; + +static int vinohash_wait(int); + +static const void * +vinotab_get_key(struct vinotab *vinotab, const void *key, size_t *sizep) +{ + const struct vinotab_ops *const ops = vinotab->vt_ops; + + return (*ops->vto_get_key)(ops->vto_context, key, sizep); +} + +static const void * +vinotab_vnode_key(struct vinotab *vinotab, struct vnode *vp, size_t *sizep) +{ + const struct vinotab_ops *const ops = vinotab->vt_ops; + + return (*ops->vto_vnode_key)(ops->vto_context, vp, sizep); +} + +static size_t +vinotab_hash(struct vinotab *vinotab, struct vinohashtab *hashtab, + const void *data, size_t size) +{ + + return (hash32_buf(data, size, HASH32_STR_INIT) & hashtab->vht_mask); +} + +static struct vinohashtab * +vinohashtab_create(void) +{ + struct vinohashtab *const hashtab = pool_cache_get(vinohashtab_pc, + PR_WAITOK); + + hashtab->vht_buckets = hashinit(desiredvnodes, HASH_LIST, + true, &hashtab->vht_mask); + + return hashtab; +} + +static void +vinohashtab_destroy(struct vinohashtab *hashtab) +{ + + hashdone(hashtab->vht_buckets, HASH_LIST, hashtab->vht_mask); + pool_cache_put(vinohashtab_pc, hashtab); +} + +void +vinotabs_init(void) +{ + + vinotab_pc = pool_cache_init(sizeof(struct vinotab), 0, 0, 0, + "vinotab", NULL, IPL_NONE, NULL, NULL, NULL); + vinohashtab_pc = pool_cache_init(sizeof(struct vinohashtab), 0, 0, 0, + "vinoht", NULL, IPL_NONE, NULL, NULL, NULL); + + mutex_init(&vinohash.lock, MUTEX_DEFAULT, IPL_NONE); + cv_init(&vinohash.cv, "vinohash"); + vinohash.hashtab = vinohashtab_create(); + + mutex_init(&vinotabs.lock, MUTEX_DEFAULT, IPL_NONE); + LIST_INIT(&vinotabs.head); +} + +void +vinotabs_reinit(void) +{ + struct vinotab *vinotab; + struct vinohashtab *oldhashtab, *newhashtab; + struct vinotab_head *oldbuckets, *newbuckets; + struct vinotab_entry *entry, *next; + size_t size; + const void *data; + size_t hash; + size_t i; + + /* Allocate a new hash table. */ + newhashtab = vinohashtab_create(); + newbuckets = newhashtab->vht_buckets; + + /* Detach the old hash table. */ + mutex_enter(&vinohash.lock); + (void)vinohash_wait(0); + oldhashtab = vinohash.hashtab; + KASSERT(oldhashtab != NULL); + oldbuckets = oldhashtab->vht_buckets; + vinohash.hashtab = NULL; + mutex_exit(&vinohash.lock); + + /* For each vinotab, rehash its entries. */ + mutex_enter(&vinotabs.lock); + LIST_FOREACH(vinotab, &vinotabs.head, vt_entry) { + for (i = 0; i <= oldhashtab->vht_mask; i++) { + LIST_FOREACH_SAFE(entry, &oldbuckets[i], vte_entry, + next) { + if (entry->vte_vnode->v_mount != + vinotab->vt_mount) + continue; + LIST_REMOVE(entry, vte_entry); + data = vinotab_vnode_key(vinotab, + entry->vte_vnode, &size); + hash = vinotab_hash(vinotab, newhashtab, data, + size); + LIST_INSERT_HEAD(&newbuckets[hash], entry, + vte_entry); + } + } + } + mutex_exit(&vinotabs.lock); + +#if DIAGNOSTIC + /* We had better have removed all the entries. */ + for (i = 0; i <= oldhashtab->vht_mask; i++) + KASSERT(LIST_EMPTY(&oldbuckets[i])); +#endif + + /* Attach the new hash table. */ + mutex_enter(&vinohash.lock); + KASSERT(vinohash.hashtab == NULL); + vinohash.hashtab = newhashtab; + cv_broadcast(&vinohash.cv); + mutex_exit(&vinohash.lock); + + /* Deallocate the old hash table. */ + vinohashtab_destroy(oldhashtab); +} + +static int +vinohash_wait(int flags) +{ + int error = 0; + + KASSERT(mutex_owned(&vinohash.lock)); + while (vinohash.hashtab == NULL) { + if (ISSET(flags, LK_NOWAIT)) { + error = EWOULDBLOCK; + break; +#if notyet + } else if (ISSET(flags, LK_INTR)) { + error = cv_wait_sig(&vinohash.cv, &vinohash.lock); + if (error) + break; +#endif + } else { + cv_wait(&vinohash.cv, &vinohash.lock); + } + } + + return error; +} + +struct vinotab * +vinotab_create(struct mount *mp, const struct vinotab_ops *ops) +{ + struct vinotab *const vinotab = pool_cache_get(vinotab_pc, PR_WAITOK); + + vinotab->vt_mount = mp; + vinotab->vt_ops = ops; + + mutex_enter(&vinotabs.lock); + LIST_INSERT_HEAD(&vinotabs.head, vinotab, vt_entry); + mutex_exit(&vinotabs.lock); + + return vinotab; +} + +void +vinotab_destroy(struct vinotab *vinotab) +{ + + mutex_enter(&vinohash.lock); + LIST_REMOVE(vinotab, vt_entry); + mutex_exit(&vinohash.lock); + + /* Paranoia. */ + vinotab->vt_ops = NULL; + vinotab->vt_mount = NULL; + + pool_cache_put(vinotab_pc, vinotab); +} + +int +vinotab_get(struct vinotab *vinotab, const void *key, int flags, + struct vnode **vpp) +{ + struct vinohashtab *hashtab; + size_t hash; + struct vinotab_head *head; + struct vinotab_entry *entry; + size_t size, size0; + const void *data, *data0; + int error; + + data = vinotab_get_key(vinotab, key, &size); + + /* Get the current hash table. Wait if someone's rehashing. */ + mutex_enter(&vinohash.lock); + error = vinohash_wait(flags); + if (error) { + mutex_exit(&vinohash.lock); + goto fail; + } + hashtab = vinohash.hashtab; + KASSERT(hashtab != NULL); + + /* Look it up. */ + hash = vinotab_hash(vinotab, hashtab, data, size); + head = &hashtab->vht_buckets[hash]; + LIST_FOREACH(entry, head, vte_entry) { + if (entry->vte_vnode->v_mount != vinotab->vt_mount) + continue; + data0 = vinotab_vnode_key(vinotab, entry->vte_vnode, &size0); + if ((size == size0) && memcmp(data, data0, size) == 0) { + struct vnode *const vp = entry->vte_vnode; + + mutex_enter(vp->v_interlock); + mutex_exit(&vinohash.lock); + error = vget(vp, flags); + if (error) + goto fail; + *vpp = vp; + return 0; + } + } + mutex_exit(&vinohash.lock); + error = ENOENT; + +fail: KASSERT(error); + *vpp = NULL; + return error; +} + +int +vinotab_insert(struct vinotab *vinotab, struct vnode *vp, int flags) +{ + struct vinohashtab *hashtab; + size_t hash; + struct vinotab_head *head; + struct vinotab_entry *entry; + const void *data, *data0; + size_t size, size0; + int error; + + KASSERT(vp->v_mount == vinotab->vt_mount); + + data = vinotab_vnode_key(vinotab, vp, &size); + + mutex_enter(&vinohash.lock); + error = vinohash_wait(flags); + if (error) + goto out; + hashtab = vinohash.hashtab; + hash = vinotab_hash(vinotab, hashtab, data, size); + head = &hashtab->vht_buckets[hash]; + LIST_FOREACH(entry, head, vte_entry) { + if (entry->vte_vnode->v_mount != vinotab->vt_mount) + continue; + data0 = vinotab_vnode_key(vinotab, entry->vte_vnode, &size0); + if ((size == size0) && memcmp(data, data0, size) == 0) { + error = EEXIST; + goto out; + } + } + entry = (void *)((char *)vp->v_data + + vinotab->vt_ops->vto_entry_offset); + entry->vte_vnode = vp; + LIST_INSERT_HEAD(head, entry, vte_entry); + error = 0; +out: mutex_exit(&vinohash.lock); + + return error; +} + +void +vinotab_remove(struct vinotab *vinotab, struct vnode *vp) +{ + struct vinotab_entry *const entry = (void *)((char *)vp->v_data + + vinotab->vt_ops->vto_entry_offset); + + mutex_enter(&vinohash.lock); + (void)vinohash_wait(0); + LIST_REMOVE(entry, vte_entry); + mutex_exit(&vinohash.lock); +} diff --git a/sys/rump/librump/rumpvfs/Makefile.rumpvfs b/sys/rump/librump/rumpvfs/Makefile.rumpvfs index e0f6eef..b942b0f 100644 --- a/sys/rump/librump/rumpvfs/Makefile.rumpvfs +++ b/sys/rump/librump/rumpvfs/Makefile.rumpvfs @@ -28,8 +28,8 @@ SRCS+= kern_physio.c # sys/kern vfs SRCS+= vfs_bio.c vfs_cache.c vfs_cwd.c vfs_dirhash.c vfs_getcwd.c \ vfs_hooks.c vfs_init.c vfs_lockf.c vfs_lookup.c vfs_mount.c \ - vfs_subr.c vfs_syscalls.c vfs_trans.c vfs_vnode.c vfs_vnops.c \ - vfs_wapbl.c vfs_xattr.c + vfs_subr.c vfs_syscalls.c vfs_trans.c vfs_vinotab.c vfs_vnode.c \ + vfs_vnops.c vfs_wapbl.c vfs_xattr.c # sys/kern module support SRCS+= kern_module_vfs.c subr_kobj_vfs.c diff --git a/sys/sys/vinotab.h b/sys/sys/vinotab.h new file mode 100644 index 0000000..1e68bd3 --- /dev/null +++ b/sys/sys/vinotab.h @@ -0,0 +1,78 @@ +/* $NetBSD$ */ + +/*- + * Copyright (c) 2014 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Taylor R Campbell. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_VINOTAB_H_ +#define _SYS_VINOTAB_H_ + +#include +#include + +#ifdef DIAGNOSTIC +#include +#endif + +struct vinotab; +struct vnode; + +struct vinotab_entry { + LIST_ENTRY(vinotab_entry) vte_entry; + struct vnode *vte_vnode; +}; + +struct vinotab_ops { + const void *(*vto_get_key)(void *, const void *, size_t *); + const void *(*vto_vnode_key)(void *, struct vnode *, size_t *); + size_t vto_entry_offset; + void *vto_context; +}; + +void vinotabs_init(void); +void vinotabs_reinit(void); + +struct vinotab * + vinotab_create(struct mount *, const struct vinotab_ops *); +void vinotab_destroy(struct vinotab *); +int vinotab_get(struct vinotab *, const void *, int, struct vnode **); +int vinotab_insert(struct vinotab *, struct vnode *, int); +void vinotab_remove(struct vinotab *, struct vnode *); + +/* For ITOV-type operations. Caller must have a reference already. */ +static inline struct vnode * +vinotab_entry_vnode(const struct vinotab_entry *entry) +{ + + KASSERT(entry->vte_vnode != NULL); + KASSERT(0 < entry->vte_vnode->v_usecount); + + return entry->vte_vnode; +} + +#endif /* _SYS_VINOTAB_H_ */ -- 1.8.3.1