commit eb11550b51d8142d43fe2a2a3ab56e5cdffd4fe5 Author: Ryota Ozaki Date: Thu Dec 25 19:22:10 2014 +0900 psz bridge rt diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c index 5bcfd6c..9d60662 100644 --- a/sys/net/if_bridge.c +++ b/sys/net/if_bridge.c @@ -175,9 +175,43 @@ __KERNEL_RCSID(0, "$NetBSD: if_bridge.c,v 1.93 2014/12/24 08:55:09 ozaki-r Exp $ #define BRIDGE_RTABLE_PRUNE_PERIOD (5 * 60) #endif +#define BRIDGE_RT_INTR_LOCK(_sc) mutex_enter((_sc)->sc_rtlist_lock) +#define BRIDGE_RT_INTR_UNLOCK(_sc) mutex_exit((_sc)->sc_rtlist_lock) +#define BRIDGE_RT_INTR_LOCKED(_sc) mutex_owned((_sc)->sc_rtlist_lock) + +#define BRIDGE_RT_LOCK(_sc) if ((_sc)->sc_rtlist_psz_lock) \ + mutex_enter((_sc)->sc_rtlist_psz_lock) +#define BRIDGE_RT_UNLOCK(_sc) if ((_sc)->sc_rtlist_psz_lock) \ + mutex_exit((_sc)->sc_rtlist_psz_lock) +#define BRIDGE_RT_LOCKED(_sc) (!(_sc)->sc_rtlist_psz_lock || \ + mutex_owned((_sc)->sc_rtlist_psz_lock)) + +#define BRIDGE_RT_PSZ_PERFORM(_sc) \ + if ((_sc)->sc_rtlist_psz != NULL) \ + pserialize_perform((_sc)->sc_rtlist_psz); + +#ifdef BRIDGE_MPSAFE +#define BRIDGE_RT_RENTER(__s) do { \ + if (!cpu_intr_p()) \ + __s = pserialize_read_enter(); \ + else \ + __s = splhigh(); \ + } while (0) +#define BRIDGE_RT_REXIT(__s) do { \ + if (!cpu_intr_p()) \ + pserialize_read_exit(__s); \ + else \ + splx(__s); \ + } while (0) +#else /* BRIDGE_MPSAFE */ +#define BRIDGE_RT_RENTER(__s) do { __s = 0; } while (0) +#define BRIDGE_RT_REXIT(__s) do { (void)__s; } while (0) +#endif /* BRIDGE_MPSAFE */ + int bridge_rtable_prune_period = BRIDGE_RTABLE_PRUNE_PERIOD; static struct pool bridge_rtnode_pool; +static struct work bridge_rtage_wk; void bridgeattach(int); @@ -202,6 +236,7 @@ static int bridge_rtupdate(struct bridge_softc *, const uint8_t *, static struct ifnet *bridge_rtlookup(struct bridge_softc *, const uint8_t *); static void bridge_rttrim(struct bridge_softc *); static void bridge_rtage(struct bridge_softc *); +static void bridge_rtage_work(struct work *, void *); static void bridge_rtflush(struct bridge_softc *, int); static int bridge_rtdaddr(struct bridge_softc *, const uint8_t *); static void bridge_rtdelete(struct bridge_softc *, struct ifnet *ifp); @@ -213,8 +248,9 @@ static struct bridge_rtnode *bridge_rtnode_lookup(struct bridge_softc *, const uint8_t *); static int bridge_rtnode_insert(struct bridge_softc *, struct bridge_rtnode *); -static void bridge_rtnode_destroy(struct bridge_softc *, - struct bridge_rtnode *); +static void bridge_rtnode_remove(struct bridge_softc *, + struct bridge_rtnode *); +static void bridge_rtnode_destroy(struct bridge_rtnode *); static struct bridge_iflist *bridge_lookup_member(struct bridge_softc *, const char *name); @@ -348,6 +384,7 @@ bridge_clone_create(struct if_clone *ifc, int unit) { struct bridge_softc *sc; struct ifnet *ifp; + int error, flags; sc = kmem_zalloc(sizeof(*sc), KM_SLEEP); ifp = &sc->sc_if; @@ -364,6 +401,16 @@ bridge_clone_create(struct if_clone *ifc, int unit) /* Initialize our routing table. */ bridge_rtable_init(sc); +#ifdef BRIDGE_MPSAFE + flags = WQ_MPSAFE; +#else + flags = 0; +#endif + error = workqueue_create(&sc->sc_rtage_wq, "bridge_rtage", + bridge_rtage_work, sc, PRI_SOFTNET, IPL_SOFTNET, flags); + if (error) + panic("%s: workqueue_create %d\n", __func__, error); + callout_init(&sc->sc_brcallout, 0); callout_init(&sc->sc_bstpcallout, 0); @@ -454,6 +501,8 @@ bridge_clone_destroy(struct ifnet *ifp) if (sc->sc_iflist_psz_lock) mutex_obj_free(sc->sc_iflist_psz_lock); + workqueue_destroy(sc->sc_rtage_wq); + kmem_free(sc, sizeof(*sc)); return (0); @@ -1977,6 +2026,44 @@ next: m_freem(m); } +static struct bridge_rtnode * +bridge_rtalloc(struct bridge_softc *sc, const uint8_t *dst, int *error) +{ + struct bridge_rtnode *brt; + + if (sc->sc_brtcnt >= sc->sc_brtmax) { + *error = ENOSPC; + return NULL; + } + + /* + * Allocate a new bridge forwarding node, and + * initialize the expiration time and Ethernet + * address. + */ + brt = pool_get(&bridge_rtnode_pool, PR_NOWAIT); + if (brt == NULL) { + *error = ENOMEM; + return NULL; + } + + memset(brt, 0, sizeof(*brt)); + brt->brt_expire = time_uptime + sc->sc_brttimeout; + brt->brt_flags = IFBAF_DYNAMIC; + memcpy(brt->brt_addr, dst, ETHER_ADDR_LEN); + + BRIDGE_RT_INTR_LOCK(sc); + *error = bridge_rtnode_insert(sc, brt); + BRIDGE_RT_INTR_UNLOCK(sc); + + if (*error != 0) { + pool_put(&bridge_rtnode_pool, brt); + return NULL; + } + + return brt; +} + /* * bridge_rtupdate: * @@ -1988,54 +2075,36 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t *dst, { struct bridge_rtnode *brt; int error = 0; + int s; - mutex_enter(sc->sc_rtlist_lock); - +again: /* * A route for this destination might already exist. If so, * update it, otherwise create a new one. */ - if ((brt = bridge_rtnode_lookup(sc, dst)) == NULL) { - if (sc->sc_brtcnt >= sc->sc_brtmax) { - error = ENOSPC; - goto out; - } + BRIDGE_RT_RENTER(s); + brt = bridge_rtnode_lookup(sc, dst); - /* - * Allocate a new bridge forwarding node, and - * initialize the expiration time and Ethernet - * address. - */ - brt = pool_get(&bridge_rtnode_pool, PR_NOWAIT); - if (brt == NULL) { - error = ENOMEM; - goto out; - } - - memset(brt, 0, sizeof(*brt)); - brt->brt_expire = time_uptime + sc->sc_brttimeout; - brt->brt_flags = IFBAF_DYNAMIC; - memcpy(brt->brt_addr, dst, ETHER_ADDR_LEN); - - if ((error = bridge_rtnode_insert(sc, brt)) != 0) { - pool_put(&bridge_rtnode_pool, brt); - goto out; + if (brt != NULL) { + brt->brt_ifp = dst_if; + if (setflags) { + brt->brt_flags = flags; + if (flags & IFBAF_STATIC) + brt->brt_expire = 0; + else + brt->brt_expire = time_uptime + sc->sc_brttimeout; } } + BRIDGE_RT_REXIT(s); - brt->brt_ifp = dst_if; - if (setflags) { - brt->brt_flags = flags; - if (flags & IFBAF_STATIC) - brt->brt_expire = 0; - else - brt->brt_expire = time_uptime + sc->sc_brttimeout; + if (brt == NULL) { + brt = bridge_rtalloc(sc, dst, &error); + if (brt == NULL) + return error; + goto again; } -out: - mutex_exit(sc->sc_rtlist_lock); - - return error; + return 0; } /* @@ -2048,18 +2117,60 @@ bridge_rtlookup(struct bridge_softc *sc, const uint8_t *addr) { struct bridge_rtnode *brt; struct ifnet *ifs = NULL; + int s; - mutex_enter(sc->sc_rtlist_lock); - + BRIDGE_RT_RENTER(s); brt = bridge_rtnode_lookup(sc, addr); if (brt != NULL) ifs = brt->brt_ifp; - - mutex_exit(sc->sc_rtlist_lock); + BRIDGE_RT_REXIT(s); return ifs; } +static void +bridge_rttrim0(struct bridge_softc *sc) +{ + struct bridge_rtnode *brt, *nbrt; + struct bridge_rtnode **brt_list; + int i, count; + +retry: + count = sc->sc_brtcnt; + if (count == 0) + return; + brt_list = kmem_alloc(sizeof(struct bridge_rtnode *) * count, KM_SLEEP); + + BRIDGE_RT_LOCK(sc); + BRIDGE_RT_INTR_LOCK(sc); + if (__predict_false(sc->sc_brtcnt > count)) { + BRIDGE_RT_INTR_UNLOCK(sc); + BRIDGE_RT_UNLOCK(sc); + kmem_free(brt_list, sizeof(*brt_list) * count); + goto retry; + } + + i = 0; + LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { + if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { + bridge_rtnode_remove(sc, brt); + brt_list[i++] = brt; + } + if (sc->sc_brtcnt <= sc->sc_brtmax) + break; + } + BRIDGE_RT_INTR_UNLOCK(sc); + + if (i > 0) + BRIDGE_RT_PSZ_PERFORM(sc); + BRIDGE_RT_UNLOCK(sc); + + while (--i >= 0) + bridge_rtnode_destroy(brt_list[i]); + + kmem_free(brt_list, sizeof(*brt_list) * count); +} + /* * bridge_rttrim: * @@ -2070,29 +2181,17 @@ bridge_rtlookup(struct bridge_softc *sc, const uint8_t *addr) static void bridge_rttrim(struct bridge_softc *sc) { - struct bridge_rtnode *brt, *nbrt; - - mutex_enter(sc->sc_rtlist_lock); /* Make sure we actually need to do this. */ if (sc->sc_brtcnt <= sc->sc_brtmax) - goto out; + return; /* Force an aging cycle; this might trim enough addresses. */ bridge_rtage(sc); if (sc->sc_brtcnt <= sc->sc_brtmax) - goto out; + return; - LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { - if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { - bridge_rtnode_destroy(sc, brt); - if (sc->sc_brtcnt <= sc->sc_brtmax) - goto out; - } - } - -out: - mutex_exit(sc->sc_rtlist_lock); + bridge_rttrim0(sc); return; } @@ -2107,15 +2206,21 @@ bridge_timer(void *arg) { struct bridge_softc *sc = arg; - mutex_enter(sc->sc_rtlist_lock); + workqueue_enqueue(sc->sc_rtage_wq, &bridge_rtage_wk, NULL); +} + +static void +bridge_rtage_work(struct work *wk, void *arg) +{ + struct bridge_softc *sc = arg; + + KASSERT(wk == &bridge_rtage_wk); bridge_rtage(sc); if (sc->sc_if.if_flags & IFF_RUNNING) callout_reset(&sc->sc_brcallout, bridge_rtable_prune_period * hz, bridge_timer, sc); - - mutex_exit(sc->sc_rtlist_lock); } /* @@ -2127,15 +2232,42 @@ static void bridge_rtage(struct bridge_softc *sc) { struct bridge_rtnode *brt, *nbrt; + struct bridge_rtnode **brt_list; + int i, count; - KASSERT(mutex_owned(sc->sc_rtlist_lock)); +retry: + count = sc->sc_brtcnt; + if (count == 0) + return; + brt_list = kmem_alloc(sizeof(struct bridge_rtnode *) * count, KM_SLEEP); + BRIDGE_RT_LOCK(sc); + BRIDGE_RT_INTR_LOCK(sc); + if (__predict_false(sc->sc_brtcnt > count)) { + BRIDGE_RT_INTR_UNLOCK(sc); + BRIDGE_RT_UNLOCK(sc); + kmem_free(brt_list, sizeof(*brt_list) * count); + goto retry; + } + + i = 0; LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { - if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { - if (time_uptime >= brt->brt_expire) - bridge_rtnode_destroy(sc, brt); + if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC && + time_uptime >= brt->brt_expire) { + bridge_rtnode_remove(sc, brt); + brt_list[i++] = brt; } } + BRIDGE_RT_INTR_UNLOCK(sc); + + if (i > 0) + BRIDGE_RT_PSZ_PERFORM(sc); + BRIDGE_RT_UNLOCK(sc); + + while (--i >= 0) + bridge_rtnode_destroy(brt_list[i]); + + kmem_free(brt_list, sizeof(*brt_list) * count); } /* @@ -2147,15 +2279,41 @@ static void bridge_rtflush(struct bridge_softc *sc, int full) { struct bridge_rtnode *brt, *nbrt; + struct bridge_rtnode **brt_list; + int i, count; - mutex_enter(sc->sc_rtlist_lock); +retry: + count = sc->sc_brtcnt; + if (count == 0) + return; + brt_list = kmem_alloc(sizeof(struct bridge_rtnode *) * count, KM_SLEEP); + BRIDGE_RT_LOCK(sc); + BRIDGE_RT_INTR_LOCK(sc); + if (__predict_false(sc->sc_brtcnt > count)) { + BRIDGE_RT_INTR_UNLOCK(sc); + BRIDGE_RT_UNLOCK(sc); + kmem_free(brt_list, sizeof(*brt_list) * count); + goto retry; + } + + i = 0; LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { - if (full || (brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) - bridge_rtnode_destroy(sc, brt); + if (full || (brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { + bridge_rtnode_remove(sc, brt); + brt_list[i++] = brt; + } } + BRIDGE_RT_INTR_UNLOCK(sc); - mutex_exit(sc->sc_rtlist_lock); + if (i > 0) + BRIDGE_RT_PSZ_PERFORM(sc); + BRIDGE_RT_UNLOCK(sc); + + while (--i >= 0) + bridge_rtnode_destroy(brt_list[i]); + + kmem_free(brt_list, sizeof(*brt_list) * count); } /* @@ -2167,21 +2325,23 @@ static int bridge_rtdaddr(struct bridge_softc *sc, const uint8_t *addr) { struct bridge_rtnode *brt; - int error = 0; - - mutex_enter(sc->sc_rtlist_lock); + BRIDGE_RT_LOCK(sc); + BRIDGE_RT_INTR_LOCK(sc); if ((brt = bridge_rtnode_lookup(sc, addr)) == NULL) { - error = ENOENT; - goto out; + BRIDGE_RT_INTR_UNLOCK(sc); + BRIDGE_RT_UNLOCK(sc); + return ENOENT; } + bridge_rtnode_remove(sc, brt); + BRIDGE_RT_INTR_UNLOCK(sc); + if (sc->sc_rtlist_psz != NULL) + pserialize_perform(sc->sc_rtlist_psz); + BRIDGE_RT_UNLOCK(sc); - bridge_rtnode_destroy(sc, brt); + bridge_rtnode_destroy(brt); -out: - mutex_exit(sc->sc_rtlist_lock); - - return error; + return 0; } /* @@ -2194,14 +2354,24 @@ bridge_rtdelete(struct bridge_softc *sc, struct ifnet *ifp) { struct bridge_rtnode *brt, *nbrt; - mutex_enter(sc->sc_rtlist_lock); - + BRIDGE_RT_LOCK(sc); + BRIDGE_RT_INTR_LOCK(sc); LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { if (brt->brt_ifp == ifp) - bridge_rtnode_destroy(sc, brt); + break; } + if (brt == NULL) { + BRIDGE_RT_INTR_UNLOCK(sc); + BRIDGE_RT_UNLOCK(sc); + return; + } + bridge_rtnode_remove(sc, brt); + BRIDGE_RT_INTR_UNLOCK(sc); + if (sc->sc_rtlist_psz != NULL) + pserialize_perform(sc->sc_rtlist_psz); + BRIDGE_RT_UNLOCK(sc); - mutex_exit(sc->sc_rtlist_lock); + bridge_rtnode_destroy(brt); } /* @@ -2225,6 +2395,13 @@ bridge_rtable_init(struct bridge_softc *sc) LIST_INIT(&sc->sc_rtlist); sc->sc_rtlist_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); +#ifdef BRIDGE_MPSAFE + sc->sc_rtlist_psz = pserialize_create(); + sc->sc_rtlist_psz_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); +#else + sc->sc_rtlist_psz = NULL; + sc->sc_rtlist_psz_lock = NULL; +#endif } /* @@ -2239,6 +2416,10 @@ bridge_rtable_fini(struct bridge_softc *sc) kmem_free(sc->sc_rthash, sizeof(*sc->sc_rthash) * BRIDGE_RTHASH_SIZE); if (sc->sc_rtlist_lock) mutex_obj_free(sc->sc_rtlist_lock); + if (sc->sc_rtlist_psz_lock) + mutex_obj_free(sc->sc_rtlist_lock); + if (sc->sc_rtlist_psz) + pserialize_destroy(sc->sc_rtlist_psz); } /* @@ -2289,8 +2470,6 @@ bridge_rtnode_lookup(struct bridge_softc *sc, const uint8_t *addr) uint32_t hash; int dir; - KASSERT(mutex_owned(sc->sc_rtlist_lock)); - hash = bridge_rthash(sc, addr); LIST_FOREACH(brt, &sc->sc_rthash[hash], brt_hash) { dir = memcmp(addr, brt->brt_addr, ETHER_ADDR_LEN); @@ -2316,7 +2495,7 @@ bridge_rtnode_insert(struct bridge_softc *sc, struct bridge_rtnode *brt) uint32_t hash; int dir; - KASSERT(mutex_owned(sc->sc_rtlist_lock)); + KASSERT(BRIDGE_RT_INTR_LOCKED(sc)); hash = bridge_rthash(sc, brt->brt_addr); @@ -2353,20 +2532,30 @@ bridge_rtnode_insert(struct bridge_softc *sc, struct bridge_rtnode *brt) } /* - * bridge_rtnode_destroy: + * bridge_rtnode_remove: * - * Destroy a bridge rtnode. + * Remove a bridge rtnode from the rthash and the rtlist of a bridge. */ static void -bridge_rtnode_destroy(struct bridge_softc *sc, struct bridge_rtnode *brt) +bridge_rtnode_remove(struct bridge_softc *sc, struct bridge_rtnode *brt) { - KASSERT(mutex_owned(sc->sc_rtlist_lock)); + KASSERT(BRIDGE_RT_INTR_LOCKED(sc)); LIST_REMOVE(brt, brt_hash); - LIST_REMOVE(brt, brt_list); sc->sc_brtcnt--; +} + +/* + * bridge_rtnode_destroy: + * + * Destroy a bridge rtnode. + */ +static void +bridge_rtnode_destroy(struct bridge_rtnode *brt) +{ + pool_put(&bridge_rtnode_pool, brt); } diff --git a/sys/net/if_bridgevar.h b/sys/net/if_bridgevar.h index 1d3a291..1bb5e23 100644 --- a/sys/net/if_bridgevar.h +++ b/sys/net/if_bridgevar.h @@ -209,6 +209,7 @@ struct ifbrparam { #ifdef _KERNEL #include +#include #include @@ -312,7 +313,10 @@ struct bridge_softc { kmutex_t *sc_iflist_psz_lock; LIST_HEAD(, bridge_rtnode) *sc_rthash; /* our forwarding table */ LIST_HEAD(, bridge_rtnode) sc_rtlist; /* list version of above */ + pserialize_t sc_rtlist_psz; + kmutex_t *sc_rtlist_psz_lock; kmutex_t *sc_rtlist_lock; + struct workqueue *sc_rtage_wq; uint32_t sc_rthash_key; /* key for hash */ uint32_t sc_filter_flags; /* ipf and flags */ pktqueue_t * sc_fwd_pktq;