commit 393bbc7fecfe8b3e87ca0fffa638af203345dbf6 Author: Ryota Ozaki Date: Tue Sep 19 12:33:19 2017 +0900 Invalidate rtcache based on a global generation counter diff --git a/sys/net/route.c b/sys/net/route.c index 41b83e2574f..3481d36b115 100644 --- a/sys/net/route.c +++ b/sys/net/route.c @@ -239,6 +239,8 @@ static krwlock_t rtcache_lock __cacheline_aligned; #define RTCACHE_WLOCKED() false #endif +static uint64_t rtcache_generation; + /* * mutex and cv that are used to wait for references to a rtentry left * before updating the rtentry. @@ -271,14 +273,11 @@ static int _rtcache_debug = 0; static kauth_listener_t route_listener; static int rtdeletemsg(struct rtentry *); -static void rtflushall(int); static void rt_maskedcopy(const struct sockaddr *, struct sockaddr *, const struct sockaddr *); -static void rtcache_clear(struct route *); -static void rtcache_clear_rtentry(int, struct rtentry *); -static void rtcache_invalidate(struct dom_rtlist *); +static void rtcache_invalidate(void); static void rt_ref(struct rtentry *); @@ -491,40 +490,17 @@ rt_init(void) } static void -rtflushall(int family) +rtcache_invalidate(void) { - struct domain *dom; if (rtcache_debug()) printf("%s: enter\n", __func__); - if ((dom = pffinddomain(family)) == NULL) - return; - RTCACHE_WLOCK(); - rtcache_invalidate(&dom->dom_rtcache); + rtcache_generation++; RTCACHE_UNLOCK(); } -static void -rtcache(struct route *ro) -{ - struct domain *dom; - - RTCACHE_ASSERT_WLOCK(); - - rtcache_invariants(ro); - KASSERT(ro->_ro_rt != NULL); - KASSERT(ro->ro_invalid == false); - KASSERT(rtcache_getdst(ro) != NULL); - - if ((dom = pffinddomain(rtcache_getdst(ro)->sa_family)) == NULL) - return; - - LIST_INSERT_HEAD(&dom->dom_rtcache, ro, ro_rtcache_next); - rtcache_invariants(ro); -} - #ifdef RT_DEBUG static void dump_rt(const struct rtentry *rt) @@ -1251,7 +1227,7 @@ rtrequest1(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt) RT_UNLOCK(); need_unlock = false; rt_timer_remove_all(rt); - rtcache_clear_rtentry(dst->sa_family, rt); + rtcache_invalidate(); #if defined(INET) || defined(INET6) if (netmask != NULL) lltable_prefix_free(dst->sa_family, dst, netmask, 0); @@ -1346,7 +1322,7 @@ rtrequest1(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt) } RT_UNLOCK(); need_unlock = false; - rtflushall(dst->sa_family); + rtcache_invalidate(); break; case RTM_GET: if (netmask != NULL) { @@ -1893,14 +1869,13 @@ _rtcache_init(struct route *ro, int flag) if (rtcache_getdst(ro) == NULL) return NULL; - ro->ro_invalid = false; rt = rtalloc1(rtcache_getdst(ro), flag); if (rt != NULL && ISSET(rt->rt_flags, RTF_UP)) { ro->_ro_rt = rt; + ro->ro_rtcache_generation = rtcache_generation; KASSERT(!ISSET(rt->rt_flags, RTF_UPDATING)); rtcache_ref(rt, ro); rt_unref(rt); - rtcache(ro); } else if (rt != NULL) rt_unref(rt); @@ -1933,7 +1908,7 @@ rtcache_update(struct route *ro, int clone) { struct rtentry *rt; RTCACHE_WLOCK(); - rtcache_clear(ro); + ro->_ro_rt = NULL; rt = _rtcache_init(ro, clone); RTCACHE_UNLOCK(); return rt; @@ -1958,9 +1933,8 @@ rtcache_copy(struct route *new_ro, struct route *old_ro) goto out; RTCACHE_WLOCK(); - new_ro->ro_invalid = false; if ((new_ro->_ro_rt = rt) != NULL) - rtcache(new_ro); + new_ro->ro_rtcache_generation = rtcache_generation; rtcache_invariants(new_ro); RTCACHE_UNLOCK(); out: @@ -1968,8 +1942,6 @@ out: return; } -static struct dom_rtlist invalid_routes = LIST_HEAD_INITIALIZER(dom_rtlist); - #if defined(RT_DEBUG) && defined(NET_MPSAFE) static void rtcache_trace(const char *func, struct rtentry *rt, struct route *ro) @@ -2023,7 +1995,7 @@ retry: rt = ro->_ro_rt; rtcache_invariants(ro); - if (ro->ro_invalid) { + if (ro->ro_rtcache_generation != rtcache_generation) { rt = NULL; goto out; } @@ -2063,57 +2035,6 @@ rtcache_validate(struct route *ro) return rt; } -static void -rtcache_invalidate(struct dom_rtlist *rtlist) -{ - struct route *ro; - - RTCACHE_ASSERT_WLOCK(); - - while ((ro = LIST_FIRST(rtlist)) != NULL) { - rtcache_invariants(ro); - KASSERT(ro->_ro_rt != NULL); - ro->ro_invalid = true; - LIST_REMOVE(ro, ro_rtcache_next); - LIST_INSERT_HEAD(&invalid_routes, ro, ro_rtcache_next); - rtcache_invariants(ro); - } -} - -static void -rtcache_clear_rtentry(int family, struct rtentry *rt) -{ - struct domain *dom; - struct route *ro, *nro; - - if ((dom = pffinddomain(family)) == NULL) - return; - - RTCACHE_WLOCK(); - LIST_FOREACH_SAFE(ro, &dom->dom_rtcache, ro_rtcache_next, nro) { - if (ro->_ro_rt == rt) - rtcache_clear(ro); - } - RTCACHE_UNLOCK(); -} - -static void -rtcache_clear(struct route *ro) -{ - - RTCACHE_ASSERT_WLOCK(); - - rtcache_invariants(ro); - if (ro->_ro_rt == NULL) - return; - - LIST_REMOVE(ro, ro_rtcache_next); - - ro->_ro_rt = NULL; - ro->ro_invalid = false; - rtcache_invariants(ro); -} - struct rtentry * rtcache_lookup2(struct route *ro, const struct sockaddr *dst, int clone, int *hitp) @@ -2140,7 +2061,7 @@ rtcache_lookup2(struct route *ro, const struct sockaddr *dst, if (rt == NULL) { RTCACHE_UNLOCK(); RTCACHE_WLOCK(); - rtcache_clear(ro); + ro->_ro_rt = NULL; goto miss; } @@ -2167,7 +2088,8 @@ rtcache_free_locked(struct route *ro) { RTCACHE_ASSERT_WLOCK(); - rtcache_clear(ro); + + ro->_ro_rt = NULL; if (ro->ro_sa != NULL) { sockaddr_free(ro->ro_sa); ro->ro_sa = NULL; @@ -2194,7 +2116,7 @@ rtcache_setdst_locked(struct route *ro, const struct sockaddr *sa) rtcache_invariants(ro); if (ro->ro_sa != NULL) { if (ro->ro_sa->sa_family == sa->sa_family) { - rtcache_clear(ro); + ro->_ro_rt = NULL; sockaddr_copy(ro->ro_sa, ro->ro_sa->sa_len, sa); rtcache_invariants(ro); return 0; diff --git a/sys/net/route.h b/sys/net/route.h index 5f18f6228bb..c9d3c57bd1e 100644 --- a/sys/net/route.h +++ b/sys/net/route.h @@ -64,8 +64,7 @@ struct route { struct rtentry *_ro_rt; struct sockaddr *ro_sa; - LIST_ENTRY(route) ro_rtcache_next; - bool ro_invalid; + uint64_t ro_rtcache_generation; struct psref ro_psref; int ro_bound; }; @@ -458,8 +457,8 @@ struct rtentry * static inline void rtcache_invariants(const struct route *ro) { + KASSERT(ro->ro_sa != NULL || ro->_ro_rt == NULL); - KASSERT(!ro->ro_invalid || ro->_ro_rt != NULL); } static inline struct rtentry * diff --git a/sys/netatalk/at_proto.c b/sys/netatalk/at_proto.c index ae44aacc4a5..f43cc9ff608 100644 --- a/sys/netatalk/at_proto.c +++ b/sys/netatalk/at_proto.c @@ -78,7 +78,6 @@ struct domain atalkdomain = { .dom_mowner = MOWNER_INIT("",""), .dom_sa_cmpofs = offsetof(struct sockaddr_at, sat_addr), .dom_sa_cmplen = sizeof(struct at_addr), - .dom_rtcache = LIST_HEAD_INITIALIZER(atalkdomain.dom_rtcache) }; int diff --git a/sys/netinet/in_proto.c b/sys/netinet/in_proto.c index aae37fb99cd..c1d3de768cf 100644 --- a/sys/netinet/in_proto.c +++ b/sys/netinet/in_proto.c @@ -489,7 +489,6 @@ struct domain inetdomain = { .dom_sa_any = (const struct sockaddr *)&in_any, .dom_sockaddr_const_addr = sockaddr_in_const_addr, .dom_sockaddr_addr = sockaddr_in_addr, - .dom_rtcache = LIST_HEAD_INITIALIZER(inetdomain.dom_rtcache) }; u_char ip_protox[IPPROTO_MAX]; diff --git a/sys/netinet6/in6_proto.c b/sys/netinet6/in6_proto.c index 3253cfa09c3..f68189f60e9 100644 --- a/sys/netinet6/in6_proto.c +++ b/sys/netinet6/in6_proto.c @@ -490,7 +490,6 @@ struct domain inet6domain = { .dom_sa_cmplen = sizeof(struct in6_addr), .dom_sa_any = (const struct sockaddr *)&in6_any, .dom_sockaddr_externalize = sockaddr_in6_externalize, - .dom_rtcache = LIST_HEAD_INITIALIZER(inet6domain.dom_rtcache) }; #if 0 diff --git a/sys/netmpls/mpls_proto.c b/sys/netmpls/mpls_proto.c index 86996c94ba7..e583da9e274 100644 --- a/sys/netmpls/mpls_proto.c +++ b/sys/netmpls/mpls_proto.c @@ -391,5 +391,4 @@ struct domain mplsdomain = { .dom_mowner = MOWNER_INIT("MPLS", ""), .dom_sa_cmpofs = offsetof(struct sockaddr_mpls, smpls_addr), .dom_sa_cmplen = sizeof(union mpls_shim), - .dom_rtcache = LIST_HEAD_INITIALIZER(mplsdomain.dom_rtcache) }; diff --git a/sys/netnatm/natm_proto.c b/sys/netnatm/natm_proto.c index 46ffb217af3..67e1ecb9fb6 100644 --- a/sys/netnatm/natm_proto.c +++ b/sys/netnatm/natm_proto.c @@ -105,7 +105,6 @@ struct domain natmdomain = { .dom_protosw = natmsw, .dom_protoswNPROTOSW = &natmsw[sizeof(natmsw)/sizeof(natmsw[0])], .dom_ifqueues = { &natmintrq, NULL }, - .dom_rtcache = LIST_HEAD_INITIALIZER(natmdomain.dom_rtcache) }; #ifdef NATM_STAT u_int natm_sodropcnt = 0; /* # mbufs dropped due to full sb */ diff --git a/sys/rump/net/lib/libsockin/sockin.c b/sys/rump/net/lib/libsockin/sockin.c index b48bbfb3def..940b4cfcc5e 100644 --- a/sys/rump/net/lib/libsockin/sockin.c +++ b/sys/rump/net/lib/libsockin/sockin.c @@ -162,7 +162,6 @@ struct domain sockindomain = { .dom_ifqueues = { NULL }, .dom_link = { NULL }, .dom_mowner = MOWNER_INIT("",""), - .dom_rtcache = { NULL }, .dom_sockaddr_cmp = NULL }; struct domain sockin6domain = { @@ -181,7 +180,6 @@ struct domain sockin6domain = { .dom_ifqueues = { NULL }, .dom_link = { NULL }, .dom_mowner = MOWNER_INIT("",""), - .dom_rtcache = { NULL }, .dom_sockaddr_cmp = NULL }; diff --git a/sys/sys/domain.h b/sys/sys/domain.h index cc710b90a80..53afe7987fd 100644 --- a/sys/sys/domain.h +++ b/sys/sys/domain.h @@ -90,7 +90,6 @@ struct domain { struct mowner dom_mowner; uint_fast8_t dom_sa_cmpofs; uint_fast8_t dom_sa_cmplen; - struct dom_rtlist dom_rtcache; }; STAILQ_HEAD(domainhead,domain);