commit cf4cfa126987aa03b643fa26566a6b1e84f16970 Author: Ryota Ozaki Date: Wed Jan 27 11:42:27 2016 +0900 Implement softint-based if_input (with softint + ifqueue) diff --git a/sys/dev/pci/if_iwm.c b/sys/dev/pci/if_iwm.c index 12c60bb..ecde05c 100644 --- a/sys/dev/pci/if_iwm.c +++ b/sys/dev/pci/if_iwm.c @@ -6825,7 +6825,7 @@ iwm_attach(device_t parent, device_t self, void *aux) IFQ_SET_READY(&ifp->if_snd); memcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); - if_initialize(ifp); + if_initialize(ifp, 0); #if 0 ieee80211_ifattach(ic); #else diff --git a/sys/dev/pci/if_rtwn.c b/sys/dev/pci/if_rtwn.c index 99c7d46..fe60c68 100644 --- a/sys/dev/pci/if_rtwn.c +++ b/sys/dev/pci/if_rtwn.c @@ -355,7 +355,7 @@ rtwn_attach(device_t parent, device_t self, void *aux) IFQ_SET_READY(&ifp->if_snd); memcpy(ifp->if_xname, device_xname(sc->sc_dev), IFNAMSIZ); - if_initialize(ifp); + if_initialize(ifp, 0); ieee80211_ifattach(ic); if_register(ifp); diff --git a/sys/dev/pci/if_wm.c b/sys/dev/pci/if_wm.c index 4124859..6a19ffe 100644 --- a/sys/dev/pci/if_wm.c +++ b/sys/dev/pci/if_wm.c @@ -7121,7 +7121,7 @@ wm_rxeof(struct wm_rxqueue *rxq) bpf_mtap(ifp, m); /* Pass it on. */ - (*ifp->if_input)(ifp, m); + if_input(ifp, m); WM_RX_LOCK(rxq); diff --git a/sys/net/if.c b/sys/net/if.c index 56e3d98..40d65fc 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -117,6 +117,7 @@ __KERNEL_RCSID(0, "$NetBSD: if.c,v 1.322 2016/01/21 15:41:29 riastradh Exp $"); #include #include #include +#include #include #include @@ -198,6 +199,7 @@ static void if_attachdomain1(struct ifnet *); static int ifconf(u_long, void *); static int if_clone_create(const char *); static int if_clone_destroy(const char *); +static void if_input_softint(void *); #if defined(INET) || defined(INET6) static void sysctl_net_pktq_setup(struct sysctllog **, int); @@ -564,6 +566,15 @@ skip: ifindex2ifnet[ifp->if_index] = ifp; } +static void +if_input_init_ifq(void *p, void *arg __unused, struct cpu_info *ci __unused) +{ + struct ifqueue *const ifq = p; + + memset(ifq, 0, sizeof(*ifq)); + ifq->ifq_maxlen = IFQ_MAXLEN; +} + /* * Initialize an interface and assign an index for it. * @@ -571,12 +582,12 @@ skip: * (e.g., ether_ifattach and ieee80211_ifattach) or if_alloc_sadl, * and be followed by if_register: * - * if_initialize(ifp); + * if_initialize(ifp, 0); * ether_ifattach(ifp, enaddr); * if_register(ifp); */ void -if_initialize(ifnet_t *ifp) +if_initialize(ifnet_t *ifp, int flags) { KASSERT(if_indexlim > 0); TAILQ_INIT(&ifp->if_addrlist); @@ -618,6 +629,13 @@ if_initialize(ifnet_t *ifp) IF_AFDATA_LOCK_INIT(ifp); if_getindex(ifp); + + if ((flags & IF_INPUTF_NO_SOFTINT) == 0) { + ifp->if_input_si = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE, + if_input_softint, ifp); + ifp->if_input_ifqs = percpu_alloc(sizeof(struct ifqueue)); + percpu_foreach(ifp->if_input_ifqs, &if_input_init_ifq, NULL); + } } /* @@ -655,7 +673,7 @@ if_register(ifnet_t *ifp) void if_attach(ifnet_t *ifp) { - if_initialize(ifp); + if_initialize(ifp, 0); if_register(ifp); } @@ -728,6 +746,14 @@ if_purgeaddrs(struct ifnet *ifp, int family, void (*purgeaddr)(struct ifaddr *)) } } +static void +if_input_purge_ifq(void *p, void *arg __unused, struct cpu_info *ci __unused) +{ + struct ifqueue *const ifq = p; + + IF_PURGE(ifq); +} + /* * Detach an interface from the list of "active" interfaces, * freeing any resources as we go along. @@ -925,6 +951,12 @@ again: xc = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL); xc_wait(xc); + if (ifp->if_input_ifqs != NULL) { + percpu_foreach(ifp->if_input_ifqs, &if_input_purge_ifq, NULL); + percpu_free(ifp->if_input_ifqs, sizeof(struct ifqueue)); + softint_disestablish(ifp->if_input_si); + } + splx(s); } @@ -2442,6 +2474,54 @@ if_mcast_op(ifnet_t *ifp, const unsigned long cmd, const struct sockaddr *sa) } static void +if_input_softint(void *arg) +{ + struct ifnet *ifp = arg; + struct mbuf *m; + struct ifqueue *ifq; + int s; + + s = splnet(); + ifq = percpu_getref(ifp->if_input_ifqs); + while (!IF_IS_EMPTY(ifq)) { + IF_DEQUEUE(ifq, m); + ifp->if_input(ifp, m); + } + percpu_putref(ifp->if_input_ifqs); + splx(s); +} + +/* + * The common interface input routine that is called by device drivers. + * We ensure that the upper layer routines, e.g., ether_input and bridge_input, + * aren't run in hardware interrupt context. + */ +void +if_input(struct ifnet *ifp, struct mbuf *m) +{ + + if (ifp->if_input_ifqs) { + struct ifqueue *ifq = percpu_getref(ifp->if_input_ifqs); + + KASSERT(cpu_intr_p()); + + if (IF_QFULL(ifq)) { + IF_DROP(ifq); + m_freem(m); + return; + } + IF_ENQUEUE(ifq, m); + percpu_putref(ifp->if_input_ifqs); + + softint_schedule(ifp->if_input_si); + } else { + KASSERT(!cpu_intr_p()); + + ifp->if_input(ifp, m); + } +} + +static void sysctl_sndq_setup(struct sysctllog **clog, const char *ifname, struct ifaltq *ifq) { diff --git a/sys/net/if.h b/sys/net/if.h index f8051e4..803dc6d 100644 --- a/sys/net/if.h +++ b/sys/net/if.h @@ -85,6 +85,7 @@ #include #ifdef _KERNEL #include +#include #endif /* @@ -351,12 +352,14 @@ typedef struct ifnet { struct ifnet_lock *if_ioctl_lock; #ifdef _KERNEL /* XXX kvm(3) */ struct callout *if_slowtimo_ch; -#endif #ifdef GATEWAY struct kmutex *if_afdata_lock; #else struct krwlock *if_afdata_lock; #endif + void *if_input_si; + percpu_t *if_input_ifqs; +#endif } ifnet_t; #define if_mtu if_data.ifi_mtu @@ -924,7 +927,8 @@ void if_activate_sadl(struct ifnet *, struct ifaddr *, const struct sockaddr_dl *); void if_set_sadl(struct ifnet *, const void *, u_char, bool); void if_alloc_sadl(struct ifnet *); -void if_initialize(struct ifnet *); +void if_initialize(struct ifnet *, int); +#define IF_INPUTF_NO_SOFTINT 1 /* Don't use softint in if_input */ void if_register(struct ifnet *); void if_attach(struct ifnet *); /* Deprecated. Use if_initialize and if_register */ void if_attachdomain(void); @@ -946,6 +950,7 @@ int if_do_dad(struct ifnet *); int if_mcast_op(ifnet_t *, const unsigned long, const struct sockaddr *); int if_flags_set(struct ifnet *, const short); int if_clone_list(int, char *, int *); +void if_input(struct ifnet *, struct mbuf *); void ifa_insert(struct ifnet *, struct ifaddr *); void ifa_remove(struct ifnet *, struct ifaddr *); diff --git a/sys/net/if_tap.c b/sys/net/if_tap.c index c7a6aa9..f0168e1 100644 --- a/sys/net/if_tap.c +++ b/sys/net/if_tap.c @@ -342,7 +342,7 @@ tap_attach(device_t parent, device_t self, void *aux) sc->sc_ec.ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU; /* Those steps are mandatory for an Ethernet driver. */ - if_initialize(ifp); + if_initialize(ifp, IF_INPUTF_NO_SOFTINT); ether_ifattach(ifp, enaddr); if_register(ifp); @@ -1061,7 +1061,7 @@ tap_dev_write(int unit, struct uio *uio, int flags) bpf_mtap(ifp, m); s = splnet(); - (*ifp->if_input)(ifp, m); + if_input(ifp, m); splx(s); return (0);