diff --git a/sys/altq/altq_cbq.c b/sys/altq/altq_cbq.c index 548d9c2..6471cad 100644 --- a/sys/altq/altq_cbq.c +++ b/sys/altq/altq_cbq.c @@ -592,7 +592,7 @@ cbqrestart(struct ifaltq *ifq) ifp = ifq->altq_ifp; if (ifp->if_start && cbqp->cbq_qlen > 0 && (ifp->if_flags & IFF_OACTIVE) == 0) - (*ifp->if_start)(ifp); + if_start_lock(ifp); } static void diff --git a/sys/altq/altq_subr.c b/sys/altq/altq_subr.c index 5231333..2a45204 100644 --- a/sys/altq/altq_subr.c +++ b/sys/altq/altq_subr.c @@ -366,7 +366,7 @@ tbr_timeout(void *arg) active++; if (!IFQ_IS_EMPTY(&ifp->if_snd) && ifp->if_start != NULL) { int _s = splnet(); - (*ifp->if_start)(ifp); + if_start_lock(ifp); splx(_s); } diff --git a/sys/dev/pci/if_wm.c b/sys/dev/pci/if_wm.c index 34ca76a..71eaaba 100644 --- a/sys/dev/pci/if_wm.c +++ b/sys/dev/pci/if_wm.c @@ -2407,6 +2407,7 @@ alloc_retry: strlcpy(ifp->if_xname, xname, IFNAMSIZ); ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_extflags = IFEF_START_MPSAFE; ifp->if_ioctl = wm_ioctl; if ((sc->sc_flags & WM_F_NEWQUEUE) != 0) { ifp->if_start = wm_nq_start; @@ -6080,6 +6081,8 @@ wm_start(struct ifnet *ifp) struct wm_softc *sc = ifp->if_softc; struct wm_txqueue *txq = &sc->sc_queue[0].wmq_txq; + KASSERT(ifp->if_extflags & IFEF_START_MPSAFE); + mutex_enter(txq->txq_lock); if (!sc->sc_stopping) wm_start_locked(ifp); @@ -6597,6 +6600,8 @@ wm_nq_start(struct ifnet *ifp) struct wm_softc *sc = ifp->if_softc; struct wm_txqueue *txq = &sc->sc_queue[0].wmq_txq; + KASSERT(ifp->if_extflags & IFEF_START_MPSAFE); + mutex_enter(txq->txq_lock); if (!sc->sc_stopping) wm_nq_start_locked(ifp); diff --git a/sys/dist/pf/net/pf.c b/sys/dist/pf/net/pf.c index 39ba12d..1919854 100644 --- a/sys/dist/pf/net/pf.c +++ b/sys/dist/pf/net/pf.c @@ -5381,7 +5381,7 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, else if (m0->m_pkthdr.csum_flags & M_UDPV4_CSUM_OUT) udpstat.udps_outhwcsum++; #endif /* !__NetBSD__ */ - error = (*ifp->if_output)(ifp, m0, dst, NULL); + error = if_output_lock(ifp, ifp, m0, dst, NULL); goto done; } diff --git a/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c b/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c index f2d6d90..89838fa 100644 --- a/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c +++ b/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c @@ -1228,9 +1228,7 @@ ipf_fastroute(mb_t *m0, mb_t **mpp, fr_info_t *fin, frdest_t *fdp) ip->ip_sum = in_cksum(m, hlen); # endif /* M_CSUM_IPv4 */ - KERNEL_LOCK(1, NULL); - error = (*ifp->if_output)(ifp, m, dst, rt); - KERNEL_UNLOCK_ONE(NULL); + error = if_output_lock(ifp, ifp, m, dst, rt); goto done; } diff --git a/sys/kern/kern_pmf.c b/sys/kern/kern_pmf.c index dbafa98..4aceda7 100644 --- a/sys/kern/kern_pmf.c +++ b/sys/kern/kern_pmf.c @@ -911,7 +911,7 @@ pmf_class_network_resume(device_t dev, const pmf_qual_t *qual) ifp->if_flags &= ~IFF_RUNNING; if ((*ifp->if_init)(ifp) != 0) aprint_normal_ifnet(ifp, "resume failed\n"); - (*ifp->if_start)(ifp); + if_start_lock(ifp); } splx(s); diff --git a/sys/net/agr/if_agr.c b/sys/net/agr/if_agr.c index fa21b44..6ebf5f1 100644 --- a/sys/net/agr/if_agr.c +++ b/sys/net/agr/if_agr.c @@ -209,7 +209,7 @@ agr_xmit_frame(struct ifnet *ifp_port, struct mbuf *m) m_copydata(m, 0, hdrlen, &dst->sa_data); m_adj(m, hdrlen); - error = (*ifp_port->if_output)(ifp_port, m, dst, NULL); + error = if_output_lock(ifp_port, ifp_port, m, dst, NULL); return error; } diff --git a/sys/net/bpf.c b/sys/net/bpf.c index eac083b..87b3651 100644 --- a/sys/net/bpf.c +++ b/sys/net/bpf.c @@ -723,7 +723,7 @@ bpf_write(struct file *fp, off_t *offp, struct uio *uio, mc = NULL; s = splsoftnet(); - error = (*ifp->if_output)(ifp, m, (struct sockaddr *) &dst, NULL); + error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); if (mc != NULL) { if (error == 0) diff --git a/sys/net/if.c b/sys/net/if.c index 8b076e9..277ae2e 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -2788,7 +2788,7 @@ if_transmit(struct ifnet *ifp, struct mbuf *m) ifp->if_omcasts++; if ((ifp->if_flags & IFF_OACTIVE) == 0) - (*ifp->if_start)(ifp); + if_start_lock(ifp); out: splx(s); diff --git a/sys/net/if.h b/sys/net/if.h index 8c10b21..9ba786e 100644 --- a/sys/net/if.h +++ b/sys/net/if.h @@ -242,7 +242,7 @@ typedef struct ifnet { u_short if_index; /* numeric abbreviation for this if */ short if_timer; /* time 'til if_slowtimo called */ short if_flags; /* up/down, broadcast, etc. */ - short if__pad1; /* be nice to m68k ports */ + short if_extflags; /* if_output MP-safe, etc. */ struct if_data if_data; /* statistics and other data about if */ /* * Procedure handles. If you add more of these, don't forget the @@ -255,7 +255,7 @@ typedef struct ifnet { (struct ifnet *, struct mbuf *); void (*if_start) /* initiate output routine */ (struct ifnet *); - int (*if_transmit) /* output routine (direct) */ + int (*if_transmit) /* output routine, must be MP-safe */ (struct ifnet *, struct mbuf *); int (*if_ioctl) /* ioctl routine */ (struct ifnet *, u_long, void *); @@ -379,6 +379,53 @@ typedef struct ifnet { #define IFF_LINK2 0x4000 /* per link layer defined bit */ #define IFF_MULTICAST 0x8000 /* supports multicast */ +#define IFEF_OUTPUT_MPSAFE 0x0001 /* if_output() can run parallel */ +#define IFEF_START_MPSAFE 0x0002 /* if_start() can run parallel */ + +static inline bool +if_output_is_mpsafe(struct ifnet *ifp) +{ + + return ((ifp->if_extflags & IFEF_OUTPUT_MPSAFE) != 0); +} + +static inline int +if_output_lock(struct ifnet *cifp, struct ifnet *ifp, struct mbuf *m, + const struct sockaddr *dst, const struct rtentry *rt) +{ + + if (if_output_is_mpsafe(cifp)) { + return (*cifp->if_output)(ifp, m, dst, rt); + } else { + int ret; + + KERNEL_LOCK(1, NULL); + ret = (*cifp->if_output)(ifp, m, dst, rt); + KERNEL_UNLOCK_ONE(NULL); + return ret; + } +} + +static inline bool +if_start_is_mpsafe(struct ifnet *ifp) +{ + + return ((ifp->if_extflags & IFEF_START_MPSAFE) != 0); +} + +static inline void +if_start_lock(struct ifnet *ifp) +{ + + if (if_start_is_mpsafe(ifp)) { + (*ifp->if_start)(ifp); + } else { + KERNEL_LOCK(1, NULL); + (*ifp->if_start)(ifp); + KERNEL_UNLOCK_ONE(NULL); + } +} + #define IFFBITS \ "\020\1UP\2BROADCAST\3DEBUG\4LOOPBACK\5POINTOPOINT\6NOTRAILERS" \ "\7RUNNING\10NOARP\11PROMISC\12ALLMULTI\13OACTIVE\14SIMPLEX" \ diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c index 64354d6..5093d9d 100644 --- a/sys/net/if_bridge.c +++ b/sys/net/if_bridge.c @@ -424,6 +424,7 @@ bridge_clone_create(struct if_clone *ifc, int unit) if_initname(ifp, ifc->ifc_name, unit); ifp->if_softc = sc; + ifp->if_extflags = IFEF_OUTPUT_MPSAFE; ifp->if_mtu = ETHERMTU; ifp->if_ioctl = bridge_ioctl; ifp->if_output = bridge_output; @@ -1378,6 +1379,7 @@ bridge_enqueue(struct bridge_softc *sc, struct ifnet *dst_ifp, struct mbuf *m, } #ifdef ALTQ + KERNEL_LOCK(1, NULL); /* * If ALTQ is enabled on the member interface, do * classification; the queueing discipline might @@ -1388,6 +1390,7 @@ bridge_enqueue(struct bridge_softc *sc, struct ifnet *dst_ifp, struct mbuf *m, /* XXX IFT_ETHER */ altq_etherclassify(&dst_ifp->if_snd, m); } + KERNEL_UNLOCK_ONE(NULL); #endif /* ALTQ */ len = m->m_pkthdr.len; @@ -1425,6 +1428,8 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa, struct bridge_softc *sc; int s; + KASSERT(ifp->if_extflags & IFEF_OUTPUT_MPSAFE); + if (m->m_len < ETHER_HDR_LEN) { m = m_pullup(m, ETHER_HDR_LEN); if (m == NULL) diff --git a/sys/net/if_ecosubr.c b/sys/net/if_ecosubr.c index 0987661..838d01c 100644 --- a/sys/net/if_ecosubr.c +++ b/sys/net/if_ecosubr.c @@ -410,7 +410,7 @@ eco_input(struct ifnet *ifp, struct mbuf *m) /* dst->sa_len??? */ dst->sa_family = AF_UNSPEC; memcpy(dst->sa_data, eh, ECO_HDR_LEN); - ifp->if_output(ifp, m, dst, NULL); + if_output_lock(ifp, ifp, m, dst, NULL); return; } default: @@ -782,7 +782,7 @@ eco_inputidle(struct ifnet *ifp) break; } ec->ec_state = ECO_IDLE; - ifp->if_start(ifp); + if_start_lock(ifp); } /* diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c index 95a248f..b64f50c 100644 --- a/sys/net/if_ethersubr.c +++ b/sys/net/if_ethersubr.c @@ -204,9 +204,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, struct at_ifaddr *aa; #endif /* NETATALK */ -#ifndef NET_MPSAFE - KASSERT(KERNEL_LOCKED_P()); -#endif + KASSERT(ifp->if_extflags & IFEF_OUTPUT_MPSAFE); #ifdef MBUFTRACE m_claimm(m, ifp->if_mowner); @@ -238,13 +236,17 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, #ifdef INET case AF_INET: + KERNEL_LOCK(1, NULL); if (m->m_flags & M_BCAST) (void)memcpy(edst, etherbroadcastaddr, sizeof(edst)); else if (m->m_flags & M_MCAST) ETHER_MAP_IP_MULTICAST(&satocsin(dst)->sin_addr, edst); else if ((error = arpresolve(ifp, rt, m, dst, edst, - sizeof(edst))) != 0) + sizeof(edst))) != 0) { + KERNEL_UNLOCK_ONE(NULL); return error == EWOULDBLOCK ? 0 : error; + } + KERNEL_UNLOCK_ONE(NULL); /* If broadcasting on a simplex interface, loopback a copy */ if ((m->m_flags & M_BCAST) && (ifp->if_flags & IFF_SIMPLEX)) mcopy = m_copy(m, 0, (int)M_COPYALL); @@ -292,10 +294,12 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, #endif #ifdef NETATALK case AF_APPLETALK: + KERNEL_LOCK(1, NULL); if (aarpresolve(ifp, m, (const struct sockaddr_at *)dst, edst)) { #ifdef NETATALKDEBUG printf("aarpresolv failed\n"); #endif /* NETATALKDEBUG */ + KERNEL_UNLOCK_ONE(NULL); return (0); } /* @@ -303,8 +307,10 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, */ aa = (struct at_ifaddr *) at_ifawithnet( (const struct sockaddr_at *)dst, ifp); - if (aa == NULL) + if (aa == NULL) { + KERNEL_UNLOCK_ONE(NULL); goto bad; + } /* * In the phase 2 case, we need to prepend an mbuf for the @@ -325,6 +331,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, } else { etype = htons(ETHERTYPE_ATALK); } + KERNEL_UNLOCK_ONE(NULL); break; #endif /* NETATALK */ case pseudo_AF_HDRCMPLT: @@ -349,6 +356,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, } #ifdef MPLS + KERNEL_LOCK(1, NULL); { struct m_tag *mtag; mtag = m_tag_find(m, PACKET_TAG_MPLS, NULL); @@ -358,6 +366,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, m_tag_delete(m, mtag); } } + KERNEL_UNLOCK_ONE(NULL); #endif if (mcopy) @@ -410,6 +419,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, #endif /* NCARP > 0 */ #ifdef ALTQ + KERNEL_LOCK(1, NULL); /* * If ALTQ is enabled on the parent interface, do * classification; the queueing discipline might not @@ -418,6 +428,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, */ if (ALTQ_IS_ENABLED(&ifp->if_snd)) altq_etherclassify(&ifp->if_snd, m); + KERNEL_UNLOCK_ONE(NULL); #endif return ifq_enqueue(ifp, m); @@ -910,6 +921,7 @@ ether_ifattach(struct ifnet *ifp, const uint8_t *lla) { struct ethercom *ec = (struct ethercom *)ifp; + ifp->if_extflags |= IFEF_OUTPUT_MPSAFE; ifp->if_type = IFT_ETHER; ifp->if_hdrlen = ETHER_HDR_LEN; ifp->if_dlt = DLT_EN10MB; diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c index a62c058..735141e 100644 --- a/sys/net/if_loop.c +++ b/sys/net/if_loop.c @@ -157,6 +157,7 @@ loop_clone_create(struct if_clone *ifc, int unit) ifp->if_mtu = LOMTU; ifp->if_flags = IFF_LOOPBACK | IFF_MULTICAST | IFF_RUNNING; + ifp->if_extflags = IFEF_OUTPUT_MPSAFE; ifp->if_ioctl = loioctl; ifp->if_output = looutput; #ifdef ALTQ @@ -211,12 +212,12 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct ifqueue *ifq = NULL; int s, isr = -1; int csum_flags; + int error = 0; size_t pktlen; MCLAIM(m, ifp->if_mowner); -#ifndef NET_MPSAFE - KASSERT(KERNEL_LOCKED_P()); -#endif + + KERNEL_LOCK(1, NULL); if ((m->m_flags & M_PKTHDR) == 0) panic("looutput: no header mbuf"); @@ -226,8 +227,9 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, if (rt && rt->rt_flags & (RTF_REJECT|RTF_BLACKHOLE)) { m_freem(m); - return (rt->rt_flags & RTF_BLACKHOLE ? 0 : + error = (rt->rt_flags & RTF_BLACKHOLE ? 0 : rt->rt_flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH); + goto out; } pktlen = m->m_pkthdr.len; @@ -241,8 +243,6 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, */ if ((ALTQ_IS_ENABLED(&ifp->if_snd) || TBR_IS_ENABLED(&ifp->if_snd)) && ifp->if_start == lostart) { - int error; - /* * If the queueing discipline needs packet classification, * do it before prepending the link headers. @@ -250,12 +250,14 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); M_PREPEND(m, sizeof(uint32_t), M_DONTWAIT); - if (m == NULL) - return (ENOBUFS); + if (m == NULL) { + error = ENOBUFS; + goto out; + } *(mtod(m, uint32_t *)) = dst->sa_family; error = ifp->if_transmit(ifp, m); - return (error); + goto out; } #endif /* ALTQ */ @@ -310,12 +312,13 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, printf("%s: can't handle af%d\n", ifp->if_xname, dst->sa_family); m_freem(m); - return (EAFNOSUPPORT); + error = EAFNOSUPPORT; + goto out; } s = splnet(); if (__predict_true(pktq)) { - int error = 0; + error = 0; if (__predict_true(pktq_enqueue(pktq, m, 0))) { ifp->if_ipackets++; @@ -325,20 +328,23 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, error = ENOBUFS; } splx(s); - return error; + goto out; } if (IF_QFULL(ifq)) { IF_DROP(ifq); m_freem(m); splx(s); - return (ENOBUFS); + error = ENOBUFS; + goto out; } IF_ENQUEUE(ifq, m); schednetisr(isr); ifp->if_ipackets++; ifp->if_ibytes += m->m_pkthdr.len; splx(s); - return (0); +out: + KERNEL_UNLOCK_ONE(NULL); + return error; } #ifdef ALTQ diff --git a/sys/net/if_mpls.c b/sys/net/if_mpls.c index 96dd7f2..310d9e0 100644 --- a/sys/net/if_mpls.c +++ b/sys/net/if_mpls.c @@ -478,9 +478,7 @@ mpls_send_frame(struct mbuf *m, struct ifnet *ifp, const struct rtentry *rt) #ifdef INET ret = ip_if_output(ifp, m, rt->rt_gateway, rt); #else - KERNEL_LOCK(1, NULL); - ret = (*ifp->if_output)(ifp, m, rt->rt_gateway, rt); - KERNEL_UNLOCK_ONE(NULL); + if_output_lock(ifp, ifp, m, rt->rt_gateway, rt); #endif return ret; break; diff --git a/sys/net/if_pppoe.c b/sys/net/if_pppoe.c index c4b75d3..d550cdd 100644 --- a/sys/net/if_pppoe.c +++ b/sys/net/if_pppoe.c @@ -880,7 +880,7 @@ pppoe_output(struct pppoe_softc *sc, struct mbuf *m) m->m_flags &= ~(M_BCAST|M_MCAST); sc->sc_sppp.pp_if.if_opackets++; - return sc->sc_eth_if->if_output(sc->sc_eth_if, m, &dst, NULL); + return if_output_lock(sc->sc_eth_if, sc->sc_eth_if, m, &dst, NULL); } static int @@ -1377,7 +1377,7 @@ pppoe_send_padt(struct ifnet *outgoing_if, u_int session, const uint8_t *dest) memcpy(&eh->ether_dhost, dest, ETHER_ADDR_LEN); m0->m_flags &= ~(M_BCAST|M_MCAST); - return outgoing_if->if_output(outgoing_if, m0, &dst, NULL); + return if_output_lock(outgoing_if, outgoing_if, m0, &dst, NULL); } #ifdef PPPOE_SERVER diff --git a/sys/net/if_spppsubr.c b/sys/net/if_spppsubr.c index 41afd19..a1afd42 100644 --- a/sys/net/if_spppsubr.c +++ b/sys/net/if_spppsubr.c @@ -832,7 +832,7 @@ sppp_output(struct ifnet *ifp, struct mbuf *m, * framing according to RFC 1333. */ if (!(ifp->if_flags & IFF_OACTIVE)) - (*ifp->if_start)(ifp); + if_start_lock(ifp); ifp->if_obytes += m->m_pkthdr.len + sp->pp_framebytes; } splx(s); @@ -1214,7 +1214,7 @@ sppp_cisco_send(struct sppp *sp, int type, int32_t par1, int32_t par2) } else IF_ENQUEUE(&sp->pp_cpq, m); if (! (ifp->if_flags & IFF_OACTIVE)) - (*ifp->if_start)(ifp); + if_start_lock(ifp); ifp->if_obytes += m->m_pkthdr.len + sp->pp_framebytes; } @@ -1279,7 +1279,7 @@ sppp_cp_send(struct sppp *sp, u_short proto, u_char type, } else IF_ENQUEUE(&sp->pp_cpq, m); if (! (ifp->if_flags & IFF_OACTIVE)) - (*ifp->if_start)(ifp); + if_start_lock(ifp); ifp->if_obytes += m->m_pkthdr.len + sp->pp_framebytes; } @@ -4715,7 +4715,7 @@ sppp_auth_send(const struct cp *cp, struct sppp *sp, } else IF_ENQUEUE(&sp->pp_cpq, m); if (! (ifp->if_flags & IFF_OACTIVE)) - (*ifp->if_start)(ifp); + if_start_lock(ifp); ifp->if_obytes += m->m_pkthdr.len + 3; } diff --git a/sys/net/if_srt.c b/sys/net/if_srt.c index 117b1c9..e656f9c 100644 --- a/sys/net/if_srt.c +++ b/sys/net/if_srt.c @@ -232,7 +232,7 @@ srt_if_output( return 0; /* XXX ENETDOWN? */ } /* XXX is 0 the right last arg here? */ - return (*r->u.dstifp->if_output)(r->u.dstifp,m,&r->dst.sa,0); + return if_output_lock(r->u.dstifp, r->u.dstifp, m, &r->dst.sa, 0); } static int diff --git a/sys/net/ppp_tty.c b/sys/net/ppp_tty.c index 865e60f..ac2bbde 100644 --- a/sys/net/ppp_tty.c +++ b/sys/net/ppp_tty.c @@ -425,7 +425,7 @@ pppwrite(struct tty *tp, struct uio *uio, int flag) dst.sa_family = AF_UNSPEC; bcopy(mtod(m0, u_char *), dst.sa_data, PPP_HDRLEN); m_adj(m0, PPP_HDRLEN); - return ((*sc->sc_if.if_output)(&sc->sc_if, m0, &dst, (struct rtentry *)0)); + return if_output_lock(&sc->sc_if, &sc->sc_if, m0, &dst, (struct rtentry *)0); } /* diff --git a/sys/net80211/ieee80211_output.c b/sys/net80211/ieee80211_output.c index 1ce0782..04b9d3a 100644 --- a/sys/net80211/ieee80211_output.c +++ b/sys/net80211/ieee80211_output.c @@ -221,7 +221,7 @@ ieee80211_mgmt_output(struct ieee80211com *ic, struct ieee80211_node *ni, ic->ic_mgt_timer = timer; ifp->if_timer = 1; } - (*ifp->if_start)(ifp); + if_start_lock(ifp); return 0; } @@ -268,7 +268,7 @@ ieee80211_send_nulldata(struct ieee80211_node *ni) wh->i_fc[1] & IEEE80211_FC1_PWR_MGT ? "ena" : "dis"); IF_ENQUEUE(&ic->ic_mgtq, m); /* cheat */ - (*ifp->if_start)(ifp); + if_start_lock(ifp); return 0; } @@ -1362,7 +1362,7 @@ ieee80211_send_probereq(struct ieee80211_node *ni, ieee80211_chan2ieee(ic, ic->ic_curchan)); IF_ENQUEUE(&ic->ic_mgtq, m); - (*ic->ic_ifp->if_start)(ic->ic_ifp); + if_start_lock(ic->ic_ifp); return 0; } diff --git a/sys/net80211/ieee80211_proto.c b/sys/net80211/ieee80211_proto.c index 9a48ea8..a028411 100644 --- a/sys/net80211/ieee80211_proto.c +++ b/sys/net80211/ieee80211_proto.c @@ -1121,7 +1121,7 @@ ieee80211_newstate(struct ieee80211com *ic, enum ieee80211_state nstate, int arg if (ic->ic_opmode == IEEE80211_M_STA) ieee80211_notify_node_join(ic, ni, arg == IEEE80211_FC0_SUBTYPE_ASSOC_RESP); - (*ifp->if_start)(ifp); /* XXX not authorized yet */ + if_start_lock(ifp); /* XXX not authorized yet */ break; } /* diff --git a/sys/netatalk/aarp.c b/sys/netatalk/aarp.c index 4160d98..a32926e 100644 --- a/sys/netatalk/aarp.c +++ b/sys/netatalk/aarp.c @@ -238,7 +238,7 @@ aarpwhohas(struct ifnet *ifp, const struct sockaddr_at *sat) sa.sa_len = sizeof(struct sockaddr); sa.sa_family = AF_UNSPEC; - (*ifp->if_output) (ifp, m, &sa, NULL); /* XXX NULL should be routing */ + if_output_lock(ifp, ifp, m, &sa, NULL); /* XXX NULL should be routing */ /* information */ } @@ -429,7 +429,7 @@ at_aarpinput(struct ifnet *ifp, struct mbuf *m) sat.sat_len = sizeof(struct sockaddr_at); sat.sat_family = AF_APPLETALK; sat.sat_addr = spa; - (*ifp->if_output)(ifp, aat->aat_hold, + if_output_lock(ifp, ifp, aat->aat_hold, (struct sockaddr *) & sat, NULL); /* XXX */ aat->aat_hold = 0; } diff --git a/sys/netatalk/ddp_output.c b/sys/netatalk/ddp_output.c index aea1794..7db172b 100644 --- a/sys/netatalk/ddp_output.c +++ b/sys/netatalk/ddp_output.c @@ -203,5 +203,5 @@ ddp_route(struct mbuf *m, struct route *ro) #endif looutput(lo0ifp, copym, rtcache_getdst(ro), NULL); } - return (*ifp->if_output)(ifp, m, (struct sockaddr *)&gate, NULL); + return if_output_lock(ifp, ifp, m, (struct sockaddr *)&gate, NULL); } diff --git a/sys/netinet/if_arp.c b/sys/netinet/if_arp.c index 0070f65..473fb86 100644 --- a/sys/netinet/if_arp.c +++ b/sys/netinet/if_arp.c @@ -674,7 +674,7 @@ arprequest(struct ifnet *ifp, arps[ARP_STAT_SNDTOTAL]++; arps[ARP_STAT_SENDREQUEST]++; ARP_STAT_PUTREF(); - (*ifp->if_output)(ifp, m, &sa, NULL); + if_output_lock(ifp, ifp, m, &sa, NULL); } /* @@ -1251,7 +1251,7 @@ in_arpinput(struct mbuf *m) for (; m_hold != NULL; m_hold = m_hold_next) { m_hold_next = m_hold->m_nextpkt; m_hold->m_nextpkt = NULL; - (*ifp->if_output)(ifp, m_hold, sintosa(&sin), NULL); + if_output_lock(ifp, ifp, m_hold, sintosa(&sin), NULL); } } else LLE_WUNLOCK(la); @@ -1334,7 +1334,7 @@ reply: arps[ARP_STAT_SNDTOTAL]++; arps[ARP_STAT_SNDREPLY]++; ARP_STAT_PUTREF(); - (*ifp->if_output)(ifp, m, &sa, NULL); + if_output_lock(ifp, ifp, m, &sa, NULL); if (rcvif != NULL) m_put_rcvif_psref(rcvif, &psref); return; @@ -1888,9 +1888,7 @@ revarprequest(struct ifnet *ifp) sa.sa_len = 2; m->m_flags |= M_BCAST; - KERNEL_LOCK(1, NULL); - (*ifp->if_output)(ifp, m, &sa, NULL); - KERNEL_UNLOCK_ONE(NULL); + if_output_lock(ifp, ifp, m, &sa, NULL); } /* diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c index 7b9adad3..1733018 100644 --- a/sys/netinet/ip_carp.c +++ b/sys/netinet/ip_carp.c @@ -2113,7 +2113,7 @@ carp_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa, KASSERT(KERNEL_LOCKED_P()); if (sc->sc_carpdev != NULL && sc->sc_state == MASTER) { - return (sc->sc_carpdev->if_output(ifp, m, sa, rt)); + return if_output_lock(sc->sc_carpdev, ifp, m, sa, rt)); } else { m_freem(m); return (ENETUNREACH); diff --git a/sys/netinet/ip_flow.c b/sys/netinet/ip_flow.c index 0ff4145..609383a 100644 --- a/sys/netinet/ip_flow.c +++ b/sys/netinet/ip_flow.c @@ -323,14 +323,12 @@ ipflow_fastforward(struct mbuf *m) else dst = rtcache_getdst(&ipf->ipf_ro); - KERNEL_LOCK(1, NULL); - if ((error = (*rt->rt_ifp->if_output)(rt->rt_ifp, m, dst, rt)) != 0) { + if ((error = if_output_lock(rt->rt_ifp, rt->rt_ifp, m, dst, rt)) != 0) { if (error == ENOBUFS) ipf->ipf_dropped++; else ipf->ipf_errors++; } - KERNEL_UNLOCK_ONE(NULL); ret = 1; out: mutex_exit(&ipflow_lock); diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c index 466e2e5..f36d8cf 100644 --- a/sys/netinet/ip_output.c +++ b/sys/netinet/ip_output.c @@ -212,15 +212,8 @@ ip_if_output(struct ifnet * const ifp, struct mbuf * const m, return error; } -#ifndef NET_MPSAFE - KERNEL_LOCK(1, NULL); -#endif - - error = (*ifp->if_output)(ifp, m, dst, rt); + error = if_output_lock(ifp, ifp, m, dst, rt); -#ifndef NET_MPSAFE - KERNEL_UNLOCK_ONE(NULL); -#endif return error; } diff --git a/sys/netinet6/ip6_flow.c b/sys/netinet6/ip6_flow.c index 2394203..1b6c472 100644 --- a/sys/netinet6/ip6_flow.c +++ b/sys/netinet6/ip6_flow.c @@ -336,14 +336,12 @@ ip6flow_fastforward(struct mbuf **mp) ip6f->ip6f_uses++; - KERNEL_LOCK(1, NULL); /* Send on its way - straight to the interface output routine. */ - if ((error = (*rt->rt_ifp->if_output)(rt->rt_ifp, m, dst, rt)) != 0) { + if ((error = if_output_lock(rt->rt_ifp, rt->rt_ifp, m, dst, rt)) != 0) { ip6f->ip6f_dropped++; } else { ip6f->ip6f_forwarded++; } - KERNEL_UNLOCK_ONE(NULL); ret = 1; out: mutex_exit(&ip6flow_lock); diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index 8f5b246..d369f91 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -2291,16 +2291,10 @@ nd6_output(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m, if (ln != NULL) LLE_WUNLOCK(ln); -#ifndef NET_MPSAFE - KERNEL_LOCK(1, NULL); -#endif if ((ifp->if_flags & IFF_LOOPBACK) != 0) - error = (*ifp->if_output)(origifp, m, sin6tocsa(dst), rt); + error = if_output_lock(origifp, origifp, m, sin6tocsa(dst), rt); else - error = (*ifp->if_output)(ifp, m, sin6tocsa(dst), rt); -#ifndef NET_MPSAFE - KERNEL_UNLOCK_ONE(NULL); -#endif + error = if_output_lock(ifp, ifp, m, sin6tocsa(dst), rt); goto exit; bad: