diff --git a/sys/dev/ic/hd64570.c b/sys/dev/ic/hd64570.c index 359b03f..9df2225 100644 --- a/sys/dev/ic/hd64570.c +++ b/sys/dev/ic/hd64570.c @@ -171,6 +171,9 @@ static struct mbuf *sca_mbuf_alloc(struct sca_softc *, void *, u_int); static void sca_frame_print(sca_port_t *, sca_desc_t *, u_int8_t *); #endif +static IFNET_WRAP_IF_OUTPUT_DECL(sca_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(sca_output); +#define SCA_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(sca_output) #define sca_read_1(sc, reg) (sc)->sc_read_1(sc, reg) #define sca_read_2(sc, reg) (sc)->sc_read_2(sc, reg) @@ -446,7 +449,7 @@ sca_port_attach(struct sca_softc *sc, u_int port) ifp->if_type = IFT_PTPSERIAL; ifp->if_hdrlen = HDLC_HDRLEN; ifp->if_ioctl = sca_ioctl; - ifp->if_output = sca_output; + ifp->if_output = SCA_OUTPUT_LOCK; ifp->if_watchdog = sca_watchdog; ifp->if_snd.ifq_maxlen = IFQ_MAXLEN; scp->linkq.ifq_maxlen = 5; /* if we exceed this we are hosed already */ @@ -799,6 +802,8 @@ sca_output( int s, error, len; short mflags; + KASSERT(KERNEL_LOCKED_P()); + error = 0; if ((ifp->if_flags & IFF_UP) != IFF_UP) { diff --git a/sys/dev/ic/midway.c b/sys/dev/ic/midway.c index 83bf0f1..df057e7 100644 --- a/sys/dev/ic/midway.c +++ b/sys/dev/ic/midway.c @@ -765,7 +765,7 @@ done_probe: #endif ifp->if_flags = IFF_SIMPLEX|IFF_NOTRAILERS; ifp->if_ioctl = en_ioctl; - ifp->if_output = atm_output; + ifp->if_output = ATM_OUTPUT_LOCK; ifp->if_start = en_start; IFQ_SET_READY(&ifp->if_snd); diff --git a/sys/dev/ic/pdq_ifsubr.c b/sys/dev/ic/pdq_ifsubr.c index f117850..3b81413 100644 --- a/sys/dev/ic/pdq_ifsubr.c +++ b/sys/dev/ic/pdq_ifsubr.c @@ -496,7 +496,7 @@ pdq_ifattach( ifp->if_ioctl = pdq_ifioctl; #if !defined(__NetBSD__) - ifp->if_output = fddi_output; + ifp->if_output = FDDI_OUTPUT_LOCK; #endif ifp->if_start = pdq_ifstart; IFQ_SET_READY(&ifp->if_snd); diff --git a/sys/dev/pci/if_lmc.c b/sys/dev/pci/if_lmc.c index a40fb80..5afa064 100644 --- a/sys/dev/pci/if_lmc.c +++ b/sys/dev/pci/if_lmc.c @@ -3360,6 +3360,8 @@ ifnet_output(struct ifnet *ifp, struct mbuf *m, softc_t *sc = IFP2SC(ifp); int error = 0; + KASSERT(KERNEL_LOCKED_P()); + /* Fail if the link is down. */ if (sc->status.link_state != STATE_UP) { @@ -3393,6 +3395,10 @@ ifnet_output(struct ifnet *ifp, struct mbuf *m, return error; } +static IFNET_WRAP_IF_OUTPUT_DECL(ifnet_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(ifnet_output); +#define IFNET_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(ifnet_output) + static int /* context: process */ ifnet_ioctl(struct ifnet *ifp, u_long cmd, void *data) { @@ -3511,7 +3517,7 @@ ifnet_setup(struct ifnet *ifp) ifp->if_flags |= IFF_SIMPLEX; ifp->if_flags |= IFF_NOARP; ifp->_if_input = ifnet_input; - ifp->if_output = ifnet_output; + ifp->if_output = IFNET_OUTPUT_LOCK; ifp->if_start = ifnet_start; ifp->if_ioctl = ifnet_ioctl; ifp->if_watchdog = ifnet_watchdog; diff --git a/sys/dev/pci/if_wm.c b/sys/dev/pci/if_wm.c index 71d4104..6778cc8 100644 --- a/sys/dev/pci/if_wm.c +++ b/sys/dev/pci/if_wm.c @@ -2414,6 +2414,7 @@ alloc_retry: strlcpy(ifp->if_xname, xname, IFNAMSIZ); ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_extflags = IFEF_MPSAFE; ifp->if_ioctl = wm_ioctl; if ((sc->sc_flags & WM_F_NEWQUEUE) != 0) { ifp->if_start = wm_nq_start; diff --git a/sys/dev/ppbus/if_plip.c b/sys/dev/ppbus/if_plip.c index 96f2201..a3f4897 100644 --- a/sys/dev/ppbus/if_plip.c +++ b/sys/dev/ppbus/if_plip.c @@ -198,6 +198,9 @@ static int lpoutput(struct ifnet *, struct mbuf *, const struct sockaddr *, static void lpstart(struct ifnet *); static void lp_intr(void *); +static IFNET_WRAP_IF_OUTPUT_DECL(lpoutput); +static IFNET_WRAP_IF_OUTPUT_IMPL(lpoutput); +#define LPOUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(lpoutput) static int lp_probe(device_t parent, cfdata_t match, void *aux) @@ -230,7 +233,7 @@ lp_attach(device_t parent, device_t self, void *aux) ifp->if_mtu = LPMTU; ifp->if_flags = IFF_SIMPLEX | IFF_POINTOPOINT | IFF_MULTICAST; ifp->if_ioctl = lpioctl; - ifp->if_output = lpoutput; + ifp->if_output = LPOUTOUT_LOCK; ifp->if_start = lpstart; ifp->if_type = IFT_PARA; ifp->if_hdrlen = 0; @@ -707,6 +710,8 @@ lpoutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, int err; int s; + KASSERT(KERNEL_LOCKED_P()); + s = splnet(); if(sc->sc_dev_ok) { diff --git a/sys/dev/qbus/if_dmc.c b/sys/dev/qbus/if_dmc.c index ba3c987..3d969cc 100644 --- a/sys/dev/qbus/if_dmc.c +++ b/sys/dev/qbus/if_dmc.c @@ -182,6 +182,11 @@ static int dmcoutput(struct ifnet *, struct mbuf *, struct sockaddr *, struct rtentry *); static void dmcreset(device_t); +static IFNET_WRAP_IF_OUTPUT_DECL(dmcoutput); +static IFNET_WRAP_IF_OUTPUT_IMPL(dmcoutput); +#define DMCOUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(dmcoutput) + + CFATTACH_DECL_NEW(dmc, sizeof(struct dmc_softc), dmcmatch, dmcattach, NULL, NULL); @@ -259,7 +264,7 @@ dmcattach(device_t parent, device_t self, void *aux) strlcpy(sc->sc_if.if_xname, device_xname(sc->sc_dev), IFNAMSIZ); sc->sc_if.if_mtu = DMCMTU; sc->sc_if.if_init = dmcinit; - sc->sc_if.if_output = dmcoutput; + sc->sc_if.if_output = DMCOUTPUT_LOCK; sc->sc_if.if_ioctl = dmcioctl; sc->sc_if.if_watchdog = dmctimeout; sc->sc_if.if_flags = IFF_POINTOPOINT; @@ -426,6 +431,8 @@ dmcstart(struct ifnet *ifp) struct dmcbufs *rp; int n; + KASSERT(KERNEL_LOCKED_P()); + /* * Dequeue up to NXMT requests and map them to the UNIBUS. * If no more requests, or no dmc buffers available, just return. diff --git a/sys/dev/usb/if_upl.c b/sys/dev/usb/if_upl.c index 8e9dcc0..6b2817b 100644 --- a/sys/dev/usb/if_upl.c +++ b/sys/dev/usb/if_upl.c @@ -196,6 +196,11 @@ Static int upl_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); Static void upl_input(struct ifnet *, struct mbuf *); +Static IFNET_WRAP_IF_OUTPUT_DECL(upl_output); +Static IFNET_WRAP_IF_OUTPUT_IMPL(upl_output); +#define UPL_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(upl_output) + + /* * Probe for a Prolific chip. */ @@ -298,7 +303,7 @@ upl_attach(device_t parent, device_t self, void *aux) ifp->if_type = IFT_OTHER; ifp->if_addrlen = 0; ifp->if_hdrlen = 0; - ifp->if_output = upl_output; + ifp->if_output = UPL_OUTPUT_LOCK; ifp->_if_input = upl_input; ifp->if_baudrate = 12000000; ifp->if_dlt = DLT_RAW; @@ -989,6 +994,8 @@ upl_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, { int error; + KASSERT(KERNEL_LOCKED_P()); + DPRINTFN(10,("%s: %s: enter\n", device_xname(((struct upl_softc *)ifp->if_softc)->sc_dev), __func__)); diff --git a/sys/dev/usb/uhso.c b/sys/dev/usb/uhso.c index cf153f0..83207be 100644 --- a/sys/dev/usb/uhso.c +++ b/sys/dev/usb/uhso.c @@ -424,6 +424,10 @@ Static void uhso_ifnet_start(struct ifnet *); Static int uhso_ifnet_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); +Static IFNET_WRAP_IF_OUTPUT_DECL(uhso_ifnet_output); +Static IFNET_WRAP_IF_OUTPUT_IMPL(uhso_ifnet_output); +#define UHSO_IFNET_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(uhso_ifnet_output) + /******************************************************************************* * @@ -1937,7 +1941,7 @@ uhso_ifnet_attach(struct uhso_softc *sc, struct usbd_interface *ifh, int index) ifp->if_flags = IFF_NOARP | IFF_SIMPLEX; ifp->if_ioctl = uhso_ifnet_ioctl; ifp->if_start = uhso_ifnet_start; - ifp->if_output = uhso_ifnet_output; + ifp->if_output = UHSO_IFNET_OUTPUT_LOCK; IFQ_SET_READY(&ifp->if_snd); hp->hp_sc = sc; @@ -2344,6 +2348,8 @@ uhso_ifnet_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, { int error; + KASSERT(KERNEL_LOCKED_P()); + if (!ISSET(ifp->if_flags, IFF_RUNNING)) return EIO; diff --git a/sys/net/if.c b/sys/net/if.c index 8b076e9..277ae2e 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -2788,7 +2788,7 @@ if_transmit(struct ifnet *ifp, struct mbuf *m) ifp->if_omcasts++; if ((ifp->if_flags & IFF_OACTIVE) == 0) - (*ifp->if_start)(ifp); + if_start_lock(ifp); out: splx(s); diff --git a/sys/net/if.h b/sys/net/if.h index 8c10b21..046c2b8 100644 --- a/sys/net/if.h +++ b/sys/net/if.h @@ -242,7 +242,8 @@ typedef struct ifnet { u_short if_index; /* numeric abbreviation for this if */ short if_timer; /* time 'til if_slowtimo called */ short if_flags; /* up/down, broadcast, etc. */ - short if__pad1; /* be nice to m68k ports */ + short if_extflags; /* device driver MP-safe, etc. */ + /* be nice to m68k ports */ struct if_data if_data; /* statistics and other data about if */ /* * Procedure handles. If you add more of these, don't forget the @@ -379,6 +380,28 @@ typedef struct ifnet { #define IFF_LINK2 0x4000 /* per link layer defined bit */ #define IFF_MULTICAST 0x8000 /* supports multicast */ +#define IFEF_MPSAFE 0x0001 /* if_start() can run parallel */ + +static inline bool +if_is_mpsafe(struct ifnet *ifp) +{ + + return ((ifp->if_extflags & IFEF_MPSAFE) != 0); +} + +static inline void +if_start_lock(struct ifnet *ifp) +{ + + if (if_is_mpsafe(ifp)) { + (*ifp->if_start)(ifp); + } else { + KERNEL_LOCK(1, NULL); + (*ifp->if_start)(ifp); + KERNEL_UNLOCK_ONE(NULL); + } +} + #define IFFBITS \ "\020\1UP\2BROADCAST\3DEBUG\4LOOPBACK\5POINTOPOINT\6NOTRAILERS" \ "\7RUNNING\10NOARP\11PROMISC\12ALLMULTI\13OACTIVE\14SIMPLEX" \ @@ -1028,6 +1051,25 @@ __END_DECLS } \ } while (0) +#define IFNET_WRAP_IF_OUTPUT_DECL(name) \ + int \ + name##_wrapper(struct ifnet *, struct mbuf *, \ + const struct sockaddr *, const struct rtentry *) + +#define IFNET_WRAP_IF_OUTPUT_IMPL(name) \ + int \ + name##_wrapper(struct ifnet *__ifp, struct mbuf *__m, \ + const struct sockaddr *__sa, const struct rtentry *__rt) \ + { \ + int __ret; \ + KERNEL_LOCK(1, NULL); \ + __ret = name(__ifp, __m, __sa, __rt); \ + KERNEL_UNLOCK_ONE(NULL); \ + return __ret; \ + } + +#define IFNET_WRAP_IF_OUTPUT_CALL(name) name##_wrapper + extern struct pslist_head ifnet_pslist; extern struct psref_class *ifnet_psref_class; extern kmutex_t ifnet_mtx; diff --git a/sys/net/if_arcsubr.c b/sys/net/if_arcsubr.c index 953a35a..46c5334 100644 --- a/sys/net/if_arcsubr.c +++ b/sys/net/if_arcsubr.c @@ -105,6 +105,10 @@ static int arc_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); static void arc_input(struct ifnet *, struct mbuf *); +static IFNET_WRAP_IF_OUTPUT_DECL(arc_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(arc_output); +#define ARC_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(arc_output) + /* * ARCnet output routine. * Encapsulate a packet of type family for the local net. @@ -123,6 +127,8 @@ arc_output(struct ifnet *ifp, struct mbuf *m0, const struct sockaddr *dst, uint8_t atype, adst, myself; int tfrags, sflag, fsflag, rsflag; + KASSERT(KERNEL_LOCKED_P()); + if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) return (ENETDOWN); /* m, m1 aren't initialized yet */ @@ -620,7 +626,7 @@ arc_ifattach(struct ifnet *ifp, uint8_t lla) "%s: arc_ipmtu is %d, but must not exceed %d\n", ifp->if_xname, arc_ipmtu, ARC_PHDS_MAXMTU); - ifp->if_output = arc_output; + ifp->if_output = ARC_OUTPUT_LOCK; ifp->_if_input = arc_input; ac = (struct arccom *)ifp; ac->ac_seqid = (time_second) & 0xFFFF; /* try to make seqid unique */ diff --git a/sys/net/if_atm.h b/sys/net/if_atm.h index 78daeb3..3f82860 100644 --- a/sys/net/if_atm.h +++ b/sys/net/if_atm.h @@ -137,6 +137,10 @@ void atm_input(struct ifnet *, struct atm_pseudohdr *, struct mbuf *, void *); int atm_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); + +IFNET_WRAP_IF_OUTPUT_DECL(atm_output); +#define ATM_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(atm_output) + #endif #ifdef ATM_PVCEXT #ifdef _KERNEL diff --git a/sys/net/if_atmsubr.c b/sys/net/if_atmsubr.c index 2c54589..8a40528 100644 --- a/sys/net/if_atmsubr.c +++ b/sys/net/if_atmsubr.c @@ -71,6 +71,8 @@ __KERNEL_RCSID(0, "$NetBSD: if_atmsubr.c,v 1.59 2016/06/10 13:27:15 ozaki-r Exp #include #endif +IFNET_WRAP_IF_OUTPUT_IMPL(atm_output); + #define senderr(e) { error = (e); goto bad;} /* @@ -99,6 +101,8 @@ atm_output(struct ifnet *ifp, struct mbuf *m0, const struct sockaddr *dst, struct atmllc *atmllc; uint32_t atm_flags; + KASSERT(KERNEL_LOCKED_P()); + if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) senderr(ENETDOWN); @@ -294,7 +298,7 @@ atm_ifattach(struct ifnet *ifp) ifp->if_hdrlen = 0; ifp->if_dlt = DLT_ATM_RFC1483; ifp->if_mtu = ATMMTU; - ifp->if_output = atm_output; + ifp->if_output = ATM_OUTPUT_LOCK; #if 0 /* XXX XXX XXX */ ifp->if_input = atm_input; #endif diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c index 64354d6..5e421e0 100644 --- a/sys/net/if_bridge.c +++ b/sys/net/if_bridge.c @@ -1378,6 +1378,7 @@ bridge_enqueue(struct bridge_softc *sc, struct ifnet *dst_ifp, struct mbuf *m, } #ifdef ALTQ + KERNEL_LOCK(1, NULL); /* * If ALTQ is enabled on the member interface, do * classification; the queueing discipline might @@ -1388,6 +1389,7 @@ bridge_enqueue(struct bridge_softc *sc, struct ifnet *dst_ifp, struct mbuf *m, /* XXX IFT_ETHER */ altq_etherclassify(&dst_ifp->if_snd, m); } + KERNEL_UNLOCK_ONE(NULL); #endif /* ALTQ */ len = m->m_pkthdr.len; diff --git a/sys/net/if_ecosubr.c b/sys/net/if_ecosubr.c index 0987661..b0c403c 100644 --- a/sys/net/if_ecosubr.c +++ b/sys/net/if_ecosubr.c @@ -111,6 +111,10 @@ static void eco_defer(struct ifnet *, struct mbuf *, int); static void eco_retry_free(struct eco_retry *er); static void eco_retry(void *); +static IFNET_WRAP_IF_OUTPUT_DECL(eco_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(eco_output); +#define ECO_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(eco_output) + void eco_ifattach(struct ifnet *ifp, const uint8_t *lla) { @@ -122,7 +126,7 @@ eco_ifattach(struct ifnet *ifp, const uint8_t *lla) ifp->if_dlt = DLT_ECONET; ifp->if_mtu = ECO_MTU; - ifp->if_output = eco_output; + ifp->if_output = ECO_OUTPUT_LOCK; ifp->if_input = eco_input; ifp->if_start = eco_start; ifp->if_ioctl = eco_ioctl; @@ -179,6 +183,8 @@ eco_output(struct ifnet *ifp, struct mbuf *m0, const struct sockaddr *dst, struct eco_arp *ecah; #endif + KASSERT(KERNEL_LOCKED_P()); + if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) senderr(ENETDOWN); /* diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c index 95a248f..20a812d 100644 --- a/sys/net/if_ethersubr.c +++ b/sys/net/if_ethersubr.c @@ -204,10 +204,6 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, struct at_ifaddr *aa; #endif /* NETATALK */ -#ifndef NET_MPSAFE - KASSERT(KERNEL_LOCKED_P()); -#endif - #ifdef MBUFTRACE m_claimm(m, ifp->if_mowner); #endif @@ -238,13 +234,17 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, #ifdef INET case AF_INET: + KERNEL_LOCK(1, NULL); if (m->m_flags & M_BCAST) (void)memcpy(edst, etherbroadcastaddr, sizeof(edst)); else if (m->m_flags & M_MCAST) ETHER_MAP_IP_MULTICAST(&satocsin(dst)->sin_addr, edst); else if ((error = arpresolve(ifp, rt, m, dst, edst, - sizeof(edst))) != 0) + sizeof(edst))) != 0) { + KERNEL_UNLOCK_ONE(NULL); return error == EWOULDBLOCK ? 0 : error; + } + KERNEL_UNLOCK_ONE(NULL); /* If broadcasting on a simplex interface, loopback a copy */ if ((m->m_flags & M_BCAST) && (ifp->if_flags & IFF_SIMPLEX)) mcopy = m_copy(m, 0, (int)M_COPYALL); @@ -292,10 +292,12 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, #endif #ifdef NETATALK case AF_APPLETALK: + KERNEL_LOCK(1, NULL); if (aarpresolve(ifp, m, (const struct sockaddr_at *)dst, edst)) { #ifdef NETATALKDEBUG printf("aarpresolv failed\n"); #endif /* NETATALKDEBUG */ + KERNEL_UNLOCK_ONE(NULL); return (0); } /* @@ -303,8 +305,10 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, */ aa = (struct at_ifaddr *) at_ifawithnet( (const struct sockaddr_at *)dst, ifp); - if (aa == NULL) + if (aa == NULL) { + KERNEL_UNLOCK_ONE(NULL); goto bad; + } /* * In the phase 2 case, we need to prepend an mbuf for the @@ -325,6 +329,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, } else { etype = htons(ETHERTYPE_ATALK); } + KERNEL_UNLOCK_ONE(NULL); break; #endif /* NETATALK */ case pseudo_AF_HDRCMPLT: @@ -349,6 +354,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, } #ifdef MPLS + KERNEL_LOCK(1, NULL); { struct m_tag *mtag; mtag = m_tag_find(m, PACKET_TAG_MPLS, NULL); @@ -358,6 +364,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, m_tag_delete(m, mtag); } } + KERNEL_UNLOCK_ONE(NULL); #endif if (mcopy) @@ -410,6 +417,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, #endif /* NCARP > 0 */ #ifdef ALTQ + KERNEL_LOCK(1, NULL); /* * If ALTQ is enabled on the parent interface, do * classification; the queueing discipline might not @@ -418,6 +426,7 @@ ether_output(struct ifnet * const ifp0, struct mbuf * const m0, */ if (ALTQ_IS_ENABLED(&ifp->if_snd)) altq_etherclassify(&ifp->if_snd, m); + KERNEL_UNLOCK_ONE(NULL); #endif return ifq_enqueue(ifp, m); diff --git a/sys/net/if_faith.c b/sys/net/if_faith.c index 7c5e191..3f8f781 100644 --- a/sys/net/if_faith.c +++ b/sys/net/if_faith.c @@ -98,6 +98,11 @@ static struct if_clone faith_cloner = #define FAITHMTU 1500 +static IFNET_WRAP_IF_OUTPUT_DECL(faithoutput); +static IFNET_WRAP_IF_OUTPUT_IMPL(faithoutput); +#define FAITHOUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(faithoutput) + + /* ARGSUSED */ void faithattach(int count) @@ -119,7 +124,7 @@ faith_clone_create(struct if_clone *ifc, int unit) /* Change to BROADCAST experimentaly to announce its prefix. */ ifp->if_flags = /* IFF_LOOPBACK */ IFF_BROADCAST | IFF_MULTICAST; ifp->if_ioctl = faithioctl; - ifp->if_output = faithoutput; + ifp->if_output = FAITHOUTPUT_LOCK; ifp->if_type = IFT_FAITH; ifp->if_hdrlen = 0; ifp->if_addrlen = 0; @@ -150,6 +155,8 @@ faithoutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, int s, error; uint32_t af; + KASSERT(KERNEL_LOCKED_P()); + if ((m->m_flags & M_PKTHDR) == 0) panic("faithoutput no HDR"); af = dst->sa_family; diff --git a/sys/net/if_fddisubr.c b/sys/net/if_fddisubr.c index 4ce90f2..86571bc 100644 --- a/sys/net/if_fddisubr.c +++ b/sys/net/if_fddisubr.c @@ -179,6 +179,10 @@ static int fddi_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); static void fddi_input(struct ifnet *, struct mbuf *); +static IFNET_WRAP_IF_OUTPUT_DECL(fddi_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(fddi_output); +#define FDDI_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(fddi_output) + /* * FDDI output routine. * Encapsulate a packet of type family for the local net. @@ -196,6 +200,8 @@ fddi_output(struct ifnet *ifp0, struct mbuf *m0, const struct sockaddr *dst, struct mbuf *mcopy = NULL; struct ifnet *ifp = ifp0; + KASSERT(KERNEL_LOCKED_P()); + MCLAIM(m, ifp->if_mowner); #if NCARP > 0 @@ -596,7 +602,7 @@ fddi_ifattach(struct ifnet *ifp, void *lla) ifp->if_hdrlen = 21; ifp->if_dlt = DLT_FDDI; ifp->if_mtu = FDDIMTU; - ifp->if_output = fddi_output; + ifp->if_output = FDDI_OUTPUT_LOCK; ifp->_if_input = fddi_input; ifp->if_baudrate = IF_Mbps(100); #ifdef IFF_NOTRAILERS diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c index 83b97c8..5d1e380 100644 --- a/sys/net/if_gif.c +++ b/sys/net/if_gif.c @@ -35,6 +35,7 @@ __KERNEL_RCSID(0, "$NetBSD: if_gif.c,v 1.110 2016/06/10 13:27:16 ozaki-r Exp $") #ifdef _KERNEL_OPT #include "opt_inet.h" +#include "opt_net_mpsafe.h" #endif #include @@ -85,6 +86,12 @@ __KERNEL_RCSID(0, "$NetBSD: if_gif.c,v 1.110 2016/06/10 13:27:16 ozaki-r Exp $") #include "ioconf.h" +#if 0 /* NOTYET */ +#ifdef NET_MPSAFE +#define GIF_MPSAFE 1 +#endif +#endif + static void gifintr(void *); /* @@ -125,6 +132,14 @@ static struct if_clone gif_cloner = #endif static int max_gif_nesting = MAX_GIF_NEST; +#ifndef GIF_MPSAFE +static IFNET_WRAP_IF_OUTPUT_DECL(gif_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(gif_output); +#define GIF_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(gif_output) +#else +#define GIF_OUTPUT_LOCK gif_output +#endif + static void gif_sysctl_setup(struct sysctllog **clog) { @@ -187,7 +202,7 @@ gifattach0(struct gif_softc *sc) sc->gif_if.if_mtu = GIF_MTU; sc->gif_if.if_flags = IFF_POINTOPOINT | IFF_MULTICAST; sc->gif_if.if_ioctl = gif_ioctl; - sc->gif_if.if_output = gif_output; + sc->gif_if.if_output = GIF_OUTPUT_LOCK; sc->gif_if.if_type = IFT_GIF; sc->gif_if.if_dlt = DLT_NULL; sc->gif_if.if_softc = sc; @@ -320,6 +335,10 @@ gif_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, int error = 0; int s; +#ifndef GIF_MPSAFE + KASSERT(KERNEL_LOCKED_P()); +#endif + IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); if ((error = gif_check_nesting(ifp, m)) != 0) { diff --git a/sys/net/if_gre.c b/sys/net/if_gre.c index 9391bbb..a74212f 100644 --- a/sys/net/if_gre.c +++ b/sys/net/if_gre.c @@ -167,6 +167,10 @@ static bool gre_fp_send(struct gre_softc *, enum gre_msg, file_t *); static bool gre_fp_recv(struct gre_softc *); static void gre_fp_recvloop(void *); +static IFNET_WRAP_IF_OUTPUT_DECL(gre_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(gre_output); +#define GRE_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(gre_output) + static void gre_bufq_init(struct gre_bufq *bq, size_t len0) { @@ -295,7 +299,7 @@ gre_clone_create(struct if_clone *ifc, int unit) sc->sc_if.if_dlt = DLT_NULL; sc->sc_if.if_mtu = GREMTU; sc->sc_if.if_flags = IFF_POINTOPOINT|IFF_MULTICAST; - sc->sc_if.if_output = gre_output; + sc->sc_if.if_output = GRE_OUTPUT_LOCK; sc->sc_if.if_ioctl = gre_ioctl; sp = &sc->sc_soparm; sockaddr_copy(sstosa(&sp->sp_dst), sizeof(sp->sp_dst), any); @@ -885,6 +889,8 @@ gre_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct gre_h *gh; uint16_t etype = 0; + KASSERT(KERNEL_LOCKED_P()); + if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) { m_freem(m); error = ENETDOWN; diff --git a/sys/net/if_hippisubr.c b/sys/net/if_hippisubr.c index 8ec5c69..5b15320 100644 --- a/sys/net/if_hippisubr.c +++ b/sys/net/if_hippisubr.c @@ -76,6 +76,10 @@ static int hippi_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); static void hippi_input(struct ifnet *, struct mbuf *); +static IFNET_WRAP_IF_OUTPUT_DECL(hippi_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(hippi_output); +#define HIPPI_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(hippi_output) + /* * HIPPI output routine. * Encapsulate a packet of type family for the local net. @@ -95,6 +99,8 @@ hippi_output(struct ifnet *ifp, struct mbuf *m0, const struct sockaddr *dst, uint32_t *cci; uint32_t d2_len; + KASSERT(KERNEL_LOCKED_P()); + if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) senderr(ENETDOWN); @@ -286,7 +292,7 @@ hippi_ifattach(struct ifnet *ifp, void *lla) ifp->if_hdrlen = sizeof(struct hippi_header) + 8; /* add CCI */ ifp->if_dlt = DLT_HIPPI; ifp->if_mtu = HIPPIMTU; - ifp->if_output = hippi_output; + ifp->if_output = HIPPI_OUTPUT_LOCK; ifp->_if_input = hippi_input; ifp->if_baudrate = IF_Mbps(800); /* XXX double-check */ diff --git a/sys/net/if_ieee1394subr.c b/sys/net/if_ieee1394subr.c index 58ca37b..f4de8a7 100644 --- a/sys/net/if_ieee1394subr.c +++ b/sys/net/if_ieee1394subr.c @@ -82,6 +82,10 @@ static int ieee1394_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); static struct mbuf *ieee1394_reass(struct ifnet *, struct mbuf *, uint16_t); +static IFNET_WRAP_IF_OUTPUT_DECL(ieee1394_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(ieee1394_output); +#define IEEE1394_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(ieee1394_output) + static int ieee1394_output(struct ifnet *ifp, struct mbuf *m0, const struct sockaddr *dst, const struct rtentry *rt) @@ -98,6 +102,8 @@ ieee1394_output(struct ifnet *ifp, struct mbuf *m0, const struct sockaddr *dst, struct m_tag *mtag; int unicast; + KASSERT(KERNEL_LOCKED_P()); + if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) senderr(ENETDOWN); @@ -637,7 +643,7 @@ ieee1394_ifattach(struct ifnet *ifp, const struct ieee1394_hwaddr *hwaddr) ifp->if_hdrlen = sizeof(struct ieee1394_header); ifp->if_dlt = DLT_EN10MB; /* XXX */ ifp->if_mtu = IEEE1394MTU; - ifp->if_output = ieee1394_output; + ifp->if_output = IEEE1394_OUTPUT_LOCK; ifp->if_drain = ieee1394_drain; ifp->if_watchdog = ieee1394_watchdog; ifp->if_timer = 1; diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c index a62c058..9f824d1 100644 --- a/sys/net/if_loop.c +++ b/sys/net/if_loop.c @@ -211,12 +211,12 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct ifqueue *ifq = NULL; int s, isr = -1; int csum_flags; + int error = 0; size_t pktlen; MCLAIM(m, ifp->if_mowner); -#ifndef NET_MPSAFE - KASSERT(KERNEL_LOCKED_P()); -#endif + + KERNEL_LOCK(1, NULL); if ((m->m_flags & M_PKTHDR) == 0) panic("looutput: no header mbuf"); @@ -226,8 +226,9 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, if (rt && rt->rt_flags & (RTF_REJECT|RTF_BLACKHOLE)) { m_freem(m); - return (rt->rt_flags & RTF_BLACKHOLE ? 0 : + error = (rt->rt_flags & RTF_BLACKHOLE ? 0 : rt->rt_flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH); + goto out; } pktlen = m->m_pkthdr.len; @@ -241,8 +242,6 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, */ if ((ALTQ_IS_ENABLED(&ifp->if_snd) || TBR_IS_ENABLED(&ifp->if_snd)) && ifp->if_start == lostart) { - int error; - /* * If the queueing discipline needs packet classification, * do it before prepending the link headers. @@ -250,12 +249,14 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); M_PREPEND(m, sizeof(uint32_t), M_DONTWAIT); - if (m == NULL) - return (ENOBUFS); + if (m == NULL) { + error = ENOBUFS; + goto out; + } *(mtod(m, uint32_t *)) = dst->sa_family; error = ifp->if_transmit(ifp, m); - return (error); + goto out; } #endif /* ALTQ */ @@ -310,12 +311,13 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, printf("%s: can't handle af%d\n", ifp->if_xname, dst->sa_family); m_freem(m); - return (EAFNOSUPPORT); + error = EAFNOSUPPORT; + goto out; } s = splnet(); if (__predict_true(pktq)) { - int error = 0; + error = 0; if (__predict_true(pktq_enqueue(pktq, m, 0))) { ifp->if_ipackets++; @@ -325,20 +327,23 @@ looutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, error = ENOBUFS; } splx(s); - return error; + goto out;; } if (IF_QFULL(ifq)) { IF_DROP(ifq); m_freem(m); splx(s); - return (ENOBUFS); + error = ENOBUFS; + goto out; } IF_ENQUEUE(ifq, m); schednetisr(isr); ifp->if_ipackets++; ifp->if_ibytes += m->m_pkthdr.len; splx(s); - return (0); +out: + KERNEL_UNLOCK_ONE(NULL); + return error; } #ifdef ALTQ diff --git a/sys/net/if_mpls.c b/sys/net/if_mpls.c index 96dd7f2..5111dcb 100644 --- a/sys/net/if_mpls.c +++ b/sys/net/if_mpls.c @@ -111,6 +111,10 @@ extern int mpls_defttl, mpls_mapttl_inet, mpls_mapttl_inet6, mpls_icmp_respond, mpls_forwarding, mpls_frame_accept, mpls_mapprec_inet, mpls_mapclass_inet6, mpls_rfc4182; +static IFNET_WRAP_IF_OUTPUT_DECL(mpls_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(mpls_output); +#define MPLS_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(mpls_output) + /* ARGSUSED */ void ifmplsattach(int count) @@ -134,7 +138,7 @@ mpls_clone_create(struct if_clone *ifc, int unit) sc->sc_if.if_mtu = 1500; sc->sc_if.if_flags = 0; sc->sc_if._if_input = mpls_input; - sc->sc_if.if_output = mpls_output; + sc->sc_if.if_output = MPLS_OUTPUT_LOCK; sc->sc_if.if_ioctl = mpls_ioctl; if_attach(&sc->sc_if); diff --git a/sys/net/if_ppp.c b/sys/net/if_ppp.c index 30e3aa9..89b021f 100644 --- a/sys/net/if_ppp.c +++ b/sys/net/if_ppp.c @@ -223,6 +223,7 @@ static struct compressor *ppp_get_compressor(uint8_t); static void ppp_compressor_rele(struct compressor *); #endif /* PPP_COMPRESS */ +IFNET_WRAP_IF_OUTPUT_IMPL(pppoutput); /* * Called from boot code to establish ppp interfaces. @@ -295,7 +296,7 @@ ppp_create(const char *name, int unit) sc->sc_if.if_hdrlen = PPP_HDRLEN; sc->sc_if.if_dlt = DLT_NULL; sc->sc_if.if_ioctl = pppsioctl; - sc->sc_if.if_output = pppoutput; + sc->sc_if.if_output = PPPOUTPUT_LOCK; #ifdef ALTQ sc->sc_if.if_start = ppp_ifstart; #endif @@ -852,6 +853,8 @@ pppoutput(struct ifnet *ifp, struct mbuf *m0, const struct sockaddr *dst, enum NPmode mode; int len; + KASSERT(KERNEL_LOCKED_P()); + if (sc->sc_devp == NULL || (ifp->if_flags & IFF_RUNNING) == 0 || ((ifp->if_flags & IFF_UP) == 0 && dst->sa_family != AF_UNSPEC)) { error = ENETDOWN; /* sort of */ diff --git a/sys/net/if_pppvar.h b/sys/net/if_pppvar.h index e8e2ebd..0addcff 100644 --- a/sys/net/if_pppvar.h +++ b/sys/net/if_pppvar.h @@ -160,6 +160,11 @@ void ppppktin(struct ppp_softc *, struct mbuf *, int); struct mbuf *ppp_dequeue(struct ppp_softc *); int pppoutput(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); + +IFNET_WRAP_IF_OUTPUT_DECL(pppoutput); +#define PPPOUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(pppoutput) + + #endif /* _KERNEL */ #endif /* !_NET_IF_PPPVAR_H_ */ diff --git a/sys/net/if_sl.c b/sys/net/if_sl.c index 460d3ab..5875e86 100644 --- a/sys/net/if_sl.c +++ b/sys/net/if_sl.c @@ -202,6 +202,10 @@ static int sloutput(struct ifnet *, struct mbuf *, const struct sockaddr *, static int slstart(struct tty *); static int sltioctl(struct tty *, u_long, void *, int, struct lwp *); +static IFNET_WRAP_IF_OUTPUT_DECL(sloutput); +static IFNET_WRAP_IF_OUTPUT_IMPL(sloutput); +#define SLOUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(sloutput) + static struct linesw slip_disc = { .l_name = "slip", .l_open = slopen, @@ -238,7 +242,7 @@ sl_clone_create(struct if_clone *ifc, int unit) sc->sc_if.if_flags = IFF_POINTOPOINT | SC_AUTOCOMP | IFF_MULTICAST; sc->sc_if.if_type = IFT_SLIP; sc->sc_if.if_ioctl = slioctl; - sc->sc_if.if_output = sloutput; + sc->sc_if.if_output = SLOUTPUT_LOCK; sc->sc_if.if_dlt = DLT_SLIP; sc->sc_fastq.ifq_maxlen = 32; IFQ_SET_READY(&sc->sc_if.if_snd); @@ -440,6 +444,8 @@ sloutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct ifqueue *ifq = NULL; int s, error; + KASSERT(KERNEL_LOCKED_P()); + IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); /* diff --git a/sys/net/if_spppsubr.c b/sys/net/if_spppsubr.c index 41afd19..ed4d0f5 100644 --- a/sys/net/if_spppsubr.c +++ b/sys/net/if_spppsubr.c @@ -380,6 +380,10 @@ static void sppp_gen_ip6_addr(struct sppp *sp, const struct in6_addr *src); static void sppp_suggest_ip6_addr(struct sppp *sp, struct in6_addr *src); #endif +static IFNET_WRAP_IF_OUTPUT_DECL(sppp_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(sppp_output); +#define SPPP_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(sppp_output) + /* our control protocol descriptors */ static const struct cp lcp = { PPP_LCP, IDX_LCP, CP_LCP, "lcp", @@ -648,6 +652,8 @@ sppp_output(struct ifnet *ifp, struct mbuf *m, int s, error = 0; uint16_t protocol; + KASSERT(KERNEL_LOCKED_P()); + s = splnet(); sp->pp_last_activity = time_uptime; @@ -855,7 +861,7 @@ sppp_attach(struct ifnet *ifp) spppq = sp; sp->pp_if.if_type = IFT_PPP; - sp->pp_if.if_output = sppp_output; + sp->pp_if.if_output = SPPP_OUTPUT_LOCK; sp->pp_fastq.ifq_maxlen = 32; sp->pp_cpq.ifq_maxlen = 20; sp->pp_loopcnt = 0; diff --git a/sys/net/if_srt.c b/sys/net/if_srt.c index 117b1c9..ef2f2f9 100644 --- a/sys/net/if_srt.c +++ b/sys/net/if_srt.c @@ -235,6 +235,9 @@ srt_if_output( return (*r->u.dstifp->if_output)(r->u.dstifp,m,&r->dst.sa,0); } +static IFNET_WRAP_IF_OUTPUT_IMPL(srt_if_output); +#define SRT_IF_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(srf_if_output) + static int srt_clone_create(struct if_clone *cl, int unit) { @@ -256,7 +259,7 @@ srt_clone_create(struct if_clone *cl, int unit) sc->intf.if_flags = IFF_POINTOPOINT; sc->intf.if_type = IFT_OTHER; sc->intf.if_ioctl = &srt_if_ioctl; - sc->intf.if_output = &srt_if_output; + sc->intf.if_output = &SRT_IF_OUTPUT_LOCK; sc->intf.if_dlt = DLT_RAW; if_attach(&sc->intf); if_alloc_sadl(&sc->intf); diff --git a/sys/net/if_stf.c b/sys/net/if_stf.c index 25c76f4..35d796c 100644 --- a/sys/net/if_stf.c +++ b/sys/net/if_stf.c @@ -177,6 +177,10 @@ static int stf_checkaddr6(struct stf_softc *, const struct in6_addr *, static void stf_rtrequest(int, struct rtentry *, const struct rt_addrinfo *); static int stf_ioctl(struct ifnet *, u_long, void *); +static IFNET_WRAP_IF_OUTPUT_DECL(stf_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(stf_output); +#define STF_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(stf_output) + /* ARGSUSED */ void stfattach(int count) @@ -211,7 +215,7 @@ stf_clone_create(struct if_clone *ifc, int unit) sc->sc_if.if_mtu = STF_MTU; sc->sc_if.if_flags = 0; sc->sc_if.if_ioctl = stf_ioctl; - sc->sc_if.if_output = stf_output; + sc->sc_if.if_output = STF_OUTPUT_LOCK; sc->sc_if.if_type = IFT_STF; sc->sc_if.if_dlt = DLT_NULL; if_attach(&sc->sc_if); @@ -340,6 +344,8 @@ stf_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct sockaddr_in dst4; } u; + KASSERT(KERNEL_LOCKED_P()); + sc = (struct stf_softc*)ifp; dst6 = (const struct sockaddr_in6 *)dst; diff --git a/sys/net/if_strip.c b/sys/net/if_strip.c index bc36188b..ba28196 100644 --- a/sys/net/if_strip.c +++ b/sys/net/if_strip.c @@ -337,6 +337,10 @@ static int stripoutput(struct ifnet *, static int stripstart(struct tty *); static int striptioctl(struct tty *, u_long, void *, int, struct lwp *); +static IFNET_WRAP_IF_OUTPUT_DECL(stripoutput); +static IFNET_WRAP_IF_OUTPUT_IMPL(stripoutput); +#define STRIPOUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(stripoutput) + static struct linesw strip_disc = { .l_name = "strip", .l_open = stripopen, @@ -376,7 +380,7 @@ strip_clone_create(struct if_clone *ifc, int unit) #endif sc->sc_if.if_type = IFT_SLIP; sc->sc_if.if_ioctl = stripioctl; - sc->sc_if.if_output = stripoutput; + sc->sc_if.if_output = STRIPOUTPUT_LOCK; sc->sc_if.if_dlt = DLT_SLIP; sc->sc_fastq.ifq_maxlen = 32; IFQ_SET_READY(&sc->sc_if.if_snd); @@ -732,6 +736,8 @@ stripoutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, int s, error; u_char dl_addrbuf[STARMODE_ADDR_LEN+1]; + KASSERT(KERNEL_LOCKED_P()); + /* * Verify tty line is up and alive. */ diff --git a/sys/net/if_tokensubr.c b/sys/net/if_tokensubr.c index 8ddd016..d75739b 100644 --- a/sys/net/if_tokensubr.c +++ b/sys/net/if_tokensubr.c @@ -146,6 +146,10 @@ static int token_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); static void token_input(struct ifnet *, struct mbuf *); +static IFNET_WRAP_IF_OUTPUT_DECL(token_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(token_output); +#define TOKEN_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(token_output) + /* * Token Ring output routine. * Encapsulate a packet of type family for the local net. @@ -170,6 +174,8 @@ token_output(struct ifnet *ifp0, struct mbuf *m0, const struct sockaddr *dst, struct ifnet *ifp = ifp0; size_t riflen = 0; + KASSERT(KERNEL_LOCKED_P()); + #if NCARP > 0 if (ifp->if_type == IFT_CARP) { struct ifaddr *ifa; @@ -499,7 +505,7 @@ token_ifattach(struct ifnet *ifp, void *lla) ifp->if_hdrlen = 14; ifp->if_dlt = DLT_IEEE802; ifp->if_mtu = ISO88025_MTU; - ifp->if_output = token_output; + ifp->if_output = TOKEN_OUTPUT_LOCK; ifp->_if_input = token_input; ifp->if_broadcastaddr = tokenbroadcastaddr; #ifdef IFF_NOTRAILERS diff --git a/sys/net/if_tun.c b/sys/net/if_tun.c index dacf4e7..c9ddd40 100644 --- a/sys/net/if_tun.c +++ b/sys/net/if_tun.c @@ -90,6 +90,10 @@ static void tunstart(struct ifnet *); static struct tun_softc *tun_find_unit(dev_t); static struct tun_softc *tun_find_zunit(int); +static IFNET_WRAP_IF_OUTPUT_DECL(tun_output); +static IFNET_WRAP_IF_OUTPUT_IMPL(tun_output); +#define TUN_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(tun_output) + static dev_type_open(tunopen); static dev_type_close(tunclose); static dev_type_read(tunread); @@ -208,7 +212,7 @@ tunattach0(struct tun_softc *tp) ifp->if_softc = tp; ifp->if_mtu = TUNMTU; ifp->if_ioctl = tun_ioctl; - ifp->if_output = tun_output; + ifp->if_output = TUN_OUTPUT_LOCK; #ifdef ALTQ ifp->if_start = tunstart; #endif @@ -504,6 +508,8 @@ tun_output(struct ifnet *ifp, struct mbuf *m0, const struct sockaddr *dst, uint32_t *af; #endif + KASSERT(KERNEL_LOCKED_P()); + s = splnet(); mutex_enter(&tp->tun_lock); TUNDEBUG ("%s: tun_output\n", ifp->if_xname); diff --git a/sys/net80211/ieee80211_output.c b/sys/net80211/ieee80211_output.c index 1ce0782..04b9d3a 100644 --- a/sys/net80211/ieee80211_output.c +++ b/sys/net80211/ieee80211_output.c @@ -221,7 +221,7 @@ ieee80211_mgmt_output(struct ieee80211com *ic, struct ieee80211_node *ni, ic->ic_mgt_timer = timer; ifp->if_timer = 1; } - (*ifp->if_start)(ifp); + if_start_lock(ifp); return 0; } @@ -268,7 +268,7 @@ ieee80211_send_nulldata(struct ieee80211_node *ni) wh->i_fc[1] & IEEE80211_FC1_PWR_MGT ? "ena" : "dis"); IF_ENQUEUE(&ic->ic_mgtq, m); /* cheat */ - (*ifp->if_start)(ifp); + if_start_lock(ifp); return 0; } @@ -1362,7 +1362,7 @@ ieee80211_send_probereq(struct ieee80211_node *ni, ieee80211_chan2ieee(ic, ic->ic_curchan)); IF_ENQUEUE(&ic->ic_mgtq, m); - (*ic->ic_ifp->if_start)(ic->ic_ifp); + if_start_lock(ic->ic_ifp); return 0; } diff --git a/sys/net80211/ieee80211_proto.c b/sys/net80211/ieee80211_proto.c index 9a48ea8..a028411 100644 --- a/sys/net80211/ieee80211_proto.c +++ b/sys/net80211/ieee80211_proto.c @@ -1121,7 +1121,7 @@ ieee80211_newstate(struct ieee80211com *ic, enum ieee80211_state nstate, int arg if (ic->ic_opmode == IEEE80211_M_STA) ieee80211_notify_node_join(ic, ni, arg == IEEE80211_FC0_SUBTYPE_ASSOC_RESP); - (*ifp->if_start)(ifp); /* XXX not authorized yet */ + if_start_lock(ifp); /* XXX not authorized yet */ break; } /* diff --git a/sys/netinet/if_arp.c b/sys/netinet/if_arp.c index 0070f65..79ebd54 100644 --- a/sys/netinet/if_arp.c +++ b/sys/netinet/if_arp.c @@ -1888,9 +1888,7 @@ revarprequest(struct ifnet *ifp) sa.sa_len = 2; m->m_flags |= M_BCAST; - KERNEL_LOCK(1, NULL); (*ifp->if_output)(ifp, m, &sa, NULL); - KERNEL_UNLOCK_ONE(NULL); } /* diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c index 7b9adad3..1d20cf5 100644 --- a/sys/netinet/ip_carp.c +++ b/sys/netinet/ip_carp.c @@ -236,6 +236,8 @@ static void sysctl_net_inet_carp_setup(struct sysctllog **); struct if_clone carp_cloner = IF_CLONE_INITIALIZER("carp", carp_clone_create, carp_clone_destroy); +IFNET_WRAP_IF_OUTPUT_IMPL(carp_output); + static __inline u_int16_t carp_cksum(struct mbuf *m, int len) { @@ -809,7 +811,7 @@ carp_clone_create(struct if_clone *ifc, int unit) ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = carp_ioctl; ifp->if_start = carp_start; - ifp->if_output = carp_output; + ifp->if_output = CARP_OUTPUT_LOCK; ifp->if_type = IFT_CARP; ifp->if_addrlen = ETHER_ADDR_LEN; ifp->if_hdrlen = ETHER_HDR_LEN; diff --git a/sys/netinet/ip_carp.h b/sys/netinet/ip_carp.h index 8996e79..eac7a91 100644 --- a/sys/netinet/ip_carp.h +++ b/sys/netinet/ip_carp.h @@ -164,5 +164,9 @@ struct ifnet *carp_ourether(void *, struct ether_header *, u_char, int); int carp_input(struct mbuf *, u_int8_t *, u_int8_t *, u_int16_t); int carp_output(struct ifnet *, struct mbuf *, const struct sockaddr *, const struct rtentry *); + +IFNET_WRAP_IF_OUTPUT_DECL(carp_output); +#define CARP_OUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(carp_output) + #endif /* _KERNEL */ #endif /* _NETINET_IP_CARP_H_ */ diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c index 466e2e5..f360d2e 100644 --- a/sys/netinet/ip_output.c +++ b/sys/netinet/ip_output.c @@ -212,15 +212,8 @@ ip_if_output(struct ifnet * const ifp, struct mbuf * const m, return error; } -#ifndef NET_MPSAFE - KERNEL_LOCK(1, NULL); -#endif - error = (*ifp->if_output)(ifp, m, dst, rt); -#ifndef NET_MPSAFE - KERNEL_UNLOCK_ONE(NULL); -#endif return error; } diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index 8f5b246..291c7e7 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -2291,16 +2291,10 @@ nd6_output(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m, if (ln != NULL) LLE_WUNLOCK(ln); -#ifndef NET_MPSAFE - KERNEL_LOCK(1, NULL); -#endif if ((ifp->if_flags & IFF_LOOPBACK) != 0) error = (*ifp->if_output)(origifp, m, sin6tocsa(dst), rt); else error = (*ifp->if_output)(ifp, m, sin6tocsa(dst), rt); -#ifndef NET_MPSAFE - KERNEL_UNLOCK_ONE(NULL); -#endif goto exit; bad: diff --git a/sys/netisdn/i4b_ipr.c b/sys/netisdn/i4b_ipr.c index 9d9d0da..6a6a0c4 100644 --- a/sys/netisdn/i4b_ipr.c +++ b/sys/netisdn/i4b_ipr.c @@ -269,6 +269,10 @@ static void ipr_dialresponse(void *softc, int status, cause_t cause); static void ipr_updown(void *softc, int updown); static void* ipr_get_softc(int unit); +static IFNET_WRAP_IF_OUTPUT_DECL(iripoutput); +static IFNET_WRAP_IF_OUTPUT_IMPL(iripoutput); +#define IRIPOUTPUT_LOCK IFNET_WRAP_IF_OUTPUT_CALL(iripoutput) + static const struct isdn_l4_driver_functions ipr_l4_functions = { ipr_rx_data_rdy, @@ -339,7 +343,7 @@ iripattach(void) sc->sc_if.if_mtu = I4BIPRMTU; sc->sc_if.if_type = IFT_ISDNBASIC; sc->sc_if.if_ioctl = iripioctl; - sc->sc_if.if_output = iripoutput; + sc->sc_if.if_output = IRIPOUTPUT_LOCK; IFQ_SET_MAXLEN(&sc->sc_if.if_snd, I4BIPRMAXQLEN); sc->sc_fastq.ifq_maxlen = I4BIPRMAXQLEN; @@ -423,6 +427,8 @@ iripoutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct ifqueue *ifq = NULL; struct ip *ip; + KASSERT(KERNEL_LOCKED_P()); + s = splnet(); #if defined(__FreeBSD__) || defined(__bsdi__) diff --git a/sys/netnatm/natm.c b/sys/netnatm/natm.c index ca1e329..f14f592 100644 --- a/sys/netnatm/natm.c +++ b/sys/netnatm/natm.c @@ -155,7 +155,7 @@ natm_connect(struct socket *so, struct sockaddr *nam, struct lwp *l) if (ifp == NULL || (ifp->if_flags & IFF_RUNNING) == 0) { return ENXIO; } - if (ifp->if_output != atm_output) { + if (ifp->if_output != ATM_OUTPUT_LOCK) { return EAFNOSUPPORT; } @@ -368,7 +368,7 @@ natm_send(struct socket *so, struct mbuf *m, struct sockaddr *nam, ATM_PH_VPI(aph) = npcb->npcb_vpi; ATM_PH_SETVCI(aph, npcb->npcb_vci); ATM_PH_FLAGS(aph) = (proto == PROTO_NATMAAL5) ? ATM_PH_AAL5 : 0; - error = atm_output(npcb->npcb_ifp, m, NULL, NULL); + error = ATM_OUTPUT_LOCK(npcb->npcb_ifp, m, NULL, NULL); splx(s); return error; @@ -404,7 +404,7 @@ natm_send(struct socket *so, struct mbuf *m, struct mbuf *nam, ATM_PH_VPI(aph) = npcb->npcb_vpi; ATM_PH_SETVCI(aph, npcb->npcb_vci); ATM_PH_FLAGS(aph) = (proto == PROTO_NATMAAL5) ? ATM_PH_AAL5 : 0; - error = atm_output(npcb->npcb_ifp, m, NULL, NULL); + error = ATM_OUTPUT_LOCK(npcb->npcb_ifp, m, NULL, NULL); splx(s); return error;