Index: dev/netif/em/if_em.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/dev/netif/em/if_em.c,v retrieving revision 1.68 diff -u -p -r1.68 if_em.c --- dev/netif/em/if_em.c 3 Apr 2008 12:55:15 -0000 1.68 +++ dev/netif/em/if_em.c 12 Apr 2008 05:52:44 -0000 @@ -100,6 +100,7 @@ #include #include #include +#include #include #include #include @@ -472,12 +473,14 @@ static int em_attach(device_t dev) { struct adapter *adapter; + struct ifnet *ifp; int tsize, rsize; int error = 0; INIT_DEBUGOUT("em_attach: begin"); adapter = device_get_softc(dev); + ifp = &adapter->interface_data.ac_if; callout_init(&adapter->timer); callout_init(&adapter->tx_fifo_timer); @@ -681,14 +684,15 @@ em_attach(device_t dev) error = bus_setup_intr(dev, adapter->res_interrupt, INTR_NETSAFE, em_intr, adapter, - &adapter->int_handler_tag, - adapter->interface_data.ac_if.if_serializer); + &adapter->int_handler_tag, ifp->if_serializer); if (error) { device_printf(dev, "Error registering interrupt handler!\n"); - ether_ifdetach(&adapter->interface_data.ac_if); + ether_ifdetach(ifp); goto fail; } + ifp->if_cpuid = ithread_cpuid(rman_get_start(adapter->res_interrupt)); + KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); INIT_DEBUGOUT("em_attach: end"); return(0); @@ -822,20 +826,22 @@ em_start(struct ifnet *ifp) if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) return; - if (!adapter->link_active) + if (!adapter->link_active) { + ifq_purge(&ifp->if_snd); return; + } while (!ifq_is_empty(&ifp->if_snd)) { - m_head = ifq_poll(&ifp->if_snd); - + m_head = ifq_dequeue(&ifp->if_snd, NULL); if (m_head == NULL) break; logif(pkt_txqueue); if (em_encap(adapter, m_head)) { ifp->if_flags |= IFF_OACTIVE; + ifp->if_oerrors++; + m_freem(m_head); break; } - ifq_dequeue(&ifp->if_snd, m_head); /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, m_head); Index: kern/kern_intr.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/kern/kern_intr.c,v retrieving revision 1.52 diff -u -p -r1.52 kern_intr.c --- kern/kern_intr.c 30 Mar 2008 14:40:13 -0000 1.52 +++ kern/kern_intr.c 12 Apr 2008 05:52:44 -0000 @@ -951,6 +951,19 @@ emergency_intr_timer_callback(systimer_t lwkt_schedule(info->data); } +int +ithread_cpuid(int intr) +{ + const struct intr_info *info; + + KKASSERT(intr >= 0 && intr < MAX_INTS); + info = &intr_info_ary[intr]; + + if (info->i_state == ISTATE_NOTHREAD) + return -1; + return info->i_thread.td_gd->gd_cpuid; +} + /* * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. * The data for this machine dependent, and the declarations are in machine Index: net/bpf.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/bpf.c,v retrieving revision 1.44 diff -u -p -r1.44 bpf.c --- net/bpf.c 14 Mar 2008 09:52:10 -0000 1.44 +++ net/bpf.c 12 Apr 2008 05:52:44 -0000 @@ -530,9 +530,7 @@ bpfwrite(struct dev_write_args *ap) dst.sa_family = pseudo_AF_HDRCMPLT; crit_enter(); - lwkt_serialize_enter(ifp->if_serializer); error = (*ifp->if_output)(ifp, m, &dst, (struct rtentry *)NULL); - lwkt_serialize_exit(ifp->if_serializer); crit_exit(); /* * The driver frees the mbuf. Index: net/if.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/if.c,v retrieving revision 1.63 diff -u -p -r1.63 if.c --- net/if.c 29 Mar 2008 03:38:53 -0000 1.63 +++ net/if.c 12 Apr 2008 05:52:44 -0000 @@ -59,6 +59,8 @@ #include #include #include +#include +#include #include #include @@ -69,6 +71,7 @@ #include #include #include +#include #include #include @@ -97,15 +100,6 @@ struct netmsg_ifaddr { }; /* - * Support for non-ALTQ interfaces. - */ -static int ifq_classic_enqueue(struct ifaltq *, struct mbuf *, - struct altq_pktattr *); -static struct mbuf * - ifq_classic_dequeue(struct ifaltq *, struct mbuf *, int); -static int ifq_classic_request(struct ifaltq *, int, void *); - -/* * System initialization */ static void if_attachdomain(void *); @@ -170,6 +164,69 @@ ifinit(void *dummy) if_slowtimo(0); } +static int +if_start_cpuid(struct ifnet *ifp) +{ + return ifp->if_cpuid; +} + +#ifdef DEVICE_POLLING +static int +if_start_cpuid_poll(struct ifnet *ifp) +{ + int poll_cpuid = ifp->if_poll_cpuid; + + if (poll_cpuid >= 0) + return poll_cpuid; + else + return ifp->if_cpuid; +} +#endif + +static void +if_start_dispatch(struct netmsg *nmsg) +{ + struct lwkt_msg *lmsg = &nmsg->nm_lmsg; + struct ifnet *ifp = lmsg->u.ms_resultp; + int oactive = 0; + + crit_enter(); + lwkt_replymsg(lmsg, 0); + crit_exit(); + + if (ifp->if_flags & IFF_UP) { + lwkt_serialize_enter(ifp->if_serializer); + if ((ifp->if_flags & IFF_OACTIVE) == 0) + ifp->if_start(ifp); + + if (ifp->if_flags & IFF_OACTIVE) + oactive = 1; + lwkt_serialize_exit(ifp->if_serializer); + + if (!oactive && (ifp->if_flags & IFF_UP) && + !ifq_is_empty(&ifp->if_snd)) { + crit_enter(); + if (lmsg->ms_flags & MSGF_DONE) + lwkt_sendmsg(cpu_portfn(mycpuid), lmsg); + crit_exit(); + return; + } + } + atomic_poll_release_int(&ifp->if_start_running); +} + +static void +if_start_ipifunc(void *arg) +{ + struct ifnet *ifp = arg; + struct lwkt_msg *lmsg = &ifp->if_start_nmsg[mycpuid].nm_lmsg; + + crit_enter(); + if (lmsg->ms_flags & MSGF_DONE) + lwkt_sendmsg(cpu_portfn(mycpuid), lmsg); + crit_exit(); +} + /* * Attach an interface to the list of "active" interfaces. * @@ -200,11 +257,25 @@ if_attach(struct ifnet *ifp, lwkt_serial } ifp->if_serializer = serializer; + ifp->if_start_cpuid = if_start_cpuid; + ifp->if_cpuid = 0; + ifp->if_start_running = 0; + #ifdef DEVICE_POLLING /* Device is not in polling mode by default */ ifp->if_poll_cpuid = -1; + if (ifp->if_poll != NULL) + ifp->if_start_cpuid = if_start_cpuid_poll; #endif + ifp->if_start_nmsg = kmalloc(ncpus * sizeof(struct netmsg), + M_IFADDR /* XXX */, M_WAITOK); + for (i = 0; i < ncpus; ++i) { + netmsg_init(&ifp->if_start_nmsg[i], &netisr_adone_rport, 0, + if_start_dispatch); + ifp->if_start_nmsg[i].nm_lmsg.u.ms_resultp = ifp; + } + TAILQ_INSERT_TAIL(&ifnet, ifp, if_link); ifp->if_index = ++if_index; @@ -275,6 +346,7 @@ if_attach(struct ifnet *ifp, lwkt_serial EVENTHANDLER_INVOKE(ifnet_attach_event, ifp); ifq = &ifp->if_snd; + spin_init(&ifq->altq_lock); ifq->altq_type = 0; ifq->altq_disc = NULL; ifq->altq_flags &= ALTQF_CANTCHANGE; @@ -475,6 +547,7 @@ if_detach(struct ifnet *ifp) TAILQ_REMOVE(&ifnet, ifp, if_link); kfree(ifp->if_addrheads, M_IFADDR); + kfree(ifp->if_start_nmsg, M_IFADDR); crit_exit(); } @@ -1771,28 +1844,24 @@ ifq_set_classic(struct ifaltq *ifq) ifq->altq_request = ifq_classic_request; } -static int +int ifq_classic_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pa __unused) { - crit_enter(); if (IF_QFULL(ifq)) { m_freem(m); - crit_exit(); return(ENOBUFS); } else { IF_ENQUEUE(ifq, m); - crit_exit(); return(0); } } -static struct mbuf * +struct mbuf * ifq_classic_dequeue(struct ifaltq *ifq, struct mbuf *mpolled, int op) { struct mbuf *m; - crit_enter(); switch (op) { case ALTDQ_POLL: IF_POLL(ifq, m); @@ -1803,15 +1872,13 @@ ifq_classic_dequeue(struct ifaltq *ifq, default: panic("unsupported ALTQ dequeue op: %d", op); } - crit_exit(); KKASSERT(mpolled == NULL || mpolled == m); return(m); } -static int +int ifq_classic_request(struct ifaltq *ifq, int req, void *arg) { - crit_enter(); switch (req) { case ALTRQ_PURGE: IF_DRAIN(ifq); @@ -1819,10 +1886,31 @@ ifq_classic_request(struct ifaltq *ifq, default: panic("unsupported ALTQ request: %d", req); } - crit_exit(); return(0); } +int +ifq_dispatch(struct ifnet *ifp, struct mbuf *m, struct altq_pktattr *pa) +{ + int error; + + error = ifq_enqueue(&ifp->if_snd, m, pa); + if (!error) { + ifp->if_obytes += m->m_pkthdr.len; + if (m->m_flags & M_MCAST) + ifp->if_omcasts++; + + if (atomic_poll_acquire_int(&ifp->if_start_running)) { + int cpu; + + cpu = ifp->if_start_cpuid(ifp); + lwkt_send_ipiq(globaldata_find(cpu), + if_start_ipifunc, ifp); + } + } + return error; +} + void * ifa_create(int size, int flags) { Index: net/if_ethersubr.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/if_ethersubr.c,v retrieving revision 1.57 diff -u -p -r1.57 if_ethersubr.c --- net/if_ethersubr.c 20 Mar 2008 14:08:45 -0000 1.57 +++ net/if_ethersubr.c 12 Apr 2008 05:52:44 -0000 @@ -174,8 +174,6 @@ ether_output(struct ifnet *ifp, struct m struct arpcom *ac = IFP2AC(ifp); int error; - ASSERT_SERIALIZED(ifp->if_serializer); - if (ifp->if_flags & IFF_MONITOR) gotoerr(ENETDOWN); if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) != (IFF_UP | IFF_RUNNING)) @@ -321,7 +319,9 @@ ether_output(struct ifnet *ifp, struct m if (ifp->if_bridge) { KASSERT(bridge_output_p != NULL, ("%s: if_bridge not loaded!", __func__)); - return ((*bridge_output_p)(ifp, m, NULL, NULL)); + lwkt_serialize_enter(ifp->if_serializer); + error = bridge_output_p(ifp, m, NULL, NULL); + lwkt_serialize_exit(ifp->if_serializer); } /* @@ -396,8 +396,6 @@ ether_output_frame(struct ifnet *ifp, st struct altq_pktattr pktattr; struct m_tag *mtag; - ASSERT_SERIALIZED(ifp->if_serializer); - /* Extract info from dummynet tag */ mtag = m_tag_find(m, PACKET_TAG_DUMMYNET, NULL); if (mtag != NULL) { @@ -438,7 +436,7 @@ ether_output_frame(struct ifnet *ifp, st * Queue message on interface, update output statistics if * successful, and start output if interface not yet active. */ - error = ifq_handoff(ifp, m, &pktattr); + error = ifq_dispatch(ifp, m, &pktattr); return (error); } Index: net/if_loop.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/if_loop.c,v retrieving revision 1.20 diff -u -p -r1.20 if_loop.c --- net/if_loop.c 22 Dec 2006 23:44:54 -0000 1.20 +++ net/if_loop.c 12 Apr 2008 05:52:44 -0000 @@ -262,7 +262,9 @@ if_simloop(struct ifnet *ifp, struct mbu */ crit_enter(); error = ifq_enqueue(&ifp->if_snd, m, &pktattr); - (*ifp->if_start)(ifp); + lwkt_serialize_enter(ifp->if_serializer); + ifp->if_start(ifp); + lwkt_serialize_exit(ifp->if_serializer); crit_exit(); return (error); } Index: net/if_var.h =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/if_var.h,v retrieving revision 1.48 diff -u -p -r1.48 if_var.h --- net/if_var.h 20 Mar 2008 12:55:32 -0000 1.48 +++ net/if_var.h 12 Apr 2008 05:52:44 -0000 @@ -87,6 +87,7 @@ struct lwkt_serialize; struct ifaddr_container; struct ifaddr; struct lwkt_port; +struct netmsg; #include /* get TAILQ macros */ @@ -211,6 +212,8 @@ struct ifnet { (void *); int (*if_resolvemulti) /* validate/resolve multicast */ (struct ifnet *, struct sockaddr **, struct sockaddr *); + int (*if_start_cpuid) /* cpuid to run if_start */ + (struct ifnet *); #ifdef DEVICE_POLLING void (*if_poll) /* IFF_POLLING support */ (struct ifnet *, enum poll_cmd, int); @@ -228,6 +231,9 @@ struct ifnet { struct ifaddr *if_lladdr; struct lwkt_serialize *if_serializer; /* serializer or MP lock */ struct lwkt_serialize if_default_serializer; /* if not supplied */ + int if_cpuid; + u_int if_start_running; /* synchronization to avoid excess IPIs */ + struct netmsg *if_start_nmsg; /* percpu messages to schedule if_start */ }; typedef void if_init_f_t (void *); Index: net/ifq_var.h =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/ifq_var.h,v retrieving revision 1.9 diff -u -p -r1.9 ifq_var.h --- net/ifq_var.h 21 Aug 2007 19:21:54 -0000 1.9 +++ net/ifq_var.h 12 Apr 2008 05:52:44 -0000 @@ -59,6 +59,12 @@ #ifndef _SYS_MBUF_H_ #include #endif +#ifndef _SYS_SPINLOCK_H_ +#include +#endif +#ifndef _SYS_SPINLOCK2_H_ +#include +#endif #ifndef _NET_IF_VAR_H_ #include #endif @@ -68,6 +74,18 @@ struct ifaltq; +/* + * Support for non-ALTQ interfaces. + */ +int ifq_classic_enqueue(struct ifaltq *, struct mbuf *, + struct altq_pktattr *); +struct mbuf *ifq_classic_dequeue(struct ifaltq *, struct mbuf *, int); +int ifq_classic_request(struct ifaltq *, int, void *); +void ifq_set_classic(struct ifaltq *); + +int ifq_dispatch(struct ifnet *, struct mbuf *, + struct altq_pktattr *); + #ifdef ALTQ static __inline int ifq_is_enabled(struct ifaltq *_ifq) @@ -118,7 +136,17 @@ ifq_set_ready(struct ifaltq *_ifq) static __inline int ifq_enqueue(struct ifaltq *_ifq, struct mbuf *_m, struct altq_pktattr *_pa) { - return((*_ifq->altq_enqueue)(_ifq, _m, _pa)); + int _e; + + spin_lock_wr(&_ifq->altq_lock); +#ifdef ALTQ + if (!ifq_is_enabled(_ifq)) + _e = ifq_classic_enqueue(_ifq, _m, _pa); + else +#endif + _e = _ifq->altq_enqueue(_ifq, _m, _pa); + spin_unlock_wr(&_ifq->altq_lock); + return _e; } /* @@ -127,11 +155,19 @@ ifq_enqueue(struct ifaltq *_ifq, struct static __inline struct mbuf * ifq_dequeue(struct ifaltq *_ifq, struct mbuf *_mpolled) { + struct mbuf *_m; + + spin_lock_wr(&_ifq->altq_lock); #ifdef ALTQ if (_ifq->altq_tbr != NULL) - return(tbr_dequeue(_ifq, _mpolled, ALTDQ_REMOVE)); + _m = tbr_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); + else if (!ifq_is_enabled(_ifq)) + _m = ifq_classic_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); + else #endif - return((*_ifq->altq_dequeue)(_ifq, _mpolled, ALTDQ_REMOVE)); + _m = _ifq->altq_dequeue(_ifq, _mpolled, ALTDQ_REMOVE); + spin_unlock_wr(&_ifq->altq_lock); + return _m; } /* @@ -140,23 +176,41 @@ ifq_dequeue(struct ifaltq *_ifq, struct static __inline struct mbuf * ifq_poll(struct ifaltq *_ifq) { + struct mbuf *_m; + + spin_lock_wr(&_ifq->altq_lock); #ifdef ALTQ if (_ifq->altq_tbr != NULL) - return(tbr_dequeue(_ifq, NULL, ALTDQ_POLL)); + _m = tbr_dequeue(_ifq, NULL, ALTDQ_POLL); + else if (!ifq_is_enabled(_ifq)) + _m = ifq_classic_dequeue(_ifq, NULL, ALTDQ_POLL); + else #endif - return((*_ifq->altq_dequeue)(_ifq, NULL, ALTDQ_POLL)); + _m = _ifq->altq_dequeue(_ifq, NULL, ALTDQ_POLL); + spin_unlock_wr(&_ifq->altq_lock); + return _m; } /* * WARNING: Should only be called in an MPSAFE manner. */ static __inline void -ifq_purge(struct ifaltq *_ifq) +ifq_purge_locked(struct ifaltq *_ifq) { - if (ifq_is_enabled(_ifq)) - (*_ifq->altq_request)(_ifq, ALTRQ_PURGE, NULL); +#ifdef ALTQ + if (!ifq_is_enabled(_ifq)) + ifq_classic_request(_ifq, ALTRQ_PURGE, NULL); else - IF_DRAIN(_ifq); +#endif + _ifq->altq_request(_ifq, ALTRQ_PURGE, NULL); +} + +static __inline void +ifq_purge(struct ifaltq *_ifq) +{ + spin_lock_wr(&_ifq->altq_lock); + ifq_purge_locked(_ifq); + spin_unlock_wr(&_ifq->altq_lock); } /* @@ -166,12 +220,16 @@ static __inline void ifq_classify(struct ifaltq *_ifq, struct mbuf *_m, uint8_t _af, struct altq_pktattr *_pa) { - if (!ifq_is_enabled(_ifq)) - return; - _pa->pattr_af = _af; - _pa->pattr_hdr = mtod(_m, caddr_t); - if (_ifq->altq_flags & ALTQF_CLASSIFY) - (*_ifq->altq_classify)(_ifq, _m, _pa); +#ifdef ALTQ + spin_lock_wr(&_ifq->altq_lock); + if (ifq_is_enabled(_ifq)) { + _pa->pattr_af = _af; + _pa->pattr_hdr = mtod(_m, caddr_t); + if (_ifq->altq_flags & ALTQF_CLASSIFY) + _ifq->altq_classify(_ifq, _m, _pa); + } + spin_unlock_wr(&_ifq->altq_lock); +#endif } /* @@ -212,7 +270,5 @@ ifq_set_maxlen(struct ifaltq *_ifq, int _ifq->ifq_maxlen = _len; } -void ifq_set_classic(struct ifaltq *); - #endif /* _KERNEL */ #endif /* _NET_IFQ_VAR_H_ */ Index: net/altq/altq_cbq.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/altq/altq_cbq.c,v retrieving revision 1.6 diff -u -p -r1.6 altq_cbq.c --- net/altq/altq_cbq.c 22 Dec 2006 23:44:55 -0000 1.6 +++ net/altq/altq_cbq.c 12 Apr 2008 05:52:44 -0000 @@ -201,18 +201,10 @@ get_class_stats(class_stats_t *statsp, s } int -cbq_pfattach(struct pf_altq *a) +cbq_pfattach(struct pf_altq *a, struct ifaltq *ifq) { - struct ifnet *ifp; - int error; - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - crit_enter(); - error = altq_attach(&ifp->if_snd, ALTQT_CBQ, a->altq_disc, + return altq_attach(ifq, ALTQT_CBQ, a->altq_disc, cbq_enqueue, cbq_dequeue, cbq_request, NULL, NULL); - crit_exit(); - return (error); } int Index: net/altq/altq_fairq.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/altq/altq_fairq.c,v retrieving revision 1.1 diff -u -p -r1.1 altq_fairq.c --- net/altq/altq_fairq.c 6 Apr 2008 18:58:15 -0000 1.1 +++ net/altq/altq_fairq.c 12 Apr 2008 05:52:44 -0000 @@ -133,18 +133,10 @@ static void get_class_stats(struct fairq static struct fairq_class *clh_to_clp(struct fairq_if *, uint32_t); int -fairq_pfattach(struct pf_altq *a) +fairq_pfattach(struct pf_altq *a, struct ifaltq *ifq) { - struct ifnet *ifp; - int error; - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - crit_enter(); - error = altq_attach(&ifp->if_snd, ALTQT_FAIRQ, a->altq_disc, + return altq_attach(ifq, ALTQT_FAIRQ, a->altq_disc, fairq_enqueue, fairq_dequeue, fairq_request, NULL, NULL); - crit_exit(); - return (error); } int Index: net/altq/altq_hfsc.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/altq/altq_hfsc.c,v retrieving revision 1.8 diff -u -p -r1.8 altq_hfsc.c --- net/altq/altq_hfsc.c 22 Dec 2006 23:44:55 -0000 1.8 +++ net/altq/altq_hfsc.c 12 Apr 2008 05:52:44 -0000 @@ -139,18 +139,10 @@ static struct hfsc_class *clh_to_clp(str #define HT_INFINITY 0xffffffffffffffffLL /* infinite time value */ int -hfsc_pfattach(struct pf_altq *a) +hfsc_pfattach(struct pf_altq *a, struct ifaltq *ifq) { - struct ifnet *ifp; - int error; - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - crit_enter(); - error = altq_attach(&ifp->if_snd, ALTQT_HFSC, a->altq_disc, + return altq_attach(ifq, ALTQT_HFSC, a->altq_disc, hfsc_enqueue, hfsc_dequeue, hfsc_request, NULL, NULL); - crit_exit(); - return (error); } int Index: net/altq/altq_priq.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/altq/altq_priq.c,v retrieving revision 1.8 diff -u -p -r1.8 altq_priq.c --- net/altq/altq_priq.c 22 Dec 2006 23:44:55 -0000 1.8 +++ net/altq/altq_priq.c 12 Apr 2008 05:52:44 -0000 @@ -78,18 +78,10 @@ static void get_class_stats(struct priq_ static struct priq_class *clh_to_clp(struct priq_if *, uint32_t); int -priq_pfattach(struct pf_altq *a) +priq_pfattach(struct pf_altq *a, struct ifaltq *ifq) { - struct ifnet *ifp; - int error; - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - crit_enter(); - error = altq_attach(&ifp->if_snd, ALTQT_PRIQ, a->altq_disc, + return altq_attach(ifq, ALTQT_PRIQ, a->altq_disc, priq_enqueue, priq_dequeue, priq_request, NULL, NULL); - crit_exit(); - return (error); } int Index: net/altq/altq_subr.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/altq/altq_subr.c,v retrieving revision 1.10 diff -u -p -r1.10 altq_subr.c --- net/altq/altq_subr.c 6 Apr 2008 18:58:15 -0000 1.10 +++ net/altq/altq_subr.c 12 Apr 2008 05:52:44 -0000 @@ -45,6 +45,8 @@ #include #include #include +#include +#include #include #include @@ -74,6 +76,11 @@ * internal function prototypes */ static void tbr_timeout(void *); +static int altq_enable_locked(struct ifaltq *); +static int altq_disable_locked(struct ifaltq *); +static int altq_detach_locked(struct ifaltq *); +static int tbr_set_locked(struct ifaltq *, struct tb_profile *); + int (*altq_input)(struct mbuf *, int) = NULL; static int tbr_timer = 0; /* token bucket regulator timer */ static struct callout tbr_callout; @@ -123,8 +130,8 @@ altq_attach(struct ifaltq *ifq, int type return 0; } -int -altq_detach(struct ifaltq *ifq) +static int +altq_detach_locked(struct ifaltq *ifq) { if (!ifq_is_ready(ifq)) return ENXIO; @@ -143,38 +150,67 @@ altq_detach(struct ifaltq *ifq) } int -altq_enable(struct ifaltq *ifq) +altq_detach(struct ifaltq *ifq) +{ + int error; + + spin_lock_wr(&ifq->altq_lock); + error = altq_detach_locked(ifq); + spin_unlock_wr(&ifq->altq_lock); + return error; +} + +static int +altq_enable_locked(struct ifaltq *ifq) { if (!ifq_is_ready(ifq)) return ENXIO; if (ifq_is_enabled(ifq)) return 0; - crit_enter(); - ifq_purge(ifq); + ifq_purge_locked(ifq); KKASSERT(ifq->ifq_len == 0); + ifq->altq_flags |= ALTQF_ENABLED; if (ifq->altq_clfier != NULL) ifq->altq_flags |= ALTQF_CLASSIFY; - crit_exit(); - return 0; } int -altq_disable(struct ifaltq *ifq) +altq_enable(struct ifaltq *ifq) +{ + int error; + + spin_lock_wr(&ifq->altq_lock); + error = altq_enable_locked(ifq); + spin_unlock_wr(&ifq->altq_lock); + return error; +} + +static int +altq_disable_locked(struct ifaltq *ifq) { if (!ifq_is_enabled(ifq)) return 0; - crit_enter(); - ifq_purge(ifq); + ifq_purge_locked(ifq); KKASSERT(ifq->ifq_len == 0); ifq->altq_flags &= ~(ALTQF_ENABLED|ALTQF_CLASSIFY); - crit_exit(); return 0; } +int +altq_disable(struct ifaltq *ifq) +{ + int error; + + spin_lock_wr(&ifq->altq_lock); + error = altq_disable_locked(ifq); + spin_unlock_wr(&ifq->altq_lock); + return error; +} + /* * internal representation of token bucket parameters * rate: byte_per_unittime << 32 @@ -239,15 +275,15 @@ tbr_dequeue(struct ifaltq *ifq, struct m * set a token bucket regulator. * if the specified rate is zero, the token bucket regulator is deleted. */ -int -tbr_set(struct ifaltq *ifq, struct tb_profile *profile) +static int +tbr_set_locked(struct ifaltq *ifq, struct tb_profile *profile) { struct tb_regulator *tbr, *otbr; if (machclk_freq == 0) init_machclk(); if (machclk_freq == 0) { - kprintf("tbr_set: no cpu clock available!\n"); + kprintf("%s: no cpu clock available!\n", __func__); return (ENXIO); } @@ -283,6 +319,17 @@ tbr_set(struct ifaltq *ifq, struct tb_pr return (0); } +int +tbr_set(struct ifaltq *ifq, struct tb_profile *profile) +{ + int error; + + spin_lock_wr(&ifq->altq_lock); + error = tbr_set_locked(ifq, profile); + spin_unlock_wr(&ifq->altq_lock); + return error; +} + /* * tbr_timeout goes through the interface list, and kicks the drivers * if necessary. @@ -338,54 +385,64 @@ tbr_get(struct ifaltq *ifq, struct tb_pr int altq_pfattach(struct pf_altq *a) { + struct ifaltq *ifq; struct ifnet *ifp; - struct tb_profile tb; - int error = 0; + int error; + + if (a->scheduler == ALTQT_NONE) + return 0; + + if (a->altq_disc == NULL) + return EINVAL; + + ifp = ifunit(a->ifname); + if (ifp == NULL) + return EINVAL; + ifq = &ifp->if_snd; + + spin_lock_wr(&ifq->altq_lock); switch (a->scheduler) { - case ALTQT_NONE: - break; #ifdef ALTQ_CBQ case ALTQT_CBQ: - error = cbq_pfattach(a); + error = cbq_pfattach(a, ifq); break; #endif #ifdef ALTQ_PRIQ case ALTQT_PRIQ: - error = priq_pfattach(a); + error = priq_pfattach(a, ifq); break; #endif #ifdef ALTQ_HFSC case ALTQT_HFSC: - error = hfsc_pfattach(a); + error = hfsc_pfattach(a, ifq); break; #endif #ifdef ALTQ_FAIRQ case ALTQT_FAIRQ: - error = fairq_pfattach(a); + error = fairq_pfattach(a, ifq); break; #endif default: error = ENXIO; + goto back; } - ifp = ifunit(a->ifname); - /* if the state is running, enable altq */ - if (error == 0 && pfaltq_running && - ifp != NULL && ifp->if_snd.altq_type != ALTQT_NONE && - !ifq_is_enabled(&ifp->if_snd)) - error = altq_enable(&ifp->if_snd); + if (error == 0 && pfaltq_running && ifq->altq_type != ALTQT_NONE && + !ifq_is_enabled(ifq)) + error = altq_enable_locked(ifq); /* if altq is already enabled, reset set tokenbucket regulator */ - if (error == 0 && ifp != NULL && ifq_is_enabled(&ifp->if_snd)) { + if (error == 0 && ifq_is_enabled(ifq)) { + struct tb_profile tb; + tb.rate = a->ifbandwidth; tb.depth = a->tbrsize; - crit_enter(); - error = tbr_set(&ifp->if_snd, &tb); - crit_exit(); + error = tbr_set_locked(ifq, &tb); } - +back: + spin_unlock_wr(&ifq->altq_lock); return (error); } @@ -398,22 +455,30 @@ int altq_pfdetach(struct pf_altq *a) { struct ifnet *ifp; + struct ifaltq *ifq; int error = 0; - if ((ifp = ifunit(a->ifname)) == NULL) + ifp = ifunit(a->ifname); + if (ifp == NULL) return (EINVAL); + ifq = &ifp->if_snd; /* if this discipline is no longer referenced, just return */ - if (a->altq_disc == NULL || a->altq_disc != ifp->if_snd.altq_disc) + if (a->altq_disc == NULL) return (0); - crit_enter(); - if (ifq_is_enabled(&ifp->if_snd)) - error = altq_disable(&ifp->if_snd); + spin_lock_wr(&ifq->altq_lock); + + if (a->altq_disc != ifq->altq_disc) + goto back; + + if (ifq_is_enabled(ifq)) + error = altq_disable_locked(ifq); if (error == 0) - error = altq_detach(&ifp->if_snd); - crit_exit(); + error = altq_detach_locked(ifq); +back: + spin_unlock_wr(&ifq->altq_lock); return (error); } Index: net/altq/altq_var.h =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/altq/altq_var.h,v retrieving revision 1.3 diff -u -p -r1.3 altq_var.h --- net/altq/altq_var.h 6 Apr 2008 18:58:15 -0000 1.3 +++ net/altq/altq_var.h 12 Apr 2008 05:52:44 -0000 @@ -79,28 +79,28 @@ int altq_add_queue(struct pf_altq *); int altq_remove_queue(struct pf_altq *); int altq_getqstats(struct pf_altq *, void *, int *); -int cbq_pfattach(struct pf_altq *); +int cbq_pfattach(struct pf_altq *, struct ifaltq *); int cbq_add_altq(struct pf_altq *); int cbq_remove_altq(struct pf_altq *); int cbq_add_queue(struct pf_altq *); int cbq_remove_queue(struct pf_altq *); int cbq_getqstats(struct pf_altq *, void *, int *); -int priq_pfattach(struct pf_altq *); +int priq_pfattach(struct pf_altq *, struct ifaltq *); int priq_add_altq(struct pf_altq *); int priq_remove_altq(struct pf_altq *); int priq_add_queue(struct pf_altq *); int priq_remove_queue(struct pf_altq *); int priq_getqstats(struct pf_altq *, void *, int *); -int hfsc_pfattach(struct pf_altq *); +int hfsc_pfattach(struct pf_altq *, struct ifaltq *); int hfsc_add_altq(struct pf_altq *); int hfsc_remove_altq(struct pf_altq *); int hfsc_add_queue(struct pf_altq *); int hfsc_remove_queue(struct pf_altq *); int hfsc_getqstats(struct pf_altq *, void *, int *); -int fairq_pfattach(struct pf_altq *); +int fairq_pfattach(struct pf_altq *, struct ifaltq *); int fairq_add_altq(struct pf_altq *); int fairq_remove_altq(struct pf_altq *); int fairq_add_queue(struct pf_altq *); Index: net/altq/if_altq.h =================================================================== RCS file: /data/repos/df_cvs/src/sys/net/altq/if_altq.h,v retrieving revision 1.3 diff -u -p -r1.3 if_altq.h --- net/altq/if_altq.h 20 May 2006 02:42:09 -0000 1.3 +++ net/altq/if_altq.h 12 Apr 2008 05:52:44 -0000 @@ -29,6 +29,10 @@ #ifndef _NET_ALTQ_IF_ALTQ_H_ #define _NET_ALTQ_IF_ALTQ_H_ +#ifndef _SYS_SPINLOCK_H_ +#include +#endif + struct altq_pktattr; /* @@ -60,6 +64,7 @@ struct ifaltq { /* token bucket regulator */ struct tb_regulator *altq_tbr; + struct spinlock altq_lock; }; Index: netinet/if_ether.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/netinet/if_ether.c,v retrieving revision 1.45 diff -u -p -r1.45 if_ether.c --- netinet/if_ether.c 7 Mar 2008 11:34:20 -0000 1.45 +++ netinet/if_ether.c 12 Apr 2008 05:52:44 -0000 @@ -224,12 +224,10 @@ arp_rtrequest(int req, struct rtentry *r } /* Announce a new entry if requested. */ if (rt->rt_flags & RTF_ANNOUNCE) { - lwkt_serialize_enter(rt->rt_ifp->if_serializer); arprequest(rt->rt_ifp, &SIN(rt_key(rt))->sin_addr, &SIN(rt_key(rt))->sin_addr, LLADDR(SDL(gate))); - lwkt_serialize_exit(rt->rt_ifp->if_serializer); } /*FALLTHROUGH*/ case RTM_RESOLVE: @@ -327,8 +325,6 @@ arprequest(struct ifnet *ifp, struct in_ struct sockaddr sa; u_short ar_hrd; - ASSERT_SERIALIZED(ifp->if_serializer); - if ((m = m_gethdr(MB_DONTWAIT, MT_DATA)) == NULL) return; m->m_pkthdr.rcvif = (struct ifnet *)NULL; @@ -367,7 +363,7 @@ arprequest(struct ifnet *ifp, struct in_ sa.sa_family = AF_UNSPEC; sa.sa_len = sizeof sa; - (*ifp->if_output)(ifp, m, &sa, (struct rtentry *)NULL); + ifp->if_output(ifp, m, &sa, (struct rtentry *)NULL); } /* @@ -634,9 +630,7 @@ arp_update_oncpu(struct mbuf *m, in_addr */ if (la->la_hold != NULL) { m_adj(la->la_hold, sizeof(struct ether_header)); - lwkt_serialize_enter(ifp->if_serializer); - (*ifp->if_output)(ifp, la->la_hold, rt_key(rt), rt); - lwkt_serialize_exit(ifp->if_serializer); + ifp->if_output(ifp, la->la_hold, rt_key(rt), rt); la->la_hold = NULL; } } @@ -859,9 +853,7 @@ reply: } sa.sa_family = AF_UNSPEC; sa.sa_len = sizeof sa; - lwkt_serialize_enter(ifp->if_serializer); - (*ifp->if_output)(ifp, m, &sa, (struct rtentry *)0); - lwkt_serialize_exit(ifp->if_serializer); + ifp->if_output(ifp, m, &sa, (struct rtentry *)0); return; } Index: netinet/ip_flow.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/netinet/ip_flow.c,v retrieving revision 1.13 diff -u -p -r1.13 ip_flow.c --- netinet/ip_flow.c 3 Apr 2008 16:02:32 -0000 1.13 +++ netinet/ip_flow.c 12 Apr 2008 05:52:44 -0000 @@ -180,10 +180,6 @@ ipflow_fastforward(struct mbuf *m, lwkt_ else dst = &ipf->ipf_ro.ro_dst; - if (serializer) - lwkt_serialize_exit(serializer); - - lwkt_serialize_enter(ifp->if_serializer); error = ifp->if_output(ifp, m, dst, rt); if (error) { if (error == ENOBUFS) @@ -191,10 +187,6 @@ ipflow_fastforward(struct mbuf *m, lwkt_ else ipf->ipf_errors++; } - lwkt_serialize_exit(ifp->if_serializer); - - if (serializer) - lwkt_serialize_enter(serializer); return 1; } Index: netinet/ip_output.c =================================================================== RCS file: /data/repos/df_cvs/src/sys/netinet/ip_output.c,v retrieving revision 1.41 diff -u -p -r1.41 ip_output.c --- netinet/ip_output.c 6 Jan 2008 16:55:52 -0000 1.41 +++ netinet/ip_output.c 12 Apr 2008 05:52:44 -0000 @@ -1017,10 +1017,8 @@ pass: m->m_pkthdr.len = tmp; } #endif - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m, (struct sockaddr *)dst, - ro->ro_rt); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m, (struct sockaddr *)dst, + ro->ro_rt); goto done; } @@ -1062,11 +1060,8 @@ pass: ia->ia_ifa.if_opackets++; ia->ia_ifa.if_obytes += m->m_pkthdr.len; } - lwkt_serialize_enter(ifp->if_serializer); - error = (*ifp->if_output)(ifp, m, - (struct sockaddr *)dst, - ro->ro_rt); - lwkt_serialize_exit(ifp->if_serializer); + error = ifp->if_output(ifp, m, (struct sockaddr *)dst, + ro->ro_rt); } else { m_freem(m); } Index: sys/interrupt.h =================================================================== RCS file: /data/repos/df_cvs/src/sys/sys/interrupt.h,v retrieving revision 1.18 diff -u -p -r1.18 interrupt.h --- sys/interrupt.h 11 Jan 2007 23:23:57 -0000 1.18 +++ sys/interrupt.h 12 Apr 2008 05:52:44 -0000 @@ -124,6 +124,7 @@ void unregister_randintr(int intr); int next_registered_randintr(int intr); void sched_ithd(int intr); /* procedure called from MD */ void forward_fastint_remote(void *arg); /* MD procedure (SMP) */ +int ithread_cpuid(int intr); extern char eintrnames[]; /* end of intrnames[] */ extern char intrnames[]; /* string table containing device names */