Index: dev/pci/if_myx.c =================================================================== RCS file: /cvs/src/sys/dev/pci/if_myx.c,v retrieving revision 1.90 diff -u -p -r1.90 if_myx.c --- dev/pci/if_myx.c 3 Dec 2015 12:45:56 -0000 1.90 +++ dev/pci/if_myx.c 5 Dec 2015 11:06:59 -0000 @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -151,6 +152,7 @@ struct myx_softc { u_int sc_tx_prod; u_int sc_tx_cons; struct myx_slot *sc_tx_slots; + struct task sc_tx_restart; struct ifmedia sc_media; @@ -527,6 +529,8 @@ myx_attachhook(void *arg) IFCAP_CSUM_UDPv4; #endif + task_set(&sc->sc_tx_restart, if_restart_task, ifp); + ifmedia_init(&sc->sc_media, 0, myx_media_change, myx_media_status); ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); @@ -1702,9 +1706,8 @@ myx_txeof(struct myx_softc *sc, u_int32_ sc->sc_tx_ring_cons = idx; sc->sc_tx_cons = cons; - ifq_clr_oactive(&ifp->if_snd); if (!ifq_empty(&ifp->if_snd)) - if_start(ifp); + if_restart(ifp, &sc->sc_tx_restart); } void Index: kern/kern_task.c =================================================================== RCS file: /cvs/src/sys/kern/kern_task.c,v retrieving revision 1.15 diff -u -p -r1.15 kern_task.c --- kern/kern_task.c 19 Nov 2015 13:19:24 -0000 1.15 +++ kern/kern_task.c 5 Dec 2015 11:06:59 -0000 @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #define TASK_ONQUEUE 1 @@ -73,6 +76,10 @@ int taskq_sleep(const volatile void *, s int taskq_next_work(struct taskq *, struct task *, sleepfn); void taskq_thread(void *); +void taskctx_run(struct taskctx *); + +void task_barrier(void *); + void taskq_init(void) { @@ -179,6 +186,30 @@ taskq_create_thread(void *arg) } void +taskq_barrier(struct taskq *tq) +{ + struct sleep_state sls; + unsigned int notdone = 1; + struct task t = TASK_INITIALIZER(task_barrier, ¬done); + + task_add(tq, &t); + + while (notdone) { + sleep_setup(&sls, ¬done, PWAIT, "tqbar"); + sleep_finish(&sls, notdone); + } +} + +void +task_barrier(void *p) +{ + unsigned int *notdone = p; + + *notdone = 0; + wakeup_one(notdone); +} + +void task_set(struct task *t, void (*fn)(void *), void *arg) { t->t_func = fn; @@ -305,4 +336,96 @@ taskq_thread(void *xtq) wakeup_one(&tq->tq_running); kthread_exit(0); +} + +void +taskctx_init(struct taskctx *tc, int ipl) +{ + mtx_init(&tc->tc_mtx, ipl); + TAILQ_INIT(&tc->tc_worklist); + tc->tc_serializer = 0; +} + +void +task_dispatch(struct taskctx *tc, struct task *t) +{ + if (ISSET(t->t_flags, TASK_ONQUEUE)) + return; + + mtx_enter(&tc->tc_mtx); + if (!ISSET(t->t_flags, TASK_ONQUEUE)) { + SET(t->t_flags, TASK_ONQUEUE); + TAILQ_INSERT_TAIL(&tc->tc_worklist, t, t_entry); + } + mtx_leave(&tc->tc_mtx); + + taskctx_run(tc); +} + +void +taskctx_barrier(struct taskctx *tc) +{ + struct sleep_state sls; + unsigned int notdone = 1; + struct task t = TASK_INITIALIZER(task_barrier, ¬done); + + task_dispatch(tc, &t); + + while (notdone) { + sleep_setup(&sls, ¬done, PWAIT, "tqbar"); + sleep_finish(&sls, notdone); + } +} + +static inline unsigned int +taskctx_enter(struct taskctx *tc) +{ + return (atomic_inc_int_nv(&tc->tc_serializer) == 1); +} + +static inline unsigned int +taskctx_leave(struct taskctx *tc) +{ + if (atomic_cas_uint(&tc->tc_serializer, 1, 0) == 1) + return (1); + + tc->tc_serializer = 1; + + return (0); +} + +static inline int +taskctx_next_work(struct taskctx *tc, struct task *work) +{ + struct task *t; + int rv = 0; + + mtx_enter(&tc->tc_mtx); + t = TAILQ_FIRST(&tc->tc_worklist); + if (t != NULL) { + TAILQ_REMOVE(&tc->tc_worklist, t, t_entry); + CLR(t->t_flags, TASK_ONQUEUE); + + *work = *t; /* copy to caller to avoid races */ + + rv = 1; + } + mtx_leave(&tc->tc_mtx); + + return (rv); +} + +void +taskctx_run(struct taskctx *tc) +{ + struct task work; + + if (!taskctx_enter(tc)) + return; + + do { + while (taskctx_next_work(tc, &work)) + (*work.t_func)(work.t_arg); + + } while (!taskctx_leave(tc)); } Index: net/if.c =================================================================== RCS file: /cvs/src/sys/net/if.c,v retrieving revision 1.418 diff -u -p -r1.418 if.c --- net/if.c 3 Dec 2015 12:22:51 -0000 1.418 +++ net/if.c 5 Dec 2015 11:06:59 -0000 @@ -153,8 +153,9 @@ void if_input_process(void *); void ifa_print_all(void); #endif -void if_start_mpsafe(struct ifnet *ifp); -void if_start_locked(struct ifnet *ifp); +void if_start_mpsafe(struct ifnet *); +void if_start_locked(struct ifnet *); +void if_start_task(void *); /* * interface index map @@ -513,6 +514,7 @@ if_attach_common(struct ifnet *ifp) TAILQ_INIT(&ifp->if_maddrlist); ifq_init(&ifp->if_snd); + task_set(&ifp->if_start_ctx, if_start_task, ifp); ifp->if_addrhooks = malloc(sizeof(*ifp->if_addrhooks), M_TEMP, M_WAITOK); @@ -557,66 +559,40 @@ if_start_locked(struct ifnet *ifp) KERNEL_UNLOCK(); } -static inline unsigned int -ifq_enter(struct ifqueue *ifq) +void +if_start_mpsafe(struct ifnet *ifp) { - return (atomic_inc_int_nv(&ifq->ifq_serializer) == 1); + task_dispatch(&ifp->if_snd.ifq_serializer, &ifp->if_start_ctx); } -static inline unsigned int -ifq_leave(struct ifqueue *ifq) +void +if_start_task(void *p) { - if (atomic_cas_uint(&ifq->ifq_serializer, 1, 0) == 1) - return (1); - - ifq->ifq_serializer = 1; + struct ifnet *ifp = p; - return (0); + ifp->if_start(ifp); } void -if_start_mpsafe(struct ifnet *ifp) +if_start_barrier(struct ifnet *ifp) { - struct ifqueue *ifq = &ifp->if_snd; - - if (!ifq_enter(ifq)) - return; - - do { - if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) { - ifq->ifq_serializer = 0; - wakeup_one(&ifq->ifq_serializer); - return; - } - - if (ifq_empty(ifq) || ifq_is_oactive(ifq)) - continue; - - ifp->if_start(ifp); + taskctx_barrier(&ifp->if_snd.ifq_serializer); +} - } while (!ifq_leave(ifq)); +void +if_restart(struct ifnet *ifp, struct task *t) +{ + task_dispatch(&ifp->if_snd.ifq_serializer, t); } void -if_start_barrier(struct ifnet *ifp) +if_restart_task(void *p) { - struct sleep_state sls; - struct ifqueue *ifq = &ifp->if_snd; + struct ifnet *ifp = p; - /* this should only be called from converted drivers */ + ifq_clr_oactive(&ifp->if_snd); KASSERT(ISSET(ifp->if_xflags, IFXF_MPSAFE)); - - /* drivers should only call this on the way down */ - KASSERT(!ISSET(ifp->if_flags, IFF_RUNNING)); - - if (ifq->ifq_serializer == 0) - return; - - if_start_mpsafe(ifp); /* spin the wheel to guarantee a wakeup */ - do { - sleep_setup(&sls, &ifq->ifq_serializer, PWAIT, "ifbar"); - sleep_finish(&sls, ifq->ifq_serializer != 0); - } while (ifq->ifq_serializer != 0); + if_start_mpsafe(ifp); } int @@ -830,8 +806,8 @@ if_input_process(void *xmq) struct mbuf *m; struct ifnet *ifp; struct ifih *ifih; + struct ifref r; struct srpl_iter i; - int s; mq_delist(mq, &ml); if (ml_empty(&ml)) @@ -839,12 +815,11 @@ if_input_process(void *xmq) add_net_randomness(ml_len(&ml)); - s = splnet(); while ((m = ml_dequeue(&ml)) != NULL) { - ifp = if_get(m->m_pkthdr.ph_ifidx); + ifp = if_enter(&r, m->m_pkthdr.ph_ifidx); if (ifp == NULL) { m_freem(m); - continue; + goto next; } /* @@ -860,9 +835,9 @@ if_input_process(void *xmq) if (ifih == NULL) m_freem(m); - if_put(ifp); +next: + if_leave(&r, ifp); } - splx(s); } void @@ -1539,28 +1514,49 @@ ifunit(const char *name) return (NULL); } -/* - * Map interface index to interface structure pointer. - */ struct ifnet * -if_get(unsigned int index) +if_enter(struct ifref *r, unsigned int index) { struct if_map *if_map; struct srp *map; - struct ifnet *ifp = NULL; if_map = srp_enter(&if_idxmap.map); if (index < if_map->limit) { map = (struct srp *)(if_map + 1); - ifp = srp_follow(&if_idxmap.map, if_map, &map[index]); - if (ifp != NULL) { - KASSERT(ifp->if_index == index); - if_ref(ifp); - } - srp_leave(&map[index], ifp); - } else - srp_leave(&if_idxmap.map, if_map); + r->p = &map[index]; + return (srp_follow(&if_idxmap.map, if_map, r->p)); + } + srp_leave(&if_idxmap.map, if_map); + + r->p = NULL; + return (NULL); +} + +void +if_leave(struct ifref *r, struct ifnet *ifp) +{ + if (r->p == NULL) + return; + + srp_leave(r->p, ifp); +} + +/* + * Map interface index to interface structure pointer. + */ +struct ifnet * +if_get(unsigned int index) +{ + struct ifref r; + struct ifnet *ifp; + + ifp = if_enter(&r, index); + if (ifp != NULL) { + KASSERT(ifp->if_index == index); + if_ref(ifp); + } + if_leave(&r, ifp); return (ifp); } @@ -2997,6 +2993,7 @@ ifq_purge(struct ifqueue *ifq) void ifq_init(struct ifqueue *ifq) { + taskctx_init(&ifq->ifq_serializer, IPL_NET); mtx_init(&ifq->ifq_mtx, IPL_NET); ifq->ifq_drops = 0; @@ -3004,7 +3001,6 @@ ifq_init(struct ifqueue *ifq) ifq->ifq_ops = &priq_ops; ifq->ifq_q = priq_ops.ifqop_alloc(NULL); - ifq->ifq_serializer = 0; ifq->ifq_len = 0; if (ifq->ifq_maxlen == 0) Index: net/if.h =================================================================== RCS file: /cvs/src/sys/net/if.h,v retrieving revision 1.174 diff -u -p -r1.174 if.h --- net/if.h 3 Dec 2015 12:22:51 -0000 1.174 +++ net/if.h 5 Dec 2015 11:06:59 -0000 @@ -451,6 +451,10 @@ struct if_afreq { struct socket; struct ifnet; +struct ifref { + struct srp *p; +}; + void if_alloc_sadl(struct ifnet *); void if_free_sadl(struct ifnet *); void if_attach(struct ifnet *); @@ -472,6 +476,8 @@ int if_addgroup(struct ifnet *, const ch int if_delgroup(struct ifnet *, const char *); void if_group_routechange(struct sockaddr *, struct sockaddr *); struct ifnet *ifunit(const char *); +struct ifnet *if_enter(struct ifref *, unsigned int); +void if_leave(struct ifref *, struct ifnet *); struct ifnet *if_get(unsigned int); void if_put(struct ifnet *); void ifnewlladdr(struct ifnet *); Index: net/if_var.h =================================================================== RCS file: /cvs/src/sys/net/if_var.h,v retrieving revision 1.61 diff -u -p -r1.61 if_var.h --- net/if_var.h 3 Dec 2015 12:22:51 -0000 1.61 +++ net/if_var.h 5 Dec 2015 11:06:59 -0000 @@ -40,6 +40,7 @@ #include #include #include +#include /* * Structures defining a network interface, providing a packet @@ -107,12 +108,13 @@ struct ifq_ops { }; struct ifqueue { + struct taskctx ifq_serializer; + struct mutex ifq_mtx; uint64_t ifq_drops; const struct ifq_ops *ifq_ops; void *ifq_q; unsigned int ifq_len; - unsigned int ifq_serializer; unsigned int ifq_oactive; unsigned int ifq_maxlen; @@ -172,6 +174,7 @@ struct ifnet { /* and the entries */ int (*if_ll_output)(struct ifnet *, struct mbuf *, struct sockaddr *, struct rtentry *); /* initiate output routine */ + struct task if_start_ctx; void (*if_start)(struct ifnet *); /* ioctl routine */ int (*if_ioctl)(struct ifnet *, u_long, caddr_t); @@ -364,6 +367,8 @@ extern unsigned int lo0ifidx; void if_start(struct ifnet *); void if_start_barrier(struct ifnet *); +void if_restart(struct ifnet *, struct task *); +void if_restart_task(void *); int if_enqueue_try(struct ifnet *, struct mbuf *); int if_enqueue(struct ifnet *, struct mbuf *); void if_input(struct ifnet *, struct mbuf_list *); Index: sys/task.h =================================================================== RCS file: /cvs/src/sys/sys/task.h,v retrieving revision 1.8 diff -u -p -r1.8 task.h --- sys/task.h 9 Feb 2015 03:15:41 -0000 1.8 +++ sys/task.h 5 Dec 2015 11:06:59 -0000 @@ -39,6 +39,7 @@ extern struct taskq *const systqmp; struct taskq *taskq_create(const char *, unsigned int, int, unsigned int); void taskq_destroy(struct taskq *); +void taskq_barrier(struct taskq *); void task_set(struct task *, void (*)(void *), void *); int task_add(struct taskq *, struct task *); Index: sys/taskctx.h =================================================================== RCS file: sys/taskctx.h diff -N sys/taskctx.h --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ sys/taskctx.h 5 Dec 2015 11:06:59 -0000 @@ -0,0 +1,42 @@ +/* $OpenBSD: task.h,v 1.8 2015/02/09 03:15:41 dlg Exp $ */ + +/* + * Copyright (c) 2013 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef _SYS_TASKCTX_H_ +#define _SYS_TASKCTX_H_ + +#include +#include + +struct taskctx { + struct mutex tc_mtx; + TAILQ_HEAD(, task) tc_worklist; + unsigned int tc_serializer; +}; + +#ifdef _KERNEL + +void taskctx_init(struct taskctx *, int); +void taskctx_barrier(struct taskctx *); + +void task_dispatch(struct taskctx *, struct task *); + +#define TASKCTX_INITIALIZER(_tc, _i) \ + { MUTEX_INITIALIZER(_i), TAILQ_HEAD_INITIALIZER(_tc.tc_worklist), 0 } + +#endif /* _KERNEL */ +#endif /* _SYS_TASKCTX_H_ */