Index: net/ifq.c =================================================================== RCS file: /cvs/src/sys/net/ifq.c,v retrieving revision 1.40 diff -u -p -r1.40 ifq.c --- net/ifq.c 17 Jun 2020 06:45:22 -0000 1.40 +++ net/ifq.c 25 Jun 2020 23:51:54 -0000 @@ -17,6 +17,11 @@ */ #include "bpfilter.h" +#if 0 +#include "kstat.h" +#else +#define NKSTAT 0 +#endif #include #include @@ -32,6 +37,10 @@ #include #endif +#if NKSTAT > 0 +#include +#endif + /* * priq glue */ @@ -122,7 +131,11 @@ ifq_is_serialized(struct ifqueue *ifq) void ifq_start(struct ifqueue *ifq) { - if (ifq_len(ifq) >= min(ifq->ifq_if->if_txmit, ifq->ifq_maxlen)) { + struct ifnet *ifp = ifq->ifq_if; + + if (ISSET(ifp->if_xflags, IFXF_MPSAFE) && + ifq_len(ifq) >= min(ifp->if_txmit, ifq->ifq_maxlen)) { + atomic_inc_int(&ifq->ifq_dispatch); task_del(ifq->ifq_softnet, &ifq->ifq_bundle); ifq_run_start(ifq); } else @@ -157,6 +170,7 @@ ifq_bundle_task(void *p) { struct ifqueue *ifq = p; + atomic_inc_int(&ifq->ifq_defers); ifq_run_start(ifq); } @@ -188,11 +202,48 @@ ifq_barrier_task(void *p) * ifqueue mbuf queue API */ +#if NKSTAT > 0 +struct ifq_kstat_data { + struct kstat_kv kd_packets; + struct kstat_kv kd_bytes; + struct kstat_kv kd_qdrops; + struct kstat_kv kd_errors; + struct kstat_kv kd_qlen; + struct kstat_kv kd_maxqlen; + struct kstat_kv kd_oactive; + + struct kstat_kv kd_dispatch; + struct kstat_kv kd_defers; +}; + +static const struct ifq_kstat_data ifq_kstat_tpl = { + KSTAT_KV_UNIT_INITIALIZER("packets", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("bytes", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), + KSTAT_KV_UNIT_INITIALIZER("qdrops", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("errors", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("qlen", + KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("maxqlen", + KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS), + KSTAT_KV_INITIALIZER("oactive", KSTAT_KV_T_BOOL), + + KSTAT_KV_INITIALIZER("dispatch", KSTAT_KV_T_COUNTER32), + KSTAT_KV_INITIALIZER("defers", KSTAT_KV_T_COUNTER32), +}; + +static int ifq_kstat_copy(struct kstat *, void *); +#endif + void ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx) { ifq->ifq_if = ifp; - ifq->ifq_softnet = net_tq(ifp->if_index); /* + idx */ + ifq->ifq_softnet = ISSET(ifp->if_xflags, IFXF_MPSAFE) ? + net_tq(ifp->if_index /* + idx */) : systq; ifq->ifq_softc = NULL; mtx_init(&ifq->ifq_mtx, IPL_NET); @@ -222,6 +273,18 @@ ifq_init(struct ifqueue *ifq, struct ifn ifq_set_maxlen(ifq, IFQ_MAXLEN); ifq->ifq_idx = idx; + +#if NKSTAT > 0 + /* XXX xname vs driver name and unit */ + ifq->ifq_kstat = kstat_create(ifp->if_xname, 0, + "txq", ifq->ifq_idx, KSTAT_T_KV, 0); + KASSERT(ifq->ifq_kstat != NULL); + kstat_set_mutex(ifq->ifq_kstat, &ifq->ifq_mtx); + ifq->ifq_kstat->ks_softc = ifq; + ifq->ifq_kstat->ks_datalen = sizeof(ifq_kstat_tpl); + ifq->ifq_kstat->ks_copy = ifq_kstat_copy; + kstat_install(ifq->ifq_kstat); +#endif } void @@ -265,6 +328,10 @@ ifq_destroy(struct ifqueue *ifq) { struct mbuf_list ml = MBUF_LIST_INITIALIZER(); +#if NKSTAT > 0 + kstat_destroy(ifq->ifq_kstat); +#endif + NET_ASSERT_UNLOCKED(); if (!task_del(ifq->ifq_softnet, &ifq->ifq_bundle)) taskq_barrier(ifq->ifq_softnet); @@ -289,6 +356,29 @@ ifq_add_data(struct ifqueue *ifq, struct mtx_leave(&ifq->ifq_mtx); } +#if NKSTAT > 0 +static int +ifq_kstat_copy(struct kstat *ks, void *dst) +{ + struct ifqueue *ifq = ks->ks_softc; + struct ifq_kstat_data *kd = dst; + + *kd = ifq_kstat_tpl; + kd->kd_packets.kv_v.v_u64 = ifq->ifq_packets; + kd->kd_bytes.kv_v.v_u64 = ifq->ifq_bytes; + kd->kd_qdrops.kv_v.v_u64 = ifq->ifq_qdrops; + kd->kd_errors.kv_v.v_u64 = ifq->ifq_errors; + kd->kd_qlen.kv_v.v_u32 = ifq->ifq_len; + kd->kd_maxqlen.kv_v.v_u32 = ifq->ifq_maxlen; + kd->kd_oactive.kv_v.v_bool = ifq->ifq_oactive; + + kd->kd_dispatch.kv_v.v_u32 = ifq->ifq_dispatch; + kd->kd_defers.kv_v.v_u32 = ifq->ifq_defers; + + return (0); +} +#endif + int ifq_enqueue(struct ifqueue *ifq, struct mbuf *m) { @@ -505,6 +595,64 @@ ifq_mfreeml(struct ifqueue *ifq, struct * ifiq */ +void +ifiql_init(struct ifiq_list *ifiql) +{ + ifiql->ifiql_packets = 0; + ifiql->ifiql_bytes = 0; + ifiql->ifiql_errors = 0; + ml_init(&ifiql->ifiql_ml); +} + +void +ifiql_enqueue(struct ifiq_list *ifiql, struct ifnet *ifp, struct mbuf *m) +{ +#if NBPFILTER > 0 + caddr_t if_bpf; +#endif + + m->m_pkthdr.ph_ifidx = ifp->if_index; + m->m_pkthdr.ph_rtableid = ifp->if_rdomain; + + ifiql->ifiql_packets++; + ifiql->ifiql_bytes += m->m_pkthdr.len; + +#if NBPFILTER > 0 + if_bpf = READ_ONCE(ifp->if_bpf); + if (if_bpf && bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_IN)) { + m_freem(m); + return; + } +#endif + + ml_enqueue(&ifiql->ifiql_ml, m); +} + +#if NKSTAT > 0 +struct ifiq_kstat_data { + struct kstat_kv kd_packets; + struct kstat_kv kd_bytes; + struct kstat_kv kd_qdrops; + struct kstat_kv kd_errors; + struct kstat_kv kd_qlen; +}; + +static const struct ifiq_kstat_data ifiq_kstat_tpl = { + KSTAT_KV_UNIT_INITIALIZER("packets", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("bytes", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), + KSTAT_KV_UNIT_INITIALIZER("qdrops", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("errors", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("qlen", + KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS), +}; + +static int ifiq_kstat_copy(struct kstat *, void *); +#endif + static void ifiq_process(void *); void @@ -525,11 +673,27 @@ ifiq_init(struct ifiqueue *ifiq, struct ifiq->ifiq_errors = 0; ifiq->ifiq_idx = idx; + +#if NKSTAT > 0 + /* XXX xname vs driver name and unit */ + ifiq->ifiq_kstat = kstat_create(ifp->if_xname, 0, + "rxq", ifiq->ifiq_idx, KSTAT_T_KV, 0); + KASSERT(ifiq->ifiq_kstat != NULL); + kstat_set_mutex(ifiq->ifiq_kstat, &ifiq->ifiq_mtx); + ifiq->ifiq_kstat->ks_softc = ifiq; + ifiq->ifiq_kstat->ks_datalen = sizeof(ifiq_kstat_tpl); + ifiq->ifiq_kstat->ks_copy = ifiq_kstat_copy; + kstat_install(ifiq->ifiq_kstat); +#endif } void ifiq_destroy(struct ifiqueue *ifiq) { +#if NKSTAT > 0 + kstat_destroy(ifiq->ifiq_kstat); +#endif + NET_ASSERT_UNLOCKED(); if (!task_del(ifiq->ifiq_softnet, &ifiq->ifiq_task)) taskq_barrier(ifiq->ifiq_softnet); @@ -545,52 +709,31 @@ int ifiq_input(struct ifiqueue *ifiq, struct mbuf_list *ml) { struct ifnet *ifp = ifiq->ifiq_if; + struct ifiq_list ifiql = IFIQ_LIST_INITIALIZER(); struct mbuf *m; - uint64_t packets; - uint64_t bytes = 0; - unsigned int len; -#if NBPFILTER > 0 - caddr_t if_bpf; -#endif if (ml_empty(ml)) return (0); - MBUF_LIST_FOREACH(ml, m) { - m->m_pkthdr.ph_ifidx = ifp->if_index; - m->m_pkthdr.ph_rtableid = ifp->if_rdomain; - bytes += m->m_pkthdr.len; - } - packets = ml_len(ml); + while ((m = ml_dequeue(ml)) != NULL) + ifiql_enqueue(&ifiql, ifp, m); -#if NBPFILTER > 0 - if_bpf = ifp->if_bpf; - if (if_bpf) { - struct mbuf_list ml0 = *ml; - - ml_init(ml); - - while ((m = ml_dequeue(&ml0)) != NULL) { - if (bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_IN)) - m_freem(m); - else - ml_enqueue(ml, m); - } + return (ifiq_list_input(ifiq, &ifiql)); +} - if (ml_empty(ml)) { - mtx_enter(&ifiq->ifiq_mtx); - ifiq->ifiq_packets += packets; - ifiq->ifiq_bytes += bytes; - mtx_leave(&ifiq->ifiq_mtx); +int +ifiq_list_input(struct ifiqueue *ifiq, struct ifiq_list *ifiql) +{ + unsigned int len; + struct mbuf_list *ml = &ifiql->ifiql_ml; + int empty; - return (0); - } - } -#endif + empty = ml_empty(ml); mtx_enter(&ifiq->ifiq_mtx); - ifiq->ifiq_packets += packets; - ifiq->ifiq_bytes += bytes; + ifiq->ifiq_packets += ifiql->ifiql_packets; + ifiq->ifiq_bytes += ifiql->ifiql_bytes; + ifiq->ifiq_errors += ifiql->ifiql_errors; len = ml_len(&ifiq->ifiq_ml); if (len > ifiq_maxlen_drop) @@ -599,9 +742,10 @@ ifiq_input(struct ifiqueue *ifiq, struct ml_enlist(&ifiq->ifiq_ml, ml); mtx_leave(&ifiq->ifiq_mtx); - if (ml_empty(ml)) - task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task); - else + if (ml_empty(ml)) { + if (!empty) + task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task); + } else ml_purge(ml); return (len > ifiq_maxlen_return); @@ -616,6 +760,24 @@ ifiq_add_data(struct ifiqueue *ifiq, str data->ifi_iqdrops += ifiq->ifiq_qdrops; mtx_leave(&ifiq->ifiq_mtx); } + +#if NKSTAT > 0 +static int +ifiq_kstat_copy(struct kstat *ks, void *dst) +{ + struct ifiqueue *ifiq = ks->ks_softc; + struct ifiq_kstat_data *kd = dst; + + *kd = ifiq_kstat_tpl; + kd->kd_packets.kv_v.v_u64 = ifiq->ifiq_packets; + kd->kd_bytes.kv_v.v_u64 = ifiq->ifiq_bytes; + kd->kd_qdrops.kv_v.v_u64 = ifiq->ifiq_qdrops; + kd->kd_errors.kv_v.v_u64 = ifiq->ifiq_errors; + kd->kd_qlen.kv_v.v_u32 = ml_len(&ifiq->ifiq_ml); + + return (0); +} +#endif int ifiq_enqueue(struct ifiqueue *ifiq, struct mbuf *m) Index: net/ifq.h =================================================================== RCS file: /cvs/src/sys/net/ifq.h,v retrieving revision 1.31 diff -u -p -r1.31 ifq.h --- net/ifq.h 22 May 2020 07:02:24 -0000 1.31 +++ net/ifq.h 25 Jun 2020 23:51:54 -0000 @@ -20,6 +20,7 @@ #define _NET_IFQ_H_ struct ifnet; +struct kstat; struct ifq_ops; @@ -54,6 +55,11 @@ struct ifqueue { uint64_t ifq_errors; uint64_t ifq_mcasts; + unsigned int ifq_dispatch; + unsigned int ifq_defers; + + struct kstat *ifq_kstat; + /* work serialisation */ struct mutex ifq_task_mtx; struct task_list ifq_task_list; @@ -92,6 +98,8 @@ struct ifiqueue { uint64_t ifiq_mcasts; uint64_t ifiq_noproto; + struct kstat *ifiq_kstat; + /* properties */ unsigned int ifiq_idx; }; @@ -478,9 +486,34 @@ ifq_idx(struct ifqueue *ifq, unsigned in /* ifiq */ +struct ifiq_list { + uint64_t ifiql_packets; + uint64_t ifiql_bytes; + uint64_t ifiql_errors; + struct mbuf_list ifiql_ml; +}; + +#define IFIQ_LIST_INITIALIZER() { \ + .ifiql_packets = 0, \ + .ifiql_bytes = 0, \ + .ifiql_errors = 0, \ + .ifiql_ml = MBUF_LIST_INITIALIZER(), \ +} + +void ifiql_init(struct ifiq_list *); +void ifiql_enqueue(struct ifiq_list *, struct ifnet *, + struct mbuf *); + +static inline void +ifiql_inc_errors(struct ifiq_list *ifiql) +{ + ifiql->ifiql_errors++; +} + void ifiq_init(struct ifiqueue *, struct ifnet *, unsigned int); void ifiq_destroy(struct ifiqueue *); int ifiq_input(struct ifiqueue *, struct mbuf_list *); +int ifiq_list_input(struct ifiqueue *, struct ifiq_list *); int ifiq_enqueue(struct ifiqueue *, struct mbuf *); void ifiq_add_data(struct ifiqueue *, struct if_data *); Index: dev/pci/if_ixl.c =================================================================== RCS file: /cvs/src/sys/dev/pci/if_ixl.c,v retrieving revision 1.57 diff -u -p -r1.57 if_ixl.c --- dev/pci/if_ixl.c 25 Jun 2020 09:11:08 -0000 1.57 +++ dev/pci/if_ixl.c 25 Jun 2020 23:51:54 -0000 @@ -3085,7 +3085,7 @@ ixl_rxeof(struct ixl_softc *sc, struct i struct ixl_rx_map *rxm; bus_dmamap_t map; unsigned int cons, prod; - struct mbuf_list ml = MBUF_LIST_INITIALIZER(); + struct ifiq_list ifiql = IFIQ_LIST_INITIALIZER(); struct mbuf *m; uint64_t word; unsigned int len; @@ -3138,9 +3138,9 @@ ixl_rxeof(struct ixl_softc *sc, struct i if (ISSET(word, IXL_RX_DESC_EOP)) { if (!ISSET(word, IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { - ml_enqueue(&ml, m); + ifiql_enqueue(&ifiql, ifp, m); } else { - ifp->if_ierrors++; /* XXX */ + ifiql_inc_errors(&ifiql); m_freem(m); } @@ -3156,7 +3156,7 @@ ixl_rxeof(struct ixl_softc *sc, struct i if (done) { rxr->rxr_cons = cons; - if (ifiq_input(ifiq, &ml)) + if (ifiq_list_input(ifiq, &ifiql)) if_rxr_livelocked(&rxr->rxr_acct); ixl_rxfill(sc, rxr); }