Index: ifq.c =================================================================== RCS file: /cvs/src/sys/net/ifq.c,v diff -u -p -r1.54 ifq.c --- ifq.c 9 Nov 2024 04:09:56 -0000 1.54 +++ ifq.c 6 Dec 2024 01:57:36 -0000 @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -121,7 +122,10 @@ ifq_serialize(struct ifqueue *ifq, struc void ifq_start(struct ifqueue *ifq) { - if (ifq_len(ifq) >= min(ifq->ifq_if->if_txmit, ifq->ifq_maxlen)) { + struct ifnet *ifp = ifq->ifq_if; + + if (ISSET(ifp->if_xflags, IFXF_MPSAFE) && + ifq_len(ifq) >= min(ifp->if_txmit, ifq->ifq_maxlen)) { task_del(ifq->ifq_softnet, &ifq->ifq_bundle); ifq_run_start(ifq); } else @@ -138,6 +142,7 @@ ifq_start_task(void *p) ifq_empty(ifq) || ifq_is_oactive(ifq)) return; + LLTRACE(lltrace_count, LLTRACE_COUNT_T_PKTS_IFQ, ifq_len(ifq)); ifp->if_qstart(ifq); } @@ -162,6 +167,7 @@ ifq_restart_task(void *p) struct ifnet *ifp = ifq->ifq_if; ifq_clr_oactive(ifq); + LLTRACE(lltrace_count, LLTRACE_COUNT_T_PKTS_IFQ, ifq_len(ifq)); ifp->if_qstart(ifq); } @@ -254,7 +260,8 @@ void ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx) { ifq->ifq_if = ifp; - ifq->ifq_softnet = net_tq(idx); + ifq->ifq_softnet = ISSET(ifp->if_xflags, IFXF_MPSAFE) ? + net_tq(idx) : systq; ifq->ifq_softc = NULL; mtx_init(&ifq->ifq_mtx, IPL_NET); @@ -595,8 +602,11 @@ ifq_mfreeml(struct ifqueue *ifq, struct #if NKSTAT > 0 struct ifiq_kstat_data { + struct kstat_kv kd_histogram[IFIQ_HISTOGRAM_BUCKETS]; + struct kstat_kv kd_packets; struct kstat_kv kd_bytes; + struct kstat_kv kd_hdrops; struct kstat_kv kd_fdrops; struct kstat_kv kd_qdrops; struct kstat_kv kd_errors; @@ -607,10 +617,35 @@ struct ifiq_kstat_data { }; static const struct ifiq_kstat_data ifiq_kstat_tpl = { + { + KSTAT_KV_UNIT_INITIALIZER("histo1", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo2", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo4", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo8", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo16", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo32", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo64", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo128", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo256", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + KSTAT_KV_UNIT_INITIALIZER("histo512", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), + }, + KSTAT_KV_UNIT_INITIALIZER("packets", KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), KSTAT_KV_UNIT_INITIALIZER("bytes", KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), + KSTAT_KV_UNIT_INITIALIZER("hdrops", + KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), KSTAT_KV_UNIT_INITIALIZER("fdrops", KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), KSTAT_KV_UNIT_INITIALIZER("qdrops", @@ -631,10 +666,15 @@ ifiq_kstat_copy(struct kstat *ks, void * { struct ifiqueue *ifiq = ks->ks_softc; struct ifiq_kstat_data *kd = dst; + size_t i; *kd = ifiq_kstat_tpl; + for (i = 0; i < IFIQ_HISTOGRAM_BUCKETS; i++) + kstat_kv_u64(&kd->kd_histogram[i]) = ifiq->ifiq_histogram[i]; + kstat_kv_u64(&kd->kd_packets) = ifiq->ifiq_packets; kstat_kv_u64(&kd->kd_bytes) = ifiq->ifiq_bytes; + kstat_kv_u64(&kd->kd_hdrops) = ifiq->ifiq_hdrops; kstat_kv_u64(&kd->kd_fdrops) = ifiq->ifiq_fdrops; kstat_kv_u64(&kd->kd_qdrops) = ifiq->ifiq_qdrops; kstat_kv_u64(&kd->kd_errors) = ifiq->ifiq_errors; @@ -653,7 +693,9 @@ void ifiq_init(struct ifiqueue *ifiq, struct ifnet *ifp, unsigned int idx) { ifiq->ifiq_if = ifp; - ifiq->ifiq_softnet = net_tq(idx); + ifiq->ifiq_bpfp = NULL; + ifiq->ifiq_softnet = ISSET(ifp->if_xflags, IFXF_MPSAFE) ? + net_tq(idx) : systq; ifiq->ifiq_softc = NULL; mtx_init(&ifiq->ifiq_mtx, IPL_NET); @@ -697,8 +739,8 @@ ifiq_destroy(struct ifiqueue *ifiq) ml_purge(&ifiq->ifiq_ml); } -unsigned int ifiq_maxlen_drop = 2048 * 5; -unsigned int ifiq_maxlen_return = 2048 * 3; +unsigned int ifiq_pressure_drop = 8; +unsigned int ifiq_pressure_return = 6; int ifiq_input(struct ifiqueue *ifiq, struct mbuf_list *ml) @@ -707,10 +749,12 @@ ifiq_input(struct ifiqueue *ifiq, struct struct mbuf *m; uint64_t packets; uint64_t bytes = 0; - uint64_t fdrops = 0; - unsigned int len; + unsigned int pressure = 0; + unsigned int fdrops = 0; + unsigned int hdrops = 0; + unsigned int bucket; #if NBPFILTER > 0 - caddr_t if_bpf; + caddr_t *ifiq_bpfp, ifiq_bpf = NULL, if_bpf; #endif if (ml_empty(ml)) @@ -720,18 +764,35 @@ ifiq_input(struct ifiqueue *ifiq, struct m->m_pkthdr.ph_ifidx = ifp->if_index; m->m_pkthdr.ph_rtableid = ifp->if_rdomain; bytes += m->m_pkthdr.len; + hdrops += m->m_pkthdr.ph_drops; } packets = ml_len(ml); + bucket = fls(packets); + if (bucket > IFIQ_HISTOGRAM_BUCKETS) + bucket = IFIQ_HISTOGRAM_BUCKETS; + bucket--; + + LLTRACE(lltrace_count, LLTRACE_COUNT_T_PKTS_IFIQ, packets); + if (hdrops) + LLTRACE(lltrace_count, LLTRACE_COUNT_T_PKTS_HDROP, hdrops); #if NBPFILTER > 0 - if_bpf = ifp->if_bpf; - if (if_bpf) { + ifiq_bpfp = READ_ONCE(ifiq->ifiq_bpfp); + if (ifiq_bpfp != NULL) + ifiq_bpf = READ_ONCE(*ifiq_bpfp); + if_bpf = READ_ONCE(ifp->if_bpf); + if (ifiq_bpf || if_bpf) { struct mbuf_list ml0 = *ml; ml_init(ml); while ((m = ml_dequeue(&ml0)) != NULL) { - if ((*ifp->if_bpf_mtap)(if_bpf, m, BPF_DIRECTION_IN)) { + int drop = 0; + if ((*ifp->if_bpf_mtap)(ifiq_bpf, m, BPF_DIRECTION_IN)) + drop = 1; + if ((*ifp->if_bpf_mtap)(if_bpf, m, BPF_DIRECTION_IN)) + drop = 1; + if (drop) { m_freem(m); fdrops++; } else @@ -743,6 +804,7 @@ ifiq_input(struct ifiqueue *ifiq, struct ifiq->ifiq_packets += packets; ifiq->ifiq_bytes += bytes; ifiq->ifiq_fdrops += fdrops; + ifiq->ifiq_hdrops += hdrops; mtx_leave(&ifiq->ifiq_mtx); return (0); @@ -751,13 +813,15 @@ ifiq_input(struct ifiqueue *ifiq, struct #endif mtx_enter(&ifiq->ifiq_mtx); + ifiq->ifiq_histogram[bucket]++; ifiq->ifiq_packets += packets; ifiq->ifiq_bytes += bytes; + ifiq->ifiq_hdrops += hdrops; ifiq->ifiq_fdrops += fdrops; - len = ml_len(&ifiq->ifiq_ml); if (__predict_true(!ISSET(ifp->if_xflags, IFXF_MONITOR))) { - if (len > ifiq_maxlen_drop) + pressure = ++ifiq->ifiq_pressure; + if (pressure > ifiq_pressure_drop) ifiq->ifiq_qdrops += ml_len(ml); else { ifiq->ifiq_enqueues++; @@ -768,10 +832,12 @@ ifiq_input(struct ifiqueue *ifiq, struct if (ml_empty(ml)) task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task); - else + else { + LLTRACE(lltrace_count, LLTRACE_COUNT_T_PKTS_QDROP, ml_len(ml)); ml_purge(ml); + } - return (len > ifiq_maxlen_return); + return (pressure > ifiq_pressure_return); } void @@ -781,6 +847,7 @@ ifiq_add_data(struct ifiqueue *ifiq, str data->ifi_ipackets += ifiq->ifiq_packets; data->ifi_ibytes += ifiq->ifiq_bytes; data->ifi_iqdrops += ifiq->ifiq_qdrops; + data->ifi_collisions += ifiq->ifiq_hdrops; mtx_leave(&ifiq->ifiq_mtx); } @@ -833,11 +900,14 @@ ifiq_process(void *arg) return; mtx_enter(&ifiq->ifiq_mtx); + ifiq->ifiq_pressure = 0; ifiq->ifiq_dequeues++; ml = ifiq->ifiq_ml; ml_init(&ifiq->ifiq_ml); mtx_leave(&ifiq->ifiq_mtx); + LLTRACE(lltrace_count, LLTRACE_COUNT_T_PKTS_NETTQ, ml_len(&ml)); + if_input_process(ifiq->ifiq_if, &ml); } @@ -846,8 +916,6 @@ net_ifiq_sysctl(int *name, u_int namelen void *newp, size_t newlen) { int error = EOPNOTSUPP; -/* pressure is disabled for 6.6-release */ -#if 0 int val; if (namelen != 1) @@ -876,7 +944,6 @@ net_ifiq_sysctl(int *name, u_int namelen error = EOPNOTSUPP; break; } -#endif return (error); } Index: ifq.h =================================================================== RCS file: /cvs/src/sys/net/ifq.h,v diff -u -p -r1.41 ifq.h --- ifq.h 10 Nov 2023 15:51:24 -0000 1.41 +++ ifq.h 6 Dec 2024 01:57:36 -0000 @@ -73,8 +73,11 @@ struct ifqueue { unsigned int ifq_idx; }; +#define IFIQ_HISTOGRAM_BUCKETS 10 + struct ifiqueue { struct ifnet *ifiq_if; + caddr_t *ifiq_bpfp; struct taskq *ifiq_softnet; union { void *_ifiq_softc; @@ -89,8 +92,10 @@ struct ifiqueue { unsigned int ifiq_pressure; /* counters */ + uint64_t ifiq_histogram[IFIQ_HISTOGRAM_BUCKETS]; uint64_t ifiq_packets; uint64_t ifiq_bytes; + uint64_t ifiq_hdrops; uint64_t ifiq_fdrops; uint64_t ifiq_qdrops; uint64_t ifiq_errors;