Index: kern/uipc_mbuf.c
===================================================================
RCS file: /cvs/src/sys/kern/uipc_mbuf.c,v
retrieving revision 1.287
diff -u -p -r1.287 uipc_mbuf.c
--- kern/uipc_mbuf.c	23 Jun 2023 04:36:49 -0000	1.287
+++ kern/uipc_mbuf.c	4 Sep 2023 06:02:46 -0000
@@ -1573,6 +1573,18 @@ ml_enqueue(struct mbuf_list *ml, struct 
 }
 
 void
+ml_requeue(struct mbuf_list *ml, struct mbuf *m)
+{
+	m->m_nextpkt = ml->ml_head;
+	ml->ml_head = m;
+
+	if (ml->ml_tail == NULL)
+		ml->ml_tail = m;
+
+	ml->ml_len++;
+}
+
+void
 ml_enlist(struct mbuf_list *mla, struct mbuf_list *mlb)
 {
 	if (!ml_empty(mlb)) {
Index: net/if.c
===================================================================
RCS file: /cvs/src/sys/net/if.c,v
retrieving revision 1.707
diff -u -p -r1.707 if.c
--- net/if.c	18 Aug 2023 08:10:16 -0000	1.707
+++ net/if.c	4 Sep 2023 06:02:46 -0000
@@ -70,6 +70,7 @@
 #include "ppp.h"
 #include "pppoe.h"
 #include "if_wg.h"
+#include "kstat.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -90,6 +91,7 @@
 #include <sys/smr.h>
 
 #include <net/if.h>
+#include <net/if_var.h>
 #include <net/if_dl.h>
 #include <net/if_types.h>
 #include <net/route.h>
@@ -138,6 +140,10 @@
 #include <net/pfvar.h>
 #endif
 
+#if NKSTAT > 0
+#include <sys/kstat.h>
+#endif
+
 #include <sys/device.h>
 
 void	if_attachsetup(struct ifnet *);
@@ -179,6 +185,11 @@ void	ifa_print_all(void);
 
 void	if_qstart_compat(struct ifqueue *);
 
+#if NKSTAT > 0
+static void	if_kstat_attach(struct ifnet *);
+static void	if_kstat_detach(struct ifnet *);
+#endif
+
 /*
  * interface index map
  *
@@ -247,13 +258,18 @@ int	ifq_congestion;
 int		 netisr;
 
 struct softnet {
-	char		 sn_name[16];
-	struct taskq	*sn_taskq;
+	char			 sn_name[16];
+	struct taskq		*sn_taskq;
+
+	struct mbuf_queue	 sn_proto_mq;
+	struct task		 sn_proto_task;
 };
 
 #define	NET_TASKQ	4
 struct softnet	softnets[NET_TASKQ];
 
+static void net_proto_process(void *);
+
 struct task if_input_task_locked = TASK_INITIALIZER(if_netisr, NULL);
 
 /*
@@ -283,6 +299,9 @@ ifinit(void)
 		    TASKQ_MPSAFE);
 		if (sn->sn_taskq == NULL)
 			panic("unable to create network taskq %d", i);
+
+		mq_init(&sn->sn_proto_mq, 1024, IPL_SOFTNET);
+		task_set(&sn->sn_proto_task, net_proto_process, sn);
 	}
 }
 
@@ -500,6 +519,10 @@ if_attachsetup(struct ifnet *ifp)
 
 	/* Announce the interface. */
 	rtm_ifannounce(ifp, IFAN_ARRIVAL);
+
+#if NKSTAT > 0
+	if_kstat_attach(ifp);
+#endif
 }
 
 /*
@@ -845,28 +868,10 @@ if_input_local(struct ifnet *ifp, struct
 	ifp->if_ipackets++;
 	ifp->if_ibytes += m->m_pkthdr.len;
 
-	switch (af) {
-	case AF_INET:
-		if (ISSET(keepcksum, M_IPV4_CSUM_OUT))
-			m->m_pkthdr.csum_flags |= M_IPV4_CSUM_IN_OK;
-		ipv4_input(ifp, m);
-		break;
-#ifdef INET6
-	case AF_INET6:
-		ipv6_input(ifp, m);
-		break;
-#endif /* INET6 */
-#ifdef MPLS
-	case AF_MPLS:
-		mpls_input(ifp, m);
-		break;
-#endif /* MPLS */
-	default:
-		printf("%s: can't handle af%d\n", ifp->if_xname, af);
-		m_freem(m);
-		return (EAFNOSUPPORT);
-	}
+	if (af == AF_INET && ISSET(keepcksum, M_IPV4_CSUM_OUT))
+		m->m_pkthdr.csum_flags |= M_IPV4_CSUM_IN_OK;
 
+	net_proto_input(ifp, af, m);
 	return (0);
 }
 
@@ -967,28 +972,109 @@ if_output_local(struct ifnet *ifp, struc
 	struct ifiqueue *ifiq;
 	unsigned int flow = 0;
 
-	m->m_pkthdr.ph_family = af;
-	m->m_pkthdr.ph_ifidx = ifp->if_index;
-	m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
-
 	if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
 		flow = m->m_pkthdr.ph_flowid;
-
 	ifiq = ifp->if_iqs[flow % ifp->if_niqs];
 
+	m->m_pkthdr.ph_family = af;
 	return (ifiq_enqueue(ifiq, m) == 0 ? 0 : ENOBUFS);
 }
 
+struct netstack {
+	struct mbuf_list	*ns_ifinput;
+	struct mbuf_list	 ns_proto;
+	struct mbuf_list	 ns_junk;
+};
+
 void
-if_input_process(struct ifnet *ifp, struct mbuf_list *ml)
+netstack_vinput(struct netstack *ns, struct ifnet *ifp, struct mbuf *m)
 {
-	struct mbuf *m;
+#if NBPFILTER > 0
+	caddr_t if_bpf;
+#endif
+ 
+ 	m->m_pkthdr.ph_ifidx = ifp->if_index;
+ 	m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
+ 
+	counters_pkt(ifp->if_counters,
+	    ifc_ipackets, ifc_ibytes, m->m_pkthdr.len);
 
-	if (ml_empty(ml))
+#if NPF > 0
+	pf_pkt_addr_changed(m);
+#endif
+ 
+#if NBPFILTER > 0
+	if_bpf = ifp->if_bpf;
+	if (if_bpf) {
+		if ((*ifp->if_bpf_mtap)(if_bpf, m, BPF_DIRECTION_IN)) {
+			m_freem(m);
+			return;
+		}
+	}
+#endif
+ 
+	ml_requeue(ns->ns_ifinput, m);
+}
+
+void
+netstack_proto_input(struct netstack *ns, struct ifnet *ifp, int af,
+    struct mbuf *m)
+{
+	switch (af) {
+	case AF_INET:
+		m = ipv4_check(ifp, m);
+		break;
+#if INET6
+	case AF_INET6:
+		m = ipv6_check(ifp, m);
+		break;
+#endif /* INET6 */
+	}
+	if (m == NULL)
 		return;
+ 
+	m->m_pkthdr.ph_family = af;
+	ml_enqueue(&ns->ns_proto, m);
+}
 
-	if (!ISSET(ifp->if_xflags, IFXF_CLONED))
-		enqueue_randomness(ml_len(ml) ^ (uintptr_t)MBUF_LIST_FIRST(ml));
+void
+netstack_mfreem(struct netstack *ns, struct mbuf *m)
+{
+	ml_enqueue(&ns->ns_junk, m);
+}
+
+static void
+netstack_proto_process(struct netstack *ns, struct ifnet *ifp, struct mbuf *m)
+{
+	void (*input)(struct ifnet *, struct mbuf *);
+
+	switch (m->m_pkthdr.ph_family) {
+	case AF_INET:
+		input = ipv4_input;
+		break;
+#ifdef INET6
+	case AF_INET6:
+		input = ipv6_input;
+		break;
+#endif
+#ifdef MPLS
+	case AF_MPLS:
+		input = mpls_input;
+		break;
+#endif
+	default:
+		ml_enqueue(&ns->ns_junk, m);
+		return;
+	}
+
+	(*input)(ifp, m);
+}
+
+static void
+netstack_process(struct netstack *ns)
+{
+	struct ifnet *ifp;
+	struct mbuf *m;
 
 	/*
 	 * We grab the shared netlock for packet processing in the softnet
@@ -997,43 +1083,70 @@ if_input_process(struct ifnet *ifp, stru
 	 * read only or MP safe.  Usually they hold the exclusive net lock.
 	 */
 
-	NET_LOCK_SHARED();
-	while ((m = ml_dequeue(ml)) != NULL)
-		(*ifp->if_input)(ifp, m);
-	NET_UNLOCK_SHARED();
+	do {
+		while ((m = ml_dequeue(ns->ns_ifinput)) != NULL) {
+			ifp = if_get(m->m_pkthdr.ph_ifidx);
+			if (ifp != NULL)
+				(*ifp->if_input)(ns, ifp, m);
+			else
+				ml_enqueue(&ns->ns_junk, m);
+			if_put(ifp);
+		}
+
+		m = ml_dequeue(&ns->ns_proto);
+		if (m == NULL)
+			break;
+
+		NET_LOCK_SHARED();
+		do {
+			ifp = if_get(m->m_pkthdr.ph_ifidx);
+			if (ifp != NULL)
+				netstack_proto_process(ns, ifp, m);
+			else
+				ml_enqueue(&ns->ns_junk, m);
+			if_put(ifp);
+
+			m = ml_dequeue(&ns->ns_proto);
+		} while (m != NULL);
+		NET_UNLOCK_SHARED();
+	} while (!ml_empty(ns->ns_ifinput));
+
+	ml_purge(&ns->ns_junk);
 }
 
 void
-if_vinput(struct ifnet *ifp, struct mbuf *m)
+if_input_process(struct ifnet *ifp, struct mbuf_list *ml)
 {
-#if NBPFILTER > 0
-	caddr_t if_bpf;
-#endif
+	struct netstack ns = {
+		.ns_ifinput = ml,
+		.ns_proto = MBUF_LIST_INITIALIZER(),
+		.ns_junk = MBUF_LIST_INITIALIZER(),
+	};
 
-	m->m_pkthdr.ph_ifidx = ifp->if_index;
-	m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
+	if (ml_empty(ml))
+		return;
 
-	counters_pkt(ifp->if_counters,
-	    ifc_ipackets, ifc_ibytes, m->m_pkthdr.len);
+	if (!ISSET(ifp->if_xflags, IFXF_CLONED))
+		enqueue_randomness(ml_len(ml) ^ (uintptr_t)MBUF_LIST_FIRST(ml));
+
+	netstack_process(&ns);
+}
+
+void
+if_vinput(struct ifnet *ifp, struct mbuf *m)
+{
+	struct ifiqueue *ifiq;
+	unsigned int flow = 0;
 
 #if NPF > 0
 	pf_pkt_addr_changed(m);
 #endif
 
-#if NBPFILTER > 0
-	if_bpf = ifp->if_bpf;
-	if (if_bpf) {
-		if ((*ifp->if_bpf_mtap)(if_bpf, m, BPF_DIRECTION_IN)) {
-			m_freem(m);
-			return;
-		}
-	}
-#endif
+	if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
+		flow = m->m_pkthdr.ph_flowid;
+	ifiq = ifp->if_iqs[flow % ifp->if_niqs];
 
-	if (__predict_true(!ISSET(ifp->if_xflags, IFXF_MONITOR)))
-		(*ifp->if_input)(ifp, m);
-	else
-		m_freem(m);
+	ifiq_enqueue(ifiq, m);
 }
 
 void
@@ -1193,6 +1306,10 @@ if_detach(struct ifnet *ifp)
 #endif
 
 	NET_LOCK();
+#if NKSTAT > 0
+	if_kstat_detach(ifp);
+#endif
+
 	s = splnet();
 	ifp->if_ioctl = if_detached_ioctl;
 	ifp->if_watchdog = NULL;
@@ -1678,30 +1795,9 @@ p2p_bpf_mtap(caddr_t if_bpf, const struc
 }
 
 void
-p2p_input(struct ifnet *ifp, struct mbuf *m)
+p2p_input(struct netstack *ns, struct ifnet *ifp, struct mbuf *m)
 {
-	void (*input)(struct ifnet *, struct mbuf *);
-
-	switch (m->m_pkthdr.ph_family) {
-	case AF_INET:
-		input = ipv4_input;
-		break;
-#ifdef INET6
-	case AF_INET6:
-		input = ipv6_input;
-		break;
-#endif
-#ifdef MPLS
-	case AF_MPLS:
-		input = mpls_input;
-		break;
-#endif
-	default:
-		m_freem(m);
-		return;
-	}
-
-	(*input)(ifp, m);
+	netstack_proto_input(ns, ifp, m->m_pkthdr.ph_family, m);
 }
 
 /*
@@ -2826,6 +2922,127 @@ if_getdata(struct ifnet *ifp, struct if_
 	}
 }
 
+#if NKSTAT
+struct if_kstat_data {
+	struct kstat_kv	kd_up;
+	struct kstat_kv	kd_link;
+	struct kstat_kv	kd_baudrate;
+	struct kstat_kv	kd_ibytes;
+	struct kstat_kv	kd_ipackets;
+	struct kstat_kv	kd_ierrors;
+	struct kstat_kv	kd_iqdrops;
+	struct kstat_kv	kd_obytes;
+	struct kstat_kv	kd_opackets;
+	struct kstat_kv	kd_oerrors;
+	struct kstat_kv	kd_oqdrops;
+};
+
+static const struct if_kstat_data if_kstat_data_template = {
+	KSTAT_KV_INITIALIZER("up", KSTAT_KV_T_BOOL),
+	KSTAT_KV_INITIALIZER("link", KSTAT_KV_T_BOOL),
+	KSTAT_KV_UNIT_INITIALIZER("baudrate",
+	    KSTAT_KV_T_UINT64, KSTAT_KV_U_NONE),
+
+	KSTAT_KV_UNIT_INITIALIZER("ibytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES),
+	KSTAT_KV_UNIT_INITIALIZER("ipackets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("ierrors",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("iqdrops",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+
+	KSTAT_KV_UNIT_INITIALIZER("obytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES),
+	KSTAT_KV_UNIT_INITIALIZER("opackets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("oerrors",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("oqdrops",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+};
+
+static int if_kstat_read(struct kstat *);
+
+static void
+if_kstat_attach(struct ifnet *ifp)
+{
+	struct kstat *ks;
+	struct if_kstat_data *kd;
+
+	kd = malloc(sizeof(*kd), M_DEVBUF, M_WAITOK|M_CANFAIL|M_ZERO);
+	if (kd == NULL)
+		return;
+
+	ks = kstat_create(ifp->if_xname, 0, "ifstat", 0, KSTAT_T_KV, 0);
+	if (ks == NULL) {
+		free(kd, M_DEVBUF, sizeof(*kd));
+		return;
+	}
+
+	*kd = if_kstat_data_template;
+
+	/* which lock? */
+	ks->ks_softc = ifp;
+	ks->ks_data = kd;
+	ks->ks_datalen = sizeof(*kd);
+	ks->ks_read = if_kstat_read;
+
+	ifp->if_kstat = ks;
+
+	kstat_install(ks);
+}
+
+static int
+if_kstat_read(struct kstat *ks)
+{
+	struct ifnet *ifp = ks->ks_softc;
+	struct if_kstat_data *kd = ks->ks_data;
+	struct if_data data;
+
+	memset(&data, 0, sizeof(data));
+
+	NET_LOCK_SHARED();
+	if_getdata(ifp, &data);
+
+	kstat_kv_bool(&kd->kd_up) = !!ISSET(ifp->if_flags, IFF_RUNNING);
+	kstat_kv_bool(&kd->kd_link) = LINK_STATE_IS_UP(ifp->if_link_state);
+	NET_UNLOCK_SHARED();
+
+	nanouptime(&ks->ks_updated);
+
+	kstat_kv_u64(&kd->kd_baudrate) = data.ifi_baudrate;
+
+	kstat_kv_u64(&kd->kd_ibytes) = data.ifi_ibytes;
+	kstat_kv_u64(&kd->kd_ipackets) = data.ifi_ipackets;
+	kstat_kv_u64(&kd->kd_ierrors) = data.ifi_ierrors;
+	kstat_kv_u64(&kd->kd_iqdrops) = data.ifi_iqdrops;
+
+	kstat_kv_u64(&kd->kd_obytes) = data.ifi_obytes;
+	kstat_kv_u64(&kd->kd_opackets) = data.ifi_opackets;
+	kstat_kv_u64(&kd->kd_oerrors) = data.ifi_oerrors;
+	kstat_kv_u64(&kd->kd_oqdrops) = data.ifi_oqdrops;
+
+	return (0);
+}
+
+static void
+if_kstat_detach(struct ifnet *ifp)
+{
+	struct kstat *ks = ifp->if_kstat;
+	struct if_kstat_data *kd;
+
+	if (ks == NULL)
+		return;
+
+	kstat_remove(ks);
+	kd = ks->ks_data;
+	kstat_destroy(ks);
+
+	free(kd, M_DEVBUF, sizeof(*kd));
+}
+#endif
+
 /*
  * Dummy functions replaced in ifnet during detach (if protocols decide to
  * fiddle with the if during detach.
@@ -3546,11 +3763,21 @@ unhandled_af(int af)
 	panic("unhandled af %d", af);
 }
 
+static int nettaskqs;
+
+unsigned int
+net_tq_count(void)
+{
+	if (nettaskqs == 0)
+		nettaskqs = min(NET_TASKQ, ncpus);
+
+	return (nettaskqs);
+}
+
 struct taskq *
 net_tq(unsigned int ifindex)
 {
 	struct softnet *sn;
-	static int nettaskqs;
 
 	if (nettaskqs == 0)
 		nettaskqs = min(NET_TASKQ, ncpus);
@@ -3574,4 +3801,41 @@ net_tq_barriers(const char *wmesg)
 	}
  
 	refcnt_finalize(&r, wmesg);
+}
+
+void
+net_proto_input(struct ifnet *ifp, int af, struct mbuf *m)
+{
+	struct softnet *sn;
+	unsigned int flow;
+
+	if (nettaskqs == 0)
+		nettaskqs = min(NET_TASKQ, ncpus);
+
+	if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
+		flow = m->m_pkthdr.ph_flowid;
+	else
+		flow = ifp->if_index;
+
+	sn = &softnets[flow % nettaskqs];
+
+	m->m_pkthdr.ph_family = af;
+	if (mq_enqueue(&sn->sn_proto_mq, m))
+		task_add(sn->sn_taskq, &sn->sn_proto_task);
+}
+
+static void
+net_proto_process(void *arg)
+{
+	struct softnet *sn = arg;
+	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
+	struct netstack ns = {
+		.ns_ifinput = &ml,
+		.ns_proto = MBUF_LIST_INITIALIZER(),
+		.ns_junk = MBUF_LIST_INITIALIZER(),
+	};
+
+	mq_delist(&sn->sn_proto_mq, &ns.ns_proto);
+
+	netstack_process(&ns);
 }
Index: net/if_aggr.c
===================================================================
RCS file: /cvs/src/sys/net/if_aggr.c,v
retrieving revision 1.40
diff -u -p -r1.40 if_aggr.c
--- net/if_aggr.c	16 May 2023 14:32:54 -0000	1.40
+++ net/if_aggr.c	4 Sep 2023 06:02:46 -0000
@@ -333,7 +333,7 @@ struct aggr_port {
 	uint32_t		 p_mtu;
 
 	int (*p_ioctl)(struct ifnet *, u_long, caddr_t);
-	void (*p_input)(struct ifnet *, struct mbuf *);
+	void (*p_input)(struct netstack *, struct ifnet *, struct mbuf *);
 	int (*p_output)(struct ifnet *, struct mbuf *, struct sockaddr *,
 	    struct rtentry *);
 
@@ -728,7 +728,7 @@ aggr_eh_is_slow(const struct ether_heade
 }
 
 static void
-aggr_input(struct ifnet *ifp0, struct mbuf *m)
+aggr_input(struct netstack *ns, struct ifnet *ifp0, struct mbuf *m)
 {
 	struct arpcom *ac0 = (struct arpcom *)ifp0;
 	struct aggr_port *p = ac0->ac_trunkport;
@@ -773,12 +773,12 @@ aggr_input(struct ifnet *ifp0, struct mb
 	if (!ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
 		m->m_pkthdr.ph_flowid = ifp0->if_index ^ sc->sc_mix;
 
-	if_vinput(ifp, m);
+	netstack_vinput(ns, ifp, m);
 
 	return;
 
 drop:
-	m_freem(m);
+	netstack_mfreem(ns, m);
 }
 
 static int
Index: net/if_bpe.c
===================================================================
RCS file: /cvs/src/sys/net/if_bpe.c,v
retrieving revision 1.19
diff -u -p -r1.19 if_bpe.c
--- net/if_bpe.c	8 Nov 2021 04:54:44 -0000	1.19
+++ net/if_bpe.c	4 Sep 2023 06:02:46 -0000
@@ -727,7 +727,7 @@ bpe_find(struct ifnet *ifp0, uint32_t is
 }
 
 void
-bpe_input(struct ifnet *ifp0, struct mbuf *m)
+bpe_input(struct netstack *ns, struct ifnet *ifp0, struct mbuf *m)
 {
 	struct bpe_softc *sc;
 	struct ifnet *ifp;
@@ -777,7 +777,7 @@ bpe_input(struct ifnet *ifp0, struct mbu
 	if (!ALIGNED_POINTER(mtod(n, caddr_t) + off, uint32_t)) {
 		/* unaligned ++ */
 		n = m_dup_pkt(m, ETHER_ALIGN, M_NOWAIT);
-		m_freem(m);
+		netstack_mfreem(ns, m);
 		if (n == NULL)
 			return;
 
@@ -800,18 +800,16 @@ bpe_input(struct ifnet *ifp0, struct mbu
 	}
 
 	m->m_flags &= ~(M_BCAST|M_MCAST);
-	m->m_pkthdr.ph_ifidx = ifp->if_index;
-	m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
 
 #if NPF > 0
 	pf_pkt_addr_changed(m);
 #endif
 
-	if_vinput(ifp, m);
+	netstack_vinput(ns, ifp, m);
 	return;
 
 drop:
-	m_freem(m);
+	netstack_mfreem(ns, m);
 }
 
 void
Index: net/if_bpe.h
===================================================================
RCS file: /cvs/src/sys/net/if_bpe.h,v
retrieving revision 1.1
diff -u -p -r1.1 if_bpe.h
--- net/if_bpe.h	20 Dec 2018 23:00:55 -0000	1.1
+++ net/if_bpe.h	4 Sep 2023 06:02:46 -0000
@@ -20,7 +20,7 @@
 #define _NET_IF_BPE_H
 
 #ifdef _KERNEL
-void	bpe_input(struct ifnet *, struct mbuf *);
+void	bpe_input(struct netstack *ns, struct ifnet *, struct mbuf *);
 #endif
 
 #endif /* _NET_IF_GRE_H_ */
Index: net/if_ethersubr.c
===================================================================
RCS file: /cvs/src/sys/net/if_ethersubr.c,v
retrieving revision 1.291
diff -u -p -r1.291 if_ethersubr.c
--- net/if_ethersubr.c	27 Jul 2023 20:21:25 -0000	1.291
+++ net/if_ethersubr.c	4 Sep 2023 06:02:46 -0000
@@ -377,15 +377,15 @@ ether_output(struct ifnet *ifp, struct m
  * to ether_input().
  */
 void
-ether_input(struct ifnet *ifp, struct mbuf *m)
+ether_input(struct netstack *ns, struct ifnet *ifp, struct mbuf *m)
 {
 	struct ether_header *eh;
-	void (*input)(struct ifnet *, struct mbuf *);
 	u_int16_t etype;
 	struct arpcom *ac;
 	const struct ether_brport *eb;
 	unsigned int sdelim = 0;
 	uint64_t dst, self;
+	int af;
 
 	/* Drop short frames */
 	if (m->m_len < ETHER_HDR_LEN)
@@ -407,7 +407,7 @@ ether_input(struct ifnet *ifp, struct mb
 	if (ISSET(m->m_flags, M_VLANTAG) ||
 	    etype == ETHERTYPE_VLAN || etype == ETHERTYPE_QINQ) {
 #if NVLAN > 0
-		m = vlan_input(ifp, m, &sdelim);
+		m = vlan_input(ns, ifp, m, &sdelim);
 		if (m == NULL)
 			return;
 #else
@@ -465,7 +465,7 @@ ether_input(struct ifnet *ifp, struct mb
 		 */
 		if (ifp->if_type != IFT_CARP &&
 		    !SRPL_EMPTY_LOCKED(&ifp->if_carp)) {
-			m = carp_input(ifp, m, dst);
+			m = carp_input(ns, ifp, m, dst);
 			if (m == NULL)
 				return;
 
@@ -504,27 +504,29 @@ ether_input(struct ifnet *ifp, struct mb
 
 	switch (etype) {
 	case ETHERTYPE_IP:
-		input = ipv4_input;
+		af = AF_INET;
 		break;
 
 	case ETHERTYPE_ARP:
 		if (ifp->if_flags & IFF_NOARP)
 			goto dropanyway;
-		input = arpinput;
-		break;
+		m_adj(m, sizeof(*eh));
+		arpinput(ifp, m);
+		return;
 
 	case ETHERTYPE_REVARP:
 		if (ifp->if_flags & IFF_NOARP)
 			goto dropanyway;
-		input = revarpinput;
-		break;
+		m_adj(m, sizeof(*eh));
+		revarpinput(ifp, m);
+		return;
 
 #ifdef INET6
 	/*
 	 * Schedule IPv6 software interrupt for incoming IPv6 packet.
 	 */
 	case ETHERTYPE_IPV6:
-		input = ipv6_input;
+		af = AF_INET6;
 		break;
 #endif /* INET6 */
 #if NPPPOE > 0 || defined(PIPEX)
@@ -555,12 +557,12 @@ ether_input(struct ifnet *ifp, struct mb
 #ifdef MPLS
 	case ETHERTYPE_MPLS:
 	case ETHERTYPE_MPLS_MCAST:
-		input = mpls_input;
+		af = AF_MPLS;
 		break;
 #endif
 #if NBPE > 0
 	case ETHERTYPE_PBB:
-		bpe_input(ifp, m);
+		bpe_input(ns, ifp, m);
 		return;
 #endif
 	default:
@@ -568,10 +570,10 @@ ether_input(struct ifnet *ifp, struct mb
 	}
 
 	m_adj(m, sizeof(*eh));
-	(*input)(ifp, m);
+	netstack_proto_input(ns, ifp, af, m);
 	return;
 dropanyway:
-	m_freem(m);
+	netstack_mfreem(ns, m);
 	return;
 }
 
Index: net/if_loop.c
===================================================================
RCS file: /cvs/src/sys/net/if_loop.c,v
retrieving revision 1.97
diff -u -p -r1.97 if_loop.c
--- net/if_loop.c	21 Jul 2023 22:24:41 -0000	1.97
+++ net/if_loop.c	4 Sep 2023 06:02:46 -0000
@@ -143,7 +143,7 @@
 int	loioctl(struct ifnet *, u_long, caddr_t);
 void	loopattach(int);
 void	lortrequest(struct ifnet *, int, struct rtentry *);
-void	loinput(struct ifnet *, struct mbuf *);
+void	loinput(struct netstack *, struct ifnet *, struct mbuf *);
 int	looutput(struct ifnet *,
 	    struct mbuf *, struct sockaddr *, struct rtentry *);
 int	lo_bpf_mtap(caddr_t, const struct mbuf *, u_int);
@@ -241,7 +241,7 @@ lo_bpf_mtap(caddr_t if_bpf, const struct
 }
 
 void
-loinput(struct ifnet *ifp, struct mbuf *m)
+loinput(struct netstack *ns, struct ifnet *ifp, struct mbuf *m)
 {
 	int error;
 
Index: net/if_rport.c
===================================================================
RCS file: net/if_rport.c
diff -N net/if_rport.c
Index: net/if_trunk.c
===================================================================
RCS file: /cvs/src/sys/net/if_trunk.c,v
retrieving revision 1.152
diff -u -p -r1.152 if_trunk.c
--- net/if_trunk.c	2 Aug 2021 21:10:55 -0000	1.152
+++ net/if_trunk.c	4 Sep 2023 06:02:46 -0000
@@ -76,7 +76,7 @@ int	 trunk_ether_delmulti(struct trunk_s
 void	 trunk_ether_purgemulti(struct trunk_softc *);
 int	 trunk_ether_cmdmulti(struct trunk_port *, u_long);
 int	 trunk_ioctl_allports(struct trunk_softc *, u_long, caddr_t);
-void	 trunk_input(struct ifnet *, struct mbuf *);
+void	 trunk_input(struct netstack *, struct ifnet *, struct mbuf *);
 void	 trunk_start(struct ifnet *);
 void	 trunk_init(struct ifnet *);
 void	 trunk_stop(struct ifnet *);
@@ -1142,20 +1142,21 @@ trunk_stop(struct ifnet *ifp)
 }
 
 void
-trunk_input(struct ifnet *ifp, struct mbuf *m)
+trunk_input(struct netstack *ns, struct ifnet *ifp, struct mbuf *m)
 {
 	struct arpcom *ac0 = (struct arpcom *)ifp;
 	struct trunk_port *tp;
 	struct trunk_softc *tr;
 	struct ifnet *trifp = NULL;
 	struct ether_header *eh;
+	int rv;
 
 	if (m->m_len < sizeof(*eh))
 		goto bad;
 
 	eh = mtod(m, struct ether_header *);
 	if (ETHER_IS_MULTICAST(eh->ether_dhost))
-		ifp->if_imcasts++;
+		ifp->if_imcasts++; /* counters_int */
 
 	/* Should be checked by the caller */
 	if (ifp->if_type != IFT_IEEE8023ADLAG)
@@ -1169,7 +1170,10 @@ trunk_input(struct ifnet *ifp, struct mb
 	if (tr->tr_proto == TRUNK_PROTO_NONE)
 		goto bad;
 
-	if ((*tr->tr_input)(tr, tp, m)) {
+	NET_LOCK_SHARED();
+	rv = (*tr->tr_input)(tr, tp, m);
+	NET_UNLOCK_SHARED();
+	if (rv) {
 		/*
 		 * We stop here if the packet has been consumed
 		 * by the protocol routine.
@@ -1195,13 +1199,13 @@ trunk_input(struct ifnet *ifp, struct mb
 	}
 
 
-	if_vinput(trifp, m);
+	netstack_vinput(ns, trifp, m);
 	return;
 
  bad:
 	if (trifp != NULL)
-		trifp->if_ierrors++;
-	m_freem(m);
+		trifp->if_ierrors++; /* counters_inc */
+	m_freem(m); /* netstack_mfreem */
 }
 
 int
Index: net/if_trunk.h
===================================================================
RCS file: /cvs/src/sys/net/if_trunk.h,v
retrieving revision 1.30
diff -u -p -r1.30 if_trunk.h
--- net/if_trunk.h	22 Jul 2020 02:16:02 -0000	1.30
+++ net/if_trunk.h	4 Sep 2023 06:02:46 -0000
@@ -172,7 +172,7 @@ struct trunk_port {
 	int	(*tp_ioctl)(struct ifnet *, u_long, caddr_t);
 	int	(*tp_output)(struct ifnet *, struct mbuf *, struct sockaddr *,
 		    struct rtentry *);
-	void	(*tp_input)(struct ifnet *, struct mbuf *);
+	void	(*tp_input)(struct netstack *, struct ifnet *, struct mbuf *);
 
 	SLIST_ENTRY(trunk_port)		tp_entries;
 };
Index: net/if_tun.c
===================================================================
RCS file: /cvs/src/sys/net/if_tun.c,v
retrieving revision 1.238
diff -u -p -r1.238 if_tun.c
--- net/if_tun.c	10 Feb 2023 14:39:18 -0000	1.238
+++ net/if_tun.c	4 Sep 2023 06:02:46 -0000
@@ -114,7 +114,7 @@ int	tun_dev_write(dev_t, struct uio *, i
 int	tun_dev_kqfilter(dev_t, struct knote *);
 
 int	tun_ioctl(struct ifnet *, u_long, caddr_t);
-void	tun_input(struct ifnet *, struct mbuf *);
+void	tun_input(struct netstack *, struct ifnet *, struct mbuf *);
 int	tun_output(struct ifnet *, struct mbuf *, struct sockaddr *,
 	    struct rtentry *);
 int	tun_enqueue(struct ifnet *, struct mbuf *);
@@ -924,34 +924,17 @@ put:
 }
 
 void
-tun_input(struct ifnet *ifp, struct mbuf *m0)
+tun_input(struct netstack *ns, struct ifnet *ifp, struct mbuf *m)
 {
 	uint32_t		af;
 
-	KASSERT(m0->m_len >= sizeof(af));
+	KASSERT(m->m_len >= sizeof(af));
 
-	af = *mtod(m0, uint32_t *);
+	af = *mtod(m, uint32_t *);
 	/* strip the tunnel header */
-	m_adj(m0, sizeof(af));
+	m_adj(m, sizeof(af));
 
-	switch (ntohl(af)) {
-	case AF_INET:
-		ipv4_input(ifp, m0);
-		break;
-#ifdef INET6
-	case AF_INET6:
-		ipv6_input(ifp, m0);
-		break;
-#endif
-#ifdef MPLS
-	case AF_MPLS:
-		mpls_input(ifp, m0);
-		break;
-#endif
-	default:
-		m_freem(m0);
-		break;
-	}
+	netstack_proto_input(ns, ifp, af, m);
 }
 
 int
Index: net/if_var.h
===================================================================
RCS file: /cvs/src/sys/net/if_var.h,v
retrieving revision 1.129
diff -u -p -r1.129 if_var.h
--- net/if_var.h	7 Jul 2023 08:05:02 -0000	1.129
+++ net/if_var.h	4 Sep 2023 06:02:46 -0000
@@ -89,6 +89,8 @@ struct rtentry;
 struct ifnet;
 struct task;
 struct cpumem;
+struct kstat;
+struct netstack;
 
 /*
  * Structure describing a `cloning' interface.
@@ -160,7 +162,7 @@ struct ifnet {				/* and the entries */
 	struct	task if_linkstatetask;	/* [I] task to do route updates */
 
 	/* procedure handles */
-	void	(*if_input)(struct ifnet *, struct mbuf *);
+	void	(*if_input)(struct netstack *, struct ifnet *, struct mbuf *);
 	int	(*if_bpf_mtap)(caddr_t, const struct mbuf *, u_int);
 	int	(*if_output)(struct ifnet *, struct mbuf *, struct sockaddr *,
 		     struct rtentry *);	/* output routine (enqueue) */
@@ -183,6 +185,7 @@ struct ifnet {				/* and the entries */
 	struct	ifiqueue if_rcv;	/* rx/input queue */
 	struct	ifiqueue **if_iqs;	/* [I] pointer to the array of iqs */
 	unsigned int if_niqs;		/* [I] number of input queues */
+	struct kstat *if_kstat;
 
 	struct sockaddr_dl *if_sadl;	/* [N] pointer to our sockaddr_dl */
 
@@ -334,7 +337,7 @@ int	if_output_tso(struct ifnet *, struct
 int	if_output_local(struct ifnet *, struct mbuf *, sa_family_t);
 void	if_rtrequest_dummy(struct ifnet *, int, struct rtentry *);
 void	p2p_rtrequest(struct ifnet *, int, struct rtentry *);
-void	p2p_input(struct ifnet *, struct mbuf *);
+void	p2p_input(struct netstack *, struct ifnet *, struct mbuf *);
 int	p2p_bpf_mtap(caddr_t, const struct mbuf *, u_int);
 
 struct	ifaddr *ifa_ifwithaddr(struct sockaddr *, u_int);
@@ -386,6 +389,13 @@ int	if_txhprio_l2_check(int);
 int	if_txhprio_l3_check(int);
 int	if_rxhprio_l2_check(int);
 int	if_rxhprio_l3_check(int);
+
+void	netstack_vinput(struct netstack *, struct ifnet *, struct mbuf *);
+void	netstack_proto_input(struct netstack *, struct ifnet *, int,
+	    struct mbuf *);
+void	netstack_mfreem(struct netstack *, struct mbuf *);
+
+void	net_proto_input(struct ifnet *, int, struct mbuf *);
 
 #endif /* _KERNEL */
 
Index: net/if_veb.c
===================================================================
RCS file: /cvs/src/sys/net/if_veb.c,v
retrieving revision 1.31
diff -u -p -r1.31 if_veb.c
--- net/if_veb.c	16 May 2023 14:32:54 -0000	1.31
+++ net/if_veb.c	4 Sep 2023 06:02:46 -0000
@@ -168,7 +168,7 @@ static int	veb_clone_create(struct if_cl
 static int	veb_clone_destroy(struct ifnet *);
 
 static int	veb_ioctl(struct ifnet *, u_long, caddr_t);
-static void	veb_input(struct ifnet *, struct mbuf *);
+static void	veb_input(struct netstack *, struct ifnet *, struct mbuf *);
 static int	veb_enqueue(struct ifnet *, struct mbuf *);
 static int	veb_output(struct ifnet *, struct mbuf *, struct sockaddr *,
 		    struct rtentry *);
@@ -612,6 +612,7 @@ veb_pf(struct ifnet *ifp0, int dir, stru
 {
 	struct ether_header *eh, copy;
 	const struct veb_pf_ip_family *fam;
+	int pfact;
 
 	/*
 	 * pf runs on vport interfaces when they enter or leave the
@@ -646,7 +647,10 @@ veb_pf(struct ifnet *ifp0, int dir, stru
 			return (NULL);
 	}
 
-	if (pf_test(fam->af, dir, ifp0, &m) != PF_PASS) {
+	NET_LOCK_SHARED();
+	pfact = pf_test(fam->af, dir, ifp0, &m);
+	NET_UNLOCK_SHARED();
+	if (pfact != PF_PASS) {
 		m_freem(m);
 		return (NULL);
 	}
@@ -656,7 +660,7 @@ veb_pf(struct ifnet *ifp0, int dir, stru
 	if (dir == PF_IN && ISSET(m->m_pkthdr.pf.flags, PF_TAG_DIVERTED)) {
 		pf_mbuf_unlink_state_key(m);
 		pf_mbuf_unlink_inpcb(m);
-		(*fam->ip_input)(ifp0, m);
+		net_proto_input(ifp0, fam->af, m);
 		return (NULL);
 	}
 
@@ -1191,7 +1195,7 @@ drop:
 }
 
 static void
-veb_input(struct ifnet *ifp, struct mbuf *m)
+veb_input(struct netstack *ns, struct ifnet *ifp, struct mbuf *m)
 {
 	m_freem(m);
 }
@@ -2351,6 +2355,9 @@ vport_clone_create(struct if_clone *ifc,
 	if_attach(ifp);
 	ether_ifattach(ifp);
 
+	ifq_set_maxlen(&ifp->if_snd, 8192); /* XXX */
+	if_attach_queues(ifp, net_tq_count());
+
 	return (0);
 }
 
@@ -2454,12 +2461,11 @@ vport_if_enqueue(struct ifnet *ifp, stru
 static int
 vport_enqueue(struct ifnet *ifp, struct mbuf *m)
 {
+	struct ifqueue *ifq;
 	struct arpcom *ac;
 	const struct ether_brport *eb;
+	unsigned int idx;
 	int error = ENETDOWN;
-#if NBPFILTER > 0
-	caddr_t if_bpf;
-#endif
 
 	/*
 	 * a packet sent from the l3 stack out a vport goes into
@@ -2478,20 +2484,54 @@ vport_enqueue(struct ifnet *ifp, struct 
 
 	smr_read_enter();
 	eb = SMR_PTR_GET(&ac->ac_brport);
+	smr_read_leave();
+	if (eb == NULL) {
+		m_freem(m);
+		return (ENETDOWN);
+	}
+
+	idx = ifq_idx(&ifp->if_snd, ifp->if_nifqs, m);
+	ifq = ifp->if_ifqs[idx];
+
+	error = ifq_enqueue(ifq, m);
+	if (error != 0)
+		return (error);
+
+	task_add(ifq->ifq_softnet, &ifq->ifq_bundle);
+	return (0);
+}
+
+static void
+vport_start(struct ifqueue *ifq)
+{
+	struct ifnet *ifp = ifq->ifq_if;
+	struct arpcom *ac = (struct arpcom *)ifp;
+	const struct ether_brport *eb;
+	struct mbuf *(*input)(struct ifnet *, struct mbuf *, uint64_t, void *);
+	struct ether_header *eh;
+	uint64_t dst;
+	struct mbuf *m;
+
+	ac = (struct arpcom *)ifp;
+
+	smr_read_enter();
+	eb = SMR_PTR_GET(&ac->ac_brport);
 	if (eb != NULL)
 		eb->eb_port_take(eb->eb_port);
 	smr_read_leave();
-	if (eb != NULL) {
-		struct mbuf *(*input)(struct ifnet *, struct mbuf *,
-		    uint64_t, void *) = eb->eb_input;
-		struct ether_header *eh;
-		uint64_t dst;
 
-		counters_pkt(ifp->if_counters, ifc_opackets, ifc_obytes,
-		    m->m_pkthdr.len);
+	if (eb == NULL) {
+		ifq_purge(ifq);
+		return;
+	}
+
+	input = eb->eb_input;
+	if (input == veb_vport_input)
+		input = veb_port_input;
 
+	while ((m = ifq_dequeue(ifq)) != NULL) {
 #if NBPFILTER > 0
-		if_bpf = READ_ONCE(ifp->if_bpf);
+		caddr_t if_bpf = READ_ONCE(ifp->if_bpf);
 		if (if_bpf != NULL)
 			bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_OUT);
 #endif
@@ -2499,22 +2539,10 @@ vport_enqueue(struct ifnet *ifp, struct 
 		eh = mtod(m, struct ether_header *);
 		dst = ether_addr_to_e64((struct ether_addr *)eh->ether_dhost);
 
-		if (input == veb_vport_input)
-			input = veb_port_input;
 		m = (*input)(ifp, m, dst, eb->eb_port);
-
-		error = 0;
-
-		eb->eb_port_rele(eb->eb_port);
+		if (m != NULL)
+			m_freem(m);
 	}
 
-	m_freem(m);
-
-	return (error);
-}
-
-static void
-vport_start(struct ifqueue *ifq)
-{
-	ifq_purge(ifq);
+	eb->eb_port_rele(eb->eb_port);
 }
Index: net/if_vlan.c
===================================================================
RCS file: /cvs/src/sys/net/if_vlan.c,v
retrieving revision 1.215
diff -u -p -r1.215 if_vlan.c
--- net/if_vlan.c	16 May 2023 14:32:54 -0000	1.215
+++ net/if_vlan.c	4 Sep 2023 06:02:46 -0000
@@ -374,7 +374,8 @@ vlan_inject(struct mbuf *m, uint16_t typ
 }
 
 struct mbuf *
-vlan_input(struct ifnet *ifp0, struct mbuf *m, unsigned int *sdelim)
+vlan_input(struct netstack *ns, struct ifnet *ifp0, struct mbuf *m,
+    unsigned int *sdelim)
 {
 	struct vlan_softc *sc;
 	struct ifnet *ifp;
@@ -434,7 +435,7 @@ vlan_input(struct ifnet *ifp0, struct mb
 			eh = mtod(m, struct ether_header *);
 			if (eh->ether_type == htons(ETHERTYPE_VLAN) ||
 			    eh->ether_type == htons(ETHERTYPE_QINQ)) {
-				m_freem(m);
+				netstack_mfreem(ns, m);
 				return (NULL);
 			}
 		} else
@@ -445,7 +446,8 @@ vlan_input(struct ifnet *ifp0, struct mb
 
 	ifp = &sc->sc_if;
 	if (!ISSET(ifp->if_flags, IFF_RUNNING)) {
-		m_freem(m);
+		/* maybe we should decline in this situation */
+		netstack_mfreem(ns, m);
 		goto leave;
 	}
 
@@ -471,7 +473,7 @@ vlan_input(struct ifnet *ifp0, struct mb
 		break;
 	}
 
-	if_vinput(ifp, m);
+	netstack_vinput(ns, ifp, m);
 leave:
 	refcnt_rele_wake(&sc->sc_refcnt);
 	return (NULL);
Index: net/if_vlan_var.h
===================================================================
RCS file: /cvs/src/sys/net/if_vlan_var.h,v
retrieving revision 1.46
diff -u -p -r1.46 if_vlan_var.h
--- net/if_vlan_var.h	26 Apr 2023 00:14:21 -0000	1.46
+++ net/if_vlan_var.h	4 Sep 2023 06:02:46 -0000
@@ -47,7 +47,8 @@ struct	vlanreq {
 };
 
 #ifdef _KERNEL
-struct mbuf	*vlan_input(struct ifnet *, struct mbuf *, unsigned int *);
+struct mbuf	*vlan_input(struct netstack *, struct ifnet *, struct mbuf *,
+		     unsigned int *);
 struct mbuf	*vlan_inject(struct mbuf *, uint16_t, uint16_t);
 void		 vlan_flags_from_parent(struct ifnet *, int);
 #endif /* _KERNEL */
Index: netinet/if_ether.h
===================================================================
RCS file: /cvs/src/sys/netinet/if_ether.h,v
retrieving revision 1.90
diff -u -p -r1.90 if_ether.h
--- netinet/if_ether.h	27 Jul 2023 20:21:25 -0000	1.90
+++ netinet/if_ether.h	4 Sep 2023 06:02:46 -0000
@@ -278,7 +278,7 @@ int	ether_multiaddr(struct sockaddr *, u
 void	ether_ifattach(struct ifnet *);
 void	ether_ifdetach(struct ifnet *);
 int	ether_ioctl(struct ifnet *, struct arpcom *, u_long, caddr_t);
-void	ether_input(struct ifnet *, struct mbuf *);
+void	ether_input(struct netstack *, struct ifnet *, struct mbuf *);
 int	ether_resolve(struct ifnet *, struct mbuf *, struct sockaddr *,
 	    struct rtentry *, struct ether_header *);
 struct mbuf *
Index: netinet/ip_carp.c
===================================================================
RCS file: /cvs/src/sys/netinet/ip_carp.c,v
retrieving revision 1.357
diff -u -p -r1.357 ip_carp.c
--- netinet/ip_carp.c	16 May 2023 14:32:54 -0000	1.357
+++ netinet/ip_carp.c	4 Sep 2023 06:02:46 -0000
@@ -1378,7 +1378,8 @@ carp_vhe_match(struct carp_softc *sc, ui
 }
 
 struct mbuf *
-carp_input(struct ifnet *ifp0, struct mbuf *m, uint64_t dst)
+carp_input(struct netstack *ns, struct ifnet *ifp0, struct mbuf *m,
+    uint64_t dst)
 {
 	struct srpl *cif;
 	struct carp_softc *sc;
@@ -1432,14 +1433,14 @@ carp_input(struct ifnet *ifp0, struct mb
 			if (m0 == NULL)
 				continue;
 
-			if_vinput(&sc->sc_if, m0);
+			netstack_vinput(ns, &sc->sc_if, m0);
 		}
 		SRPL_LEAVE(&sr);
 
 		return (m);
 	}
 
-	if_vinput(&sc->sc_if, m);
+	netstack_vinput(ns, &sc->sc_if, m);
 out:
 	SRPL_LEAVE(&sr);
 
Index: netinet/ip_carp.h
===================================================================
RCS file: /cvs/src/sys/netinet/ip_carp.h,v
retrieving revision 1.51
diff -u -p -r1.51 ip_carp.h
--- netinet/ip_carp.h	7 Mar 2021 06:02:32 -0000	1.51
+++ netinet/ip_carp.h	4 Sep 2023 06:02:46 -0000
@@ -209,7 +209,8 @@ carp_strict_addr_chk(struct ifnet *ifp_a
 	    ifp_a->if_carpdevidx == ifp_b->if_carpdevidx));
 }
 
-struct mbuf	*carp_input(struct ifnet *, struct mbuf *, uint64_t);
+struct mbuf	*carp_input(struct netstack *, struct ifnet *, struct mbuf *,
+		     uint64_t);
 
 int		 carp_proto_input(struct mbuf **, int *, int, int);
 void		 carp_carpdev_state(void *);
Index: sys/mbuf.h
===================================================================
RCS file: /cvs/src/sys/sys/mbuf.h,v
retrieving revision 1.261
diff -u -p -r1.261 mbuf.h
--- sys/mbuf.h	16 Jul 2023 03:01:31 -0000	1.261
+++ sys/mbuf.h	4 Sep 2023 06:02:46 -0000
@@ -505,6 +505,7 @@ struct m_tag *m_tag_next(struct mbuf *, 
 
 void			ml_init(struct mbuf_list *);
 void			ml_enqueue(struct mbuf_list *, struct mbuf *);
+void			ml_requeue(struct mbuf_list *, struct mbuf *);
 struct mbuf *		ml_dequeue(struct mbuf_list *);
 void			ml_enlist(struct mbuf_list *, struct mbuf_list *);
 struct mbuf *		ml_dechain(struct mbuf_list *);