? arch/i386/stand/mbr/assym.h.tmp
Index: arch/i386/conf/GENERIC.MP
===================================================================
RCS file: /cvs/src/sys/arch/i386/conf/GENERIC.MP,v
retrieving revision 1.9
diff -u -p -r1.9 GENERIC.MP
--- arch/i386/conf/GENERIC.MP	3 Sep 2014 07:44:33 -0000	1.9
+++ arch/i386/conf/GENERIC.MP	10 Sep 2015 15:19:31 -0000
@@ -5,6 +5,7 @@
 
 include "arch/i386/conf/GENERIC"
 
+makeoptions DEBUG="-g"
 option		MULTIPROCESSOR	# Multiple processor support
 #option		MP_LOCKDEBUG
 
Index: net/if.c
===================================================================
RCS file: /cvs/src/sys/net/if.c,v
retrieving revision 1.367
diff -u -p -r1.367 if.c
--- net/if.c	10 Sep 2015 14:06:43 -0000	1.367
+++ net/if.c	10 Sep 2015 15:19:33 -0000
@@ -151,6 +151,21 @@ void	if_input_process(void *);
 void	ifa_print_all(void);
 #endif
 
+struct if_map {
+	unsigned long		 limit;
+	/* followed by limit x ifnet * */
+};
+
+struct if_idxmap {
+	unsigned int		 serial;
+	unsigned int		 count;
+	struct if_map		*map;
+};
+
+void	if_idxmap_init(void);
+void	if_idxmap_insert(struct ifnet *);
+void	if_idxmap_remove(struct ifnet *);
+
 TAILQ_HEAD(, ifg_group) ifg_head = TAILQ_HEAD_INITIALIZER(ifg_head);
 LIST_HEAD(, if_clone) if_cloners = LIST_HEAD_INITIALIZER(if_cloners);
 int if_cloners_count;
@@ -171,6 +186,8 @@ struct taskq *softnettq;
 void
 ifinit()
 {
+	if_idxmap_init();
+
 	timeout_set(&net_tick_to, net_tick, &net_tick_to);
 
 	softnettq = taskq_create("softnet", 1, IPL_NET,
@@ -181,90 +198,122 @@ ifinit()
 	net_tick(&net_tick_to);
 }
 
-static unsigned int if_index = 0;
-static unsigned int if_indexlim = 0;
-struct ifnet **ifindex2ifnet = NULL;
+static struct if_idxmap if_idxmap = {
+	0,
+	0,
+	NULL
+};
+
 struct ifnet_head ifnet = TAILQ_HEAD_INITIALIZER(ifnet);
 struct ifnet_head iftxlist = TAILQ_HEAD_INITIALIZER(iftxlist);
 struct ifnet *lo0ifp;
 
-/*
- * Attach an interface to the
- * list of "active" interfaces.
- */
 void
-if_attachsetup(struct ifnet *ifp)
+if_idxmap_init(void)
 {
-	int wrapped = 0;
+	if_idxmap.serial = 1; /* skip ifidx 0 */
+	if_idxmap.map = malloc(sizeof(*if_idxmap.map) +
+	    8 * sizeof(struct ifnet *), M_IFADDR, M_WAITOK | M_ZERO);
+	if_idxmap.map->limit = 8;
+}
 
-	/*
-	 * Always increment the index to avoid races.
-	 */
-	if_index++;
+void
+if_idxmap_insert(struct ifnet *ifp)
+{
+	struct if_map *if_map;
+	struct ifnet **map;
+	u_int index, i;
 
-	/*
-	 * If we hit USHRT_MAX, we skip back to 1 since there are a
-	 * number of places where the value of ifp->if_index or
-	 * if_index itself is compared to or stored in an unsigned
-	 * short.  By jumping back, we won't botch those assignments
-	 * or comparisons.
-	 */
-	if (if_index == USHRT_MAX) {
-		if_index = 1;
-		wrapped++;
-	}
-
-	while (if_index < if_indexlim && ifindex2ifnet[if_index] != NULL) {
-		if_index++;
-
-		if (if_index == USHRT_MAX) {
-			/*
-			 * If we have to jump back to 1 twice without
-			 * finding an empty slot then there are too many
-			 * interfaces.
-			 */
-			if (wrapped)
-				panic("too many interfaces");
+	ifp->if_refcnt = 1; /* give the ifnet a reference for existing */
 
-			if_index = 1;
-			wrapped++;
-		}
+	/* lock if_idxmap */
+	KERNEL_ASSERT_LOCKED();
+
+	if (++if_idxmap.count > USHRT_MAX)
+		panic("too many interfaces");
+
+	if_map = if_idxmap.map;
+	map = (struct ifnet **)(if_map + 1);
+
+	index = if_idxmap.serial++ & USHRT_MAX;
+
+	if (index >= if_map->limit) {
+		struct if_map *nif_map;
+		struct ifnet **nmap;
+		u_int nlimit;
+
+		nlimit = if_map->limit * 2;
+		nif_map = malloc(sizeof(*nif_map) + nlimit * sizeof(*nmap),
+		    M_IFADDR, M_WAITOK);
+		nmap = (struct ifnet **)(nif_map + 1);
+
+		nif_map->limit = nlimit;
+		for (i = 0; i < if_map->limit; i++)
+			nmap[i] = map[i];
+
+		while (i < nlimit)
+			nmap[i] = NULL;
+
+		if_idxmap.map = nif_map;
+		free(if_map, M_IFADDR, sizeof(*nif_map) +
+		    if_map->limit * sizeof(*map));
+		if_map = nif_map;
+		map = nmap;
 	}
-	ifp->if_index = if_index;
 
-	/*
-	 * We have some arrays that should be indexed by if_index.
-	 * since if_index will grow dynamically, they should grow too.
-	 *	struct ifnet **ifindex2ifnet
-	 */
-	if (ifindex2ifnet == NULL || if_index >= if_indexlim) {
-		size_t m, n, oldlim;
-		caddr_t q;
-
-		oldlim = if_indexlim;
-		if (if_indexlim == 0)
-			if_indexlim = 8;
-		while (if_index >= if_indexlim)
-			if_indexlim <<= 1;
-
-		/* grow ifindex2ifnet */
-		m = oldlim * sizeof(struct ifnet *);
-		n = if_indexlim * sizeof(struct ifnet *);
-		q = (caddr_t)malloc(n, M_IFADDR, M_WAITOK|M_ZERO);
-		if (ifindex2ifnet) {
-			bcopy((caddr_t)ifindex2ifnet, q, m);
-			free((caddr_t)ifindex2ifnet, M_IFADDR, 0);
-		}
-		ifindex2ifnet = (struct ifnet **)q;
+	for (i = 0; i < USHRT_MAX; i++) {
+		/* index 0 is special */
+		if (index != 0 && map[index] == NULL)
+			break;
+
+		index = if_idxmap.serial++ & USHRT_MAX;
 	}
 
+	/* commit */
+	ifp->if_index = index;
+	map[index] = if_ref(ifp);
+
+	/* unlock if_idxmap */
+}
+
+void
+if_idxmap_remove(struct ifnet *ifp)
+{
+	struct if_map *if_map;
+	struct ifnet **map;
+	u_int index, r;
+
+	index = ifp->if_index & USHRT_MAX;
+
+	/* lock if_idxmap */
+	KERNEL_ASSERT_LOCKED();
+	if_map = if_idxmap.map;
+	map = (struct ifnet **)(if_map + 1);
+	KASSERT(ifp == map[index]);
+
+	map[index] = NULL;
+	if_put(ifp);
+	if_idxmap.count--;
+	/* unlock if_idxmap */
+
+	r = atomic_dec_int_nv(&ifp->if_refcnt); /* drop the last reference */
+	if (r != 0)
+		printf("%s: refcnt %u\n", ifp->if_xname, r);
+
+	/* sleep until refcnt == 0 */
+}
+
+/*
+ * Attach an interface to the
+ * list of "active" interfaces.
+ */
+void
+if_attachsetup(struct ifnet *ifp)
+{
 	TAILQ_INIT(&ifp->if_groups);
 
 	if_addgroup(ifp, IFG_ALL);
 
-	ifp->if_refcnt = 0;
-	ifindex2ifnet[if_index] = if_ref(ifp);
-
 	if (ifp->if_snd.ifq_maxlen == 0)
 		IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN);
 
@@ -278,6 +327,9 @@ if_attachsetup(struct ifnet *ifp)
 
 	task_set(ifp->if_linkstatetask, if_link_state_change_task, ifp);
 
+	if_idxmap_insert(ifp);
+	KASSERT(if_get(0) == NULL);
+
 	/* Announce the interface. */
 	rt_ifannouncemsg(ifp, IFAN_ARRIVAL);
 }
@@ -760,8 +812,7 @@ if_detach(struct ifnet *ifp)
 	/* Announce that the interface is gone. */
 	rt_ifannouncemsg(ifp, IFAN_DEPARTURE);
 
-	ifindex2ifnet[ifp->if_index] = NULL;
-	if_put(ifp);
+	if_idxmap_remove(ifp);
 	splx(s);
 }
 
@@ -1324,12 +1375,16 @@ ifunit(const char *name)
 struct ifnet *
 if_get(unsigned int index)
 {
+	struct if_map *if_map = if_idxmap.map;
+	struct ifnet **map = (struct ifnet **)(if_map + 1);
 	struct ifnet *ifp = NULL;
 
-	if (index < if_indexlim) {
-		ifp = ifindex2ifnet[index];
-		if (ifp != NULL)
+	if (index < if_map->limit) {
+		ifp = map[index];
+		if (ifp != NULL) {
+			KASSERT(ifp->if_index == index);
 			if_ref(ifp);
+		}
 	}
 
 	return (ifp);
Index: net/if_bridge.c
===================================================================
RCS file: /cvs/src/sys/net/if_bridge.c,v
retrieving revision 1.262
diff -u -p -r1.262 if_bridge.c
--- net/if_bridge.c	10 Sep 2015 13:32:19 -0000	1.262
+++ net/if_bridge.c	10 Sep 2015 15:19:34 -0000
@@ -1264,24 +1264,17 @@ bridge_input(struct ifnet *ifp, struct m
 	return (NULL);
 }
 
-void
-bridge_process(struct mbuf *m)
+static inline void
+_bridge_process(struct ifnet *ifp, struct mbuf *m)
 {
 	struct bridge_softc *sc;
 	struct bridge_iflist *ifl;
 	struct bridge_iflist *srcifl;
 	struct ether_header *eh;
-	struct ifnet *ifp;
 	struct arpcom *ac;
 	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
 	struct mbuf *mc;
 
-	ifp = if_get(m->m_pkthdr.ph_ifidx);
-	if (ifp == NULL) {
-		m_freem(m);
-		return;
-	}
-
 	ifl = (struct bridge_iflist *)ifp->if_bridgeport;
 	if (ifl == NULL)
 		goto reenqueue;
@@ -1408,6 +1401,22 @@ reenqueue:
 	m->m_flags |= M_PROTO1;
 	ml_enqueue(&ml, m);
 	if_input(ifp, &ml);
+}
+
+void
+bridge_process(struct mbuf *m)
+{
+	struct ifnet *ifp;
+
+	ifp = if_get(m->m_pkthdr.ph_ifidx);
+	if (ifp == NULL) {
+		m_freem(m);
+		return;
+	}
+
+	_bridge_process(ifp, m);
+
+	if_put(ifp);
 }
 
 /*
Index: net/if_tun.c
===================================================================
RCS file: /cvs/src/sys/net/if_tun.c,v
retrieving revision 1.154
diff -u -p -r1.154 if_tun.c
--- net/if_tun.c	9 Sep 2015 20:18:03 -0000	1.154
+++ net/if_tun.c	10 Sep 2015 15:19:34 -0000
@@ -734,6 +734,7 @@ tunread(dev_t dev, struct uio *uio, int 
 	TUNDEBUG(("%s: read\n", ifp->if_xname));
 	if ((tp->tun_flags & TUN_READY) != TUN_READY) {
 		TUNDEBUG(("%s: not ready %#x\n", ifp->if_xname, tp->tun_flags));
+		if_put(ifp);
 		return (EHOSTDOWN);
 	}
 
Index: net/if_var.h
===================================================================
RCS file: /cvs/src/sys/net/if_var.h,v
retrieving revision 1.37
diff -u -p -r1.37 if_var.h
--- net/if_var.h	10 Sep 2015 14:56:41 -0000	1.37
+++ net/if_var.h	10 Sep 2015 15:19:34 -0000
@@ -140,7 +140,7 @@ struct ifnet {				/* and the entries */
 	} if_carp_ptr;
 #define if_carp		if_carp_ptr.carp_s
 #define if_carpdev	if_carp_ptr.carp_d
-	u_short	if_index;		/* numeric abbreviation for this if */
+	unsigned int if_index;		/* numeric abbreviation for this if */
 	short	if_timer;		/* time 'til if_watchdog called */
 	short	if_flags;		/* up/down, broadcast, etc. */
 	int	if_xflags;		/* extra softnet flags */
@@ -225,7 +225,7 @@ struct ifaddr {
  */
 struct ifmaddr {
 	struct sockaddr		*ifma_addr;	/* Protocol address */
-	unsigned short		 ifma_ifidx;	/* Index of the interface */
+	unsigned int		 ifma_ifidx;	/* Index of the interface */
 	unsigned int		 ifma_refcnt;	/* Count of references */
 	TAILQ_ENTRY(ifmaddr)	 ifma_list;	/* Per-interface list */
 };
Index: net/if_vlan.c
===================================================================
RCS file: /cvs/src/sys/net/if_vlan.c,v
retrieving revision 1.136
diff -u -p -r1.136 if_vlan.c
--- net/if_vlan.c	10 Sep 2015 13:32:19 -0000	1.136
+++ net/if_vlan.c	10 Sep 2015 15:19:34 -0000
@@ -172,6 +172,24 @@ vlan_ifdetach(void *ptr)
 	vlan_clone_destroy(&ifv->ifv_if);
 }
 
+static inline int
+vlan_mplstunnel(int ifidx)
+{
+#if NMPW > 0
+	struct ifnet *ifp;
+	int rv = 0;
+
+	ifp = if_get(ifidx);
+	if (ifp != NULL) {
+		rv = ifp->if_type == IFT_MPLSTUNNEL;
+		if_put(ifp);
+	}
+	return (rv);
+#else
+	return (0);
+#endif
+}
+
 void
 vlan_start(struct ifnet *ifp)
 {
@@ -206,21 +224,18 @@ vlan_start(struct ifnet *ifp)
 		if (prio <= 1)
 			prio = !prio;
 
-#if NMPW > 0
-		struct ifnet *ifpn = if_get(m->m_pkthdr.ph_ifidx);
 		/*
 		 * If this packet came from a pseudowire it means it already
 		 * has all tags it needs, so just output it.
 		 */
-		if (ifpn && ifpn->if_type == IFT_MPLSTUNNEL) {
+		if (vlan_mplstunnel(m->m_pkthdr.ph_ifidx)) {
 			/* NOTHING */
-		} else
-#endif /* NMPW */
+
 		/*
 		 * If the underlying interface cannot do VLAN tag insertion
 		 * itself, create an encapsulation header.
 		 */
-		if ((p->if_capabilities & IFCAP_VLAN_HWTAGGING) &&
+		} else if ((p->if_capabilities & IFCAP_VLAN_HWTAGGING) &&
 		    (ifv->ifv_type == ETHERTYPE_VLAN)) {
 			m->m_pkthdr.ether_vtag = ifv->ifv_tag +
 			    (prio << EVL_PRIO_BITS);
Index: net/route.c
===================================================================
RCS file: /cvs/src/sys/net/route.c,v
retrieving revision 1.231
diff -u -p -r1.231 route.c
--- net/route.c	10 Sep 2015 09:37:52 -0000	1.231
+++ net/route.c	10 Sep 2015 15:19:34 -0000
@@ -684,8 +684,10 @@ ifa_ifwithroute(int flags, struct sockad
 
 			if (ifp == NULL)
 				ifp = ifunit(sdl->sdl_data);
-			if (ifp != NULL)
+			if (ifp != NULL) {
 				ifa = ifaof_ifpforaddr(dst, ifp);
+				if_put(ifp);
+			}
 		} else {
 			ifa = ifa_ifwithnet(gateway, rtableid);
 		}
@@ -764,6 +766,8 @@ rt_getifa(struct rt_addrinfo *info, u_in
 			info->rti_ifa = ifa_ifwithroute(info->rti_flags,
 			    sa, sa, rtid);
 	}
+
+	if_put(ifp);
 
 	if (info->rti_ifa == NULL)
 		return (ENETUNREACH);
Index: netinet/ip_input.c
===================================================================
RCS file: /cvs/src/sys/netinet/ip_input.c,v
retrieving revision 1.253
diff -u -p -r1.253 ip_input.c
--- netinet/ip_input.c	19 Aug 2015 15:30:25 -0000	1.253
+++ netinet/ip_input.c	10 Sep 2015 15:19:35 -0000
@@ -233,7 +233,7 @@ ipv4_input(struct mbuf *m)
 	if (m->m_len < sizeof (struct ip) &&
 	    (m = m_pullup(m, sizeof (struct ip))) == NULL) {
 		ipstat.ips_toosmall++;
-		return;
+		goto out;
 	}
 	ip = mtod(m, struct ip *);
 	if (ip->ip_v != IPVERSION) {
@@ -248,7 +248,7 @@ ipv4_input(struct mbuf *m)
 	if (hlen > m->m_len) {
 		if ((m = m_pullup(m, hlen)) == NULL) {
 			ipstat.ips_badhlen++;
-			return;
+			goto out;
 		}
 		ip = mtod(m, struct ip *);
 	}
@@ -318,7 +318,7 @@ ipv4_input(struct mbuf *m)
 	if (pf_test(AF_INET, PF_IN, ifp, &m) != PF_PASS)
 		goto bad;
 	if (m == NULL)
-		return;
+		goto out;
 
 	ip = mtod(m, struct ip *);
 	hlen = ip->ip_hl << 2;
@@ -332,12 +332,12 @@ ipv4_input(struct mbuf *m)
 	 * to be sent and the original packet to be freed).
 	 */
 	if (hlen > sizeof (struct ip) && ip_dooptions(m, ifp)) {
-	        return;
+	        goto out;
 	}
 
 	if (in_ouraddr(m, ifp, ip->ip_dst)) {
 		ip_ours(m);
-		return;
+		goto out;
 	}
 
 	if (IN_MULTICAST(ip->ip_dst.s_addr)) {
@@ -355,7 +355,7 @@ ipv4_input(struct mbuf *m)
 			if (m->m_flags & M_EXT) {
 				if ((m = m_pullup(m, hlen)) == NULL) {
 					ipstat.ips_toosmall++;
-					return;
+					goto out;
 				}
 				ip = mtod(m, struct ip *);
 			}
@@ -383,7 +383,7 @@ ipv4_input(struct mbuf *m)
 			 */
 			if (ip->ip_p == IPPROTO_IGMP) {
 				ip_ours(m);
-				return;
+				goto out;
 			}
 			ipstat.ips_forward++;
 		}
@@ -400,13 +400,13 @@ ipv4_input(struct mbuf *m)
 			goto bad;
 		}
 		ip_ours(m);
-		return;
+		goto out;
 	}
 
 	if (ip->ip_dst.s_addr == INADDR_BROADCAST ||
 	    ip->ip_dst.s_addr == INADDR_ANY) {
 		ip_ours(m);
-		return;
+		goto out;
 	}
 
 #if NCARP > 0
@@ -451,9 +451,12 @@ ipv4_input(struct mbuf *m)
 #endif /* IPSEC */
 
 	ip_forward(m, ifp, pfrdr);
+	if_put(ifp);
 	return;
 bad:
 	m_freem(m);
+out:
+	if_put(ifp);
 }
 
 /*