Index: dev/pci/if_ix.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_ix.c,v
retrieving revision 1.150
diff -u -p -r1.150 if_ix.c
--- dev/pci/if_ix.c	24 Jan 2017 03:57:35 -0000	1.150
+++ dev/pci/if_ix.c	7 Feb 2017 11:21:52 -0000
@@ -133,14 +133,17 @@ void	ixgbe_rxrefill(void *);
 void	ixgbe_enable_intr(struct ix_softc *);
 void	ixgbe_disable_intr(struct ix_softc *);
 void	ixgbe_update_stats_counters(struct ix_softc *);
-int	ixgbe_txeof(struct tx_ring *);
-int	ixgbe_rxeof(struct ix_queue *);
+void	ixgbe_txeof(void *);
+void	ixgbe_rxeof(void *);
+void	ixgbe_rxfill_tmo(void *);
+void	ixgbe_rxfill_task(void *);
 void	ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t);
 void	ixgbe_iff(struct ix_softc *);
 #ifdef IX_DEBUG
 void	ixgbe_print_hw_stats(struct ix_softc *);
 #endif
-void	ixgbe_update_link_status(struct ix_softc *);
+//void	ixgbe_update_link_status(struct ix_softc *);
+void	ixgbe_update_link_status(void *);
 int	ixgbe_get_buf(struct rx_ring *, int);
 int	ixgbe_encap(struct tx_ring *, struct mbuf *);
 int	ixgbe_dma_malloc(struct ix_softc *, bus_size_t,
@@ -227,7 +230,6 @@ ixgbe_attach(struct device *parent, stru
 
 	/* Set up the timer callout */
 	timeout_set(&sc->timer, ixgbe_local_timer, sc);
-	timeout_set(&sc->rx_refill, ixgbe_rxrefill, sc);
 
 	/* Determine hardware revision */
 	ixgbe_identify_hardware(sc);
@@ -357,7 +359,7 @@ ixgbe_detach(struct device *self, int fl
 	if_detach(ifp);
 
 	timeout_del(&sc->timer);
-	timeout_del(&sc->rx_refill);
+	//timeout_del(&sc->rx_refill);
 	ixgbe_free_pci_resources(sc);
 
 	ixgbe_free_transmit_structures(sc);
@@ -383,14 +385,14 @@ ixgbe_start(struct ifqueue *ifq)
 {
 	struct ifnet		*ifp = ifq->ifq_if;
 	struct ix_softc		*sc = ifp->if_softc;
-	struct tx_ring		*txr = sc->tx_rings;
+	struct tx_ring		*txr = ifq->ifq_softc;
 	struct mbuf  		*m_head;
 	int			 post = 0;
 
-	if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(ifq))
-		return;
-	if (!sc->link_up)
+	if (!sc->link_up) {
+		ifq_purge(ifq);
 		return;
+	}
 
 	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0,
 	    txr->txdma.dma_map->dm_mapsize,
@@ -521,7 +523,7 @@ ixgbe_rxrinfo(struct ix_softc *sc, struc
 	for (i = 0; i < sc->num_queues; i++) {
 		rxr = &sc->rx_rings[i];
 		ifr[n].ifr_size = sc->rx_mbuf_sz;
-		snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "/%d", i);
+		snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i);
 		ifr[n].ifr_info = rxr->rx_ring;
 		n++;
 	}
@@ -714,9 +716,10 @@ ixgbe_init(void *arg)
 
 	timeout_add_sec(&sc->timer, 1);
 
+	/* set up interrupt routing */
+	ixgbe_configure_ivars(sc);
 	/* Set up MSI/X routing */
 	if (sc->msix > 1) {
-		ixgbe_configure_ivars(sc);
 		/* Set up auto-mask */
 		if (sc->hw.mac.type == ixgbe_mac_82598EB)
 			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
@@ -725,8 +728,8 @@ ixgbe_init(void *arg)
 			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
 		}
 	} else {  /* Simple settings for Legacy/MSI */
-		ixgbe_set_ivar(sc, 0, 0, 0);
-		ixgbe_set_ivar(sc, 0, 0, 1);
+		//ixgbe_set_ivar(sc, 0, 0, 0);
+		//ixgbe_set_ivar(sc, 0, 0, 1);
 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
 	}
 
@@ -744,7 +747,7 @@ ixgbe_init(void *arg)
 	itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8;
 	if (sc->hw.mac.type != ixgbe_mac_82598EB)
 		itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS;
-	IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr);
+//	IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr);
 
 	/* Enable power to the phy */
 	if (sc->hw.phy.ops.set_phy_power)
@@ -917,12 +920,12 @@ int
 ixgbe_intr(void *arg)
 {
 	struct ix_softc	*sc = (struct ix_softc *)arg;
-	struct ix_queue *que = sc->queues;
 	struct ifnet	*ifp = &sc->arpcom.ac_if;
-	struct tx_ring	*txr = sc->tx_rings;
 	struct ixgbe_hw	*hw = &sc->hw;
 	uint32_t	 reg_eicr, mod_mask, msf_mask;
-	int		 i, refill = 0;
+	int		 i;
+	struct tx_ring *txr;
+	struct rx_ring *rxr;
 
 	reg_eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR);
 	if (reg_eicr == 0) {
@@ -930,28 +933,25 @@ ixgbe_intr(void *arg)
 		return (0);
 	}
 
-	if (ISSET(ifp->if_flags, IFF_RUNNING)) {
-		ixgbe_rxeof(que);
-		ixgbe_txeof(txr);
-		refill = 1;
-	}
+	if (ISSET(ifp->if_flags, IFF_RUNNING) &&
+	    (reg_eicr & IXGBE_EICR_RTX_QUEUE)) {
+		struct ix_queue *que;
 
-	if (refill) {
-		if (ixgbe_rxfill(que->rxr)) {
-			/* Advance the Rx Queue "Tail Pointer" */
-			IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(que->rxr->me),
-			    que->rxr->last_desc_filled);
-		} else
-			timeout_add(&sc->rx_refill, 1);
+		for (i = 0; i < sc->num_queues; i++) {
+			txr = &sc->tx_rings[i];
+			rxr = &sc->rx_rings[i];
+			que = &sc->queues[i];
+
+			if (ISSET(reg_eicr, 1 << rxr->msix))
+				task_add(que->tq, &rxr->rxeof);
+			if (ISSET(reg_eicr, 1 << txr->msix))
+				task_add(que->tq, &txr->txeof);
+		}
 	}
 
 	/* Link status change */
-	if (reg_eicr & IXGBE_EICR_LSC) {
-		KERNEL_LOCK();
-		ixgbe_update_link_status(sc);
-		KERNEL_UNLOCK();
-		ifq_start(&ifp->if_snd);
-	}
+	if (reg_eicr & IXGBE_EICR_LSC)
+		task_add(systq, &sc->linkch);
 
 	if (hw->mac.type != ixgbe_mac_82598EB) {
 		if (reg_eicr & IXGBE_EICR_ECC) {
@@ -1015,8 +1015,14 @@ ixgbe_intr(void *arg)
 		KERNEL_UNLOCK();
 	}
 
-	for (i = 0; i < sc->num_queues; i++, que++)
-		ixgbe_enable_queue(sc, que->msix);
+#if 1
+	for (i = 0; i < sc->num_queues; i++) {
+		txr = &sc->tx_rings[i];
+		rxr = &sc->rx_rings[i];
+		ixgbe_enable_queue(sc, txr->msix);
+		ixgbe_enable_queue(sc, rxr->msix);
+	}
+#endif
 
 	return (1);
 }
@@ -1343,8 +1349,9 @@ ixgbe_local_timer(void *arg)
 }
 
 void
-ixgbe_update_link_status(struct ix_softc *sc)
+ixgbe_update_link_status(void *arg)
 {
+	struct ix_softc *sc = arg;
 	struct ifnet	*ifp = &sc->arpcom.ac_if;
 	int		link_state = LINK_STATE_DOWN;
 
@@ -1407,7 +1414,7 @@ ixgbe_stop(void *arg)
 	if (sc->hw.mac.ops.disable_tx_laser)
 		sc->hw.mac.ops.disable_tx_laser(&sc->hw);
 	timeout_del(&sc->timer);
-	timeout_del(&sc->rx_refill);
+	/* stop queues */ //timeout_del(&sc->rx_refill);
 
 	/* reprogram the RAR[0] in case user changed it. */
 	ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);
@@ -1564,7 +1571,7 @@ ixgbe_allocate_pci_resources(struct ix_s
 	sc->hw.hw_addr = (uint8_t *)os->os_membase;
 
 	/* Legacy defaults */
-	sc->num_queues = 1;
+	sc->num_queues = 8; /* XXX */
 	sc->hw.back = os;
 
 #ifdef notyet
@@ -1607,6 +1614,7 @@ void
 ixgbe_setup_interface(struct ix_softc *sc)
 {
 	struct ifnet   *ifp = &sc->arpcom.ac_if;
+	int i;
 
 	strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ);
 	ifp->if_softc = sc;
@@ -1639,9 +1647,20 @@ ixgbe_setup_interface(struct ix_softc *s
 	ixgbe_add_media_types(sc);
 	ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
 
+	task_set(&sc->linkch, ixgbe_update_link_status, sc);
+
 	if_attach(ifp);
 	ether_ifattach(ifp);
 
+	if_attach_queues(ifp, sc->num_queues);
+	for (i = 0; i < sc->num_queues; i++) {
+		struct ifqueue *ifq = ifp->if_ifqs[i];
+		struct tx_ring *txr = &sc->tx_rings[i];
+
+		ifq->ifq_softc = txr;
+		txr->ifq = ifq;
+	}
+
 	sc->max_frame_size = IXGBE_MAX_FRAME_SIZE;
 }
 
@@ -1738,7 +1757,7 @@ ixgbe_dma_malloc(struct ix_softc *sc, bu
 
 	dma->dma_tag = os->os_pa.pa_dmat;
 	r = bus_dmamap_create(dma->dma_tag, size, 1,
-	    size, 0, BUS_DMA_NOWAIT, &dma->dma_map);
+	    size, 0, BUS_DMA_NOWAIT | BUS_DMA_BUS4, &dma->dma_map);
 	if (r != 0) {
 		printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; "
 		       "error %u\n", ifp->if_xname, r);
@@ -1853,6 +1872,7 @@ ixgbe_allocate_queues(struct ix_softc *s
 		txr = &sc->tx_rings[i];
 		txr->sc = sc;
 		txr->me = i;
+		txr->msix = i;
 
 		if (ixgbe_dma_malloc(sc, tsize,
 		    &txr->txdma, BUS_DMA_NOWAIT)) {
@@ -1862,6 +1882,8 @@ ixgbe_allocate_queues(struct ix_softc *s
 		}
 		txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
 		bzero((void *)txr->tx_base, tsize);
+
+		task_set(&txr->txeof, ixgbe_txeof, txr);
 	}
 
 	/*
@@ -1874,6 +1896,8 @@ ixgbe_allocate_queues(struct ix_softc *s
 		/* Set up some basics */
 		rxr->sc = sc;
 		rxr->me = i;
+		rxr->msix = i + 8;
+		rxr->que = &sc->queues[i];
 
 		if (ixgbe_dma_malloc(sc, rsize,
 			&rxr->rxdma, BUS_DMA_NOWAIT)) {
@@ -1883,6 +1907,10 @@ ixgbe_allocate_queues(struct ix_softc *s
 		}
 		rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
 		bzero((void *)rxr->rx_base, rsize);
+
+		task_set(&rxr->rxeof, ixgbe_rxeof, rxr);
+		task_set(&rxr->rxfill, ixgbe_rxfill_task, rxr);
+		timeout_set(&rxr->refill, ixgbe_rxfill_tmo, rxr);
 	}
 
 	/*
@@ -1893,6 +1921,11 @@ ixgbe_allocate_queues(struct ix_softc *s
 		que->sc = sc;
 		que->txr = &sc->tx_rings[i];
 		que->rxr = &sc->rx_rings[i];
+		snprintf(que->tqname, sizeof(que->tqname), "%s.%d",
+		    sc->dev.dv_xname, i);
+		que->tq = taskq_create(que->tqname, 1, IPL_NET, TASKQ_MPSAFE);
+		if (que->tq == NULL)
+			goto err_rx_desc;
 	}
 
 	return (0);
@@ -1941,7 +1974,7 @@ ixgbe_allocate_transmit_buffers(struct t
 		txbuf = &txr->tx_buffers[i];
 		error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE,
 			    sc->num_segs, PAGE_SIZE, 0,
-			    BUS_DMA_NOWAIT, &txbuf->map);
+			    BUS_DMA_NOWAIT | BUS_DMA_BUS4, &txbuf->map);
 
 		if (error != 0) {
 			printf("%s: Unable to create TX DMA map\n",
@@ -2313,37 +2346,35 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, 
  *  tx_buffer is put back on the free queue.
  *
  **********************************************************************/
-int
-ixgbe_txeof(struct tx_ring *txr)
+void
+ixgbe_txeof(void *arg)
 {
+	struct tx_ring			*txr = arg;
 	struct ix_softc			*sc = txr->sc;
-	struct ifnet			*ifp = &sc->arpcom.ac_if;
+	struct ifqueue			*ifq = txr->ifq;
+	struct ifnet			*ifp = ifq->ifq_if;
 	uint32_t			 first, last, done, processed;
 	uint32_t			 num_avail;
 	struct ixgbe_tx_buf		*tx_buffer;
 	struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
 
-	if (!ISSET(ifp->if_flags, IFF_RUNNING))
-		return FALSE;
-
 	if (txr->tx_avail == sc->num_tx_desc) {
 		txr->queue_status = IXGBE_QUEUE_IDLE;
-		return FALSE;
+		return;
 	}
 
-	membar_consumer();
-
 	processed = 0;
 	first = txr->next_to_clean;
 	/* was the txt queue cleaned up in the meantime */
 	if (txr->tx_buffers == NULL)
-		return FALSE;
+		return;
+
 	tx_buffer = &txr->tx_buffers[first];
 	/* For cleanup we just use legacy struct */
 	tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
 	last = tx_buffer->eop_index;
 	if (last == -1)
-		return FALSE;
+		return;
 	eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
 
 	/*
@@ -2357,7 +2388,7 @@ ixgbe_txeof(struct tx_ring *txr)
 
 	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 	    0, txr->txdma.dma_map->dm_mapsize,
-	    BUS_DMASYNC_POSTREAD);
+	    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 
 	while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
 		/* We clean the range of the packet */
@@ -2411,10 +2442,10 @@ ixgbe_txeof(struct tx_ring *txr)
 	if (num_avail == sc->num_tx_desc)
 		ifp->if_timer = 0;
 
-	if (ifq_is_oactive(&ifp->if_snd))
-		ifq_restart(&ifp->if_snd);
+	ixgbe_enable_queue(sc, txr->msix);
 
-	return TRUE;
+	if (ifq_is_oactive(ifq))
+		ifq_restart(ifq);
 }
 
 /*********************************************************************
@@ -2430,7 +2461,6 @@ ixgbe_get_buf(struct rx_ring *rxr, int i
 	struct mbuf		*mp;
 	int			error;
 	union ixgbe_adv_rx_desc	*rxdesc;
-	size_t			 dsize = sizeof(union ixgbe_adv_rx_desc);
 
 	rxbuf = &rxr->rx_buffers[i];
 	rxdesc = &rxr->rx_base[i];
@@ -2457,15 +2487,9 @@ ixgbe_get_buf(struct rx_ring *rxr, int i
 
 	bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map,
 	    0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD);
-	rxbuf->buf = mp;
-
-	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
-	    dsize * i, dsize, BUS_DMASYNC_POSTWRITE);
 
-	rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr);
-
-	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
-	    dsize * i, dsize, BUS_DMASYNC_PREWRITE);
+	rxbuf->buf = mp;
+	htolem64(&rxdesc->read.pkt_addr, rxbuf->map->dm_segs[0].ds_addr);
 
 	return (0);
 }
@@ -2497,7 +2521,7 @@ ixgbe_allocate_receive_buffers(struct rx
 	rxbuf = rxr->rx_buffers;
 	for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) {
 		error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1,
-		    16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map);
+		    16 * 1024, 0, BUS_DMA_NOWAIT | BUS_DMA_BUS4, &rxbuf->map);
 		if (error) {
 			printf("%s: Unable to create Pack DMA map\n",
 			    ifp->if_xname);
@@ -2560,6 +2584,8 @@ ixgbe_rxfill(struct rx_ring *rxr)
 	int		 i;
 
 	i = rxr->last_desc_filled;
+	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0,
+	    rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
 	for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc);
 	    slots > 0; slots--) {
 		if (++i == sc->num_rx_desc)
@@ -2571,6 +2597,8 @@ ixgbe_rxfill(struct rx_ring *rxr)
 		rxr->last_desc_filled = i;
 		post = 1;
 	}
+	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0,
+	    rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
 
 	if_rxr_put(&rxr->rx_ring, slots);
 
@@ -2578,20 +2606,25 @@ ixgbe_rxfill(struct rx_ring *rxr)
 }
 
 void
-ixgbe_rxrefill(void *xsc)
+ixgbe_rxfill_tmo(void *arg)
 {
-	struct ix_softc *sc = xsc;
-	struct ix_queue *que = sc->queues;
-	int s;
+	struct rx_ring *rxr = arg;
+	/* serialise the refill with rxeof */
+	task_add(rxr->que->tq, &rxr->rxfill);
+}
 
-	s = splnet();
-	if (ixgbe_rxfill(que->rxr)) {
+void
+ixgbe_rxfill_task(void *arg)
+{
+	struct rx_ring *rxr = arg;
+	struct ix_softc *sc = rxr->sc;
+
+	if (ixgbe_rxfill(rxr)) {
 		/* Advance the Rx Queue "Tail Pointer" */
-		IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(que->rxr->me),
-		    que->rxr->last_desc_filled);
+		IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me),
+		    rxr->last_desc_filled);
 	} else
-		timeout_add(&sc->rx_refill, 1);
-	splx(s);
+		timeout_add(&rxr->refill, 1);
 }
 
 /*********************************************************************
@@ -2822,11 +2855,11 @@ ixgbe_free_receive_buffers(struct rx_rin
  *  dma'ed into host memory to upper layer.
  *
  *********************************************************************/
-int
-ixgbe_rxeof(struct ix_queue *que)
+void
+ixgbe_rxeof(void *arg)
 {
-	struct ix_softc 	*sc = que->sc;
-	struct rx_ring		*rxr = que->rxr;
+	struct rx_ring		*rxr = arg;
+	struct ix_softc 	*sc = rxr->sc;
 	struct ifnet   		*ifp = &sc->arpcom.ac_if;
 	struct mbuf_list	 ml = MBUF_LIST_INITIALIZER();
 	struct mbuf    		*mp, *sendmp;
@@ -2838,16 +2871,13 @@ ixgbe_rxeof(struct ix_queue *que)
 	size_t			 dsize = sizeof(union ixgbe_adv_rx_desc);
 	int			 i, nextp;
 
-	if (!ISSET(ifp->if_flags, IFF_RUNNING))
-		return FALSE;
-
 	i = rxr->next_to_check;
+	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0,
+	    rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
 	while (if_rxr_inuse(&rxr->rx_ring) > 0) {
-		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
-		    dsize * i, dsize, BUS_DMASYNC_POSTREAD);
 
 		rxdesc = &rxr->rx_base[i];
-		staterr = letoh32(rxdesc->wb.upper.status_error);
+		staterr = lemtoh32(&rxdesc->wb.upper.status_error);
 		if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) {
 			bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 			    dsize * i, dsize,
@@ -2865,10 +2895,10 @@ ixgbe_rxeof(struct ix_queue *que)
 		bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map);
 
 		mp = rxbuf->buf;
-		len = letoh16(rxdesc->wb.upper.length);
-		ptype = letoh32(rxdesc->wb.lower.lo_dword.data) &
+		len = lemtoh16(&rxdesc->wb.upper.length);
+		ptype = lemtoh32(&rxdesc->wb.lower.lo_dword.data) &
 		    IXGBE_RXDADV_PKTTYPE_MASK;
-		vtag = letoh16(rxdesc->wb.upper.vlan);
+		vtag = lemtoh16(&rxdesc->wb.upper.vlan);
 		eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
 
 		if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) {
@@ -2948,22 +2978,20 @@ ixgbe_rxeof(struct ix_queue *que)
 		}
 next_desc:
 		if_rxr_put(&rxr->rx_ring, 1);
-		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
-		    dsize * i, dsize,
-		    BUS_DMASYNC_PREREAD);
 
 		/* Advance our pointers to the next descriptor. */
 		if (++i == sc->num_rx_desc)
 			i = 0;
 	}
+	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0,
+	    rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
 	rxr->next_to_check = i;
 
-	if_input(ifp, &ml);
+	ixgbe_rxfill_task(rxr);
 
-	if (!(staterr & IXGBE_RXD_STAT_DD))
-		return FALSE;
+	if_input(ifp, &ml);
 
-	return TRUE;
+	ixgbe_enable_queue(sc, rxr->msix);
 }
 
 /*********************************************************************
@@ -3035,11 +3063,11 @@ void
 ixgbe_enable_intr(struct ix_softc *sc)
 {
 	struct ixgbe_hw *hw = &sc->hw;
-	struct ix_queue *que = sc->queues;
 	uint32_t	mask, fwsm;
 	int i;
 
 	mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
+	//mask = IXGBE_EIMS_ENABLE_MASK;
 	/* Enable Fan Failure detection */
 	if (hw->device_id == IXGBE_DEV_ID_82598AT)
 		    mask |= IXGBE_EIMS_GPI_SDP1;
@@ -3089,8 +3117,12 @@ ixgbe_enable_intr(struct ix_softc *sc)
 	 * allow for handling the extended (beyond 32) MSIX
 	 * vectors that can be used by 82599
 	 */
-	for (i = 0; i < sc->num_queues; i++, que++)
-		ixgbe_enable_queue(sc, que->msix);
+	for (i = 0; i < sc->num_queues; i++) {
+		struct rx_ring *rxr = &sc->rx_rings[i];
+		struct tx_ring *txr = &sc->tx_rings[i];
+		ixgbe_enable_queue(sc, rxr->msix);
+		ixgbe_enable_queue(sc, txr->msix);
+	}
 
 	IXGBE_WRITE_FLUSH(hw);
 }
@@ -3167,7 +3199,6 @@ ixgbe_set_ivar(struct ix_softc *sc, uint
 	vector |= IXGBE_IVAR_ALLOC_VAL;
 
 	switch (hw->mac.type) {
-
 	case ixgbe_mac_82598EB:
 		if (type == -1)
 			entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
@@ -3193,8 +3224,10 @@ ixgbe_set_ivar(struct ix_softc *sc, uint
 		} else {	/* RX/TX IVARS */
 			index = (16 * (entry & 1)) + (8 * type);
 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
+printf("%s: ivar entry %u type %u pre %u=%08x\n", sc->dev.dv_xname, entry, type, IXGBE_IVAR(entry >> 1), ivar);
 			ivar &= ~(0xFF << index);
 			ivar |= (vector << index);
+printf("%s: ivar entry %u type %u post %u=%08x\n", sc->dev.dv_xname, entry, type, IXGBE_IVAR(entry >> 1), ivar);
 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
 		}
 
@@ -3206,29 +3239,24 @@ ixgbe_set_ivar(struct ix_softc *sc, uint
 void
 ixgbe_configure_ivars(struct ix_softc *sc)
 {
-#if notyet
 	struct ix_queue *que = sc->queues;
 	uint32_t newitr;
 	int i;
 
-	if (ixgbe_max_interrupt_rate > 0)
-		newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
-	else
-		newitr = 0;
+	newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8;
 
 	for (i = 0; i < sc->num_queues; i++, que++) {
 		/* First the RX queue entry */
-		ixgbe_set_ivar(sc, i, que->msix, 0);
+		ixgbe_set_ivar(sc, i, que->rxr->msix, 0);
 		/* ... and the TX */
-		ixgbe_set_ivar(sc, i, que->msix, 1);
+		ixgbe_set_ivar(sc, i, que->txr->msix, 1);
 		/* Set an Initial EITR value */
-		IXGBE_WRITE_REG(&sc->hw,
-		    IXGBE_EITR(que->msix), newitr);
+		IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(que->rxr->msix), newitr);
+		IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(que->txr->msix), newitr);
 	}
 
 	/* For the Link interrupt */
-	ixgbe_set_ivar(sc, 1, sc->linkvec, -1);
-#endif
+//	ixgbe_set_ivar(sc, 1, sc->linkvec, -1);
 }
 
 /*
Index: dev/pci/if_ix.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_ix.h,v
retrieving revision 1.32
diff -u -p -r1.32 if_ix.h
--- dev/pci/if_ix.h	21 Nov 2016 17:21:33 -0000	1.32
+++ dev/pci/if_ix.h	7 Feb 2017 11:21:52 -0000
@@ -151,12 +151,13 @@ struct ixgbe_dma_alloc {
  */
 struct ix_queue {
 	struct ix_softc         *sc;
-	uint32_t		msix;           /* This queue's MSIX vector */
 	uint32_t		eims;           /* This queue's EIMS bit */
 	uint32_t		eitr_setting;
 	void			*tag;
 	struct tx_ring		*txr;
 	struct rx_ring		*rxr;
+	char			 tqname[8];
+	struct taskq		*tq;		/* fake multiple cpus */
 };
 
 /*
@@ -165,6 +166,7 @@ struct ix_queue {
 struct tx_ring {
 	struct ix_softc		*sc;
 	uint32_t		me;
+	uint32_t		msix;           /* This queue's MSIX vector */
 	uint32_t		watchdog_timer;
 	union ixgbe_adv_tx_desc	*tx_base;
 	struct ixgbe_tx_buf	*tx_buffers;
@@ -183,6 +185,9 @@ struct tx_ring {
 	uint32_t		packets;
 	/* Soft Stats */
 	uint64_t		tx_packets;
+
+	struct ifqueue		*ifq;
+	struct task		txeof;
 };
 
 
@@ -191,7 +196,9 @@ struct tx_ring {
  */
 struct rx_ring {
 	struct ix_softc		*sc;
+	struct ix_queue		*que;
 	uint32_t		me;
+	uint32_t		msix;           /* This queue's MSIX vector */
 	union ixgbe_adv_rx_desc	*rx_base;
 	struct ixgbe_dma_alloc	rxdma;
 #if 0
@@ -215,6 +222,10 @@ struct rx_ring {
 	uint64_t		rx_bytes;
 	uint64_t		rx_discarded;
 	uint64_t		rsc_num;
+
+	struct timeout		refill;
+	struct task		rxfill;
+	struct task		rxeof;
 };
 
 /* Our adapter structure */
@@ -229,7 +240,7 @@ struct ix_softc {
 
 	struct ifmedia		media;
 	struct timeout		timer;
-	struct timeout		rx_refill;
+	struct task		linkch;
 	int			msix;
 	int			if_flags;
 
Index: arch/sparc64/dev/iommu.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/iommu.c,v
retrieving revision 1.73
diff -u -p -r1.73 iommu.c
--- arch/sparc64/dev/iommu.c	4 May 2016 18:26:12 -0000	1.73
+++ arch/sparc64/dev/iommu.c	7 Feb 2017 11:21:52 -0000
@@ -72,9 +72,9 @@ int iommudebug = IDB_INFO;
 #define DPRINTF(l, s)
 #endif
 
-void iommu_enter(struct iommu_state *, struct strbuf_ctl *, bus_addr_t,
+static inline void iommu_enter(struct iommu_state *, struct strbuf_ctl *, bus_addr_t,
     paddr_t, int);
-void iommu_remove(struct iommu_state *, struct strbuf_ctl *, bus_addr_t);
+//void iommu_remove(struct iommu_state *, struct strbuf_ctl *, bus_addr_t);
 int iommu_dvmamap_sync_range(struct strbuf_ctl*, bus_addr_t, bus_size_t);
 int iommu_strbuf_flush_done(struct iommu_map_state *);
 int iommu_dvmamap_load_seg(bus_dma_tag_t, struct iommu_state *,
@@ -151,7 +151,8 @@ iommu_init(char *name, struct iommu_stat
 	 */
 	if (strncmp(name, "pyro", 4) == 0) {
 		is->is_cr = IOMMUREG_READ(is, iommu_cr);
-		is->is_cr &= ~IOMMUCR_FIRE_BE;
+		//is->is_cr &= ~IOMMUCR_FIRE_BE;
+		is->is_cr |= IOMMUCR_FIRE_BE;
 		is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
 		    IOMMUCR_FIRE_TE);
 	} else 
@@ -319,14 +320,14 @@ strbuf_reset(struct strbuf_ctl *sb)
  * The entry is marked streaming if an STC was detected and 
  * the BUS_DMA_STREAMING flag is set.
  */
-void
+static inline void
 iommu_enter(struct iommu_state *is, struct strbuf_ctl *sb, bus_addr_t va,
     paddr_t pa, int flags)
 {
 	int64_t tte;
 	volatile int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)];
 
-#ifdef DIAGNOSTIC
+#if 0 //def DIAGNOSTIC
 	if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
 		panic("iommu_enter: va %#lx not in DVMA space", va);
 
@@ -378,13 +379,13 @@ iommu_enter(struct iommu_state *is, stru
  * flag, but better to be safe.  (The IOMMU will be ignored as long
  * as an STC entry exists.)
  */
-void
+static inline void
 iommu_remove(struct iommu_state *is, struct strbuf_ctl *sb, bus_addr_t va)
 {
-	int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)];
-	int64_t tte;
+	uint64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)];
+	uint64_t tte;
 
-#ifdef DIAGNOSTIC
+#if 0 //def DIAGNOSTIC
 	if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
 		panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
 	if (va != trunc_page(va)) {
@@ -394,10 +395,12 @@ iommu_remove(struct iommu_state *is, str
 #endif
 	tte = *tte_ptr;
 
+#if 0
 	DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%llx]@%p\n",
 	    va, tte, tte_ptr));
+#endif
 
-#ifdef DIAGNOSTIC
+#if 0 //def DIAGNOSTIC
 	if ((tte & IOTTE_V) == 0) {
 		printf("Removing invalid tte entry (dva %lx &tte %p "
 		    "tte %llx)\n", va, tte_ptr, tte);
@@ -412,7 +415,6 @@ iommu_remove(struct iommu_state *is, str
 	 * IO operations are strongly ordered WRT each other.  It is
 	 * unclear how they relate to normal memory accesses.
 	 */
-	membar(StoreStore);
 
 	IOMMUREG_WRITE(is, iommu_flush, va);
 
@@ -610,6 +612,11 @@ iommu_dvmamap_create(bus_dma_tag_t t, bu
 	if (ret)
 		return (ret);
 
+	if (flags & BUS_DMA_BUS4) {
+		*dmamap = map;
+		return (0);
+	}
+
 	ims = iommu_iomap_create(atop(round_page(size)));
 
 	if (ims == NULL) {
@@ -670,6 +677,27 @@ iommu_dvmamap_load(bus_dma_tag_t t, bus_
 	struct iommu_map_state *ims = map->_dm_cookie;
 	pmap_t pmap;
 
+	if (map->_dm_flags & BUS_DMA_BUS4) {
+		int i;
+
+		BUS_DMA_FIND_PARENT(t, _dmamap_load);
+		err = (*t->_dmamap_load)(t, t0, map, buf, buflen, p, flags);
+		if (err != 0)
+			return (err);
+
+		/* Bypass translation by the IOMMU. */
+		for (i = 0; i < map->dm_nsegs; i++) {
+			map->dm_segs[i].ds_addr |= 0xfffc000000000000UL;
+//			map->dm_segs[i].ds_addr |= 1ULL << 47;
+#if 0
+			printf("%s: %u %016lx %lu\n", __func__, i,
+			    map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len);
+#endif
+		}
+
+		return (0);
+	}
+
 #ifdef DIAGNOSTIC
 	if (ims == NULL)
 		panic("iommu_dvmamap_load: null map state");
@@ -878,11 +906,34 @@ iommu_dvmamap_load_raw(bus_dma_tag_t t, 
 	struct iommu_state *is;
 	struct iommu_map_state *ims = map->_dm_cookie;
 
-#ifdef DIAGNOSTIC
+	if (map->_dm_flags & BUS_DMA_BUS4) {
+		/* Bypass translation by the IOMMU. */
+		for (i = 0; i < nsegs; i++) {
+#if 0
+			printf("%s: %u %016lx %lu\n", __func__, i,
+			    map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len);
+#endif
+			map->dm_segs[i].ds_addr = segs[i].ds_addr |
+			    0xfffc000000000000UL;
+			map->dm_segs[i].ds_len = segs[i].ds_len;
+//			map->dm_segs[i].ds_addr |= 1ULL << 47;
+#if 0
+			printf("%s: %u %016lx %lu\n", __func__, i,
+			    map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len);
+#endif
+		}
+
+		map->dm_nsegs = nsegs;
+		map->dm_mapsize = size;
+
+		return (0);
+	}
+
+#if 0 //def DIAGNOSTIC
 	if (ims == NULL)
 		panic("iommu_dvmamap_load_raw: null map state");
 #endif
-#ifdef DEBUG
+#if 0//def DEBUG
 	if (ims->ims_sb == NULL)
 		panic("iommu_dvmamap_load_raw: null sb");
 	if (ims->ims_sb->sb_iommu == NULL)
@@ -890,6 +941,8 @@ iommu_dvmamap_load_raw(bus_dma_tag_t t, 
 #endif /* DEBUG */
 	is = ims->ims_sb->sb_iommu;
 
+	KASSERT(map->dm_nsegs == 0);
+#if 0
 	if (map->dm_nsegs) {
 		/* Already in use?? */
 #ifdef DIAGNOSTIC
@@ -897,6 +950,7 @@ iommu_dvmamap_load_raw(bus_dma_tag_t t, 
 #endif
 		bus_dmamap_unload(t0, map);
 	}
+#endif
 
 	/*
 	 * A boundary presented to bus_dmamem_alloc() takes precedence
@@ -1303,6 +1357,11 @@ iommu_dvmamap_unload(bus_dma_tag_t t, bu
 	bus_size_t sgsize = map->_dm_dvmasize;
 	int error;
 
+	if (map->_dm_flags & BUS_DMA_BUS4) {
+		bus_dmamap_unload(t->_parent, map);
+		return;
+	}
+
 #ifdef DEBUG
 	if (ims == NULL)
 		panic("iommu_dvmamap_unload: null map state");
@@ -1548,6 +1607,12 @@ iommu_dvmamap_sync(bus_dma_tag_t t, bus_
 {
 	struct iommu_map_state *ims = map->_dm_cookie;
 
+	if (map->_dm_flags & BUS_DMA_BUS4) {
+		if (ops & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD))
+			membar(MemIssue);
+		return;
+	}
+
 #ifdef DIAGNOSTIC
 	if (ims == NULL)
 		panic("iommu_dvmamap_sync: null map state");
@@ -1622,9 +1687,13 @@ iommu_dvmamem_alloc(bus_dma_tag_t t, bus
 	    "bound %llx segp %p flags %d\n", (unsigned long long)size,
 	    (unsigned long long)alignment, (unsigned long long)boundary,
 	    segs, flags));
+
+	if ((flags & BUS_DMA_BUS4) == 0)
+		flags |= BUS_DMA_DVMA;
+
 	BUS_DMA_FIND_PARENT(t, _dmamem_alloc);
 	return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary,
-	    segs, nsegs, rsegs, flags | BUS_DMA_DVMA));
+	    segs, nsegs, rsegs, flags));
 }
 
 void
@@ -1782,6 +1851,8 @@ iommu_iomap_unload_map(struct iommu_stat
 	struct iommu_page_entry *e;
 	struct strbuf_ctl *sb = ims->ims_sb;
 	int i, slot;
+
+	membar(StoreStore);
 
 	for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) {
 		iommu_remove(is, sb, e->ipe_va);
Index: arch/sparc64/dev/iommureg.h
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/iommureg.h,v
retrieving revision 1.17
diff -u -p -r1.17 iommureg.h
--- arch/sparc64/dev/iommureg.h	17 Aug 2012 20:46:50 -0000	1.17
+++ arch/sparc64/dev/iommureg.h	7 Feb 2017 11:21:52 -0000
@@ -72,42 +72,42 @@ struct iommu_strbuf {
 #define STRBUFREG(x)    (offsetof(struct iommu_strbuf, x))
 
 /* streaming buffer control register */
-#define STRBUF_EN		0x000000000000000001LL
-#define STRBUF_D		0x000000000000000002LL
+#define STRBUF_EN		0x0000000000000001ULL
+#define STRBUF_D		0x0000000000000002ULL
 
 /* control register bits */
-#define IOMMUCR_TSB1K		0x000000000000000000LL	/* Nummber of entries in IOTSB */
-#define IOMMUCR_TSB2K		0x000000000000010000LL
-#define IOMMUCR_TSB4K		0x000000000000020000LL
-#define IOMMUCR_TSB8K		0x000000000000030000LL
-#define IOMMUCR_TSB16K		0x000000000000040000LL
-#define IOMMUCR_TSB32K		0x000000000000050000LL
-#define IOMMUCR_TSB64K		0x000000000000060000LL
-#define IOMMUCR_TSB128K		0x000000000000070000LL
-#define IOMMUCR_TSBMASK		0xfffffffffffff8ffffLL	/* Mask for above */
-#define IOMMUCR_8KPG		0x000000000000000000LL	/* 8K iommu page size */
-#define IOMMUCR_64KPG		0x000000000000000004LL	/* 64K iommu page size */
-#define IOMMUCR_DE		0x000000000000000002LL	/* Diag enable */
-#define IOMMUCR_EN		0x000000000000000001LL	/* Enable IOMMU */
+#define IOMMUCR_TSB1K		0x0000000000000000ULL	/* Nummber of entries in IOTSB */
+#define IOMMUCR_TSB2K		0x0000000000010000ULL
+#define IOMMUCR_TSB4K		0x0000000000020000ULL
+#define IOMMUCR_TSB8K		0x0000000000030000ULL
+#define IOMMUCR_TSB16K		0x0000000000040000ULL
+#define IOMMUCR_TSB32K		0x0000000000050000ULL
+#define IOMMUCR_TSB64K		0x0000000000060000ULL
+#define IOMMUCR_TSB128K		0x0000000000070000ULL
+#define IOMMUCR_TSBMASK		0xfffffffffff8ffffULL	/* Mask for above */
+#define IOMMUCR_8KPG		0x0000000000000000ULL	/* 8K iommu page size */
+#define IOMMUCR_64KPG		0x0000000000000004ULL	/* 64K iommu page size */
+#define IOMMUCR_DE		0x0000000000000002ULL	/* Diag enable */
+#define IOMMUCR_EN		0x0000000000000001ULL	/* Enable IOMMU */
 
-#define IOMMUCR_FIRE_SE		0x000000000000000400LL	/* Snoop enable */
-#define IOMMUCR_FIRE_CM_EN	0x000000000000000300LL  /* Cache mode enable */
-#define IOMMUCR_FIRE_BE		0x000000000000000002LL	/* Bypass enable */
-#define IOMMUCR_FIRE_TE		0x000000000000000001LL	/* Translation enabled */
+#define IOMMUCR_FIRE_SE		0x0000000000000400ULL	/* Snoop enable */
+#define IOMMUCR_FIRE_CM_EN	0x0000000000000300ULL	/* Cache mode enable */
+#define IOMMUCR_FIRE_BE		0x0000000000000002ULL	/* Bypass enable */
+#define IOMMUCR_FIRE_TE		0x0000000000000001ULL	/* Translation enabled */
 
 /*
  * IOMMU stuff
  */
-#define	IOTTE_V		0x8000000000000000LL	/* Entry valid */
-#define IOTTE_64K	0x2000000000000000LL	/* 8K or 64K page? */
-#define IOTTE_8K	0x0000000000000000LL
-#define IOTTE_STREAM	0x1000000000000000LL	/* Is page streamable? */
-#define	IOTTE_LOCAL	0x0800000000000000LL	/* Accesses to same bus segment? */
-#define	IOTTE_CONTEXT	0x07ff800000000000LL	/* context number */
-#define IOTTE_PAMASK	0x00007fffffffe000LL	/* Let's assume this is correct (bits 42..13) */
-#define IOTTE_C		0x0000000000000010LL	/* Accesses to cacheable space */
-#define IOTTE_W		0x0000000000000002LL	/* Writeable */
-#define IOTTE_SOFTWARE	0x0000000000001f80LL	/* For software use (bits 12..7) */
+#define	IOTTE_V		0x8000000000000000ULL	/* Entry valid */
+#define IOTTE_64K	0x2000000000000000ULL	/* 8K or 64K page? */
+#define IOTTE_8K	0x0000000000000000ULL
+#define IOTTE_STREAM	0x1000000000000000ULL	/* Is page streamable? */
+#define	IOTTE_LOCAL	0x0800000000000000ULL	/* Accesses to same bus segment? */
+#define	IOTTE_CONTEXT	0x07ff800000000000ULL	/* context number */
+#define IOTTE_PAMASK	0x00007fffffffe000ULL	/* Let's assume this is correct (bits 42..13) */
+#define IOTTE_C		0x0000000000000010ULL	/* Accesses to cacheable space */
+#define IOTTE_W		0x0000000000000002ULL	/* Writeable */
+#define IOTTE_SOFTWARE	0x0000000000001f80ULL	/* For software use (bits 12..7) */
 
 
 /*
Index: arch/sparc64/sparc64/machdep.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/sparc64/machdep.c,v
retrieving revision 1.183
diff -u -p -r1.183 machdep.c
--- arch/sparc64/sparc64/machdep.c	21 Jan 2017 05:42:03 -0000	1.183
+++ arch/sparc64/sparc64/machdep.c	7 Feb 2017 11:21:52 -0000
@@ -1105,6 +1105,12 @@ _bus_dmamap_load_mbuf(t, t0, map, m, fla
 			}
 			segs[i].ds_addr = pa;
 			segs[i].ds_len = incr;
+#if 0
+			if (map->_dm_flags & BUS_DMA_BUS4) {
+				printf("%s: %u %016lx %lu\n", __func__, i,
+				    segs[i].ds_addr, segs[i].ds_len);
+			}
+#endif
 			segs[i]._ds_boundary = 0;
 			segs[i]._ds_align = 0;
 			segs[i]._ds_mlist = NULL;