Index: alpha/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/alpha/include/bus.h,v
retrieving revision 1.30
diff -u -p -r1.30 bus.h
--- alpha/include/bus.h	3 May 2016 11:22:08 -0000	1.30
+++ alpha/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -464,6 +464,7 @@ struct alpha_bus_space {
 #define	BUS_DMA_READ		0x0200	/* mapping is device -> memory only */
 #define	BUS_DMA_WRITE		0x0400	/* mapping is memory -> device only */
 #define	BUS_DMA_ZERO		0x1000	/* zero memory in dmamem_alloc */
+#define BUS_DMA_64BIT		0x0000
 
 /*
  * Private flags stored in the DMA map.
Index: amd64/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/bus.h,v
retrieving revision 1.32
diff -u -p -r1.32 bus.h
--- amd64/include/bus.h	24 Jan 2015 15:13:55 -0000	1.32
+++ amd64/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -525,6 +525,7 @@ extern const struct x86_bus_space_ops x8
 #define	BUS_DMA_WRITE		0x0400	/* mapping is memory -> device only */
 #define	BUS_DMA_NOCACHE		0x0800	/* map memory uncached */
 #define	BUS_DMA_ZERO		0x1000	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_64BIT		0x2000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
Index: arm/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/arm/include/bus.h,v
retrieving revision 1.16
diff -u -p -r1.16 bus.h
--- arm/include/bus.h	5 Oct 2016 07:44:24 -0000	1.16
+++ arm/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -617,6 +617,7 @@ bs_c_8_proto(f);
 #define	BUS_DMA_WRITE		0x200	/* mapping is memory -> device only */
 #define	BUS_DMA_NOCACHE		0x400	/* hint: map non-cached memory */
 #define	BUS_DMA_ZERO		0x800	/* dmamem_alloc returns zeroed mem */
+#define	BUS_DMA_64BIT		0x1000
 
 /*
  * Private flags stored in the DMA map.
Index: arm64/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/arm64/include/bus.h,v
retrieving revision 1.1
diff -u -p -r1.1 bus.h
--- arm64/include/bus.h	17 Dec 2016 23:38:33 -0000	1.1
+++ arm64/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -315,6 +315,7 @@ bus_space_barrier(bus_space_tag_t t, bus
 #define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
 #define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
 #define	BUS_DMA_NOCACHE		0x1000
+#define	BUS_DMA_64BIT		0x2000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
Index: hppa/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/hppa/include/bus.h,v
retrieving revision 1.31
diff -u -p -r1.31 bus.h
--- hppa/include/bus.h	14 Nov 2011 14:29:53 -0000	1.31
+++ hppa/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -299,6 +299,7 @@ extern const struct hppa_bus_space_tag h
 #define	BUS_DMA_READ		0x0400	/* mapping is device -> memory only */
 #define	BUS_DMA_WRITE		0x0800	/* mapping is memory -> device only */
 #define	BUS_DMA_ZERO		0x1000	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_64BIT		0x2000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
Index: i386/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/i386/include/bus.h,v
retrieving revision 1.66
diff -u -p -r1.66 bus.h
--- i386/include/bus.h	27 Feb 2015 18:59:43 -0000	1.66
+++ i386/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -528,6 +528,7 @@ extern const struct i386_bus_space_ops i
 #define	BUS_DMA_WRITE		0x0400	/* mapping is memory -> device only */
 #define	BUS_DMA_NOCACHE		0x0800	/* map memory uncached */
 #define	BUS_DMA_ZERO		0x1000	/* dmamem_alloc return zeroed mem */
+#define	BUS_DMA_64BIT		0x2000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
Index: landisk/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/landisk/include/bus.h,v
retrieving revision 1.8
diff -u -p -r1.8 bus.h
--- landisk/include/bus.h	23 Mar 2011 16:54:35 -0000	1.8
+++ landisk/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -434,6 +434,7 @@ struct _bus_space {
 #define	BUS_DMA_WRITE		0x200	/* mapping is memory -> device only */
 #define	BUS_DMA_NOCACHE		0x400	/* hint: map non-cached memory */
 #define	BUS_DMA_ZERO		0x800	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_64BIT		0x1000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
Index: loongson/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/loongson/include/bus.h,v
retrieving revision 1.6
diff -u -p -r1.6 bus.h
--- loongson/include/bus.h	24 May 2014 21:11:01 -0000	1.6
+++ loongson/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -350,6 +350,7 @@ bus_space_barrier(bus_space_tag_t t, bus
 #define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
 #define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
 #define	BUS_DMA_NOCACHE		0x1000
+#define	BUS_DMA_64BIT		0x2000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
Index: macppc/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/macppc/include/bus.h,v
retrieving revision 1.25
diff -u -p -r1.25 bus.h
--- macppc/include/bus.h	3 May 2016 12:23:25 -0000	1.25
+++ macppc/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -489,6 +489,7 @@ bus_space_copy_4(void *v, bus_space_hand
 #define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
 #define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
 #define	BUS_DMA_NOCACHE		0x1000	/* map memory uncached */
+#define	BUS_DMA_64BIT		0x2000
 
 
 /* Forwards needed by prototypes below. */
Index: octeon/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/octeon/include/bus.h,v
retrieving revision 1.6
diff -u -p -r1.6 bus.h
--- octeon/include/bus.h	29 Mar 2014 18:09:30 -0000	1.6
+++ octeon/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -316,6 +316,7 @@ bus_space_barrier(bus_space_tag_t t, bus
 #define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
 #define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
 #define	BUS_DMA_NOCACHE		0x1000
+#define	BUS_DMA_64BIT		0x2000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
Index: sgi/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/sgi/include/bus.h,v
retrieving revision 1.26
diff -u -p -r1.26 bus.h
--- sgi/include/bus.h	10 Mar 2014 21:32:15 -0000	1.26
+++ sgi/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -318,6 +318,7 @@ bus_space_barrier(bus_space_tag_t t, bus
 #define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
 #define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
 #define	BUS_DMA_NOCACHE		0x1000
+#define	BUS_DMA_64BIT		0x2000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
Index: socppc/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/socppc/include/bus.h,v
retrieving revision 1.11
diff -u -p -r1.11 bus.h
--- socppc/include/bus.h	24 Jan 2015 20:59:42 -0000	1.11
+++ socppc/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -461,6 +461,7 @@ bus_space_copy_4(void *v, bus_space_hand
 #define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
 #define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
 #define	BUS_DMA_NOCACHE		0x1000	/* map memory uncached */
+#define	BUS_DMA_64BIT		0x2000
 
 
 /* Forwards needed by prototypes below. */
Index: sparc64/dev/iommu.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/iommu.c,v
retrieving revision 1.73
diff -u -p -r1.73 iommu.c
--- sparc64/dev/iommu.c	4 May 2016 18:26:12 -0000	1.73
+++ sparc64/dev/iommu.c	8 Feb 2017 13:35:07 -0000
@@ -100,6 +100,25 @@ void iommu_iomap_clear_pages(struct iomm
 void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t,
     bus_addr_t, bus_size_t, int);
 
+void iommu_hw_enable(struct iommu_state *);
+
+const struct iommu_hw iommu_hw_default = {
+	.ihw_enable	= iommu_hw_enable,
+
+	.ihw_dvma_pa	= IOTTE_PAMASK,
+
+	.ihw_bypass	= 0x3fffUL << 50,
+	.ihw_bypass_nc	= 0,
+	.ihw_bypass_ro	= 0,
+};
+
+void
+iommu_hw_enable(struct iommu_state *is)
+{
+	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
+	IOMMUREG_WRITE(is, iommu_cr, IOMMUCR_EN | (is->is_tsbsize << 16));
+}
+
 /*
  * Initiate an STC entry flush.
  */
@@ -125,7 +144,8 @@ iommu_strbuf_flush(struct strbuf_ctl *sb
  *	- create a private DVMA map.
  */
 void
-iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
+iommu_init(char *name, const struct iommu_hw *ihw, struct iommu_state *is,
+    int tsbsize, u_int32_t iovabase)
 {
 	psize_t size;
 	vaddr_t va;
@@ -149,13 +169,9 @@ iommu_init(char *name, struct iommu_stat
 	 * be hard-wired, so we read the start and size from the PROM and
 	 * just use those values.
 	 */
-	if (strncmp(name, "pyro", 4) == 0) {
-		is->is_cr = IOMMUREG_READ(is, iommu_cr);
-		is->is_cr &= ~IOMMUCR_FIRE_BE;
-		is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
-		    IOMMUCR_FIRE_TE);
-	} else 
-		is->is_cr = IOMMUCR_EN;
+
+	is->is_hw = ihw;
+
 	is->is_tsbsize = tsbsize;
 	if (iovabase == (u_int32_t)-1) {
 		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
@@ -237,15 +253,6 @@ iommu_init(char *name, struct iommu_stat
 	mtx_init(&is->is_mtx, IPL_HIGH);
 
 	/*
-	 * Set the TSB size.  The relevant bits were moved to the TSB
-	 * base register in the PCIe host bridges.
-	 */
-	if (strncmp(name, "pyro", 4) == 0)
-		is->is_ptsb |= is->is_tsbsize;
-	else
-		is->is_cr |= (is->is_tsbsize << 16);
-
-	/*
 	 * Now actually start up the IOMMU.
 	 */
 	iommu_reset(is);
@@ -262,10 +269,7 @@ iommu_reset(struct iommu_state *is)
 {
 	int i;
 
-	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
-
-	/* Enable IOMMU */
-	IOMMUREG_WRITE(is, iommu_cr, is->is_cr);
+	(*is->is_hw->ihw_enable)(is);
 
 	for (i = 0; i < 2; ++i) {
 		struct strbuf_ctl *sb = is->is_sb[i];
@@ -280,7 +284,7 @@ iommu_reset(struct iommu_state *is)
 			printf(", STC%d enabled", i);
 	}
 
-	if (is->is_flags & IOMMU_FLUSH_CACHE)
+	if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE))
 		IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
 }
 
@@ -433,7 +437,7 @@ iommu_extract(struct iommu_state *is, bu
 	if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
 		tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
 
-	return (tte & IOTTE_PAMASK);
+	return (tte & is->is_hw->ihw_dvma_pa);
 }
 
 /*
@@ -610,6 +614,12 @@ iommu_dvmamap_create(bus_dma_tag_t t, bu
 	if (ret)
 		return (ret);
 
+	if (flags & BUS_DMA_64BIT) {
+		map->_dm_cookie = sb->sb_iommu;
+		*dmamap = map;
+		return (0);
+	}
+
 	ims = iommu_iomap_create(atop(round_page(size)));
 
 	if (ims == NULL) {
@@ -641,8 +651,10 @@ iommu_dvmamap_destroy(bus_dma_tag_t t, b
 	if (map->dm_nsegs)
 		bus_dmamap_unload(t0, map);
 
-        if (map->_dm_cookie)
-                iommu_iomap_destroy(map->_dm_cookie);
+	if (!ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+	        if (map->_dm_cookie)
+			iommu_iomap_destroy(map->_dm_cookie);
+	}
 	map->_dm_cookie = NULL;
 
 	BUS_DMA_FIND_PARENT(t, _dmamap_destroy);
@@ -667,36 +679,36 @@ iommu_dvmamap_load(bus_dma_tag_t t, bus_
 	u_long dvmaddr, sgstart, sgend;
 	bus_size_t align, boundary;
 	struct iommu_state *is;
-	struct iommu_map_state *ims = map->_dm_cookie;
+	struct iommu_map_state *ims;
 	pmap_t pmap;
 
-#ifdef DIAGNOSTIC
-	if (ims == NULL)
-		panic("iommu_dvmamap_load: null map state");
-#endif
-#ifdef DEBUG
-	if (ims->ims_sb == NULL)
-		panic("iommu_dvmamap_load: null sb");
-	if (ims->ims_sb->sb_iommu == NULL)
-		panic("iommu_dvmamap_load: null iommu");
-#endif /* DEBUG */
-	is = ims->ims_sb->sb_iommu;
-
-	if (map->dm_nsegs) {
-		/*
-		 * Is it still in use? _bus_dmamap_load should have taken care
-		 * of this.
-		 */
-#ifdef DIAGNOSTIC
-		panic("iommu_dvmamap_load: map still in use");
-#endif
-		bus_dmamap_unload(t0, map);
-	}
-
 	/*
 	 * Make sure that on error condition we return "no valid mappings".
 	 */
-	map->dm_nsegs = 0;
+	KASSERTMSG(map->dm_nsegs == 0, "map still in use");
+
+	if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+		unsigned long bypass;
+		int i;
+
+		is = map->_dm_cookie;
+		bypass = is->is_hw->ihw_bypass;
+
+		/* Bypass translation by the IOMMU. */
+
+		BUS_DMA_FIND_PARENT(t, _dmamap_load);
+		err = (*t->_dmamap_load)(t, t0, map, buf, buflen, p, flags);
+		if (err != 0)
+			return (err);
+
+		for (i = 0; i < map->dm_nsegs; i++)
+			map->dm_segs[i].ds_addr |= bypass;
+
+		return (0);
+	}
+
+	ims = map->_dm_cookie;
+	is = ims->ims_sb->sb_iommu;
 
 	if (buflen < 1 || buflen > map->_dm_size) {
 		DPRINTF(IDB_BUSDMA,
@@ -876,28 +888,31 @@ iommu_dvmamap_load_raw(bus_dma_tag_t t, 
 	bus_size_t boundary, align;
 	u_long dvmaddr, sgstart, sgend;
 	struct iommu_state *is;
-	struct iommu_map_state *ims = map->_dm_cookie;
+	struct iommu_map_state *ims;
 
-#ifdef DIAGNOSTIC
-	if (ims == NULL)
-		panic("iommu_dvmamap_load_raw: null map state");
-#endif
-#ifdef DEBUG
-	if (ims->ims_sb == NULL)
-		panic("iommu_dvmamap_load_raw: null sb");
-	if (ims->ims_sb->sb_iommu == NULL)
-		panic("iommu_dvmamap_load_raw: null iommu");
-#endif /* DEBUG */
-	is = ims->ims_sb->sb_iommu;
+	KASSERTMSG(map->dm_nsegs == 0, "map stil in use");
 
-	if (map->dm_nsegs) {
-		/* Already in use?? */
-#ifdef DIAGNOSTIC
-		panic("iommu_dvmamap_load_raw: map still in use");
-#endif
-		bus_dmamap_unload(t0, map);
+	if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+		unsigned long bypass;
+
+		is = map->_dm_cookie;
+		bypass = is->is_hw->ihw_bypass;
+
+		/* Bypass translation by the IOMMU. */
+		for (i = 0; i < nsegs; i++) {
+			map->dm_segs[i].ds_addr = bypass | segs[i].ds_addr;
+			map->dm_segs[i].ds_len = segs[i].ds_len;
+		}
+
+		map->dm_nsegs = nsegs;
+		map->dm_mapsize = size;
+
+		return (0);
 	}
 
+	ims = map->_dm_cookie;
+	is = ims->ims_sb->sb_iommu;
+
 	/*
 	 * A boundary presented to bus_dmamem_alloc() takes precedence
 	 * over boundary in the map.
@@ -1088,11 +1103,6 @@ iommu_dvmamap_append_range(bus_dma_tag_t
 	bus_dma_segment_t *seg = NULL;
 	int i = map->dm_nsegs;
 
-#ifdef DEBUG
-	if (ims == NULL)
-		panic("iommu_dvmamap_append_range: null map state");
-#endif
-
 	sgstart = iommu_iomap_translate(ims, pa);
 	sgend = sgstart + length - 1;
 
@@ -1298,20 +1308,17 @@ void
 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
 {
 	struct iommu_state *is;
-	struct iommu_map_state *ims = map->_dm_cookie;
+	struct iommu_map_state *ims;
 	bus_addr_t dvmaddr = map->_dm_dvmastart;
 	bus_size_t sgsize = map->_dm_dvmasize;
 	int error;
 
-#ifdef DEBUG
-	if (ims == NULL)
-		panic("iommu_dvmamap_unload: null map state");
-	if (ims->ims_sb == NULL)
-		panic("iommu_dvmamap_unload: null sb");
-	if (ims->ims_sb->sb_iommu == NULL)
-		panic("iommu_dvmamap_unload: null iommu");
-#endif /* DEBUG */
+	if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+		bus_dmamap_unload(t->_parent, map);
+		return;
+	}
 
+	ims = map->_dm_cookie;
 	is = ims->ims_sb->sb_iommu;
 
 	/* Flush the iommu */
@@ -1488,7 +1495,7 @@ iommu_dvmamap_print_map(bus_dma_tag_t t,
 		break;
 	}
 
-	if (map->_dm_cookie) {
+	if (!ISSET(map->_dm_flags, BUS_DMA_64BIT) && map->_dm_cookie != NULL) {
 		struct iommu_map_state *ims = map->_dm_cookie;
 		struct iommu_page_map *ipm = &ims->ims_map;
 
@@ -1546,19 +1553,19 @@ void
 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
     bus_addr_t offset, bus_size_t len, int ops)
 {
-	struct iommu_map_state *ims = map->_dm_cookie;
+	struct iommu_map_state *ims;
 
-#ifdef DIAGNOSTIC
-	if (ims == NULL)
-		panic("iommu_dvmamap_sync: null map state");
-	if (ims->ims_sb == NULL)
-		panic("iommu_dvmamap_sync: null sb");
-	if (ims->ims_sb->sb_iommu == NULL)
-		panic("iommu_dvmamap_sync: null iommu");
-#endif
 	if (len == 0)
 		return;
 
+	if (map->_dm_flags & BUS_DMA_64BIT) {
+		if (ops & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD))
+			membar(MemIssue);
+		return;
+	}
+
+	ims = map->_dm_cookie;
+
 	if (ops & BUS_DMASYNC_PREWRITE)
 		membar(MemIssue);
 
@@ -1622,9 +1629,13 @@ iommu_dvmamem_alloc(bus_dma_tag_t t, bus
 	    "bound %llx segp %p flags %d\n", (unsigned long long)size,
 	    (unsigned long long)alignment, (unsigned long long)boundary,
 	    segs, flags));
+
+	if ((flags & BUS_DMA_64BIT) == 0)
+		flags |= BUS_DMA_DVMA;
+
 	BUS_DMA_FIND_PARENT(t, _dmamem_alloc);
 	return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary,
-	    segs, nsegs, rsegs, flags | BUS_DMA_DVMA));
+	    segs, nsegs, rsegs, flags));
 }
 
 void
@@ -1763,7 +1774,7 @@ iommu_iomap_load_map(struct iommu_state 
 
 		/* Flush cache if necessary. */
 		slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize);
-		if (is->is_flags & IOMMU_FLUSH_CACHE &&
+		if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) &&
 		    (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7))
 			IOMMUREG_WRITE(is, iommu_cache_flush,
 			    is->is_ptsb + slot * 8);
@@ -1788,7 +1799,7 @@ iommu_iomap_unload_map(struct iommu_stat
 
 		/* Flush cache if necessary. */
 		slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize);
-		if (is->is_flags & IOMMU_FLUSH_CACHE &&
+		if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) &&
 		    (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7))
 			IOMMUREG_WRITE(is, iommu_cache_flush,
 			    is->is_ptsb + slot * 8);
Index: sparc64/dev/iommuvar.h
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/iommuvar.h,v
retrieving revision 1.17
diff -u -p -r1.17 iommuvar.h
--- sparc64/dev/iommuvar.h	4 May 2016 18:26:12 -0000	1.17
+++ sparc64/dev/iommuvar.h	8 Feb 2017 13:35:07 -0000
@@ -100,6 +100,21 @@ struct iommu_map_state {
 };
 #define IOMMU_MAP_STREAM	1
 
+struct iommu_hw {
+	void			(*ihw_enable)(struct iommu_state *);
+
+	unsigned long		ihw_dvma_pa;
+
+	unsigned long		ihw_bypass;
+	unsigned long		ihw_bypass_nc;		/* non-cached */
+	unsigned long		ihw_bypass_ro;		/* relaxed ordering */
+
+	unsigned int		ihw_flags;
+#define IOMMU_HW_FLUSH_CACHE		(1 << 0)
+};
+
+extern const struct iommu_hw iommu_hw_default;
+
 /*
  * per-IOMMU state
  */
@@ -112,8 +127,7 @@ struct iommu_state {
 	int64_t			is_cr;		/* Control register value */
 	struct mutex		is_mtx;
 	struct extent		*is_dvmamap;	/* DVMA map for this instance */
-	int			is_flags;
-#define IOMMU_FLUSH_CACHE	0x00000001
+	const struct iommu_hw	*is_hw;
 
 	struct strbuf_ctl	*is_sb[2];	/* Streaming buffers if any */
 
@@ -126,7 +140,8 @@ struct iommu_state {
 };
 
 /* interfaces for PCI/SBus code */
-void	iommu_init(char *, struct iommu_state *, int, u_int32_t);
+void	iommu_init(char *, const struct iommu_hw *, struct iommu_state *,
+    int, u_int32_t);
 void	iommu_reset(struct iommu_state *);
 paddr_t iommu_extract(struct iommu_state *, bus_addr_t);
 int64_t iommu_lookup_tte(struct iommu_state *, bus_addr_t);
@@ -146,6 +161,7 @@ int	iommu_dvmamem_alloc(bus_dma_tag_t, b
 	    bus_size_t, bus_size_t, bus_dma_segment_t *, int, int *, int);
 void	iommu_dvmamem_free(bus_dma_tag_t, bus_dma_tag_t, bus_dma_segment_t *,
 	    int);
+
 
 #define IOMMUREG_READ(is, reg)				\
 	bus_space_read_8((is)->is_bustag,		\
Index: sparc64/dev/psycho.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/psycho.c,v
retrieving revision 1.74
diff -u -p -r1.74 psycho.c
--- sparc64/dev/psycho.c	23 Aug 2016 03:28:01 -0000	1.74
+++ sparc64/dev/psycho.c	8 Feb 2017 13:35:07 -0000
@@ -902,7 +902,7 @@ psycho_iommu_init(struct psycho_softc *s
 		panic("couldn't malloc iommu name");
 	snprintf(name, 32, "%s dvma", sc->sc_dev.dv_xname);
 
-	iommu_init(name, is, tsbsize, iobase);
+	iommu_init(name, &iommu_hw_default, is, tsbsize, iobase);
 }
 
 /*
Index: sparc64/dev/pyro.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/pyro.c,v
retrieving revision 1.30
diff -u -p -r1.30 pyro.c
--- sparc64/dev/pyro.c	20 Dec 2016 13:40:50 -0000	1.30
+++ sparc64/dev/pyro.c	8 Feb 2017 13:35:07 -0000
@@ -131,6 +131,30 @@ int pyro_msi_eq_intr(void *);
 int pyro_dmamap_create(bus_dma_tag_t, bus_dma_tag_t, bus_size_t, int,
     bus_size_t, bus_size_t, int, bus_dmamap_t *);
 
+void pyro_iommu_enable(struct iommu_state *);
+
+const struct iommu_hw iommu_hw_pyro = {
+	.ihw_enable	= pyro_iommu_enable,
+
+	.ihw_dvma_pa	= 0x7ffffffffffUL,
+
+	.ihw_bypass	= 0xfffc000000000000UL,
+	.ihw_bypass_nc	= 1UL << 43,
+	.ihw_bypass_ro	= 0,
+};
+
+const struct iommu_hw iommu_hw_oberon = {
+	.ihw_enable	= pyro_iommu_enable,
+
+	.ihw_dvma_pa	= 0x7fffffffffUL,
+
+	.ihw_bypass	= 0x7ffc000000000000UL,
+	.ihw_bypass_nc	= 1UL << 47,
+	.ihw_bypass_ro	= 0x8000000000000000UL,
+
+	.ihw_flags	= IOMMU_HW_FLUSH_CACHE,
+};
+
 #ifdef DDB
 void pyro_xir(void *, int);
 #endif
@@ -266,6 +290,7 @@ pyro_init_iommu(struct pyro_softc *sc, s
 	int tsbsize = 7;
 	u_int32_t iobase = -1;
 	char *name;
+	const struct iommu_hw *ihw = &iommu_hw_pyro;
 
 	is->is_bustag = sc->sc_bust;
 
@@ -282,11 +307,23 @@ pyro_init_iommu(struct pyro_softc *sc, s
 		panic("couldn't malloc iommu name");
 	snprintf(name, 32, "%s dvma", sc->sc_dv.dv_xname);
 
-	/* On Oberon, we need to flush the cache. */
 	if (sc->sc_oberon)
-		is->is_flags |= IOMMU_FLUSH_CACHE;
+		ihw = &iommu_hw_oberon;
+
+	iommu_init(name, ihw, is, tsbsize, iobase);
+}
+
+void
+pyro_iommu_enable(struct iommu_state *is)
+{
+	unsigned long cr;
+
+	cr = IOMMUREG_READ(is, iommu_cr);
+	cr |= IOMMUCR_FIRE_BE | IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
+	    IOMMUCR_FIRE_TE;
 
-	iommu_init(name, is, tsbsize, iobase);
+	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb | is->is_tsbsize);
+	IOMMUREG_WRITE(is, iommu_cr, cr);
 }
 
 void
Index: sparc64/dev/sbus.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/sbus.c,v
retrieving revision 1.44
diff -u -p -r1.44 sbus.c
--- sparc64/dev/sbus.c	19 Sep 2015 21:07:04 -0000	1.44
+++ sparc64/dev/sbus.c	8 Feb 2017 13:35:07 -0000
@@ -349,7 +349,7 @@ sbus_mb_attach(struct device *parent, st
 	snprintf(name, 32, "%s dvma", sc->sc_dev.dv_xname);
 
 	printf("%s: ", sc->sc_dev.dv_xname);
-	iommu_init(name, &sc->sc_is, 0, -1);
+	iommu_init(name, &iommu_hw_default, &sc->sc_is, 0, -1);
 
 	/* Initialize Starfire PC interrupt translation. */
 	if (OF_getprop(findroot(), "name", buf, sizeof(buf)) > 0 &&
Index: sparc64/dev/schizo.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/schizo.c,v
retrieving revision 1.67
diff -u -p -r1.67 schizo.c
--- sparc64/dev/schizo.c	23 Aug 2016 03:28:01 -0000	1.67
+++ sparc64/dev/schizo.c	8 Feb 2017 13:35:07 -0000
@@ -451,7 +451,7 @@ schizo_init_iommu(struct schizo_softc *s
 		    "using iobase=0x%x, tsbsize=%d\n", iobase, tsbsize));
 	}
 
-	iommu_init(name, is, tsbsize, iobase);
+	iommu_init(name, &iommu_hw_default, is, tsbsize, iobase);
 }
 
 int
Index: sparc64/include/bus.h
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/include/bus.h,v
retrieving revision 1.30
diff -u -p -r1.30 bus.h
--- sparc64/include/bus.h	4 May 2016 18:26:12 -0000	1.30
+++ sparc64/include/bus.h	8 Feb 2017 13:35:07 -0000
@@ -369,7 +369,8 @@ bus_space_barrier(t, h, o, s, f)
 #define	BUS_DMA_READ		0x0200	/* mapping is device -> memory only */
 #define	BUS_DMA_WRITE		0x0400	/* mapping is memory -> device only */
 #define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
-#define BUS_DMA_OVERRUN		0x1000  /* tolerate DMA overruns */
+#define	BUS_DMA_OVERRUN		0x1000  /* tolerate DMA overruns */
+#define	BUS_DMA_64BIT		0x2000  /* may bypass IOMMU */
 
 #define	BUS_DMA_NOCACHE		BUS_DMA_BUS1
 #define	BUS_DMA_DVMA		BUS_DMA_BUS2	/* Don't bother with alignment */