Index: arch/alpha/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/alpha/include/bus.h,v retrieving revision 1.30 diff -u -p -r1.30 bus.h --- arch/alpha/include/bus.h 3 May 2016 11:22:08 -0000 1.30 +++ arch/alpha/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -464,6 +464,7 @@ struct alpha_bus_space { #define BUS_DMA_READ 0x0200 /* mapping is device -> memory only */ #define BUS_DMA_WRITE 0x0400 /* mapping is memory -> device only */ #define BUS_DMA_ZERO 0x1000 /* zero memory in dmamem_alloc */ +#define BUS_DMA_64BIT 0x0000 /* * Private flags stored in the DMA map. Index: arch/amd64/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/amd64/include/bus.h,v retrieving revision 1.32 diff -u -p -r1.32 bus.h --- arch/amd64/include/bus.h 24 Jan 2015 15:13:55 -0000 1.32 +++ arch/amd64/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -525,6 +525,7 @@ extern const struct x86_bus_space_ops x8 #define BUS_DMA_WRITE 0x0400 /* mapping is memory -> device only */ #define BUS_DMA_NOCACHE 0x0800 /* map memory uncached */ #define BUS_DMA_ZERO 0x1000 /* zero memory in dmamem_alloc */ +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ struct mbuf; Index: arch/arm/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/arm/include/bus.h,v retrieving revision 1.16 diff -u -p -r1.16 bus.h --- arch/arm/include/bus.h 5 Oct 2016 07:44:24 -0000 1.16 +++ arch/arm/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -617,6 +617,7 @@ bs_c_8_proto(f); #define BUS_DMA_WRITE 0x200 /* mapping is memory -> device only */ #define BUS_DMA_NOCACHE 0x400 /* hint: map non-cached memory */ #define BUS_DMA_ZERO 0x800 /* dmamem_alloc returns zeroed mem */ +#define BUS_DMA_64BIT 0x1000 /* * Private flags stored in the DMA map. Index: arch/arm64/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/arm64/include/bus.h,v retrieving revision 1.1 diff -u -p -r1.1 bus.h --- arch/arm64/include/bus.h 17 Dec 2016 23:38:33 -0000 1.1 +++ arch/arm64/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -315,6 +315,7 @@ bus_space_barrier(bus_space_tag_t t, bus #define BUS_DMA_STREAMING 0x0400 /* hint: sequential, unidirectional */ #define BUS_DMA_ZERO 0x0800 /* zero memory in dmamem_alloc */ #define BUS_DMA_NOCACHE 0x1000 +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ struct mbuf; Index: arch/hppa/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/hppa/include/bus.h,v retrieving revision 1.31 diff -u -p -r1.31 bus.h --- arch/hppa/include/bus.h 14 Nov 2011 14:29:53 -0000 1.31 +++ arch/hppa/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -299,6 +299,7 @@ extern const struct hppa_bus_space_tag h #define BUS_DMA_READ 0x0400 /* mapping is device -> memory only */ #define BUS_DMA_WRITE 0x0800 /* mapping is memory -> device only */ #define BUS_DMA_ZERO 0x1000 /* zero memory in dmamem_alloc */ +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ struct mbuf; Index: arch/i386/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/i386/include/bus.h,v retrieving revision 1.66 diff -u -p -r1.66 bus.h --- arch/i386/include/bus.h 27 Feb 2015 18:59:43 -0000 1.66 +++ arch/i386/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -528,6 +528,7 @@ extern const struct i386_bus_space_ops i #define BUS_DMA_WRITE 0x0400 /* mapping is memory -> device only */ #define BUS_DMA_NOCACHE 0x0800 /* map memory uncached */ #define BUS_DMA_ZERO 0x1000 /* dmamem_alloc return zeroed mem */ +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ struct mbuf; Index: arch/landisk/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/landisk/include/bus.h,v retrieving revision 1.8 diff -u -p -r1.8 bus.h --- arch/landisk/include/bus.h 23 Mar 2011 16:54:35 -0000 1.8 +++ arch/landisk/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -434,6 +434,7 @@ struct _bus_space { #define BUS_DMA_WRITE 0x200 /* mapping is memory -> device only */ #define BUS_DMA_NOCACHE 0x400 /* hint: map non-cached memory */ #define BUS_DMA_ZERO 0x800 /* zero memory in dmamem_alloc */ +#define BUS_DMA_64BIT 0x1000 /* Forwards needed by prototypes below. */ struct mbuf; Index: arch/loongson/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/loongson/include/bus.h,v retrieving revision 1.6 diff -u -p -r1.6 bus.h --- arch/loongson/include/bus.h 24 May 2014 21:11:01 -0000 1.6 +++ arch/loongson/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -350,6 +350,7 @@ bus_space_barrier(bus_space_tag_t t, bus #define BUS_DMA_STREAMING 0x0400 /* hint: sequential, unidirectional */ #define BUS_DMA_ZERO 0x0800 /* zero memory in dmamem_alloc */ #define BUS_DMA_NOCACHE 0x1000 +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ struct mbuf; Index: arch/macppc/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/macppc/include/bus.h,v retrieving revision 1.25 diff -u -p -r1.25 bus.h --- arch/macppc/include/bus.h 3 May 2016 12:23:25 -0000 1.25 +++ arch/macppc/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -489,6 +489,7 @@ bus_space_copy_4(void *v, bus_space_hand #define BUS_DMA_STREAMING 0x0400 /* hint: sequential, unidirectional */ #define BUS_DMA_ZERO 0x0800 /* zero memory in dmamem_alloc */ #define BUS_DMA_NOCACHE 0x1000 /* map memory uncached */ +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ Index: arch/octeon/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/octeon/include/bus.h,v retrieving revision 1.6 diff -u -p -r1.6 bus.h --- arch/octeon/include/bus.h 29 Mar 2014 18:09:30 -0000 1.6 +++ arch/octeon/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -316,6 +316,7 @@ bus_space_barrier(bus_space_tag_t t, bus #define BUS_DMA_STREAMING 0x0400 /* hint: sequential, unidirectional */ #define BUS_DMA_ZERO 0x0800 /* zero memory in dmamem_alloc */ #define BUS_DMA_NOCACHE 0x1000 +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ struct mbuf; Index: arch/sgi/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/sgi/include/bus.h,v retrieving revision 1.26 diff -u -p -r1.26 bus.h --- arch/sgi/include/bus.h 10 Mar 2014 21:32:15 -0000 1.26 +++ arch/sgi/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -318,6 +318,7 @@ bus_space_barrier(bus_space_tag_t t, bus #define BUS_DMA_STREAMING 0x0400 /* hint: sequential, unidirectional */ #define BUS_DMA_ZERO 0x0800 /* zero memory in dmamem_alloc */ #define BUS_DMA_NOCACHE 0x1000 +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ struct mbuf; Index: arch/socppc/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/socppc/include/bus.h,v retrieving revision 1.11 diff -u -p -r1.11 bus.h --- arch/socppc/include/bus.h 24 Jan 2015 20:59:42 -0000 1.11 +++ arch/socppc/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -461,6 +461,7 @@ bus_space_copy_4(void *v, bus_space_hand #define BUS_DMA_STREAMING 0x0400 /* hint: sequential, unidirectional */ #define BUS_DMA_ZERO 0x0800 /* zero memory in dmamem_alloc */ #define BUS_DMA_NOCACHE 0x1000 /* map memory uncached */ +#define BUS_DMA_64BIT 0x2000 /* Forwards needed by prototypes below. */ Index: arch/sparc64/dev/iommu.c =================================================================== RCS file: /cvs/src/sys/arch/sparc64/dev/iommu.c,v retrieving revision 1.73 diff -u -p -r1.73 iommu.c --- arch/sparc64/dev/iommu.c 4 May 2016 18:26:12 -0000 1.73 +++ arch/sparc64/dev/iommu.c 13 Feb 2017 02:05:16 -0000 @@ -100,6 +100,25 @@ void iommu_iomap_clear_pages(struct iomm void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, bus_size_t, int); +void iommu_hw_enable(struct iommu_state *); + +const struct iommu_hw iommu_hw_default = { + .ihw_enable = iommu_hw_enable, + + .ihw_dvma_pa = IOTTE_PAMASK, + + .ihw_bypass = 0x3fffUL << 50, + .ihw_bypass_nc = 0, + .ihw_bypass_ro = 0, +}; + +void +iommu_hw_enable(struct iommu_state *is) +{ + IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb); + IOMMUREG_WRITE(is, iommu_cr, IOMMUCR_EN | (is->is_tsbsize << 16)); +} + /* * Initiate an STC entry flush. */ @@ -125,7 +144,8 @@ iommu_strbuf_flush(struct strbuf_ctl *sb * - create a private DVMA map. */ void -iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase) +iommu_init(char *name, const struct iommu_hw *ihw, struct iommu_state *is, + int tsbsize, u_int32_t iovabase) { psize_t size; vaddr_t va; @@ -149,13 +169,9 @@ iommu_init(char *name, struct iommu_stat * be hard-wired, so we read the start and size from the PROM and * just use those values. */ - if (strncmp(name, "pyro", 4) == 0) { - is->is_cr = IOMMUREG_READ(is, iommu_cr); - is->is_cr &= ~IOMMUCR_FIRE_BE; - is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN | - IOMMUCR_FIRE_TE); - } else - is->is_cr = IOMMUCR_EN; + + is->is_hw = ihw; + is->is_tsbsize = tsbsize; if (iovabase == (u_int32_t)-1) { is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); @@ -237,15 +253,6 @@ iommu_init(char *name, struct iommu_stat mtx_init(&is->is_mtx, IPL_HIGH); /* - * Set the TSB size. The relevant bits were moved to the TSB - * base register in the PCIe host bridges. - */ - if (strncmp(name, "pyro", 4) == 0) - is->is_ptsb |= is->is_tsbsize; - else - is->is_cr |= (is->is_tsbsize << 16); - - /* * Now actually start up the IOMMU. */ iommu_reset(is); @@ -262,10 +269,7 @@ iommu_reset(struct iommu_state *is) { int i; - IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb); - - /* Enable IOMMU */ - IOMMUREG_WRITE(is, iommu_cr, is->is_cr); + (*is->is_hw->ihw_enable)(is); for (i = 0; i < 2; ++i) { struct strbuf_ctl *sb = is->is_sb[i]; @@ -279,9 +283,6 @@ iommu_reset(struct iommu_state *is) if (sb->sb_flush) printf(", STC%d enabled", i); } - - if (is->is_flags & IOMMU_FLUSH_CACHE) - IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL); } /* @@ -433,7 +434,7 @@ iommu_extract(struct iommu_state *is, bu if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; - return (tte & IOTTE_PAMASK); + return (tte & is->is_hw->ihw_dvma_pa); } /* @@ -610,6 +611,12 @@ iommu_dvmamap_create(bus_dma_tag_t t, bu if (ret) return (ret); + if (ISSET(flags, BUS_DMA_64BIT)) { + map->_dm_cookie = sb->sb_iommu; + *dmamap = map; + return (0); + } + ims = iommu_iomap_create(atop(round_page(size))); if (ims == NULL) { @@ -641,8 +648,10 @@ iommu_dvmamap_destroy(bus_dma_tag_t t, b if (map->dm_nsegs) bus_dmamap_unload(t0, map); - if (map->_dm_cookie) - iommu_iomap_destroy(map->_dm_cookie); + if (!ISSET(map->_dm_flags, BUS_DMA_64BIT)) { + if (map->_dm_cookie) + iommu_iomap_destroy(map->_dm_cookie); + } map->_dm_cookie = NULL; BUS_DMA_FIND_PARENT(t, _dmamap_destroy); @@ -667,36 +676,36 @@ iommu_dvmamap_load(bus_dma_tag_t t, bus_ u_long dvmaddr, sgstart, sgend; bus_size_t align, boundary; struct iommu_state *is; - struct iommu_map_state *ims = map->_dm_cookie; + struct iommu_map_state *ims; pmap_t pmap; -#ifdef DIAGNOSTIC - if (ims == NULL) - panic("iommu_dvmamap_load: null map state"); -#endif -#ifdef DEBUG - if (ims->ims_sb == NULL) - panic("iommu_dvmamap_load: null sb"); - if (ims->ims_sb->sb_iommu == NULL) - panic("iommu_dvmamap_load: null iommu"); -#endif /* DEBUG */ - is = ims->ims_sb->sb_iommu; - - if (map->dm_nsegs) { - /* - * Is it still in use? _bus_dmamap_load should have taken care - * of this. - */ -#ifdef DIAGNOSTIC - panic("iommu_dvmamap_load: map still in use"); -#endif - bus_dmamap_unload(t0, map); - } - /* * Make sure that on error condition we return "no valid mappings". */ - map->dm_nsegs = 0; + KASSERTMSG(map->dm_nsegs == 0, "map still in use"); + + if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) { + unsigned long bypass; + int i; + + /* Bypass translation by the IOMMU. */ + + BUS_DMA_FIND_PARENT(t, _dmamap_load); + err = (*t->_dmamap_load)(t, t0, map, buf, buflen, p, flags); + if (err != 0) + return (err); + + is = map->_dm_cookie; + bypass = is->is_hw->ihw_bypass; + + for (i = 0; i < map->dm_nsegs; i++) + map->dm_segs[i].ds_addr |= bypass; + + return (0); + } + + ims = map->_dm_cookie; + is = ims->ims_sb->sb_iommu; if (buflen < 1 || buflen > map->_dm_size) { DPRINTF(IDB_BUSDMA, @@ -876,28 +885,31 @@ iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_size_t boundary, align; u_long dvmaddr, sgstart, sgend; struct iommu_state *is; - struct iommu_map_state *ims = map->_dm_cookie; + struct iommu_map_state *ims; -#ifdef DIAGNOSTIC - if (ims == NULL) - panic("iommu_dvmamap_load_raw: null map state"); -#endif -#ifdef DEBUG - if (ims->ims_sb == NULL) - panic("iommu_dvmamap_load_raw: null sb"); - if (ims->ims_sb->sb_iommu == NULL) - panic("iommu_dvmamap_load_raw: null iommu"); -#endif /* DEBUG */ - is = ims->ims_sb->sb_iommu; + KASSERTMSG(map->dm_nsegs == 0, "map stil in use"); - if (map->dm_nsegs) { - /* Already in use?? */ -#ifdef DIAGNOSTIC - panic("iommu_dvmamap_load_raw: map still in use"); -#endif - bus_dmamap_unload(t0, map); + if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) { + unsigned long bypass; + + is = map->_dm_cookie; + bypass = is->is_hw->ihw_bypass; + + /* Bypass translation by the IOMMU. */ + for (i = 0; i < nsegs; i++) { + map->dm_segs[i].ds_addr = bypass | segs[i].ds_addr; + map->dm_segs[i].ds_len = segs[i].ds_len; + } + + map->dm_nsegs = nsegs; + map->dm_mapsize = size; + + return (0); } + ims = map->_dm_cookie; + is = ims->ims_sb->sb_iommu; + /* * A boundary presented to bus_dmamem_alloc() takes precedence * over boundary in the map. @@ -1088,11 +1100,6 @@ iommu_dvmamap_append_range(bus_dma_tag_t bus_dma_segment_t *seg = NULL; int i = map->dm_nsegs; -#ifdef DEBUG - if (ims == NULL) - panic("iommu_dvmamap_append_range: null map state"); -#endif - sgstart = iommu_iomap_translate(ims, pa); sgend = sgstart + length - 1; @@ -1298,20 +1305,17 @@ void iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) { struct iommu_state *is; - struct iommu_map_state *ims = map->_dm_cookie; + struct iommu_map_state *ims; bus_addr_t dvmaddr = map->_dm_dvmastart; bus_size_t sgsize = map->_dm_dvmasize; int error; -#ifdef DEBUG - if (ims == NULL) - panic("iommu_dvmamap_unload: null map state"); - if (ims->ims_sb == NULL) - panic("iommu_dvmamap_unload: null sb"); - if (ims->ims_sb->sb_iommu == NULL) - panic("iommu_dvmamap_unload: null iommu"); -#endif /* DEBUG */ + if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) { + bus_dmamap_unload(t->_parent, map); + return; + } + ims = map->_dm_cookie; is = ims->ims_sb->sb_iommu; /* Flush the iommu */ @@ -1488,7 +1492,7 @@ iommu_dvmamap_print_map(bus_dma_tag_t t, break; } - if (map->_dm_cookie) { + if (!ISSET(map->_dm_flags, BUS_DMA_64BIT) && map->_dm_cookie != NULL) { struct iommu_map_state *ims = map->_dm_cookie; struct iommu_page_map *ipm = &ims->ims_map; @@ -1542,23 +1546,26 @@ _iommu_dvmamap_sync(bus_dma_tag_t t, bus iommu_strbuf_flush_done(ims); } +void jbus_stst_order(void); + void iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) { - struct iommu_map_state *ims = map->_dm_cookie; + struct iommu_map_state *ims; -#ifdef DIAGNOSTIC - if (ims == NULL) - panic("iommu_dvmamap_sync: null map state"); - if (ims->ims_sb == NULL) - panic("iommu_dvmamap_sync: null sb"); - if (ims->ims_sb->sb_iommu == NULL) - panic("iommu_dvmamap_sync: null iommu"); -#endif if (len == 0) return; + if (map->_dm_flags & BUS_DMA_64BIT) { + jbus_stst_order(); + if (ops & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD)) + membar(MemIssue); + return; + } + + ims = map->_dm_cookie; + if (ops & BUS_DMASYNC_PREWRITE) membar(MemIssue); @@ -1622,9 +1629,13 @@ iommu_dvmamem_alloc(bus_dma_tag_t t, bus "bound %llx segp %p flags %d\n", (unsigned long long)size, (unsigned long long)alignment, (unsigned long long)boundary, segs, flags)); + + if ((flags & BUS_DMA_64BIT) == 0) + flags |= BUS_DMA_DVMA; + BUS_DMA_FIND_PARENT(t, _dmamem_alloc); return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary, - segs, nsegs, rsegs, flags | BUS_DMA_DVMA)); + segs, nsegs, rsegs, flags)); } void @@ -1763,7 +1774,7 @@ iommu_iomap_load_map(struct iommu_state /* Flush cache if necessary. */ slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize); - if (is->is_flags & IOMMU_FLUSH_CACHE && + if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) && (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7)) IOMMUREG_WRITE(is, iommu_cache_flush, is->is_ptsb + slot * 8); @@ -1788,7 +1799,7 @@ iommu_iomap_unload_map(struct iommu_stat /* Flush cache if necessary. */ slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize); - if (is->is_flags & IOMMU_FLUSH_CACHE && + if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) && (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7)) IOMMUREG_WRITE(is, iommu_cache_flush, is->is_ptsb + slot * 8); Index: arch/sparc64/dev/iommuvar.h =================================================================== RCS file: /cvs/src/sys/arch/sparc64/dev/iommuvar.h,v retrieving revision 1.17 diff -u -p -r1.17 iommuvar.h --- arch/sparc64/dev/iommuvar.h 4 May 2016 18:26:12 -0000 1.17 +++ arch/sparc64/dev/iommuvar.h 13 Feb 2017 02:05:16 -0000 @@ -100,6 +100,21 @@ struct iommu_map_state { }; #define IOMMU_MAP_STREAM 1 +struct iommu_hw { + void (*ihw_enable)(struct iommu_state *); + + bus_addr_t ihw_dvma_pa; + + bus_addr_t ihw_bypass; + bus_addr_t ihw_bypass_nc; /* non-cached */ + bus_addr_t ihw_bypass_ro; /* relaxed ordering */ + + unsigned int ihw_flags; +#define IOMMU_HW_FLUSH_CACHE (1 << 0) +}; + +extern const struct iommu_hw iommu_hw_default; + /* * per-IOMMU state */ @@ -109,11 +124,9 @@ struct iommu_state { int is_tsbsize; /* 0 = 8K, ... */ u_int is_dvmabase; u_int is_dvmaend; - int64_t is_cr; /* Control register value */ struct mutex is_mtx; struct extent *is_dvmamap; /* DVMA map for this instance */ - int is_flags; -#define IOMMU_FLUSH_CACHE 0x00000001 + const struct iommu_hw *is_hw; struct strbuf_ctl *is_sb[2]; /* Streaming buffers if any */ @@ -126,7 +139,8 @@ struct iommu_state { }; /* interfaces for PCI/SBus code */ -void iommu_init(char *, struct iommu_state *, int, u_int32_t); +void iommu_init(char *, const struct iommu_hw *, struct iommu_state *, + int, u_int32_t); void iommu_reset(struct iommu_state *); paddr_t iommu_extract(struct iommu_state *, bus_addr_t); int64_t iommu_lookup_tte(struct iommu_state *, bus_addr_t); Index: arch/sparc64/dev/psycho.c =================================================================== RCS file: /cvs/src/sys/arch/sparc64/dev/psycho.c,v retrieving revision 1.74 diff -u -p -r1.74 psycho.c --- arch/sparc64/dev/psycho.c 23 Aug 2016 03:28:01 -0000 1.74 +++ arch/sparc64/dev/psycho.c 13 Feb 2017 02:05:16 -0000 @@ -902,7 +902,7 @@ psycho_iommu_init(struct psycho_softc *s panic("couldn't malloc iommu name"); snprintf(name, 32, "%s dvma", sc->sc_dev.dv_xname); - iommu_init(name, is, tsbsize, iobase); + iommu_init(name, &iommu_hw_default, is, tsbsize, iobase); } /* Index: arch/sparc64/dev/pyro.c =================================================================== RCS file: /cvs/src/sys/arch/sparc64/dev/pyro.c,v retrieving revision 1.30 diff -u -p -r1.30 pyro.c --- arch/sparc64/dev/pyro.c 20 Dec 2016 13:40:50 -0000 1.30 +++ arch/sparc64/dev/pyro.c 13 Feb 2017 02:05:16 -0000 @@ -131,6 +131,30 @@ int pyro_msi_eq_intr(void *); int pyro_dmamap_create(bus_dma_tag_t, bus_dma_tag_t, bus_size_t, int, bus_size_t, bus_size_t, int, bus_dmamap_t *); +void pyro_iommu_enable(struct iommu_state *); + +const struct iommu_hw iommu_hw_fire = { + .ihw_enable = pyro_iommu_enable, + + .ihw_dvma_pa = 0x7ffffffffffUL, + + .ihw_bypass = 0xfffc000000000000UL, + .ihw_bypass_nc = 1UL << 43, + .ihw_bypass_ro = 0, +}; + +const struct iommu_hw iommu_hw_oberon = { + .ihw_enable = pyro_iommu_enable, + + .ihw_dvma_pa = 0x7fffffffffUL, + + .ihw_bypass = 0x7ffc000000000000UL, + .ihw_bypass_nc = 1UL << 47, + .ihw_bypass_ro = 0x8000000000000000UL, + + .ihw_flags = IOMMU_HW_FLUSH_CACHE, +}; + #ifdef DDB void pyro_xir(void *, int); #endif @@ -266,6 +290,7 @@ pyro_init_iommu(struct pyro_softc *sc, s int tsbsize = 7; u_int32_t iobase = -1; char *name; + const struct iommu_hw *ihw = &iommu_hw_fire; is->is_bustag = sc->sc_bust; @@ -282,11 +307,24 @@ pyro_init_iommu(struct pyro_softc *sc, s panic("couldn't malloc iommu name"); snprintf(name, 32, "%s dvma", sc->sc_dv.dv_xname); - /* On Oberon, we need to flush the cache. */ if (sc->sc_oberon) - is->is_flags |= IOMMU_FLUSH_CACHE; + ihw = &iommu_hw_oberon; + + iommu_init(name, ihw, is, tsbsize, iobase); +} + +void +pyro_iommu_enable(struct iommu_state *is) +{ + uint64_t cr; - iommu_init(name, is, tsbsize, iobase); + cr = IOMMUREG_READ(is, iommu_cr); + cr |= IOMMUCR_FIRE_BE | IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN | + IOMMUCR_FIRE_TE; + + IOMMUREG_WRITE(is, iommu_cache_invalidate, ~0ULL); + IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb | is->is_tsbsize); + IOMMUREG_WRITE(is, iommu_cr, cr); } void Index: arch/sparc64/dev/sbus.c =================================================================== RCS file: /cvs/src/sys/arch/sparc64/dev/sbus.c,v retrieving revision 1.44 diff -u -p -r1.44 sbus.c --- arch/sparc64/dev/sbus.c 19 Sep 2015 21:07:04 -0000 1.44 +++ arch/sparc64/dev/sbus.c 13 Feb 2017 02:05:16 -0000 @@ -349,7 +349,7 @@ sbus_mb_attach(struct device *parent, st snprintf(name, 32, "%s dvma", sc->sc_dev.dv_xname); printf("%s: ", sc->sc_dev.dv_xname); - iommu_init(name, &sc->sc_is, 0, -1); + iommu_init(name, &iommu_hw_default, &sc->sc_is, 0, -1); /* Initialize Starfire PC interrupt translation. */ if (OF_getprop(findroot(), "name", buf, sizeof(buf)) > 0 && Index: arch/sparc64/dev/schizo.c =================================================================== RCS file: /cvs/src/sys/arch/sparc64/dev/schizo.c,v retrieving revision 1.67 diff -u -p -r1.67 schizo.c --- arch/sparc64/dev/schizo.c 23 Aug 2016 03:28:01 -0000 1.67 +++ arch/sparc64/dev/schizo.c 13 Feb 2017 02:05:16 -0000 @@ -451,7 +451,7 @@ schizo_init_iommu(struct schizo_softc *s "using iobase=0x%x, tsbsize=%d\n", iobase, tsbsize)); } - iommu_init(name, is, tsbsize, iobase); + iommu_init(name, &iommu_hw_default, is, tsbsize, iobase); } int Index: arch/sparc64/include/bus.h =================================================================== RCS file: /cvs/src/sys/arch/sparc64/include/bus.h,v retrieving revision 1.30 diff -u -p -r1.30 bus.h --- arch/sparc64/include/bus.h 4 May 2016 18:26:12 -0000 1.30 +++ arch/sparc64/include/bus.h 13 Feb 2017 02:05:16 -0000 @@ -370,6 +370,7 @@ bus_space_barrier(t, h, o, s, f) #define BUS_DMA_WRITE 0x0400 /* mapping is memory -> device only */ #define BUS_DMA_ZERO 0x0800 /* zero memory in dmamem_alloc */ #define BUS_DMA_OVERRUN 0x1000 /* tolerate DMA overruns */ +#define BUS_DMA_64BIT 0x2000 /* allow iommu bypass */ #define BUS_DMA_NOCACHE BUS_DMA_BUS1 #define BUS_DMA_DVMA BUS_DMA_BUS2 /* Don't bother with alignment */ Index: arch/sparc64/sparc64/locore.s =================================================================== RCS file: /cvs/src/sys/arch/sparc64/sparc64/locore.s,v retrieving revision 1.184 diff -u -p -r1.184 locore.s --- arch/sparc64/sparc64/locore.s 18 Oct 2016 00:43:57 -0000 1.184 +++ arch/sparc64/sparc64/locore.s 13 Feb 2017 02:05:16 -0000 @@ -7747,3 +7747,25 @@ _C_LABEL(dlflush_start): .xword dlflush4 .xword dlflush5 .xword 0 + +#define VIS_BLOCKSIZE 64 + + .data + .align VIS_BLOCKSIZE + .type sync_buf, #object +sync_buf: + .skip VIS_BLOCKSIZE + .size sync_buf, VIS_BLOCKSIZE + .text + +ENTRY(jbus_stst_order) + set sync_buf, %o1 + + rd %fprs, %o2 ! %o2 = saved fprs + or %o2, FPRS_FEF, %o3 + wr %g0, %o3, %fprs ! make sure fp is enabled + stda %f0, [%o1] 0xe0 ! ASI_BLOCK_COMMIT_PRIMARY + wr %o2, 0, %fprs ! restore fprs + + retl + membar #Sync Index: dev/pci/if_bge.c =================================================================== RCS file: /cvs/src/sys/dev/pci/if_bge.c,v retrieving revision 1.385 diff -u -p -r1.385 if_bge.c --- dev/pci/if_bge.c 13 Feb 2017 00:56:32 -0000 1.385 +++ dev/pci/if_bge.c 13 Feb 2017 02:05:16 -0000 @@ -1231,7 +1231,8 @@ bge_init_rx_ring_std(struct bge_softc *s for (i = 0; i < BGE_STD_RX_RING_CNT; i++) { if (bus_dmamap_create(sc->bge_dmatag, sc->bge_rx_std_len, 1, - sc->bge_rx_std_len, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, + sc->bge_rx_std_len, 0, + BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, &sc->bge_cdata.bge_rx_std_map[i]) != 0) { printf("%s: unable to create dmamap for slot %d\n", sc->bge_dev.dv_xname, i); @@ -1357,7 +1358,7 @@ bge_init_rx_ring_jumbo(struct bge_softc for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) { if (bus_dmamap_create(sc->bge_dmatag, BGE_JLEN, 4, BGE_JLEN, 0, - BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, + BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, &sc->bge_cdata.bge_rx_jumbo_map[i]) != 0) { printf("%s: unable to create dmamap for slot %d\n", sc->bge_dev.dv_xname, i); @@ -1506,7 +1507,8 @@ bge_init_tx_ring(struct bge_softc *sc) for (i = 0; i < BGE_TX_RING_CNT; i++) { if (bus_dmamap_create(sc->bge_dmatag, txmaxsegsz, - BGE_NTXSEG, txsegsz, 0, BUS_DMA_NOWAIT, &sc->bge_txdma[i])) + BGE_NTXSEG, txsegsz, 0, BUS_DMA_NOWAIT | BUS_DMA_64BIT, + &sc->bge_txdma[i])) return (ENOBUFS); } @@ -2961,7 +2963,7 @@ bge_attach(struct device *parent, struct DPRINTFN(5, ("bus_dmamem_create\n")); if (bus_dmamap_create(sc->bge_dmatag, sizeof(struct bge_ring_data), 1, sizeof(struct bge_ring_data), 0, - BUS_DMA_NOWAIT, &sc->bge_ring_map)) { + BUS_DMA_NOWAIT | BUS_DMA_64BIT, &sc->bge_ring_map)) { printf(": can't create dma map\n"); goto fail_4; } Index: dev/pci/if_bgereg.h =================================================================== RCS file: /cvs/src/sys/dev/pci/if_bgereg.h,v retrieving revision 1.130 diff -u -p -r1.130 if_bgereg.h --- dev/pci/if_bgereg.h 29 Nov 2015 20:19:35 -0000 1.130 +++ dev/pci/if_bgereg.h 13 Feb 2017 02:05:16 -0000 @@ -2785,9 +2785,15 @@ struct bge_ring_data { struct bge_gib bge_info; }; +static inline unsigned long +_dma_addr(unsigned long dva, const char *f, unsigned int l) +{ + printf("%s[%u]: 0x%016lx\n", f, l, dva); + return (dva); +} + #define BGE_RING_DMA_ADDR(sc, offset) \ - ((sc)->bge_ring_map->dm_segs[0].ds_addr + \ - offsetof(struct bge_ring_data, offset)) + _dma_addr(sc->bge_ring_map->dm_segs[0].ds_addr + offsetof(struct bge_ring_data, offset), __func__, __LINE__) /* * Number of DMA segments in a TxCB. Note that this is carefully