Index: arch/arm64/conf/GENERIC =================================================================== RCS file: /cvs/src/sys/arch/arm64/conf/GENERIC,v retrieving revision 1.193 diff -u -p -r1.193 GENERIC --- arch/arm64/conf/GENERIC 8 Mar 2021 20:56:10 -0000 1.193 +++ arch/arm64/conf/GENERIC 18 Apr 2021 00:42:50 -0000 @@ -85,7 +85,8 @@ sdmmc* at sdhc? bwfm* at sdmmc? # Broadcom FullMAC #smmu* at fdt? xhci* at fdt? -ccp* at fdt? # AMD Cryptographic Co-processor +ccpbus* at fdt? # AMD Cryptographic Co-processor +ccp* at ccpbus? ipmi* at fdt? # NS16550 compatible serial ports @@ -136,6 +137,8 @@ aplintc* at fdt? early 1 aplpcie* at fdt? pci* at aplpcie? exuart* at fdt? +aplns* at fdt? # Apple NVME Storage controllers +nvme* at aplns? # iMX imxccm* at fdt? early 1 Index: arch/arm64/conf/files.arm64 =================================================================== RCS file: /cvs/src/sys/arch/arm64/conf/files.arm64,v retrieving revision 1.38 diff -u -p -r1.38 files.arm64 --- arch/arm64/conf/files.arm64 28 Feb 2021 21:40:11 -0000 1.38 +++ arch/arm64/conf/files.arm64 18 Apr 2021 00:42:50 -0000 @@ -152,6 +152,12 @@ device aplpcie: pcibus attach aplpcie at fdt file arch/arm64/dev/aplpcie.c aplpcie +# Apple NVME Storage +device aplns {} +attach aplns at fdt +attach nvme at aplns with nvme_ans +file arch/arm64/dev/aplns.c aplns | nvme_ans + device bcmintc attach bcmintc at fdt file arch/arm64/dev/bcm2836_intr.c bcmintc Index: arch/arm64/dev/aplns.c =================================================================== RCS file: arch/arm64/dev/aplns.c diff -N arch/arm64/dev/aplns.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ arch/arm64/dev/aplns.c 18 Apr 2021 00:42:50 -0000 @@ -0,0 +1,288 @@ +/* $OpenBSD$ */ +/* + * Copyright (c) 2014, 2021 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include + +#define ANS_MODESEL_REG 0x01304 +#define ANS_LINEAR_ASQ_DB 0x2490c +#define ANS_LINEAR_IOSQ_DB 0x24910 +#define ANS_NVMMU_TCB_INVAL 0x28118 +#define ANS_NVMMU_TCB_STAT 0x28120 +#define ANS_NVMMU_NUM 0x28100 +#define ANS_NVMMU_BASE_ASQ 0x28108 +#define ANS_NVMMU_BASE_IOSQ 0x28110 +#define ANS_LINEAR_BAR_SIZE 0x40000 + +#define ANS_NVMMU_TCB_SIZE 0x4000 +#define ANS_NVMMU_TCB_PITCH 0x80 + +struct ans_nvmmu_tcb { + uint8_t tcb_opcode; + uint8_t tcb_flags; +#define ANS_NVMMU_TCB_WRITE (1 << 0) +#define ANS_NVMMU_TCB_READ (1 << 1) + uint8_t tcb_cid; + uint8_t tcb_pad0[1]; + + uint32_t tcb_prpl_len; + uint8_t tcb_pad1[16]; + + uint64_t tcb_prp[2]; +}; + +int aplns_match(struct device *, void *, void *); +void aplns_attach(struct device *, struct device *, void *); + +struct cfattach aplns_ca = { + sizeof(struct device), + aplns_match, + aplns_attach +}; + +struct cfdriver aplns_cd = { + NULL, "aplns", DV_DULL +}; + +int +aplns_match(struct device *parent, void *match, void *aux) +{ + struct fdt_attach_args *faa = aux; + + return (OF_is_compatible(faa->fa_node, "apple,nvme-m1")); +} + +void +aplns_attach(struct device *parent, struct device *self, void *aux) +{ + struct fdt_attach_args *faa = aux; + + printf("\n"); + + config_found(self, faa, NULL); +} + +struct nvme_ans_softc { + struct nvme_softc asc_nvme; + + struct nvme_dmamem *asc_nvmmu; +}; + +int nvme_ans_match(struct device *, void *, void *); +void nvme_ans_attach(struct device *, struct device *, void *); + +struct cfattach nvme_ans_ca = { + sizeof(struct nvme_ans_softc), + nvme_ans_match, + nvme_ans_attach, +}; + +void nvme_ans_enable(struct nvme_softc *); + +int nvme_ans_q_alloc(struct nvme_softc *, + struct nvme_queue *); +void nvme_ans_q_free(struct nvme_softc *, + struct nvme_queue *); + +uint32_t nvme_ans_sq_enter(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +void nvme_ans_sq_leave(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + +static const struct nvme_ops nvme_ans_ops = { + .op_enable = nvme_ans_enable, + + .op_q_alloc = nvme_ans_q_alloc, + .op_q_free = nvme_ans_q_free, + + .op_sq_enter = nvme_ans_sq_enter, + .op_sq_leave = nvme_ans_sq_leave, + .op_sq_enter_locked = nvme_ans_sq_enter, + .op_sq_leave_locked = nvme_ans_sq_leave, +}; + +int +nvme_ans_match(struct device *parent, void *match, void *aux) +{ + struct fdt_attach_args *faa = aux; + + return (OF_is_compatible(faa->fa_node, "apple,nvme-m1")); +} + +void +nvme_ans_attach(struct device *parent, struct device *self, void *aux) +{ + struct nvme_ans_softc *asc = (struct nvme_ans_softc *)self; + struct nvme_softc *sc = &asc->asc_nvme; + struct fdt_attach_args *faa = aux; + + if (bus_space_map(faa->fa_iot, + faa->fa_reg[0].addr, faa->fa_reg[0].size, + 0, &sc->sc_ioh) != 0) { + printf(": unable to map registers\n"); + return; + } + + sc->sc_ih = fdt_intr_establish(faa->fa_node, IPL_BIO, + nvme_intr, sc, sc->sc_dev.dv_xname); + if (sc->sc_ih == NULL) { + printf(": unable to establish interrupt\n"); + goto unmap; + } + + sc->sc_dmat = faa->fa_dmat; + sc->sc_iot = faa->fa_iot; + sc->sc_ios = faa->fa_reg[0].size; + sc->sc_ops = &nvme_ans_ops; + + + printf(":"); + if (nvme_attach(sc) != 0) { + /* error printed by nvme_attach() */ + goto disestablish; + } + + return; + +disestablish: + fdt_intr_disestablish(sc->sc_ih); + sc->sc_ih = NULL; + +unmap: + bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios); + sc->sc_ios = 0; +} + +int +nvme_ans_q_alloc(struct nvme_softc *sc, + struct nvme_queue *q) +{ + bus_size_t db = ANS_LINEAR_IOSQ_DB; + bus_size_t base = ANS_NVMMU_BASE_IOSQ; + + KASSERT(q->q_entries <= (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH)); + + q->q_nvmmu_dmamem = nvme_dmamem_alloc(sc, ANS_NVMMU_TCB_SIZE); + if (q->q_nvmmu_dmamem == NULL) + return (-1); + + memset(NVME_DMA_KVA(q->q_nvmmu_dmamem), + 0, NVME_DMA_LEN(q->q_nvmmu_dmamem)); + + switch (q->q_id) { + case NVME_IO_Q: + case NVME_HIB_Q: + break; + case NVME_ADMIN_Q: + db = ANS_LINEAR_ASQ_DB; + base = ANS_NVMMU_BASE_ASQ; + break; + default: + panic("unsupported queue id %u", q->q_id); + /* NOTREACHED */ + } + + q->q_sqtdbl = db; + + nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_PREWRITE); + nvme_write8(sc, base, NVME_DMA_DVA(q->q_nvmmu_dmamem)); + + return (0); +} + +void +nvme_ans_enable(struct nvme_softc *sc) +{ + nvme_write4(sc, ANS_NVMMU_NUM, + (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1); + nvme_write4(sc, ANS_MODESEL_REG, 0); +} + +void +nvme_ans_q_free(struct nvme_softc *sc, + struct nvme_queue *q) +{ + nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_POSTWRITE); + nvme_dmamem_free(sc, q->q_nvmmu_dmamem); +} + +uint32_t +nvme_ans_sq_enter(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + return (ccb->ccb_id); +} + +static inline struct ans_nvmmu_tcb * +nvme_ans_tcb(struct nvme_queue *q, unsigned int qid) +{ + caddr_t ptr = NVME_DMA_KVA(q->q_nvmmu_dmamem); + ptr += qid * ANS_NVMMU_TCB_PITCH; + return ((struct ans_nvmmu_tcb *)ptr); +} + +void +nvme_ans_sq_leave(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + unsigned int id = ccb->ccb_id; + struct nvme_sqe_io *sqe; + struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id); + + sqe = NVME_DMA_KVA(q->q_sq_dmamem); + sqe += id; + + bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), + ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE); + + memset(tcb, 0, sizeof(*tcb)); + tcb->tcb_opcode = sqe->opcode; + tcb->tcb_flags = ANS_NVMMU_TCB_WRITE | ANS_NVMMU_TCB_READ; + tcb->tcb_cid = id; + tcb->tcb_prpl_len = sqe->nlb; + tcb->tcb_prp[0] = sqe->entry.prp[0]; + tcb->tcb_prp[1] = sqe->entry.prp[1]; + + bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), + ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE); + + nvme_write4(sc, q->q_sqtdbl, id); +} + +void +nvme_ans_cq_done() Index: dev/ic/nvme.c =================================================================== RCS file: /cvs/src/sys/dev/ic/nvme.c,v retrieving revision 1.91 diff -u -p -r1.91 nvme.c --- dev/ic/nvme.c 25 Feb 2021 07:30:36 -0000 1.91 +++ dev/ic/nvme.c 18 Apr 2021 00:42:50 -0000 @@ -79,11 +79,6 @@ void nvme_q_submit(struct nvme_softc *, int nvme_q_complete(struct nvme_softc *, struct nvme_queue *); void nvme_q_free(struct nvme_softc *, struct nvme_queue *); -struct nvme_dmamem * - nvme_dmamem_alloc(struct nvme_softc *, size_t); -void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); -void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, int); - void nvme_scsi_cmd(struct scsi_xfer *); void nvme_minphys(struct buf *, struct scsi_link *); int nvme_scsi_probe(struct scsi_link *); @@ -117,15 +112,27 @@ void nvme_scsi_inquiry(struct scsi_xfer void nvme_scsi_capacity16(struct scsi_xfer *); void nvme_scsi_capacity(struct scsi_xfer *); -#define nvme_read4(_s, _r) \ - bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) -#define nvme_write4(_s, _r, _v) \ - bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) +uint32_t nvme_op_sq_enter(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +void nvme_op_sq_leave(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +uint32_t nvme_op_sq_enter_locked(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +void nvme_op_sq_leave_locked(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + +static const struct nvme_ops nvme_ops = { + .op_sq_enter = nvme_op_sq_enter, + .op_sq_leave = nvme_op_sq_leave, + .op_sq_enter_locked = nvme_op_sq_enter_locked, + .op_sq_leave_locked = nvme_op_sq_leave_locked, +}; + /* * Some controllers, at least Apple NVMe, always require split * transfers, so don't use bus_space_{read,write}_8() on LP64. */ -static inline u_int64_t +u_int64_t nvme_read8(struct nvme_softc *sc, bus_size_t r) { u_int64_t v; @@ -142,7 +149,7 @@ nvme_read8(struct nvme_softc *sc, bus_si return (v); } -static inline void +void nvme_write8(struct nvme_softc *sc, bus_size_t r, u_int64_t v) { u_int32_t *a = (u_int32_t *)&v; @@ -221,6 +228,9 @@ nvme_enable(struct nvme_softc *sc) if (ISSET(cc, NVME_CC_EN)) return (nvme_ready(sc, NVME_CSTS_RDY)); + if (sc->sc_ops->op_enable != NULL) + sc->sc_ops->op_enable(sc); + nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); @@ -280,6 +290,8 @@ nvme_attach(struct nvme_softc *sc) mtx_init(&sc->sc_ccb_mtx, IPL_BIO); SIMPLEQ_INIT(&sc->sc_ccb_list); scsi_iopool_init(&sc->sc_iopool, sc, nvme_ccb_get, nvme_ccb_put); + if (sc->sc_ops == NULL) + sc->sc_ops = &nvme_ops; reg = nvme_read4(sc, NVME_VS); if (reg == 0xffffffff) { @@ -879,6 +891,42 @@ nvme_scsi_free(struct scsi_link *link) free(identify, M_DEVBUF, sizeof(*identify)); } +uint32_t +nvme_op_sq_enter(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + mtx_enter(&q->q_sq_mtx); + return (nvme_op_sq_enter_locked(sc, q, ccb)); +} + +uint32_t +nvme_op_sq_enter_locked(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + return (q->q_sq_tail); +} + +void +nvme_op_sq_leave_locked(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + uint32_t tail; + + tail = ++q->q_sq_tail; + if (tail >= q->q_entries) + tail = 0; + q->q_sq_tail = tail; + nvme_write4(sc, q->q_sqtdbl, tail); +} + +void +nvme_op_sq_leave(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + nvme_op_sq_leave_locked(sc, q, ccb); + mtx_leave(&q->q_sq_mtx); +} + void nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, void (*fill)(struct nvme_softc *, struct nvme_ccb *, void *)) @@ -886,10 +934,7 @@ nvme_q_submit(struct nvme_softc *sc, str struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); u_int32_t tail; - mtx_enter(&q->q_sq_mtx); - tail = q->q_sq_tail; - if (++q->q_sq_tail >= q->q_entries) - q->q_sq_tail = 0; + tail = sc->sc_ops->op_sq_enter(sc, q, ccb); sqe += tail; @@ -901,8 +946,7 @@ nvme_q_submit(struct nvme_softc *sc, str bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); - nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); - mtx_leave(&q->q_sq_mtx); + sc->sc_ops->op_sq_leave(sc, q, ccb); } struct nvme_poll_state { @@ -1300,11 +1344,18 @@ nvme_q_alloc(struct nvme_softc *sc, u_in q->q_cq_head = 0; q->q_cq_phase = NVME_CQE_PHASE; + if (sc->sc_ops->op_q_alloc != NULL) { + if (sc->sc_ops->op_q_alloc(sc, q) != 0) + goto free_cq; + } + nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); return (q); +free_cq: + nvme_dmamem_free(sc, q->q_cq_dmamem); free_sq: nvme_dmamem_free(sc, q->q_sq_dmamem); free: @@ -1319,9 +1370,6 @@ nvme_q_reset(struct nvme_softc *sc, stru memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); - q->q_sqtdbl = NVME_SQTDBL(q->q_id, sc->sc_dstrd); - q->q_cqhdbl = NVME_CQHDBL(q->q_id, sc->sc_dstrd); - q->q_sq_tail = 0; q->q_cq_head = 0; q->q_cq_phase = NVME_CQE_PHASE; @@ -1337,6 +1385,10 @@ nvme_q_free(struct nvme_softc *sc, struc { nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); + + if (sc->sc_ops->op_q_alloc != NULL) + sc->sc_ops->op_q_free(sc, q); + nvme_dmamem_free(sc, q->q_cq_dmamem); nvme_dmamem_free(sc, q->q_sq_dmamem); free(q, M_DEVBUF, sizeof *q); @@ -1441,9 +1493,7 @@ nvme_hibernate_admin_cmd(struct nvme_sof u_int16_t flags; /* submit command */ - tail = q->q_sq_tail; - if (++q->q_sq_tail >= q->q_entries) - q->q_sq_tail = 0; + tail = sc->sc_ops->op_sq_enter_locked(sc, q, ccb); asqe += tail; bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), @@ -1453,7 +1503,8 @@ nvme_hibernate_admin_cmd(struct nvme_sof bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); - nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); + nvme_write4(sc, q->q_sqtdbl, tail); + sc->sc_ops->op_sq_leave_locked(sc, q, ccb); /* wait for completion */ acqe += q->q_cq_head; Index: dev/ic/nvmevar.h =================================================================== RCS file: /cvs/src/sys/dev/ic/nvmevar.h,v retrieving revision 1.20 diff -u -p -r1.20 nvmevar.h --- dev/ic/nvmevar.h 22 Jul 2020 13:16:04 -0000 1.20 +++ dev/ic/nvmevar.h 18 Apr 2021 00:42:50 -0000 @@ -56,6 +56,7 @@ struct nvme_queue { struct mutex q_cq_mtx; struct nvme_dmamem *q_sq_dmamem; struct nvme_dmamem *q_cq_dmamem; + struct nvme_dmamem *q_nvmmu_dmamem; /* for aplns(4) */ bus_size_t q_sqtdbl; /* submission queue tail doorbell */ bus_size_t q_cqhdbl; /* completion queue head doorbell */ u_int16_t q_id; @@ -69,9 +70,29 @@ struct nvme_namespace { struct nvm_identify_namespace *ident; }; +struct nvme_ops { + void (*op_enable)(struct nvme_softc *); + + int (*op_q_alloc)(struct nvme_softc *, + struct nvme_queue *); + void (*op_q_free)(struct nvme_softc *, + struct nvme_queue *); + + uint32_t (*op_sq_enter)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + void (*op_sq_leave)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + uint32_t (*op_sq_enter_locked)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + void (*op_sq_leave_locked)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +}; + struct nvme_softc { struct device sc_dev; + const struct nvme_ops *sc_ops; + bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; bus_size_t sc_ios; @@ -108,3 +129,17 @@ int nvme_intr(void *); int nvme_intr_intx(void *); #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname) + +#define nvme_read4(_s, _r) \ + bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) +#define nvme_write4(_s, _r, _v) \ + bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) + +u_int64_t + nvme_read8(struct nvme_softc *, bus_size_t); +void nvme_write8(struct nvme_softc *, bus_size_t, uint64_t); + +struct nvme_dmamem * + nvme_dmamem_alloc(struct nvme_softc *, size_t); +void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); +void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, int);