mcx0:0:mcx-queues:0 RQ SW prod: 326001345 RQ HW prod: 25281 RQ HW cons: 25281 RQ HW state: RDY SQ SW prod: 0 SQ SW cons: 0 SQ HW prod: 0 SQ HW cons: 0 SQ HW state: RDY CQ SW cons: 326001228 CQ HW prod: 7234241 CQ HW cons: 7232344 CQ HW diff: 1897 CQ HW notify: 7234139 CQ HW solicit: 7230143 CQ HW status: OK CQ HW state: fired CQ Doorbell 0: 7234124 CQ Doorbell 1: 275669580 CQ uval: 1183991830602055696 EQ SW cons: 34655865 EQ HW prod: 1101433 EQ HW cons: 1101433 EQ HW status: OK EQ HW state: armed mcx0:0:mcx-queues:0 RQ SW prod: 188405894 0x000000000b3ad886 RQ HW prod: 55430 0x000000000000d886 RQ HW cons: 55430 0x000000000000d886 RQ HW state: RDY SQ SW prod: 0 0x0000000000000000 SQ SW cons: 0 0x0000000000000000 SQ HW prod: 0 0x0000000000000000 SQ HW cons: 0 0x0000000000000000 SQ HW state: RDY CQ SW cons: 188405780 0x0b3ad814 CQ SW cons next: 2068 0x00000814 CQ SW cons m: 3856404 0x003ad814 CQ SW cons owne: 1 0x00000001 CQ SW count: 23714608 0x0169db30 CQ SW count sn: 0 0x00000000 CQ HW prod: 3856518 0x003ad886 CQ HW prod m: 2182 0x00000886 CQ HW cons: 3854579 0x003ad0f3 CQ HW cons m: 243 0x000000f3 CQ HW diff: 1939 0x00000793 CQ HW notify: 3856404 0x003ad814 CQ HW notify m: 2068 0x00000814 CQ HW solicit: 3852420 0x003ac884 CQ HW solicit m: 2180 0x00000884 CQ HW status: OK CQ HW state: fired CQ DB[0]: 3856404 0x003ad814 CQ DB[1]: 3856404 0x003ad814 CQ DB[1] arm: 3856404 0x003ad814 CQ DB[1] sn: 0 0x00000000 CQ uval: 16563129060163600 0x003ad81400000010 CQ uval hi: 3856404 0x003ad814 CQ uval lo: 16 0x00000010 EQ SW cons: 23714608 0x000000000169db30 EQ HW prod: 6937392 0x000000000069db30 EQ HW cons: 6937392 0x000000000069db30 EQ HW status: OK EQ HW state: armed Index: if_mcx.c =================================================================== RCS file: /cvs/src/sys/dev/pci/if_mcx.c,v retrieving revision 1.75 diff -u -p -r1.75 if_mcx.c --- if_mcx.c 6 Nov 2020 02:50:02 -0000 1.75 +++ if_mcx.c 24 Dec 2020 01:08:02 -0000 @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -2171,6 +2172,8 @@ struct mcx_cq { int cq_n; struct mcx_dmamem cq_mem; uint32_t *cq_doorbell; + bus_addr_t cq_doorbell_offset; + uint64_t cq_uval; uint32_t cq_cons; uint32_t cq_count; }; @@ -2226,6 +2229,9 @@ struct mcx_queues { struct mcx_tx q_tx; struct mcx_cq q_cq; struct mcx_eq q_eq; + + uint64_t q_interrupts; + uint64_t q_events;; #if NKSTAT > 0 struct kstat *q_kstat; #endif @@ -2485,6 +2491,7 @@ static void mcx_process_cq(struct mcx_so struct mcx_cq *); static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int); +static void mcx_update_cq_ci(struct mcx_softc *, struct mcx_cq *); static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int); static int mcx_admin_intr(void *); static int mcx_cq_intr(void *); @@ -2842,15 +2849,16 @@ mcx_attach(struct device *parent, struct tx->tx_ifq = ifq; ifq->ifq_softc = tx; - if (pci_intr_map_msix(pa, i + 1, &ih) != 0) { + if (pci_intr_map_msix(pa, vec, &ih) != 0) { printf("%s: unable to map queue interrupt %d\n", DEVNAME(sc), i); goto teardown; } snprintf(q->q_name, sizeof(q->q_name), "%s:%d", DEVNAME(sc), i); - q->q_ihc = pci_intr_establish(sc->sc_pc, ih, - IPL_NET | IPL_MPSAFE, mcx_cq_intr, q, q->q_name); + q->q_ihc = pci_intr_establish_cpu(sc->sc_pc, ih, + IPL_NET | IPL_MPSAFE, intrmap_one(&sc->sc_dev), + mcx_cq_intr, q, q->q_name); } timeout_set(&sc->sc_calibrate, mcx_calibrate, sc); @@ -4382,12 +4390,16 @@ mcx_create_cq(struct mcx_softc *sc, stru int error; uint64_t *pas; int insize, npages, paslen, i, token; + caddr_t doorbell; npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), MCX_PAGE_SIZE); paslen = npages * sizeof(*pas); insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen; + cq->cq_doorbell_offset = MCX_CQ_DOORBELL_BASE + + (MCX_CQ_DOORBELL_STRIDE * db); + if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE, MCX_PAGE_SIZE) != 0) { printf("%s: unable to allocate completion queue memory\n", @@ -4423,8 +4435,7 @@ mcx_create_cq(struct mcx_softc *sc, stru (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) | MCX_CQ_MOD_COUNTER); mbin->cmd_cq_ctx.cq_doorbell = htobe64( - MCX_DMA_DVA(&sc->sc_doorbell_mem) + - MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db)); + MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell_offset); /* physical addresses follow the mailbox in data */ mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem); @@ -4448,11 +4459,21 @@ mcx_create_cq(struct mcx_softc *sc, stru goto free; } + doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); + doorbell += cq->cq_doorbell_offset; + cq->cq_n = mcx_get_id(out->cmd_cqn); cq->cq_cons = 0; cq->cq_count = 0; - cq->cq_doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem) + - MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db); + cq->cq_doorbell = (uint32_t *)doorbell; + + bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), + 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); + + bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), + cq->cq_doorbell_offset, 2 * sizeof(*cq->cq_doorbell), + BUS_DMASYNC_PREWRITE); + mcx_arm_cq(sc, cq, uar); free: @@ -4496,6 +4517,9 @@ mcx_destroy_cq(struct mcx_softc *sc, str return -1; } + bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), + 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); + cq->cq_n = 0; mcx_dmamem_free(sc, &cq->cq_mem); cq->cq_cons = 0; @@ -6577,6 +6601,7 @@ mcx_process_rx(struct mcx_softc *sc, str ms->ms_m = NULL; m->m_pkthdr.len = m->m_len = bemtoh32(&cqe->cq_byte_cnt); + m->m_pkthdr.ph_drops = cqe->cq_rx_drops; if (cqe->cq_rx_hash_type) { m->m_pkthdr.ph_flowid = betoh32(cqe->cq_rx_hash); @@ -6611,7 +6636,7 @@ mcx_process_rx(struct mcx_softc *sc, str return (1); } -static struct mcx_cq_entry * +static inline struct mcx_cq_entry * mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq) { struct mcx_cq_entry *cqe; @@ -6625,6 +6650,15 @@ mcx_next_cq_entry(struct mcx_softc *sc, return (&cqe[next]); } + bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), + 0, MCX_DMA_LEN(&cq->cq_mem), + BUS_DMASYNC_PREREAD|BUS_DMASYNC_POSTREAD); + + if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) == + ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) { + return (&cqe[next]); + } + return (NULL); } @@ -6639,18 +6673,32 @@ mcx_arm_cq(struct mcx_softc *sc, struct val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT; val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); + bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), + cq->cq_doorbell_offset, 2 * sizeof(*cq->cq_doorbell), + BUS_DMASYNC_POSTWRITE); + cq->cq_doorbell[0] = htobe32(cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); cq->cq_doorbell[1] = htobe32(val); - uval = val; - uval <<= 32; + bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), + cq->cq_doorbell_offset, 2 * sizeof(*cq->cq_doorbell), + BUS_DMASYNC_PREWRITE); + + uval = (uint64_t)val << 32; uval |= cq->cq_n; + cq->cq_uval = uval; bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, offset + MCX_UAR_CQ_DOORBELL, htobe64(uval)); mcx_bar(sc, offset + MCX_UAR_CQ_DOORBELL, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE); } +static void +mcx_update_cq_ci(struct mcx_softc *sc, struct mcx_cq *cq) +{ + cq->cq_doorbell[0] = htobe32(cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); +} + void mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq) { @@ -6667,6 +6715,9 @@ mcx_process_cq(struct mcx_softc *sc, str membar_consumer(); c = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; + bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), + 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); + rxfree = 0; txfree = 0; while ((cqe = mcx_next_cq_entry(sc, cq))) { @@ -6682,19 +6733,31 @@ mcx_process_cq(struct mcx_softc *sc, str case MCX_CQ_ENTRY_OPCODE_REQ_ERR: case MCX_CQ_ENTRY_OPCODE_SEND_ERR: cqp = (uint8_t *)cqe; - /* printf("%s: cq completion error: %x\n", - DEVNAME(sc), cqp[0x37]); */ + printf("%s: cq completion error: %x\n", DEVNAME(sc), + cqp[0x37]); break; default: + printf("%s: %s opcode %x\n", DEVNAME(sc), __func__, + opcode); /* printf("%s: cq completion opcode %x??\n", DEVNAME(sc), opcode); */ break; } cq->cq_cons++; + + /* + * occasionally update the consumer counter without rearming + * to prevent overflow, maybe + */ + if ((cq->cq_cons & 0xf) == 0) + mcx_update_cq_ci(sc, cq); } + bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), + 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); + cq->cq_count++; mcx_arm_cq(sc, cq, q->q_uar); @@ -6725,7 +6788,8 @@ mcx_arm_eq(struct mcx_softc *sc, struct val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff); mcx_wr(sc, offset, val); - /* barrier? */ + mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t), + BUS_SPACE_BARRIER_WRITE); } static struct mcx_eq_entry * @@ -6752,6 +6816,7 @@ mcx_admin_intr(void *xsc) while ((eqe = mcx_next_eq_entry(sc, &sc->sc_admin_eq))) { switch (eqe->eq_event_type) { +#if 0 case MCX_EVENT_TYPE_LAST_WQE: /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */ break; @@ -6759,6 +6824,7 @@ mcx_admin_intr(void *xsc) case MCX_EVENT_TYPE_CQ_ERROR: /* printf("%s: cq error\n", DEVNAME(sc)); */ break; +#endif case MCX_EVENT_TYPE_CMD_COMPLETION: /* wakeup probably */ @@ -6769,6 +6835,8 @@ mcx_admin_intr(void *xsc) break; default: + printf("%s: %s event type %x\n", DEVNAME(sc), __func__, + eqe->eq_event_type); /* printf("%s: something happened\n", DEVNAME(sc)); */ break; } @@ -6785,13 +6853,20 @@ mcx_cq_intr(void *xq) struct mcx_eq_entry *eqe; int cqn; + q->q_interrupts++; + while ((eqe = mcx_next_eq_entry(sc, &q->q_eq))) { + q->q_events++; switch (eqe->eq_event_type) { case MCX_EVENT_TYPE_COMPLETION: cqn = betoh32(eqe->eq_event_data[6]); if (cqn == q->q_cq.cq_n) mcx_process_cq(sc, q, &q->q_cq); break; + default: + printf("%s: %s event type %x\n", DEVNAME(sc), __func__, + eqe->eq_event_type); + break; } } @@ -8271,10 +8346,14 @@ static const struct mcx_queuestat mcx_qu { "CQ SW cons", KSTAT_KV_T_COUNTER64 }, { "CQ HW prod", KSTAT_KV_T_COUNTER64 }, { "CQ HW cons", KSTAT_KV_T_COUNTER64 }, + { "CQ HW diff", KSTAT_KV_T_UINT32 }, { "CQ HW notify", KSTAT_KV_T_COUNTER64 }, { "CQ HW solicit", KSTAT_KV_T_COUNTER64 }, { "CQ HW status", KSTAT_KV_T_ISTR }, { "CQ HW state", KSTAT_KV_T_ISTR }, + { "CQ Doorbell 0", KSTAT_KV_T_UINT32 }, + { "CQ Doorbell 1", KSTAT_KV_T_UINT32 }, + { "CQ uval", KSTAT_KV_T_UINT64 }, { "EQ SW cons", KSTAT_KV_T_COUNTER64 }, { "EQ HW prod", KSTAT_KV_T_COUNTER64 }, @@ -8398,6 +8477,8 @@ mcx_kstat_queue_read(struct kstat *ks) kstat_kv_u64(kvs++) = q->q_cq.cq_cons; kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter); kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter); + kstat_kv_u32(kvs++) = bemtoh32(&u.cq.cq_producer_counter) - + bemtoh32(&u.cq.cq_consumer_counter); kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified); kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit); @@ -8436,6 +8517,9 @@ mcx_kstat_queue_read(struct kstat *ks) } strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); kvs++; + kstat_kv_u32(kvs++) = betoh32(q->q_cq.cq_doorbell[0]); + kstat_kv_u32(kvs++) = betoh32(q->q_cq.cq_doorbell[1]); + kstat_kv_u64(kvs++) = q->q_cq.cq_uval; if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) { error = EIO; @@ -8475,6 +8559,10 @@ mcx_kstat_queue_read(struct kstat *ks) } strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); kvs++; + + kstat_kv_u64(kvs++) = q->q_interrupts; + kstat_kv_u64(kvs++) = q->q_events; + nanouptime(&ks->ks_updated); out: