--- /home/dlg/tmp/kern_intrmap.c.orig Thu Jun 11 13:07:37 2020 +++ kern_intrmap.c Thu Jun 11 13:02:00 2020 @@ -42,47 +42,133 @@ #include #include #include +#include #include -#define intrmap_ncpu ncpus +struct intrmap_cpus { + struct refcnt ic_refs; + unsigned int ic_count; + unsigned int *ic_cpumap; +}; struct intrmap { unsigned int im_count; unsigned int im_grid; + struct intrmap_cpus * + im_cpus; unsigned int *im_cpumap; }; +/* + * The CPUs that should be used for interrupts may be a subset of all CPUs. + */ + +struct rwlock intrmap_lock = RWLOCK_INITIALIZER("intrcpus"); +struct intrmap_cpus *intrmap_cpus = NULL; +int intrmap_ncpu = 0; + +static void +intrmap_cpus_put(struct intrmap_cpus *ic) +{ + if (ic == NULL) + return; + + if (refcnt_rele(&ic->ic_refs)) { + free(ic->ic_cpumap, M_DEVBUF, + ic->ic_count * sizeof(*ic->ic_cpumap)); + free(ic, M_DEVBUF, sizeof(*ic)); + } +} + +static struct intrmap_cpus * +intrmap_cpus_get(void) +{ + struct intrmap_cpus *oic = NULL; + struct intrmap_cpus *ic; + + rw_enter_write(&intrmap_lock); + if (intrmap_ncpu != ncpus) { + unsigned int icpus = 0; + unsigned int *cpumap; + CPU_INFO_ITERATOR cii; + struct cpu_info *ci; + + /* + * there's a new "version" of the set of CPUs available, so + * we need to figure out which ones we can use for interrupts. + */ + + cpumap = mallocarray(ncpus, sizeof(*cpumap), + M_DEVBUF, M_WAITOK); + + CPU_INFO_FOREACH(cii, ci) { +#ifdef __HAVE_CPU_TOPOLOGY + if (ci->ci_smt_id > 0) + continue; +#endif + cpumap[icpus++] = CPU_INFO_UNIT(ci); + } + + if (icpus < ncpus) { + /* this is mostly about free(9) needing a size */ + unsigned int *icpumap = mallocarray(icpus, + sizeof(*icpumap), M_DEVBUF, M_WAITOK); + memcpy(icpumap, cpumap, icpus * sizeof(*icpumap)); + free(cpumap, M_DEVBUF, ncpus * sizeof(*cpumap)); + cpumap = icpumap; + } + + ic = malloc(sizeof(*ic), M_DEVBUF, M_WAITOK); + refcnt_init(&ic->ic_refs); + ic->ic_count = icpus; + ic->ic_cpumap = cpumap; + + oic = intrmap_cpus; + intrmap_cpus = ic; /* give this ref to the global. */ + } else + ic = intrmap_cpus; + + refcnt_take(&ic->ic_refs); /* take a ref for the caller */ + rw_exit_write(&intrmap_lock); + + intrmap_cpus_put(oic); + + return (ic); +} + static int -intrmap_nintrs(unsigned int nintrs, unsigned int maxintrs) +intrmap_nintrs(const struct intrmap_cpus *ic, unsigned int nintrs, + unsigned int maxintrs) { KASSERTMSG(maxintrs > 0, "invalid maximum interrupt count %u", maxintrs); if (nintrs == 0 || nintrs > maxintrs) nintrs = maxintrs; - if (nintrs > intrmap_ncpu) - nintrs = intrmap_ncpu; + if (nintrs > ic->ic_count) + nintrs = ic->ic_count; return (nintrs); } static void -intrmap_set_grid(unsigned int unit, struct intrmap *im, unsigned int grid) +intrmap_set_grid(struct intrmap *im, unsigned int unit, unsigned int grid) { unsigned int i, offset; unsigned int *cpumap = im->im_cpumap; + const struct intrmap_cpus *ic = im->im_cpus; KASSERTMSG(grid > 0, "invalid if_ringmap grid %u", grid); KASSERTMSG(grid >= im->im_count, "invalid intrmap grid %u, count %u", grid, im->im_count); im->im_grid = grid; - offset = (grid * unit) % intrmap_ncpu; + offset = (grid * unit) % ic->ic_count; for (i = 0; i < im->im_count; i++) { cpumap[i] = offset + i; - KASSERTMSG(cpumap[i] < intrmap_ncpu, + KASSERTMSG(cpumap[i] < ic->ic_count, "invalid cpumap[%u] = %u, offset %u (ncpu %d)", i, - cpumap[i], offset, intrmap_ncpu); + cpumap[i], offset, ic->ic_count); } } @@ -93,31 +179,35 @@ struct intrmap *im; unsigned int unit = dv->dv_unit; unsigned int i, grid = 0, prev_grid; + struct intrmap_cpus *ic; - nintrs = intrmap_nintrs(nintrs, maxintrs); + ic = intrmap_cpus_get(); + + nintrs = intrmap_nintrs(ic, nintrs, maxintrs); if (ISSET(flags, INTRMAP_POWEROF2)) nintrs = 1 << (fls(nintrs) - 1); im = malloc(sizeof(*im), M_DEVBUF, M_WAITOK | M_ZERO); im->im_count = nintrs; + im->im_cpus = ic; im->im_cpumap = mallocarray(nintrs, sizeof(*im->im_cpumap), M_DEVBUF, M_WAITOK | M_ZERO); - prev_grid = intrmap_ncpu; - for (i = 0; i < intrmap_ncpu; i++) { - if (intrmap_ncpu % (i + 1) != 0) + prev_grid = ic->ic_count; + for (i = 0; i < ic->ic_count; i++) { + if (ic->ic_count % (i + 1) != 0) continue; - grid = intrmap_ncpu / (i + 1); + grid = ic->ic_count / (i + 1); if (nintrs > grid) { grid = prev_grid; break; } - if (nintrs > intrmap_ncpu / (i + 2)) + if (nintrs > ic->ic_count / (i + 2)) break; prev_grid = grid; } - intrmap_set_grid(unit, im, grid); + intrmap_set_grid(im, unit, grid); return (im); } @@ -126,6 +216,7 @@ intrmap_destroy(struct intrmap *im) { free(im->im_cpumap, M_DEVBUF, im->im_count * sizeof(*im->im_cpumap)); + intrmap_cpus_put(im->im_cpus); free(im, M_DEVBUF, sizeof(*im)); } @@ -152,10 +243,12 @@ { unsigned int unit = dv->dv_unit; + KASSERT(im0->im_cpus == im1->im_cpus); + if (im0->im_grid > im1->im_grid) - intrmap_set_grid(unit, im1, im0->im_grid); + intrmap_set_grid(im1, unit, im0->im_grid); else if (im0->im_grid < im1->im_grid) - intrmap_set_grid(unit, im0, im1->im_grid); + intrmap_set_grid(im0, unit, im1->im_grid); } void @@ -163,10 +256,12 @@ struct intrmap *im0, struct intrmap *im1) { unsigned int unit = dv->dv_unit; + const struct intrmap_cpus *ic; unsigned int subset_grid, cnt, divisor, mod, offset, i; struct intrmap *subset_im, *im; unsigned int old_im0_grid, old_im1_grid; + KASSERT(im0->im_cpus == im1->im_cpus); if (im0->im_grid == im1->im_grid) return; @@ -205,14 +300,16 @@ return; } + ic = im0->im_cpus; + mod = cnt / subset_grid; KASSERT(mod >= 2); - divisor = intrmap_ncpu / im->im_grid; + divisor = ic->ic_count / im->im_grid; offset = ((unit / divisor) % mod) * subset_grid; for (i = 0; i < subset_im->im_count; i++) { subset_im->im_cpumap[i] += offset; - KASSERTMSG(subset_im->im_cpumap[i] < intrmap_ncpu, + KASSERTMSG(subset_im->im_cpumap[i] < ic->ic_count, "match: invalid cpumap[%d] = %d, offset %d", i, subset_im->im_cpumap[i], offset); } @@ -240,6 +337,11 @@ unsigned int intrmap_cpu(const struct intrmap *im, unsigned int ring) { - KASSERTMSG(ring >= 0 && ring < im->im_count, "invalid ring %u", ring); - return (im->im_cpumap[ring]); + const struct intrmap_cpus *ic = im->im_cpus; + unsigned int icpu; + KASSERTMSG(ring < im->im_count, "invalid ring %u", ring); + icpu = im->im_cpumap[ring]; + KASSERTMSG(icpu < ic->ic_count, "invalid interrupt cpu %u for ring %u" + " (intrmap %p)", icpu, ring, im); + return (ic->ic_cpumap[icpu]); }