Index: sys/percpu.h =================================================================== RCS file: sys/percpu.h diff -N sys/percpu.h --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ sys/percpu.h 17 May 2016 11:50:49 -0000 @@ -0,0 +1,53 @@ +/* $OpenBSD$ */ + +/* + * Copyright (c) 2016 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef _SYS_PERCPU_H_ +#define _SYS_PERCPU_H_ + +struct cpumem; + +struct cpumem_iter { + unsigned int cpu; +}; + +struct cpumem *cpumem_alloc(size_t, int); +void cpumem_free(struct cpumem *, int, size_t); + +void *cpumem_enter(struct cpumem *); +void cpumem_leave(struct cpumem *, void *); + +void *cpumem_first(struct cpumem_iter *, struct cpumem *); +void *cpumem_next(struct cpumem_iter *, struct cpumem *); + +#define CPUMEM_FOREACH(_var, _iter, _cpumem) \ + for ((_var) = cpumem_first((_iter), (_cpumem)); \ + (_var) != NULL; \ + (_var) = cpumem_next((_iter), (_cpumem))) + +struct counters_ref { + uint64_t *gen; +}; + +struct cpumem *counters_alloc(unsigned int, int); +void counters_free(struct cpumem *, int, unsigned int); +uint64_t *counters_enter(struct counters_ref *, struct cpumem *); +void counters_leave(struct counters_ref *, struct cpumem *); +void counters_read(struct cpumem *, uint64_t *, unsigned int); +void counters_zero(struct cpumem *, unsigned int); + +#endif /* _SYS_PERCPU_H_ */ Index: kern/subr_percpu.c =================================================================== RCS file: kern/subr_percpu.c diff -N kern/subr_percpu.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ kern/subr_percpu.c 17 May 2016 11:50:49 -0000 @@ -0,0 +1,217 @@ +/* $OpenBSD$ */ + +/* + * Copyright (c) 2016 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#ifndef CACHELINESIZE +#define CACHELINESIZE 64 +#endif + +struct cpumem { + void *mem; +}; + +struct pool cpumem_pl; + +extern int ncpus; + +void +percpu_attach(void) +{ + pool_init(&cpumem_pl, sizeof(struct cpumem) * ncpus, 0, 0, + PR_WAITOK, "cpumem", NULL); + pool_setipl(&cpumem_pl, IPL_NONE); +} + +struct cpumem * +cpumem_alloc(size_t sz, int type) +{ + struct cpumem *cm; + unsigned int cpu; + + sz = roundup(sz, CACHELINESIZE); + + cm = pool_get(&cpumem_pl, PR_WAITOK); + + for (cpu = 0; cpu < ncpus; cpu++) + cm[cpu].mem = malloc(sz, type, M_WAITOK); + + return (cm); +} + +void +cpumem_free(struct cpumem *cm, int type, size_t sz) +{ + unsigned int cpu; + + sz = roundup(sz, CACHELINESIZE); + + for (cpu = 0; cpu < ncpus; cpu++) + free(cm[cpu].mem, type, sz); + + pool_put(&cpumem_pl, cm); +} + +void * +cpumem_enter(struct cpumem *cm) +{ + unsigned int cpu = CPU_INFO_UNIT(curcpu()); + + return (cm[cpu].mem); +} + +void +cpumem_leave(struct cpumem *cm, void *mem) +{ + KDASSERT(cpumem_enter(cm) == mem); +} + +void * +cpumem_first(struct cpumem_iter *i, struct cpumem *cm) +{ + i->cpu = 0; + + return (cm[0].mem); +} + +void * +cpumem_next(struct cpumem_iter *i, struct cpumem *cm) +{ + unsigned int cpu = ++i->cpu; + + if (cpu >= ncpus) + return (NULL); + + return (cm[cpu].mem); +} + +struct cpumem * +counters_alloc(unsigned int n, int type) +{ + struct cpumem *cm; + struct cpumem_iter cmi; + uint64_t *counters; + unsigned int i; + + KASSERT(n > 0); + + n++; /* add space for a generation number */ + cm = cpumem_alloc(n * sizeof(uint64_t), type); + + CPUMEM_FOREACH(counters, &cmi, cm) { + for (i = 0; i < n; i++) + counters[i] = 0; + } + + return (cm); +} + +void +counters_free(struct cpumem *cm, int type, unsigned int n) +{ + n++; + cpumem_free(cm, type, n * sizeof(uint64_t)); +} + +uint64_t * +counters_enter(struct counters_ref *ref, struct cpumem *cm) +{ + ref->gen = cpumem_enter(cm); + (*ref->gen)++; /* make the generation number odd */ + return (ref->gen + 1); +} + +void +counters_leave(struct counters_ref *ref, struct cpumem *cm) +{ + membar_producer(); + (*ref->gen)++; /* even the generation number off */ + cpumem_leave(cm, ref->gen); +} + +void +counters_read(struct cpumem *cm, uint64_t *output, unsigned int n) +{ + struct cpumem_iter cmi; + uint64_t *gen, *counters, *temp; + uint64_t enter, leave; + unsigned int i; + + for (i = 0; i < n; i++) + output[i] = 0; + + temp = mallocarray(n, sizeof(uint64_t), M_TEMP, M_WAITOK); + + gen = cpumem_first(&cmi, cm); + do { + counters = gen + 1; + + enter = *gen; + for (;;) { + /* the generation number is odd during an update */ + while (enter & 1) { + yield(); + membar_consumer(); + enter = *gen; + } + + for (i = 0; i < n; i++) + temp[i] = counters[i]; + + membar_consumer(); + leave = *gen; + + if (enter == leave) + break; + + enter = leave; + } + + for (i = 0; i < n; i++) + output[i] += temp[i]; + + gen = cpumem_next(&cmi, cm); + } while (gen != NULL); + + free(temp, M_TEMP, n * sizeof(uint64_t)); +} + +void +counters_zero(struct cpumem *cm, unsigned int n) +{ + struct cpumem_iter cmi; + uint64_t *counters; + unsigned int i; + + n++; /* zero the generation numbers too */ + + counters = cpumem_first(&cmi, cm); + do { + for (i = 0; i < n; i++) + counters[i] = 0; + + counters = cpumem_next(&cmi, cm); + } while (counters != NULL); +}