Index: conf/files
===================================================================
RCS file: /cvs/src/sys/conf/files,v
retrieving revision 1.632
diff -u -p -r1.632 files
--- conf/files	16 Sep 2016 19:13:17 -0000	1.632
+++ conf/files	21 Oct 2016 04:11:03 -0000
@@ -691,6 +691,7 @@ file kern/subr_evcount.c
 file kern/subr_extent.c
 file kern/subr_hibernate.c		hibernate
 file kern/subr_log.c
+file kern/subr_percpu.c
 file kern/subr_poison.c			diagnostic
 file kern/subr_pool.c
 file kern/subr_tree.c
Index: kern/init_main.c
===================================================================
RCS file: /cvs/src/sys/kern/init_main.c,v
retrieving revision 1.259
diff -u -p -r1.259 init_main.c
--- kern/init_main.c	22 Sep 2016 12:55:24 -0000	1.259
+++ kern/init_main.c	21 Oct 2016 04:11:03 -0000
@@ -146,6 +146,7 @@ void	kqueue_init(void);
 void	taskq_init(void);
 void	timeout_proc_init(void);
 void	pool_gc_pages(void *);
+void	percpu_init(void);
 
 extern char sigcode[], esigcode[], sigcoderet[];
 #ifdef SYSCALL_DEBUG
@@ -359,6 +360,9 @@ main(void *framep)
 
 	/* Configure virtual memory system, set vm rlimits. */
 	uvm_init_limits(p);
+
+	/* Per CPU memory allocation */
+	percpu_init();
 
 	/* Initialize the file systems. */
 #if defined(NFSSERVER) || defined(NFSCLIENT)
Index: kern/subr_percpu.c
===================================================================
RCS file: kern/subr_percpu.c
diff -N kern/subr_percpu.c
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ kern/subr_percpu.c	21 Oct 2016 04:11:03 -0000
@@ -0,0 +1,325 @@
+/*	$OpenBSD$ */
+
+/*
+ * Copyright (c) 2016 David Gwynne <dlg@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/pool.h>
+#include <sys/malloc.h>
+#include <sys/types.h>
+
+#include <sys/percpu.h>
+
+#ifdef MULTIPROCESSOR
+struct pool cpumem_pl;
+
+void
+percpu_init(void)
+{
+	pool_init(&cpumem_pl, sizeof(struct cpumem) * ncpus, 0, IPL_NONE,
+	    PR_WAITOK, "percpumem", &pool_allocator_single);
+}
+
+struct cpumem *
+cpumem_get(struct pool *pp)
+{
+	struct cpumem *cm;
+	unsigned int cpu;
+
+	cm = pool_get(&cpumem_pl, PR_WAITOK);
+
+	for (cpu = 0; cpu < ncpus; cpu++)
+		cm[cpu].mem = pool_get(pp, PR_WAITOK | PR_ZERO);
+
+	return (cm);
+}
+
+void
+cpumem_put(struct pool *pp, struct cpumem *cm)
+{
+	unsigned int cpu;
+
+	for (cpu = 0; cpu < ncpus; cpu++)
+		pool_put(pp, cm[cpu].mem);
+
+	pool_put(&cpumem_pl, cm);
+}
+
+struct cpumem *
+cpumem_malloc(size_t sz, int type)
+{
+	struct cpumem *cm;
+	unsigned int cpu;
+
+	sz = roundup(sz, CACHELINESIZE);
+
+	cm = pool_get(&cpumem_pl, PR_WAITOK);
+
+	for (cpu = 0; cpu < ncpus; cpu++)
+		cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO);
+
+	return (cm);
+}
+
+struct cpumem *
+cpumem_realloc(struct cpumem *bootcm, size_t sz, int type)
+{
+	struct cpumem *cm;
+	unsigned int cpu;
+
+	sz = roundup(sz, CACHELINESIZE);
+
+	cm = pool_get(&cpumem_pl, PR_WAITOK);
+
+	cm[0].mem = bootcm[0].mem;
+	for (cpu = 1; cpu < ncpus; cpu++)
+		cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO);
+
+	return (cm);
+}
+
+void
+cpumem_free(struct cpumem *cm, int type, size_t sz)
+{
+	unsigned int cpu;
+
+	sz = roundup(sz, CACHELINESIZE);
+
+	for (cpu = 0; cpu < ncpus; cpu++)
+		free(cm[cpu].mem, type, sz);
+
+	pool_put(&cpumem_pl, cm);
+}
+
+void *
+cpumem_first(struct cpumem_iter *i, struct cpumem *cm)
+{
+	i->cpu = 0;
+
+	return (cm[0].mem);
+}
+
+void *
+cpumem_next(struct cpumem_iter *i, struct cpumem *cm)
+{
+	unsigned int cpu = ++i->cpu;
+
+	if (cpu >= ncpus)
+		return (NULL);
+
+	return (cm[cpu].mem);
+}
+
+struct cpumem *
+counters_alloc(unsigned int n, int type)
+{
+	struct cpumem *cm;
+	struct cpumem_iter cmi;
+	uint64_t *counters;
+	unsigned int i;
+
+	KASSERT(n > 0);
+
+	n++; /* add space for a generation number */
+	cm = cpumem_malloc(n * sizeof(uint64_t), type);
+
+	CPUMEM_FOREACH(counters, &cmi, cm) {
+		for (i = 0; i < n; i++)
+			counters[i] = 0;
+	}
+
+	return (cm);
+}
+
+struct cpumem *
+counters_realloc(struct cpumem *cm, unsigned int n, int type)
+{
+	n++; /* the generation number */
+	return (cpumem_realloc(cm, n * sizeof(uint64_t), type));
+}
+
+void
+counters_free(struct cpumem *cm, int type, unsigned int n)
+{
+	n++; /* generation number */
+	cpumem_free(cm, type, n * sizeof(uint64_t));
+}
+
+void
+counters_read(struct cpumem *cm, uint64_t *output, unsigned int n)
+{
+	struct cpumem_iter cmi;
+	uint64_t *gen, *counters, *temp;
+	uint64_t enter, leave;
+	unsigned int i;
+
+	for (i = 0; i < n; i++)
+		output[i] = 0;
+
+	temp = mallocarray(n, sizeof(uint64_t), M_TEMP, M_WAITOK);
+
+	gen = cpumem_first(&cmi, cm);
+	do {
+		counters = gen + 1;
+
+		enter = *gen;
+		for (;;) {
+			/* the generation number is odd during an update */
+			while (enter & 1) {
+				yield();
+				membar_consumer();
+				enter = *gen;
+			}
+
+			for (i = 0; i < n; i++)
+				temp[i] = counters[i];
+
+			membar_consumer();
+			leave = *gen;
+
+			if (enter == leave)
+				break;
+
+			enter = leave;
+		}
+
+		for (i = 0; i < n; i++)
+			output[i] += temp[i];
+
+		gen = cpumem_next(&cmi, cm);
+	} while (gen != NULL);
+
+	free(temp, M_TEMP, n * sizeof(uint64_t));
+}
+
+void
+counters_zero(struct cpumem *cm, unsigned int n)
+{
+	struct cpumem_iter cmi;
+	uint64_t *counters;
+	unsigned int i;
+
+	n++; /* zero the generation numbers too */
+
+	counters = cpumem_first(&cmi, cm);
+	do {
+		for (i = 0; i < n; i++)
+			counters[i] = 0;
+
+		counters = cpumem_next(&cmi, cm);
+	} while (counters != NULL);
+}
+
+#else /* MULTIPROCESSOR */
+
+/*
+ * Uniprocessor implementation of per-CPU data structures.
+ *
+ * UP percpu memory is a single memory allocation cast to/from the
+ * cpumem struct. It is not scaled up to the size of cacheline because
+ * there's no other cache to contend with.
+ */
+
+void
+percpu_init(void)
+{
+	/* nop */
+}
+
+struct cpumem *
+cpumem_get(struct pool *pp)
+{
+	return (pool_get(pp, PR_WAITOK));
+}
+
+void
+cpumem_put(struct pool *pp, struct cpumem *cm)
+{
+	pool_put(pp, cm);
+}
+
+struct cpumem *
+cpumem_malloc(size_t sz, int type)
+{
+	return (malloc(sz, type, M_WAITOK));
+}
+
+struct cpumem *
+cpumem_realloc(struct cpumem *cm, size_t sz, int type)
+{
+	return (cm);
+}
+
+void
+cpumem_free(struct cpumem *cm, int type, size_t sz)
+{
+	free(cm, type, sz);
+}
+
+struct cpumem *
+counters_alloc(unsigned int n, int type)
+{
+	KASSERT(n > 0);
+
+	return (cpumem_malloc(n * sizeof(uint64_t), type));
+}
+
+struct cpumem *
+counters_realloc(struct cpumem *cm, unsigned int n, int type)
+{
+	/* this is unecessary, but symmetrical */
+	return (cpumem_realloc(cm, n * sizeof(uint64_t), type));
+}
+
+void
+counters_free(struct cpumem *cm, int type, unsigned int n)
+{
+	cpumem_free(cm, type, n * sizeof(uint64_t));
+}
+
+void
+counters_read(struct cpumem *cm, uint64_t *output, unsigned int n)
+{
+	uint64_t *counters;
+	unsigned int i;
+	int s;
+
+	counters = (uint64_t *)cm;
+
+	s = splhigh();
+	for (i = 0; i < n; i++)
+		output[i] = counters[i];
+	splx(s);
+}
+
+void
+counters_zero(struct cpumem *cm, unsigned int n)
+{
+	uint64_t *counters;
+	unsigned int i;
+	int s;
+
+	counters = (uint64_t *)cm;
+
+	s = splhigh();
+	for (i = 0; i < n; i++)
+		counters[i] = 0;
+	splx(s);
+}
+
+#endif /* MULTIPROCESSOR */
+
Index: sys/percpu.h
===================================================================
RCS file: sys/percpu.h
diff -N sys/percpu.h
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ sys/percpu.h	21 Oct 2016 04:11:03 -0000
@@ -0,0 +1,172 @@
+/*	$OpenBSD$ */
+
+/*
+ * Copyright (c) 2016 David Gwynne <dlg@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _SYS_PERCPU_H_
+#define _SYS_PERCPU_H_
+
+#ifndef CACHELINESIZE
+#define CACHELINESIZE 64
+#endif
+
+#ifndef __upunused /* this should go in param.h */
+#ifdef MULTIPROCESSOR
+#define __upunused
+#else
+#define __upunused __attribute__((__unused__))
+#endif
+#endif
+
+struct cpumem {
+	void		*mem;
+};
+
+struct cpumem_iter {
+	unsigned int	cpu;
+} __upunused;
+
+struct counters_ref {
+	uint64_t	 g;
+	uint64_t	*c;
+};
+
+#ifdef _KERNEL
+
+#include <sys/atomic.h>
+
+struct pool;
+
+struct cpumem	*cpumem_get(struct pool *);
+void		 cpumem_put(struct pool *, struct cpumem *);
+
+struct cpumem	*cpumem_malloc(size_t, int);
+struct cpumem	*cpumem_realloc(struct cpumem *, size_t, int);
+void		 cpumem_free(struct cpumem *, int, size_t);
+
+#ifdef MULTIPROCESSOR
+static inline void *
+cpumem_enter(struct cpumem *cm)
+{
+	return (cm[cpu_number()].mem);
+}
+
+static inline void
+cpumem_leave(struct cpumem *cm, void *mem)
+{
+	/* KDASSERT? */
+}
+
+void		*cpumem_first(struct cpumem_iter *, struct cpumem *);
+void		*cpumem_next(struct cpumem_iter *, struct cpumem *);
+
+#define CPUMEM_BOOT_MEMORY(_name, _sz)					\
+static struct {								\
+	unsigned char	mem[_sz];					\
+	struct cpumem	cpumem;						\
+} __aligned(CACHELINESIZE) _name##_boot_cpumem = {			\
+	.cpumem = { _name##_boot_cpumem.mem }				\
+}
+
+#define CPUMEM_BOOT_INITIALIZER(_name)					\
+	{ &_name##_boot_cpumem.cpumem }
+
+#else /* MULTIPROCESSOR */
+static inline void *
+cpumem_enter(struct cpumem *cm)
+{
+	return (cm);
+}
+
+static inline void
+cpumem_leave(struct cpumem *cm, void *mem)
+{
+	/* KDASSERT? */
+}
+
+static inline void *
+cpumem_first(struct cpumem_iter *i, struct cpumem *cm)
+{
+	return (cm);
+}
+
+static inline void *
+cpumem_next(struct cpumem_iter *i, struct cpumem *cm)
+{
+	return (NULL);
+}
+
+#define CPUMEM_BOOT_MEMORY(_name, _sz)					\
+static struct {								\
+	unsigned char	mem[_sz];					\
+} _name##_boot_cpumem
+
+#define CPUMEM_BOOT_INITIALIZER(_name)					\
+	{ (struct cpumem *)&_name##_boot_cpumem.mem }
+
+#endif /* MULTIPROCESSOR */
+
+#define CPUMEM_FOREACH(_var, _iter, _cpumem)				\
+	for ((_var) = cpumem_first((_iter), (_cpumem));			\
+	    (_var) != NULL;						\
+	    (_var) = cpumem_next((_iter), (_cpumem)))
+
+struct cpumem	*counters_alloc(unsigned int, int);
+struct cpumem	*counters_realloc(struct cpumem *, unsigned int, int);
+void		 counters_free(struct cpumem *, int, unsigned int);
+void		 counters_read(struct cpumem *, uint64_t *, unsigned int);
+void		 counters_zero(struct cpumem *, unsigned int);
+
+#ifdef MULTIPROCESSOR
+static inline uint64_t *
+counters_enter(struct counters_ref *ref, struct cpumem *cm)
+{
+	ref->c = cpumem_enter(cm);
+	ref->g = ++(*ref->c); /* make the generation number odd */
+	return (ref->c + 1);
+}
+
+static inline void
+counters_leave(struct counters_ref *ref, struct cpumem *cm)
+{
+	membar_producer();
+	(*ref->c) = ++ref->g; /* make the generation number even again */
+	cpumem_leave(cm, ref->c);
+}
+#define COUNTERS_BOOT_MEMORY(_name, _n)					\
+	CPUMEM_BOOT_MEMORY(_name, ((_n) + 1) * sizeof(uint64_t))
+#else
+static inline uint64_t *
+counters_enter(struct counters_ref *r, struct cpumem *cm)
+{
+	r->c = cpumem_enter(cm);
+	return (r->c);
+}
+
+static inline void
+counters_leave(struct counters_ref *r, struct cpumem *cm)
+{
+	cpumem_leave(cm, r->c);
+}
+
+#define COUNTERS_BOOT_MEMORY(_name, _n)					\
+	CPUMEM_BOOT_MEMORY(_name, (_n) * sizeof(uint64_t))
+#endif
+
+#define COUNTERS_BOOT_INITIALIZER(_name)	CPUMEM_BOOT_INITIALIZER(_name)
+
+#endif /* _KERNEL */
+#endif /* _SYS_PERCPU_H_ */
Index: sys/srp.h
===================================================================
RCS file: /cvs/src/sys/sys/srp.h,v
retrieving revision 1.11
diff -u -p -r1.11 srp.h
--- sys/srp.h	7 Jun 2016 07:53:33 -0000	1.11
+++ sys/srp.h	21 Oct 2016 04:11:03 -0000
@@ -21,11 +21,13 @@
 
 #include <sys/refcnt.h>
 
+#ifndef __upunused
 #ifdef MULTIPROCESSOR
 #define __upunused
 #else
 #define __upunused __attribute__((__unused__))
 #endif
+#endif /* __upunused */
 
 struct srp {
 	void			*ref;