Index: sys/pool.h
===================================================================
RCS file: /cvs/src/sys/sys/pool.h,v
retrieving revision 1.69
diff -u -p -r1.69 pool.h
--- sys/pool.h	7 Feb 2017 05:39:17 -0000	1.69
+++ sys/pool.h	27 Mar 2017 04:58:53 -0000
@@ -70,10 +70,10 @@ struct kinfo_pool {
 
 #include <sys/queue.h>
 #include <sys/tree.h>
-#include <sys/mutex.h>
 
 struct pool;
 struct pool_request;
+struct pool_lock;
 TAILQ_HEAD(pool_requests, pool_request);
 
 struct pool_allocator {
@@ -115,7 +115,9 @@ TAILQ_HEAD(pool_cache_lists, pool_cache_
 struct cpumem;
 
 struct pool {
-	struct mutex	pr_mtx;
+	const struct pool_lock *
+			pr_lock_ops;
+	void *		pr_lock;
 	SIMPLEQ_ENTRY(pool)
 			pr_poollist;
 	struct pool_pagelist
@@ -156,7 +158,7 @@ struct pool {
 
 	struct cpumem *	pr_cache;
 	unsigned long	pr_cache_magic[2];
-	struct mutex	pr_cache_mtx;
+	void *		pr_cache_lock;
 	struct pool_cache_lists
 			pr_cache_lists;
 	u_int		pr_cache_nlist;
@@ -179,7 +181,7 @@ struct pool {
 	/*
 	 * pool item requests queue
 	 */
-	struct mutex	pr_requests_mtx;
+	void *		pr_requests_lock;
 	struct pool_requests
 			pr_requests;
 	unsigned int	pr_requesting;
Index: kern/subr_pool.c
===================================================================
RCS file: /cvs/src/sys/kern/subr_pool.c,v
retrieving revision 1.207
diff -u -p -r1.207 subr_pool.c
--- kern/subr_pool.c	20 Feb 2017 00:43:25 -0000	1.207
+++ kern/subr_pool.c	27 Mar 2017 04:58:53 -0000
@@ -39,6 +39,7 @@
 #include <sys/pool.h>
 #include <sys/syslog.h>
 #include <sys/rwlock.h>
+#include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/task.h>
 #include <sys/timeout.h>
@@ -75,6 +76,14 @@ struct rwlock pool_lock = RWLOCK_INITIAL
 /* Private pool for page header structures */
 struct pool phpool;
 
+struct pool_lock {
+	void *		(*pl_alloc)(struct pool *);
+	void		(*pl_free)(void *);
+	void		(*pl_enter)(void *);
+	int		(*pl_enter_try)(void *);
+	void		(*pl_leave)(void *);
+};
+
 struct pool_item {
 	u_long				pi_magic;
 	XSIMPLEQ_ENTRY(pool_item)	pi_list;
@@ -199,6 +208,18 @@ void	 pool_print1(struct pool *, const c
 	     __attribute__((__format__(__kprintf__,1,2))));
 #endif
 
+void	*pool_lock_mtx_alloc(struct pool *);
+void	 pool_lock_mtx_free(void *);
+void	 pool_lock_mtx_enter(void *);
+int	 pool_lock_mtx_enter_try(void *);
+void	 pool_lock_mtx_leave(void *);
+
+void	*pool_lock_rw_alloc(struct pool *);
+void	 pool_lock_rw_free(void *);
+void	 pool_lock_rw_enter(void *);
+int	 pool_lock_rw_enter_try(void *);
+void	 pool_lock_rw_leave(void *);
+
 /* stale page garbage collectors */
 void	pool_gc_sched(void *);
 struct timeout pool_gc_tick = TIMEOUT_INITIALIZER(pool_gc_sched, NULL);
@@ -255,6 +276,22 @@ pr_find_pagehead(struct pool *pp, void *
 	return (ph);
 }
 
+const struct pool_lock pool_lock_mtx = {
+	pool_lock_mtx_alloc,
+	pool_lock_mtx_free,
+	pool_lock_mtx_enter,
+	pool_lock_mtx_enter_try,
+	pool_lock_mtx_leave,
+};
+
+const struct pool_lock pool_lock_rw = {
+	pool_lock_rw_alloc,
+	pool_lock_rw_free,
+	pool_lock_rw_enter,
+	pool_lock_rw_enter_try,
+	pool_lock_rw_leave,
+};
+
 /*
  * Initialize the given pool resource structure.
  *
@@ -333,6 +370,11 @@ pool_init(struct pool *pp, size_t size, 
 
 	KASSERT(items > 0);
 
+	if (ISSET(flags, PR_WAITOK))
+		pp->pr_lock_ops = &pool_lock_rw;
+	else
+		pp->pr_lock_ops = &pool_lock_mtx;
+
 	/*
 	 * Initialize the pool structure.
 	 */
@@ -380,8 +422,8 @@ pool_init(struct pool *pp, size_t size, 
 	pp->pr_nidle = 0;
 
 	pp->pr_ipl = ipl;
-	mtx_init(&pp->pr_mtx, pp->pr_ipl);
-	mtx_init(&pp->pr_requests_mtx, pp->pr_ipl);
+	pp->pr_lock = (*pp->pr_lock_ops->pl_alloc)(pp);
+	pp->pr_requests_lock = (*pp->pr_lock_ops->pl_alloc)(pp);
 	TAILQ_INIT(&pp->pr_requests);
 
 	if (phpool.pr_size == 0) {
@@ -452,9 +494,9 @@ pool_destroy(struct pool *pp)
 
 	/* Remove all pages */
 	while ((ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL) {
-		mtx_enter(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 		pool_p_remove(pp, ph);
-		mtx_leave(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 		pool_p_free(pp, ph);
 	}
 	KASSERT(TAILQ_EMPTY(&pp->pr_fullpages));
@@ -473,10 +515,10 @@ pool_request_init(struct pool_request *p
 void
 pool_request(struct pool *pp, struct pool_request *pr)
 {
-	mtx_enter(&pp->pr_requests_mtx);
+	(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 	TAILQ_INSERT_TAIL(&pp->pr_requests, pr, pr_entry);
 	pool_runqueue(pp, PR_NOWAIT);
-	mtx_leave(&pp->pr_requests_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 }
 
 struct pool_get_memory {
@@ -503,7 +545,7 @@ pool_get(struct pool *pp, int flags)
 
 	KASSERT(flags & (PR_WAITOK | PR_NOWAIT));
 
-	mtx_enter(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 	if (pp->pr_nout >= pp->pr_hardlimit) {
 		if (ISSET(flags, PR_NOWAIT|PR_LIMITFAIL))
 			goto fail;
@@ -511,7 +553,7 @@ pool_get(struct pool *pp, int flags)
 		if (ISSET(flags, PR_NOWAIT))
 			goto fail;
 	}
-	mtx_leave(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 
 	if ((slowdown || pool_debug == 2) && ISSET(flags, PR_WAITOK))
 		yield();
@@ -543,7 +585,7 @@ good:
 
 fail:
 	pp->pr_nfail++;
-	mtx_leave(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 	return (NULL);
 }
 
@@ -565,9 +607,6 @@ pool_runqueue(struct pool *pp, int flags
 	struct pool_requests prl = TAILQ_HEAD_INITIALIZER(prl);
 	struct pool_request *pr;
 
-	MUTEX_ASSERT_UNLOCKED(&pp->pr_mtx);
-	MUTEX_ASSERT_LOCKED(&pp->pr_requests_mtx);
-
 	if (pp->pr_requesting++)
 		return;
 
@@ -582,9 +621,9 @@ pool_runqueue(struct pool *pp, int flags
 		if (TAILQ_EMPTY(&prl))
 			continue;
 
-		mtx_leave(&pp->pr_requests_mtx);
+		(*pp->pr_lock_ops->pl_leave)(&pp->pr_requests_lock);
 
-		mtx_enter(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 		pr = TAILQ_FIRST(&prl);
 		while (pr != NULL) {
 			int slowdown = 0;
@@ -598,7 +637,7 @@ pool_runqueue(struct pool *pp, int flags
 
 			pr = TAILQ_NEXT(pr, pr_entry);
 		}
-		mtx_leave(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 
 		while ((pr = TAILQ_FIRST(&prl)) != NULL &&
 		    pr->pr_item != NULL) {
@@ -606,7 +645,7 @@ pool_runqueue(struct pool *pp, int flags
 			(*pr->pr_handler)(pr->pr_cookie, pr->pr_item);
 		}
 
-		mtx_enter(&pp->pr_requests_mtx);
+		(*pp->pr_lock_ops->pl_enter)(&pp->pr_requests_lock);
 	} while (--pp->pr_requesting);
 
 	/* no TAILQ_JOIN :( */
@@ -622,20 +661,18 @@ pool_do_get(struct pool *pp, int flags, 
 	struct pool_item *pi;
 	struct pool_page_header *ph;
 
-	MUTEX_ASSERT_LOCKED(&pp->pr_mtx);
-
 	splassert(pp->pr_ipl);
 
 	/*
 	 * Account for this item now to avoid races if we need to give up
-	 * pr_mtx to allocate a page.
+	 * pr_lock to allocate a page.
 	 */
 	pp->pr_nout++;
 
 	if (pp->pr_curpage == NULL) {
-		mtx_leave(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 		ph = pool_p_alloc(pp, flags, slowdown);
-		mtx_enter(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 
 		if (ph == NULL) {
 			pp->pr_nout--;
@@ -721,7 +758,7 @@ pool_put(struct pool *pp, void *v)
 	}
 #endif
 
-	mtx_enter(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 
 	splassert(pp->pr_ipl);
 
@@ -777,15 +814,15 @@ pool_put(struct pool *pp, void *v)
 		freeph = ph;
 		pool_p_remove(pp, freeph);
 	}
-	mtx_leave(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 
 	if (freeph != NULL)
 		pool_p_free(pp, freeph);
 
 	if (!TAILQ_EMPTY(&pp->pr_requests)) {
-		mtx_enter(&pp->pr_requests_mtx);
+		(*pp->pr_lock_ops->pl_enter)(&pp->pr_requests_lock);
 		pool_runqueue(pp, PR_NOWAIT);
-		mtx_leave(&pp->pr_requests_mtx);
+		(*pp->pr_lock_ops->pl_leave)(&pp->pr_requests_lock);
 	}
 }
 
@@ -811,12 +848,12 @@ pool_prime(struct pool *pp, int n)
 		TAILQ_INSERT_TAIL(&pl, ph, ph_entry);
 	}
 
-	mtx_enter(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 	while ((ph = TAILQ_FIRST(&pl)) != NULL) {
 		TAILQ_REMOVE(&pl, ph, ph_entry);
 		pool_p_insert(pp, ph);
 	}
-	mtx_leave(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 
 	return (0);
 }
@@ -829,7 +866,6 @@ pool_p_alloc(struct pool *pp, int flags,
 	caddr_t addr;
 	int n;
 
-	MUTEX_ASSERT_UNLOCKED(&pp->pr_mtx);
 	KASSERT(pp->pr_size >= sizeof(*pi));
 
 	addr = pool_allocator_alloc(pp, flags, slowdown);
@@ -882,7 +918,6 @@ pool_p_free(struct pool *pp, struct pool
 {
 	struct pool_item *pi;
 
-	MUTEX_ASSERT_UNLOCKED(&pp->pr_mtx);
 	KASSERT(ph->ph_nmissing == 0);
 
 	XSIMPLEQ_FOREACH(pi, &ph->ph_items, pi_list) {
@@ -918,8 +953,6 @@ pool_p_free(struct pool *pp, struct pool
 void
 pool_p_insert(struct pool *pp, struct pool_page_header *ph)
 {
-	MUTEX_ASSERT_LOCKED(&pp->pr_mtx);
-
 	/* If the pool was depleted, point at the new page */
 	if (pp->pr_curpage == NULL)
 		pp->pr_curpage = ph;
@@ -939,8 +972,6 @@ pool_p_insert(struct pool *pp, struct po
 void
 pool_p_remove(struct pool *pp, struct pool_page_header *ph)
 {
-	MUTEX_ASSERT_LOCKED(&pp->pr_mtx);
-
 	pp->pr_npagefree++;
 	pp->pr_npages--;
 	pp->pr_nidle--;
@@ -967,7 +998,7 @@ pool_setlowat(struct pool *pp, int n)
 {
 	int prime = 0;
 
-	mtx_enter(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 	pp->pr_minitems = n;
 	pp->pr_minpages = (n == 0)
 		? 0
@@ -975,7 +1006,7 @@ pool_setlowat(struct pool *pp, int n)
 
 	if (pp->pr_nitems < n)
 		prime = n - pp->pr_nitems;
-	mtx_leave(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 
 	if (prime > 0)
 		pool_prime(pp, prime);
@@ -1026,7 +1057,7 @@ pool_reclaim(struct pool *pp)
 	struct pool_page_header *ph, *phnext;
 	struct pool_pagelist pl = TAILQ_HEAD_INITIALIZER(pl);
 
-	mtx_enter(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 	for (ph = TAILQ_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
 		phnext = TAILQ_NEXT(ph, ph_entry);
 
@@ -1045,7 +1076,7 @@ pool_reclaim(struct pool *pp)
 		pool_p_remove(pp, ph);
 		TAILQ_INSERT_TAIL(&pl, ph, ph_entry);
 	}
-	mtx_leave(&pp->pr_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 
 	if (TAILQ_EMPTY(&pl))
 		return (0);
@@ -1404,7 +1435,7 @@ sysctl_dopool(int *name, u_int namelen, 
 	case KERN_POOL_POOL:
 		memset(&pi, 0, sizeof(pi));
 
-		mtx_enter(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_enter)(&pp->pr_lock);
 		pi.pr_size = pp->pr_size;
 		pi.pr_pgsize = pp->pr_pgsize;
 		pi.pr_itemsperpage = pp->pr_itemsperpage;
@@ -1421,7 +1452,7 @@ sysctl_dopool(int *name, u_int namelen, 
 		pi.pr_npagefree = pp->pr_npagefree;
 		pi.pr_hiwat = pp->pr_hiwat;
 		pi.pr_nidle = pp->pr_nidle;
-		mtx_leave(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 
 		pool_cache_info(pp, &pi);
 
@@ -1452,7 +1483,7 @@ pool_gc_pages(void *null)
 	s = splvm(); /* XXX go to splvm until all pools _setipl properly */
 	SIMPLEQ_FOREACH(pp, &pool_head, pr_poollist) {
 		if (pp->pr_nidle <= pp->pr_minpages || /* guess */
-		    !mtx_enter_try(&pp->pr_mtx)) /* try */
+		   !(*pp->pr_lock_ops->pl_enter_try)(&pp->pr_lock)) /* try */
 			continue;
 
 		/* is it time to free a page? */
@@ -1464,7 +1495,7 @@ pool_gc_pages(void *null)
 		} else
 			freeph = NULL;
 
-		mtx_leave(&pp->pr_mtx);
+		(*pp->pr_lock_ops->pl_leave)(&pp->pr_lock);
 
 		if (freeph != NULL)
 			pool_p_free(pp, freeph);
@@ -1612,7 +1643,7 @@ pool_cache_init(struct pool *pp)
 
 	cm = cpumem_get(&pool_caches);
 
-	mtx_init(&pp->pr_cache_mtx, pp->pr_ipl);
+	pp->pr_cache_lock = (*pp->pr_lock_ops->pl_alloc)(pp);
 	arc4random_buf(pp->pr_cache_magic, sizeof(pp->pr_cache_magic));
 	TAILQ_INIT(&pp->pr_cache_lists);
 	pp->pr_cache_nlist = 0;
@@ -1669,8 +1700,8 @@ fail:
 static inline void
 pool_list_enter(struct pool *pp)
 {
-	if (mtx_enter_try(&pp->pr_cache_mtx) == 0) {
-		mtx_enter(&pp->pr_cache_mtx);
+	if ((*pp->pr_lock_ops->pl_enter_try)(&pp->pr_cache_lock) == 0) {
+		(*pp->pr_lock_ops->pl_enter)(&pp->pr_cache_lock);
 		pp->pr_cache_contention++;
 	}
 }
@@ -1678,7 +1709,7 @@ pool_list_enter(struct pool *pp)
 static inline void
 pool_list_leave(struct pool *pp)
 {
-	mtx_leave(&pp->pr_cache_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_cache_lock);
 }
 
 static inline struct pool_cache_item *
@@ -1883,7 +1914,7 @@ pool_cache_info(struct pool *pp, struct 
 
 	/* loop through the caches twice to collect stats */
 
-	/* once without the mtx so we can yield while reading nget/nput */
+	/* once without the lock so we can yield while reading nget/nput */
 	CPUMEM_FOREACH(pc, &i, pp->pr_cache) {
 		uint64_t gen, nget, nput;
 
@@ -1900,12 +1931,12 @@ pool_cache_info(struct pool *pp, struct 
 	}
 
 	/* and once with the mtx so we can get consistent nout values */
-	mtx_enter(&pp->pr_cache_mtx);
+	(*pp->pr_lock_ops->pl_enter)(&pp->pr_cache_lock);
 	CPUMEM_FOREACH(pc, &i, pp->pr_cache)
 		pi->pr_nout += pc->pc_nout;
 
 	pi->pr_nout += pp->pr_cache_nout;
-	mtx_leave(&pp->pr_cache_mtx);
+	(*pp->pr_lock_ops->pl_leave)(&pp->pr_cache_lock);
 }
 #else /* MULTIPROCESSOR */
 void
@@ -1920,3 +1951,89 @@ pool_cache_info(struct pool *pp, struct 
 	/* nop */
 }
 #endif /* MULTIPROCESSOR */
+
+void *
+pool_lock_mtx_alloc(struct pool *pp)
+{
+	struct mutex *mtx;
+
+	mtx = malloc(sizeof(*mtx), M_DEVBUF /* ? */, M_WAITOK);
+	mtx_init(mtx, pp->pr_ipl);
+
+	return (mtx);
+}
+
+void
+pool_lock_mtx_free(void *lock)
+{
+	struct mutex *mtx = lock;
+
+	free(mtx, M_DEVBUF /* ? */, sizeof(*mtx));
+}
+
+void
+pool_lock_mtx_enter(void *lock)
+{
+	struct mutex *mtx = lock;
+
+	mtx_enter(mtx);
+}
+
+int
+pool_lock_mtx_enter_try(void *lock)
+{
+	struct mutex *mtx = lock;
+
+	return (mtx_enter_try(mtx));
+}
+
+void
+pool_lock_mtx_leave(void *lock)
+{
+	struct mutex *mtx = lock;
+
+	mtx_leave(mtx);
+}
+
+void *
+pool_lock_rw_alloc(struct pool *pp)
+{
+	struct rwlock *rw;
+
+	rw = malloc(sizeof(*rw), M_DEVBUF /* ? */, M_WAITOK);
+	rw_init(rw, pp->pr_wchan); /* XXX */
+
+	return (rw);
+}
+
+void
+pool_lock_rw_free(void *lock)
+{
+	struct rwlock *rw = lock;
+
+	free(rw, M_DEVBUF /* ? */, sizeof(*rw));
+}
+
+void
+pool_lock_rw_enter(void *lock)
+{
+	struct rwlock *rw = lock;
+
+	rw_enter_write(rw);
+}
+
+int
+pool_lock_rw_enter_try(void *lock)
+{
+	struct rwlock *rw = lock;
+
+	return (rw_enter(rw, RW_WRITE | RW_NOSLEEP) == 0);
+}
+
+void
+pool_lock_rw_leave(void *lock)
+{
+	struct rwlock *rw = lock;
+
+	rw_exit(rw);
+}