Index: sys/pool.h =================================================================== RCS file: /cvs/src/sys/sys/pool.h,v retrieving revision 1.54 diff -u -p -r1.54 pool.h --- sys/pool.h 1 Nov 2014 23:58:07 -0000 1.54 +++ sys/pool.h 1 Dec 2014 00:42:14 -0000 @@ -82,7 +82,7 @@ struct pool_allocator { int pa_pagesz; }; -LIST_HEAD(pool_pagelist, pool_item_header); +TAILQ_HEAD(pool_pagelist, pool_item_header); struct pool { struct mutex pr_mtx; @@ -189,7 +189,7 @@ void pool_set_constraints(struct pool * void *pool_get(struct pool *, int) __malloc; void pool_request_init(struct pool_request *, void (*)(void *, void *), void *); -void pool_request(struct pool *, struct pool_request *); +void pool_request(struct pool *, struct pool_request *, int); void pool_put(struct pool *, void *); int pool_reclaim(struct pool *); void pool_reclaim_all(void); Index: kern/subr_pool.c =================================================================== RCS file: /cvs/src/sys/kern/subr_pool.c,v retrieving revision 1.168 diff -u -p -r1.168 subr_pool.c --- kern/subr_pool.c 18 Nov 2014 02:37:31 -0000 1.168 +++ kern/subr_pool.c 1 Dec 2014 00:42:14 -0000 @@ -32,6 +32,7 @@ */ #include +#include #include #include #include @@ -74,7 +75,7 @@ struct pool phpool; struct pool_item_header { /* Page headers */ - LIST_ENTRY(pool_item_header) + TAILQ_ENTRY(pool_item_header) ph_pagelist; /* pool page list */ XSIMPLEQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */ RB_ENTRY(pool_item_header) @@ -82,6 +83,7 @@ struct pool_item_header { int ph_nmissing; /* # of chunks in use */ caddr_t ph_page; /* this page's address */ u_long ph_magic; + int ph_tick; }; #define POOL_MAGICBIT (1 << 3) /* keep away from perturbed low bits */ #define POOL_PHPOISON(ph) ISSET((ph)->ph_magic, POOL_MAGICBIT) @@ -231,7 +233,7 @@ pool_init(struct pool *pp, size_t size, size = roundup(size, align); if (palloc == NULL) { - while (size > pgsize) + while (size * 8 > pgsize) pgsize <<= 1; if (pgsize > PAGE_SIZE) { @@ -263,9 +265,9 @@ pool_init(struct pool *pp, size_t size, * Initialize the pool structure. */ memset(pp, 0, sizeof(*pp)); - LIST_INIT(&pp->pr_emptypages); - LIST_INIT(&pp->pr_fullpages); - LIST_INIT(&pp->pr_partpages); + TAILQ_INIT(&pp->pr_emptypages); + TAILQ_INIT(&pp->pr_fullpages); + TAILQ_INIT(&pp->pr_partpages); pp->pr_curpage = NULL; pp->pr_npages = 0; pp->pr_minitems = 0; @@ -375,14 +377,14 @@ pool_destroy(struct pool *pp) rw_exit_write(&pool_lock); /* Remove all pages */ - while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) { + while ((ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL) { mtx_enter(&pp->pr_mtx); pool_p_remove(pp, ph); mtx_leave(&pp->pr_mtx); pool_p_free(pp, ph); } - KASSERT(LIST_EMPTY(&pp->pr_fullpages)); - KASSERT(LIST_EMPTY(&pp->pr_partpages)); + KASSERT(TAILQ_EMPTY(&pp->pr_fullpages)); + KASSERT(TAILQ_EMPTY(&pp->pr_partpages)); } void @@ -395,18 +397,15 @@ pool_request_init(struct pool_request *p } void -pool_request(struct pool *pp, struct pool_request *pr) +pool_request(struct pool *pp, struct pool_request *pr, int flags) { mtx_enter(&pp->pr_requests_mtx); TAILQ_INSERT_TAIL(&pp->pr_requests, pr, pr_entry); - pool_runqueue(pp, PR_NOWAIT); + pool_runqueue(pp, flags); mtx_leave(&pp->pr_requests_mtx); } -struct pool_get_memory { - struct mutex mtx; - void * volatile v; -}; +u_int pool_sleeps; /* * Grab an item from the pool. @@ -419,34 +418,34 @@ pool_get(struct pool *pp, int flags) KASSERT(flags & (PR_WAITOK | PR_NOWAIT)); + if ((flags & (PR_WAITOK|PR_LIMITFAIL)) != PR_WAITOK) { + mtx_enter(&pp->pr_mtx); + if (pp->pr_nout >= pp->pr_hardlimit) { + if (ISSET(flags, PR_NOWAIT|PR_LIMITFAIL)) + goto fail; + } else if ((v = pool_do_get(pp, flags, &slowdown)) == NULL) { + if (ISSET(flags, PR_NOWAIT)) + goto fail; + } + mtx_leave(&pp->pr_mtx); - mtx_enter(&pp->pr_mtx); - if (pp->pr_nout >= pp->pr_hardlimit) { - if (ISSET(flags, PR_NOWAIT|PR_LIMITFAIL)) - goto fail; - } else if ((v = pool_do_get(pp, flags, &slowdown)) == NULL) { - if (ISSET(flags, PR_NOWAIT)) - goto fail; + if (slowdown && ISSET(flags, PR_WAITOK)) + yield(); } - mtx_leave(&pp->pr_mtx); - - if (slowdown && ISSET(flags, PR_WAITOK)) - yield(); if (v == NULL) { - struct pool_get_memory mem = - { MUTEX_INITIALIZER(pp->pr_ipl), NULL }; + struct sleep_state sls; struct pool_request pr; - pool_request_init(&pr, pool_get_done, &mem); - pool_request(pp, &pr); + pool_request_init(&pr, pool_get_done, &v); + pool_request(pp, &pr, flags); - mtx_enter(&mem.mtx); - while (mem.v == NULL) - msleep(&mem, &mem.mtx, PSWP, pp->pr_wchan, 0); - mtx_leave(&mem.mtx); + atomic_inc_int(&pool_sleeps); - v = mem.v; + while (v == NULL) { + sleep_setup(&sls, &v, PWAIT, pp->pr_wchan); + sleep_finish(&sls, (v == NULL)); + } } if (ISSET(flags, PR_ZERO)) @@ -461,15 +460,12 @@ fail: } void -pool_get_done(void *xmem, void *v) +pool_get_done(void *ctx, void *v) { - struct pool_get_memory *mem = xmem; + void **vp = ctx; - mtx_enter(&mem->mtx); - mem->v = v; - mtx_leave(&mem->mtx); - - wakeup_one(mem); + *vp = v; + wakeup_one(vp); } void @@ -485,6 +481,7 @@ pool_runqueue(struct pool *pp, int flags return; do { + int slowdown = 0; pp->pr_requesting = 1; /* no TAILQ_JOIN? :( */ @@ -500,19 +497,20 @@ pool_runqueue(struct pool *pp, int flags mtx_enter(&pp->pr_mtx); pr = TAILQ_FIRST(&prl); while (pr != NULL) { - int slowdown = 0; - if (pp->pr_nout >= pp->pr_hardlimit) break; pr->pr_item = pool_do_get(pp, flags, &slowdown); - if (pr->pr_item == NULL) /* || slowdown ? */ + if (pr->pr_item == NULL) break; pr = TAILQ_NEXT(pr, pr_entry); } mtx_leave(&pp->pr_mtx); + if (slowdown && ISSET(flags, PR_WAITOK)) + yield(); + while ((pr = TAILQ_FIRST(&prl)) != NULL && pr->pr_item != NULL) { TAILQ_REMOVE(&prl, pr, pr_entry); @@ -590,8 +588,8 @@ pool_do_get(struct pool *pp, int flags, * This page was previously empty. Move it to the list of * partially-full pages. This page is already curpage. */ - LIST_REMOVE(ph, ph_pagelist); - LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); + TAILQ_REMOVE(&pp->pr_emptypages, ph, ph_pagelist); + TAILQ_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); pp->pr_nidle--; } @@ -601,8 +599,8 @@ pool_do_get(struct pool *pp, int flags, * This page is now full. Move it to the full list * and select a new current page. */ - LIST_REMOVE(ph, ph_pagelist); - LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist); + TAILQ_REMOVE(&pp->pr_partpages, ph, ph_pagelist); + TAILQ_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist); pool_update_curpage(pp); } @@ -619,6 +617,7 @@ pool_put(struct pool *pp, void *v) { struct pool_item *pi = v; struct pool_item_header *ph, *freeph = NULL; + extern int ticks; #ifdef DIAGNOSTIC if (v == NULL) @@ -653,8 +652,8 @@ pool_put(struct pool *pp, void *v) * The page was previously completely full, move it to the * partially-full list. */ - LIST_REMOVE(ph, ph_pagelist); - LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); + TAILQ_REMOVE(&pp->pr_fullpages, ph, ph_pagelist); + TAILQ_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); } if (ph->ph_nmissing == 0) { @@ -663,8 +662,9 @@ pool_put(struct pool *pp, void *v) */ pp->pr_nidle++; - LIST_REMOVE(ph, ph_pagelist); - LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); + ph->ph_tick = ticks; + TAILQ_REMOVE(&pp->pr_partpages, ph, ph_pagelist); + TAILQ_INSERT_TAIL(&pp->pr_emptypages, ph, ph_pagelist); pool_update_curpage(pp); } @@ -673,8 +673,11 @@ pool_put(struct pool *pp, void *v) /* is it time to free a page? */ if (pp->pr_nidle > pp->pr_maxpages && - (freeph = LIST_FIRST(&pp->pr_emptypages)) != NULL) + (ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL && + (ticks - ph->ph_tick) > hz) { + freeph = ph; pool_p_remove(pp, freeph); + } mtx_leave(&pp->pr_mtx); if (freeph != NULL) @@ -691,7 +694,7 @@ pool_put(struct pool *pp, void *v) int pool_prime(struct pool *pp, int n) { - struct pool_pagelist pl = LIST_HEAD_INITIALIZER(pl); + struct pool_pagelist pl = TAILQ_HEAD_INITIALIZER(pl); struct pool_item_header *ph; int newpages; @@ -704,12 +707,12 @@ pool_prime(struct pool *pp, int n) if (ph == NULL) /* or slowdown? */ break; - LIST_INSERT_HEAD(&pl, ph, ph_pagelist); + TAILQ_INSERT_HEAD(&pl, ph, ph_pagelist); } mtx_enter(&pp->pr_mtx); - while ((ph = LIST_FIRST(&pl)) != NULL) { - LIST_REMOVE(ph, ph_pagelist); + while ((ph = TAILQ_FIRST(&pl)) != NULL) { + TAILQ_REMOVE(&pl, ph, ph_pagelist); pool_p_insert(pp, ph); } mtx_leave(&pp->pr_mtx); @@ -818,7 +821,7 @@ pool_p_insert(struct pool *pp, struct po if (pp->pr_curpage == NULL) pp->pr_curpage = ph; - LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); + TAILQ_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); if (!POOL_INPGHDR(pp)) RB_INSERT(phtree, &pp->pr_phtree, ph); @@ -842,7 +845,7 @@ pool_p_remove(struct pool *pp, struct po if (!POOL_INPGHDR(pp)) RB_REMOVE(phtree, &pp->pr_phtree, ph); - LIST_REMOVE(ph, ph_pagelist); + TAILQ_REMOVE(&pp->pr_emptypages, ph, ph_pagelist); pool_update_curpage(pp); } @@ -850,10 +853,9 @@ pool_p_remove(struct pool *pp, struct po void pool_update_curpage(struct pool *pp) { - pp->pr_curpage = LIST_FIRST(&pp->pr_partpages); - if (pp->pr_curpage == NULL) { - pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); - } + pp->pr_curpage = TAILQ_FIRST(&pp->pr_partpages); + if (pp->pr_curpage == NULL) + pp->pr_curpage = TAILQ_FIRST(&pp->pr_emptypages); } void @@ -918,11 +920,11 @@ int pool_reclaim(struct pool *pp) { struct pool_item_header *ph, *phnext; - struct pool_pagelist pl = LIST_HEAD_INITIALIZER(pl); + struct pool_pagelist pl = TAILQ_HEAD_INITIALIZER(pl); mtx_enter(&pp->pr_mtx); - for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { - phnext = LIST_NEXT(ph, ph_pagelist); + for (ph = TAILQ_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { + phnext = TAILQ_NEXT(ph, ph_pagelist); /* Check our minimum page claim */ if (pp->pr_npages <= pp->pr_minpages) @@ -937,15 +939,15 @@ pool_reclaim(struct pool *pp) break; pool_p_remove(pp, ph); - LIST_INSERT_HEAD(&pl, ph, ph_pagelist); + TAILQ_INSERT_HEAD(&pl, ph, ph_pagelist); } mtx_leave(&pp->pr_mtx); - if (LIST_EMPTY(&pl)) + if (TAILQ_EMPTY(&pl)) return (0); - while ((ph = LIST_FIRST(&pl)) != NULL) { - LIST_REMOVE(ph, ph_pagelist); + while ((ph = TAILQ_FIRST(&pl)) != NULL) { + TAILQ_REMOVE(&pl, ph, ph_pagelist); pool_p_free(pp, ph); } @@ -989,7 +991,7 @@ pool_print_pagelist(struct pool_pagelist struct pool_item_header *ph; struct pool_item *pi; - LIST_FOREACH(ph, pl, ph_pagelist) { + TAILQ_FOREACH(ph, pl, ph_pagelist) { (*pr)("\t\tpage %p, nmissing %d\n", ph->ph_page, ph->ph_nmissing); XSIMPLEQ_FOREACH(pi, &ph->ph_itemlist, pi_list) { @@ -1032,13 +1034,13 @@ pool_print1(struct pool *pp, const char if (print_pagelist == 0) return; - if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) + if ((ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL) (*pr)("\n\tempty page list:\n"); pool_print_pagelist(&pp->pr_emptypages, pr); - if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL) + if ((ph = TAILQ_FIRST(&pp->pr_fullpages)) != NULL) (*pr)("\n\tfull page list:\n"); pool_print_pagelist(&pp->pr_fullpages, pr); - if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL) + if ((ph = TAILQ_FIRST(&pp->pr_partpages)) != NULL) (*pr)("\n\tpartial-page list:\n"); pool_print_pagelist(&pp->pr_partpages, pr); @@ -1202,11 +1204,11 @@ pool_chk(struct pool *pp) struct pool_item_header *ph; int r = 0; - LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) + TAILQ_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) r += pool_chk_page(pp, ph, pp->pr_itemsperpage); - LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) + TAILQ_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) r += pool_chk_page(pp, ph, 0); - LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) + TAILQ_FOREACH(ph, &pp->pr_partpages, ph_pagelist) r += pool_chk_page(pp, ph, -1); return (r); @@ -1225,7 +1227,7 @@ pool_walk(struct pool *pp, int full, caddr_t cp; int n; - LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { + TAILQ_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { cp = ph->ph_page; n = ph->ph_nmissing; @@ -1235,7 +1237,7 @@ pool_walk(struct pool *pp, int full, } } - LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { + TAILQ_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { cp = ph->ph_page; n = ph->ph_nmissing; Index: kern/uipc_mbuf.c =================================================================== RCS file: /cvs/src/sys/kern/uipc_mbuf.c,v retrieving revision 1.198 diff -u -p -r1.198 uipc_mbuf.c --- kern/uipc_mbuf.c 5 Nov 2014 00:28:15 -0000 1.198 +++ kern/uipc_mbuf.c 1 Dec 2014 00:42:14 -0000 @@ -156,7 +156,7 @@ mbinit(void) for (i = 0; i < nitems(mclsizes); i++) { snprintf(mclnames[i], sizeof(mclnames[0]), "mcl%dk", mclsizes[i] >> 10); - pool_init(&mclpools[i], mclsizes[i], 0, 0, 0, + pool_init(&mclpools[i], mclsizes[i], 1024, 0, 0, mclnames[i], NULL); pool_setipl(&mclpools[i], IPL_NET); pool_set_constraints(&mclpools[i], &kp_dma_contig); @@ -1309,6 +1309,35 @@ ml_dechain(struct mbuf_list *ml) return (m0); } +struct mbuf * +ml_filter(struct mbuf_list *ml, + int (*filter)(void *, const struct mbuf *), void *ctx) +{ + struct mbuf_list matches = MBUF_LIST_INITIALIZER(); + struct mbuf *m, *n; + struct mbuf **mp; + + mp = &ml->ml_head; + + for (m = ml->ml_head; m != NULL; m = n) { + n = m->m_nextpkt; + if ((*filter)(ctx, m)) { + *mp = n; + ml_enqueue(&matches, m); + } else { + mp = &m->m_nextpkt; + ml->ml_tail = m; + } + } + + /* fixup ml */ + if (ml->ml_head == NULL) + ml->ml_tail = NULL; + ml->ml_len -= ml_len(&matches); + + return (matches.ml_head); /* ml_dechain */ +} + /* * mbuf queues */ @@ -1382,6 +1411,19 @@ mq_dechain(struct mbuf_queue *mq) mtx_enter(&mq->mq_mtx); m0 = ml_dechain(&mq->mq_list); + mtx_leave(&mq->mq_mtx); + + return (m0); +} + +struct mbuf * +mq_filter(struct mbuf_queue *mq, + int (*filter)(void *, const struct mbuf *), void *ctx) +{ + struct mbuf *m0; + + mtx_enter(&mq->mq_mtx); + m0 = ml_filter(&mq->mq_list, filter, ctx); mtx_leave(&mq->mq_mtx); return (m0);