Index: sys/mbuf.h =================================================================== RCS file: /cvs/src/sys/sys/mbuf.h,v retrieving revision 1.182 diff -u -p -r1.182 mbuf.h --- sys/mbuf.h 18 Aug 2014 05:11:03 -0000 1.182 +++ sys/mbuf.h 30 Sep 2014 11:07:02 -0000 @@ -245,8 +245,6 @@ struct mbuf { /* * Macros for tracking external storage associated with an mbuf. - * - * Note: add and delete reference must be called at splnet(). */ #ifdef DEBUG #define MCLREFDEBUGN(m, file, line) do { \ @@ -264,16 +262,7 @@ struct mbuf { #define MCLISREFERENCED(m) ((m)->m_ext.ext_nextref != (m)) -#define MCLADDREFERENCE(o, n) do { \ - int ms = splnet(); \ - (n)->m_flags |= ((o)->m_flags & (M_EXT|M_EXTWR)); \ - (n)->m_ext.ext_nextref = (o)->m_ext.ext_nextref; \ - (n)->m_ext.ext_prevref = (o); \ - (o)->m_ext.ext_nextref = (n); \ - (n)->m_ext.ext_nextref->m_ext.ext_prevref = (n); \ - splx(ms); \ - MCLREFDEBUGN((n), __FILE__, __LINE__); \ - } while (/* CONSTCOND */ 0) +#define MCLADDREFERENCE(o, n) m_extref((o), (n)) #define MCLINITREFERENCE(m) do { \ (m)->m_ext.ext_prevref = (m); \ @@ -411,7 +400,6 @@ void mbinit(void); struct mbuf *m_copym2(struct mbuf *, int, int, int); struct mbuf *m_copym(struct mbuf *, int, int, int); struct mbuf *m_free(struct mbuf *); -struct mbuf *m_free_unlocked(struct mbuf *); struct mbuf *m_get(int, int); struct mbuf *m_getclr(int, int); struct mbuf *m_gethdr(int, int); @@ -426,6 +414,7 @@ struct mbuf *m_getptr(struct mbuf *, in int m_leadingspace(struct mbuf *); int m_trailingspace(struct mbuf *); struct mbuf *m_clget(struct mbuf *, int, struct ifnet *, u_int); +void m_extref(struct mbuf *, struct mbuf *); void m_extfree_pool(caddr_t, u_int, void *); void m_adj(struct mbuf *, int); int m_copyback(struct mbuf *, int, int, const void *, int); Index: kern/uipc_mbuf.c =================================================================== RCS file: /cvs/src/sys/kern/uipc_mbuf.c,v retrieving revision 1.194 diff -u -p -r1.194 uipc_mbuf.c --- kern/uipc_mbuf.c 14 Sep 2014 14:17:26 -0000 1.194 +++ kern/uipc_mbuf.c 30 Sep 2014 11:07:02 -0000 @@ -81,12 +81,11 @@ #include #include #include - #include #include +#include #include - #include #ifdef DDB @@ -95,6 +94,7 @@ #endif struct mbstat mbstat; /* mbuf stats */ +struct mutex mbstatmtx = MUTEX_INITIALIZER(IPL_NET); struct pool mbpool; /* mbuf pool */ struct pool mtagpool; @@ -117,6 +117,8 @@ int max_linkhdr; /* largest link-level int max_protohdr; /* largest protocol header */ int max_hdr; /* largest link+protocol header */ +struct mutex m_extref_mtx = MUTEX_INITIALIZER(IPL_NET); + void m_extfree(struct mbuf *); struct mbuf *m_copym0(struct mbuf *, int, int, int, int); void nmbclust_update(void); @@ -135,11 +137,14 @@ mbinit(void) int i; #if DIAGNOSTIC + if (mclsizes[0] != MCLBYTES) + panic("mbinit: the smallest cluster size != MCLBYTES"); if (mclsizes[nitems(mclsizes) - 1] != MAXMCLBYTES) panic("mbinit: the largest cluster size != MAXMCLBYTES"); #endif pool_init(&mbpool, MSIZE, 0, 0, 0, "mbpl", NULL); + pool_setipl(&mbpool, IPL_NET); pool_set_constraints(&mbpool, &kp_dma_contig); pool_setlowat(&mbpool, mblowat); @@ -152,6 +157,7 @@ mbinit(void) mclsizes[i] >> 10); pool_init(&mclpools[i], mclsizes[i], 0, 0, 0, mclnames[i], NULL); + pool_setipl(&mclpools[i], IPL_NET); pool_set_constraints(&mclpools[i], &kp_dma_contig); pool_setlowat(&mclpools[i], mcllowat); } @@ -183,21 +189,6 @@ nmbclust_update(void) pool_sethiwat(&mbpool, nmbclust); } -void -m_reclaim(void *arg, int flags) -{ - struct domain *dp; - struct protosw *pr; - int s = splnet(); - - for (dp = domains; dp; dp = dp->dom_next) - for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) - if (pr->pr_drain) - (*pr->pr_drain)(); - mbstat.m_drain++; - splx(s); -} - /* * Space allocation routines. */ @@ -205,20 +196,21 @@ struct mbuf * m_get(int nowait, int type) { struct mbuf *m; - int s; - s = splnet(); m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : PR_NOWAIT); - if (m) - mbstat.m_mtypes[type]++; - splx(s); - if (m) { - m->m_type = type; - m->m_next = NULL; - m->m_nextpkt = NULL; - m->m_data = m->m_dat; - m->m_flags = 0; - } + if (m == NULL) + return (NULL); + + mtx_enter(&mbstatmtx); + mbstat.m_mtypes[type]++; + mtx_leave(&mbstatmtx); + + m->m_type = type; + m->m_next = NULL; + m->m_nextpkt = NULL; + m->m_data = m->m_dat; + m->m_flags = 0; + return (m); } @@ -230,25 +222,18 @@ struct mbuf * m_gethdr(int nowait, int type) { struct mbuf *m; - int s; - s = splnet(); m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : PR_NOWAIT); - if (m) - mbstat.m_mtypes[type]++; - splx(s); - if (m) { - m->m_type = type; - - /* keep in sync with m_inithdr */ - m->m_next = NULL; - m->m_nextpkt = NULL; - m->m_data = m->m_pktdat; - m->m_flags = M_PKTHDR; - memset(&m->m_pkthdr, 0, sizeof(m->m_pkthdr)); - m->m_pkthdr.pf.prio = IFQ_DEFPRIO; - } - return (m); + if (m == NULL) + return (NULL); + + mtx_enter(&mbstatmtx); + mbstat.m_mtypes[type]++; + mtx_leave(&mbstatmtx); + + m->m_type = type; + + return (m_inithdr(m)); } struct mbuf * @@ -298,7 +283,6 @@ m_clget(struct mbuf *m, int how, struct struct mbuf *m0 = NULL; struct pool *pp; caddr_t buf; - int s; pp = m_clpool(pktlen); #ifdef DIAGNOSTIC @@ -306,23 +290,19 @@ m_clget(struct mbuf *m, int how, struct panic("m_clget: request for %u byte cluster", pktlen); #endif - s = splnet(); if (m == NULL) { - MGETHDR(m0, M_DONTWAIT, MT_DATA); - if (m0 == NULL) { - splx(s); + m0 = m_gethdr(how, MT_DATA); + if (m0 == NULL) return (NULL); - } + m = m0; } buf = pool_get(pp, how == M_WAIT ? PR_WAITOK : PR_NOWAIT); if (buf == NULL) { if (m0) m_freem(m0); - splx(s); return (NULL); } - splx(s); MEXTADD(m, buf, pp->pr_size, M_EXTWR, m_extfree_pool, pp); return (m); @@ -331,16 +311,18 @@ m_clget(struct mbuf *m, int how, struct void m_extfree_pool(caddr_t buf, u_int size, void *pp) { - splassert(IPL_NET); pool_put(pp, buf); } struct mbuf * -m_free_unlocked(struct mbuf *m) +m_free(struct mbuf *m) { struct mbuf *n; + mtx_enter(&mbstatmtx); mbstat.m_mtypes[m->m_type]--; + mtx_leave(&mbstatmtx); + n = m->m_next; if (m->m_flags & M_ZEROIZE) { m_zero(m); @@ -352,54 +334,64 @@ m_free_unlocked(struct mbuf *m) m_tag_delete_chain(m); if (m->m_flags & M_EXT) m_extfree(m); + pool_put(&mbpool, m); return (n); } -struct mbuf * -m_free(struct mbuf *m) +void +m_extref(struct mbuf *o, struct mbuf *n) { - struct mbuf *n; - int s; + n->m_flags |= o->m_flags & (M_EXT|M_EXTWR); - s = splnet(); - n = m_free_unlocked(m); - splx(s); + mtx_enter(&m_extref_mtx); + n->m_ext.ext_nextref = o->m_ext.ext_nextref; + n->m_ext.ext_prevref = o; + o->m_ext.ext_nextref = n; + n->m_ext.ext_nextref->m_ext.ext_prevref = n; + mtx_leave(&m_extref_mtx); - return (n); + MCLREFDEBUGN((n), __FILE__, __LINE__); \ } -void -m_extfree(struct mbuf *m) +static inline u_int +m_extunref(struct mbuf *m) { + int refs = 1; + + if (!MCLISREFERENCED(m)) + return (0); + + mtx_enter(&m_extref_mtx); if (MCLISREFERENCED(m)) { m->m_ext.ext_nextref->m_ext.ext_prevref = m->m_ext.ext_prevref; m->m_ext.ext_prevref->m_ext.ext_nextref = m->m_ext.ext_nextref; - } else if (m->m_ext.ext_free) + } else + refs = 0; + mtx_leave(&m_extref_mtx); + + return (refs); +} + +void +m_extfree(struct mbuf *m) +{ + if (m_extunref(m) == 0) { (*(m->m_ext.ext_free))(m->m_ext.ext_buf, m->m_ext.ext_size, m->m_ext.ext_arg); - else - panic("unknown type of extension buffer"); - m->m_ext.ext_size = 0; + } + m->m_flags &= ~(M_EXT|M_EXTWR); } void m_freem(struct mbuf *m) { - struct mbuf *n; - int s; - - if (m == NULL) - return; - s = splnet(); - do { - n = m_free_unlocked(m); - } while ((m = n) != NULL); - splx(s); + while (m != NULL) + m = m_free(m); } /* @@ -435,12 +427,8 @@ m_defrag(struct mbuf *m, int how) /* free chain behind and possible ext buf on the first mbuf */ m_freem(m->m_next); m->m_next = NULL; - - if (m->m_flags & M_EXT) { - int s = splnet(); + if (m->m_flags & M_EXT) m_extfree(m); - splx(s); - } /* * Bounce copy mbuf over to the original mbuf and set everything up.