Index: kern/init_main.c =================================================================== RCS file: /cvs/src/sys/kern/init_main.c,v retrieving revision 1.260 diff -u -p -r1.260 init_main.c --- kern/init_main.c 21 Oct 2016 06:27:50 -0000 1.260 +++ kern/init_main.c 21 Oct 2016 07:01:18 -0000 @@ -415,6 +415,8 @@ main(void *framep) prof_init(); #endif + mbcpuinit(); /* enable per cpu mbuf data */ + /* init exec and emul */ init_exec(); Index: kern/kern_sysctl.c =================================================================== RCS file: /cvs/src/sys/kern/kern_sysctl.c,v retrieving revision 1.316 diff -u -p -r1.316 kern_sysctl.c --- kern/kern_sysctl.c 8 Oct 2016 21:31:56 -0000 1.316 +++ kern/kern_sysctl.c 21 Oct 2016 07:01:18 -0000 @@ -62,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -390,9 +391,24 @@ kern_sysctl(int *name, u_int namelen, vo case KERN_FILE: return (sysctl_file(name + 1, namelen - 1, oldp, oldlenp, p)); #endif - case KERN_MBSTAT: - return (sysctl_rdstruct(oldp, oldlenp, newp, &mbstat, - sizeof(mbstat))); + case KERN_MBSTAT: { + extern struct cpumem *mbstat; + uint64_t counters[MBSTAT_COUNT]; + struct mbstat mbs; + unsigned int i; + + memset(&mbs, 0, sizeof(mbs)); + counters_read(mbstat, counters, MBSTAT_COUNT); + for (i = 0; i < MBSTAT_TYPES; i++) + mbs.m_mtypes[i] = counters[i]; + + mbs.m_drops = counters[MBSTAT_DROPS]; + mbs.m_wait = counters[MBSTAT_WAIT]; + mbs.m_drain = counters[MBSTAT_DRAIN]; + + return (sysctl_rdstruct(oldp, oldlenp, newp, + &mbs, sizeof(mbs))); + } #if defined(GPROF) || defined(DDBPROF) case KERN_PROF: return (sysctl_doprof(name + 1, namelen - 1, oldp, oldlenp, Index: kern/uipc_mbuf.c =================================================================== RCS file: /cvs/src/sys/kern/uipc_mbuf.c,v retrieving revision 1.233 diff -u -p -r1.233 uipc_mbuf.c --- kern/uipc_mbuf.c 10 Oct 2016 00:41:17 -0000 1.233 +++ kern/uipc_mbuf.c 21 Oct 2016 07:01:18 -0000 @@ -83,6 +83,7 @@ #include #include #include +#include #include #include @@ -99,9 +100,11 @@ #include #endif /* NPF > 0 */ -struct mbstat mbstat; /* mbuf stats */ -struct mutex mbstatmtx = MUTEX_INITIALIZER(IPL_NET); -struct pool mbpool; /* mbuf pool */ +/* mbuf stats */ +COUNTERS_BOOT_MEMORY(mbstat_boot, MBSTAT_COUNT); +struct cpumem *mbstat = COUNTERS_BOOT_INITIALIZER(mbstat_boot); +/* mbuf pools */ +struct pool mbpool; struct pool mtagpool; /* mbuf cluster pools */ @@ -173,6 +176,12 @@ mbinit(void) } void +mbcpuinit() +{ + mbstat = counters_realloc(mbstat, MBSTAT_COUNT, M_DEVBUF); +} + +void nmbclust_update(void) { unsigned int i, n; @@ -204,14 +213,21 @@ struct mbuf * m_get(int nowait, int type) { struct mbuf *m; + struct counters_ref cr; + uint64_t *counters; + int s; + + KDASSERT(type < MT_NTYPES); m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : PR_NOWAIT); if (m == NULL) return (NULL); - mtx_enter(&mbstatmtx); - mbstat.m_mtypes[type]++; - mtx_leave(&mbstatmtx); + s = splnet(); + counters = counters_enter(&cr, mbstat); + counters[type]++; + counters_leave(&cr, mbstat); + splx(s); m->m_type = type; m->m_next = NULL; @@ -230,14 +246,21 @@ struct mbuf * m_gethdr(int nowait, int type) { struct mbuf *m; + struct counters_ref cr; + uint64_t *counters; + int s; + + KDASSERT(type < MT_NTYPES); m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : PR_NOWAIT); if (m == NULL) return (NULL); - mtx_enter(&mbstatmtx); - mbstat.m_mtypes[type]++; - mtx_leave(&mbstatmtx); + s = splnet(); + counters = counters_enter(&cr, mbstat); + counters[type]++; + counters_leave(&cr, mbstat); + splx(s); m->m_type = type; @@ -349,13 +372,18 @@ struct mbuf * m_free(struct mbuf *m) { struct mbuf *n; + struct counters_ref cr; + uint64_t *counters; + int s; if (m == NULL) return (NULL); - mtx_enter(&mbstatmtx); - mbstat.m_mtypes[m->m_type]--; - mtx_leave(&mbstatmtx); + s = splnet(); + counters = counters_enter(&cr, mbstat); + counters[m->m_type]--; + counters_leave(&cr, mbstat); + splx(s); n = m->m_next; if (m->m_flags & M_ZEROIZE) { Index: sys/mbuf.h =================================================================== RCS file: /cvs/src/sys/sys/mbuf.h,v retrieving revision 1.221 diff -u -p -r1.221 mbuf.h --- sys/mbuf.h 17 Oct 2016 02:53:47 -0000 1.221 +++ sys/mbuf.h 21 Oct 2016 07:01:19 -0000 @@ -236,6 +236,7 @@ struct mbuf { #define MT_FTABLE 5 /* fragment reassembly header */ #define MT_CONTROL 6 /* extra-data protocol message */ #define MT_OOBDATA 7 /* expedited data */ +#define MT_NTYPES 8 /* flowid field */ #define M_FLOWID_VALID 0x8000 /* is the flowid set */ @@ -393,6 +394,12 @@ struct mbstat { u_short m_mtypes[256]; /* type specific mbuf allocations */ }; +#define MBSTAT_TYPES MT_NTYPES +#define MBSTAT_DROPS (MBSTAT_TYPES + 0) +#define MBSTAT_WAIT (MBSTAT_TYPES + 1) +#define MBSTAT_DRAIN (MBSTAT_TYPES + 2) +#define MBSTAT_COUNT (MBSTAT_TYPES + 3) + #include struct mbuf_list { @@ -410,7 +417,6 @@ struct mbuf_queue { #ifdef _KERNEL -extern struct mbstat mbstat; extern int nmbclust; /* limit on the # of clusters */ extern int mblowat; /* mbuf low water mark */ extern int mcllowat; /* mbuf cluster low water mark */ @@ -419,6 +425,7 @@ extern int max_protohdr; /* largest pro extern int max_hdr; /* largest link+protocol header */ void mbinit(void); +void mbcpuinit(void); struct mbuf *m_copym(struct mbuf *, int, int, int); struct mbuf *m_free(struct mbuf *); struct mbuf *m_get(int, int);