Index: sys/arch/amd64/amd64/conf.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/conf.c,v
retrieving revision 1.70
diff -u -p -r1.70 conf.c
--- sys/arch/amd64/amd64/conf.c	25 May 2020 06:37:52 -0000	1.70
+++ sys/arch/amd64/amd64/conf.c	8 Jun 2020 07:43:44 -0000
@@ -142,6 +142,7 @@ cdev_decl(cy);
 #include "pctr.h"
 #include "bktr.h"
 #include "ksyms.h"
+#include "kstat.h"
 #include "usb.h"
 #include "uhid.h"
 #include "fido.h"
@@ -238,7 +239,7 @@ struct cdevsw	cdevsw[] =
 	cdev_notdef(),			/* 48 */
 	cdev_bktr_init(NBKTR,bktr),     /* 49: Bt848 video capture device */
 	cdev_ksyms_init(NKSYMS,ksyms),	/* 50: Kernel symbols device */
-	cdev_notdef(),			/* 51 */
+	cdev_kstat_init(NKSTAT,kstat),	/* 51: Kernel statistics */
 	cdev_midi_init(NMIDI,midi),	/* 52: MIDI I/O */
 	cdev_notdef(),			/* 53 was: sequencer I/O */
 	cdev_notdef(),			/* 54 was: RAIDframe disk driver */
Index: sys/conf/GENERIC
===================================================================
RCS file: /cvs/src/sys/conf/GENERIC,v
retrieving revision 1.269
diff -u -p -r1.269 GENERIC
--- sys/conf/GENERIC	9 May 2020 19:48:45 -0000	1.269
+++ sys/conf/GENERIC	8 Jun 2020 07:43:45 -0000
@@ -82,6 +82,7 @@ pseudo-device	msts	1	# MSTS line discipl
 pseudo-device	endrun	1	# EndRun line discipline
 pseudo-device	vnd	4	# vnode disk devices
 pseudo-device	ksyms	1	# kernel symbols device
+pseudo-device	kstat	1	# kernel statistics
 #pseudo-device	dt		# Dynamic Tracer
 
 # clonable devices
Index: sys/conf/files
===================================================================
RCS file: /cvs/src/sys/conf/files,v
retrieving revision 1.686
diff -u -p -r1.686 files
--- sys/conf/files	15 Apr 2020 09:26:49 -0000	1.686
+++ sys/conf/files	8 Jun 2020 07:43:45 -0000
@@ -570,6 +571,9 @@ pseudo-device switch: ifnet, ether
 pseudo-device ksyms
 file	dev/ksyms.c			ksyms needs-flag
 
+pseudo-device kstat
+file	dev/kstat.c			kstat needs-flag
+
 pseudo-device fuse
 file	miscfs/fuse/fuse_device.c	fuse	needs-flag
 file	miscfs/fuse/fuse_file.c		fuse
Index: sys/dev/kstat.c
===================================================================
RCS file: sys/dev/kstat.c
diff -N sys/dev/kstat.c
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ sys/dev/kstat.c	8 Jun 2020 07:43:45 -0000
@@ -0,0 +1,626 @@
+/* $OpenBSD$ */
+
+/*
+ * Copyright (c) 2020 David Gwynne <dlg@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+#include <sys/malloc.h>
+#include <sys/pool.h>
+#include <sys/time.h>
+
+#include <sys/kstat.h>
+
+RBT_HEAD(kstat_id_tree, kstat);
+
+static inline int
+kstat_id_cmp(const struct kstat *a, const struct kstat *b)
+{
+	if (a->ks_id > b->ks_id)
+		return (1);
+	if (a->ks_id < b->ks_id)
+		return (-1);
+
+	return (0);
+}
+
+RBT_PROTOTYPE(kstat_id_tree, kstat, ks_id_entry, kstat_id_cmp);
+
+RBT_HEAD(kstat_pv_tree, kstat);
+
+static inline int
+kstat_pv_cmp(const struct kstat *a, const struct kstat *b)
+{
+	int rv;
+
+	rv = strcmp(a->ks_provider, b->ks_provider);
+	if (rv != 0)
+		return (rv);
+
+	if (a->ks_instance > b->ks_instance)
+		return (1);
+	if (a->ks_instance < b->ks_instance)
+		return (-1);
+
+	rv = strcmp(a->ks_name, b->ks_name);
+	if (rv != 0)
+		return (rv);
+
+	if (a->ks_unit > b->ks_unit)
+		return (1);
+	if (a->ks_unit < b->ks_unit)
+		return (-1);
+
+	return (0);
+}
+
+RBT_PROTOTYPE(kstat_pv_tree, kstat, ks_pv_entry, kstat_pv_cmp);
+
+RBT_HEAD(kstat_nm_tree, kstat);
+
+static inline int
+kstat_nm_cmp(const struct kstat *a, const struct kstat *b)
+{
+	int rv;
+
+	rv = strcmp(a->ks_name, b->ks_name);
+	if (rv != 0)
+		return (rv);
+
+	if (a->ks_unit > b->ks_unit)
+		return (1);
+	if (a->ks_unit < b->ks_unit)
+		return (-1);
+
+	rv = strcmp(a->ks_provider, b->ks_provider);
+	if (rv != 0)
+		return (rv);
+
+	if (a->ks_instance > b->ks_instance)
+		return (1);
+	if (a->ks_instance < b->ks_instance)
+		return (-1);
+
+	return (0);
+}
+
+RBT_PROTOTYPE(kstat_nm_tree, kstat, ks_nm_entry, kstat_nm_cmp);
+
+struct kstat_lock_ops {
+	void	(*enter)(void *);
+	void	(*leave)(void *);
+};
+
+#define kstat_enter(_ks) (_ks)->ks_lock_ops->enter((_ks)->ks_lock)
+#define kstat_leave(_ks) (_ks)->ks_lock_ops->leave((_ks)->ks_lock)
+
+static const struct kstat_lock_ops kstat_rlock_ops = {
+	(void (*)(void *))rw_enter_read,
+	(void (*)(void *))rw_exit_read,
+};
+
+static const struct kstat_lock_ops kstat_wlock_ops = {
+	(void (*)(void *))rw_enter_write,
+	(void (*)(void *))rw_exit_write,
+};
+
+static const struct kstat_lock_ops kstat_mutex_ops = {
+	(void (*)(void *))mtx_enter,
+	(void (*)(void *))mtx_leave,
+};
+
+static struct rwlock		kstat_lock = RWLOCK_INITIALIZER("kstat");
+
+/*
+ * The global state is versioned so changes to the set of kstats
+ * can be detected. This is an int so it can be read atomically on
+ * any arch, which is ridiculous optimisation, really.
+ */
+static unsigned int		kstat_version = 0;
+
+/*
+ * kstat structures have a unique identifier so they can be found
+ * quickly. Identifiers are 64bit in the hope that it won't wrap
+ * during the runtime of a system. The identifiers start at 1 so that
+ * 0 can be used as the first value for userland to iterate with.
+ */
+static uint64_t			kstat_next_id = 1;
+
+static struct kstat_id_tree	kstat_id_tree = RBT_INITIALIZER();
+static struct kstat_pv_tree	kstat_pv_tree = RBT_INITIALIZER();
+static struct kstat_nm_tree	kstat_nm_tree = RBT_INITIALIZER();
+static struct pool		kstat_pool;
+
+static struct rwlock		kstat_default_lock = 
+				    RWLOCK_INITIALIZER("kstatlk");
+
+static int	kstat_read(struct kstat *);
+static int	kstat_copy(struct kstat *, void *);
+
+int
+kstatattach(int num)
+{
+	/* XXX install system stats here */
+	return (0);
+}
+
+int
+kstatopen(dev_t dev, int flag, int mode, struct proc *p)
+{
+	return (0);
+}
+
+int
+kstatclose(dev_t dev, int flag, int mode, struct proc *p)
+{
+	return (0);
+}
+
+static int
+kstatioc_enter(struct kstat_req *ksreq)
+{
+	int error;
+
+	error = rw_enter(&kstat_lock, RW_READ | RW_INTR);
+	if (error != 0)
+		return (error);
+
+	if (!ISSET(ksreq->ks_rflags, KSTATIOC_F_IGNVER) &&
+	    ksreq->ks_version != kstat_version) {
+		error = EBUSY;
+		goto error;
+	}
+
+	return (0);
+
+error:
+	rw_exit(&kstat_lock);
+	return (error);
+}
+
+#define sstrlcpy(_dst, _src) \
+    (strlcpy((_dst), (_src), sizeof((_dst))) >= sizeof((_dst)))
+
+static int
+kstatioc_leave(struct kstat_req *ksreq, struct kstat *ks)
+{
+	void *buf = NULL;
+	size_t klen = 0, ulen = 0;
+	struct timespec updated;
+	int error = 0;
+
+	if (ks == NULL) {
+		error = ENOENT;
+		goto error;
+	}
+
+	switch (ks->ks_state) {
+	case KSTAT_S_CREATED:
+		ksreq->ks_updated = ks->ks_created;
+		ksreq->ks_interval.tv_sec = 0;
+		ksreq->ks_interval.tv_nsec = 0;
+		ksreq->ks_datalen = 0;
+		ksreq->ks_dataver = 0;
+		break;
+
+	case KSTAT_S_INSTALLED:
+		ksreq->ks_dataver = ks->ks_dataver;
+		ksreq->ks_interval = ks->ks_interval;
+
+		if (ksreq->ks_data == NULL) {
+			/* userland doesn't want actual data, so shortcut */ 
+			kstat_enter(ks);
+			ksreq->ks_datalen = ks->ks_datalen;
+			ksreq->ks_updated = ks->ks_updated;
+			kstat_leave(ks);
+			break;
+		}
+
+		klen = ks->ks_datalen; /* KSTAT_F_REALLOC */
+		buf = malloc(klen, M_TEMP, M_WAITOK|M_CANFAIL);
+		if (buf == NULL) {
+			error = ENOMEM;
+			goto error;
+		}
+
+		kstat_enter(ks);
+		error = (*ks->ks_read)(ks);
+		if (error == 0) {
+			updated = ks->ks_updated;
+
+			/* KSTAT_F_REALLOC */
+			KASSERTMSG(ks->ks_datalen == klen,
+			    "kstat doesnt support resized data yet");
+
+			error = (*ks->ks_copy)(ks, buf);
+		}
+		kstat_leave(ks);
+
+		if (error != 0)
+			goto error;
+
+		ulen = ksreq->ks_datalen;
+		ksreq->ks_datalen = klen; /* KSTAT_F_REALLOC */
+		ksreq->ks_updated = updated;
+		break;
+	default:
+		panic("ks %p unexpected state %u", ks, ks->ks_state);
+	}
+
+	ksreq->ks_version = kstat_version;
+	ksreq->ks_id = ks->ks_id;
+
+	if (sstrlcpy(ksreq->ks_provider, ks->ks_provider) != 0)
+		panic("kstat provider string has grown");
+	ksreq->ks_instance = ks->ks_instance;
+	if (sstrlcpy(ksreq->ks_name, ks->ks_name) != 0)
+		panic("kstat name string has grown");
+	ksreq->ks_unit = ks->ks_unit;
+
+	ksreq->ks_created = ks->ks_created;
+	ksreq->ks_type = ks->ks_type;
+	ksreq->ks_state = ks->ks_state;
+
+error:
+	rw_exit(&kstat_lock);
+
+	if (buf != NULL) {
+		if (error == 0)
+			error = copyout(buf, ksreq->ks_data, min(klen, ulen));
+
+		free(buf, M_TEMP, klen);
+	}
+
+	return (error);
+}
+
+static int
+kstatioc_find_id(struct kstat_req *ksreq)
+{
+	struct kstat *ks, key;
+	int error;
+
+	error = kstatioc_enter(ksreq);
+	if (error != 0)
+		return (error);
+
+	key.ks_id = ksreq->ks_id;
+
+	ks = RBT_FIND(kstat_id_tree, &kstat_id_tree, &key);
+
+	return (kstatioc_leave(ksreq, ks));
+}
+
+static int
+kstatioc_nfind_id(struct kstat_req *ksreq)
+{
+	struct kstat *ks, key;
+	int error;
+
+	error = kstatioc_enter(ksreq);
+	if (error != 0)
+		return (error);
+
+	key.ks_id = ksreq->ks_id;
+
+	ks = RBT_NFIND(kstat_id_tree, &kstat_id_tree, &key);
+
+	return (kstatioc_leave(ksreq, ks));
+}
+
+static int
+kstatioc_find_pv(struct kstat_req *ksreq)
+{
+	struct kstat *ks, key;
+	int error;
+
+	error = kstatioc_enter(ksreq);
+	if (error != 0)
+		return (error);
+
+	key.ks_provider = ksreq->ks_provider;
+	key.ks_instance = ksreq->ks_instance;
+	key.ks_name = ksreq->ks_name;
+	key.ks_unit = ksreq->ks_unit;
+
+	ks = RBT_FIND(kstat_pv_tree, &kstat_pv_tree, &key);
+
+	return (kstatioc_leave(ksreq, ks));
+}
+
+static int
+kstatioc_nfind_pv(struct kstat_req *ksreq)
+{
+	struct kstat *ks, key;
+	int error;
+
+	error = kstatioc_enter(ksreq);
+	if (error != 0)
+		return (error);
+
+	key.ks_provider = ksreq->ks_provider;
+	key.ks_instance = ksreq->ks_instance;
+	key.ks_name = ksreq->ks_name;
+	key.ks_unit = ksreq->ks_unit;
+
+	ks = RBT_NFIND(kstat_pv_tree, &kstat_pv_tree, &key);
+
+	return (kstatioc_leave(ksreq, ks));
+}
+
+static int
+kstatioc_find_nm(struct kstat_req *ksreq)
+{
+	struct kstat *ks, key;
+	int error;
+
+	error = kstatioc_enter(ksreq);
+	if (error != 0)
+		return (error);
+
+	key.ks_name = ksreq->ks_name;
+	key.ks_unit = ksreq->ks_unit;
+	key.ks_provider = ksreq->ks_provider;
+	key.ks_instance = ksreq->ks_instance;
+
+	ks = RBT_FIND(kstat_nm_tree, &kstat_nm_tree, &key);
+
+	return (kstatioc_leave(ksreq, ks));
+}
+
+static int
+kstatioc_nfind_nm(struct kstat_req *ksreq)
+{
+	struct kstat *ks, key;
+	int error;
+
+	error = kstatioc_enter(ksreq);
+	if (error != 0)
+		return (error);
+
+	key.ks_name = ksreq->ks_name;
+	key.ks_unit = ksreq->ks_unit;
+	key.ks_provider = ksreq->ks_provider;
+	key.ks_instance = ksreq->ks_instance;
+
+	ks = RBT_NFIND(kstat_nm_tree, &kstat_nm_tree, &key);
+
+	return (kstatioc_leave(ksreq, ks));
+}
+
+int
+kstatioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
+{
+	struct kstat_req *ksreq = (struct kstat_req *)data;
+	int error = 0;
+
+	KERNEL_UNLOCK();
+
+	switch (cmd) {
+	case KSTATIOC_VERSION:
+		*(unsigned int *)data = kstat_version;
+		break;
+
+	case KSTATIOC_FIND_ID:
+		error = kstatioc_find_id(ksreq);
+		break;
+	case KSTATIOC_NFIND_ID:
+		error = kstatioc_nfind_id(ksreq);
+		break;
+	case KSTATIOC_FIND_PROVIDER:
+		error = kstatioc_find_pv(ksreq);
+		break;
+	case KSTATIOC_NFIND_PROVIDER:
+		error = kstatioc_nfind_pv(ksreq);
+		break;
+	case KSTATIOC_FIND_NAME:
+		error = kstatioc_find_nm(ksreq);
+		break;
+	case KSTATIOC_NFIND_NAME:
+		error = kstatioc_nfind_nm(ksreq);
+		break;
+
+	default:
+		error = ENOTTY;
+		break;
+	}
+
+	KERNEL_LOCK();
+
+	return (error);
+}
+
+static void
+kstat_init(void)
+{
+	static int initialized = 0;
+
+	if (initialized)
+		return;
+
+	pool_init(&kstat_pool, sizeof(struct kstat), 0, IPL_NONE,
+	    PR_WAITOK | PR_RWLOCK, "kstatmem", NULL);
+
+	initialized = 1;
+}
+
+struct kstat *
+kstat_create(const char *provider, unsigned int instance,
+    const char *name, unsigned int unit,
+    unsigned int type, unsigned int flags)
+{
+	struct kstat *ks, *oks;
+
+	KASSERT(strlen(provider) <= KSTAT_STRLEN);
+	KASSERT(strlen(name) <= KSTAT_STRLEN);
+
+	kstat_init();
+
+	ks = pool_get(&kstat_pool, PR_WAITOK|PR_ZERO);
+
+	ks->ks_provider = provider;
+	ks->ks_instance = instance;
+	ks->ks_name = name;
+	ks->ks_unit = unit;
+	ks->ks_flags = flags;
+	ks->ks_type = type;
+	ks->ks_state = KSTAT_S_CREATED;
+
+	getnanouptime(&ks->ks_created);
+	ks->ks_updated = ks->ks_created;
+
+	ks->ks_lock = &kstat_default_lock;
+	ks->ks_lock_ops = &kstat_wlock_ops;
+	ks->ks_read = kstat_read;
+	ks->ks_copy = kstat_copy;
+
+	rw_enter_write(&kstat_lock);
+	ks->ks_id = kstat_next_id;
+
+	oks = RBT_INSERT(kstat_pv_tree, &kstat_pv_tree, ks);
+	if (oks == NULL) {
+		/* commit */
+		kstat_next_id++;
+		kstat_version++;
+
+		oks = RBT_INSERT(kstat_nm_tree, &kstat_nm_tree, ks);
+		if (oks != NULL)
+			panic("kstat name collision! (%llu)", ks->ks_id);
+
+		oks = RBT_INSERT(kstat_id_tree, &kstat_id_tree, ks);
+		if (oks != NULL)
+			panic("kstat id collision! (%llu)", ks->ks_id);
+	}
+	rw_exit_write(&kstat_lock);
+
+	if (oks != NULL) {
+		pool_put(&kstat_pool, ks);
+		return (NULL);
+	}
+
+	return (ks);
+}
+
+void
+kstat_set_rlock(struct kstat *ks, struct rwlock *rwl)
+{
+	KASSERT(ks->ks_state == KSTAT_S_CREATED);
+
+	ks->ks_lock = rwl;
+	ks->ks_lock_ops = &kstat_rlock_ops;
+}
+
+void
+kstat_set_wlock(struct kstat *ks, struct rwlock *rwl)
+{
+	KASSERT(ks->ks_state == KSTAT_S_CREATED);
+
+	ks->ks_lock = rwl;
+	ks->ks_lock_ops = &kstat_wlock_ops;
+}
+
+void
+kstat_set_mutex(struct kstat *ks, struct mutex *mtx)
+{
+	KASSERT(ks->ks_state == KSTAT_S_CREATED);
+
+	ks->ks_lock = mtx;
+	ks->ks_lock_ops = &kstat_mutex_ops;
+}
+
+int
+kstat_read_nop(struct kstat *ks)
+{
+	return (0);
+}
+
+void
+kstat_install(struct kstat *ks)
+{
+	if (!ISSET(ks->ks_flags, KSTAT_F_REALLOC)) {
+		KASSERTMSG(ks->ks_copy != NULL || ks->ks_data != NULL,
+		    "kstat %s:%u:%s:%u must provide ks_copy or ks_data",
+		    ks->ks_provider, ks->ks_instance, ks->ks_name, ks->ks_unit);
+		KASSERT(ks->ks_datalen > 0);
+	}
+
+	rw_enter_write(&kstat_lock);
+	ks->ks_state = KSTAT_S_INSTALLED;
+	rw_exit_write(&kstat_lock);
+}
+
+void
+kstat_destroy(struct kstat *ks)
+{
+	rw_enter_write(&kstat_lock);
+	RBT_REMOVE(kstat_id_tree, &kstat_id_tree, ks);
+	RBT_REMOVE(kstat_pv_tree, &kstat_pv_tree, ks);
+	RBT_REMOVE(kstat_nm_tree, &kstat_nm_tree, ks);
+	kstat_version++;
+	rw_exit_write(&kstat_lock);
+
+	pool_put(&kstat_pool, ks);
+}
+
+static int
+kstat_read(struct kstat *ks)
+{
+	getnanouptime(&ks->ks_updated);
+	return (0);
+}
+
+static int
+kstat_copy(struct kstat *ks, void *buf)
+{
+	memcpy(buf, ks->ks_data, ks->ks_datalen);
+	return (0);
+}
+
+RBT_GENERATE(kstat_id_tree, kstat, ks_id_entry, kstat_id_cmp);
+RBT_GENERATE(kstat_pv_tree, kstat, ks_pv_entry, kstat_pv_cmp);
+RBT_GENERATE(kstat_nm_tree, kstat, ks_nm_entry, kstat_nm_cmp);
+
+void
+kstat_kv_init(struct kstat_kv *kv, const char *name, enum kstat_kv_type type)
+{
+	memset(kv, 0, sizeof(*kv));
+	strlcpy(kv->kv_key, name, sizeof(kv->kv_key)); /* XXX truncated? */
+	kv->kv_type = type;
+	kv->kv_unit = KSTAT_KV_U_NONE;
+}
+
+void
+kstat_kv_unit_init(struct kstat_kv *kv, const char *name,
+    enum kstat_kv_type type, enum kstat_kv_unit unit)
+{
+	switch (type) {
+	case KSTAT_KV_T_COUNTER64:
+	case KSTAT_KV_T_COUNTER32:
+	case KSTAT_KV_T_UINT64:
+	case KSTAT_KV_T_INT64:
+	case KSTAT_KV_T_UINT32:
+	case KSTAT_KV_T_INT32:
+		break;
+	default:
+		panic("kv unit init %s: unit for non-integer type", name);
+	}
+
+	memset(kv, 0, sizeof(*kv));
+	strlcpy(kv->kv_key, name, sizeof(kv->kv_key)); /* XXX truncated? */
+	kv->kv_type = type;
+	kv->kv_unit = unit;
+}
Index: sys/dev/pci/if_ix.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_ix.c,v
retrieving revision 1.166
diff -u -p -r1.166 if_ix.c
--- sys/dev/pci/if_ix.c	7 Jun 2020 23:52:05 -0000	1.166
+++ sys/dev/pci/if_ix.c	8 Jun 2020 07:43:45 -0000
@@ -145,15 +145,11 @@ void	ixgbe_rxrefill(void *);
 int	ixgbe_intr(struct ix_softc *sc);
 void	ixgbe_enable_intr(struct ix_softc *);
 void	ixgbe_disable_intr(struct ix_softc *);
-void	ixgbe_update_stats_counters(struct ix_softc *);
 int	ixgbe_txeof(struct tx_ring *);
 int	ixgbe_rxeof(struct rx_ring *);
 void	ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t);
 void	ixgbe_iff(struct ix_softc *);
-#ifdef IX_DEBUG
-void	ixgbe_print_hw_stats(struct ix_softc *);
 void	ixgbe_map_queue_statistics(struct ix_softc *);
-#endif
 void	ixgbe_update_link_status(struct ix_softc *);
 int	ixgbe_get_buf(struct rx_ring *, int);
 int	ixgbe_encap(struct tx_ring *, struct mbuf *);
@@ -186,6 +182,13 @@ void	ixgbe_rearm_queue(struct ix_softc *
 int	ixgbe_link_intr(void *);
 int	ixgbe_queue_intr(void *);
 
+#if NKSTAT > 0
+static void	ix_kstats(struct ix_softc *);
+static void	ix_rxq_kstats(struct ix_softc *, struct rx_ring *);
+static void	ix_txq_kstats(struct ix_softc *, struct tx_ring *);
+static void	ix_kstats_tick(void *);
+#endif
+
 /*********************************************************************
  *  OpenBSD Device Interface Entry Points
  *********************************************************************/
@@ -246,8 +249,9 @@ ixgbe_attach(struct device *parent, stru
 
 	rw_init(&sc->sfflock, "ixsff");
 
-	/* Set up the timer callout */
-	timeout_set(&sc->timer, ixgbe_local_timer, sc);
+#if NKSTAT > 0
+	ix_kstats(sc);
+#endif
 
 	/* Determine hardware revision */
 	ixgbe_identify_hardware(sc);
@@ -319,9 +323,6 @@ ixgbe_attach(struct device *parent, stru
 	/* Setup OS specific network interface */
 	ixgbe_setup_interface(sc);
 
-	/* Initialize statistics */
-	ixgbe_update_stats_counters(sc);
-
 	/* Get the PCI-E bus info and determine LAN ID */
 	hw->mac.ops.get_bus_info(hw);
 
@@ -376,7 +377,6 @@ ixgbe_detach(struct device *self, int fl
 	ether_ifdetach(ifp);
 	if_detach(ifp);
 
-	timeout_del(&sc->timer);
 	ixgbe_free_pci_resources(sc);
 
 	ixgbe_free_transmit_structures(sc);
@@ -384,6 +384,8 @@ ixgbe_detach(struct device *self, int fl
 	free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS *
 	    MAX_NUM_MULTICAST_ADDRESSES);
 
+	/* XXX kstat */
+
 	return (0);
 }
 
@@ -666,7 +668,10 @@ ixgbe_watchdog(struct ifnet * ifp)
 		    i, txr->next_to_clean);
 	}
 	ifp->if_flags &= ~IFF_RUNNING;
-	sc->watchdog_events++;
+
+#if NKSTAT > 0
+	//sc->watchdog_events++;
+#endif
 
 	ixgbe_init(sc);
 }
@@ -795,8 +800,6 @@ ixgbe_init(void *arg)
 	rxctrl |= IXGBE_RXCTRL_RXEN;
 	sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl);
 
-	timeout_add_sec(&sc->timer, 1);
-
 	/* Set up MSI/X routing */
 	if (sc->msix > 1) {
 		ixgbe_configure_ivars(sc);
@@ -857,6 +860,10 @@ ixgbe_init(void *arg)
 	for (i = 0; i < sc->num_queues; i++)
 		ifq_clr_oactive(ifp->if_ifqs[i]);
 
+#if NKSTAT > 0
+	ix_kstats_tick(sc);
+#endif
+
 	splx(s);
 }
 
@@ -1402,7 +1409,9 @@ ixgbe_encap(struct tx_ring *txr, struct 
 			break;
 		/* FALLTHROUGH */
 	default:
-		sc->no_tx_dma_setup++;
+#if NKSTAT > 0
+		//sc->no_tx_dma_setup++;
+#endif
 		return (0);
 	}
 
@@ -1445,7 +1454,9 @@ ixgbe_encap(struct tx_ring *txr, struct 
 
 	txr->next_avail_desc = i;
 
-	++txr->tx_packets;
+#if NKSTAT > 0
+	//++txr->tx_packets;
+#endif
 	return (ntxc + j);
 
 xmit_fail:
@@ -1518,30 +1529,6 @@ ixgbe_mc_array_itr(struct ixgbe_hw *hw, 
 }
 
 void
-ixgbe_local_timer(void *arg)
-{
-	struct ix_softc *sc = arg;
-#ifdef IX_DEBUG
-	struct ifnet	*ifp = &sc->arpcom.ac_if;
-#endif
-	int		 s;
-
-	s = splnet();
-
-	ixgbe_update_stats_counters(sc);
-
-#ifdef IX_DEBUG
-	if ((ifp->if_flags & (IFF_RUNNING|IFF_DEBUG)) ==
-	    (IFF_RUNNING|IFF_DEBUG))
-		ixgbe_print_hw_stats(sc);
-#endif
-
-	timeout_add_sec(&sc->timer, 1);
-
-	splx(s);
-}
-
-void
 ixgbe_update_link_status(struct ix_softc *sc)
 {
 	struct ifnet	*ifp = &sc->arpcom.ac_if;
@@ -1595,6 +1582,10 @@ ixgbe_stop(void *arg)
 	/* Tell the stack that the interface is no longer active */
 	ifp->if_flags &= ~IFF_RUNNING;
 
+#if NKSTAT > 0
+	timeout_del(&sc->sc_kstat_tmo);
+#endif
+
 	INIT_DEBUGOUT("ixgbe_stop: begin\n");
 	ixgbe_disable_intr(sc);
 
@@ -1606,7 +1597,6 @@ ixgbe_stop(void *arg)
 	/* Turn off the laser */
 	if (sc->hw.mac.ops.disable_tx_laser)
 		sc->hw.mac.ops.disable_tx_laser(&sc->hw);
-	timeout_del(&sc->timer);
 
 	/* reprogram the RAR[0] in case user changed it. */
 	ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);
@@ -1902,6 +1892,11 @@ ixgbe_setup_interface(struct ix_softc *s
 
 		ifiq->ifiq_softc = rxr;
 		rxr->ifiq = ifiq;
+
+#if NKSTAT > 0
+		ix_txq_kstats(sc, txr);
+		ix_rxq_kstats(sc, rxr);
+#endif
 	}
 
 	sc->max_frame_size = IXGBE_MAX_FRAME_SIZE;
@@ -2905,10 +2900,8 @@ ixgbe_initialize_receive_units(struct ix
 		rxcsum |= IXGBE_RXCSUM_PCSD;
 	}
 
-#ifdef IX_DEBUG
 	/* Map QPRC/QPRDC/QPTC on a per queue basis */
 	ixgbe_map_queue_statistics(sc);
-#endif
 
 	/* This is useful for calculating UDP/IP fragment checksums */
 	if (!(rxcsum & IXGBE_RXCSUM_PCSD))
@@ -3091,7 +3084,9 @@ ixgbe_rxeof(struct rx_ring *rxr)
 		eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
 
 		if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) {
-			sc->dropped_pkts++;
+#if NKSTAT > 0
+			//sc->dropped_pkts++;
+#endif
 
 			if (rxbuf->fmp) {
 				m_freem(rxbuf->fmp);
@@ -3156,10 +3151,12 @@ ixgbe_rxeof(struct rx_ring *rxr)
 			sendmp = NULL;
 			mp->m_next = nxbuf->buf;
 		} else { /* Sending this frame? */
-			rxr->rx_packets++;
+#if NKSTAT > 0
+			//rxr->rx_packets++;
 			/* capture data for AIM */
-			rxr->bytes += sendmp->m_pkthdr.len;
-			rxr->rx_bytes += sendmp->m_pkthdr.len;
+			//rxr->bytes += sendmp->m_pkthdr.len;
+			//rxr->rx_bytes += sendmp->m_pkthdr.len;
+#endif
 
 			ixgbe_rx_checksum(staterr, sendmp, ptype);
 
@@ -3507,164 +3504,378 @@ ixgbe_handle_phy(struct ix_softc *sc)
 
 }
 
+#if NKSTAT > 0
+struct ix_kstats {
+	struct kstat_kv crcerrs;
+
+	struct kstat_kv	lxontxc;
+	struct kstat_kv	lxonrxc;
+	struct kstat_kv	lxofftxc;
+	struct kstat_kv	lxoffrxc;
+
+	struct kstat_kv	prc64;
+	struct kstat_kv	prc127;
+	struct kstat_kv	prc255;
+	struct kstat_kv	prc511;
+	struct kstat_kv	prc1023;
+	struct kstat_kv	prc1522;
+
+	struct kstat_kv	gptc;
+	struct kstat_kv	gorc;
+	struct kstat_kv	gotc;
+	struct kstat_kv ruc;
+	struct kstat_kv rfc;
+	struct kstat_kv roc;
+	struct kstat_kv rjc;
+
+	struct kstat_kv tor;
+	struct kstat_kv tpr;
+	struct kstat_kv tpt;
+
+	struct kstat_kv	gprc;
+	struct kstat_kv	bprc;
+	struct kstat_kv	mprc;
+
+	struct kstat_kv	ptc64;
+	struct kstat_kv	ptc127;
+	struct kstat_kv	ptc255;
+	struct kstat_kv	ptc511;
+	struct kstat_kv	ptc1023;
+	struct kstat_kv	ptc1522;
+
+	struct kstat_kv	mptc;
+	struct kstat_kv	bptc;
+};
+
+struct ix_rxq_kstats {
+	struct kstat_kv	qprc;
+	struct kstat_kv	qbrc;
+	struct kstat_kv	qprdc;
+};
+
+struct ix_txq_kstats {
+	struct kstat_kv	qptc;
+	struct kstat_kv	qbtc;
+};
+
+static int	ix_kstats_read(struct kstat *ks);
+static int	ix_rxq_kstats_read(struct kstat *ks);
+static int	ix_txq_kstats_read(struct kstat *ks);
+
+static void
+ix_kstats(struct ix_softc *sc)
+{
+	struct ix_kstats *stats;
+	struct kstat *ks;
+
+	mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK);
+	timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc);
+
+	ks = kstat_create(sc->dev.dv_xname, 0, "ix", 0,
+	    KSTAT_T_KV, 0);
+	if (ks == NULL)
+		return;
+
+	stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO);
+
+	kstat_kv_unit_init(&stats->crcerrs, "Rx CRC Errs",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_kv_unit_init(&stats->lxontxc, "Link XON Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->lxonrxc, "Link XON Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->lxofftxc, "Link XOFF Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->lxoffrxc, "Link XOFF Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_kv_unit_init(&stats->prc64, "64B Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->prc127, "65-127B Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->prc255, "128-255B Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->prc511, "256-511B Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->prc1023, "512-1023B Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->prc1522, "1024-MaxB Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_kv_unit_init(&stats->gptc, "Good Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->gorc, "Good Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&stats->gotc, "Good Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+
+	kstat_kv_unit_init(&stats->ruc, "Rx Undersize",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->rfc, "Rx Fragment",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->roc, "Rx Oversize",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->rjc, "Rx Jabber",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_kv_unit_init(&stats->tor, "Total Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&stats->tpr, "Total Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->tpt, "Total Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_kv_unit_init(&stats->gprc, "Good Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->bprc, "Broadcast Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->mprc, "Multicast Rx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_kv_unit_init(&stats->ptc64, "64B Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->ptc127, "65-127B Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->ptc255, "128-255B Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->ptc511, "256-511B Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->ptc1023, "512-1023B Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->ptc1522, "1024-MaxB Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_kv_unit_init(&stats->mptc, "Multicast Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->bptc, "Broadcast Tx",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_set_mutex(ks, &sc->sc_kstat_mtx);
+	ks->ks_softc = sc;
+	ks->ks_data = stats;
+	ks->ks_datalen = sizeof(*stats);
+	ks->ks_read = ix_kstats_read;
+
+	sc->sc_kstat = ks;
+	kstat_install(ks);
+}
+
+static void
+ix_rxq_kstats(struct ix_softc *sc, struct rx_ring *rxr)
+{
+	struct ix_rxq_kstats *stats;
+	struct kstat *ks;
+
+	ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me,
+	    KSTAT_T_KV, 0);
+	if (ks == NULL)
+		return;
+
+	stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO);
+
+	kstat_kv_unit_init(&stats->qprc, "packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->qbrc, "bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&stats->qprdc, "qdrops",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_set_mutex(ks, &sc->sc_kstat_mtx);
+	ks->ks_softc = rxr;
+	ks->ks_data = stats;
+	ks->ks_datalen = sizeof(*stats);
+	ks->ks_read = ix_rxq_kstats_read;
+
+	rxr->kstat = ks;
+	kstat_install(ks);
+}
+
+static void
+ix_txq_kstats(struct ix_softc *sc, struct tx_ring *txr)
+{
+	struct ix_txq_kstats *stats;
+	struct kstat *ks;
+
+	ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me,
+	    KSTAT_T_KV, 0);
+	if (ks == NULL)
+		return;
+
+	stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO);
+
+	kstat_kv_unit_init(&stats->qptc, "packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&stats->qbtc, "bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+
+	kstat_set_mutex(ks, &sc->sc_kstat_mtx);
+	ks->ks_softc = txr;
+	ks->ks_data = stats;
+	ks->ks_datalen = sizeof(*stats);
+	ks->ks_read = ix_txq_kstats_read;
+
+	txr->kstat = ks;
+	kstat_install(ks);
+}
+
 /**********************************************************************
  *
  *  Update the board statistics counters.
  *
  **********************************************************************/
-void
-ixgbe_update_stats_counters(struct ix_softc *sc)
+
+static void
+ix_kstats_tick(void *arg)
 {
-	struct ifnet	*ifp = &sc->arpcom.ac_if;
+	struct ix_softc *sc = arg;
+	int i;
+
+	timeout_add_sec(&sc->sc_kstat_tmo, 1);
+
+	mtx_enter(&sc->sc_kstat_mtx);
+	ix_kstats_read(sc->sc_kstat);
+	for (i = 0; i < sc->num_queues; i++) {
+		ix_rxq_kstats_read(sc->rx_rings[i].kstat);
+		ix_txq_kstats_read(sc->tx_rings[i].kstat);
+	}
+	mtx_leave(&sc->sc_kstat_mtx);
+}
+
+static uint64_t
+ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg)
+{
+	uint64_t lo, hi;
+
+	lo = IXGBE_READ_REG(hw, loreg);
+	hi = IXGBE_READ_REG(hw, hireg);
+
+	return (((hi & 0xf) << 32) | lo);
+}
+
+static int
+ix_kstats_read(struct kstat *ks)
+{
+	struct ix_kstats *stats = ks->ks_data;
+	struct ix_softc *sc = ks->ks_softc;
 	struct ixgbe_hw	*hw = &sc->hw;
-	uint64_t	crcerrs, rlec, total_missed_rx = 0;
-#ifdef IX_DEBUG
-	uint32_t	missed_rx = 0, bprc, lxon, lxoff, total;
-	int		i;
-#endif
 
-	crcerrs = IXGBE_READ_REG(hw, IXGBE_CRCERRS);
-	sc->stats.crcerrs += crcerrs;
-	rlec = IXGBE_READ_REG(hw, IXGBE_RLEC);
-	sc->stats.rlec += rlec;
-
-#ifdef IX_DEBUG
-	for (i = 0; i < 8; i++) {
-		uint32_t mp;
-		mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
-		/* missed_rx tallies misses for the gprc workaround */
-		missed_rx += mp;
-		/* global total per queue */
-		sc->stats.mpc[i] += mp;
-		/* running comprehensive total for stats display */
-		total_missed_rx += sc->stats.mpc[i];
-		if (hw->mac.type == ixgbe_mac_82598EB)
-			sc->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
-	}
-
-	/* Hardware workaround, gprc counts missed packets */
-	sc->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
-	sc->stats.gprc -= missed_rx;
+	stats->crcerrs.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
 
-	if (hw->mac.type != ixgbe_mac_82598EB) {
-		sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
-		    ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
-		sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
-		    ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
-		sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
-		    ((uint64_t)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
-		sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
-		sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
+	stats->lxontxc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_LXONTXC);
+	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
+		stats->lxonrxc.kv_v.v_u64 +=
+		    IXGBE_READ_REG(hw, IXGBE_LXONRXC);
 	} else {
-		sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
-		sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
-		/* 82598 only has a counter in the high register */
-		sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
-		sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
-		sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
+		stats->lxonrxc.kv_v.v_u64 +=
+		    IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
+	}
+	stats->lxofftxc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
+	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
+		stats->lxoffrxc.kv_v.v_u64 +=
+		    IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
+	} else {
+		stats->lxoffrxc.kv_v.v_u64 +=
+		    IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
 	}
 
-	/*
-	 * Workaround: mprc hardware is incorrectly counting
-	 * broadcasts, so for now we subtract those.
-	 */
-	bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
-	sc->stats.bprc += bprc;
-	sc->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
-	if (hw->mac.type == ixgbe_mac_82598EB)
-		sc->stats.mprc -= bprc;
-
-	sc->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
-	sc->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
-	sc->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
-	sc->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
-	sc->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
-	sc->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
-	sc->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
-
-	lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
-	sc->stats.lxontxc += lxon;
-	lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
-	sc->stats.lxofftxc += lxoff;
-	total = lxon + lxoff;
-
-	sc->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
-	sc->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
-	sc->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
-	sc->stats.gptc -= total;
-	sc->stats.mptc -= total;
-	sc->stats.ptc64 -= total;
-	sc->stats.gotc -= total * ETHER_MIN_LEN;
-
-	sc->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
-	sc->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
-	sc->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
-	sc->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
-	sc->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
-	sc->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
-	sc->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
-	sc->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
-	sc->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
-	sc->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
-	for (i = 0; i < 16; i++) {
-		uint32_t dropped;
-
-		dropped = IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
-		sc->stats.qprdc[i] += dropped;
-		missed_rx += dropped;
-		sc->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
-		sc->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
+	stats->prc64.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
+	stats->prc127.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PRC127);
+	stats->prc255.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PRC255);
+	stats->prc511.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PRC511);
+	stats->prc1023.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
+	stats->prc1522.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
+
+	stats->gptc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_GPTC);
+	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
+		stats->gorc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_GORCH);
+		stats->gotc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_GOTCH);
+	} else {
+		stats->gorc.kv_v.v_u64 +=
+		    ix_read36(hw, IXGBE_GORCL, IXGBE_GORCH);
+		stats->gotc.kv_v.v_u64 +=
+		    ix_read36(hw, IXGBE_GOTCL, IXGBE_GOTCH);
 	}
-#endif
 
-	/* Fill out the OS statistics structure */
-	ifp->if_collisions = 0;
-	ifp->if_oerrors = sc->watchdog_events;
-	ifp->if_ierrors = total_missed_rx + crcerrs + rlec;
+	stats->ruc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_RUC);
+	stats->rfc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_RFC);
+	stats->roc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_ROC);
+	stats->rjc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_RJC);
+
+	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
+		stats->tor.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_TORH);
+	} else {
+		stats->tor.kv_v.v_u64 +=
+		    ix_read36(hw, IXGBE_TORL, IXGBE_TORH);
+	}
+	stats->tpr.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_TPR);
+	stats->tpt.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_TPT);
+
+	stats->gprc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_GPRC);
+	stats->bprc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_BPRC);
+	stats->mprc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_MPRC);
+
+	stats->ptc64.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
+	stats->ptc127.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PTC127);
+	stats->ptc255.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PTC255);
+	stats->ptc511.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PTC511);
+	stats->ptc1023.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
+	stats->ptc1522.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
+
+	stats->mptc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_MPTC);
+	stats->bptc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_BPTC);
+
+	getnanouptime(&ks->ks_updated);
+
+	return (0);
 }
 
-#ifdef IX_DEBUG
-/**********************************************************************
- *
- *  This routine is called only when ixgbe_display_debug_stats is enabled.
- *  This routine provides a way to take a look at important statistics
- *  maintained by the driver and hardware.
- *
- **********************************************************************/
-void
-ixgbe_print_hw_stats(struct ix_softc * sc)
+int
+ix_rxq_kstats_read(struct kstat *ks)
 {
-	struct ifnet   *ifp = &sc->arpcom.ac_if;
-	int i;
+	struct ix_rxq_kstats *stats = ks->ks_data;
+	struct rx_ring *rxr = ks->ks_softc;
+	struct ix_softc *sc = rxr->sc;
+	struct ixgbe_hw	*hw = &sc->hw;
+	uint32_t i = rxr->me;
 
-	printf("%s: missed pkts %llu, rx len errs %llu, crc errs %llu, "
-	    "dropped pkts %lu, watchdog timeouts %ld, "
-	    "XON rx %llu, XON tx %llu, XOFF rx %llu, XOFF tx %llu, "
-	    "total pkts rx %llu, good pkts rx %llu, good pkts tx %llu, "
-	    "tso tx %lu\n",
-	    ifp->if_xname,
-	    (long long)sc->stats.mpc[0],
-	    (long long)sc->stats.roc + (long long)sc->stats.ruc,
-	    (long long)sc->stats.crcerrs,
-	    sc->dropped_pkts,
-	    sc->watchdog_events,
-	    (long long)sc->stats.lxonrxc,
-	    (long long)sc->stats.lxontxc,
-	    (long long)sc->stats.lxoffrxc,
-	    (long long)sc->stats.lxofftxc,
-	    (long long)sc->stats.tpr,
-	    (long long)sc->stats.gprc,
-	    (long long)sc->stats.gptc,
-	    sc->tso_tx);
+	stats->qprc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
+	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
+		stats->qprdc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
+		stats->qbrc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
+	} else {
+		stats->qprdc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
+		stats->qbrc.kv_v.v_u64 +=
+		    ix_read36(hw, IXGBE_QBRC_L(i), IXGBE_QBRC_H(i));
+	}
 
-	printf("%s: per queue statistics\n", ifp->if_xname);
-	for (i = 0; i < sc->num_queues; i++) {
-		printf("\tqueue %d: rx pkts %llu, rx drops %llu, "
-		    "tx pkts %llu\n",
-		    i, sc->stats.qprc[i], sc->stats.qprdc[i],
-		    sc->stats.qptc[i]);
+	getnanouptime(&ks->ks_updated);
+
+	return (0);
+}
+
+int
+ix_txq_kstats_read(struct kstat *ks)
+{
+	struct ix_txq_kstats *stats = ks->ks_data;
+	struct rx_ring *txr = ks->ks_softc;
+	struct ix_softc *sc = txr->sc;
+	struct ixgbe_hw	*hw = &sc->hw;
+	uint32_t i = txr->me;
+
+	stats->qptc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
+	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
+		stats->qbtc.kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
+	} else {
+		stats->qbtc.kv_v.v_u64 +=
+		    ix_read36(hw, IXGBE_QBTC_L(i), IXGBE_QBTC_H(i));
 	}
+
+	getnanouptime(&ks->ks_updated);
+
+	return (0);
 }
+#endif /* NKVSTAT > 0 */
 
 void
 ixgbe_map_queue_statistics(struct ix_softc *sc)
@@ -3692,4 +3903,3 @@ ixgbe_map_queue_statistics(struct ix_sof
 		IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r);
 	}
 }
-#endif
Index: sys/dev/pci/if_ix.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_ix.h,v
retrieving revision 1.41
diff -u -p -r1.41 if_ix.h
--- sys/dev/pci/if_ix.h	7 Jun 2020 23:52:05 -0000	1.41
+++ sys/dev/pci/if_ix.h	8 Jun 2020 07:43:45 -0000
@@ -182,10 +182,8 @@ struct tx_ring {
 	}			queue_status;
 	uint32_t		txd_cmd;
 	bus_dma_tag_t		txtag;
-	uint32_t		bytes; /* Used for AIM calc */
-	uint32_t		packets;
-	/* Soft Stats */
-	uint64_t		tx_packets;
+
+	struct kstat		*kstat;
 };
 
 
@@ -211,15 +209,7 @@ struct rx_ring {
 	struct if_rxring	rx_ring;
 	struct ixgbe_rx_buf	*rx_buffers;
 
-	uint32_t		bytes; /* Used for AIM calc */
-	uint32_t		packets;
-
-	/* Soft stats */
-	uint64_t		rx_irq;
-	uint64_t		rx_packets;
-	uint64_t		rx_bytes;
-	uint64_t		rx_discarded;
-	uint64_t		rsc_num;
+	struct kstat		*kstat;
 };
 
 /* Our adapter structure */
@@ -233,7 +223,6 @@ struct ix_softc {
 	void			*tag;
 
 	struct ifmedia		media;
-	struct timeout		timer;
 	int			msix;
 	int			if_flags;
 
@@ -288,14 +277,9 @@ struct ix_softc {
 	uint8_t			*mta;
 
 	/* Misc stats maintained by the driver */
-	unsigned long		dropped_pkts;
-	unsigned long		no_tx_map_avail;
-	unsigned long		no_tx_dma_setup;
-	unsigned long		watchdog_events;
-	unsigned long		tso_tx;
-	unsigned long		link_irq;
-
-	struct ixgbe_hw_stats 	stats;
+	struct mutex		 sc_kstat_mtx;
+	struct timeout		 sc_kstat_tmo;
+	struct kstat		*sc_kstat;
 };
 
 #endif /* _IX_H_ */
Index: sys/dev/pci/if_vmx.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_vmx.c,v
retrieving revision 1.56
diff -u -p -r1.56 if_vmx.c
--- sys/dev/pci/if_vmx.c	28 May 2020 07:21:56 -0000	1.56
+++ sys/dev/pci/if_vmx.c	8 Jun 2020 07:43:45 -0000
@@ -17,6 +17,7 @@
  */
 
 #include "bpfilter.h"
+#include "kstat.h"
 
 #include <sys/param.h>
 #include <sys/device.h>
@@ -25,8 +26,10 @@
 #include <sys/sockio.h>
 #include <sys/systm.h>
 #include <sys/atomic.h>
+#include <sys/kstat.h>
 
 #include <net/bpf.h>
+#include <net/toeplitz.h>
 #include <net/if.h>
 #include <net/if_media.h>
 
@@ -42,7 +45,7 @@
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcidevs.h>
 
-#define VMX_MAX_QUEUES	1
+#define VMX_MAX_QUEUES	4
 
 #define NTXDESC 512 /* tx ring size */
 #define NTXSEGS 8 /* tx descriptors per packet */
@@ -90,18 +93,50 @@ struct vmxnet3_comp_ring {
 	u_int32_t gen;
 };
 
+struct vmx_txstats_kv {
+	struct kstat_kv	tso_packets;
+	struct kstat_kv	tso_bytes;
+	struct kstat_kv	ucast_packets;
+	struct kstat_kv	ucast_bytes;
+	struct kstat_kv	mcast_packets;
+	struct kstat_kv	mcast_bytes;
+	struct kstat_kv	bcast_packets;
+	struct kstat_kv	bcast_bytes;
+	struct kstat_kv	errors;
+	struct kstat_kv	discards;
+};
+
 struct vmxnet3_txqueue {
+	struct vmxnet3_softc *sc; /* sigh */
 	struct vmxnet3_txring cmd_ring;
 	struct vmxnet3_comp_ring comp_ring;
 	struct vmxnet3_txq_shared *ts;
 	struct ifqueue *ifq;
+
+	struct kstat *txkstat;
+};
+
+struct vmx_rxstats_kv {
+	struct kstat_kv	lro_packets;
+	struct kstat_kv	lro_bytes;
+	struct kstat_kv	ucast_packets;
+	struct kstat_kv	ucast_bytes;
+	struct kstat_kv	mcast_packets;
+	struct kstat_kv	mcast_bytes;
+	struct kstat_kv	bcast_packets;
+	struct kstat_kv	bcast_bytes;
+	struct kstat_kv	nobuffers;
+	struct kstat_kv	errors;
 };
 
 struct vmxnet3_rxqueue {
+	struct vmxnet3_softc *sc; /* sigh */
 	struct vmxnet3_rxring cmd_ring[2];
 	struct vmxnet3_comp_ring comp_ring;
 	struct vmxnet3_rxq_shared *rs;
 	struct ifiqueue *ifiq;
+
+	struct kstat *rxkstat;
 };
 
 struct vmxnet3_queue {
@@ -112,6 +147,14 @@ struct vmxnet3_queue {
 	int intr;
 };
 
+struct vmx_kstats {
+	struct rwlock		lock;
+	struct timeval		updated;
+
+	struct vmx_txstats_kv	txstats;
+	struct vmx_rxstats_kv	rxstats;
+};
+
 struct vmxnet3_softc {
 	struct device sc_dev;
 	struct arpcom sc_arpcom;
@@ -130,6 +173,11 @@ struct vmxnet3_softc {
 	struct vmxnet3_queue sc_q[VMX_MAX_QUEUES];
 	struct vmxnet3_driver_shared *sc_ds;
 	u_int8_t *sc_mcast;
+	struct vmxnet3_upt1_rss_conf *sc_rss;
+
+#if NKSTAT > 0
+	struct vmx_kstats	sc_kstats;
+#endif
 };
 
 #define VMXNET3_STAT
@@ -196,6 +244,14 @@ void vmxnet3_media_status(struct ifnet *
 int vmxnet3_media_change(struct ifnet *);
 void *vmxnet3_dma_allocmem(struct vmxnet3_softc *, u_int, u_int, bus_addr_t *);
 
+#if NKSTAT > 0
+static void	vmx_kstat_init(struct vmxnet3_softc *);
+static void	vmx_kstat_txstats(struct vmxnet3_softc *,
+		    struct vmxnet3_txqueue *, int);
+static void	vmx_kstat_rxstats(struct vmxnet3_softc *,
+		    struct vmxnet3_rxqueue *, int);
+#endif
+
 const struct pci_matchid vmx_devices[] = {
 	{ PCI_VENDOR_VMWARE, PCI_PRODUCT_VMWARE_NET_3 }
 };
@@ -249,7 +305,7 @@ vmxnet3_attach(struct device *parent, st
 
 	ver = READ_BAR1(sc, VMXNET3_BAR1_UVRS);
 	if ((ver & 0x1) == 0) {
-		printf(": incompatiable UPT version 0x%x\n", ver);
+		printf(": incompatible UPT version 0x%x\n", ver);
 		return;
 	}
 	WRITE_BAR1(sc, VMXNET3_BAR1_UVRS, 1);
@@ -267,6 +323,7 @@ vmxnet3_attach(struct device *parent, st
 	case VMXNET3_INTRCFG_TYPE_MSIX:
 		if (pci_intr_map_msix(pa, 0, &ih) == 0) {
 			isr = vmxnet3_intr_event;
+			sc->sc_nqueues = VMX_MAX_QUEUES;
 			sc->sc_nintr = sc->sc_nqueues + 1;
 			break;
 		}
@@ -367,12 +424,21 @@ vmxnet3_attach(struct device *parent, st
 	ether_ifattach(ifp);
 	vmxnet3_link_state(sc);
 
-	if_attach_queues(ifp, sc->sc_nqueues);
-	if_attach_iqueues(ifp, sc->sc_nqueues);
+#if NKSTAT > 0
+	vmx_kstat_init(sc);
+#endif
+
+        if_attach_queues(ifp, sc->sc_nqueues);
+        if_attach_iqueues(ifp, sc->sc_nqueues);
 	for (i = 0; i < sc->sc_nqueues; i++) {
 		ifp->if_ifqs[i]->ifq_softc = &sc->sc_q[i].tx;
 		sc->sc_q[i].tx.ifq = ifp->if_ifqs[i];
 		sc->sc_q[i].rx.ifiq = ifp->if_iqs[i];
+
+#if NKSTAT > 0
+		vmx_kstat_txstats(sc, &sc->sc_q[i].tx, i);
+		vmx_kstat_rxstats(sc, &sc->sc_q[i].rx, i);
+#endif
 	}
 }
 
@@ -441,7 +507,7 @@ vmxnet3_dma_init(struct vmxnet3_softc *s
 #endif
 	ds->vmxnet3_revision = 1;
 	ds->upt_version = 1;
-	ds->upt_features = UPT1_F_CSUM | UPT1_F_VLAN;
+	ds->upt_features = UPT1_F_CSUM | UPT1_F_VLAN | UPT1_F_RSS;
 	ds->driver_data = vtophys(sc);
 	ds->driver_data_len = sizeof(struct vmxnet3_softc);
 	ds->queue_shared = qs_pa;
@@ -456,6 +522,33 @@ vmxnet3_dma_init(struct vmxnet3_softc *s
 	ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
 	for (i = 0; i < sc->sc_nintr; i++)
 		ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
+
+	if (sc->sc_nqueues > 0) {
+		struct vmxnet3_upt1_rss_conf *rsscfg;
+		bus_addr_t rss_pa;
+
+		rsscfg = vmxnet3_dma_allocmem(sc, sizeof(*rsscfg), 8, &rss_pa);
+
+		rsscfg->hash_type = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
+		    UPT1_RSS_HASH_TYPE_IPV4 |
+		    UPT1_RSS_HASH_TYPE_TCP_IPV6 |
+		    UPT1_RSS_HASH_TYPE_IPV6;
+		rsscfg->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
+		rsscfg->hash_key_size = sizeof(rsscfg->hash_key);
+		stoeplitz_to_key(rsscfg->hash_key, sizeof(rsscfg->hash_key));
+
+		rsscfg->ind_table_size = sizeof(rsscfg->ind_table);
+		for (i = 0; i < sizeof(rsscfg->ind_table); i++)
+			rsscfg->ind_table[i] = i % sc->sc_nqueues;
+
+		ds->upt_features |= UPT1_F_RSS;
+		ds->rss.version = 1;
+		ds->rss.len = sizeof(*rsscfg);
+		ds->rss.paddr = rss_pa;
+
+		sc->sc_rss = rsscfg;
+	}
+
 	WRITE_BAR1(sc, VMXNET3_BAR1_DSL, ds_pa);
 	WRITE_BAR1(sc, VMXNET3_BAR1_DSH, (u_int64_t)ds_pa >> 32);
 	return 0;
@@ -635,6 +728,157 @@ vmxnet3_rxfill(struct vmxnet3_rxring *ri
 		timeout_add(&ring->refill, 1);
 }
 
+#if NKSTAT > 0
+static const struct timeval vmx_kstat_rate = { 1, 0 };
+
+static void
+vmx_kstat_init(struct vmxnet3_softc *sc)
+{
+	struct vmx_txstats_kv *txkvs = &sc->sc_kstats.txstats;
+	struct vmx_rxstats_kv *rxkvs = &sc->sc_kstats.rxstats;
+
+	rw_init(&sc->sc_kstats.lock, "vmxstats");
+
+	kstat_kv_unit_init(&txkvs->tso_packets, "TSO packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&txkvs->tso_bytes, "TSO bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&txkvs->ucast_packets, "ucast packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&txkvs->ucast_bytes, "ucast bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&txkvs->mcast_packets, "mcast packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&txkvs->mcast_bytes, "mcast bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&txkvs->bcast_packets, "bcast packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&txkvs->bcast_bytes, "bcast bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&txkvs->errors, "errors",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&txkvs->discards, "discards",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+
+	kstat_kv_unit_init(&rxkvs->lro_packets, "LRO packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&rxkvs->lro_bytes, "LRO bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&rxkvs->ucast_packets, "ucast packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&rxkvs->ucast_bytes, "ucast bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&rxkvs->mcast_packets, "mcast packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&rxkvs->mcast_bytes, "mcast bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&rxkvs->bcast_packets, "bcast packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&rxkvs->bcast_bytes, "bcast bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES);
+	kstat_kv_unit_init(&rxkvs->nobuffers, "no buffers",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+	kstat_kv_unit_init(&rxkvs->errors, "errors",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS);
+}
+
+static int
+vmx_txstats_read(struct kstat *ks)
+{
+	struct vmxnet3_txqueue *tq = ks->ks_softc;
+	struct vmxnet3_softc *sc = tq->sc;
+	struct vmx_txstats_kv *txkvs = ks->ks_data;
+	struct UPT1_TxStats *txstats = &tq->ts->stats;
+
+	if (ratecheck(&sc->sc_kstats.updated, &vmx_kstat_rate))
+		WRITE_CMD(sc, VMXNET3_CMD_GET_STATS);
+
+	txkvs->tso_packets.kv_v.v_u64 = txstats->TSO_packets;
+	txkvs->tso_bytes.kv_v.v_u64 = txstats->TSO_bytes;
+	txkvs->ucast_packets.kv_v.v_u64 = txstats->ucast_packets;
+	txkvs->ucast_bytes.kv_v.v_u64 = txstats->ucast_bytes;
+	txkvs->mcast_packets.kv_v.v_u64 = txstats->mcast_packets;
+	txkvs->mcast_bytes.kv_v.v_u64 = txstats->mcast_bytes;
+	txkvs->bcast_packets.kv_v.v_u64 = txstats->bcast_packets;
+	txkvs->bcast_bytes.kv_v.v_u64 = txstats->bcast_bytes;
+	txkvs->errors.kv_v.v_u64 = txstats->error;
+	txkvs->discards.kv_v.v_u64 = txstats->discard;
+
+	TIMEVAL_TO_TIMESPEC(&sc->sc_kstats.updated, &ks->ks_updated);
+
+	return (0);
+}
+
+static void
+vmx_kstat_txstats(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq, int i)
+{
+	tq->sc = sc;
+
+	tq->txkstat = kstat_create(sc->sc_dev.dv_xname, 0, "vmx-txstats", i,
+	    KSTAT_T_KV, 0);
+	if (tq->txkstat == NULL)
+		return;
+
+	kstat_set_wlock(tq->txkstat, &sc->sc_kstats.lock);
+
+	tq->txkstat->ks_softc = tq;
+	tq->txkstat->ks_data = &sc->sc_kstats.txstats;
+	tq->txkstat->ks_datalen = sizeof(sc->sc_kstats.txstats);
+	tq->txkstat->ks_read = vmx_txstats_read;
+	TIMEVAL_TO_TIMESPEC(&vmx_kstat_rate, &tq->txkstat->ks_interval);
+
+	kstat_install(tq->txkstat);
+}
+
+static int
+vmx_rxstats_read(struct kstat *ks)
+{
+	struct vmxnet3_rxqueue *rq = ks->ks_softc;
+	struct vmxnet3_softc *sc = rq->sc;
+	struct vmx_rxstats_kv *rxkvs = ks->ks_data;
+	struct UPT1_RxStats *rxstats = &rq->rs->stats;
+
+	if (ratecheck(&sc->sc_kstats.updated, &vmx_kstat_rate))
+		WRITE_CMD(sc, VMXNET3_CMD_GET_STATS);
+
+	rxkvs->lro_packets.kv_v.v_u64 = rxstats->LRO_packets;
+	rxkvs->lro_bytes.kv_v.v_u64 = rxstats->LRO_bytes;
+	rxkvs->ucast_packets.kv_v.v_u64 = rxstats->ucast_packets;
+	rxkvs->ucast_bytes.kv_v.v_u64 = rxstats->ucast_bytes;
+	rxkvs->mcast_packets.kv_v.v_u64 = rxstats->mcast_packets;
+	rxkvs->mcast_bytes.kv_v.v_u64 = rxstats->mcast_bytes;
+	rxkvs->bcast_packets.kv_v.v_u64 = rxstats->bcast_packets;
+	rxkvs->bcast_bytes.kv_v.v_u64 = rxstats->bcast_bytes;
+	rxkvs->nobuffers.kv_v.v_u64 = rxstats->nobuffer;
+	rxkvs->errors.kv_v.v_u64 = rxstats->error;
+
+	TIMEVAL_TO_TIMESPEC(&sc->sc_kstats.updated, &ks->ks_updated);
+
+	return (0);
+}
+
+static void
+vmx_kstat_rxstats(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq, int i)
+{
+	rq->sc = sc;
+
+	rq->rxkstat = kstat_create(sc->sc_dev.dv_xname, 0, "vmx-rxstats", i,
+	    KSTAT_T_KV, 0);
+	if (rq->rxkstat == NULL)
+		return;
+
+	kstat_set_wlock(rq->rxkstat, &rq->sc->sc_kstats.lock);
+
+	rq->rxkstat->ks_softc = rq;
+	rq->rxkstat->ks_data = &sc->sc_kstats.rxstats;
+	rq->rxkstat->ks_datalen = sizeof(sc->sc_kstats.rxstats);
+	rq->rxkstat->ks_read = vmx_rxstats_read;
+	TIMEVAL_TO_TIMESPEC(&vmx_kstat_rate, &rq->rxkstat->ks_interval);
+
+	kstat_install(rq->rxkstat);
+}
+#endif
+
 void
 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq)
 {
@@ -964,6 +1208,10 @@ vmxnet3_rxintr(struct vmxnet3_softc *sc,
 			m->m_flags |= M_VLANTAG;
 			m->m_pkthdr.ether_vtag = letoh32((rxcd->rxc_word2 >>
 			    VMXNET3_RXC_VLANTAG_S) & VMXNET3_RXC_VLANTAG_M);
+		}
+		if (((letoh32(rxcd->rxc_word0) >> VMXNET3_RXC_RSSTYPE_S) &
+		    VMXNET3_RXC_RSSTYPE_M) != VMXNET3_RXC_RSSTYPE_NONE) {
+			m->m_pkthdr.ph_flowid = letoh32(rxcd->rxc_word1);
 		}
 
 		ml_enqueue(&ml, m);
Index: sys/dev/pci/if_vmxreg.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_vmxreg.h,v
retrieving revision 1.6
diff -u -p -r1.6 if_vmxreg.h
--- sys/dev/pci/if_vmxreg.h	27 Oct 2019 22:24:40 -0000	1.6
+++ sys/dev/pci/if_vmxreg.h	8 Jun 2020 07:43:45 -0000
@@ -73,6 +73,7 @@ struct UPT1_RxStats {
 #define VMXNET3_CMD_SET_RXMODE	0xcafe0003	/* set interface flags */
 #define VMXNET3_CMD_SET_FILTER	0xcafe0004	/* set address filter */
 #define VMXNET3_CMD_GET_STATUS	0xf00d0000	/* get queue errors */
+#define VMXNET3_CMD_GET_STATS	0xf00d0001
 #define VMXNET3_CMD_GET_LINK	0xf00d0002	/* get link status */
 #define VMXNET3_CMD_GET_MACL	0xf00d0003
 #define VMXNET3_CMD_GET_MACH	0xf00d0004
@@ -177,6 +178,7 @@ struct vmxnet3_rxcompdesc {
 #define VMXNET3_RXC_QID_S	16
 #define VMXNET3_RXC_RSSTYPE_M	0x0000000f
 #define VMXNET3_RXC_RSSTYPE_S	26
+#define VMXNET3_RXC_RSSTYPE_NONE 0
 #define VMXNET3_RXC_NOCSUM	0x40000000	/* no checksum calculated */
 #define VMXNET3_RXC_RES1	0x80000000
 
@@ -223,7 +225,6 @@ struct vmxnet3_rxcompdesc {
 #define VMXNET3_MAX_TX_QUEUES 8
 #define VMXNET3_MAX_RX_QUEUES 16
 #define VMXNET3_MAX_INTRS (VMXNET3_MAX_TX_QUEUES + VMXNET3_MAX_RX_QUEUES + 1)
-#define VMXNET3_NINTR 1
 
 #define VMXNET3_ICTRL_DISABLE_ALL 0x01
 
@@ -334,4 +335,22 @@ struct vmxnet3_rxq_shared {
 	struct UPT1_RxStats stats;
 
 	u_int8_t pad4[88];
+} __packed;
+
+#define UPT1_RSS_MAX_KEY_SIZE		40
+#define UPT1_RSS_MAX_IND_TABLE_SIZE	128
+
+struct vmxnet3_upt1_rss_conf {
+	u_int16_t hash_type;
+#define UPT1_RSS_HASH_TYPE_NONE		0
+#define UPT1_RSS_HASH_TYPE_IPV4		1
+#define UPT1_RSS_HASH_TYPE_TCP_IPV4	2
+#define UPT1_RSS_HASH_TYPE_IPV6		4
+#define UPT1_RSS_HASH_TYPE_TCP_IPV6	8
+	u_int16_t hash_func;
+#define UPT1_RSS_HASH_FUNC_TOEPLITZ	1
+	u_int16_t hash_key_size;
+	u_int16_t ind_table_size;
+	u_int8_t hash_key[UPT1_RSS_MAX_KEY_SIZE];
+	u_int8_t ind_table[UPT1_RSS_MAX_IND_TABLE_SIZE];
 } __packed;
Index: sys/dev/pci/ixgbe.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/ixgbe.h,v
retrieving revision 1.29
diff -u -p -r1.29 ixgbe.h
--- sys/dev/pci/ixgbe.h	2 Mar 2020 01:59:01 -0000	1.29
+++ sys/dev/pci/ixgbe.h	8 Jun 2020 07:43:45 -0000
@@ -41,6 +41,7 @@
 
 #include "bpfilter.h"
 #include "vlan.h"
+#include "kstat.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -54,6 +55,7 @@
 #include <sys/pool.h>
 #include <sys/rwlock.h>
 #include <sys/atomic.h>
+#include <sys/kstat.h>
 
 #include <net/if.h>
 #include <net/if_media.h>
Index: sys/net/ifq.c
===================================================================
RCS file: /cvs/src/sys/net/ifq.c,v
retrieving revision 1.39
diff -u -p -r1.39 ifq.c
--- sys/net/ifq.c	21 May 2020 00:06:16 -0000	1.39
+++ sys/net/ifq.c	8 Jun 2020 07:43:45 -0000
@@ -17,6 +17,7 @@
  */
 
 #include "bpfilter.h"
+#include "kstat.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -32,6 +33,10 @@
 #include <net/bpf.h>
 #endif
 
+#if NKSTAT > 0
+#include <sys/kstat.h>
+#endif
+
 /*
  * priq glue
  */
@@ -122,7 +127,10 @@ ifq_is_serialized(struct ifqueue *ifq)
 void
 ifq_start(struct ifqueue *ifq)
 {
-	if (ifq_len(ifq) >= min(ifq->ifq_if->if_txmit, ifq->ifq_maxlen)) {
+	struct ifnet *ifp = ifq->ifq_if;
+
+	if (ISSET(ifp->if_xflags, IFXF_MPSAFE) &&
+	    ifq_len(ifq) >= min(ifp->if_txmit, ifq->ifq_maxlen)) {
 		task_del(ifq->ifq_softnet, &ifq->ifq_bundle);
 		ifq_run_start(ifq);
 	} else
@@ -188,11 +196,42 @@ ifq_barrier_task(void *p)
  * ifqueue mbuf queue API
  */
 
+#if NKSTAT > 0
+struct ifq_kstat_data {
+	struct kstat_kv kd_packets;
+	struct kstat_kv kd_bytes;
+	struct kstat_kv kd_qdrops;
+	struct kstat_kv kd_errors;
+	struct kstat_kv kd_qlen;
+	struct kstat_kv kd_maxqlen;
+	struct kstat_kv kd_oactive;
+};
+
+static const struct ifq_kstat_data ifq_kstat_tpl = {
+	KSTAT_KV_UNIT_INITIALIZER("packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES),
+	KSTAT_KV_UNIT_INITIALIZER("qdrops",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("errors",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("qlen",
+	    KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("maxqlen",
+	    KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_INITIALIZER("oactive", KSTAT_KV_T_BOOL),
+};
+
+static int	ifq_kstat_copy(struct kstat *, void *);
+#endif
+
 void
 ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
 {
 	ifq->ifq_if = ifp;
-	ifq->ifq_softnet = net_tq(ifp->if_index); /* + idx */
+	ifq->ifq_softnet = ISSET(ifp->if_xflags, IFXF_MPSAFE) ?
+	    net_tq(ifp->if_index /* + idx */) : systq;
 	ifq->ifq_softc = NULL;
 
 	mtx_init(&ifq->ifq_mtx, IPL_NET);
@@ -222,6 +261,18 @@ ifq_init(struct ifqueue *ifq, struct ifn
 		ifq_set_maxlen(ifq, IFQ_MAXLEN);
 
 	ifq->ifq_idx = idx;
+
+#if NKSTAT > 0
+	/* XXX xname vs driver name and unit */
+	ifq->ifq_kstat = kstat_create(ifp->if_xname, 0,
+	    "txq", ifq->ifq_idx, KSTAT_T_KV, 0);
+	KASSERT(ifq->ifq_kstat != NULL);
+	kstat_set_mutex(ifq->ifq_kstat, &ifq->ifq_mtx);
+	ifq->ifq_kstat->ks_softc = ifq;
+	ifq->ifq_kstat->ks_datalen = sizeof(ifq_kstat_tpl);
+	ifq->ifq_kstat->ks_copy = ifq_kstat_copy;
+	kstat_install(ifq->ifq_kstat);
+#endif
 }
 
 void
@@ -265,6 +316,10 @@ ifq_destroy(struct ifqueue *ifq)
 {
 	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
 
+#if NKSTAT > 0
+	kstat_destroy(ifq->ifq_kstat);
+#endif
+
 	NET_ASSERT_UNLOCKED();
 	if (!task_del(ifq->ifq_softnet, &ifq->ifq_bundle))
 		taskq_barrier(ifq->ifq_softnet);
@@ -289,6 +344,26 @@ ifq_add_data(struct ifqueue *ifq, struct
 	mtx_leave(&ifq->ifq_mtx);
 }
 
+#if NKSTAT > 0
+static int
+ifq_kstat_copy(struct kstat *ks, void *dst)
+{
+	struct ifqueue *ifq = ks->ks_softc;
+	struct ifq_kstat_data *kd = dst;
+
+	*kd = ifq_kstat_tpl;
+	kd->kd_packets.kv_v.v_u64 = ifq->ifq_packets;
+	kd->kd_bytes.kv_v.v_u64 = ifq->ifq_bytes;
+	kd->kd_qdrops.kv_v.v_u64 = ifq->ifq_qdrops;
+	kd->kd_errors.kv_v.v_u64 = ifq->ifq_errors;
+	kd->kd_qlen.kv_v.v_u32 = ifq->ifq_len;
+	kd->kd_maxqlen.kv_v.v_u32 = ifq->ifq_maxlen;
+	kd->kd_oactive.kv_v.v_bool = ifq->ifq_oactive;
+
+	return (0);
+}
+#endif
+
 int
 ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
 {
@@ -505,6 +580,31 @@ ifq_mfreeml(struct ifqueue *ifq, struct 
  * ifiq
  */
 
+#if NKSTAT > 0
+struct ifiq_kstat_data {
+	struct kstat_kv kd_packets;
+	struct kstat_kv kd_bytes;
+	struct kstat_kv kd_qdrops;
+	struct kstat_kv kd_errors;
+	struct kstat_kv kd_qlen;
+};
+
+static const struct ifiq_kstat_data ifiq_kstat_tpl = {
+	KSTAT_KV_UNIT_INITIALIZER("packets",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("bytes",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES),
+	KSTAT_KV_UNIT_INITIALIZER("qdrops",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("errors",
+	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
+	KSTAT_KV_UNIT_INITIALIZER("qlen",
+	    KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS),
+};
+
+static int	ifiq_kstat_copy(struct kstat *, void *);
+#endif
+
 static void	ifiq_process(void *);
 
 void
@@ -525,11 +625,27 @@ ifiq_init(struct ifiqueue *ifiq, struct 
 	ifiq->ifiq_errors = 0;
 
 	ifiq->ifiq_idx = idx;
+
+#if NKSTAT > 0
+	/* XXX xname vs driver name and unit */
+	ifiq->ifiq_kstat = kstat_create(ifp->if_xname, 0,
+	    "rxq", ifiq->ifiq_idx, KSTAT_T_KV, 0);
+	KASSERT(ifiq->ifiq_kstat != NULL);
+	kstat_set_mutex(ifiq->ifiq_kstat, &ifiq->ifiq_mtx);
+	ifiq->ifiq_kstat->ks_softc = ifiq;
+	ifiq->ifiq_kstat->ks_datalen = sizeof(ifiq_kstat_tpl);
+	ifiq->ifiq_kstat->ks_copy = ifiq_kstat_copy;
+	kstat_install(ifiq->ifiq_kstat);
+#endif
 }
 
 void
 ifiq_destroy(struct ifiqueue *ifiq)
 {
+#if NKSTAT > 0
+	kstat_destroy(ifiq->ifiq_kstat);
+#endif
+
 	NET_ASSERT_UNLOCKED();
 	if (!task_del(ifiq->ifiq_softnet, &ifiq->ifiq_task))
 		taskq_barrier(ifiq->ifiq_softnet);
@@ -616,6 +732,24 @@ ifiq_add_data(struct ifiqueue *ifiq, str
 	data->ifi_iqdrops += ifiq->ifiq_qdrops;
 	mtx_leave(&ifiq->ifiq_mtx);
 }
+
+#if NKSTAT > 0
+static int
+ifiq_kstat_copy(struct kstat *ks, void *dst)
+{
+	struct ifiqueue *ifiq = ks->ks_softc;
+	struct ifiq_kstat_data *kd = dst;
+
+	*kd = ifiq_kstat_tpl;
+	kd->kd_packets.kv_v.v_u64 = ifiq->ifiq_packets;
+	kd->kd_bytes.kv_v.v_u64 = ifiq->ifiq_bytes;
+	kd->kd_qdrops.kv_v.v_u64 = ifiq->ifiq_qdrops;
+	kd->kd_errors.kv_v.v_u64 = ifiq->ifiq_errors;
+	kd->kd_qlen.kv_v.v_u32 = ml_len(&ifiq->ifiq_ml);
+
+	return (0);
+}
+#endif
 
 int
 ifiq_enqueue(struct ifiqueue *ifiq, struct mbuf *m)
Index: sys/net/ifq.h
===================================================================
RCS file: /cvs/src/sys/net/ifq.h,v
retrieving revision 1.31
diff -u -p -r1.31 ifq.h
--- sys/net/ifq.h	22 May 2020 07:02:24 -0000	1.31
+++ sys/net/ifq.h	8 Jun 2020 07:43:45 -0000
@@ -20,6 +20,7 @@
 #define _NET_IFQ_H_
 
 struct ifnet;
+struct kstat;
 
 struct ifq_ops;
 
@@ -54,6 +55,11 @@ struct ifqueue {
 	uint64_t		 ifq_errors;
 	uint64_t		 ifq_mcasts;
 
+	uint64_t		 ifq_dispatch;
+	uint64_t		 ifq_defers;
+
+	struct kstat		*ifq_kstat;
+
 	/* work serialisation */
 	struct mutex		 ifq_task_mtx;
 	struct task_list	 ifq_task_list;
@@ -91,6 +97,8 @@ struct ifiqueue {
 	uint64_t		 ifiq_errors;
 	uint64_t		 ifiq_mcasts;
 	uint64_t		 ifiq_noproto;
+
+	struct kstat		*ifiq_kstat;
 
 	/* properties */
 	unsigned int		 ifiq_idx;
Index: sys/sys/conf.h
===================================================================
RCS file: /cvs/src/sys/sys/conf.h,v
retrieving revision 1.152
diff -u -p -r1.152 conf.h
--- sys/sys/conf.h	26 May 2020 07:53:00 -0000	1.152
+++ sys/sys/conf.h	8 Jun 2020 07:43:45 -0000
@@ -328,6 +328,13 @@ extern struct cdevsw cdevsw[];
 	(dev_type_stop((*))) enodev, 0, seltrue, \
 	(dev_type_mmap((*))) enodev, 0, 0, seltrue_kqfilter }
 
+/* open, close, ioctl */
+#define cdev_kstat_init(c,n) { \
+	dev_init(c,n,open), dev_init(c,n,close), (dev_type_read((*))) enodev, \
+	(dev_type_write((*))) enodev, dev_init(c,n,ioctl), \
+	(dev_type_stop((*))) enodev, 0, selfalse, \
+	(dev_type_mmap((*))) enodev }
+
 /* open, close, read, write, ioctl, stop, tty, poll, mmap, kqfilter */
 #define	cdev_wsdisplay_init(c,n) { \
 	dev_init(c,n,open), dev_init(c,n,close), dev_init(c,n,read), \
@@ -605,6 +612,7 @@ cdev_decl(wsmouse);
 cdev_decl(wsmux);
 
 cdev_decl(ksyms);
+cdev_decl(kstat);
 
 cdev_decl(bio);
 cdev_decl(vscsi);
Index: sys/sys/kstat.h
===================================================================
RCS file: sys/sys/kstat.h
diff -N sys/sys/kstat.h
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ sys/sys/kstat.h	8 Jun 2020 07:43:45 -0000
@@ -0,0 +1,195 @@
+/* $OpenBSD$ */
+
+/*
+ * Copyright (c) 2020 David Gwynne <dlg@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _SYS_KSTAT_H_
+#define _SYS_KSTAT_H_
+
+#include <sys/ioccom.h>
+
+#define KSTAT_STRLEN		32
+
+#define KSTAT_T_RAW		0
+#define KSTAT_T_KV		1
+#define KSTAT_T_COUNTERS	2
+
+struct kstat_req {
+	unsigned int		 ks_rflags;
+#define KSTATIOC_F_IGNVER		(1 << 0)
+	/* the current version of the kstat subsystem */
+	unsigned int		 ks_version;
+
+	uint64_t		 ks_id;
+
+	char			 ks_provider[KSTAT_STRLEN];
+	unsigned int		 ks_instance;
+	char			 ks_name[KSTAT_STRLEN];
+	unsigned int		 ks_unit;
+
+	struct timespec		 ks_created;
+	struct timespec		 ks_updated;
+	struct timespec		 ks_interval;
+	unsigned int		 ks_type;
+	unsigned int		 ks_state;
+
+	void			*ks_data;
+	size_t			 ks_datalen;
+	unsigned int		 ks_dataver;
+};
+
+/* ioctls */
+
+#define KSTATIOC_VERSION	_IOR('k', 1, unsigned int)
+#define KSTATIOC_FIND_ID	_IOWR('k', 2, struct kstat_req)
+#define KSTATIOC_NFIND_ID	_IOWR('k', 3, struct kstat_req)
+#define KSTATIOC_FIND_PROVIDER	_IOWR('k', 4, struct kstat_req)
+#define KSTATIOC_NFIND_PROVIDER	_IOWR('k', 5, struct kstat_req)
+#define KSTATIOC_FIND_NAME	_IOWR('k', 6, struct kstat_req)
+#define KSTATIOC_NFIND_NAME	_IOWR('k', 7, struct kstat_req)
+
+/* named data */
+
+#define KSTAT_KV_NAMELEN	16
+#define KSTAT_KV_ALIGN		sizeof(uint64_t)
+
+enum kstat_kv_type {
+	KSTAT_KV_T_NULL,
+	KSTAT_KV_T_BOOL,
+	KSTAT_KV_T_COUNTER64,
+	KSTAT_KV_T_COUNTER32,
+	KSTAT_KV_T_UINT64,
+	KSTAT_KV_T_INT64,
+	KSTAT_KV_T_UINT32,
+	KSTAT_KV_T_INT32,
+	KSTAT_KV_T_ISTR,	/* inline string */
+	KSTAT_KV_T_STR,		/* trailing string */
+	KSTAT_KV_T_BYTES,	/* trailing bytes */
+};
+
+/* units only apply to integer types */
+enum kstat_kv_unit {
+	KSTAT_KV_U_NONE = 0,
+	KSTAT_KV_U_PACKETS,	/* packets */
+	KSTAT_KV_U_BYTES,	/* bytes */
+	KSTAT_KV_U_TEMP,	/* temperature (uK) */
+	KSTAT_KV_U_FANRPM,	/* fan revolution speed */
+	KSTAT_KV_U_VOLTS_DC,	/* voltage (uV DC) */
+	KSTAT_KV_U_VOLTS_AC,	/* voltage (uV AC) */
+	KSTAT_KV_U_OHMS,	/* resistance */
+	KSTAT_KV_U_WATTS,	/* power (uW) */
+	KSTAT_KV_U_AMPS,	/* current (uA) */
+	KSTAT_KV_U_WATTHOUR,	/* power capacity (uWh) */
+	KSTAT_KV_U_AMPHOUR,	/* power capacity (uAh) */
+	KSTAT_KV_U_PERCENT,	/* percent (m%) */
+	KSTAT_KV_U_LUX,		/* illuminance (ulx) */
+	KSTAT_KV_U_TIMEDELTA,	/* system time error (nSec) */
+	KSTAT_KV_U_HUMIDITY,	/* humidity (m%RH) */
+	KSTAT_KV_U_FREQ,	/* frequency (uHz) */
+	KSTAT_KV_U_ANGLE,	/* angle (uDegrees) */
+	KSTAT_KV_U_DISTANCE,	/* distance (uMeter) */
+	KSTAT_KV_U_PRESSURE,	/* pressure (mPa) */
+	KSTAT_KV_U_ACCEL,	/* acceleration (u m/s^2) */
+	KSTAT_KV_U_VELOCITY,	/* velocity (u m/s) */
+};
+
+struct kstat_kv {
+	char			 kv_key[KSTAT_KV_NAMELEN];
+	union {
+		char			v_istr[16];
+		unsigned int		v_bool;	
+		uint64_t		v_u64;
+		int64_t			v_s64;
+		uint32_t		v_u32;
+		int32_t			v_s32;
+		size_t			v_len;
+	}			 kv_v;
+	enum kstat_kv_type	 kv_type;
+	enum kstat_kv_unit	 kv_unit;
+} __aligned(KSTAT_KV_ALIGN);
+
+#define KSTAT_KV_UNIT_INITIALIZER(_key, _type, _unit) {	\
+	.kv_key = (_key),				\
+	.kv_type = (_type),				\
+	.kv_unit = (_unit),				\
+}
+
+#define KSTAT_KV_INITIALIZER(_key, _type)		\
+    KSTAT_KV_UNIT_INITIALIZER((_key), (_type), KSTAT_KV_U_NONE)
+
+void	kstat_kv_init(struct kstat_kv *, const char *, enum kstat_kv_type);
+void	kstat_kv_unit_init(struct kstat_kv *, const char *,
+	    enum kstat_kv_type, enum kstat_kv_unit);
+
+#ifdef _KERNEL
+
+#include <sys/tree.h>
+
+struct kstat_lock_ops;
+
+struct kstat {
+	uint64_t		  ks_id;
+
+	const char		 *ks_provider;
+	unsigned int		  ks_instance;
+	const char		 *ks_name;
+	unsigned int		  ks_unit;
+
+	unsigned int		  ks_type;
+	unsigned int		  ks_flags;
+#define KSTAT_F_REALLOC			(1 << 0)
+	unsigned int		  ks_state;
+#define KSTAT_S_CREATED			0
+#define KSTAT_S_INSTALLED		1
+
+	struct timespec		  ks_created;
+	RBT_ENTRY(kstat)	  ks_id_entry;
+	RBT_ENTRY(kstat)	  ks_pv_entry;
+	RBT_ENTRY(kstat)	  ks_nm_entry;
+
+	/* the driver can update these between kstat creation and install */
+	unsigned int		  ks_dataver;
+	void			 *ks_softc;
+	int			(*ks_read)(struct kstat *);
+	int			(*ks_copy)(struct kstat *, void *);
+
+	const struct kstat_lock_ops *
+				  ks_lock_ops;
+	void			 *ks_lock;
+
+	/* the data that is updated by ks_read */
+	void			 *ks_data;
+	size_t			  ks_datalen;
+	struct timespec		  ks_updated;
+	struct timespec		  ks_interval;
+};
+
+struct kstat	*kstat_create(const char *, unsigned int,
+		     const char *, unsigned int,
+		     unsigned int, unsigned int);
+
+void		 kstat_set_rlock(struct kstat *, struct rwlock *);
+void		 kstat_set_wlock(struct kstat *, struct rwlock *);
+void		 kstat_set_mutex(struct kstat *, struct mutex *);
+
+int		 kstat_read_nop(struct kstat *);
+
+void		 kstat_install(struct kstat *);
+void		 kstat_destroy(struct kstat *);
+
+#endif /* _KERNEL */
+
+#endif /* _SYS_KSTAT_H_ */
Index: usr.bin/kstat/Makefile
===================================================================
RCS file: usr.bin/kstat/Makefile
diff -N usr.bin/kstat/Makefile
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ usr.bin/kstat/Makefile	8 Jun 2020 07:43:45 -0000
@@ -0,0 +1,9 @@
+# $OpenBSD$
+
+PROG=	kstat
+SRCS=	kstat.c
+MAN=
+WARNINGS=Yes
+DEBUG=-g
+
+.include <bsd.prog.mk>
Index: usr.bin/kstat/kstat.c
===================================================================
RCS file: usr.bin/kstat/kstat.c
diff -N usr.bin/kstat/kstat.c
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ usr.bin/kstat/kstat.c	8 Jun 2020 07:43:45 -0000
@@ -0,0 +1,321 @@
+/* $OpenBSD$ */
+
+/*
+ * Copyright (c) 2020 David Gwynne <dlg@openbsd.org>
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <err.h>
+#include <vis.h>
+
+#include <sys/tree.h>
+#include <sys/ioctl.h>
+#include <sys/time.h>
+
+#include "/usr/src/sys/sys/kstat.h"
+
+#ifndef roundup
+#define roundup(x, y)   ((((x)+((y)-1))/(y))*(y))
+#endif
+
+#define DEV_KSTAT "/dev/kstat"
+
+static void	kstat_list(int, unsigned int);
+
+#if 0
+__dead static void
+usage(void)
+{
+	extern char *__progname;
+	fprintf(stderr, "usage: %s\n", __progname);
+	exit(1);
+}
+#endif
+
+int
+main(int argc, char *argv[])
+{
+	unsigned int version;
+	int fd;
+
+	fd = open(DEV_KSTAT, O_RDONLY);
+	if (fd == -1)
+		err(1, "%s", DEV_KSTAT);
+
+	if (ioctl(fd, KSTATIOC_VERSION, &version) == -1)
+		err(1, "kstat version");
+
+	//printf("version %u\n", version);
+
+	kstat_list(fd, version);
+
+	return (0);
+}
+
+struct kstat_entry {
+	struct kstat_req	kstat;
+	RBT_ENTRY(kstat_entry)	entry;
+};
+
+RBT_HEAD(kstat_tree, kstat_entry);
+
+static inline int
+kstat_cmp(const struct kstat_entry *ea, const struct kstat_entry *eb)
+{
+	const struct kstat_req *a = &ea->kstat;
+	const struct kstat_req *b = &eb->kstat;
+	int rv;
+
+	rv = strncmp(a->ks_provider, b->ks_provider, sizeof(a->ks_provider));
+	if (rv != 0)
+		return (rv);
+	if (a->ks_instance > b->ks_instance)
+		return (1);
+	if (a->ks_instance < b->ks_instance)
+		return (-1);
+
+	rv = strncmp(a->ks_name, b->ks_name, sizeof(a->ks_name));
+	if (rv != 0)
+		return (rv);
+	if (a->ks_unit > b->ks_unit)
+		return (1);
+	if (a->ks_unit < b->ks_unit)
+		return (-1);
+
+	return (0);
+}
+
+RBT_PROTOTYPE(kstat_tree, kstat_entry, entry, kstat_cmp);
+RBT_GENERATE(kstat_tree, kstat_entry, entry, kstat_cmp);
+
+static int
+printable(int ch)
+{
+        if (ch == '\0')
+                return ('_');
+        if (!isprint(ch))
+                return ('~');
+        return (ch);
+}
+
+static void
+hexdump(const void *d, size_t datalen)
+{
+        const uint8_t *data = d;
+        size_t i, j = 0;
+
+        for (i = 0; i < datalen; i += j) {
+                printf("%4zu: ", i);
+
+                for (j = 0; j < 16 && i+j < datalen; j++)
+                        printf("%02x ", data[i + j]);
+                while (j++ < 16)
+                        printf("   ");
+                printf("|");
+
+                for (j = 0; j < 16 && i+j < datalen; j++)
+                        putchar(printable(data[i + j]));
+                printf("|\n");
+        }
+}
+
+static void
+strdump(const void *s, size_t len)
+{
+	const char *str = s;
+	char dst[8];
+	size_t i;
+
+	for (i = 0; i < len; i++) {
+		char ch = str[i];
+		if (ch == '\0')
+			break;
+
+		vis(dst, ch, VIS_TAB | VIS_NL, 0);
+		printf("%s", dst);
+	}
+	printf("\n");
+}
+
+static void
+kstat_kv(const void *d, ssize_t len)
+{
+	const uint8_t *buf;
+	const struct kstat_kv *kv;
+	ssize_t blen;
+	void (*trailer)(const void *, size_t);
+
+	if (len < (ssize_t)sizeof(*kv)) {
+		warn("short kv (len %zu < size %zu)", len, sizeof(*kv));
+		return;
+	}
+
+	buf = d;
+	do {
+		kv = (const struct kstat_kv *)buf;
+
+		buf += sizeof(*kv);
+		len -= sizeof(*kv);
+
+		blen = 0;
+		trailer = hexdump;
+
+		printf("%16.16s: ", kv->kv_key);
+
+		switch (kv->kv_type) {
+		case KSTAT_KV_T_NULL:
+			printf("null");
+			break;
+		case KSTAT_KV_T_BOOL:
+			printf("%s", kv->kv_v.v_bool ? "true" : "false");
+			break;
+		case KSTAT_KV_T_COUNTER64:
+		case KSTAT_KV_T_UINT64:
+			printf("%" PRIu64, kv->kv_v.v_u64);
+			break;
+		case KSTAT_KV_T_INT64:
+			printf("%" PRId64, kv->kv_v.v_s64);
+			break;
+		case KSTAT_KV_T_COUNTER32:
+		case KSTAT_KV_T_UINT32:
+			printf("%" PRIu32, kv->kv_v.v_u32);
+			break;
+		case KSTAT_KV_T_INT32:
+			printf("%" PRId32, kv->kv_v.v_s32);
+			break;
+		case KSTAT_KV_T_STR:
+			blen = kv->kv_v.v_len;
+			trailer = strdump;
+			break;
+		case KSTAT_KV_T_BYTES:
+			blen = kv->kv_v.v_len;
+			trailer = hexdump;
+
+			printf("\n");
+			break;
+
+		case KSTAT_KV_T_ISTR:
+			printf("%.*s\n", (int)sizeof(kv->kv_v.v_istr),
+			    kv->kv_v.v_istr);
+			break;
+		default:
+			printf("unknown type %u, stopping\n", kv->kv_type);
+			return;
+		}
+
+		switch (kv->kv_unit) {
+		case KSTAT_KV_U_NONE:
+			break;
+		case KSTAT_KV_U_PACKETS:
+			printf(" packets");
+			break;
+		case KSTAT_KV_U_BYTES:
+			printf(" bytes");
+			break;
+
+		default:
+			printf(" unit-type-%u", kv->kv_unit);
+			break;
+		}
+
+		if (blen > 0) {
+			if (blen > len) {
+				blen = len;
+			}
+
+			(*trailer)(buf, blen);
+		} else
+			printf("\n");
+
+		blen = roundup(blen, KSTAT_KV_ALIGN);
+		buf += blen;
+		len -= blen;
+	} while (len >= (ssize_t)sizeof(*kv));
+}
+
+static void
+kstat_list(int fd, unsigned int version)
+{
+	struct kstat_entry *kse;
+	struct kstat_req *ksreq;
+	size_t len;
+	uint64_t id = 0;
+	struct kstat_tree kstat_tree = RBT_INITIALIZER();
+
+	for (;;) {
+		kse = malloc(sizeof(*kse));
+		if (kse == NULL)
+			err(1, NULL);
+
+		memset(kse, 0, sizeof(*kse));
+		ksreq = &kse->kstat;
+		ksreq->ks_version = version;
+		ksreq->ks_id = ++id;
+
+		ksreq->ks_datalen = len = 64; /* magic */
+		ksreq->ks_data = malloc(len);
+		if (ksreq->ks_data == NULL)
+			err(1, "data alloc");
+
+		if (ioctl(fd, KSTATIOC_NFIND_ID, ksreq) == -1) {
+			if (errno == ENOENT) {
+				free(ksreq->ks_data);
+				free(kse);
+				break;
+			}
+
+			err(1, "nfind id %llu", id);
+		}
+
+		while (ksreq->ks_datalen > len) {
+			len = ksreq->ks_datalen;
+			ksreq->ks_data = realloc(ksreq->ks_data, len);
+			if (ksreq->ks_data == NULL)
+				err(1, "data resize (%zu)", len);
+
+			if (ioctl(fd, KSTATIOC_FIND_ID, ksreq) == -1)
+				err(1, "find id %llu", id);
+		}
+
+		if (RBT_INSERT(kstat_tree, &kstat_tree, kse) != NULL)
+			errx(1, "duplicate kstat entry");
+
+		id = ksreq->ks_id;
+	}
+
+	RBT_FOREACH(kse, kstat_tree, &kstat_tree) {
+		ksreq = &kse->kstat;
+		printf("%s:%u:%s:%u\n",
+		    ksreq->ks_provider, ksreq->ks_instance,
+		    ksreq->ks_name, ksreq->ks_unit);
+		switch (ksreq->ks_type) {
+		case KSTAT_T_RAW:
+			hexdump(ksreq->ks_data, ksreq->ks_datalen);
+			break;
+		case KSTAT_T_KV:
+			kstat_kv(ksreq->ks_data, ksreq->ks_datalen);
+			break;
+		default:
+			hexdump(ksreq->ks_data, ksreq->ks_datalen);
+			break;
+		}
+	}
+}
+