/*	$OpenBSD$ */

/*
 * Copyright (c) 2022, 2023 David Gwynne <dlg@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include "kstat.h"

#if NKSTAT == 0
#error cpumon(4) requires kstat(4)
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/signalvar.h>
#include <sys/malloc.h>
#include <sys/device.h>
#include <sys/pool.h>
#include <sys/proc.h>
#include <sys/user.h>
#include <sys/ioctl.h>
#include <sys/queue.h>
#include <sys/rwlock.h>
#include <sys/kstat.h>
#include <sys/task.h>
#include <sys/timeout.h>

#include <machine/cpufunc.h>

#define INTEL_F6_SANDYBRIDGE_X		0x2d
#define INTEL_F6_HASWELL_X		0x3f
#define INTEL_F6_HASWELL_L		0x45

#define MSR_RAPL_PWR_UNIT	0xc0010299
#define  MSR_RAPL_PWR_UNIT_ESU_SHIFT	8
#define  MSR_RAPL_PWR_UNIT_ESU_MASK	0x1f
#define MSR_CORE_ENERGY_STATE   0xc001029a
#define MSR_PKG_ENERGY_STATE    0xc001029b

#define CPUMON_MSR_AMD_RAPL_PWR_UNIT		0xC0010299
#define  CPUMON_MSR_AMD_RAPL_PWR_UNIT_ESU_SHIFT		8
#define  CPUMON_MSR_AMD_SR_RAPL_PWR_UNIT_ESU_MASK	0x1f
#define CPUMON_MSR_AMD_CORE_ENERGY_STAT		0xC001029A
#define CPUMON_MSR_AMD_PKG_ENERGY_STAT		0xC001029B

struct cpumon_rapl {
	uint64_t		 rapl_energy;		/* accumulator */
	uint32_t		 rapl_energy_prev;
	unsigned int		 rapl_energy_unit;
	uint32_t		 rapl_energy_msr;
};

enum cpumon_core_map {
	CPUMON_CORE_MAP_TSC,
	CPUMON_CORE_MAP_EFF_PERF,
	CPUMON_CORE_MAP_SMI,
	CPUMON_CORE_MAP_IRPERF,

	CPUMON_CORE_MAP_TEMP,
	CPUMON_CORE_MAP_RAPL,

	CPUMON_CORE_MAP_COUNT
};

struct cpumon_core {
	struct cpumon_softc		*c_sc;
	struct cpu_info			*c_ci;
	struct kstat			*c_ks;

	TAILQ_ENTRY(cpumon_core)	 c_entry;

	unsigned int			 c_nkvs;
	int8_t				 c_map[CPUMON_CORE_MAP_COUNT];

	int				 c_temp_max;

	struct cpumon_rapl		 c_rapl;
	struct task			 c_rapl_xcall;
};

TAILQ_HEAD(cpumon_cores, cpumon_core);

enum cpumon_pkg_map {
	CPUMON_PKG_MAP_RAPL_PP0,
	CPUMON_PKG_MAP_RAPL_PKG,
	CPUMON_PKG_MAP_RAPL_RAM,
	CPUMON_PKG_MAP_RAPL_PP1,
	CPUMON_PKG_MAP_RAPL_PSYS,

	CPUMON_PKG_MAP_COUNT,

	CPUMON_PKG_MAP_RAPL_RAM_FIXED, /* intel has a weird RAM unit */
};

static const char *cpump_pkg_names[] = {
	[CPUMON_PKG_MAP_RAPL_PP0]	= "pp0",
	[CPUMON_PKG_MAP_RAPL_PKG]	= "pkg",
	[CPUMON_PKG_MAP_RAPL_RAM]	= "ram",
	[CPUMON_PKG_MAP_RAPL_PP1]	= "pp1",
	[CPUMON_PKG_MAP_RAPL_PSYS]	= "psys",
};

struct cpumon_pkg {
	struct cpumon_softc		*p_sc;
	struct cpu_info			*p_ci;
	struct kstat			*p_ks;

	TAILQ_ENTRY(cpumon_pkg)		 p_entry;

	unsigned int			 p_nkvs;
	int8_t				 p_map[CPUMON_PKG_MAP_COUNT];

	struct cpumon_rapl		 p_rapl[CPUMON_PKG_MAP_COUNT];
	struct task			 p_rapl_xcall;
};

TAILQ_HEAD(cpumon_pkgs, cpumon_pkg);

struct cpumon_softc {
	struct device			 sc_dev;
	struct task			 sc_deferred;

	struct cpumon_cores		 sc_cores;
	struct cpumon_pkgs		 sc_pkgs;

	/* used by the ticks below to wait for all cores/pkgs to read stuff */
	struct refcnt			 sc_rapl_refs;

	struct timeout			 sc_core_rapl_tick;
	struct timeout			 sc_pkg_rapl_tick;
};

static int	cpumon_match(struct device *, void *, void *);
static void	cpumon_attach(struct device *, struct device *, void *);

struct cfdriver cpumon_cd = {
	NULL, "cpumon", DV_DULL, CD_SKIPHIBERNATE
};

const struct cfattach cpumon_ca = {
	sizeof(struct cpumon_softc), cpumon_match, cpumon_attach, NULL, NULL
};

static void	cpumon_deferred(void *);
static struct cpumon_core *
		cpumon_attach_core(struct cpumon_softc *, struct cpu_info *);
static struct cpumon_pkg *
		cpumon_attach_pkg(struct cpumon_softc *, struct cpu_info *);

static void	cpumon_core_rapl_tick(void *);
static void	cpumon_pkg_rapl_tick(void *);

static int
cpumon_match(struct device *parent, void *match, void *aux)
{
	const char **busname = (const char **)aux;

	if (strcmp(*busname, cpumon_cd.cd_name) != 0)
		return (0);

	return (1);
}

static void
cpumon_attach(struct device *parent, struct device *self, void *aux)
{
	struct cpumon_softc *sc = (struct cpumon_softc *)self;

	printf("\n");

	task_set(&sc->sc_deferred, cpumon_deferred, sc);
	TAILQ_INIT(&sc->sc_cores);
	TAILQ_INIT(&sc->sc_pkgs);
	timeout_set_proc(&sc->sc_core_rapl_tick, cpumon_core_rapl_tick, sc);
	timeout_set_proc(&sc->sc_pkg_rapl_tick, cpumon_pkg_rapl_tick, sc);

	task_add(systqmp, &sc->sc_deferred);
}

static inline uint32_t
cpumon_rapl_read_msr(const struct cpumon_rapl *rapl)
{
	return (rdmsr(rapl->rapl_energy_msr));
}

static void
cpumon_core_rapl_xcall(void *arg)
{
	struct cpumon_core *c = arg;
	struct cpumon_softc *sc = c->c_sc;
	struct cpumon_rapl *rapl = &c->c_rapl;
	uint32_t energy_now;
	uint32_t diff;

	energy_now = cpumon_rapl_read_msr(rapl);
	diff = energy_now - rapl->rapl_energy_prev;

	rapl->rapl_energy_prev = energy_now;
	rapl->rapl_energy += diff;

	refcnt_rele_wake(&sc->sc_rapl_refs);
}

static void
cpumon_pkg_rapl_xcall(void *arg)
{
	struct cpumon_pkg *p = arg;
	struct cpumon_softc *sc = p->p_sc;
	struct cpumon_rapl *rapl;
	uint32_t energy_now;
	uint32_t diff;
	size_t i;

	for (i = 0; i < nitems(p->p_rapl); i++) {
		if (p->p_map[i] == 0)
			continue;

		rapl = &p->p_rapl[i];

		energy_now = cpumon_rapl_read_msr(rapl);
		diff = energy_now - rapl->rapl_energy_prev;

		rapl->rapl_energy_prev = energy_now;
		rapl->rapl_energy += diff;
	}

	refcnt_rele_wake(&sc->sc_rapl_refs);
}

static uint64_t
cpumon_rapl_read(struct cpumon_rapl *rapl, uint32_t energy_now)
{
	uint32_t diff = energy_now - rapl->rapl_energy_prev;
	uint64_t energy = rapl->rapl_energy + diff;

	rapl->rapl_energy_prev = energy_now;
	rapl->rapl_energy = energy;

	/* XXX i feel like this will overflow */
	return ((energy * 1000000) >> rapl->rapl_energy_unit);
}

static void
cpumon_probe_core_effperf(struct cpumon_core *c)
{
	uint32_t eax, ebx, ecx, edx;

	CPUID(0x06, eax, ebx, ecx, edx);

	if (ecx & (1 << 0)) {
		c->c_map[CPUMON_CORE_MAP_EFF_PERF] = c->c_nkvs;
		c->c_nkvs += 2;
	}
}

static void
cpumon_probe_core_intel(struct cpumon_core *c)
{
	struct cpu_info *ci = c->c_ci;

	if (cpuid_level >= 0x06) {
		cpumon_probe_core_effperf(c);

		switch (ci->ci_model) {
		case INTEL_F6_SANDYBRIDGE_X:
		case 0x45: /* Haswell mobile */
			c->c_map[CPUMON_CORE_MAP_SMI] = c->c_nkvs;
			c->c_nkvs += 1;
			break;
		}
	}

	if (ISSET(ci->ci_feature_tpmflags, TPM_SENSOR)) {
		c->c_map[CPUMON_CORE_MAP_TEMP] = c->c_nkvs;
		c->c_nkvs += 1;

		c->c_temp_max = 100;

		/* Only some Core family chips have MSR_TEMPERATURE_TARGET. */
		if (ci->ci_model == 0x0e &&
		    (rdmsr(MSR_TEMPERATURE_TARGET_UNDOCUMENTED) &
		     MSR_TEMPERATURE_TARGET_LOW_BIT_UNDOCUMENTED))
			c->c_temp_max = 85;

		/*
		 * Newer CPUs can tell you what their max temperature is.
		 * See: '64-ia-32-architectures-software-developer-
		 * vol-3c-part-3-manual.pdf'
		 */
		if (ci->ci_model > 0x17 && ci->ci_model != 0x1c &&
		    ci->ci_model != 0x26 && ci->ci_model != 0x27 &&
		    ci->ci_model != 0x35 && ci->ci_model != 0x36)
			c->c_temp_max = MSR_TEMPERATURE_TARGET_TJMAX(
			    rdmsr(MSR_TEMPERATURE_TARGET));
	}
}

static void
cpumon_probe_core_amd(struct cpumon_core *c)
{
	cpumon_probe_core_effperf(c);

	if (c->c_ci->ci_family >= 0x17) {
		uint32_t eax, ebx, ecx, edx;

		CPUID(0x80000008, eax, ebx, ecx, edx);
		if (ebx & (1 << 1)) {
			c->c_map[CPUMON_CORE_MAP_IRPERF] = c->c_nkvs;
			c->c_nkvs += 1;
		}

		CPUID(0x80000007, eax, ebx, ecx, edx);
		if (edx & (1 << 14)) {
			c->c_map[CPUMON_CORE_MAP_RAPL] = c->c_nkvs;
			c->c_nkvs += 1;
		}
	}
}

static void
cpumon_deferred(void *arg)
{
	struct cpumon_softc *sc = arg;
	struct cpu_info *ci;
	CPU_INFO_ITERATOR cii;
	struct cpumon_core *c;
	int rapl = 0;

	CPU_INFO_FOREACH(cii, ci) {
		sched_peg_curproc(ci);

		c = cpumon_attach_core(sc, ci);
		if (c && c->c_map[CPUMON_CORE_MAP_RAPL])
			rapl = 1;

		cpumon_attach_pkg(sc, ci);
	}

	atomic_clearbits_int(&curproc->p_flag, P_CPUPEG);

	if (rapl)
		timeout_add_sec(&sc->sc_core_rapl_tick, 53);

	if (!TAILQ_EMPTY(&sc->sc_pkgs))
		timeout_add_sec(&sc->sc_pkg_rapl_tick, 5);
}

static void
cpumon_core_rapl_tick(void *arg)
{
	struct cpumon_softc *sc = arg;
	struct cpumon_core *c;

	refcnt_init(&sc->sc_rapl_refs);

	TAILQ_FOREACH(c, &sc->sc_cores, c_entry) {
		if (c->c_map[CPUMON_CORE_MAP_RAPL] == 0) {
			/* is this even possible? */
			continue;
		}

		refcnt_take(&sc->sc_rapl_refs);
		cpu_xcall(c->c_ci, &c->c_rapl_xcall);
	}

	refcnt_finalize(&sc->sc_rapl_refs, "raplcore");

	/* this doesnt have to be accurate */
	timeout_add_sec(&sc->sc_core_rapl_tick, 53);
}

static void
cpumon_pkg_rapl_tick(void *arg)
{
	struct cpumon_softc *sc = arg;
	struct cpumon_pkg *p;

	refcnt_init(&sc->sc_rapl_refs);

	TAILQ_FOREACH(p, &sc->sc_pkgs, p_entry) {
		refcnt_take(&sc->sc_rapl_refs);
		cpu_xcall(p->p_ci, &p->p_rapl_xcall);
	}

	refcnt_finalize(&sc->sc_rapl_refs, "raplpkg");

	/* this doesnt have to be accurate */
	timeout_add_sec(&sc->sc_core_rapl_tick, 7);
}

struct cpumon_xcall {
	struct kstat	*cx_ks;
	struct cond	 cx_c;
};

static void
cpumon_read_core_xcall(void *arg)
{
	struct cpumon_xcall *cx = arg;
	struct kstat *ks = cx->cx_ks;
	struct kstat_kv *kvs = ks->ks_data;
	struct cpumon_core *c = ks->ks_softc;
	unsigned long s;
	uint32_t energy_now;
	int idx, rapl;

	/* this isn't timing critical */
	idx = c->c_map[CPUMON_CORE_MAP_TEMP];
	if (idx) {
		uint64_t msr = rdmsr(MSR_THERM_STATUS);

		if (msr & MSR_THERM_STATUS_VALID_BIT) {
			uint64_t v;

			v = c->c_temp_max - MSR_THERM_STATUS_TEMP(msr);
			/* micro degrees */
			v *= 1000000;
			/* kelvin */
			v += 273150000;

			kvs[idx].kv_type = KSTAT_KV_T_TEMP;
			kstat_kv_temp(&kvs[idx]) = v;
		} else
			kvs[idx].kv_type = KSTAT_KV_T_NULL;
	}

	s = intr_disable();
	idx = c->c_map[CPUMON_CORE_MAP_TSC];
	if (idx)
		kstat_kv_u64(&kvs[idx]) = rdtsc_lfence();

	idx = c->c_map[CPUMON_CORE_MAP_EFF_PERF];
	if (idx) {
		kstat_kv_u64(&kvs[idx + 0]) = rdmsr(0xe7);
		kstat_kv_u64(&kvs[idx + 1]) = rdmsr(0xe8);
	}

	idx = c->c_map[CPUMON_CORE_MAP_SMI];
	if (idx)
		kstat_kv_u32(&kvs[idx]) = rdmsr(0x34);

	idx = c->c_map[CPUMON_CORE_MAP_IRPERF];
	if (idx)
		kstat_kv_u64(&kvs[idx]) = rdmsr(0xe7);

	rapl = c->c_map[CPUMON_CORE_MAP_RAPL];
	if (rapl)
		energy_now = cpumon_rapl_read_msr(&c->c_rapl);

	nanouptime(&ks->ks_updated);
	intr_restore(s);

	if (rapl) {
		kstat_kv_u64(&kvs[rapl]) =
		    cpumon_rapl_read(&c->c_rapl, energy_now);
	}

	cond_signal(&cx->cx_c);
}

static int
cpumon_read_core(struct kstat *ks)
{
	struct timespec now, diff;

	/* rate limit the updates to roughly twice a second */
	getnanouptime(&now);
	timespecsub(&now, &ks->ks_updated, &diff);
	if (diff.tv_sec > 0 || diff.tv_nsec > 500000000) {
		struct cpumon_xcall cx = { ks, COND_INITIALIZER() };
		struct task t = TASK_INITIALIZER(cpumon_read_core_xcall, &cx);
		struct cpumon_core *c = ks->ks_softc;

		cpu_xcall(c->c_ci, &t);

		cond_wait(&cx.cx_c, "cpumonc");
	}

	return (0);
}

static struct cpumon_core *
cpumon_attach_core(struct cpumon_softc *sc, struct cpu_info *ci)
{
	struct kstat *ks;
	struct kstat_kv *kvs;
	struct cpumon_core *c;
	int idx;

	TAILQ_FOREACH(c, &sc->sc_cores, c_entry) {
		if (ci->ci_pkg_id == c->c_ci->ci_pkg_id &&
		    ci->ci_core_id == c->c_ci->ci_core_id) {
			/* core is already being monitored */

			if (ci->ci_smt_id < c->c_ci->ci_smt_id) {
				/* prefer low threads */
				c->c_ci = ci;
			}

			return (NULL);
		}
	}

	ks = kstat_create("cpu-core", ci->ci_pkg_id << 24 | ci->ci_core_id,
	    "cpumon", 0, KSTAT_T_KV, 0);
	if (ks == NULL) {
		printf("unable to create cpu-core kstat for pkg %u core %d\n",
		    ci->ci_pkg_id, ci->ci_core_id);
		return (NULL);
	}

	c = malloc(sizeof(*c), M_DEVBUF, M_WAITOK|M_ZERO);
	c->c_sc = sc;
	c->c_ci = ci;
	c->c_ks = ks;
	c->c_nkvs = 2; /* pkg and core ids */

	/* assume we have tsc */
	c->c_map[CPUMON_CORE_MAP_TSC] = c->c_nkvs;
	c->c_nkvs += 1;

	if (strcmp(cpu_vendor, "GenuineIntel") == 0)
		cpumon_probe_core_intel(c);
	else if (strcmp(cpu_vendor, "AuthenticAMD") == 0)
		cpumon_probe_core_amd(c);

	kvs = mallocarray(c->c_nkvs, sizeof(*kvs), M_DEVBUF, M_WAITOK|M_ZERO);

	kstat_kv_init(&kvs[0], "package", KSTAT_KV_T_UINT32);
	kstat_kv_u32(&kvs[0]) = ci->ci_pkg_id;
	kstat_kv_init(&kvs[1], "core", KSTAT_KV_T_UINT32);
	kstat_kv_u32(&kvs[1]) = ci->ci_core_id;

	idx = c->c_map[CPUMON_CORE_MAP_TSC];
	if (idx) {
		kstat_kv_unit_init(&kvs[idx], "tsc",
		    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_CYCLES);
	}

	idx = c->c_map[CPUMON_CORE_MAP_EFF_PERF];
	if (idx) {
		kstat_kv_init(&kvs[idx + 0], "mperf", KSTAT_KV_T_COUNTER64);
		kstat_kv_init(&kvs[idx + 1], "aperf", KSTAT_KV_T_COUNTER64);
	}

	idx = c->c_map[CPUMON_CORE_MAP_SMI];
	if (idx)
		kstat_kv_init(&kvs[idx], "smi", KSTAT_KV_T_COUNTER32);

	idx = c->c_map[CPUMON_CORE_MAP_IRPERF];
	if (idx) {
		uint64_t msr;

		msr = rdmsr(0xC0010015);
		SET(msr, (1 << 30));
		wrmsr(0xC0010015, msr);

		kstat_kv_unit_init(&kvs[idx], "irperf",
		    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_INSTR);
	}

	idx = c->c_map[CPUMON_CORE_MAP_TEMP];
	if (idx)
		kstat_kv_init(&kvs[idx], "temperature", KSTAT_KV_T_TEMP);

	idx = c->c_map[CPUMON_CORE_MAP_RAPL];
	if (idx) {
		uint64_t rapl_pwr_unit;
		unsigned int unit;

		rapl_pwr_unit = rdmsr(CPUMON_MSR_AMD_RAPL_PWR_UNIT);
		unit = rapl_pwr_unit >> CPUMON_MSR_AMD_RAPL_PWR_UNIT_ESU_SHIFT;
		unit &= CPUMON_MSR_AMD_SR_RAPL_PWR_UNIT_ESU_MASK;

		task_set(&c->c_rapl_xcall, cpumon_core_rapl_xcall, c);

		c->c_rapl.rapl_energy_msr = CPUMON_MSR_AMD_CORE_ENERGY_STAT;
		c->c_rapl.rapl_energy_prev = rdmsr(c->c_rapl.rapl_energy_msr);
		c->c_rapl.rapl_energy_unit = unit;

		kstat_kv_unit_init(&kvs[idx], "energy",
		    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_UJOULES);
	}

	ks->ks_data = kvs;
	ks->ks_datalen = c->c_nkvs * sizeof(*kvs);
	ks->ks_read = cpumon_read_core;
	ks->ks_softc = c;

	kstat_install(ks);

	TAILQ_INSERT_TAIL(&sc->sc_cores, c, c_entry);

	return (c);
}

static void
cpumon_read_pkg_xcall(void *arg)
{
	struct cpumon_xcall *cx = arg;
	struct kstat *ks = cx->cx_ks;
	struct kstat_kv *kvs = ks->ks_data;
	struct cpumon_pkg *p = ks->ks_softc;
	unsigned long s;
	uint32_t energy_now[nitems(p->p_map)];
	size_t i;
	int idx;

	s = intr_disable();
	for (i = 0; i < nitems(p->p_map); i++) {
		if (p->p_map[i] == 0)
			continue;

		energy_now[i] = cpumon_rapl_read_msr(&p->p_rapl[i]);
	}

	nanouptime(&ks->ks_updated);
	intr_restore(s);

	for (i = 0; i < nitems(p->p_map); i++) {
		idx = p->p_map[i];
		if (idx == 0)
			continue;

		energy_now[i] = cpumon_rapl_read_msr(&p->p_rapl[i]);
		kstat_kv_u64(&kvs[idx]) =
		    cpumon_rapl_read(&p->p_rapl[i], energy_now[i]);
	}

	cond_signal(&cx->cx_c);
}

static int
cpumon_read_pkg(struct kstat *ks)
{
	struct timespec now, diff;

	/* rate limit the updates to roughly twice a second */
	getnanouptime(&now);
	timespecsub(&now, &ks->ks_updated, &diff);
	if (diff.tv_sec > 0 || diff.tv_nsec > 500000000) {
		struct cpumon_xcall cx = { ks, COND_INITIALIZER() };
		struct task t = TASK_INITIALIZER(cpumon_read_pkg_xcall, &cx);
		struct cpumon_pkg *p = ks->ks_softc;

		cpu_xcall(p->p_ci, &t);

		cond_wait(&cx.cx_c, "cpumonp");
	}

	return (0);
}

struct cpumon_intel_f6_pkg {
	unsigned int			model;
	unsigned int			rapls;
};

static const struct cpumon_intel_f6_pkg cpumon_intel_f6_pkgs[] = {
	{ INTEL_F6_SANDYBRIDGE_X,	(1 << CPUMON_PKG_MAP_RAPL_PP0) |
					(1 << CPUMON_PKG_MAP_RAPL_PKG) |
					(1 << CPUMON_PKG_MAP_RAPL_RAM) },

	{ INTEL_F6_HASWELL_X,		(1 << CPUMON_PKG_MAP_RAPL_PP0) |
					(1 << CPUMON_PKG_MAP_RAPL_PKG) |
					(1 << CPUMON_PKG_MAP_RAPL_RAM_FIXED) },

	{ INTEL_F6_HASWELL_L,		(1 << CPUMON_PKG_MAP_RAPL_PP0) |
					(1 << CPUMON_PKG_MAP_RAPL_PKG) |
					(1 << CPUMON_PKG_MAP_RAPL_RAM) |
					(1 << CPUMON_PKG_MAP_RAPL_PP1) },
};

static uint32_t cpumon_intel_rapl_msrs[] = {
	[CPUMON_PKG_MAP_RAPL_PP0]	= 0x00000639,
	[CPUMON_PKG_MAP_RAPL_PKG]	= 0x00000611,
	[CPUMON_PKG_MAP_RAPL_RAM]	= 0x00000619,
	[CPUMON_PKG_MAP_RAPL_PP1]	= 0x00000641,
	[CPUMON_PKG_MAP_RAPL_PSYS]	= 0x0000064D,
};

static int
cpumon_probe_pkg_intel(struct cpumon_pkg *p)
{
	struct cpu_info *ci = p->p_ci;
	uint64_t rapl_pwr_unit;
	unsigned int unit;
	struct cpumon_rapl *rapl;
	const struct cpumon_intel_f6_pkg *pkg;
	int rv = 0;
	size_t i;
	unsigned int r;

	if (ci->ci_family < 0x06)
		return (0);

	for (i = 0; i < nitems(cpumon_intel_f6_pkgs); i++) {
		pkg = &cpumon_intel_f6_pkgs[i];
		if (pkg->model != ci->ci_model)
			continue;

		rapl_pwr_unit = rdmsr(0x00000606);
		unit = rapl_pwr_unit >>
		    CPUMON_MSR_AMD_RAPL_PWR_UNIT_ESU_SHIFT;
		unit &= CPUMON_MSR_AMD_SR_RAPL_PWR_UNIT_ESU_MASK;

		for (r = 0; r < CPUMON_PKG_MAP_COUNT; r++) {
			if (!ISSET(1 << r, pkg->rapls))
				continue;

			p->p_map[r] = p->p_nkvs;
			p->p_nkvs += 1;

			rapl = &p->p_rapl[r];
			rapl->rapl_energy_msr = cpumon_intel_rapl_msrs[r];
			rapl->rapl_energy_prev = rdmsr(rapl->rapl_energy_msr);
			rapl->rapl_energy_unit = unit;
		}

		if (ISSET(1 << CPUMON_PKG_MAP_RAPL_RAM_FIXED, pkg->rapls)) {
			r = CPUMON_PKG_MAP_RAPL_RAM;

			p->p_map[r] = p->p_nkvs;
			p->p_nkvs += 1;

			rapl = &p->p_rapl[r];
			rapl->rapl_energy_msr = cpumon_intel_rapl_msrs[r];
			rapl->rapl_energy_prev = rdmsr(rapl->rapl_energy_msr);
			rapl->rapl_energy_unit = 16;
		}

		rv = 1;
		break;
	}

	return (rv);
}

static int
cpumon_probe_pkg_amd(struct cpumon_pkg *p)
{
	uint64_t rapl_pwr_unit;
	unsigned int unit;
	struct cpumon_rapl *rapl;
	int rv = 0;

	if (p->p_ci->ci_family >= 0x17) {
		uint32_t eax, ebx, ecx, edx;

		CPUID(0x80000007, eax, ebx, ecx, edx);
		if (edx & (1 << 14)) {
			p->p_map[CPUMON_PKG_MAP_RAPL_PKG] = p->p_nkvs;
			p->p_nkvs += 1;

			rapl_pwr_unit = rdmsr(CPUMON_MSR_AMD_RAPL_PWR_UNIT);
			unit = rapl_pwr_unit >>
			    CPUMON_MSR_AMD_RAPL_PWR_UNIT_ESU_SHIFT;
			unit &= CPUMON_MSR_AMD_SR_RAPL_PWR_UNIT_ESU_MASK;

			rapl = &p->p_rapl[CPUMON_PKG_MAP_RAPL_PKG];

			rapl->rapl_energy_msr = CPUMON_MSR_AMD_PKG_ENERGY_STAT;
			rapl->rapl_energy_prev = rdmsr(rapl->rapl_energy_msr);
			rapl->rapl_energy_unit = unit;

			rv = 1;
		}
	}

	return (rv);
}

static struct cpumon_pkg *
cpumon_attach_pkg(struct cpumon_softc *sc, struct cpu_info *ci)
{
	struct kstat *ks;
	struct kstat_kv *kvs;
	struct cpumon_pkg *p;
	int rv = 0;
	size_t i;
	int idx;

	TAILQ_FOREACH(p, &sc->sc_pkgs, p_entry) {
		if (ci->ci_pkg_id == p->p_ci->ci_pkg_id) {
			/* pkg is already being monitored */

			return (NULL);
		}
	}

	p = malloc(sizeof(*p), M_DEVBUF, M_WAITOK|M_ZERO);
	p->p_sc = sc;
	p->p_ci = ci;
	p->p_nkvs = 1; /* pkg id */
	task_set(&p->p_rapl_xcall, cpumon_pkg_rapl_xcall, p);

	if (strcmp(cpu_vendor, "GenuineIntel") == 0)
		rv = cpumon_probe_pkg_intel(p);
	else if (strcmp(cpu_vendor, "AuthenticAMD") == 0)
		rv = cpumon_probe_pkg_amd(p);

	if (rv == 0) {
		free(p, M_DEVBUF, sizeof(*p));
		return (NULL);
	}

	ks = kstat_create("cpu-pkg", ci->ci_pkg_id, "cpumon", 0, KSTAT_T_KV, 0);
	if (ks == NULL) {
		printf("unable to create cpu-pkg kstat for pkg %u\n",
		    ci->ci_pkg_id);
		return (NULL);
	}

	kvs = mallocarray(p->p_nkvs, sizeof(*kvs), M_DEVBUF, M_WAITOK|M_ZERO);

	kstat_kv_init(&kvs[0], "package", KSTAT_KV_T_UINT32);
	kstat_kv_u32(&kvs[0]) = ci->ci_pkg_id;

	for (i = 0; i < nitems(p->p_map); i++) {
		idx = p->p_map[i];
		if (idx == 0)
			continue;

		kstat_kv_unit_init(&kvs[idx], cpump_pkg_names[i],
		    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_UJOULES);
	}

	ks->ks_data = kvs;
	ks->ks_datalen = p->p_nkvs * sizeof(*kvs);
	ks->ks_read = cpumon_read_pkg;
	ks->ks_softc = p;

	kstat_install(ks);

	TAILQ_INSERT_TAIL(&sc->sc_pkgs, p, p_entry);

	return (p);
}