hwpmc: Implement IBS process sampling

This change enables process-wide sampling to work with IBS by ensuring
that read/write only gets or sets the current counter.

Reviewed by:	mhorne
Sponsored by:	Netflix
Pull Request:	https://github.com/freebsd/freebsd-src/pull/2131

2 files changed, 33 insertions, 3 deletions

diff --git a/sys/dev/hwpmc/hwpmc_ibs.c b/sys/dev/hwpmc/hwpmc_ibs.c
index a230288f157e..574df12093f8 100644
--- a/sys/dev/hwpmc/hwpmc_ibs.c
+++ b/sys/dev/hwpmc/hwpmc_ibs.c
@@ -82,13 +82,13 @@ ibs_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
 	KASSERT(ibs_pcpu[cpu],
 	    ("[ibs,%d] null per-cpu, cpu %d", __LINE__, cpu));
 
-	/* read the IBS ctl */
+	/* read the IBS count */
 	switch (ri) {
 	case IBS_PMC_FETCH:
-		*v = rdmsr(IBS_FETCH_CTL);
+		*v = IBS_FETCH_CTL_TO_COUNT(rdmsr(IBS_FETCH_CTL));
 		break;
 	case IBS_PMC_OP:
-		*v = rdmsr(IBS_OP_CTL);
+		*v = IBS_OP_CTL_TO_COUNT(rdmsr(IBS_OP_CTL));
 		break;
 	}
 
@@ -103,12 +103,31 @@ ibs_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
 static int
 ibs_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
 {
+	pmc_value_t m;
 
 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 	    ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
 	KASSERT(ri >= 0 && ri < IBS_NPMCS,
 	    ("[ibs,%d] illegal row-index %d", __LINE__, ri));
 
+	/* write the IBS count */
+	switch (ri) {
+	case IBS_PMC_FETCH:
+		m = rdmsr(IBS_FETCH_CTL) & ~IBS_FETCH_CTL_CURCNTMASK;
+		/* Setting a count greater than interval is undefined. */
+		if (IBS_FETCH_CTL_TO_INTERVAL(m) > v)
+			m |= IBS_FETCH_COUNT_TO_CTL(v);
+		wrmsr(IBS_FETCH_CTL, m);
+		break;
+	case IBS_PMC_OP:
+		m = rdmsr(IBS_OP_CTL) & ~IBS_OP_CTL_CURCNTMASK;
+		/* Setting a count greater than interval is undefined */
+		if (IBS_OP_CTL_TO_INTERVAL(m) > v)
+			m |= IBS_OP_COUNT_TO_CTL(v);
+		wrmsr(IBS_OP_CTL, m);
+		break;
+	}
+
 	PMCDBG3(MDP, WRI, 1, "ibs-write cpu=%d ri=%d v=%jx", cpu, ri, v);
 
 	return (0);
@@ -177,6 +196,9 @@ ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
 	if ((caps & PMC_CAP_SYSTEM) == 0)
 		return (EINVAL);
 
+	if (!PMC_IS_SAMPLING_MODE(a->pm_mode))
+		return (EINVAL);
+
 	config = a->pm_md.pm_ibs.ibs_ctl;
 	pm->pm_md.pm_ibs.ibs_ctl = config;
 
diff --git a/sys/dev/hwpmc/hwpmc_ibs.h b/sys/dev/hwpmc/hwpmc_ibs.h
index 1616a746ffef..4cb69503bd8b 100644
--- a/sys/dev/hwpmc/hwpmc_ibs.h
+++ b/sys/dev/hwpmc/hwpmc_ibs.h
@@ -92,9 +92,13 @@
 #define IBS_FETCH_CTL_VALID		(1ULL << 49) /* Valid */
 #define IBS_FETCH_CTL_ENABLE		(1ULL << 48) /* Enable */
 #define IBS_FETCH_CTL_MAXCNTMASK	0x0000FFFFULL
+#define IBS_FETCH_CTL_CURCNTMASK	0xFFFF0000ULL
 
 #define IBS_FETCH_INTERVAL_TO_CTL(_c)	(((_c) >> 4) & 0x0000FFFF)
+#define IBS_FETCH_CTL_TO_INTERVAL(_c)	(((_c) & IBS_FETCH_CTL_MAXCNTMASK) << 4)
 #define IBS_FETCH_CTL_TO_LAT(_c)	(((_c) >> 32) & 0x0000FFFF)
+#define IBS_FETCH_COUNT_TO_CTL(_c)	(((_c) << 12) & IBS_FETCH_CTL_CURCNTMASK)
+#define IBS_FETCH_CTL_TO_COUNT(_c)	(((_c) & IBS_FETCH_CTL_CURCNTMASK) >> 12)
 
 #define IBS_FETCH_LINADDR		0xC0011031 /* Fetch Linear Address */
 #define IBS_FETCH_PHYSADDR		0xC0011032 /* Fetch Physical Address */
@@ -113,9 +117,13 @@
 #define IBS_OP_CTL_ENABLE		(1ULL << 17) /* Enable */
 #define IBS_OP_CTL_L3MISSONLY		(1ULL << 16) /* L3 Miss Filtering */
 #define IBS_OP_CTL_MAXCNTMASK		0x07F0FFFFULL
+#define IBS_OP_CTL_CURCNTMASK		0x07FFFFFF00000000ULL
 
 #define IBS_OP_CTL_LDLAT_TO_CTL(_c)	((((ldlat) >> 7) - 1) << 59)
 #define IBS_OP_INTERVAL_TO_CTL(_c)	((((_c) >> 4) & 0x0000FFFFULL) | ((_c) & 0x07F00000))
+#define IBS_OP_CTL_TO_INTERVAL(_c)	((((_c) & 0x0000FFFFULL) << 4) | ((_c) & 0x07F00000))
+#define IBS_OP_COUNT_TO_CTL(_c)		(((_c) << 32) & IBS_OP_CTL_CURCNTMASK)
+#define IBS_OP_CTL_TO_COUNT(_c)		(((_c) & IBS_OP_CTL_CURCNTMASK) >> 32)
 
 #define IBS_OP_RIP			0xC0011034 /* IBS Op RIP */
 #define IBS_OP_DATA			0xC0011035 /* IBS Op Data */