diff options
| -rw-r--r-- | lib/libpmc/libpmc_pmu_util.c | 71 | ||||
| -rw-r--r-- | lib/libpmc/pmu-events/jevents.c | 40 | ||||
| -rw-r--r-- | lib/libpmc/pmu-events/json.c | 14 | ||||
| -rw-r--r-- | lib/libpmc/pmu-events/json.h | 1 | ||||
| -rw-r--r-- | sys/dev/hwpmc/hwpmc_amd.c | 19 | ||||
| -rw-r--r-- | sys/dev/hwpmc/hwpmc_amd.h | 93 |
6 files changed, 204 insertions, 34 deletions
diff --git a/lib/libpmc/libpmc_pmu_util.c b/lib/libpmc/libpmc_pmu_util.c index de642aa71a18..0832ab32e2f1 100644 --- a/lib/libpmc/libpmc_pmu_util.c +++ b/lib/libpmc/libpmc_pmu_util.c @@ -121,6 +121,24 @@ pmu_events_mfr(void) return (mfr); } +static int +pmu_events_x86_family(void) +{ + char buf[PMC_CPUID_LEN]; + size_t s = sizeof(buf); + char *cpuid, *family; + + if (sysctlbyname("kern.hwpmc.cpuid", buf, &s, + (void *)NULL, 0) == -1) + return (-1); + cpuid = &buf[0]; + + strsep(&cpuid, "-"); + family = strsep(&cpuid, "-"); + + return (strtol(family, NULL, 10)); +} + /* * The Intel fixed mode counters are: * "inst_retired.any", @@ -208,6 +226,10 @@ struct pmu_event_desc { uint64_t ped_offcore_rsp; uint64_t ped_l3_thread; uint64_t ped_l3_slice; + uint32_t ped_sourceid; + uint32_t ped_coreid; + uint32_t ped_allsources; + uint32_t ped_allcores; uint32_t ped_event; uint32_t ped_frontend; uint32_t ped_ldlat; @@ -347,6 +369,14 @@ pmu_parse_event(struct pmu_event_desc *ped, const char *eventin) ped->ped_l3_thread = strtol(value, NULL, 16); else if (strcmp(key, "l3_slice_mask") == 0) ped->ped_l3_slice = strtol(value, NULL, 16); + else if (strcmp(key, "sourceid") == 0) + ped->ped_sourceid = strtol(value, NULL, 16); + else if (strcmp(key, "coreid") == 0) + ped->ped_coreid = strtol(value, NULL, 16); + else if (strcmp(key, "allcores") == 0) + ped->ped_allcores = strtol(value, NULL, 10); + else if (strcmp(key, "allsources") == 0) + ped->ped_allsources = strtol(value, NULL, 10); else { debug = getenv("PMUDEBUG"); if (debug != NULL && strcmp(debug, "true") == 0 && value != NULL) @@ -486,20 +516,23 @@ static int pmc_pmu_amd_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm, struct pmu_event_desc *ped) { + int cpu_family; struct pmc_md_amd_op_pmcallocate *amd; const struct pmu_event *pe; int idx = -1; + cpu_family = pmu_events_x86_family(); + amd = &pm->pm_md.pm_amd; if (ped->ped_umask > 0) { pm->pm_caps |= PMC_CAP_QUALIFIER; - amd->pm_amd_config |= AMD_PMC_TO_UNITMASK(ped->ped_umask); } pm->pm_class = PMC_CLASS_K8; pe = pmu_event_get(NULL, event_name, &idx); if (pe->pmu == NULL) { amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK(ped->ped_event); + amd->pm_amd_config |= AMD_PMC_TO_UNITMASK(ped->ped_umask); amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_CORE; if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 || (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == @@ -515,14 +548,42 @@ pmc_pmu_amd_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm, amd->pm_amd_config |= AMD_PMC_INVERT; if (pm->pm_caps & PMC_CAP_INTERRUPT) amd->pm_amd_config |= AMD_PMC_INT; + if (pm->pm_caps & PMC_CAP_PRECISE) + amd->pm_amd_config |= AMD_PMC_PRECISERETIRE; } else if (strcmp("amd_l3", pe->pmu) == 0) { - amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK(ped->ped_event); + amd->pm_amd_config |= AMD_PMC_L3_TO_EVENTMASK(ped->ped_event); + amd->pm_amd_config |= AMD_PMC_L3_TO_UNITMASK(ped->ped_umask); amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_L3_CACHE; - amd->pm_amd_config |= AMD_PMC_TO_L3SLICE(ped->ped_l3_slice); - amd->pm_amd_config |= AMD_PMC_TO_L3CORE(ped->ped_l3_thread); + if (cpu_family <= 0x17) { + amd->pm_amd_config |= + AMD_PMC_L31_TO_SLICE(ped->ped_l3_slice); + amd->pm_amd_config |= + AMD_PMC_L31_TO_CORE(ped->ped_l3_thread); + } else { + amd->pm_amd_config |= + AMD_PMC_L32_TO_THREAD(ped->ped_l3_thread); + amd->pm_amd_config |= + AMD_PMC_L32_TO_SOURCEID(ped->ped_sourceid); + amd->pm_amd_config |= + AMD_PMC_L32_TO_COREID(ped->ped_coreid); + if (ped->ped_allcores) + amd->pm_amd_config |= AMD_PMC_L32_ALLCORES; + if (ped->ped_allsources) + amd->pm_amd_config |= AMD_PMC_L32_ALLSOURCES; + } } else if (strcmp("amd_df", pe->pmu) == 0) { - amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK_DF(ped->ped_event); amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_DATA_FABRIC; + if (cpu_family <= 19) { + amd->pm_amd_config |= + AMD_PMC_DF1_TO_EVENTMASK(ped->ped_event); + amd->pm_amd_config |= + AMD_PMC_DF1_TO_UNITMASK(ped->ped_umask); + } else { + amd->pm_amd_config |= + AMD_PMC_DF2_TO_EVENTMASK(ped->ped_event); + amd->pm_amd_config |= + AMD_PMC_DF2_TO_UNITMASK(ped->ped_umask); + } } else { printf("PMC pmu '%s' is not supported!\n", pe->pmu); return (EOPNOTSUPP); diff --git a/lib/libpmc/pmu-events/jevents.c b/lib/libpmc/pmu-events/jevents.c index 628ed26c6f9d..4e4f53a0c0d0 100644 --- a/lib/libpmc/pmu-events/jevents.c +++ b/lib/libpmc/pmu-events/jevents.c @@ -560,6 +560,10 @@ static int json_events(const char *fn, jsmntok_t *obj = tok++; bool configcode_present = false; char *umask = NULL; + char *allcores = NULL; + char *allslices = NULL; + char *sliceid = NULL; + char *threadmask = NULL; char *cmask = NULL; char *inv = NULL; char *any = NULL; @@ -585,6 +589,22 @@ static int json_events(const char *fn, /* match_field */ if (json_streq(map, field, "UMask") && nz) { addfield(map, &umask, "", "umask=", val); + } else if (json_streq(map, field, "EnAllCores") && nz) { + addfield(map, &allcores, "", "allcores=", val); + } else if (json_streq(map, field, "EnAllSlices") && nz) { + addfield(map, &allslices, "", "allslices=", val); + } else if (json_streq(map, field, "SliceId") && nz) { + /* + * We use sourceid because there's a + * descripency where the JSON from linux calls + * it a SliceId, which is not the name used by + * AMD in the PPRs. The field name from Family + * 19h and below that calls it slicemask see + * the references in hwpmc_amd.h. + */ + addfield(map, &sliceid, "", "sourceid=", val); + } else if (json_streq(map, field, "ThreadMask") && nz) { + addfield(map, &threadmask, "", "l3_thread_mask=", val); } else if (json_streq(map, field, "CounterMask") && nz) { addfield(map, &cmask, "", "cmask=", val); } else if (json_streq(map, field, "Invert") && nz) { @@ -675,8 +695,14 @@ static int json_events(const char *fn, addfield(map, &arch_std, "", "", val); for (s = arch_std; *s; s++) *s = tolower(*s); + } else { + /* + * We shouldn't ignore unknown fields that + * makes the counter invalid! + */ + json_copystr(map, field, buf, sizeof(buf)); + fprintf(stderr, "Unknown field '%s'!\n", buf); } - /* ignore unknown fields */ } if (precise && je.desc && !strstr(je.desc, "(Precise Event)")) { if (json_streq(map, precise, "2")) @@ -707,6 +733,14 @@ static int json_events(const char *fn, addfield(map, &event, ",", period, NULL); if (umask) addfield(map, &event, ",", umask, NULL); + if (allcores) + addfield(map, &event, ",", allcores, NULL); + if (allslices) + addfield(map, &event, ",", allslices, NULL); + if (sliceid) + addfield(map, &event, ",", sliceid, NULL); + if (threadmask) + addfield(map, &event, ",", threadmask, NULL); if (je.desc && extra_desc) addfield(map, &je.desc, " ", extra_desc, NULL); @@ -737,6 +771,10 @@ static int json_events(const char *fn, err = func(data, &je); free_strings: free(umask); + free(allcores); + free(allslices); + free(sliceid); + free(threadmask); free(cmask); free(inv); free(any); diff --git a/lib/libpmc/pmu-events/json.c b/lib/libpmc/pmu-events/json.c index 89cafbc04fb6..66eaf0a74ba6 100644 --- a/lib/libpmc/pmu-events/json.c +++ b/lib/libpmc/pmu-events/json.c @@ -160,3 +160,17 @@ int json_streq(char *map, jsmntok_t *t, const char *s) unsigned len = json_len(t); return len == strlen(s) && !strncasecmp(map + t->start, s, len); } + +int json_copystr(char *map, jsmntok_t *t, char *s, int len) +{ + int jlen; + + jlen = json_len(t); + if (jlen > len) + jlen = len - 1; + + memcpy(s, map + t->start, jlen); + s[jlen] = '\0'; + + return (jlen); +} diff --git a/lib/libpmc/pmu-events/json.h b/lib/libpmc/pmu-events/json.h index 278ebd32cfb6..89b9c2fba617 100644 --- a/lib/libpmc/pmu-events/json.h +++ b/lib/libpmc/pmu-events/json.h @@ -9,6 +9,7 @@ int json_line(char *map, jsmntok_t *t); const char *json_name(jsmntok_t *t); int json_streq(char *map, jsmntok_t *t, const char *s); int json_len(jsmntok_t *t); +int json_copystr(char *map, jsmntok_t *t, char *s, int len); extern int verbose; diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index 801b75b39595..b34cbffcffa8 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -347,6 +347,10 @@ amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, caps = pm->pm_caps; + if (((caps & PMC_CAP_PRECISE) != 0) && + ((pd->pd_caps & PMC_CAP_PRECISE) == 0)) + return (EINVAL); + PMCDBG2(MDP, ALL, 1,"amd-allocate ri=%d caps=0x%x", ri, caps); /* Validate sub-class. */ @@ -360,6 +364,9 @@ amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, return (0); } + /* + * Everything below this is for supporting older processors. + */ pe = a->pm_ev; /* map ev to the correct event mask code */ @@ -817,6 +824,14 @@ pmc_amd_initialize(void) "K8-%d", i); d->pm_descr.pd_class = PMC_CLASS_K8; d->pm_descr.pd_caps = AMD_PMC_CAPS; + /* + * Zen 5 can precisely count retire events. + * + * Refer to PPR Vol 1 for AMD Family 1Ah Model 02h C1 57238 + * Rev. 0.24 September 29, 2024. + */ + if ((family >= 0x1a) && (i == 2)) + d->pm_descr.pd_caps |= PMC_CAP_PRECISE; d->pm_descr.pd_width = 48; if ((amd_feature2 & AMDID2_PCXC) != 0) { d->pm_evsel = AMD_PMC_CORE_BASE + 2 * i; @@ -836,7 +851,7 @@ pmc_amd_initialize(void) snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, "K8-L3-%d", i); d->pm_descr.pd_class = PMC_CLASS_K8; - d->pm_descr.pd_caps = AMD_PMC_CAPS; + d->pm_descr.pd_caps = AMD_PMC_L3_CAPS; d->pm_descr.pd_width = 48; d->pm_evsel = AMD_PMC_L3_BASE + 2 * i; d->pm_perfctr = AMD_PMC_L3_BASE + 2 * i + 1; @@ -852,7 +867,7 @@ pmc_amd_initialize(void) snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, "K8-DF-%d", i); d->pm_descr.pd_class = PMC_CLASS_K8; - d->pm_descr.pd_caps = AMD_PMC_CAPS; + d->pm_descr.pd_caps = AMD_PMC_DF_CAPS; d->pm_descr.pd_width = 48; d->pm_evsel = AMD_PMC_DF_BASE + 2 * i; d->pm_perfctr = AMD_PMC_DF_BASE + 2 * i + 1; diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h index be484a1111a2..6d8ab8203942 100644 --- a/sys/dev/hwpmc/hwpmc_amd.h +++ b/sys/dev/hwpmc/hwpmc_amd.h @@ -62,21 +62,10 @@ #define AMD_PMC_CORE_DEFAULT 6 #define AMD_PMC_CORE_MAX 16 -/* L3 */ -#define AMD_PMC_L3_BASE 0xC0010230 -#define AMD_PMC_L3_DEFAULT 6 -#define AMD_PMC_L3_MAX 6 - -/* DF */ -#define AMD_PMC_DF_BASE 0xC0010240 -#define AMD_PMC_DF_DEFAULT 4 -#define AMD_PMC_DF_MAX 64 - -#define AMD_NPMCS_K8 4 -#define AMD_NPMCS_MAX (AMD_PMC_CORE_MAX + AMD_PMC_L3_MAX + \ - AMD_PMC_DF_MAX) - #define AMD_PMC_COUNTERMASK 0xFF000000 +#define AMD_PMC_PRECISERETIRE (1ULL << 43) /* Only valid for PERF_CTL2 */ +#define AMD_PMC_HOST (1ULL << 41) +#define AMD_PMC_GUEST (1ULL << 40) #define AMD_PMC_TO_COUNTER(x) (((x) << 24) & AMD_PMC_COUNTERMASK) #define AMD_PMC_INVERT (1 << 23) #define AMD_PMC_ENABLE (1 << 22) @@ -85,24 +74,13 @@ #define AMD_PMC_EDGE (1 << 18) #define AMD_PMC_OS (1 << 17) #define AMD_PMC_USR (1 << 16) -#define AMD_PMC_L3SLICEMASK (0x000F000000000000) -#define AMD_PMC_L3COREMASK (0xFF00000000000000) -#define AMD_PMC_TO_L3SLICE(x) (((x) << 48) & AMD_PMC_L3SLICEMASK) -#define AMD_PMC_TO_L3CORE(x) (((x) << 56) & AMD_PMC_L3COREMASK) - -#define AMD_PMC_UNITMASK_M 0x10 -#define AMD_PMC_UNITMASK_O 0x08 -#define AMD_PMC_UNITMASK_E 0x04 -#define AMD_PMC_UNITMASK_S 0x02 -#define AMD_PMC_UNITMASK_I 0x01 -#define AMD_PMC_UNITMASK_MOESI 0x1F #define AMD_PMC_UNITMASK 0xFF00 #define AMD_PMC_EVENTMASK 0xF000000FF #define AMD_PMC_TO_UNITMASK(x) (((x) << 8) & AMD_PMC_UNITMASK) #define AMD_PMC_TO_EVENTMASK(x) (((x) & 0xFF) | (((uint64_t)(x) & 0xF00) << 24)) -#define AMD_PMC_TO_EVENTMASK_DF(x) (((x) & 0xFF) | (((uint64_t)(x) & 0x0F00) << 24)) | (((uint64_t)(x) & 0x3000) << 47) + #define AMD_VALID_BITS (AMD_PMC_COUNTERMASK | AMD_PMC_INVERT | \ AMD_PMC_ENABLE | AMD_PMC_INT | AMD_PMC_PC | AMD_PMC_EDGE | \ AMD_PMC_OS | AMD_PMC_USR | AMD_PMC_UNITMASK | AMD_PMC_EVENTMASK) @@ -111,6 +89,69 @@ PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_THRESHOLD | \ PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER) +/* L3 */ +#define AMD_PMC_L3_BASE 0xC0010230 +#define AMD_PMC_L3_DEFAULT 6 +#define AMD_PMC_L3_MAX 6 + +/* + * L3 counters change their encoding slightly between Family 17h and Family 19h + * processors. + * + * Refer to the following documents for the L3 fields: + * PPR for AMD Family 17h Model 20h A1 55772-A1 Rev. 3.08 April 14, 2021 + * PPR for AMD Family 19h Model 51h A1 56569-A1 Rev. 3.03 September 21, 2021 + * PPR for AMD Family 1Ah Model 02h C1 57238 Rev. 0.24 September 29, 2024 + */ +#define AMD_PMC_L31_SLICEMASK (0x000F000000000000ULL) +#define AMD_PMC_L31_COREMASK (0xFF00000000000000ULL) + +#define AMD_PMC_L31_TO_SLICE(x) (((uint64_t)(x) << 48) & AMD_PMC_L31_SLICEMASK) +#define AMD_PMC_L31_TO_CORE(x) (((uint64_t)(x) << 56) & AMD_PMC_L31_COREMASK) + +#define AMD_PMC_L32_THREADMASK (0x0F00000000000000ULL) +#define AMD_PMC_L32_SOURCEMASK (0x0007000000000000ULL) +#define AMD_PMC_L32_ALLCORES (1ULL << 47) +#define AMD_PMC_L32_ALLSOURCES (1ULL << 46) +#define AMD_PMC_L32_COREMASK (0x00001C0000000000ULL) + +#define AMD_PMC_L32_TO_THREAD(x) (((uint64_t)(x) << 56) & AMD_PMC_L32_THREADMASK) +#define AMD_PMC_L32_TO_SOURCEID(x) (((uint64_t)(x) << 48) & AMD_PMC_L32_SOURCEMASK) +#define AMD_PMC_L32_TO_COREID(x) (((uint64_t)(x) << 42) & AMD_PMC_L32_COREMASK) + +#define AMD_PMC_L3_TO_UNITMASK(x) (((x) << 8) & AMD_PMC_UNITMASK) +#define AMD_PMC_L3_TO_EVENTMASK(x) ((x) & 0xFF) + +#define AMD_PMC_L3_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | \ + PMC_CAP_QUALIFIER | PMC_CAP_DOMWIDE) + +/* DF */ +#define AMD_PMC_DF_BASE 0xC0010240 +#define AMD_PMC_DF_DEFAULT 4 +#define AMD_PMC_DF_MAX 64 + +#define AMD_PMC_DF_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | \ + PMC_CAP_QUALIFIER | PMC_CAP_DOMWIDE) + +/* + * DF counters change their encoding between Family 19h and Family 1Ah + * processors. + * + * Refer to the same documents as the L3 counters. + */ +#define AMD_PMC_DF1_TO_EVENTMASK(x) (((x) & 0xFF) | \ + (((uint64_t)(x) & 0x0F00) << 24) | (((uint64_t)(x) & 0x3000) << 47)) +#define AMD_PMC_DF1_TO_UNITMASK(x) (((x) & 0xFF) << 8) + +#define AMD_PMC_DF2_TO_EVENTMASK(x) (((x) & 0xFF) | \ + (((uint64_t)(x) & 0x7F00) << 24)) +#define AMD_PMC_DF2_TO_UNITMASK(x) ((((x) & 0xFF) << 8) | \ + (((uint64_t)(x) & 0x0F00) << 16)) + +#define AMD_NPMCS_K8 4 +#define AMD_NPMCS_MAX (AMD_PMC_CORE_MAX + AMD_PMC_L3_MAX + \ + AMD_PMC_DF_MAX) + #define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) #define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) |
