diff options
| author | Konstantin Belousov <kib@FreeBSD.org> | 2021-01-06 15:05:09 +0000 |
|---|---|---|
| committer | Konstantin Belousov <kib@FreeBSD.org> | 2021-01-10 02:42:34 +0000 |
| commit | cd240c9cf100bec3def38ceb4a320611b1d02693 (patch) | |
| tree | 65b39ddd7527608442455d3f7d75898e82ba85c8 | |
| parent | 5bf4bafd13a4044b7c3d2e8246684a597c6f2134 (diff) | |
| download | src-cd240c9cf100bec3def38ceb4a320611b1d02693.tar.gz src-cd240c9cf100bec3def38ceb4a320611b1d02693.zip | |
x86 vdso gettc: Add RDTSCP support
Detect and use RDTSCP if available, instead of fence+RDTSC. For AMD Zens+,
use LFENCE+RDTSC instead of RDTSCP (or MFENCE;RDTSC previously).
Reviewed by: gallatin, markj
Tested by: pho
MFC after: 1 week
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D27986
| -rw-r--r-- | lib/libc/x86/sys/__vdso_gettc.c | 25 |
1 files changed, 25 insertions, 0 deletions
diff --git a/lib/libc/x86/sys/__vdso_gettc.c b/lib/libc/x86/sys/__vdso_gettc.c index 4e0fc9f7e841..7f224f8758cb 100644 --- a/lib/libc/x86/sys/__vdso_gettc.c +++ b/lib/libc/x86/sys/__vdso_gettc.c @@ -63,6 +63,16 @@ rdtsc_low(const struct vdso_timehands *th) return (rv); } +static inline u_int +rdtscp_low(const struct vdso_timehands *th) +{ + u_int rv; + + __asm __volatile("rdtscp; movl %%edi,%%ecx; shrd %%cl, %%edx, %0" + : "=a" (rv) : "D" (th->th_x86_shift) : "ecx", "edx"); + return (rv); +} + static u_int rdtsc_low_mb_lfence(const struct vdso_timehands *th) { @@ -103,6 +113,12 @@ rdtsc32_mb_none(void) return (rdtsc32()); } +static u_int +rdtscp32_(void) +{ + return (rdtscp32()); +} + struct tsc_selector_tag { u_int (*ts_rdtsc32)(void); u_int (*ts_rdtsc_low)(const struct vdso_timehands *); @@ -121,6 +137,10 @@ static const struct tsc_selector_tag tsc_selector[] = { .ts_rdtsc32 = rdtsc32_mb_none, .ts_rdtsc_low = rdtsc_low_mb_none, }, + [3] = { /* RDTSCP */ + .ts_rdtsc32 = rdtscp32_, + .ts_rdtsc_low = rdtscp_low, + }, }; static int @@ -144,6 +164,9 @@ tsc_selector_idx(u_int cpu_feature) do_cpuid(1, p); cpu_id = p[0]; + if (amd_cpu && CPUID_TO_FAMILY(cpu_id) >= 0x17) + return (0); + if (cpu_feature != 0) { do_cpuid(0x80000000, p); cpu_exthigh = p[0]; @@ -157,6 +180,8 @@ tsc_selector_idx(u_int cpu_feature) amd_feature = 0; } + if ((amd_feature & AMDID_RDTSCP) != 0) + return (3); if ((cpu_feature & CPUID_SSE2) == 0) return (2); return (amd_cpu ? 1 : 0); |