aboutsummaryrefslogtreecommitdiff
path: root/lib/libc
diff options
context:
space:
mode:
authorKonstantin Belousov <kib@FreeBSD.org>2021-01-06 15:05:09 +0000
committerKonstantin Belousov <kib@FreeBSD.org>2021-01-10 02:42:34 +0000
commitcd240c9cf100bec3def38ceb4a320611b1d02693 (patch)
tree65b39ddd7527608442455d3f7d75898e82ba85c8 /lib/libc
parent5bf4bafd13a4044b7c3d2e8246684a597c6f2134 (diff)
downloadsrc-cd240c9cf100bec3def38ceb4a320611b1d02693.tar.gz
src-cd240c9cf100bec3def38ceb4a320611b1d02693.zip
x86 vdso gettc: Add RDTSCP support
Detect and use RDTSCP if available, instead of fence+RDTSC. For AMD Zens+, use LFENCE+RDTSC instead of RDTSCP (or MFENCE;RDTSC previously). Reviewed by: gallatin, markj Tested by: pho MFC after: 1 week Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D27986
Diffstat (limited to 'lib/libc')
-rw-r--r--lib/libc/x86/sys/__vdso_gettc.c25
1 files changed, 25 insertions, 0 deletions
diff --git a/lib/libc/x86/sys/__vdso_gettc.c b/lib/libc/x86/sys/__vdso_gettc.c
index 4e0fc9f7e841..7f224f8758cb 100644
--- a/lib/libc/x86/sys/__vdso_gettc.c
+++ b/lib/libc/x86/sys/__vdso_gettc.c
@@ -63,6 +63,16 @@ rdtsc_low(const struct vdso_timehands *th)
return (rv);
}
+static inline u_int
+rdtscp_low(const struct vdso_timehands *th)
+{
+ u_int rv;
+
+ __asm __volatile("rdtscp; movl %%edi,%%ecx; shrd %%cl, %%edx, %0"
+ : "=a" (rv) : "D" (th->th_x86_shift) : "ecx", "edx");
+ return (rv);
+}
+
static u_int
rdtsc_low_mb_lfence(const struct vdso_timehands *th)
{
@@ -103,6 +113,12 @@ rdtsc32_mb_none(void)
return (rdtsc32());
}
+static u_int
+rdtscp32_(void)
+{
+ return (rdtscp32());
+}
+
struct tsc_selector_tag {
u_int (*ts_rdtsc32)(void);
u_int (*ts_rdtsc_low)(const struct vdso_timehands *);
@@ -121,6 +137,10 @@ static const struct tsc_selector_tag tsc_selector[] = {
.ts_rdtsc32 = rdtsc32_mb_none,
.ts_rdtsc_low = rdtsc_low_mb_none,
},
+ [3] = { /* RDTSCP */
+ .ts_rdtsc32 = rdtscp32_,
+ .ts_rdtsc_low = rdtscp_low,
+ },
};
static int
@@ -144,6 +164,9 @@ tsc_selector_idx(u_int cpu_feature)
do_cpuid(1, p);
cpu_id = p[0];
+ if (amd_cpu && CPUID_TO_FAMILY(cpu_id) >= 0x17)
+ return (0);
+
if (cpu_feature != 0) {
do_cpuid(0x80000000, p);
cpu_exthigh = p[0];
@@ -157,6 +180,8 @@ tsc_selector_idx(u_int cpu_feature)
amd_feature = 0;
}
+ if ((amd_feature & AMDID_RDTSCP) != 0)
+ return (3);
if ((cpu_feature & CPUID_SSE2) == 0)
return (2);
return (amd_cpu ? 1 : 0);