aboutsummaryrefslogtreecommitdiff
path: root/lib/libc
diff options
context:
space:
mode:
authorKonstantin Belousov <kib@FreeBSD.org>2021-01-05 21:14:11 +0000
committerKonstantin Belousov <kib@FreeBSD.org>2021-01-10 02:42:34 +0000
commitde898cb96042a026ef703d81aea6cdf1ffce8f32 (patch)
tree5f3457b18dcd6d52d4bb563ae6ea40e064943dc2 /lib/libc
parent9e680e4005b77e3028d28377ee3722a5260f4422 (diff)
downloadsrc-de898cb96042a026ef703d81aea6cdf1ffce8f32.tar.gz
src-de898cb96042a026ef703d81aea6cdf1ffce8f32.zip
x86 vdso gettc: reorganize ifunctions.
Instead of providing ifuncs for each kind of fence, define ifuncs that combine fence and invocation of RDTSC. This refactoring makes introduction of RDTSCP use possible. Reviewed by: gallatin, markj MFC after: 1 week Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D27986
Diffstat (limited to 'lib/libc')
-rw-r--r--lib/libc/x86/sys/__vdso_gettc.c68
1 files changed, 47 insertions, 21 deletions
diff --git a/lib/libc/x86/sys/__vdso_gettc.c b/lib/libc/x86/sys/__vdso_gettc.c
index 7a958ca256da..b08b11e9c714 100644
--- a/lib/libc/x86/sys/__vdso_gettc.c
+++ b/lib/libc/x86/sys/__vdso_gettc.c
@@ -53,57 +53,83 @@ __FBSDID("$FreeBSD$");
#include <x86/ifunc.h>
#include "libc_private.h"
-static void
-rdtsc_mb_lfence(void)
+static inline u_int
+rdtsc_low(const struct vdso_timehands *th)
{
+ u_int rv;
- lfence();
+ __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
+ : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
+ return (rv);
}
-static void
-rdtsc_mb_mfence(void)
+static u_int
+rdtsc_low_mb_lfence(const struct vdso_timehands *th)
{
+ lfence();
+ return (rdtsc_low(th));
+}
+static u_int
+rdtsc_low_mb_mfence(const struct vdso_timehands *th)
+{
mfence();
+ return (rdtsc_low(th));
}
-static void
-rdtsc_mb_none(void)
+static u_int
+rdtsc_low_mb_none(const struct vdso_timehands *th)
{
+ return (rdtsc_low(th));
}
-DEFINE_UIFUNC(static, void, rdtsc_mb, (void))
+DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc_low,
+ (const struct vdso_timehands *th))
{
u_int p[4];
/* Not a typo, string matches our do_cpuid() registers use. */
static const char intel_id[] = "GenuntelineI";
if ((cpu_feature & CPUID_SSE2) == 0)
- return (rdtsc_mb_none);
+ return (rdtsc_low_mb_none);
do_cpuid(0, p);
return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ?
- rdtsc_mb_lfence : rdtsc_mb_mfence);
+ rdtsc_low_mb_lfence : rdtsc_low_mb_mfence);
}
static u_int
-__vdso_gettc_rdtsc_low(const struct vdso_timehands *th)
+rdtsc32_mb_lfence(void)
{
- u_int rv;
-
- rdtsc_mb();
- __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
- : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
- return (rv);
+ lfence();
+ return (rdtsc32());
}
static u_int
-__vdso_rdtsc32(void)
+rdtsc32_mb_mfence(void)
{
+ mfence();
+ return (rdtsc32());
+}
- rdtsc_mb();
+static u_int
+rdtsc32_mb_none(void)
+{
return (rdtsc32());
}
+DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc32, (void))
+{
+ u_int p[4];
+ /* Not a typo, string matches our do_cpuid() registers use. */
+ static const char intel_id[] = "GenuntelineI";
+
+ if ((cpu_feature & CPUID_SSE2) == 0)
+ return (rdtsc32_mb_none);
+ do_cpuid(0, p);
+ return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ?
+ rdtsc32_mb_lfence : rdtsc32_mb_mfence);
+}
+
#define HPET_DEV_MAP_MAX 10
static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX];
@@ -199,7 +225,7 @@ __vdso_hyperv_tsc(struct hyperv_reftsc *tsc_ref, u_int *tc)
scale = tsc_ref->tsc_scale;
ofs = tsc_ref->tsc_ofs;
- rdtsc_mb();
+ mfence(); /* XXXKIB */
tsc = rdtsc();
/* ret = ((tsc * scale) >> 64) + ofs */
@@ -231,7 +257,7 @@ __vdso_gettc(const struct vdso_timehands *th, u_int *tc)
switch (th->th_algo) {
case VDSO_TH_ALGO_X86_TSC:
*tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) :
- __vdso_rdtsc32();
+ __vdso_gettc_rdtsc32();
return (0);
case VDSO_TH_ALGO_X86_HPET:
idx = th->th_x86_hpet_idx;