diff options
| author | Adam Fenn <adam@fenn.io> | 2021-07-28 17:12:00 +0000 |
|---|---|---|
| committer | Konstantin Belousov <kib@FreeBSD.org> | 2021-10-12 16:01:26 +0000 |
| commit | 946f96d13b013963cc11de1eca2ede4c0658fc61 (patch) | |
| tree | c317442eabbcaec190017d33ffda773cd6d55aca | |
| parent | e088ad2d0efec94462d5c10c992f2ac0b5aa22fd (diff) | |
x86: cpufunc: Add rdtsc_ordered()
Add a variant of 'rdtsc()' that performs the ordered version of 'rdtsc'
appropriate for the invoking x86 variant.
Also, expose the 'lfence'-ed and 'mfence'-ed 'rdtsc()' variants needed
by 'rdtsc_ordered()' for general use.
Sponsored by: Juniper Networks, Inc.
Sponsored by: Klara, Inc.
Reviewed by: kib
Differential Revision: https://reviews.freebsd.org/D31416
(cherry picked from commit 652ae7b11485d9f991ea66a56aa79d4dd9cd1103)
| -rw-r--r-- | sys/amd64/include/cpufunc.h | 14 | ||||
| -rw-r--r-- | sys/i386/include/cpufunc.h | 14 | ||||
| -rw-r--r-- | sys/x86/include/x86_var.h | 1 | ||||
| -rw-r--r-- | sys/x86/x86/cpu_machdep.c | 16 |
4 files changed, 45 insertions, 0 deletions
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h index 93e62c0df73a..95086469d51d 100644 --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -395,6 +395,20 @@ rdtsc(void) } static __inline uint64_t +rdtsc_ordered_lfence(void) +{ + lfence(); + return (rdtsc()); +} + +static __inline uint64_t +rdtsc_ordered_mfence(void) +{ + mfence(); + return (rdtsc()); +} + +static __inline uint64_t rdtscp(void) { uint32_t low, high; diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h index 7b99978facc3..79cdd3004b77 100644 --- a/sys/i386/include/cpufunc.h +++ b/sys/i386/include/cpufunc.h @@ -395,6 +395,20 @@ rdtsc(void) } static __inline uint64_t +rdtsc_ordered_lfence(void) +{ + lfence(); + return (rdtsc()); +} + +static __inline uint64_t +rdtsc_ordered_mfence(void) +{ + mfence(); + return (rdtsc()); +} + +static __inline uint64_t rdtscp(void) { uint64_t rv; diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h index 3e146b44338f..5b8e41bd76d7 100644 --- a/sys/x86/include/x86_var.h +++ b/sys/x86/include/x86_var.h @@ -159,6 +159,7 @@ int pti_get_default(void); int user_dbreg_trap(register_t dr6); int minidumpsys(struct dumperinfo *); struct pcb *get_pcb_td(struct thread *td); +uint64_t rdtsc_ordered(void); #define MSR_OP_ANDNOT 0x00000001 #define MSR_OP_OR 0x00000002 diff --git a/sys/x86/x86/cpu_machdep.c b/sys/x86/x86/cpu_machdep.c index 8bceaee913db..26423228d5dc 100644 --- a/sys/x86/x86/cpu_machdep.c +++ b/sys/x86/x86/cpu_machdep.c @@ -76,6 +76,7 @@ __FBSDID("$FreeBSD$"); #include <machine/clock.h> #include <machine/cpu.h> +#include <machine/cpufunc.h> #include <machine/cputypes.h> #include <machine/specialreg.h> #include <machine/md_var.h> @@ -88,6 +89,7 @@ __FBSDID("$FreeBSD$"); #include <machine/elan_mmcr.h> #endif #include <x86/acpica_machdep.h> +#include <x86/ifunc.h> #include <vm/vm.h> #include <vm/vm_extern.h> @@ -1421,3 +1423,17 @@ acpi_get_fadt_bootflags(uint16_t *flagsp) return (false); #endif } + +DEFINE_IFUNC(, uint64_t, rdtsc_ordered, (void), static) +{ + bool cpu_is_amd = cpu_vendor_id == CPU_VENDOR_AMD || + cpu_vendor_id == CPU_VENDOR_HYGON; + + if ((amd_feature & AMDID_RDTSCP) != 0) + return (rdtscp); + else if ((cpu_feature & CPUID_SSE2) != 0) + return (cpu_is_amd ? rdtsc_ordered_mfence : + rdtsc_ordered_lfence); + else + return (rdtsc); +} |