x86: Probe the TSC frequency earlier

This lets us use the TSC to implement early DELAY, limiting the use of
the sometimes-unreliable 8254 PIT.

PR:		262155
Reviewed by:	emaste
Tested by:	emaste, mike tancsa <mike@sentex.net>, Stefan Hegnauer <stefan.hegnauer@gmx.ch>
MFC after:	1 month
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D34367

5 files changed, 94 insertions, 61 deletions

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index c629db566528..6eda6c9c8352 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -169,6 +169,9 @@ extern u_int64_t hammer_time(u_int64_t, u_int64_t);
 static void cpu_startup(void *);
 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 
+/* Probe 8254 PIT and TSC. */
+static void native_clock_source_init(void);
+
 /* Preload data parse function */
 static caddr_t native_parse_preload_data(u_int64_t);
 
@@ -177,8 +180,8 @@ static void native_parse_memmap(caddr_t, vm_paddr_t *, int *);
 
 /* Default init_ops implementation. */
 struct init_ops init_ops = {
-	.parse_preload_data =	native_parse_preload_data,
-	.early_clock_source_init =	i8254_init,
+	.parse_preload_data =		native_parse_preload_data,
+	.early_clock_source_init =	native_clock_source_init,
 	.early_delay =			i8254_delay,
 	.parse_memmap =			native_parse_memmap,
 };
@@ -1161,6 +1164,13 @@ native_parse_preload_data(u_int64_t modulep)
 }
 
 static void
+native_clock_source_init(void)
+{
+	i8254_init();
+	tsc_init();
+}
+
+static void
 amd64_kdb_init(void)
 {
 	kdb_init();
diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c
index ee6752861c9e..6913c0691fd4 100644
--- a/sys/i386/i386/machdep.c
+++ b/sys/i386/i386/machdep.c
@@ -188,6 +188,8 @@ struct kva_md_info kmi;
 static struct trapframe proc0_tf;
 struct pcpu __pcpu[MAXCPU];
 
+static void i386_clock_source_init(void);
+
 struct mtx icu_lock;
 
 struct mem_range_softc mem_range_softc;
@@ -198,11 +200,18 @@ extern struct sysentvec elf32_freebsd_sysvec;
 
 /* Default init_ops implementation. */
 struct init_ops init_ops = {
-	.early_clock_source_init =	i8254_init,
+	.early_clock_source_init =	i386_clock_source_init,
 	.early_delay =			i8254_delay,
 };
 
 static void
+i386_clock_source_init(void)
+{
+	i8254_init();
+	tsc_init();
+}
+
+static void
 cpu_startup(dummy)
 	void *dummy;
 {
diff --git a/sys/x86/include/clock.h b/sys/x86/include/clock.h
index 83c8351ed31c..9aeccadf89aa 100644
--- a/sys/x86/include/clock.h
+++ b/sys/x86/include/clock.h
@@ -28,6 +28,7 @@ void	i8254_init(void);
 void	i8254_delay(int);
 void	clock_init(void);
 void	lapic_calibrate(void);
+void	tsc_init(void);
 void	tsc_calibrate(void);
 
 /*
@@ -35,7 +36,7 @@ void	tsc_calibrate(void);
  */
 
 void	startrtclock(void);
-void	init_TSC(void);
+void	start_TSC(void);
 void	resume_TSC(void);
 
 #define	HAS_TIMER_SPKR 1
diff --git a/sys/x86/isa/clock.c b/sys/x86/isa/clock.c
index 1178d35979c1..f21f847709cd 100644
--- a/sys/x86/isa/clock.c
+++ b/sys/x86/isa/clock.c
@@ -398,10 +398,10 @@ i8254_init(void)
 }
 
 void
-startrtclock()
+startrtclock(void)
 {
 
-	init_TSC();
+	start_TSC();
 }
 
 void
diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
index 317be8979feb..82ee358b6895 100644
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -265,17 +265,42 @@ tsc_freq_8254(uint64_t *res)
 static void
 probe_tsc_freq(void)
 {
-	if (cpu_power_ecx & CPUID_PERF_STAT) {
-		/*
-		 * XXX Some emulators expose host CPUID without actual support
-		 * for these MSRs.  We must test whether they really work.
-		 */
-		wrmsr(MSR_MPERF, 0);
-		wrmsr(MSR_APERF, 0);
-		DELAY(10);
-		if (rdmsr(MSR_MPERF) > 0 && rdmsr(MSR_APERF) > 0)
-			tsc_perf_stat = 1;
+#ifdef __i386__
+	/* The TSC is known to be broken on certain CPUs. */
+	switch (cpu_vendor_id) {
+	case CPU_VENDOR_AMD:
+		switch (cpu_id & 0xFF0) {
+		case 0x500:
+			/* K5 Model 0 */
+			tsc_disabled = 1;
+			return;
+		}
+		break;
+	case CPU_VENDOR_CENTAUR:
+		switch (cpu_id & 0xff0) {
+		case 0x540:
+			/*
+			 * http://www.centtech.com/c6_data_sheet.pdf
+			 *
+			 * I-12 RDTSC may return incoherent values in EDX:EAX
+			 * I-13 RDTSC hangs when certain event counters are used
+			 */
+			tsc_disabled = 1;
+			return;
+		}
+		break;
+	case CPU_VENDOR_NSC:
+		switch (cpu_id & 0xff0) {
+		case 0x540:
+			if ((cpu_id & CPUID_STEPPING) == 0) {
+				tsc_disabled = 1;
+				return;
+			}
+			break;
+		}
+		break;
 	}
+#endif
 
 	switch (cpu_vendor_id) {
 	case CPU_VENDOR_AMD:
@@ -315,15 +340,18 @@ probe_tsc_freq(void)
 		break;
 	}
 
-	if (tsc_freq_cpuid_vm())
-		return;
-
-	if (vm_guest == VM_GUEST_VMWARE) {
+	if (tsc_freq_cpuid_vm()) {
+		if (bootverbose)
+			printf(
+		    "Early TSC frequency %juHz derived from hypervisor CPUID\n",
+			    (uintmax_t)tsc_freq);
+	} else if (vm_guest == VM_GUEST_VMWARE) {
 		tsc_freq_vmware();
-		return;
-	}
-
-	if (tsc_freq_cpuid(&tsc_freq)) {
+		if (bootverbose)
+			printf(
+		    "Early TSC frequency %juHz derived from VMWare hypercall\n",
+			    (uintmax_t)tsc_freq);
+	} else if (tsc_freq_cpuid(&tsc_freq)) {
 		/*
 		 * If possible, use the value obtained from CPUID as the initial
 		 * frequency.  This will be refined later during boot but is
@@ -361,50 +389,26 @@ probe_tsc_freq(void)
 		    "Early TSC frequency %juHz calibrated from 8254 PIT\n",
 			    (uintmax_t)tsc_freq);
 	}
+
+	if (cpu_power_ecx & CPUID_PERF_STAT) {
+		/*
+		 * XXX Some emulators expose host CPUID without actual support
+		 * for these MSRs.  We must test whether they really work.
+		 */
+		wrmsr(MSR_MPERF, 0);
+		wrmsr(MSR_APERF, 0);
+		DELAY(10);
+		if (rdmsr(MSR_MPERF) > 0 && rdmsr(MSR_APERF) > 0)
+			tsc_perf_stat = 1;
+	}
 }
 
 void
-init_TSC(void)
+start_TSC(void)
 {
-
 	if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled)
 		return;
 
-#ifdef __i386__
-	/* The TSC is known to be broken on certain CPUs. */
-	switch (cpu_vendor_id) {
-	case CPU_VENDOR_AMD:
-		switch (cpu_id & 0xFF0) {
-		case 0x500:
-			/* K5 Model 0 */
-			return;
-		}
-		break;
-	case CPU_VENDOR_CENTAUR:
-		switch (cpu_id & 0xff0) {
-		case 0x540:
-			/*
-			 * http://www.centtech.com/c6_data_sheet.pdf
-			 *
-			 * I-12 RDTSC may return incoherent values in EDX:EAX
-			 * I-13 RDTSC hangs when certain event counters are used
-			 */
-			return;
-		}
-		break;
-	case CPU_VENDOR_NSC:
-		switch (cpu_id & 0xff0) {
-		case 0x540:
-			if ((cpu_id & CPUID_STEPPING) == 0)
-				return;
-			break;
-		}
-		break;
-	}
-#endif
-
-	probe_tsc_freq();
-
 	/*
 	 * Inform CPU accounting about our boot-time clock rate.  This will
 	 * be updated if someone loads a cpufreq driver after boot that
@@ -708,6 +712,15 @@ tsc_update_freq(uint64_t new_freq)
 	    new_freq >> (int)(intptr_t)tsc_timecounter.tc_priv);
 }
 
+void
+tsc_init(void)
+{
+	if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled)
+		return;
+
+	probe_tsc_freq();
+}
+
 /*
  * Perform late calibration of the TSC frequency once ACPI-based timecounters
  * are available.  At this point timehands are not set up, so we read the