x86: Always use clean FPU and segment base state for new kthreads.

Reviewed by:	kib
MFC after:	1 week
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D29208

3 files changed, 81 insertions, 30 deletions

diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
index f10d0339a65a..6e60f2b3faff 100644
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -164,10 +164,12 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 		return;
 	}
 
-	/* Ensure that td1's pcb is up to date. */
-	fpuexit(td1);
-	if (td1 == curthread)
+	/* Ensure that td1's pcb is up to date for user processes. */
+	if ((td2->td_pflags & TDP_KTHREAD) == 0) {
+		MPASS(td1 == curthread);
+		fpuexit(td1);
 		update_pcb_bases(td1->td_pcb);
+	}
 
 	/* Point the stack and pcb to the actual location */
 	set_top_of_stack_td(td2);
@@ -178,8 +180,18 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 
 	/* Properly initialize pcb_save */
 	pcb2->pcb_save = get_pcb_user_save_pcb(pcb2);
-	bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2),
-	    cpu_max_ext_state_size);
+
+	/* Kernel processes start with clean FPU and segment bases. */
+	if ((td2->td_pflags & TDP_KTHREAD) != 0) {
+		pcb2->pcb_fsbase = 0;
+		pcb2->pcb_gsbase = 0;
+		clear_pcb_flags(pcb2, PCB_FPUINITDONE | PCB_USERFPUINITDONE |
+		    PCB_KERNFPU | PCB_KERNFPU_THR);
+	} else {
+		MPASS((pcb2->pcb_flags & (PCB_KERNFPU | PCB_KERNFPU_THR)) == 0);
+		bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2),
+		    cpu_max_ext_state_size);
+	}
 
 	/* Point mdproc and then copy over td1's contents */
 	mdp2 = &p2->p_md;
@@ -564,10 +576,12 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
 
 	pcb2 = td->td_pcb;
 
-	/* Ensure that td0's pcb is up to date. */
-	fpuexit(td0);
-	if (td0 == curthread)
+	/* Ensure that td0's pcb is up to date for user threads. */
+	if ((td->td_pflags & TDP_KTHREAD) == 0) {
+		MPASS(td0 == curthread);
+		fpuexit(td0);
 		update_pcb_bases(td0->td_pcb);
+	}
 
 	/*
 	 * Copy the upcall pcb.  This loads kernel regs.
@@ -575,12 +589,22 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
 	 * values here.
 	 */
 	bcopy(td0->td_pcb, pcb2, sizeof(*pcb2));
-	clear_pcb_flags(pcb2, PCB_KERNFPU);
 	pcb2->pcb_save = get_pcb_user_save_pcb(pcb2);
-	bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save,
-	    cpu_max_ext_state_size);
+
+	/* Kernel threads start with clean FPU and segment bases. */
+	if ((td->td_pflags & TDP_KTHREAD) != 0) {
+		pcb2->pcb_fsbase = 0;
+		pcb2->pcb_gsbase = 0;
+		clear_pcb_flags(pcb2, PCB_FPUINITDONE | PCB_USERFPUINITDONE |
+		    PCB_KERNFPU | PCB_KERNFPU_THR);
+	} else {
+		MPASS((pcb2->pcb_flags & (PCB_KERNFPU | PCB_KERNFPU_THR)) == 0);
+		bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save,
+		    cpu_max_ext_state_size);
+	}
 	set_pcb_flags_raw(pcb2, PCB_FULL_IRET);
 
+
 	/*
 	 * Create a new fresh stack for the new thread.
 	 */
diff --git a/sys/i386/i386/sys_machdep.c b/sys/i386/i386/sys_machdep.c
index 3f650b65e160..a0a1c273f467 100644
--- a/sys/i386/i386/sys_machdep.c
+++ b/sys/i386/i386/sys_machdep.c
@@ -108,7 +108,8 @@ set_fsbase(struct thread *td, uint32_t base)
 	fill_based_sd(&sd, base);
 	critical_enter();
 	td->td_pcb->pcb_fsd = sd;
-	PCPU_GET(fsgs_gdt)[0] = sd;
+	if (td == curthread)
+		PCPU_GET(fsgs_gdt)[0] = sd;
 	critical_exit();
 }
 
@@ -120,7 +121,8 @@ set_gsbase(struct thread *td, uint32_t base)
 	fill_based_sd(&sd, base);
 	critical_enter();
 	td->td_pcb->pcb_gsd = sd;
-	PCPU_GET(fsgs_gdt)[1] = sd;
+	if (td == curthread)
+		PCPU_GET(fsgs_gdt)[1] = sd;
 	critical_exit();
 }
 
diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c
index 502de6e7f38f..471128e1713d 100644
--- a/sys/i386/i386/vm_machdep.c
+++ b/sys/i386/i386/vm_machdep.c
@@ -167,13 +167,15 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 		return;
 	}
 
-	/* Ensure that td1's pcb is up to date. */
-	if (td1 == curthread)
+	/* Ensure that td1's pcb is up to date for user processes. */
+	if ((td2->td_pflags & TDP_KTHREAD) == 0) {
+		MPASS(td1 == curthread);
 		td1->td_pcb->pcb_gs = rgs();
-	critical_enter();
-	if (PCPU_GET(fpcurthread) == td1)
-		npxsave(td1->td_pcb->pcb_save);
-	critical_exit();
+		critical_enter();
+		if (PCPU_GET(fpcurthread) == td1)
+			npxsave(td1->td_pcb->pcb_save);
+		critical_exit();
+	}
 
 	/* Point the pcb to the top of the stack */
 	pcb2 = get_pcb_td(td2);
@@ -184,8 +186,19 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 
 	/* Properly initialize pcb_save */
 	pcb2->pcb_save = get_pcb_user_save_pcb(pcb2);
-	bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2),
-	    cpu_max_ext_state_size);
+
+	/* Kernel processes start with clean NPX and segment bases. */
+	if ((td2->td_pflags & TDP_KTHREAD) != 0) {
+		pcb2->pcb_gs = _udatasel;
+		set_fsbase(td2, 0);
+		set_gsbase(td2, 0);
+		pcb2->pcb_flags &= ~(PCB_NPXINITDONE | PCB_NPXUSERINITDONE |
+		    PCB_KERNNPX | PCB_KERNNPX_THR);
+	} else {
+		MPASS((pcb2->pcb_flags & (PCB_KERNNPX | PCB_KERNNPX_THR)) == 0);
+		bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2),
+		    cpu_max_ext_state_size);
+	}
 
 	/* Point mdproc and then copy over td1's contents */
 	mdp2 = &p2->p_md;
@@ -428,13 +441,15 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
 	/* Point the pcb to the top of the stack. */
 	pcb2 = td->td_pcb;
 
-	/* Ensure that td0's pcb is up to date. */
-	if (td0 == curthread)
+	/* Ensure that td0's pcb is up to date for user threads. */
+	if ((td->td_pflags & TDP_KTHREAD) == 0) {
+		MPASS(td0 == curthread);
 		td0->td_pcb->pcb_gs = rgs();
-	critical_enter();
-	if (PCPU_GET(fpcurthread) == td0)
-		npxsave(td0->td_pcb->pcb_save);
-	critical_exit();
+		critical_enter();
+		if (PCPU_GET(fpcurthread) == td0)
+			npxsave(td0->td_pcb->pcb_save);
+		critical_exit();
+	}
 
 	/*
 	 * Copy the upcall pcb.  This loads kernel regs.
@@ -442,10 +457,20 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
 	 * values here.
 	 */
 	bcopy(td0->td_pcb, pcb2, sizeof(*pcb2));
-	pcb2->pcb_flags &= ~PCB_KERNNPX;
 	pcb2->pcb_save = get_pcb_user_save_pcb(pcb2);
-	bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save,
-	    cpu_max_ext_state_size);
+
+	/* Kernel threads start with clean NPX and segment bases. */
+	if ((td->td_pflags & TDP_KTHREAD) != 0) {
+		pcb2->pcb_gs = _udatasel;
+		set_fsbase(td, 0);
+		set_gsbase(td, 0);
+		pcb2->pcb_flags &= ~(PCB_NPXINITDONE | PCB_NPXUSERINITDONE |
+		    PCB_KERNNPX | PCB_KERNNPX_THR);
+	} else {
+		MPASS((pcb2->pcb_flags & (PCB_KERNNPX | PCB_KERNNPX_THR)) == 0);
+		bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save,
+		    cpu_max_ext_state_size);
+	}
 
 	/*
 	 * Create a new fresh stack for the new thread.