dtrace: fix userspace access on boxes with SMAP

dtrace has its own routines which were not updated after SMAP support got
implemented. Use ifunc just like for other routines.

This in particular fixes ustack().

Reviewed by:	markj
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D18542

2 files changed, 161 insertions, 14 deletions

diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_asm.S b/sys/cddl/dev/dtrace/amd64/dtrace_asm.S
index a1e9b9c66521..22e1d6964fb8 100644
--- a/sys/cddl/dev/dtrace/amd64/dtrace_asm.S
+++ b/sys/cddl/dev/dtrace/amd64/dtrace_asm.S
@@ -208,7 +208,7 @@ dtrace_caller(int aframes)
 void
 dtrace_copy(uintptr_t src, uintptr_t dest, size_t size)
 */
-	ENTRY(dtrace_copy)
+	ENTRY(dtrace_copy_nosmap)
 	pushq	%rbp
 	movq	%rsp, %rbp
 
@@ -218,14 +218,28 @@ dtrace_copy(uintptr_t src, uintptr_t dest, size_t size)
 	smovb				/*   move from %ds:rsi to %ed:rdi */
 	leave
 	ret
-	END(dtrace_copy)
+	END(dtrace_copy_nosmap)
+
+	ENTRY(dtrace_copy_smap)
+	pushq	%rbp
+	movq	%rsp, %rbp
+
+	xchgq	%rdi, %rsi		/* make %rsi source, %rdi dest */
+	movq	%rdx, %rcx		/* load count */
+	stac
+	repz				/* repeat for count ... */
+	smovb				/*   move from %ds:rsi to %ed:rdi */
+	clac
+	leave
+	ret
+	END(dtrace_copy_smap)
 
 /*
 void
 dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
     volatile uint16_t *flags)
 */
-	ENTRY(dtrace_copystr)
+	ENTRY(dtrace_copystr_nosmap)
 	pushq	%rbp
 	movq	%rsp, %rbp
 
@@ -248,55 +262,120 @@ dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
 	leave
 	ret
 
-	END(dtrace_copystr)
+	END(dtrace_copystr_nosmap)
+
+	ENTRY(dtrace_copystr_smap)
+	pushq	%rbp
+	movq	%rsp, %rbp
+
+	stac
+0:
+	movb	(%rdi), %al		/* load from source */
+	movb	%al, (%rsi)		/* store to destination */
+	addq	$1, %rdi		/* increment source pointer */
+	addq	$1, %rsi		/* increment destination pointer */
+	subq	$1, %rdx		/* decrement remaining count */
+	cmpb	$0, %al
+	je	2f
+	testq	$0xfff, %rdx		/* test if count is 4k-aligned */
+	jnz	1f			/* if not, continue with copying */
+	testq	$CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */
+	jnz	2f
+1:
+	cmpq	$0, %rdx
+	jne	0b
+2:
+	clac
+	leave
+	ret
+
+	END(dtrace_copystr_smap)
 
 /*
 uintptr_t
 dtrace_fulword(void *addr)
 */
-	ENTRY(dtrace_fulword)
+	ENTRY(dtrace_fulword_nosmap)
 	movq	(%rdi), %rax
 	ret
-	END(dtrace_fulword)
+	END(dtrace_fulword_nosmap)
+
+	ENTRY(dtrace_fulword_smap)
+	stac
+	movq	(%rdi), %rax
+	clac
+	ret
+	END(dtrace_fulword_smap)
 
 /*
 uint8_t
 dtrace_fuword8_nocheck(void *addr)
 */
-	ENTRY(dtrace_fuword8_nocheck)
+	ENTRY(dtrace_fuword8_nocheck_nosmap)
 	xorq	%rax, %rax
 	movb	(%rdi), %al
 	ret
-	END(dtrace_fuword8_nocheck)
+	END(dtrace_fuword8_nocheck_nosmap)
+
+	ENTRY(dtrace_fuword8_nocheck_smap)
+	stac
+	xorq	%rax, %rax
+	movb	(%rdi), %al
+	clac
+	ret
+	END(dtrace_fuword8_nocheck_smap)
 
 /*
 uint16_t
 dtrace_fuword16_nocheck(void *addr)
 */
-	ENTRY(dtrace_fuword16_nocheck)
+	ENTRY(dtrace_fuword16_nocheck_nosmap)
+	xorq	%rax, %rax
+	movw	(%rdi), %ax
+	ret
+	END(dtrace_fuword16_nocheck_nosmap)
+
+	ENTRY(dtrace_fuword16_nocheck_smap)
+	stac
 	xorq	%rax, %rax
 	movw	(%rdi), %ax
+	clac
 	ret
-	END(dtrace_fuword16_nocheck)
+	END(dtrace_fuword16_nocheck_smap)
 
 /*
 uint32_t
 dtrace_fuword32_nocheck(void *addr)
 */
-	ENTRY(dtrace_fuword32_nocheck)
+	ENTRY(dtrace_fuword32_nocheck_nosmap)
+	xorq	%rax, %rax
+	movl	(%rdi), %eax
+	ret
+	END(dtrace_fuword32_nocheck_nosmap)
+
+	ENTRY(dtrace_fuword32_nocheck_smap)
+	stac
 	xorq	%rax, %rax
 	movl	(%rdi), %eax
+	clac
 	ret
-	END(dtrace_fuword32_nocheck)
+	END(dtrace_fuword32_nocheck_smap)
 
 /*
 uint64_t
 dtrace_fuword64_nocheck(void *addr)
 */
-	ENTRY(dtrace_fuword64_nocheck)
+	ENTRY(dtrace_fuword64_nocheck_nosmap)
+	movq	(%rdi), %rax
+	ret
+	END(dtrace_fuword64_nocheck_nosmap)
+
+	ENTRY(dtrace_fuword64_nocheck_smap)
+	stac
 	movq	(%rdi), %rax
+	clac
 	ret
-	END(dtrace_fuword64_nocheck)
+	END(dtrace_fuword64_nocheck_smap)
 
 /*
 void
diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_isa.c b/sys/cddl/dev/dtrace/amd64/dtrace_isa.c
index 603921a35d81..e8fa16184398 100644
--- a/sys/cddl/dev/dtrace/amd64/dtrace_isa.c
+++ b/sys/cddl/dev/dtrace/amd64/dtrace_isa.c
@@ -37,6 +37,7 @@
 #include <machine/md_var.h>
 #include <machine/reg.h>
 #include <machine/stack.h>
+#include <x86/ifunc.h>
 
 #include <vm/vm.h>
 #include <vm/vm_param.h>
@@ -664,3 +665,70 @@ dtrace_fuword64(void *uaddr)
 	}
 	return (dtrace_fuword64_nocheck(uaddr));
 }
+
+/*
+ * ifunc resolvers for SMAP support
+ */
+void dtrace_copy_nosmap(uintptr_t, uintptr_t, size_t);
+void dtrace_copy_smap(uintptr_t, uintptr_t, size_t);
+DEFINE_IFUNC(, void, dtrace_copy, (uintptr_t, uintptr_t, size_t), static)
+{
+
+	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+	    dtrace_copy_smap : dtrace_copy_nosmap);
+}
+
+void dtrace_copystr_nosmap(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
+void dtrace_copystr_smap(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
+DEFINE_IFUNC(, void, dtrace_copystr, (uintptr_t, uintptr_t, size_t,
+    volatile uint16_t *), static)
+{
+
+	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+	    dtrace_copystr_smap : dtrace_copystr_nosmap);
+}
+
+uintptr_t dtrace_fulword_nosmap(void *);
+uintptr_t dtrace_fulword_smap(void *);
+DEFINE_IFUNC(, uintptr_t, dtrace_fulword, (void *), static)
+{
+
+	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+	    dtrace_fulword_smap : dtrace_fulword_nosmap);
+}
+
+uint8_t dtrace_fuword8_nocheck_nosmap(void *);
+uint8_t dtrace_fuword8_nocheck_smap(void *);
+DEFINE_IFUNC(, uint8_t, dtrace_fuword8_nocheck, (void *), static)
+{
+
+	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+	    dtrace_fuword8_nocheck_smap : dtrace_fuword8_nocheck_nosmap);
+}
+
+uint16_t dtrace_fuword16_nocheck_nosmap(void *);
+uint16_t dtrace_fuword16_nocheck_smap(void *);
+DEFINE_IFUNC(, uint16_t, dtrace_fuword16_nocheck, (void *), static)
+{
+
+	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+	    dtrace_fuword16_nocheck_smap : dtrace_fuword16_nocheck_nosmap);
+}
+
+uint32_t dtrace_fuword32_nocheck_nosmap(void *);
+uint32_t dtrace_fuword32_nocheck_smap(void *);
+DEFINE_IFUNC(, uint32_t, dtrace_fuword32_nocheck, (void *), static)
+{
+
+	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+	    dtrace_fuword32_nocheck_smap : dtrace_fuword32_nocheck_nosmap);
+}
+
+uint64_t dtrace_fuword64_nocheck_nosmap(void *);
+uint64_t dtrace_fuword64_nocheck_smap(void *);
+DEFINE_IFUNC(, uint64_t, dtrace_fuword64_nocheck, (void *), static)
+{
+
+	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+	    dtrace_fuword64_nocheck_smap : dtrace_fuword64_nocheck_nosmap);
+}