Simplify kernel sanitizer interceptors

KASAN and KCSAN implement interceptors for various primitive operations
that are not instrumented by the compiler.  KMSAN requires them as well.
Rather than adding new cases for each sanitizer which requires
interceptors, implement the following protocol:
- When interceptor definitions are required, define
  SAN_NEEDS_INTERCEPTORS and SANITIZER_INTERCEPTOR_PREFIX.
- In headers that declare functions which need to be intercepted by a
  sanitizer runtime, use SANITIZER_INTERCEPTOR_PREFIX to provide
  declarations.
- When SAN_RUNTIME is defined, do not redefine the names of intercepted
  functions.  This is typically the case in files which implement
  sanitizer runtimes but is also needed in, for example, files which
  define ifunc selectors for intercepted operations.

MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation

11 files changed, 45 insertions, 80 deletions

diff --git a/sys/amd64/amd64/copyout.c b/sys/amd64/amd64/copyout.c
index 0e96944c4a23..36e817635694 100644
--- a/sys/amd64/amd64/copyout.c
+++ b/sys/amd64/amd64/copyout.c
@@ -32,6 +32,10 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#ifdef SAN_NEEDS_INTERCEPTORS
+#define	SAN_RUNTIME
+#endif
+
 #include <sys/param.h>
 #include <sys/systm.h>
 
@@ -146,10 +150,6 @@ DEFINE_IFUNC(, int, casueword, (volatile u_long *, u_long, u_long *, u_long))
 	    casueword_smap : casueword_nosmap);
 }
 
-#undef copyinstr
-#undef copyin
-#undef copyout
-
 int	copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len,
 	    size_t *lencopied);
 int	copyinstr_smap(const void *udaddr, void *kaddr, size_t len,
diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h
index c821e77e5b8b..4a9095ca831b 100644
--- a/sys/amd64/include/atomic.h
+++ b/sys/amd64/include/atomic.h
@@ -68,15 +68,7 @@
 #define	OFFSETOF_MONITORBUF	0x100
 #endif
 
-#ifndef SAN_RUNTIME
-#if defined(KASAN)
-#define	ATOMIC_SAN_PREFIX	kasan
-#elif defined(KCSAN)
-#define	ATOMIC_SAN_PREFIX	kcsan
-#endif
-#endif
-
-#ifdef ATOMIC_SAN_PREFIX
+#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
 #include <sys/atomic_san.h>
 #else
 #include <sys/atomic_common.h>
@@ -687,6 +679,6 @@ u_long	atomic_swap_long(volatile u_long *p, u_long v);
 
 #endif /* !WANT_FUNCTIONS */
 
-#endif /* !ATOMIC_SAN_PREFIX */
+#endif /* !SAN_NEEDS_INTERCEPTORS || SAN_RUNTIME */
 
 #endif /* !_MACHINE_ATOMIC_H_ */
diff --git a/sys/arm64/include/bus.h b/sys/arm64/include/bus.h
index 61573b27728d..a87577b00f0c 100644
--- a/sys/arm64/include/bus.h
+++ b/sys/arm64/include/bus.h
@@ -92,15 +92,7 @@
 #define	BUS_SPACE_BARRIER_READ	0x01
 #define	BUS_SPACE_BARRIER_WRITE	0x02
 
-#ifndef SAN_RUNTIME
-#if defined(KASAN)
-#define	BUS_SAN_PREFIX	kasan
-#elif defined(KCSAN)
-#define	BUS_SAN_PREFIX	kcsan
-#endif
-#endif
-
-#ifdef BUS_SAN_PREFIX
+#ifdef SAN_NEEDS_INTERCEPTORS
 #include <sys/bus_san.h>
 #else
 
@@ -506,7 +498,7 @@ struct bus_space {
 #define	bus_space_peek_4(t, h, o, vp)	__bs_peek(4, (t), (h), (o), (vp))
 #define	bus_space_peek_8(t, h, o, vp)	__bs_peek(8, (t), (h), (o), (vp))
 
-#endif
+#endif /* !SAN_NEEDS_INTERCEPTORS */
 
 #include <machine/bus_dma.h>
 
diff --git a/sys/conf/kern.pre.mk b/sys/conf/kern.pre.mk
index 539605f60ca0..221c0aa22d9d 100644
--- a/sys/conf/kern.pre.mk
+++ b/sys/conf/kern.pre.mk
@@ -94,7 +94,8 @@ ASM_CFLAGS= -x assembler-with-cpp -DLOCORE ${CFLAGS} ${ASM_CFLAGS.${.IMPSRC:T}}
 
 KASAN_ENABLED!=	grep KASAN opt_global.h || true ; echo
 .if !empty(KASAN_ENABLED)
-SAN_CFLAGS+=	-fsanitize=kernel-address \
+SAN_CFLAGS+=	-DSAN_NEEDS_INTERCEPTORS -DSAN_INTERCEPTOR_PREFIX=kasan \
+		-fsanitize=kernel-address \
 		-mllvm -asan-stack=true \
 		-mllvm -asan-instrument-dynamic-allocas=true \
 		-mllvm -asan-globals=true \
@@ -104,7 +105,8 @@ SAN_CFLAGS+=	-fsanitize=kernel-address \
 
 KCSAN_ENABLED!=	grep KCSAN opt_global.h || true ; echo
 .if !empty(KCSAN_ENABLED)
-SAN_CFLAGS+=	-fsanitize=thread
+SAN_CFLAGS+=	-DSAN_NEEDS_INTERCEPTORS -DSAN_INTERCEPTOR_PREFIX=kcsan \
+		-fsanitize=thread
 .endif
 
 KUBSAN_ENABLED!=	grep KUBSAN opt_global.h || true ; echo
diff --git a/sys/kern/subr_asan.c b/sys/kern/subr_asan.c
index 825d7d4228c0..7083a8e64540 100644
--- a/sys/kern/subr_asan.c
+++ b/sys/kern/subr_asan.c
@@ -523,7 +523,6 @@ kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 /* -------------------------------------------------------------------------- */
 
 #include <machine/atomic.h>
-#define	ATOMIC_SAN_PREFIX	kasan
 #include <sys/atomic_san.h>
 
 #define _ASAN_ATOMIC_FUNC_ADD(name, type)				\
@@ -785,7 +784,6 @@ kasan_atomic_interrupt_fence(void)
 
 #include <sys/bus.h>
 #include <machine/bus.h>
-#define	BUS_SAN_PREFIX	kasan
 #include <sys/bus_san.h>
 
 int
diff --git a/sys/kern/subr_csan.c b/sys/kern/subr_csan.c
index 56d2e59ff12c..7bed25076fa9 100644
--- a/sys/kern/subr_csan.c
+++ b/sys/kern/subr_csan.c
@@ -374,7 +374,6 @@ kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
 /* -------------------------------------------------------------------------- */
 
 #include <machine/atomic.h>
-#define	ATOMIC_SAN_PREFIX	kcsan
 #include <sys/atomic_san.h>
 
 #define	_CSAN_ATOMIC_FUNC_ADD(name, type)				\
@@ -689,7 +688,6 @@ kcsan_atomic_interrupt_fence(void)
 
 #include <sys/bus.h>
 #include <machine/bus.h>
-#define	BUS_SAN_PREFIX		kcsan
 #include <sys/bus_san.h>
 
 int
diff --git a/sys/sys/atomic_san.h b/sys/sys/atomic_san.h
index 5d10f58f7565..30bd01c97cf2 100644
--- a/sys/sys/atomic_san.h
+++ b/sys/sys/atomic_san.h
@@ -43,10 +43,6 @@
 #error do not include this header, use machine/atomic.h
 #endif
 
-#ifndef ATOMIC_SAN_PREFIX
-#error No sanitizer prefix available
-#endif
-
 #define	ATOMIC_SAN_FUNC_1(sp, op, name, type)				\
 	void sp##_atomic_##op##_##name(volatile type *, type);		\
 	void sp##_atomic_##op##_acq_##name(volatile type *, type);	\
@@ -104,7 +100,7 @@
 	ATOMIC_SAN_THREAD_FENCE(sp);
 
 #define	ATOMIC_SAN_FUNCS(name, type)					\
-	_ATOMIC_SAN_FUNCS(ATOMIC_SAN_PREFIX, name, type)
+	_ATOMIC_SAN_FUNCS(SAN_INTERCEPTOR_PREFIX, name, type)
 
 ATOMIC_SAN_FUNCS(char, uint8_t);
 ATOMIC_SAN_FUNCS(short, uint16_t);
@@ -123,8 +119,8 @@ ATOMIC_SAN_FUNCS(64, uint64_t);
  * For instance, KASAN callers of atomic_add_char() will be redirected to
  * kasan_atomic_add_char().
  */
-#define	ATOMIC_SAN(func)		\
-	__CONCAT(ATOMIC_SAN_PREFIX, __CONCAT(_atomic_, func))
+#define	ATOMIC_SAN(func)						\
+	__CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_atomic_, func))
 
 #define	atomic_add_char			ATOMIC_SAN(add_char)
 #define	atomic_add_acq_char		ATOMIC_SAN(add_acq_char)
diff --git a/sys/sys/bus_san.h b/sys/sys/bus_san.h
index 05d5ecd4b844..fd3c4c803675 100644
--- a/sys/sys/bus_san.h
+++ b/sys/sys/bus_san.h
@@ -43,10 +43,6 @@
 #error do not include this header, use machine/bus.h
 #endif
 
-#ifndef BUS_SAN_PREFIX
-#error No sanitizer prefix defined
-#endif
-
 #define	BUS_SAN_MULTI(sp, rw, width, type)				\
 	void sp##_bus_space_##rw##_multi_##width(bus_space_tag_t, 	\
 	    bus_space_handle_t, bus_size_t, type *, bus_size_t);	\
@@ -122,7 +118,7 @@
 	BUS_SAN_MISC(sp);
 
 #define	BUS_SAN_FUNCS(width, type)					\
-	_BUS_SAN_FUNCS(BUS_SAN_PREFIX, width, type)
+	_BUS_SAN_FUNCS(SAN_INTERCEPTOR_PREFIX, width, type)
 
 BUS_SAN_FUNCS(1, uint8_t);
 BUS_SAN_FUNCS(2, uint16_t);
@@ -131,7 +127,8 @@ BUS_SAN_FUNCS(8, uint64_t);
 
 #ifndef SAN_RUNTIME
 
-#define	BUS_SAN(func)	__CONCAT(BUS_SAN_PREFIX, __CONCAT(_bus_space_, func))
+#define	BUS_SAN(func)							\
+	__CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_bus_space_, func))
 
 #define	bus_space_map			BUS_SAN(map)
 #define	bus_space_unmap			BUS_SAN(unmap)
@@ -224,6 +221,6 @@ BUS_SAN_FUNCS(8, uint64_t);
 #define	bus_space_poke_8		BUS_SAN(poke_8)
 #define	bus_space_peek_8		BUS_SAN(peek_8)
 
-#endif /* !KCSAN_RUNTIME */
+#endif /* !SAN_RUNTIME */
 
 #endif /* !_SYS_BUS_SAN_H_ */
diff --git a/sys/sys/libkern.h b/sys/sys/libkern.h
index d8d3dce1b705..147eba9bd1bd 100644
--- a/sys/sys/libkern.h
+++ b/sys/sys/libkern.h
@@ -193,7 +193,11 @@ size_t	 strspn(const char *, const char *);
 char	*strstr(const char *, const char *);
 int	 strvalid(const char *, size_t);
 
-#ifdef SAN_PREFIX
+#ifdef SAN_NEEDS_INTERCEPTORS
+#ifndef SAN_INTERCEPTOR
+#define	SAN_INTERCEPTOR(func)	\
+	__CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_, func))
+#endif
 char	*SAN_INTERCEPTOR(strcpy)(char *, const char *);
 int	SAN_INTERCEPTOR(strcmp)(const char *, const char *);
 size_t	SAN_INTERCEPTOR(strlen)(const char *);
@@ -202,11 +206,11 @@ size_t	SAN_INTERCEPTOR(strlen)(const char *);
 #define	strcmp(s1, s2)	SAN_INTERCEPTOR(strcmp)((s1), (s2))
 #define	strlen(s)	SAN_INTERCEPTOR(strlen)(s)
 #endif /* !SAN_RUNTIME */
-#else
-#define strcpy(d, s) __builtin_strcpy((d), (s))
-#define strcmp(s1, s2) __builtin_strcmp((s1), (s2))
-#define strlen(s) __builtin_strlen((s))
-#endif /* SAN_PREFIX */
+#else /* !SAN_NEEDS_INTERCEPTORS */
+#define strcpy(d, s)	__builtin_strcpy((d), (s))
+#define strcmp(s1, s2)	__builtin_strcmp((s1), (s2))
+#define strlen(s)	__builtin_strlen((s))
+#endif /* SAN_NEEDS_INTERCEPTORS */
 
 static __inline char *
 index(const char *p, int ch)
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index 8080f22266e2..671b5c2b8d38 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -351,15 +351,9 @@ void	*memcpy(void * _Nonnull to, const void * _Nonnull from, size_t len);
 void	*memmove(void * _Nonnull dest, const void * _Nonnull src, size_t n);
 int	memcmp(const void *b1, const void *b2, size_t len);
 
-#if defined(KASAN)
-#define	SAN_PREFIX	kasan_
-#elif defined(KCSAN)
-#define	SAN_PREFIX	kcsan_
-#endif
-
-#ifdef SAN_PREFIX
-#define	SAN_INTERCEPTOR(func)	__CONCAT(SAN_PREFIX, func)
-
+#ifdef SAN_NEEDS_INTERCEPTORS
+#define	SAN_INTERCEPTOR(func)	\
+	__CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_, func))
 void	*SAN_INTERCEPTOR(memset)(void *, int, size_t);
 void	*SAN_INTERCEPTOR(memcpy)(void *, const void *, size_t);
 void	*SAN_INTERCEPTOR(memmove)(void *, const void *, size_t);
@@ -373,15 +367,15 @@ int	SAN_INTERCEPTOR(memcmp)(const void *, const void *, size_t);
 #define memmove(dest, src, n)	SAN_INTERCEPTOR(memmove)((dest), (src), (n))
 #define memcmp(b1, b2, len)	SAN_INTERCEPTOR(memcmp)((b1), (b2), (len))
 #endif /* !SAN_RUNTIME */
-#else
-#define bcopy(from, to, len) __builtin_memmove((to), (from), (len))
-#define bzero(buf, len) __builtin_memset((buf), 0, (len))
-#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
-#define memset(buf, c, len) __builtin_memset((buf), (c), (len))
-#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len))
-#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
-#define memcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
-#endif /* !SAN_PREFIX */
+#else /* !SAN_NEEDS_INTERCEPTORS */
+#define bcopy(from, to, len)	__builtin_memmove((to), (from), (len))
+#define bzero(buf, len)		__builtin_memset((buf), 0, (len))
+#define bcmp(b1, b2, len)	__builtin_memcmp((b1), (b2), (len))
+#define memset(buf, c, len)	__builtin_memset((buf), (c), (len))
+#define memcpy(to, from, len)	__builtin_memcpy((to), (from), (len))
+#define memmove(dest, src, n)	__builtin_memmove((dest), (src), (n))
+#define memcmp(b1, b2, len)	__builtin_memcmp((b1), (b2), (len))
+#endif /* SAN_NEEDS_INTERCEPTORS */
 
 void	*memset_early(void * _Nonnull buf, int c, size_t len);
 #define bzero_early(buf, len) memset_early((buf), 0, (len))
@@ -412,7 +406,7 @@ int	copyout(const void * _Nonnull __restrict kaddr,
 int	copyout_nofault(const void * _Nonnull __restrict kaddr,
 	    void * __restrict udaddr, size_t len);
 
-#ifdef SAN_PREFIX
+#ifdef SAN_NEEDS_INTERCEPTORS
 int	SAN_INTERCEPTOR(copyin)(const void *, void *, size_t);
 int	SAN_INTERCEPTOR(copyinstr)(const void *, void *, size_t, size_t *);
 int	SAN_INTERCEPTOR(copyout)(const void *, void *, size_t);
@@ -421,7 +415,7 @@ int	SAN_INTERCEPTOR(copyout)(const void *, void *, size_t);
 #define	copyinstr(u, k, l, lc)	SAN_INTERCEPTOR(copyinstr)((u), (k), (l), (lc))
 #define	copyout(k, u, l)	SAN_INTERCEPTOR(copyout)((k), (u), (l))
 #endif /* !SAN_RUNTIME */
-#endif /* SAN_PREFIX */
+#endif /* SAN_NEEDS_INTERCEPTORS */
 
 int	fubyte(volatile const void *base);
 long	fuword(volatile const void *base);
diff --git a/sys/x86/include/bus.h b/sys/x86/include/bus.h
index 0ba68e250b3d..9522e5db7c78 100644
--- a/sys/x86/include/bus.h
+++ b/sys/x86/include/bus.h
@@ -135,15 +135,7 @@
 #define	BUS_SPACE_BARRIER_READ	0x01		/* force read barrier */
 #define	BUS_SPACE_BARRIER_WRITE	0x02		/* force write barrier */
 
-#ifndef SAN_RUNTIME
-#if defined(KASAN)
-#define	BUS_SAN_PREFIX	kasan
-#elif defined(KCSAN)
-#define	BUS_SAN_PREFIX	kcsan
-#endif
-#endif
-
-#ifdef BUS_SAN_PREFIX
+#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
 #include <sys/bus_san.h>
 #else
 
@@ -1129,6 +1121,6 @@ BUS_POKE_FUNC(4, uint32_t)
 BUS_POKE_FUNC(8, uint64_t)
 #endif
 
-#endif /* !BUS_SAN_PREFIX */
+#endif /* !SAN_NEEDS_INTERCEPTORS && SAN_RUNTIME */
 
 #endif /* !_MACHINE_BUS_H_ */