Add WITH_LLVM_BINUTILS to install LLVM binutils instead of Elftoolchain

When WITH_LLVM_BINUTILS is set, we will install the LLVM binutils as
ar/ranlib/nm/objcopy/etc. instead of the elftoolchain ones.
Having the LLVM binutils instead of the elftoolchain ones allows us to use
features such as LTO that depend on binutils that understand LLVM IR.
Another benefit will be an improved user-experience when compiling with
AddressSanitizer, since ASAN does not symbolize backtraces correctly if
addr2line is elftoolchain addr2line instead of llvm-symbolizer.
See https://lists.freebsd.org/archives/freebsd-toolchain/2021-July/000062.html
for more details.

This is currently off by default but will be turned on by default at some
point in the near future.

Reviewed By:	emaste

Differential Revision: https://reviews.freebsd.org/D31060

14 files changed, 85 insertions, 15 deletions

diff --git a/Makefile.inc1 b/Makefile.inc1
index b59c1913f8ce..478824675382 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -771,6 +771,7 @@ XMAKE=		${BMAKE} \
 		TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		MK_CLANG_IS_CC=${MK_CLANG_BOOTSTRAP} \
 		MK_LLDB=no \
+		MK_LLVM_BINUTILS=no \
 		MK_TESTS=no
 
 # kernel-tools stage
@@ -2339,9 +2340,10 @@ _gensnmptree=	usr.sbin/bsnmpd/gensnmptree
 
 # We need to build tblgen when we're building clang or lld, either as
 # bootstrap tools, or as the part of the normal build.
+# llvm-tblgen is also needed for various llvm binutils (e.g. objcopy).
 .if ${MK_CLANG_BOOTSTRAP} != "no" || ${MK_CLANG} != "no" || \
     ${MK_LLD_BOOTSTRAP} != "no" || ${MK_LLD} != "no" || \
-    ${MK_LLDB} != "no"
+    ${MK_LLDB} != "no" || ${MK_LLVM_BINUTILS} != "no"
 _clang_tblgen= \
 	lib/clang/libllvmminimal \
 	usr.bin/clang/llvm-tblgen
diff --git a/lib/Makefile b/lib/Makefile
index 674368a19ffd..1e375bb456e6 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -157,7 +157,10 @@ SUBDIR.${MK_BLUETOOTH}+=libbluetooth libsdp
 SUBDIR.${MK_BSNMP}+=	libbsnmp
 
 .if !defined(COMPAT_32BIT) && !defined(COMPAT_SOFTFP)
-SUBDIR.${MK_CLANG}+=	clang
+.if ${MK_CLANG} != "no" || ${MK_LLD} != "no" || \
+    ${MK_LLDB} != "no" || ${MK_LLVM_BINUTILS} != "no"
+SUBDIR+=	clang
+.endif
 .endif
 
 SUBDIR.${MK_CUSE}+= 	libcuse
diff --git a/lib/clang/Makefile b/lib/clang/Makefile
index bc09ea62dc67..df4aa01a2653 100644
--- a/lib/clang/Makefile
+++ b/lib/clang/Makefile
@@ -4,10 +4,12 @@
 
 # These have to be built in order.
 SUBDIR=		libllvm
+.if ${MK_CLANG} != "no"
 SUBDIR+=	libclang
+SUBDIR+=	headers
+.endif
 .if ${MK_LLDB} != "no"
 SUBDIR+=	liblldb
 .endif
-SUBDIR+=	headers
 
 .include <bsd.subdir.mk>
diff --git a/lib/clang/libllvm/Makefile b/lib/clang/libllvm/Makefile
index 09d6336c01d9..7eaedf65dcb3 100644
--- a/lib/clang/libllvm/Makefile
+++ b/lib/clang/libllvm/Makefile
@@ -830,7 +830,7 @@ SRCS_MIN+=	Object/IRObjectFile.cpp
 SRCS_MIN+=	Object/IRSymtab.cpp
 SRCS_MIN+=	Object/MachOObjectFile.cpp
 SRCS_MIW+=	Object/MachOUniversal.cpp
-SRCS_EXT+=	Object/MachOUniversalWriter.cpp
+SRCS_MIW+=	Object/MachOUniversalWriter.cpp
 SRCS_MIW+=	Object/Minidump.cpp
 SRCS_MIN+=	Object/ModuleSymbolTable.cpp
 SRCS_EXT+=	Object/Object.cpp
@@ -920,7 +920,7 @@ SRCS_MIN+=	Support/Errno.cpp
 SRCS_MIN+=	Support/Error.cpp
 SRCS_MIN+=	Support/ErrorHandling.cpp
 SRCS_MIN+=	Support/FileCollector.cpp
-SRCS_EXL+=	Support/FileOutputBuffer.cpp
+SRCS_MIW+=	Support/FileOutputBuffer.cpp
 SRCS_MIN+=	Support/FileUtilities.cpp
 SRCS_MIN+=	Support/FoldingSet.cpp
 SRCS_MIN+=	Support/FormatVariadic.cpp
@@ -945,7 +945,7 @@ SRCS_MIN+=	Support/MD5.cpp
 SRCS_MIN+=	Support/ManagedStatic.cpp
 SRCS_MIN+=	Support/MathExtras.cpp
 SRCS_MIN+=	Support/MemAlloc.cpp
-SRCS_XDL+=	Support/Memory.cpp
+SRCS_MIW+=	Support/Memory.cpp
 SRCS_MIN+=	Support/MemoryBuffer.cpp
 SRCS_MIN+=	Support/MemoryBufferRef.cpp
 SRCS_MIN+=	Support/NativeFormatting.cpp
diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk
index 32346e31a752..5363cb6e78f9 100644
--- a/share/mk/src.opts.mk
+++ b/share/mk/src.opts.mk
@@ -203,6 +203,7 @@ __DEFAULT_NO_OPTIONS = \
     LOADER_FIREWIRE \
     LOADER_VERBOSE \
     LOADER_VERIEXEC_PASS_MANIFEST \
+    LLVM_BINUTILS \
     MALLOC_PRODUCTION \
     OFED_EXTRA \
     OPENLDAP \
@@ -459,6 +460,7 @@ MK_CLANG:=	no
 MK_INCLUDES:=	no
 MK_LLD:=	no
 MK_LLDB:=	no
+MK_LLVM_BINUTILS:=	no
 .endif
 
 .if ${MK_CLANG} == "no"
@@ -468,6 +470,12 @@ MK_CLANG_FULL:= no
 MK_LLVM_COV:= no
 .endif
 
+.if ${MK_LLVM_BINUTILS} == "yes"
+# MK_LLVM_CXXFILT is a subset of MK_LLVM_BINUTILS and should therefore be
+# enabled if MK_LLVM_BINUTILS is set.
+MK_LLVM_CXXFILT:=	yes
+.endif
+
 .if ${MK_LOADER_VERIEXEC} == "no"
 MK_LOADER_VERIEXEC_PASS_MANIFEST := no
 .endif
diff --git a/tools/build/options/WITH_LLVM_BINUTILS b/tools/build/options/WITH_LLVM_BINUTILS
new file mode 100644
index 000000000000..8fa2c55f31a9
--- /dev/null
+++ b/tools/build/options/WITH_LLVM_BINUTILS
@@ -0,0 +1,2 @@
+.\" $FreeBSD$
+Install LLVM's binutils (ar, addr2line, nm, etc.) instead of ELF Tool Chain's.
diff --git a/usr.bin/Makefile b/usr.bin/Makefile
index f1b07a7007be..e8be161db01a 100644
--- a/usr.bin/Makefile
+++ b/usr.bin/Makefile
@@ -195,7 +195,10 @@ SUBDIR.${MK_ATM}+=	atm
 SUBDIR.${MK_BLUETOOTH}+=	bluetooth
 SUBDIR.${MK_BSD_CPIO}+=	cpio
 SUBDIR.${MK_CALENDAR}+=	calendar
-SUBDIR.${MK_CLANG}+=	clang
+.if ${MK_CLANG} != "no" || ${MK_LLVM_BINUTILS} != "no" || \
+    ${MK_LLD} != "no" || ${MK_LLDB} != "no"
+SUBDIR+=	clang
+.endif
 SUBDIR.${MK_DIALOG}+=	dpv
 SUBDIR.${MK_EE}+=	ee
 SUBDIR.${MK_FILE}+=	file
@@ -251,25 +254,28 @@ SUBDIR.${MK_TESTS_SUPPORT}.${MK_CXX}+=	kyua
 SUBDIR.${MK_TESTS}+=	tests
 SUBDIR.${MK_TEXTPROC}+=	ul
 SUBDIR.${MK_TFTP}+=	tftp
+.if ${MK_LLVM_BINUTILS} == "no"
+# Only build the elftoolchain tools if we aren't using the LLVM ones.
 SUBDIR.${MK_TOOLCHAIN}+=	addr2line
 SUBDIR.${MK_TOOLCHAIN}+=	ar
+SUBDIR.${MK_TOOLCHAIN}+=	nm
+SUBDIR.${MK_TOOLCHAIN}+=	objcopy
+SUBDIR.${MK_TOOLCHAIN}+=	readelf
+SUBDIR.${MK_TOOLCHAIN}+=	size
+.endif
 SUBDIR.${MK_TOOLCHAIN}+=	c89
 SUBDIR.${MK_TOOLCHAIN}+=	c99
 SUBDIR.${MK_TOOLCHAIN}+=	ctags
 .if ${MK_LLVM_CXXFILT} == "no"
 SUBDIR.${MK_TOOLCHAIN}+=	cxxfilt
 .endif
-SUBDIR.${MK_TOOLCHAIN}+=	objcopy
 SUBDIR.${MK_TOOLCHAIN}+=	file2c
 SUBDIR.${MK_TOOLCHAIN}+=	gprof
 SUBDIR.${MK_TOOLCHAIN}+=	indent
 SUBDIR.${MK_TOOLCHAIN}+=	lex
 SUBDIR.${MK_TOOLCHAIN}+=	mkstr
-SUBDIR.${MK_TOOLCHAIN}+=	nm
-SUBDIR.${MK_TOOLCHAIN}+=	readelf
 SUBDIR.${MK_TOOLCHAIN}+=	rpcgen
 SUBDIR.${MK_TOOLCHAIN}+=	unifdef
-SUBDIR.${MK_TOOLCHAIN}+=	size
 SUBDIR.${MK_TOOLCHAIN}+=	xstr
 SUBDIR.${MK_TOOLCHAIN}+=	yacc
 SUBDIR.${MK_VI}+=	vi
diff --git a/usr.bin/clang/Makefile b/usr.bin/clang/Makefile
index 1c53e94965f7..7fc31e8df194 100644
--- a/usr.bin/clang/Makefile
+++ b/usr.bin/clang/Makefile
@@ -7,12 +7,20 @@ SUBDIR+=	clang
 .endif
 
 .if !defined(TOOLS_PREFIX)
+# LLVM binutils are needed to support features such as LTO, so we build them
+# by default if clang is enabled. If MK_LLVM_BINUTILS is set, we also use them
+# as the default binutils (ar,nm,addr2line, etc.).
+.if ${MK_CLANG} != "no" || ${MK_LLVM_BINUTILS} != "no"
 SUBDIR+=	llvm-ar
 SUBDIR+=	llvm-nm
+SUBDIR+=	llvm-objcopy
 SUBDIR+=	llvm-objdump
+SUBDIR+=	llvm-size
+SUBDIR+=	llvm-strings
 SUBDIR+=	llvm-symbolizer
+.endif
 
-.if ${MK_CLANG_EXTRAS} != "no" || ${MK_LLVM_CXXFILT} != "no"
+.if ${MK_LLVM_BINUTILS} != "no" || ${MK_LLVM_CXXFILT} != "no"
 SUBDIR+=	llvm-cxxfilt
 .endif
 
@@ -34,11 +42,8 @@ SUBDIR+=	llvm-lto2
 SUBDIR+=	llvm-mc
 SUBDIR+=	llvm-mca
 SUBDIR+=	llvm-modextract
-SUBDIR+=	llvm-objcopy
 SUBDIR+=	llvm-pdbutil
 SUBDIR+=	llvm-rtdyld
-SUBDIR+=	llvm-size
-SUBDIR+=	llvm-strings
 SUBDIR+=	llvm-xray
 SUBDIR+=	opt
 .endif
diff --git a/usr.bin/clang/llvm-ar/Makefile b/usr.bin/clang/llvm-ar/Makefile
index 86d4538271a1..bd43194153d7 100644
--- a/usr.bin/clang/llvm-ar/Makefile
+++ b/usr.bin/clang/llvm-ar/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-ar
 MAN=		llvm-ar.1 llvm-ranlib.1
 
@@ -10,4 +12,9 @@ LIBADD+=	z
 
 LINKS+=		${BINDIR}/llvm-ar ${BINDIR}/llvm-ranlib
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-ar ${BINDIR}/ar ${BINDIR}/llvm-ar ${BINDIR}/ranlib
+MLINKS=		llvm-ar.1 ar.1 llvm-ar.1 ranlib.1
+.endif
+
 .include "../llvm.prog.mk"
diff --git a/usr.bin/clang/llvm-nm/Makefile b/usr.bin/clang/llvm-nm/Makefile
index 02f16887a7cf..4a6990cfa22d 100644
--- a/usr.bin/clang/llvm-nm/Makefile
+++ b/usr.bin/clang/llvm-nm/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-nm
 
 SRCDIR=		llvm/tools/llvm-nm
@@ -7,4 +9,9 @@ SRCS+=		llvm-nm.cpp
 
 LIBADD+=	z
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-nm ${BINDIR}/nm
+MLINKS=		llvm-nm.1 nm.1
+.endif
+
 .include "../llvm.prog.mk"
diff --git a/usr.bin/clang/llvm-objcopy/Makefile b/usr.bin/clang/llvm-objcopy/Makefile
index 645bbbd7c4e9..2e6fc8aba356 100644
--- a/usr.bin/clang/llvm-objcopy/Makefile
+++ b/usr.bin/clang/llvm-objcopy/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-objcopy
 
 SRCDIR=		llvm/tools/llvm-objcopy
@@ -42,4 +44,9 @@ CLEANFILES+=	${TGHDRS} ${TGHDRS:C/$/.d/}
 
 LIBADD+=	z
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-objcopy ${BINDIR}/objcopy
+MLINKS=		llvm-objcopy.1 objcopy.1
+.endif
+
 .include "../llvm.prog.mk"
diff --git a/usr.bin/clang/llvm-objdump/Makefile b/usr.bin/clang/llvm-objdump/Makefile
index 34b015042718..53b63bd7200e 100644
--- a/usr.bin/clang/llvm-objdump/Makefile
+++ b/usr.bin/clang/llvm-objdump/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-objdump
 
 SRCDIR=		llvm/tools/llvm-objdump
@@ -12,4 +14,9 @@ SRCS+=		llvm-objdump.cpp
 
 LIBADD+=	z
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-objdump ${BINDIR}/objdump
+MLINKS=		llvm-objdump.1 objdump.1
+.endif
+
 .include "../llvm.prog.mk"
diff --git a/usr.bin/clang/llvm-size/Makefile b/usr.bin/clang/llvm-size/Makefile
index a4a387aabf6f..4ddaaf86dca3 100644
--- a/usr.bin/clang/llvm-size/Makefile
+++ b/usr.bin/clang/llvm-size/Makefile
@@ -1,8 +1,15 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-size
 
 SRCDIR=		llvm/tools/llvm-size
 SRCS+=		llvm-size.cpp
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-size ${BINDIR}/size
+MLINKS=		llvm-size.1 size.1
+.endif
+
 .include "../llvm.prog.mk"
diff --git a/usr.bin/clang/llvm-symbolizer/Makefile b/usr.bin/clang/llvm-symbolizer/Makefile
index 6352d6a67b94..52a48d428600 100644
--- a/usr.bin/clang/llvm-symbolizer/Makefile
+++ b/usr.bin/clang/llvm-symbolizer/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-symbolizer
 MAN=		llvm-symbolizer.1 llvm-addr2line.1
 
@@ -26,4 +28,9 @@ LIBADD+=	z
 
 LINKS+=		${BINDIR}/llvm-symbolizer ${BINDIR}/llvm-addr2line
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-symbolizer ${BINDIR}/addr2line
+MLINKS=		llvm-addr2line.1 addr2line.1
+.endif
+
 .include "../llvm.prog.mk"