Fix scripted installation from media without local distfiles.

The bsdinstall script target did not have the infrastructure to fetch
distfiles from a remote server the way the interactive installer does
on e.g. bootonly media. Solve this by factoring out the parts of the
installer that deal with fetching missing distributions into a new
install stage called 'fetchmissingdists', which is called by both the
interactive and scripted installer frontends.

In the course of these changes, cleaned up a few other issues with
the fetching of missing distribution files and added a warning if
fetching the MANIFEST file, which is used to verify the integrity of
the distribution files. We should at some point add cryptographic
signatures to MANIFEST so that it can be fetched safely if not present
on the install media (which it is for bootonly media).

Initial patch by: Vinícius Zavam
PR:		255659, 250928
Reviewed by:	dteske
MFC after:	4 weeks
Differential Revision:	https://reviews.freebsd.org/D27121

(cherry picked from commit 40923b0c81cc2c151388ec5ead59f4bed89ac432)

5 files changed, 154 insertions, 96 deletions

diff --git a/usr.sbin/bsdinstall/scripts/Makefile b/usr.sbin/bsdinstall/scripts/Makefile
index 11c4564ea909..147f49609921 100644
--- a/usr.sbin/bsdinstall/scripts/Makefile
+++ b/usr.sbin/bsdinstall/scripts/Makefile
@@ -1,9 +1,9 @@
 # $FreeBSD$
 
-SCRIPTS= auto adduser bootconfig checksum config docsinstall entropy hardening \
-	 hostname jail keymap mirrorselect mount netconfig netconfig_ipv4 \
-	 netconfig_ipv6 rootpass script services time umount wlanconfig \
-	 zfsboot
+SCRIPTS= auto adduser bootconfig checksum config docsinstall entropy \
+         fetchmissingdists hardening hostname jail keymap mirrorselect mount \
+         netconfig netconfig_ipv4 netconfig_ipv6 rootpass script services time \
+         umount wlanconfig zfsboot
 BINDIR= ${LIBEXECDIR}/bsdinstall
 
 MAN=
diff --git a/usr.sbin/bsdinstall/scripts/auto b/usr.sbin/bsdinstall/scripts/auto
index 58e74acb2d92..68809a22671b 100755
--- a/usr.sbin/bsdinstall/scripts/auto
+++ b/usr.sbin/bsdinstall/scripts/auto
@@ -77,7 +77,6 @@ error()
 	local prompt="${1:+$1\n\n}$msg_an_installation_step_has_been_aborted"
 	local hline="$hline_arrows_tab_space_enter"
 
-	[ "$DISTDIR_IS_UNIONFS" ] && umount -f "$BSDINSTALL_DISTDIR"
 	[ -f "$PATH_FSTAB" ] && bsdinstall umount
 
 	local height width
@@ -170,17 +169,12 @@ if [ -f $BSDINSTALL_DISTDIR/MANIFEST ]; then
 	done
 fi
 
-LOCAL_DISTRIBUTIONS="MANIFEST"
 FETCH_DISTRIBUTIONS=""
 for dist in $DISTRIBUTIONS; do
 	if [ ! -f $BSDINSTALL_DISTDIR/$dist ]; then
 		FETCH_DISTRIBUTIONS="$FETCH_DISTRIBUTIONS $dist"
-	else
-		LOCAL_DISTRIBUTIONS="$LOCAL_DISTRIBUTIONS $dist"
 	fi
 done
-LOCAL_DISTRIBUTIONS=`echo $LOCAL_DISTRIBUTIONS`	# Trim white space
-FETCH_DISTRIBUTIONS=`echo $FETCH_DISTRIBUTIONS`	# Trim white space
 
 if [ -n "$FETCH_DISTRIBUTIONS" -a -n "$BSDINSTALL_CONFIGCURRENT" ]; then
 	dialog --backtitle "FreeBSD Installer" --title "Network Installation" --msgbox "Some installation files were not found on the boot volume. The next few screens will allow you to configure networking so that they can be downloaded from the Internet." 0 0
@@ -188,15 +182,6 @@ if [ -n "$FETCH_DISTRIBUTIONS" -a -n "$BSDINSTALL_CONFIGCURRENT" ]; then
 	NETCONFIG_DONE=yes
 fi
 
-if [ -n "$FETCH_DISTRIBUTIONS" ]; then
-	exec 3>&1
-	BSDINSTALL_DISTSITE=$(`dirname $0`/mirrorselect 2>&1 1>&3)
-	MIRROR_BUTTON=$?
-	exec 3>&-
-	test $MIRROR_BUTTON -eq 0 || error "No mirror selected"
-	export BSDINSTALL_DISTSITE
-fi
-
 rm -f $PATH_FSTAB
 touch $PATH_FSTAB
 
@@ -347,79 +332,14 @@ case "$PARTMODE" in
 	;;
 esac
 
-if [ ! -z "$FETCH_DISTRIBUTIONS" ]; then
-	ALL_DISTRIBUTIONS="$DISTRIBUTIONS"
-	WANT_DEBUG=
-
-	# Download to a directory in the new system as scratch space
-	BSDINSTALL_FETCHDEST="$BSDINSTALL_CHROOT/usr/freebsd-dist"
-	mkdir -p "$BSDINSTALL_FETCHDEST" || error "Could not create directory $BSDINSTALL_FETCHDEST"
-
-	export DISTRIBUTIONS="$FETCH_DISTRIBUTIONS"
-	# Try to use any existing distfiles
-	if [ -d $BSDINSTALL_DISTDIR ]; then
-		DISTDIR_IS_UNIONFS=1
-		mount_nullfs -o union "$BSDINSTALL_FETCHDEST" "$BSDINSTALL_DISTDIR"
-	else
-		export DISTRIBUTIONS="$FETCH_DISTRIBUTIONS"
-		export BSDINSTALL_DISTDIR="$BSDINSTALL_FETCHDEST"
-	fi
-
-	export FTP_PASSIVE_MODE=YES
-	# Iterate through the distribution list and set a flag if debugging
-	# distributions have been selected.
-	for _DISTRIBUTION in $DISTRIBUTIONS; do
-		case $_DISTRIBUTION in
-			*-dbg.*)
-				[ -e $BSDINSTALL_DISTDIR/$_DISTRIBUTION ] \
-					&& continue
-				WANT_DEBUG=1
-				DEBUG_LIST="\n$DEBUG_LIST\n$_DISTRIBUTION"
-				;;
-			*)
-				;;
-		esac
-	done
-
-	# Fetch the distributions.
-	bsdinstall distfetch
-	rc=$?
-
-	if [ $rc -ne 0 ]; then
-		# If unable to fetch the remote distributions, recommend
-		# deselecting the debugging distributions, and retrying the
-		# installation, since failure to fetch *-dbg.txz should not
-		# be considered a fatal installation error.
-		msg="Failed to fetch remote distribution"
-		if [ ! -z "$WANT_DEBUG" ]; then
-			# Trim leading and trailing newlines.
-			DEBUG_LIST="${DEBUG_LIST%%\n}"
-			DEBUG_LIST="${DEBUG_LIST##\n}"
-			msg="$msg\n\nPlease deselect the following distributions"
-			msg="$msg and retry the installation:"
-			msg="$msg\n$DEBUG_LIST"
-		fi
-		error "$msg"
-	fi
-	export DISTRIBUTIONS="$ALL_DISTRIBUTIONS"
-fi
+if [ -n "$FETCH_DISTRIBUTIONS" ]; then
+	exec 3>&1
+	export BSDINSTALL_DISTDIR=$(`dirname $0`/fetchmissingdists 2>&1 1>&3)
+	FETCH_RESULT=$?
+	exec 3>&-
 
-if [ ! -z "$LOCAL_DISTRIBUTIONS" ]; then
-	# Download to a directory in the new system as scratch space
-	BSDINSTALL_FETCHDEST="$BSDINSTALL_CHROOT/usr/freebsd-dist"
-	mkdir -p "$BSDINSTALL_FETCHDEST" || error "Could not create directory $BSDINSTALL_FETCHDEST"
-	# Try to use any existing distfiles
-	if [ -d $BSDINSTALL_DISTDIR ]; then
-		DISTDIR_IS_UNIONFS=1
-		mount_nullfs -o union "$BSDINSTALL_FETCHDEST" "$BSDINSTALL_DISTDIR"
-		export BSDINSTALL_DISTDIR="$BSDINSTALL_FETCHDEST"
-	fi
-	env DISTRIBUTIONS="$LOCAL_DISTRIBUTIONS" \
-		BSDINSTALL_DISTSITE="file:///usr/freebsd-dist" \
-		bsdinstall distfetch || \
-		error "Failed to fetch distribution from local media"
+	[ $FETCH_RESULT -ne 0 ] && error "Could not fetch remote distributions"
 fi
-
 bsdinstall checksum || error "Distribution checksum failed"
 bsdinstall distextract || error "Distribution extract failed"
 
@@ -498,10 +418,6 @@ finalconfig
 trap error SIGINT	# SIGINT is bad again
 bsdinstall config  || error "Failed to save config"
 
-if [ -n "$DISTDIR_IS_UNIONFS" ]; then
-	umount -f $BSDINSTALL_DISTDIR
-fi
-
 if [ ! -z "$BSDINSTALL_FETCHDEST" ]; then
 	rm -rf "$BSDINSTALL_FETCHDEST"
 fi
diff --git a/usr.sbin/bsdinstall/scripts/bootconfig b/usr.sbin/bsdinstall/scripts/bootconfig
index a592142d87a2..c0a0ff4f77ee 100755
--- a/usr.sbin/bsdinstall/scripts/bootconfig
+++ b/usr.sbin/bsdinstall/scripts/bootconfig
@@ -83,8 +83,10 @@ if [ -n "$(awk '{if ($2=="/boot/efi") printf("%s\n",$1);}' $PATH_FSTAB)" ]; then
 
 	bootlabel="FreeBSD"
 
-	f_dprintf "Creating UEFI boot entry"
-	efibootmgr --create --activate --label "$bootlabel" --loader "${mntpt}/${FREEBSD_BOOTNAME}" > /dev/null
+	if [ "$BSDINSTALL_CONFIGCURRENT" ]; then
+		f_dprintf "Creating UEFI boot entry"
+		efibootmgr --create --activate --label "$bootlabel" --loader "${mntpt}/${FREEBSD_BOOTNAME}" > /dev/null
+	fi
 
 	f_dprintf "Finished configuring ESP"
 fi
diff --git a/usr.sbin/bsdinstall/scripts/fetchmissingdists b/usr.sbin/bsdinstall/scripts/fetchmissingdists
new file mode 100644
index 000000000000..768217e2fb8c
--- /dev/null
+++ b/usr.sbin/bsdinstall/scripts/fetchmissingdists
@@ -0,0 +1,132 @@
+#!/bin/sh
+#-
+# Copyright (c) 2011 Nathan Whitehorn
+# Copyright (c) 2013-2018 Devin Teske
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+error()
+{
+	dialog --backtitle "FreeBSD Installer" --title "Error" --msgbox $1 0 0
+	exit 1
+}
+
+FETCH_DISTRIBUTIONS=""
+LOCAL_DISTRIBUTIONS=""
+for dist in $DISTRIBUTIONS; do
+	if [ ! -f $BSDINSTALL_DISTDIR/$dist ]; then
+		FETCH_DISTRIBUTIONS="$FETCH_DISTRIBUTIONS $dist"
+	else
+		LOCAL_DISTRIBUTIONS="$LOCAL_DISTRIBUTIONS $dist"
+	fi
+done
+LOCAL_DISTRIBUTIONS=`echo $LOCAL_DISTRIBUTIONS`	# Trim white space
+FETCH_DISTRIBUTIONS=`echo $FETCH_DISTRIBUTIONS`	# Trim white space
+
+if [ -z "$FETCH_DISTRIBUTIONS" ]; then
+	echo $BSDINSTALL_DISTDIR >&2
+	exit 0
+fi
+
+ALL_DISTRIBUTIONS="$DISTRIBUTIONS"
+WANT_DEBUG=
+
+# Download to a directory in the new system as scratch space
+BSDINSTALL_FETCHDEST="$BSDINSTALL_CHROOT/usr/freebsd-dist"
+mkdir -p "$BSDINSTALL_FETCHDEST" || error "Could not create directory $BSDINSTALL_FETCHDEST"
+
+if [ -z "$BSDINSTALL_DISTSITE" ]; then
+	exec 3>&1
+	BSDINSTALL_DISTSITE=$(`dirname $0`/mirrorselect 2>&1 1>&3)
+	MIRROR_BUTTON=$?
+	exec 3>&-
+	test $MIRROR_BUTTON -eq 0 || error "No mirror selected"
+	export BSDINSTALL_DISTSITE
+fi
+
+BSDINSTALL_DISTDIR_ORIG="$BSDINSTALL_DISTDIR"
+export BSDINSTALL_DISTDIR="$BSDINSTALL_FETCHDEST"
+export FTP_PASSIVE_MODE=YES
+
+if [ -f "$BSDINSTALL_DISTDIR_ORIG/MANIFEST" ]; then
+	cp "$BSDINSTALL_DISTDIR_ORIG/MANIFEST" "$BSDINSTALL_DISTDIR/MANIFEST"
+	VERIFY_MANIFEST_SIG=0
+else
+	FETCH_DISTRIBUTIONS="MANIFEST $FETCH_DISTRIBUTIONS"
+	VERIFY_MANIFEST_SIG=1
+
+	# XXX actually verify signature on manifest
+	dialog --backtitle "FreeBSD Installer" --title "Warning" --msgbox "Manifest not found on local disk and will be fetched from an unverified source. This is a potential security risk. If you do not wish to proceed, press control-C now." 0 0
+fi
+
+if [ ! -z "$LOCAL_DISTRIBUTIONS" ]; then
+	# Copy local stuff first
+	env DISTRIBUTIONS="$LOCAL_DISTRIBUTIONS" \
+		BSDINSTALL_DISTSITE="file://$BSDINSTALL_DISTDIR" \
+		bsdinstall distfetch || \
+		error "Failed to fetch distribution from local media"
+fi
+
+export DISTRIBUTIONS="$FETCH_DISTRIBUTIONS"
+
+# Iterate through the distribution list and set a flag if debugging
+# distributions have been selected.
+for _DISTRIBUTION in $DISTRIBUTIONS; do
+	case $_DISTRIBUTION in
+		*-dbg.*)
+			[ -e $BSDINSTALL_DISTDIR/$_DISTRIBUTION ] \
+				&& continue
+			WANT_DEBUG=1
+			DEBUG_LIST="\n$DEBUG_LIST\n$_DISTRIBUTION"
+			;;
+		*)
+			;;
+	esac
+done
+
+# Fetch the distributions.
+bsdinstall distfetch
+rc=$?
+
+if [ $rc -ne 0 ]; then
+	# If unable to fetch the remote distributions, recommend
+	# deselecting the debugging distributions, and retrying the
+	# installation, since failure to fetch *-dbg.txz should not
+	# be considered a fatal installation error.
+	msg="Failed to fetch remote distribution"
+	if [ ! -z "$WANT_DEBUG" ]; then
+		# Trim leading and trailing newlines.
+		DEBUG_LIST="${DEBUG_LIST%%\n}"
+		DEBUG_LIST="${DEBUG_LIST##\n}"
+		msg="$msg\n\nPlease deselect the following distributions"
+		msg="$msg and retry the installation:"
+		msg="$msg\n$DEBUG_LIST"
+	fi
+	error "$msg"
+fi
+
+echo $BSDINSTALL_DISTDIR >&2
+
diff --git a/usr.sbin/bsdinstall/scripts/script b/usr.sbin/bsdinstall/scripts/script
index 6a03e88bba39..131ef008d35f 100755
--- a/usr.sbin/bsdinstall/scripts/script
+++ b/usr.sbin/bsdinstall/scripts/script
@@ -114,6 +114,14 @@ else
 fi
 bsdinstall mount
 
+# Fetch missing distribution files, if any
+exec 3>&1
+export BSDINSTALL_DISTDIR=$(`dirname $0`/fetchmissingdists 2>&1 1>&3)
+FETCH_RESULT=$?
+exec 3>&-
+
+[ $FETCH_RESULT -ne 0 ] && error "Could not fetch remote distributions"
+
 # Unpack distributions
 bsdinstall checksum
 if [ -t 0 ]; then