Add support for adding default routes for other FIBs

Make rc.d/routing read defaultrouter_fibN and ipv6_defaultrouter_fibN, and
set it as the default gateway for FIB N, where N is from 1 to (net.fibs - 1)
This allows adding gateways for multiple FIBs in the same format as the main
gateway. (FIB 0)

Reviewed by:	olivier, rgrimes, bcr (man page)
Sponsored by:	ScaleEngine Inc.
Differential Revision:	https://reviews.freebsd.org/D22706

3 files changed, 48 insertions, 3 deletions

diff --git a/libexec/rc/rc.conf b/libexec/rc/rc.conf
index 998cd3fd14d6..d941a20b6e07 100644
--- a/libexec/rc/rc.conf
+++ b/libexec/rc/rc.conf
@@ -437,6 +437,7 @@ bsnmpd_flags=""			# Flags for bsnmpd.
 
 ### Network routing options: ###
 defaultrouter="NO"		# Set to default gateway (or NO).
+#defaultrouter_fibN="192.0.2.1"	# Use this form to set a gateway for FIB N
 static_arp_pairs=""		# Set to static ARP list (or leave empty).
 static_ndp_pairs=""		# Set to static NDP list (or leave empty).
 static_routes=""		# Set to static route list (or leave empty).
@@ -499,6 +500,7 @@ ipv6_activate_all_interfaces="NO"	# If NO, interfaces which have no
 					# reason.
 ipv6_defaultrouter="NO"		# Set to IPv6 default gateway (or NO).
 #ipv6_defaultrouter="2002:c058:6301::"	# Use this for 6to4 (RFC 3068)
+#ipv6_defaultrouter_fibN="2001:db8::"	# Use this form to set a gateway for FIB N
 ipv6_static_routes=""		# Set to static route list (or leave empty).
 #ipv6_static_routes="xxx"	# An example to set fec0:0000:0000:0006::/64
 				#  route toward loopback interface.
diff --git a/libexec/rc/rc.d/routing b/libexec/rc/rc.d/routing
index 043c5b15fbaa..37b3da0f0cef 100755
--- a/libexec/rc/rc.d/routing
+++ b/libexec/rc/rc.d/routing
@@ -140,11 +140,12 @@ get_fibmod()
 
 static_inet()
 {
-	local _action _if _skip _fibmod
+	local _action _if _skip _fibmod _fibs
 	_action=$1
 	_if=$2
 
 	_fibmod=`get_fibmod`
+	_fibs=$((`${SYSCTL_N} net.fibs` - 1))
 
 	# Provide loopback route in all routing tables.  This has to come
 	# first so that any following routes can be added.
@@ -161,6 +162,22 @@ static_inet()
 		;;
 	esac
 
+	# Add default routes for fibs
+	if [ ${_fibs} -gt 0 ]; then
+		for _fibnum in `jot ${_fibs}` ; do
+			eval _fib_gw=\${defaultrouter_fib${_fibnum}}
+			case ${_fib_gw} in
+			[Nn][Oo] | '')
+				;;
+			*)
+				static_routes="${static_routes} _default_fib${_fibnum}"
+				eval route__default_fib${fibnum}="'default ${_fib_gw} -fib ${_fibnum}'"
+				;;
+			esac
+		done
+	fi
+
+
 	# Install configured routes.
 	if [ -n "${static_routes}" ]; then
 		for i in ${static_routes}; do
@@ -185,11 +202,12 @@ static_inet()
 
 static_inet6()
 {
-	local _action _if _skip fibmod allfibs
+	local _action _if _skip fibmod _fibs
 	_action=$1
 	_if=$2
 
 	fibmod=`get_fibmod`
+	_fibs=$((`${SYSCTL_N} net.fibs` - 1))
 
 	# Add pre-defined static routes first.
 	ipv6_static_routes="_v4mapped _v4compat ${ipv6_static_routes}"
@@ -221,6 +239,22 @@ static_inet6()
 		;;
 	esac
 
+	# Add default routes for fibs
+	if [ ${_fibs} -gt 0 ]; then
+		for _fibnum in `jot ${_fibs}` ; do
+			eval _fib_gw=\${ipv6_defaultrouter_fib${_fibnum}}
+			case ${_fib_gw} in
+			[Nn][Oo] | '')
+				;;
+			*)
+				ipv6_static_routes="${static_routes} _default_fib${_fibnum}"
+				eval ipv6_route__default_fib${fibnum}="'default ${_fib_gw} -fib ${_fibnum}'"
+				;;
+			esac
+		done
+	fi
+
+
 	# Install configured routes.
 	if [ -n "${ipv6_static_routes}" ]; then
 		for i in ${ipv6_static_routes}; do
diff --git a/share/man/man5/rc.conf.5 b/share/man/man5/rc.conf.5
index 01c4a8436496..6baf3b22b024 100644
--- a/share/man/man5/rc.conf.5
+++ b/share/man/man5/rc.conf.5
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 21, 2021
+.Dd May 11, 2021
 .Dt RC.CONF 5
 .Os
 .Sh NAME
@@ -2818,10 +2818,19 @@ If not set to
 create a default route to this host name or IP address
 (use an IP address if this router is also required to get to the
 name server!).
+.It Va defaultrouter_fibN
+.Pq Vt str
+If not set to
+.Dq Li NO ,
+create a default route in FIB N to this host name or IP address.
 .It Va ipv6_defaultrouter
 .Pq Vt str
 The IPv6 equivalent of
 .Va defaultrouter .
+.It Va ipv6_defaultrouter_fibN
+.Pq Vt str
+The IPv6 equivalent of
+.Va defaultrouter_fibN .
 .It Va static_arp_pairs
 .Pq Vt str
 Set to the list of static ARP pairs that are to be added at system