route(8): convert to netlink

This change converts all kernel rtsock interactions in route(8)
 to Netlink.

Based on the WITHOUT_NETLINK_SUPPORT src.conf(5) variable, route(8)
 now fully operates either via Netlink or via rtsock/sysctl.
The default (compile-time) is Netlink.

The output for route delete/add/get/flush is targeted to be exactly
 the same (apart from some error handling cases).
The output for the route monitor has been changed to improve
 readability and support netlink models.

Other behaviour changes:
* exact prefix lookup (route -n get a.b.c.d/e) is not yet supported.
* route monitor does not show the change originator yet.

Differential Revision:	https://reviews.freebsd.org/D39007

3 files changed, 930 insertions, 28 deletions

diff --git a/sbin/route/Makefile b/sbin/route/Makefile
index e65030f805bb..aec222310d46 100644
--- a/sbin/route/Makefile
+++ b/sbin/route/Makefile
@@ -19,6 +19,12 @@ CFLAGS+= -DINET6
 .endif
 CFLAGS+= -I.
 
+.if ${MK_NETLINK_SUPPORT} != "no"
+SRCS+=	route_netlink.c
+.else
+CFLAGS+=-DWITHOUT_NETLINK
+.endif
+
 HAS_TESTS=
 SUBDIR.${MK_TESTS}+= tests
 
diff --git a/sbin/route/route.c b/sbin/route/route.c
index 5f33cecb1b20..947c97ce794a 100644
--- a/sbin/route/route.c
+++ b/sbin/route/route.c
@@ -90,12 +90,11 @@ static struct keytab {
 	{0, 0}
 };
 
+int	verbose, debugonly;
 static struct sockaddr_storage so[RTAX_MAX];
 static int	pid, rtm_addrs;
-static int	s;
-static int	nflag, af, qflag, tflag;
-static int	verbose, aflen;
-static int	locking, lockrest, debugonly;
+static int	nflag, af, aflen, qflag, tflag;
+static int	locking, lockrest;
 static struct rt_metrics rt_metrics;
 static u_long  rtm_inits;
 static uid_t	uid;
@@ -103,18 +102,30 @@ static int	defaultfib;
 static int	numfibs;
 static char	domain[MAXHOSTNAMELEN + 1];
 static bool	domain_initialized;
-static int	rtm_seq;
 static char	rt_line[NI_MAXHOST];
 static char	net_line[MAXHOSTNAMELEN + 1];
 
+#ifdef WITHOUT_NETLINK
+static int	s;
+static int	rtm_seq;
+
 static struct {
 	struct	rt_msghdr m_rtm;
 	char	m_space[512];
 } m_rtmsg;
 
+static int	rtmsg_rtsock(int, int, int);
+static int	flushroutes_fib_rtsock(int);
+static void	monitor_rtsock(void);
+#else
+int		rtmsg_nl(int, int, int, struct sockaddr_storage *, struct rt_metrics *);
+int		flushroutes_fib_nl(int, int);
+void		monitor_nl(int);
+#endif
+
 static TAILQ_HEAD(fibl_head_t, fibl) fibl_head;
 
-static void	printb(int, const char *);
+void	printb(int, const char *);
 static void	flushroutes(int argc, char *argv[]);
 static int	flushroutes_fib(int);
 static int	getaddr(int, char *, int);
@@ -127,7 +138,7 @@ static int	inet6_makenetandmask(struct sockaddr_in6 *, const char *);
 #endif
 static void	interfaces(void);
 static void	monitor(int, char*[]);
-static const char	*netname(struct sockaddr *);
+const char	*netname(struct sockaddr *);
 static void	newroute(int, char **);
 static int	newroute_fib(int, char *, int);
 static void	pmsg_addrs(char *, int, size_t);
@@ -135,7 +146,7 @@ static void	pmsg_common(struct rt_msghdr *, size_t);
 static int	prefixlen(const char *);
 static void	print_getmsg(struct rt_msghdr *, int, int);
 static void	print_rtmsg(struct rt_msghdr *, size_t);
-static const char	*routename(struct sockaddr *);
+const char	*routename(struct sockaddr *);
 static int	rtmsg(int, int, int);
 static void	set_metric(char *, int);
 static int	set_sofib(int);
@@ -216,12 +227,14 @@ main(int argc, char **argv)
 
 	pid = getpid();
 	uid = geteuid();
+#ifdef WITHOUT_NETLINK
 	if (tflag)
 		s = open(_PATH_DEVNULL, O_WRONLY, 0);
 	else
 		s = socket(PF_ROUTE, SOCK_RAW, 0);
 	if (s < 0)
 		err(EX_OSERR, "socket");
+#endif
 
 	len = sizeof(numfibs);
 	if (sysctlbyname("net.fibs", (void *)&numfibs, &len, NULL, 0) == -1)
@@ -264,10 +277,14 @@ static int
 set_sofib(int fib)
 {
 
+#ifdef WITHOUT_NETLINK
 	if (fib < 0)
 		return (0);
 	return (setsockopt(s, SOL_SOCKET, SO_SETFIB, (void *)&fib,
 	    sizeof(fib)));
+#else
+	return (0);
+#endif
 }
 
 static int
@@ -395,7 +412,9 @@ flushroutes(int argc, char *argv[])
 
 	if (uid != 0 && !debugonly && !tflag)
 		errx(EX_NOPERM, "must be root to alter routing table");
+#ifdef WITHOUT_NETLINK
 	shutdown(s, SHUT_RD); /* Don't want to read back our messages */
+#endif
 
 	TAILQ_INIT(&fibl_head);
 	while (argc > 1) {
@@ -442,6 +461,17 @@ flushroutes(int argc, char *argv[])
 static int
 flushroutes_fib(int fib)
 {
+#ifdef WITHOUT_NETLINK
+	return (flushroutes_fib_rtsock(fib));
+#else
+	return (flushroutes_fib_nl(fib, af));
+#endif
+}
+
+#ifdef WITHOUT_NETLINK
+static int
+flushroutes_fib_rtsock(int fib)
+{
 	struct rt_msghdr *rtm;
 	size_t needed;
 	char *buf, *next, *lim;
@@ -525,8 +555,9 @@ retry:
 	free(buf);
 	return (error);
 }
+#endif
 
-static const char *
+const char *
 routename(struct sockaddr *sa)
 {
 	struct sockaddr_dl *sdl;
@@ -645,7 +676,7 @@ routename(struct sockaddr *sa)
  * Return the name of the network whose address is given.
  * The address is assumed to be that of a net, not a host.
  */
-static const char *
+const char *
 netname(struct sockaddr *sa)
 {
 	struct sockaddr_dl *sdl;
@@ -810,8 +841,10 @@ newroute(int argc, char **argv)
 		warn("sigaction SIGALRM");
 
 	cmd = argv[0];
+#ifdef WITHOUT_NETLINK
 	if (*cmd != 'g' && *cmd != 's')
 		shutdown(s, SHUT_RD); /* Don't want to read back our messages */
+#endif
 	while (--argc > 0) {
 		if (**(++argv)== '-') {
 			switch (key = keyword(1 + *argv)) {
@@ -1398,8 +1431,8 @@ retry2:
 static void
 monitor(int argc, char *argv[])
 {
-	int n, fib, error;
-	char msg[2048], *endptr;
+	int fib, error;
+	char *endptr;
 
 	fib = defaultfib;
 	while (argc > 1) {
@@ -1435,6 +1468,19 @@ monitor(int argc, char *argv[])
 		interfaces();
 		exit(0);
 	}
+#ifdef WITHOUT_NETLINK
+	monitor_rtsock();
+#else
+	monitor_nl(fib);
+#endif
+}
+
+#ifdef WITHOUT_NETLINK
+static void
+monitor_rtsock(void)
+{
+	char msg[2048];
+	int n;
 
 #ifdef SO_RERROR
 	n = 1;
@@ -1454,25 +1500,12 @@ monitor(int argc, char *argv[])
 		print_rtmsg((struct rt_msghdr *)(void *)msg, n);
 	}
 }
+#endif
 
 static int
 rtmsg(int cmd, int flags, int fib)
 {
-	int rlen;
-	char *cp = m_rtmsg.m_space;
-	int l;
-
-#define NEXTADDR(w, u)							\
-	if (rtm_addrs & (w)) {						\
-		l = SA_SIZE(&(u));					\
-		memmove(cp, (char *)&(u), l);				\
-		cp += l;						\
-		if (verbose)						\
-			sodump((struct sockaddr *)&(u), #w);		\
-	}
-
 	errno = 0;
-	memset(&m_rtmsg, 0, sizeof(m_rtmsg));
 	if (cmd == 'a')
 		cmd = RTM_ADD;
 	else if (cmd == 'c')
@@ -1488,6 +1521,33 @@ rtmsg(int cmd, int flags, int fib)
 		cmd = RTM_DELETE;
 		flags |= RTF_PINNED;
 	}
+#ifdef WITHOUT_NETLINK
+	return (rtmsg_rtsock(cmd, flags, fib));
+#else
+	errno = rtmsg_nl(cmd, flags, fib, so, &rt_metrics);
+	return (errno == 0 ? 0 : -1);
+#endif
+}
+
+#ifdef WITHOUT_NETLINK
+static int
+rtmsg_rtsock(int cmd, int flags, int fib)
+{
+	int rlen;
+	char *cp = m_rtmsg.m_space;
+	int l;
+
+	memset(&m_rtmsg, 0, sizeof(m_rtmsg));
+
+#define NEXTADDR(w, u)							\
+	if (rtm_addrs & (w)) {						\
+		l = SA_SIZE(&(u));					\
+		memmove(cp, (char *)&(u), l);				\
+		cp += l;						\
+		if (verbose)						\
+			sodump((struct sockaddr *)&(u), #w);		\
+	}
+
 #define rtm m_rtmsg.m_rtm
 	rtm.rtm_type = cmd;
 	rtm.rtm_flags = flags;
@@ -1545,6 +1605,7 @@ rtmsg(int cmd, int flags, int fib)
 #undef rtm
 	return (0);
 }
+#endif
 
 static const char *const msgtypes[] = {
 	"",
@@ -1571,7 +1632,7 @@ static const char *const msgtypes[] = {
 static const char metricnames[] =
     "\011weight\010rttvar\7rtt\6ssthresh\5sendpipe\4recvpipe\3expire"
     "\1mtu";
-static const char routeflags[] =
+const char routeflags[] =
     "\1UP\2GATEWAY\3HOST\4REJECT\5DYNAMIC\6MODIFIED\7DONE"
     "\012XRESOLVE\013LLINFO\014STATIC\015BLACKHOLE"
     "\017PROTO2\020PROTO1\021PRCLONING\022WASCLONED\023PROTO3"
@@ -1812,7 +1873,7 @@ pmsg_addrs(char *cp, int addrs, size_t len)
 	(void)fflush(stdout);
 }
 
-static void
+void
 printb(int b, const char *str)
 {
 	int i;
diff --git a/sbin/route/route_netlink.c b/sbin/route/route_netlink.c
new file mode 100644
index 000000000000..648d866670fc
--- /dev/null
+++ b/sbin/route/route_netlink.c
@@ -0,0 +1,835 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <err.h>
+#include <errno.h>
+
+#include <sys/bitcount.h>
+#include <sys/param.h>
+#include <sys/socket.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+#include <sys/types.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_dl.h>
+#include <net/if_types.h>
+#include <netlink/netlink.h>
+#include <netlink/netlink_route.h>
+#include <netlink/netlink_snl.h>
+#include <netlink/netlink_snl_route.h>
+#include <netlink/netlink_snl_route_compat.h>
+#include <netlink/netlink_snl_route_parsers.h>
+
+const char *routename(struct sockaddr *);
+const char *netname(struct sockaddr *);
+void printb(int, const char *);
+extern const char routeflags[];
+extern int verbose, debugonly;
+
+int rtmsg_nl(int cmd, int rtm_flags, int fib, struct sockaddr_storage *so,
+    struct rt_metrics *rt_metrics);
+int flushroutes_fib_nl(int fib, int af);
+void monitor_nl(int fib);
+
+struct nl_helper;
+static void print_getmsg(struct nl_helper *h, struct nlmsghdr *hdr, struct sockaddr *dst);
+static void print_nlmsg(struct nl_helper *h, struct nlmsghdr *hdr);
+
+#define s6_addr32 __u6_addr.__u6_addr32
+#define	bitcount32(x)	__bitcount32((uint32_t)(x))
+static int
+inet6_get_plen(const struct in6_addr *addr)
+{
+
+	return (bitcount32(addr->s6_addr32[0]) + bitcount32(addr->s6_addr32[1]) +
+	    bitcount32(addr->s6_addr32[2]) + bitcount32(addr->s6_addr32[3]));
+}
+
+static void
+ip6_writemask(struct in6_addr *addr6, uint8_t mask)
+{
+	uint32_t *cp;
+
+	for (cp = (uint32_t *)addr6; mask >= 32; mask -= 32)
+		*cp++ = 0xFFFFFFFF;
+	if (mask > 0)
+		*cp = htonl(mask ? ~((1 << (32 - mask)) - 1) : 0);
+}
+
+static struct sockaddr *
+get_netmask(struct snl_state *ss, int family, int plen)
+{
+	if (family == AF_INET) {
+		if (plen == 32)
+			return (NULL);
+
+		struct sockaddr_in *sin = snl_allocz(ss, sizeof(*sin));
+
+		sin->sin_len = sizeof(*sin);
+		sin->sin_family = family;
+		sin->sin_addr.s_addr = htonl(plen ? ~((1 << (32 - plen)) - 1) : 0);
+
+		return (struct sockaddr *)sin;
+	} else if (family == AF_INET6) {
+		if (plen == 128)
+			return (NULL);
+
+		struct sockaddr_in6 *sin6 = snl_allocz(ss, sizeof(*sin6));
+
+		sin6->sin6_len = sizeof(*sin6);
+		sin6->sin6_family = family;
+		ip6_writemask(&sin6->sin6_addr, plen);
+
+		return (struct sockaddr *)sin6;
+	}
+	return (NULL);
+}
+
+struct nl_helper {
+	struct snl_state ss_cmd;
+};
+
+static void
+nl_helper_init(struct nl_helper *h)
+{
+	if (!snl_init(&h->ss_cmd, NETLINK_ROUTE))
+		err(1, "unable to open netlink socket");
+}
+
+static void
+nl_helper_free(struct nl_helper *h)
+{
+	snl_free(&h->ss_cmd);
+}
+
+static int
+rtmsg_nl_int(struct nl_helper *h, int cmd, int rtm_flags, int fib,
+    struct sockaddr_storage *so, struct rt_metrics *rt_metrics)
+{
+	struct snl_state *ss = &h->ss_cmd;
+	struct snl_writer nw;
+	int nl_type = 0, nl_flags = 0;
+
+	snl_init_writer(ss, &nw);
+
+	switch (cmd) {
+	case RTSOCK_RTM_ADD:
+		nl_type = RTM_NEWROUTE;
+		nl_flags = NLM_F_CREATE | NLM_F_APPEND; /* Do append by default */
+		break;
+	case RTSOCK_RTM_CHANGE:
+		nl_type = RTM_NEWROUTE;
+		nl_flags = NLM_F_REPLACE;
+		break;
+	case RTSOCK_RTM_DELETE:
+		nl_type = RTM_DELROUTE;
+		break;
+	case RTSOCK_RTM_GET:
+		nl_type = RTM_GETROUTE;
+		break;
+	default:
+		exit(1);
+	}
+
+	struct sockaddr *dst = (struct sockaddr *)&so[RTAX_DST];
+	struct sockaddr *mask = (struct sockaddr *)&so[RTAX_NETMASK];
+	struct sockaddr *gw = (struct sockaddr *)&so[RTAX_GATEWAY];
+
+	if (dst == NULL)
+		return (EINVAL);
+
+	struct nlmsghdr *hdr = snl_create_msg_request(&nw, nl_type);
+	hdr->nlmsg_flags |= nl_flags;
+
+	int plen = 0;
+	int rtm_type = RTN_UNICAST;
+
+	switch (dst->sa_family) {
+	case AF_INET:
+	    {
+		struct sockaddr_in *mask4 = (struct sockaddr_in *)mask;
+
+		plen = mask4 ? bitcount32(mask4->sin_addr.s_addr) : 32;
+		break;
+	    }
+	case AF_INET6:
+	    {
+		struct sockaddr_in6 *mask6 = (struct sockaddr_in6 *)mask;
+
+		plen = mask6 ? inet6_get_plen(&mask6->sin6_addr) : 128;
+		break;
+	    }
+	default:
+		return (ENOTSUP);
+	}
+
+	if (rtm_flags & RTF_REJECT)
+		rtm_type = RTN_PROHIBIT;
+	else if (rtm_flags & RTF_BLACKHOLE)
+		rtm_type = RTN_BLACKHOLE;
+
+	struct rtmsg *rtm = snl_reserve_msg_object(&nw, struct rtmsg);
+	rtm->rtm_family = dst->sa_family;
+	rtm->rtm_protocol = RTPROT_STATIC;
+	rtm->rtm_type = rtm_type;
+	rtm->rtm_dst_len = plen;
+
+	snl_add_msg_attr_ip(&nw, RTA_DST, dst);
+	snl_add_msg_attr_u32(&nw, RTA_TABLE, fib);
+
+	if (rtm_flags & RTF_GATEWAY) {
+		if (gw->sa_family == dst->sa_family)
+			snl_add_msg_attr_ip(&nw, RTA_GATEWAY, gw);
+		else
+			snl_add_msg_attr_ipvia(&nw, RTA_VIA, gw);
+	} else if (gw != NULL) {
+		/* Should be AF_LINK */
+		struct sockaddr_dl *sdl = (struct sockaddr_dl *)gw;
+		if (sdl->sdl_index != 0)
+			snl_add_msg_attr_u32(&nw, RTA_OIF, sdl->sdl_index);
+	}
+
+	if (rtm_flags != 0)
+		snl_add_msg_attr_u32(&nw, NL_RTA_RTFLAGS, rtm_flags);
+
+	if (rt_metrics->rmx_mtu > 0) {
+		int off = snl_add_msg_attr_nested(&nw, RTA_METRICS);
+		snl_add_msg_attr_u32(&nw, RTAX_MTU, rt_metrics->rmx_mtu);
+		snl_end_attr_nested(&nw, off);
+	}
+
+	if (rt_metrics->rmx_weight > 0)
+		snl_add_msg_attr_u32(&nw, NL_RTA_WEIGHT, rt_metrics->rmx_weight);
+
+	if (snl_finalize_msg(&nw) && snl_send_message(ss, hdr)) {
+		struct snl_errmsg_data e = {};
+
+		hdr = snl_read_reply(ss, hdr->nlmsg_seq);
+		if (nl_type == NL_RTM_GETROUTE) {
+			if (hdr->nlmsg_type == NL_RTM_NEWROUTE)
+				print_getmsg(h, hdr, dst);
+			else {
+				snl_parse_errmsg(ss, hdr, &e);
+				if (e.error == ESRCH)
+					warn("route has not been found");
+				else
+					warn("message indicates error %d", e.error);
+			}
+
+			return (0);
+		}
+
+		if (snl_parse_errmsg(ss, hdr, &e))
+			return (e.error);
+	}
+	return (EINVAL);
+}
+
+int
+rtmsg_nl(int cmd, int rtm_flags, int fib, struct sockaddr_storage *so,
+    struct rt_metrics *rt_metrics)
+{
+	struct nl_helper h = {};
+
+	nl_helper_init(&h);
+	int error = rtmsg_nl_int(&h, cmd, rtm_flags, fib, so, rt_metrics);
+	nl_helper_free(&h);
+
+	return (error);
+}
+
+static void
+get_ifdata(struct nl_helper *h, uint32_t ifindex, struct snl_parsed_link_simple *link)
+{
+	struct snl_state *ss = &h->ss_cmd;
+	struct snl_writer nw;
+
+	snl_init_writer(ss, &nw);
+	struct nlmsghdr *hdr = snl_create_msg_request(&nw, NL_RTM_GETLINK);
+	struct ifinfomsg *ifmsg = snl_reserve_msg_object(&nw, struct ifinfomsg);
+	if (ifmsg != NULL)
+		ifmsg->ifi_index = ifindex;
+	if (!snl_finalize_msg(&nw) || !snl_send_message(ss, hdr))
+		return;
+
+	hdr = snl_read_reply(ss, hdr->nlmsg_seq);
+
+	if (hdr != NULL && hdr->nlmsg_type == RTM_NEWLINK) {
+		snl_parse_nlmsg(ss, hdr, &snl_rtm_link_parser_simple, link);
+	}
+
+	if (link->ifla_ifname == NULL) {
+		char ifname[16];
+
+		snprintf(ifname, sizeof(ifname), "if#%u", ifindex);
+		int len = strlen(ifname);
+		char *buf = snl_allocz(ss, len + 1);
+		strlcpy(buf, ifname, len + 1);
+		link->ifla_ifname = buf;
+	}
+}
+
+static void
+print_getmsg(struct nl_helper *h, struct nlmsghdr *hdr, struct sockaddr *dst)
+{
+	struct snl_state *ss = &h->ss_cmd;
+	struct timespec ts;
+	struct snl_parsed_route r = { .rtax_weight = RT_DEFAULT_WEIGHT };
+
+	if (!snl_parse_nlmsg(ss, hdr, &snl_rtm_route_parser, &r))
+		return;
+
+	struct snl_parsed_link_simple link = {};
+	get_ifdata(h, r.rta_oif, &link);
+
+	if (r.rtax_mtu == 0)
+		r.rtax_mtu = link.ifla_mtu;
+	r.rta_rtflags |= (RTF_UP | RTF_DONE);
+
+	(void)printf("   route to: %s\n", routename(dst));
+
+	if (r.rta_dst)
+		(void)printf("destination: %s\n", routename(r.rta_dst));
+	struct sockaddr *mask = get_netmask(ss, r.rtm_family, r.rtm_dst_len);
+	if (mask)
+		(void)printf("       mask: %s\n", routename(mask));
+	if (r.rta_gw && (r.rta_rtflags & RTF_GATEWAY))
+		(void)printf("    gateway: %s\n", routename(r.rta_gw));
+	(void)printf("        fib: %u\n", (unsigned int)r.rta_table);
+	if (link.ifla_ifname)
+		(void)printf("  interface: %s\n", link.ifla_ifname);
+	(void)printf("      flags: ");
+	printb(r.rta_rtflags, routeflags);
+
+	struct rt_metrics rmx = {
+		.rmx_mtu = r.rtax_mtu,
+		.rmx_weight = r.rtax_weight,
+		.rmx_expire = r.rta_expire,
+	};
+
+	printf("\n%9s %9s %9s %9s %9s %10s %9s\n", "recvpipe",
+	    "sendpipe", "ssthresh", "rtt,msec", "mtu   ", "weight", "expire");
+	printf("%8lu  ", rmx.rmx_recvpipe);
+	printf("%8lu  ", rmx.rmx_sendpipe);
+	printf("%8lu  ", rmx.rmx_ssthresh);
+	printf("%8lu  ", 0UL);
+	printf("%8lu  ", rmx.rmx_mtu);
+	printf("%8lu  ", rmx.rmx_weight);
+	if (rmx.rmx_expire > 0)
+		clock_gettime(CLOCK_REALTIME_FAST, &ts);
+	else
+		ts.tv_sec = 0;
+	printf("%8ld \n", (long)(rmx.rmx_expire - ts.tv_sec));
+}
+
+static void
+print_prefix(struct nl_helper *h, char *buf, int bufsize, struct sockaddr *sa, int plen)
+{
+	int sz = 0;
+
+	if (sa == NULL) {
+		snprintf(buf, bufsize, "<NULL>");
+		return;
+	}
+
+	switch (sa->sa_family) {
+	case AF_INET:
+		{
+			struct sockaddr_in *sin = (struct sockaddr_in *)sa;
+			char abuf[INET_ADDRSTRLEN];
+
+			inet_ntop(AF_INET, &sin->sin_addr, abuf, sizeof(abuf));
+			sz = snprintf(buf, bufsize, "%s", abuf);
+			break;
+		}
+	case AF_INET6:
+		{
+			struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sa;
+			char abuf[INET6_ADDRSTRLEN];
+			char *ifname = NULL;
+
+			inet_ntop(AF_INET6, &sin6->sin6_addr, abuf, sizeof(abuf));
+			if (IN6_IS_ADDR_LINKLOCAL(&sin6->sin6_addr)) {
+				struct snl_parsed_link_simple link = {};
+
+				if (sin6->sin6_scope_id != 0) {
+					get_ifdata(h, sin6->sin6_scope_id, &link);
+					ifname = link.ifla_ifname;
+				}
+			}
+			if (ifname == NULL)
+				sz = snprintf(buf, bufsize, "%s", abuf);
+			else
+				sz = snprintf(buf, bufsize, "%s%%%s", abuf, ifname);
+			break;
+		}
+	default:
+		snprintf(buf, bufsize, "unknown_af#%d", sa->sa_family);
+		plen = -1;
+	}
+
+	if (plen >= 0)
+		snprintf(buf + sz, bufsize - sz, "/%d", plen);
+}
+
+
+static int
+print_line_prefix(const char *cmd, const char *name)
+{
+	struct timespec tp;
+	struct tm tm;
+	char buf[32];
+
+	clock_gettime(CLOCK_REALTIME, &tp);
+	localtime_r(&tp.tv_sec, &tm);
+
+	strftime(buf, sizeof(buf), "%T", &tm);
+	int len = printf("%s.%03ld %s %s ", buf, tp.tv_nsec / 1000000, cmd, name);
+
+	return (len);
+}
+
+static const char *
+get_action_name(struct nlmsghdr *hdr, int new_cmd)
+{
+	if (hdr->nlmsg_type == new_cmd) {
+		//return ((hdr->nlmsg_flags & NLM_F_REPLACE) ? "replace" : "add");
+		return ("add/repl");
+	} else
+		return ("delete");
+}
+
+static void
+print_nlmsg_route_nhop(struct nl_helper *h, struct snl_parsed_route *r,
+    struct rta_mpath_nh *nh, bool first)
+{
+	// gw 10.0.0.1 ifp vtnet0 mtu 1500 table inet.0
+	if (nh->gw != NULL) {
+		char gwbuf[128];
+		print_prefix(h, gwbuf, sizeof(gwbuf), nh->gw, -1);
+		printf("gw %s ", gwbuf);
+	}
+
+	if (nh->ifindex != 0) {
+		struct snl_parsed_link_simple link = {};
+
+		get_ifdata(h, nh->ifindex, &link);
+		if (nh->rtax_mtu == 0)
+			nh->rtax_mtu = link.ifla_mtu;
+		printf("iface %s ", link.ifla_ifname);
+		if (nh->rtax_mtu != 0)
+			printf("mtu %d ", nh->rtax_mtu);
+	}
+
+	if (first) {
+		switch (r->rtm_family) {
+			case AF_INET:
+				printf("table inet.%d", r->rta_table);
+				break;
+			case AF_INET6:
+				printf("table inet6.%d", r->rta_table);
+				break;
+		}
+	}
+
+	printf("\n");
+}
+
+static void
+print_nlmsg_route(struct nl_helper *h, struct nlmsghdr *hdr)
+{
+	struct snl_parsed_route r = { .rtax_weight = RT_DEFAULT_WEIGHT };
+	struct snl_state *ss = &h->ss_cmd;
+
+	if (!snl_parse_nlmsg(ss, hdr, &snl_rtm_route_parser, &r))
+		return;
+
+	// 20:19:41.333 add route 10.0.0.0/24 gw 10.0.0.1 ifp vtnet0 mtu 1500 table inet.0
+
+	const char *cmd = get_action_name(hdr, RTM_NEWROUTE);
+	int len = print_line_prefix(cmd, "route");
+
+	char buf[128];
+	print_prefix(h, buf, sizeof(buf), r.rta_dst, r.rtm_dst_len);
+	len += strlen(buf) + 1;
+	printf("%s ", buf);
+
+	switch (r.rtm_type) {
+	case RTN_BLACKHOLE:
+		printf("blackhole\n");
+		return;
+	case RTN_UNREACHABLE:
+		printf("unreach(reject)\n");
+		return;
+	case RTN_PROHIBIT:
+		printf("prohibit(reject)\n");
+		return;
+	}
+
+	if (r.rta_multipath != NULL) {
+		bool first = true;
+
+		memset(buf, ' ', sizeof(buf));
+		buf[len] = '\0';
+
+		for (int i = 0; i < r.rta_multipath->num_nhops; i++) {
+			struct rta_mpath_nh *nh = &r.rta_multipath->nhops[i];
+
+			if (!first)
+				printf("%s", buf);
+			print_nlmsg_route_nhop(h, &r, nh, first);
+			first = false;
+		}
+	} else {
+		struct rta_mpath_nh nh = {
+			.gw = r.rta_gw,
+			.ifindex = r.rta_oif,
+			.rtax_mtu = r.rtax_mtu,
+		};
+
+		print_nlmsg_route_nhop(h, &r, &nh, true);
+	}
+}
+
+static const char *operstate[] = {
+	"UNKNOWN",	/* 0, IF_OPER_UNKNOWN */
+	"NOTPRESENT",	/* 1, IF_OPER_NOTPRESENT */
+	"DOWN",		/* 2, IF_OPER_DOWN */
+	"LLDOWN",	/* 3, IF_OPER_LOWERLAYERDOWN */
+	"TESTING",	/* 4, IF_OPER_TESTING */
+	"DORMANT",	/* 5, IF_OPER_DORMANT */
+	"UP",		/* 6, IF_OPER_UP */
+};
+
+static void
+print_nlmsg_link(struct nl_helper *h, struct nlmsghdr *hdr)
+{
+	struct snl_parsed_link l = {};
+	struct snl_state *ss = &h->ss_cmd;
+
+	if (!snl_parse_nlmsg(ss, hdr, &snl_rtm_link_parser, &l))
+		return;
+
+	// 20:19:41.333 add iface#3 vtnet0 admin UP oper UP mtu 1500 table inet.0
+	const char *cmd = get_action_name(hdr, RTM_NEWLINK);
+	print_line_prefix(cmd, "iface");
+
+	printf("iface#%u %s ", l.ifi_index, l.ifla_ifname);
+	printf("admin %s ", (l.ifi_flags & IFF_UP) ? "UP" : "DOWN");
+	if (l.ifla_operstate < NL_ARRAY_LEN(operstate))
+		printf("oper %s ", operstate[l.ifla_operstate]);
+	if (l.ifla_mtu > 0)
+		printf("mtu %u ", l.ifla_mtu);
+
+	printf("\n");
+}
+
+static void
+print_nlmsg_addr(struct nl_helper *h, struct nlmsghdr *hdr)
+{
+	struct snl_parsed_addr attrs = {};
+	struct snl_state *ss = &h->ss_cmd;
+
+	if (!snl_parse_nlmsg(ss, hdr, &snl_rtm_addr_parser, &attrs))
+		return;
+
+	// add addr 192.168.1.1/24 iface vtnet0
+	const char *cmd = get_action_name(hdr, RTM_NEWADDR);
+	print_line_prefix(cmd, "addr");
+
+	char buf[128];
+	struct sockaddr *addr = attrs.ifa_local ? attrs.ifa_local : attrs.ifa_address;
+	print_prefix(h, buf, sizeof(buf), addr, attrs.ifa_prefixlen);
+	printf("%s ", buf);
+
+	struct snl_parsed_link_simple link = {};
+	get_ifdata(h, attrs.ifa_index, &link);
+
+	if (link.ifi_flags & IFF_POINTOPOINT) {
+		char buf[64];
+		print_prefix(h, buf, sizeof(buf), attrs.ifa_address, -1);
+		printf("-> %s ", buf);
+	}
+
+	printf("iface %s ", link.ifla_ifname);
+
+	printf("\n");
+}
+
+static const char *nudstate[] = {
+	"INCOMPLETE",		/* 0x01(0) */
+	"REACHABLE",		/* 0x02(1) */
+	"STALE",		/* 0x04(2) */
+	"DELAY",		/* 0x08(3) */
+	"PROBE",		/* 0x10(4) */
+	"FAILED",		/* 0x20(5) */
+};
+
+#define	NUD_INCOMPLETE		0x01	/* No lladdr, address resolution in progress */
+#define	NUD_REACHABLE		0x02	/* reachable & recently resolved */
+#define	NUD_STALE		0x04	/* has lladdr but it's stale */
+#define	NUD_DELAY		0x08	/* has lladdr, is stale, probes delayed */
+#define	NUD_PROBE		0x10	/* has lladdr, is stale, probes sent */
+#define	NUD_FAILED		0x20	/* unused */
+
+
+static void
+print_nlmsg_neigh(struct nl_helper *h, struct nlmsghdr *hdr)
+{
+	struct snl_parsed_neigh attrs = {};
+	struct snl_state *ss = &h->ss_cmd;
+
+	if (!snl_parse_nlmsg(ss, hdr, &snl_rtm_neigh_parser, &attrs))
+		return;
+
+	// add addr 192.168.1.1 state %s lladdr %s iface vtnet0
+	const char *cmd = get_action_name(hdr, RTM_NEWNEIGH);
+	print_line_prefix(cmd, "neigh");
+
+	char buf[128];
+	print_prefix(h, buf, sizeof(buf), attrs.nda_dst, -1);
+	printf("%s ", buf);
+
+	struct snl_parsed_link_simple link = {};
+	get_ifdata(h, attrs.nda_ifindex, &link);
+
+	for (unsigned int i = 0; i < NL_ARRAY_LEN(nudstate); i++) {
+		if ((1 << i) & attrs.ndm_state) {
+			printf("state %s ", nudstate[i]);
+			break;
+		}
+	}
+
+	if (attrs.nda_lladdr != NULL) {
+		int if_type = link.ifi_type;
+
+		if ((if_type == IFT_ETHER || if_type == IFT_L2VLAN || if_type == IFT_BRIDGE) &&
+		    NLA_DATA_LEN(attrs.nda_lladdr) == ETHER_ADDR_LEN) {
+			struct ether_addr *ll;
+
+			ll = (struct ether_addr *)NLA_DATA(attrs.nda_lladdr);
+			printf("lladdr %s ", ether_ntoa(ll));
+		} else {
+			struct sockaddr_dl sdl = {
+				.sdl_len = sizeof(sdl),
+				.sdl_family = AF_LINK,
+				.sdl_index = attrs.nda_ifindex,
+				.sdl_type = if_type,
+				.sdl_alen = NLA_DATA_LEN(attrs.nda_lladdr),
+			};
+			if (sdl.sdl_alen < sizeof(sdl.sdl_data)) {
+				void *ll = NLA_DATA(attrs.nda_lladdr);
+
+				memcpy(sdl.sdl_data, ll, sdl.sdl_alen);
+				printf("lladdr %s ", link_ntoa(&sdl));
+			}
+		}
+	}
+
+	if (link.ifla_ifname != NULL)
+		printf("iface %s ", link.ifla_ifname);
+	printf("\n");
+}
+
+static void
+print_nlmsg_generic(struct nl_helper *h, struct nlmsghdr *hdr)
+{
+}
+
+static void
+print_nlmsg(struct nl_helper *h, struct nlmsghdr *hdr)
+{
+	switch (hdr->nlmsg_type) {
+	case RTM_NEWLINK:
+	case RTM_DELLINK:
+		print_nlmsg_link(h, hdr);
+		break;
+	case RTM_NEWADDR:
+	case RTM_DELADDR:
+		print_nlmsg_addr(h, hdr);
+		break;
+	case RTM_NEWROUTE:
+	case RTM_DELROUTE:
+		print_nlmsg_route(h, hdr);
+		break;
+	case RTM_NEWNEIGH:
+	case RTM_DELNEIGH:
+		print_nlmsg_neigh(h, hdr);
+		break;
+	default:
+		print_nlmsg_generic(h, hdr);
+	}
+
+	snl_clear_lb(&h->ss_cmd);
+}
+
+void
+monitor_nl(int fib)
+{
+	struct snl_state ss_event = {};
+	struct nl_helper h;
+
+	if (!snl_init(&ss_event, NETLINK_ROUTE))
+		err(1, "unable to open netlink socket");
+	nl_helper_init(&h);
+
+	int groups[] = {
+		RTNLGRP_LINK,
+		RTNLGRP_NEIGH,
+		RTNLGRP_NEXTHOP,
+#ifdef INET
+		RTNLGRP_IPV4_IFADDR,
+		RTNLGRP_IPV4_ROUTE,
+#endif
+#ifdef INET6
+		RTNLGRP_IPV6_IFADDR,
+		RTNLGRP_IPV6_ROUTE,
+#endif
+	};
+
+	for (unsigned int i = 0; i < NL_ARRAY_LEN(groups); i++) {
+		int error;
+		int optval = groups[i];
+		socklen_t optlen = sizeof(optval);
+		error = setsockopt(ss_event.fd, SOL_NETLINK,
+		    NETLINK_ADD_MEMBERSHIP, &optval, optlen);
+		if (error != 0)
+			warn("Unable to subscribe to group %d", optval);
+	}
+
+	struct nlmsghdr *hdr;
+	while ((hdr = snl_read_message(&ss_event)) != NULL)
+	{
+		// printf("-- MSG type %d--\n", hdr->nlmsg_type);
+		print_nlmsg(&h, hdr);
+		snl_clear_lb(&h.ss_cmd);
+		snl_clear_lb(&ss_event);
+	}
+
+	snl_free(&ss_event);
+	nl_helper_free(&h);
+	exit(0);
+}
+
+static void
+print_flushed_route(struct snl_parsed_route *r, struct sockaddr *gw)
+{
+	struct sockaddr *sa = r->rta_dst;
+
+	printf("%-20.20s ", r->rta_rtflags & RTF_HOST ?
+	    routename(sa) : netname(sa));
+	sa = gw;
+	printf("%-20.20s ", routename(sa));
+	if (r->rta_table >= 0)
+		printf("-fib %-3d ", r->rta_table);
+	printf("done\n");
+}
+
+static int
+flushroute_one(struct nl_helper *h, struct snl_parsed_route *r)
+{
+	struct snl_state *ss = &h->ss_cmd;
+	struct snl_errmsg_data e = {};
+	struct snl_writer nw;
+
+	snl_init_writer(ss, &nw);
+
+	struct nlmsghdr *hdr = snl_create_msg_request(&nw, NL_RTM_DELROUTE);
+	struct rtmsg *rtm = snl_reserve_msg_object(&nw, struct rtmsg);
+	rtm->rtm_family = r->rtm_family;
+	rtm->rtm_dst_len = r->rtm_dst_len;
+
+	snl_add_msg_attr_u32(&nw, RTA_TABLE, r->rta_table);
+	snl_add_msg_attr_ip(&nw, RTA_DST, r->rta_dst);
+
+	if (!snl_finalize_msg(&nw) || !snl_send_message(ss, hdr))
+		return (ENOMEM);
+
+	if (!snl_read_reply_code(ss, hdr->nlmsg_seq, &e)) {
+		return (e.error);
+		if (e.error == EPERM)
+			errc(1, e.error, "RTM_DELROUTE failed:");
+		else
+			warnc(e.error, "RTM_DELROUTE failed:");
+		return (true);
+	};
+
+	if (verbose)
+		print_nlmsg(h, hdr);
+	else {
+		if (r->rta_multipath != NULL) {
+			for (int i = 0; i < r->rta_multipath->num_nhops; i++) {
+				struct rta_mpath_nh *nh = &r->rta_multipath->nhops[i];
+
+				print_flushed_route(r, nh->gw);
+			}
+
+		} else
+			print_flushed_route(r, r->rta_gw);
+	}
+
+	return (0);
+}
+
+int
+flushroutes_fib_nl(int fib, int af)
+{
+	struct snl_state ss = {};
+	struct snl_writer nw;
+	struct nl_helper h = {};
+
+	if (!snl_init(&ss, NETLINK_ROUTE))
+		err(1, "unable to open netlink socket");
+	snl_init_writer(&ss, &nw);
+
+	struct nlmsghdr *hdr = snl_create_msg_request(&nw, NL_RTM_GETROUTE);
+	hdr->nlmsg_flags |= NLM_F_DUMP;
+	struct rtmsg *rtm = snl_reserve_msg_object(&nw, struct rtmsg);
+	rtm->rtm_family = af;
+	snl_add_msg_attr_u32(&nw, RTA_TABLE, fib);
+
+	if (!snl_finalize_msg(&nw) || !snl_send_message(&ss, hdr)) {
+		snl_free(&ss);
+		return (EINVAL);
+	}
+
+	struct snl_errmsg_data e = {};
+	uint32_t nlm_seq = hdr->nlmsg_seq;
+
+	nl_helper_init(&h);
+	
+	while ((hdr = snl_read_reply_multi(&ss, nlm_seq, &e)) != NULL) {
+		struct snl_parsed_route r = { .rtax_weight = RT_DEFAULT_WEIGHT };
+		int error;
+
+		if (!snl_parse_nlmsg(&ss, hdr, &snl_rtm_route_parser, &r))
+			continue;
+		if (verbose)
+			print_nlmsg(&h, hdr);
+		if (r.rta_table != (uint32_t)fib || r.rtm_family != af)
+			continue;
+		if ((r.rta_rtflags & RTF_GATEWAY) == 0)
+			continue;
+		if (debugonly)
+			continue;
+
+		if ((error = flushroute_one(&h, &r)) != 0) {
+			if (error == EPERM)
+				errc(1, error, "RTM_DELROUTE failed:");
+			else
+				warnc(error, "RTM_DELROUTE failed:");
+		}
+		snl_clear_lb(&h.ss_cmd);
+	}
+
+	snl_free(&ss);
+	nl_helper_free(&h);
+
+	return (e.error);
+}
+