Don't insert a flowtable entry if the lle isn't yet valid.

Some of the collisions that are occuring are due to flowtable lookups
that succeed but have an invalid lle - typically because the L2 adjacency
lookup hasn't completed.  This would lead to a follow-up insert which
would then fail (ie, collision) and the code would fall through to doing
a slow-path L2/L3 lookup in the netinet/netinet6 code.

This patch simply aborts storing a new flowtable entry if the lle isn't
yet valid.

Whilst I'm here, add a new pcpu counter for the item so the number of
failures can be tracked separately from generic "collisions."

Reviewed by:	glebius
MFC after:	10 days
Sponsored by:	Netflix, Inc.

3 files changed, 11 insertions, 0 deletions

diff --git a/sys/net/flowtable.c b/sys/net/flowtable.c
index f20a535aecbc..f79f53381f88 100644
--- a/sys/net/flowtable.c
+++ b/sys/net/flowtable.c
@@ -966,6 +966,15 @@ flowtable_lookup_common(struct flowtable *ft, struct sockaddr_storage *ssa,
 		RTFREE(rt);
 		return (NULL);
 	}
+
+	/* Don't insert the entry if the ARP hasn't yet finished resolving */
+	if ((lle->la_flags & LLE_VALID) == 0) {
+		RTFREE(rt);
+		LLE_FREE(lle);
+		FLOWSTAT_INC(ft, ft_fail_lle_invalid);
+		return (NULL);
+	}
+
 	ro->ro_lle = lle;
 
 	if (flowtable_insert(ft, hash, key, fibnum, ro, flags) != 0) {
diff --git a/sys/net/flowtable.h b/sys/net/flowtable.h
index 6609ab279da5..819640ecb9a5 100644
--- a/sys/net/flowtable.h
+++ b/sys/net/flowtable.h
@@ -39,6 +39,7 @@ struct flowtable_stat {
 	uint64_t	ft_frees;
 	uint64_t	ft_hits;
 	uint64_t	ft_lookups;
+	uint64_t	ft_fail_lle_invalid;
 };
 
 #ifdef	_KERNEL
diff --git a/usr.bin/netstat/flowtable.c b/usr.bin/netstat/flowtable.c
index 715d343edaec..2392a1ee55d0 100644
--- a/usr.bin/netstat/flowtable.c
+++ b/usr.bin/netstat/flowtable.c
@@ -55,6 +55,7 @@ print_stats(struct flowtable_stat *stat)
 	p(ft_collisions, "\t%ju collision%s\n");
 	p(ft_free_checks, "\t%ju free check%s\n");
 	p(ft_frees, "\t%ju free%s\n");
+	p(ft_fail_lle_invalid, "\t%ju lookups w/ no resolved ARP%s\n");
 
 #undef	p2
 #undef	p