krpc: Display stats of TLS usage

This patch adds some sysctls:
kern.rpc.unenc.tx_msgcnt
kern.rpc.unenc.tx_msgbytes
kern.rpc.unenc.rx_msgcnt
kern.rpc.unenc.rx_msgbytes
kern.rpc.tls.tx_msgcnt
kern.rpc.tls.tx_msgbytes
kern.rpc.tls.rx_msgcnt
kern.rpc.tls.rx_msgbytes
kern.rpc.tls.handshake_success
kern.rpc.tls.handshake_failed
kern.rpc.tls.alerts
which allow a NFS server sysadmin to determine how much
NFS-over-TLS is being used.  A large number of failed
handshakes might also indicate an NFS confirguration
problem.

This patch moves the definition of "kern.rpc" from the
kgssapi module to the krpc module.  As such, both modules
need to be rebuilt from sources.  Since __FreeBSD_version
was bumped yesterday, I will not bump it again.

(cherry picked from commit 1a878807006cc10a5698cbca9e24a38b3412d7ed)

4 files changed, 104 insertions, 5 deletions

diff --git a/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c b/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
index 90aa9e0d7d4f..89526544639a 100644
--- a/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
+++ b/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
@@ -174,8 +174,7 @@ struct svc_rpc_gss_cookedcred {
 u_int svc_rpc_gss_client_max = CLIENT_MAX;
 u_int svc_rpc_gss_client_hash_size = CLIENT_HASH_SIZE;
 
-SYSCTL_NODE(_kern, OID_AUTO, rpc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
-    "RPC");
+SYSCTL_DECL(_kern_rpc);
 SYSCTL_NODE(_kern_rpc, OID_AUTO, gss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
     "GSS");
 
diff --git a/sys/rpc/rpcsec_tls.h b/sys/rpc/rpcsec_tls.h
index 1445c5c35f19..e3eed64863a1 100644
--- a/sys/rpc/rpcsec_tls.h
+++ b/sys/rpc/rpcsec_tls.h
@@ -86,10 +86,14 @@ bool		rpctls_getinfo(u_int *maxlen, bool rpctlscd_run,
 
 /* Macros for VIMAGE. */
 /* Just define the KRPC_VNETxxx() macros as VNETxxx() macros. */
+#define	KRPC_VNET_NAME(n)		VNET_NAME(n)
+#define	KRPC_VNET_DECLARE(t, n)		VNET_DECLARE(t, n)
 #define	KRPC_VNET_DEFINE(t, n)		VNET_DEFINE(t, n)
 #define	KRPC_VNET_DEFINE_STATIC(t, n)	VNET_DEFINE_STATIC(t, n)
 #define	KRPC_VNET(n)			VNET(n)
 
+#define	CTLFLAG_KRPC_VNET		CTLFLAG_VNET
+
 #define	KRPC_CURVNET_SET(n)		CURVNET_SET(n)
 #define	KRPC_CURVNET_SET_QUIET(n)	CURVNET_SET_QUIET(n)
 #define	KRPC_CURVNET_RESTORE()		CURVNET_RESTORE()
diff --git a/sys/rpc/rpcsec_tls/rpctls_impl.c b/sys/rpc/rpcsec_tls/rpctls_impl.c
index c0e269e55932..64111eed62c0 100644
--- a/sys/rpc/rpcsec_tls/rpctls_impl.c
+++ b/sys/rpc/rpcsec_tls/rpctls_impl.c
@@ -78,6 +78,9 @@ static CLIENT		*rpctls_connect_cl = NULL;
 static struct mtx	rpctls_server_lock;
 static struct opaque_auth rpctls_null_verf;
 
+KRPC_VNET_DECLARE(uint64_t, svc_vc_tls_handshake_success);
+KRPC_VNET_DECLARE(uint64_t, svc_vc_tls_handshake_failed);
+
 KRPC_VNET_DEFINE_STATIC(CLIENT **, rpctls_server_handle);
 KRPC_VNET_DEFINE_STATIC(struct socket *, rpctls_server_so) = NULL;
 KRPC_VNET_DEFINE_STATIC(SVCXPRT *, rpctls_server_xprt) = NULL;
@@ -748,25 +751,33 @@ _svcauth_rpcsec_tls(struct svc_req *rqst, struct rpc_msg *msg)
 	u_int maxlen;
 #endif
 	
+	KRPC_CURVNET_SET_QUIET(KRPC_TD_TO_VNET(curthread));
+	KRPC_VNET(svc_vc_tls_handshake_failed)++;
 	/* Initialize reply. */
 	rqst->rq_verf = rpctls_null_verf;
 
 	/* Check client credentials. */
 	if (rqst->rq_cred.oa_length != 0 ||
 	    msg->rm_call.cb_verf.oa_length != 0 ||
-	    msg->rm_call.cb_verf.oa_flavor != AUTH_NULL)
+	    msg->rm_call.cb_verf.oa_flavor != AUTH_NULL) {
+		KRPC_CURVNET_RESTORE();
 		return (AUTH_BADCRED);
+	}
 	
-	if (rqst->rq_proc != NULLPROC)
+	if (rqst->rq_proc != NULLPROC) {
+		KRPC_CURVNET_RESTORE();
 		return (AUTH_REJECTEDCRED);
+	}
 
 	call_stat = FALSE;
 #ifdef KERN_TLS
 	if (rpctls_getinfo(&maxlen, false, true))
 		call_stat = TRUE;
 #endif
-	if (!call_stat)
+	if (!call_stat) {
+		KRPC_CURVNET_RESTORE();
 		return (AUTH_REJECTEDCRED);
+	}
 
 	/*
 	 * Disable reception for the krpc so that the TLS handshake can
@@ -787,6 +798,7 @@ _svcauth_rpcsec_tls(struct svc_req *rqst, struct rpc_msg *msg)
 		xprt->xp_dontrcv = FALSE;
 		sx_xunlock(&xprt->xp_lock);
 		xprt_active(xprt);	/* Harmless if already active. */
+		KRPC_CURVNET_RESTORE();
 		return (AUTH_REJECTEDCRED);
 	}
 
@@ -809,9 +821,12 @@ _svcauth_rpcsec_tls(struct svc_req *rqst, struct rpc_msg *msg)
 			xprt->xp_uid = uid;
 			xprt->xp_gidp = gidp;
 		}
+		KRPC_VNET(svc_vc_tls_handshake_failed)--;
+		KRPC_VNET(svc_vc_tls_handshake_success)++;
 	}
 	sx_xunlock(&xprt->xp_lock);
 	xprt_active(xprt);		/* Harmless if already active. */
+	KRPC_CURVNET_RESTORE();
 
 	return (RPCSEC_GSS_NODISPATCH);
 }
diff --git a/sys/rpc/svc_vc.c b/sys/rpc/svc_vc.c
index e98e28de6982..69960079883b 100644
--- a/sys/rpc/svc_vc.c
+++ b/sys/rpc/svc_vc.c
@@ -74,6 +74,66 @@ static char *sccsid = "@(#)svc_tcp.c	2.2 88/08/01 4.0 RPCSRC";
 
 #include <security/mac/mac_framework.h>
 
+SYSCTL_NODE(_kern, OID_AUTO, rpc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+    "RPC");
+SYSCTL_NODE(_kern_rpc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+    "TLS");
+SYSCTL_NODE(_kern_rpc, OID_AUTO, unenc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+    "unencrypted");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_rx_msgbytes) = 0;
+SYSCTL_U64(_kern_rpc_unenc, OID_AUTO, rx_msgbytes, CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_rx_msgbytes), 0, "Count of non-TLS rx bytes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_rx_msgcnt) = 0;
+SYSCTL_U64(_kern_rpc_unenc, OID_AUTO, rx_msgcnt, CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_rx_msgcnt), 0, "Count of non-TLS rx messages");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tx_msgbytes) = 0;
+SYSCTL_U64(_kern_rpc_unenc, OID_AUTO, tx_msgbytes, CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_tx_msgbytes), 0, "Count of non-TLS tx bytes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tx_msgcnt) = 0;
+SYSCTL_U64(_kern_rpc_unenc, OID_AUTO, tx_msgcnt, CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_tx_msgcnt), 0, "Count of non-TLS tx messages");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_alerts) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, alerts,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_alerts), 0,
+    "Count of TLS alert messages");
+
+KRPC_VNET_DEFINE(uint64_t, svc_vc_tls_handshake_failed) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, handshake_failed,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_tls_handshake_failed), 0,
+    "Count of TLS failed handshakes");
+
+KRPC_VNET_DEFINE(uint64_t, svc_vc_tls_handshake_success) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, handshake_success,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_tls_handshake_success), 0,
+    "Count of TLS successful handshakes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_rx_msgbytes) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, rx_msgbytes,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_rx_msgbytes), 0,
+    "Count of TLS rx bytes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_rx_msgcnt) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, rx_msgcnt,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_rx_msgcnt), 0,
+    "Count of TLS rx messages");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_tx_msgbytes) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, tx_msgbytes,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_tx_msgbytes), 0,
+    "Count of TLS tx bytes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_tx_msgcnt) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, tx_msgcnt,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_tx_msgcnt), 0,
+    "Count of TLS tx messages");
+
 static bool_t svc_vc_rendezvous_recv(SVCXPRT *, struct rpc_msg *,
     struct sockaddr **, struct mbuf **);
 static enum xprt_stat svc_vc_rendezvous_stat(SVCXPRT *);
@@ -808,8 +868,11 @@ tryagain:
 		 * This record needs to be handled in userland
 		 * via an SSL_read() call, so do an upcall to the daemon.
 		 */
+		KRPC_CURVNET_SET(so->so_vnet);
 		if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0 &&
 		    error == ENXIO) {
+			KRPC_VNET(svc_vc_tls_alerts)++;
+			KRPC_CURVNET_RESTORE();
 			/* Disable reception. */
 			xprt->xp_dontrcv = TRUE;
 			sx_xunlock(&xprt->xp_lock);
@@ -832,6 +895,7 @@ tryagain:
 		}
 
 		if (error) {
+			KRPC_CURVNET_RESTORE();
 			SOCKBUF_LOCK(&so->so_rcv);
 			if (xprt->xp_upcallset) {
 				xprt->xp_upcallset = 0;
@@ -845,6 +909,7 @@ tryagain:
 		}
 
 		if (!m) {
+			KRPC_CURVNET_RESTORE();
 			/*
 			 * EOF - the other end has closed the socket.
 			 */
@@ -870,11 +935,20 @@ tryagain:
 					m_freem(m);
 					m_free(ctrl);
 					rcvflag = MSG_DONTWAIT | MSG_TLSAPPDATA;
+					KRPC_CURVNET_RESTORE();
 					goto tryagain;
 				}
+				KRPC_VNET(svc_vc_tls_rx_msgcnt)++;
+				KRPC_VNET(svc_vc_tls_rx_msgbytes) +=
+				    1000000000 - uio.uio_resid;
 			}
 			m_free(ctrl);
+		} else {
+			KRPC_VNET(svc_vc_rx_msgcnt)++;
+			KRPC_VNET(svc_vc_rx_msgbytes) += 1000000000 -
+			    uio.uio_resid;
 		}
+		KRPC_CURVNET_RESTORE();
 
 		if (cd->mpending)
 			m_last(cd->mpending)->m_next = m;
@@ -963,6 +1037,7 @@ svc_vc_reply(SVCXPRT *xprt, struct rpc_msg *msg,
 			htonl(0x80000000 | (len - sizeof(uint32_t)));
 
 		/* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */
+		KRPC_CURVNET_SET(xprt->xp_socket->so_vnet);
 		if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) {
 			/*
 			 * Copy the mbuf chain to a chain of
@@ -974,7 +1049,13 @@ svc_vc_reply(SVCXPRT *xprt, struct rpc_msg *msg,
 				maxextsiz = min(maxextsiz, maxlen);
 #endif
 			mrep = _rpc_copym_into_ext_pgs(mrep, maxextsiz);
+			KRPC_VNET(svc_vc_tls_tx_msgcnt)++;
+			KRPC_VNET(svc_vc_tls_tx_msgbytes) += len;
+		} else {
+			KRPC_VNET(svc_vc_tx_msgcnt)++;
+			KRPC_VNET(svc_vc_tx_msgbytes) += len;
 		}
+		KRPC_CURVNET_RESTORE();
 		atomic_add_32(&xprt->xp_snd_cnt, len);
 		/*
 		 * sosend consumes mreq.