rpcsec_tls: Avoid a socket reference underflow in rpctls_server()

The upcall_sockets tree owns a ref on any resident socket.  When a
socket is removed after a TLS handshake failure, rpctls_rpc_failed()
thus calls soclose().

rpctls_server() does not acquire an extra ref to compensate for this.
So, if the upcall fails, e.g., because rpc.tlsservd is not running,
we'll call soclose() to drop the reference, but this effectively
releases the xprt layer's reference.

Fix the problem by explicitly acquiring a socket reference when adding
a socket to the upcall tree.

PR:		289734
Reviewed by:	rmacklem, glebius
MFC after:	1 week
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D57555

1 files changed, 20 insertions, 3 deletions

diff --git a/sys/rpc/rpcsec_tls/rpctls_impl.c b/sys/rpc/rpcsec_tls/rpctls_impl.c
index e1b80217263f..621f6bcac3e4 100644
--- a/sys/rpc/rpcsec_tls/rpctls_impl.c
+++ b/sys/rpc/rpcsec_tls/rpctls_impl.c
@@ -190,7 +190,6 @@ sys_rpctls_syscall(struct thread *td, struct rpctls_syscall_args *uap)
 		CURVNET_RESTORE();
 		return (error);
 	}
-	soref(ups.so);
 	if (ups.server) {
 		/*
 		 * Once this file descriptor is associated
@@ -277,6 +276,7 @@ rpctls_connect(CLIENT *newclient, char *certname, struct socket *so,
 	if (stat != RPC_SUCCESS)
 		return (RPC_SYSTEMERROR);
 
+	soref(so);
 	mtx_lock(&rpctls_lock);
 	RB_INSERT(upsock_t, &upcall_sockets, &ups);
 	mtx_unlock(&rpctls_lock);
@@ -294,9 +294,16 @@ rpctls_connect(CLIENT *newclient, char *certname, struct socket *so,
 	stat = rpctlscd_connect_2(&arg, &res, rpctls_connect_handle);
 	if (stat == RPC_SUCCESS)
 		*reterr = res.reterr;
-	else
+	else {
 		rpctls_rpc_failed(&ups, so);
 
+		/*
+		 * The socket was closed, make sure the krpc code doesn't close
+		 * it a second time.
+		 */
+		CLNT_CONTROL(newclient, CLSET_TLS, &(int){RPCTLS_INHANDSHAKE});
+	}
+
 	/* Unblock reception. */
 	CLNT_CONTROL(newclient, CLSET_BLOCKRCV, &(int){0});
 
@@ -388,6 +395,7 @@ rpctls_server(SVCXPRT *xprt, uint32_t *flags, uid_t *uid, int *ngrps,
 	uint32_t *gidv;
 	int i;
 
+	soref(xprt->xp_socket);
 	mtx_lock(&rpctls_lock);
 	RB_INSERT(upsock_t, &upcall_sockets, &ups);
 	mtx_unlock(&rpctls_lock);
@@ -407,9 +415,18 @@ rpctls_server(SVCXPRT *xprt, uint32_t *flags, uid_t *uid, int *ngrps,
 			for (i = 0; i < *ngrps; i++)
 				*gidp++ = *gidv++;
 		}
-	} else
+	} else {
 		rpctls_rpc_failed(&ups, xprt->xp_socket);
 
+		/*
+		 * The socket was closed, make sure the krpc code doesn't close
+		 * it a second time.
+		 */
+		sx_xlock(&ups.xp->xp_lock);
+		ups.xp->xp_tls = RPCTLS_FLAGS_HANDSHFAIL;
+		sx_xunlock(&ups.xp->xp_lock);
+	}
+
 	mem_free(res.gid.gid_val, 0);
 
 #ifdef INVARIANTS