cuse(3): Optimise small reads and writes.

When doing small reads and writes use an intermediate buffer to store the
data to save locking the remote process to access data.

Reviewed by:	imp @
Differential Revision:	https://reviews.freebsd.org/D36633
MFC after:	1 week
Sponsored by:	NVIDIA Networking

1 files changed, 130 insertions, 6 deletions

diff --git a/sys/fs/cuse/cuse.c b/sys/fs/cuse/cuse.c
index a870df1b07dc..41f8bdbe2bfe 100644
--- a/sys/fs/cuse/cuse.c
+++ b/sys/fs/cuse/cuse.c
@@ -1,6 +1,6 @@
 /* $FreeBSD$ */
 /*-
- * Copyright (c) 2010-2022 Hans Petter Selasky. All rights reserved.
+ * Copyright (c) 2010-2022 Hans Petter Selasky
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -64,6 +64,10 @@
 #include <fs/cuse/cuse_defs.h>
 #include <fs/cuse/cuse_ioctl.h>
 
+/* set this define to zero to disable this feature */
+#define	CUSE_COPY_BUFFER_MAX \
+	CUSE_BUFFER_MAX
+
 #define	CUSE_ALLOC_PAGES_MAX \
 	(CUSE_ALLOC_BYTES_MAX / PAGE_SIZE)
 
@@ -154,6 +158,12 @@ struct cuse_client {
 	struct cuse_server *server;
 	struct cuse_server_dev *server_dev;
 
+	uintptr_t read_base;
+	uintptr_t write_base;
+	int read_length;
+	int write_length;
+	uint8_t	read_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4);
+	uint8_t	write_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4);
 	uint8_t	ioctl_buffer[CUSE_BUFFER_MAX] __aligned(4);
 
 	int	fflags;			/* file flags */
@@ -265,6 +275,12 @@ cuse_server_unlock(struct cuse_server *pcs)
 	mtx_unlock(&pcs->mtx);
 }
 
+static bool
+cuse_server_is_locked(struct cuse_server *pcs)
+{
+	return (mtx_owned(&pcs->mtx));
+}
+
 static void
 cuse_cmd_lock(struct cuse_client_command *pccmd)
 {
@@ -968,6 +984,48 @@ cuse_server_data_copy_locked(struct cuse_server *pcs,
 }
 
 static int
+cuse_server_data_copy_optimized_locked(struct cuse_server *pcs,
+    struct cuse_client_command *pccmd,
+    struct cuse_data_chunk *pchk, bool isread)
+{
+	uintptr_t offset;
+	int error;
+
+	/*
+	 * Check if data is stored locally to avoid accessing
+	 * other process's data space:
+	 */
+	if (isread) {
+		offset = pchk->peer_ptr - pccmd->client->write_base;
+
+		if (offset < (uintptr_t)pccmd->client->write_length &&
+		    pchk->length <= (unsigned long)pccmd->client->write_length &&
+		    offset + pchk->length <= (uintptr_t)pccmd->client->write_length) {
+			cuse_server_unlock(pcs);
+			error = copyout(pccmd->client->write_buffer + offset,
+			    (void *)pchk->local_ptr, pchk->length);
+			goto done;
+		}
+	} else {
+		offset = pchk->peer_ptr - pccmd->client->read_base;
+
+		if (offset < (uintptr_t)pccmd->client->read_length &&
+		    pchk->length <= (unsigned long)pccmd->client->read_length &&
+		    offset + pchk->length <= (uintptr_t)pccmd->client->read_length) {
+			cuse_server_unlock(pcs);
+			error = copyin((void *)pchk->local_ptr,
+			    pccmd->client->read_buffer + offset, pchk->length);
+			goto done;
+		}
+	}
+
+	/* use process to process copy function */
+	error = cuse_server_data_copy_locked(pcs, pccmd, pchk, isread);
+done:
+	return (error);
+}
+
+static int
 cuse_alloc_unit_by_id_locked(struct cuse_server *pcs, int id)
 {
 	int n;
@@ -1294,14 +1352,22 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
 			error = ENXIO;	/* invalid request */
 		} else if (pchk->peer_ptr < CUSE_BUF_MIN_PTR) {
 			error = EFAULT;	/* NULL pointer */
+		} else if (pchk->length == 0) {
+			/* NOP */
 		} else if (pchk->peer_ptr < CUSE_BUF_MAX_PTR) {
 			error = cuse_server_ioctl_copy_locked(pcs, pccmd,
 			    pchk, cmd == CUSE_IOCTL_READ_DATA);
 		} else {
-			error = cuse_server_data_copy_locked(pcs, pccmd,
-			    pchk, cmd == CUSE_IOCTL_READ_DATA);
+			error = cuse_server_data_copy_optimized_locked(
+			    pcs, pccmd, pchk, cmd == CUSE_IOCTL_READ_DATA);
 		}
-		cuse_server_unlock(pcs);
+
+		/*
+		 * Sometimes the functions above drop the server lock
+		 * early as an optimization:
+		 */
+		if (cuse_server_is_locked(pcs))
+			cuse_server_unlock(pcs);
 		break;
 
 	case CUSE_IOCTL_SELWAKEUP:
@@ -1581,6 +1647,7 @@ cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag)
 	struct cuse_client *pcc;
 	struct cuse_server *pcs;
 	int error;
+	int temp;
 	int len;
 
 	error = cuse_client_get(&pcc);
@@ -1605,13 +1672,41 @@ cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag)
 		len = uio->uio_iov->iov_len;
 
 		cuse_server_lock(pcs);
+		if (len <= CUSE_COPY_BUFFER_MAX) {
+			/* set read buffer region for small reads */
+			pcc->read_base = (uintptr_t)uio->uio_iov->iov_base;
+			pcc->read_length = len;
+		}
 		cuse_client_send_command_locked(pccmd,
 		    (uintptr_t)uio->uio_iov->iov_base,
 		    (unsigned long)(unsigned int)len, pcc->fflags, ioflag);
 
 		error = cuse_client_receive_command_locked(pccmd, 0, 0);
+		/*
+		 * After finishing reading data, disable the read
+		 * region for the cuse_server_data_copy_optimized_locked()
+		 * function:
+		 */
+		pcc->read_base = 0;
+		pcc->read_length = 0;
 		cuse_server_unlock(pcs);
 
+		/*
+		 * The return value indicates the read length, when
+		 * not negative. Range check it just in case to avoid
+		 * passing invalid length values to uiomove().
+		 */
+		if (error > len) {
+			error = ERANGE;
+			break;
+		} else if (error > 0 && len <= CUSE_COPY_BUFFER_MAX) {
+			temp = copyout(pcc->read_buffer,
+			    uio->uio_iov->iov_base, error);
+			if (temp != 0) {
+				error = temp;
+				break;
+			}
+		}
 		if (error < 0) {
 			error = cuse_convert_error(error);
 			break;
@@ -1664,15 +1759,43 @@ cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag)
 		}
 		len = uio->uio_iov->iov_len;
 
+		if (len <= CUSE_COPY_BUFFER_MAX) {
+			error = copyin(uio->uio_iov->iov_base,
+			    pcc->write_buffer, len);
+			if (error != 0)
+				break;
+		}
+
 		cuse_server_lock(pcs);
+		if (len <= CUSE_COPY_BUFFER_MAX) {
+			/* set write buffer region for small writes */
+			pcc->write_base = (uintptr_t)uio->uio_iov->iov_base;
+			pcc->write_length = len;
+		}
 		cuse_client_send_command_locked(pccmd,
 		    (uintptr_t)uio->uio_iov->iov_base,
 		    (unsigned long)(unsigned int)len, pcc->fflags, ioflag);
 
 		error = cuse_client_receive_command_locked(pccmd, 0, 0);
+
+		/*
+		 * After finishing writing data, disable the write
+		 * region for the cuse_server_data_copy_optimized_locked()
+		 * function:
+		 */
+		pcc->write_base = 0;
+		pcc->write_length = 0;
 		cuse_server_unlock(pcs);
 
-		if (error < 0) {
+		/*
+		 * The return value indicates the write length, when
+		 * not negative. Range check it just in case to avoid
+		 * passing invalid length values to uiomove().
+		 */
+		if (error > len) {
+			error = ERANGE;
+			break;
+		} else if (error < 0) {
 			error = cuse_convert_error(error);
 			break;
 		} else if (error == len) {
@@ -1686,7 +1809,8 @@ cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag)
 	}
 	cuse_cmd_unlock(pccmd);
 
-	uio->uio_segflg = UIO_USERSPACE;/* restore segment flag */
+	/* restore segment flag */
+	uio->uio_segflg = UIO_USERSPACE;
 
 	if (error == EWOULDBLOCK)
 		cuse_client_kqfilter_poll(dev, pcc);