uipc_shm: Implements fspacectl(2) support

This implements fspacectl(2) support on shared memory objects. The
semantic of SPACECTL_DEALLOC is equivalent to clearing the backing
store and free the pages within the affected range. If the call
succeeds, subsequent reads on the affected range return all zero.

tests/sys/posixshm/posixshm_tests.c is expanded to include a
fspacectl(2) functional test.

Sponsored by:	The FreeBSD Foundation
Reviewed by:	kevans, kib
Differential Revision:	https://reviews.freebsd.org/D31490

1 files changed, 155 insertions, 42 deletions

diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
index 58c9f8cec239..16d1e22a898b 100644
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -131,6 +131,8 @@ static int	shm_dotruncate_locked(struct shmfd *shmfd, off_t length,
     void *rl_cookie);
 static int	shm_copyin_path(struct thread *td, const char *userpath_in,
     char **path_out);
+static int	shm_deallocate(struct shmfd *shmfd, off_t *offset,
+    off_t *length, int flags);
 
 static fo_rdwr_t	shm_read;
 static fo_rdwr_t	shm_write;
@@ -146,6 +148,7 @@ static fo_mmap_t	shm_mmap;
 static fo_get_seals_t	shm_get_seals;
 static fo_add_seals_t	shm_add_seals;
 static fo_fallocate_t	shm_fallocate;
+static fo_fspacectl_t	shm_fspacectl;
 
 /* File descriptor operations. */
 struct fileops shm_ops = {
@@ -166,6 +169,7 @@ struct fileops shm_ops = {
 	.fo_get_seals = shm_get_seals,
 	.fo_add_seals = shm_add_seals,
 	.fo_fallocate = shm_fallocate,
+	.fo_fspacectl = shm_fspacectl,
 	.fo_flags = DFLAG_PASSABLE | DFLAG_SEEKABLE,
 };
 
@@ -627,13 +631,63 @@ out:
 }
 
 static int
+shm_partial_page_invalidate(vm_object_t object, vm_pindex_t idx, int base,
+    int end)
+{
+	vm_page_t m;
+	int rv;
+
+	VM_OBJECT_ASSERT_WLOCKED(object);
+	KASSERT(base >= 0, ("%s: base %d", __func__, base));
+	KASSERT(end - base <= PAGE_SIZE, ("%s: base %d end %d", __func__, base,
+	    end));
+
+retry:
+	m = vm_page_grab(object, idx, VM_ALLOC_NOCREAT);
+	if (m != NULL) {
+		MPASS(vm_page_all_valid(m));
+	} else if (vm_pager_has_page(object, idx, NULL, NULL)) {
+		m = vm_page_alloc(object, idx,
+		    VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL);
+		if (m == NULL)
+			goto retry;
+		vm_object_pip_add(object, 1);
+		VM_OBJECT_WUNLOCK(object);
+		rv = vm_pager_get_pages(object, &m, 1, NULL, NULL);
+		VM_OBJECT_WLOCK(object);
+		vm_object_pip_wakeup(object);
+		if (rv == VM_PAGER_OK) {
+			/*
+			 * Since the page was not resident, and therefore not
+			 * recently accessed, immediately enqueue it for
+			 * asynchronous laundering.  The current operation is
+			 * not regarded as an access.
+			 */
+			vm_page_launder(m);
+		} else {
+			vm_page_free(m);
+			VM_OBJECT_WUNLOCK(object);
+			return (EIO);
+		}
+	}
+	if (m != NULL) {
+		pmap_zero_page_area(m, base, end - base);
+		KASSERT(vm_page_all_valid(m), ("%s: page %p is invalid",
+		    __func__, m));
+		vm_page_set_dirty(m);
+		vm_page_xunbusy(m);
+	}
+
+	return (0);
+}
+
+static int
 shm_dotruncate_locked(struct shmfd *shmfd, off_t length, void *rl_cookie)
 {
 	vm_object_t object;
-	vm_page_t m;
-	vm_pindex_t idx, nobjsize;
+	vm_pindex_t nobjsize;
 	vm_ooffset_t delta;
-	int base, rv;
+	int base, error;
 
 	KASSERT(length >= 0, ("shm_dotruncate: length < 0"));
 	object = shmfd->shm_object;
@@ -660,45 +714,10 @@ shm_dotruncate_locked(struct shmfd *shmfd, off_t length, void *rl_cookie)
 		 */
 		base = length & PAGE_MASK;
 		if (base != 0) {
-			idx = OFF_TO_IDX(length);
-retry:
-			m = vm_page_grab(object, idx, VM_ALLOC_NOCREAT);
-			if (m != NULL) {
-				MPASS(vm_page_all_valid(m));
-			} else if (vm_pager_has_page(object, idx, NULL, NULL)) {
-				m = vm_page_alloc(object, idx,
-				    VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL);
-				if (m == NULL)
-					goto retry;
-				vm_object_pip_add(object, 1);
-				VM_OBJECT_WUNLOCK(object);
-				rv = vm_pager_get_pages(object, &m, 1, NULL,
-				    NULL);
-				VM_OBJECT_WLOCK(object);
-				vm_object_pip_wakeup(object);
-				if (rv == VM_PAGER_OK) {
-					/*
-					 * Since the page was not resident,
-					 * and therefore not recently
-					 * accessed, immediately enqueue it
-					 * for asynchronous laundering.  The
-					 * current operation is not regarded
-					 * as an access.
-					 */
-					vm_page_launder(m);
-				} else {
-					vm_page_free(m);
-					VM_OBJECT_WUNLOCK(object);
-					return (EIO);
-				}
-			}
-			if (m != NULL) {
-				pmap_zero_page_area(m, base, PAGE_SIZE - base);
-				KASSERT(vm_page_all_valid(m),
-				    ("shm_dotruncate: page %p is invalid", m));
-				vm_page_set_dirty(m);
-				vm_page_xunbusy(m);
-			}
+			error = shm_partial_page_invalidate(object,
+			    OFF_TO_IDX(length), base, PAGE_SIZE);
+			if (error)
+				return (error);
 		}
 		delta = IDX_TO_OFF(object->size - nobjsize);
 
@@ -1875,6 +1894,100 @@ shm_get_seals(struct file *fp, int *seals)
 }
 
 static int
+shm_deallocate(struct shmfd *shmfd, off_t *offset, off_t *length, int flags)
+{
+	vm_object_t object;
+	vm_pindex_t pistart, pi, piend;
+	vm_ooffset_t off, len;
+	int startofs, endofs, end;
+	int error;
+
+	off = *offset;
+	len = *length;
+	KASSERT(off + len <= (vm_ooffset_t)OFF_MAX, ("off + len overflows"));
+	object = shmfd->shm_object;
+	startofs = off & PAGE_MASK;
+	endofs = (off + len) & PAGE_MASK;
+	pistart = OFF_TO_IDX(off);
+	piend = OFF_TO_IDX(off + len);
+	pi = OFF_TO_IDX(off + PAGE_MASK);
+	error = 0;
+
+	VM_OBJECT_WLOCK(object);
+
+	if (startofs != 0) {
+		end = pistart != piend ? PAGE_SIZE : endofs;
+		error = shm_partial_page_invalidate(object, pistart, startofs,
+		    end);
+		if (error)
+			goto out;
+		off += end - startofs;
+		len -= end - startofs;
+	}
+
+	if (pi < piend) {
+		vm_object_page_remove(object, pi, piend, 0);
+		off += IDX_TO_OFF(piend - pi);
+		len -= IDX_TO_OFF(piend - pi);
+	}
+
+	if (endofs != 0 && pistart != piend) {
+		error = shm_partial_page_invalidate(object, piend, 0, endofs);
+		if (error)
+			goto out;
+		off += endofs;
+		len -= endofs;
+	}
+
+out:
+	VM_OBJECT_WUNLOCK(shmfd->shm_object);
+	*offset = off;
+	*length = len;
+	return (error);
+}
+
+static int
+shm_fspacectl(struct file *fp, int cmd, off_t *offset, off_t *length, int flags,
+    struct ucred *active_cred, struct thread *td)
+{
+	void *rl_cookie;
+	struct shmfd *shmfd;
+	off_t off, len;
+	int error;
+
+	/* This assumes that the caller already checked for overflow. */
+	error = EINVAL;
+	shmfd = fp->f_data;
+	off = *offset;
+	len = *length;
+
+	if (cmd != SPACECTL_DEALLOC || off < 0 || len <= 0 ||
+	    len > OFF_MAX - off || flags != 0)
+		return (EINVAL);
+
+	rl_cookie = rangelock_wlock(&shmfd->shm_rl, off, off + len,
+	    &shmfd->shm_mtx);
+	switch (cmd) {
+	case SPACECTL_DEALLOC:
+		if ((shmfd->shm_seals & F_SEAL_WRITE) != 0) {
+			error = EPERM;
+			break;
+		}
+		error = shm_deallocate(shmfd, &off, &len, flags);
+		if (error != 0)
+			break;
+		*offset = off;
+		*length = len;
+		break;
+	default:
+		__assert_unreachable();
+	}
+	rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx);
+	return (error);
+}
+
+
+static int
 shm_fallocate(struct file *fp, off_t offset, off_t len, struct thread *td)
 {
 	void *rl_cookie;