vm: Fix racy checks for swap objects

Commit 4b8365d752ef introduced the ability to dynamically register
VM object types, for use by tmpfs, which creates swap-backed objects.
As a part of this, checks for such objects changed from

  object->type == OBJT_DEFAULT || object->type == OBJT_SWAP

to

  object->type == OBJT_DEFAULT || (object->flags & OBJ_SWAP) != 0

In particular, objects of type OBJT_DEFAULT do not have OBJ_SWAP set;
the swap pager sets this flag when converting from OBJT_DEFAULT to
OBJT_SWAP.

A few of these checks are done without the object lock held.  It turns
out that this can result in false negatives since the swap pager
converts objects like so:

  object->type = OBJT_SWAP;
  object->flags |= OBJ_SWAP;

Fix the problem by adding explicit tests for OBJT_SWAP objects in
unlocked checks.

PR:		258932
Fixes:		4b8365d752ef ("Add OBJT_SWAP_TMPFS pager")
Reported by:	bdrewery
Reviewed by:	kib
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D35470

3 files changed, 11 insertions, 8 deletions

diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index 1842b6e1a314..93182051abb7 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -2824,9 +2824,6 @@ again:
 			continue;
 		}
 
-		if (obj->type != OBJT_DEFAULT &&
-		    (obj->flags & OBJ_SWAP) == 0)
-			continue;
 		VM_OBJECT_WLOCK(obj);
 		if (obj->type != OBJT_DEFAULT &&
 		    (obj->flags & OBJ_SWAP) == 0) {
@@ -4139,7 +4136,13 @@ vm_map_copy_entry(
 		 */
 		size = src_entry->end - src_entry->start;
 		if ((src_object = src_entry->object.vm_object) != NULL) {
+			/*
+			 * Swap-backed objects need special handling.  Note that
+			 * this is an unlocked check, so it is possible to race
+			 * with an OBJT_DEFAULT -> OBJT_SWAP conversion.
+			 */
 			if (src_object->type == OBJT_DEFAULT ||
+			    src_object->type == OBJT_SWAP ||
 			    (src_object->flags & OBJ_SWAP) != 0) {
 				vm_map_copy_swap_object(src_entry, dst_entry,
 				    size, fork_charge);
diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c
index db9a32d1c9bc..5f27b550b78d 100644
--- a/sys/vm/vm_mmap.c
+++ b/sys/vm/vm_mmap.c
@@ -1368,9 +1368,8 @@ vm_mmap_vnode(struct thread *td, vm_size_t objsize,
 			goto done;
 		}
 	} else {
-		KASSERT(obj->type == OBJT_DEFAULT ||
-		    (obj->flags & OBJ_SWAP) != 0,
-		    ("wrong object type"));
+		KASSERT(obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP ||
+		    (obj->flags & OBJ_SWAP) != 0, ("wrong object type"));
 		vm_object_reference(obj);
 #if VM_NRESERVLEVEL > 0
 		if ((obj->flags & OBJ_COLORED) == 0) {
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index 36d5f3275800..74439d5884ef 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -1886,8 +1886,9 @@ vm_pageout_oom_pagecount(struct vmspace *vmspace)
 		if ((entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0 &&
 		    obj->ref_count != 1)
 			continue;
-		if (obj->type == OBJT_DEFAULT || obj->type == OBJT_PHYS ||
-		    obj->type == OBJT_VNODE || (obj->flags & OBJ_SWAP) != 0)
+		if (obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP ||
+		    obj->type == OBJT_PHYS || obj->type == OBJT_VNODE ||
+		    (obj->flags & OBJ_SWAP) != 0)
 			res += obj->resident_page_count;
 	}
 	return (res);