Add a blocking counter KPI.

refcount(9) was recently extended to support waiting on a refcount to
drop to zero, as this was needed for a lockless VM object
paging-in-progress counter.  However, this adds overhead to all uses of
refcount(9) and doesn't really match traditional refcounting semantics:
once a counter has dropped to zero, the protected object may be freed at
any point and it is not safe to dereference the counter.

This change removes that extension and instead adds a new set of KPIs,
blockcount_*, for use by VM object PIP and busy.

Reviewed by:	jeff, kib, mjg
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D23723

5 files changed, 33 insertions, 35 deletions

diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index b99b430af4fe..22c9b54e4b8b 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -377,7 +377,7 @@ vm_fault_restore_map_lock(struct faultstate *fs)
 {
 
 	VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
-	MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0);
+	MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0);
 
 	if (!vm_map_trylock_read(fs->map)) {
 		VM_OBJECT_WUNLOCK(fs->first_object);
@@ -428,7 +428,7 @@ vm_fault_populate(struct faultstate *fs)
 
 	MPASS(fs->object == fs->first_object);
 	VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
-	MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0);
+	MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0);
 	MPASS(fs->first_object->backing_object == NULL);
 	MPASS(fs->lookup_still_valid);
 
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 0169d06b730c..85a59287fb01 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -71,6 +71,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/blockcount.h>
 #include <sys/cpuset.h>
 #include <sys/lock.h>
 #include <sys/mman.h>
@@ -201,12 +202,11 @@ vm_object_zdtor(void *mem, int size, void *arg)
 	    ("object %p has reservations",
 	    object));
 #endif
-	KASSERT(REFCOUNT_COUNT(object->paging_in_progress) == 0,
+	KASSERT(blockcount_read(&object->paging_in_progress) == 0,
 	    ("object %p paging_in_progress = %d",
-	    object, REFCOUNT_COUNT(object->paging_in_progress)));
-	KASSERT(object->busy == 0,
-	    ("object %p busy = %d",
-	    object, object->busy));
+	    object, blockcount_read(&object->paging_in_progress)));
+	KASSERT(!vm_object_busied(object),
+	    ("object %p busy = %d", object, blockcount_read(&object->busy)));
 	KASSERT(object->resident_page_count == 0,
 	    ("object %p resident_page_count = %d",
 	    object, object->resident_page_count));
@@ -231,8 +231,8 @@ vm_object_zinit(void *mem, int size, int flags)
 	object->type = OBJT_DEAD;
 	vm_radix_init(&object->rtree);
 	refcount_init(&object->ref_count, 0);
-	refcount_init(&object->paging_in_progress, 0);
-	refcount_init(&object->busy, 0);
+	blockcount_init(&object->paging_in_progress);
+	blockcount_init(&object->busy);
 	object->resident_page_count = 0;
 	object->shadow_count = 0;
 	object->flags = OBJ_DEAD;
@@ -363,34 +363,36 @@ void
 vm_object_pip_add(vm_object_t object, short i)
 {
 
-	refcount_acquiren(&object->paging_in_progress, i);
+	if (i > 0)
+		blockcount_acquire(&object->paging_in_progress, i);
 }
 
 void
 vm_object_pip_wakeup(vm_object_t object)
 {
 
-	refcount_release(&object->paging_in_progress);
+	vm_object_pip_wakeupn(object, 1);
 }
 
 void
 vm_object_pip_wakeupn(vm_object_t object, short i)
 {
 
-	refcount_releasen(&object->paging_in_progress, i);
+	if (i > 0)
+		blockcount_release(&object->paging_in_progress, i);
 }
 
 /*
- * Atomically drop the interlock and wait for pip to drain.  This protects
- * from sleep/wakeup races due to identity changes.  The lock is not
- * re-acquired on return.
+ * Atomically drop the object lock and wait for pip to drain.  This protects
+ * from sleep/wakeup races due to identity changes.  The lock is not re-acquired
+ * on return.
  */
 static void
 vm_object_pip_sleep(vm_object_t object, const char *waitid)
 {
 
-	refcount_sleep_interlock(&object->paging_in_progress,
-	    &object->lock, waitid, PVM);
+	(void)blockcount_sleep(&object->paging_in_progress, &object->lock,
+	    waitid, PVM | PDROP);
 }
 
 void
@@ -399,10 +401,8 @@ vm_object_pip_wait(vm_object_t object, const char *waitid)
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
 
-	while (REFCOUNT_COUNT(object->paging_in_progress) > 0) {
-		vm_object_pip_sleep(object, waitid);
-		VM_OBJECT_WLOCK(object);
-	}
+	blockcount_wait(&object->paging_in_progress, &object->lock, waitid,
+	    PVM);
 }
 
 void
@@ -411,8 +411,7 @@ vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)
 
 	VM_OBJECT_ASSERT_UNLOCKED(object);
 
-	while (REFCOUNT_COUNT(object->paging_in_progress) > 0)
-		refcount_wait(&object->paging_in_progress, waitid, PVM);
+	blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);
 }
 
 /*
@@ -955,7 +954,7 @@ vm_object_terminate(vm_object_t object)
 	 */
 	vm_object_pip_wait(object, "objtrm");
 
-	KASSERT(!REFCOUNT_COUNT(object->paging_in_progress),
+	KASSERT(!blockcount_read(&object->paging_in_progress),
 	    ("vm_object_terminate: pageout in progress"));
 
 	KASSERT(object->ref_count == 0,
@@ -2458,7 +2457,7 @@ vm_object_busy(vm_object_t obj)
 
 	VM_OBJECT_ASSERT_LOCKED(obj);
 
-	refcount_acquire(&obj->busy);
+	blockcount_acquire(&obj->busy, 1);
 	/* The fence is required to order loads of page busy. */
 	atomic_thread_fence_acq_rel();
 }
@@ -2467,8 +2466,7 @@ void
 vm_object_unbusy(vm_object_t obj)
 {
 
-
-	refcount_release(&obj->busy);
+	blockcount_release(&obj->busy, 1);
 }
 
 void
@@ -2477,8 +2475,7 @@ vm_object_busy_wait(vm_object_t obj, const char *wmesg)
 
 	VM_OBJECT_ASSERT_UNLOCKED(obj);
 
-	if (obj->busy)
-		refcount_sleep(&obj->busy, wmesg, PVM);
+	(void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM);
 }
 
 /*
diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
index 71ddbe85bf3d..007e945daa77 100644
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -70,6 +70,7 @@
 #define	_VM_OBJECT_
 
 #include <sys/queue.h>
+#include <sys/_blockcount.h>
 #include <sys/_lock.h>
 #include <sys/_mutex.h>
 #include <sys/_pctrie.h>
@@ -113,8 +114,8 @@ struct vm_object {
 	objtype_t type;			/* type of pager */
 	u_short flags;			/* see below */
 	u_short pg_color;		/* (c) color of first page in obj */
-	volatile u_int paging_in_progress; /* Paging (in or out) so don't collapse or destroy */
-	volatile u_int busy;		/* (a) object is busy, disallow page busy. */
+	blockcount_t paging_in_progress; /* (a) Paging (in or out) so don't collapse or destroy */
+	blockcount_t busy;		/* (a) object is busy, disallow page busy. */
 	int resident_page_count;	/* number of resident pages */
 	struct vm_object *backing_object; /* object that I'm a shadow of */
 	vm_ooffset_t backing_object_offset;/* Offset in backing object */
@@ -265,7 +266,7 @@ extern struct vm_object kernel_object_store;
 	lock_class_rw.lc_lock(&(object)->lock.lock_object, (state))
 
 #define	VM_OBJECT_ASSERT_PAGING(object)					\
-	KASSERT((object)->paging_in_progress != 0,			\
+	KASSERT(blockcount_read(&(object)->paging_in_progress) != 0,	\
 	    ("vm_object %p is not paging", object))
 #define	VM_OBJECT_ASSERT_REFERENCE(object)				\
 	KASSERT((object)->reference_count != 0,				\
@@ -348,7 +349,7 @@ static inline bool
 vm_object_busied(vm_object_t object)
 {
 
-	return (object->busy != 0);
+	return (blockcount_read(&object->busy) != 0);
 }
 #define	VM_OBJECT_ASSERT_BUSY(object)	MPASS(vm_object_busied((object)))
 
diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h
index 5ceaaf5bc8a4..a1177510a256 100644
--- a/sys/vm/vm_pager.h
+++ b/sys/vm/vm_pager.h
@@ -168,7 +168,7 @@ vm_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type,
 
 	MPASS((object->flags & OBJ_POPULATE) != 0);
 	MPASS(pidx < object->size);
-	MPASS(object->paging_in_progress > 0);
+	MPASS(blockcount_read(&object->paging_in_progress) > 0);
 	return ((*pagertab[object->type]->pgo_populate)(object, pidx,
 	    fault_type, max_prot, first, last));
 }
diff --git a/sys/vm/vm_swapout.c b/sys/vm/vm_swapout.c
index 6dd8f75d3865..bf4a248d7188 100644
--- a/sys/vm/vm_swapout.c
+++ b/sys/vm/vm_swapout.c
@@ -218,7 +218,7 @@ vm_swapout_object_deactivate(pmap_t pmap, vm_object_t first_object,
 			goto unlock_return;
 		VM_OBJECT_ASSERT_LOCKED(object);
 		if ((object->flags & OBJ_UNMANAGED) != 0 ||
-		    REFCOUNT_COUNT(object->paging_in_progress) > 0)
+		    blockcount_read(&object->paging_in_progress) > 0)
 			goto unlock_return;
 
 		unmap = true;