Different consumers of the struct vm_page abuse pageq member to keep

additional information, when the page is guaranteed to not belong to a
paging queue.  Usually, this results in a lot of type casts which make
reasoning about the code correctness harder.

Sometimes m->object is used instead of pageq, which could cause real
and confusing bugs if non-NULL m->object is leaked.  See r141955 and
r253140 for examples.

Change the pageq member into a union containing explicitly-typed
members.  Use them instead of type-punning or abusing m->object in x86
pmaps, uma and vm_page_alloc_contig().

Requested and reviewed by:	alc
Sponsored by:	The FreeBSD Foundation

9 files changed, 73 insertions, 80 deletions

diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
index ba193e9f3db4..8113f0394b33 100644
--- a/sys/vm/device_pager.c
+++ b/sys/vm/device_pager.c
@@ -226,7 +226,7 @@ dev_pager_free_page(vm_object_t object, vm_page_t m)
 	KASSERT((object->type == OBJT_DEVICE &&
 	    (m->oflags & VPO_UNMANAGED) != 0),
 	    ("Managed device or page obj %p m %p", object, m));
-	TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, pageq);
+	TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, plinks.q);
 	vm_page_putfake(m);
 }
 
@@ -281,7 +281,7 @@ dev_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int reqpage)
 		    ("Wrong page type %p %p", ma[reqpage], object));
 		if (object->type == OBJT_DEVICE) {
 			TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist,
-			    ma[reqpage], pageq);
+			    ma[reqpage], plinks.q);
 		}
 	}
 
diff --git a/sys/vm/memguard.c b/sys/vm/memguard.c
index ea2d9259a80a..8b021612283c 100644
--- a/sys/vm/memguard.c
+++ b/sys/vm/memguard.c
@@ -261,7 +261,7 @@ v2sizep(vm_offset_t va)
 	p = PHYS_TO_VM_PAGE(pa);
 	KASSERT(p->wire_count != 0 && p->queue == PQ_NONE,
 	    ("MEMGUARD: Expected wired page %p in vtomgfifo!", p));
-	return ((u_long *)&p->pageq.tqe_next);
+	return (&p->plinks.memguard.p);
 }
 
 static u_long *
@@ -276,7 +276,7 @@ v2sizev(vm_offset_t va)
 	p = PHYS_TO_VM_PAGE(pa);
 	KASSERT(p->wire_count != 0 && p->queue == PQ_NONE,
 	    ("MEMGUARD: Expected wired page %p in vtomgfifo!", p));
-	return ((u_long *)&p->pageq.tqe_prev);
+	return (&p->plinks.memguard.v);
 }
 
 /*
diff --git a/sys/vm/sg_pager.c b/sys/vm/sg_pager.c
index 2b3f1173fc3c..08e8fc7d2060 100644
--- a/sys/vm/sg_pager.c
+++ b/sys/vm/sg_pager.c
@@ -124,7 +124,7 @@ sg_pager_dealloc(vm_object_t object)
 	 * Free up our fake pages.
 	 */
 	while ((m = TAILQ_FIRST(&object->un_pager.sgp.sgp_pglist)) != 0) {
-		TAILQ_REMOVE(&object->un_pager.sgp.sgp_pglist, m, pageq);
+		TAILQ_REMOVE(&object->un_pager.sgp.sgp_pglist, m, plinks.q);
 		vm_page_putfake(m);
 	}
 	
@@ -182,7 +182,7 @@ sg_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
 	/* Construct a new fake page. */
 	page = vm_page_getfake(paddr, memattr);
 	VM_OBJECT_WLOCK(object);
-	TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, pageq);
+	TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, plinks.q);
 
 	/* Free the original pages and insert this fake page into the object. */
 	for (i = 0; i < count; i++) {
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 5db18163a1ff..6627a07b60ef 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -718,18 +718,6 @@ keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
 			keg->uk_fini(slab->us_data + (keg->uk_rsize * i),
 			    keg->uk_size);
 	}
-	if (keg->uk_flags & UMA_ZONE_VTOSLAB) {
-		vm_object_t obj;
-
-		if (flags & UMA_SLAB_KMEM)
-			obj = kmem_object;
-		else if (flags & UMA_SLAB_KERNEL)
-			obj = kernel_object;
-		else
-			obj = NULL;
-		for (i = 0; i < keg->uk_ppera; i++)
-			vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), obj);
-	}
 	if (keg->uk_flags & UMA_ZONE_OFFPAGE)
 		zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
 #ifdef UMA_DEBUG
@@ -3112,7 +3100,7 @@ uma_large_malloc(int size, int wait)
 void
 uma_large_free(uma_slab_t slab)
 {
-	vsetobj((vm_offset_t)slab->us_data, kmem_object);
+
 	page_free(slab->us_data, slab->us_size, slab->us_flags);
 	zone_free_item(slabzone, slab, NULL, SKIP_NONE);
 }
diff --git a/sys/vm/uma_int.h b/sys/vm/uma_int.h
index 9aa05609114c..ec88e58e679e 100644
--- a/sys/vm/uma_int.h
+++ b/sys/vm/uma_int.h
@@ -407,7 +407,7 @@ vtoslab(vm_offset_t va)
 	uma_slab_t slab;
 
 	p = PHYS_TO_VM_PAGE(pmap_kextract(va));
-	slab = (uma_slab_t )p->object;
+	slab = (uma_slab_t )p->plinks.s.pv;
 
 	if (p->flags & PG_SLAB)
 		return (slab);
@@ -421,20 +421,10 @@ vsetslab(vm_offset_t va, uma_slab_t slab)
 	vm_page_t p;
 
 	p = PHYS_TO_VM_PAGE(pmap_kextract(va));
-	p->object = (vm_object_t)slab;
+	p->plinks.s.pv = slab;
 	p->flags |= PG_SLAB;
 }
 
-static __inline void
-vsetobj(vm_offset_t va, vm_object_t obj)
-{
-	vm_page_t p;
-
-	p = PHYS_TO_VM_PAGE(pmap_kextract(va));
-	p->object = obj;
-	p->flags &= ~PG_SLAB;
-}
-
 /*
  * The following two functions may be defined by architecture specific code
  * if they can provide more effecient allocation functions.  This is useful
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 39f16c067a2c..b204ae2b54fb 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1643,6 +1643,16 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 	return (m);
 }
 
+static void
+vm_page_alloc_contig_vdrop(struct spglist *lst)
+{
+
+	while (!SLIST_EMPTY(lst)) {
+		vdrop((struct vnode *)SLIST_FIRST(lst)-> plinks.s.pv);
+		SLIST_REMOVE_HEAD(lst, plinks.s.ss);
+	}
+}
+
 /*
  *	vm_page_alloc_contig:
  *
@@ -1687,7 +1697,8 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
     vm_paddr_t boundary, vm_memattr_t memattr)
 {
 	struct vnode *drop;
-	vm_page_t deferred_vdrop_list, m, m_tmp, m_ret;
+	struct spglist deferred_vdrop_list;
+	vm_page_t m, m_tmp, m_ret;
 	u_int flags, oflags;
 	int req_class;
 
@@ -1712,7 +1723,7 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
 	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
 		req_class = VM_ALLOC_SYSTEM;
 
-	deferred_vdrop_list = NULL;
+	SLIST_INIT(&deferred_vdrop_list);
 	mtx_lock(&vm_page_queue_free_mtx);
 	if (cnt.v_free_count + cnt.v_cache_count >= npages +
 	    cnt.v_free_reserved || (req_class == VM_ALLOC_SYSTEM &&
@@ -1744,9 +1755,9 @@ retry:
 				 * page list, "pageq" can be safely abused to
 				 * construct a short-lived list of vnodes.
 				 */
-				m->pageq.tqe_prev = (void *)drop;
-				m->pageq.tqe_next = deferred_vdrop_list;
-				deferred_vdrop_list = m;
+				m->plinks.s.pv = drop;
+				SLIST_INSERT_HEAD(&deferred_vdrop_list, m,
+				    plinks.s.ss);
 			}
 		}
 	else {
@@ -1792,11 +1803,8 @@ retry:
 		m->oflags = oflags;
 		if (object != NULL) {
 			if (vm_page_insert(m, object, pindex)) {
-				while (deferred_vdrop_list != NULL) {
-		vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev);
-					deferred_vdrop_list =
-					    deferred_vdrop_list->pageq.tqe_next;
-				}
+				vm_page_alloc_contig_vdrop(
+				    &deferred_vdrop_list);
 				if (vm_paging_needed())
 					pagedaemon_wakeup();
 				for (m = m_ret, m_tmp = m_ret;
@@ -1815,10 +1823,7 @@ retry:
 			pmap_page_set_memattr(m, memattr);
 		pindex++;
 	}
-	while (deferred_vdrop_list != NULL) {
-		vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev);
-		deferred_vdrop_list = deferred_vdrop_list->pageq.tqe_next;
-	}
+	vm_page_alloc_contig_vdrop(&deferred_vdrop_list);
 	if (vm_paging_needed())
 		pagedaemon_wakeup();
 	return (m_ret);
@@ -2035,7 +2040,7 @@ vm_page_dequeue(vm_page_t m)
 	pq = vm_page_pagequeue(m);
 	vm_pagequeue_lock(pq);
 	m->queue = PQ_NONE;
-	TAILQ_REMOVE(&pq->pq_pl, m, pageq);
+	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
 	vm_pagequeue_cnt_dec(pq);
 	vm_pagequeue_unlock(pq);
 }
@@ -2056,7 +2061,7 @@ vm_page_dequeue_locked(vm_page_t m)
 	pq = vm_page_pagequeue(m);
 	vm_pagequeue_assert_locked(pq);
 	m->queue = PQ_NONE;
-	TAILQ_REMOVE(&pq->pq_pl, m, pageq);
+	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
 	vm_pagequeue_cnt_dec(pq);
 }
 
@@ -2076,7 +2081,7 @@ vm_page_enqueue(int queue, vm_page_t m)
 	pq = &vm_phys_domain(m)->vmd_pagequeues[queue];
 	vm_pagequeue_lock(pq);
 	m->queue = queue;
-	TAILQ_INSERT_TAIL(&pq->pq_pl, m, pageq);
+	TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
 	vm_pagequeue_cnt_inc(pq);
 	vm_pagequeue_unlock(pq);
 }
@@ -2098,8 +2103,8 @@ vm_page_requeue(vm_page_t m)
 	    ("vm_page_requeue: page %p is not queued", m));
 	pq = vm_page_pagequeue(m);
 	vm_pagequeue_lock(pq);
-	TAILQ_REMOVE(&pq->pq_pl, m, pageq);
-	TAILQ_INSERT_TAIL(&pq->pq_pl, m, pageq);
+	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
+	TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
 	vm_pagequeue_unlock(pq);
 }
 
@@ -2119,8 +2124,8 @@ vm_page_requeue_locked(vm_page_t m)
 	    ("vm_page_requeue_locked: page %p is not queued", m));
 	pq = vm_page_pagequeue(m);
 	vm_pagequeue_assert_locked(pq);
-	TAILQ_REMOVE(&pq->pq_pl, m, pageq);
-	TAILQ_INSERT_TAIL(&pq->pq_pl, m, pageq);
+	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
+	TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
 }
 
 /*
@@ -2413,9 +2418,9 @@ _vm_page_deactivate(vm_page_t m, int athead)
 		vm_pagequeue_lock(pq);
 		m->queue = PQ_INACTIVE;
 		if (athead)
-			TAILQ_INSERT_HEAD(&pq->pq_pl, m, pageq);
+			TAILQ_INSERT_HEAD(&pq->pq_pl, m, plinks.q);
 		else
-			TAILQ_INSERT_TAIL(&pq->pq_pl, m, pageq);
+			TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
 		vm_pagequeue_cnt_inc(pq);
 		vm_pagequeue_unlock(pq);
 	}
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 01c496718e3e..43ee04a33ee8 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -126,10 +126,19 @@ typedef uint64_t vm_page_bits_t;
 #endif
 
 struct vm_page {
-	TAILQ_ENTRY(vm_page) pageq;	/* page queue or free list (Q)	*/
-	TAILQ_ENTRY(vm_page) listq;	/* pages in same object (O) 	*/
-
-	vm_object_t object;		/* which object am I in (O,P)*/
+	union {
+		TAILQ_ENTRY(vm_page) q; /* page queue or free list (Q) */
+		struct {
+			SLIST_ENTRY(vm_page) ss; /* private slists */
+			void *pv;
+		} s;
+		struct {
+			u_long p;
+			u_long v;
+		} memguard;
+	} plinks;
+	TAILQ_ENTRY(vm_page) listq;	/* pages in same object (O) */
+	vm_object_t object;		/* which object am I in (O,P) */
 	vm_pindex_t pindex;		/* offset into object (O,P) */
 	vm_paddr_t phys_addr;		/* physical address of page */
 	struct md_page md;		/* machine dependant stuff */
@@ -145,7 +154,7 @@ struct vm_page {
 	uint16_t flags;			/* page PG_* flags (P) */
 	u_char	act_count;		/* page usage count (P) */
 	u_char __pad0;			/* unused padding */
-	/* NOTE that these must support one bit per DEV_BSIZE in a page!!! */
+	/* NOTE that these must support one bit per DEV_BSIZE in a page */
 	/* so, on normal X86 kernels, they must be at least 8 bits wide */
 	vm_page_bits_t valid;		/* map of valid DEV_BSIZE chunks (O) */
 	vm_page_bits_t dirty;		/* map of dirty DEV_BSIZE chunks (M) */
@@ -201,6 +210,7 @@ struct vm_page {
 #define	PQ_COUNT	2
 
 TAILQ_HEAD(pglist, vm_page);
+SLIST_HEAD(spglist, vm_page);
 
 struct vm_pagequeue {
 	struct mtx	pq_mutex;
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index f8016034247c..3a6399afc703 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -274,7 +274,7 @@ vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
 	pq = vm_page_pagequeue(m);
 	object = m->object;
 	
-	TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, pageq);
+	TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
 	vm_pagequeue_unlock(pq);
 	vm_page_unlock(m);
 	VM_OBJECT_WLOCK(object);
@@ -282,11 +282,11 @@ vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
 	vm_pagequeue_lock(pq);
 
 	/* Page queue might have changed. */
-	*next = TAILQ_NEXT(&marker, pageq);
+	*next = TAILQ_NEXT(&marker, plinks.q);
 	unchanged = (m->queue == queue &&
 		     m->object == object &&
-		     &marker == TAILQ_NEXT(m, pageq));
-	TAILQ_REMOVE(&pq->pq_pl, &marker, pageq);
+		     &marker == TAILQ_NEXT(m, plinks.q));
+	TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 	return (unchanged);
 }
 
@@ -315,15 +315,15 @@ vm_pageout_page_lock(vm_page_t m, vm_page_t *next)
 	vm_pageout_init_marker(&marker, queue);
 	pq = vm_page_pagequeue(m);
 
-	TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, pageq);
+	TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
 	vm_pagequeue_unlock(pq);
 	vm_page_lock(m);
 	vm_pagequeue_lock(pq);
 
 	/* Page queue might have changed. */
-	*next = TAILQ_NEXT(&marker, pageq);
-	unchanged = (m->queue == queue && &marker == TAILQ_NEXT(m, pageq));
-	TAILQ_REMOVE(&pq->pq_pl, &marker, pageq);
+	*next = TAILQ_NEXT(&marker, plinks.q);
+	unchanged = (m->queue == queue && &marker == TAILQ_NEXT(m, plinks.q));
+	TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 	return (unchanged);
 }
 
@@ -578,7 +578,7 @@ vm_pageout_launder(struct vm_pagequeue *pq, int tries, vm_paddr_t low,
 	vm_page_t m, m_tmp, next;
 
 	vm_pagequeue_lock(pq);
-	TAILQ_FOREACH_SAFE(m, &pq->pq_pl, pageq, next) {
+	TAILQ_FOREACH_SAFE(m, &pq->pq_pl, plinks.q, next) {
 		if ((m->flags & PG_MARKER) != 0)
 			continue;
 		pa = VM_PAGE_TO_PHYS(m);
@@ -963,7 +963,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 		KASSERT(m->queue == PQ_INACTIVE, ("Inactive queue %p", m));
 
 		PCPU_INC(cnt.v_pdpages);
-		next = TAILQ_NEXT(m, pageq);
+		next = TAILQ_NEXT(m, plinks.q);
 
 		/*
 		 * skip marker pages
@@ -1013,7 +1013,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 		 * 'next' pointer.  Use our marker to remember our
 		 * place.
 		 */
-		TAILQ_INSERT_AFTER(&pq->pq_pl, m, &vmd->vmd_marker, pageq);
+		TAILQ_INSERT_AFTER(&pq->pq_pl, m, &vmd->vmd_marker, plinks.q);
 		vm_pagequeue_unlock(pq);
 		queues_locked = FALSE;
 
@@ -1207,7 +1207,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 				 */
 				if (m->queue != PQ_INACTIVE ||
 				    m->object != object ||
-				    TAILQ_NEXT(m, pageq) != &vmd->vmd_marker) {
+				    TAILQ_NEXT(m, plinks.q) != &vmd->vmd_marker) {
 					vm_page_unlock(m);
 					if (object->flags & OBJ_MIGHTBEDIRTY)
 						vnodes_skipped++;
@@ -1277,8 +1277,8 @@ relock_queues:
 			vm_pagequeue_lock(pq);
 			queues_locked = TRUE;
 		}
-		next = TAILQ_NEXT(&vmd->vmd_marker, pageq);
-		TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_marker, pageq);
+		next = TAILQ_NEXT(&vmd->vmd_marker, plinks.q);
+		TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_marker, plinks.q);
 	}
 	vm_pagequeue_unlock(pq);
 
@@ -1304,7 +1304,7 @@ relock_queues:
 		KASSERT(m->queue == PQ_ACTIVE,
 		    ("vm_pageout_scan: page %p isn't active", m));
 
-		next = TAILQ_NEXT(m, pageq);
+		next = TAILQ_NEXT(m, plinks.q);
 		if ((m->flags & PG_MARKER) != 0) {
 			m = next;
 			continue;
@@ -1612,7 +1612,7 @@ vm_pageout_page_stats(struct vm_domain *vmd)
 		KASSERT(m->queue == PQ_ACTIVE,
 		    ("vm_pageout_page_stats: page %p isn't active", m));
 
-		next = TAILQ_NEXT(m, pageq);
+		next = TAILQ_NEXT(m, plinks.q);
 		if ((m->flags & PG_MARKER) != 0) {
 			m = next;
 			continue;
diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c
index 4c988db94648..ce6dfd3c161e 100644
--- a/sys/vm/vm_phys.c
+++ b/sys/vm/vm_phys.c
@@ -226,9 +226,9 @@ vm_freelist_add(struct vm_freelist *fl, vm_page_t m, int order, int tail)
 
 	m->order = order;
 	if (tail)
-		TAILQ_INSERT_TAIL(&fl[order].pl, m, pageq);
+		TAILQ_INSERT_TAIL(&fl[order].pl, m, plinks.q);
 	else
-		TAILQ_INSERT_HEAD(&fl[order].pl, m, pageq);
+		TAILQ_INSERT_HEAD(&fl[order].pl, m, plinks.q);
 	fl[order].lcnt++;
 }
 
@@ -236,7 +236,7 @@ static void
 vm_freelist_rem(struct vm_freelist *fl, vm_page_t m, int order)
 {
 
-	TAILQ_REMOVE(&fl[order].pl, m, pageq);
+	TAILQ_REMOVE(&fl[order].pl, m, plinks.q);
 	fl[order].lcnt--;
 	m->order = VM_NFREEORDER;
 }
@@ -819,7 +819,7 @@ vm_phys_zero_pages_idle(void)
 	fl = vm_phys_free_queues[domain][0][0];
 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
 	for (;;) {
-		TAILQ_FOREACH_REVERSE(m, &fl[oind].pl, pglist, pageq) {
+		TAILQ_FOREACH_REVERSE(m, &fl[oind].pl, pglist, plinks.q) {
 			for (m_tmp = m; m_tmp < &m[1 << oind]; m_tmp++) {
 				if ((m_tmp->flags & (PG_CACHED | PG_ZERO)) == 0) {
 					vm_phys_unfree_page(m_tmp);
@@ -889,7 +889,7 @@ restartdom:
 		for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) {
 			for (pind = 0; pind < VM_NFREEPOOL; pind++) {
 				fl = &vm_phys_free_queues[domain][flind][pind][0];
-				TAILQ_FOREACH(m_ret, &fl[oind].pl, pageq) {
+				TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) {
 					/*
 					 * A free list may contain physical pages
 					 * from one or more segments.