Widen the vm_page aflags field to 16 bits.

We are now out of aflags bits, whereas the "flags" field only makes use
of five of its sixteen bits, so narrow "flags" to eight bits.  I have no
intention of adding a new aflag in the near future, but would like to
combine the aflags, queue and act_count fields into a single atomically
updated word.  This will allow vm_page_pqstate_cmpset() to become much
simpler and is a step towards eliminating the use of the page lock array
in updating per-page queue state.

The change modifies the layout of struct vm_page, so bump
__FreeBSD_version.

Reviewed by:	alc, dougm, jeff, kib
Sponsored by:	Netflix, Intel
Differential Revision:	https://reviews.freebsd.org/D22397

3 files changed, 24 insertions, 24 deletions

diff --git a/sys/sys/param.h b/sys/sys/param.h
index e49cf2bdb74e..60718001f8a0 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -60,7 +60,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1300057	/* Master, propagated to newvers */
+#define __FreeBSD_version 1300058	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 048211fd1c5d..b040a1075cd1 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -5014,7 +5014,7 @@ vm_page_object_busy_assert(vm_page_t m)
 }
 
 void
-vm_page_assert_pga_writeable(vm_page_t m, uint8_t bits)
+vm_page_assert_pga_writeable(vm_page_t m, uint16_t bits)
 {
 
 	if ((bits & PGA_WRITEABLE) == 0)
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 0218064fa4a7..6a447425f5f6 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -234,15 +234,15 @@ struct vm_page {
 	struct md_page md;		/* machine dependent stuff */
 	u_int ref_count;		/* page references (A) */
 	volatile u_int busy_lock;	/* busy owners lock */
-	uint16_t flags;			/* page PG_* flags (P) */
-	uint8_t	order;			/* index of the buddy queue (F) */
+	uint16_t aflags;		/* atomic flags (A) */
+	uint8_t queue;			/* page queue index (Q) */
+	uint8_t act_count;		/* page usage count (P) */
+	uint8_t order;			/* index of the buddy queue (F) */
 	uint8_t pool;			/* vm_phys freepool index (F) */
-	uint8_t aflags;			/* atomic flags (A) */
+	uint8_t flags;			/* page PG_* flags (P) */
 	uint8_t oflags;			/* page VPO_* flags (O) */
-	uint8_t queue;			/* page queue index (Q) */
 	int8_t psind;			/* pagesizes[] index (O) */
 	int8_t segind;			/* vm_phys segment index (C) */
-	u_char	act_count;		/* page usage count (P) */
 	/* NOTE that these must support one bit per DEV_BSIZE in a page */
 	/* so, on normal X86 kernels, they must be at least 8 bits wide */
 	vm_page_bits_t valid;		/* valid DEV_BSIZE chunk map (O,B) */
@@ -414,14 +414,14 @@ extern struct mtx_padalign pa_lock[];
  * the inactive queue, thus bypassing LRU.  The page lock must be held to
  * set this flag, and the queue lock for the page must be held to clear it.
  */
-#define	PGA_WRITEABLE	0x01		/* page may be mapped writeable */
-#define	PGA_REFERENCED	0x02		/* page has been referenced */
-#define	PGA_EXECUTABLE	0x04		/* page may be mapped executable */
-#define	PGA_ENQUEUED	0x08		/* page is enqueued in a page queue */
-#define	PGA_DEQUEUE	0x10		/* page is due to be dequeued */
-#define	PGA_REQUEUE	0x20		/* page is due to be requeued */
-#define	PGA_REQUEUE_HEAD 0x40		/* page requeue should bypass LRU */
-#define	PGA_NOSYNC	0x80		/* do not collect for syncer */
+#define	PGA_WRITEABLE	0x0001		/* page may be mapped writeable */
+#define	PGA_REFERENCED	0x0002		/* page has been referenced */
+#define	PGA_EXECUTABLE	0x0004		/* page may be mapped executable */
+#define	PGA_ENQUEUED	0x0008		/* page is enqueued in a page queue */
+#define	PGA_DEQUEUE	0x0010		/* page is due to be dequeued */
+#define	PGA_REQUEUE	0x0020		/* page is due to be requeued */
+#define	PGA_REQUEUE_HEAD 0x0040		/* page requeue should bypass LRU */
+#define	PGA_NOSYNC	0x0080		/* do not collect for syncer */
 
 #define	PGA_QUEUE_STATE_MASK	(PGA_ENQUEUED | PGA_DEQUEUE | PGA_REQUEUE | \
 				PGA_REQUEUE_HEAD)
@@ -434,11 +434,11 @@ extern struct mtx_padalign pa_lock[];
  * allocated from a per-CPU cache.  It is cleared the next time that the
  * page is allocated from the physical memory allocator.
  */
-#define	PG_PCPU_CACHE	0x0001		/* was allocated from per-CPU caches */
-#define	PG_FICTITIOUS	0x0004		/* physical page doesn't exist */
-#define	PG_ZERO		0x0008		/* page is zeroed */
-#define	PG_MARKER	0x0010		/* special queue marker page */
-#define	PG_NODUMP	0x0080		/* don't include this page in a dump */
+#define	PG_PCPU_CACHE	0x01		/* was allocated from per-CPU caches */
+#define	PG_FICTITIOUS	0x02		/* physical page doesn't exist */
+#define	PG_ZERO		0x04		/* page is zeroed */
+#define	PG_MARKER	0x08		/* special queue marker page */
+#define	PG_NODUMP	0x10		/* don't include this page in a dump */
 
 /*
  * Misc constants.
@@ -716,7 +716,7 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line);
 #ifdef INVARIANTS
 void vm_page_object_busy_assert(vm_page_t m);
 #define	VM_PAGE_OBJECT_BUSY_ASSERT(m)	vm_page_object_busy_assert(m)
-void vm_page_assert_pga_writeable(vm_page_t m, uint8_t bits);
+void vm_page_assert_pga_writeable(vm_page_t m, uint16_t bits);
 #define	VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits)				\
 	vm_page_assert_pga_writeable(m, bits)
 #else
@@ -749,7 +749,7 @@ _Static_assert(sizeof(((struct vm_page *)NULL)->queue) == 1,
 #define	VM_PAGE_AFLAG_SHIFT	0
 #define	VM_PAGE_QUEUE_SHIFT	16
 #else
-#define	VM_PAGE_AFLAG_SHIFT	24
+#define	VM_PAGE_AFLAG_SHIFT	16
 #define	VM_PAGE_QUEUE_SHIFT	8
 #endif
 #define	VM_PAGE_QUEUE_MASK	(0xff << VM_PAGE_QUEUE_SHIFT)
@@ -758,7 +758,7 @@ _Static_assert(sizeof(((struct vm_page *)NULL)->queue) == 1,
  *	Clear the given bits in the specified page.
  */
 static inline void
-vm_page_aflag_clear(vm_page_t m, uint8_t bits)
+vm_page_aflag_clear(vm_page_t m, uint16_t bits)
 {
 	uint32_t *addr, val;
 
@@ -782,7 +782,7 @@ vm_page_aflag_clear(vm_page_t m, uint8_t bits)
  *	Set the given bits in the specified page.
  */
 static inline void
-vm_page_aflag_set(vm_page_t m, uint8_t bits)
+vm_page_aflag_set(vm_page_t m, uint16_t bits)
 {
 	uint32_t *addr, val;