aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJohn Baldwin <jhb@FreeBSD.org>2022-08-17 19:10:35 +0000
committerJohn Baldwin <jhb@FreeBSD.org>2022-08-17 19:10:35 +0000
commit3499df29c2a1bbd6755f4ebe4b4a0083fda7a666 (patch)
tree9ef344dc0f969dfaebc8c6677171f25f05cdfcf7
parent5567d6b4419b02a2099527228b1a51cc55a5b47d (diff)
downloadsrc-3499df29c2a1bbd6755f4ebe4b4a0083fda7a666.tar.gz
src-3499df29c2a1bbd6755f4ebe4b4a0083fda7a666.zip
arm64 pmap: Convert PC_IS_FREE into an inline function.
This permits inlining the comparisons even in the 16K page case. Note that since PC_FREEN is -1, values can be compared efficiently without having to fetch words of pc_freemask from memory via the 'cmn <reg>, #0x1' instruction. Reviewed by: markj Sponsored by: DARPA Differential Revision: https://reviews.freebsd.org/D36218
-rw-r--r--sys/arm64/arm64/pmap.c21
1 files changed, 11 insertions, 10 deletions
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 9c47c043d251..deea00bc5d13 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -2462,14 +2462,6 @@ pv_to_chunk(pv_entry_t pv)
#define PC_FREEN 0xfffffffffffffffful
#define PC_FREEL ((1ul << (_NPCPV % 64)) - 1)
-#if _NPCM == 3
-#define PC_IS_FREE(pc) ((pc)->pc_map[0] == PC_FREEN && \
- (pc)->pc_map[1] == PC_FREEN && (pc)->pc_map[2] == PC_FREEL)
-#else
-#define PC_IS_FREE(pc) \
- (memcmp((pc)->pc_map, pc_freemask, sizeof(pc_freemask)) == 0)
-#endif
-
static const uint64_t pc_freemask[] = { PC_FREEN, PC_FREEN,
#if _NPCM > 3
PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN,
@@ -2489,6 +2481,15 @@ pc_is_full(struct pv_chunk *pc)
return (true);
}
+static __inline bool
+pc_is_free(struct pv_chunk *pc)
+{
+ for (u_int i = 0; i < _NPCM - 1; i++)
+ if (pc->pc_map[i] != PC_FREEN)
+ return (false);
+ return (pc->pc_map[_NPCM - 1] == PC_FREEL);
+}
+
#ifdef PV_STATS
static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
@@ -2653,7 +2654,7 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
PV_STAT(atomic_add_int(&pv_entry_spare, freed));
PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
- if (PC_IS_FREE(pc)) {
+ if (pc_is_free(pc)) {
PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
@@ -2722,7 +2723,7 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv)
field = idx / 64;
bit = idx % 64;
pc->pc_map[field] |= 1ul << bit;
- if (!PC_IS_FREE(pc)) {
+ if (!pc_is_free(pc)) {
/* 98% of the time, pc is already at the head of the list. */
if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);