aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMark Johnston <markj@FreeBSD.org>2020-09-19 15:22:04 +0000
committerMark Johnston <markj@FreeBSD.org>2020-09-19 15:22:04 +0000
commitd26ab2bec024b00bbb753248256fe68265280bfa (patch)
tree2ae636f698a4050b335935958ca4d7ec8e483448
parent918d5f5908907c63ed1c8054e805a6a7d4850917 (diff)
downloadsrc-d26ab2bec024b00bbb753248256fe68265280bfa.tar.gz
src-d26ab2bec024b00bbb753248256fe68265280bfa.zip
Fix some nits in 1G page support in the amd64 pmap.
- Move assertions out of the main loop to avoid duplicate conditional expressions, and improve assertion messages. - Fix va_next updates. In some cases we were not doing the wraparound check before continuing the loop. - Use the right va_next. In pmap_advise() and pmap_copy() we would step through 1G pages 2M at a time. - Copy 1G mappings in pmap_copy(). Reviewed by: alc, kib MFC with: r365518 Sponsored by: Juniper Networks, Inc., Klara, Inc. Differential Revision: https://reviews.freebsd.org/D26463
Notes
Notes: svn path=/head/; revision=365906
-rw-r--r--sys/amd64/amd64/pmap.c112
1 files changed, 66 insertions, 46 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index d04d2457a1da..f5a129f50d21 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -5935,22 +5935,20 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
continue;
}
- pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
va_next = (sva + NBPDP) & ~PDPMASK;
- if ((*pdpe & PG_V) == 0) {
- if (va_next < sva)
- va_next = eva;
+ if (va_next < sva)
+ va_next = eva;
+ pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
+ if ((*pdpe & PG_V) == 0)
continue;
- }
-
- KASSERT((*pdpe & PG_PS) == 0 || va_next <= eva,
- ("pmap_remove of non-transient 1G page "
- "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
- *pdpe, sva, eva, va_next));
if ((*pdpe & PG_PS) != 0) {
+ KASSERT(va_next <= eva,
+ ("partial update of non-transparent 1G mapping "
+ "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
+ *pdpe, sva, eva, va_next));
MPASS(pmap != kernel_pmap); /* XXXKIB */
MPASS((*pdpe & (PG_MANAGED | PG_G)) == 0);
- anyvalid = 1;
+ anyvalid = 1;
*pdpe = 0;
pmap_resident_count_dec(pmap, NBPDP / PAGE_SIZE);
mt = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, sva) & PG_FRAME);
@@ -6228,19 +6226,17 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
continue;
}
- pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
va_next = (sva + NBPDP) & ~PDPMASK;
- if ((*pdpe & PG_V) == 0) {
- if (va_next < sva)
- va_next = eva;
+ if (va_next < sva)
+ va_next = eva;
+ pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
+ if ((*pdpe & PG_V) == 0)
continue;
- }
-
- KASSERT((*pdpe & PG_PS) == 0 || va_next <= eva,
- ("pmap_remove of non-transient 1G page "
- "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
- *pdpe, sva, eva, va_next));
if ((*pdpe & PG_PS) != 0) {
+ KASSERT(va_next <= eva,
+ ("partial update of non-transparent 1G mapping "
+ "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
+ *pdpe, sva, eva, va_next));
retry_pdpe:
obits = pbits = *pdpe;
MPASS((pbits & (PG_MANAGED | PG_G)) == 0);
@@ -7373,17 +7369,18 @@ pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
va_next = eva;
continue;
}
- pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
+
va_next = (sva + NBPDP) & ~PDPMASK;
if (va_next < sva)
va_next = eva;
+ pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
if ((*pdpe & PG_V) == 0)
continue;
- KASSERT((*pdpe & PG_PS) == 0 || va_next <= eva,
- ("pmap_unwire of non-transient 1G page "
- "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
- *pdpe, sva, eva, va_next));
if ((*pdpe & PG_PS) != 0) {
+ KASSERT(va_next <= eva,
+ ("partial update of non-transparent 1G mapping "
+ "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
+ *pdpe, sva, eva, va_next));
MPASS(pmap != kernel_pmap); /* XXXKIB */
MPASS((*pdpe & (PG_MANAGED | PG_G)) == 0);
atomic_clear_long(pdpe, PG_W);
@@ -7498,21 +7495,42 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
continue;
}
+ va_next = (addr + NBPDP) & ~PDPMASK;
+ if (va_next < addr)
+ va_next = end_addr;
pdpe = pmap_pml4e_to_pdpe(pml4e, addr);
- if ((*pdpe & PG_V) == 0) {
- va_next = (addr + NBPDP) & ~PDPMASK;
- if (va_next < addr)
- va_next = end_addr;
+ if ((*pdpe & PG_V) == 0)
+ continue;
+ if ((*pdpe & PG_PS) != 0) {
+ KASSERT(va_next <= end_addr,
+ ("partial update of non-transparent 1G mapping "
+ "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
+ *pdpe, addr, end_addr, va_next));
+ MPASS((addr & PDPMASK) == 0);
+ MPASS((*pdpe & PG_MANAGED) == 0);
+ srcptepaddr = *pdpe;
+ pdpe = pmap_pdpe(dst_pmap, addr);
+ if (pdpe == NULL) {
+ if (_pmap_allocpte(dst_pmap,
+ pmap_pml4e_pindex(addr), NULL, addr) ==
+ NULL)
+ break;
+ pdpe = pmap_pdpe(dst_pmap, addr);
+ } else {
+ pml4e = pmap_pml4e(dst_pmap, addr);
+ dst_pdpg = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME);
+ dst_pdpg->ref_count++;
+ }
+ KASSERT(*pdpe == 0,
+ ("1G mapping present in dst pmap "
+ "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
+ *pdpe, addr, end_addr, va_next));
+ *pdpe = srcptepaddr & ~PG_W;
+ pmap_resident_count_inc(dst_pmap, NBPDP / PAGE_SIZE);
continue;
}
va_next = (addr + NBPDR) & ~PDRMASK;
- KASSERT((*pdpe & PG_PS) == 0 || va_next <= end_addr,
- ("pmap_copy of partial non-transient 1G page "
- "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
- *pdpe, addr, end_addr, va_next));
- if ((*pdpe & PG_PS) != 0)
- continue;
if (va_next < addr)
va_next = end_addr;
@@ -8559,22 +8577,24 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
va_next = eva;
continue;
}
+
+ va_next = (sva + NBPDP) & ~PDPMASK;
+ if (va_next < sva)
+ va_next = eva;
pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
- if ((*pdpe & PG_V) == 0) {
- va_next = (sva + NBPDP) & ~PDPMASK;
- if (va_next < sva)
- va_next = eva;
+ if ((*pdpe & PG_V) == 0)
+ continue;
+ if ((*pdpe & PG_PS) != 0) {
+ KASSERT(va_next <= eva,
+ ("partial update of non-transparent 1G mapping "
+ "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
+ *pdpe, sva, eva, va_next));
continue;
}
+
va_next = (sva + NBPDR) & ~PDRMASK;
if (va_next < sva)
va_next = eva;
- KASSERT((*pdpe & PG_PS) == 0 || va_next <= eva,
- ("pmap_advise of non-transient 1G page "
- "pdpe %#lx sva %#lx eva %#lx va_next %#lx",
- *pdpe, sva, eva, va_next));
- if ((*pdpe & PG_PS) != 0)
- continue;
pde = pmap_pdpe_to_pde(pdpe, sva);
oldpde = *pde;
if ((oldpde & PG_V) == 0)