diff options
Diffstat (limited to 'sys/amd64/include/pmap.h')
-rw-r--r-- | sys/amd64/include/pmap.h | 100 |
1 files changed, 47 insertions, 53 deletions
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 3d51803d82b7..591244e19b7c 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -38,10 +38,6 @@ * map the page tables using the pagetables themselves. This is done to * reduce the impact on kernel virtual memory for lots of sparse address * space, and to reduce the cost of memory to each process. - * - * from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90 - * from: @(#)pmap.h 7.4 (Berkeley) 5/12/91 - * $FreeBSD$ */ #ifdef __i386__ @@ -130,7 +126,7 @@ * (PTE) page mappings have identical settings for the following fields: */ #define PG_PTE_PROMOTE (PG_NX | PG_MANAGED | PG_W | PG_G | PG_PTE_CACHE | \ - PG_M | PG_A | PG_U | PG_RW | PG_V | PG_PKU_MASK) + PG_M | PG_U | PG_RW | PG_V | PG_PKU_MASK) /* * Page Protection Exception bits @@ -286,11 +282,14 @@ #ifndef LOCORE +#include <sys/kassert.h> #include <sys/queue.h> #include <sys/_cpuset.h> #include <sys/_lock.h> #include <sys/_mutex.h> #include <sys/_pctrie.h> +#include <machine/_pmap.h> +#include <sys/_pv_entry.h> #include <sys/_rangeset.h> #include <sys/_smr.h> @@ -353,8 +352,6 @@ extern pt_entry_t pg_nx; /* * Pmap stuff */ -struct pv_entry; -struct pv_chunk; /* * Locks @@ -372,11 +369,6 @@ enum pmap_type { PT_RVI, /* AMD's nested page tables */ }; -struct pmap_pcids { - uint32_t pm_pcid; - uint32_t pm_gen; -}; - /* * The kernel virtual address (KVA) of the level 4 page table page is always * within the direct map (DMAP) region. @@ -395,7 +387,7 @@ struct pmap { long pm_eptgen; /* EPT pmap generation id */ smr_t pm_eptsmr; int pm_flags; - struct pmap_pcids pm_pcids[MAXCPU]; + struct pmap_pcid *pm_pcidp; struct rangeset pm_pkru; }; @@ -424,39 +416,6 @@ extern struct pmap kernel_pmap_store; int pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags); int pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype); -#endif - -/* - * For each vm_page_t, there is a list of all currently valid virtual - * mappings of that page. An entry is a pv_entry_t, the list is pv_list. - */ -typedef struct pv_entry { - vm_offset_t pv_va; /* virtual address for mapping */ - TAILQ_ENTRY(pv_entry) pv_next; -} *pv_entry_t; - -/* - * pv_entries are allocated in chunks per-process. This avoids the - * need to track per-pmap assignments. - */ -#define _NPCM 3 -#define _NPCPV 168 -#define PV_CHUNK_HEADER \ - pmap_t pc_pmap; \ - TAILQ_ENTRY(pv_chunk) pc_list; \ - uint64_t pc_map[_NPCM]; /* bitmap; 1 = free */ \ - TAILQ_ENTRY(pv_chunk) pc_lru; - -struct pv_chunk_header { - PV_CHUNK_HEADER -}; - -struct pv_chunk { - PV_CHUNK_HEADER - struct pv_entry pc_pventry[_NPCPV]; -}; - -#ifdef _KERNEL extern caddr_t CADDR1; extern pt_entry_t *CMAP1; @@ -465,6 +424,8 @@ extern vm_offset_t virtual_end; extern vm_paddr_t dmaplimit; extern int pmap_pcid_enabled; extern int invpcid_works; +extern int pmap_pcid_invlpg_workaround; +extern int pmap_pcid_invlpg_workaround_uena; #define pmap_page_get_memattr(m) ((vm_memattr_t)(m)->md.pat_mode) #define pmap_page_is_write_mapped(m) (((m)->a.flags & PGA_WRITEABLE) != 0) @@ -482,10 +443,10 @@ void pmap_activate_boot(pmap_t pmap); void pmap_activate_sw(struct thread *); void pmap_allow_2m_x_ept_recalculate(void); void pmap_bootstrap(vm_paddr_t *); -int pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde); +int pmap_cache_bits(pmap_t pmap, int mode, bool is_pde); int pmap_change_attr(vm_offset_t, vm_size_t, int); int pmap_change_prot(vm_offset_t, vm_size_t, vm_prot_t); -void pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, boolean_t invalidate); +void pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, bool invalidate); void pmap_flush_cache_range(vm_offset_t, vm_offset_t); void pmap_flush_cache_phys_range(vm_paddr_t, vm_paddr_t, vm_memattr_t); void pmap_init_pat(void); @@ -501,13 +462,13 @@ void *pmap_mapdev(vm_paddr_t, vm_size_t); void *pmap_mapdev_attr(vm_paddr_t, vm_size_t, int); void *pmap_mapdev_pciecfg(vm_paddr_t pa, vm_size_t size); bool pmap_not_in_di(void); -boolean_t pmap_page_is_mapped(vm_page_t m); +bool pmap_page_is_mapped(vm_page_t m); void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma); void pmap_page_set_memattr_noflush(vm_page_t m, vm_memattr_t ma); void pmap_pinit_pml4(vm_page_t); void pmap_pinit_pml5(vm_page_t); bool pmap_ps_enabled(pmap_t pmap); -void pmap_unmapdev(vm_offset_t, vm_size_t); +void pmap_unmapdev(void *, vm_size_t); void pmap_invalidate_page(pmap_t, vm_offset_t); void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t); void pmap_invalidate_all(pmap_t); @@ -516,8 +477,9 @@ void pmap_invalidate_cache_pages(vm_page_t *pages, int count); void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva); void pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva); void pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num); -boolean_t pmap_map_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t); -void pmap_unmap_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t); +bool pmap_map_io_transient(vm_page_t *, vm_offset_t *, int, bool); +void pmap_unmap_io_transient(vm_page_t *, vm_offset_t *, int, bool); +void pmap_map_delete(pmap_t, vm_offset_t, vm_offset_t); void pmap_pti_add_kva(vm_offset_t sva, vm_offset_t eva, bool exec); void pmap_pti_remove_kva(vm_offset_t sva, vm_offset_t eva); void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3); @@ -533,7 +495,6 @@ void pmap_page_array_startup(long count); vm_page_t pmap_page_alloc_below_4g(bool zeroed); #if defined(KASAN) || defined(KMSAN) -void pmap_san_bootstrap(void); void pmap_san_enter(vm_offset_t); #endif @@ -548,6 +509,39 @@ pmap_invalidate_cpu_mask(pmap_t pmap) return (&pmap->pm_active); } +#if defined(_SYS_PCPU_H_) && defined(_MACHINE_CPUFUNC_H_) +/* + * It seems that AlderLake+ small cores have some microarchitectural + * bug, which results in the INVLPG instruction failing to flush all + * global TLB entries when PCID is enabled. Work around it for now, + * by doing global invalidation on small cores instead of INVLPG. + */ +static __inline void +pmap_invlpg(pmap_t pmap, vm_offset_t va) +{ + if (pmap == kernel_pmap && PCPU_GET(pcid_invlpg_workaround)) { + struct invpcid_descr d = { 0 }; + + invpcid(&d, INVPCID_CTXGLOB); + } else { + invlpg(va); + } +} +#endif /* sys/pcpu.h && machine/cpufunc.h */ + +#if defined(_SYS_PCPU_H_) +/* Return pcid for the pmap pmap on current cpu */ +static __inline uint32_t +pmap_get_pcid(pmap_t pmap) +{ + struct pmap_pcid *pcidp; + + MPASS(pmap_pcid_enabled); + pcidp = zpcpu_get(pmap->pm_pcidp); + return (pcidp->pm_pcid); +} +#endif /* sys/pcpu.h */ + #endif /* _KERNEL */ /* Return various clipped indexes for a given VA */ |