diff options
author | Jeff Roberson <jeff@FreeBSD.org> | 2019-08-18 23:44:23 +0000 |
---|---|---|
committer | Jeff Roberson <jeff@FreeBSD.org> | 2019-08-18 23:44:23 +0000 |
commit | a5e5548c88909938c2e38a754c747efb392acc64 (patch) | |
tree | 47c2319b67051c08335a311680e0ec08ec672e51 | |
parent | 6565f65ce17a1df1a782b39ff5a911dd4b69fac8 (diff) | |
download | src-a5e5548c88909938c2e38a754c747efb392acc64.tar.gz src-a5e5548c88909938c2e38a754c747efb392acc64.zip |
Allocate all per-cpu datastructures in domain correct memory.
Reviewed by: kib, gallatin (some objections)
Tested by: pho
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D21242
Notes
Notes:
svn path=/head/; revision=351200
-rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 67 | ||||
-rw-r--r-- | sys/dev/acpica/acpi_pxm.c | 11 | ||||
-rw-r--r-- | sys/dev/acpica/acpivar.h | 1 |
3 files changed, 68 insertions, 11 deletions
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 3c402b227ea9..3f7f17791d0e 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/bus.h> #include <sys/cpuset.h> +#include <sys/domainset.h> #ifdef GPROF #include <sys/gmon.h> #endif @@ -59,6 +60,8 @@ __FBSDID("$FreeBSD$"); #include <vm/pmap.h> #include <vm/vm_kern.h> #include <vm/vm_extern.h> +#include <vm/vm_page.h> +#include <vm/vm_phys.h> #include <x86/apicreg.h> #include <machine/clock.h> @@ -75,6 +78,9 @@ __FBSDID("$FreeBSD$"); #include <machine/cpu.h> #include <x86/init.h> +#include <contrib/dev/acpica/include/acpi.h> +#include <dev/acpica/acpivar.h> + #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) @@ -384,6 +390,25 @@ init_secondary(void) * local functions and data */ +#ifdef NUMA +static void +mp_realloc_pcpu(int cpuid, int domain) +{ + vm_page_t m; + vm_offset_t oa, na; + + oa = (vm_offset_t)&__pcpu[cpuid]; + if (_vm_phys_domain(pmap_kextract(oa)) == domain) + return; + m = vm_page_alloc_domain(NULL, 0, domain, + VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_ZERO); + na = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); + pagecopy((void *)oa, (void *)na); + pmap_enter(kernel_pmap, oa, m, VM_PROT_READ | VM_PROT_WRITE, 0, 0); + /* XXX old pcpu page leaked. */ +} +#endif + /* * start each AP in our list */ @@ -392,7 +417,7 @@ native_start_all_aps(void) { u_int64_t *pt4, *pt3, *pt2; u_int32_t mpbioswarmvec; - int apic_id, cpu, i; + int apic_id, cpu, domain, i; u_char mpbiosreason; mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); @@ -431,21 +456,41 @@ native_start_all_aps(void) outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ + /* Relocate pcpu areas to the correct domain. */ +#ifdef NUMA + if (vm_ndomains > 1) + for (cpu = 1; cpu < mp_ncpus; cpu++) { + apic_id = cpu_apic_ids[cpu]; + domain = acpi_pxm_get_cpu_locality(apic_id); + mp_realloc_pcpu(cpu, domain); + } +#endif + /* start each AP */ + domain = 0; for (cpu = 1; cpu < mp_ncpus; cpu++) { apic_id = cpu_apic_ids[cpu]; - +#ifdef NUMA + if (vm_ndomains > 1) + domain = acpi_pxm_get_cpu_locality(apic_id); +#endif /* allocate and set up an idle stack data page */ - bootstacks[cpu] = (void *)kmem_malloc(kstack_pages * PAGE_SIZE, + bootstacks[cpu] = (void *)kmem_malloc_domainset( + DOMAINSET_FIXED(domain), kstack_pages * PAGE_SIZE, M_WAITOK | M_ZERO); - doublefault_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | - M_ZERO); - mce_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO); - nmi_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO); - dbg_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO); - dpcpu = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); - - bootSTK = (char *)bootstacks[cpu] + kstack_pages * PAGE_SIZE - 8; + doublefault_stack = (char *)kmem_malloc_domainset( + DOMAINSET_FIXED(domain), PAGE_SIZE, M_WAITOK | M_ZERO); + mce_stack = (char *)kmem_malloc_domainset( + DOMAINSET_FIXED(domain), PAGE_SIZE, M_WAITOK | M_ZERO); + nmi_stack = (char *)kmem_malloc_domainset( + DOMAINSET_FIXED(domain), PAGE_SIZE, M_WAITOK | M_ZERO); + dbg_stack = (char *)kmem_malloc_domainset( + DOMAINSET_FIXED(domain), PAGE_SIZE, M_WAITOK | M_ZERO); + dpcpu = (void *)kmem_malloc_domainset(DOMAINSET_FIXED(domain), + DPCPU_SIZE, M_WAITOK | M_ZERO); + + bootSTK = (char *)bootstacks[cpu] + + kstack_pages * PAGE_SIZE - 8; bootAP = cpu; /* attempt to start the Application Processor */ diff --git a/sys/dev/acpica/acpi_pxm.c b/sys/dev/acpica/acpi_pxm.c index a4dbbfbf2a4b..a0fb5554096c 100644 --- a/sys/dev/acpica/acpi_pxm.c +++ b/sys/dev/acpica/acpi_pxm.c @@ -653,6 +653,17 @@ acpi_pxm_set_cpu_locality(void) } } +int +acpi_pxm_get_cpu_locality(int apic_id) +{ + struct cpu_info *cpu; + + cpu = cpu_find(apic_id); + if (cpu == NULL) + panic("SRAT: CPU with ID %u is not known", apic_id); + return (cpu->domain); +} + /* * Free data structures allocated during acpi_pxm_init. */ diff --git a/sys/dev/acpica/acpivar.h b/sys/dev/acpica/acpivar.h index ec1cee6fb623..e6f10cb4e41d 100644 --- a/sys/dev/acpica/acpivar.h +++ b/sys/dev/acpica/acpivar.h @@ -532,6 +532,7 @@ int acpi_pxm_init(int ncpus, vm_paddr_t maxphys); void acpi_pxm_parse_tables(void); void acpi_pxm_set_mem_locality(void); void acpi_pxm_set_cpu_locality(void); +int acpi_pxm_get_cpu_locality(int apic_id); void acpi_pxm_free(void); /* |