diff options
Diffstat (limited to 'sys/arm64')
48 files changed, 1580 insertions, 631 deletions
diff --git a/sys/arm64/arm64/cpu_errata.c b/sys/arm64/arm64/cpu_errata.c index 989924bc0567..b876703a2a15 100644 --- a/sys/arm64/arm64/cpu_errata.c +++ b/sys/arm64/arm64/cpu_errata.c @@ -52,56 +52,11 @@ struct cpu_quirks { u_int flags; }; -static enum { - SSBD_FORCE_ON, - SSBD_FORCE_OFF, - SSBD_KERNEL, -} ssbd_method = SSBD_KERNEL; - -static cpu_quirk_install install_psci_bp_hardening; -static cpu_quirk_install install_ssbd_workaround; static cpu_quirk_install install_thunderx_bcast_tlbi_workaround; static struct cpu_quirks cpu_quirks[] = { { .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, - .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A57,0,0), - .quirk_install = install_psci_bp_hardening, - .flags = CPU_QUIRK_POST_DEVICE, - }, - { - .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, - .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A72,0,0), - .quirk_install = install_psci_bp_hardening, - .flags = CPU_QUIRK_POST_DEVICE, - }, - { - .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, - .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A73,0,0), - .quirk_install = install_psci_bp_hardening, - .flags = CPU_QUIRK_POST_DEVICE, - }, - { - .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, - .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A75,0,0), - .quirk_install = install_psci_bp_hardening, - .flags = CPU_QUIRK_POST_DEVICE, - }, - { - .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, - .midr_value = - CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX2, 0,0), - .quirk_install = install_psci_bp_hardening, - .flags = CPU_QUIRK_POST_DEVICE, - }, - { - .midr_mask = 0, - .midr_value = 0, - .quirk_install = install_ssbd_workaround, - .flags = CPU_QUIRK_POST_DEVICE, - }, - { - .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, .midr_value = CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX, 0, 0), .quirk_install = install_thunderx_bcast_tlbi_workaround, @@ -114,57 +69,6 @@ static struct cpu_quirks cpu_quirks[] = { }, }; -static void -install_psci_bp_hardening(void) -{ - /* SMCCC depends on PSCI. If PSCI is missing so is SMCCC */ - if (!psci_present) - return; - - if (smccc_arch_features(SMCCC_ARCH_WORKAROUND_1) != SMCCC_RET_SUCCESS) - return; - - PCPU_SET(bp_harden, smccc_arch_workaround_1); -} - -static void -install_ssbd_workaround(void) -{ - char *env; - - if (PCPU_GET(cpuid) == 0) { - env = kern_getenv("kern.cfg.ssbd"); - if (env != NULL) { - if (strcmp(env, "force-on") == 0) { - ssbd_method = SSBD_FORCE_ON; - } else if (strcmp(env, "force-off") == 0) { - ssbd_method = SSBD_FORCE_OFF; - } - } - } - - /* SMCCC depends on PSCI. If PSCI is missing so is SMCCC */ - if (!psci_present) - return; - - /* Enable the workaround on this CPU if it's enabled in the firmware */ - if (smccc_arch_features(SMCCC_ARCH_WORKAROUND_2) != SMCCC_RET_SUCCESS) - return; - - switch(ssbd_method) { - case SSBD_FORCE_ON: - smccc_arch_workaround_2(1); - break; - case SSBD_FORCE_OFF: - smccc_arch_workaround_2(0); - break; - case SSBD_KERNEL: - default: - PCPU_SET(ssbd, smccc_arch_workaround_2); - break; - } -} - /* * Workaround Cavium erratum 27456. * diff --git a/sys/arm64/arm64/cpu_feat.c b/sys/arm64/arm64/cpu_feat.c index cc262394913d..94114d47f846 100644 --- a/sys/arm64/arm64/cpu_feat.c +++ b/sys/arm64/arm64/cpu_feat.c @@ -32,16 +32,21 @@ #include <machine/cpu.h> #include <machine/cpu_feat.h> +SYSCTL_NODE(_hw, OID_AUTO, feat, CTLFLAG_RD, 0, "CPU features/errata"); + /* TODO: Make this a list if we ever grow a callback other than smccc_errata */ static cpu_feat_errata_check_fn cpu_feat_check_cb = NULL; void enable_cpu_feat(uint32_t stage) { + char tunable[32]; struct cpu_feat **featp, *feat; uint32_t midr; u_int errata_count, *errata_list; cpu_feat_errata errata_status; + cpu_feat_en check_status; + bool val; MPASS((stage & ~CPU_FEAT_STAGE_MASK) == 0); @@ -49,6 +54,21 @@ enable_cpu_feat(uint32_t stage) SET_FOREACH(featp, cpu_feat_set) { feat = *featp; + /* Read any tunable the user may have set */ + if (stage == CPU_FEAT_EARLY_BOOT && PCPU_GET(cpuid) == 0) { + snprintf(tunable, sizeof(tunable), "hw.feat.%s", + feat->feat_name); + if (TUNABLE_BOOL_FETCH(tunable, &val)) { + if (val) { + feat->feat_flags |= + CPU_FEAT_USER_ENABLED; + } else { + feat->feat_flags |= + CPU_FEAT_USER_DISABLED; + } + } + } + /* Run the enablement code at the correct stage of boot */ if ((feat->feat_flags & CPU_FEAT_STAGE_MASK) != stage) continue; @@ -58,8 +78,26 @@ enable_cpu_feat(uint32_t stage) PCPU_GET(cpuid) != 0) continue; - if (feat->feat_check != NULL && !feat->feat_check(feat, midr)) - continue; + if (feat->feat_check != NULL) { + check_status = feat->feat_check(feat, midr); + } else { + check_status = FEAT_DEFAULT_ENABLE; + } + /* Ignore features that are not present */ + if (check_status == FEAT_ALWAYS_DISABLE) + goto next; + + /* The user disabled the feature */ + if ((feat->feat_flags & CPU_FEAT_USER_DISABLED) != 0) + goto next; + + /* + * The feature was disabled by default and the user + * didn't enable it then skip. + */ + if (check_status == FEAT_DEFAULT_DISABLE && + (feat->feat_flags & CPU_FEAT_USER_ENABLED) == 0) + goto next; /* * Check if the feature has any errata that may need a @@ -97,8 +135,13 @@ enable_cpu_feat(uint32_t stage) /* Shouldn't be possible */ MPASS(errata_status != ERRATA_UNKNOWN); - feat->feat_enable(feat, errata_status, errata_list, - errata_count); + if (feat->feat_enable(feat, errata_status, errata_list, + errata_count)) + feat->feat_enabled = true; + +next: + if (!feat->feat_enabled && feat->feat_disabled != NULL) + feat->feat_disabled(feat); } } diff --git a/sys/arm64/arm64/db_disasm.c b/sys/arm64/arm64/db_disasm.c index ab1002560b20..14ae2acc2ce6 100644 --- a/sys/arm64/arm64/db_disasm.c +++ b/sys/arm64/arm64/db_disasm.c @@ -31,6 +31,7 @@ #include <ddb/db_access.h> #include <ddb/db_sym.h> +#include <machine/armreg.h> #include <machine/disassem.h> static u_int db_disasm_read_word(vm_offset_t); diff --git a/sys/arm64/arm64/efirt_machdep.c b/sys/arm64/arm64/efirt_machdep.c index 0301eb91c9ef..bde0d4f784dc 100644 --- a/sys/arm64/arm64/efirt_machdep.c +++ b/sys/arm64/arm64/efirt_machdep.c @@ -106,7 +106,8 @@ efi_1t1_l3(vm_offset_t va) if (*l0 == 0) { m = efi_1t1_page(); mphys = VM_PAGE_TO_PHYS(m); - *l0 = PHYS_TO_PTE(mphys) | L0_TABLE; + *l0 = PHYS_TO_PTE(mphys) | TATTR_UXN_TABLE | + TATTR_AP_TABLE_NO_EL0 | L0_TABLE; } else { mphys = PTE_TO_PHYS(*l0); } @@ -117,7 +118,8 @@ efi_1t1_l3(vm_offset_t va) if (*l1 == 0) { m = efi_1t1_page(); mphys = VM_PAGE_TO_PHYS(m); - *l1 = PHYS_TO_PTE(mphys) | L1_TABLE; + *l1 = PHYS_TO_PTE(mphys) | TATTR_UXN_TABLE | + TATTR_AP_TABLE_NO_EL0 | L1_TABLE; } else { mphys = PTE_TO_PHYS(*l1); } @@ -128,7 +130,8 @@ efi_1t1_l3(vm_offset_t va) if (*l2 == 0) { m = efi_1t1_page(); mphys = VM_PAGE_TO_PHYS(m); - *l2 = PHYS_TO_PTE(mphys) | L2_TABLE; + *l2 = PHYS_TO_PTE(mphys) | TATTR_UXN_TABLE | + TATTR_AP_TABLE_NO_EL0 | L2_TABLE; } else { mphys = PTE_TO_PHYS(*l2); } @@ -218,8 +221,9 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz) p->md_phys, mode, p->md_pages); } - l3_attr = ATTR_AF | pmap_sh_attr | ATTR_S1_IDX(mode) | - ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_nG | L3_PAGE; + l3_attr = ATTR_S1_UXN | ATTR_AF | pmap_sh_attr | + ATTR_S1_IDX(mode) | ATTR_S1_AP(ATTR_S1_AP_RW) | + ATTR_S1_nG | L3_PAGE; if (mode == VM_MEMATTR_DEVICE || p->md_attr & EFI_MD_ATTR_XP) l3_attr |= ATTR_S1_XN; diff --git a/sys/arm64/arm64/elf32_machdep.c b/sys/arm64/arm64/elf32_machdep.c index 5c81c6cdce3d..4cb8ee5f57ef 100644 --- a/sys/arm64/arm64/elf32_machdep.c +++ b/sys/arm64/arm64/elf32_machdep.c @@ -210,7 +210,7 @@ freebsd32_fetch_syscall_args(struct thread *td) sa->code = *ap++; nap--; } else if (sa->code == SYS___syscall) { - sa->code = ap[1]; + sa->code = ap[_QUAD_LOWWORD]; nap -= 2; ap += 2; } @@ -225,7 +225,7 @@ freebsd32_fetch_syscall_args(struct thread *td) sa->args[i] = ap[i]; if (narg > nap) { if (narg - nap > nitems(args)) - panic("Too many system call arguiments"); + panic("Too many system call arguments"); error = copyin((void *)td->td_frame->tf_x[13], args, (narg - nap) * sizeof(int)); if (error != 0) diff --git a/sys/arm64/arm64/elf_machdep.c b/sys/arm64/arm64/elf_machdep.c index 13af5c5065d6..207b37180a26 100644 --- a/sys/arm64/arm64/elf_machdep.c +++ b/sys/arm64/arm64/elf_machdep.c @@ -121,7 +121,7 @@ static struct sysentvec elf64_freebsd_sysvec = { }; INIT_SYSENTVEC(elf64_sysvec, &elf64_freebsd_sysvec); -static Elf64_Brandinfo freebsd_brand_info = { +static const Elf64_Brandinfo freebsd_brand_info = { .brand = ELFOSABI_FREEBSD, .machine = EM_AARCH64, .compat_3_brand = "FreeBSD", @@ -131,8 +131,7 @@ static Elf64_Brandinfo freebsd_brand_info = { .brand_note = &elf64_freebsd_brandnote, .flags = BI_CAN_EXEC_DYN | BI_BRAND_NOTE }; - -SYSINIT(elf64, SI_SUB_EXEC, SI_ORDER_FIRST, +C_SYSINIT(elf64, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)elf64_insert_brand_entry, &freebsd_brand_info); static bool @@ -336,7 +335,7 @@ elf_cpu_parse_dynamic(caddr_t loadbase __unused, Elf_Dyn *dynamic __unused) return (0); } -static Elf_Note gnu_property_note = { +static const Elf_Note gnu_property_note = { .n_namesz = sizeof(GNU_ABI_VENDOR), .n_descsz = 16, .n_type = NT_GNU_PROPERTY_TYPE_0, diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S index 13095def8b00..5a4181348a54 100644 --- a/sys/arm64/arm64/exception.S +++ b/sys/arm64/arm64/exception.S @@ -42,10 +42,9 @@ */ .macro save_registers_head el .if \el == 1 - mov x18, sp - stp x0, x1, [sp, #(TF_X - TF_SIZE - 128)]! + stp x0, x1, [sp, #-(TF_SIZE - TF_X + 128)]! .else - stp x0, x1, [sp, #(TF_X - TF_SIZE)]! + stp x0, x1, [sp, #-(TF_SIZE - TF_X)]! .endif stp x2, x3, [sp, #(2 * 8)] stp x4, x5, [sp, #(4 * 8)] @@ -61,7 +60,9 @@ stp x24, x25, [sp, #(24 * 8)] stp x26, x27, [sp, #(26 * 8)] stp x28, x29, [sp, #(28 * 8)] -.if \el == 0 +.if \el == 1 + add x18, sp, #(TF_SIZE - TF_X + 128) +.else mrs x18, sp_el0 .endif mrs x10, elr_el1 diff --git a/sys/arm64/arm64/identcpu.c b/sys/arm64/arm64/identcpu.c index bcacea43ad2f..2d07420bcdb0 100644 --- a/sys/arm64/arm64/identcpu.c +++ b/sys/arm64/arm64/identcpu.c @@ -232,6 +232,10 @@ static const struct cpu_parts cpu_parts_arm[] = { { CPU_PART_CORTEX_X2, "Cortex-X2" }, { CPU_PART_CORTEX_X3, "Cortex-X3" }, { CPU_PART_CORTEX_X4, "Cortex-X4" }, + { CPU_PART_C1_NANO, "C1-Nano" }, + { CPU_PART_C1_PRO, "C1-Pro" }, + { CPU_PART_C1_PREMIUM, "C1-Premium" }, + { CPU_PART_C1_ULTRA, "C1-Ultra" }, { CPU_PART_NEOVERSE_E1, "Neoverse-E1" }, { CPU_PART_NEOVERSE_N1, "Neoverse-N1" }, { CPU_PART_NEOVERSE_N2, "Neoverse-N2" }, @@ -2272,37 +2276,25 @@ static const struct mrs_user_reg user_regs[] = { static bool user_ctr_has_neoverse_n1_1542419(uint32_t midr, uint64_t ctr) { - /* Skip non-Neoverse-N1 */ - if (!CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK, CPU_IMPL_ARM, - CPU_PART_NEOVERSE_N1, 0, 0)) - return (false); - - switch (CPU_VAR(midr)) { - default: - break; - case 4: - /* Fixed in r4p1 */ - if (CPU_REV(midr) > 0) - break; - /* FALLTHROUGH */ - case 3: - /* If DIC is enabled (coherent icache) then we are affected */ - return (CTR_DIC_VAL(ctr) != 0); - } - - return (false); + /* + * Neoverse-N1 erratum 1542419 + * Present in r3p0 - r4p0 + * Fixed in r4p1 + */ + return (midr_check_var_part_range(midr, CPU_IMPL_ARM, + CPU_PART_NEOVERSE_N1, 3, 0, 4, 0) && CTR_DIC_VAL(ctr) != 0); } -static bool -user_ctr_check(const struct cpu_feat *feat __unused, u_int midr __unused) +static cpu_feat_en +user_ctr_check(const struct cpu_feat *feat __unused, u_int midr) { if (emulate_ctr) - return (true); + return (FEAT_DEFAULT_ENABLE); if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0))) - return (true); + return (FEAT_DEFAULT_ENABLE); - return (false); + return (FEAT_ALWAYS_DISABLE); } static bool @@ -2320,7 +2312,7 @@ user_ctr_has_errata(const struct cpu_feat *feat __unused, u_int midr, return (false); } -static void +static bool user_ctr_enable(const struct cpu_feat *feat __unused, cpu_feat_errata errata_status, u_int *errata_list, u_int errata_count) { @@ -2356,16 +2348,13 @@ user_ctr_enable(const struct cpu_feat *feat __unused, WRITE_SPECIALREG(sctlr_el1, READ_SPECIALREG(sctlr_el1) & ~SCTLR_UCT); isb(); + + return (true); } -static struct cpu_feat user_ctr = { - .feat_name = "Trap CTR_EL0", - .feat_check = user_ctr_check, - .feat_has_errata = user_ctr_has_errata, - .feat_enable = user_ctr_enable, - .feat_flags = CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU, -}; -DATA_SET(cpu_feat_set, user_ctr); +CPU_FEAT(trap_ctr, "Trap CTR_EL0", + user_ctr_check, user_ctr_has_errata, user_ctr_enable, NULL, + CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); static bool user_ctr_handler(uint64_t esr, struct trapframe *frame) diff --git a/sys/arm64/arm64/kexec_support.c b/sys/arm64/arm64/kexec_support.c new file mode 100644 index 000000000000..8b9719c05b67 --- /dev/null +++ b/sys/arm64/arm64/kexec_support.c @@ -0,0 +1,188 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Juniper Networks, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/systm.h> +#include <sys/kexec.h> +#include <vm/vm.h> +#include <vm/vm_extern.h> +#include <vm/vm_object.h> +#include <vm/vm_phys.h> +#include <vm/vm_radix.h> +#include <vm/pmap.h> +#include <vm/vm_page.h> + +#include <machine/armreg.h> +#include <machine/pmap.h> +#include <machine/pte.h> + +/* + * Idea behind this: + * + * kexec_load_md(): + * - Update boot page tables (identity map) to include all pages needed before + * disabling MMU. + * + * kexec_reboot_md(): + * - Copy pages into target(s) + * - Do "other stuff" + * - Does not return + */ + +extern pt_entry_t pagetable_l0_ttbr0_bootstrap[]; +extern unsigned long initstack_end[]; +void switch_stack(void *, void (*)(void *, void *, struct kexec_image *), void *); + +#define SCTLR_EL1_NO_MMU (SCTLR_RES1 | SCTLR_LSMAOE | SCTLR_nTLSMD | \ + SCTLR_EIS | SCTLR_TSCXT | SCTLR_EOS) +#define vm_page_offset(m) ((vm_offset_t)(m) - vm_page_base) +static inline vm_page_t +phys_vm_page(vm_page_t m, vm_offset_t vm_page_v, vm_paddr_t vm_page_p) +{ + return ((vm_page_t)((vm_offset_t)m - vm_page_v + vm_page_p)); +} + +/* First 2 args are filler for switch_stack() */ +static void __aligned(16) __dead2 +kexec_reboot_bottom( void *arg1 __unused, void *arg2 __unused, + struct kexec_image *image) +{ + void (*e)(void) = (void *)image->entry; + vm_offset_t vm_page_base = (vm_offset_t)vm_page_array; + vm_paddr_t vm_page_phys = pmap_kextract((vm_offset_t)vm_page_array); + struct kexec_segment_stage *phys_segs = + (void *)pmap_kextract((vm_offset_t)&image->segments); + vm_paddr_t from_pa, to_pa; + vm_size_t size; + vm_page_t first, m, mp; + struct pctrie_iter pct_i; + + /* + * Create a linked list of all pages in the object before we disable the + * MMU. Once the MMU is disabled we can't use the vm_radix iterators, + * as they rely on virtual address pointers. + */ + first = NULL; + vm_radix_iter_init(&pct_i, &image->map_obj->rtree); + VM_RADIX_FORALL(m, &pct_i) { + if (first == NULL) + first = m; + else + SLIST_INSERT_AFTER(mp, m, plinks.s.ss); + mp = m; + } + + /* + * We're running out of the identity map now, disable the MMU before we + * continue. It's possible page tables can be overwritten, which would + * be very bad if we were running with the MMU enabled. + */ + WRITE_SPECIALREG(sctlr_el1, SCTLR_EL1_NO_MMU); + isb(); + for (int i = 0; i < KEXEC_SEGMENT_MAX; i++) { + if (phys_segs[i].size == 0) + break; + to_pa = phys_segs[i].target; + /* Copy the segment here... */ + for (vm_page_t p = phys_segs[i].first_page; + p != NULL && to_pa - phys_segs[i].target < phys_segs[i].size; + p = SLIST_NEXT(p, plinks.s.ss)) { + p = phys_vm_page(p, vm_page_base, vm_page_phys); + from_pa = p->phys_addr; + if (p->phys_addr == to_pa) { + to_pa += PAGE_SIZE; + continue; + } + for (size = PAGE_SIZE / sizeof(register_t); + size > 0; --size) { + *(register_t *)to_pa = *(register_t *)from_pa; + to_pa += sizeof(register_t); + from_pa += sizeof(register_t); + } + } + } + invalidate_icache(); + e(); + while (1) + ; +} + +void +kexec_reboot_md(struct kexec_image *image) +{ + uintptr_t ptr; + register_t reg; + + for (int i = 0; i < KEXEC_SEGMENT_MAX; i++) { + if (image->segments[i].size > 0) + cpu_dcache_inv_range((void *)PHYS_TO_DMAP(image->segments[i].target), + image->segments[i].size); + } + ptr = pmap_kextract((vm_offset_t)kexec_reboot_bottom); + serror_disable(); + + reg = pmap_kextract((vm_offset_t)pagetable_l0_ttbr0_bootstrap); + set_ttbr0(reg); + cpu_tlb_flushID(); + + typeof(kexec_reboot_bottom) *p = (void *)ptr; + switch_stack((void *)pmap_kextract((vm_offset_t)initstack_end), + p, image); + while (1) + ; +} + +int +kexec_load_md(struct kexec_image *image) +{ + vm_paddr_t tmp; + pt_entry_t *pte; + + /* Create L2 page blocks for the trampoline. L0/L1 are from the startup. */ + + /* + * There are exactly 2 pages before the pagetable_l0_ttbr0_bootstrap, so + * move to there. + */ + pte = pagetable_l0_ttbr0_bootstrap; + pte -= (Ln_ENTRIES * 2); /* move to start of L2 pages */ + + /* + * Populate the identity map with symbols we know we'll need before we + * turn off the MMU. + */ + tmp = pmap_kextract((vm_offset_t)kexec_reboot_bottom); + pte[pmap_l2_index(tmp)] = (tmp | L2_BLOCK | ATTR_AF | ATTR_S1_UXN); + tmp = pmap_kextract((vm_offset_t)initstack_end); + pte[pmap_l2_index(tmp)] = (tmp | L2_BLOCK | ATTR_AF | ATTR_S1_UXN); + /* We'll need vm_page_array for doing offset calculations. */ + tmp = pmap_kextract((vm_offset_t)&vm_page_array); + pte[pmap_l2_index(tmp)] = (tmp | L2_BLOCK | ATTR_AF | ATTR_S1_UXN); + + return (0); +} diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S index 4a10a2b4f2d3..3ec12140f139 100644 --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -39,6 +39,23 @@ #define VIRT_BITS 48 +/* + * Loads a 64-bit value into reg using 1 to 4 mov/movk instructions. + * This can be used early on when we don't know the CPUs endianness. + */ +.macro mov_q reg, val + mov \reg, :abs_g0_nc:\val +.if (\val >> 16) & 0xffff != 0 + movk \reg, :abs_g1_nc:\val +.endif +.if (\val >> 32) & 0xffff != 0 + movk \reg, :abs_g2_nc:\val +.endif +.if (\val >> 48) & 0xffff != 0 + movk \reg, :abs_g3:\val +.endif +.endm + #if PAGE_SIZE == PAGE_SIZE_16K /* * The number of level 3 tables to create. 32 will allow for 1G of address @@ -308,6 +325,19 @@ mp_virtdone: b init_secondary LEND(mpentry_common) + +ENTRY(mp_cpu_spinloop) +0: + wfe + ldr x0, mp_cpu_spin_table_release_addr + cbz x0, 0b + blr x0 + .globl mp_cpu_spin_table_release_addr +mp_cpu_spin_table_release_addr: + .quad 0 + .globl mp_cpu_spinloop_end +mp_cpu_spinloop_end: +END(mp_cpu_spinloop) #endif /* @@ -324,15 +354,23 @@ LENTRY(enter_kernel_el) cmp x23, #(CURRENTEL_EL_EL2) b.eq 1f - ldr x2, =SCTLR_MMU_OFF + /* + * Ensure there are no memory operations here. If the boot loader + * enters the kernel in big-endian mode then loading sctlr will + * be incorrect. As instructions are the same in both endians it is + * safe to use mov instructions. + */ + mov_q x2, SCTLR_MMU_OFF msr sctlr_el1, x2 - /* SCTLR_EOS is set so eret is a context synchronizing event so we + /* + * SCTLR_EOS is set to make eret a context synchronizing event. We * need an isb here to ensure it's observed by later instructions, * but don't need it in the eret below. */ isb - /* Ensure SPSR_EL1 and pstate are in sync. The only wat to set the + /* + * Ensure SPSR_EL1 and pstate are in sync. The only way to set the * latter is to set the former and return from an exception with eret. */ mov x2, #(PSR_DAIF | PSR_M_EL1h) @@ -346,11 +384,19 @@ LENTRY(enter_kernel_el) * Set just the reserved bits in sctlr_el2. This will disable the * MMU which may have broken the kernel if we enter the kernel in * EL2, e.g. when using VHE. + * + * As with sctlr_el1 above use mov instructions to ensure there are + * no memory operations. */ - ldr x2, =(SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS) + mov_q x2, (SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS) msr sctlr_el2, x2 isb + /* + * The hardware is now in little-endian mode so memory operations + * are safe. + */ + /* Configure the Hypervisor */ ldr x2, =(HCR_RW | HCR_APK | HCR_API) msr hcr_el2, x2 @@ -385,7 +431,7 @@ LENTRY(enter_kernel_el) msr SCTLR_EL12_REG, x2 mov x2, xzr /* CPTR_EL2 is managed by vfp.c */ - ldr x3, =(CNTHCTL_E2H_EL1PCTEN | CNTHCTL_E2H_EL1PTEN) + ldr x3, =(CNTHCTL_E2H_EL1PCTEN_NOTRAP | CNTHCTL_E2H_EL1PTEN_NOTRAP) ldr x5, =(PSR_DAIF | PSR_M_EL2h) b .Ldone_vhe @@ -396,7 +442,7 @@ LENTRY(enter_kernel_el) msr vbar_el2, x2 ldr x2, =(CPTR_RES1) - ldr x3, =(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) + ldr x3, =(CNTHCTL_EL1PCTEN_NOTRAP | CNTHCTL_EL1PCEN_NOTRAP) ldr x5, =(PSR_DAIF | PSR_M_EL1h) .Ldone_vhe: @@ -442,6 +488,29 @@ LENTRY(enter_kernel_el) eret LEND(enter_kernel_el) +/* Turn off the MMU. Install ttbr0 from the bootstrap page table, and go there. + * Does not return. + * - x0 - target address to jump to after stopping the MMU. + * - x1 - kernel load address + */ +ENTRY(stop_mmu) + mov x16, x0 /* Save target. */ + ldr x2, =(1f - KERNBASE) + add x17, x1, x2 + ldr x3, =(pagetable_l0_ttbr0_bootstrap - KERNBASE) + add x1, x1, x3 + msr ttbr0_el1, x1 + isb + br x17 +1: + BTI_J + mrs x0, sctlr_el1 + bic x0, x0, SCTLR_M + bic x0, x0, SCTLR_C + msr sctlr_el1, x0 + isb + br x16 +END(stop_mmu) /* * Get the physical address the kernel was loaded at. */ @@ -1061,12 +1130,19 @@ tcr: TCR_SH0_IS | TCR_ORGN0_WBWA | TCR_IRGN0_WBWA) LEND(start_mmu) +ENTRY(switch_stack) + mov sp, x0 + mov x16, x1 + br x16 +END(switch_stack) + ENTRY(abort) b abort END(abort) .bss .align PAGE_SHIFT + .globl initstack_end initstack: .space BOOT_STACK_SIZE initstack_end: @@ -1083,6 +1159,7 @@ initstack_end: * L0 for user */ .globl pagetable_l0_ttbr1 + .globl pagetable_l0_ttbr0_bootstrap pagetable: pagetable_l3_ttbr1: .space (PAGE_SIZE * L3_PAGE_COUNT) diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c index 53856dd90cae..322bad273a08 100644 --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -173,16 +173,20 @@ SYSINIT(ssp_warn, SI_SUB_COPYRIGHT, SI_ORDER_ANY, print_ssp_warning, NULL); SYSINIT(ssp_warn2, SI_SUB_LAST, SI_ORDER_ANY, print_ssp_warning, NULL); #endif -static bool +static cpu_feat_en pan_check(const struct cpu_feat *feat __unused, u_int midr __unused) { uint64_t id_aa64mfr1; - id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); - return (ID_AA64MMFR1_PAN_VAL(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE); + if (!get_kernel_reg(ID_AA64MMFR1_EL1, &id_aa64mfr1)) + return (FEAT_ALWAYS_DISABLE); + if (ID_AA64MMFR1_PAN_VAL(id_aa64mfr1) == ID_AA64MMFR1_PAN_NONE) + return (FEAT_ALWAYS_DISABLE); + + return (FEAT_DEFAULT_ENABLE); } -static void +static bool pan_enable(const struct cpu_feat *feat __unused, cpu_feat_errata errata_status __unused, u_int *errata_list __unused, u_int errata_count __unused) @@ -200,15 +204,20 @@ pan_enable(const struct cpu_feat *feat __unused, ".arch_extension pan \n" "msr pan, #1 \n" ".arch_extension nopan \n"); + + return (true); } -static struct cpu_feat feat_pan = { - .feat_name = "FEAT_PAN", - .feat_check = pan_check, - .feat_enable = pan_enable, - .feat_flags = CPU_FEAT_EARLY_BOOT | CPU_FEAT_PER_CPU, -}; -DATA_SET(cpu_feat_set, feat_pan); +static void +pan_disabled(const struct cpu_feat *feat __unused) +{ + if (PCPU_GET(cpuid) == 0) + update_special_reg(ID_AA64MMFR1_EL1, ID_AA64MMFR1_PAN_MASK, 0); +} + +CPU_FEAT(feat_pan, "Privileged access never", + pan_check, NULL, pan_enable, pan_disabled, + CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); bool has_hyp(void) @@ -857,7 +866,7 @@ initarm(struct arm64_bootparams *abp) cninit(); set_ttbr0(abp->kern_ttbr0); - cpu_tlb_flushID(); + pmap_s1_invalidate_all_kernel(); if (!valid) panic("Invalid bus configuration: %s", diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c index e4d011df3a06..0bdd2ecfd8a7 100644 --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -60,6 +60,7 @@ #include <machine/debug_monitor.h> #include <machine/intr.h> #include <machine/smp.h> +#include <machine/vmparam.h> #ifdef VFP #include <machine/vfp.h> #endif @@ -103,6 +104,7 @@ static void ipi_hardclock(void *); static void ipi_preempt(void *); static void ipi_rendezvous(void *); static void ipi_stop(void *); +static void ipi_off(void *); #ifdef FDT static u_int fdt_cpuid; @@ -193,6 +195,7 @@ release_aps(void *dummy __unused) intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); + intr_ipi_setup(IPI_OFF, "off", ipi_off, NULL); atomic_store_int(&aps_started, 0); atomic_store_rel_int(&aps_ready, 1); @@ -390,6 +393,34 @@ ipi_stop(void *dummy __unused) CTR0(KTR_SMP, "IPI_STOP (restart)"); } +void stop_mmu(vm_paddr_t, vm_paddr_t) __dead2; +extern uint32_t mp_cpu_spinloop[]; +extern uint32_t mp_cpu_spinloop_end[]; +extern uint64_t mp_cpu_spin_table_release_addr; +static void +ipi_off(void *dummy __unused) +{ + CTR0(KTR_SMP, "IPI_OFF"); + if (psci_present) + psci_cpu_off(); + else { + uint64_t release_addr; + vm_size_t size; + + size = (vm_offset_t)&mp_cpu_spin_table_release_addr - + (vm_offset_t)mp_cpu_spinloop; + release_addr = PCPU_GET(release_addr) - size; + isb(); + invalidate_icache(); + /* Go catatonic, don't take any interrupts. */ + intr_disable(); + stop_mmu(release_addr, pmap_kextract(KERNBASE)); + + + } + CTR0(KTR_SMP, "IPI_OFF failed"); +} + struct cpu_group * cpu_topo(void) { @@ -511,6 +542,7 @@ start_cpu(u_int cpuid, uint64_t target_cpu, int domain, vm_paddr_t release_addr) pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); pcpup->pc_mpidr = target_cpu & CPU_AFF_MASK; bootpcpu = pcpup; + pcpup->pc_release_addr = release_addr; dpcpu[cpuid - 1] = (void *)(pcpup + 1); dpcpu_init(dpcpu[cpuid - 1], cpuid); @@ -752,6 +784,52 @@ cpu_mp_start(void) } } +void +cpu_mp_stop(void) +{ + + /* Short-circuit for single-CPU */ + if (CPU_COUNT(&all_cpus) == 1) + return; + + KASSERT(PCPU_GET(cpuid) == CPU_FIRST(), ("Not on the first CPU!\n")); + + /* + * If we use spin-table, assume U-boot method for now (single address + * shared by all CPUs). + */ + if (!psci_present) { + int cpu; + vm_paddr_t release_addr; + void *release_vaddr; + vm_size_t size; + + /* Find the shared release address. */ + CPU_FOREACH(cpu) { + release_addr = pcpu_find(cpu)->pc_release_addr; + if (release_addr != 0) + break; + } + /* No release address? No way of notifying other CPUs. */ + if (release_addr == 0) + return; + + size = (vm_offset_t)&mp_cpu_spinloop_end - + (vm_offset_t)&mp_cpu_spinloop; + + release_addr -= (vm_offset_t)&mp_cpu_spin_table_release_addr - + (vm_offset_t)mp_cpu_spinloop; + + release_vaddr = pmap_mapdev(release_addr, size); + bcopy(mp_cpu_spinloop, release_vaddr, size); + cpu_dcache_wbinv_range(release_vaddr, size); + pmap_unmapdev(release_vaddr, size); + invalidate_icache(); + } + ipi_all_but_self(IPI_OFF); + DELAY(1000000); +} + /* Introduce rest of cores to the world */ void cpu_mp_announce(void) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index ec89c4573799..dbf5c820d20b 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -190,6 +190,8 @@ pt_entry_t __read_mostly pmap_gp_attr; #define PMAP_SAN_PTE_BITS (ATTR_AF | ATTR_S1_XN | pmap_sh_attr | \ ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_AP(ATTR_S1_AP_RW)) +static bool __read_mostly pmap_multiple_tlbi = false; + struct pmap_large_md_page { struct rwlock pv_lock; struct md_page pv_page; @@ -1297,7 +1299,7 @@ pmap_bootstrap_dmap(vm_size_t kernlen) } } - cpu_tlb_flushID(); + pmap_s1_invalidate_all_kernel(); bs_state.dmap_valid = true; @@ -1399,7 +1401,7 @@ pmap_bootstrap(void) /* And the l3 tables for the early devmap */ pmap_bootstrap_l3(VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE)); - cpu_tlb_flushID(); + pmap_s1_invalidate_all_kernel(); #define alloc_pages(var, np) \ (var) = bs_state.freemempos; \ @@ -1656,14 +1658,17 @@ pmap_init_pv_table(void) } } -static bool +static cpu_feat_en pmap_dbm_check(const struct cpu_feat *feat __unused, u_int midr __unused) { uint64_t id_aa64mmfr1; id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); - return (ID_AA64MMFR1_HAFDBS_VAL(id_aa64mmfr1) >= - ID_AA64MMFR1_HAFDBS_AF_DBS); + if (ID_AA64MMFR1_HAFDBS_VAL(id_aa64mmfr1) >= + ID_AA64MMFR1_HAFDBS_AF_DBS) + return (FEAT_DEFAULT_ENABLE); + + return (FEAT_ALWAYS_DISABLE); } static bool @@ -1671,8 +1676,8 @@ pmap_dbm_has_errata(const struct cpu_feat *feat __unused, u_int midr, u_int **errata_list, u_int *errata_count) { /* Disable on Cortex-A55 for erratum 1024718 - all revisions */ - if (CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK, CPU_IMPL_ARM, - CPU_PART_CORTEX_A55, 0, 0)) { + if (CPU_IMPL(midr) == CPU_IMPL_ARM && + CPU_PART(midr) == CPU_PART_CORTEX_A55) { static u_int errata_id = 1024718; *errata_list = &errata_id; @@ -1681,21 +1686,19 @@ pmap_dbm_has_errata(const struct cpu_feat *feat __unused, u_int midr, } /* Disable on Cortex-A510 for erratum 2051678 - r0p0 to r0p2 */ - if (CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK | CPU_VAR_MASK, - CPU_IMPL_ARM, CPU_PART_CORTEX_A510, 0, 0)) { - if (CPU_REV(PCPU_GET(midr)) < 3) { - static u_int errata_id = 2051678; + if (midr_check_var_part_range(midr, CPU_IMPL_ARM, CPU_PART_CORTEX_A510, + 0, 0, 0, 2)) { + static u_int errata_id = 2051678; - *errata_list = &errata_id; - *errata_count = 1; - return (true); - } + *errata_list = &errata_id; + *errata_count = 1; + return (true); } return (false); } -static void +static bool pmap_dbm_enable(const struct cpu_feat *feat __unused, cpu_feat_errata errata_status, u_int *errata_list __unused, u_int errata_count) @@ -1704,7 +1707,7 @@ pmap_dbm_enable(const struct cpu_feat *feat __unused, /* Skip if there is an erratum affecting DBM */ if (errata_status != ERRATA_NONE) - return; + return (false); tcr = READ_SPECIALREG(tcr_el1) | TCR_HD; WRITE_SPECIALREG(tcr_el1, tcr); @@ -1714,16 +1717,58 @@ pmap_dbm_enable(const struct cpu_feat *feat __unused, __asm __volatile("tlbi vmalle1"); dsb(nsh); isb(); + + return (true); } -static struct cpu_feat feat_dbm = { - .feat_name = "FEAT_HAFDBS (DBM)", - .feat_check = pmap_dbm_check, - .feat_has_errata = pmap_dbm_has_errata, - .feat_enable = pmap_dbm_enable, - .feat_flags = CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU, -}; -DATA_SET(cpu_feat_set, feat_dbm); +CPU_FEAT(feat_hafdbs, "Hardware management of the Access flag and dirty state", + pmap_dbm_check, pmap_dbm_has_errata, pmap_dbm_enable, NULL, + CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); + +static cpu_feat_en +pmap_multiple_tlbi_check(const struct cpu_feat *feat __unused, u_int midr) +{ + /* + * Cortex-A55 erratum 2441007 (Cat B rare) + * Present in all revisions + */ + if (CPU_IMPL(midr) == CPU_IMPL_ARM && + CPU_PART(midr) == CPU_PART_CORTEX_A55) + return (FEAT_DEFAULT_DISABLE); + + /* + * Cortex-A76 erratum 1286807 (Cat B rare) + * Present in r0p0 - r3p0 + * Fixed in r3p1 + */ + if (midr_check_var_part_range(midr, CPU_IMPL_ARM, CPU_PART_CORTEX_A76, + 0, 0, 3, 0)) + return (FEAT_DEFAULT_DISABLE); + + /* + * Cortex-A510 erratum 2441009 (Cat B rare) + * Present in r0p0 - r1p1 + * Fixed in r1p2 + */ + if (midr_check_var_part_range(midr, CPU_IMPL_ARM, CPU_PART_CORTEX_A510, + 0, 0, 1, 1)) + return (FEAT_DEFAULT_DISABLE); + + return (FEAT_ALWAYS_DISABLE); +} + +static bool +pmap_multiple_tlbi_enable(const struct cpu_feat *feat __unused, + cpu_feat_errata errata_status, u_int *errata_list __unused, + u_int errata_count __unused) +{ + pmap_multiple_tlbi = true; + return (true); +} + +CPU_FEAT(errata_multi_tlbi, "Multiple TLBI errata", + pmap_multiple_tlbi_check, NULL, pmap_multiple_tlbi_enable, NULL, + CPU_FEAT_EARLY_BOOT | CPU_FEAT_PER_CPU); /* * Initialize the pmap module. @@ -1878,9 +1923,17 @@ pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) r = TLBI_VA(va); if (pmap == kernel_pmap) { pmap_s1_invalidate_kernel(r, final_only); + if (pmap_multiple_tlbi) { + dsb(ish); + pmap_s1_invalidate_kernel(r, final_only); + } } else { r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); pmap_s1_invalidate_user(r, final_only); + if (pmap_multiple_tlbi) { + dsb(ish); + pmap_s1_invalidate_user(r, final_only); + } } dsb(ish); isb(); @@ -1922,12 +1975,24 @@ pmap_s1_invalidate_strided(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, end = TLBI_VA(eva); for (r = start; r < end; r += TLBI_VA(stride)) pmap_s1_invalidate_kernel(r, final_only); + + if (pmap_multiple_tlbi) { + dsb(ish); + for (r = start; r < end; r += TLBI_VA(stride)) + pmap_s1_invalidate_kernel(r, final_only); + } } else { start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); start |= TLBI_VA(sva); end |= TLBI_VA(eva); for (r = start; r < end; r += TLBI_VA(stride)) pmap_s1_invalidate_user(r, final_only); + + if (pmap_multiple_tlbi) { + dsb(ish); + for (r = start; r < end; r += TLBI_VA(stride)) + pmap_s1_invalidate_user(r, final_only); + } } dsb(ish); isb(); @@ -1963,6 +2028,19 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pmap_s2_invalidate_range(pmap, sva, eva, final_only); } +void +pmap_s1_invalidate_all_kernel(void) +{ + dsb(ishst); + __asm __volatile("tlbi vmalle1is"); + dsb(ish); + if (pmap_multiple_tlbi) { + __asm __volatile("tlbi vmalle1is"); + dsb(ish); + } + isb(); +} + /* * Invalidates all cached intermediate- and final-level TLB entries for the * given virtual address space. @@ -1977,9 +2055,17 @@ pmap_s1_invalidate_all(pmap_t pmap) dsb(ishst); if (pmap == kernel_pmap) { __asm __volatile("tlbi vmalle1is"); + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi vmalle1is"); + } } else { r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); __asm __volatile("tlbi aside1is, %0" : : "r" (r)); + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi aside1is, %0" : : "r" (r)); + } } dsb(ish); isb(); @@ -7967,7 +8053,7 @@ pmap_mapbios(vm_paddr_t pa, vm_size_t size) pa += L2_SIZE; } if ((old_l2e & ATTR_DESCR_VALID) != 0) - pmap_s1_invalidate_all(kernel_pmap); + pmap_s1_invalidate_all_kernel(); else { /* * Because the old entries were invalid and the new @@ -8058,7 +8144,7 @@ pmap_unmapbios(void *p, vm_size_t size) } } if (preinit_map) { - pmap_s1_invalidate_all(kernel_pmap); + pmap_s1_invalidate_all_kernel(); return; } diff --git a/sys/arm64/arm64/ptrauth.c b/sys/arm64/arm64/ptrauth.c index dbe0c69b8d60..ab40b72887e9 100644 --- a/sys/arm64/arm64/ptrauth.c +++ b/sys/arm64/arm64/ptrauth.c @@ -82,7 +82,7 @@ ptrauth_disable(void) return (false); } -static bool +static cpu_feat_en ptrauth_check(const struct cpu_feat *feat __unused, u_int midr __unused) { uint64_t isar; @@ -97,11 +97,11 @@ ptrauth_check(const struct cpu_feat *feat __unused, u_int midr __unused) if (!pac_enable) { if (boothowto & RB_VERBOSE) printf("Pointer authentication is disabled\n"); - goto out; + return (FEAT_ALWAYS_DISABLE); } if (ptrauth_disable()) - goto out; + return (FEAT_ALWAYS_DISABLE); /* * This assumes if there is pointer authentication on the boot CPU @@ -116,32 +116,21 @@ ptrauth_check(const struct cpu_feat *feat __unused, u_int midr __unused) if (get_kernel_reg(ID_AA64ISAR1_EL1, &isar)) { if (ID_AA64ISAR1_APA_VAL(isar) > 0 || ID_AA64ISAR1_API_VAL(isar) > 0) { - return (true); + return (FEAT_DEFAULT_ENABLE); } } /* The QARMA3 algorithm is reported in ID_AA64ISAR2_EL1. */ if (get_kernel_reg(ID_AA64ISAR2_EL1, &isar)) { if (ID_AA64ISAR2_APA3_VAL(isar) > 0) { - return (true); + return (FEAT_DEFAULT_ENABLE); } } -out: - /* - * Pointer authentication may be disabled, mask out the ID fields we - * expose to userspace and the rest of the kernel so they don't try - * to use it. - */ - update_special_reg(ID_AA64ISAR1_EL1, ID_AA64ISAR1_API_MASK | - ID_AA64ISAR1_APA_MASK | ID_AA64ISAR1_GPA_MASK | - ID_AA64ISAR1_GPI_MASK, 0); - update_special_reg(ID_AA64ISAR2_EL1, ID_AA64ISAR2_APA3_MASK, 0); - - return (false); + return (FEAT_ALWAYS_DISABLE); } -static void +static bool ptrauth_enable(const struct cpu_feat *feat __unused, cpu_feat_errata errata_status __unused, u_int *errata_list __unused, u_int errata_count __unused) @@ -153,16 +142,30 @@ ptrauth_enable(const struct cpu_feat *feat __unused, elf64_addr_mask_14.code |= PAC_ADDR_MASK_14; elf64_addr_mask_14.data |= PAC_ADDR_MASK_14; #endif + + return (true); } +static void +ptrauth_disabled(const struct cpu_feat *feat __unused) +{ + /* + * Pointer authentication may be disabled, mask out the ID fields we + * expose to userspace and the rest of the kernel so they don't try + * to use it. + */ + if (PCPU_GET(cpuid) == 0) { + update_special_reg(ID_AA64ISAR1_EL1, ID_AA64ISAR1_API_MASK | + ID_AA64ISAR1_APA_MASK | ID_AA64ISAR1_GPA_MASK | + ID_AA64ISAR1_GPI_MASK, 0); + update_special_reg(ID_AA64ISAR2_EL1, ID_AA64ISAR2_APA3_MASK, 0); + } + +} -static struct cpu_feat feat_pauth = { - .feat_name = "FEAT_PAuth", - .feat_check = ptrauth_check, - .feat_enable = ptrauth_enable, - .feat_flags = CPU_FEAT_EARLY_BOOT | CPU_FEAT_SYSTEM, -}; -DATA_SET(cpu_feat_set, feat_pauth); +CPU_FEAT(feat_pauth, "Pointer Authentication", + ptrauth_check, NULL, ptrauth_enable, ptrauth_disabled, + CPU_FEAT_EARLY_BOOT | CPU_FEAT_SYSTEM); /* Copy the keys when forking a new process */ void diff --git a/sys/arm64/arm64/spec_workaround.c b/sys/arm64/arm64/spec_workaround.c new file mode 100644 index 000000000000..7f4f86cdb48c --- /dev/null +++ b/sys/arm64/arm64/spec_workaround.c @@ -0,0 +1,166 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Arm Ltd + * Copyright (c) 2018 Andrew Turner + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/pcpu.h> +#include <sys/systm.h> + +#include <machine/cpu.h> +#include <machine/cpu_feat.h> + +#include <dev/psci/psci.h> +#include <dev/psci/smccc.h> + +static enum { + SSBD_FORCE_ON, + SSBD_FORCE_OFF, + SSBD_KERNEL, +} ssbd_method = SSBD_KERNEL; + +struct psci_bp_hardening_impl { + u_int midr_mask; + u_int midr_value; +}; + +static struct psci_bp_hardening_impl psci_bp_hardening_impl[] = { + { + .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, + .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A57,0,0), + }, + { + .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, + .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A72,0,0), + }, + { + .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, + .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A73,0,0), + }, + { + .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, + .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A75,0,0), + }, + { + .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK, + .midr_value = + CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX2, 0,0), + } +}; + +static cpu_feat_en +psci_bp_hardening_check(const struct cpu_feat *feat __unused, u_int midr) +{ + size_t i; + + for (i = 0; i < nitems(psci_bp_hardening_impl); i++) { + if ((midr & psci_bp_hardening_impl[i].midr_mask) == + psci_bp_hardening_impl[i].midr_value) { + /* SMCCC depends on PSCI. If PSCI is missing so is SMCCC */ + if (!psci_present) + return (FEAT_ALWAYS_DISABLE); + + if (smccc_arch_features(SMCCC_ARCH_WORKAROUND_1) != + SMCCC_RET_SUCCESS) + return (FEAT_ALWAYS_DISABLE); + + return (FEAT_DEFAULT_ENABLE); + } + } + + return (FEAT_ALWAYS_DISABLE); +} + +static bool +psci_bp_hardening_enable(const struct cpu_feat *feat __unused, + cpu_feat_errata errata_status __unused, u_int *errata_list __unused, + u_int errata_count __unused) +{ + PCPU_SET(bp_harden, smccc_arch_workaround_1); + + return (true); +} + +CPU_FEAT(feat_csv2_missing, "Branch Predictor Hardening", + psci_bp_hardening_check, NULL, psci_bp_hardening_enable, NULL, + CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); + +static cpu_feat_en +ssbd_workaround_check(const struct cpu_feat *feat __unused, u_int midr __unused) +{ + char *env; + + if (PCPU_GET(cpuid) == 0) { + env = kern_getenv("kern.cfg.ssbd"); + if (env != NULL) { + if (strcmp(env, "force-on") == 0) { + ssbd_method = SSBD_FORCE_ON; + } else if (strcmp(env, "force-off") == 0) { + ssbd_method = SSBD_FORCE_OFF; + } + } + } + + /* SMCCC depends on PSCI. If PSCI is missing so is SMCCC */ + if (!psci_present) + return (FEAT_ALWAYS_DISABLE); + + /* Enable the workaround on this CPU if it's enabled in the firmware */ + if (smccc_arch_features(SMCCC_ARCH_WORKAROUND_2) != SMCCC_RET_SUCCESS) + return (FEAT_ALWAYS_DISABLE); + + return (FEAT_DEFAULT_ENABLE); +} + +static bool +ssbd_workaround_enable(const struct cpu_feat *feat __unused, + cpu_feat_errata errata_status __unused, u_int *errata_list __unused, + u_int errata_count __unused) +{ + switch(ssbd_method) { + case SSBD_FORCE_ON: + smccc_arch_workaround_2(1); + break; + case SSBD_FORCE_OFF: + smccc_arch_workaround_2(0); + break; + case SSBD_KERNEL: + default: + PCPU_SET(ssbd, smccc_arch_workaround_2); + break; + } + + return (true); +} + +CPU_FEAT(feat_ssbs_missing, "Speculator Store Bypass Disable Workaround", + ssbd_workaround_check, NULL, ssbd_workaround_enable, NULL, + CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); diff --git a/sys/arm64/arm64/trap.c b/sys/arm64/arm64/trap.c index bed58095201a..75c9b5f87892 100644 --- a/sys/arm64/arm64/trap.c +++ b/sys/arm64/arm64/trap.c @@ -246,6 +246,7 @@ external_abort(struct thread *td, struct trapframe *frame, uint64_t esr, print_registers(frame); print_gp_register("far", far); + printf(" esr: 0x%.16lx\n", esr); panic("Unhandled external data abort"); } diff --git a/sys/arm64/conf/std.arm b/sys/arm64/conf/std.arm index fb5561506531..309059a096eb 100644 --- a/sys/arm64/conf/std.arm +++ b/sys/arm64/conf/std.arm @@ -21,3 +21,6 @@ device arm_doorbell # ARM Message Handling Unit (MHU) options FDT device acpi + +# DTBs +makeoptions MODULES_EXTRA+="dtb/arm" diff --git a/sys/arm64/conf/std.arm64 b/sys/arm64/conf/std.arm64 index c83e98c17a33..a0568466cfaf 100644 --- a/sys/arm64/conf/std.arm64 +++ b/sys/arm64/conf/std.arm64 @@ -7,6 +7,7 @@ makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support options SCHED_ULE # ULE scheduler options NUMA # Non-Uniform Memory Architecture support options PREEMPTION # Enable kernel thread preemption +options EXTERR_STRINGS options VIMAGE # Subsystem virtualization, e.g. VNET options INET # InterNETworking options INET6 # IPv6 communications protocols diff --git a/sys/arm64/conf/std.dev b/sys/arm64/conf/std.dev index 719f272426dd..c5c364ffda04 100644 --- a/sys/arm64/conf/std.dev +++ b/sys/arm64/conf/std.dev @@ -115,7 +115,6 @@ device mmcsd # mmc/sd flash cards options HID_DEBUG # enable debug msgs device hid # Generic HID support device hidbus # Generic HID Bus -options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ # Firmware device mmio_sram # Generic on-chip SRAM diff --git a/sys/arm64/coresight/coresight.c b/sys/arm64/coresight/coresight.c index 5928c153f4ae..9b9d3c65ecc9 100644 --- a/sys/arm64/coresight/coresight.c +++ b/sys/arm64/coresight/coresight.c @@ -113,7 +113,7 @@ coresight_get_output_device(struct endpoint *endp, struct endpoint **out_endp) } static void -coresight_init(void) +coresight_init(void *dummy __unused) { mtx_init(&cs_mtx, "ARM Coresight", NULL, MTX_DEF); diff --git a/sys/arm64/include/_armreg.h b/sys/arm64/include/_armreg.h new file mode 100644 index 000000000000..0f5134e5a978 --- /dev/null +++ b/sys/arm64/include/_armreg.h @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2013, 2014 Andrew Turner + * Copyright (c) 2015,2021 The FreeBSD Foundation + * + * Portions of this software were developed by Andrew Turner + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if !defined(_MACHINE_ARMREG_H_) && \ + !defined(_MACHINE_CPU_H_) && \ + !defined(_MACHINE_HYPERVISOR_H_) +#error Do not include this file directly +#endif + +#ifndef _MACHINE__ARMREG_H_ +#define _MACHINE__ARMREG_H_ + +#define __MRS_REG_ALT_NAME(op0, op1, crn, crm, op2) \ + S##op0##_##op1##_C##crn##_C##crm##_##op2 +#define _MRS_REG_ALT_NAME(op0, op1, crn, crm, op2) \ + __MRS_REG_ALT_NAME(op0, op1, crn, crm, op2) +#define MRS_REG_ALT_NAME(reg) \ + _MRS_REG_ALT_NAME(reg##_op0, reg##_op1, reg##_CRn, reg##_CRm, reg##_op2) + + +#define READ_SPECIALREG(reg) \ +({ uint64_t _val; \ + __asm __volatile("mrs %0, " __STRING(reg) : "=&r" (_val)); \ + _val; \ +}) +#define WRITE_SPECIALREG(reg, _val) \ + __asm __volatile("msr " __STRING(reg) ", %0" : : "r"((uint64_t)_val)) + +#define UL(x) UINT64_C(x) + +#endif /* !_MACHINE__ARMREG_H_ */ diff --git a/sys/arm64/include/armreg.h b/sys/arm64/include/armreg.h index 500f35c48787..aa9b672ad85a 100644 --- a/sys/arm64/include/armreg.h +++ b/sys/arm64/include/armreg.h @@ -34,25 +34,9 @@ #ifndef _MACHINE_ARMREG_H_ #define _MACHINE_ARMREG_H_ -#define INSN_SIZE 4 - -#define __MRS_REG_ALT_NAME(op0, op1, crn, crm, op2) \ - S##op0##_##op1##_C##crn##_C##crm##_##op2 -#define _MRS_REG_ALT_NAME(op0, op1, crn, crm, op2) \ - __MRS_REG_ALT_NAME(op0, op1, crn, crm, op2) -#define MRS_REG_ALT_NAME(reg) \ - _MRS_REG_ALT_NAME(reg##_op0, reg##_op1, reg##_CRn, reg##_CRm, reg##_op2) - +#include <machine/_armreg.h> -#define READ_SPECIALREG(reg) \ -({ uint64_t _val; \ - __asm __volatile("mrs %0, " __STRING(reg) : "=&r" (_val)); \ - _val; \ -}) -#define WRITE_SPECIALREG(reg, _val) \ - __asm __volatile("msr " __STRING(reg) ", %0" : : "r"((uint64_t)_val)) - -#define UL(x) UINT64_C(x) +#define INSN_SIZE 4 /* AFSR0_EL1 - Auxiliary Fault Status Register 0 */ #define AFSR0_EL1_REG MRS_REG_ALT_NAME(AFSR0_EL1) @@ -232,6 +216,14 @@ #define CNTP_CTL_IMASK (1 << 1) #define CNTP_CTL_ISTATUS (1 << 2) +/* CNTP_CTL_EL02 - Counter-timer Physical Timer Control register */ +#define CNTP_CTL_EL02_REG MRS_REG_ALT_NAME(CNTP_CTL_EL02) +#define CNTP_CTL_EL02_op0 3 +#define CNTP_CTL_EL02_op1 5 +#define CNTP_CTL_EL02_CRn 14 +#define CNTP_CTL_EL02_CRm 2 +#define CNTP_CTL_EL02_op2 1 + /* CNTP_CVAL_EL0 - Counter-timer Physical Timer CompareValue register */ #define CNTP_CVAL_EL0_op0 3 #define CNTP_CVAL_EL0_op1 3 @@ -239,6 +231,14 @@ #define CNTP_CVAL_EL0_CRm 2 #define CNTP_CVAL_EL0_op2 2 +/* CNTP_CVAL_EL02 - Counter-timer Physical Timer CompareValue register */ +#define CNTP_CVAL_EL02_REG MRS_REG_ALT_NAME(CNTP_CVAL_EL02) +#define CNTP_CVAL_EL02_op0 3 +#define CNTP_CVAL_EL02_op1 5 +#define CNTP_CVAL_EL02_CRn 14 +#define CNTP_CVAL_EL02_CRm 2 +#define CNTP_CVAL_EL02_op2 2 + /* CNTP_TVAL_EL0 - Counter-timer Physical Timer TimerValue register */ #define CNTP_TVAL_EL0_op0 3 #define CNTP_TVAL_EL0_op1 3 @@ -254,6 +254,14 @@ #define CNTPCT_EL0_CRm 0 #define CNTPCT_EL0_op2 1 +/* CNTPCTSS_EL0 - Counter-timer Self-Synchronized Physical Count register */ +#define CNTPCTSS_EL0_REG MRS_REG_ALT_NAME(CNTPCTSS_EL0) +#define CNTPCTSS_EL0_op0 3 +#define CNTPCTSS_EL0_op1 3 +#define CNTPCTSS_EL0_CRn 14 +#define CNTPCTSS_EL0_CRm 0 +#define CNTPCTSS_EL0_op2 5 + /* CNTV_CTL_EL0 - Counter-timer Virtual Timer Control register */ #define CNTV_CTL_EL0_op0 3 #define CNTV_CTL_EL0_op1 3 @@ -282,6 +290,14 @@ #define CNTV_CVAL_EL02_CRm 3 #define CNTV_CVAL_EL02_op2 2 +/* CNTVCTSS_EL0 - Counter-timer Self-Synchronized Virtual Count register */ +#define CNTVCTSS_EL0_REG MRS_REG_ALT_NAME(CNTVCTSS_EL0) +#define CNTVCTSS_EL0_op0 3 +#define CNTVCTSS_EL0_op1 3 +#define CNTVCTSS_EL0_CRn 14 +#define CNTVCTSS_EL0_CRm 0 +#define CNTVCTSS_EL0_op2 6 + /* CONTEXTIDR_EL1 - Context ID register */ #define CONTEXTIDR_EL1_REG MRS_REG_ALT_NAME(CONTEXTIDR_EL1) #define CONTEXTIDR_EL1_op0 3 @@ -2148,6 +2164,7 @@ #define OSLAR_EL1_CRn 1 #define OSLAR_EL1_CRm 0 #define OSLAR_EL1_op2 4 +#define OSLAR_OSLK (0x1ul << 0) /* OSLSR_EL1 */ #define OSLSR_EL1_op0 2 @@ -2155,6 +2172,10 @@ #define OSLSR_EL1_CRn 1 #define OSLSR_EL1_CRm 1 #define OSLSR_EL1_op2 4 +#define OSLSR_OSLM_1 (0x1ul << 3) +#define OSLSR_nTT (0x1ul << 2) +#define OSLSR_OSLK (0x1ul << 1) +#define OSLSR_OSLM_0 (0x1ul << 0) /* PAR_EL1 - Physical Address Register */ #define PAR_F_SHIFT 0 @@ -2241,6 +2262,11 @@ #define PMBSR_DL (UL(0x1) << PMBSR_DL_SHIFT) #define PMBSR_EC_SHIFT 26 #define PMBSR_EC_MASK (UL(0x3f) << PMBSR_EC_SHIFT) +#define PMBSR_EC_VAL(x) (((x) & PMBSR_EC_MASK) >> PMBSR_EC_SHIFT) +#define PMBSR_EC_OTHER_BUF_MGMT 0x00 +#define PMBSR_EC_GRAN_PROT_CHK 0x1e +#define PMBSR_EC_STAGE1_DA 0x24 +#define PMBSR_EC_STAGE2_DA 0x25 /* PMCCFILTR_EL0 */ #define PMCCFILTR_EL0_op0 3 @@ -2476,6 +2502,15 @@ #define PMSIDR_FnE (UL(0x1) << PMSIDR_FnE_SHIFT) #define PMSIDR_Interval_SHIFT 8 #define PMSIDR_Interval_MASK (UL(0xf) << PMSIDR_Interval_SHIFT) +#define PMSIDR_Interval_VAL(x) (((x) & PMSIDR_Interval_MASK) >> PMSIDR_Interval_SHIFT) +#define PMSIDR_Interval_256 0 +#define PMSIDR_Interval_512 2 +#define PMSIDR_Interval_768 3 +#define PMSIDR_Interval_1024 4 +#define PMSIDR_Interval_1536 5 +#define PMSIDR_Interval_2048 6 +#define PMSIDR_Interval_3072 7 +#define PMSIDR_Interval_4096 8 #define PMSIDR_MaxSize_SHIFT 12 #define PMSIDR_MaxSize_MASK (UL(0xf) << PMSIDR_MaxSize_SHIFT) #define PMSIDR_CountSize_SHIFT 16 @@ -2612,10 +2647,12 @@ (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_EIS | SCTLR_TSCXT | SCTLR_EOS) #define SCTLR_MMU_ON \ (SCTLR_MMU_OFF | \ + SCTLR_EPAN | \ SCTLR_BT1 | \ SCTLR_BT0 | \ SCTLR_UCI | \ SCTLR_SPAN | \ + SCTLR_IESB | \ SCTLR_nTWE | \ SCTLR_nTWI | \ SCTLR_UCT | \ diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h index 59cda36f275e..b15210633d37 100644 --- a/sys/arm64/include/cpu.h +++ b/sys/arm64/include/cpu.h @@ -43,10 +43,10 @@ #define _MACHINE_CPU_H_ #if !defined(__ASSEMBLER__) +#include <machine/_armreg.h> #include <machine/atomic.h> #include <machine/frame.h> #endif -#include <machine/armreg.h> #define TRAPF_PC(tfp) ((tfp)->tf_elr) #define TRAPF_USERMODE(tfp) (((tfp)->tf_spsr & PSR_M_MASK) == PSR_M_EL0t) @@ -125,7 +125,11 @@ #define CPU_PART_NEOVERSE_V3 0xD84 #define CPU_PART_CORTEX_X925 0xD85 #define CPU_PART_CORTEX_A725 0xD87 +#define CPU_PART_C1_NANO 0xD8A +#define CPU_PART_C1_PRO 0xD8B +#define CPU_PART_C1_ULTRA 0xD8C #define CPU_PART_NEOVERSE_N3 0xD8E +#define CPU_PART_C1_PREMIUM 0xD90 /* Cavium Part numbers */ #define CPU_PART_THUNDERX 0x0A1 @@ -193,8 +197,30 @@ (((mask) & PCPU_GET(midr)) == \ ((mask) & CPU_ID_RAW((impl), (part), (var), (rev)))) -#define CPU_MATCH_RAW(mask, devid) \ - (((mask) & PCPU_GET(midr)) == ((mask) & (devid))) +#if !defined(__ASSEMBLER__) +static inline bool +midr_check_var_part_range(u_int midr, u_int impl, u_int part, u_int var_low, + u_int part_low, u_int var_high, u_int part_high) +{ + /* Check for the correct part */ + if (CPU_IMPL(midr) != impl || CPU_PART(midr) != part) + return (false); + + /* Check if the variant is between var_low and var_high inclusive */ + if (CPU_VAR(midr) < var_low || CPU_VAR(midr) > var_high) + return (false); + + /* If the variant is the low value, check if the part is high enough */ + if (CPU_VAR(midr) == var_low && CPU_PART(midr) < part_low) + return (false); + + /* If the variant is the high value, check if the part is low enough */ + if (CPU_VAR(midr) == var_high && CPU_PART(midr) > part_high) + return (false); + + return (true); +} +#endif /* * Chip-specific errata. This defines are intended to be diff --git a/sys/arm64/include/cpu_feat.h b/sys/arm64/include/cpu_feat.h index 9fe6a9dd95d9..6a311d4000bb 100644 --- a/sys/arm64/include/cpu_feat.h +++ b/sys/arm64/include/cpu_feat.h @@ -29,6 +29,7 @@ #define _MACHINE_CPU_FEAT_H_ #include <sys/linker_set.h> +#include <sys/sysctl.h> typedef enum { ERRATA_UNKNOWN, /* Unknown erratum */ @@ -39,6 +40,31 @@ typedef enum { /* kernel component. */ } cpu_feat_errata; +typedef enum { + /* + * Don't implement the feature or erratum wrokarount, + * e.g. the feature is not implemented or erratum is + * for another CPU. + */ + FEAT_ALWAYS_DISABLE, + + /* + * Disable by default, but allow the user to enable, + * e.g. For a rare erratum with a workaround, Arm + * Category B (rare) or similar. + */ + FEAT_DEFAULT_DISABLE, + + /* + * Enabled by default, bit allow the user to disable, + * e.g. For a common erratum with a workaround, Arm + * Category A or B or similar. + */ + FEAT_DEFAULT_ENABLE, + + /* We could add FEAT_ALWAYS_ENABLE if a need was found. */ +} cpu_feat_en; + #define CPU_FEAT_STAGE_MASK 0x00000001 #define CPU_FEAT_EARLY_BOOT 0x00000000 #define CPU_FEAT_AFTER_DEV 0x00000001 @@ -47,23 +73,45 @@ typedef enum { #define CPU_FEAT_PER_CPU 0x00000000 #define CPU_FEAT_SYSTEM 0x00000010 +#define CPU_FEAT_USER_ENABLED 0x40000000 +#define CPU_FEAT_USER_DISABLED 0x80000000 + struct cpu_feat; -typedef bool (cpu_feat_check)(const struct cpu_feat *, u_int); +typedef cpu_feat_en (cpu_feat_check)(const struct cpu_feat *, u_int); typedef bool (cpu_feat_has_errata)(const struct cpu_feat *, u_int, u_int **, u_int *); -typedef void (cpu_feat_enable)(const struct cpu_feat *, cpu_feat_errata, +typedef bool (cpu_feat_enable)(const struct cpu_feat *, cpu_feat_errata, u_int *, u_int); +typedef void (cpu_feat_disabled)(const struct cpu_feat *); struct cpu_feat { const char *feat_name; cpu_feat_check *feat_check; cpu_feat_has_errata *feat_has_errata; cpu_feat_enable *feat_enable; + cpu_feat_disabled *feat_disabled; uint32_t feat_flags; + bool feat_enabled; }; SET_DECLARE(cpu_feat_set, struct cpu_feat); +SYSCTL_DECL(_hw_feat); + +#define CPU_FEAT(name, descr, check, has_errata, enable, disabled, flags) \ +static struct cpu_feat name = { \ + .feat_name = #name, \ + .feat_check = check, \ + .feat_has_errata = has_errata, \ + .feat_enable = enable, \ + .feat_disabled = disabled, \ + .feat_flags = flags, \ + .feat_enabled = false, \ +}; \ +DATA_SET(cpu_feat_set, name); \ +SYSCTL_BOOL(_hw_feat, OID_AUTO, name, CTLFLAG_RD, &name.feat_enabled, \ + 0, descr) + /* * Allow drivers to mark an erratum as worked around, e.g. the Errata * Management ABI may know the workaround isn't needed on a given system. diff --git a/sys/arm64/include/cpufunc.h b/sys/arm64/include/cpufunc.h index e6e1f682794e..e9eee643216b 100644 --- a/sys/arm64/include/cpufunc.h +++ b/sys/arm64/include/cpufunc.h @@ -96,6 +96,13 @@ serror_enable(void) __asm __volatile("msr daifclr, #(" __XSTRING(DAIF_A) ")"); } +static __inline void +serror_disable(void) +{ + + __asm __volatile("msr daifset, #(" __XSTRING(DAIF_A) ")"); +} + static __inline register_t get_midr(void) { diff --git a/sys/arm64/include/db_machdep.h b/sys/arm64/include/db_machdep.h index 5dc496ca851d..3ef95f7802ea 100644 --- a/sys/arm64/include/db_machdep.h +++ b/sys/arm64/include/db_machdep.h @@ -31,7 +31,6 @@ #ifndef _MACHINE_DB_MACHDEP_H_ #define _MACHINE_DB_MACHDEP_H_ -#include <machine/armreg.h> #include <machine/frame.h> #include <machine/trap.h> diff --git a/sys/arm64/include/hypervisor.h b/sys/arm64/include/hypervisor.h index e3a880afbe9c..7d405e63cd8d 100644 --- a/sys/arm64/include/hypervisor.h +++ b/sys/arm64/include/hypervisor.h @@ -30,26 +30,85 @@ #ifndef _MACHINE_HYPERVISOR_H_ #define _MACHINE_HYPERVISOR_H_ +#include <machine/_armreg.h> + /* * These registers are only useful when in hypervisor context, * e.g. specific to EL2, or controlling the hypervisor. */ /* CNTHCTL_EL2 - Counter-timer Hypervisor Control register */ -#define CNTHCTL_EVNTI_MASK (0xf << 4) /* Bit to trigger event stream */ /* Valid if HCR_EL2.E2H == 0 */ -#define CNTHCTL_EL1PCTEN (1 << 0) /* Allow physical counter access */ -#define CNTHCTL_EL1PCEN (1 << 1) /* Allow physical timer access */ +#define CNTHCTL_EL1PCTEN_SHIFT 0 +#define CNTHCTL_EL1PCTEN_MASK (0x1ul << CNTHCTL_E2H_EL1PCTEN_SHIFT) +#define CNTHCTL_EL1PCTEN_TRAP (0x0ul << CNTHCTL_E2H_EL1PCTEN_SHIFT) +#define CNTHCTL_EL1PCTEN_NOTRAP (0x1ul << CNTHCTL_EL1PCTEN_SHIFT) +#define CNTHCTL_EL1PCEN_SHIFT 1 +#define CNTHCTL_EL1PCEN_MASK (0x1ul << CNTHCTL_EL1PCEN_SHIFT) +#define CNTHCTL_EL1PCEN_TRAP (0x0ul << CNTHCTL_EL1PCEN_SHIFT) +#define CNTHCTL_EL1PCEN_NOTRAP (0x1ul << CNTHCTL_EL1PCEN_SHIFT) /* Valid if HCR_EL2.E2H == 1 */ -#define CNTHCTL_E2H_EL0PCTEN (1 << 0) /* Allow EL0 physical counter access */ -#define CNTHCTL_E2H_EL0VCTEN (1 << 1) /* Allow EL0 virtual counter access */ -#define CNTHCTL_E2H_EL0VTEN (1 << 8) -#define CNTHCTL_E2H_EL0PTEN (1 << 9) -#define CNTHCTL_E2H_EL1PCTEN (1 << 10) /* Allow physical counter access */ -#define CNTHCTL_E2H_EL1PTEN (1 << 11) /* Allow physical timer access */ +#define CNTHCTL_E2H_EL0PCTEN_SHIFT 0 +#define CNTHCTL_E2H_EL0PCTEN_MASK (0x1ul << CNTHCTL_E2H_EL0PCTEN_SHIFT) +#define CNTHCTL_E2H_EL0PCTEN_TRAP (0x0ul << CNTHCTL_E2H_EL0PCTEN_SHIFT) +#define CNTHCTL_E2H_EL0PCTEN_NOTRAP (0x1ul << CNTHCTL_E2H_EL0PCTEN_SHIFT) +#define CNTHCTL_E2H_EL0VCTEN_SHIFT 1 +#define CNTHCTL_E2H_EL0VCTEN_MASK (0x1ul << CNTHCTL_E2H_EL0VCTEN_SHIFT) +#define CNTHCTL_E2H_EL0VCTEN_TRAP (0x0ul << CNTHCTL_E2H_EL0VCTEN_SHIFT) +#define CNTHCTL_E2H_EL0VCTEN_NOTRAP (0x1ul << CNTHCTL_E2H_EL0VCTEN_SHIFT) +#define CNTHCTL_E2H_EL0VTEN_SHIFT 8 +#define CNTHCTL_E2H_EL0VTEN_MASK (0x1ul << CNTHCTL_E2H_EL0VTEN_SHIFT) +#define CNTHCTL_E2H_EL0VTEN_TRAP (0x0ul << CNTHCTL_E2H_EL0VTEN_SHIFT) +#define CNTHCTL_E2H_EL0VTEN_NOTRAP (0x1ul << CNTHCTL_E2H_EL0VTEN_SHIFT) +#define CNTHCTL_E2H_EL0PTEN_SHIFT 9 +#define CNTHCTL_E2H_EL0PTEN_MASK (0x1ul << CNTHCTL_E2H_EL0PTEN_SHIFT) +#define CNTHCTL_E2H_EL0PTEN_TRAP (0x0ul << CNTHCTL_E2H_EL0PTEN_SHIFT) +#define CNTHCTL_E2H_EL0PTEN_NOTRAP (0x1ul << CNTHCTL_E2H_EL0PTEN_SHIFT) +#define CNTHCTL_E2H_EL1PCTEN_SHIFT 10 +#define CNTHCTL_E2H_EL1PCTEN_MASK (0x1ul << CNTHCTL_E2H_EL1PCTEN_SHIFT) +#define CNTHCTL_E2H_EL1PCTEN_TRAP (0x0ul << CNTHCTL_E2H_EL1PCTEN_SHIFT) +#define CNTHCTL_E2H_EL1PCTEN_NOTRAP (0x1ul << CNTHCTL_E2H_EL1PCTEN_SHIFT) +#define CNTHCTL_E2H_EL1PTEN_SHIFT 11 +#define CNTHCTL_E2H_EL1PTEN_MASK (0x1ul << CNTHCTL_E2H_EL1PTEN_SHIFT) +#define CNTHCTL_E2H_EL1PTEN_TRAP (0x0ul << CNTHCTL_E2H_EL1PTEN_SHIFT) +#define CNTHCTL_E2H_EL1PTEN_NOTRAP (0x1ul << CNTHCTL_E2H_EL1PTEN_SHIFT) /* Unconditionally valid */ -#define CNTHCTL_EVNTDIR (1 << 3) /* Control transition trigger bit */ -#define CNTHCTL_EVNTEN (1 << 2) /* Enable event stream */ +#define CNTHCTL_EVNTEN_SHIFT 2 +#define CNTHCTL_EVNTEN_MASK (0x1ul << CNTHCTL_EVNTEN_SHIFT) +#define CNTHCTL_EVNTEN_DIS (0x0ul << CNTHCTL_EVNTEN_SHIFT) +#define CNTHCTL_EVNTEN_EN (0x1ul << CNTHCTL_EVNTEN_SHIFT) +#define CNTHCTL_EVNTDIR_SHIFT 3 +#define CNTHCTL_EVNTDIR_MASK (0x1ul << CNTHCTL_EVNTDIR_SHIFT) +#define CNTHCTL_EVNTDIR_HIGH (0x0ul << CNTHCTL_EVNTDIR_SHIFT) +#define CNTHCTL_EVNTDIR_LOW (0x1ul << CNTHCTL_EVNTDIR_SHIFT) +#define CNTHCTL_EVNTI_SHIFT 4 +#define CNTHCTL_EVNTI_MASK (0xful << CNTHCTL_EVNTI_SHIFT) +#define CNTHCTL_ECV_SHIFT 12 +#define CNTHCTL_ECV_MASK (0x1ul << CNTHCTL_ECV_SHIFT) +#define CNTHCTL_ECV_DIS (0x0ul << CNTHCTL_ECV_SHIFT) +#define CNTHCTL_ECV_EN (0x1ul << CNTHCTL_ECV_SHIFT) +#define CNTHCTL_EL1TVT_SHIFT 13 +#define CNTHCTL_EL1TVT_MASK (0x1ul << CNTHCTL_EL1TVT_SHIFT) +#define CNTHCTL_EL1TVT_NOTRAP (0x0ul << CNTHCTL_EL1TVT_SHIFT) +#define CNTHCTL_EL1TVT_TRAP (0x1ul << CNTHCTL_EL1TVT_SHIFT) +#define CNTHCTL_EL1TVCT_SHIFT 14 +#define CNTHCTL_EL1TVCT_MASK (0x1ul << CNTHCTL_EL1TVCT_SHIFT) +#define CNTHCTL_EL1TVCT_NOTRAP (0x0ul << CNTHCTL_EL1TVCT_SHIFT) +#define CNTHCTL_EL1TVCT_TRAP (0x1ul << CNTHCTL_EL1TVCT_SHIFT) +#define CNTHCTL_EL1NVPCT_SHIFT 15 +#define CNTHCTL_EL1NVPCT_MASK (0x1ul << CNTHCTL_EL1NVPCT_SHIFT) +#define CNTHCTL_EL1NVPCT_NOTRAP (0x0ul << CNTHCTL_EL1NVPCT_SHIFT) +#define CNTHCTL_EL1NVPCT_TRAP (0x1ul << CNTHCTL_EL1NVPCT_SHIFT) +#define CNTHCTL_EL1NVVCT_SHIFT 16 +#define CNTHCTL_EL1NVVCT_MASK (0x1ul << CNTHCTL_EL1NVVCT_SHIFT) +#define CNTHCTL_EL1NVVCT_NOTRAP (0x0ul << CNTHCTL_EL1NVVCT_SHIFT) +#define CNTHCTL_EL1NVVCT_TRAP (0x1ul << CNTHCTL_EL1NVVCT_SHIFT) +#define CNTHCTL_EVNTIS_SHIFT 17 +#define CNTHCTL_EVNTIS_MASK (0x1ul << CNTHCTL_EVNTIS_SHIFT) +#define CNTHCTL_CNTVMASK_SHIFT 18 +#define CNTHCTL_CNTVMASK_MASK (0x1ul << CNTHCTL_CNTVMASK_SHIFT) +#define CNTHCTL_CNTPMASK_SHIFT 19 +#define CNTHCTL_CNTPMASK_MASK (0x1ul << CNTHCTL_CNTPMASK_SHIFT) /* CNTPOFF_EL2 - Counter-timer Physical Offset Register */ #define CNTPOFF_EL2_REG MRS_REG_ALT_NAME(CNTPOFF_EL2) @@ -190,6 +249,54 @@ #define ICC_SRE_EL2_SRE (1UL << 0) #define ICC_SRE_EL2_EN (1UL << 3) +/* MDCR_EL2 - Hyp Debug Control Register */ +#define MDCR_EL2_HPMN_MASK 0x1f +#define MDCR_EL2_HPMN_SHIFT 0 +#define MDCR_EL2_TPMCR_SHIFT 5 +#define MDCR_EL2_TPMCR (0x1UL << MDCR_EL2_TPMCR_SHIFT) +#define MDCR_EL2_TPM_SHIFT 6 +#define MDCR_EL2_TPM (0x1UL << MDCR_EL2_TPM_SHIFT) +#define MDCR_EL2_HPME_SHIFT 7 +#define MDCR_EL2_HPME (0x1UL << MDCR_EL2_HPME_SHIFT) +#define MDCR_EL2_TDE_SHIFT 8 +#define MDCR_EL2_TDE (0x1UL << MDCR_EL2_TDE_SHIFT) +#define MDCR_EL2_TDA_SHIFT 9 +#define MDCR_EL2_TDA (0x1UL << MDCR_EL2_TDA_SHIFT) +#define MDCR_EL2_TDOSA_SHIFT 10 +#define MDCR_EL2_TDOSA (0x1UL << MDCR_EL2_TDOSA_SHIFT) +#define MDCR_EL2_TDRA_SHIFT 11 +#define MDCR_EL2_TDRA (0x1UL << MDCR_EL2_TDRA_SHIFT) +#define MDCR_EL2_E2PB_SHIFT 12 +#define MDCR_EL2_E2PB_MASK (0x3UL << MDCR_EL2_E2PB_SHIFT) +#define MDCR_EL2_TPMS_SHIFT 14 +#define MDCR_EL2_TPMS (0x1UL << MDCR_EL2_TPMS_SHIFT) +#define MDCR_EL2_EnSPM_SHIFT 15 +#define MDCR_EL2_EnSPM (0x1UL << MDCR_EL2_EnSPM_SHIFT) +#define MDCR_EL2_HPMD_SHIFT 17 +#define MDCR_EL2_HPMD (0x1UL << MDCR_EL2_HPMD_SHIFT) +#define MDCR_EL2_TTRF_SHIFT 19 +#define MDCR_EL2_TTRF (0x1UL << MDCR_EL2_TTRF_SHIFT) +#define MDCR_EL2_HCCD_SHIFT 23 +#define MDCR_EL2_HCCD (0x1UL << MDCR_EL2_HCCD_SHIFT) +#define MDCR_EL2_E2TB_SHIFT 24 +#define MDCR_EL2_E2TB_MASK (0x3UL << MDCR_EL2_E2TB_SHIFT) +#define MDCR_EL2_HLP_SHIFT 26 +#define MDCR_EL2_HLP (0x1UL << MDCR_EL2_HLP_SHIFT) +#define MDCR_EL2_TDCC_SHIFT 27 +#define MDCR_EL2_TDCC (0x1UL << MDCR_EL2_TDCC_SHIFT) +#define MDCR_EL2_MTPME_SHIFT 28 +#define MDCR_EL2_MTPME (0x1UL << MDCR_EL2_MTPME_SHIFT) +#define MDCR_EL2_HPMFZO_SHIFT 29 +#define MDCR_EL2_HPMFZO (0x1UL << MDCR_EL2_HPMFZO_SHIFT) +#define MDCR_EL2_PMSSE_SHIFT 30 +#define MDCR_EL2_PMSSE_MASK (0x3UL << MDCR_EL2_PMSSE_SHIFT) +#define MDCR_EL2_HPMFZS_SHIFT 36 +#define MDCR_EL2_HPMFZS (0x1UL << MDCR_EL2_HPMFZS_SHIFT) +#define MDCR_EL2_PMEE_SHIFT 40 +#define MDCR_EL2_PMEE_MASK (0x3UL << MDCR_EL2_PMEE_SHIFT) +#define MDCR_EL2_EBWE_SHIFT 43 +#define MDCR_EL2_EBWE (0x1UL << MDCR_EL2_EBWE_SHIFT) + /* SCTLR_EL2 - System Control Register */ #define SCTLR_EL2_RES1 0x30c50830 #define SCTLR_EL2_M_SHIFT 0 @@ -299,52 +406,4 @@ /* Assumed to be 0 by locore.S */ #define VTTBR_HOST 0x0000000000000000 -/* MDCR_EL2 - Hyp Debug Control Register */ -#define MDCR_EL2_HPMN_MASK 0x1f -#define MDCR_EL2_HPMN_SHIFT 0 -#define MDCR_EL2_TPMCR_SHIFT 5 -#define MDCR_EL2_TPMCR (0x1UL << MDCR_EL2_TPMCR_SHIFT) -#define MDCR_EL2_TPM_SHIFT 6 -#define MDCR_EL2_TPM (0x1UL << MDCR_EL2_TPM_SHIFT) -#define MDCR_EL2_HPME_SHIFT 7 -#define MDCR_EL2_HPME (0x1UL << MDCR_EL2_HPME_SHIFT) -#define MDCR_EL2_TDE_SHIFT 8 -#define MDCR_EL2_TDE (0x1UL << MDCR_EL2_TDE_SHIFT) -#define MDCR_EL2_TDA_SHIFT 9 -#define MDCR_EL2_TDA (0x1UL << MDCR_EL2_TDA_SHIFT) -#define MDCR_EL2_TDOSA_SHIFT 10 -#define MDCR_EL2_TDOSA (0x1UL << MDCR_EL2_TDOSA_SHIFT) -#define MDCR_EL2_TDRA_SHIFT 11 -#define MDCR_EL2_TDRA (0x1UL << MDCR_EL2_TDRA_SHIFT) -#define MDCR_E2PB_SHIFT 12 -#define MDCR_E2PB_MASK (0x3UL << MDCR_E2PB_SHIFT) -#define MDCR_TPMS_SHIFT 14 -#define MDCR_TPMS (0x1UL << MDCR_TPMS_SHIFT) -#define MDCR_EnSPM_SHIFT 15 -#define MDCR_EnSPM (0x1UL << MDCR_EnSPM_SHIFT) -#define MDCR_HPMD_SHIFT 17 -#define MDCR_HPMD (0x1UL << MDCR_HPMD_SHIFT) -#define MDCR_TTRF_SHIFT 19 -#define MDCR_TTRF (0x1UL << MDCR_TTRF_SHIFT) -#define MDCR_HCCD_SHIFT 23 -#define MDCR_HCCD (0x1UL << MDCR_HCCD_SHIFT) -#define MDCR_E2TB_SHIFT 24 -#define MDCR_E2TB_MASK (0x3UL << MDCR_E2TB_SHIFT) -#define MDCR_HLP_SHIFT 26 -#define MDCR_HLP (0x1UL << MDCR_HLP_SHIFT) -#define MDCR_TDCC_SHIFT 27 -#define MDCR_TDCC (0x1UL << MDCR_TDCC_SHIFT) -#define MDCR_MTPME_SHIFT 28 -#define MDCR_MTPME (0x1UL << MDCR_MTPME_SHIFT) -#define MDCR_HPMFZO_SHIFT 29 -#define MDCR_HPMFZO (0x1UL << MDCR_HPMFZO_SHIFT) -#define MDCR_PMSSE_SHIFT 30 -#define MDCR_PMSSE_MASK (0x3UL << MDCR_PMSSE_SHIFT) -#define MDCR_HPMFZS_SHIFT 36 -#define MDCR_HPMFZS (0x1UL << MDCR_HPMFZS_SHIFT) -#define MDCR_PMEE_SHIFT 40 -#define MDCR_PMEE_MASK (0x3UL << MDCR_PMEE_SHIFT) -#define MDCR_EBWE_SHIFT 43 -#define MDCR_EBWE (0x1UL << MDCR_EBWE_SHIFT) - #endif /* !_MACHINE_HYPERVISOR_H_ */ diff --git a/sys/arm64/include/kexec.h b/sys/arm64/include/kexec.h new file mode 100644 index 000000000000..0a8c7a053331 --- /dev/null +++ b/sys/arm64/include/kexec.h @@ -0,0 +1,33 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Juniper Networks, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _ARM64_KEXEC_H_ +#define _ARM64_KEXEC_H_ + +#define KEXEC_MD_PAGES(x) 0 + +#endif /* _ARM64_KEXEC_H_ */ diff --git a/sys/arm64/include/pcpu.h b/sys/arm64/include/pcpu.h index 09bd8fa8a966..73399d2c3f8c 100644 --- a/sys/arm64/include/pcpu.h +++ b/sys/arm64/include/pcpu.h @@ -50,7 +50,8 @@ struct debug_monitor_state; struct pmap *pc_curvmpmap; \ uint64_t pc_mpidr; \ u_int pc_bcast_tlbi_workaround; \ - char __pad[197] + uint64_t pc_release_addr; \ + char __pad[189] #ifdef _KERNEL diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h index 0f23f200f0f6..406b6e2c5e0a 100644 --- a/sys/arm64/include/pmap.h +++ b/sys/arm64/include/pmap.h @@ -69,6 +69,7 @@ struct md_page { TAILQ_HEAD(,pv_entry) pv_list; int pv_gen; vm_memattr_t pv_memattr; + uint8_t pv_reserve[3]; }; enum pmap_stage { @@ -174,6 +175,8 @@ int pmap_fault(pmap_t, uint64_t, uint64_t); struct pcb *pmap_switch(struct thread *); +void pmap_s1_invalidate_all_kernel(void); + extern void (*pmap_clean_stage2_tlbi)(void); extern void (*pmap_stage2_invalidate_range)(uint64_t, vm_offset_t, vm_offset_t, bool); diff --git a/sys/arm64/include/proc.h b/sys/arm64/include/proc.h index 184743d4cc80..b40990e89385 100644 --- a/sys/arm64/include/proc.h +++ b/sys/arm64/include/proc.h @@ -75,6 +75,7 @@ struct mdthread { struct mdproc { uint64_t md_tcr; /* TCR_EL1 fields to update */ + uint64_t md_reserved[2]; }; #endif /* !LOCORE */ diff --git a/sys/arm64/include/smp.h b/sys/arm64/include/smp.h index 500cd1ef4f02..4a5bfda3ac1c 100644 --- a/sys/arm64/include/smp.h +++ b/sys/arm64/include/smp.h @@ -40,6 +40,7 @@ enum { IPI_STOP, IPI_STOP_HARD, IPI_HARDCLOCK, + IPI_OFF, INTR_IPI_COUNT, }; diff --git a/sys/arm64/include/vmm.h b/sys/arm64/include/vmm.h index 73b5b4a09591..e67540eac66d 100644 --- a/sys/arm64/include/vmm.h +++ b/sys/arm64/include/vmm.h @@ -42,6 +42,7 @@ enum vm_suspend_how { VM_SUSPEND_RESET, VM_SUSPEND_POWEROFF, VM_SUSPEND_HALT, + VM_SUSPEND_DESTROY, VM_SUSPEND_LAST }; @@ -105,27 +106,6 @@ enum vm_reg_name { #define VM_GUEST_BASE_IPA 0x80000000UL /* Guest kernel start ipa */ -/* - * The VM name has to fit into the pathname length constraints of devfs, - * governed primarily by SPECNAMELEN. The length is the total number of - * characters in the full path, relative to the mount point and not - * including any leading '/' characters. - * A prefix and a suffix are added to the name specified by the user. - * The prefix is usually "vmm/" or "vmm.io/", but can be a few characters - * longer for future use. - * The suffix is a string that identifies a bootrom image or some similar - * image that is attached to the VM. A separator character gets added to - * the suffix automatically when generating the full path, so it must be - * accounted for, reducing the effective length by 1. - * The effective length of a VM name is 229 bytes for FreeBSD 13 and 37 - * bytes for FreeBSD 12. A minimum length is set for safety and supports - * a SPECNAMELEN as small as 32 on old systems. - */ -#define VM_MAX_PREFIXLEN 10 -#define VM_MAX_SUFFIXLEN 15 -#define VM_MAX_NAMELEN \ - (SPECNAMELEN - VM_MAX_PREFIXLEN - VM_MAX_SUFFIXLEN - 1) - #ifdef _KERNEL struct vm; struct vm_exception; @@ -142,10 +122,41 @@ struct vm_eventinfo { int *iptr; /* reqidle cookie */ }; +#define DECLARE_VMMOPS_FUNC(ret_type, opname, args) \ + ret_type vmmops_##opname args + +DECLARE_VMMOPS_FUNC(int, modinit, (int ipinum)); +DECLARE_VMMOPS_FUNC(int, modcleanup, (void)); +DECLARE_VMMOPS_FUNC(void *, init, (struct vm *vm, struct pmap *pmap)); +DECLARE_VMMOPS_FUNC(int, gla2gpa, (void *vcpui, struct vm_guest_paging *paging, + uint64_t gla, int prot, uint64_t *gpa, int *is_fault)); +DECLARE_VMMOPS_FUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap, + struct vm_eventinfo *info)); +DECLARE_VMMOPS_FUNC(void, cleanup, (void *vmi)); +DECLARE_VMMOPS_FUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu, + int vcpu_id)); +DECLARE_VMMOPS_FUNC(void, vcpu_cleanup, (void *vcpui)); +DECLARE_VMMOPS_FUNC(int, exception, (void *vcpui, uint64_t esr, uint64_t far)); +DECLARE_VMMOPS_FUNC(int, getreg, (void *vcpui, int num, uint64_t *retval)); +DECLARE_VMMOPS_FUNC(int, setreg, (void *vcpui, int num, uint64_t val)); +DECLARE_VMMOPS_FUNC(int, getcap, (void *vcpui, int num, int *retval)); +DECLARE_VMMOPS_FUNC(int, setcap, (void *vcpui, int num, int val)); +DECLARE_VMMOPS_FUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min, + vm_offset_t max)); +DECLARE_VMMOPS_FUNC(void, vmspace_free, (struct vmspace *vmspace)); +#ifdef notyet +#ifdef BHYVE_SNAPSHOT +DECLARE_VMMOPS_FUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta *meta)); +DECLARE_VMMOPS_FUNC(int, vcpu_snapshot, (void *vcpui, + struct vm_snapshot_meta *meta)); +DECLARE_VMMOPS_FUNC(int, restore_tsc, (void *vcpui, uint64_t now)); +#endif +#endif + int vm_create(const char *name, struct vm **retvm); struct vcpu *vm_alloc_vcpu(struct vm *vm, int vcpuid); void vm_disable_vcpu_creation(struct vm *vm); -void vm_slock_vcpus(struct vm *vm); +void vm_lock_vcpus(struct vm *vm); void vm_unlock_vcpus(struct vm *vm); void vm_destroy(struct vm *vm); int vm_reinit(struct vm *vm); @@ -231,7 +242,6 @@ vcpu_should_yield(struct vcpu *vcpu) void *vcpu_stats(struct vcpu *vcpu); void vcpu_notify_event(struct vcpu *vcpu); -struct vmspace *vm_vmspace(struct vm *vm); struct vm_mem *vm_mem(struct vm *vm); enum vm_reg_name vm_segment_name(int seg_encoding); diff --git a/sys/arm64/include/vmm_dev.h b/sys/arm64/include/vmm_dev.h index 219f1116c728..289ff0fe1fc9 100644 --- a/sys/arm64/include/vmm_dev.h +++ b/sys/arm64/include/vmm_dev.h @@ -31,6 +31,8 @@ #include <machine/vmm.h> +#include <dev/vmm/vmm_param.h> + struct vm_memmap { vm_paddr_t gpa; int segid; /* memory segment */ diff --git a/sys/arm64/linux/linux_sysvec.c b/sys/arm64/linux/linux_sysvec.c index 084b7a11b01f..ac05820f89bc 100644 --- a/sys/arm64/linux/linux_sysvec.c +++ b/sys/arm64/linux/linux_sysvec.c @@ -584,7 +584,7 @@ linux_vdso_reloc(char *mapping, Elf_Addr offset) } } -static Elf_Brandnote linux64_brandnote = { +static const Elf_Brandnote linux64_brandnote = { .hdr.n_namesz = sizeof(GNU_ABI_VENDOR), .hdr.n_descsz = 16, .hdr.n_type = 1, @@ -593,7 +593,7 @@ static Elf_Brandnote linux64_brandnote = { .trans_osrel = linux_trans_osrel }; -static Elf64_Brandinfo linux_glibc2brand = { +static const Elf64_Brandinfo linux_glibc2brand = { .brand = ELFOSABI_LINUX, .machine = EM_AARCH64, .compat_3_brand = "Linux", @@ -604,7 +604,7 @@ static Elf64_Brandinfo linux_glibc2brand = { .flags = BI_CAN_EXEC_DYN | BI_BRAND_NOTE }; -Elf64_Brandinfo *linux_brandlist[] = { +const Elf64_Brandinfo *linux_brandlist[] = { &linux_glibc2brand, NULL }; @@ -612,8 +612,8 @@ Elf64_Brandinfo *linux_brandlist[] = { static int linux64_elf_modevent(module_t mod, int type, void *data) { - Elf64_Brandinfo **brandinfo; - struct linux_ioctl_handler**lihp; + const Elf64_Brandinfo **brandinfo; + struct linux_ioctl_handler **lihp; int error; error = 0; diff --git a/sys/arm64/nvidia/tegra210/max77620_regulators.c b/sys/arm64/nvidia/tegra210/max77620_regulators.c index af1a5af20ec3..d52aeaef1287 100644 --- a/sys/arm64/nvidia/tegra210/max77620_regulators.c +++ b/sys/arm64/nvidia/tegra210/max77620_regulators.c @@ -364,7 +364,7 @@ max77620_get_sel(struct max77620_reg_sc *sc, uint8_t *sel) rv = RD1(sc->base_sc, sc->def->volt_reg, sel); if (rv != 0) { - printf("%s: cannot read volatge selector: %d\n", + printf("%s: cannot read voltage selector: %d\n", regnode_get_name(sc->regnode), rv); return (rv); } @@ -384,7 +384,7 @@ max77620_set_sel(struct max77620_reg_sc *sc, uint8_t sel) rv = RM1(sc->base_sc, sc->def->volt_reg, sc->def->volt_vsel_mask, sel); if (rv != 0) { - printf("%s: cannot set volatge selector: %d\n", + printf("%s: cannot set voltage selector: %d\n", regnode_get_name(sc->regnode), rv); return (rv); } diff --git a/sys/arm64/rockchip/rk_gpio.c b/sys/arm64/rockchip/rk_gpio.c index 61614f532634..8da37d516802 100644 --- a/sys/arm64/rockchip/rk_gpio.c +++ b/sys/arm64/rockchip/rk_gpio.c @@ -90,6 +90,11 @@ struct rk_pin_irqsrc { uint32_t mode; }; +struct rk_gpio_reg { + uint8_t single; + uint8_t offset; +}; + struct rk_gpio_softc { device_t sc_dev; device_t sc_busdev; @@ -103,7 +108,7 @@ struct rk_gpio_softc { uint32_t swporta_ddr; uint32_t version; struct pin_cached pin_cached[RK_GPIO_MAX_PINS]; - uint8_t regs[RK_GPIO_REGNUM]; + struct rk_gpio_reg regs[RK_GPIO_REGNUM]; void *ihandle; struct rk_pin_irqsrc isrcs[RK_GPIO_MAX_PINS]; }; @@ -138,14 +143,15 @@ static int rk_gpio_detach(device_t dev); static int rk_gpio_read_bit(struct rk_gpio_softc *sc, int reg, int bit) { - int offset = sc->regs[reg]; + struct rk_gpio_reg *rk_reg = &sc->regs[reg]; uint32_t value; - if (sc->version == RK_GPIO_TYPE_V1) { - value = RK_GPIO_READ(sc, offset); + if (rk_reg->single) { + value = RK_GPIO_READ(sc, rk_reg->offset); value >>= bit; } else { - value = RK_GPIO_READ(sc, bit > 15 ? offset + 4 : offset); + value = RK_GPIO_READ(sc, bit > 15 ? + rk_reg->offset + 4 : rk_reg->offset); value >>= (bit % 16); } return (value & 1); @@ -154,50 +160,53 @@ rk_gpio_read_bit(struct rk_gpio_softc *sc, int reg, int bit) static void rk_gpio_write_bit(struct rk_gpio_softc *sc, int reg, int bit, int data) { - int offset = sc->regs[reg]; + struct rk_gpio_reg *rk_reg = &sc->regs[reg]; uint32_t value; - if (sc->version == RK_GPIO_TYPE_V1) { - value = RK_GPIO_READ(sc, offset); + if (rk_reg->single) { + value = RK_GPIO_READ(sc, rk_reg->offset); if (data) value |= (1 << bit); else value &= ~(1 << bit); - RK_GPIO_WRITE(sc, offset, value); + RK_GPIO_WRITE(sc, rk_reg->offset, value); } else { if (data) value = (1 << (bit % 16)); else value = 0; value |= (1 << ((bit % 16) + 16)); - RK_GPIO_WRITE(sc, bit > 15 ? offset + 4 : offset, value); + RK_GPIO_WRITE(sc, bit > 15 ? + rk_reg->offset + 4 : rk_reg->offset, value); } } static uint32_t rk_gpio_read_4(struct rk_gpio_softc *sc, int reg) { - int offset = sc->regs[reg]; + struct rk_gpio_reg *rk_reg = &sc->regs[reg]; uint32_t value; - if (sc->version == RK_GPIO_TYPE_V1) - value = RK_GPIO_READ(sc, offset); + if (rk_reg->single) + value = RK_GPIO_READ(sc, rk_reg->offset); else - value = (RK_GPIO_READ(sc, offset) & 0xffff) | - (RK_GPIO_READ(sc, offset + 4) << 16); + value = (RK_GPIO_READ(sc, rk_reg->offset) & 0xffff) | + (RK_GPIO_READ(sc, rk_reg->offset + 4) << 16); return (value); } static void rk_gpio_write_4(struct rk_gpio_softc *sc, int reg, uint32_t value) { - int offset = sc->regs[reg]; + struct rk_gpio_reg *rk_reg = &sc->regs[reg]; - if (sc->version == RK_GPIO_TYPE_V1) - RK_GPIO_WRITE(sc, offset, value); + if (rk_reg->single) + RK_GPIO_WRITE(sc, rk_reg->offset, value); else { - RK_GPIO_WRITE(sc, offset, (value & 0xffff) | 0xffff0000); - RK_GPIO_WRITE(sc, offset + 4, (value >> 16) | 0xffff0000); + RK_GPIO_WRITE(sc, rk_reg->offset, + (value & 0xffff) | 0xffff0000); + RK_GPIO_WRITE(sc, rk_reg->offset + 4, + (value >> 16) | 0xffff0000); } } @@ -313,31 +322,31 @@ rk_gpio_attach(device_t dev) switch (sc->version) { case RK_GPIO_TYPE_V1: - sc->regs[RK_GPIO_SWPORTA_DR] = 0x00; - sc->regs[RK_GPIO_SWPORTA_DDR] = 0x04; - sc->regs[RK_GPIO_INTEN] = 0x30; - sc->regs[RK_GPIO_INTMASK] = 0x34; - sc->regs[RK_GPIO_INTTYPE_LEVEL] = 0x38; - sc->regs[RK_GPIO_INT_POLARITY] = 0x3c; - sc->regs[RK_GPIO_INT_STATUS] = 0x40; - sc->regs[RK_GPIO_INT_RAWSTATUS] = 0x44; - sc->regs[RK_GPIO_DEBOUNCE] = 0x48; - sc->regs[RK_GPIO_PORTA_EOI] = 0x4c; - sc->regs[RK_GPIO_EXT_PORTA] = 0x50; + sc->regs[RK_GPIO_SWPORTA_DR] = (struct rk_gpio_reg){ 1, 0x00 }; + sc->regs[RK_GPIO_SWPORTA_DDR] = (struct rk_gpio_reg){ 1, 0x04 }; + sc->regs[RK_GPIO_INTEN] = (struct rk_gpio_reg){ 1, 0x30 }; + sc->regs[RK_GPIO_INTMASK] = (struct rk_gpio_reg){ 1, 0x34 }; + sc->regs[RK_GPIO_INTTYPE_LEVEL] = (struct rk_gpio_reg){ 1, 0x38 }; + sc->regs[RK_GPIO_INT_POLARITY] = (struct rk_gpio_reg){ 1, 0x3c }; + sc->regs[RK_GPIO_INT_STATUS] = (struct rk_gpio_reg){ 1, 0x40 }; + sc->regs[RK_GPIO_INT_RAWSTATUS] = (struct rk_gpio_reg){ 1, 0x44 }; + sc->regs[RK_GPIO_DEBOUNCE] = (struct rk_gpio_reg){ 1, 0x48 }; + sc->regs[RK_GPIO_PORTA_EOI] = (struct rk_gpio_reg){ 1, 0x4c }; + sc->regs[RK_GPIO_EXT_PORTA] = (struct rk_gpio_reg){ 1, 0x50 }; break; case RK_GPIO_TYPE_V2: - sc->regs[RK_GPIO_SWPORTA_DR] = 0x00; - sc->regs[RK_GPIO_SWPORTA_DDR] = 0x08; - sc->regs[RK_GPIO_INTEN] = 0x10; - sc->regs[RK_GPIO_INTMASK] = 0x18; - sc->regs[RK_GPIO_INTTYPE_LEVEL] = 0x20; - sc->regs[RK_GPIO_INTTYPE_BOTH] = 0x30; - sc->regs[RK_GPIO_INT_POLARITY] = 0x28; - sc->regs[RK_GPIO_INT_STATUS] = 0x50; - sc->regs[RK_GPIO_INT_RAWSTATUS] = 0x58; - sc->regs[RK_GPIO_DEBOUNCE] = 0x38; - sc->regs[RK_GPIO_PORTA_EOI] = 0x60; - sc->regs[RK_GPIO_EXT_PORTA] = 0x70; + sc->regs[RK_GPIO_SWPORTA_DR] = (struct rk_gpio_reg){ 0, 0x00 }; + sc->regs[RK_GPIO_SWPORTA_DDR] = (struct rk_gpio_reg){ 0, 0x08 }; + sc->regs[RK_GPIO_INTEN] = (struct rk_gpio_reg){ 0, 0x10 }; + sc->regs[RK_GPIO_INTMASK] = (struct rk_gpio_reg){ 0, 0x18 }; + sc->regs[RK_GPIO_INTTYPE_LEVEL] = (struct rk_gpio_reg){ 0, 0x20 }; + sc->regs[RK_GPIO_INTTYPE_BOTH] = (struct rk_gpio_reg){ 0, 0x30 }; + sc->regs[RK_GPIO_INT_POLARITY] = (struct rk_gpio_reg){ 0, 0x28 }; + sc->regs[RK_GPIO_INT_STATUS] = (struct rk_gpio_reg){ 1, 0x50 }; + sc->regs[RK_GPIO_INT_RAWSTATUS] = (struct rk_gpio_reg){ 1, 0x58 }; + sc->regs[RK_GPIO_DEBOUNCE] = (struct rk_gpio_reg){ 0, 0x38 }; + sc->regs[RK_GPIO_PORTA_EOI] = (struct rk_gpio_reg){ 0, 0x60 }; + sc->regs[RK_GPIO_EXT_PORTA] = (struct rk_gpio_reg){ 1, 0x70 }; break; default: device_printf(dev, "Unknown gpio version %08x\n", sc->version); @@ -394,7 +403,7 @@ rk_gpio_detach(device_t dev) mtx_destroy(&sc->sc_mtx); clk_disable(sc->clk); - return(0); + return (0); } static device_t @@ -471,7 +480,7 @@ rk_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { if (pin >= RK_GPIO_MAX_PINS) - return EINVAL; + return (EINVAL); *caps = RK_GPIO_DEFAULT_CAPS; return (0); @@ -654,46 +663,108 @@ rk_gpio_get_node(device_t bus, device_t dev) } static int -rk_pic_map_intr(device_t dev, struct intr_map_data *data, - struct intr_irqsrc **isrcp) +rk_gpio_pic_map_fdt(struct rk_gpio_softc *sc, + struct intr_map_data_fdt *daf, + u_int *irqp, uint32_t *modep) { - struct rk_gpio_softc *sc = device_get_softc(dev); - struct intr_map_data_gpio *gdata; uint32_t irq; + uint32_t mode; - if (data->type != INTR_MAP_DATA_GPIO) { - device_printf(dev, "Wrong type\n"); - return (ENOTSUP); - } - gdata = (struct intr_map_data_gpio *)data; - irq = gdata->gpio_pin_num; + if (daf->ncells != 2) + return (EINVAL); + + irq = daf->cells[0]; + if (irq >= RK_GPIO_MAX_PINS) + return (EINVAL); + + /* Only reasonable modes are supported. */ + if (daf->cells[1] == 1) + mode = GPIO_INTR_EDGE_RISING; + else if (daf->cells[1] == 2) + mode = GPIO_INTR_EDGE_FALLING; + else if (daf->cells[1] == 3) + mode = GPIO_INTR_EDGE_BOTH; + else if (daf->cells[1] == 4) + mode = GPIO_INTR_LEVEL_HIGH; + else if (daf->cells[1] == 8) + mode = GPIO_INTR_LEVEL_LOW; + else + return (EINVAL); + + *irqp = irq; + if (modep != NULL) + *modep = mode; + return (0); +} + +static int +rk_gpio_pic_map_gpio(struct rk_gpio_softc *sc, + struct intr_map_data_gpio *dag, + u_int *irqp, uint32_t *modep) +{ + uint32_t irq; + irq = dag->gpio_pin_num; if (irq >= RK_GPIO_MAX_PINS) { - device_printf(dev, "Invalid interrupt %u\n", irq); + device_printf(sc->sc_dev, "Invalid interrupt %u\n", + irq); return (EINVAL); } - *isrcp = RK_GPIO_ISRC(sc, irq); + + *irqp = irq; + if (modep != NULL) + *modep = dag->gpio_intr_mode; return (0); } static int +rk_gpio_pic_map(struct rk_gpio_softc *sc, struct intr_map_data *data, + u_int *irqp, uint32_t *modep) +{ + switch (data->type) { + case INTR_MAP_DATA_FDT: + return (rk_gpio_pic_map_fdt(sc, + (struct intr_map_data_fdt *)data, irqp, modep)); + case INTR_MAP_DATA_GPIO: + return (rk_gpio_pic_map_gpio(sc, + (struct intr_map_data_gpio *)data, irqp, modep)); + default: + device_printf(sc->sc_dev, "Wrong type\n"); + return (ENOTSUP); + } +} + +static int +rk_pic_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + int error; + struct rk_gpio_softc *sc = device_get_softc(dev); + uint32_t irq; + + error = rk_gpio_pic_map(sc, data, &irq, NULL); + if (error == 0) + *isrcp = RK_GPIO_ISRC(sc, irq); + return (error); +} + +static int rk_pic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct rk_gpio_softc *sc = device_get_softc(dev); struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc; - struct intr_map_data_gpio *gdata; uint32_t mode; - uint8_t pin; + uint32_t pin; if (!data) { device_printf(dev, "No map data\n"); return (ENOTSUP); } - gdata = (struct intr_map_data_gpio *)data; - mode = gdata->gpio_intr_mode; - pin = gdata->gpio_pin_num; - if (rkisrc->irq != gdata->gpio_pin_num) { + if (rk_gpio_pic_map(sc, data, &pin, &mode) != 0) + return (EINVAL); + + if (rkisrc->irq != pin) { device_printf(dev, "Interrupts don't match\n"); return (EINVAL); } @@ -780,6 +851,10 @@ static device_method_t rk_gpio_methods[] = { DEVMETHOD(device_attach, rk_gpio_attach), DEVMETHOD(device_detach, rk_gpio_detach), + /* Bus interface */ + DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), + DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), + /* GPIO protocol */ DEVMETHOD(gpio_get_bus, rk_gpio_get_bus), DEVMETHOD(gpio_pin_max, rk_gpio_pin_max), diff --git a/sys/arm64/rockchip/rk_tsadc.c b/sys/arm64/rockchip/rk_tsadc.c index e6cbad36f697..d83b09480a0c 100644 --- a/sys/arm64/rockchip/rk_tsadc.c +++ b/sys/arm64/rockchip/rk_tsadc.c @@ -484,7 +484,7 @@ tsadc_init_tsensor(struct tsadc_softc *sc, struct tsensor *sensor) WR4(sc, TSADC_INT_EN, val); /* Shutdown temperature */ - val = tsadc_raw_to_temp(sc, sc->shutdown_temp); + val = tsadc_temp_to_raw(sc, sc->shutdown_temp); WR4(sc, TSADC_COMP_SHUT(sensor->channel), val); val = RD4(sc, TSADC_AUTO_CON); val |= TSADC_AUTO_SRC_EN(sensor->channel); diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h index 6a0c4c78e568..f530dab05331 100644 --- a/sys/arm64/vmm/arm64.h +++ b/sys/arm64/vmm/arm64.h @@ -78,14 +78,16 @@ struct hypctx { uint64_t pmcr_el0; /* Performance Monitors Control Register */ uint64_t pmccntr_el0; uint64_t pmccfiltr_el0; + uint64_t pmuserenr_el0; + uint64_t pmselr_el0; + uint64_t pmxevcntr_el0; uint64_t pmcntenset_el0; uint64_t pmintenset_el1; uint64_t pmovsset_el0; - uint64_t pmselr_el0; - uint64_t pmuserenr_el0; uint64_t pmevcntr_el0[31]; uint64_t pmevtyper_el0[31]; + uint64_t dbgclaimset_el1; uint64_t dbgbcr_el1[16]; /* Debug Breakpoint Control Registers */ uint64_t dbgbvr_el1[16]; /* Debug Breakpoint Value Registers */ uint64_t dbgwcr_el1[16]; /* Debug Watchpoint Control Registers */ @@ -117,6 +119,7 @@ struct hypctx { struct vgic_v3_regs vgic_v3_regs; struct vgic_v3_cpu *vgic_cpu; bool has_exception; + bool dbg_oslock; }; struct hyp { @@ -125,42 +128,14 @@ struct hyp { uint64_t vmid_generation; uint64_t vttbr_el2; uint64_t el2_addr; /* The address of this in el2 space */ + uint64_t feats; /* Which features are enabled */ +#define HYP_FEAT_HCX (0x1ul << 0) +#define HYP_FEAT_ECV_POFF (0x1ul << 1) bool vgic_attached; struct vgic_v3 *vgic; struct hypctx *ctx[]; }; -#define DEFINE_VMMOPS_IFUNC(ret_type, opname, args) \ - ret_type vmmops_##opname args; - -DEFINE_VMMOPS_IFUNC(int, modinit, (int ipinum)) -DEFINE_VMMOPS_IFUNC(int, modcleanup, (void)) -DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap)) -DEFINE_VMMOPS_IFUNC(int, gla2gpa, (void *vcpui, struct vm_guest_paging *paging, - uint64_t gla, int prot, uint64_t *gpa, int *is_fault)) -DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap, - struct vm_eventinfo *info)) -DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi)) -DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu, - int vcpu_id)) -DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vcpui)) -DEFINE_VMMOPS_IFUNC(int, exception, (void *vcpui, uint64_t esr, uint64_t far)) -DEFINE_VMMOPS_IFUNC(int, getreg, (void *vcpui, int num, uint64_t *retval)) -DEFINE_VMMOPS_IFUNC(int, setreg, (void *vcpui, int num, uint64_t val)) -DEFINE_VMMOPS_IFUNC(int, getcap, (void *vcpui, int num, int *retval)) -DEFINE_VMMOPS_IFUNC(int, setcap, (void *vcpui, int num, int val)) -DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min, - vm_offset_t max)) -DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace)) -#ifdef notyet -#ifdef BHYVE_SNAPSHOT -DEFINE_VMMOPS_IFUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta *meta)) -DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vcpui, - struct vm_snapshot_meta *meta)) -DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vcpui, uint64_t now)) -#endif -#endif - uint64_t vmm_call_hyp(uint64_t, ...); #if 0 diff --git a/sys/arm64/vmm/hyp.h b/sys/arm64/vmm/hyp.h index 0b2977c73960..0c8d0fb28b18 100644 --- a/sys/arm64/vmm/hyp.h +++ b/sys/arm64/vmm/hyp.h @@ -80,7 +80,6 @@ #define HYP_ENTER_GUEST 0x00000002 #define HYP_READ_REGISTER 0x00000003 #define HYP_REG_ICH_VTR 0x1 -#define HYP_REG_CNTHCTL 0x2 #define HYP_CLEAN_S2_TLBI 0x00000004 #define HYP_DC_CIVAC 0x00000005 #define HYP_EL2_TLBI 0x00000006 diff --git a/sys/arm64/vmm/io/vgic_v3.c b/sys/arm64/vmm/io/vgic_v3.c index 67afb3374815..023406c64182 100644 --- a/sys/arm64/vmm/io/vgic_v3.c +++ b/sys/arm64/vmm/io/vgic_v3.c @@ -47,7 +47,6 @@ #include <dev/ofw/openfirm.h> -#include <machine/armreg.h> #include <machine/atomic.h> #include <machine/bus.h> #include <machine/cpufunc.h> diff --git a/sys/arm64/vmm/io/vtimer.c b/sys/arm64/vmm/io/vtimer.c index f59d7ebc1ad4..7c7fbb49e691 100644 --- a/sys/arm64/vmm/io/vtimer.c +++ b/sys/arm64/vmm/io/vtimer.c @@ -36,6 +36,7 @@ #include <sys/module.h> #include <sys/mutex.h> #include <sys/rman.h> +#include <sys/sysctl.h> #include <sys/time.h> #include <sys/timeet.h> #include <sys/timetc.h> @@ -43,7 +44,6 @@ #include <machine/bus.h> #include <machine/machdep.h> #include <machine/vmm.h> -#include <machine/armreg.h> #include <arm64/vmm/arm64.h> @@ -55,11 +55,18 @@ #define timer_enabled(ctl) \ (!((ctl) & CNTP_CTL_IMASK) && ((ctl) & CNTP_CTL_ENABLE)) -static uint64_t cnthctl_el2_reg; static uint32_t tmr_frq; #define timer_condition_met(ctl) ((ctl) & CNTP_CTL_ISTATUS) +SYSCTL_DECL(_hw_vmm); +SYSCTL_NODE(_hw_vmm, OID_AUTO, vtimer, CTLFLAG_RW, NULL, NULL); + +static bool allow_ecv_phys = false; +SYSCTL_BOOL(_hw_vmm_vtimer, OID_AUTO, allow_ecv_phys, CTLFLAG_RW, + &allow_ecv_phys, 0, + "Enable hardware access to the physical timer if FEAT_ECV_POFF is supported"); + static void vtimer_schedule_irq(struct hypctx *hypctx, bool phys); static int @@ -111,9 +118,8 @@ out: } int -vtimer_init(uint64_t cnthctl_el2) +vtimer_init(void) { - cnthctl_el2_reg = cnthctl_el2; /* * The guest *MUST* use the same timer frequency as the host. The * register CNTFRQ_EL0 is accessible to the guest and a different value @@ -128,8 +134,12 @@ void vtimer_vminit(struct hyp *hyp) { uint64_t now; + bool ecv_poff; - hyp->vtimer.cnthctl_el2 = cnthctl_el2_reg; + ecv_poff = false; + + if (allow_ecv_phys && (hyp->feats & HYP_FEAT_ECV_POFF) != 0) + ecv_poff = true; /* * Configure the Counter-timer Hypervisor Control Register for the VM. @@ -137,35 +147,58 @@ vtimer_vminit(struct hyp *hyp) if (in_vhe()) { /* * CNTHCTL_E2H_EL0PCTEN: trap EL0 access to CNTP{CT,CTSS}_EL0 - * CNTHCTL_E2H_EL1VCTEN: don't trap EL0 access to - * CNTV{CT,CTSS}_EL0 + * CNTHCTL_E2H_EL0VCTEN: don't trap EL0 access to + * CNTV{CT,CTXX}_EL0 * CNTHCTL_E2H_EL0VTEN: don't trap EL0 access to * CNTV_{CTL,CVAL,TVAL}_EL0 * CNTHCTL_E2H_EL0PTEN: trap EL0 access to * CNTP_{CTL,CVAL,TVAL}_EL0 - * CNTHCTL_E2H_EL1PCEN: trap EL1 access to - CNTP_{CTL,CVAL,TVAL}_EL0 * CNTHCTL_E2H_EL1PCTEN: trap access to CNTPCT_EL0 + * CNTHCTL_E2H_EL1PTEN: trap access to + * CNTP_{CTL,CVAL,TVAL}_EL0 + * CNTHCTL_E2H_EL1VCTEN: don't trap EL0 access to + * CNTV{CT,CTSS}_EL0 + * CNTHCTL_E2H_EL1PCEN: trap EL1 access to + * CNTP_{CTL,CVAL,TVAL}_EL0 * * TODO: Don't trap when FEAT_ECV is present */ - hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL0PCTEN; - hyp->vtimer.cnthctl_el2 |= CNTHCTL_E2H_EL0VCTEN; - hyp->vtimer.cnthctl_el2 |= CNTHCTL_E2H_EL0VTEN; - hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL0PTEN; - - hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL1PTEN; - hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL1PCTEN; + hyp->vtimer.cnthctl_el2 = + CNTHCTL_E2H_EL0VCTEN_NOTRAP | + CNTHCTL_E2H_EL0VTEN_NOTRAP; + if (ecv_poff) { + hyp->vtimer.cnthctl_el2 |= + CNTHCTL_E2H_EL0PCTEN_NOTRAP | + CNTHCTL_E2H_EL0PTEN_NOTRAP | + CNTHCTL_E2H_EL1PCTEN_NOTRAP | + CNTHCTL_E2H_EL1PTEN_NOTRAP; + } else { + hyp->vtimer.cnthctl_el2 |= + CNTHCTL_E2H_EL0PCTEN_TRAP | + CNTHCTL_E2H_EL0PTEN_TRAP | + CNTHCTL_E2H_EL1PCTEN_TRAP | + CNTHCTL_E2H_EL1PTEN_TRAP; + } } else { /* * CNTHCTL_EL1PCEN: trap access to CNTP_{CTL, CVAL, TVAL}_EL0 * from EL1 * CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0 */ - hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCEN; - hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCTEN; + if (ecv_poff) { + hyp->vtimer.cnthctl_el2 = + CNTHCTL_EL1PCTEN_NOTRAP | + CNTHCTL_EL1PCEN_NOTRAP; + } else { + hyp->vtimer.cnthctl_el2 = + CNTHCTL_EL1PCTEN_TRAP | + CNTHCTL_EL1PCEN_TRAP; + } } + if (ecv_poff) + hyp->vtimer.cnthctl_el2 |= CNTHCTL_ECV_EN; + now = READ_SPECIALREG(cntpct_el0); hyp->vtimer.cntvoff_el2 = now; @@ -231,15 +264,10 @@ vtimer_cleanup(void) { } -void -vtimer_sync_hwstate(struct hypctx *hypctx) +static void +vtime_sync_timer(struct hypctx *hypctx, struct vtimer_timer *timer, + uint64_t cntpct_el0) { - struct vtimer_timer *timer; - uint64_t cntpct_el0; - - timer = &hypctx->vtimer_cpu.virt_timer; - cntpct_el0 = READ_SPECIALREG(cntpct_el0) - - hypctx->hyp->vtimer.cntvoff_el2; if (!timer_enabled(timer->cntx_ctl_el0)) { vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), timer->irqid, false); @@ -253,6 +281,21 @@ vtimer_sync_hwstate(struct hypctx *hypctx) } } +void +vtimer_sync_hwstate(struct hypctx *hypctx) +{ + uint64_t cntpct_el0; + + cntpct_el0 = READ_SPECIALREG(cntpct_el0) - + hypctx->hyp->vtimer.cntvoff_el2; + vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.virt_timer, cntpct_el0); + /* If FEAT_ECV_POFF is in use then we need to sync the physical timer */ + if ((hypctx->hyp->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0) { + vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.phys_timer, + cntpct_el0); + } +} + static void vtimer_inject_irq_callout_phys(void *context) { diff --git a/sys/arm64/vmm/io/vtimer.h b/sys/arm64/vmm/io/vtimer.h index 71a20344d05e..92ce025968d2 100644 --- a/sys/arm64/vmm/io/vtimer.h +++ b/sys/arm64/vmm/io/vtimer.h @@ -66,7 +66,7 @@ struct vtimer_cpu { uint32_t cntkctl_el1; }; -int vtimer_init(uint64_t cnthctl_el2); +int vtimer_init(void); void vtimer_vminit(struct hyp *); void vtimer_cpuinit(struct hypctx *); void vtimer_cpucleanup(struct hypctx *); diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c index 3082d2941221..31d2fb3f516b 100644 --- a/sys/arm64/vmm/vmm.c +++ b/sys/arm64/vmm/vmm.c @@ -33,7 +33,6 @@ #include <sys/linker.h> #include <sys/lock.h> #include <sys/malloc.h> -#include <sys/module.h> #include <sys/mutex.h> #include <sys/pcpu.h> #include <sys/proc.h> @@ -51,7 +50,6 @@ #include <vm/vm_extern.h> #include <vm/vm_param.h> -#include <machine/armreg.h> #include <machine/cpu.h> #include <machine/fpu.h> #include <machine/machdep.h> @@ -88,7 +86,6 @@ struct vcpu { struct vfpstate *guestfpu; /* (a,i) guest fpu state */ }; -#define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx)) #define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) #define vcpu_lock_destroy(v) mtx_destroy(&((v)->mtx)) #define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) @@ -126,9 +123,8 @@ struct vm { bool dying; /* (o) is dying */ volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */ volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */ - struct vmspace *vmspace; /* (o) guest's address space */ struct vm_mem mem; /* (i) guest memory */ - char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */ + char name[VM_MAX_NAMELEN + 1]; /* (o) virtual machine name */ struct vcpu **vcpu; /* (i) guest vcpus */ struct vmm_mmio_region mmio_region[VM_MAX_MMIO_REGIONS]; /* (o) guest MMIO regions */ @@ -141,8 +137,6 @@ struct vm { struct sx vcpus_init_lock; /* (o) */ }; -static bool vmm_initialized = false; - static int vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu); @@ -211,10 +205,6 @@ static const struct vmm_regs vmm_arch_regs_masks = { /* Host registers masked by vmm_arch_regs_masks. */ static struct vmm_regs vmm_arch_regs; -u_int vm_maxcpu; -SYSCTL_UINT(_hw_vmm, OID_AUTO, maxcpu, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, - &vm_maxcpu, 0, "Maximum number of vCPUs"); - static void vcpu_notify_event_locked(struct vcpu *vcpu); /* global statistics */ @@ -234,12 +224,6 @@ VMM_STAT(VMEXIT_SS, "number of vmexits for a single-step exception"); VMM_STAT(VMEXIT_UNHANDLED_EL2, "number of vmexits for an unhandled EL2 exception"); VMM_STAT(VMEXIT_UNHANDLED, "number of vmexits for an unhandled exception"); -/* - * Upper limit on vm_maxcpu. We could increase this to 28 bits, but this - * is a safe value for now. - */ -#define VM_MAXCPU MIN(0xffff - 1, CPU_SETSIZE) - static int vmm_regs_init(struct vmm_regs *regs, const struct vmm_regs *masks) { @@ -274,6 +258,7 @@ vcpu_cleanup(struct vcpu *vcpu, bool destroy) vmm_stat_free(vcpu->stats); fpu_save_area_free(vcpu->guestfpu); vcpu_lock_destroy(vcpu); + free(vcpu, M_VMM); } } @@ -325,20 +310,14 @@ vmm_unsupported_quirk(void) return (0); } -static int -vmm_init(void) +int +vmm_modinit(void) { int error; - vm_maxcpu = mp_ncpus; - TUNABLE_INT_FETCH("hw.vmm.maxcpu", &vm_maxcpu); - - if (vm_maxcpu > VM_MAXCPU) { - printf("vmm: vm_maxcpu clamped to %u\n", VM_MAXCPU); - vm_maxcpu = VM_MAXCPU; - } - if (vm_maxcpu == 0) - vm_maxcpu = 1; + error = vmm_unsupported_quirk(); + if (error != 0) + return (error); error = vmm_regs_init(&vmm_arch_regs, &vmm_arch_regs_masks); if (error != 0) @@ -347,67 +326,18 @@ vmm_init(void) return (vmmops_modinit(0)); } -static int -vmm_handler(module_t mod, int what, void *arg) +int +vmm_modcleanup(void) { - int error; - - switch (what) { - case MOD_LOAD: - error = vmm_unsupported_quirk(); - if (error != 0) - break; - error = vmmdev_init(); - if (error != 0) - break; - error = vmm_init(); - if (error == 0) - vmm_initialized = true; - else - (void)vmmdev_cleanup(); - break; - case MOD_UNLOAD: - error = vmmdev_cleanup(); - if (error == 0 && vmm_initialized) { - error = vmmops_modcleanup(); - if (error) { - /* - * Something bad happened - prevent new - * VMs from being created - */ - vmm_initialized = false; - } - } - break; - default: - error = 0; - break; - } - return (error); + return (vmmops_modcleanup()); } -static moduledata_t vmm_kmod = { - "vmm", - vmm_handler, - NULL -}; - -/* - * vmm initialization has the following dependencies: - * - * - HYP initialization requires smp_rendezvous() and therefore must happen - * after SMP is fully functional (after SI_SUB_SMP). - * - vmm device initialization requires an initialized devfs. - */ -DECLARE_MODULE(vmm, vmm_kmod, MAX(SI_SUB_SMP, SI_SUB_DEVFS) + 1, SI_ORDER_ANY); -MODULE_VERSION(vmm, 1); - static void vm_init(struct vm *vm, bool create) { int i; - vm->cookie = vmmops_init(vm, vmspace_pmap(vm->vmspace)); + vm->cookie = vmmops_init(vm, vmspace_pmap(vm_vmspace(vm))); MPASS(vm->cookie != NULL); CPU_ZERO(&vm->active_cpus); @@ -443,10 +373,6 @@ vm_alloc_vcpu(struct vm *vm, int vcpuid) if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(vm)) return (NULL); - /* Some interrupt controllers may have a CPU limit */ - if (vcpuid >= vgic_max_cpu_count(vm->cookie)) - return (NULL); - vcpu = (struct vcpu *) atomic_load_acq_ptr((uintptr_t *)&vm->vcpu[vcpuid]); if (__predict_true(vcpu != NULL)) @@ -455,6 +381,12 @@ vm_alloc_vcpu(struct vm *vm, int vcpuid) sx_xlock(&vm->vcpus_init_lock); vcpu = vm->vcpu[vcpuid]; if (vcpu == NULL && !vm->dying) { + /* Some interrupt controllers may have a CPU limit */ + if (vcpuid >= vgic_max_cpu_count(vm->cookie)) { + sx_xunlock(&vm->vcpus_init_lock); + return (NULL); + } + vcpu = vcpu_alloc(vm, vcpuid); vcpu_init(vcpu); @@ -470,9 +402,9 @@ vm_alloc_vcpu(struct vm *vm, int vcpuid) } void -vm_slock_vcpus(struct vm *vm) +vm_lock_vcpus(struct vm *vm) { - sx_slock(&vm->vcpus_init_lock); + sx_xlock(&vm->vcpus_init_lock); } void @@ -485,26 +417,15 @@ int vm_create(const char *name, struct vm **retvm) { struct vm *vm; - struct vmspace *vmspace; - - /* - * If vmm.ko could not be successfully initialized then don't attempt - * to create the virtual machine. - */ - if (!vmm_initialized) - return (ENXIO); - - if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) - return (EINVAL); - - vmspace = vmmops_vmspace_alloc(0, 1ul << 39); - if (vmspace == NULL) - return (ENOMEM); + int error; vm = malloc(sizeof(struct vm), M_VMM, M_WAITOK | M_ZERO); + error = vm_mem_init(&vm->mem, 0, 1ul << 39); + if (error != 0) { + free(vm, M_VMM); + return (error); + } strcpy(vm->name, name); - vm->vmspace = vmspace; - vm_mem_init(&vm->mem); sx_init(&vm->vcpus_init_lock, "vm vcpus"); vm->sockets = 1; @@ -558,7 +479,7 @@ vm_cleanup(struct vm *vm, bool destroy) if (destroy) { vm_xlock_memsegs(vm); - pmap = vmspace_pmap(vm->vmspace); + pmap = vmspace_pmap(vm_vmspace(vm)); sched_pin(); PCPU_SET(curvmpmap, NULL); sched_unpin(); @@ -582,11 +503,6 @@ vm_cleanup(struct vm *vm, bool destroy) if (destroy) { vm_mem_destroy(vm); - vmmops_vmspace_free(vm->vmspace); - vm->vmspace = NULL; - - for (i = 0; i < vm->maxcpus; i++) - free(vm->vcpu[i], M_VMM); free(vm->vcpu, M_VMM); sx_destroy(&vm->vcpus_init_lock); } @@ -651,6 +567,33 @@ vmm_reg_wi(struct vcpu *vcpu, uint64_t wval, void *arg) return (0); } +static int +vmm_write_oslar_el1(struct vcpu *vcpu, uint64_t wval, void *arg) +{ + struct hypctx *hypctx; + + hypctx = vcpu_get_cookie(vcpu); + /* All other fields are RES0 & we don't do anything with this */ + /* TODO: Disable access to other debug state when locked */ + hypctx->dbg_oslock = (wval & OSLAR_OSLK) == OSLAR_OSLK; + return (0); +} + +static int +vmm_read_oslsr_el1(struct vcpu *vcpu, uint64_t *rval, void *arg) +{ + struct hypctx *hypctx; + uint64_t val; + + hypctx = vcpu_get_cookie(vcpu); + val = OSLSR_OSLM_1; + if (hypctx->dbg_oslock) + val |= OSLSR_OSLK; + *rval = val; + + return (0); +} + static const struct vmm_special_reg vmm_special_regs[] = { #define SPECIAL_REG(_reg, _read, _write) \ { \ @@ -707,6 +650,13 @@ static const struct vmm_special_reg vmm_special_regs[] = { SPECIAL_REG(CNTP_TVAL_EL0, vtimer_phys_tval_read, vtimer_phys_tval_write), SPECIAL_REG(CNTPCT_EL0, vtimer_phys_cnt_read, vtimer_phys_cnt_write), + + /* Debug registers */ + SPECIAL_REG(DBGPRCR_EL1, vmm_reg_raz, vmm_reg_wi), + SPECIAL_REG(OSDLR_EL1, vmm_reg_raz, vmm_reg_wi), + /* TODO: Exceptions on invalid access */ + SPECIAL_REG(OSLAR_EL1, vmm_reg_raz, vmm_write_oslar_el1), + SPECIAL_REG(OSLSR_EL1, vmm_read_oslsr_el1, vmm_reg_wi), #undef SPECIAL_REG }; @@ -1056,12 +1006,6 @@ vcpu_notify_event(struct vcpu *vcpu) vcpu_unlock(vcpu); } -struct vmspace * -vm_vmspace(struct vm *vm) -{ - return (vm->vmspace); -} - struct vm_mem * vm_mem(struct vm *vm) { @@ -1258,8 +1202,7 @@ vcpu_get_state(struct vcpu *vcpu, int *hostcpu) int vm_get_register(struct vcpu *vcpu, int reg, uint64_t *retval) { - - if (reg >= VM_REG_LAST) + if (reg < 0 || reg >= VM_REG_LAST) return (EINVAL); return (vmmops_getreg(vcpu->cookie, reg, retval)); @@ -1270,7 +1213,7 @@ vm_set_register(struct vcpu *vcpu, int reg, uint64_t val) { int error; - if (reg >= VM_REG_LAST) + if (reg < 0 || reg >= VM_REG_LAST) return (EINVAL); error = vmmops_setreg(vcpu->cookie, reg, val); if (error || reg != VM_REG_GUEST_PC) @@ -1342,8 +1285,14 @@ vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu) static int vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu) { + struct vm *vm; + + vm = vcpu->vm; vcpu_lock(vcpu); while (1) { + if (vm->suspend) + break; + if (vgic_has_pending_irq(vcpu->cookie)) break; @@ -1376,7 +1325,7 @@ vm_handle_paging(struct vcpu *vcpu, bool *retu) vme = &vcpu->exitinfo; - pmap = vmspace_pmap(vcpu->vm->vmspace); + pmap = vmspace_pmap(vm_vmspace(vcpu->vm)); addr = vme->u.paging.gpa; esr = vme->u.paging.esr; @@ -1393,7 +1342,7 @@ vm_handle_paging(struct vcpu *vcpu, bool *retu) panic("%s: Invalid exception (esr = %lx)", __func__, esr); } - map = &vm->vmspace->vm_map; + map = &vm_vmspace(vm)->vm_map; rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL, NULL); if (rv != KERN_SUCCESS) return (EFAULT); @@ -1467,7 +1416,7 @@ vm_run(struct vcpu *vcpu) if (CPU_ISSET(vcpuid, &vm->suspended_cpus)) return (EINVAL); - pmap = vmspace_pmap(vm->vmspace); + pmap = vmspace_pmap(vm_vmspace(vm)); vme = &vcpu->exitinfo; evinfo.rptr = NULL; evinfo.sptr = &vm->suspend; diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c index e293c99a6646..006239431f29 100644 --- a/sys/arm64/vmm/vmm_arm64.c +++ b/sys/arm64/vmm/vmm_arm64.c @@ -47,7 +47,6 @@ #include <vm/vm_page.h> #include <vm/vm_param.h> -#include <machine/armreg.h> #include <machine/vm.h> #include <machine/cpufunc.h> #include <machine/cpu.h> @@ -238,7 +237,6 @@ vmmops_modinit(int ipinum) vm_offset_t next_hyp_va; vm_paddr_t vmm_base; uint64_t id_aa64mmfr0_el1, pa_range_bits, pa_range_field; - uint64_t cnthctl_el2; int cpu, i; bool rv __diagused; @@ -444,10 +442,9 @@ vmmops_modinit(int ipinum) vmem_add(el2_mem_alloc, next_hyp_va, HYP_VM_MAX_ADDRESS - next_hyp_va, M_WAITOK); } - cnthctl_el2 = vmm_read_reg(HYP_REG_CNTHCTL); vgic_init(); - vtimer_init(cnthctl_el2); + vtimer_init(); return (0); } @@ -517,6 +514,7 @@ vmmops_init(struct vm *vm, pmap_t pmap) { struct hyp *hyp; vm_size_t size; + uint64_t idreg; size = el2_hyp_size(vm); hyp = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO); @@ -524,6 +522,16 @@ vmmops_init(struct vm *vm, pmap_t pmap) hyp->vm = vm; hyp->vgic_attached = false; + if (get_kernel_reg(ID_AA64MMFR0_EL1, &idreg)) { + if (ID_AA64MMFR0_ECV_VAL(idreg) >= ID_AA64MMFR0_ECV_POFF) + hyp->feats |= HYP_FEAT_ECV_POFF; + } + + if (get_kernel_reg(ID_AA64MMFR1_EL1, &idreg)) { + if (ID_AA64MMFR1_HCX_VAL(idreg) >= ID_AA64MMFR1_HCX_IMPL) + hyp->feats |= HYP_FEAT_HCX; + } + vtimer_vminit(hyp); vgic_vminit(hyp); diff --git a/sys/arm64/vmm/vmm_dev_machdep.c b/sys/arm64/vmm/vmm_dev_machdep.c index 926a74fa528b..29d14e1ba952 100644 --- a/sys/arm64/vmm/vmm_dev_machdep.c +++ b/sys/arm64/vmm/vmm_dev_machdep.c @@ -68,19 +68,13 @@ int vmmdev_machdep_ioctl(struct vm *vm, struct vcpu *vcpu, u_long cmd, caddr_t data, int fflag, struct thread *td) { - struct vm_run *vmrun; - struct vm_vgic_version *vgv; - struct vm_vgic_descr *vgic; - struct vm_irq *vi; - struct vm_exception *vmexc; - struct vm_gla2gpa *gg; - struct vm_msi *vmsi; int error; error = 0; switch (cmd) { case VM_RUN: { struct vm_exit *vme; + struct vm_run *vmrun; vmrun = (struct vm_run *)data; vme = vm_exitinfo(vcpu); @@ -94,41 +88,62 @@ vmmdev_machdep_ioctl(struct vm *vm, struct vcpu *vcpu, u_long cmd, caddr_t data, break; break; } - case VM_INJECT_EXCEPTION: + case VM_INJECT_EXCEPTION: { + struct vm_exception *vmexc; + vmexc = (struct vm_exception *)data; error = vm_inject_exception(vcpu, vmexc->esr, vmexc->far); break; - case VM_GLA2GPA_NOFAULT: + } + case VM_GLA2GPA_NOFAULT: { + struct vm_gla2gpa *gg; + gg = (struct vm_gla2gpa *)data; error = vm_gla2gpa_nofault(vcpu, &gg->paging, gg->gla, gg->prot, &gg->gpa, &gg->fault); KASSERT(error == 0 || error == EFAULT, ("%s: vm_gla2gpa unknown error %d", __func__, error)); break; - case VM_GET_VGIC_VERSION: + } + case VM_GET_VGIC_VERSION: { + struct vm_vgic_version *vgv; + vgv = (struct vm_vgic_version *)data; /* TODO: Query the vgic driver for this */ vgv->version = 3; vgv->flags = 0; error = 0; break; - case VM_ATTACH_VGIC: + } + case VM_ATTACH_VGIC: { + struct vm_vgic_descr *vgic; + vgic = (struct vm_vgic_descr *)data; error = vm_attach_vgic(vm, vgic); break; - case VM_RAISE_MSI: + } + case VM_RAISE_MSI: { + struct vm_msi *vmsi; + vmsi = (struct vm_msi *)data; error = vm_raise_msi(vm, vmsi->msg, vmsi->addr, vmsi->bus, vmsi->slot, vmsi->func); break; - case VM_ASSERT_IRQ: + } + case VM_ASSERT_IRQ: { + struct vm_irq *vi; + vi = (struct vm_irq *)data; error = vm_assert_irq(vm, vi->irq); break; - case VM_DEASSERT_IRQ: + } + case VM_DEASSERT_IRQ: { + struct vm_irq *vi; + vi = (struct vm_irq *)data; error = vm_deassert_irq(vm, vi->irq); break; + } default: error = ENOTTY; break; diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c index d61885c15871..0ad7930e9a87 100644 --- a/sys/arm64/vmm/vmm_hyp.c +++ b/sys/arm64/vmm/vmm_hyp.c @@ -32,7 +32,6 @@ #include <sys/types.h> #include <sys/proc.h> -#include <machine/armreg.h> #include "arm64.h" #include "hyp.h" @@ -42,11 +41,11 @@ struct hypctx; uint64_t VMM_HYP_FUNC(do_call_guest)(struct hypctx *); static void -vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest) +vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest, + bool ecv_poff) { uint64_t dfr0; - /* Store the guest VFP registers */ if (guest) { /* Store the timer registers */ hypctx->vtimer_cpu.cntkctl_el1 = @@ -55,7 +54,20 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest) READ_SPECIALREG(EL0_REG(CNTV_CVAL)); hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 = READ_SPECIALREG(EL0_REG(CNTV_CTL)); + } + if (guest_or_nonvhe(guest) && ecv_poff) { + /* + * If we have ECV then the guest could modify these registers. + * If VHE is enabled then the kernel will see a different view + * of the registers, so doesn't need to handle them. + */ + hypctx->vtimer_cpu.phys_timer.cntx_cval_el0 = + READ_SPECIALREG(EL0_REG(CNTP_CVAL)); + hypctx->vtimer_cpu.phys_timer.cntx_ctl_el0 = + READ_SPECIALREG(EL0_REG(CNTP_CTL)); + } + if (guest) { /* Store the GICv3 registers */ hypctx->vgic_v3_regs.ich_eisr_el2 = READ_SPECIALREG(ich_eisr_el2); @@ -108,6 +120,8 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest) } } + hypctx->dbgclaimset_el1 = READ_SPECIALREG(dbgclaimset_el1); + dfr0 = READ_SPECIALREG(id_aa64dfr0_el1); switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) { #define STORE_DBG_BRP(x) \ @@ -167,10 +181,13 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest) hypctx->pmcr_el0 = READ_SPECIALREG(pmcr_el0); hypctx->pmccntr_el0 = READ_SPECIALREG(pmccntr_el0); hypctx->pmccfiltr_el0 = READ_SPECIALREG(pmccfiltr_el0); + hypctx->pmuserenr_el0 = READ_SPECIALREG(pmuserenr_el0); + hypctx->pmselr_el0 = READ_SPECIALREG(pmselr_el0); + hypctx->pmxevcntr_el0 = READ_SPECIALREG(pmxevcntr_el0); hypctx->pmcntenset_el0 = READ_SPECIALREG(pmcntenset_el0); hypctx->pmintenset_el1 = READ_SPECIALREG(pmintenset_el1); hypctx->pmovsset_el0 = READ_SPECIALREG(pmovsset_el0); - hypctx->pmuserenr_el0 = READ_SPECIALREG(pmuserenr_el0); + switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) { #define STORE_PMU(x) \ case (x + 1): \ @@ -259,29 +276,20 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest) hypctx->hcr_el2 = READ_SPECIALREG(hcr_el2); hypctx->vpidr_el2 = READ_SPECIALREG(vpidr_el2); hypctx->vmpidr_el2 = READ_SPECIALREG(vmpidr_el2); - -#ifndef VMM_VHE - /* hcrx_el2 depends on feat_hcx */ - uint64_t mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); - if (ID_AA64MMFR1_HCX_VAL(mmfr1) >> ID_AA64MMFR1_HCX_SHIFT) { - hypctx->hcrx_el2 = READ_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2)); - } -#endif } static void -vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest) +vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest, + bool ecv_poff) { uint64_t dfr0; /* Restore the special registers */ WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2); - if (guest_or_nonvhe(guest)) { - uint64_t mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); - if (ID_AA64MMFR1_HCX_VAL(mmfr1) >> ID_AA64MMFR1_HCX_SHIFT) { - WRITE_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2), hypctx->hcrx_el2); - } + if (guest) { + if ((hyp->feats & HYP_FEAT_HCX) != 0) + WRITE_SPECIALREG(HCRX_EL2_REG, hypctx->hcrx_el2); } isb(); @@ -333,12 +341,15 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest) WRITE_SPECIALREG(pmcr_el0, hypctx->pmcr_el0); WRITE_SPECIALREG(pmccntr_el0, hypctx->pmccntr_el0); WRITE_SPECIALREG(pmccfiltr_el0, hypctx->pmccfiltr_el0); + WRITE_SPECIALREG(pmuserenr_el0, hypctx->pmuserenr_el0); + WRITE_SPECIALREG(pmselr_el0, hypctx->pmselr_el0); + WRITE_SPECIALREG(pmxevcntr_el0, hypctx->pmxevcntr_el0); /* Clear all events/interrupts then enable them */ - WRITE_SPECIALREG(pmcntenclr_el0, 0xfffffffful); + WRITE_SPECIALREG(pmcntenclr_el0, ~0ul); WRITE_SPECIALREG(pmcntenset_el0, hypctx->pmcntenset_el0); - WRITE_SPECIALREG(pmintenclr_el1, 0xfffffffful); + WRITE_SPECIALREG(pmintenclr_el1, ~0ul); WRITE_SPECIALREG(pmintenset_el1, hypctx->pmintenset_el1); - WRITE_SPECIALREG(pmovsclr_el0, 0xfffffffful); + WRITE_SPECIALREG(pmovsclr_el0, ~0ul); WRITE_SPECIALREG(pmovsset_el0, hypctx->pmovsset_el0); switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) { @@ -384,6 +395,9 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest) #undef LOAD_PMU } + WRITE_SPECIALREG(dbgclaimclr_el1, ~0ul); + WRITE_SPECIALREG(dbgclaimclr_el1, hypctx->dbgclaimset_el1); + dfr0 = READ_SPECIALREG(id_aa64dfr0_el1); switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) { #define LOAD_DBG_BRP(x) \ @@ -450,6 +464,29 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest) WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2); WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2); + if (ecv_poff) { + /* + * Load the same offset as the virtual timer + * to keep in sync. + */ + WRITE_SPECIALREG(CNTPOFF_EL2_REG, + hyp->vtimer.cntvoff_el2); + isb(); + } + } + if (guest_or_nonvhe(guest) && ecv_poff) { + /* + * If we have ECV then the guest could modify these registers. + * If VHE is enabled then the kernel will see a different view + * of the registers, so doesn't need to handle them. + */ + WRITE_SPECIALREG(EL0_REG(CNTP_CVAL), + hypctx->vtimer_cpu.phys_timer.cntx_cval_el0); + WRITE_SPECIALREG(EL0_REG(CNTP_CTL), + hypctx->vtimer_cpu.phys_timer.cntx_ctl_el0); + } + + if (guest) { /* Load the GICv3 registers */ WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2); WRITE_SPECIALREG(ich_vmcr_el2, @@ -502,11 +539,19 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx) struct hypctx host_hypctx; uint64_t cntvoff_el2; uint64_t ich_hcr_el2, ich_vmcr_el2, cnthctl_el2, cntkctl_el1; +#ifndef VMM_VHE + uint64_t hcrx_el2; +#endif uint64_t ret; uint64_t s1e1r, hpfar_el2; - bool hpfar_valid; + bool ecv_poff, hpfar_valid; - vmm_hyp_reg_store(&host_hypctx, NULL, false); + ecv_poff = (hyp->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0; + vmm_hyp_reg_store(&host_hypctx, NULL, false, ecv_poff); +#ifndef VMM_VHE + if ((hyp->feats & HYP_FEAT_HCX) != 0) + hcrx_el2 = READ_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2)); +#endif /* Save the host special registers */ cnthctl_el2 = READ_SPECIALREG(cnthctl_el2); @@ -516,7 +561,7 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx) ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2); ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2); - vmm_hyp_reg_restore(hypctx, hyp, true); + vmm_hyp_reg_restore(hypctx, hyp, true, ecv_poff); /* Load the common hypervisor registers */ WRITE_SPECIALREG(vttbr_el2, hyp->vttbr_el2); @@ -532,7 +577,7 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx) /* Store the exit info */ hypctx->exit_info.far_el2 = READ_SPECIALREG(far_el2); - vmm_hyp_reg_store(hypctx, hyp, true); + vmm_hyp_reg_store(hypctx, hyp, true, ecv_poff); hpfar_valid = true; if (ret == EXCP_TYPE_EL1_SYNC) { @@ -582,7 +627,12 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx) } } - vmm_hyp_reg_restore(&host_hypctx, NULL, false); + vmm_hyp_reg_restore(&host_hypctx, NULL, false, ecv_poff); + +#ifndef VMM_VHE + if ((hyp->feats & HYP_FEAT_HCX) != 0) + WRITE_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2), hcrx_el2); +#endif /* Restore the host special registers */ WRITE_SPECIALREG(ich_hcr_el2, ich_hcr_el2); @@ -613,8 +663,6 @@ VMM_HYP_FUNC(read_reg)(uint64_t reg) switch (reg) { case HYP_REG_ICH_VTR: return (READ_SPECIALREG(ich_vtr_el2)); - case HYP_REG_CNTHCTL: - return (READ_SPECIALREG(cnthctl_el2)); } return (0); diff --git a/sys/arm64/vmm/vmm_reset.c b/sys/arm64/vmm/vmm_reset.c index 79d022cf33e8..0e4910ea87b4 100644 --- a/sys/arm64/vmm/vmm_reset.c +++ b/sys/arm64/vmm/vmm_reset.c @@ -31,7 +31,6 @@ #include <sys/kernel.h> #include <sys/lock.h> -#include <machine/armreg.h> #include <machine/cpu.h> #include <machine/hypervisor.h> @@ -100,10 +99,12 @@ reset_vm_el01_regs(void *vcpu) el2ctx->pmcr_el0 |= PMCR_LC; set_arch_unknown(el2ctx->pmccntr_el0); set_arch_unknown(el2ctx->pmccfiltr_el0); + set_arch_unknown(el2ctx->pmuserenr_el0); + set_arch_unknown(el2ctx->pmselr_el0); + set_arch_unknown(el2ctx->pmxevcntr_el0); set_arch_unknown(el2ctx->pmcntenset_el0); set_arch_unknown(el2ctx->pmintenset_el1); set_arch_unknown(el2ctx->pmovsset_el0); - set_arch_unknown(el2ctx->pmuserenr_el0); memset(el2ctx->pmevcntr_el0, 0, sizeof(el2ctx->pmevcntr_el0)); memset(el2ctx->pmevtyper_el0, 0, sizeof(el2ctx->pmevtyper_el0)); } @@ -143,7 +144,8 @@ reset_vm_el2_regs(void *vcpu) /* Set the Extended Hypervisor Configuration Register */ el2ctx->hcrx_el2 = 0; /* TODO: Trap all extensions we don't support */ - el2ctx->mdcr_el2 = 0; + el2ctx->mdcr_el2 = MDCR_EL2_TDOSA | MDCR_EL2_TDRA | MDCR_EL2_TPMS | + MDCR_EL2_TTRF; /* PMCR_EL0.N is read from MDCR_EL2.HPMN */ el2ctx->mdcr_el2 |= (el2ctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT; |