diff options
Diffstat (limited to 'sys')
508 files changed, 16561 insertions, 8141 deletions
diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S index d7e954f573b0..17ff8005e3cd 100644 --- a/sys/amd64/amd64/cpu_switch.S +++ b/sys/amd64/amd64/cpu_switch.S @@ -35,7 +35,6 @@ #include <machine/specialreg.h> #include "assym.inc" -#include "opt_sched.h" /*****************************************************************************/ /* Scheduling */ @@ -136,13 +135,11 @@ ctx_switch_fpusave_done: movq %r15,TD_LOCK(%r13) /* Release the old thread */ sw1: leaq TD_MD_PCB(%r12),%r8 -#if defined(SCHED_ULE) movq $blocked_lock, %rdx movq TD_LOCK(%r12),%rcx cmpq %rcx, %rdx je sw1wait sw1cont: -#endif /* * At this point, we've switched address spaces and are ready * to load up the rest of the next context. @@ -492,7 +489,6 @@ ENTRY(resumectx) END(resumectx) /* Wait for the new thread to become unblocked */ -#if defined(SCHED_ULE) sw1wait: 1: pause @@ -500,4 +496,3 @@ sw1wait: cmpq %rcx, %rdx je 1b jmp sw1cont -#endif diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index cae58181000f..ae5df475f046 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -322,7 +322,6 @@ late_ifunc_resolve(void *dummy __unused) } SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL); - void cpu_setregs(void) { @@ -828,15 +827,6 @@ native_parse_memmap(vm_paddr_t *physmap, int *physmap_idx) { struct bios_smap *smap; struct efi_map_header *efihdr; - u_int32_t size; - - /* - * Memory map from INT 15:E820. - * - * subr_module.c says: - * "Consumer may safely assume that size value precedes data." - * ie: an int32_t immediately precedes smap. - */ efihdr = (struct efi_map_header *)preload_search_info(preload_kmdp, MODINFO_METADATA | MODINFOMD_EFI_MAP); @@ -849,7 +839,15 @@ native_parse_memmap(vm_paddr_t *physmap, int *physmap_idx) add_efi_map_entries(efihdr, physmap, physmap_idx); strlcpy(bootmethod, "UEFI", sizeof(bootmethod)); } else { - size = *((u_int32_t *)smap - 1); + /* + * Memory map from INT 15:E820. + * + * subr_module.c says: + * "Consumer may safely assume that size value precedes data." + * ie: an int32_t immediately precedes smap. + */ + u_int32_t size = *((u_int32_t *)smap - 1); + bios_add_smap_entries(smap, size, physmap, physmap_idx); strlcpy(bootmethod, "BIOS", sizeof(bootmethod)); } @@ -1353,6 +1351,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); } + sched_instance_select(); + link_elf_ireloc(); /* diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index c763ff58680e..2e180003e93d 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -93,8 +93,7 @@ set_top_of_stack_td(struct thread *td) struct savefpu * get_pcb_user_save_td(struct thread *td) { - KASSERT(((vm_offset_t)td->td_md.md_usr_fpu_save % - XSAVE_AREA_ALIGN) == 0, + KASSERT(__is_aligned(td->td_md.md_usr_fpu_save, XSAVE_AREA_ALIGN), ("Unaligned pcb_user_save area ptr %p td %p", td->td_md.md_usr_fpu_save, td)); return (td->td_md.md_usr_fpu_save); diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index 1003572805fb..c390ff7596b6 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -24,6 +24,7 @@ makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support options SCHED_ULE # ULE scheduler +options SCHED_4BSD # Original 4.xBSD scheduler options NUMA # Non-Uniform Memory Architecture support options PREEMPTION # Enable kernel thread preemption options EXTERR_STRINGS diff --git a/sys/amd64/include/ifunc.h b/sys/amd64/include/ifunc.h new file mode 100644 index 000000000000..1af46757b836 --- /dev/null +++ b/sys/amd64/include/ifunc.h @@ -0,0 +1,5 @@ +/* + * This file is in the public domain. + */ + +#include <x86/ifunc.h> diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h index b6d8c469cdf6..19eab48168f7 100644 --- a/sys/amd64/include/md_var.h +++ b/sys/amd64/include/md_var.h @@ -58,6 +58,7 @@ extern vm_paddr_t KERNend; extern bool efi_boot; struct __mcontext; +struct pcpu; struct savefpu; struct sysentvec; diff --git a/sys/amd64/vmm/vmm_host.h b/sys/amd64/vmm/vmm_host.h index eebb794843b6..adca53271448 100644 --- a/sys/amd64/vmm/vmm_host.h +++ b/sys/amd64/vmm/vmm_host.h @@ -29,6 +29,8 @@ #ifndef _VMM_HOST_H_ #define _VMM_HOST_H_ +#include <sys/pcpu.h> + #ifndef _KERNEL #error "no user-serviceable parts inside" #endif diff --git a/sys/arm/arm/elf_machdep.c b/sys/arm/arm/elf_machdep.c index 881c4fcff475..cf2add0c367c 100644 --- a/sys/arm/arm/elf_machdep.c +++ b/sys/arm/arm/elf_machdep.c @@ -150,7 +150,7 @@ bool elf_is_ifunc_reloc(Elf_Size r_info __unused) { - return (false); + return (ELF_R_TYPE(r_info) == R_ARM_IRELATIVE); } /* @@ -253,6 +253,12 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, case R_ARM_RELATIVE: break; + case R_ARM_IRELATIVE: + addr = relocbase + addend; + addr = ((Elf_Addr (*)(void))addr)(); + if (*where != addr) + *where = addr; + break; default: printf("kldload: unexpected relocation type %d, " "symbol index %d\n", rtype, symidx); diff --git a/sys/arm/arm/gic.c b/sys/arm/arm/gic.c index e33bda4886b9..64f19c848ea2 100644 --- a/sys/arm/arm/gic.c +++ b/sys/arm/arm/gic.c @@ -151,6 +151,8 @@ static struct arm_gic_softc *gic_sc = NULL; /* CPU Interface */ #define gic_c_read_4(_sc, _reg) \ bus_read_4((_sc)->gic_res[GIC_RES_CPU], (_reg)) +#define gic_c_peek_4(_sc, _reg, _val) \ + bus_peek_4((_sc)->gic_res[GIC_RES_CPU], (_reg), (_val)) #define gic_c_write_4(_sc, _reg, _val) \ bus_write_4((_sc)->gic_res[GIC_RES_CPU], (_reg), (_val)) /* Distributor Interface */ @@ -347,7 +349,18 @@ arm_gic_attach(device_t dev) goto cleanup; } - icciidr = gic_c_read_4(sc, GICC_IIDR); + /* + * Try accessing a CPU interface register. On some broken + * virtualization environments this will raise an external + * data abort. When this happens we can detect it using + * by peeking at the register & checking for the fault. + * As there is no way to continue with a normal boot we + * panic. + */ + if (gic_c_peek_4(sc, GICC_IIDR, &icciidr) != 0) + panic("Unable to access %s CPU registers, " + "broken hardware or hypervisor configuration", + device_get_nameunit(dev)); device_printf(dev, "pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", GICD_IIDR_PROD(icciidr), GICD_IIDR_VAR(icciidr), diff --git a/sys/arm/arm/machdep.c b/sys/arm/arm/machdep.c index cfc0b32f5102..dc2205023820 100644 --- a/sys/arm/arm/machdep.c +++ b/sys/arm/arm/machdep.c @@ -374,7 +374,7 @@ pcpu0_init(void) /* * Initialize proc0 */ -void +static void init_proc0(vm_offset_t kstack) { proc_linkup0(&proc0, &thread0); @@ -447,6 +447,8 @@ initarm(struct arm_boot_params *abp) set_cpufuncs(); cpuinfo_init(); + sched_instance_select(); + link_elf_ireloc(); /* * Find the dtb passed in by the boot loader. diff --git a/sys/arm/arm/swtch-v6.S b/sys/arm/arm/swtch-v6.S index 97d863b6d4de..98c8e5c41ec5 100644 --- a/sys/arm/arm/swtch-v6.S +++ b/sys/arm/arm/swtch-v6.S @@ -79,7 +79,6 @@ */ #include "assym.inc" -#include "opt_sched.h" #include <machine/asm.h> #include <machine/asmacros.h> @@ -432,11 +431,7 @@ sw1: * r11 = newtd */ -#if defined(SMP) && defined(SCHED_ULE) - /* - * 386 and amd64 do the blocked lock test only for SMP and SCHED_ULE - * QQQ: What does it mean in reality and why is it done? - */ +#if defined(SMP) ldr r6, =blocked_lock 1: ldr r3, [r11, #TD_LOCK] /* atomic write regular read */ diff --git a/sys/arm/include/ifunc.h b/sys/arm/include/ifunc.h new file mode 100644 index 000000000000..98cc354ae6ca --- /dev/null +++ b/sys/arm/include/ifunc.h @@ -0,0 +1,28 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025, Michal Meloun <mmel@freebsd.org> + * + */ + +#ifndef __ARM_IFUNC_H +#define __ARM_IFUNC_H + +#define DEFINE_IFUNC(qual, ret_type, name, args) \ + static ret_type (*name##_resolver(void))args __used; \ + qual ret_type name args __attribute__((ifunc(#name "_resolver"))); \ + static ret_type (*name##_resolver(void))args + +#ifdef __not_yet__ +#define DEFINE_UIFUNC(qual, ret_type, name, args) \ + static ret_type (*name##_resolver(uint32_t, uint32_t, uint32_t, \ + uint32_t))args __used; \ + qual ret_type name args __attribute__((ifunc(#name "_resolver"))); \ + static ret_type (*name##_resolver( \ + uint32_t elf_hwcap __unused, \ + uint32_t elf_hwcap2 __unused, \ + uint32_t arg3 __unused, \ + uint32_t arg4 __unused))args +#endif + +#endif diff --git a/sys/arm/include/machdep.h b/sys/arm/include/machdep.h index 45e44a65368b..ada2dfa502aa 100644 --- a/sys/arm/include/machdep.h +++ b/sys/arm/include/machdep.h @@ -15,7 +15,6 @@ extern vm_offset_t abtstack; /* misc prototypes used by the many arm machdeps */ struct trapframe; -void init_proc0(vm_offset_t kstack); void halt(void); void abort_handler(struct trapframe *, int ); void set_stackptrs(int cpu); diff --git a/sys/arm64/arm64/gic_v3_fdt.c b/sys/arm64/arm64/gic_v3_fdt.c index 871516145168..a71114881906 100644 --- a/sys/arm64/arm64/gic_v3_fdt.c +++ b/sys/arm64/arm64/gic_v3_fdt.c @@ -86,7 +86,7 @@ EARLY_DRIVER_MODULE(gic_v3, ofwbus, gic_v3_fdt_driver, 0, 0, /* * Helper functions declarations. */ -static int gic_v3_ofw_bus_attach(device_t); +static void gic_v3_ofw_bus_attach(device_t); /* * Device interface. @@ -193,12 +193,7 @@ gic_v3_fdt_attach(device_t dev) * GIC will act as a bus in that case. * Failure here will not affect main GIC functionality. */ - if (gic_v3_ofw_bus_attach(dev) != 0) { - if (bootverbose) { - device_printf(dev, - "Failed to attach ITS to this GIC\n"); - } - } + gic_v3_ofw_bus_attach(dev); if (device_get_children(dev, &sc->gic_children, &sc->gic_nchildren) != 0) sc->gic_nchildren = 0; @@ -298,7 +293,7 @@ gic_v3_ofw_fill_ranges(phandle_t parent, struct gic_v3_softc *sc, * Collects and configures device informations and finally * adds ITS device as a child of GICv3 in Newbus hierarchy. */ -static int +static void gic_v3_ofw_bus_attach(device_t dev) { struct gic_v3_ofw_devinfo *di; @@ -313,8 +308,13 @@ gic_v3_ofw_bus_attach(device_t dev) if (parent > 0) { rv = gic_v3_ofw_fill_ranges(parent, sc, &addr_cells, &size_cells); - if (rv != 0) - return (rv); + if (rv != 0) { + if (bootverbose) { + device_printf(dev, + "Failed to attach ITS to this GIC\n"); + } + goto vgic; + } /* Iterate through all GIC subordinates */ for (node = OF_child(parent); node > 0; node = OF_peer(node)) { @@ -369,28 +369,31 @@ gic_v3_ofw_bus_attach(device_t dev) sc->gic_nchildren++; device_set_ivars(child, di); } - } - /* - * If there is a vgic maintanance interrupt add a virtual gic - * child so we can use this in the vmm module for bhyve. - */ - if (OF_hasprop(parent, "interrupts")) { - child = device_add_child(dev, "vgic", DEVICE_UNIT_ANY); - if (child == NULL) { - device_printf(dev, "Could not add vgic child\n"); - } else { - di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); - resource_list_init(&di->di_rl); - di->di_gic_dinfo.gic_domain = -1; - di->di_gic_dinfo.is_vgic = 1; - device_set_ivars(child, di); - sc->gic_nchildren++; +vgic: + + /* + * If there is a vgic maintanance interrupt add a virtual gic + * child so we can use this in the vmm module for bhyve. + */ + if (OF_hasprop(parent, "interrupts")) { + child = device_add_child(dev, "vgic", DEVICE_UNIT_ANY); + if (child == NULL) { + device_printf(dev, + "Could not add vgic child\n"); + } else { + di = malloc(sizeof(*di), M_GIC_V3, + M_WAITOK | M_ZERO); + resource_list_init(&di->di_rl); + di->di_gic_dinfo.gic_domain = -1; + di->di_gic_dinfo.is_vgic = 1; + device_set_ivars(child, di); + sc->gic_nchildren++; + } } } bus_attach_children(dev); - return (0); } static struct resource_list * diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c index 6790f47a0f82..b1e22c900f3f 100644 --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -825,6 +825,7 @@ initarm(struct arm64_bootparams *abp) PCPU_SET(curthread, &thread0); PCPU_SET(midr, get_midr()); + sched_instance_select(); link_elf_ireloc(); #ifdef FDT try_load_dtb(); diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S index a461fded929c..b3bf88135e57 100644 --- a/sys/arm64/arm64/swtch.S +++ b/sys/arm64/arm64/swtch.S @@ -31,7 +31,6 @@ #include "assym.inc" #include "opt_kstack_pages.h" -#include "opt_sched.h" #include <sys/elf_common.h> @@ -197,7 +196,7 @@ ENTRY(cpu_switch) * Release the old thread. */ stlr x2, [x0, #TD_LOCK] -#if defined(SCHED_ULE) && defined(SMP) +#if defined(SMP) /* Spin if TD_LOCK points to a blocked_lock */ ldr x2, =_C_LABEL(blocked_lock) 1: diff --git a/sys/arm64/arm64/vfp.c b/sys/arm64/arm64/vfp.c index bcddebfaf66e..64f13458e2d9 100644 --- a/sys/arm64/arm64/vfp.c +++ b/sys/arm64/arm64/vfp.c @@ -934,6 +934,9 @@ get_arm64_sve(struct regset *rs, struct thread *td, void *buf, pcb = td->td_pcb; + if (td == curthread && (pcb->pcb_fpflags & PCB_FP_STARTED) != 0) + vfp_save_state(td, pcb); + /* If there is no SVE support in HW then we don't support NT_ARM_SVE */ if (pcb->pcb_sve_len == 0) return (false); @@ -955,9 +958,6 @@ get_arm64_sve(struct regset *rs, struct thread *td, void *buf, KASSERT(*sizep == sizeof(struct svereg_header) + buf_size, ("%s: invalid size", __func__)); - if (td == curthread && (pcb->pcb_fpflags & PCB_FP_STARTED) != 0) - vfp_save_state(td, pcb); - header = buf; memset(header, 0, sizeof(*header)); diff --git a/sys/arm64/include/hypervisor.h b/sys/arm64/include/hypervisor.h index f3d7027269c9..3ee5c12f2265 100644 --- a/sys/arm64/include/hypervisor.h +++ b/sys/arm64/include/hypervisor.h @@ -134,6 +134,15 @@ /* Unconditionally valid */ #define CPTR_TCPAC 0x80000000 +/* HAFGRTR_EL2 */ +#define HAFGRTR_EL2_REG MRS_REG_ALT_NAME(HAFGRTR_EL2) +#define HAFGRTR_EL2_op0 3 +#define HAFGRTR_EL2_op1 4 +#define HAFGRTR_EL2_CRn 3 +#define HAFGRTR_EL2_CRm 1 +#define HAFGRTR_EL2_op2 6 +#define HAFGRTR_EL2_TRAP_ALL UL(0x0003fffffffe001f) + /* HCR_EL2 - Hypervisor Config Register */ #define HCR_VM (UL(0x1) << 0) #define HCR_SWIO (UL(0x1) << 1) @@ -234,6 +243,1741 @@ /* Bit 25 is reserved */ #define HCRX_SRMASKEn (UL(0x1) << 26) +/* HDFGRTR2_EL2 */ +#define HDFGRTR2_EL2_REG MRS_REG_ALT_NAME(HDFGRTR2_EL2) +#define HDFGRTR2_EL2_op0 3 +#define HDFGRTR2_EL2_op1 4 +#define HDFGRTR2_EL2_CRn 3 +#define HDFGRTR2_EL2_CRm 1 +#define HDFGRTR2_EL2_op2 0 +#define HDFGRTR2_EL2_nMDSTEPOP_EL1_SHIFT 23 +#define HDFGRTR2_EL2_nMDSTEPOP_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nMDSTEPOP_EL1_SHIFT) +#define HDFGRTR2_EL2_nMDSTEPOP_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nMDSTEPOP_EL1_MASK) +#define HDFGRTR2_EL2_nMDSTEPOP_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nMDSTEPOP_EL1_SHIFT) +#define HDFGRTR2_EL2_nMDSTEPOP_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nMDSTEPOP_EL1_SHIFT) +#define HDFGRTR2_EL2_nTRBMPAM_EL1_SHIFT 22 +#define HDFGRTR2_EL2_nTRBMPAM_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nTRBMPAM_EL1_SHIFT) +#define HDFGRTR2_EL2_nTRBMPAM_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nTRBMPAM_EL1_MASK) +#define HDFGRTR2_EL2_nTRBMPAM_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nTRBMPAM_EL1_SHIFT) +#define HDFGRTR2_EL2_nTRBMPAM_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nTRBMPAM_EL1_SHIFT) +#define HDFGRTR2_EL2_nTRCITECR_EL1_SHIFT 20 +#define HDFGRTR2_EL2_nTRCITECR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nTRCITECR_EL1_SHIFT) +#define HDFGRTR2_EL2_nTRCITECR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nTRCITECR_EL1_MASK) +#define HDFGRTR2_EL2_nTRCITECR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nTRCITECR_EL1_SHIFT) +#define HDFGRTR2_EL2_nTRCITECR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nTRCITECR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMSDSFR_EL1_SHIFT 19 +#define HDFGRTR2_EL2_nPMSDSFR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nPMSDSFR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMSDSFR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nPMSDSFR_EL1_MASK) +#define HDFGRTR2_EL2_nPMSDSFR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nPMSDSFR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMSDSFR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nPMSDSFR_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMDEVAFF_EL1_SHIFT 18 +#define HDFGRTR2_EL2_nSPMDEVAFF_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMDEVAFF_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMDEVAFF_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nSPMDEVAFF_EL1_MASK) +#define HDFGRTR2_EL2_nSPMDEVAFF_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMDEVAFF_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMDEVAFF_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMDEVAFF_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMID_SHIFT 17 +#define HDFGRTR2_EL2_nSPMID_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMID_SHIFT) +#define HDFGRTR2_EL2_nSPMID_VAL(x) ((x) & HDFGRTR2_EL2_nSPMID_MASK) +#define HDFGRTR2_EL2_nSPMID_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMID_SHIFT) +#define HDFGRTR2_EL2_nSPMID_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMID_SHIFT) +#define HDFGRTR2_EL2_nSPMSCR_EL1_SHIFT 16 +#define HDFGRTR2_EL2_nSPMSCR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMSCR_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMSCR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nSPMSCR_EL1_MASK) +#define HDFGRTR2_EL2_nSPMSCR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMSCR_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMSCR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMSCR_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMACCESSR_EL1_SHIFT 15 +#define HDFGRTR2_EL2_nSPMACCESSR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMACCESSR_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMACCESSR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nSPMACCESSR_EL1_MASK) +#define HDFGRTR2_EL2_nSPMACCESSR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMACCESSR_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMACCESSR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMACCESSR_EL1_SHIFT) +#define HDFGRTR2_EL2_nSPMCR_EL0_SHIFT 14 +#define HDFGRTR2_EL2_nSPMCR_EL0_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMCR_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMCR_EL0_VAL(x) ((x) & HDFGRTR2_EL2_nSPMCR_EL0_MASK) +#define HDFGRTR2_EL2_nSPMCR_EL0_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMCR_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMCR_EL0_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMCR_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMOVS_SHIFT 13 +#define HDFGRTR2_EL2_nSPMOVS_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMOVS_SHIFT) +#define HDFGRTR2_EL2_nSPMOVS_VAL(x) ((x) & HDFGRTR2_EL2_nSPMOVS_MASK) +#define HDFGRTR2_EL2_nSPMOVS_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMOVS_SHIFT) +#define HDFGRTR2_EL2_nSPMOVS_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMOVS_SHIFT) +#define HDFGRTR2_EL2_nSPMINTEN_SHIFT 12 +#define HDFGRTR2_EL2_nSPMINTEN_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMINTEN_SHIFT) +#define HDFGRTR2_EL2_nSPMINTEN_VAL(x) ((x) & HDFGRTR2_EL2_nSPMINTEN_MASK) +#define HDFGRTR2_EL2_nSPMINTEN_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMINTEN_SHIFT) +#define HDFGRTR2_EL2_nSPMINTEN_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMINTEN_SHIFT) +#define HDFGRTR2_EL2_nSPMCNTEN_SHIFT 11 +#define HDFGRTR2_EL2_nSPMCNTEN_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMCNTEN_SHIFT) +#define HDFGRTR2_EL2_nSPMCNTEN_VAL(x) ((x) & HDFGRTR2_EL2_nSPMCNTEN_MASK) +#define HDFGRTR2_EL2_nSPMCNTEN_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMCNTEN_SHIFT) +#define HDFGRTR2_EL2_nSPMCNTEN_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMCNTEN_SHIFT) +#define HDFGRTR2_EL2_nSPMSELR_EL0_SHIFT 10 +#define HDFGRTR2_EL2_nSPMSELR_EL0_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMSELR_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMSELR_EL0_VAL(x) ((x) & HDFGRTR2_EL2_nSPMSELR_EL0_MASK) +#define HDFGRTR2_EL2_nSPMSELR_EL0_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMSELR_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMSELR_EL0_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMSELR_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMEVTYPERn_EL0_SHIFT 9 +#define HDFGRTR2_EL2_nSPMEVTYPERn_EL0_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMEVTYPERn_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMEVTYPERn_EL0_VAL(x) ((x) & HDFGRTR2_EL2_nSPMEVTYPERn_EL0_MASK) +#define HDFGRTR2_EL2_nSPMEVTYPERn_EL0_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMEVTYPERn_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMEVTYPERn_EL0_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMEVTYPERn_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMEVCNTRn_EL0_SHIFT 8 +#define HDFGRTR2_EL2_nSPMEVCNTRn_EL0_MASK (UL(0x1) << HDFGRTR2_EL2_nSPMEVCNTRn_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMEVCNTRn_EL0_VAL(x) ((x) & HDFGRTR2_EL2_nSPMEVCNTRn_EL0_MASK) +#define HDFGRTR2_EL2_nSPMEVCNTRn_EL0_TRAP (UL(0x0) << HDFGRTR2_EL2_nSPMEVCNTRn_EL0_SHIFT) +#define HDFGRTR2_EL2_nSPMEVCNTRn_EL0_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nSPMEVCNTRn_EL0_SHIFT) +#define HDFGRTR2_EL2_nPMSSCR_EL1_SHIFT 7 +#define HDFGRTR2_EL2_nPMSSCR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nPMSSCR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMSSCR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nPMSSCR_EL1_MASK) +#define HDFGRTR2_EL2_nPMSSCR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nPMSSCR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMSSCR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nPMSSCR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMSSDATA_SHIFT 6 +#define HDFGRTR2_EL2_nPMSSDATA_MASK (UL(0x1) << HDFGRTR2_EL2_nPMSSDATA_SHIFT) +#define HDFGRTR2_EL2_nPMSSDATA_VAL(x) ((x) & HDFGRTR2_EL2_nPMSSDATA_MASK) +#define HDFGRTR2_EL2_nPMSSDATA_TRAP (UL(0x0) << HDFGRTR2_EL2_nPMSSDATA_SHIFT) +#define HDFGRTR2_EL2_nPMSSDATA_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nPMSSDATA_SHIFT) +#define HDFGRTR2_EL2_nMDSELR_EL1_SHIFT 5 +#define HDFGRTR2_EL2_nMDSELR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nMDSELR_EL1_SHIFT) +#define HDFGRTR2_EL2_nMDSELR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nMDSELR_EL1_MASK) +#define HDFGRTR2_EL2_nMDSELR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nMDSELR_EL1_SHIFT) +#define HDFGRTR2_EL2_nMDSELR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nMDSELR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMUACR_EL1_SHIFT 4 +#define HDFGRTR2_EL2_nPMUACR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nPMUACR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMUACR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nPMUACR_EL1_MASK) +#define HDFGRTR2_EL2_nPMUACR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nPMUACR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMUACR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nPMUACR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMICFILTR_EL0_SHIFT 3 +#define HDFGRTR2_EL2_nPMICFILTR_EL0_MASK (UL(0x1) << HDFGRTR2_EL2_nPMICFILTR_EL0_SHIFT) +#define HDFGRTR2_EL2_nPMICFILTR_EL0_VAL(x) ((x) & HDFGRTR2_EL2_nPMICFILTR_EL0_MASK) +#define HDFGRTR2_EL2_nPMICFILTR_EL0_TRAP (UL(0x0) << HDFGRTR2_EL2_nPMICFILTR_EL0_SHIFT) +#define HDFGRTR2_EL2_nPMICFILTR_EL0_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nPMICFILTR_EL0_SHIFT) +#define HDFGRTR2_EL2_nPMICNTR_EL0_SHIFT 2 +#define HDFGRTR2_EL2_nPMICNTR_EL0_MASK (UL(0x1) << HDFGRTR2_EL2_nPMICNTR_EL0_SHIFT) +#define HDFGRTR2_EL2_nPMICNTR_EL0_VAL(x) ((x) & HDFGRTR2_EL2_nPMICNTR_EL0_MASK) +#define HDFGRTR2_EL2_nPMICNTR_EL0_TRAP (UL(0x0) << HDFGRTR2_EL2_nPMICNTR_EL0_SHIFT) +#define HDFGRTR2_EL2_nPMICNTR_EL0_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nPMICNTR_EL0_SHIFT) +#define HDFGRTR2_EL2_nPMIAR_EL1_SHIFT 1 +#define HDFGRTR2_EL2_nPMIAR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nPMIAR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMIAR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nPMIAR_EL1_MASK) +#define HDFGRTR2_EL2_nPMIAR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nPMIAR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMIAR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nPMIAR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMECR_EL1_SHIFT 0 +#define HDFGRTR2_EL2_nPMECR_EL1_MASK (UL(0x1) << HDFGRTR2_EL2_nPMECR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMECR_EL1_VAL(x) ((x) & HDFGRTR2_EL2_nPMECR_EL1_MASK) +#define HDFGRTR2_EL2_nPMECR_EL1_TRAP (UL(0x0) << HDFGRTR2_EL2_nPMECR_EL1_SHIFT) +#define HDFGRTR2_EL2_nPMECR_EL1_NOTRAP (UL(0x1) << HDFGRTR2_EL2_nPMECR_EL1_SHIFT) + +/* HDFGRTR_EL2 */ +#define HDFGRTR_EL2_REG MRS_REG_ALT_NAME(HDFGRTR_EL2) +#define HDFGRTR_EL2_op0 3 +#define HDFGRTR_EL2_op1 4 +#define HDFGRTR_EL2_CRn 3 +#define HDFGRTR_EL2_CRm 1 +#define HDFGRTR_EL2_op2 4 +#define HDFGRTR_EL2_PMBIDR_EL1_SHIFT 63 +#define HDFGRTR_EL2_PMBIDR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMBIDR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBIDR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMBIDR_EL1_MASK) +#define HDFGRTR_EL2_PMBIDR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMBIDR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBIDR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMBIDR_EL1_SHIFT) +#define HDFGRTR_EL2_nPMSNEVFR_EL1_SHIFT 62 +#define HDFGRTR_EL2_nPMSNEVFR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_nPMSNEVFR_EL1_SHIFT) +#define HDFGRTR_EL2_nPMSNEVFR_EL1_VAL(x) ((x) & HDFGRTR_EL2_nPMSNEVFR_EL1_MASK) +#define HDFGRTR_EL2_nPMSNEVFR_EL1_TRAP (UL(0x0) << HDFGRTR_EL2_nPMSNEVFR_EL1_SHIFT) +#define HDFGRTR_EL2_nPMSNEVFR_EL1_NOTRAP (UL(0x1) << HDFGRTR_EL2_nPMSNEVFR_EL1_SHIFT) +#define HDFGRTR_EL2_nBRBDATA_SHIFT 61 +#define HDFGRTR_EL2_nBRBDATA_MASK (UL(0x1) << HDFGRTR_EL2_nBRBDATA_SHIFT) +#define HDFGRTR_EL2_nBRBDATA_VAL(x) ((x) & HDFGRTR_EL2_nBRBDATA_MASK) +#define HDFGRTR_EL2_nBRBDATA_TRAP (UL(0x0) << HDFGRTR_EL2_nBRBDATA_SHIFT) +#define HDFGRTR_EL2_nBRBDATA_NOTRAP (UL(0x1) << HDFGRTR_EL2_nBRBDATA_SHIFT) +#define HDFGRTR_EL2_nBRBCTL_SHIFT 60 +#define HDFGRTR_EL2_nBRBCTL_MASK (UL(0x1) << HDFGRTR_EL2_nBRBCTL_SHIFT) +#define HDFGRTR_EL2_nBRBCTL_VAL(x) ((x) & HDFGRTR_EL2_nBRBCTL_MASK) +#define HDFGRTR_EL2_nBRBCTL_TRAP (UL(0x0) << HDFGRTR_EL2_nBRBCTL_SHIFT) +#define HDFGRTR_EL2_nBRBCTL_NOTRAP (UL(0x1) << HDFGRTR_EL2_nBRBCTL_SHIFT) +#define HDFGRTR_EL2_nBRBIDR_SHIFT 59 +#define HDFGRTR_EL2_nBRBIDR_MASK (UL(0x1) << HDFGRTR_EL2_nBRBIDR_SHIFT) +#define HDFGRTR_EL2_nBRBIDR_VAL(x) ((x) & HDFGRTR_EL2_nBRBIDR_MASK) +#define HDFGRTR_EL2_nBRBIDR_TRAP (UL(0x0) << HDFGRTR_EL2_nBRBIDR_SHIFT) +#define HDFGRTR_EL2_nBRBIDR_NOTRAP (UL(0x1) << HDFGRTR_EL2_nBRBIDR_SHIFT) +#define HDFGRTR_EL2_PMCEIDn_EL0_SHIFT 58 +#define HDFGRTR_EL2_PMCEIDn_EL0_MASK (UL(0x1) << HDFGRTR_EL2_PMCEIDn_EL0_SHIFT) +#define HDFGRTR_EL2_PMCEIDn_EL0_VAL(x) ((x) & HDFGRTR_EL2_PMCEIDn_EL0_MASK) +#define HDFGRTR_EL2_PMCEIDn_EL0_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMCEIDn_EL0_SHIFT) +#define HDFGRTR_EL2_PMCEIDn_EL0_TRAP (UL(0x1) << HDFGRTR_EL2_PMCEIDn_EL0_SHIFT) +#define HDFGRTR_EL2_PMUSERENR_EL0_SHIFT 57 +#define HDFGRTR_EL2_PMUSERENR_EL0_MASK (UL(0x1) << HDFGRTR_EL2_PMUSERENR_EL0_SHIFT) +#define HDFGRTR_EL2_PMUSERENR_EL0_VAL(x) ((x) & HDFGRTR_EL2_PMUSERENR_EL0_MASK) +#define HDFGRTR_EL2_PMUSERENR_EL0_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMUSERENR_EL0_SHIFT) +#define HDFGRTR_EL2_PMUSERENR_EL0_TRAP (UL(0x1) << HDFGRTR_EL2_PMUSERENR_EL0_SHIFT) +#define HDFGRTR_EL2_TRBTRG_EL1_SHIFT 56 +#define HDFGRTR_EL2_TRBTRG_EL1_MASK (UL(0x1) << HDFGRTR_EL2_TRBTRG_EL1_SHIFT) +#define HDFGRTR_EL2_TRBTRG_EL1_VAL(x) ((x) & HDFGRTR_EL2_TRBTRG_EL1_MASK) +#define HDFGRTR_EL2_TRBTRG_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRBTRG_EL1_SHIFT) +#define HDFGRTR_EL2_TRBTRG_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_TRBTRG_EL1_SHIFT) +#define HDFGRTR_EL2_TRBSR_EL1_SHIFT 55 +#define HDFGRTR_EL2_TRBSR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_TRBSR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBSR_EL1_VAL(x) ((x) & HDFGRTR_EL2_TRBSR_EL1_MASK) +#define HDFGRTR_EL2_TRBSR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRBSR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBSR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_TRBSR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBPTR_EL1_SHIFT 54 +#define HDFGRTR_EL2_TRBPTR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_TRBPTR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBPTR_EL1_VAL(x) ((x) & HDFGRTR_EL2_TRBPTR_EL1_MASK) +#define HDFGRTR_EL2_TRBPTR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRBPTR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBPTR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_TRBPTR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBMAR_EL1_SHIFT 53 +#define HDFGRTR_EL2_TRBMAR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_TRBMAR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBMAR_EL1_VAL(x) ((x) & HDFGRTR_EL2_TRBMAR_EL1_MASK) +#define HDFGRTR_EL2_TRBMAR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRBMAR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBMAR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_TRBMAR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBLIMITR_EL1_SHIFT 52 +#define HDFGRTR_EL2_TRBLIMITR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_TRBLIMITR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBLIMITR_EL1_VAL(x) ((x) & HDFGRTR_EL2_TRBLIMITR_EL1_MASK) +#define HDFGRTR_EL2_TRBLIMITR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRBLIMITR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBLIMITR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_TRBLIMITR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBIDR_EL1_SHIFT 51 +#define HDFGRTR_EL2_TRBIDR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_TRBIDR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBIDR_EL1_VAL(x) ((x) & HDFGRTR_EL2_TRBIDR_EL1_MASK) +#define HDFGRTR_EL2_TRBIDR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRBIDR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBIDR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_TRBIDR_EL1_SHIFT) +#define HDFGRTR_EL2_TRBBASER_EL1_SHIFT 50 +#define HDFGRTR_EL2_TRBBASER_EL1_MASK (UL(0x1) << HDFGRTR_EL2_TRBBASER_EL1_SHIFT) +#define HDFGRTR_EL2_TRBBASER_EL1_VAL(x) ((x) & HDFGRTR_EL2_TRBBASER_EL1_MASK) +#define HDFGRTR_EL2_TRBBASER_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRBBASER_EL1_SHIFT) +#define HDFGRTR_EL2_TRBBASER_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_TRBBASER_EL1_SHIFT) +#define HDFGRTR_EL2_TRCVICTLR_SHIFT 48 +#define HDFGRTR_EL2_TRCVICTLR_MASK (UL(0x1) << HDFGRTR_EL2_TRCVICTLR_SHIFT) +#define HDFGRTR_EL2_TRCVICTLR_VAL(x) ((x) & HDFGRTR_EL2_TRCVICTLR_MASK) +#define HDFGRTR_EL2_TRCVICTLR_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCVICTLR_SHIFT) +#define HDFGRTR_EL2_TRCVICTLR_TRAP (UL(0x1) << HDFGRTR_EL2_TRCVICTLR_SHIFT) +#define HDFGRTR_EL2_TRCSTATR_SHIFT 47 +#define HDFGRTR_EL2_TRCSTATR_MASK (UL(0x1) << HDFGRTR_EL2_TRCSTATR_SHIFT) +#define HDFGRTR_EL2_TRCSTATR_VAL(x) ((x) & HDFGRTR_EL2_TRCSTATR_MASK) +#define HDFGRTR_EL2_TRCSTATR_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCSTATR_SHIFT) +#define HDFGRTR_EL2_TRCSTATR_TRAP (UL(0x1) << HDFGRTR_EL2_TRCSTATR_SHIFT) +#define HDFGRTR_EL2_TRCSSCSRn_SHIFT 46 +#define HDFGRTR_EL2_TRCSSCSRn_MASK (UL(0x1) << HDFGRTR_EL2_TRCSSCSRn_SHIFT) +#define HDFGRTR_EL2_TRCSSCSRn_VAL(x) ((x) & HDFGRTR_EL2_TRCSSCSRn_MASK) +#define HDFGRTR_EL2_TRCSSCSRn_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCSSCSRn_SHIFT) +#define HDFGRTR_EL2_TRCSSCSRn_TRAP (UL(0x1) << HDFGRTR_EL2_TRCSSCSRn_SHIFT) +#define HDFGRTR_EL2_TRCSEQSTR_SHIFT 45 +#define HDFGRTR_EL2_TRCSEQSTR_MASK (UL(0x1) << HDFGRTR_EL2_TRCSEQSTR_SHIFT) +#define HDFGRTR_EL2_TRCSEQSTR_VAL(x) ((x) & HDFGRTR_EL2_TRCSEQSTR_MASK) +#define HDFGRTR_EL2_TRCSEQSTR_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCSEQSTR_SHIFT) +#define HDFGRTR_EL2_TRCSEQSTR_TRAP (UL(0x1) << HDFGRTR_EL2_TRCSEQSTR_SHIFT) +#define HDFGRTR_EL2_TRCPRGCTLR_SHIFT 44 +#define HDFGRTR_EL2_TRCPRGCTLR_MASK (UL(0x1) << HDFGRTR_EL2_TRCPRGCTLR_SHIFT) +#define HDFGRTR_EL2_TRCPRGCTLR_VAL(x) ((x) & HDFGRTR_EL2_TRCPRGCTLR_MASK) +#define HDFGRTR_EL2_TRCPRGCTLR_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCPRGCTLR_SHIFT) +#define HDFGRTR_EL2_TRCPRGCTLR_TRAP (UL(0x1) << HDFGRTR_EL2_TRCPRGCTLR_SHIFT) +#define HDFGRTR_EL2_TRCOSLSR_SHIFT 43 +#define HDFGRTR_EL2_TRCOSLSR_MASK (UL(0x1) << HDFGRTR_EL2_TRCOSLSR_SHIFT) +#define HDFGRTR_EL2_TRCOSLSR_VAL(x) ((x) & HDFGRTR_EL2_TRCOSLSR_MASK) +#define HDFGRTR_EL2_TRCOSLSR_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCOSLSR_SHIFT) +#define HDFGRTR_EL2_TRCOSLSR_TRAP (UL(0x1) << HDFGRTR_EL2_TRCOSLSR_SHIFT) +#define HDFGRTR_EL2_TRCIMSPECn_SHIFT 41 +#define HDFGRTR_EL2_TRCIMSPECn_MASK (UL(0x1) << HDFGRTR_EL2_TRCIMSPECn_SHIFT) +#define HDFGRTR_EL2_TRCIMSPECn_VAL(x) ((x) & HDFGRTR_EL2_TRCIMSPECn_MASK) +#define HDFGRTR_EL2_TRCIMSPECn_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCIMSPECn_SHIFT) +#define HDFGRTR_EL2_TRCIMSPECn_TRAP (UL(0x1) << HDFGRTR_EL2_TRCIMSPECn_SHIFT) +#define HDFGRTR_EL2_TRCID_SHIFT 40 +#define HDFGRTR_EL2_TRCID_MASK (UL(0x1) << HDFGRTR_EL2_TRCID_SHIFT) +#define HDFGRTR_EL2_TRCID_VAL(x) ((x) & HDFGRTR_EL2_TRCID_MASK) +#define HDFGRTR_EL2_TRCID_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCID_SHIFT) +#define HDFGRTR_EL2_TRCID_TRAP (UL(0x1) << HDFGRTR_EL2_TRCID_SHIFT) +#define HDFGRTR_EL2_TRCCNTVRn_SHIFT 37 +#define HDFGRTR_EL2_TRCCNTVRn_MASK (UL(0x1) << HDFGRTR_EL2_TRCCNTVRn_SHIFT) +#define HDFGRTR_EL2_TRCCNTVRn_VAL(x) ((x) & HDFGRTR_EL2_TRCCNTVRn_MASK) +#define HDFGRTR_EL2_TRCCNTVRn_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCCNTVRn_SHIFT) +#define HDFGRTR_EL2_TRCCNTVRn_TRAP (UL(0x1) << HDFGRTR_EL2_TRCCNTVRn_SHIFT) +#define HDFGRTR_EL2_TRCCLAIM_SHIFT 36 +#define HDFGRTR_EL2_TRCCLAIM_MASK (UL(0x1) << HDFGRTR_EL2_TRCCLAIM_SHIFT) +#define HDFGRTR_EL2_TRCCLAIM_VAL(x) ((x) & HDFGRTR_EL2_TRCCLAIM_MASK) +#define HDFGRTR_EL2_TRCCLAIM_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCCLAIM_SHIFT) +#define HDFGRTR_EL2_TRCCLAIM_TRAP (UL(0x1) << HDFGRTR_EL2_TRCCLAIM_SHIFT) +#define HDFGRTR_EL2_TRCAUXCTLR_SHIFT 35 +#define HDFGRTR_EL2_TRCAUXCTLR_MASK (UL(0x1) << HDFGRTR_EL2_TRCAUXCTLR_SHIFT) +#define HDFGRTR_EL2_TRCAUXCTLR_VAL(x) ((x) & HDFGRTR_EL2_TRCAUXCTLR_MASK) +#define HDFGRTR_EL2_TRCAUXCTLR_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCAUXCTLR_SHIFT) +#define HDFGRTR_EL2_TRCAUXCTLR_TRAP (UL(0x1) << HDFGRTR_EL2_TRCAUXCTLR_SHIFT) +#define HDFGRTR_EL2_TRCAUTHSTATUS_SHIFT 34 +#define HDFGRTR_EL2_TRCAUTHSTATUS_MASK (UL(0x1) << HDFGRTR_EL2_TRCAUTHSTATUS_SHIFT) +#define HDFGRTR_EL2_TRCAUTHSTATUS_VAL(x) ((x) & HDFGRTR_EL2_TRCAUTHSTATUS_MASK) +#define HDFGRTR_EL2_TRCAUTHSTATUS_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRCAUTHSTATUS_SHIFT) +#define HDFGRTR_EL2_TRCAUTHSTATUS_TRAP (UL(0x1) << HDFGRTR_EL2_TRCAUTHSTATUS_SHIFT) +#define HDFGRTR_EL2_TRC_SHIFT 33 +#define HDFGRTR_EL2_TRC_MASK (UL(0x1) << HDFGRTR_EL2_TRC_SHIFT) +#define HDFGRTR_EL2_TRC_VAL(x) ((x) & HDFGRTR_EL2_TRC_MASK) +#define HDFGRTR_EL2_TRC_NOTRAP (UL(0x0) << HDFGRTR_EL2_TRC_SHIFT) +#define HDFGRTR_EL2_TRC_TRAP (UL(0x1) << HDFGRTR_EL2_TRC_SHIFT) +#define HDFGRTR_EL2_PMSLATFR_EL1_SHIFT 32 +#define HDFGRTR_EL2_PMSLATFR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMSLATFR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSLATFR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMSLATFR_EL1_MASK) +#define HDFGRTR_EL2_PMSLATFR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMSLATFR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSLATFR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMSLATFR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSIRR_EL1_SHIFT 31 +#define HDFGRTR_EL2_PMSIRR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMSIRR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSIRR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMSIRR_EL1_MASK) +#define HDFGRTR_EL2_PMSIRR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMSIRR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSIRR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMSIRR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSIDR_EL1_SHIFT 30 +#define HDFGRTR_EL2_PMSIDR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMSIDR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSIDR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMSIDR_EL1_MASK) +#define HDFGRTR_EL2_PMSIDR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMSIDR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSIDR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMSIDR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSICR_EL1_SHIFT 29 +#define HDFGRTR_EL2_PMSICR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMSICR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSICR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMSICR_EL1_MASK) +#define HDFGRTR_EL2_PMSICR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMSICR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSICR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMSICR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSFCR_EL1_SHIFT 28 +#define HDFGRTR_EL2_PMSFCR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMSFCR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSFCR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMSFCR_EL1_MASK) +#define HDFGRTR_EL2_PMSFCR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMSFCR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSFCR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMSFCR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSEVFR_EL1_SHIFT 27 +#define HDFGRTR_EL2_PMSEVFR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMSEVFR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSEVFR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMSEVFR_EL1_MASK) +#define HDFGRTR_EL2_PMSEVFR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMSEVFR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSEVFR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMSEVFR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSCR_EL1_SHIFT 26 +#define HDFGRTR_EL2_PMSCR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMSCR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSCR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMSCR_EL1_MASK) +#define HDFGRTR_EL2_PMSCR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMSCR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSCR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMSCR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBSR_EL1_SHIFT 25 +#define HDFGRTR_EL2_PMBSR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMBSR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBSR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMBSR_EL1_MASK) +#define HDFGRTR_EL2_PMBSR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMBSR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBSR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMBSR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBPTR_EL1_SHIFT 24 +#define HDFGRTR_EL2_PMBPTR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMBPTR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBPTR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMBPTR_EL1_MASK) +#define HDFGRTR_EL2_PMBPTR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMBPTR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBPTR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMBPTR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBLIMITR_EL1_SHIFT 23 +#define HDFGRTR_EL2_PMBLIMITR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMBLIMITR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBLIMITR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMBLIMITR_EL1_MASK) +#define HDFGRTR_EL2_PMBLIMITR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMBLIMITR_EL1_SHIFT) +#define HDFGRTR_EL2_PMBLIMITR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMBLIMITR_EL1_SHIFT) +#define HDFGRTR_EL2_PMMIR_EL1_SHIFT 22 +#define HDFGRTR_EL2_PMMIR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_PMMIR_EL1_SHIFT) +#define HDFGRTR_EL2_PMMIR_EL1_VAL(x) ((x) & HDFGRTR_EL2_PMMIR_EL1_MASK) +#define HDFGRTR_EL2_PMMIR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMMIR_EL1_SHIFT) +#define HDFGRTR_EL2_PMMIR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_PMMIR_EL1_SHIFT) +#define HDFGRTR_EL2_PMSELR_EL0_SHIFT 19 +#define HDFGRTR_EL2_PMSELR_EL0_MASK (UL(0x1) << HDFGRTR_EL2_PMSELR_EL0_SHIFT) +#define HDFGRTR_EL2_PMSELR_EL0_VAL(x) ((x) & HDFGRTR_EL2_PMSELR_EL0_MASK) +#define HDFGRTR_EL2_PMSELR_EL0_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMSELR_EL0_SHIFT) +#define HDFGRTR_EL2_PMSELR_EL0_TRAP (UL(0x1) << HDFGRTR_EL2_PMSELR_EL0_SHIFT) +#define HDFGRTR_EL2_PMOVS_SHIFT 18 +#define HDFGRTR_EL2_PMOVS_MASK (UL(0x1) << HDFGRTR_EL2_PMOVS_SHIFT) +#define HDFGRTR_EL2_PMOVS_VAL(x) ((x) & HDFGRTR_EL2_PMOVS_MASK) +#define HDFGRTR_EL2_PMOVS_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMOVS_SHIFT) +#define HDFGRTR_EL2_PMOVS_TRAP (UL(0x1) << HDFGRTR_EL2_PMOVS_SHIFT) +#define HDFGRTR_EL2_PMINTEN_SHIFT 17 +#define HDFGRTR_EL2_PMINTEN_MASK (UL(0x1) << HDFGRTR_EL2_PMINTEN_SHIFT) +#define HDFGRTR_EL2_PMINTEN_VAL(x) ((x) & HDFGRTR_EL2_PMINTEN_MASK) +#define HDFGRTR_EL2_PMINTEN_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMINTEN_SHIFT) +#define HDFGRTR_EL2_PMINTEN_TRAP (UL(0x1) << HDFGRTR_EL2_PMINTEN_SHIFT) +#define HDFGRTR_EL2_PMCNTEN_SHIFT 16 +#define HDFGRTR_EL2_PMCNTEN_MASK (UL(0x1) << HDFGRTR_EL2_PMCNTEN_SHIFT) +#define HDFGRTR_EL2_PMCNTEN_VAL(x) ((x) & HDFGRTR_EL2_PMCNTEN_MASK) +#define HDFGRTR_EL2_PMCNTEN_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMCNTEN_SHIFT) +#define HDFGRTR_EL2_PMCNTEN_TRAP (UL(0x1) << HDFGRTR_EL2_PMCNTEN_SHIFT) +#define HDFGRTR_EL2_PMCCNTR_EL0_SHIFT 15 +#define HDFGRTR_EL2_PMCCNTR_EL0_MASK (UL(0x1) << HDFGRTR_EL2_PMCCNTR_EL0_SHIFT) +#define HDFGRTR_EL2_PMCCNTR_EL0_VAL(x) ((x) & HDFGRTR_EL2_PMCCNTR_EL0_MASK) +#define HDFGRTR_EL2_PMCCNTR_EL0_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMCCNTR_EL0_SHIFT) +#define HDFGRTR_EL2_PMCCNTR_EL0_TRAP (UL(0x1) << HDFGRTR_EL2_PMCCNTR_EL0_SHIFT) +#define HDFGRTR_EL2_PMCCFILTR_EL0_SHIFT 14 +#define HDFGRTR_EL2_PMCCFILTR_EL0_MASK (UL(0x1) << HDFGRTR_EL2_PMCCFILTR_EL0_SHIFT) +#define HDFGRTR_EL2_PMCCFILTR_EL0_VAL(x) ((x) & HDFGRTR_EL2_PMCCFILTR_EL0_MASK) +#define HDFGRTR_EL2_PMCCFILTR_EL0_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMCCFILTR_EL0_SHIFT) +#define HDFGRTR_EL2_PMCCFILTR_EL0_TRAP (UL(0x1) << HDFGRTR_EL2_PMCCFILTR_EL0_SHIFT) +#define HDFGRTR_EL2_PMEVTYPERn_EL0_SHIFT 13 +#define HDFGRTR_EL2_PMEVTYPERn_EL0_MASK (UL(0x1) << HDFGRTR_EL2_PMEVTYPERn_EL0_SHIFT) +#define HDFGRTR_EL2_PMEVTYPERn_EL0_VAL(x) ((x) & HDFGRTR_EL2_PMEVTYPERn_EL0_MASK) +#define HDFGRTR_EL2_PMEVTYPERn_EL0_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMEVTYPERn_EL0_SHIFT) +#define HDFGRTR_EL2_PMEVTYPERn_EL0_TRAP (UL(0x1) << HDFGRTR_EL2_PMEVTYPERn_EL0_SHIFT) +#define HDFGRTR_EL2_PMEVCNTRn_EL0_SHIFT 12 +#define HDFGRTR_EL2_PMEVCNTRn_EL0_MASK (UL(0x1) << HDFGRTR_EL2_PMEVCNTRn_EL0_SHIFT) +#define HDFGRTR_EL2_PMEVCNTRn_EL0_VAL(x) ((x) & HDFGRTR_EL2_PMEVCNTRn_EL0_MASK) +#define HDFGRTR_EL2_PMEVCNTRn_EL0_NOTRAP (UL(0x0) << HDFGRTR_EL2_PMEVCNTRn_EL0_SHIFT) +#define HDFGRTR_EL2_PMEVCNTRn_EL0_TRAP (UL(0x1) << HDFGRTR_EL2_PMEVCNTRn_EL0_SHIFT) +#define HDFGRTR_EL2_OSDLR_EL1_SHIFT 11 +#define HDFGRTR_EL2_OSDLR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_OSDLR_EL1_SHIFT) +#define HDFGRTR_EL2_OSDLR_EL1_VAL(x) ((x) & HDFGRTR_EL2_OSDLR_EL1_MASK) +#define HDFGRTR_EL2_OSDLR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_OSDLR_EL1_SHIFT) +#define HDFGRTR_EL2_OSDLR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_OSDLR_EL1_SHIFT) +#define HDFGRTR_EL2_OSECCR_EL1_SHIFT 10 +#define HDFGRTR_EL2_OSECCR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_OSECCR_EL1_SHIFT) +#define HDFGRTR_EL2_OSECCR_EL1_VAL(x) ((x) & HDFGRTR_EL2_OSECCR_EL1_MASK) +#define HDFGRTR_EL2_OSECCR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_OSECCR_EL1_SHIFT) +#define HDFGRTR_EL2_OSECCR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_OSECCR_EL1_SHIFT) +#define HDFGRTR_EL2_OSLSR_EL1_SHIFT 9 +#define HDFGRTR_EL2_OSLSR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_OSLSR_EL1_SHIFT) +#define HDFGRTR_EL2_OSLSR_EL1_VAL(x) ((x) & HDFGRTR_EL2_OSLSR_EL1_MASK) +#define HDFGRTR_EL2_OSLSR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_OSLSR_EL1_SHIFT) +#define HDFGRTR_EL2_OSLSR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_OSLSR_EL1_SHIFT) +#define HDFGRTR_EL2_DBGPRCR_EL1_SHIFT 7 +#define HDFGRTR_EL2_DBGPRCR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_DBGPRCR_EL1_SHIFT) +#define HDFGRTR_EL2_DBGPRCR_EL1_VAL(x) ((x) & HDFGRTR_EL2_DBGPRCR_EL1_MASK) +#define HDFGRTR_EL2_DBGPRCR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_DBGPRCR_EL1_SHIFT) +#define HDFGRTR_EL2_DBGPRCR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_DBGPRCR_EL1_SHIFT) +#define HDFGRTR_EL2_DBGAUTHSTATUS_EL1_SHIFT 6 +#define HDFGRTR_EL2_DBGAUTHSTATUS_EL1_MASK (UL(0x1) << HDFGRTR_EL2_DBGAUTHSTATUS_EL1_SHIFT) +#define HDFGRTR_EL2_DBGAUTHSTATUS_EL1_VAL(x) ((x) & HDFGRTR_EL2_DBGAUTHSTATUS_EL1_MASK) +#define HDFGRTR_EL2_DBGAUTHSTATUS_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_DBGAUTHSTATUS_EL1_SHIFT) +#define HDFGRTR_EL2_DBGAUTHSTATUS_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_DBGAUTHSTATUS_EL1_SHIFT) +#define HDFGRTR_EL2_DBGCLAIM_SHIFT 5 +#define HDFGRTR_EL2_DBGCLAIM_MASK (UL(0x1) << HDFGRTR_EL2_DBGCLAIM_SHIFT) +#define HDFGRTR_EL2_DBGCLAIM_VAL(x) ((x) & HDFGRTR_EL2_DBGCLAIM_MASK) +#define HDFGRTR_EL2_DBGCLAIM_NOTRAP (UL(0x0) << HDFGRTR_EL2_DBGCLAIM_SHIFT) +#define HDFGRTR_EL2_DBGCLAIM_TRAP (UL(0x1) << HDFGRTR_EL2_DBGCLAIM_SHIFT) +#define HDFGRTR_EL2_MDSCR_EL1_SHIFT 4 +#define HDFGRTR_EL2_MDSCR_EL1_MASK (UL(0x1) << HDFGRTR_EL2_MDSCR_EL1_SHIFT) +#define HDFGRTR_EL2_MDSCR_EL1_VAL(x) ((x) & HDFGRTR_EL2_MDSCR_EL1_MASK) +#define HDFGRTR_EL2_MDSCR_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_MDSCR_EL1_SHIFT) +#define HDFGRTR_EL2_MDSCR_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_MDSCR_EL1_SHIFT) +#define HDFGRTR_EL2_DBGWVRn_EL1_SHIFT 3 +#define HDFGRTR_EL2_DBGWVRn_EL1_MASK (UL(0x1) << HDFGRTR_EL2_DBGWVRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGWVRn_EL1_VAL(x) ((x) & HDFGRTR_EL2_DBGWVRn_EL1_MASK) +#define HDFGRTR_EL2_DBGWVRn_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_DBGWVRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGWVRn_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_DBGWVRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGWCRn_EL1_SHIFT 2 +#define HDFGRTR_EL2_DBGWCRn_EL1_MASK (UL(0x1) << HDFGRTR_EL2_DBGWCRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGWCRn_EL1_VAL(x) ((x) & HDFGRTR_EL2_DBGWCRn_EL1_MASK) +#define HDFGRTR_EL2_DBGWCRn_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_DBGWCRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGWCRn_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_DBGWCRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGBVRn_EL1_SHIFT 1 +#define HDFGRTR_EL2_DBGBVRn_EL1_MASK (UL(0x1) << HDFGRTR_EL2_DBGBVRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGBVRn_EL1_VAL(x) ((x) & HDFGRTR_EL2_DBGBVRn_EL1_MASK) +#define HDFGRTR_EL2_DBGBVRn_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_DBGBVRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGBVRn_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_DBGBVRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGBCRn_EL1_SHIFT 0 +#define HDFGRTR_EL2_DBGBCRn_EL1_MASK (UL(0x1) << HDFGRTR_EL2_DBGBCRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGBCRn_EL1_VAL(x) ((x) & HDFGRTR_EL2_DBGBCRn_EL1_MASK) +#define HDFGRTR_EL2_DBGBCRn_EL1_NOTRAP (UL(0x0) << HDFGRTR_EL2_DBGBCRn_EL1_SHIFT) +#define HDFGRTR_EL2_DBGBCRn_EL1_TRAP (UL(0x1) << HDFGRTR_EL2_DBGBCRn_EL1_SHIFT) + +/* HDFGWTR2_EL2 */ +#define HDFGWTR2_EL2_REG MRS_REG_ALT_NAME(HDFGWTR2_EL2) +#define HDFGWTR2_EL2_op0 3 +#define HDFGWTR2_EL2_op1 4 +#define HDFGWTR2_EL2_CRn 3 +#define HDFGWTR2_EL2_CRm 1 +#define HDFGWTR2_EL2_op2 1 +#define HDFGWTR2_EL2_nMDSTEPOP_EL1_SHIFT 23 +#define HDFGWTR2_EL2_nMDSTEPOP_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nMDSTEPOP_EL1_SHIFT) +#define HDFGWTR2_EL2_nMDSTEPOP_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nMDSTEPOP_EL1_MASK) +#define HDFGWTR2_EL2_nMDSTEPOP_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nMDSTEPOP_EL1_SHIFT) +#define HDFGWTR2_EL2_nMDSTEPOP_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nMDSTEPOP_EL1_SHIFT) +#define HDFGWTR2_EL2_nTRBMPAM_EL1_SHIFT 22 +#define HDFGWTR2_EL2_nTRBMPAM_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nTRBMPAM_EL1_SHIFT) +#define HDFGWTR2_EL2_nTRBMPAM_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nTRBMPAM_EL1_MASK) +#define HDFGWTR2_EL2_nTRBMPAM_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nTRBMPAM_EL1_SHIFT) +#define HDFGWTR2_EL2_nTRBMPAM_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nTRBMPAM_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMZR_EL0_SHIFT 21 +#define HDFGWTR2_EL2_nPMZR_EL0_MASK (UL(0x1) << HDFGWTR2_EL2_nPMZR_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMZR_EL0_VAL(x) ((x) & HDFGWTR2_EL2_nPMZR_EL0_MASK) +#define HDFGWTR2_EL2_nPMZR_EL0_TRAP (UL(0x0) << HDFGWTR2_EL2_nPMZR_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMZR_EL0_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nPMZR_EL0_SHIFT) +#define HDFGWTR2_EL2_nTRCITECR_EL1_SHIFT 20 +#define HDFGWTR2_EL2_nTRCITECR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nTRCITECR_EL1_SHIFT) +#define HDFGWTR2_EL2_nTRCITECR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nTRCITECR_EL1_MASK) +#define HDFGWTR2_EL2_nTRCITECR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nTRCITECR_EL1_SHIFT) +#define HDFGWTR2_EL2_nTRCITECR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nTRCITECR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMSDSFR_EL1_SHIFT 19 +#define HDFGWTR2_EL2_nPMSDSFR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nPMSDSFR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMSDSFR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nPMSDSFR_EL1_MASK) +#define HDFGWTR2_EL2_nPMSDSFR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nPMSDSFR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMSDSFR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nPMSDSFR_EL1_SHIFT) +#define HDFGWTR2_EL2_nSPMSCR_EL1_SHIFT 16 +#define HDFGWTR2_EL2_nSPMSCR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMSCR_EL1_SHIFT) +#define HDFGWTR2_EL2_nSPMSCR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nSPMSCR_EL1_MASK) +#define HDFGWTR2_EL2_nSPMSCR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMSCR_EL1_SHIFT) +#define HDFGWTR2_EL2_nSPMSCR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMSCR_EL1_SHIFT) +#define HDFGWTR2_EL2_nSPMACCESSR_EL1_SHIFT 15 +#define HDFGWTR2_EL2_nSPMACCESSR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMACCESSR_EL1_SHIFT) +#define HDFGWTR2_EL2_nSPMACCESSR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nSPMACCESSR_EL1_MASK) +#define HDFGWTR2_EL2_nSPMACCESSR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMACCESSR_EL1_SHIFT) +#define HDFGWTR2_EL2_nSPMACCESSR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMACCESSR_EL1_SHIFT) +#define HDFGWTR2_EL2_nSPMCR_EL0_SHIFT 14 +#define HDFGWTR2_EL2_nSPMCR_EL0_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMCR_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMCR_EL0_VAL(x) ((x) & HDFGWTR2_EL2_nSPMCR_EL0_MASK) +#define HDFGWTR2_EL2_nSPMCR_EL0_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMCR_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMCR_EL0_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMCR_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMOVS_SHIFT 13 +#define HDFGWTR2_EL2_nSPMOVS_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMOVS_SHIFT) +#define HDFGWTR2_EL2_nSPMOVS_VAL(x) ((x) & HDFGWTR2_EL2_nSPMOVS_MASK) +#define HDFGWTR2_EL2_nSPMOVS_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMOVS_SHIFT) +#define HDFGWTR2_EL2_nSPMOVS_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMOVS_SHIFT) +#define HDFGWTR2_EL2_nSPMINTEN_SHIFT 12 +#define HDFGWTR2_EL2_nSPMINTEN_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMINTEN_SHIFT) +#define HDFGWTR2_EL2_nSPMINTEN_VAL(x) ((x) & HDFGWTR2_EL2_nSPMINTEN_MASK) +#define HDFGWTR2_EL2_nSPMINTEN_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMINTEN_SHIFT) +#define HDFGWTR2_EL2_nSPMINTEN_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMINTEN_SHIFT) +#define HDFGWTR2_EL2_nSPMCNTEN_SHIFT 11 +#define HDFGWTR2_EL2_nSPMCNTEN_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMCNTEN_SHIFT) +#define HDFGWTR2_EL2_nSPMCNTEN_VAL(x) ((x) & HDFGWTR2_EL2_nSPMCNTEN_MASK) +#define HDFGWTR2_EL2_nSPMCNTEN_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMCNTEN_SHIFT) +#define HDFGWTR2_EL2_nSPMCNTEN_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMCNTEN_SHIFT) +#define HDFGWTR2_EL2_nSPMSELR_EL0_SHIFT 10 +#define HDFGWTR2_EL2_nSPMSELR_EL0_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMSELR_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMSELR_EL0_VAL(x) ((x) & HDFGWTR2_EL2_nSPMSELR_EL0_MASK) +#define HDFGWTR2_EL2_nSPMSELR_EL0_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMSELR_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMSELR_EL0_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMSELR_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMEVTYPERn_EL0_SHIFT 9 +#define HDFGWTR2_EL2_nSPMEVTYPERn_EL0_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMEVTYPERn_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMEVTYPERn_EL0_VAL(x) ((x) & HDFGWTR2_EL2_nSPMEVTYPERn_EL0_MASK) +#define HDFGWTR2_EL2_nSPMEVTYPERn_EL0_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMEVTYPERn_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMEVTYPERn_EL0_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMEVTYPERn_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMEVCNTRn_EL0_SHIFT 8 +#define HDFGWTR2_EL2_nSPMEVCNTRn_EL0_MASK (UL(0x1) << HDFGWTR2_EL2_nSPMEVCNTRn_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMEVCNTRn_EL0_VAL(x) ((x) & HDFGWTR2_EL2_nSPMEVCNTRn_EL0_MASK) +#define HDFGWTR2_EL2_nSPMEVCNTRn_EL0_TRAP (UL(0x0) << HDFGWTR2_EL2_nSPMEVCNTRn_EL0_SHIFT) +#define HDFGWTR2_EL2_nSPMEVCNTRn_EL0_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nSPMEVCNTRn_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMSSCR_EL1_SHIFT 7 +#define HDFGWTR2_EL2_nPMSSCR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nPMSSCR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMSSCR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nPMSSCR_EL1_MASK) +#define HDFGWTR2_EL2_nPMSSCR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nPMSSCR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMSSCR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nPMSSCR_EL1_SHIFT) +#define HDFGWTR2_EL2_nMDSELR_EL1_SHIFT 5 +#define HDFGWTR2_EL2_nMDSELR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nMDSELR_EL1_SHIFT) +#define HDFGWTR2_EL2_nMDSELR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nMDSELR_EL1_MASK) +#define HDFGWTR2_EL2_nMDSELR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nMDSELR_EL1_SHIFT) +#define HDFGWTR2_EL2_nMDSELR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nMDSELR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMUACR_EL1_SHIFT 4 +#define HDFGWTR2_EL2_nPMUACR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nPMUACR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMUACR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nPMUACR_EL1_MASK) +#define HDFGWTR2_EL2_nPMUACR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nPMUACR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMUACR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nPMUACR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMICFILTR_EL0_SHIFT 3 +#define HDFGWTR2_EL2_nPMICFILTR_EL0_MASK (UL(0x1) << HDFGWTR2_EL2_nPMICFILTR_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMICFILTR_EL0_VAL(x) ((x) & HDFGWTR2_EL2_nPMICFILTR_EL0_MASK) +#define HDFGWTR2_EL2_nPMICFILTR_EL0_TRAP (UL(0x0) << HDFGWTR2_EL2_nPMICFILTR_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMICFILTR_EL0_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nPMICFILTR_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMICNTR_EL0_SHIFT 2 +#define HDFGWTR2_EL2_nPMICNTR_EL0_MASK (UL(0x1) << HDFGWTR2_EL2_nPMICNTR_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMICNTR_EL0_VAL(x) ((x) & HDFGWTR2_EL2_nPMICNTR_EL0_MASK) +#define HDFGWTR2_EL2_nPMICNTR_EL0_TRAP (UL(0x0) << HDFGWTR2_EL2_nPMICNTR_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMICNTR_EL0_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nPMICNTR_EL0_SHIFT) +#define HDFGWTR2_EL2_nPMIAR_EL1_SHIFT 1 +#define HDFGWTR2_EL2_nPMIAR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nPMIAR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMIAR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nPMIAR_EL1_MASK) +#define HDFGWTR2_EL2_nPMIAR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nPMIAR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMIAR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nPMIAR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMECR_EL1_SHIFT 0 +#define HDFGWTR2_EL2_nPMECR_EL1_MASK (UL(0x1) << HDFGWTR2_EL2_nPMECR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMECR_EL1_VAL(x) ((x) & HDFGWTR2_EL2_nPMECR_EL1_MASK) +#define HDFGWTR2_EL2_nPMECR_EL1_TRAP (UL(0x0) << HDFGWTR2_EL2_nPMECR_EL1_SHIFT) +#define HDFGWTR2_EL2_nPMECR_EL1_NOTRAP (UL(0x1) << HDFGWTR2_EL2_nPMECR_EL1_SHIFT) + +/* HDFGWTR_EL2 */ +#define HDFGWTR_EL2_REG MRS_REG_ALT_NAME(HDFGWTR_EL2) +#define HDFGWTR_EL2_op0 3 +#define HDFGWTR_EL2_op1 4 +#define HDFGWTR_EL2_CRn 3 +#define HDFGWTR_EL2_CRm 1 +#define HDFGWTR_EL2_op2 5 +#define HDFGWTR_EL2_nPMSNEVFR_EL1_SHIFT 62 +#define HDFGWTR_EL2_nPMSNEVFR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_nPMSNEVFR_EL1_SHIFT) +#define HDFGWTR_EL2_nPMSNEVFR_EL1_VAL(x) ((x) & HDFGWTR_EL2_nPMSNEVFR_EL1_MASK) +#define HDFGWTR_EL2_nPMSNEVFR_EL1_TRAP (UL(0x0) << HDFGWTR_EL2_nPMSNEVFR_EL1_SHIFT) +#define HDFGWTR_EL2_nPMSNEVFR_EL1_NOTRAP (UL(0x1) << HDFGWTR_EL2_nPMSNEVFR_EL1_SHIFT) +#define HDFGWTR_EL2_nBRBDATA_SHIFT 61 +#define HDFGWTR_EL2_nBRBDATA_MASK (UL(0x1) << HDFGWTR_EL2_nBRBDATA_SHIFT) +#define HDFGWTR_EL2_nBRBDATA_VAL(x) ((x) & HDFGWTR_EL2_nBRBDATA_MASK) +#define HDFGWTR_EL2_nBRBDATA_TRAP (UL(0x0) << HDFGWTR_EL2_nBRBDATA_SHIFT) +#define HDFGWTR_EL2_nBRBDATA_NOTRAP (UL(0x1) << HDFGWTR_EL2_nBRBDATA_SHIFT) +#define HDFGWTR_EL2_nBRBCTL_SHIFT 60 +#define HDFGWTR_EL2_nBRBCTL_MASK (UL(0x1) << HDFGWTR_EL2_nBRBCTL_SHIFT) +#define HDFGWTR_EL2_nBRBCTL_VAL(x) ((x) & HDFGWTR_EL2_nBRBCTL_MASK) +#define HDFGWTR_EL2_nBRBCTL_TRAP (UL(0x0) << HDFGWTR_EL2_nBRBCTL_SHIFT) +#define HDFGWTR_EL2_nBRBCTL_NOTRAP (UL(0x1) << HDFGWTR_EL2_nBRBCTL_SHIFT) +#define HDFGWTR_EL2_PMUSERENR_EL0_SHIFT 57 +#define HDFGWTR_EL2_PMUSERENR_EL0_MASK (UL(0x1) << HDFGWTR_EL2_PMUSERENR_EL0_SHIFT) +#define HDFGWTR_EL2_PMUSERENR_EL0_VAL(x) ((x) & HDFGWTR_EL2_PMUSERENR_EL0_MASK) +#define HDFGWTR_EL2_PMUSERENR_EL0_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMUSERENR_EL0_SHIFT) +#define HDFGWTR_EL2_PMUSERENR_EL0_TRAP (UL(0x1) << HDFGWTR_EL2_PMUSERENR_EL0_SHIFT) +#define HDFGWTR_EL2_TRBTRG_EL1_SHIFT 56 +#define HDFGWTR_EL2_TRBTRG_EL1_MASK (UL(0x1) << HDFGWTR_EL2_TRBTRG_EL1_SHIFT) +#define HDFGWTR_EL2_TRBTRG_EL1_VAL(x) ((x) & HDFGWTR_EL2_TRBTRG_EL1_MASK) +#define HDFGWTR_EL2_TRBTRG_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRBTRG_EL1_SHIFT) +#define HDFGWTR_EL2_TRBTRG_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_TRBTRG_EL1_SHIFT) +#define HDFGWTR_EL2_TRBSR_EL1_SHIFT 55 +#define HDFGWTR_EL2_TRBSR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_TRBSR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBSR_EL1_VAL(x) ((x) & HDFGWTR_EL2_TRBSR_EL1_MASK) +#define HDFGWTR_EL2_TRBSR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRBSR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBSR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_TRBSR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBPTR_EL1_SHIFT 54 +#define HDFGWTR_EL2_TRBPTR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_TRBPTR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBPTR_EL1_VAL(x) ((x) & HDFGWTR_EL2_TRBPTR_EL1_MASK) +#define HDFGWTR_EL2_TRBPTR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRBPTR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBPTR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_TRBPTR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBMAR_EL1_SHIFT 53 +#define HDFGWTR_EL2_TRBMAR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_TRBMAR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBMAR_EL1_VAL(x) ((x) & HDFGWTR_EL2_TRBMAR_EL1_MASK) +#define HDFGWTR_EL2_TRBMAR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRBMAR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBMAR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_TRBMAR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBLIMITR_EL1_SHIFT 52 +#define HDFGWTR_EL2_TRBLIMITR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_TRBLIMITR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBLIMITR_EL1_VAL(x) ((x) & HDFGWTR_EL2_TRBLIMITR_EL1_MASK) +#define HDFGWTR_EL2_TRBLIMITR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRBLIMITR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBLIMITR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_TRBLIMITR_EL1_SHIFT) +#define HDFGWTR_EL2_TRBBASER_EL1_SHIFT 50 +#define HDFGWTR_EL2_TRBBASER_EL1_MASK (UL(0x1) << HDFGWTR_EL2_TRBBASER_EL1_SHIFT) +#define HDFGWTR_EL2_TRBBASER_EL1_VAL(x) ((x) & HDFGWTR_EL2_TRBBASER_EL1_MASK) +#define HDFGWTR_EL2_TRBBASER_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRBBASER_EL1_SHIFT) +#define HDFGWTR_EL2_TRBBASER_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_TRBBASER_EL1_SHIFT) +#define HDFGWTR_EL2_TRFCR_EL1_SHIFT 49 +#define HDFGWTR_EL2_TRFCR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_TRFCR_EL1_SHIFT) +#define HDFGWTR_EL2_TRFCR_EL1_VAL(x) ((x) & HDFGWTR_EL2_TRFCR_EL1_MASK) +#define HDFGWTR_EL2_TRFCR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRFCR_EL1_SHIFT) +#define HDFGWTR_EL2_TRFCR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_TRFCR_EL1_SHIFT) +#define HDFGWTR_EL2_TRCVICTLR_SHIFT 48 +#define HDFGWTR_EL2_TRCVICTLR_MASK (UL(0x1) << HDFGWTR_EL2_TRCVICTLR_SHIFT) +#define HDFGWTR_EL2_TRCVICTLR_VAL(x) ((x) & HDFGWTR_EL2_TRCVICTLR_MASK) +#define HDFGWTR_EL2_TRCVICTLR_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCVICTLR_SHIFT) +#define HDFGWTR_EL2_TRCVICTLR_TRAP (UL(0x1) << HDFGWTR_EL2_TRCVICTLR_SHIFT) +#define HDFGWTR_EL2_TRCSSCSRn_SHIFT 46 +#define HDFGWTR_EL2_TRCSSCSRn_MASK (UL(0x1) << HDFGWTR_EL2_TRCSSCSRn_SHIFT) +#define HDFGWTR_EL2_TRCSSCSRn_VAL(x) ((x) & HDFGWTR_EL2_TRCSSCSRn_MASK) +#define HDFGWTR_EL2_TRCSSCSRn_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCSSCSRn_SHIFT) +#define HDFGWTR_EL2_TRCSSCSRn_TRAP (UL(0x1) << HDFGWTR_EL2_TRCSSCSRn_SHIFT) +#define HDFGWTR_EL2_TRCSEQSTR_SHIFT 45 +#define HDFGWTR_EL2_TRCSEQSTR_MASK (UL(0x1) << HDFGWTR_EL2_TRCSEQSTR_SHIFT) +#define HDFGWTR_EL2_TRCSEQSTR_VAL(x) ((x) & HDFGWTR_EL2_TRCSEQSTR_MASK) +#define HDFGWTR_EL2_TRCSEQSTR_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCSEQSTR_SHIFT) +#define HDFGWTR_EL2_TRCSEQSTR_TRAP (UL(0x1) << HDFGWTR_EL2_TRCSEQSTR_SHIFT) +#define HDFGWTR_EL2_TRCPRGCTLR_SHIFT 44 +#define HDFGWTR_EL2_TRCPRGCTLR_MASK (UL(0x1) << HDFGWTR_EL2_TRCPRGCTLR_SHIFT) +#define HDFGWTR_EL2_TRCPRGCTLR_VAL(x) ((x) & HDFGWTR_EL2_TRCPRGCTLR_MASK) +#define HDFGWTR_EL2_TRCPRGCTLR_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCPRGCTLR_SHIFT) +#define HDFGWTR_EL2_TRCPRGCTLR_TRAP (UL(0x1) << HDFGWTR_EL2_TRCPRGCTLR_SHIFT) +#define HDFGWTR_EL2_TRCOSLAR_SHIFT 42 +#define HDFGWTR_EL2_TRCOSLAR_MASK (UL(0x1) << HDFGWTR_EL2_TRCOSLAR_SHIFT) +#define HDFGWTR_EL2_TRCOSLAR_VAL(x) ((x) & HDFGWTR_EL2_TRCOSLAR_MASK) +#define HDFGWTR_EL2_TRCOSLAR_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCOSLAR_SHIFT) +#define HDFGWTR_EL2_TRCOSLAR_TRAP (UL(0x1) << HDFGWTR_EL2_TRCOSLAR_SHIFT) +#define HDFGWTR_EL2_TRCIMSPECn_SHIFT 41 +#define HDFGWTR_EL2_TRCIMSPECn_MASK (UL(0x1) << HDFGWTR_EL2_TRCIMSPECn_SHIFT) +#define HDFGWTR_EL2_TRCIMSPECn_VAL(x) ((x) & HDFGWTR_EL2_TRCIMSPECn_MASK) +#define HDFGWTR_EL2_TRCIMSPECn_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCIMSPECn_SHIFT) +#define HDFGWTR_EL2_TRCIMSPECn_TRAP (UL(0x1) << HDFGWTR_EL2_TRCIMSPECn_SHIFT) +#define HDFGWTR_EL2_TRCCNTVRn_SHIFT 37 +#define HDFGWTR_EL2_TRCCNTVRn_MASK (UL(0x1) << HDFGWTR_EL2_TRCCNTVRn_SHIFT) +#define HDFGWTR_EL2_TRCCNTVRn_VAL(x) ((x) & HDFGWTR_EL2_TRCCNTVRn_MASK) +#define HDFGWTR_EL2_TRCCNTVRn_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCCNTVRn_SHIFT) +#define HDFGWTR_EL2_TRCCNTVRn_TRAP (UL(0x1) << HDFGWTR_EL2_TRCCNTVRn_SHIFT) +#define HDFGWTR_EL2_TRCCLAIM_SHIFT 36 +#define HDFGWTR_EL2_TRCCLAIM_MASK (UL(0x1) << HDFGWTR_EL2_TRCCLAIM_SHIFT) +#define HDFGWTR_EL2_TRCCLAIM_VAL(x) ((x) & HDFGWTR_EL2_TRCCLAIM_MASK) +#define HDFGWTR_EL2_TRCCLAIM_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCCLAIM_SHIFT) +#define HDFGWTR_EL2_TRCCLAIM_TRAP (UL(0x1) << HDFGWTR_EL2_TRCCLAIM_SHIFT) +#define HDFGWTR_EL2_TRCAUXCTLR_SHIFT 35 +#define HDFGWTR_EL2_TRCAUXCTLR_MASK (UL(0x1) << HDFGWTR_EL2_TRCAUXCTLR_SHIFT) +#define HDFGWTR_EL2_TRCAUXCTLR_VAL(x) ((x) & HDFGWTR_EL2_TRCAUXCTLR_MASK) +#define HDFGWTR_EL2_TRCAUXCTLR_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRCAUXCTLR_SHIFT) +#define HDFGWTR_EL2_TRCAUXCTLR_TRAP (UL(0x1) << HDFGWTR_EL2_TRCAUXCTLR_SHIFT) +#define HDFGWTR_EL2_TRC_SHIFT 33 +#define HDFGWTR_EL2_TRC_MASK (UL(0x1) << HDFGWTR_EL2_TRC_SHIFT) +#define HDFGWTR_EL2_TRC_VAL(x) ((x) & HDFGWTR_EL2_TRC_MASK) +#define HDFGWTR_EL2_TRC_NOTRAP (UL(0x0) << HDFGWTR_EL2_TRC_SHIFT) +#define HDFGWTR_EL2_TRC_TRAP (UL(0x1) << HDFGWTR_EL2_TRC_SHIFT) +#define HDFGWTR_EL2_PMSLATFR_EL1_SHIFT 32 +#define HDFGWTR_EL2_PMSLATFR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMSLATFR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSLATFR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMSLATFR_EL1_MASK) +#define HDFGWTR_EL2_PMSLATFR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMSLATFR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSLATFR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMSLATFR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSIRR_EL1_SHIFT 31 +#define HDFGWTR_EL2_PMSIRR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMSIRR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSIRR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMSIRR_EL1_MASK) +#define HDFGWTR_EL2_PMSIRR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMSIRR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSIRR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMSIRR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSICR_EL1_SHIFT 29 +#define HDFGWTR_EL2_PMSICR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMSICR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSICR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMSICR_EL1_MASK) +#define HDFGWTR_EL2_PMSICR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMSICR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSICR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMSICR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSFCR_EL1_SHIFT 28 +#define HDFGWTR_EL2_PMSFCR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMSFCR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSFCR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMSFCR_EL1_MASK) +#define HDFGWTR_EL2_PMSFCR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMSFCR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSFCR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMSFCR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSEVFR_EL1_SHIFT 27 +#define HDFGWTR_EL2_PMSEVFR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMSEVFR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSEVFR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMSEVFR_EL1_MASK) +#define HDFGWTR_EL2_PMSEVFR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMSEVFR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSEVFR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMSEVFR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSCR_EL1_SHIFT 26 +#define HDFGWTR_EL2_PMSCR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMSCR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSCR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMSCR_EL1_MASK) +#define HDFGWTR_EL2_PMSCR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMSCR_EL1_SHIFT) +#define HDFGWTR_EL2_PMSCR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMSCR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBSR_EL1_SHIFT 25 +#define HDFGWTR_EL2_PMBSR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMBSR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBSR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMBSR_EL1_MASK) +#define HDFGWTR_EL2_PMBSR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMBSR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBSR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMBSR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBPTR_EL1_SHIFT 24 +#define HDFGWTR_EL2_PMBPTR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMBPTR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBPTR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMBPTR_EL1_MASK) +#define HDFGWTR_EL2_PMBPTR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMBPTR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBPTR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMBPTR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBLIMITR_EL1_SHIFT 23 +#define HDFGWTR_EL2_PMBLIMITR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_PMBLIMITR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBLIMITR_EL1_VAL(x) ((x) & HDFGWTR_EL2_PMBLIMITR_EL1_MASK) +#define HDFGWTR_EL2_PMBLIMITR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMBLIMITR_EL1_SHIFT) +#define HDFGWTR_EL2_PMBLIMITR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_PMBLIMITR_EL1_SHIFT) +#define HDFGWTR_EL2_PMCR_EL0_SHIFT 21 +#define HDFGWTR_EL2_PMCR_EL0_MASK (UL(0x1) << HDFGWTR_EL2_PMCR_EL0_SHIFT) +#define HDFGWTR_EL2_PMCR_EL0_VAL(x) ((x) & HDFGWTR_EL2_PMCR_EL0_MASK) +#define HDFGWTR_EL2_PMCR_EL0_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMCR_EL0_SHIFT) +#define HDFGWTR_EL2_PMCR_EL0_TRAP (UL(0x1) << HDFGWTR_EL2_PMCR_EL0_SHIFT) +#define HDFGWTR_EL2_PMSWINC_EL0_SHIFT 20 +#define HDFGWTR_EL2_PMSWINC_EL0_MASK (UL(0x1) << HDFGWTR_EL2_PMSWINC_EL0_SHIFT) +#define HDFGWTR_EL2_PMSWINC_EL0_VAL(x) ((x) & HDFGWTR_EL2_PMSWINC_EL0_MASK) +#define HDFGWTR_EL2_PMSWINC_EL0_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMSWINC_EL0_SHIFT) +#define HDFGWTR_EL2_PMSWINC_EL0_TRAP (UL(0x1) << HDFGWTR_EL2_PMSWINC_EL0_SHIFT) +#define HDFGWTR_EL2_PMSELR_EL0_SHIFT 19 +#define HDFGWTR_EL2_PMSELR_EL0_MASK (UL(0x1) << HDFGWTR_EL2_PMSELR_EL0_SHIFT) +#define HDFGWTR_EL2_PMSELR_EL0_VAL(x) ((x) & HDFGWTR_EL2_PMSELR_EL0_MASK) +#define HDFGWTR_EL2_PMSELR_EL0_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMSELR_EL0_SHIFT) +#define HDFGWTR_EL2_PMSELR_EL0_TRAP (UL(0x1) << HDFGWTR_EL2_PMSELR_EL0_SHIFT) +#define HDFGWTR_EL2_PMOVS_SHIFT 18 +#define HDFGWTR_EL2_PMOVS_MASK (UL(0x1) << HDFGWTR_EL2_PMOVS_SHIFT) +#define HDFGWTR_EL2_PMOVS_VAL(x) ((x) & HDFGWTR_EL2_PMOVS_MASK) +#define HDFGWTR_EL2_PMOVS_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMOVS_SHIFT) +#define HDFGWTR_EL2_PMOVS_TRAP (UL(0x1) << HDFGWTR_EL2_PMOVS_SHIFT) +#define HDFGWTR_EL2_PMINTEN_SHIFT 17 +#define HDFGWTR_EL2_PMINTEN_MASK (UL(0x1) << HDFGWTR_EL2_PMINTEN_SHIFT) +#define HDFGWTR_EL2_PMINTEN_VAL(x) ((x) & HDFGWTR_EL2_PMINTEN_MASK) +#define HDFGWTR_EL2_PMINTEN_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMINTEN_SHIFT) +#define HDFGWTR_EL2_PMINTEN_TRAP (UL(0x1) << HDFGWTR_EL2_PMINTEN_SHIFT) +#define HDFGWTR_EL2_PMCNTEN_SHIFT 16 +#define HDFGWTR_EL2_PMCNTEN_MASK (UL(0x1) << HDFGWTR_EL2_PMCNTEN_SHIFT) +#define HDFGWTR_EL2_PMCNTEN_VAL(x) ((x) & HDFGWTR_EL2_PMCNTEN_MASK) +#define HDFGWTR_EL2_PMCNTEN_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMCNTEN_SHIFT) +#define HDFGWTR_EL2_PMCNTEN_TRAP (UL(0x1) << HDFGWTR_EL2_PMCNTEN_SHIFT) +#define HDFGWTR_EL2_PMCCNTR_EL0_SHIFT 15 +#define HDFGWTR_EL2_PMCCNTR_EL0_MASK (UL(0x1) << HDFGWTR_EL2_PMCCNTR_EL0_SHIFT) +#define HDFGWTR_EL2_PMCCNTR_EL0_VAL(x) ((x) & HDFGWTR_EL2_PMCCNTR_EL0_MASK) +#define HDFGWTR_EL2_PMCCNTR_EL0_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMCCNTR_EL0_SHIFT) +#define HDFGWTR_EL2_PMCCNTR_EL0_TRAP (UL(0x1) << HDFGWTR_EL2_PMCCNTR_EL0_SHIFT) +#define HDFGWTR_EL2_PMCCFILTR_EL0_SHIFT 14 +#define HDFGWTR_EL2_PMCCFILTR_EL0_MASK (UL(0x1) << HDFGWTR_EL2_PMCCFILTR_EL0_SHIFT) +#define HDFGWTR_EL2_PMCCFILTR_EL0_VAL(x) ((x) & HDFGWTR_EL2_PMCCFILTR_EL0_MASK) +#define HDFGWTR_EL2_PMCCFILTR_EL0_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMCCFILTR_EL0_SHIFT) +#define HDFGWTR_EL2_PMCCFILTR_EL0_TRAP (UL(0x1) << HDFGWTR_EL2_PMCCFILTR_EL0_SHIFT) +#define HDFGWTR_EL2_PMEVTYPERn_EL0_SHIFT 13 +#define HDFGWTR_EL2_PMEVTYPERn_EL0_MASK (UL(0x1) << HDFGWTR_EL2_PMEVTYPERn_EL0_SHIFT) +#define HDFGWTR_EL2_PMEVTYPERn_EL0_VAL(x) ((x) & HDFGWTR_EL2_PMEVTYPERn_EL0_MASK) +#define HDFGWTR_EL2_PMEVTYPERn_EL0_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMEVTYPERn_EL0_SHIFT) +#define HDFGWTR_EL2_PMEVTYPERn_EL0_TRAP (UL(0x1) << HDFGWTR_EL2_PMEVTYPERn_EL0_SHIFT) +#define HDFGWTR_EL2_PMEVCNTRn_EL0_SHIFT 12 +#define HDFGWTR_EL2_PMEVCNTRn_EL0_MASK (UL(0x1) << HDFGWTR_EL2_PMEVCNTRn_EL0_SHIFT) +#define HDFGWTR_EL2_PMEVCNTRn_EL0_VAL(x) ((x) & HDFGWTR_EL2_PMEVCNTRn_EL0_MASK) +#define HDFGWTR_EL2_PMEVCNTRn_EL0_NOTRAP (UL(0x0) << HDFGWTR_EL2_PMEVCNTRn_EL0_SHIFT) +#define HDFGWTR_EL2_PMEVCNTRn_EL0_TRAP (UL(0x1) << HDFGWTR_EL2_PMEVCNTRn_EL0_SHIFT) +#define HDFGWTR_EL2_OSDLR_EL1_SHIFT 11 +#define HDFGWTR_EL2_OSDLR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_OSDLR_EL1_SHIFT) +#define HDFGWTR_EL2_OSDLR_EL1_VAL(x) ((x) & HDFGWTR_EL2_OSDLR_EL1_MASK) +#define HDFGWTR_EL2_OSDLR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_OSDLR_EL1_SHIFT) +#define HDFGWTR_EL2_OSDLR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_OSDLR_EL1_SHIFT) +#define HDFGWTR_EL2_OSECCR_EL1_SHIFT 10 +#define HDFGWTR_EL2_OSECCR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_OSECCR_EL1_SHIFT) +#define HDFGWTR_EL2_OSECCR_EL1_VAL(x) ((x) & HDFGWTR_EL2_OSECCR_EL1_MASK) +#define HDFGWTR_EL2_OSECCR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_OSECCR_EL1_SHIFT) +#define HDFGWTR_EL2_OSECCR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_OSECCR_EL1_SHIFT) +#define HDFGWTR_EL2_OSLAR_EL1_SHIFT 8 +#define HDFGWTR_EL2_OSLAR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_OSLAR_EL1_SHIFT) +#define HDFGWTR_EL2_OSLAR_EL1_VAL(x) ((x) & HDFGWTR_EL2_OSLAR_EL1_MASK) +#define HDFGWTR_EL2_OSLAR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_OSLAR_EL1_SHIFT) +#define HDFGWTR_EL2_OSLAR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_OSLAR_EL1_SHIFT) +#define HDFGWTR_EL2_DBGPRCR_EL1_SHIFT 7 +#define HDFGWTR_EL2_DBGPRCR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_DBGPRCR_EL1_SHIFT) +#define HDFGWTR_EL2_DBGPRCR_EL1_VAL(x) ((x) & HDFGWTR_EL2_DBGPRCR_EL1_MASK) +#define HDFGWTR_EL2_DBGPRCR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_DBGPRCR_EL1_SHIFT) +#define HDFGWTR_EL2_DBGPRCR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_DBGPRCR_EL1_SHIFT) +#define HDFGWTR_EL2_DBGCLAIM_SHIFT 5 +#define HDFGWTR_EL2_DBGCLAIM_MASK (UL(0x1) << HDFGWTR_EL2_DBGCLAIM_SHIFT) +#define HDFGWTR_EL2_DBGCLAIM_VAL(x) ((x) & HDFGWTR_EL2_DBGCLAIM_MASK) +#define HDFGWTR_EL2_DBGCLAIM_NOTRAP (UL(0x0) << HDFGWTR_EL2_DBGCLAIM_SHIFT) +#define HDFGWTR_EL2_DBGCLAIM_TRAP (UL(0x1) << HDFGWTR_EL2_DBGCLAIM_SHIFT) +#define HDFGWTR_EL2_MDSCR_EL1_SHIFT 4 +#define HDFGWTR_EL2_MDSCR_EL1_MASK (UL(0x1) << HDFGWTR_EL2_MDSCR_EL1_SHIFT) +#define HDFGWTR_EL2_MDSCR_EL1_VAL(x) ((x) & HDFGWTR_EL2_MDSCR_EL1_MASK) +#define HDFGWTR_EL2_MDSCR_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_MDSCR_EL1_SHIFT) +#define HDFGWTR_EL2_MDSCR_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_MDSCR_EL1_SHIFT) +#define HDFGWTR_EL2_DBGWVRn_EL1_SHIFT 3 +#define HDFGWTR_EL2_DBGWVRn_EL1_MASK (UL(0x1) << HDFGWTR_EL2_DBGWVRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGWVRn_EL1_VAL(x) ((x) & HDFGWTR_EL2_DBGWVRn_EL1_MASK) +#define HDFGWTR_EL2_DBGWVRn_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_DBGWVRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGWVRn_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_DBGWVRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGWCRn_EL1_SHIFT 2 +#define HDFGWTR_EL2_DBGWCRn_EL1_MASK (UL(0x1) << HDFGWTR_EL2_DBGWCRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGWCRn_EL1_VAL(x) ((x) & HDFGWTR_EL2_DBGWCRn_EL1_MASK) +#define HDFGWTR_EL2_DBGWCRn_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_DBGWCRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGWCRn_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_DBGWCRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGBVRn_EL1_SHIFT 1 +#define HDFGWTR_EL2_DBGBVRn_EL1_MASK (UL(0x1) << HDFGWTR_EL2_DBGBVRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGBVRn_EL1_VAL(x) ((x) & HDFGWTR_EL2_DBGBVRn_EL1_MASK) +#define HDFGWTR_EL2_DBGBVRn_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_DBGBVRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGBVRn_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_DBGBVRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGBCRn_EL1_SHIFT 0 +#define HDFGWTR_EL2_DBGBCRn_EL1_MASK (UL(0x1) << HDFGWTR_EL2_DBGBCRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGBCRn_EL1_VAL(x) ((x) & HDFGWTR_EL2_DBGBCRn_EL1_MASK) +#define HDFGWTR_EL2_DBGBCRn_EL1_NOTRAP (UL(0x0) << HDFGWTR_EL2_DBGBCRn_EL1_SHIFT) +#define HDFGWTR_EL2_DBGBCRn_EL1_TRAP (UL(0x1) << HDFGWTR_EL2_DBGBCRn_EL1_SHIFT) + +/* HFGITR2_EL2 */ +#define HFGITR2_EL2_REG MRS_REG_ALT_NAME(HFGITR2_EL2) +#define HFGITR2_EL2_op0 3 +#define HFGITR2_EL2_op1 4 +#define HFGITR2_EL2_CRn 3 +#define HFGITR2_EL2_CRm 1 +#define HFGITR2_EL2_op2 7 + +/* HFGITR_EL2 */ +#define HFGITR_EL2_REG MRS_REG_ALT_NAME(HFGITR_EL2) +#define HFGITR_EL2_op0 3 +#define HFGITR_EL2_op1 4 +#define HFGITR_EL2_CRn 1 +#define HFGITR_EL2_CRm 1 +#define HFGITR_EL2_op2 6 +#define HFGITR_EL2_ATS1E1A_SHIFT 62 +#define HFGITR_EL2_ATS1E1A_MASK (UL(0x1) << HFGITR_EL2_ATS1E1A_SHIFT) +#define HFGITR_EL2_ATS1E1A_VAL(x) ((x) & HFGITR_EL2_ATS1E1A_MASK) +#define HFGITR_EL2_ATS1E1A_NOTRAP (UL(0x0) << HFGITR_EL2_ATS1E1A_SHIFT) +#define HFGITR_EL2_ATS1E1A_TRAP (UL(0x1) << HFGITR_EL2_ATS1E1A_SHIFT) +#define HFGITR_EL2_COSPRCTX_SHIFT 60 +#define HFGITR_EL2_COSPRCTX_MASK (UL(0x1) << HFGITR_EL2_COSPRCTX_SHIFT) +#define HFGITR_EL2_COSPRCTX_VAL(x) ((x) & HFGITR_EL2_COSPRCTX_MASK) +#define HFGITR_EL2_COSPRCTX_NOTRAP (UL(0x0) << HFGITR_EL2_COSPRCTX_SHIFT) +#define HFGITR_EL2_COSPRCTX_TRAP (UL(0x1) << HFGITR_EL2_COSPRCTX_SHIFT) +#define HFGITR_EL2_nGCSEPP_SHIFT 59 +#define HFGITR_EL2_nGCSEPP_MASK (UL(0x1) << HFGITR_EL2_nGCSEPP_SHIFT) +#define HFGITR_EL2_nGCSEPP_VAL(x) ((x) & HFGITR_EL2_nGCSEPP_MASK) +#define HFGITR_EL2_nGCSEPP_TRAP (UL(0x0) << HFGITR_EL2_nGCSEPP_SHIFT) +#define HFGITR_EL2_nGCSEPP_NOTRAP (UL(0x1) << HFGITR_EL2_nGCSEPP_SHIFT) +#define HFGITR_EL2_nGCSSTR_EL1_SHIFT 58 +#define HFGITR_EL2_nGCSSTR_EL1_MASK (UL(0x1) << HFGITR_EL2_nGCSSTR_EL1_SHIFT) +#define HFGITR_EL2_nGCSSTR_EL1_VAL(x) ((x) & HFGITR_EL2_nGCSSTR_EL1_MASK) +#define HFGITR_EL2_nGCSSTR_EL1_TRAP (UL(0x0) << HFGITR_EL2_nGCSSTR_EL1_SHIFT) +#define HFGITR_EL2_nGCSSTR_EL1_NOTRAP (UL(0x1) << HFGITR_EL2_nGCSSTR_EL1_SHIFT) +#define HFGITR_EL2_nGCSPUSHM_EL1_SHIFT 57 +#define HFGITR_EL2_nGCSPUSHM_EL1_MASK (UL(0x1) << HFGITR_EL2_nGCSPUSHM_EL1_SHIFT) +#define HFGITR_EL2_nGCSPUSHM_EL1_VAL(x) ((x) & HFGITR_EL2_nGCSPUSHM_EL1_MASK) +#define HFGITR_EL2_nGCSPUSHM_EL1_TRAP (UL(0x0) << HFGITR_EL2_nGCSPUSHM_EL1_SHIFT) +#define HFGITR_EL2_nGCSPUSHM_EL1_NOTRAP (UL(0x1) << HFGITR_EL2_nGCSPUSHM_EL1_SHIFT) +#define HFGITR_EL2_nBRBIALL_SHIFT 56 +#define HFGITR_EL2_nBRBIALL_MASK (UL(0x1) << HFGITR_EL2_nBRBIALL_SHIFT) +#define HFGITR_EL2_nBRBIALL_VAL(x) ((x) & HFGITR_EL2_nBRBIALL_MASK) +#define HFGITR_EL2_nBRBIALL_TRAP (UL(0x0) << HFGITR_EL2_nBRBIALL_SHIFT) +#define HFGITR_EL2_nBRBIALL_NOTRAP (UL(0x1) << HFGITR_EL2_nBRBIALL_SHIFT) +#define HFGITR_EL2_nBRBINJ_SHIFT 55 +#define HFGITR_EL2_nBRBINJ_MASK (UL(0x1) << HFGITR_EL2_nBRBINJ_SHIFT) +#define HFGITR_EL2_nBRBINJ_VAL(x) ((x) & HFGITR_EL2_nBRBINJ_MASK) +#define HFGITR_EL2_nBRBINJ_TRAP (UL(0x0) << HFGITR_EL2_nBRBINJ_SHIFT) +#define HFGITR_EL2_nBRBINJ_NOTRAP (UL(0x1) << HFGITR_EL2_nBRBINJ_SHIFT) +#define HFGITR_EL2_DCCVAC_SHIFT 54 +#define HFGITR_EL2_DCCVAC_MASK (UL(0x1) << HFGITR_EL2_DCCVAC_SHIFT) +#define HFGITR_EL2_DCCVAC_VAL(x) ((x) & HFGITR_EL2_DCCVAC_MASK) +#define HFGITR_EL2_DCCVAC_NOTRAP (UL(0x0) << HFGITR_EL2_DCCVAC_SHIFT) +#define HFGITR_EL2_DCCVAC_TRAP (UL(0x1) << HFGITR_EL2_DCCVAC_SHIFT) +#define HFGITR_EL2_SVC_EL1_SHIFT 53 +#define HFGITR_EL2_SVC_EL1_MASK (UL(0x1) << HFGITR_EL2_SVC_EL1_SHIFT) +#define HFGITR_EL2_SVC_EL1_VAL(x) ((x) & HFGITR_EL2_SVC_EL1_MASK) +#define HFGITR_EL2_SVC_EL1_NOTRAP (UL(0x0) << HFGITR_EL2_SVC_EL1_SHIFT) +#define HFGITR_EL2_SVC_EL1_TRAP (UL(0x1) << HFGITR_EL2_SVC_EL1_SHIFT) +#define HFGITR_EL2_SVC_EL0_SHIFT 52 +#define HFGITR_EL2_SVC_EL0_MASK (UL(0x1) << HFGITR_EL2_SVC_EL0_SHIFT) +#define HFGITR_EL2_SVC_EL0_VAL(x) ((x) & HFGITR_EL2_SVC_EL0_MASK) +#define HFGITR_EL2_SVC_EL0_NOTRAP (UL(0x0) << HFGITR_EL2_SVC_EL0_SHIFT) +#define HFGITR_EL2_SVC_EL0_TRAP (UL(0x1) << HFGITR_EL2_SVC_EL0_SHIFT) +#define HFGITR_EL2_ERET_SHIFT 51 +#define HFGITR_EL2_ERET_MASK (UL(0x1) << HFGITR_EL2_ERET_SHIFT) +#define HFGITR_EL2_ERET_VAL(x) ((x) & HFGITR_EL2_ERET_MASK) +#define HFGITR_EL2_ERET_NOTRAP (UL(0x0) << HFGITR_EL2_ERET_SHIFT) +#define HFGITR_EL2_ERET_TRAP (UL(0x1) << HFGITR_EL2_ERET_SHIFT) +#define HFGITR_EL2_CPPRCTX_SHIFT 50 +#define HFGITR_EL2_CPPRCTX_MASK (UL(0x1) << HFGITR_EL2_CPPRCTX_SHIFT) +#define HFGITR_EL2_CPPRCTX_VAL(x) ((x) & HFGITR_EL2_CPPRCTX_MASK) +#define HFGITR_EL2_CPPRCTX_NOTRAP (UL(0x0) << HFGITR_EL2_CPPRCTX_SHIFT) +#define HFGITR_EL2_CPPRCTX_TRAP (UL(0x1) << HFGITR_EL2_CPPRCTX_SHIFT) +#define HFGITR_EL2_DVPRCTX_SHIFT 49 +#define HFGITR_EL2_DVPRCTX_MASK (UL(0x1) << HFGITR_EL2_DVPRCTX_SHIFT) +#define HFGITR_EL2_DVPRCTX_VAL(x) ((x) & HFGITR_EL2_DVPRCTX_MASK) +#define HFGITR_EL2_DVPRCTX_NOTRAP (UL(0x0) << HFGITR_EL2_DVPRCTX_SHIFT) +#define HFGITR_EL2_DVPRCTX_TRAP (UL(0x1) << HFGITR_EL2_DVPRCTX_SHIFT) +#define HFGITR_EL2_CFPRCTX_SHIFT 48 +#define HFGITR_EL2_CFPRCTX_MASK (UL(0x1) << HFGITR_EL2_CFPRCTX_SHIFT) +#define HFGITR_EL2_CFPRCTX_VAL(x) ((x) & HFGITR_EL2_CFPRCTX_MASK) +#define HFGITR_EL2_CFPRCTX_NOTRAP (UL(0x0) << HFGITR_EL2_CFPRCTX_SHIFT) +#define HFGITR_EL2_CFPRCTX_TRAP (UL(0x1) << HFGITR_EL2_CFPRCTX_SHIFT) +#define HFGITR_EL2_TLBIVAALE1_SHIFT 47 +#define HFGITR_EL2_TLBIVAALE1_MASK (UL(0x1) << HFGITR_EL2_TLBIVAALE1_SHIFT) +#define HFGITR_EL2_TLBIVAALE1_VAL(x) ((x) & HFGITR_EL2_TLBIVAALE1_MASK) +#define HFGITR_EL2_TLBIVAALE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAALE1_SHIFT) +#define HFGITR_EL2_TLBIVAALE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAALE1_SHIFT) +#define HFGITR_EL2_TLBIVALE1_SHIFT 46 +#define HFGITR_EL2_TLBIVALE1_MASK (UL(0x1) << HFGITR_EL2_TLBIVALE1_SHIFT) +#define HFGITR_EL2_TLBIVALE1_VAL(x) ((x) & HFGITR_EL2_TLBIVALE1_MASK) +#define HFGITR_EL2_TLBIVALE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVALE1_SHIFT) +#define HFGITR_EL2_TLBIVALE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIVALE1_SHIFT) +#define HFGITR_EL2_TLBIVAAE1_SHIFT 45 +#define HFGITR_EL2_TLBIVAAE1_MASK (UL(0x1) << HFGITR_EL2_TLBIVAAE1_SHIFT) +#define HFGITR_EL2_TLBIVAAE1_VAL(x) ((x) & HFGITR_EL2_TLBIVAAE1_MASK) +#define HFGITR_EL2_TLBIVAAE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAAE1_SHIFT) +#define HFGITR_EL2_TLBIVAAE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAAE1_SHIFT) +#define HFGITR_EL2_TLBIASIDE1_SHIFT 44 +#define HFGITR_EL2_TLBIASIDE1_MASK (UL(0x1) << HFGITR_EL2_TLBIASIDE1_SHIFT) +#define HFGITR_EL2_TLBIASIDE1_VAL(x) ((x) & HFGITR_EL2_TLBIASIDE1_MASK) +#define HFGITR_EL2_TLBIASIDE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIASIDE1_SHIFT) +#define HFGITR_EL2_TLBIASIDE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIASIDE1_SHIFT) +#define HFGITR_EL2_TLBIVAE1_SHIFT 43 +#define HFGITR_EL2_TLBIVAE1_MASK (UL(0x1) << HFGITR_EL2_TLBIVAE1_SHIFT) +#define HFGITR_EL2_TLBIVAE1_VAL(x) ((x) & HFGITR_EL2_TLBIVAE1_MASK) +#define HFGITR_EL2_TLBIVAE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAE1_SHIFT) +#define HFGITR_EL2_TLBIVAE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAE1_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1_SHIFT 42 +#define HFGITR_EL2_TLBIVMALLE1_MASK (UL(0x1) << HFGITR_EL2_TLBIVMALLE1_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1_VAL(x) ((x) & HFGITR_EL2_TLBIVMALLE1_MASK) +#define HFGITR_EL2_TLBIVMALLE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVMALLE1_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIVMALLE1_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1_SHIFT 41 +#define HFGITR_EL2_TLBIRVAALE1_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAALE1_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1_VAL(x) ((x) & HFGITR_EL2_TLBIRVAALE1_MASK) +#define HFGITR_EL2_TLBIRVAALE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAALE1_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAALE1_SHIFT) +#define HFGITR_EL2_TLBIRVALE1_SHIFT 40 +#define HFGITR_EL2_TLBIRVALE1_MASK (UL(0x1) << HFGITR_EL2_TLBIRVALE1_SHIFT) +#define HFGITR_EL2_TLBIRVALE1_VAL(x) ((x) & HFGITR_EL2_TLBIRVALE1_MASK) +#define HFGITR_EL2_TLBIRVALE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVALE1_SHIFT) +#define HFGITR_EL2_TLBIRVALE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVALE1_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1_SHIFT 39 +#define HFGITR_EL2_TLBIRVAAE1_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAAE1_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1_VAL(x) ((x) & HFGITR_EL2_TLBIRVAAE1_MASK) +#define HFGITR_EL2_TLBIRVAAE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAAE1_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAAE1_SHIFT) +#define HFGITR_EL2_TLBIRVAE1_SHIFT 38 +#define HFGITR_EL2_TLBIRVAE1_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAE1_SHIFT) +#define HFGITR_EL2_TLBIRVAE1_VAL(x) ((x) & HFGITR_EL2_TLBIRVAE1_MASK) +#define HFGITR_EL2_TLBIRVAE1_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAE1_SHIFT) +#define HFGITR_EL2_TLBIRVAE1_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAE1_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1IS_SHIFT 37 +#define HFGITR_EL2_TLBIRVAALE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAALE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIRVAALE1IS_MASK) +#define HFGITR_EL2_TLBIRVAALE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAALE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAALE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVALE1IS_SHIFT 36 +#define HFGITR_EL2_TLBIRVALE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIRVALE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVALE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIRVALE1IS_MASK) +#define HFGITR_EL2_TLBIRVALE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVALE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVALE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVALE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1IS_SHIFT 35 +#define HFGITR_EL2_TLBIRVAAE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAAE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIRVAAE1IS_MASK) +#define HFGITR_EL2_TLBIRVAAE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAAE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAAE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAE1IS_SHIFT 34 +#define HFGITR_EL2_TLBIRVAE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIRVAE1IS_MASK) +#define HFGITR_EL2_TLBIRVAE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAALE1IS_SHIFT 33 +#define HFGITR_EL2_TLBIVAALE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIVAALE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAALE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIVAALE1IS_MASK) +#define HFGITR_EL2_TLBIVAALE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAALE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAALE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAALE1IS_SHIFT) +#define HFGITR_EL2_TLBIVALE1IS_SHIFT 32 +#define HFGITR_EL2_TLBIVALE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIVALE1IS_SHIFT) +#define HFGITR_EL2_TLBIVALE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIVALE1IS_MASK) +#define HFGITR_EL2_TLBIVALE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVALE1IS_SHIFT) +#define HFGITR_EL2_TLBIVALE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVALE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAAE1IS_SHIFT 31 +#define HFGITR_EL2_TLBIVAAE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIVAAE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAAE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIVAAE1IS_MASK) +#define HFGITR_EL2_TLBIVAAE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAAE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAAE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAAE1IS_SHIFT) +#define HFGITR_EL2_TLBIASIDE1IS_SHIFT 30 +#define HFGITR_EL2_TLBIASIDE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIASIDE1IS_SHIFT) +#define HFGITR_EL2_TLBIASIDE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIASIDE1IS_MASK) +#define HFGITR_EL2_TLBIASIDE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIASIDE1IS_SHIFT) +#define HFGITR_EL2_TLBIASIDE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIASIDE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAE1IS_SHIFT 29 +#define HFGITR_EL2_TLBIVAE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIVAE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIVAE1IS_MASK) +#define HFGITR_EL2_TLBIVAE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAE1IS_SHIFT) +#define HFGITR_EL2_TLBIVAE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAE1IS_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1IS_SHIFT 28 +#define HFGITR_EL2_TLBIVMALLE1IS_MASK (UL(0x1) << HFGITR_EL2_TLBIVMALLE1IS_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1IS_VAL(x) ((x) & HFGITR_EL2_TLBIVMALLE1IS_MASK) +#define HFGITR_EL2_TLBIVMALLE1IS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVMALLE1IS_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1IS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVMALLE1IS_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1OS_SHIFT 27 +#define HFGITR_EL2_TLBIRVAALE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAALE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIRVAALE1OS_MASK) +#define HFGITR_EL2_TLBIRVAALE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAALE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVAALE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAALE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVALE1OS_SHIFT 26 +#define HFGITR_EL2_TLBIRVALE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIRVALE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVALE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIRVALE1OS_MASK) +#define HFGITR_EL2_TLBIRVALE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVALE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVALE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVALE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1OS_SHIFT 25 +#define HFGITR_EL2_TLBIRVAAE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAAE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIRVAAE1OS_MASK) +#define HFGITR_EL2_TLBIRVAAE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAAE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVAAE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAAE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVAE1OS_SHIFT 24 +#define HFGITR_EL2_TLBIRVAE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIRVAE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVAE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIRVAE1OS_MASK) +#define HFGITR_EL2_TLBIRVAE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIRVAE1OS_SHIFT) +#define HFGITR_EL2_TLBIRVAE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIRVAE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAALE1OS_SHIFT 23 +#define HFGITR_EL2_TLBIVAALE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIVAALE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAALE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIVAALE1OS_MASK) +#define HFGITR_EL2_TLBIVAALE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAALE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAALE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAALE1OS_SHIFT) +#define HFGITR_EL2_TLBIVALE1OS_SHIFT 22 +#define HFGITR_EL2_TLBIVALE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIVALE1OS_SHIFT) +#define HFGITR_EL2_TLBIVALE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIVALE1OS_MASK) +#define HFGITR_EL2_TLBIVALE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVALE1OS_SHIFT) +#define HFGITR_EL2_TLBIVALE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVALE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAAE1OS_SHIFT 21 +#define HFGITR_EL2_TLBIVAAE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIVAAE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAAE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIVAAE1OS_MASK) +#define HFGITR_EL2_TLBIVAAE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAAE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAAE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAAE1OS_SHIFT) +#define HFGITR_EL2_TLBIASIDE1OS_SHIFT 20 +#define HFGITR_EL2_TLBIASIDE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIASIDE1OS_SHIFT) +#define HFGITR_EL2_TLBIASIDE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIASIDE1OS_MASK) +#define HFGITR_EL2_TLBIASIDE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIASIDE1OS_SHIFT) +#define HFGITR_EL2_TLBIASIDE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIASIDE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAE1OS_SHIFT 19 +#define HFGITR_EL2_TLBIVAE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIVAE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIVAE1OS_MASK) +#define HFGITR_EL2_TLBIVAE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVAE1OS_SHIFT) +#define HFGITR_EL2_TLBIVAE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVAE1OS_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1OS_SHIFT 18 +#define HFGITR_EL2_TLBIVMALLE1OS_MASK (UL(0x1) << HFGITR_EL2_TLBIVMALLE1OS_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1OS_VAL(x) ((x) & HFGITR_EL2_TLBIVMALLE1OS_MASK) +#define HFGITR_EL2_TLBIVMALLE1OS_NOTRAP (UL(0x0) << HFGITR_EL2_TLBIVMALLE1OS_SHIFT) +#define HFGITR_EL2_TLBIVMALLE1OS_TRAP (UL(0x1) << HFGITR_EL2_TLBIVMALLE1OS_SHIFT) +#define HFGITR_EL2_ATS1E1WP_SHIFT 17 +#define HFGITR_EL2_ATS1E1WP_MASK (UL(0x1) << HFGITR_EL2_ATS1E1WP_SHIFT) +#define HFGITR_EL2_ATS1E1WP_VAL(x) ((x) & HFGITR_EL2_ATS1E1WP_MASK) +#define HFGITR_EL2_ATS1E1WP_NOTRAP (UL(0x0) << HFGITR_EL2_ATS1E1WP_SHIFT) +#define HFGITR_EL2_ATS1E1WP_TRAP (UL(0x1) << HFGITR_EL2_ATS1E1WP_SHIFT) +#define HFGITR_EL2_ATS1E1RP_SHIFT 16 +#define HFGITR_EL2_ATS1E1RP_MASK (UL(0x1) << HFGITR_EL2_ATS1E1RP_SHIFT) +#define HFGITR_EL2_ATS1E1RP_VAL(x) ((x) & HFGITR_EL2_ATS1E1RP_MASK) +#define HFGITR_EL2_ATS1E1RP_NOTRAP (UL(0x0) << HFGITR_EL2_ATS1E1RP_SHIFT) +#define HFGITR_EL2_ATS1E1RP_TRAP (UL(0x1) << HFGITR_EL2_ATS1E1RP_SHIFT) +#define HFGITR_EL2_ATS1E0W_SHIFT 15 +#define HFGITR_EL2_ATS1E0W_MASK (UL(0x1) << HFGITR_EL2_ATS1E0W_SHIFT) +#define HFGITR_EL2_ATS1E0W_VAL(x) ((x) & HFGITR_EL2_ATS1E0W_MASK) +#define HFGITR_EL2_ATS1E0W_NOTRAP (UL(0x0) << HFGITR_EL2_ATS1E0W_SHIFT) +#define HFGITR_EL2_ATS1E0W_TRAP (UL(0x1) << HFGITR_EL2_ATS1E0W_SHIFT) +#define HFGITR_EL2_ATS1E0R_SHIFT 14 +#define HFGITR_EL2_ATS1E0R_MASK (UL(0x1) << HFGITR_EL2_ATS1E0R_SHIFT) +#define HFGITR_EL2_ATS1E0R_VAL(x) ((x) & HFGITR_EL2_ATS1E0R_MASK) +#define HFGITR_EL2_ATS1E0R_NOTRAP (UL(0x0) << HFGITR_EL2_ATS1E0R_SHIFT) +#define HFGITR_EL2_ATS1E0R_TRAP (UL(0x1) << HFGITR_EL2_ATS1E0R_SHIFT) +#define HFGITR_EL2_ATS1E1W_SHIFT 13 +#define HFGITR_EL2_ATS1E1W_MASK (UL(0x1) << HFGITR_EL2_ATS1E1W_SHIFT) +#define HFGITR_EL2_ATS1E1W_VAL(x) ((x) & HFGITR_EL2_ATS1E1W_MASK) +#define HFGITR_EL2_ATS1E1W_NOTRAP (UL(0x0) << HFGITR_EL2_ATS1E1W_SHIFT) +#define HFGITR_EL2_ATS1E1W_TRAP (UL(0x1) << HFGITR_EL2_ATS1E1W_SHIFT) +#define HFGITR_EL2_ATS1E1R_SHIFT 12 +#define HFGITR_EL2_ATS1E1R_MASK (UL(0x1) << HFGITR_EL2_ATS1E1R_SHIFT) +#define HFGITR_EL2_ATS1E1R_VAL(x) ((x) & HFGITR_EL2_ATS1E1R_MASK) +#define HFGITR_EL2_ATS1E1R_NOTRAP (UL(0x0) << HFGITR_EL2_ATS1E1R_SHIFT) +#define HFGITR_EL2_ATS1E1R_TRAP (UL(0x1) << HFGITR_EL2_ATS1E1R_SHIFT) +#define HFGITR_EL2_DCZVA_SHIFT 11 +#define HFGITR_EL2_DCZVA_MASK (UL(0x1) << HFGITR_EL2_DCZVA_SHIFT) +#define HFGITR_EL2_DCZVA_VAL(x) ((x) & HFGITR_EL2_DCZVA_MASK) +#define HFGITR_EL2_DCZVA_NOTRAP (UL(0x0) << HFGITR_EL2_DCZVA_SHIFT) +#define HFGITR_EL2_DCZVA_TRAP (UL(0x1) << HFGITR_EL2_DCZVA_SHIFT) +#define HFGITR_EL2_DCCIVAC_SHIFT 10 +#define HFGITR_EL2_DCCIVAC_MASK (UL(0x1) << HFGITR_EL2_DCCIVAC_SHIFT) +#define HFGITR_EL2_DCCIVAC_VAL(x) ((x) & HFGITR_EL2_DCCIVAC_MASK) +#define HFGITR_EL2_DCCIVAC_NOTRAP (UL(0x0) << HFGITR_EL2_DCCIVAC_SHIFT) +#define HFGITR_EL2_DCCIVAC_TRAP (UL(0x1) << HFGITR_EL2_DCCIVAC_SHIFT) +#define HFGITR_EL2_DCCVADP_SHIFT 9 +#define HFGITR_EL2_DCCVADP_MASK (UL(0x1) << HFGITR_EL2_DCCVADP_SHIFT) +#define HFGITR_EL2_DCCVADP_VAL(x) ((x) & HFGITR_EL2_DCCVADP_MASK) +#define HFGITR_EL2_DCCVADP_NOTRAP (UL(0x0) << HFGITR_EL2_DCCVADP_SHIFT) +#define HFGITR_EL2_DCCVADP_TRAP (UL(0x1) << HFGITR_EL2_DCCVADP_SHIFT) +#define HFGITR_EL2_DCCVAP_SHIFT 8 +#define HFGITR_EL2_DCCVAP_MASK (UL(0x1) << HFGITR_EL2_DCCVAP_SHIFT) +#define HFGITR_EL2_DCCVAP_VAL(x) ((x) & HFGITR_EL2_DCCVAP_MASK) +#define HFGITR_EL2_DCCVAP_NOTRAP (UL(0x0) << HFGITR_EL2_DCCVAP_SHIFT) +#define HFGITR_EL2_DCCVAP_TRAP (UL(0x1) << HFGITR_EL2_DCCVAP_SHIFT) +#define HFGITR_EL2_DCCVAU_SHIFT 7 +#define HFGITR_EL2_DCCVAU_MASK (UL(0x1) << HFGITR_EL2_DCCVAU_SHIFT) +#define HFGITR_EL2_DCCVAU_VAL(x) ((x) & HFGITR_EL2_DCCVAU_MASK) +#define HFGITR_EL2_DCCVAU_NOTRAP (UL(0x0) << HFGITR_EL2_DCCVAU_SHIFT) +#define HFGITR_EL2_DCCVAU_TRAP (UL(0x1) << HFGITR_EL2_DCCVAU_SHIFT) +#define HFGITR_EL2_DCCISW_SHIFT 6 +#define HFGITR_EL2_DCCISW_MASK (UL(0x1) << HFGITR_EL2_DCCISW_SHIFT) +#define HFGITR_EL2_DCCISW_VAL(x) ((x) & HFGITR_EL2_DCCISW_MASK) +#define HFGITR_EL2_DCCISW_NOTRAP (UL(0x0) << HFGITR_EL2_DCCISW_SHIFT) +#define HFGITR_EL2_DCCISW_TRAP (UL(0x1) << HFGITR_EL2_DCCISW_SHIFT) +#define HFGITR_EL2_DCCSW_SHIFT 5 +#define HFGITR_EL2_DCCSW_MASK (UL(0x1) << HFGITR_EL2_DCCSW_SHIFT) +#define HFGITR_EL2_DCCSW_VAL(x) ((x) & HFGITR_EL2_DCCSW_MASK) +#define HFGITR_EL2_DCCSW_NOTRAP (UL(0x0) << HFGITR_EL2_DCCSW_SHIFT) +#define HFGITR_EL2_DCCSW_TRAP (UL(0x1) << HFGITR_EL2_DCCSW_SHIFT) +#define HFGITR_EL2_DCISW_SHIFT 4 +#define HFGITR_EL2_DCISW_MASK (UL(0x1) << HFGITR_EL2_DCISW_SHIFT) +#define HFGITR_EL2_DCISW_VAL(x) ((x) & HFGITR_EL2_DCISW_MASK) +#define HFGITR_EL2_DCISW_NOTRAP (UL(0x0) << HFGITR_EL2_DCISW_SHIFT) +#define HFGITR_EL2_DCISW_TRAP (UL(0x1) << HFGITR_EL2_DCISW_SHIFT) +#define HFGITR_EL2_DCIVAC_SHIFT 3 +#define HFGITR_EL2_DCIVAC_MASK (UL(0x1) << HFGITR_EL2_DCIVAC_SHIFT) +#define HFGITR_EL2_DCIVAC_VAL(x) ((x) & HFGITR_EL2_DCIVAC_MASK) +#define HFGITR_EL2_DCIVAC_NOTRAP (UL(0x0) << HFGITR_EL2_DCIVAC_SHIFT) +#define HFGITR_EL2_DCIVAC_TRAP (UL(0x1) << HFGITR_EL2_DCIVAC_SHIFT) +#define HFGITR_EL2_ICIVAU_SHIFT 2 +#define HFGITR_EL2_ICIVAU_MASK (UL(0x1) << HFGITR_EL2_ICIVAU_SHIFT) +#define HFGITR_EL2_ICIVAU_VAL(x) ((x) & HFGITR_EL2_ICIVAU_MASK) +#define HFGITR_EL2_ICIVAU_NOTRAP (UL(0x0) << HFGITR_EL2_ICIVAU_SHIFT) +#define HFGITR_EL2_ICIVAU_TRAP (UL(0x1) << HFGITR_EL2_ICIVAU_SHIFT) +#define HFGITR_EL2_ICIALLU_SHIFT 1 +#define HFGITR_EL2_ICIALLU_MASK (UL(0x1) << HFGITR_EL2_ICIALLU_SHIFT) +#define HFGITR_EL2_ICIALLU_VAL(x) ((x) & HFGITR_EL2_ICIALLU_MASK) +#define HFGITR_EL2_ICIALLU_NOTRAP (UL(0x0) << HFGITR_EL2_ICIALLU_SHIFT) +#define HFGITR_EL2_ICIALLU_TRAP (UL(0x1) << HFGITR_EL2_ICIALLU_SHIFT) +#define HFGITR_EL2_ICIALLUIS_SHIFT 0 +#define HFGITR_EL2_ICIALLUIS_MASK (UL(0x1) << HFGITR_EL2_ICIALLUIS_SHIFT) +#define HFGITR_EL2_ICIALLUIS_VAL(x) ((x) & HFGITR_EL2_ICIALLUIS_MASK) +#define HFGITR_EL2_ICIALLUIS_NOTRAP (UL(0x0) << HFGITR_EL2_ICIALLUIS_SHIFT) +#define HFGITR_EL2_ICIALLUIS_TRAP (UL(0x1) << HFGITR_EL2_ICIALLUIS_SHIFT) + +/* HFGRTR2_EL2 */ +#define HFGRTR2_EL2_REG MRS_REG_ALT_NAME(HFGRTR2_EL2) +#define HFGRTR2_EL2_op0 3 +#define HFGRTR2_EL2_op1 4 +#define HFGRTR2_EL2_CRn 3 +#define HFGRTR2_EL2_CRm 1 +#define HFGRTR2_EL2_op2 2 +#define HFGRTR2_EL2_nRCWSMASK_EL1_SHIFT 2 +#define HFGRTR2_EL2_nRCWSMASK_EL1_MASK (UL(0x1) << HFGRTR2_EL2_nRCWSMASK_EL1_SHIFT) +#define HFGRTR2_EL2_nRCWSMASK_EL1_VAL(x) ((x) & HFGRTR2_EL2_nRCWSMASK_EL1_MASK) +#define HFGRTR2_EL2_nRCWSMASK_EL1_TRAP (UL(0x0) << HFGRTR2_EL2_nRCWSMASK_EL1_SHIFT) +#define HFGRTR2_EL2_nRCWSMASK_EL1_NOTRAP (UL(0x1) << HFGRTR2_EL2_nRCWSMASK_EL1_SHIFT) +#define HFGRTR2_EL2_nERXGSR_EL1_SHIFT 1 +#define HFGRTR2_EL2_nERXGSR_EL1_MASK (UL(0x1) << HFGRTR2_EL2_nERXGSR_EL1_SHIFT) +#define HFGRTR2_EL2_nERXGSR_EL1_VAL(x) ((x) & HFGRTR2_EL2_nERXGSR_EL1_MASK) +#define HFGRTR2_EL2_nERXGSR_EL1_TRAP (UL(0x0) << HFGRTR2_EL2_nERXGSR_EL1_SHIFT) +#define HFGRTR2_EL2_nERXGSR_EL1_NOTRAP (UL(0x1) << HFGRTR2_EL2_nERXGSR_EL1_SHIFT) +#define HFGRTR2_EL2_nPFAR_EL1_SHIFT 0 +#define HFGRTR2_EL2_nPFAR_EL1_MASK (UL(0x1) << HFGRTR2_EL2_nPFAR_EL1_SHIFT) +#define HFGRTR2_EL2_nPFAR_EL1_VAL(x) ((x) & HFGRTR2_EL2_nPFAR_EL1_MASK) +#define HFGRTR2_EL2_nPFAR_EL1_TRAP (UL(0x0) << HFGRTR2_EL2_nPFAR_EL1_SHIFT) +#define HFGRTR2_EL2_nPFAR_EL1_NOTRAP (UL(0x1) << HFGRTR2_EL2_nPFAR_EL1_SHIFT) + +/* HFGRTR_EL2 */ +#define HFGRTR_EL2_REG MRS_REG_ALT_NAME(HFGRTR_EL2) +#define HFGRTR_EL2_op0 3 +#define HFGRTR_EL2_op1 4 +#define HFGRTR_EL2_CRn 1 +#define HFGRTR_EL2_CRm 1 +#define HFGRTR_EL2_op2 4 +#define HFGRTR_EL2_nAMAIR2_EL1_SHIFT 63 +#define HFGRTR_EL2_nAMAIR2_EL1_MASK (UL(0x1) << HFGRTR_EL2_nAMAIR2_EL1_SHIFT) +#define HFGRTR_EL2_nAMAIR2_EL1_VAL(x) ((x) & HFGRTR_EL2_nAMAIR2_EL1_MASK) +#define HFGRTR_EL2_nAMAIR2_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nAMAIR2_EL1_SHIFT) +#define HFGRTR_EL2_nAMAIR2_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nAMAIR2_EL1_SHIFT) +#define HFGRTR_EL2_nMAIR2_EL1_SHIFT 62 +#define HFGRTR_EL2_nMAIR2_EL1_MASK (UL(0x1) << HFGRTR_EL2_nMAIR2_EL1_SHIFT) +#define HFGRTR_EL2_nMAIR2_EL1_VAL(x) ((x) & HFGRTR_EL2_nMAIR2_EL1_MASK) +#define HFGRTR_EL2_nMAIR2_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nMAIR2_EL1_SHIFT) +#define HFGRTR_EL2_nMAIR2_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nMAIR2_EL1_SHIFT) +#define HFGRTR_EL2_nS2POR_EL1_SHIFT 61 +#define HFGRTR_EL2_nS2POR_EL1_MASK (UL(0x1) << HFGRTR_EL2_nS2POR_EL1_SHIFT) +#define HFGRTR_EL2_nS2POR_EL1_VAL(x) ((x) & HFGRTR_EL2_nS2POR_EL1_MASK) +#define HFGRTR_EL2_nS2POR_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nS2POR_EL1_SHIFT) +#define HFGRTR_EL2_nS2POR_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nS2POR_EL1_SHIFT) +#define HFGRTR_EL2_nPOR_EL1_SHIFT 60 +#define HFGRTR_EL2_nPOR_EL1_MASK (UL(0x1) << HFGRTR_EL2_nPOR_EL1_SHIFT) +#define HFGRTR_EL2_nPOR_EL1_VAL(x) ((x) & HFGRTR_EL2_nPOR_EL1_MASK) +#define HFGRTR_EL2_nPOR_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nPOR_EL1_SHIFT) +#define HFGRTR_EL2_nPOR_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nPOR_EL1_SHIFT) +#define HFGRTR_EL2_nPOR_EL0_SHIFT 59 +#define HFGRTR_EL2_nPOR_EL0_MASK (UL(0x1) << HFGRTR_EL2_nPOR_EL0_SHIFT) +#define HFGRTR_EL2_nPOR_EL0_VAL(x) ((x) & HFGRTR_EL2_nPOR_EL0_MASK) +#define HFGRTR_EL2_nPOR_EL0_TRAP (UL(0x0) << HFGRTR_EL2_nPOR_EL0_SHIFT) +#define HFGRTR_EL2_nPOR_EL0_NOTRAP (UL(0x1) << HFGRTR_EL2_nPOR_EL0_SHIFT) +#define HFGRTR_EL2_nPIR_EL1_SHIFT 58 +#define HFGRTR_EL2_nPIR_EL1_MASK (UL(0x1) << HFGRTR_EL2_nPIR_EL1_SHIFT) +#define HFGRTR_EL2_nPIR_EL1_VAL(x) ((x) & HFGRTR_EL2_nPIR_EL1_MASK) +#define HFGRTR_EL2_nPIR_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nPIR_EL1_SHIFT) +#define HFGRTR_EL2_nPIR_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nPIR_EL1_SHIFT) +#define HFGRTR_EL2_nPIRE0_EL1_SHIFT 57 +#define HFGRTR_EL2_nPIRE0_EL1_MASK (UL(0x1) << HFGRTR_EL2_nPIRE0_EL1_SHIFT) +#define HFGRTR_EL2_nPIRE0_EL1_VAL(x) ((x) & HFGRTR_EL2_nPIRE0_EL1_MASK) +#define HFGRTR_EL2_nPIRE0_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nPIRE0_EL1_SHIFT) +#define HFGRTR_EL2_nPIRE0_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nPIRE0_EL1_SHIFT) +#define HFGRTR_EL2_nRCWMASK_EL1_SHIFT 56 +#define HFGRTR_EL2_nRCWMASK_EL1_MASK (UL(0x1) << HFGRTR_EL2_nRCWMASK_EL1_SHIFT) +#define HFGRTR_EL2_nRCWMASK_EL1_VAL(x) ((x) & HFGRTR_EL2_nRCWMASK_EL1_MASK) +#define HFGRTR_EL2_nRCWMASK_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nRCWMASK_EL1_SHIFT) +#define HFGRTR_EL2_nRCWMASK_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nRCWMASK_EL1_SHIFT) +#define HFGRTR_EL2_nTPIDR2_EL0_SHIFT 55 +#define HFGRTR_EL2_nTPIDR2_EL0_MASK (UL(0x1) << HFGRTR_EL2_nTPIDR2_EL0_SHIFT) +#define HFGRTR_EL2_nTPIDR2_EL0_VAL(x) ((x) & HFGRTR_EL2_nTPIDR2_EL0_MASK) +#define HFGRTR_EL2_nTPIDR2_EL0_TRAP (UL(0x0) << HFGRTR_EL2_nTPIDR2_EL0_SHIFT) +#define HFGRTR_EL2_nTPIDR2_EL0_NOTRAP (UL(0x1) << HFGRTR_EL2_nTPIDR2_EL0_SHIFT) +#define HFGRTR_EL2_nSMPRI_EL1_SHIFT 54 +#define HFGRTR_EL2_nSMPRI_EL1_MASK (UL(0x1) << HFGRTR_EL2_nSMPRI_EL1_SHIFT) +#define HFGRTR_EL2_nSMPRI_EL1_VAL(x) ((x) & HFGRTR_EL2_nSMPRI_EL1_MASK) +#define HFGRTR_EL2_nSMPRI_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nSMPRI_EL1_SHIFT) +#define HFGRTR_EL2_nSMPRI_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nSMPRI_EL1_SHIFT) +#define HFGRTR_EL2_nGCS_EL1_SHIFT 53 +#define HFGRTR_EL2_nGCS_EL1_MASK (UL(0x1) << HFGRTR_EL2_nGCS_EL1_SHIFT) +#define HFGRTR_EL2_nGCS_EL1_VAL(x) ((x) & HFGRTR_EL2_nGCS_EL1_MASK) +#define HFGRTR_EL2_nGCS_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nGCS_EL1_SHIFT) +#define HFGRTR_EL2_nGCS_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nGCS_EL1_SHIFT) +#define HFGRTR_EL2_nGCS_EL0_SHIFT 52 +#define HFGRTR_EL2_nGCS_EL0_MASK (UL(0x1) << HFGRTR_EL2_nGCS_EL0_SHIFT) +#define HFGRTR_EL2_nGCS_EL0_VAL(x) ((x) & HFGRTR_EL2_nGCS_EL0_MASK) +#define HFGRTR_EL2_nGCS_EL0_TRAP (UL(0x0) << HFGRTR_EL2_nGCS_EL0_SHIFT) +#define HFGRTR_EL2_nGCS_EL0_NOTRAP (UL(0x1) << HFGRTR_EL2_nGCS_EL0_SHIFT) +#define HFGRTR_EL2_nACCDATA_EL1_SHIFT 50 +#define HFGRTR_EL2_nACCDATA_EL1_MASK (UL(0x1) << HFGRTR_EL2_nACCDATA_EL1_SHIFT) +#define HFGRTR_EL2_nACCDATA_EL1_VAL(x) ((x) & HFGRTR_EL2_nACCDATA_EL1_MASK) +#define HFGRTR_EL2_nACCDATA_EL1_TRAP (UL(0x0) << HFGRTR_EL2_nACCDATA_EL1_SHIFT) +#define HFGRTR_EL2_nACCDATA_EL1_NOTRAP (UL(0x1) << HFGRTR_EL2_nACCDATA_EL1_SHIFT) +#define HFGRTR_EL2_ERXADDR_EL1_SHIFT 49 +#define HFGRTR_EL2_ERXADDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERXADDR_EL1_SHIFT) +#define HFGRTR_EL2_ERXADDR_EL1_VAL(x) ((x) & HFGRTR_EL2_ERXADDR_EL1_MASK) +#define HFGRTR_EL2_ERXADDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERXADDR_EL1_SHIFT) +#define HFGRTR_EL2_ERXADDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERXADDR_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGCDN_EL1_SHIFT 48 +#define HFGRTR_EL2_ERXPFGCDN_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERXPFGCDN_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGCDN_EL1_VAL(x) ((x) & HFGRTR_EL2_ERXPFGCDN_EL1_MASK) +#define HFGRTR_EL2_ERXPFGCDN_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERXPFGCDN_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGCDN_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERXPFGCDN_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGCTL_EL1_SHIFT 47 +#define HFGRTR_EL2_ERXPFGCTL_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERXPFGCTL_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGCTL_EL1_VAL(x) ((x) & HFGRTR_EL2_ERXPFGCTL_EL1_MASK) +#define HFGRTR_EL2_ERXPFGCTL_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERXPFGCTL_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGCTL_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERXPFGCTL_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGF_EL1_SHIFT 46 +#define HFGRTR_EL2_ERXPFGF_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERXPFGF_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGF_EL1_VAL(x) ((x) & HFGRTR_EL2_ERXPFGF_EL1_MASK) +#define HFGRTR_EL2_ERXPFGF_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERXPFGF_EL1_SHIFT) +#define HFGRTR_EL2_ERXPFGF_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERXPFGF_EL1_SHIFT) +#define HFGRTR_EL2_ERXMISCn_EL1_SHIFT 45 +#define HFGRTR_EL2_ERXMISCn_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERXMISCn_EL1_SHIFT) +#define HFGRTR_EL2_ERXMISCn_EL1_VAL(x) ((x) & HFGRTR_EL2_ERXMISCn_EL1_MASK) +#define HFGRTR_EL2_ERXMISCn_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERXMISCn_EL1_SHIFT) +#define HFGRTR_EL2_ERXMISCn_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERXMISCn_EL1_SHIFT) +#define HFGRTR_EL2_ERXSTATUS_EL1_SHIFT 44 +#define HFGRTR_EL2_ERXSTATUS_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERXSTATUS_EL1_SHIFT) +#define HFGRTR_EL2_ERXSTATUS_EL1_VAL(x) ((x) & HFGRTR_EL2_ERXSTATUS_EL1_MASK) +#define HFGRTR_EL2_ERXSTATUS_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERXSTATUS_EL1_SHIFT) +#define HFGRTR_EL2_ERXSTATUS_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERXSTATUS_EL1_SHIFT) +#define HFGRTR_EL2_ERXCTLR_EL1_SHIFT 43 +#define HFGRTR_EL2_ERXCTLR_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERXCTLR_EL1_SHIFT) +#define HFGRTR_EL2_ERXCTLR_EL1_VAL(x) ((x) & HFGRTR_EL2_ERXCTLR_EL1_MASK) +#define HFGRTR_EL2_ERXCTLR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERXCTLR_EL1_SHIFT) +#define HFGRTR_EL2_ERXCTLR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERXCTLR_EL1_SHIFT) +#define HFGRTR_EL2_ERXFR_EL1_SHIFT 42 +#define HFGRTR_EL2_ERXFR_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERXFR_EL1_SHIFT) +#define HFGRTR_EL2_ERXFR_EL1_VAL(x) ((x) & HFGRTR_EL2_ERXFR_EL1_MASK) +#define HFGRTR_EL2_ERXFR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERXFR_EL1_SHIFT) +#define HFGRTR_EL2_ERXFR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERXFR_EL1_SHIFT) +#define HFGRTR_EL2_ERRSELR_EL1_SHIFT 41 +#define HFGRTR_EL2_ERRSELR_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERRSELR_EL1_SHIFT) +#define HFGRTR_EL2_ERRSELR_EL1_VAL(x) ((x) & HFGRTR_EL2_ERRSELR_EL1_MASK) +#define HFGRTR_EL2_ERRSELR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERRSELR_EL1_SHIFT) +#define HFGRTR_EL2_ERRSELR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERRSELR_EL1_SHIFT) +#define HFGRTR_EL2_ERRIDR_EL1_SHIFT 40 +#define HFGRTR_EL2_ERRIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_ERRIDR_EL1_SHIFT) +#define HFGRTR_EL2_ERRIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_ERRIDR_EL1_MASK) +#define HFGRTR_EL2_ERRIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ERRIDR_EL1_SHIFT) +#define HFGRTR_EL2_ERRIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ERRIDR_EL1_SHIFT) +#define HFGRTR_EL2_ICC_IGRPENn_EL1_SHIFT 39 +#define HFGRTR_EL2_ICC_IGRPENn_EL1_MASK (UL(0x1) << HFGRTR_EL2_ICC_IGRPENn_EL1_SHIFT) +#define HFGRTR_EL2_ICC_IGRPENn_EL1_VAL(x) ((x) & HFGRTR_EL2_ICC_IGRPENn_EL1_MASK) +#define HFGRTR_EL2_ICC_IGRPENn_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ICC_IGRPENn_EL1_SHIFT) +#define HFGRTR_EL2_ICC_IGRPENn_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ICC_IGRPENn_EL1_SHIFT) +#define HFGRTR_EL2_VBAR_EL1_SHIFT 38 +#define HFGRTR_EL2_VBAR_EL1_MASK (UL(0x1) << HFGRTR_EL2_VBAR_EL1_SHIFT) +#define HFGRTR_EL2_VBAR_EL1_VAL(x) ((x) & HFGRTR_EL2_VBAR_EL1_MASK) +#define HFGRTR_EL2_VBAR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_VBAR_EL1_SHIFT) +#define HFGRTR_EL2_VBAR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_VBAR_EL1_SHIFT) +#define HFGRTR_EL2_TTBR1_EL1_SHIFT 37 +#define HFGRTR_EL2_TTBR1_EL1_MASK (UL(0x1) << HFGRTR_EL2_TTBR1_EL1_SHIFT) +#define HFGRTR_EL2_TTBR1_EL1_VAL(x) ((x) & HFGRTR_EL2_TTBR1_EL1_MASK) +#define HFGRTR_EL2_TTBR1_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_TTBR1_EL1_SHIFT) +#define HFGRTR_EL2_TTBR1_EL1_TRAP (UL(0x1) << HFGRTR_EL2_TTBR1_EL1_SHIFT) +#define HFGRTR_EL2_TTBR0_EL1_SHIFT 36 +#define HFGRTR_EL2_TTBR0_EL1_MASK (UL(0x1) << HFGRTR_EL2_TTBR0_EL1_SHIFT) +#define HFGRTR_EL2_TTBR0_EL1_VAL(x) ((x) & HFGRTR_EL2_TTBR0_EL1_MASK) +#define HFGRTR_EL2_TTBR0_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_TTBR0_EL1_SHIFT) +#define HFGRTR_EL2_TTBR0_EL1_TRAP (UL(0x1) << HFGRTR_EL2_TTBR0_EL1_SHIFT) +#define HFGRTR_EL2_TPIDR_EL0_SHIFT 35 +#define HFGRTR_EL2_TPIDR_EL0_MASK (UL(0x1) << HFGRTR_EL2_TPIDR_EL0_SHIFT) +#define HFGRTR_EL2_TPIDR_EL0_VAL(x) ((x) & HFGRTR_EL2_TPIDR_EL0_MASK) +#define HFGRTR_EL2_TPIDR_EL0_NOTRAP (UL(0x0) << HFGRTR_EL2_TPIDR_EL0_SHIFT) +#define HFGRTR_EL2_TPIDR_EL0_TRAP (UL(0x1) << HFGRTR_EL2_TPIDR_EL0_SHIFT) +#define HFGRTR_EL2_TPIDRRO_EL0_SHIFT 34 +#define HFGRTR_EL2_TPIDRRO_EL0_MASK (UL(0x1) << HFGRTR_EL2_TPIDRRO_EL0_SHIFT) +#define HFGRTR_EL2_TPIDRRO_EL0_VAL(x) ((x) & HFGRTR_EL2_TPIDRRO_EL0_MASK) +#define HFGRTR_EL2_TPIDRRO_EL0_NOTRAP (UL(0x0) << HFGRTR_EL2_TPIDRRO_EL0_SHIFT) +#define HFGRTR_EL2_TPIDRRO_EL0_TRAP (UL(0x1) << HFGRTR_EL2_TPIDRRO_EL0_SHIFT) +#define HFGRTR_EL2_TPIDR_EL1_SHIFT 33 +#define HFGRTR_EL2_TPIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_TPIDR_EL1_SHIFT) +#define HFGRTR_EL2_TPIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_TPIDR_EL1_MASK) +#define HFGRTR_EL2_TPIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_TPIDR_EL1_SHIFT) +#define HFGRTR_EL2_TPIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_TPIDR_EL1_SHIFT) +#define HFGRTR_EL2_TCR_EL1_SHIFT 32 +#define HFGRTR_EL2_TCR_EL1_MASK (UL(0x1) << HFGRTR_EL2_TCR_EL1_SHIFT) +#define HFGRTR_EL2_TCR_EL1_VAL(x) ((x) & HFGRTR_EL2_TCR_EL1_MASK) +#define HFGRTR_EL2_TCR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_TCR_EL1_SHIFT) +#define HFGRTR_EL2_TCR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_TCR_EL1_SHIFT) +#define HFGRTR_EL2_SCXTNUM_EL0_SHIFT 31 +#define HFGRTR_EL2_SCXTNUM_EL0_MASK (UL(0x1) << HFGRTR_EL2_SCXTNUM_EL0_SHIFT) +#define HFGRTR_EL2_SCXTNUM_EL0_VAL(x) ((x) & HFGRTR_EL2_SCXTNUM_EL0_MASK) +#define HFGRTR_EL2_SCXTNUM_EL0_NOTRAP (UL(0x0) << HFGRTR_EL2_SCXTNUM_EL0_SHIFT) +#define HFGRTR_EL2_SCXTNUM_EL0_TRAP (UL(0x1) << HFGRTR_EL2_SCXTNUM_EL0_SHIFT) +#define HFGRTR_EL2_SCXTNUM_EL1_SHIFT 30 +#define HFGRTR_EL2_SCXTNUM_EL1_MASK (UL(0x1) << HFGRTR_EL2_SCXTNUM_EL1_SHIFT) +#define HFGRTR_EL2_SCXTNUM_EL1_VAL(x) ((x) & HFGRTR_EL2_SCXTNUM_EL1_MASK) +#define HFGRTR_EL2_SCXTNUM_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_SCXTNUM_EL1_SHIFT) +#define HFGRTR_EL2_SCXTNUM_EL1_TRAP (UL(0x1) << HFGRTR_EL2_SCXTNUM_EL1_SHIFT) +#define HFGRTR_EL2_SCTLR_EL1_SHIFT 29 +#define HFGRTR_EL2_SCTLR_EL1_MASK (UL(0x1) << HFGRTR_EL2_SCTLR_EL1_SHIFT) +#define HFGRTR_EL2_SCTLR_EL1_VAL(x) ((x) & HFGRTR_EL2_SCTLR_EL1_MASK) +#define HFGRTR_EL2_SCTLR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_SCTLR_EL1_SHIFT) +#define HFGRTR_EL2_SCTLR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_SCTLR_EL1_SHIFT) +#define HFGRTR_EL2_REVIDR_EL1_SHIFT 28 +#define HFGRTR_EL2_REVIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_REVIDR_EL1_SHIFT) +#define HFGRTR_EL2_REVIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_REVIDR_EL1_MASK) +#define HFGRTR_EL2_REVIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_REVIDR_EL1_SHIFT) +#define HFGRTR_EL2_REVIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_REVIDR_EL1_SHIFT) +#define HFGRTR_EL2_PAR_EL1_SHIFT 27 +#define HFGRTR_EL2_PAR_EL1_MASK (UL(0x1) << HFGRTR_EL2_PAR_EL1_SHIFT) +#define HFGRTR_EL2_PAR_EL1_VAL(x) ((x) & HFGRTR_EL2_PAR_EL1_MASK) +#define HFGRTR_EL2_PAR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_PAR_EL1_SHIFT) +#define HFGRTR_EL2_PAR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_PAR_EL1_SHIFT) +#define HFGRTR_EL2_MPIDR_EL1_SHIFT 26 +#define HFGRTR_EL2_MPIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_MPIDR_EL1_SHIFT) +#define HFGRTR_EL2_MPIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_MPIDR_EL1_MASK) +#define HFGRTR_EL2_MPIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_MPIDR_EL1_SHIFT) +#define HFGRTR_EL2_MPIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_MPIDR_EL1_SHIFT) +#define HFGRTR_EL2_MIDR_EL1_SHIFT 25 +#define HFGRTR_EL2_MIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_MIDR_EL1_SHIFT) +#define HFGRTR_EL2_MIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_MIDR_EL1_MASK) +#define HFGRTR_EL2_MIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_MIDR_EL1_SHIFT) +#define HFGRTR_EL2_MIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_MIDR_EL1_SHIFT) +#define HFGRTR_EL2_MAIR_EL1_SHIFT 24 +#define HFGRTR_EL2_MAIR_EL1_MASK (UL(0x1) << HFGRTR_EL2_MAIR_EL1_SHIFT) +#define HFGRTR_EL2_MAIR_EL1_VAL(x) ((x) & HFGRTR_EL2_MAIR_EL1_MASK) +#define HFGRTR_EL2_MAIR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_MAIR_EL1_SHIFT) +#define HFGRTR_EL2_MAIR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_MAIR_EL1_SHIFT) +#define HFGRTR_EL2_LORSA_EL1_SHIFT 23 +#define HFGRTR_EL2_LORSA_EL1_MASK (UL(0x1) << HFGRTR_EL2_LORSA_EL1_SHIFT) +#define HFGRTR_EL2_LORSA_EL1_VAL(x) ((x) & HFGRTR_EL2_LORSA_EL1_MASK) +#define HFGRTR_EL2_LORSA_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_LORSA_EL1_SHIFT) +#define HFGRTR_EL2_LORSA_EL1_TRAP (UL(0x1) << HFGRTR_EL2_LORSA_EL1_SHIFT) +#define HFGRTR_EL2_LORN_EL1_SHIFT 22 +#define HFGRTR_EL2_LORN_EL1_MASK (UL(0x1) << HFGRTR_EL2_LORN_EL1_SHIFT) +#define HFGRTR_EL2_LORN_EL1_VAL(x) ((x) & HFGRTR_EL2_LORN_EL1_MASK) +#define HFGRTR_EL2_LORN_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_LORN_EL1_SHIFT) +#define HFGRTR_EL2_LORN_EL1_TRAP (UL(0x1) << HFGRTR_EL2_LORN_EL1_SHIFT) +#define HFGRTR_EL2_LORID_EL1_SHIFT 21 +#define HFGRTR_EL2_LORID_EL1_MASK (UL(0x1) << HFGRTR_EL2_LORID_EL1_SHIFT) +#define HFGRTR_EL2_LORID_EL1_VAL(x) ((x) & HFGRTR_EL2_LORID_EL1_MASK) +#define HFGRTR_EL2_LORID_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_LORID_EL1_SHIFT) +#define HFGRTR_EL2_LORID_EL1_TRAP (UL(0x1) << HFGRTR_EL2_LORID_EL1_SHIFT) +#define HFGRTR_EL2_LOREA_EL1_SHIFT 20 +#define HFGRTR_EL2_LOREA_EL1_MASK (UL(0x1) << HFGRTR_EL2_LOREA_EL1_SHIFT) +#define HFGRTR_EL2_LOREA_EL1_VAL(x) ((x) & HFGRTR_EL2_LOREA_EL1_MASK) +#define HFGRTR_EL2_LOREA_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_LOREA_EL1_SHIFT) +#define HFGRTR_EL2_LOREA_EL1_TRAP (UL(0x1) << HFGRTR_EL2_LOREA_EL1_SHIFT) +#define HFGRTR_EL2_LORC_EL1_SHIFT 19 +#define HFGRTR_EL2_LORC_EL1_MASK (UL(0x1) << HFGRTR_EL2_LORC_EL1_SHIFT) +#define HFGRTR_EL2_LORC_EL1_VAL(x) ((x) & HFGRTR_EL2_LORC_EL1_MASK) +#define HFGRTR_EL2_LORC_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_LORC_EL1_SHIFT) +#define HFGRTR_EL2_LORC_EL1_TRAP (UL(0x1) << HFGRTR_EL2_LORC_EL1_SHIFT) +#define HFGRTR_EL2_ISR_EL1_SHIFT 18 +#define HFGRTR_EL2_ISR_EL1_MASK (UL(0x1) << HFGRTR_EL2_ISR_EL1_SHIFT) +#define HFGRTR_EL2_ISR_EL1_VAL(x) ((x) & HFGRTR_EL2_ISR_EL1_MASK) +#define HFGRTR_EL2_ISR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ISR_EL1_SHIFT) +#define HFGRTR_EL2_ISR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ISR_EL1_SHIFT) +#define HFGRTR_EL2_FAR_EL1_SHIFT 17 +#define HFGRTR_EL2_FAR_EL1_MASK (UL(0x1) << HFGRTR_EL2_FAR_EL1_SHIFT) +#define HFGRTR_EL2_FAR_EL1_VAL(x) ((x) & HFGRTR_EL2_FAR_EL1_MASK) +#define HFGRTR_EL2_FAR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_FAR_EL1_SHIFT) +#define HFGRTR_EL2_FAR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_FAR_EL1_SHIFT) +#define HFGRTR_EL2_ESR_EL1_SHIFT 16 +#define HFGRTR_EL2_ESR_EL1_MASK (UL(0x1) << HFGRTR_EL2_ESR_EL1_SHIFT) +#define HFGRTR_EL2_ESR_EL1_VAL(x) ((x) & HFGRTR_EL2_ESR_EL1_MASK) +#define HFGRTR_EL2_ESR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_ESR_EL1_SHIFT) +#define HFGRTR_EL2_ESR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_ESR_EL1_SHIFT) +#define HFGRTR_EL2_DCZID_EL0_SHIFT 15 +#define HFGRTR_EL2_DCZID_EL0_MASK (UL(0x1) << HFGRTR_EL2_DCZID_EL0_SHIFT) +#define HFGRTR_EL2_DCZID_EL0_VAL(x) ((x) & HFGRTR_EL2_DCZID_EL0_MASK) +#define HFGRTR_EL2_DCZID_EL0_NOTRAP (UL(0x0) << HFGRTR_EL2_DCZID_EL0_SHIFT) +#define HFGRTR_EL2_DCZID_EL0_TRAP (UL(0x1) << HFGRTR_EL2_DCZID_EL0_SHIFT) +#define HFGRTR_EL2_CTR_EL0_SHIFT 14 +#define HFGRTR_EL2_CTR_EL0_MASK (UL(0x1) << HFGRTR_EL2_CTR_EL0_SHIFT) +#define HFGRTR_EL2_CTR_EL0_VAL(x) ((x) & HFGRTR_EL2_CTR_EL0_MASK) +#define HFGRTR_EL2_CTR_EL0_NOTRAP (UL(0x0) << HFGRTR_EL2_CTR_EL0_SHIFT) +#define HFGRTR_EL2_CTR_EL0_TRAP (UL(0x1) << HFGRTR_EL2_CTR_EL0_SHIFT) +#define HFGRTR_EL2_CSSELR_EL1_SHIFT 13 +#define HFGRTR_EL2_CSSELR_EL1_MASK (UL(0x1) << HFGRTR_EL2_CSSELR_EL1_SHIFT) +#define HFGRTR_EL2_CSSELR_EL1_VAL(x) ((x) & HFGRTR_EL2_CSSELR_EL1_MASK) +#define HFGRTR_EL2_CSSELR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_CSSELR_EL1_SHIFT) +#define HFGRTR_EL2_CSSELR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_CSSELR_EL1_SHIFT) +#define HFGRTR_EL2_CPACR_EL1_SHIFT 12 +#define HFGRTR_EL2_CPACR_EL1_MASK (UL(0x1) << HFGRTR_EL2_CPACR_EL1_SHIFT) +#define HFGRTR_EL2_CPACR_EL1_VAL(x) ((x) & HFGRTR_EL2_CPACR_EL1_MASK) +#define HFGRTR_EL2_CPACR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_CPACR_EL1_SHIFT) +#define HFGRTR_EL2_CPACR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_CPACR_EL1_SHIFT) +#define HFGRTR_EL2_CONTEXTIDR_EL1_SHIFT 11 +#define HFGRTR_EL2_CONTEXTIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_CONTEXTIDR_EL1_SHIFT) +#define HFGRTR_EL2_CONTEXTIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_CONTEXTIDR_EL1_MASK) +#define HFGRTR_EL2_CONTEXTIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_CONTEXTIDR_EL1_SHIFT) +#define HFGRTR_EL2_CONTEXTIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_CONTEXTIDR_EL1_SHIFT) +#define HFGRTR_EL2_CLIDR_EL1_SHIFT 10 +#define HFGRTR_EL2_CLIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_CLIDR_EL1_SHIFT) +#define HFGRTR_EL2_CLIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_CLIDR_EL1_MASK) +#define HFGRTR_EL2_CLIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_CLIDR_EL1_SHIFT) +#define HFGRTR_EL2_CLIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_CLIDR_EL1_SHIFT) +#define HFGRTR_EL2_CCSIDR_EL1_SHIFT 9 +#define HFGRTR_EL2_CCSIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_CCSIDR_EL1_SHIFT) +#define HFGRTR_EL2_CCSIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_CCSIDR_EL1_MASK) +#define HFGRTR_EL2_CCSIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_CCSIDR_EL1_SHIFT) +#define HFGRTR_EL2_CCSIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_CCSIDR_EL1_SHIFT) +#define HFGRTR_EL2_APIBKey_SHIFT 8 +#define HFGRTR_EL2_APIBKey_MASK (UL(0x1) << HFGRTR_EL2_APIBKey_SHIFT) +#define HFGRTR_EL2_APIBKey_VAL(x) ((x) & HFGRTR_EL2_APIBKey_MASK) +#define HFGRTR_EL2_APIBKey_NOTRAP (UL(0x0) << HFGRTR_EL2_APIBKey_SHIFT) +#define HFGRTR_EL2_APIBKey_TRAP (UL(0x1) << HFGRTR_EL2_APIBKey_SHIFT) +#define HFGRTR_EL2_APIAKey_SHIFT 7 +#define HFGRTR_EL2_APIAKey_MASK (UL(0x1) << HFGRTR_EL2_APIAKey_SHIFT) +#define HFGRTR_EL2_APIAKey_VAL(x) ((x) & HFGRTR_EL2_APIAKey_MASK) +#define HFGRTR_EL2_APIAKey_NOTRAP (UL(0x0) << HFGRTR_EL2_APIAKey_SHIFT) +#define HFGRTR_EL2_APIAKey_TRAP (UL(0x1) << HFGRTR_EL2_APIAKey_SHIFT) +#define HFGRTR_EL2_APGAKey_SHIFT 6 +#define HFGRTR_EL2_APGAKey_MASK (UL(0x1) << HFGRTR_EL2_APGAKey_SHIFT) +#define HFGRTR_EL2_APGAKey_VAL(x) ((x) & HFGRTR_EL2_APGAKey_MASK) +#define HFGRTR_EL2_APGAKey_NOTRAP (UL(0x0) << HFGRTR_EL2_APGAKey_SHIFT) +#define HFGRTR_EL2_APGAKey_TRAP (UL(0x1) << HFGRTR_EL2_APGAKey_SHIFT) +#define HFGRTR_EL2_APDBKey_SHIFT 5 +#define HFGRTR_EL2_APDBKey_MASK (UL(0x1) << HFGRTR_EL2_APDBKey_SHIFT) +#define HFGRTR_EL2_APDBKey_VAL(x) ((x) & HFGRTR_EL2_APDBKey_MASK) +#define HFGRTR_EL2_APDBKey_NOTRAP (UL(0x0) << HFGRTR_EL2_APDBKey_SHIFT) +#define HFGRTR_EL2_APDBKey_TRAP (UL(0x1) << HFGRTR_EL2_APDBKey_SHIFT) +#define HFGRTR_EL2_APDAKey_SHIFT 4 +#define HFGRTR_EL2_APDAKey_MASK (UL(0x1) << HFGRTR_EL2_APDAKey_SHIFT) +#define HFGRTR_EL2_APDAKey_VAL(x) ((x) & HFGRTR_EL2_APDAKey_MASK) +#define HFGRTR_EL2_APDAKey_NOTRAP (UL(0x0) << HFGRTR_EL2_APDAKey_SHIFT) +#define HFGRTR_EL2_APDAKey_TRAP (UL(0x1) << HFGRTR_EL2_APDAKey_SHIFT) +#define HFGRTR_EL2_AMAIR_EL1_SHIFT 3 +#define HFGRTR_EL2_AMAIR_EL1_MASK (UL(0x1) << HFGRTR_EL2_AMAIR_EL1_SHIFT) +#define HFGRTR_EL2_AMAIR_EL1_VAL(x) ((x) & HFGRTR_EL2_AMAIR_EL1_MASK) +#define HFGRTR_EL2_AMAIR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_AMAIR_EL1_SHIFT) +#define HFGRTR_EL2_AMAIR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_AMAIR_EL1_SHIFT) +#define HFGRTR_EL2_AIDR_EL1_SHIFT 2 +#define HFGRTR_EL2_AIDR_EL1_MASK (UL(0x1) << HFGRTR_EL2_AIDR_EL1_SHIFT) +#define HFGRTR_EL2_AIDR_EL1_VAL(x) ((x) & HFGRTR_EL2_AIDR_EL1_MASK) +#define HFGRTR_EL2_AIDR_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_AIDR_EL1_SHIFT) +#define HFGRTR_EL2_AIDR_EL1_TRAP (UL(0x1) << HFGRTR_EL2_AIDR_EL1_SHIFT) +#define HFGRTR_EL2_AFSR1_EL1_SHIFT 1 +#define HFGRTR_EL2_AFSR1_EL1_MASK (UL(0x1) << HFGRTR_EL2_AFSR1_EL1_SHIFT) +#define HFGRTR_EL2_AFSR1_EL1_VAL(x) ((x) & HFGRTR_EL2_AFSR1_EL1_MASK) +#define HFGRTR_EL2_AFSR1_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_AFSR1_EL1_SHIFT) +#define HFGRTR_EL2_AFSR1_EL1_TRAP (UL(0x1) << HFGRTR_EL2_AFSR1_EL1_SHIFT) +#define HFGRTR_EL2_AFSR0_EL1_SHIFT 0 +#define HFGRTR_EL2_AFSR0_EL1_MASK (UL(0x1) << HFGRTR_EL2_AFSR0_EL1_SHIFT) +#define HFGRTR_EL2_AFSR0_EL1_VAL(x) ((x) & HFGRTR_EL2_AFSR0_EL1_MASK) +#define HFGRTR_EL2_AFSR0_EL1_NOTRAP (UL(0x0) << HFGRTR_EL2_AFSR0_EL1_SHIFT) +#define HFGRTR_EL2_AFSR0_EL1_TRAP (UL(0x1) << HFGRTR_EL2_AFSR0_EL1_SHIFT) + +/* HFGWTR2_EL2 */ +#define HFGWTR2_EL2_REG MRS_REG_ALT_NAME(HFGWTR2_EL2) +#define HFGWTR2_EL2_op0 3 +#define HFGWTR2_EL2_op1 4 +#define HFGWTR2_EL2_CRn 3 +#define HFGWTR2_EL2_CRm 1 +#define HFGWTR2_EL2_op2 3 +#define HFGWTR2_EL2_nRCWSMASK_EL1_SHIFT 2 +#define HFGWTR2_EL2_nRCWSMASK_EL1_MASK (UL(0x1) << HFGWTR2_EL2_nRCWSMASK_EL1_SHIFT) +#define HFGWTR2_EL2_nRCWSMASK_EL1_VAL(x) ((x) & HFGWTR2_EL2_nRCWSMASK_EL1_MASK) +#define HFGWTR2_EL2_nRCWSMASK_EL1_TRAP (UL(0x0) << HFGWTR2_EL2_nRCWSMASK_EL1_SHIFT) +#define HFGWTR2_EL2_nRCWSMASK_EL1_NOTRAP (UL(0x1) << HFGWTR2_EL2_nRCWSMASK_EL1_SHIFT) +#define HFGWTR2_EL2_nPFAR_EL1_SHIFT 0 +#define HFGWTR2_EL2_nPFAR_EL1_MASK (UL(0x1) << HFGWTR2_EL2_nPFAR_EL1_SHIFT) +#define HFGWTR2_EL2_nPFAR_EL1_VAL(x) ((x) & HFGWTR2_EL2_nPFAR_EL1_MASK) +#define HFGWTR2_EL2_nPFAR_EL1_TRAP (UL(0x0) << HFGWTR2_EL2_nPFAR_EL1_SHIFT) +#define HFGWTR2_EL2_nPFAR_EL1_NOTRAP (UL(0x1) << HFGWTR2_EL2_nPFAR_EL1_SHIFT) + +/* HFGWTR_EL2 */ +#define HFGWTR_EL2_REG MRS_REG_ALT_NAME(HFGWTR_EL2) +#define HFGWTR_EL2_op0 3 +#define HFGWTR_EL2_op1 4 +#define HFGWTR_EL2_CRn 1 +#define HFGWTR_EL2_CRm 1 +#define HFGWTR_EL2_op2 5 +#define HFGWTR_EL2_nAMAIR2_EL1_SHIFT 63 +#define HFGWTR_EL2_nAMAIR2_EL1_MASK (UL(0x1) << HFGWTR_EL2_nAMAIR2_EL1_SHIFT) +#define HFGWTR_EL2_nAMAIR2_EL1_VAL(x) ((x) & HFGWTR_EL2_nAMAIR2_EL1_MASK) +#define HFGWTR_EL2_nAMAIR2_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nAMAIR2_EL1_SHIFT) +#define HFGWTR_EL2_nAMAIR2_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nAMAIR2_EL1_SHIFT) +#define HFGWTR_EL2_nMAIR2_EL1_SHIFT 62 +#define HFGWTR_EL2_nMAIR2_EL1_MASK (UL(0x1) << HFGWTR_EL2_nMAIR2_EL1_SHIFT) +#define HFGWTR_EL2_nMAIR2_EL1_VAL(x) ((x) & HFGWTR_EL2_nMAIR2_EL1_MASK) +#define HFGWTR_EL2_nMAIR2_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nMAIR2_EL1_SHIFT) +#define HFGWTR_EL2_nMAIR2_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nMAIR2_EL1_SHIFT) +#define HFGWTR_EL2_nS2POR_EL1_SHIFT 61 +#define HFGWTR_EL2_nS2POR_EL1_MASK (UL(0x1) << HFGWTR_EL2_nS2POR_EL1_SHIFT) +#define HFGWTR_EL2_nS2POR_EL1_VAL(x) ((x) & HFGWTR_EL2_nS2POR_EL1_MASK) +#define HFGWTR_EL2_nS2POR_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nS2POR_EL1_SHIFT) +#define HFGWTR_EL2_nS2POR_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nS2POR_EL1_SHIFT) +#define HFGWTR_EL2_nPOR_EL1_SHIFT 60 +#define HFGWTR_EL2_nPOR_EL1_MASK (UL(0x1) << HFGWTR_EL2_nPOR_EL1_SHIFT) +#define HFGWTR_EL2_nPOR_EL1_VAL(x) ((x) & HFGWTR_EL2_nPOR_EL1_MASK) +#define HFGWTR_EL2_nPOR_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nPOR_EL1_SHIFT) +#define HFGWTR_EL2_nPOR_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nPOR_EL1_SHIFT) +#define HFGWTR_EL2_nPOR_EL0_SHIFT 59 +#define HFGWTR_EL2_nPOR_EL0_MASK (UL(0x1) << HFGWTR_EL2_nPOR_EL0_SHIFT) +#define HFGWTR_EL2_nPOR_EL0_VAL(x) ((x) & HFGWTR_EL2_nPOR_EL0_MASK) +#define HFGWTR_EL2_nPOR_EL0_TRAP (UL(0x0) << HFGWTR_EL2_nPOR_EL0_SHIFT) +#define HFGWTR_EL2_nPOR_EL0_NOTRAP (UL(0x1) << HFGWTR_EL2_nPOR_EL0_SHIFT) +#define HFGWTR_EL2_nPIR_EL1_SHIFT 58 +#define HFGWTR_EL2_nPIR_EL1_MASK (UL(0x1) << HFGWTR_EL2_nPIR_EL1_SHIFT) +#define HFGWTR_EL2_nPIR_EL1_VAL(x) ((x) & HFGWTR_EL2_nPIR_EL1_MASK) +#define HFGWTR_EL2_nPIR_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nPIR_EL1_SHIFT) +#define HFGWTR_EL2_nPIR_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nPIR_EL1_SHIFT) +#define HFGWTR_EL2_nPIRE0_EL1_SHIFT 57 +#define HFGWTR_EL2_nPIRE0_EL1_MASK (UL(0x1) << HFGWTR_EL2_nPIRE0_EL1_SHIFT) +#define HFGWTR_EL2_nPIRE0_EL1_VAL(x) ((x) & HFGWTR_EL2_nPIRE0_EL1_MASK) +#define HFGWTR_EL2_nPIRE0_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nPIRE0_EL1_SHIFT) +#define HFGWTR_EL2_nPIRE0_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nPIRE0_EL1_SHIFT) +#define HFGWTR_EL2_nRCWMASK_EL1_SHIFT 56 +#define HFGWTR_EL2_nRCWMASK_EL1_MASK (UL(0x1) << HFGWTR_EL2_nRCWMASK_EL1_SHIFT) +#define HFGWTR_EL2_nRCWMASK_EL1_VAL(x) ((x) & HFGWTR_EL2_nRCWMASK_EL1_MASK) +#define HFGWTR_EL2_nRCWMASK_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nRCWMASK_EL1_SHIFT) +#define HFGWTR_EL2_nRCWMASK_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nRCWMASK_EL1_SHIFT) +#define HFGWTR_EL2_nTPIDR2_EL0_SHIFT 55 +#define HFGWTR_EL2_nTPIDR2_EL0_MASK (UL(0x1) << HFGWTR_EL2_nTPIDR2_EL0_SHIFT) +#define HFGWTR_EL2_nTPIDR2_EL0_VAL(x) ((x) & HFGWTR_EL2_nTPIDR2_EL0_MASK) +#define HFGWTR_EL2_nTPIDR2_EL0_TRAP (UL(0x0) << HFGWTR_EL2_nTPIDR2_EL0_SHIFT) +#define HFGWTR_EL2_nTPIDR2_EL0_NOTRAP (UL(0x1) << HFGWTR_EL2_nTPIDR2_EL0_SHIFT) +#define HFGWTR_EL2_nSMPRI_EL1_SHIFT 54 +#define HFGWTR_EL2_nSMPRI_EL1_MASK (UL(0x1) << HFGWTR_EL2_nSMPRI_EL1_SHIFT) +#define HFGWTR_EL2_nSMPRI_EL1_VAL(x) ((x) & HFGWTR_EL2_nSMPRI_EL1_MASK) +#define HFGWTR_EL2_nSMPRI_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nSMPRI_EL1_SHIFT) +#define HFGWTR_EL2_nSMPRI_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nSMPRI_EL1_SHIFT) +#define HFGWTR_EL2_nGCS_EL1_SHIFT 53 +#define HFGWTR_EL2_nGCS_EL1_MASK (UL(0x1) << HFGWTR_EL2_nGCS_EL1_SHIFT) +#define HFGWTR_EL2_nGCS_EL1_VAL(x) ((x) & HFGWTR_EL2_nGCS_EL1_MASK) +#define HFGWTR_EL2_nGCS_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nGCS_EL1_SHIFT) +#define HFGWTR_EL2_nGCS_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nGCS_EL1_SHIFT) +#define HFGWTR_EL2_nGCS_EL0_SHIFT 52 +#define HFGWTR_EL2_nGCS_EL0_MASK (UL(0x1) << HFGWTR_EL2_nGCS_EL0_SHIFT) +#define HFGWTR_EL2_nGCS_EL0_VAL(x) ((x) & HFGWTR_EL2_nGCS_EL0_MASK) +#define HFGWTR_EL2_nGCS_EL0_TRAP (UL(0x0) << HFGWTR_EL2_nGCS_EL0_SHIFT) +#define HFGWTR_EL2_nGCS_EL0_NOTRAP (UL(0x1) << HFGWTR_EL2_nGCS_EL0_SHIFT) +#define HFGWTR_EL2_nACCDATA_EL1_SHIFT 50 +#define HFGWTR_EL2_nACCDATA_EL1_MASK (UL(0x1) << HFGWTR_EL2_nACCDATA_EL1_SHIFT) +#define HFGWTR_EL2_nACCDATA_EL1_VAL(x) ((x) & HFGWTR_EL2_nACCDATA_EL1_MASK) +#define HFGWTR_EL2_nACCDATA_EL1_TRAP (UL(0x0) << HFGWTR_EL2_nACCDATA_EL1_SHIFT) +#define HFGWTR_EL2_nACCDATA_EL1_NOTRAP (UL(0x1) << HFGWTR_EL2_nACCDATA_EL1_SHIFT) +#define HFGWTR_EL2_ERXADDR_EL1_SHIFT 49 +#define HFGWTR_EL2_ERXADDR_EL1_MASK (UL(0x1) << HFGWTR_EL2_ERXADDR_EL1_SHIFT) +#define HFGWTR_EL2_ERXADDR_EL1_VAL(x) ((x) & HFGWTR_EL2_ERXADDR_EL1_MASK) +#define HFGWTR_EL2_ERXADDR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ERXADDR_EL1_SHIFT) +#define HFGWTR_EL2_ERXADDR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ERXADDR_EL1_SHIFT) +#define HFGWTR_EL2_ERXPFGCDN_EL1_SHIFT 48 +#define HFGWTR_EL2_ERXPFGCDN_EL1_MASK (UL(0x1) << HFGWTR_EL2_ERXPFGCDN_EL1_SHIFT) +#define HFGWTR_EL2_ERXPFGCDN_EL1_VAL(x) ((x) & HFGWTR_EL2_ERXPFGCDN_EL1_MASK) +#define HFGWTR_EL2_ERXPFGCDN_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ERXPFGCDN_EL1_SHIFT) +#define HFGWTR_EL2_ERXPFGCDN_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ERXPFGCDN_EL1_SHIFT) +#define HFGWTR_EL2_ERXPFGCTL_EL1_SHIFT 47 +#define HFGWTR_EL2_ERXPFGCTL_EL1_MASK (UL(0x1) << HFGWTR_EL2_ERXPFGCTL_EL1_SHIFT) +#define HFGWTR_EL2_ERXPFGCTL_EL1_VAL(x) ((x) & HFGWTR_EL2_ERXPFGCTL_EL1_MASK) +#define HFGWTR_EL2_ERXPFGCTL_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ERXPFGCTL_EL1_SHIFT) +#define HFGWTR_EL2_ERXPFGCTL_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ERXPFGCTL_EL1_SHIFT) +#define HFGWTR_EL2_ERXMISCn_EL1_SHIFT 45 +#define HFGWTR_EL2_ERXMISCn_EL1_MASK (UL(0x1) << HFGWTR_EL2_ERXMISCn_EL1_SHIFT) +#define HFGWTR_EL2_ERXMISCn_EL1_VAL(x) ((x) & HFGWTR_EL2_ERXMISCn_EL1_MASK) +#define HFGWTR_EL2_ERXMISCn_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ERXMISCn_EL1_SHIFT) +#define HFGWTR_EL2_ERXMISCn_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ERXMISCn_EL1_SHIFT) +#define HFGWTR_EL2_ERXSTATUS_EL1_SHIFT 44 +#define HFGWTR_EL2_ERXSTATUS_EL1_MASK (UL(0x1) << HFGWTR_EL2_ERXSTATUS_EL1_SHIFT) +#define HFGWTR_EL2_ERXSTATUS_EL1_VAL(x) ((x) & HFGWTR_EL2_ERXSTATUS_EL1_MASK) +#define HFGWTR_EL2_ERXSTATUS_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ERXSTATUS_EL1_SHIFT) +#define HFGWTR_EL2_ERXSTATUS_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ERXSTATUS_EL1_SHIFT) +#define HFGWTR_EL2_ERXCTLR_EL1_SHIFT 43 +#define HFGWTR_EL2_ERXCTLR_EL1_MASK (UL(0x1) << HFGWTR_EL2_ERXCTLR_EL1_SHIFT) +#define HFGWTR_EL2_ERXCTLR_EL1_VAL(x) ((x) & HFGWTR_EL2_ERXCTLR_EL1_MASK) +#define HFGWTR_EL2_ERXCTLR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ERXCTLR_EL1_SHIFT) +#define HFGWTR_EL2_ERXCTLR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ERXCTLR_EL1_SHIFT) +#define HFGWTR_EL2_ERRSELR_EL1_SHIFT 41 +#define HFGWTR_EL2_ERRSELR_EL1_MASK (UL(0x1) << HFGWTR_EL2_ERRSELR_EL1_SHIFT) +#define HFGWTR_EL2_ERRSELR_EL1_VAL(x) ((x) & HFGWTR_EL2_ERRSELR_EL1_MASK) +#define HFGWTR_EL2_ERRSELR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ERRSELR_EL1_SHIFT) +#define HFGWTR_EL2_ERRSELR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ERRSELR_EL1_SHIFT) +#define HFGWTR_EL2_ICC_IGRPENn_EL1_SHIFT 39 +#define HFGWTR_EL2_ICC_IGRPENn_EL1_MASK (UL(0x1) << HFGWTR_EL2_ICC_IGRPENn_EL1_SHIFT) +#define HFGWTR_EL2_ICC_IGRPENn_EL1_VAL(x) ((x) & HFGWTR_EL2_ICC_IGRPENn_EL1_MASK) +#define HFGWTR_EL2_ICC_IGRPENn_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ICC_IGRPENn_EL1_SHIFT) +#define HFGWTR_EL2_ICC_IGRPENn_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ICC_IGRPENn_EL1_SHIFT) +#define HFGWTR_EL2_VBAR_EL1_SHIFT 38 +#define HFGWTR_EL2_VBAR_EL1_MASK (UL(0x1) << HFGWTR_EL2_VBAR_EL1_SHIFT) +#define HFGWTR_EL2_VBAR_EL1_VAL(x) ((x) & HFGWTR_EL2_VBAR_EL1_MASK) +#define HFGWTR_EL2_VBAR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_VBAR_EL1_SHIFT) +#define HFGWTR_EL2_VBAR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_VBAR_EL1_SHIFT) +#define HFGWTR_EL2_TTBR1_EL1_SHIFT 37 +#define HFGWTR_EL2_TTBR1_EL1_MASK (UL(0x1) << HFGWTR_EL2_TTBR1_EL1_SHIFT) +#define HFGWTR_EL2_TTBR1_EL1_VAL(x) ((x) & HFGWTR_EL2_TTBR1_EL1_MASK) +#define HFGWTR_EL2_TTBR1_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_TTBR1_EL1_SHIFT) +#define HFGWTR_EL2_TTBR1_EL1_TRAP (UL(0x1) << HFGWTR_EL2_TTBR1_EL1_SHIFT) +#define HFGWTR_EL2_TTBR0_EL1_SHIFT 36 +#define HFGWTR_EL2_TTBR0_EL1_MASK (UL(0x1) << HFGWTR_EL2_TTBR0_EL1_SHIFT) +#define HFGWTR_EL2_TTBR0_EL1_VAL(x) ((x) & HFGWTR_EL2_TTBR0_EL1_MASK) +#define HFGWTR_EL2_TTBR0_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_TTBR0_EL1_SHIFT) +#define HFGWTR_EL2_TTBR0_EL1_TRAP (UL(0x1) << HFGWTR_EL2_TTBR0_EL1_SHIFT) +#define HFGWTR_EL2_TPIDR_EL0_SHIFT 35 +#define HFGWTR_EL2_TPIDR_EL0_MASK (UL(0x1) << HFGWTR_EL2_TPIDR_EL0_SHIFT) +#define HFGWTR_EL2_TPIDR_EL0_VAL(x) ((x) & HFGWTR_EL2_TPIDR_EL0_MASK) +#define HFGWTR_EL2_TPIDR_EL0_NOTRAP (UL(0x0) << HFGWTR_EL2_TPIDR_EL0_SHIFT) +#define HFGWTR_EL2_TPIDR_EL0_TRAP (UL(0x1) << HFGWTR_EL2_TPIDR_EL0_SHIFT) +#define HFGWTR_EL2_TPIDRRO_EL0_SHIFT 34 +#define HFGWTR_EL2_TPIDRRO_EL0_MASK (UL(0x1) << HFGWTR_EL2_TPIDRRO_EL0_SHIFT) +#define HFGWTR_EL2_TPIDRRO_EL0_VAL(x) ((x) & HFGWTR_EL2_TPIDRRO_EL0_MASK) +#define HFGWTR_EL2_TPIDRRO_EL0_NOTRAP (UL(0x0) << HFGWTR_EL2_TPIDRRO_EL0_SHIFT) +#define HFGWTR_EL2_TPIDRRO_EL0_TRAP (UL(0x1) << HFGWTR_EL2_TPIDRRO_EL0_SHIFT) +#define HFGWTR_EL2_TPIDR_EL1_SHIFT 33 +#define HFGWTR_EL2_TPIDR_EL1_MASK (UL(0x1) << HFGWTR_EL2_TPIDR_EL1_SHIFT) +#define HFGWTR_EL2_TPIDR_EL1_VAL(x) ((x) & HFGWTR_EL2_TPIDR_EL1_MASK) +#define HFGWTR_EL2_TPIDR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_TPIDR_EL1_SHIFT) +#define HFGWTR_EL2_TPIDR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_TPIDR_EL1_SHIFT) +#define HFGWTR_EL2_TCR_EL1_SHIFT 32 +#define HFGWTR_EL2_TCR_EL1_MASK (UL(0x1) << HFGWTR_EL2_TCR_EL1_SHIFT) +#define HFGWTR_EL2_TCR_EL1_VAL(x) ((x) & HFGWTR_EL2_TCR_EL1_MASK) +#define HFGWTR_EL2_TCR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_TCR_EL1_SHIFT) +#define HFGWTR_EL2_TCR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_TCR_EL1_SHIFT) +#define HFGWTR_EL2_SCXTNUM_EL0_SHIFT 31 +#define HFGWTR_EL2_SCXTNUM_EL0_MASK (UL(0x1) << HFGWTR_EL2_SCXTNUM_EL0_SHIFT) +#define HFGWTR_EL2_SCXTNUM_EL0_VAL(x) ((x) & HFGWTR_EL2_SCXTNUM_EL0_MASK) +#define HFGWTR_EL2_SCXTNUM_EL0_NOTRAP (UL(0x0) << HFGWTR_EL2_SCXTNUM_EL0_SHIFT) +#define HFGWTR_EL2_SCXTNUM_EL0_TRAP (UL(0x1) << HFGWTR_EL2_SCXTNUM_EL0_SHIFT) +#define HFGWTR_EL2_SCXTNUM_EL1_SHIFT 30 +#define HFGWTR_EL2_SCXTNUM_EL1_MASK (UL(0x1) << HFGWTR_EL2_SCXTNUM_EL1_SHIFT) +#define HFGWTR_EL2_SCXTNUM_EL1_VAL(x) ((x) & HFGWTR_EL2_SCXTNUM_EL1_MASK) +#define HFGWTR_EL2_SCXTNUM_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_SCXTNUM_EL1_SHIFT) +#define HFGWTR_EL2_SCXTNUM_EL1_TRAP (UL(0x1) << HFGWTR_EL2_SCXTNUM_EL1_SHIFT) +#define HFGWTR_EL2_SCTLR_EL1_SHIFT 29 +#define HFGWTR_EL2_SCTLR_EL1_MASK (UL(0x1) << HFGWTR_EL2_SCTLR_EL1_SHIFT) +#define HFGWTR_EL2_SCTLR_EL1_VAL(x) ((x) & HFGWTR_EL2_SCTLR_EL1_MASK) +#define HFGWTR_EL2_SCTLR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_SCTLR_EL1_SHIFT) +#define HFGWTR_EL2_SCTLR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_SCTLR_EL1_SHIFT) +#define HFGWTR_EL2_PAR_EL1_SHIFT 27 +#define HFGWTR_EL2_PAR_EL1_MASK (UL(0x1) << HFGWTR_EL2_PAR_EL1_SHIFT) +#define HFGWTR_EL2_PAR_EL1_VAL(x) ((x) & HFGWTR_EL2_PAR_EL1_MASK) +#define HFGWTR_EL2_PAR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_PAR_EL1_SHIFT) +#define HFGWTR_EL2_PAR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_PAR_EL1_SHIFT) +#define HFGWTR_EL2_MAIR_EL1_SHIFT 24 +#define HFGWTR_EL2_MAIR_EL1_MASK (UL(0x1) << HFGWTR_EL2_MAIR_EL1_SHIFT) +#define HFGWTR_EL2_MAIR_EL1_VAL(x) ((x) & HFGWTR_EL2_MAIR_EL1_MASK) +#define HFGWTR_EL2_MAIR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_MAIR_EL1_SHIFT) +#define HFGWTR_EL2_MAIR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_MAIR_EL1_SHIFT) +#define HFGWTR_EL2_LORSA_EL1_SHIFT 23 +#define HFGWTR_EL2_LORSA_EL1_MASK (UL(0x1) << HFGWTR_EL2_LORSA_EL1_SHIFT) +#define HFGWTR_EL2_LORSA_EL1_VAL(x) ((x) & HFGWTR_EL2_LORSA_EL1_MASK) +#define HFGWTR_EL2_LORSA_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_LORSA_EL1_SHIFT) +#define HFGWTR_EL2_LORSA_EL1_TRAP (UL(0x1) << HFGWTR_EL2_LORSA_EL1_SHIFT) +#define HFGWTR_EL2_LORN_EL1_SHIFT 22 +#define HFGWTR_EL2_LORN_EL1_MASK (UL(0x1) << HFGWTR_EL2_LORN_EL1_SHIFT) +#define HFGWTR_EL2_LORN_EL1_VAL(x) ((x) & HFGWTR_EL2_LORN_EL1_MASK) +#define HFGWTR_EL2_LORN_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_LORN_EL1_SHIFT) +#define HFGWTR_EL2_LORN_EL1_TRAP (UL(0x1) << HFGWTR_EL2_LORN_EL1_SHIFT) +#define HFGWTR_EL2_LOREA_EL1_SHIFT 20 +#define HFGWTR_EL2_LOREA_EL1_MASK (UL(0x1) << HFGWTR_EL2_LOREA_EL1_SHIFT) +#define HFGWTR_EL2_LOREA_EL1_VAL(x) ((x) & HFGWTR_EL2_LOREA_EL1_MASK) +#define HFGWTR_EL2_LOREA_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_LOREA_EL1_SHIFT) +#define HFGWTR_EL2_LOREA_EL1_TRAP (UL(0x1) << HFGWTR_EL2_LOREA_EL1_SHIFT) +#define HFGWTR_EL2_LORC_EL1_SHIFT 19 +#define HFGWTR_EL2_LORC_EL1_MASK (UL(0x1) << HFGWTR_EL2_LORC_EL1_SHIFT) +#define HFGWTR_EL2_LORC_EL1_VAL(x) ((x) & HFGWTR_EL2_LORC_EL1_MASK) +#define HFGWTR_EL2_LORC_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_LORC_EL1_SHIFT) +#define HFGWTR_EL2_LORC_EL1_TRAP (UL(0x1) << HFGWTR_EL2_LORC_EL1_SHIFT) +#define HFGWTR_EL2_FAR_EL1_SHIFT 17 +#define HFGWTR_EL2_FAR_EL1_MASK (UL(0x1) << HFGWTR_EL2_FAR_EL1_SHIFT) +#define HFGWTR_EL2_FAR_EL1_VAL(x) ((x) & HFGWTR_EL2_FAR_EL1_MASK) +#define HFGWTR_EL2_FAR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_FAR_EL1_SHIFT) +#define HFGWTR_EL2_FAR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_FAR_EL1_SHIFT) +#define HFGWTR_EL2_ESR_EL1_SHIFT 16 +#define HFGWTR_EL2_ESR_EL1_MASK (UL(0x1) << HFGWTR_EL2_ESR_EL1_SHIFT) +#define HFGWTR_EL2_ESR_EL1_VAL(x) ((x) & HFGWTR_EL2_ESR_EL1_MASK) +#define HFGWTR_EL2_ESR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_ESR_EL1_SHIFT) +#define HFGWTR_EL2_ESR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_ESR_EL1_SHIFT) +#define HFGWTR_EL2_CSSELR_EL1_SHIFT 13 +#define HFGWTR_EL2_CSSELR_EL1_MASK (UL(0x1) << HFGWTR_EL2_CSSELR_EL1_SHIFT) +#define HFGWTR_EL2_CSSELR_EL1_VAL(x) ((x) & HFGWTR_EL2_CSSELR_EL1_MASK) +#define HFGWTR_EL2_CSSELR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_CSSELR_EL1_SHIFT) +#define HFGWTR_EL2_CSSELR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_CSSELR_EL1_SHIFT) +#define HFGWTR_EL2_CPACR_EL1_SHIFT 12 +#define HFGWTR_EL2_CPACR_EL1_MASK (UL(0x1) << HFGWTR_EL2_CPACR_EL1_SHIFT) +#define HFGWTR_EL2_CPACR_EL1_VAL(x) ((x) & HFGWTR_EL2_CPACR_EL1_MASK) +#define HFGWTR_EL2_CPACR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_CPACR_EL1_SHIFT) +#define HFGWTR_EL2_CPACR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_CPACR_EL1_SHIFT) +#define HFGWTR_EL2_CONTEXTIDR_EL1_SHIFT 11 +#define HFGWTR_EL2_CONTEXTIDR_EL1_MASK (UL(0x1) << HFGWTR_EL2_CONTEXTIDR_EL1_SHIFT) +#define HFGWTR_EL2_CONTEXTIDR_EL1_VAL(x) ((x) & HFGWTR_EL2_CONTEXTIDR_EL1_MASK) +#define HFGWTR_EL2_CONTEXTIDR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_CONTEXTIDR_EL1_SHIFT) +#define HFGWTR_EL2_CONTEXTIDR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_CONTEXTIDR_EL1_SHIFT) +#define HFGWTR_EL2_APIBKey_SHIFT 8 +#define HFGWTR_EL2_APIBKey_MASK (UL(0x1) << HFGWTR_EL2_APIBKey_SHIFT) +#define HFGWTR_EL2_APIBKey_VAL(x) ((x) & HFGWTR_EL2_APIBKey_MASK) +#define HFGWTR_EL2_APIBKey_NOTRAP (UL(0x0) << HFGWTR_EL2_APIBKey_SHIFT) +#define HFGWTR_EL2_APIBKey_TRAP (UL(0x1) << HFGWTR_EL2_APIBKey_SHIFT) +#define HFGWTR_EL2_APIAKey_SHIFT 7 +#define HFGWTR_EL2_APIAKey_MASK (UL(0x1) << HFGWTR_EL2_APIAKey_SHIFT) +#define HFGWTR_EL2_APIAKey_VAL(x) ((x) & HFGWTR_EL2_APIAKey_MASK) +#define HFGWTR_EL2_APIAKey_NOTRAP (UL(0x0) << HFGWTR_EL2_APIAKey_SHIFT) +#define HFGWTR_EL2_APIAKey_TRAP (UL(0x1) << HFGWTR_EL2_APIAKey_SHIFT) +#define HFGWTR_EL2_APGAKey_SHIFT 6 +#define HFGWTR_EL2_APGAKey_MASK (UL(0x1) << HFGWTR_EL2_APGAKey_SHIFT) +#define HFGWTR_EL2_APGAKey_VAL(x) ((x) & HFGWTR_EL2_APGAKey_MASK) +#define HFGWTR_EL2_APGAKey_NOTRAP (UL(0x0) << HFGWTR_EL2_APGAKey_SHIFT) +#define HFGWTR_EL2_APGAKey_TRAP (UL(0x1) << HFGWTR_EL2_APGAKey_SHIFT) +#define HFGWTR_EL2_APDBKey_SHIFT 5 +#define HFGWTR_EL2_APDBKey_MASK (UL(0x1) << HFGWTR_EL2_APDBKey_SHIFT) +#define HFGWTR_EL2_APDBKey_VAL(x) ((x) & HFGWTR_EL2_APDBKey_MASK) +#define HFGWTR_EL2_APDBKey_NOTRAP (UL(0x0) << HFGWTR_EL2_APDBKey_SHIFT) +#define HFGWTR_EL2_APDBKey_TRAP (UL(0x1) << HFGWTR_EL2_APDBKey_SHIFT) +#define HFGWTR_EL2_APDAKey_SHIFT 4 +#define HFGWTR_EL2_APDAKey_MASK (UL(0x1) << HFGWTR_EL2_APDAKey_SHIFT) +#define HFGWTR_EL2_APDAKey_VAL(x) ((x) & HFGWTR_EL2_APDAKey_MASK) +#define HFGWTR_EL2_APDAKey_NOTRAP (UL(0x0) << HFGWTR_EL2_APDAKey_SHIFT) +#define HFGWTR_EL2_APDAKey_TRAP (UL(0x1) << HFGWTR_EL2_APDAKey_SHIFT) +#define HFGWTR_EL2_AMAIR_EL1_SHIFT 3 +#define HFGWTR_EL2_AMAIR_EL1_MASK (UL(0x1) << HFGWTR_EL2_AMAIR_EL1_SHIFT) +#define HFGWTR_EL2_AMAIR_EL1_VAL(x) ((x) & HFGWTR_EL2_AMAIR_EL1_MASK) +#define HFGWTR_EL2_AMAIR_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_AMAIR_EL1_SHIFT) +#define HFGWTR_EL2_AMAIR_EL1_TRAP (UL(0x1) << HFGWTR_EL2_AMAIR_EL1_SHIFT) +#define HFGWTR_EL2_AFSR1_EL1_SHIFT 1 +#define HFGWTR_EL2_AFSR1_EL1_MASK (UL(0x1) << HFGWTR_EL2_AFSR1_EL1_SHIFT) +#define HFGWTR_EL2_AFSR1_EL1_VAL(x) ((x) & HFGWTR_EL2_AFSR1_EL1_MASK) +#define HFGWTR_EL2_AFSR1_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_AFSR1_EL1_SHIFT) +#define HFGWTR_EL2_AFSR1_EL1_TRAP (UL(0x1) << HFGWTR_EL2_AFSR1_EL1_SHIFT) +#define HFGWTR_EL2_AFSR0_EL1_SHIFT 0 +#define HFGWTR_EL2_AFSR0_EL1_MASK (UL(0x1) << HFGWTR_EL2_AFSR0_EL1_SHIFT) +#define HFGWTR_EL2_AFSR0_EL1_VAL(x) ((x) & HFGWTR_EL2_AFSR0_EL1_MASK) +#define HFGWTR_EL2_AFSR0_EL1_NOTRAP (UL(0x0) << HFGWTR_EL2_AFSR0_EL1_SHIFT) +#define HFGWTR_EL2_AFSR0_EL1_TRAP (UL(0x1) << HFGWTR_EL2_AFSR0_EL1_SHIFT) + /* HPFAR_EL2 - Hypervisor IPA Fault Address Register */ #define HPFAR_EL2_FIPA_SHIFT 4 #define HPFAR_EL2_FIPA_MASK 0xfffffffff0 diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h index f530dab05331..6062d1c07ae4 100644 --- a/sys/arm64/vmm/arm64.h +++ b/sys/arm64/vmm/arm64.h @@ -100,6 +100,22 @@ struct hypctx { uint64_t mdcr_el2; /* Monitor Debug Configuration Register */ uint64_t vpidr_el2; /* Virtualization Processor ID Register */ uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */ + + /* FEAT_FGT registers */ + /*uint64_t hafgrtr_el2; *//* For FEAT_AMUv1 (not supported) */ + uint64_t hdfgrtr_el2; + uint64_t hdfgwtr_el2; + uint64_t hfgitr_el2; + uint64_t hfgrtr_el2; + uint64_t hfgwtr_el2; + + /* FEAT_FGT2 registers */ + uint64_t hdfgrtr2_el2; + uint64_t hdfgwtr2_el2; + uint64_t hfgitr2_el2; + uint64_t hfgrtr2_el2; + uint64_t hfgwtr2_el2; + uint64_t el2_addr; /* The address of this in el2 space */ struct hyp *hyp; struct vcpu *vcpu; @@ -131,6 +147,8 @@ struct hyp { uint64_t feats; /* Which features are enabled */ #define HYP_FEAT_HCX (0x1ul << 0) #define HYP_FEAT_ECV_POFF (0x1ul << 1) +#define HYP_FEAT_FGT (0x1ul << 2) +#define HYP_FEAT_FGT2 (0x1ul << 3) bool vgic_attached; struct vgic_v3 *vgic; struct hypctx *ctx[]; diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c index d529f000b828..6643d32684d8 100644 --- a/sys/arm64/vmm/vmm_arm64.c +++ b/sys/arm64/vmm/vmm_arm64.c @@ -525,6 +525,18 @@ vmmops_init(struct vm *vm, pmap_t pmap) if (get_kernel_reg(ID_AA64MMFR0_EL1, &idreg)) { if (ID_AA64MMFR0_ECV_VAL(idreg) >= ID_AA64MMFR0_ECV_POFF) hyp->feats |= HYP_FEAT_ECV_POFF; + + switch (ID_AA64MMFR0_FGT_VAL(idreg)) { + case ID_AA64MMFR0_FGT_NONE: + break; + default: + case ID_AA64MMFR0_FGT_8_9: + hyp->feats |= HYP_FEAT_FGT2; + /* FALLTHROUGH */ + case ID_AA64MMFR0_FGT_8_6: + hyp->feats |= HYP_FEAT_FGT; + break; + } } if (get_kernel_reg(ID_AA64MMFR1_EL1, &idreg)) { diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c index 0ad7930e9a87..75c15f06dd08 100644 --- a/sys/arm64/vmm/vmm_hyp.c +++ b/sys/arm64/vmm/vmm_hyp.c @@ -293,6 +293,28 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest, } isb(); +#ifdef VMM_VHE + if (guest) { + /* Fine-grained trap controls */ + if ((hyp->feats & HYP_FEAT_FGT) != 0) { + WRITE_SPECIALREG(HDFGWTR_EL2_REG, hypctx->hdfgwtr_el2); + WRITE_SPECIALREG(HFGITR_EL2_REG, hypctx->hfgitr_el2); + WRITE_SPECIALREG(HFGRTR_EL2_REG, hypctx->hfgrtr_el2); + WRITE_SPECIALREG(HFGWTR_EL2_REG, hypctx->hfgwtr_el2); + } + + if ((hyp->feats & HYP_FEAT_FGT2) != 0) { + WRITE_SPECIALREG(HDFGRTR2_EL2_REG, + hypctx->hdfgrtr2_el2); + WRITE_SPECIALREG(HDFGWTR2_EL2_REG, + hypctx->hdfgwtr2_el2); + WRITE_SPECIALREG(HFGITR2_EL2_REG, hypctx->hfgitr2_el2); + WRITE_SPECIALREG(HFGRTR2_EL2_REG, hypctx->hfgrtr2_el2); + WRITE_SPECIALREG(HFGWTR2_EL2_REG, hypctx->hfgwtr2_el2); + } + } +#endif + WRITE_SPECIALREG(sp_el0, hypctx->sp_el0); WRITE_SPECIALREG(tpidr_el0, hypctx->tpidr_el0); WRITE_SPECIALREG(tpidrro_el0, hypctx->tpidrro_el0); diff --git a/sys/arm64/vmm/vmm_mmu.c b/sys/arm64/vmm/vmm_mmu.c index 42537254e27b..ab876d4b47b6 100644 --- a/sys/arm64/vmm/vmm_mmu.c +++ b/sys/arm64/vmm/vmm_mmu.c @@ -350,8 +350,8 @@ vmmpmap_remove(vm_offset_t va, vm_size_t size, bool invalidate) ("%s: Mapping is not page-sized", __func__)); if (invalidate) { - l3_list = malloc((size / PAGE_SIZE) * sizeof(l3_list[0]), - M_TEMP, M_WAITOK | M_ZERO); + l3_list = malloc(atop(size) * sizeof(l3_list[0]), M_TEMP, + M_WAITOK | M_ZERO); } sva = va; @@ -359,32 +359,17 @@ vmmpmap_remove(vm_offset_t va, vm_size_t size, bool invalidate) mtx_lock(&vmmpmap_mtx); for (i = 0; va < eva; va = va_next) { l0e = atomic_load_64(&l0[pmap_l0_index(va)]); - if (l0e == 0) { - va_next = (va + L0_SIZE) & ~L0_OFFSET; - if (va_next < va) - va_next = eva; - continue; - } + MPASS(l0e != 0); MPASS((l0e & ATTR_DESCR_MASK) == L0_TABLE); l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK); l1e = atomic_load_64(&l1[pmap_l1_index(va)]); - if (l1e == 0) { - va_next = (va + L1_SIZE) & ~L1_OFFSET; - if (va_next < va) - va_next = eva; - continue; - } + MPASS(l1e != 0); MPASS((l1e & ATTR_DESCR_MASK) == L1_TABLE); l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK); l2e = atomic_load_64(&l2[pmap_l2_index(va)]); - if (l2e == 0) { - va_next = (va + L2_SIZE) & ~L2_OFFSET; - if (va_next < va) - va_next = eva; - continue; - } + MPASS(l2e != 0); MPASS((l2e & ATTR_DESCR_MASK) == L2_TABLE); l3 = (pd_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK); @@ -419,7 +404,7 @@ vmmpmap_remove(vm_offset_t va, vm_size_t size, bool invalidate) /* Invalidate the memory from the D-cache */ vmm_call_hyp(HYP_DC_CIVAC, sva, size); - for (i = 0; i < (size / PAGE_SIZE); i++) { + for (i = 0; i < atop(size); i++) { atomic_store_64(l3_list[i], 0); } diff --git a/sys/arm64/vmm/vmm_reset.c b/sys/arm64/vmm/vmm_reset.c index 06ac6dec5af8..fd3414ecd066 100644 --- a/sys/arm64/vmm/vmm_reset.c +++ b/sys/arm64/vmm/vmm_reset.c @@ -187,4 +187,409 @@ reset_vm_el2_regs(void *vcpu) el2ctx->tf.tf_spsr = PSR_D | PSR_A | PSR_I | PSR_F; /* Use the EL1 stack when taking exceptions to EL1 */ el2ctx->tf.tf_spsr |= PSR_M_EL1h; + + /* FEAT_FGT traps */ + if ((el2ctx->hyp->feats & HYP_FEAT_FGT) != 0) { +#define HFGT_TRAP_FIELDS(read, write, read_pfx, write_pfx, name, trap) \ +do { \ + el2ctx->read |= read_pfx ## _EL2_ ## name ## _ ## trap; \ + el2ctx->write |= write_pfx ## _EL2_ ## name ## _ ## trap; \ +} while (0) + + + /* + * Traps for special registers + */ + + /* Debug registers */ + el2ctx->hdfgrtr_el2 = 0; + el2ctx->hdfgwtr_el2 = 0; + + /* FEAT_BRBE */ + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + nBRBDATA, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + nBRBCTL, TRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_nBRBIDR_TRAP; + + /* FEAT_TRBE */ + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRBTRG_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRBSR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRBPTR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRBMAR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRBLIMITR_EL1, TRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_TRBIDR_EL1_TRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRBBASER_EL1, TRAP); + + /* FEAT_TRF */ + el2ctx->hdfgwtr_el2 |= HDFGWTR_EL2_TRFCR_EL1_TRAP; + + /* FEAT_ETE */ + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRCVICTLR, TRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_TRCSTATR_TRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRCSSCSRn, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRCSEQSTR, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRCPRGCTLR, TRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_TRCOSLSR_TRAP; + el2ctx->hdfgwtr_el2 |= HDFGWTR_EL2_TRCOSLAR_TRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRCIMSPECn, TRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_TRCID_TRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRCCNTVRn, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRCCLAIM, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRCAUXCTLR, TRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_TRCAUTHSTATUS_TRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + TRC, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMSLATFR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMSIRR_EL1, TRAP); + + /* FEAT_SPE */ + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_PMBIDR_EL1_TRAP; + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_PMSIDR_EL1_TRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMSICR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMSFCR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMSEVFR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMSCR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMBSR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMBPTR_EL1, TRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMBLIMITR_EL1, TRAP); + + /* FEAT_SPE_FnE */ + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + nPMSNEVFR_EL1, TRAP); + + /* FEAT_PMUv3 */ + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_PMCEIDn_EL0_NOTRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMUSERENR_EL0, NOTRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_PMMIR_EL1_NOTRAP; + el2ctx->hdfgwtr_el2 |= HDFGWTR_EL2_PMCR_EL0_NOTRAP; + el2ctx->hdfgwtr_el2 |= HDFGWTR_EL2_PMSWINC_EL0_NOTRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMSELR_EL0, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMOVS, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMINTEN, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMCNTEN, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMCCNTR_EL0, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMCCFILTR_EL0, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMEVTYPERn_EL0, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + PMEVCNTRn_EL0, NOTRAP); + + /* FEAT_DoubleLock */ + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + OSDLR_EL1, TRAP); + + /* Base architecture */ + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + OSECCR_EL1, NOTRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_OSLSR_EL1_NOTRAP; + el2ctx->hdfgwtr_el2 |= HDFGWTR_EL2_OSLAR_EL1_NOTRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + DBGPRCR_EL1, NOTRAP); + el2ctx->hdfgrtr_el2 |= HDFGRTR_EL2_DBGAUTHSTATUS_EL1_NOTRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + DBGCLAIM, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + MDSCR_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + DBGWVRn_EL1, NOTRAP); + el2ctx->hdfgwtr_el2 |= HDFGWTR_EL2_DBGWCRn_EL1_NOTRAP; + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + DBGBVRn_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hdfgrtr_el2, hdfgwtr_el2, HDFGRTR, HDFGWTR, + DBGBCRn_EL1, NOTRAP); + + + /* Non-debug special registers */ + el2ctx->hfgrtr_el2 = 0; + el2ctx->hfgwtr_el2 = 0; + + /* FEAT_AIE */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nAMAIR2_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nMAIR2_EL1, TRAP); + + /* FEAT_S2POE */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nS2POR_EL1, TRAP); + + /* FEAT_S1POE */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nPOR_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nPOR_EL0, TRAP); + + /* FEAT_S1PIE */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nPIR_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nPIRE0_EL1, TRAP); + + /* FEAT_THE */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nRCWMASK_EL1, TRAP); + + /* FEAT_SME */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nTPIDR2_EL0, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nSMPRI_EL1, TRAP); + + /* FEAT_GCS */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nGCS_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nGCS_EL0, TRAP); + + /* FEAT_LS64_ACCDATA */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + nACCDATA_EL1, TRAP); + + /* FEAT_RASv1p1 */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ERXPFGCDN_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ERXPFGCTL_EL1, TRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_ERXPFGF_EL1_TRAP; + + /* FEAT_RAS */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ERXADDR_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ERXMISCn_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ERXSTATUS_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ERXCTLR_EL1, TRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_ERXFR_EL1_TRAP; + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ERRSELR_EL1, TRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_ERRIDR_EL1_TRAP; + + /* GICv3 */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ICC_IGRPENn_EL1, NOTRAP); + + /* FEAT_LOR */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + LORSA_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + LORN_EL1, TRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_LORID_EL1_TRAP; + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + LOREA_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + LORC_EL1, TRAP); + + /* FEAT_PAuth */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + APIBKey, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + APIAKey, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + APGAKey, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + APDBKey, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + APDAKey, TRAP); + + /* Base architecture */ + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + VBAR_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + TTBR1_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + TTBR0_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + TPIDR_EL0, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + TPIDRRO_EL0, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + TPIDR_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + TCR_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + SCXTNUM_EL0, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + SCXTNUM_EL1, TRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + SCTLR_EL1, NOTRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_REVIDR_EL1_NOTRAP; + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + PAR_EL1, NOTRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_MPIDR_EL1_NOTRAP; + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_MIDR_EL1_NOTRAP; + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + MAIR_EL1, NOTRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_ISR_EL1_NOTRAP; + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + FAR_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + ESR_EL1, NOTRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_DCZID_EL0_NOTRAP; + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_CTR_EL0_NOTRAP; + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + CSSELR_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + CPACR_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + CONTEXTIDR_EL1, NOTRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_CLIDR_EL1_NOTRAP; + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_CCSIDR_EL1_NOTRAP; + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + AMAIR_EL1, NOTRAP); + el2ctx->hfgrtr_el2 |= HFGRTR_EL2_AIDR_EL1_NOTRAP; + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + AFSR1_EL1, NOTRAP); + HFGT_TRAP_FIELDS(hfgrtr_el2, hfgwtr_el2, HFGRTR, HFGWTR, + AFSR0_EL1, NOTRAP); + + /* + * Traps for instructions + */ + + /* Enable all TLBI, cache and AT variants */ + el2ctx->hfgitr_el2 = 0; + + /* FEAT_ATS1A */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_ATS1E1A_TRAP; + + /* FEAT_SPECRES2 */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_COSPRCTX_TRAP; + + /* FEAT_GCS */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_nGCSEPP_TRAP | + HFGITR_EL2_nGCSSTR_EL1_TRAP | + HFGITR_EL2_nGCSPUSHM_EL1_TRAP; + + /* FEAT_BRBE */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_nBRBIALL_TRAP | + HFGITR_EL2_nBRBINJ_TRAP; + + /* FEAT_SPECRES */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_CPPRCTX_TRAP | + HFGITR_EL2_DVPRCTX_TRAP | + HFGITR_EL2_CFPRCTX_TRAP; + + /* FEAT_TLBIRANGE */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_TLBIRVAALE1_TRAP | + HFGITR_EL2_TLBIRVALE1_TRAP | + HFGITR_EL2_TLBIRVAAE1_TRAP | + HFGITR_EL2_TLBIRVAE1_TRAP | + HFGITR_EL2_TLBIRVAALE1IS_TRAP | + HFGITR_EL2_TLBIRVALE1IS_TRAP | + HFGITR_EL2_TLBIRVAAE1IS_TRAP | + HFGITR_EL2_TLBIRVAE1IS_TRAP; + + /* FEAT_TLBIRANGE && FEAT_TLBIOS */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_TLBIRVAALE1OS_TRAP | + HFGITR_EL2_TLBIRVALE1OS_TRAP | + HFGITR_EL2_TLBIRVAAE1OS_TRAP | + HFGITR_EL2_TLBIRVAE1OS_TRAP; + + /* FEAT_TLBIOS */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_TLBIVAALE1OS_TRAP | + HFGITR_EL2_TLBIVALE1OS_TRAP | + HFGITR_EL2_TLBIVAAE1OS_TRAP | + HFGITR_EL2_TLBIASIDE1OS_TRAP | + HFGITR_EL2_TLBIVAE1OS_TRAP | + HFGITR_EL2_TLBIVMALLE1OS_TRAP; + + /* FEAT_PAN2 */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_ATS1E1WP_TRAP | + HFGITR_EL2_ATS1E1RP_TRAP; + + /* FEAT_DPB2 */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_DCCVADP_TRAP; + + /* Base architecture */ + el2ctx->hfgitr_el2 |= + HFGITR_EL2_DCCVAC_NOTRAP | + HFGITR_EL2_SVC_EL1_NOTRAP | + HFGITR_EL2_SVC_EL0_NOTRAP | + HFGITR_EL2_ERET_NOTRAP; + + el2ctx->hfgitr_el2 |= + HFGITR_EL2_TLBIVAALE1_NOTRAP | + HFGITR_EL2_TLBIVALE1_NOTRAP | + HFGITR_EL2_TLBIVAAE1_NOTRAP | + HFGITR_EL2_TLBIASIDE1_NOTRAP | + HFGITR_EL2_TLBIVAE1_NOTRAP | + HFGITR_EL2_TLBIVMALLE1_NOTRAP | + HFGITR_EL2_TLBIVAALE1IS_NOTRAP | + HFGITR_EL2_TLBIVALE1IS_NOTRAP | + HFGITR_EL2_TLBIVAAE1IS_NOTRAP | + HFGITR_EL2_TLBIASIDE1IS_NOTRAP | + HFGITR_EL2_TLBIVAE1IS_NOTRAP | + HFGITR_EL2_TLBIVMALLE1IS_NOTRAP; + + el2ctx->hfgitr_el2 |= + HFGITR_EL2_ATS1E0W_NOTRAP | + HFGITR_EL2_ATS1E0R_NOTRAP | + HFGITR_EL2_ATS1E1W_NOTRAP | + HFGITR_EL2_ATS1E1R_NOTRAP | + HFGITR_EL2_DCZVA_NOTRAP | + HFGITR_EL2_DCCIVAC_NOTRAP | + HFGITR_EL2_DCCVAP_NOTRAP | + HFGITR_EL2_DCCVAU_NOTRAP | + HFGITR_EL2_DCCISW_NOTRAP | + HFGITR_EL2_DCCSW_NOTRAP | + HFGITR_EL2_DCISW_NOTRAP | + HFGITR_EL2_DCIVAC_NOTRAP | + HFGITR_EL2_ICIVAU_NOTRAP | + HFGITR_EL2_ICIALLU_NOTRAP | + HFGITR_EL2_ICIALLUIS_NOTRAP; + + } + + /* FEAT_FGT2 traps */ + if ((el2ctx->hyp->feats & HYP_FEAT_FGT2) != 0) { + /* Trap everything here until we support the feature */ + el2ctx->hdfgrtr2_el2 = 0; + el2ctx->hdfgwtr2_el2 = 0; + el2ctx->hfgitr2_el2 = 0; + el2ctx->hfgrtr2_el2 = 0; + el2ctx->hfgwtr2_el2 = 0; + } } diff --git a/sys/bsm/audit_kevents.h b/sys/bsm/audit_kevents.h index 9381396f247c..ac6f5d69f07d 100644 --- a/sys/bsm/audit_kevents.h +++ b/sys/bsm/audit_kevents.h @@ -664,6 +664,7 @@ #define AUE_TIMERFD 43270 /* FreeBSD/Linux. */ #define AUE_SETCRED 43271 /* FreeBSD-specific. */ #define AUE_INOTIFY 43272 /* FreeBSD/Linux. */ +#define AUE_PDRFORK 43273 /* FreeBSD-specific. */ /* * Darwin BSM uses a number of AUE_O_* definitions, which are aliased to the diff --git a/sys/cam/cam_periph.c b/sys/cam/cam_periph.c index 7f6855a5d51d..91ec7a71f612 100644 --- a/sys/cam/cam_periph.c +++ b/sys/cam/cam_periph.c @@ -1791,7 +1791,6 @@ camperiphscsisenseerror(union ccb *ccb, union ccb **orig, /*start*/TRUE, /*load/eject*/le, /*immediate*/FALSE, - /*power_condition*/SSS_PC_START_VALID, SSD_FULL_SIZE, /*timeout*/50000); break; diff --git a/sys/cam/scsi/scsi_all.c b/sys/cam/scsi/scsi_all.c index d8bba97e79bc..76fcd3fee188 100644 --- a/sys/cam/scsi/scsi_all.c +++ b/sys/cam/scsi/scsi_all.c @@ -9009,6 +9009,40 @@ void scsi_start_stop(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int start, int load_eject, + int immediate, uint8_t sense_len, uint32_t timeout) +{ + struct scsi_start_stop_unit *scsi_cmd; + int extra_flags = 0; + + scsi_cmd = (struct scsi_start_stop_unit *)&csio->cdb_io.cdb_bytes; + bzero(scsi_cmd, sizeof(*scsi_cmd)); + scsi_cmd->opcode = START_STOP_UNIT; + if (start != 0) { + scsi_cmd->how |= SSS_START; + /* it takes a lot of power to start a drive */ + extra_flags |= CAM_HIGH_POWER; + } + if (load_eject != 0) + scsi_cmd->how |= SSS_LOEJ; + if (immediate != 0) + scsi_cmd->byte2 |= SSS_IMMED; + + cam_fill_csio(csio, + retries, + cbfcnp, + /*flags*/CAM_DIR_NONE | extra_flags, + tag_action, + /*data_ptr*/NULL, + /*dxfer_len*/0, + sense_len, + sizeof(*scsi_cmd), + timeout); +} + +void +scsi_start_stop_pc(struct ccb_scsiio *csio, uint32_t retries, + void (*cbfcnp)(struct cam_periph *, union ccb *), + uint8_t tag_action, int start, int load_eject, int immediate, uint8_t power_condition, uint8_t sense_len, uint32_t timeout) { diff --git a/sys/cam/scsi/scsi_all.h b/sys/cam/scsi/scsi_all.h index cadf2d1f2835..8aa87abce370 100644 --- a/sys/cam/scsi/scsi_all.h +++ b/sys/cam/scsi/scsi_all.h @@ -4350,6 +4350,10 @@ void scsi_unmap(struct ccb_scsiio *csio, uint32_t retries, void scsi_start_stop(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int start, int load_eject, + int immediate, uint8_t sense_len, uint32_t timeout); +void scsi_start_stop_pc(struct ccb_scsiio *csio, uint32_t retries, + void (*cbfcnp)(struct cam_periph *, union ccb *), + uint8_t tag_action, int start, int load_eject, int immediate, uint8_t power_condition, uint8_t sense_len, uint32_t timeout); void scsi_read_attribute(struct ccb_scsiio *csio, uint32_t retries, diff --git a/sys/cam/scsi/scsi_cd.c b/sys/cam/scsi/scsi_cd.c index bc4634562dc6..e622a96ec77e 100644 --- a/sys/cam/scsi/scsi_cd.c +++ b/sys/cam/scsi/scsi_cd.c @@ -3406,7 +3406,6 @@ cdstartunit(struct cam_periph *periph, int load) /* start */ TRUE, /* load_eject */ load, /* immediate */ FALSE, - /* power_condition */ SSS_PC_START_VALID, /* sense_len */ SSD_FULL_SIZE, /* timeout */ 50000); @@ -3435,7 +3434,6 @@ cdstopunit(struct cam_periph *periph, uint32_t eject) /* start */ FALSE, /* load_eject */ eject, /* immediate */ FALSE, - /* power_condition */ SSS_PC_START_VALID, /* sense_len */ SSD_FULL_SIZE, /* timeout */ 50000); diff --git a/sys/compat/freebsd32/freebsd32.h b/sys/compat/freebsd32/freebsd32.h index 7d21a5be5570..25703859a7bb 100644 --- a/sys/compat/freebsd32/freebsd32.h +++ b/sys/compat/freebsd32/freebsd32.h @@ -30,6 +30,9 @@ #define _COMPAT_FREEBSD32_FREEBSD32_H_ #include <sys/abi_compat.h> +#include <sys/devicestat.h> +#include <sys/event.h> +#include <sys/mount.h> #include <sys/procfs.h> #include <sys/socket.h> #include <sys/user.h> @@ -540,4 +543,28 @@ struct ptrace_sc_remote32 { uint32_t pscr_args; }; +struct devstat32 { + u_int sequence0; + int allocated; + u_int start_count; + u_int end_count; + struct bintime32 busy_from; + struct { u_int32_t stqe_next; } dev_links; + u_int32_t device_number; + char device_name[DEVSTAT_NAME_LEN]; + int unit_number; + freebsd32_uint64_t bytes[DEVSTAT_N_TRANS_FLAGS]; + freebsd32_uint64_t operations[DEVSTAT_N_TRANS_FLAGS]; + struct bintime32 duration[DEVSTAT_N_TRANS_FLAGS]; + struct bintime32 busy_time; + struct bintime32 creation_time; + u_int32_t block_size; + freebsd32_uint64_t tag_types[3]; + devstat_support_flags flags; + devstat_type_flags device_type; + devstat_priority priority; + u_int32_t id; + u_int sequence1; +}; + #endif /* !_COMPAT_FREEBSD32_FREEBSD32_H_ */ diff --git a/sys/compat/freebsd32/freebsd32_misc.c b/sys/compat/freebsd32/freebsd32_misc.c index 7913940338c2..c76c9d5c1838 100644 --- a/sys/compat/freebsd32/freebsd32_misc.c +++ b/sys/compat/freebsd32/freebsd32_misc.c @@ -203,6 +203,7 @@ void freebsd32_rusage_out(const struct rusage *s, struct rusage32 *s32) { + bzero(s32, sizeof(*s32)); TV_CP(*s, *s32, ru_utime); TV_CP(*s, *s32, ru_stime); CP(*s, *s32, ru_maxrss); @@ -280,6 +281,37 @@ freebsd32_wait6(struct thread *td, struct freebsd32_wait6_args *uap) return (error); } +int +freebsd32_pdwait(struct thread *td, struct freebsd32_pdwait_args *uap) +{ + struct __wrusage32 wru32; + struct __wrusage wru, *wrup; + struct __siginfo32 si32; + struct __siginfo si, *sip; + int error, status; + + wrup = uap->wrusage != NULL ? &wru : NULL; + if (uap->info != NULL) { + sip = &si; + bzero(sip, sizeof(*sip)); + } else { + sip = NULL; + } + error = kern_pdwait(td, uap->fd, &status, uap->options, wrup, sip); + if (uap->status != NULL && error == 0) + error = copyout(&status, uap->status, sizeof(status)); + if (uap->wrusage != NULL && error == 0) { + freebsd32_rusage_out(&wru.wru_self, &wru32.wru_self); + freebsd32_rusage_out(&wru.wru_children, &wru32.wru_children); + error = copyout(&wru32, uap->wrusage, sizeof(wru32)); + } + if (uap->info != NULL && error == 0) { + siginfo_to_siginfo32 (&si, &si32); + error = copyout(&si32, uap->info, sizeof(si32)); + } + return (error); +} + #ifdef COMPAT_FREEBSD4 static void copy_statfs(struct statfs *in, struct ostatfs32 *out) diff --git a/sys/compat/freebsd32/freebsd32_proto.h b/sys/compat/freebsd32/freebsd32_proto.h index 5c0efc64f8a7..12458ed4cc4d 100644 --- a/sys/compat/freebsd32/freebsd32_proto.h +++ b/sys/compat/freebsd32/freebsd32_proto.h @@ -699,6 +699,13 @@ struct freebsd32_setcred_args { char wcred_l_[PADL_(const struct setcred32 *)]; const struct setcred32 * wcred; char wcred_r_[PADR_(const struct setcred32 *)]; char size_l_[PADL_(size_t)]; size_t size; char size_r_[PADR_(size_t)]; }; +struct freebsd32_pdwait_args { + char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)]; + char status_l_[PADL_(int *)]; int * status; char status_r_[PADR_(int *)]; + char options_l_[PADL_(int)]; int options; char options_r_[PADR_(int)]; + char wrusage_l_[PADL_(struct __wrusage32 *)]; struct __wrusage32 * wrusage; char wrusage_r_[PADR_(struct __wrusage32 *)]; + char info_l_[PADL_(struct __siginfo32 *)]; struct __siginfo32 * info; char info_r_[PADR_(struct __siginfo32 *)]; +}; int freebsd32_wait4(struct thread *, struct freebsd32_wait4_args *); int freebsd32_ptrace(struct thread *, struct freebsd32_ptrace_args *); int freebsd32_recvmsg(struct thread *, struct freebsd32_recvmsg_args *); @@ -817,6 +824,7 @@ int freebsd32_aio_readv(struct thread *, struct freebsd32_aio_readv_args *); int freebsd32_timerfd_gettime(struct thread *, struct freebsd32_timerfd_gettime_args *); int freebsd32_timerfd_settime(struct thread *, struct freebsd32_timerfd_settime_args *); int freebsd32_setcred(struct thread *, struct freebsd32_setcred_args *); +int freebsd32_pdwait(struct thread *, struct freebsd32_pdwait_args *); #ifdef COMPAT_43 @@ -1319,6 +1327,7 @@ int freebsd11_freebsd32_fstatat(struct thread *, struct freebsd11_freebsd32_fsta #define FREEBSD32_SYS_AUE_freebsd32_timerfd_gettime AUE_TIMERFD #define FREEBSD32_SYS_AUE_freebsd32_timerfd_settime AUE_TIMERFD #define FREEBSD32_SYS_AUE_freebsd32_setcred AUE_SETCRED +#define FREEBSD32_SYS_AUE_freebsd32_pdwait AUE_PDWAIT #undef PAD_ #undef PADL_ diff --git a/sys/compat/freebsd32/freebsd32_syscall.h b/sys/compat/freebsd32/freebsd32_syscall.h index f8ef7e4a20d3..67ff022922a8 100644 --- a/sys/compat/freebsd32/freebsd32_syscall.h +++ b/sys/compat/freebsd32/freebsd32_syscall.h @@ -517,4 +517,6 @@ #define FREEBSD32_SYS_setgroups 596 #define FREEBSD32_SYS_jail_attach_jd 597 #define FREEBSD32_SYS_jail_remove_jd 598 -#define FREEBSD32_SYS_MAXSYSCALL 600 +#define FREEBSD32_SYS_pdrfork 600 +#define FREEBSD32_SYS_freebsd32_pdwait 601 +#define FREEBSD32_SYS_MAXSYSCALL 602 diff --git a/sys/compat/freebsd32/freebsd32_syscalls.c b/sys/compat/freebsd32/freebsd32_syscalls.c index 645cdccbc02d..54b826098a9d 100644 --- a/sys/compat/freebsd32/freebsd32_syscalls.c +++ b/sys/compat/freebsd32/freebsd32_syscalls.c @@ -605,4 +605,6 @@ const char *freebsd32_syscallnames[] = { "jail_attach_jd", /* 597 = jail_attach_jd */ "jail_remove_jd", /* 598 = jail_remove_jd */ "#599", /* 599 = kexec_load */ + "pdrfork", /* 600 = pdrfork */ + "freebsd32_pdwait", /* 601 = freebsd32_pdwait */ }; diff --git a/sys/compat/freebsd32/freebsd32_sysent.c b/sys/compat/freebsd32/freebsd32_sysent.c index 240b54ae9011..2b5609e8a317 100644 --- a/sys/compat/freebsd32/freebsd32_sysent.c +++ b/sys/compat/freebsd32/freebsd32_sysent.c @@ -667,4 +667,6 @@ struct sysent freebsd32_sysent[] = { { .sy_narg = AS(jail_attach_jd_args), .sy_call = (sy_call_t *)sys_jail_attach_jd, .sy_auevent = AUE_JAIL_ATTACH, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 597 = jail_attach_jd */ { .sy_narg = AS(jail_remove_jd_args), .sy_call = (sy_call_t *)sys_jail_remove_jd, .sy_auevent = AUE_JAIL_REMOVE, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 598 = jail_remove_jd */ { .sy_narg = 0, .sy_call = (sy_call_t *)nosys, .sy_auevent = AUE_NULL, .sy_flags = 0, .sy_thrcnt = SY_THR_ABSENT }, /* 599 = freebsd32_kexec_load */ + { .sy_narg = AS(pdrfork_args), .sy_call = (sy_call_t *)sys_pdrfork, .sy_auevent = AUE_PDRFORK, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 600 = pdrfork */ + { .sy_narg = AS(freebsd32_pdwait_args), .sy_call = (sy_call_t *)freebsd32_pdwait, .sy_auevent = AUE_PDWAIT, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 601 = freebsd32_pdwait */ }; diff --git a/sys/compat/freebsd32/freebsd32_systrace_args.c b/sys/compat/freebsd32/freebsd32_systrace_args.c index 29a5497e9efa..59a74d365e1c 100644 --- a/sys/compat/freebsd32/freebsd32_systrace_args.c +++ b/sys/compat/freebsd32/freebsd32_systrace_args.c @@ -3427,6 +3427,26 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) *n_args = 1; break; } + /* pdrfork */ + case 600: { + struct pdrfork_args *p = params; + uarg[a++] = (intptr_t)p->fdp; /* int * */ + iarg[a++] = p->pdflags; /* int */ + iarg[a++] = p->rfflags; /* int */ + *n_args = 3; + break; + } + /* freebsd32_pdwait */ + case 601: { + struct freebsd32_pdwait_args *p = params; + iarg[a++] = p->fd; /* int */ + uarg[a++] = (intptr_t)p->status; /* int * */ + iarg[a++] = p->options; /* int */ + uarg[a++] = (intptr_t)p->wrusage; /* struct __wrusage32 * */ + uarg[a++] = (intptr_t)p->info; /* struct __siginfo32 * */ + *n_args = 5; + break; + } default: *n_args = 0; break; @@ -9256,6 +9276,44 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) break; }; break; + /* pdrfork */ + case 600: + switch (ndx) { + case 0: + p = "userland int *"; + break; + case 1: + p = "int"; + break; + case 2: + p = "int"; + break; + default: + break; + }; + break; + /* freebsd32_pdwait */ + case 601: + switch (ndx) { + case 0: + p = "int"; + break; + case 1: + p = "userland int *"; + break; + case 2: + p = "int"; + break; + case 3: + p = "userland struct __wrusage32 *"; + break; + case 4: + p = "userland struct __siginfo32 *"; + break; + default: + break; + }; + break; default: break; }; @@ -11174,6 +11232,16 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) if (ndx == 0 || ndx == 1) p = "int"; break; + /* pdrfork */ + case 600: + if (ndx == 0 || ndx == 1) + p = "int"; + break; + /* freebsd32_pdwait */ + case 601: + if (ndx == 0 || ndx == 1) + p = "int"; + break; default: break; }; diff --git a/sys/compat/lindebugfs/lindebugfs.c b/sys/compat/lindebugfs/lindebugfs.c index 8cddc6f390bc..857546f61e55 100644 --- a/sys/compat/lindebugfs/lindebugfs.c +++ b/sys/compat/lindebugfs/lindebugfs.c @@ -272,6 +272,9 @@ debugfs_create_dir(const char *name, struct dentry *parent) struct dentry *dnode; struct pfs_node *pnode; + if (name == NULL) + return (NULL); + dm = malloc(sizeof(*dm), M_DFSINT, M_NOWAIT | M_ZERO); if (dm == NULL) return (NULL); diff --git a/sys/compat/linuxkpi/common/include/linux/bitops.h b/sys/compat/linuxkpi/common/include/linux/bitops.h index ebe9aa120094..125081ab5b74 100644 --- a/sys/compat/linuxkpi/common/include/linux/bitops.h +++ b/sys/compat/linuxkpi/common/include/linux/bitops.h @@ -51,12 +51,6 @@ #define BITS_PER_TYPE(t) (sizeof(t) * BITS_PER_BYTE) #define BITS_TO_BYTES(n) howmany((n), BITS_PER_BYTE) -#define hweight8(x) bitcount((uint8_t)(x)) -#define hweight16(x) bitcount16(x) -#define hweight32(x) bitcount32(x) -#define hweight64(x) bitcount64(x) -#define hweight_long(x) bitcountl(x) - #if __has_builtin(__builtin_popcountg) #define HWEIGHT8(x) (__builtin_popcountg((uint8_t)(x))) #define HWEIGHT16(x) (__builtin_popcountg((uint16_t)(x))) @@ -70,6 +64,12 @@ #define HWEIGHT64(x) (__const_bitcount64((uint64_t)(x))) #endif +#define hweight8(x) (__builtin_constant_p(x) ? HWEIGHT8(x) : bitcount((uint8_t)(x))) +#define hweight16(x) (__builtin_constant_p(x) ? HWEIGHT16(x) : bitcount16(x)) +#define hweight32(x) (__builtin_constant_p(x) ? HWEIGHT32(x) : bitcount32(x)) +#define hweight64(x) (__builtin_constant_p(x) ? HWEIGHT64(x) : bitcount64(x)) +#define hweight_long(x) bitcountl(x) + static inline int __ffs(int mask) { diff --git a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h index 2d8e1196d3d3..76efbfd51074 100644 --- a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h +++ b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h @@ -96,6 +96,8 @@ void *linux_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag); void *linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag); +void linuxkpi_dmam_free_coherent(struct device *dev, size_t size, + void *addr, dma_addr_t dma_handle); dma_addr_t linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len); /* backward compat */ dma_addr_t lkpi_dma_map_phys(struct device *, vm_paddr_t, size_t, enum dma_data_direction, unsigned long); @@ -181,6 +183,13 @@ dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, kmem_free(cpu_addr, size); } +static inline void +dmam_free_coherent(struct device *dev, size_t size, void *addr, + dma_addr_t dma_handle) +{ + linuxkpi_dmam_free_coherent(dev, size, addr, dma_handle); +} + static inline dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, size_t offset, size_t size, enum dma_data_direction direction, unsigned long attrs) diff --git a/sys/compat/linuxkpi/common/include/linux/eventfd.h b/sys/compat/linuxkpi/common/include/linux/eventfd.h new file mode 100644 index 000000000000..d167d4b7d189 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/eventfd.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2025 The FreeBSD Foundation + * Copyright (c) 2025 Jean-Sébastien Pédron + * + * This software was developed by Jean-Sébastien Pédron under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _LINUXKPI_LINUX_EVENTFD_H_ +#define _LINUXKPI_LINUX_EVENTFD_H_ + +#include <sys/eventfd.h> + +#include <linux/wait.h> +#include <linux/err.h> +#include <linux/percpu-defs.h> +#include <linux/percpu.h> +#include <linux/sched.h> + +/* + * Linux uses `struct eventfd_ctx`, but FreeBSD defines `struct eventfd`. Here, + * we define a synonym to the FreeBSD structure. This allows to keep Linux code + * unmodified. + */ +#define eventfd_ctx eventfd + +#define eventfd_ctx_fdget lkpi_eventfd_ctx_fdget +struct eventfd_ctx *lkpi_eventfd_ctx_fdget(int fd); + +#define eventfd_ctx_put lkpi_eventfd_ctx_put +void lkpi_eventfd_ctx_put(struct eventfd_ctx *ctx); + +#endif /* _LINUXKPI_LINUX_EVENTFD_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/ieee80211.h b/sys/compat/linuxkpi/common/include/linux/ieee80211.h index b5051a9f7d21..d1eba94a3ad8 100644 --- a/sys/compat/linuxkpi/common/include/linux/ieee80211.h +++ b/sys/compat/linuxkpi/common/include/linux/ieee80211.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2020-2025 The FreeBSD Foundation + * Copyright (c) 2020-2026 The FreeBSD Foundation * * This software was developed by Björn Zeeb under sponsorship from * the FreeBSD Foundation. @@ -51,8 +51,16 @@ extern int linuxkpi_debug_80211; #define IMPROVE(fmt, ...) if (linuxkpi_debug_80211 & D80211_IMPROVE) \ printf("%s:%d: XXX LKPI80211 IMPROVE " fmt "\n", __func__, __LINE__, ##__VA_ARGS__) - -/* 9.4.2.55 Management MIC element (CMAC-256, GMAC-128, and GMAC-256). */ +/* 802.11-2024, 9.4.2.53 MME. */ +/* BIP-CMAC-128 */ +struct ieee80211_mmie { + uint8_t element_id; + uint8_t length; + uint16_t key_id; + uint8_t ipn[6]; + uint8_t mic[8]; +}; +/* BIP-CMAC-256, BIP-GMAC-128, BIP-GMAC-256 */ struct ieee80211_mmie_16 { uint8_t element_id; uint8_t length; @@ -211,6 +219,7 @@ enum ieee80211_min_mpdu_start_spacing { #define IEEE80211_FCTL_TODS (IEEE80211_FC1_DIR_TODS << 8) #define IEEE80211_FCTL_MOREFRAGS (IEEE80211_FC1_MORE_FRAG << 8) #define IEEE80211_FCTL_PM (IEEE80211_FC1_PWR_MGT << 8) +#define IEEE80211_FCTL_MOREDATA (IEEE80211_FC1_MORE_DATA << 8) #define IEEE80211_FTYPE_MGMT IEEE80211_FC0_TYPE_MGT #define IEEE80211_FTYPE_CTL IEEE80211_FC0_TYPE_CTL @@ -472,18 +481,6 @@ enum ieee80211_tx_control_flags { IEEE80211_TX_CTRL_MLO_LINK = 0xF0000000, /* This is IEEE80211_LINK_UNSPECIFIED on the high bits. */ }; -enum ieee80211_tx_rate_flags { - /* XXX TODO .. right shift numbers */ - IEEE80211_TX_RC_40_MHZ_WIDTH = BIT(0), - IEEE80211_TX_RC_80_MHZ_WIDTH = BIT(1), - IEEE80211_TX_RC_160_MHZ_WIDTH = BIT(2), - IEEE80211_TX_RC_GREEN_FIELD = BIT(3), - IEEE80211_TX_RC_MCS = BIT(4), - IEEE80211_TX_RC_SHORT_GI = BIT(5), - IEEE80211_TX_RC_VHT_MCS = BIT(6), - IEEE80211_TX_RC_USE_SHORT_PREAMBLE = BIT(7), -}; - #define IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED -128 #define IEEE80211_HT_CTL_LEN 4 diff --git a/sys/compat/linuxkpi/common/include/linux/minmax.h b/sys/compat/linuxkpi/common/include/linux/minmax.h index d48958f0899f..fb8eb6f704b4 100644 --- a/sys/compat/linuxkpi/common/include/linux/minmax.h +++ b/sys/compat/linuxkpi/common/include/linux/minmax.h @@ -71,4 +71,7 @@ b = _swap_tmp; \ } while (0) +/* XXX would have to make sure both are unsigned. */ +#define umin(x, y) MIN(x, y) + #endif /* _LINUXKPI_LINUX_MINMAX_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/netdevice.h b/sys/compat/linuxkpi/common/include/linux/netdevice.h index cf27753bcb80..dfed5fbd61b4 100644 --- a/sys/compat/linuxkpi/common/include/linux/netdevice.h +++ b/sys/compat/linuxkpi/common/include/linux/netdevice.h @@ -160,6 +160,30 @@ struct net_device { #define SET_NETDEV_DEV(_ndev, _dev) (_ndev)->dev.parent = _dev; +enum net_device_path_type { + DEV_PATH_MTK_WDMA, +}; + +struct net_device_path { + enum net_device_path_type type; + const struct net_device *dev; + /* We assume there's a struct per type. */ + union { + struct { + uint16_t wcid; + uint8_t wdma_idx; + uint8_t queue; + uint8_t bss; + uint8_t amsdu; + } mtk_wdma; + }; +}; + +struct net_device_path_ctx { + const struct net_device *dev; +}; + + /* -------------------------------------------------------------------------- */ /* According to linux::ipoib_main.c. */ struct netdev_notifier_info { diff --git a/sys/compat/linuxkpi/common/include/linux/ptp_clock_kernel.h b/sys/compat/linuxkpi/common/include/linux/ptp_clock_kernel.h index aad46cc25b1b..6491cbeab7e2 100644 --- a/sys/compat/linuxkpi/common/include/linux/ptp_clock_kernel.h +++ b/sys/compat/linuxkpi/common/include/linux/ptp_clock_kernel.h @@ -49,6 +49,7 @@ struct ptp_clock_info { int (*adjtime)(struct ptp_clock_info *, s64); int (*getcrosststamp)(struct ptp_clock_info *, struct system_device_crosststamp *); int (*gettime64)(struct ptp_clock_info *, struct timespec *); + int (*settime64)(struct ptp_clock_info *, const struct timespec *); }; static inline struct ptp_clock * diff --git a/sys/compat/linuxkpi/common/include/linux/radix-tree.h b/sys/compat/linuxkpi/common/include/linux/radix-tree.h index 55f0fc949807..7182f4a9e407 100644 --- a/sys/compat/linuxkpi/common/include/linux/radix-tree.h +++ b/sys/compat/linuxkpi/common/include/linux/radix-tree.h @@ -38,12 +38,19 @@ #define RADIX_TREE_MAX_HEIGHT \ howmany(sizeof(long) * NBBY, RADIX_TREE_MAP_SHIFT) +#define RADIX_TREE_MAX_TAGS 3 +#define RADIX_TREE_TAG_LONGS RADIX_TREE_MAP_SIZE + #define RADIX_TREE_ENTRY_MASK 3UL #define RADIX_TREE_EXCEPTIONAL_ENTRY 2UL #define RADIX_TREE_EXCEPTIONAL_SHIFT 2 +#define RADIX_TREE_ITER_TAG_MASK 0x0f +#define RADIX_TREE_ITER_TAGGED 0x10 + struct radix_tree_node { void *slots[RADIX_TREE_MAP_SIZE]; + unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS]; int count; }; @@ -51,6 +58,8 @@ struct radix_tree_root { struct radix_tree_node *rnode; gfp_t gfp_mask; int height; + /* Linux stores root tags inside `gfp_mask`. */ + unsigned long tags[RADIX_TREE_MAX_TAGS]; }; struct radix_tree_iter { @@ -64,9 +73,16 @@ struct radix_tree_iter { #define RADIX_TREE(name, mask) \ struct radix_tree_root name = RADIX_TREE_INIT(mask) -#define radix_tree_for_each_slot(slot, root, iter, start) \ - for ((iter)->index = (start); \ - radix_tree_iter_find(root, iter, &(slot)); (iter)->index++) +#define radix_tree_for_each_slot(slot, root, iter, start) \ + for ((iter)->index = (start); \ + radix_tree_iter_find(root, iter, &(slot), 0); \ + (iter)->index++) + +#define radix_tree_for_each_slot_tagged(slot, root, iter, start, tag) \ + for ((iter)->index = (start); \ + radix_tree_iter_find(root, iter, &(slot), \ + RADIX_TREE_ITER_TAGGED | tag); \ + (iter)->index++) static inline void * radix_tree_deref_slot(void **slot) @@ -84,7 +100,12 @@ void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); void *radix_tree_delete(struct radix_tree_root *, unsigned long); int radix_tree_insert(struct radix_tree_root *, unsigned long, void *); int radix_tree_store(struct radix_tree_root *, unsigned long, void **); -bool radix_tree_iter_find(const struct radix_tree_root *, struct radix_tree_iter *, void ***); +bool radix_tree_iter_find(const struct radix_tree_root *, struct radix_tree_iter *, void ***, int); void radix_tree_iter_delete(struct radix_tree_root *, struct radix_tree_iter *, void **); +void *radix_tree_tag_set(struct radix_tree_root *root, unsigned long index, unsigned int tag); +void *radix_tree_tag_clear(struct radix_tree_root *root, unsigned long index, unsigned int tag); +int radix_tree_tagged(const struct radix_tree_root *root, unsigned int tag); +unsigned int radix_tree_gang_lookup(const struct radix_tree_root *root, void **results, unsigned long first_index, unsigned int max_items); +unsigned int radix_tree_gang_lookup_tag(const struct radix_tree_root *root, void **results, unsigned long first_index, unsigned int max_items, unsigned int tag); #endif /* _LINUXKPI_LINUX_RADIX_TREE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/seq_buf.h b/sys/compat/linuxkpi/common/include/linux/seq_buf.h new file mode 100644 index 000000000000..d6246a40e6f7 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/seq_buf.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2025-2026 The FreeBSD Foundation + * Copyright (c) 2025-2026 Jean-Sébastien Pédron <dumbbell@FreeBSD.org> + * + * This software was developed by Jean-Sébastien Pédron under sponsorship + * from the FreeBSD Foundation. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _LINUXKPI_LINUX_SEQ_BUF_H_ +#define _LINUXKPI_LINUX_SEQ_BUF_H_ + +#include <linux/bug.h> +#include <linux/minmax.h> +#include <linux/seq_file.h> +#include <linux/types.h> + +struct seq_buf { + char *buffer; + size_t size; + size_t len; +}; + +#define DECLARE_SEQ_BUF(NAME, SIZE) \ + struct seq_buf NAME = { \ + .buffer = (char[SIZE]) { 0 }, \ + .size = SIZE, \ + } + +static inline void +seq_buf_clear(struct seq_buf *s) +{ + s->len = 0; + if (s->size > 0) + s->buffer[0] = '\0'; +} + +static inline void +seq_buf_set_overflow(struct seq_buf *s) +{ + s->len = s->size + 1; +} + +static inline bool +seq_buf_has_overflowed(struct seq_buf *s) +{ + return (s->len > s->size); +} + +static inline bool +seq_buf_buffer_left(struct seq_buf *s) +{ + if (seq_buf_has_overflowed(s)) + return (0); + + return (s->size - s->len); +} + +#define seq_buf_init(s, buf, size) linuxkpi_seq_buf_init((s), (buf), (size)) +void linuxkpi_seq_buf_init(struct seq_buf *s, char *buf, unsigned int size); + +#define seq_buf_printf(s, f, ...) linuxkpi_seq_buf_printf((s), (f), __VA_ARGS__) +int linuxkpi_seq_buf_printf(struct seq_buf *s, const char *fmt, ...) \ + __printflike(2, 3); + +#define seq_buf_vprintf(s, f, a) linuxkpi_seq_buf_vprintf((s), (f), (a)) +int linuxkpi_seq_buf_vprintf(struct seq_buf *s, const char *fmt, va_list args); + +#define seq_buf_str(s) linuxkpi_seq_buf_str((s)) +const char * linuxkpi_seq_buf_str(struct seq_buf *s); + +#endif diff --git a/sys/compat/linuxkpi/common/include/linux/skbuff.h b/sys/compat/linuxkpi/common/include/linux/skbuff.h index 2e560a120e41..c43d6daff5ee 100644 --- a/sys/compat/linuxkpi/common/include/linux/skbuff.h +++ b/sys/compat/linuxkpi/common/include/linux/skbuff.h @@ -770,7 +770,7 @@ ___skb_queue_splice(const struct sk_buff_head *from, } static inline void -skb_queue_splice_init(struct sk_buff_head *from, struct sk_buff_head *to) +skb_queue_splice(const struct sk_buff_head *from, struct sk_buff_head *to) { SKB_TRACE2(from, to); @@ -780,6 +780,13 @@ skb_queue_splice_init(struct sk_buff_head *from, struct sk_buff_head *to) ___skb_queue_splice(from, (struct sk_buff *)to, to->next); to->qlen += from->qlen; +} + +static inline void +skb_queue_splice_init(struct sk_buff_head *from, struct sk_buff_head *to) +{ + + skb_queue_splice(from, to); __skb_queue_head_init(from); } diff --git a/sys/compat/linuxkpi/common/include/linux/soc/airoha/airoha_offload.h b/sys/compat/linuxkpi/common/include/linux/soc/airoha/airoha_offload.h new file mode 100644 index 000000000000..ade0b06d839f --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/soc/airoha/airoha_offload.h @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2026 Bjoern A. Zeeb + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _LINUXKPI_LINUX_SOC_AIROHA_AIROHA_OFFLOAD_H +#define _LINUXKPI_LINUX_SOC_AIROHA_AIROHA_OFFLOAD_H + +#include <linux/kernel.h> /* pr_debug */ + +enum airoha_npu_wlan_get_cmd { + __dummy_airoha_npu_wlan_get_cmd, +}; +enum airoha_npu_wlan_set_cmd { + __dummy_airoha_npu_wlan_set_cmd, +}; + +struct airoha_npu { +}; +struct airoha_npu_rx_dma_desc { +}; +struct airoha_npu_tx_dma_desc { +}; + +static __inline int +airoha_npu_wlan_send_msg(void *npu, int ifindex, + enum airoha_npu_wlan_set_cmd cmd, void *val, size_t len, gfp_t gfp) +{ + pr_debug("%s: TODO\n", __func__); + return (-EOPNOTSUPP); +} + +static __inline int +airoha_npu_wlan_get_msg(void *npu, int ifindex, + enum airoha_npu_wlan_get_cmd cmd, void *val, size_t len, gfp_t gfp) +{ + pr_debug("%s: TODO\n", __func__); + return (-EOPNOTSUPP); +} + +static __inline void +airoha_npu_wlan_enable_irq(struct airoha_npu *npu, int q) +{ + pr_debug("%s: TODO\n", __func__); +} + +#endif /* _LINUXKPI_LINUX_SOC_AIROHA_AIROHA_OFFLOAD_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/soc/mediatek/mtk_wed.h b/sys/compat/linuxkpi/common/include/linux/soc/mediatek/mtk_wed.h index 2b9c6ae4911e..64daa8c78c9d 100644 --- a/sys/compat/linuxkpi/common/include/linux/soc/mediatek/mtk_wed.h +++ b/sys/compat/linuxkpi/common/include/linux/soc/mediatek/mtk_wed.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2022-2025 Bjoern A. Zeeb + * Copyright (c) 2022-2026 Bjoern A. Zeeb * * SPDX-License-Identifier: BSD-2-Clause */ @@ -37,7 +37,13 @@ mtk_wed_device_active(struct mtk_wed_device *dev __unused) static inline bool mtk_wed_get_rx_capa(struct mtk_wed_device *dev __unused) { + pr_debug("%s: TODO\n", __func__); + return (false); +} +static inline bool +mtk_wed_is_amsdu_supported(struct mtk_wed_device *dev __unused) +{ pr_debug("%s: TODO\n", __func__); return (false); } @@ -66,6 +72,12 @@ mtk_wed_get_rx_capa(struct mtk_wed_device *dev __unused) { return (false); } + +static inline bool +mtk_wed_is_amsdu_supported(struct mtk_wed_device *dev __unused) +{ + return (false); +} #endif /* CONFIG_NET_MEDIATEK_SOC_WED */ #endif /* _LINUXKPI_LINUX_SOC_MEDIATEK_MTK_WED_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/string_choices.h b/sys/compat/linuxkpi/common/include/linux/string_choices.h index 74aa3fd019b2..db540d3e7d40 100644 --- a/sys/compat/linuxkpi/common/include/linux/string_choices.h +++ b/sys/compat/linuxkpi/common/include/linux/string_choices.h @@ -33,39 +33,33 @@ static inline const char * str_yes_no(bool value) { - if (value) - return "yes"; - else - return "no"; + return (value ? "yes" : "no"); } static inline const char * str_on_off(bool value) { - if (value) - return "on"; - else - return "off"; + return (value ? "on" : "off"); } static inline const char * str_enabled_disabled(bool value) { - if (value) - return "enabled"; - else - return "disabled"; + return (value ? "enabled" : "disabled"); } static inline const char * str_enable_disable(bool value) { - if (value) - return "enable"; - else - return "disable"; + return (value ? "enable" : "disable"); } #define str_disable_enable(_v) str_enable_disable(!(_v)) +static inline const char * +str_read_write(bool value) +{ + return (value ? "read" : "write"); +} + #endif diff --git a/sys/compat/linuxkpi/common/include/net/cfg80211.h b/sys/compat/linuxkpi/common/include/net/cfg80211.h index 4b21f82a0762..94d34fb9dc0c 100644 --- a/sys/compat/linuxkpi/common/include/net/cfg80211.h +++ b/sys/compat/linuxkpi/common/include/net/cfg80211.h @@ -124,6 +124,7 @@ enum ieee80211_channel_flags { IEEE80211_CHAN_PSD = BIT(12), IEEE80211_CHAN_ALLOW_6GHZ_VLP_AP = BIT(13), IEEE80211_CHAN_CAN_MONITOR = BIT(14), + IEEE80211_CHAN_NO_EHT = BIT(15), }; #define IEEE80211_CHAN_NO_HT40 (IEEE80211_CHAN_NO_HT40MINUS|IEEE80211_CHAN_NO_HT40PLUS) @@ -152,6 +153,8 @@ struct linuxkpi_ieee80211_channel { int orig_mpwr; }; +#define NL80211_EHT_NSS_MAX 16 + struct cfg80211_bitrate_mask { /* TODO FIXME */ struct { @@ -159,6 +162,7 @@ struct cfg80211_bitrate_mask { uint8_t ht_mcs[IEEE80211_HT_MCS_MASK_LEN]; uint16_t vht_mcs[8]; uint16_t he_mcs[8]; + uint16_t eht_mcs[NL80211_EHT_NSS_MAX]; enum nl80211_txrate_gi gi; enum nl80211_he_gi he_gi; uint8_t he_ltf; /* XXX enum? */ diff --git a/sys/compat/linuxkpi/common/include/net/mac80211.h b/sys/compat/linuxkpi/common/include/net/mac80211.h index 6e2f3f2d8781..18891d035094 100644 --- a/sys/compat/linuxkpi/common/include/net/mac80211.h +++ b/sys/compat/linuxkpi/common/include/net/mac80211.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2020-2025 The FreeBSD Foundation + * Copyright (c) 2020-2026 The FreeBSD Foundation * Copyright (c) 2020-2025 Bjoern A. Zeeb * * This software was developed by Björn Zeeb under sponsorship from @@ -789,10 +789,21 @@ struct ieee80211_tx_queue_params { struct ieee80211_he_mu_edca_param_ac_rec mu_edca_param_rec; }; +enum mac80211_rate_control_flags { + IEEE80211_TX_RC_40_MHZ_WIDTH = BIT(0), + IEEE80211_TX_RC_80_MHZ_WIDTH = BIT(1), + IEEE80211_TX_RC_160_MHZ_WIDTH = BIT(2), + IEEE80211_TX_RC_GREEN_FIELD = BIT(3), + IEEE80211_TX_RC_MCS = BIT(4), + IEEE80211_TX_RC_SHORT_GI = BIT(5), + IEEE80211_TX_RC_VHT_MCS = BIT(6), + IEEE80211_TX_RC_USE_SHORT_PREAMBLE = BIT(7), +}; + struct ieee80211_tx_rate { uint8_t idx; uint16_t count:5, - flags:11; + flags:11; /* enum mac80211_rate_control_flags */ }; enum ieee80211_vif_driver_flags { @@ -1092,6 +1103,8 @@ struct ieee80211_ops { void (*rfkill_poll)(struct ieee80211_hw *); + int (*net_fill_forward_path)(struct ieee80211_hw *, struct ieee80211_vif *, struct ieee80211_sta *, struct net_device_path_ctx *, struct net_device_path *); + /* #ifdef CONFIG_MAC80211_DEBUGFS */ /* Do not change depending on compile-time option. */ void (*sta_add_debugfs)(struct ieee80211_hw *, struct ieee80211_vif *, struct ieee80211_sta *, struct dentry *); void (*vif_add_debugfs)(struct ieee80211_hw *, struct ieee80211_vif *); diff --git a/sys/compat/linuxkpi/common/src/linux_80211.c b/sys/compat/linuxkpi/common/src/linux_80211.c index d25b32f1dae8..4ce0a2094adb 100644 --- a/sys/compat/linuxkpi/common/src/linux_80211.c +++ b/sys/compat/linuxkpi/common/src/linux_80211.c @@ -6972,7 +6972,9 @@ linuxkpi_ieee80211_ifattach(struct ieee80211_hw *hw) } if (bootverbose) { - ic_printf(ic, "netdev_features %b\n", hw->netdev_features, NETIF_F_BITS); + if (hw->netdev_features != 0) + ic_printf(ic, "netdev_features %b\n", + hw->netdev_features, NETIF_F_BITS); ieee80211_announce(ic); } diff --git a/sys/compat/linuxkpi/common/src/linux_eventfd.c b/sys/compat/linuxkpi/common/src/linux_eventfd.c new file mode 100644 index 000000000000..126c6c54b9a5 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_eventfd.c @@ -0,0 +1,63 @@ +/*- + * Copyright (c) 2025 The FreeBSD Foundation + * Copyright (c) 2025 Jean-Sébastien Pédron + * + * This software was developed by Jean-Sébastien Pédron under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/file.h> +#include <sys/filedesc.h> + +#include <linux/eventfd.h> + +struct eventfd_ctx * +lkpi_eventfd_ctx_fdget(int fd) +{ + struct file *fp; + struct eventfd_ctx *ctx; + + /* Lookup file pointer by file descriptor index. */ + if (fget_unlocked(curthread, fd, &cap_no_rights, &fp) != 0) + return (ERR_PTR(-EBADF)); + + /* + * eventfd_get() bumps the refcount, so we can safely release the + * reference on the file itself afterwards. + */ + ctx = eventfd_get(fp); + fdrop(fp, curthread); + + if (ctx == NULL) + return (ERR_PTR(-EBADF)); + + return (ctx); +} + +void +lkpi_eventfd_ctx_put(struct eventfd_ctx *ctx) +{ + eventfd_put(ctx); +} diff --git a/sys/compat/linuxkpi/common/src/linux_pci.c b/sys/compat/linuxkpi/common/src/linux_pci.c index df5255d8a6ca..9e4fad45433a 100644 --- a/sys/compat/linuxkpi/common/src/linux_pci.c +++ b/sys/compat/linuxkpi/common/src/linux_pci.c @@ -1796,6 +1796,42 @@ lkpi_dmam_free_coherent(struct device *dev, void *p) dma_free_coherent(dev, dr->size, dr->mem, *dr->handle); } +static int +lkpi_dmam_coherent_match(struct device *dev, void *dr, void *mp) +{ + struct lkpi_devres_dmam_coherent *a, *b; + + a = dr; + b = mp; + + if (a->mem != b->mem) + return (0); + if (a->size != b->size || a->handle != b->handle) + dev_WARN(dev, "for mem %p: size %zu != %zu || handle %#jx != %#jx\n", + a->mem, a->size, b->size, + (uintmax_t)a->handle, (uintmax_t)b->handle); + return (1); +} + +void +linuxkpi_dmam_free_coherent(struct device *dev, size_t size, + void *addr, dma_addr_t dma_handle) +{ + struct lkpi_devres_dmam_coherent match = { + .size = size, + .handle = &dma_handle, + .mem = addr + }; + int error; + + error = devres_destroy(dev, lkpi_dmam_free_coherent, + lkpi_dmam_coherent_match, &match); + if (error != 0) + dev_WARN(dev, "devres_destroy returned %d, size %zu addr %p " + "dma_handle %#jx\n", error, size, addr, (uintmax_t)dma_handle); + dma_free_coherent(dev, size, addr, dma_handle); +} + void * linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag) diff --git a/sys/compat/linuxkpi/common/src/linux_radix.c b/sys/compat/linuxkpi/common/src/linux_radix.c index ee6b3a63c370..760f583f25c8 100644 --- a/sys/compat/linuxkpi/common/src/linux_radix.c +++ b/sys/compat/linuxkpi/common/src/linux_radix.c @@ -52,6 +52,81 @@ radix_pos(long id, int height) return (id >> (RADIX_TREE_MAP_SHIFT * height)) & RADIX_TREE_MAP_MASK; } +static inline int +root_tag_get(const struct radix_tree_root *root, unsigned tag) +{ + return (test_bit(tag, root->tags)); +} + +static inline void +root_tag_set(struct radix_tree_root *root, unsigned tag) +{ + set_bit(tag, root->tags); +} + +static inline void +root_tag_clear(struct radix_tree_root *root, unsigned tag) +{ + clear_bit(tag, root->tags); +} + +static inline int +tag_get(const struct radix_tree_node *node, unsigned int tag, int pos) +{ + return (test_bit(pos, node->tags[tag])); +} + +static inline void +tag_set(struct radix_tree_node *node, unsigned int tag, int pos) +{ + set_bit(pos, node->tags[tag]); +} + +static inline void +tag_clear(struct radix_tree_node *node, unsigned int tag, int pos) +{ + clear_bit(pos, node->tags[tag]); +} + +static inline bool +any_tag_set(const struct radix_tree_node *node, unsigned int tag) +{ + unsigned int pos; + + for (pos = 0; pos < RADIX_TREE_TAG_LONGS; pos++) + if (node->tags[tag][pos]) + return (true); + + return (false); +} + +static void +node_tag_clear(struct radix_tree_root *root, struct radix_tree_node *stack[], + unsigned long index, unsigned int tag) +{ + struct radix_tree_node *node; + int height, pos; + + height = 1; + node = stack[height]; + + while (node && height <= root->height - 1) { + pos = radix_pos(index, height); + if (!tag_get(node, tag, pos)) + return; + + tag_clear(node, tag, pos); + if (any_tag_set(node, tag)) + return; + + height++; + node = stack[height]; + } + + if (root_tag_get(root, tag)) + root_tag_clear(root, tag); +} + static void radix_tree_clean_root_node(struct radix_tree_root *root) { @@ -84,14 +159,70 @@ out: return (item); } +unsigned int +radix_tree_gang_lookup(const struct radix_tree_root *root, void **results, + unsigned long first_index, unsigned int max_items) +{ + struct radix_tree_iter iter; + void **slot; + int count; + + count = 0; + if (max_items == 0) + return (count); + + radix_tree_for_each_slot(slot, root, &iter, first_index) { + results[count] = *slot; + if (results[count] == NULL) + continue; + + count++; + if (count == max_items) + break; + } + + return (count); +} + +unsigned int +radix_tree_gang_lookup_tag(const struct radix_tree_root *root, + void **results, unsigned long first_index, unsigned int max_items, + unsigned int tag) +{ + struct radix_tree_iter iter; + void **slot; + int count; + + count = 0; + if (max_items == 0) + return (count); + + radix_tree_for_each_slot_tagged(slot, root, &iter, first_index, tag) { + results[count] = *slot; + if (results[count] == NULL) + continue; + + count++; + if (count == max_items) + break; + } + + return (count); +} + bool radix_tree_iter_find(const struct radix_tree_root *root, - struct radix_tree_iter *iter, void ***pppslot) + struct radix_tree_iter *iter, void ***pppslot, int flags) { struct radix_tree_node *node; unsigned long index = iter->index; + unsigned int tag; int height; + tag = flags & RADIX_TREE_ITER_TAG_MASK; + if ((flags & RADIX_TREE_ITER_TAGGED) && !root_tag_get(root, tag)) + return (false); + restart: node = root->rnode; if (node == NULL) @@ -109,7 +240,9 @@ restart: *pppslot = node->slots + pos; next = node->slots[pos]; - if (next == NULL) { + if (next == NULL || + (flags & RADIX_TREE_ITER_TAGGED && + !tag_get(next, tag, pos))) { index += step; index &= -step; if ((index & mask) == 0) @@ -131,6 +264,7 @@ radix_tree_delete(struct radix_tree_root *root, unsigned long index) void *item; int height; int idx; + int tag; item = NULL; node = root->rnode; @@ -144,9 +278,15 @@ radix_tree_delete(struct radix_tree_root *root, unsigned long index) stack[height] = node; node = node->slots[radix_pos(index, height--)]; } - idx = radix_pos(index, 0); - if (node) + + if (node) { + idx = radix_pos(index, 0); item = node->slots[idx]; + + for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) + node_tag_clear(root, stack, index, tag); + } + /* * If we removed something reduce the height of the tree. */ @@ -380,3 +520,74 @@ radix_tree_store(struct radix_tree_root *root, unsigned long index, void **ppite node->count++; return (0); } + +void * +radix_tree_tag_set(struct radix_tree_root *root, unsigned long index, + unsigned int tag) +{ + struct radix_tree_node *node; + void *item; + int height, pos; + + item = NULL; + node = root->rnode; + height = root->height - 1; + if (index > radix_max(root)) + goto out; + + while (height) { + KASSERT( + node != NULL, + ("Node at index %lu does not exist in radix tree %p", + index, root)); + + pos = radix_pos(index, height--); + node = node->slots[pos]; + + if (!tag_get(node, tag, pos)) + tag_set(node, tag, pos); + } + + item = node->slots[radix_pos(index, 0)]; + + root_tag_set(root, tag); + +out: + return (item); +} + +void * +radix_tree_tag_clear(struct radix_tree_root *root, + unsigned long index, unsigned int tag) +{ + struct radix_tree_node *stack[RADIX_TREE_MAX_HEIGHT]; + struct radix_tree_node *node; + void *item; + int height; + + item = NULL; + node = root->rnode; + height = root->height - 1; + if (index > radix_max(root)) + goto out; + + while (height && node) { + stack[height] = node; + node = node->slots[radix_pos(index, height--)]; + } + + if (node) { + item = node->slots[radix_pos(index, 0)]; + + node_tag_clear(root, stack, index, tag); + } + +out: + return (item); +} + +int +radix_tree_tagged(const struct radix_tree_root *root, unsigned int tag) +{ + return (root_tag_get(root, tag)); +} diff --git a/sys/compat/linuxkpi/common/src/linux_seq_buf.c b/sys/compat/linuxkpi/common/src/linux_seq_buf.c new file mode 100644 index 000000000000..112c53044c22 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_seq_buf.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2025-2026 The FreeBSD Foundation + * Copyright (c) 2025-2026 Jean-Sébastien Pédron <dumbbell@FreeBSD.org> + * + * This software was developed by Jean-Sébastien Pédron under sponsorship + * from the FreeBSD Foundation. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <linux/seq_buf.h> + +void +linuxkpi_seq_buf_init(struct seq_buf *s, char *buf, unsigned int size) +{ + s->buffer = buf; + s->size = size; + + seq_buf_clear(s); +} + +int +linuxkpi_seq_buf_printf(struct seq_buf *s, const char *fmt, ...) +{ + int ret; + va_list args; + + va_start(args, fmt); + ret = seq_buf_vprintf(s, fmt, args); + va_end(args); + + return (ret); +} + +int +linuxkpi_seq_buf_vprintf(struct seq_buf *s, const char *fmt, va_list args) +{ + int ret; + + if (!seq_buf_has_overflowed(s)) { + ret = vsnprintf(s->buffer + s->len, s->size - s->len, fmt, args); + if (s->len + ret < s->size) { + s->len += ret; + return (0); + } + } + + seq_buf_set_overflow(s); + return (-1); +} + +const char * +linuxkpi_seq_buf_str(struct seq_buf *s) +{ + if (s->size == 0) + return (""); + + if (seq_buf_buffer_left(s)) + s->buffer[s->len] = '\0'; + else + s->buffer[s->size - 1] = '\0'; + + return (s->buffer); +} diff --git a/sys/compat/linuxkpi/common/src/linux_xarray.c b/sys/compat/linuxkpi/common/src/linux_xarray.c index 3f07f6d7c59f..8caefbaf7e50 100644 --- a/sys/compat/linuxkpi/common/src/linux_xarray.c +++ b/sys/compat/linuxkpi/common/src/linux_xarray.c @@ -389,7 +389,7 @@ __xa_empty(struct xarray *xa) XA_ASSERT_LOCKED(xa); - return (!radix_tree_iter_find(&xa->xa_head, &iter, &temp)); + return (!radix_tree_iter_find(&xa->xa_head, &iter, &temp, 0)); } bool @@ -426,7 +426,7 @@ __xa_next(struct xarray *xa, unsigned long *pindex, bool not_first) return (NULL); } - found = radix_tree_iter_find(&xa->xa_head, &iter, &ppslot); + found = radix_tree_iter_find(&xa->xa_head, &iter, &ppslot, 0); if (likely(found)) { retval = *ppslot; if (retval == NULL_VALUE) diff --git a/sys/compat/linuxkpi/dummy/include/linux/eventfd.h b/sys/compat/linuxkpi/dummy/include/linux/eventfd.h deleted file mode 100644 index e69de29bb2d1..000000000000 --- a/sys/compat/linuxkpi/dummy/include/linux/eventfd.h +++ /dev/null diff --git a/sys/conf/NOTES b/sys/conf/NOTES index 6d3ea088b819..1f734a700cd1 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -210,7 +210,7 @@ options ROOTDEVNAME=\"ufs:da0s2e\" # options SCHED_4BSD options SCHED_STATS -#options SCHED_ULE +options SCHED_ULE ##################################################################### # SMP OPTIONS: diff --git a/sys/conf/files b/sys/conf/files index a6a76dec433a..b330727b58c6 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -776,6 +776,7 @@ dev/acpica/acpi_thermal.c optional acpi dev/acpica/acpi_throttle.c optional acpi dev/acpica/acpi_video.c optional acpi_video acpi dev/acpica/acpi_dock.c optional acpi_dock acpi +dev/acpica/acpi_spmc.c optional acpi dev/adlink/adlink.c optional adlink dev/ae/if_ae.c optional ae pci dev/age/if_age.c optional age pci @@ -3921,6 +3922,7 @@ kern/linker_if.m standard kern/p1003_1b.c standard kern/posix4_mib.c standard kern/sched_4bsd.c optional sched_4bsd +kern/sched_shim.c standard kern/sched_ule.c optional sched_ule kern/serdev_if.m standard kern/stack_protector.c standard \ @@ -4662,6 +4664,8 @@ compat/linuxkpi/common/src/linux_dmi.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_domain.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" +compat/linuxkpi/common/src/linux_eventfd.c optional compat_linuxkpi \ + compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_firmware.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_folio.c optional compat_linuxkpi \ @@ -4700,6 +4704,8 @@ compat/linuxkpi/common/src/linux_rcu.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C} -I$S/contrib/ck/include" compat/linuxkpi/common/src/linux_schedule.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" +compat/linuxkpi/common/src/linux_seq_buf.c optional compat_linuxkpi \ + compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_shmemfs.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_shrinker.c optional compat_linuxkpi \ diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index 7989f1c9bea4..d9171d33c8c0 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -390,7 +390,7 @@ powerpc/powerpc/platform.c standard powerpc/powerpc/platform_if.m standard powerpc/powerpc/ptrace_machdep.c standard powerpc/powerpc/sc_machdep.c optional sc -powerpc/powerpc/sdt_machdep.c optional powerpc64 kdtrace_hooks +powerpc/powerpc/sdt_machdep.c optional kdtrace_hooks powerpc/powerpc/setjmp.S standard powerpc/powerpc/sigcode32.S optional powerpc | compat_freebsd32 powerpc/powerpc/sigcode64.S optional powerpc64 | powerpc64le diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv index 2a80064e88ba..90a74367a54f 100644 --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -34,6 +34,8 @@ kern/subr_dummy_vdso_tc.c standard kern/subr_efi_map.c standard kern/subr_intr.c standard kern/subr_physmem.c standard +libkern/ffs.c standard +libkern/ffsdi2.c standard libkern/bcopy.c standard libkern/memcmp.c standard libkern/memset.c standard diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk index e86ffb684f9a..af7b1589c5cd 100644 --- a/sys/conf/kern.mk +++ b/sys/conf/kern.mk @@ -166,7 +166,7 @@ INLINE_LIMIT?= 8000 # code model as "medium" and "medany" respectively. # .if ${MACHINE_CPUARCH} == "riscv" -CFLAGS+= -march=rv64imafdch +CFLAGS+= -march=rv64imafdch_zifencei CFLAGS+= -mabi=lp64 CFLAGS.clang+= -mcmodel=medium CFLAGS.gcc+= -mcmodel=medany diff --git a/sys/conf/kern.pre.mk b/sys/conf/kern.pre.mk index 440ed2df5644..cf5e4a96ad49 100644 --- a/sys/conf/kern.pre.mk +++ b/sys/conf/kern.pre.mk @@ -119,11 +119,9 @@ CFLAGS+= ${CONF_CFLAGS} LDFLAGS+= --build-id=sha1 .endif -.if (${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" || \ - ${MACHINE_CPUARCH} == "i386" || ${MACHINE} == "powerpc") && \ - defined(LINKER_FEATURES) && ${LINKER_FEATURES:Mifunc} == "" && \ +.if defined(LINKER_FEATURES) && ${LINKER_FEATURES:Mifunc} == "" && \ !make(install) -.error amd64/arm64/i386/ppc* kernel requires linker ifunc support +.error kernel requires linker ifunc support .endif .if ${MACHINE_CPUARCH} == "amd64" LDFLAGS+= -z max-page-size=2097152 diff --git a/sys/conf/kmod.mk b/sys/conf/kmod.mk index edc311348971..17358428a023 100644 --- a/sys/conf/kmod.mk +++ b/sys/conf/kmod.mk @@ -383,7 +383,7 @@ afterinstall: _kldxref _kldxref: .PHONY ${KLDXREF_CMD} ${DESTDIR}${KMODDIR} .if defined(NO_ROOT) && defined(METALOG) - echo ".${DISTBASE}${KMODDIR}/linker.hints type=file mode=0644 uname=root gname=wheel" | \ + echo ".${DISTBASE}${KMODDIR}/linker.hints type=file uname=root gname=wheel mode=0644" | \ cat -l >> ${METALOG} .endif .endif diff --git a/sys/conf/options b/sys/conf/options index c9b9307718b7..d5192db59181 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -42,7 +42,7 @@ AHD_DEBUG_OPTS opt_aic79xx.h AHD_TMODE_ENABLE opt_aic79xx.h AHD_REG_PRETTY_PRINT opt_aic79xx.h -# Debugging options. +# Debugging options ALT_BREAK_TO_DEBUGGER opt_kdb.h BREAK_TO_DEBUGGER opt_kdb.h BUF_TRACKING opt_global.h @@ -70,7 +70,7 @@ TSLOG opt_global.h TSLOG_PAGEZERO opt_global.h TSLOGSIZE opt_global.h -# Miscellaneous options. +# Miscellaneous options ALQ ATSE_CFI_HACK opt_cfi.h AUDIT opt_global.h @@ -325,10 +325,10 @@ UFS_ACL opt_ufs.h UFS_EXTATTR opt_ufs.h UFS_EXTATTR_AUTOSTART opt_ufs.h -# Enable fast hash lookups for large directories on UFS-based filesystems. +# Enable fast hash lookups for large directories on UFS-based filesystems UFS_DIRHASH opt_ufs.h -# Enable gjournal-based UFS journal. +# Enable gjournal-based UFS journal UFS_GJOURNAL opt_ufs.h # The below sentence is not in English, and neither is this one. @@ -341,14 +341,14 @@ NFS_ROOT opt_nfsroot.h # SMB/CIFS requester NETSMB opt_netsmb.h -# Enable debugnet(4) networking support. +# Enable debugnet(4) networking support DEBUGNET opt_global.h -# Enable netdump(4) client support. +# Enable netdump(4) client support NETDUMP opt_global.h -# Enable netgdb(4) support. +# Enable netgdb(4) support NETGDB opt_global.h -# Options used only in subr_param.c. +# Options used only in subr_param.c HZ opt_param.h MAXFILES opt_param.h NBUF opt_param.h @@ -360,7 +360,7 @@ DFLDSIZ opt_param.h MAXDSIZ opt_param.h MAXSSIZ opt_param.h -# Generic SCSI options. +# Generic SCSI options CAM_MAX_HIGHPOWER opt_cam.h CAMDEBUG opt_cam.h CAM_DEBUG_COMPILE opt_cam.h @@ -387,7 +387,7 @@ CHANGER_MAX_BUSY_SECONDS opt_cd.h # Options used only in cam/scsi/scsi_da.c DA_TRACK_REFS opt_da.h -# Options used only in cam/scsi/scsi_sa.c. +# Options used only in cam/scsi/scsi_sa.c SA_IO_TIMEOUT opt_sa.h SA_SPACE_TIMEOUT opt_sa.h SA_REWIND_TIMEOUT opt_sa.h @@ -400,7 +400,7 @@ SCSI_PT_DEFAULT_TIMEOUT opt_pt.h # Options used only in cam/scsi/scsi_ses.c SES_ENABLE_PASSTHROUGH opt_ses.h -# Options used in dev/sym/ (Symbios SCSI driver). +# Options used in dev/sym/ (Symbios SCSI driver) SYM_SETUP_SCSI_DIFF opt_sym.h #-HVD support for 825a, 875, 885 # disabled:0 (default), enabled:1 SYM_SETUP_PCI_PARITY opt_sym.h #-PCI parity checking @@ -415,7 +415,7 @@ ISP_DEFAULT_ROLES opt_isp.h ISP_INTERNAL_TARGET opt_isp.h ISP_FCTAPE_OFF opt_isp.h -# Net stuff. +# Net stuff ACCEPT_FILTER_DATA ACCEPT_FILTER_DNS ACCEPT_FILTER_HTTP @@ -501,8 +501,8 @@ SCTP_MBUF_LOGGING opt_sctp.h # Log to KTR general mbuf aloc/free SCTP_MBCNT_LOGGING opt_sctp.h # Log to KTR mbcnt activity SCTP_PACKET_LOGGING opt_sctp.h # Log to a packet buffer last N packets SCTP_LTRACE_CHUNKS opt_sctp.h # Log to KTR chunks processed -SCTP_LTRACE_ERRORS opt_sctp.h # Log to KTR error returns. -SCTP_USE_PERCPU_STAT opt_sctp.h # Use per cpu stats. +SCTP_LTRACE_ERRORS opt_sctp.h # Log to KTR error returns +SCTP_USE_PERCPU_STAT opt_sctp.h # Use per cpu stats SCTP_MCORE_INPUT opt_sctp.h # Have multiple input threads for input mbufs SCTP_LOCAL_TRACE_BUF opt_sctp.h # Use tracebuffer exported via sysctl SCTP_DETAILED_STR_STATS opt_sctp.h # Use per PR-SCTP policy stream stats @@ -718,7 +718,7 @@ IOMMU opt_iommu.h DEV_ISA opt_isa.h ISAPNP opt_dontuse.h -# various 'device presence' options. +# various 'device presence' options DEV_BPF opt_bpf.h DEV_CARP opt_carp.h DEV_NETMAP opt_global.h @@ -946,7 +946,7 @@ RACCT_DEFAULT_TO_DISABLED opt_global.h RCTL opt_global.h # Random number generator(s) -# Alternative RNG algorithm. +# Alternative RNG algorithm RANDOM_FENESTRASX opt_global.h # With this, no entropy processor is loaded, but the entropy # harvesting infrastructure is present. This means an entropy @@ -1014,5 +1014,5 @@ U2F_DROP_UHID_ALIAS opt_hid.h # environment's contents are known to be safe. PRESERVE_EARLY_KENV opt_global.h -# Options for the Intel QuickAssist (QAT) driver. +# Options for the Intel QuickAssist (QAT) driver QAT_DISABLE_SAFE_DC_MODE opt_qat.h diff --git a/sys/contrib/dev/athk/ath10k/core.c b/sys/contrib/dev/athk/ath10k/core.c index a0407f693659..9ec08b402fd2 100644 --- a/sys/contrib/dev/athk/ath10k/core.c +++ b/sys/contrib/dev/athk/ath10k/core.c @@ -3,7 +3,6 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -1212,7 +1211,7 @@ static int ath10k_download_fw(struct ath10k *ar) u32 address, data_len; const void *data; int ret; - struct pm_qos_request latency_qos; + struct pm_qos_request latency_qos = {}; address = ar->hw_params.patch_load_addr; @@ -1246,7 +1245,6 @@ static int ath10k_download_fw(struct ath10k *ar) ret); } - memset(&latency_qos, 0, sizeof(latency_qos)); cpu_latency_qos_add_request(&latency_qos, 0); ret = ath10k_bmi_fast_download(ar, address, data, data_len); @@ -2570,8 +2568,9 @@ static int ath10k_init_hw_params(struct ath10k *ar) return 0; } -static bool ath10k_core_needs_recovery(struct ath10k *ar) +static void ath10k_core_recovery_check_work(struct work_struct *work) { + struct ath10k *ar = container_of(work, struct ath10k, recovery_check_work); long time_left; /* Sometimes the recovery will fail and then the next all recovery fail, @@ -2581,7 +2580,7 @@ static bool ath10k_core_needs_recovery(struct ath10k *ar) ath10k_err(ar, "consecutive fail %d times, will shutdown driver!", atomic_read(&ar->fail_cont_count)); ar->state = ATH10K_STATE_WEDGED; - return false; + return; } ath10k_dbg(ar, ATH10K_DBG_BOOT, "total recovery count: %d", ++ar->recovery_count); @@ -2595,27 +2594,24 @@ static bool ath10k_core_needs_recovery(struct ath10k *ar) ATH10K_RECOVERY_TIMEOUT_HZ); if (time_left) { ath10k_warn(ar, "previous recovery succeeded, skip this!\n"); - return false; + return; } /* Record the continuous recovery fail count when recovery failed. */ atomic_inc(&ar->fail_cont_count); /* Avoid having multiple recoveries at the same time. */ - return false; + return; } atomic_inc(&ar->pending_recovery); - - return true; + queue_work(ar->workqueue, &ar->restart_work); } void ath10k_core_start_recovery(struct ath10k *ar) { - if (!ath10k_core_needs_recovery(ar)) - return; - - queue_work(ar->workqueue, &ar->restart_work); + /* Use workqueue_aux to avoid blocking recovery tracking */ + queue_work(ar->workqueue_aux, &ar->recovery_check_work); } EXPORT_SYMBOL(ath10k_core_start_recovery); @@ -3440,7 +3436,7 @@ EXPORT_SYMBOL(ath10k_core_stop); */ static int ath10k_core_probe_fw(struct ath10k *ar) { - struct bmi_target_info target_info; + struct bmi_target_info target_info = {}; int ret = 0; ret = ath10k_hif_power_up(ar, ATH10K_FIRMWARE_MODE_NORMAL); @@ -3451,7 +3447,6 @@ static int ath10k_core_probe_fw(struct ath10k *ar) switch (ar->hif.bus) { case ATH10K_BUS_SDIO: - memset(&target_info, 0, sizeof(target_info)); ret = ath10k_bmi_get_target_info_sdio(ar, &target_info); if (ret) { ath10k_err(ar, "could not get target info (%d)\n", ret); @@ -3463,7 +3458,6 @@ static int ath10k_core_probe_fw(struct ath10k *ar) case ATH10K_BUS_PCI: case ATH10K_BUS_AHB: case ATH10K_BUS_USB: - memset(&target_info, 0, sizeof(target_info)); ret = ath10k_bmi_get_target_info(ar, &target_info); if (ret) { ath10k_err(ar, "could not get target info (%d)\n", ret); @@ -3473,7 +3467,6 @@ static int ath10k_core_probe_fw(struct ath10k *ar) ar->hw->wiphy->hw_version = target_info.version; break; case ATH10K_BUS_SNOC: - memset(&target_info, 0, sizeof(target_info)); ret = ath10k_hif_get_target_info(ar, &target_info); if (ret) { ath10k_err(ar, "could not get target info (%d)\n", ret); @@ -3824,6 +3817,7 @@ struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev, INIT_WORK(&ar->register_work, ath10k_core_register_work); INIT_WORK(&ar->restart_work, ath10k_core_restart); + INIT_WORK(&ar->recovery_check_work, ath10k_core_recovery_check_work); INIT_WORK(&ar->set_coverage_class_work, ath10k_core_set_coverage_class_work); diff --git a/sys/contrib/dev/athk/ath10k/core.h b/sys/contrib/dev/athk/ath10k/core.h index cb250ca6991d..eaf122d4b112 100644 --- a/sys/contrib/dev/athk/ath10k/core.h +++ b/sys/contrib/dev/athk/ath10k/core.h @@ -3,7 +3,6 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -1216,6 +1215,7 @@ struct ath10k { struct work_struct register_work; struct work_struct restart_work; + struct work_struct recovery_check_work; struct work_struct bundle_tx_work; struct work_struct tx_complete_work; @@ -1267,9 +1267,13 @@ struct ath10k { struct { /* protected by conf_mutex */ struct ath10k_fw_components utf_mode_fw; + u8 ftm_msgref; /* protected by data_lock */ bool utf_monitor; + u32 data_pos; + u32 expected_seq; + u8 *eventdata; } testmode; struct { diff --git a/sys/contrib/dev/athk/ath10k/mac.c b/sys/contrib/dev/athk/ath10k/mac.c index 6725c2c742bd..e2bda3c0d925 100644 --- a/sys/contrib/dev/athk/ath10k/mac.c +++ b/sys/contrib/dev/athk/ath10k/mac.c @@ -3,7 +3,6 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -5442,6 +5441,7 @@ static void ath10k_stop(struct ieee80211_hw *hw, bool suspend) cancel_work_sync(&ar->set_coverage_class_work); cancel_delayed_work_sync(&ar->scan.timeout); cancel_work_sync(&ar->restart_work); + cancel_work_sync(&ar->recovery_check_work); } static int ath10k_config_ps(struct ath10k *ar) diff --git a/sys/contrib/dev/athk/ath10k/qmi.c b/sys/contrib/dev/athk/ath10k/qmi.c index f1f33af0170a..8275345631a0 100644 --- a/sys/contrib/dev/athk/ath10k/qmi.c +++ b/sys/contrib/dev/athk/ath10k/qmi.c @@ -986,7 +986,7 @@ static int ath10k_qmi_new_server(struct qmi_handle *qmi_hdl, ath10k_dbg(ar, ATH10K_DBG_QMI, "wifi fw qmi service found\n"); - ret = kernel_connect(qmi_hdl->sock, (struct sockaddr *)&qmi->sq, + ret = kernel_connect(qmi_hdl->sock, (struct sockaddr_unsized *)&qmi->sq, sizeof(qmi->sq), 0); if (ret) { ath10k_err(ar, "failed to connect to a remote QMI service port\n"); diff --git a/sys/contrib/dev/athk/ath10k/testmode.c b/sys/contrib/dev/athk/ath10k/testmode.c index 3fcefc55b74f..d3bd385694d6 100644 --- a/sys/contrib/dev/athk/ath10k/testmode.c +++ b/sys/contrib/dev/athk/ath10k/testmode.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: ISC /* * Copyright (c) 2014-2017 Qualcomm Atheros, Inc. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "testmode.h" @@ -10,12 +11,17 @@ #include "debug.h" #include "wmi.h" +#include "wmi-tlv.h" #include "hif.h" #include "hw.h" #include "core.h" #include "testmode_i.h" +#define ATH10K_FTM_SEG_NONE ((u32)-1) +#define ATH10K_FTM_SEGHDR_CURRENT_SEQ GENMASK(3, 0) +#define ATH10K_FTM_SEGHDR_TOTAL_SEGMENTS GENMASK(7, 4) + static const struct nla_policy ath10k_tm_policy[ATH10K_TM_ATTR_MAX + 1] = { [ATH10K_TM_ATTR_CMD] = { .type = NLA_U32 }, [ATH10K_TM_ATTR_DATA] = { .type = NLA_BINARY, @@ -25,41 +31,19 @@ static const struct nla_policy ath10k_tm_policy[ATH10K_TM_ATTR_MAX + 1] = { [ATH10K_TM_ATTR_VERSION_MINOR] = { .type = NLA_U32 }, }; -/* Returns true if callee consumes the skb and the skb should be discarded. - * Returns false if skb is not used. Does not sleep. - */ -bool ath10k_tm_event_wmi(struct ath10k *ar, u32 cmd_id, struct sk_buff *skb) +static void ath10k_tm_event_unsegmented(struct ath10k *ar, u32 cmd_id, + struct sk_buff *skb) { struct sk_buff *nl_skb; - bool consumed; int ret; - ath10k_dbg(ar, ATH10K_DBG_TESTMODE, - "testmode event wmi cmd_id %d skb %p skb->len %d\n", - cmd_id, skb, skb->len); - - ath10k_dbg_dump(ar, ATH10K_DBG_TESTMODE, NULL, "", skb->data, skb->len); - - spin_lock_bh(&ar->data_lock); - - if (!ar->testmode.utf_monitor) { - consumed = false; - goto out; - } - - /* Only testmode.c should be handling events from utf firmware, - * otherwise all sort of problems will arise as mac80211 operations - * are not initialised. - */ - consumed = true; - nl_skb = cfg80211_testmode_alloc_event_skb(ar->hw->wiphy, 2 * sizeof(u32) + skb->len, GFP_ATOMIC); if (!nl_skb) { ath10k_warn(ar, "failed to allocate skb for testmode wmi event\n"); - goto out; + return; } ret = nla_put_u32(nl_skb, ATH10K_TM_ATTR_CMD, ATH10K_TM_CMD_WMI); @@ -68,7 +52,7 @@ bool ath10k_tm_event_wmi(struct ath10k *ar, u32 cmd_id, struct sk_buff *skb) "failed to put testmode wmi event cmd attribute: %d\n", ret); kfree_skb(nl_skb); - goto out; + return; } ret = nla_put_u32(nl_skb, ATH10K_TM_ATTR_WMI_CMDID, cmd_id); @@ -77,7 +61,7 @@ bool ath10k_tm_event_wmi(struct ath10k *ar, u32 cmd_id, struct sk_buff *skb) "failed to put testmode wmi event cmd_id: %d\n", ret); kfree_skb(nl_skb); - goto out; + return; } ret = nla_put(nl_skb, ATH10K_TM_ATTR_DATA, skb->len, skb->data); @@ -86,10 +70,122 @@ bool ath10k_tm_event_wmi(struct ath10k *ar, u32 cmd_id, struct sk_buff *skb) "failed to copy skb to testmode wmi event: %d\n", ret); kfree_skb(nl_skb); - goto out; + return; + } + + cfg80211_testmode_event(nl_skb, GFP_ATOMIC); +} + +static void ath10k_tm_event_segmented(struct ath10k *ar, u32 cmd_id, struct sk_buff *skb) +{ + struct wmi_ftm_cmd *ftm = (struct wmi_ftm_cmd *)skb->data; + u8 total_segments, current_seq; + struct sk_buff *nl_skb; + u8 const *buf_pos; + u16 datalen; + u32 data_pos; + int ret; + + if (skb->len < sizeof(*ftm)) { + ath10k_warn(ar, "Invalid ftm event length: %d\n", skb->len); + return; + } + + current_seq = FIELD_GET(ATH10K_FTM_SEGHDR_CURRENT_SEQ, + __le32_to_cpu(ftm->seg_hdr.segmentinfo)); + total_segments = FIELD_GET(ATH10K_FTM_SEGHDR_TOTAL_SEGMENTS, + __le32_to_cpu(ftm->seg_hdr.segmentinfo)); + datalen = skb->len - sizeof(*ftm); + buf_pos = ftm->data; + + if (current_seq == 0) { + ar->testmode.expected_seq = 0; + ar->testmode.data_pos = 0; + } + + data_pos = ar->testmode.data_pos; + + if ((data_pos + datalen) > ATH_FTM_EVENT_MAX_BUF_LENGTH) { + ath10k_warn(ar, "Invalid ftm event length at %u: %u\n", + data_pos, datalen); + ret = -EINVAL; + return; + } + + memcpy(&ar->testmode.eventdata[data_pos], buf_pos, datalen); + data_pos += datalen; + + if (++ar->testmode.expected_seq != total_segments) { + ar->testmode.data_pos = data_pos; + ath10k_dbg(ar, ATH10K_DBG_TESTMODE, "partial data received %u/%u\n", + current_seq + 1, total_segments); + return; + } + + ath10k_dbg(ar, ATH10K_DBG_TESTMODE, "total data length %u\n", data_pos); + + nl_skb = cfg80211_testmode_alloc_event_skb(ar->hw->wiphy, + 2 * sizeof(u32) + data_pos, + GFP_ATOMIC); + if (!nl_skb) { + ath10k_warn(ar, "failed to allocate skb for testmode wmi event\n"); + return; + } + + ret = nla_put_u32(nl_skb, ATH10K_TM_ATTR_CMD, ATH10K_TM_CMD_TLV); + if (ret) { + ath10k_warn(ar, "failed to put testmode wmi event attribute: %d\n", ret); + kfree_skb(nl_skb); + return; + } + + ret = nla_put_u32(nl_skb, ATH10K_TM_ATTR_WMI_CMDID, cmd_id); + if (ret) { + ath10k_warn(ar, "failed to put testmode wmi event cmd_id: %d\n", ret); + kfree_skb(nl_skb); + return; + } + + ret = nla_put(nl_skb, ATH10K_TM_ATTR_DATA, data_pos, &ar->testmode.eventdata[0]); + if (ret) { + ath10k_warn(ar, "failed to copy skb to testmode wmi event: %d\n", ret); + kfree_skb(nl_skb); + return; } cfg80211_testmode_event(nl_skb, GFP_ATOMIC); +} + +/* Returns true if callee consumes the skb and the skb should be discarded. + * Returns false if skb is not used. Does not sleep. + */ +bool ath10k_tm_event_wmi(struct ath10k *ar, u32 cmd_id, struct sk_buff *skb) +{ + bool consumed; + + ath10k_dbg(ar, ATH10K_DBG_TESTMODE, + "testmode event wmi cmd_id %d skb %p skb->len %d\n", + cmd_id, skb, skb->len); + + ath10k_dbg_dump(ar, ATH10K_DBG_TESTMODE, NULL, "", skb->data, skb->len); + + spin_lock_bh(&ar->data_lock); + + if (!ar->testmode.utf_monitor) { + consumed = false; + goto out; + } + + /* Only testmode.c should be handling events from utf firmware, + * otherwise all sort of problems will arise as mac80211 operations + * are not initialised. + */ + consumed = true; + + if (ar->testmode.expected_seq != ATH10K_FTM_SEG_NONE) + ath10k_tm_event_segmented(ar, cmd_id, skb); + else + ath10k_tm_event_unsegmented(ar, cmd_id, skb); out: spin_unlock_bh(&ar->data_lock); @@ -281,12 +377,18 @@ static int ath10k_tm_cmd_utf_start(struct ath10k *ar, struct nlattr *tb[]) goto err_release_utf_mode_fw; } + ar->testmode.eventdata = kzalloc(ATH_FTM_EVENT_MAX_BUF_LENGTH, GFP_KERNEL); + if (!ar->testmode.eventdata) { + ret = -ENOMEM; + goto err_power_down; + } + ret = ath10k_core_start(ar, ATH10K_FIRMWARE_MODE_UTF, &ar->testmode.utf_mode_fw); if (ret) { ath10k_err(ar, "failed to start core (testmode): %d\n", ret); ar->state = ATH10K_STATE_OFF; - goto err_power_down; + goto err_release_eventdata; } ar->state = ATH10K_STATE_UTF; @@ -302,6 +404,10 @@ static int ath10k_tm_cmd_utf_start(struct ath10k *ar, struct nlattr *tb[]) return 0; +err_release_eventdata: + kfree(ar->testmode.eventdata); + ar->testmode.eventdata = NULL; + err_power_down: ath10k_hif_power_down(ar); @@ -341,6 +447,9 @@ static void __ath10k_tm_cmd_utf_stop(struct ath10k *ar) release_firmware(ar->testmode.utf_mode_fw.fw_file.firmware); ar->testmode.utf_mode_fw.fw_file.firmware = NULL; + kfree(ar->testmode.eventdata); + ar->testmode.eventdata = NULL; + ar->state = ATH10K_STATE_OFF; } @@ -424,6 +533,85 @@ out: return ret; } +static int ath10k_tm_cmd_tlv(struct ath10k *ar, struct nlattr *tb[]) +{ + u16 total_bytes, num_segments; + u32 cmd_id, buf_len; + u8 segnumber = 0; + u8 *bufpos; + void *buf; + int ret; + + mutex_lock(&ar->conf_mutex); + + if (ar->state != ATH10K_STATE_UTF) { + ret = -ENETDOWN; + goto out; + } + + buf = nla_data(tb[ATH10K_TM_ATTR_DATA]); + buf_len = nla_len(tb[ATH10K_TM_ATTR_DATA]); + cmd_id = WMI_PDEV_UTF_CMDID; + + ath10k_dbg(ar, ATH10K_DBG_TESTMODE, + "cmd wmi ftm cmd_id %d buffer length %d\n", + cmd_id, buf_len); + ath10k_dbg_dump(ar, ATH10K_DBG_TESTMODE, NULL, "", buf, buf_len); + + bufpos = buf; + total_bytes = buf_len; + num_segments = total_bytes / MAX_WMI_UTF_LEN; + ar->testmode.expected_seq = 0; + + if (buf_len - (num_segments * MAX_WMI_UTF_LEN)) + num_segments++; + + while (buf_len) { + u16 chunk_len = min_t(u16, buf_len, MAX_WMI_UTF_LEN); + struct wmi_ftm_cmd *ftm_cmd; + struct sk_buff *skb; + u32 hdr_info; + u8 seginfo; + + skb = ath10k_wmi_alloc_skb(ar, (chunk_len + + sizeof(struct wmi_ftm_cmd))); + if (!skb) { + ret = -ENOMEM; + goto out; + } + + ftm_cmd = (struct wmi_ftm_cmd *)skb->data; + hdr_info = FIELD_PREP(WMI_TLV_TAG, WMI_TLV_TAG_ARRAY_BYTE) | + FIELD_PREP(WMI_TLV_LEN, (chunk_len + + sizeof(struct wmi_ftm_seg_hdr))); + ftm_cmd->tlv_header = __cpu_to_le32(hdr_info); + ftm_cmd->seg_hdr.len = __cpu_to_le32(total_bytes); + ftm_cmd->seg_hdr.msgref = __cpu_to_le32(ar->testmode.ftm_msgref); + seginfo = FIELD_PREP(ATH10K_FTM_SEGHDR_TOTAL_SEGMENTS, num_segments) | + FIELD_PREP(ATH10K_FTM_SEGHDR_CURRENT_SEQ, segnumber); + ftm_cmd->seg_hdr.segmentinfo = __cpu_to_le32(seginfo); + segnumber++; + + memcpy(&ftm_cmd->data, bufpos, chunk_len); + + ret = ath10k_wmi_cmd_send(ar, skb, cmd_id); + if (ret) { + ath10k_warn(ar, "failed to send wmi ftm command: %d\n", ret); + goto out; + } + + buf_len -= chunk_len; + bufpos += chunk_len; + } + + ar->testmode.ftm_msgref++; + ret = 0; + +out: + mutex_unlock(&ar->conf_mutex); + return ret; +} + int ath10k_tm_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif, void *data, int len) { @@ -439,9 +627,14 @@ int ath10k_tm_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif, if (!tb[ATH10K_TM_ATTR_CMD]) return -EINVAL; + ar->testmode.expected_seq = ATH10K_FTM_SEG_NONE; + switch (nla_get_u32(tb[ATH10K_TM_ATTR_CMD])) { case ATH10K_TM_CMD_GET_VERSION: - return ath10k_tm_cmd_get_version(ar, tb); + if (!tb[ATH10K_TM_ATTR_DATA]) + return ath10k_tm_cmd_get_version(ar, tb); + else /* ATH10K_TM_CMD_TLV */ + return ath10k_tm_cmd_tlv(ar, tb); case ATH10K_TM_CMD_UTF_START: return ath10k_tm_cmd_utf_start(ar, tb); case ATH10K_TM_CMD_UTF_STOP: diff --git a/sys/contrib/dev/athk/ath10k/testmode_i.h b/sys/contrib/dev/athk/ath10k/testmode_i.h index ee1cb27c1d60..1603f5276682 100644 --- a/sys/contrib/dev/athk/ath10k/testmode_i.h +++ b/sys/contrib/dev/athk/ath10k/testmode_i.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: ISC */ /* * Copyright (c) 2014,2017 Qualcomm Atheros, Inc. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ /* "API" level of the ath10k testmode interface. Bump it after every @@ -14,6 +15,7 @@ #define ATH10K_TESTMODE_VERSION_MINOR 0 #define ATH10K_TM_DATA_MAX_LEN 5000 +#define ATH_FTM_EVENT_MAX_BUF_LENGTH 2048 enum ath10k_tm_attr { __ATH10K_TM_ATTR_INVALID = 0, @@ -57,4 +59,17 @@ enum ath10k_tm_cmd { * ATH10K_TM_ATTR_DATA. */ ATH10K_TM_CMD_WMI = 3, + + /* The command used to transmit a test command to the firmware + * and the event to receive test events from the firmware. The data + * received only contain the TLV payload, need to add the tlv header + * and send the cmd to firmware with command id WMI_PDEV_UTF_CMDID. + * The data payload size could be large and the driver needs to + * send segmented data to firmware. + * + * This legacy testmode command shares the same value as the get-version + * command. To distinguish between them, we check whether the data attribute + * is present. + */ + ATH10K_TM_CMD_TLV = ATH10K_TM_CMD_GET_VERSION, }; diff --git a/sys/contrib/dev/athk/ath10k/wmi.h b/sys/contrib/dev/athk/ath10k/wmi.h index 0faefc0a9a40..7f50a1de6b97 100644 --- a/sys/contrib/dev/athk/ath10k/wmi.h +++ b/sys/contrib/dev/athk/ath10k/wmi.h @@ -3,7 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef _WMI_H_ @@ -7418,6 +7418,23 @@ struct wmi_pdev_bb_timing_cfg_cmd { __le32 bb_xpa_timing; } __packed; +struct wmi_ftm_seg_hdr { + __le32 len; + __le32 msgref; + __le32 segmentinfo; + __le32 pdev_id; +} __packed; + +struct wmi_ftm_cmd { + __le32 tlv_header; + struct wmi_ftm_seg_hdr seg_hdr; + u8 data[]; +} __packed; + +#define WMI_TLV_LEN GENMASK(15, 0) +#define WMI_TLV_TAG GENMASK(31, 16) +#define MAX_WMI_UTF_LEN 252 + struct ath10k; struct ath10k_vif; struct ath10k_fw_stats_pdev; diff --git a/sys/contrib/dev/mediatek/mt76/Kconfig b/sys/contrib/dev/mediatek/mt76/Kconfig new file mode 100644 index 000000000000..502303622a53 --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/Kconfig @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: BSD-3-Clause-Clear +config MT76_CORE + tristate + select PAGE_POOL + +config MT76_LEDS + bool + depends on MT76_CORE + depends on LEDS_CLASS=y || MT76_CORE=LEDS_CLASS + default y + +config MT76_USB + tristate + depends on MT76_CORE + +config MT76_SDIO + tristate + depends on MT76_CORE + +config MT76x02_LIB + tristate + select MT76_CORE + +config MT76x02_USB + tristate + select MT76_USB + +config MT76_CONNAC_LIB + tristate + select MT76_CORE + +config MT792x_LIB + tristate + select MT76_CONNAC_LIB + +config MT792x_USB + tristate + select MT76_USB + +config MT76_NPU + bool + depends on MT76_CORE + +source "drivers/net/wireless/mediatek/mt76/mt76x0/Kconfig" +source "drivers/net/wireless/mediatek/mt76/mt76x2/Kconfig" +source "drivers/net/wireless/mediatek/mt76/mt7603/Kconfig" +source "drivers/net/wireless/mediatek/mt76/mt7615/Kconfig" +source "drivers/net/wireless/mediatek/mt76/mt7915/Kconfig" +source "drivers/net/wireless/mediatek/mt76/mt7921/Kconfig" +source "drivers/net/wireless/mediatek/mt76/mt7996/Kconfig" +source "drivers/net/wireless/mediatek/mt76/mt7925/Kconfig" diff --git a/sys/contrib/dev/mediatek/mt76/Makefile b/sys/contrib/dev/mediatek/mt76/Makefile new file mode 100644 index 000000000000..1d42adfe8030 --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/Makefile @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: BSD-3-Clause-Clear +obj-$(CONFIG_MT76_CORE) += mt76.o +obj-$(CONFIG_MT76_USB) += mt76-usb.o +obj-$(CONFIG_MT76_SDIO) += mt76-sdio.o +obj-$(CONFIG_MT76x02_LIB) += mt76x02-lib.o +obj-$(CONFIG_MT76x02_USB) += mt76x02-usb.o +obj-$(CONFIG_MT76_CONNAC_LIB) += mt76-connac-lib.o +obj-$(CONFIG_MT792x_LIB) += mt792x-lib.o +obj-$(CONFIG_MT792x_USB) += mt792x-usb.o + +mt76-y := \ + mmio.o util.o trace.o dma.o mac80211.o debugfs.o eeprom.o \ + tx.o agg-rx.o mcu.o wed.o scan.o channel.o + +mt76-$(CONFIG_MT76_NPU) += npu.o +mt76-$(CONFIG_PCI) += pci.o +mt76-$(CONFIG_NL80211_TESTMODE) += testmode.o + +mt76-usb-y := usb.o usb_trace.o +mt76-sdio-y := sdio.o sdio_txrx.o + +CFLAGS_trace.o := -I$(src) +CFLAGS_usb_trace.o := -I$(src) +CFLAGS_mt76x02_trace.o := -I$(src) +CFLAGS_mt792x_trace.o := -I$(src) + +mt76x02-lib-y := mt76x02_util.o mt76x02_mac.o mt76x02_mcu.o \ + mt76x02_eeprom.o mt76x02_phy.o mt76x02_mmio.o \ + mt76x02_txrx.o mt76x02_trace.o mt76x02_debugfs.o \ + mt76x02_dfs.o mt76x02_beacon.o + +mt76x02-usb-y := mt76x02_usb_mcu.o mt76x02_usb_core.o + +mt76-connac-lib-y := mt76_connac_mcu.o mt76_connac_mac.o mt76_connac3_mac.o + +mt792x-lib-y := mt792x_core.o mt792x_mac.o mt792x_trace.o \ + mt792x_debugfs.o mt792x_dma.o +mt792x-lib-$(CONFIG_ACPI) += mt792x_acpi_sar.o +mt792x-usb-y := mt792x_usb.o + +obj-$(CONFIG_MT76x0_COMMON) += mt76x0/ +obj-$(CONFIG_MT76x2_COMMON) += mt76x2/ +obj-$(CONFIG_MT7603E) += mt7603/ +obj-$(CONFIG_MT7615_COMMON) += mt7615/ +obj-$(CONFIG_MT7915E) += mt7915/ +obj-$(CONFIG_MT7921_COMMON) += mt7921/ +obj-$(CONFIG_MT7996E) += mt7996/ +obj-$(CONFIG_MT7925_COMMON) += mt7925/ diff --git a/sys/contrib/dev/mediatek/mt76/agg-rx.c b/sys/contrib/dev/mediatek/mt76/agg-rx.c index 07c386c7b4d0..3d34caf7e4f7 100644 --- a/sys/contrib/dev/mediatek/mt76/agg-rx.c +++ b/sys/contrib/dev/mediatek/mt76/agg-rx.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Felix Fietkau <nbd@nbd.name> */ @@ -173,6 +173,8 @@ void mt76_rx_aggr_reorder(struct sk_buff *skb, struct sk_buff_head *frames) if (ackp == IEEE80211_QOS_CTL_ACK_POLICY_NOACK) return; + if (wcid->def_wcid) + wcid = wcid->def_wcid; tid = rcu_dereference(wcid->aggr[tidno]); if (!tid) return; diff --git a/sys/contrib/dev/mediatek/mt76/channel.c b/sys/contrib/dev/mediatek/mt76/channel.c index 77b75792eb48..2b705bdb7993 100644 --- a/sys/contrib/dev/mediatek/mt76/channel.c +++ b/sys/contrib/dev/mediatek/mt76/channel.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2024 Felix Fietkau <nbd@nbd.name> */ @@ -314,21 +314,24 @@ void mt76_put_vif_phy_link(struct mt76_phy *phy, struct ieee80211_vif *vif, kfree(mlink); } -static void mt76_roc_complete(struct mt76_phy *phy) +void mt76_roc_complete(struct mt76_phy *phy) { struct mt76_vif_link *mlink = phy->roc_link; + struct mt76_dev *dev = phy->dev; if (!phy->roc_vif) return; if (mlink) mlink->mvif->roc_phy = NULL; - if (phy->main_chandef.chan) + if (phy->main_chandef.chan && + !test_bit(MT76_MCU_RESET, &dev->phy.state)) mt76_set_channel(phy, &phy->main_chandef, false); mt76_put_vif_phy_link(phy, phy->roc_vif, phy->roc_link); phy->roc_vif = NULL; phy->roc_link = NULL; - ieee80211_remain_on_channel_expired(phy->hw); + if (!test_bit(MT76_MCU_RESET, &dev->phy.state)) + ieee80211_remain_on_channel_expired(phy->hw); } void mt76_roc_complete_work(struct work_struct *work) @@ -351,6 +354,7 @@ void mt76_abort_roc(struct mt76_phy *phy) mt76_roc_complete(phy); mutex_unlock(&dev->mutex); } +EXPORT_SYMBOL_GPL(mt76_abort_roc); int mt76_remain_on_channel(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_channel *chan, int duration, @@ -368,7 +372,8 @@ int mt76_remain_on_channel(struct ieee80211_hw *hw, struct ieee80211_vif *vif, mutex_lock(&dev->mutex); - if (phy->roc_vif || dev->scan.phy == phy) { + if (phy->roc_vif || dev->scan.phy == phy || + test_bit(MT76_MCU_RESET, &dev->phy.state)) { ret = -EBUSY; goto out; } diff --git a/sys/contrib/dev/mediatek/mt76/debugfs.c b/sys/contrib/dev/mediatek/mt76/debugfs.c index b6a2746c187d..a5ac6ca86735 100644 --- a/sys/contrib/dev/mediatek/mt76/debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -93,9 +93,9 @@ void mt76_seq_puts_array(struct seq_file *file, const char *str, { int i; - seq_printf(file, "%10s:", str); + seq_printf(file, "%16s:", str); for (i = 0; i < len; i++) - seq_printf(file, " %2d", val[i]); + seq_printf(file, " %4d", val[i]); seq_puts(file, "\n"); } EXPORT_SYMBOL_GPL(mt76_seq_puts_array); diff --git a/sys/contrib/dev/mediatek/mt76/dma.c b/sys/contrib/dev/mediatek/mt76/dma.c index af902a761e42..5c04a0dce1fa 100644 --- a/sys/contrib/dev/mediatek/mt76/dma.c +++ b/sys/contrib/dev/mediatek/mt76/dma.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -11,37 +11,6 @@ #include "mt76.h" #include "dma.h" -#if IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) - -#define Q_READ(_q, _field) ({ \ - u32 _offset = offsetof(struct mt76_queue_regs, _field); \ - u32 _val; \ - if ((_q)->flags & MT_QFLAG_WED) \ - _val = mtk_wed_device_reg_read((_q)->wed, \ - ((_q)->wed_regs + \ - _offset)); \ - else \ - _val = readl(&(_q)->regs->_field); \ - _val; \ -}) - -#define Q_WRITE(_q, _field, _val) do { \ - u32 _offset = offsetof(struct mt76_queue_regs, _field); \ - if ((_q)->flags & MT_QFLAG_WED) \ - mtk_wed_device_reg_write((_q)->wed, \ - ((_q)->wed_regs + _offset), \ - _val); \ - else \ - writel(_val, &(_q)->regs->_field); \ -} while (0) - -#else - -#define Q_READ(_q, _field) readl(&(_q)->regs->_field) -#define Q_WRITE(_q, _field, _val) writel(_val, &(_q)->regs->_field) - -#endif - static struct mt76_txwi_cache * mt76_alloc_txwi(struct mt76_dev *dev) { @@ -190,24 +159,61 @@ mt76_free_pending_rxwi(struct mt76_dev *dev) EXPORT_SYMBOL_GPL(mt76_free_pending_rxwi); static void +mt76_dma_queue_magic_cnt_init(struct mt76_dev *dev, struct mt76_queue *q) +{ + if (!mt76_queue_is_wed_rro(q)) + return; + + q->magic_cnt = 0; + if (mt76_queue_is_wed_rro_ind(q)) { + struct mt76_wed_rro_desc *rro_desc; + u32 data1 = FIELD_PREP(RRO_IND_DATA1_MAGIC_CNT_MASK, + MT_DMA_WED_IND_CMD_CNT - 1); + int i; + + rro_desc = (struct mt76_wed_rro_desc *)q->desc; + for (i = 0; i < q->ndesc; i++) { + struct mt76_wed_rro_ind *cmd; + + cmd = (struct mt76_wed_rro_ind *)&rro_desc[i]; + cmd->data1 = cpu_to_le32(data1); + } + } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { + struct mt76_rro_rxdmad_c *dmad = (void *)q->desc; + u32 data3 = FIELD_PREP(RRO_RXDMAD_DATA3_MAGIC_CNT_MASK, + MT_DMA_MAGIC_CNT - 1); + int i; + + for (i = 0; i < q->ndesc; i++) + dmad[i].data3 = cpu_to_le32(data3); + } +} + +static void mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) { Q_WRITE(q, desc_base, q->desc_dma); - if (q->flags & MT_QFLAG_WED_RRO_EN) + if ((q->flags & MT_QFLAG_WED_RRO_EN) && !mt76_npu_device_active(dev)) Q_WRITE(q, ring_size, MT_DMA_RRO_EN | q->ndesc); else Q_WRITE(q, ring_size, q->ndesc); + + if (mt76_queue_is_npu_tx(q)) { + writel(q->desc_dma, &q->regs->desc_base); + writel(q->ndesc, &q->regs->ring_size); + } q->head = Q_READ(q, dma_idx); q->tail = q->head; } -void __mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, - bool reset_idx) +void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, + bool reset_idx) { if (!q || !q->ndesc) return; - if (!mt76_queue_is_wed_rro_ind(q)) { + if (!mt76_queue_is_wed_rro_ind(q) && + !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) { int i; /* clear descriptors */ @@ -215,27 +221,26 @@ void __mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); } + mt76_dma_queue_magic_cnt_init(dev, q); if (reset_idx) { - Q_WRITE(q, cpu_idx, 0); + if (mt76_queue_is_emi(q)) + *q->emi_cpu_idx = 0; + else + Q_WRITE(q, cpu_idx, 0); Q_WRITE(q, dma_idx, 0); } mt76_dma_sync_idx(dev, q); } -void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q) -{ - __mt76_dma_queue_reset(dev, q, true); -} - static int mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q, struct mt76_queue_buf *buf, void *data) { struct mt76_queue_entry *entry = &q->entry[q->head]; struct mt76_txwi_cache *txwi = NULL; + u32 buf1 = 0, ctrl, info = 0; struct mt76_desc *desc; int idx = q->head; - u32 buf1 = 0, ctrl; int rx_token; if (mt76_queue_is_wed_rro_ind(q)) { @@ -244,6 +249,9 @@ mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q, rro_desc = (struct mt76_wed_rro_desc *)q->desc; data = &rro_desc[q->head]; goto done; + } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { + data = &q->desc[q->head]; + goto done; } desc = &q->desc[q->head]; @@ -252,7 +260,7 @@ mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q, buf1 = FIELD_PREP(MT_DMA_CTL_SDP0_H, buf->addr >> 32); #endif - if (mt76_queue_is_wed_rx(q)) { + if (mt76_queue_is_wed_rx(q) || mt76_queue_is_wed_rro_data(q)) { txwi = mt76_get_rxwi(dev); if (!txwi) return -ENOMEM; @@ -265,12 +273,26 @@ mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q, buf1 |= FIELD_PREP(MT_DMA_CTL_TOKEN, rx_token); ctrl |= MT_DMA_CTL_TO_HOST; + + txwi->qid = q - dev->q_rx; + } + + if (mt76_queue_is_wed_rro_msdu_pg(q) && + dev->drv->rx_rro_add_msdu_page) { + if (dev->drv->rx_rro_add_msdu_page(dev, q, buf->addr, data)) + return -ENOMEM; + } + + if (q->flags & MT_QFLAG_WED_RRO_EN) { + info |= FIELD_PREP(MT_DMA_MAGIC_MASK, q->magic_cnt); + if ((q->head + 1) == q->ndesc) + q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_MAGIC_CNT; } WRITE_ONCE(desc->buf0, cpu_to_le32(buf->addr)); WRITE_ONCE(desc->buf1, cpu_to_le32(buf1)); WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl)); - WRITE_ONCE(desc->info, 0); + WRITE_ONCE(desc->info, cpu_to_le32(info)); done: entry->dma_addr[0] = buf->addr; @@ -379,7 +401,10 @@ static void mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q) { wmb(); - Q_WRITE(q, cpu_idx, q->head); + if (mt76_queue_is_emi(q)) + *q->emi_cpu_idx = cpu_to_le16(q->head); + else + Q_WRITE(q, cpu_idx, q->head); } static void @@ -399,6 +424,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) while (q->queued > 0 && q->tail != last) { mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry); + mt76_npu_txdesc_cleanup(q, q->tail); mt76_queue_tx_complete(dev, q, &entry); if (entry.txwi) { @@ -423,15 +449,61 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) } static void * +mt76_dma_get_rxdmad_c_buf(struct mt76_dev *dev, struct mt76_queue *q, + int idx, int *len, bool *more) +{ + struct mt76_queue_entry *e = &q->entry[idx]; + struct mt76_rro_rxdmad_c *dmad = e->buf; + u32 data1 = le32_to_cpu(dmad->data1); + u32 data2 = le32_to_cpu(dmad->data2); + struct mt76_txwi_cache *t; + u16 rx_token_id; + u8 ind_reason; + void *buf; + + rx_token_id = FIELD_GET(RRO_RXDMAD_DATA2_RX_TOKEN_ID_MASK, data2); + t = mt76_rx_token_release(dev, rx_token_id); + if (!t) + return ERR_PTR(-EAGAIN); + + q = &dev->q_rx[t->qid]; + dma_sync_single_for_cpu(dev->dma_dev, t->dma_addr, + SKB_WITH_OVERHEAD(q->buf_size), + page_pool_get_dma_dir(q->page_pool)); + + if (len) + *len = FIELD_GET(RRO_RXDMAD_DATA1_SDL0_MASK, data1); + if (more) + *more = !FIELD_GET(RRO_RXDMAD_DATA1_LS_MASK, data1); + + buf = t->ptr; + ind_reason = FIELD_GET(RRO_RXDMAD_DATA2_IND_REASON_MASK, data2); + if (ind_reason == MT_DMA_WED_IND_REASON_REPEAT || + ind_reason == MT_DMA_WED_IND_REASON_OLDPKT) { + mt76_put_page_pool_buf(buf, false); + buf = ERR_PTR(-EAGAIN); + } + t->ptr = NULL; + t->dma_addr = 0; + + mt76_put_rxwi(dev, t); + + return buf; +} + +static void * mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, - int *len, u32 *info, bool *more, bool *drop) + int *len, u32 *info, bool *more, bool *drop, bool flush) { struct mt76_queue_entry *e = &q->entry[idx]; struct mt76_desc *desc = &q->desc[idx]; u32 ctrl, desc_info, buf1; void *buf = e->buf; - if (mt76_queue_is_wed_rro_ind(q)) + if (mt76_queue_is_wed_rro_rxdmad_c(q) && !flush) + buf = mt76_dma_get_rxdmad_c_buf(dev, q, idx, len, more); + + if (mt76_queue_is_wed_rro(q)) goto done; ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); @@ -486,20 +558,50 @@ mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, if (!q->queued) return NULL; - if (mt76_queue_is_wed_rro_data(q)) - return NULL; + if (mt76_queue_is_wed_rro_data(q) || mt76_queue_is_wed_rro_msdu_pg(q)) + goto done; + + if (mt76_queue_is_wed_rro_ind(q)) { + struct mt76_wed_rro_ind *cmd; + u8 magic_cnt; - if (!mt76_queue_is_wed_rro_ind(q)) { + if (flush) + goto done; + + cmd = q->entry[idx].buf; + magic_cnt = FIELD_GET(RRO_IND_DATA1_MAGIC_CNT_MASK, + le32_to_cpu(cmd->data1)); + if (magic_cnt != q->magic_cnt) + return NULL; + + if (q->tail == q->ndesc - 1) + q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_WED_IND_CMD_CNT; + } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { + struct mt76_rro_rxdmad_c *dmad; + u16 magic_cnt; + + if (flush) + goto done; + + dmad = q->entry[idx].buf; + magic_cnt = FIELD_GET(RRO_RXDMAD_DATA3_MAGIC_CNT_MASK, + le32_to_cpu(dmad->data3)); + if (magic_cnt != q->magic_cnt) + return NULL; + + if (q->tail == q->ndesc - 1) + q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_MAGIC_CNT; + } else { if (flush) q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) return NULL; } - +done: q->tail = (q->tail + 1) % q->ndesc; q->queued--; - return mt76_dma_get_buf(dev, q, idx, len, info, more, drop); + return mt76_dma_get_buf(dev, q, idx, len, info, more, drop, flush); } static int @@ -557,6 +659,10 @@ mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, if (test_bit(MT76_RESET, &phy->state)) goto free_skb; + /* TODO: Take into account unlinear skbs */ + if (mt76_npu_device_active(dev) && skb_linearize(skb)) + goto free_skb; + t = mt76_get_txwi(dev); if (!t) goto free_skb; @@ -604,6 +710,9 @@ mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, if (ret < 0) goto unmap; + if (mt76_npu_device_active(dev)) + return mt76_npu_dma_add_buf(phy, q, skb, &tx_info.buf[1], txwi); + return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf, tx_info.info, tx_info.skb, t); @@ -650,7 +759,8 @@ mt76_dma_rx_fill_buf(struct mt76_dev *dev, struct mt76_queue *q, void *buf = NULL; int offset; - if (mt76_queue_is_wed_rro_ind(q)) + if (mt76_queue_is_wed_rro_ind(q) || + mt76_queue_is_wed_rro_rxdmad_c(q)) goto done; buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); @@ -680,9 +790,6 @@ int mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, { int frames; - if (!q->ndesc) - return 0; - spin_lock_bh(&q->lock); frames = mt76_dma_rx_fill_buf(dev, q, allow_direct); spin_unlock_bh(&q->lock); @@ -708,27 +815,23 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, q->ndesc = n_desc; q->buf_size = bufsize; q->hw_idx = idx; + q->dev = dev; + + if (mt76_queue_is_wed_rro_ind(q)) + size = sizeof(struct mt76_wed_rro_desc); + else if (mt76_queue_is_npu_tx(q)) + size = sizeof(struct airoha_npu_tx_dma_desc); + else if (mt76_queue_is_npu_rx(q)) + size = sizeof(struct airoha_npu_rx_dma_desc); + else + size = sizeof(struct mt76_desc); - size = mt76_queue_is_wed_rro_ind(q) ? sizeof(struct mt76_wed_rro_desc) - : sizeof(struct mt76_desc); q->desc = dmam_alloc_coherent(dev->dma_dev, q->ndesc * size, &q->desc_dma, GFP_KERNEL); if (!q->desc) return -ENOMEM; - if (mt76_queue_is_wed_rro_ind(q)) { - struct mt76_wed_rro_desc *rro_desc; - int i; - - rro_desc = (struct mt76_wed_rro_desc *)q->desc; - for (i = 0; i < q->ndesc; i++) { - struct mt76_wed_rro_ind *cmd; - - cmd = (struct mt76_wed_rro_ind *)&rro_desc[i]; - cmd->magic_cnt = MT_DMA_WED_IND_CMD_CNT - 1; - } - } - + mt76_dma_queue_magic_cnt_init(dev, q); size = q->ndesc * sizeof(*q->entry); q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL); if (!q->entry) @@ -738,6 +841,7 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, if (ret) return ret; + mt76_npu_queue_setup(dev, q); ret = mt76_wed_dma_setup(dev, q, false); if (ret) return ret; @@ -748,7 +852,10 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, return 0; } - mt76_dma_queue_reset(dev, q); + /* HW specific driver is supposed to reset brand-new EMI queues since + * it needs to set cpu index pointer. + */ + mt76_dma_queue_reset(dev, q, !mt76_queue_is_emi(q)); return 0; } @@ -762,6 +869,11 @@ mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q) if (!q->ndesc) return; + if (mt76_queue_is_npu(q)) { + mt76_npu_queue_cleanup(dev, q); + return; + } + do { spin_lock_bh(&q->lock); buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more, NULL); @@ -791,7 +903,8 @@ mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) if (!q->ndesc) return; - if (!mt76_queue_is_wed_rro_ind(q)) { + if (!mt76_queue_is_wed_rro_ind(q) && + !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) { int i; for (i = 0; i < q->ndesc; i++) @@ -811,7 +924,10 @@ mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) return; mt76_dma_sync_idx(dev, q); - mt76_dma_rx_fill_buf(dev, q, false); + if (mt76_queue_is_npu(q)) + mt76_npu_fill_rx_queue(dev, q); + else + mt76_dma_rx_fill(dev, q, false); } static void @@ -855,8 +971,9 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) bool allow_direct = !mt76_queue_is_wed_rx(q); bool more; - if (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) && - mt76_queue_is_wed_tx_free(q)) { + if ((q->flags & MT_QFLAG_WED_RRO_EN) || + (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) && + mt76_queue_is_wed_tx_free(q))) { dma_idx = Q_READ(q, dma_idx); check_ddone = true; } @@ -878,6 +995,20 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) if (!data) break; + if (PTR_ERR(data) == -EAGAIN) { + done++; + continue; + } + + if (mt76_queue_is_wed_rro_ind(q) && dev->drv->rx_rro_ind_process) + dev->drv->rx_rro_ind_process(dev, data); + + if (mt76_queue_is_wed_rro(q) && + !mt76_queue_is_wed_rro_rxdmad_c(q)) { + done++; + continue; + } + if (drop) goto free_frag; @@ -955,6 +1086,15 @@ int mt76_dma_rx_poll(struct napi_struct *napi, int budget) } EXPORT_SYMBOL_GPL(mt76_dma_rx_poll); +static void +mt76_dma_rx_queue_init(struct mt76_dev *dev, enum mt76_rxq_id qid, + int (*poll)(struct napi_struct *napi, int budget)) +{ + netif_napi_add(dev->napi_dev, &dev->napi[qid], poll); + mt76_dma_rx_fill_buf(dev, &dev->q_rx[qid], false); + napi_enable(&dev->napi[qid]); +} + static int mt76_dma_init(struct mt76_dev *dev, int (*poll)(struct napi_struct *napi, int budget)) @@ -987,9 +1127,10 @@ mt76_dma_init(struct mt76_dev *dev, init_completion(&dev->mmio.wed_reset_complete); mt76_for_each_q_rx(dev, i) { - netif_napi_add(dev->napi_dev, &dev->napi[i], poll); - mt76_dma_rx_fill_buf(dev, &dev->q_rx[i], false); - napi_enable(&dev->napi[i]); + if (mt76_queue_is_wed_rro(&dev->q_rx[i])) + continue; + + mt76_dma_rx_queue_init(dev, i, poll); } return 0; @@ -1002,6 +1143,7 @@ static const struct mt76_queue_ops mt76_dma_ops = { .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw, .tx_queue_skb = mt76_dma_tx_queue_skb, .tx_cleanup = mt76_dma_tx_cleanup, + .rx_queue_init = mt76_dma_rx_queue_init, .rx_cleanup = mt76_dma_rx_cleanup, .rx_reset = mt76_dma_rx_reset, .kick = mt76_dma_kick_queue, diff --git a/sys/contrib/dev/mediatek/mt76/dma.h b/sys/contrib/dev/mediatek/mt76/dma.h index e3ddc7a83757..4a63de6c5bf5 100644 --- a/sys/contrib/dev/mediatek/mt76/dma.h +++ b/sys/contrib/dev/mediatek/mt76/dma.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -31,7 +31,12 @@ #define MT_DMA_CTL_PN_CHK_FAIL BIT(13) #define MT_DMA_CTL_VER_MASK BIT(7) -#define MT_DMA_RRO_EN BIT(13) +#define MT_DMA_SDP0 GENMASK(15, 0) +#define MT_DMA_TOKEN_ID GENMASK(31, 16) +#define MT_DMA_MAGIC_MASK GENMASK(31, 28) +#define MT_DMA_RRO_EN BIT(13) + +#define MT_DMA_MAGIC_CNT 16 #define MT_DMA_WED_IND_CMD_CNT 8 #define MT_DMA_WED_IND_REASON GENMASK(15, 12) @@ -41,6 +46,73 @@ #define MT_FCE_INFO_LEN 4 #define MT_RX_RXWI_LEN 32 +#if IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) + +#define Q_READ(_q, _field) ({ \ + u32 _offset = offsetof(struct mt76_queue_regs, _field); \ + u32 _val; \ + if ((_q)->flags & MT_QFLAG_WED) \ + _val = mtk_wed_device_reg_read((_q)->wed, \ + ((_q)->wed_regs + \ + _offset)); \ + else \ + _val = readl(&(_q)->regs->_field); \ + _val; \ +}) + +#define Q_WRITE(_q, _field, _val) do { \ + u32 _offset = offsetof(struct mt76_queue_regs, _field); \ + if ((_q)->flags & MT_QFLAG_WED) \ + mtk_wed_device_reg_write((_q)->wed, \ + ((_q)->wed_regs + _offset), \ + _val); \ + else \ + writel(_val, &(_q)->regs->_field); \ +} while (0) + +#elif IS_ENABLED(CONFIG_MT76_NPU) + +#define Q_READ(_q, _field) ({ \ + u32 _offset = offsetof(struct mt76_queue_regs, _field); \ + u32 _val = 0; \ + if ((_q)->flags & MT_QFLAG_NPU) { \ + struct airoha_npu *npu; \ + \ + rcu_read_lock(); \ + npu = rcu_dereference(q->dev->mmio.npu); \ + if (npu) \ + regmap_read(npu->regmap, \ + ((_q)->wed_regs + _offset), &_val); \ + rcu_read_unlock(); \ + } else { \ + _val = readl(&(_q)->regs->_field); \ + } \ + _val; \ +}) + +#define Q_WRITE(_q, _field, _val) do { \ + u32 _offset = offsetof(struct mt76_queue_regs, _field); \ + if ((_q)->flags & MT_QFLAG_NPU) { \ + struct airoha_npu *npu; \ + \ + rcu_read_lock(); \ + npu = rcu_dereference(q->dev->mmio.npu); \ + if (npu) \ + regmap_write(npu->regmap, \ + ((_q)->wed_regs + _offset), _val); \ + rcu_read_unlock(); \ + } else { \ + writel(_val, &(_q)->regs->_field); \ + } \ +} while (0) + +#else + +#define Q_READ(_q, _field) readl(&(_q)->regs->_field) +#define Q_WRITE(_q, _field, _val) writel(_val, &(_q)->regs->_field) + +#endif + struct mt76_desc { __le32 buf0; __le32 ctrl; @@ -53,6 +125,21 @@ struct mt76_wed_rro_desc { __le32 buf1; } __packed __aligned(4); +/* data1 */ +#define RRO_RXDMAD_DATA1_LS_MASK BIT(30) +#define RRO_RXDMAD_DATA1_SDL0_MASK GENMASK(29, 16) +/* data2 */ +#define RRO_RXDMAD_DATA2_RX_TOKEN_ID_MASK GENMASK(31, 16) +#define RRO_RXDMAD_DATA2_IND_REASON_MASK GENMASK(15, 12) +/* data3 */ +#define RRO_RXDMAD_DATA3_MAGIC_CNT_MASK GENMASK(31, 28) +struct mt76_rro_rxdmad_c { + __le32 data0; + __le32 data1; + __le32 data2; + __le32 data3; +}; + enum mt76_qsel { MT_QSEL_MGMT, MT_QSEL_HCCA, @@ -81,14 +168,13 @@ void mt76_dma_attach(struct mt76_dev *dev); void mt76_dma_cleanup(struct mt76_dev *dev); int mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, bool allow_direct); -void __mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, - bool reset_idx); -void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q); +void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, + bool reset_idx); static inline void mt76_dma_reset_tx_queue(struct mt76_dev *dev, struct mt76_queue *q) { - dev->queue_ops->reset_q(dev, q); + dev->queue_ops->reset_q(dev, q, true); if (mtk_wed_device_active(&dev->mmio.wed)) mt76_wed_dma_setup(dev, q, true); } diff --git a/sys/contrib/dev/mediatek/mt76/eeprom.c b/sys/contrib/dev/mediatek/mt76/eeprom.c index f2eb2cd6e509..1d5b95226a16 100644 --- a/sys/contrib/dev/mediatek/mt76/eeprom.c +++ b/sys/contrib/dev/mediatek/mt76/eeprom.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -175,14 +175,17 @@ static int mt76_get_of_eeprom(struct mt76_dev *dev, void *eep, int len) #endif } -void +int mt76_eeprom_override(struct mt76_phy *phy) { struct mt76_dev *dev = phy->dev; #if defined(CONFIG_OF) struct device_node *np = dev->dev->of_node; + int err; - of_get_mac_address(np, phy->macaddr); + err = of_get_mac_address(np, phy->macaddr); + if (err == -EPROBE_DEFER) + return err; if (!is_valid_ether_addr(phy->macaddr)) { #endif @@ -193,6 +196,8 @@ mt76_eeprom_override(struct mt76_phy *phy) #if defined(CONFIG_OF) } #endif + + return 0; } EXPORT_SYMBOL_GPL(mt76_eeprom_override); @@ -269,6 +274,19 @@ mt76_get_of_array(struct device_node *np, char *name, size_t *len, int min) return prop->value; } + +static const s8 * +mt76_get_of_array_s8(struct device_node *np, char *name, size_t *len, int min) +{ + struct property *prop = of_find_property(np, name, NULL); + + if (!prop || !prop->value || prop->length < min) + return NULL; + + *len = prop->length; + + return prop->value; +} #endif struct device_node * @@ -313,7 +331,7 @@ mt76_get_txs_delta(struct device_node *np, u8 nss) } static void -mt76_apply_array_limit(s8 *pwr, size_t pwr_len, const __be32 *data, +mt76_apply_array_limit(s8 *pwr, size_t pwr_len, const s8 *data, s8 target_power, s8 nss_delta, s8 *max_power) { int i; @@ -322,30 +340,29 @@ mt76_apply_array_limit(s8 *pwr, size_t pwr_len, const __be32 *data, return; for (i = 0; i < pwr_len; i++) { - pwr[i] = min_t(s8, target_power, - be32_to_cpu(data[i]) + nss_delta); + pwr[i] = min_t(s8, target_power, data[i] + nss_delta); *max_power = max(*max_power, pwr[i]); } } static void mt76_apply_multi_array_limit(s8 *pwr, size_t pwr_len, s8 pwr_num, - const __be32 *data, size_t len, s8 target_power, - s8 nss_delta, s8 *max_power) + const s8 *data, size_t len, s8 target_power, + s8 nss_delta) { int i, cur; + s8 max_power = -128; if (!data) return; - len /= 4; - cur = be32_to_cpu(data[0]); + cur = data[0]; for (i = 0; i < pwr_num; i++) { if (len < pwr_len + 1) break; mt76_apply_array_limit(pwr + pwr_len * i, pwr_len, data + 1, - target_power, nss_delta, max_power); + target_power, nss_delta, &max_power); if (--cur > 0) continue; @@ -354,7 +371,7 @@ mt76_apply_multi_array_limit(s8 *pwr, size_t pwr_len, s8 pwr_num, if (!len) break; - cur = be32_to_cpu(data[0]); + cur = data[0]; } } #endif @@ -367,7 +384,7 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, struct mt76_dev *dev = phy->dev; #if defined(CONFIG_OF) struct device_node *np; - const __be32 *val; + const s8 *val; char name[16]; #endif u32 mcs_rates = dev->drv->mcs_rates; @@ -379,12 +396,17 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, s8 max_power = 0; #if defined(CONFIG_OF) s8 txs_delta; + s8 max_power_backoff = -127; + s8 txs_delta; + int n_chains = hweight16(phy->chainmask); + s8 target_power_combine = target_power + mt76_tx_power_path_delta(n_chains); #endif if (!mcs_rates) mcs_rates = 10; - memset(dest, target_power, sizeof(*dest)); + memset(dest, target_power, sizeof(*dest) - sizeof(dest->path)); + memset(&dest->path, 0, sizeof(dest->path)); if (!IS_ENABLED(CONFIG_OF)) return target_power; @@ -419,24 +441,47 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, txs_delta = mt76_get_txs_delta(np, hweight16(phy->chainmask)); - val = mt76_get_of_array(np, "rates-cck", &len, ARRAY_SIZE(dest->cck)); + val = mt76_get_of_array_s8(np, "rates-cck", &len, ARRAY_SIZE(dest->cck)); mt76_apply_array_limit(dest->cck, ARRAY_SIZE(dest->cck), val, target_power, txs_delta, &max_power); - val = mt76_get_of_array(np, "rates-ofdm", - &len, ARRAY_SIZE(dest->ofdm)); + val = mt76_get_of_array_s8(np, "rates-ofdm", + &len, ARRAY_SIZE(dest->ofdm)); mt76_apply_array_limit(dest->ofdm, ARRAY_SIZE(dest->ofdm), val, target_power, txs_delta, &max_power); - val = mt76_get_of_array(np, "rates-mcs", &len, mcs_rates + 1); + val = mt76_get_of_array_s8(np, "rates-mcs", &len, mcs_rates + 1); mt76_apply_multi_array_limit(dest->mcs[0], ARRAY_SIZE(dest->mcs[0]), ARRAY_SIZE(dest->mcs), val, len, - target_power, txs_delta, &max_power); + target_power, txs_delta); - val = mt76_get_of_array(np, "rates-ru", &len, ru_rates + 1); + val = mt76_get_of_array_s8(np, "rates-ru", &len, ru_rates + 1); mt76_apply_multi_array_limit(dest->ru[0], ARRAY_SIZE(dest->ru[0]), ARRAY_SIZE(dest->ru), val, len, - target_power, txs_delta, &max_power); + target_power, txs_delta); + + max_power_backoff = max_power; + val = mt76_get_of_array_s8(np, "paths-cck", &len, ARRAY_SIZE(dest->path.cck)); + mt76_apply_array_limit(dest->path.cck, ARRAY_SIZE(dest->path.cck), val, + target_power_combine, txs_delta, &max_power_backoff); + + val = mt76_get_of_array_s8(np, "paths-ofdm", &len, ARRAY_SIZE(dest->path.ofdm)); + mt76_apply_array_limit(dest->path.ofdm, ARRAY_SIZE(dest->path.ofdm), val, + target_power_combine, txs_delta, &max_power_backoff); + + val = mt76_get_of_array_s8(np, "paths-ofdm-bf", &len, ARRAY_SIZE(dest->path.ofdm_bf)); + mt76_apply_array_limit(dest->path.ofdm_bf, ARRAY_SIZE(dest->path.ofdm_bf), val, + target_power_combine, txs_delta, &max_power_backoff); + + val = mt76_get_of_array_s8(np, "paths-ru", &len, ARRAY_SIZE(dest->path.ru[0]) + 1); + mt76_apply_multi_array_limit(dest->path.ru[0], ARRAY_SIZE(dest->path.ru[0]), + ARRAY_SIZE(dest->path.ru), val, len, + target_power_combine, txs_delta); + + val = mt76_get_of_array_s8(np, "paths-ru-bf", &len, ARRAY_SIZE(dest->path.ru_bf[0]) + 1); + mt76_apply_multi_array_limit(dest->path.ru_bf[0], ARRAY_SIZE(dest->path.ru_bf[0]), + ARRAY_SIZE(dest->path.ru_bf), val, len, + target_power_combine, txs_delta); #endif return max_power; diff --git a/sys/contrib/dev/mediatek/mt76/mac80211.c b/sys/contrib/dev/mediatek/mt76/mac80211.c index d2ace8870451..bd7aaeb4398e 100644 --- a/sys/contrib/dev/mediatek/mt76/mac80211.c +++ b/sys/contrib/dev/mediatek/mt76/mac80211.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -652,6 +652,8 @@ int mt76_create_page_pool(struct mt76_dev *dev, struct mt76_queue *q) case MT_RXQ_MAIN: case MT_RXQ_BAND1: case MT_RXQ_BAND2: + case MT_RXQ_NPU0: + case MT_RXQ_NPU1: pp_params.pool_size = 256; break; default: @@ -846,6 +848,7 @@ void mt76_free_device(struct mt76_dev *dev) destroy_workqueue(dev->wq); dev->wq = NULL; } + mt76_npu_deinit(dev); ieee80211_free_hw(dev->hw); } EXPORT_SYMBOL_GPL(mt76_free_device); @@ -856,6 +859,9 @@ static void mt76_reset_phy(struct mt76_phy *phy) return; INIT_LIST_HEAD(&phy->tx_list); + phy->num_sta = 0; + phy->chanctx = NULL; + mt76_roc_complete(phy); } void mt76_reset_device(struct mt76_dev *dev) @@ -1267,6 +1273,8 @@ mt76_rx_convert(struct mt76_dev *dev, struct sk_buff *skb, mstat = *((struct mt76_rx_status *)skb->cb); memset(status, 0, sizeof(*status)); + skb->priority = mstat.qos_ctl & IEEE80211_QOS_CTL_TID_MASK; + status->flag = mstat.flag; status->freq = mstat.freq; status->enc_flags = mstat.enc_flags; @@ -1582,7 +1590,8 @@ void mt76_rx_poll_complete(struct mt76_dev *dev, enum mt76_rxq_id q, while ((skb = __skb_dequeue(&dev->rx_skb[q])) != NULL) { mt76_check_sta(dev, skb); - if (mtk_wed_device_active(&dev->mmio.wed)) + if (mtk_wed_device_active(&dev->mmio.wed) || + mt76_npu_device_active(dev)) __skb_queue_tail(&frames, skb); else mt76_rx_aggr_reorder(skb, &frames); @@ -2096,3 +2105,55 @@ void mt76_vif_cleanup(struct mt76_dev *dev, struct ieee80211_vif *vif) mt76_abort_roc(mvif->roc_phy); } EXPORT_SYMBOL_GPL(mt76_vif_cleanup); + +u16 mt76_select_links(struct ieee80211_vif *vif, int max_active_links) +{ + unsigned long usable_links = ieee80211_vif_usable_links(vif); + struct { + u8 link_id; + enum nl80211_band band; + } data[IEEE80211_MLD_MAX_NUM_LINKS]; + unsigned int link_id; + int i, n_data = 0; + u16 sel_links = 0; + + if (!ieee80211_vif_is_mld(vif)) + return 0; + + if (vif->active_links == usable_links) + return vif->active_links; + + rcu_read_lock(); + for_each_set_bit(link_id, &usable_links, IEEE80211_MLD_MAX_NUM_LINKS) { + struct ieee80211_bss_conf *link_conf; + + link_conf = rcu_dereference(vif->link_conf[link_id]); + if (WARN_ON_ONCE(!link_conf)) + continue; + + data[n_data].link_id = link_id; + data[n_data].band = link_conf->chanreq.oper.chan->band; + n_data++; + } + rcu_read_unlock(); + + for (i = 0; i < n_data; i++) { + int j; + + if (!(BIT(data[i].link_id) & vif->active_links)) + continue; + + sel_links = BIT(data[i].link_id); + for (j = 0; j < n_data; j++) { + if (data[i].band != data[j].band) { + sel_links |= BIT(data[j].link_id); + if (hweight16(sel_links) == max_active_links) + break; + } + } + break; + } + + return sel_links; +} +EXPORT_SYMBOL_GPL(mt76_select_links); diff --git a/sys/contrib/dev/mediatek/mt76/mcu.c b/sys/contrib/dev/mediatek/mt76/mcu.c index d554eed10986..acf532c0bfe9 100644 --- a/sys/contrib/dev/mediatek/mt76/mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2019 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mmio.c b/sys/contrib/dev/mediatek/mt76/mmio.c index 48943686003c..15fb8f436ab6 100644 --- a/sys/contrib/dev/mediatek/mt76/mmio.c +++ b/sys/contrib/dev/mediatek/mt76/mmio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -41,20 +41,30 @@ static u32 mt76_mmio_rmw(struct mt76_dev *dev, u32 offset, u32 mask, u32 val) static void mt76_mmio_write_copy(struct mt76_dev *dev, u32 offset, const void *data, int len) { + int i; + + for (i = 0; i < ALIGN(len, 4); i += 4) #if defined(__linux__) - __iowrite32_copy(dev->mmio.regs + offset, data, DIV_ROUND_UP(len, 4)); + writel(get_unaligned_le32(data + i), + dev->mmio.regs + offset + i); #elif defined(__FreeBSD__) - __iowrite32_copy((u8 *)dev->mmio.regs + offset, data, DIV_ROUND_UP(len, 4)); + writel(get_unaligned_le32((const u8 *)data + i), + (u8 *)dev->mmio.regs + offset + i); #endif } static void mt76_mmio_read_copy(struct mt76_dev *dev, u32 offset, void *data, int len) { + int i; + + for (i = 0; i < ALIGN(len, 4); i += 4) #if defined(__linux__) - __ioread32_copy(data, dev->mmio.regs + offset, DIV_ROUND_UP(len, 4)); + put_unaligned_le32(readl(dev->mmio.regs + offset + i), + data + i); #elif defined(__FreeBSD__) - __ioread32_copy(data, (u8 *)dev->mmio.regs + offset, DIV_ROUND_UP(len, 4)); + put_unaligned_le32(readl((u8 *)dev->mmio.regs + offset + i), + (u8 *)data + i); #endif } diff --git a/sys/contrib/dev/mediatek/mt76/mt76.h b/sys/contrib/dev/mediatek/mt76/mt76.h index 0b7686e6c36e..840725b8b020 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76.h +++ b/sys/contrib/dev/mediatek/mt76/mt76.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -13,6 +13,7 @@ #include <linux/leds.h> #include <linux/usb.h> #include <linux/average.h> +#include <linux/soc/airoha/airoha_offload.h> #include <linux/soc/mediatek/mtk_wed.h> #if defined(__FreeBSD__) #include <linux/wait.h> @@ -40,6 +41,8 @@ #define MT_QFLAG_WED BIT(5) #define MT_QFLAG_WED_RRO BIT(6) #define MT_QFLAG_WED_RRO_EN BIT(7) +#define MT_QFLAG_EMI_EN BIT(8) +#define MT_QFLAG_NPU BIT(9) #define __MT_WED_Q(_type, _n) (MT_QFLAG_WED | \ FIELD_PREP(MT_QFLAG_WED_TYPE, _type) | \ @@ -52,6 +55,13 @@ #define MT_WED_RRO_Q_DATA(_n) __MT_WED_RRO_Q(MT76_WED_RRO_Q_DATA, _n) #define MT_WED_RRO_Q_MSDU_PG(_n) __MT_WED_RRO_Q(MT76_WED_RRO_Q_MSDU_PG, _n) #define MT_WED_RRO_Q_IND __MT_WED_RRO_Q(MT76_WED_RRO_Q_IND, 0) +#define MT_WED_RRO_Q_RXDMAD_C __MT_WED_RRO_Q(MT76_WED_RRO_Q_RXDMAD_C, 0) + +#define __MT_NPU_Q(_type, _n) (MT_QFLAG_NPU | \ + FIELD_PREP(MT_QFLAG_WED_TYPE, _type) | \ + FIELD_PREP(MT_QFLAG_WED_RING, _n)) +#define MT_NPU_Q_TX(_n) __MT_NPU_Q(MT76_WED_Q_TX, _n) +#define MT_NPU_Q_RX(_n) __MT_NPU_Q(MT76_WED_Q_RX, _n) struct mt76_dev; struct mt76_phy; @@ -78,6 +88,13 @@ enum mt76_wed_type { MT76_WED_RRO_Q_DATA, MT76_WED_RRO_Q_MSDU_PG, MT76_WED_RRO_Q_IND, + MT76_WED_RRO_Q_RXDMAD_C, +}; + +enum mt76_hwrro_mode { + MT76_HWRRO_OFF, + MT76_HWRRO_V3, + MT76_HWRRO_V3_1, }; struct mt76_bus_ops { @@ -136,6 +153,9 @@ enum mt76_rxq_id { MT_RXQ_TXFREE_BAND1, MT_RXQ_TXFREE_BAND2, MT_RXQ_RRO_IND, + MT_RXQ_RRO_RXDMAD_C, + MT_RXQ_NPU0, + MT_RXQ_NPU1, __MT_RXQ_MAX }; @@ -239,8 +259,12 @@ struct mt76_queue { u8 buf_offset; u16 flags; + u8 magic_cnt; + + __le16 *emi_cpu_idx; struct mtk_wed_device *wed; + struct mt76_dev *dev; u32 wed_regs; dma_addr_t desc_dma; @@ -293,11 +317,15 @@ struct mt76_queue_ops { void (*tx_cleanup)(struct mt76_dev *dev, struct mt76_queue *q, bool flush); + void (*rx_queue_init)(struct mt76_dev *dev, enum mt76_rxq_id qid, + int (*poll)(struct napi_struct *napi, int budget)); + void (*rx_cleanup)(struct mt76_dev *dev, struct mt76_queue *q); void (*kick)(struct mt76_dev *dev, struct mt76_queue *q); - void (*reset_q)(struct mt76_dev *dev, struct mt76_queue *q); + void (*reset_q)(struct mt76_dev *dev, struct mt76_queue *q, + bool reset_idx); }; enum mt76_phy_type { @@ -405,15 +433,16 @@ struct mt76_txq { bool aggr; }; +/* data0 */ +#define RRO_IND_DATA0_IND_REASON_MASK GENMASK(31, 28) +#define RRO_IND_DATA0_START_SEQ_MASK GENMASK(27, 16) +#define RRO_IND_DATA0_SEQ_ID_MASK GENMASK(11, 0) +/* data1 */ +#define RRO_IND_DATA1_MAGIC_CNT_MASK GENMASK(31, 29) +#define RRO_IND_DATA1_IND_COUNT_MASK GENMASK(12, 0) struct mt76_wed_rro_ind { - u32 se_id : 12; - u32 rsv : 4; - u32 start_sn : 12; - u32 ind_reason : 4; - u32 ind_cnt : 13; - u32 win_sz : 3; - u32 rsv2 : 13; - u32 magic_cnt : 3; + __le32 data0; + __le32 data1; }; struct mt76_txwi_cache { @@ -424,6 +453,8 @@ struct mt76_txwi_cache { struct sk_buff *skb; void *ptr; }; + + u8 qid; }; struct mt76_rx_tid { @@ -540,6 +571,10 @@ struct mt76_driver_ops { void (*rx_poll_complete)(struct mt76_dev *dev, enum mt76_rxq_id q); + void (*rx_rro_ind_process)(struct mt76_dev *dev, void *data); + int (*rx_rro_add_msdu_page)(struct mt76_dev *dev, struct mt76_queue *q, + dma_addr_t p, void *data); + void (*sta_ps)(struct mt76_dev *dev, struct ieee80211_sta *sta, bool ps); @@ -689,6 +724,11 @@ struct mt76_mmio { struct mtk_wed_device wed_hif2; struct completion wed_reset; struct completion wed_reset_complete; + + struct airoha_ppe_dev __rcu *ppe_dev; + struct airoha_npu __rcu *npu; + phys_addr_t phy_addr; + int npu_type; }; struct mt76_rx_status { @@ -917,6 +957,7 @@ struct mt76_dev { struct mt76_queue q_rx[__MT_RXQ_MAX]; const struct mt76_queue_ops *queue_ops; int tx_dma_idx[4]; + enum mt76_hwrro_mode hwrro_mode; struct mt76_worker tx_worker; struct napi_struct tx_napi; @@ -925,6 +966,7 @@ struct mt76_dev { struct idr token; u16 wed_token_count; u16 token_count; + u16 token_start; u16 token_size; spinlock_t rx_token_lock; @@ -1095,6 +1137,14 @@ struct mt76_power_limits { s8 mcs[4][10]; s8 ru[7][12]; s8 eht[16][16]; + + struct { + s8 cck[4]; + s8 ofdm[4]; + s8 ofdm_bf[4]; + s8 ru[7][10]; + s8 ru_bf[7][10]; + } path; }; struct mt76_ethtool_worker_info { @@ -1221,6 +1271,7 @@ static inline int mt76_wed_dma_setup(struct mt76_dev *dev, struct mt76_queue *q, #define mt76_tx_queue_skb(dev, ...) (dev)->mt76.queue_ops->tx_queue_skb(&((dev)->mphy), __VA_ARGS__) #define mt76_queue_rx_reset(dev, ...) (dev)->mt76.queue_ops->rx_reset(&((dev)->mt76), __VA_ARGS__) #define mt76_queue_tx_cleanup(dev, ...) (dev)->mt76.queue_ops->tx_cleanup(&((dev)->mt76), __VA_ARGS__) +#define mt76_queue_rx_init(dev, ...) (dev)->mt76.queue_ops->rx_queue_init(&((dev)->mt76), __VA_ARGS__) #define mt76_queue_rx_cleanup(dev, ...) (dev)->mt76.queue_ops->rx_cleanup(&((dev)->mt76), __VA_ARGS__) #define mt76_queue_kick(dev, ...) (dev)->mt76.queue_ops->kick(&((dev)->mt76), __VA_ARGS__) #define mt76_queue_reset(dev, ...) (dev)->mt76.queue_ops->reset_q(&((dev)->mt76), __VA_ARGS__) @@ -1233,6 +1284,15 @@ static inline int mt76_wed_dma_setup(struct mt76_dev *dev, struct mt76_queue *q, #define mt76_dereference(p, dev) \ rcu_dereference_protected(p, lockdep_is_held(&(dev)->mutex)) +static inline struct mt76_dev *mt76_wed_to_dev(struct mtk_wed_device *wed) +{ +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + if (wed->wlan.hif2) + return container_of(wed, struct mt76_dev, mmio.wed_hif2); +#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ + return container_of(wed, struct mt76_dev, mmio.wed); +} + static inline struct mt76_wcid * __mt76_wcid_ptr(struct mt76_dev *dev, u16 idx) { @@ -1275,7 +1335,7 @@ void mt76_seq_puts_array(struct seq_file *file, const char *str, s8 *val, int len); int mt76_eeprom_init(struct mt76_dev *dev, int len); -void mt76_eeprom_override(struct mt76_phy *phy); +int mt76_eeprom_override(struct mt76_phy *phy); int mt76_get_of_data_from_mtd(struct mt76_dev *dev, void *eep, int offset, int len); int mt76_get_of_data_from_nvmem(struct mt76_dev *dev, void *eep, const char *cell_name, int len); @@ -1579,6 +1639,109 @@ int mt76_testmode_dump(struct ieee80211_hw *hw, struct sk_buff *skb, int mt76_testmode_set_state(struct mt76_phy *phy, enum mt76_testmode_state state); int mt76_testmode_alloc_skb(struct mt76_phy *phy, u32 len); +#ifdef CONFIG_MT76_NPU +void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb, + u32 info); +int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q, + struct sk_buff *skb, struct mt76_queue_buf *buf, + void *txwi_ptr); +int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q); +int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q); +void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q); +void mt76_npu_disable_irqs(struct mt76_dev *dev); +int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type); +void mt76_npu_deinit(struct mt76_dev *dev); +void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q); +void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index); +int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct net_device *dev, enum tc_setup_type type, + void *type_data); +#else +static inline void mt76_npu_check_ppe(struct mt76_dev *dev, + struct sk_buff *skb, u32 info) +{ +} + +static inline int mt76_npu_dma_add_buf(struct mt76_phy *phy, + struct mt76_queue *q, + struct sk_buff *skb, + struct mt76_queue_buf *buf, + void *txwi_ptr) +{ + return -EOPNOTSUPP; +} + +static inline int mt76_npu_fill_rx_queue(struct mt76_dev *dev, + struct mt76_queue *q) +{ + return 0; +} + +static inline void mt76_npu_queue_cleanup(struct mt76_dev *dev, + struct mt76_queue *q) +{ +} + +static inline void mt76_npu_disable_irqs(struct mt76_dev *dev) +{ +} + +static inline int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, + int type) +{ + return 0; +} + +static inline void mt76_npu_deinit(struct mt76_dev *dev) +{ +} + +static inline void mt76_npu_queue_setup(struct mt76_dev *dev, + struct mt76_queue *q) +{ +} + +static inline void mt76_npu_txdesc_cleanup(struct mt76_queue *q, + int index) +{ +} + +static inline int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct net_device *dev, + enum tc_setup_type type, + void *type_data) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_MT76_NPU */ + +static inline bool mt76_npu_device_active(struct mt76_dev *dev) +{ + return !!rcu_access_pointer(dev->mmio.npu); +} + +static inline bool mt76_ppe_device_active(struct mt76_dev *dev) +{ + return !!rcu_access_pointer(dev->mmio.ppe_dev); +} + +static inline int mt76_npu_send_msg(struct airoha_npu *npu, int ifindex, + enum airoha_npu_wlan_set_cmd cmd, + u32 val, gfp_t gfp) +{ + return airoha_npu_wlan_send_msg(npu, ifindex, cmd, &val, sizeof(val), + gfp); +} + +static inline int mt76_npu_get_msg(struct airoha_npu *npu, int ifindex, + enum airoha_npu_wlan_get_cmd cmd, + u32 *val, gfp_t gfp) +{ + return airoha_npu_wlan_get_msg(npu, ifindex, cmd, val, sizeof(*val), + gfp); +} + static inline void mt76_testmode_reset(struct mt76_phy *phy, bool disable) { #ifdef CONFIG_NL80211_TESTMODE @@ -1624,6 +1787,7 @@ int mt76_set_channel(struct mt76_phy *phy, struct cfg80211_chan_def *chandef, void mt76_scan_work(struct work_struct *work); void mt76_abort_scan(struct mt76_dev *dev); void mt76_roc_complete_work(struct work_struct *work); +void mt76_roc_complete(struct mt76_phy *phy); void mt76_abort_roc(struct mt76_phy *phy); struct mt76_vif_link *mt76_get_vif_phy_link(struct mt76_phy *phy, struct ieee80211_vif *vif); @@ -1797,21 +1961,51 @@ static inline bool mt76_queue_is_wed_rro_ind(struct mt76_queue *q) FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_RRO_Q_IND; } +static inline bool mt76_queue_is_wed_rro_rxdmad_c(struct mt76_queue *q) +{ + return mt76_queue_is_wed_rro(q) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_RRO_Q_RXDMAD_C; +} + static inline bool mt76_queue_is_wed_rro_data(struct mt76_queue *q) { return mt76_queue_is_wed_rro(q) && - (FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_RRO_Q_DATA || - FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_RRO_Q_MSDU_PG); + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_RRO_Q_DATA; +} + +static inline bool mt76_queue_is_wed_rro_msdu_pg(struct mt76_queue *q) +{ + return mt76_queue_is_wed_rro(q) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == + MT76_WED_RRO_Q_MSDU_PG; } static inline bool mt76_queue_is_wed_rx(struct mt76_queue *q) { - if (!(q->flags & MT_QFLAG_WED)) - return false; + return (q->flags & MT_QFLAG_WED) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX; +} + +static inline bool mt76_queue_is_emi(struct mt76_queue *q) +{ + return q->flags & MT_QFLAG_EMI_EN; +} + +static inline bool mt76_queue_is_npu(struct mt76_queue *q) +{ + return q->flags & MT_QFLAG_NPU; +} - return FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX || - mt76_queue_is_wed_rro_ind(q) || mt76_queue_is_wed_rro_data(q); +static inline bool mt76_queue_is_npu_tx(struct mt76_queue *q) +{ + return mt76_queue_is_npu(q) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_TX; +} +static inline bool mt76_queue_is_npu_rx(struct mt76_queue *q) +{ + return mt76_queue_is_npu(q) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX; } struct mt76_txwi_cache * @@ -1835,7 +2029,8 @@ mt76_get_page_pool_buf(struct mt76_queue *q, u32 *offset, u32 size) { struct page *page; - page = page_pool_dev_alloc_frag(q->page_pool, offset, size); + page = page_pool_alloc_frag(q->page_pool, offset, size, + GFP_ATOMIC | __GFP_NOWARN | GFP_DMA32); if (!page) return NULL; @@ -1891,6 +2086,7 @@ mt76_vif_init(struct ieee80211_vif *vif, struct mt76_vif_data *mvif) } void mt76_vif_cleanup(struct mt76_dev *dev, struct ieee80211_vif *vif); +u16 mt76_select_links(struct ieee80211_vif *vif, int max_active_links); static inline struct mt76_vif_link * mt76_vif_link(struct mt76_dev *dev, struct ieee80211_vif *vif, int link_id) diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/Kconfig b/sys/contrib/dev/mediatek/mt76/mt7603/Kconfig new file mode 100644 index 000000000000..ae40a596e49c --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/mt7603/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: BSD-3-Clause-Clear +config MT7603E + tristate "MediaTek MT7603E (PCIe) and MT76x8 WLAN support" + select MT76_CORE + depends on MAC80211 + depends on PCI + help + This adds support for MT7603E PCIe wireless devices and the WLAN core + on MT7628/MT7688 SoC devices. This family supports IEEE 802.11n 2x2 + to 300Mbps PHY rate + + To compile this driver as a module, choose M here. diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/Makefile b/sys/contrib/dev/mediatek/mt76/mt7603/Makefile new file mode 100644 index 000000000000..e954165ee133 --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/mt7603/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause-Clear +obj-$(CONFIG_MT7603E) += mt7603e.o + +mt7603e-y := \ + pci.o soc.o main.o init.o mcu.o \ + core.o dma.o mac.o eeprom.o \ + beacon.o debugfs.o diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/beacon.c b/sys/contrib/dev/mediatek/mt76/mt7603/beacon.c index 6457ee06bb5a..300a7f9c2ef1 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/beacon.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/beacon.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include "mt7603.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/core.c b/sys/contrib/dev/mediatek/mt76/mt7603/core.c index 915b8349146a..9c2943fd904e 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/core.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/core.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include "mt7603.h" #include "../trace.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/debugfs.c b/sys/contrib/dev/mediatek/mt76/mt7603/debugfs.c index 3967f2f05774..c891ad5498e6 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include "mt7603.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/dma.c b/sys/contrib/dev/mediatek/mt76/mt7603/dma.c index e26cc78fff94..3a16851524a0 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/dma.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/dma.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include "mt7603.h" #include "mac.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/eeprom.c b/sys/contrib/dev/mediatek/mt76/mt7603/eeprom.c index f5a6b03bc61d..b89db2db6573 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/eeprom.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/eeprom.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include <linux/of.h> #include "mt7603.h" @@ -182,7 +182,6 @@ int mt7603_eeprom_init(struct mt7603_dev *dev) dev->mphy.antenna_mask = 1; dev->mphy.chainmask = dev->mphy.antenna_mask; - mt76_eeprom_override(&dev->mphy); - return 0; + return mt76_eeprom_override(&dev->mphy); } diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/eeprom.h b/sys/contrib/dev/mediatek/mt76/mt7603/eeprom.h index 4687d6dc00dc..b6b746d1e56f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/eeprom.h +++ b/sys/contrib/dev/mediatek/mt76/mt7603/eeprom.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ #ifndef __MT7603_EEPROM_H #define __MT7603_EEPROM_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/init.c b/sys/contrib/dev/mediatek/mt76/mt7603/init.c index 86617a3e4328..10f2ec70c792 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/init.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include <linux/etherdevice.h> #include "mt7603.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/mac.c b/sys/contrib/dev/mediatek/mt76/mt7603/mac.c index 6387f9e61060..d3110eeb45d7 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include <linux/etherdevice.h> #include <linux/timekeeping.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/mac.h b/sys/contrib/dev/mediatek/mt76/mt7603/mac.h index 17e34ecf2bfb..9f5fab51ff83 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt7603/mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ #ifndef __MT7603_MAC_H #define __MT7603_MAC_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/main.c b/sys/contrib/dev/mediatek/mt76/mt7603/main.c index 0d7c84941cd0..0f3a7508996c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/main.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/main.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include <linux/etherdevice.h> #include <linux/platform_device.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7603/mcu.c index 301668c3cc92..e432cce97640 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include <linux/firmware.h> #include "mt7603.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7603/mcu.h index 30df8a3fd11a..7debe76cd092 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt7603/mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ #ifndef __MT7603_MCU_H #define __MT7603_MCU_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/mt7603.h b/sys/contrib/dev/mediatek/mt76/mt7603/mt7603.h index 55a034ccbacd..071bfab3af7c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/mt7603.h +++ b/sys/contrib/dev/mediatek/mt76/mt7603/mt7603.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ #ifndef __MT7603_H #define __MT7603_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/pci.c b/sys/contrib/dev/mediatek/mt76/mt7603/pci.c index 3d94cdb4314a..5fee610597a4 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/pci.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include <linux/kernel.h> #include <linux/module.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/regs.h b/sys/contrib/dev/mediatek/mt76/mt7603/regs.h index 524bceb8e958..97942f5ebdb4 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/regs.h +++ b/sys/contrib/dev/mediatek/mt76/mt7603/regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ #ifndef __MT7603_REGS_H #define __MT7603_REGS_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/soc.c b/sys/contrib/dev/mediatek/mt76/mt7603/soc.c index 08590aa68356..b74256efba55 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7603/soc.c +++ b/sys/contrib/dev/mediatek/mt76/mt7603/soc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include <linux/kernel.h> #include <linux/module.h> @@ -48,7 +48,7 @@ mt76_wmac_probe(struct platform_device *pdev) return 0; error: - ieee80211_free_hw(mt76_hw(dev)); + mt76_free_device(mdev); return ret; } diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/Kconfig b/sys/contrib/dev/mediatek/mt76/mt7615/Kconfig new file mode 100644 index 000000000000..8cc0c1b5c24e --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/mt7615/Kconfig @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: BSD-3-Clause-Clear + +config MT7615_COMMON + tristate + select WANT_DEV_COREDUMP + select MT76_CONNAC_LIB + +config MT7615E + tristate "MediaTek MT7615E and MT7663E (PCIe) support" + select MT7615_COMMON + depends on MAC80211 + depends on PCI + help + This adds support for MT7615-based PCIe wireless devices, + which support concurrent dual-band operation at both 5GHz + and 2.4GHz, IEEE 802.11ac 4x4:4SS 1733Mbps PHY rate, wave2 + MU-MIMO up to 4 users/group and 160MHz channels. + + To compile this driver as a module, choose M here. + +config MT7622_WMAC + bool "MT7622 (SoC) WMAC support" + depends on MT7615E + depends on ARCH_MEDIATEK || COMPILE_TEST + select REGMAP + default y + help + This adds support for the built-in WMAC on MT7622 SoC devices + which has the same feature set as a MT7615, but limited to + 2.4 GHz only. + +config MT7663_USB_SDIO_COMMON + tristate + select MT7615_COMMON + +config MT7663U + tristate "MediaTek MT7663U (USB) support" + select MT76_USB + select MT7663_USB_SDIO_COMMON + depends on MAC80211 + depends on USB + help + This adds support for MT7663U 802.11ac 2x2:2 wireless devices. + + To compile this driver as a module, choose M here. + +config MT7663S + tristate "MediaTek MT7663S (SDIO) support" + select MT76_SDIO + select MT7663_USB_SDIO_COMMON + depends on MAC80211 + depends on MMC + help + This adds support for MT7663S 802.11ac 2x2:2 wireless devices. + + To compile this driver as a module, choose M here. diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/Makefile b/sys/contrib/dev/mediatek/mt76/mt7615/Makefile index 2b97b9dde477..4def3b13eae1 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/Makefile +++ b/sys/contrib/dev/mediatek/mt76/mt7615/Makefile @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear obj-$(CONFIG_MT7615_COMMON) += mt7615-common.o obj-$(CONFIG_MT7615E) += mt7615e.o diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/debugfs.c b/sys/contrib/dev/mediatek/mt76/mt7615/debugfs.c index 2a6d317db5e0..0f7b20152279 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include "mt7615.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/dma.c b/sys/contrib/dev/mediatek/mt76/mt7615/dma.c index bcf7864312d7..59d2b3e8696b 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/dma.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/dma.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Ryder Lee <ryder.lee@mediatek.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/eeprom.c b/sys/contrib/dev/mediatek/mt76/mt7615/eeprom.c index d5ec498aa9ef..740c4acba124 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/eeprom.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/eeprom.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Ryder Lee <ryder.lee@mediatek.com> @@ -366,8 +366,6 @@ int mt7615_eeprom_init(struct mt7615_dev *dev, u32 addr) #endif ETH_ALEN); - mt76_eeprom_override(&dev->mphy); - - return 0; + return mt76_eeprom_override(&dev->mphy); } EXPORT_SYMBOL_GPL(mt7615_eeprom_init); diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/eeprom.h b/sys/contrib/dev/mediatek/mt76/mt7615/eeprom.h index a67fbb90f5b3..6aed52e14181 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/eeprom.h +++ b/sys/contrib/dev/mediatek/mt76/mt7615/eeprom.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2019 MediaTek Inc. */ #ifndef __MT7615_EEPROM_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/init.c b/sys/contrib/dev/mediatek/mt76/mt7615/init.c index 06d5a3f2fa67..f3f07401e2b7 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/init.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Roy Luo <royluo@google.com> @@ -576,7 +576,10 @@ int mt7615_register_ext_phy(struct mt7615_dev *dev) ETH_ALEN); mphy->macaddr[0] |= 2; mphy->macaddr[0] ^= BIT(7); - mt76_eeprom_override(mphy); + + ret = mt76_eeprom_override(mphy); + if (ret) + return ret; /* second phy can only handle 5 GHz */ mphy->cap.has_5ghz = true; diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mac.c b/sys/contrib/dev/mediatek/mt76/mt7615/mac.c index 10bf7e5b3acb..a4a252dc0186 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Ryder Lee <ryder.lee@mediatek.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mac.h b/sys/contrib/dev/mediatek/mt76/mt7615/mac.h index d08fbe64c262..336ebce5db5f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt7615/mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2019 MediaTek Inc. */ #ifndef __MT7615_MAC_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/main.c b/sys/contrib/dev/mediatek/mt76/mt7615/main.c index 15fe155ac3f3..727266892c3d 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/main.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/main.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Roy Luo <royluo@google.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7615/mcu.c index ec2f759d407f..7f48b502effe 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Roy Luo <royluo@google.com> @@ -878,8 +878,10 @@ mt7615_mcu_wtbl_sta_add(struct mt7615_phy *phy, struct ieee80211_vif *vif, wtbl_hdr = mt76_connac_mcu_alloc_wtbl_req(&dev->mt76, &msta->wcid, WTBL_RESET_AND_SET, NULL, &wskb); - if (IS_ERR(wtbl_hdr)) + if (IS_ERR(wtbl_hdr)) { + dev_kfree_skb(sskb); return PTR_ERR(wtbl_hdr); + } if (enable) { mt76_connac_mcu_wtbl_generic_tlv(&dev->mt76, wskb, vif, sta, diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7615/mcu.h index 8e9604be0792..851b0e4839b5 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt7615/mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2019 MediaTek Inc. */ #ifndef __MT7615_MCU_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mmio.c b/sys/contrib/dev/mediatek/mt76/mt7615/mmio.c index dbb2c82407df..da7edd48ce2c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/mmio.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/mmio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/kernel.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mt7615.h b/sys/contrib/dev/mediatek/mt76/mt7615/mt7615.h index 9bdd29e8d25e..e16865dd8e52 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/mt7615.h +++ b/sys/contrib/dev/mediatek/mt76/mt7615/mt7615.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2019 MediaTek Inc. */ #ifndef __MT7615_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mt7615_trace.h b/sys/contrib/dev/mediatek/mt76/mt7615/mt7615_trace.h index 9be5a58a4e6d..697fc5d225de 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/mt7615_trace.h +++ b/sys/contrib/dev/mediatek/mt76/mt7615/mt7615_trace.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2019 Lorenzo Bianconi <lorenzo@kernel.org> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/pci.c b/sys/contrib/dev/mediatek/mt76/mt7615/pci.c index 053faacd715a..fe181d269419 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/pci.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Ryder Lee <ryder.lee@mediatek.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/pci_init.c b/sys/contrib/dev/mediatek/mt76/mt7615/pci_init.c index ec1ae85f8de1..f608d560027a 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/pci_init.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/pci_init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Roy Luo <royluo@google.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/pci_mac.c b/sys/contrib/dev/mediatek/mt76/mt7615/pci_mac.c index fe8a3d852dbf..3d471c6f8a98 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/pci_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/pci_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. * * Author: Ryder Lee <ryder.lee@mediatek.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/regs.h b/sys/contrib/dev/mediatek/mt76/mt7615/regs.h index 806b3887c541..eb3c24d51987 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/regs.h +++ b/sys/contrib/dev/mediatek/mt76/mt7615/regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2019 MediaTek Inc. */ #ifndef __MT7615_REGS_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/sdio.c b/sys/contrib/dev/mediatek/mt76/mt7615/sdio.c index f56038cd4d3a..46188951ad19 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/sdio.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/sdio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. * * Author: Felix Fietkau <nbd@nbd.name> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/soc.c b/sys/contrib/dev/mediatek/mt76/mt7615/soc.c index 06a0f2a141e8..4bd189dd67e3 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/soc.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/soc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Ryder Lee <ryder.lee@mediatek.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/testmode.c b/sys/contrib/dev/mediatek/mt76/mt7615/testmode.c index 03f5af84424b..6eb97b7eba2d 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/testmode.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/testmode.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ #include "mt7615.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/trace.c b/sys/contrib/dev/mediatek/mt76/mt7615/trace.c index 6c02d5aff68f..7ec39e0b3fb2 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/trace.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/trace.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2019 Lorenzo Bianconi <lorenzo@kernel.org> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/usb.c b/sys/contrib/dev/mediatek/mt76/mt7615/usb.c index d96e06b4fee1..d91feffadda9 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/usb.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/usb.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2019 MediaTek Inc. * * Author: Felix Fietkau <nbd@nbd.name> diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/usb_sdio.c b/sys/contrib/dev/mediatek/mt76/mt7615/usb_sdio.c index 722418e9863c..c43f3ebb4e99 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7615/usb_sdio.c +++ b/sys/contrib/dev/mediatek/mt76/mt7615/usb_sdio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. * * Author: Lorenzo Bianconi <lorenzo@kernel.org> diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac.h b/sys/contrib/dev/mediatek/mt76/mt76_connac.h index 192dcc374a64..813d61bffc2c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76_connac.h +++ b/sys/contrib/dev/mediatek/mt76/mt76_connac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT76_CONNAC_H diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac2_mac.h b/sys/contrib/dev/mediatek/mt76/mt76_connac2_mac.h index eb4765365b8c..d4e2c3140441 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76_connac2_mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt76_connac2_mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2022 MediaTek Inc. */ #ifndef __MT76_CONNAC2_MAC_H diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.c b/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.c index 2d300948308d..651fcd4169f4 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include "mt76_connac.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.h b/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.h index 1013cad57a7f..247e2e7a47d8 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef __MT76_CONNAC3_MAC_H @@ -294,6 +294,13 @@ enum tx_frag_idx { #define MT_TXP_BUF_LEN GENMASK(11, 0) #define MT_TXP_DMA_ADDR_H GENMASK(15, 12) +#define MT_TXP0_TOKEN_ID0 GENMASK(14, 0) +#define MT_TXP0_TOKEN_ID0_VALID_MASK BIT(15) + +#define MT_TXP1_TID_ADDBA GENMASK(14, 12) +#define MT_TXP3_ML0_MASK BIT(15) +#define MT_TXP3_DMA_ADDR_H GENMASK(13, 12) + #define MT_TX_RATE_STBC BIT(14) #define MT_TX_RATE_NSS GENMASK(13, 10) #define MT_TX_RATE_MODE GENMASK(9, 6) diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac_mac.c b/sys/contrib/dev/mediatek/mt76/mt76_connac_mac.c index 0db00efe88b0..3304b5971be0 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76_connac_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt76_connac_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include "mt76_connac.h" @@ -297,16 +297,18 @@ u16 mt76_connac2_mac_tx_rate_val(struct mt76_phy *mphy, struct ieee80211_bss_conf *conf, bool beacon, bool mcast) { - struct mt76_vif_link *mvif = mt76_vif_conf_link(mphy->dev, conf->vif, conf); - struct cfg80211_chan_def *chandef = mvif->ctx ? - &mvif->ctx->def : &mphy->chandef; - u8 nss = 0, mode = 0, band = chandef->chan->band; - int rateidx = 0, mcast_rate; - int offset = 0; + u8 nss = 0, mode = 0, band = NL80211_BAND_2GHZ; + int rateidx = 0, offset = 0, mcast_rate; + struct cfg80211_chan_def *chandef; + struct mt76_vif_link *mvif; if (!conf) goto legacy; + mvif = mt76_vif_conf_link(mphy->dev, conf->vif, conf); + chandef = mvif->ctx ? &mvif->ctx->def : &mphy->chandef; + band = chandef->chan->band; + if (is_mt7921(mphy->dev)) { rateidx = ffs(conf->basic_rates) - 1; goto legacy; @@ -584,8 +586,9 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi, struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; bool multicast = ieee80211_is_data(hdr->frame_control) && is_multicast_ether_addr(hdr->addr1); - u16 rate = mt76_connac2_mac_tx_rate_val(mphy, &vif->bss_conf, beacon, - multicast); + u16 rate = mt76_connac2_mac_tx_rate_val(mphy, + vif ? &vif->bss_conf : NULL, + beacon, multicast); u32 val = MT_TXD6_FIXED_BW; /* hardware won't add HTC for mgmt/ctrl frame */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.c b/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.c index ca0d022668e9..2a9af9f01388 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/firmware.h> @@ -1662,6 +1662,31 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, return err; } + if (enable && vif->bss_conf.bssid_indicator) { + struct { + struct { + u8 bss_idx; + u8 pad[3]; + } __packed hdr; + struct bss_info_uni_mbssid mbssid; + } mbssid_req = { + .hdr = { + .bss_idx = mvif->idx, + }, + .mbssid = { + .tag = cpu_to_le16(UNI_BSS_INFO_11V_MBSSID), + .len = cpu_to_le16(sizeof(struct bss_info_uni_mbssid)), + .max_indicator = vif->bss_conf.bssid_indicator, + .mbss_idx = vif->bss_conf.bssid_index, + }, + }; + + err = mt76_mcu_send_msg(mdev, MCU_UNI_CMD(BSS_INFO_UPDATE), + &mbssid_req, sizeof(mbssid_req), true); + if (err < 0) + return err; + } + return mt76_connac_mcu_uni_set_chctx(phy, mvif, ctx); } EXPORT_SYMBOL_GPL(mt76_connac_mcu_uni_add_bss); @@ -1949,7 +1974,11 @@ int mt76_connac_mcu_chip_config(struct mt76_dev *dev) .resp_type = 0, }; - memcpy(req.data, "assert", 7); +#if defined(__linux__) + strscpy(req.data, "assert"); +#elif defined(__FreeBSD__) + strscpy(req.data, "assert", sizeof(req.data)); +#endif return mt76_mcu_send_msg(dev, MCU_CE_CMD(CHIP_CONFIG), &req, sizeof(req), false); @@ -2994,7 +3023,7 @@ int mt76_connac2_load_ram(struct mt76_dev *dev, const char *fw_wm, } hdr = (const void *)(fw->data + fw->size - sizeof(*hdr)); - dev_info(dev->dev, "WM Firmware Version: %.10s, Build Time: %.15s\n", + dev_info(dev->dev, "WM Firmware Version: %.10s, Build Time: %.15s", hdr->fw_ver, hdr->build_date); ret = mt76_connac_mcu_send_ram_firmware(dev, hdr, fw->data, false); @@ -3023,7 +3052,7 @@ int mt76_connac2_load_ram(struct mt76_dev *dev, const char *fw_wm, } hdr = (const void *)(fw->data + fw->size - sizeof(*hdr)); - dev_info(dev->dev, "WA Firmware Version: %.10s, Build Time: %.15s\n", + dev_info(dev->dev, "WA Firmware Version: %.10s, Build Time: %.15s", hdr->fw_ver, hdr->build_date); ret = mt76_connac_mcu_send_ram_firmware(dev, hdr, fw->data, true); @@ -3099,7 +3128,7 @@ int mt76_connac2_load_patch(struct mt76_dev *dev, const char *fw_name) } hdr = (const void *)fw->data; - dev_info(dev->dev, "HW/SW Version: 0x%x, Build Time: %.16s\n", + dev_info(dev->dev, "HW/SW Version: 0x%x, Build Time: %.16s", be32_to_cpu(hdr->hw_sw_ver), hdr->build_date); for (i = 0; i < be32_to_cpu(hdr->desc.n_region); i++) { diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.h b/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.h index 3ba81ab77e99..5f6c7455b5ce 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT76_CONNAC_MCU_H @@ -1066,6 +1066,7 @@ enum { MCU_UNI_EVENT_ROC = 0x27, MCU_UNI_EVENT_TX_DONE = 0x2d, MCU_UNI_EVENT_THERMAL = 0x35, + MCU_UNI_EVENT_RSSI_MONITOR = 0x41, MCU_UNI_EVENT_NIC_CAPAB = 0x43, MCU_UNI_EVENT_WED_RRO = 0x57, MCU_UNI_EVENT_PER_STA_INFO = 0x6d, @@ -1304,6 +1305,7 @@ enum { MCU_UNI_CMD_THERMAL = 0x35, MCU_UNI_CMD_VOW = 0x37, MCU_UNI_CMD_FIXED_RATE_TABLE = 0x40, + MCU_UNI_CMD_RSSI_MONITOR = 0x41, MCU_UNI_CMD_TESTMODE_CTRL = 0x46, MCU_UNI_CMD_RRO = 0x57, MCU_UNI_CMD_OFFCH_SCAN_CTRL = 0x58, diff --git a/sys/contrib/dev/mediatek/mt76/mt76x0/pci.c b/sys/contrib/dev/mediatek/mt76/mt76x0/pci.c index 11c16d1fc70f..f8d206a07f99 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x0/pci.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x0/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x0/pci_mcu.c b/sys/contrib/dev/mediatek/mt76/mt76x0/pci_mcu.c index f0962507f72f..efa549dc68ec 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x0/pci_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x0/pci_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x0/usb_mcu.c b/sys/contrib/dev/mediatek/mt76/mt76x0/usb_mcu.c index 6dc1f51f5658..20a8f3659490 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x0/usb_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x0/usb_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02.h b/sys/contrib/dev/mediatek/mt76/mt76x02.h index 8d06ef8c7c62..3c98808ccf26 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Stanislaw Gruszka <stf_xl@wp.pl> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_beacon.c b/sys/contrib/dev/mediatek/mt76/mt76x02_beacon.c index d570b99bccb9..7c9b16942275 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_beacon.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_beacon.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_debugfs.c b/sys/contrib/dev/mediatek/mt76/mt76x02_debugfs.c index 8ce4bf44733d..d81f696b32c7 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_dfs.c b/sys/contrib/dev/mediatek/mt76/mt76x02_dfs.c index 7a07636d09c6..21c99ad7ef59 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_dfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_dfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_dfs.h b/sys/contrib/dev/mediatek/mt76/mt76x02_dfs.h index 491010a32247..d40051992586 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_dfs.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_dfs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_dma.h b/sys/contrib/dev/mediatek/mt76/mt76x02_dma.h index 23b0e7d10d57..2f6ba8cf51e8 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_dma.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_dma.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_eeprom.c b/sys/contrib/dev/mediatek/mt76/mt76x02_eeprom.c index a5e3392c0b48..d16be0cb0dc7 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_eeprom.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_eeprom.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_eeprom.h b/sys/contrib/dev/mediatek/mt76/mt76x02_eeprom.h index 13fa70853b0d..3cbb2977f375 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_eeprom.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_eeprom.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_mac.c b/sys/contrib/dev/mediatek/mt76/mt76x02_mac.c index 83488b2d6efb..14ee5b3b94d3 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Stanislaw Gruszka <stf_xl@wp.pl> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_mac.h b/sys/contrib/dev/mediatek/mt76/mt76x02_mac.h index 5dc6c834111e..778454ac8974 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Stanislaw Gruszka <stf_xl@wp.pl> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_mcu.c b/sys/contrib/dev/mediatek/mt76/mt76x02_mcu.c index 75978820a260..e16f06a284ca 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_mcu.h b/sys/contrib/dev/mediatek/mt76/mt76x02_mcu.h index e187ed52968e..a422cdc520cb 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_mmio.c b/sys/contrib/dev/mediatek/mt76/mt76x02_mmio.c index a683d53c7ceb..dd71c1c95cc9 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_mmio.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_mmio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_phy.c b/sys/contrib/dev/mediatek/mt76/mt76x02_phy.c index cbe7e6f0c29a..557380c9bfab 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_phy.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_phy.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_phy.h b/sys/contrib/dev/mediatek/mt76/mt76x02_phy.h index 84d8a6138b3e..09e8edee2195 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_phy.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_phy.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_regs.h b/sys/contrib/dev/mediatek/mt76/mt76x02_regs.h index fe0c5e3298bc..e87d3f8a1de9 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_regs.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_trace.c b/sys/contrib/dev/mediatek/mt76/mt76x02_trace.c index a812c3a1e258..a92b2b7391ff 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_trace.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_trace.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_trace.h b/sys/contrib/dev/mediatek/mt76/mt76x02_trace.h index 11d119cd0f6f..56eea2f68983 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_trace.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_trace.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_txrx.c b/sys/contrib/dev/mediatek/mt76/mt76x02_txrx.c index d8bc4ae185f5..301b43180006 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_txrx.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_txrx.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_usb.h b/sys/contrib/dev/mediatek/mt76/mt76x02_usb.h index b5be884b3549..49ab05c1fe73 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_usb.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_usb.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_usb_core.c b/sys/contrib/dev/mediatek/mt76/mt76x02_usb_core.c index 4840d0b500b3..3a28a8cc1338 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_usb_core.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_usb_core.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_usb_mcu.c b/sys/contrib/dev/mediatek/mt76/mt76x02_usb_mcu.c index b2cc44914294..968c73e06a5f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_usb_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_usb_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_util.c b/sys/contrib/dev/mediatek/mt76/mt76x02_util.c index 7dfcb20c692c..e5d9d1bc9415 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x02_util.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x02_util.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Stanislaw Gruszka <stf_xl@wp.pl> * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/Kconfig b/sys/contrib/dev/mediatek/mt76/mt76x2/Kconfig new file mode 100644 index 000000000000..d820510cb4bb --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/Kconfig @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: BSD-3-Clause-Clear +config MT76x2_COMMON + tristate + select MT76x02_LIB + +config MT76x2E + tristate "MediaTek MT76x2E (PCIe) support" + select MT76x2_COMMON + depends on MAC80211 + depends on PCI + help + This adds support for MT7612/MT7602/MT7662-based PCIe wireless + devices, which comply with IEEE 802.11ac standards and support + 2SS to 866Mbit/s PHY rate. + + To compile this driver as a module, choose M here. + +config MT76x2U + tristate "MediaTek MT76x2U (USB) support" + select MT76x2_COMMON + select MT76x02_USB + depends on MAC80211 + depends on USB + help + This adds support for MT7612U-based USB 3.0 wireless dongles, + which comply with IEEE 802.11ac standards and support 2SS to + 866Mbit/s PHY rate. + + To compile this driver as a module, choose M here. diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/Makefile b/sys/contrib/dev/mediatek/mt76/mt76x2/Makefile new file mode 100644 index 000000000000..cbc90a9616a6 --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: BSD-3-Clause-Clear +obj-$(CONFIG_MT76x2_COMMON) += mt76x2-common.o +obj-$(CONFIG_MT76x2E) += mt76x2e.o +obj-$(CONFIG_MT76x2U) += mt76x2u.o + +mt76x2-common-y := \ + eeprom.o mac.o init.o phy.o mcu.o + +mt76x2e-y := \ + pci.o pci_main.o pci_init.o pci_mcu.o \ + pci_phy.o + +mt76x2u-y := \ + usb.o usb_init.o usb_main.o usb_mac.o usb_mcu.o \ + usb_phy.o diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/eeprom.c b/sys/contrib/dev/mediatek/mt76/mt76x2/eeprom.c index 156b16c17b2b..782813aadc0a 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/eeprom.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/eeprom.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -499,7 +499,9 @@ int mt76x2_eeprom_init(struct mt76x02_dev *dev) mt76x02_eeprom_parse_hw_cap(dev); mt76x2_eeprom_get_macaddr(dev); - mt76_eeprom_override(&dev->mphy); + ret = mt76_eeprom_override(&dev->mphy); + if (ret) + return ret; dev->mphy.macaddr[0] &= ~BIT(1); return 0; diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/eeprom.h b/sys/contrib/dev/mediatek/mt76/mt76x2/eeprom.h index 43430ef98b11..1ee8be389b24 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/eeprom.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/eeprom.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/init.c b/sys/contrib/dev/mediatek/mt76/mt76x2/init.c index 19c139290adb..408dc08b6457 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/init.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/mac.c b/sys/contrib/dev/mediatek/mt76/mt76x2/mac.c index e08740ca3d0c..2fa34ca69095 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/mac.h b/sys/contrib/dev/mediatek/mt76/mt76x2/mac.h index d5c3d26b94c1..f8ea70074c41 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/mcu.c b/sys/contrib/dev/mediatek/mt76/mt76x2/mcu.c index ac83ce5f3e8b..769d924220e3 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/mcu.h b/sys/contrib/dev/mediatek/mt76/mt76x2/mcu.h index 41fd66563e82..16a4386aa754 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/mt76x2.h b/sys/contrib/dev/mediatek/mt76/mt76x2/mt76x2.h index f051721bb00e..984756c81bdc 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/mt76x2.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/mt76x2.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/mt76x2u.h b/sys/contrib/dev/mediatek/mt76/mt76x2/mt76x2u.h index f9d37c6cf1f0..27e478ab5b15 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/mt76x2u.h +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/mt76x2u.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/pci.c b/sys/contrib/dev/mediatek/mt76/mt76x2/pci.c index 2303019670e2..491a32921a06 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/pci.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_init.c b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_init.c index e38e8e5685c2..bec84f932311 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_init.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_main.c b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_main.c index c5dfb06d81e8..550644676201 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_main.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_main.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_mcu.c b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_mcu.c index e5b6282d1a6c..daba163802b6 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_phy.c b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_phy.c index 8831337df23e..dcf4328c1cac 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_phy.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_phy.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/phy.c b/sys/contrib/dev/mediatek/mt76/mt76x2/phy.c index e2b4cf30dc44..a5efa13a892f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/phy.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/phy.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb.c index 96cecc576a98..01cb3b2830f3 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_init.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_init.c index 3b5562811511..41778a8ef02c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_init.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_mac.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_mac.c index eaa622833f85..d0cb511ac6a2 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_main.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_main.c index 6671c53faf9f..66b06a493d95 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_main.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_main.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_mcu.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_mcu.c index dd22d8af0901..9102be1803b7 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_phy.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_phy.c index a04a98f5ce1e..b63dd7f3ee80 100644 --- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_phy.c +++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_phy.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/Kconfig b/sys/contrib/dev/mediatek/mt76/mt7915/Kconfig index 193112c49bd1..c24be8227f11 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/Kconfig +++ b/sys/contrib/dev/mediatek/mt76/mt7915/Kconfig @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear config MT7915E tristate "MediaTek MT7915E (PCIe) support" select MT76_CONNAC_LIB diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/Makefile b/sys/contrib/dev/mediatek/mt76/mt7915/Makefile index e0ca638c91a5..963fb3109af3 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/Makefile +++ b/sys/contrib/dev/mediatek/mt76/mt7915/Makefile @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear obj-$(CONFIG_MT7915E) += mt7915e.o diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/coredump.c b/sys/contrib/dev/mediatek/mt76/mt7915/coredump.c index 8c9a69837c86..e3dcf4c5557d 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/coredump.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/coredump.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2022 MediaTek Inc. */ #if defined(__FreeBSD__) diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/coredump.h b/sys/contrib/dev/mediatek/mt76/mt7915/coredump.h index a7284b391daf..251b045b5aca 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/coredump.h +++ b/sys/contrib/dev/mediatek/mt76/mt7915/coredump.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2022 MediaTek Inc. */ #ifndef _COREDUMP_H_ diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/debugfs.c b/sys/contrib/dev/mediatek/mt76/mt7915/debugfs.c index b287b7d9394e..26ed3745af43 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/relay.h> @@ -1008,7 +1008,7 @@ mt7915_rate_txpower_get(struct file *file, char __user *user_buf, if (!buf) return -ENOMEM; - ret = mt7915_mcu_get_txpower_sku(phy, txpwr, sizeof(txpwr)); + ret = mt7915_mcu_get_txpower_sku(phy, txpwr, sizeof(txpwr), TX_POWER_INFO_RATE); if (ret) goto out; @@ -1118,7 +1118,7 @@ mt7915_rate_txpower_set(struct file *file, const char __user *user_buf, mutex_lock(&dev->mt76.mutex); ret = mt7915_mcu_get_txpower_sku(phy, req.txpower_sku, - sizeof(req.txpower_sku)); + sizeof(req.txpower_sku), TX_POWER_INFO_RATE); if (ret) goto out; @@ -1160,7 +1160,7 @@ out: return ret ? ret : count; } -static const struct file_operations mt7915_rate_txpower_fops = { +static const struct file_operations mt7915_txpower_fops = { .write = mt7915_rate_txpower_set, .read = mt7915_rate_txpower_get, .open = simple_open, @@ -1169,6 +1169,70 @@ static const struct file_operations mt7915_rate_txpower_fops = { }; static int +mt7915_path_txpower_show(struct seq_file *file) +{ + struct mt7915_phy *phy = file->private; + s8 txpower[MT7915_SKU_PATH_NUM], *buf = txpower; + int ret; + +#define PATH_POWER_SHOW(_name, _len, _skip) do { \ + size_t __len = (_len); \ + if (_skip) { \ + buf -= 1; \ + *buf = 0; \ + } \ + mt76_seq_puts_array(file, _name, buf, __len); \ + buf += __len; \ + } while (0) + + seq_printf(file, "\n%*c", 18, ' '); + seq_puts(file, "1T1S/2T1S/3T1S/4T1S/2T2S/3T2S/4T2S/3T3S/4T3S/4T4S\n"); + ret = mt7915_mcu_get_txpower_sku(phy, txpower, sizeof(txpower), + TX_POWER_INFO_PATH); + if (ret) + return ret; + + PATH_POWER_SHOW("CCK", 4, 0); + PATH_POWER_SHOW("OFDM", 4, 0); + PATH_POWER_SHOW("BF-OFDM", 4, 1); + + PATH_POWER_SHOW("HT/VHT20", 10, 0); + PATH_POWER_SHOW("BF-HT/VHT20", 10, 1); + PATH_POWER_SHOW("HT/VHT40", 10, 0); + PATH_POWER_SHOW("BF-HT/VHT40", 10, 1); + + PATH_POWER_SHOW("BW20/RU242", 10, 0); + PATH_POWER_SHOW("BF-BW20/RU242", 10, 1); + PATH_POWER_SHOW("BW40/RU484", 10, 0); + PATH_POWER_SHOW("BF-BW40/RU484", 10, 1); + PATH_POWER_SHOW("BW80/RU996", 10, 0); + PATH_POWER_SHOW("BF-BW80/RU996", 10, 1); + PATH_POWER_SHOW("BW160/RU2x996", 10, 0); + PATH_POWER_SHOW("BF-BW160/RU2x996", 10, 1); + PATH_POWER_SHOW("RU26", 10, 0); + PATH_POWER_SHOW("BF-RU26", 10, 0); + PATH_POWER_SHOW("RU52", 10, 0); + PATH_POWER_SHOW("BF-RU52", 10, 0); + PATH_POWER_SHOW("RU106", 10, 0); + PATH_POWER_SHOW("BF-RU106", 10, 0); +#undef PATH_POWER_SHOW + + return 0; +} + +static int +mt7915_txpower_path_show(struct seq_file *file, void *data) +{ + struct mt7915_phy *phy = file->private; + + seq_printf(file, "\nBand %d\n", phy != &phy->dev->phy); + + return mt7915_path_txpower_show(file); +} + +DEFINE_SHOW_ATTRIBUTE(mt7915_txpower_path); + +static int mt7915_twt_stats(struct seq_file *s, void *data) { struct mt7915_dev *dev = dev_get_drvdata(s->private); @@ -1254,7 +1318,9 @@ int mt7915_init_debugfs(struct mt7915_phy *phy) debugfs_create_file("implicit_txbf", 0600, dir, dev, &fops_implicit_txbf); debugfs_create_file("txpower_sku", 0400, dir, phy, - &mt7915_rate_txpower_fops); + &mt7915_txpower_fops); + debugfs_create_file("txpower_path", 0400, dir, phy, + &mt7915_txpower_path_fops); debugfs_create_devm_seqfile(dev->mt76.dev, "twt_stats", dir, mt7915_twt_stats); debugfs_create_file("rf_regval", 0600, dir, dev, &fops_rf_regval); diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/dma.c b/sys/contrib/dev/mediatek/mt76/mt7915/dma.c index 0c62272fe7d0..aabd37366e86 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/dma.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/dma.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include "mt7915.h" @@ -624,13 +624,13 @@ int mt7915_dma_reset(struct mt7915_dev *dev, bool force) } for (i = 0; i < __MT_MCUQ_MAX; i++) - mt76_queue_reset(dev, dev->mt76.q_mcu[i]); + mt76_queue_reset(dev, dev->mt76.q_mcu[i], true); mt76_for_each_q_rx(&dev->mt76, i) { if (mt76_queue_is_wed_tx_free(&dev->mt76.q_rx[i])) continue; - mt76_queue_reset(dev, &dev->mt76.q_rx[i]); + mt76_queue_reset(dev, &dev->mt76.q_rx[i], true); } mt76_tx_status_check(&dev->mt76, true); diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.c b/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.c index 9214286ca000..0d2358e400d8 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #if defined(__FreeBSD__) @@ -292,9 +292,7 @@ int mt7915_eeprom_init(struct mt7915_dev *dev) #endif ETH_ALEN); - mt76_eeprom_override(&dev->mphy); - - return 0; + return mt76_eeprom_override(&dev->mphy); } int mt7915_eeprom_get_target_power(struct mt7915_dev *dev, diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.h b/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.h index 31aec0f40232..1dc285c72991 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.h +++ b/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7915_EEPROM_H @@ -50,9 +50,9 @@ enum mt7915_eeprom_field { #define MT_EE_CAL_GROUP_SIZE_7975 (54 * MT_EE_CAL_UNIT + 16) #define MT_EE_CAL_GROUP_SIZE_7976 (94 * MT_EE_CAL_UNIT + 16) #define MT_EE_CAL_GROUP_SIZE_7916_6G (94 * MT_EE_CAL_UNIT + 16) +#define MT_EE_CAL_GROUP_SIZE_7981 (144 * MT_EE_CAL_UNIT + 16) #define MT_EE_CAL_DPD_SIZE_V1 (54 * MT_EE_CAL_UNIT) #define MT_EE_CAL_DPD_SIZE_V2 (300 * MT_EE_CAL_UNIT) -#define MT_EE_CAL_DPD_SIZE_V2_7981 (102 * MT_EE_CAL_UNIT) /* no 6g dpd data */ #define MT_EE_WIFI_CONF0_TX_PATH GENMASK(2, 0) #define MT_EE_WIFI_CONF0_RX_PATH GENMASK(5, 3) @@ -180,6 +180,8 @@ mt7915_get_cal_group_size(struct mt7915_dev *dev) val = FIELD_GET(MT_EE_WIFI_CONF0_BAND_SEL, val); return (val == MT_EE_V2_BAND_SEL_6GHZ) ? MT_EE_CAL_GROUP_SIZE_7916_6G : MT_EE_CAL_GROUP_SIZE_7916; + } else if (is_mt7981(&dev->mt76)) { + return MT_EE_CAL_GROUP_SIZE_7981; } else if (mt7915_check_adie(dev, false)) { return MT_EE_CAL_GROUP_SIZE_7976; } else { @@ -192,8 +194,6 @@ mt7915_get_cal_dpd_size(struct mt7915_dev *dev) { if (is_mt7915(&dev->mt76)) return MT_EE_CAL_DPD_SIZE_V1; - else if (is_mt7981(&dev->mt76)) - return MT_EE_CAL_DPD_SIZE_V2_7981; else return MT_EE_CAL_DPD_SIZE_V2; } diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/init.c b/sys/contrib/dev/mediatek/mt76/mt7915/init.c index 6c0f1cdc4987..4128942361ff 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/init.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/etherdevice.h> @@ -304,6 +304,8 @@ static void __mt7915_init_txpower(struct mt7915_phy *phy, int pwr_delta = mt7915_eeprom_get_power_delta(dev, sband->band); struct mt76_power_limits limits; + phy->sku_limit_en = true; + phy->sku_path_en = true; for (i = 0; i < sband->n_channels; i++) { struct ieee80211_channel *chan = &sband->channels[i]; u32 target_power = 0; @@ -320,6 +322,11 @@ static void __mt7915_init_txpower(struct mt7915_phy *phy, target_power = mt76_get_rate_power_limits(phy->mt76, chan, &limits, target_power); + + /* MT7915N can not enable Backoff table without setting value in dts */ + if (!limits.path.ofdm[0]) + phy->sku_path_en = false; + target_power += path_delta; target_power = DIV_ROUND_UP(target_power, 2); chan->max_power = min_t(int, chan->max_reg_power, @@ -729,7 +736,9 @@ mt7915_register_ext_phy(struct mt7915_dev *dev, struct mt7915_phy *phy) mphy->macaddr[0] |= 2; mphy->macaddr[0] ^= BIT(7); } - mt76_eeprom_override(mphy); + ret = mt76_eeprom_override(mphy); + if (ret) + return ret; /* init wiphy according to mphy and phy */ mt7915_init_wiphy(phy); diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mac.c b/sys/contrib/dev/mediatek/mt76/mt7915/mac.c index dcdbde3ee5b7..fb6e03e6f34f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/etherdevice.h> @@ -1485,6 +1485,8 @@ mt7915_mac_full_reset(struct mt7915_dev *dev) if (ext_phy) cancel_delayed_work_sync(&ext_phy->mac_work); + mt76_abort_scan(&dev->mt76); + mutex_lock(&dev->mt76.mutex); for (i = 0; i < 10; i++) { if (!mt7915_mac_restart(dev)) diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mac.h b/sys/contrib/dev/mediatek/mt76/mt7915/mac.h index 448b1b380190..e39f96e00ba4 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt7915/mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7915_MAC_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/main.c b/sys/contrib/dev/mediatek/mt76/mt7915/main.c index fe0639c14bf9..90d5e79fbf74 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/main.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/main.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/etherdevice.h> @@ -73,7 +73,7 @@ int mt7915_run(struct ieee80211_hw *hw) if (ret) goto out; - ret = mt7915_mcu_set_sku_en(phy, true); + ret = mt7915_mcu_set_sku_en(phy); if (ret) goto out; diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7915/mcu.c index 17c9c261960f..87ea528915c6 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #if defined(__FreeBSD__) @@ -3066,30 +3066,15 @@ static int mt7915_dpd_freq_idx(struct mt7915_dev *dev, u16 freq, u8 bw) /* 5G BW160 */ 5250, 5570, 5815 }; - static const u16 freq_list_v2_7981[] = { - /* 5G BW20 */ - 5180, 5200, 5220, 5240, - 5260, 5280, 5300, 5320, - 5500, 5520, 5540, 5560, - 5580, 5600, 5620, 5640, - 5660, 5680, 5700, 5720, - 5745, 5765, 5785, 5805, - 5825, 5845, 5865, 5885, - /* 5G BW160 */ - 5250, 5570, 5815 - }; - const u16 *freq_list = freq_list_v1; - int n_freqs = ARRAY_SIZE(freq_list_v1); - int idx; + const u16 *freq_list; + int idx, n_freqs; if (!is_mt7915(&dev->mt76)) { - if (is_mt7981(&dev->mt76)) { - freq_list = freq_list_v2_7981; - n_freqs = ARRAY_SIZE(freq_list_v2_7981); - } else { - freq_list = freq_list_v2; - n_freqs = ARRAY_SIZE(freq_list_v2); - } + freq_list = freq_list_v2; + n_freqs = ARRAY_SIZE(freq_list_v2); + } else { + freq_list = freq_list_v1; + n_freqs = ARRAY_SIZE(freq_list_v1); } if (freq < 4000) { @@ -3365,7 +3350,8 @@ int mt7915_mcu_set_txpower_frame(struct mt7915_phy *phy, int ret; s8 txpower_sku[MT7915_SKU_RATE_NUM]; - ret = mt7915_mcu_get_txpower_sku(phy, txpower_sku, sizeof(txpower_sku)); + ret = mt7915_mcu_get_txpower_sku(phy, txpower_sku, sizeof(txpower_sku), + TX_POWER_INFO_RATE); if (ret) return ret; @@ -3405,51 +3391,136 @@ int mt7915_mcu_set_txpower_frame(struct mt7915_phy *phy, sizeof(req), true); } +static void +mt7915_update_txpower(struct mt7915_phy *phy, int tx_power) +{ + struct mt76_phy *mphy = phy->mt76; + struct ieee80211_channel *chan = mphy->main_chandef.chan; + int chain_idx, val, e2p_power_limit = 0; + + if (!chan) { + mphy->txpower_cur = tx_power; + return; + } + + for (chain_idx = 0; chain_idx < hweight16(mphy->chainmask); chain_idx++) { + val = mt7915_eeprom_get_target_power(phy->dev, chan, chain_idx); + val += mt7915_eeprom_get_power_delta(phy->dev, chan->band); + + e2p_power_limit = max_t(int, e2p_power_limit, val); + } + + if (phy->sku_limit_en) + mphy->txpower_cur = min_t(int, e2p_power_limit, tx_power); + else + mphy->txpower_cur = e2p_power_limit; +} + int mt7915_mcu_set_txpower_sku(struct mt7915_phy *phy) { +#define TX_POWER_LIMIT_TABLE_RATE 0 +#define TX_POWER_LIMIT_TABLE_PATH 1 struct mt7915_dev *dev = phy->dev; struct mt76_phy *mphy = phy->mt76; struct ieee80211_hw *hw = mphy->hw; - struct mt7915_mcu_txpower_sku req = { + struct mt7915_sku_val { + u8 format_id; + u8 limit_type; + u8 band_idx; + } __packed hdr = { .format_id = TX_POWER_LIMIT_TABLE, + .limit_type = TX_POWER_LIMIT_TABLE_RATE, .band_idx = phy->mt76->band_idx, }; - struct mt76_power_limits limits_array; - s8 *la = (s8 *)&limits_array; - int i, idx; - int tx_power; + int i, ret, tx_power; + const u8 *len = mt7915_sku_group_len; + struct mt76_power_limits la = {}; + struct sk_buff *skb; tx_power = mt76_get_power_bound(mphy, hw->conf.power_level); - tx_power = mt76_get_rate_power_limits(mphy, mphy->chandef.chan, - &limits_array, tx_power); - mphy->txpower_cur = tx_power; + if (phy->sku_limit_en) { + tx_power = mt76_get_rate_power_limits(mphy, mphy->chandef.chan, + &la, tx_power); + mt7915_update_txpower(phy, tx_power); + } else { + mt7915_update_txpower(phy, tx_power); + return 0; + } - for (i = 0, idx = 0; i < ARRAY_SIZE(mt7915_sku_group_len); i++) { - u8 mcs_num, len = mt7915_sku_group_len[i]; - int j; + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, + sizeof(hdr) + MT7915_SKU_RATE_NUM); + if (!skb) + return -ENOMEM; - if (i >= SKU_HT_BW20 && i <= SKU_VHT_BW160) { - mcs_num = 10; + skb_put_data(skb, &hdr, sizeof(hdr)); + skb_put_data(skb, &la.cck, len[SKU_CCK] + len[SKU_OFDM]); + skb_put_data(skb, &la.mcs[0], len[SKU_HT_BW20]); + skb_put_data(skb, &la.mcs[1], len[SKU_HT_BW40]); - if (i == SKU_HT_BW20 || i == SKU_VHT_BW20) - la = (s8 *)&limits_array + 12; - } else { - mcs_num = len; - } + /* vht */ + for (i = 0; i < 4; i++) { + skb_put_data(skb, &la.mcs[i], sizeof(la.mcs[i])); + skb_put_zero(skb, 2); /* padding */ + } - for (j = 0; j < min_t(u8, mcs_num, len); j++) - req.txpower_sku[idx + j] = la[j]; + /* he */ + skb_put_data(skb, &la.ru[0], sizeof(la.ru)); + ret = mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_EXT_CMD(TX_POWER_FEATURE_CTRL), true); + if (ret) + return ret; - la += mcs_num; - idx += len; + /* only set per-path power table when it's configured */ + if (!phy->sku_path_en) + return 0; + + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, + sizeof(hdr) + MT7915_SKU_PATH_NUM); + if (!skb) + return -ENOMEM; + + hdr.limit_type = TX_POWER_LIMIT_TABLE_PATH; + skb_put_data(skb, &hdr, sizeof(hdr)); + skb_put_data(skb, &la.path.cck, sizeof(la.path.cck)); + skb_put_data(skb, &la.path.ofdm, sizeof(la.path.ofdm)); + skb_put_data(skb, &la.path.ofdm_bf[1], sizeof(la.path.ofdm_bf) - 1); + + /* HT20 and HT40 */ + skb_put_data(skb, &la.path.ru[3], sizeof(la.path.ru[3])); + skb_put_data(skb, &la.path.ru_bf[3][1], sizeof(la.path.ru_bf[3]) - 1); + skb_put_data(skb, &la.path.ru[4], sizeof(la.path.ru[4])); + skb_put_data(skb, &la.path.ru_bf[4][1], sizeof(la.path.ru_bf[4]) - 1); + + /* start from non-bf and bf fields of + * BW20/RU242, BW40/RU484, BW80/RU996, BW160/RU2x996, + * RU26, RU52, and RU106 + */ + + for (i = 0; i < 8; i++) { + bool bf = i % 2; + u8 idx = (i + 6) / 2; + s8 *buf = bf ? la.path.ru_bf[idx] : la.path.ru[idx]; + /* The non-bf fields of RU26 to RU106 are special cases */ + if (bf) + skb_put_data(skb, buf + 1, 9); + else + skb_put_data(skb, buf, 10); } - return mt76_mcu_send_msg(&dev->mt76, - MCU_EXT_CMD(TX_POWER_FEATURE_CTRL), &req, - sizeof(req), true); + for (i = 0; i < 6; i++) { + bool bf = i % 2; + u8 idx = i / 2; + s8 *buf = bf ? la.path.ru_bf[idx] : la.path.ru[idx]; + + skb_put_data(skb, buf, 10); + } + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_EXT_CMD(TX_POWER_FEATURE_CTRL), true); } -int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len) +int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len, + u8 category) { #define RATE_POWER_INFO 2 struct mt7915_dev *dev = phy->dev; @@ -3460,10 +3531,9 @@ int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len) u8 _rsv; } __packed req = { .format_id = TX_POWER_LIMIT_INFO, - .category = RATE_POWER_INFO, + .category = category, .band_idx = phy->mt76->band_idx, }; - s8 txpower_sku[MT7915_SKU_RATE_NUM][2]; struct sk_buff *skb; int ret, i; @@ -3473,9 +3543,15 @@ int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len) if (ret) return ret; - memcpy(txpower_sku, skb->data + 4, sizeof(txpower_sku)); - for (i = 0; i < len; i++) - txpower[i] = txpower_sku[i][req.band_idx]; + if (category == TX_POWER_INFO_RATE) { + s8 res[MT7915_SKU_RATE_NUM][2]; + + memcpy(res, skb->data + 4, sizeof(res)); + for (i = 0; i < len; i++) + txpower[i] = res[i][req.band_idx]; + } else if (category == TX_POWER_INFO_PATH) { + memcpy(txpower, skb->data + 4, len); + } dev_kfree_skb(skb); @@ -3504,7 +3580,7 @@ int mt7915_mcu_set_test_param(struct mt7915_dev *dev, u8 param, bool test_mode, sizeof(req), false); } -int mt7915_mcu_set_sku_en(struct mt7915_phy *phy, bool enable) +int mt7915_mcu_set_sku_en(struct mt7915_phy *phy) { struct mt7915_dev *dev = phy->dev; struct mt7915_sku { @@ -3513,10 +3589,21 @@ int mt7915_mcu_set_sku_en(struct mt7915_phy *phy, bool enable) u8 band_idx; u8 rsv; } __packed req = { - .format_id = TX_POWER_LIMIT_ENABLE, .band_idx = phy->mt76->band_idx, - .sku_enable = enable, }; + int ret; + + req.sku_enable = phy->sku_limit_en; + req.format_id = TX_POWER_LIMIT_ENABLE; + + ret = mt76_mcu_send_msg(&dev->mt76, + MCU_EXT_CMD(TX_POWER_FEATURE_CTRL), &req, + sizeof(req), true); + if (ret) + return ret; + + req.sku_enable = phy->sku_path_en; + req.format_id = TX_POWER_LIMIT_PATH_ENABLE; return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(TX_POWER_FEATURE_CTRL), &req, diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7915/mcu.h index 086ad89ecd91..3af11a075a2f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt7915/mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7915_MCU_H @@ -429,6 +429,7 @@ enum { enum { TX_POWER_LIMIT_ENABLE, + TX_POWER_LIMIT_PATH_ENABLE = 0x3, TX_POWER_LIMIT_TABLE = 0x4, TX_POWER_LIMIT_INFO = 0x7, TX_POWER_LIMIT_FRAME = 0x11, @@ -436,6 +437,11 @@ enum { }; enum { + TX_POWER_INFO_PATH = 1, + TX_POWER_INFO_RATE, +}; + +enum { SPR_ENABLE = 0x1, SPR_ENABLE_SD = 0x3, SPR_ENABLE_MODE = 0x5, diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mmio.c b/sys/contrib/dev/mediatek/mt76/mt7915/mmio.c index ace2441d41a9..eb79c066fc2b 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/mmio.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/mmio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #if defined(__FreeBSD__) @@ -676,8 +676,8 @@ int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, MT_RXQ_WED_RING_BASE; wed->wlan.wpdma_rx_glo = pci_resource_start(pci_dev, 0) + MT_WPDMA_GLO_CFG; - wed->wlan.wpdma_rx = pci_resource_start(pci_dev, 0) + - MT_RXQ_WED_DATA_RING_BASE; + wed->wlan.wpdma_rx[0] = pci_resource_start(pci_dev, 0) + + MT_RXQ_WED_DATA_RING_BASE; } else { struct platform_device *plat_dev = pdev_ptr; struct resource *res; @@ -699,7 +699,7 @@ int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, wed->wlan.wpdma_tx = res->start + MT_TXQ_WED_RING_BASE; wed->wlan.wpdma_txfree = res->start + MT_RXQ_WED_RING_BASE; wed->wlan.wpdma_rx_glo = res->start + MT_WPDMA_GLO_CFG; - wed->wlan.wpdma_rx = res->start + MT_RXQ_WED_DATA_RING_BASE; + wed->wlan.wpdma_rx[0] = res->start + MT_RXQ_WED_DATA_RING_BASE; } wed->wlan.nbuf = MT7915_HW_TOKEN_SIZE; wed->wlan.tx_tbit[0] = is_mt7915(&dev->mt76) ? 4 : 30; diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mt7915.h b/sys/contrib/dev/mediatek/mt76/mt7915/mt7915.h index d674186488f8..71f417468f48 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/mt7915.h +++ b/sys/contrib/dev/mediatek/mt76/mt7915/mt7915.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7915_H @@ -73,6 +73,7 @@ #define MT7915_CDEV_THROTTLE_MAX 99 #define MT7915_SKU_RATE_NUM 161 +#define MT7915_SKU_PATH_NUM 185 #define MT7915_MAX_TWT_AGRT 16 #define MT7915_MAX_STA_TWT_AGRT 8 @@ -226,6 +227,9 @@ struct mt7915_phy { struct mt76_mib_stats mib; struct mt76_channel_state state_ts; + bool sku_limit_en:1; + bool sku_path_en:1; + #ifdef CONFIG_NL80211_TESTMODE struct { u32 *reg_backup; @@ -494,9 +498,10 @@ int mt7915_mcu_set_mac(struct mt7915_dev *dev, int band, bool enable, int mt7915_mcu_set_test_param(struct mt7915_dev *dev, u8 param, bool test_mode, u8 en); int mt7915_mcu_set_ser(struct mt7915_dev *dev, u8 action, u8 set, u8 band); -int mt7915_mcu_set_sku_en(struct mt7915_phy *phy, bool enable); +int mt7915_mcu_set_sku_en(struct mt7915_phy *phy); int mt7915_mcu_set_txpower_sku(struct mt7915_phy *phy); -int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len); +int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len, + u8 category); int mt7915_mcu_set_txpower_frame_min(struct mt7915_phy *phy, s8 txpower); int mt7915_mcu_set_txpower_frame(struct mt7915_phy *phy, struct ieee80211_vif *vif, diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/pci.c b/sys/contrib/dev/mediatek/mt76/mt7915/pci.c index 26acac3e59fd..4e94a137c077 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/pci.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. * * Author: Ryder Lee <ryder.lee@mediatek.com> diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/regs.h b/sys/contrib/dev/mediatek/mt76/mt7915/regs.h index c5ec63a25a42..307bf6a75674 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/regs.h +++ b/sys/contrib/dev/mediatek/mt76/mt7915/regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7915_REGS_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/soc.c b/sys/contrib/dev/mediatek/mt76/mt7915/soc.c index c823a7554a3a..54ff6de96f3e 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/soc.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/soc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2022 MediaTek Inc. */ #include <linux/kernel.h> @@ -284,20 +284,15 @@ static int mt798x_wmac_coninfra_check(struct mt7915_dev *dev) static int mt798x_wmac_coninfra_setup(struct mt7915_dev *dev) { struct device *pdev = dev->mt76.dev; - struct reserved_mem *rmem; - struct device_node *np; + struct resource res; u32 val; + int ret; - np = of_parse_phandle(pdev->of_node, "memory-region", 0); - if (!np) - return -EINVAL; - - rmem = of_reserved_mem_lookup(np); - of_node_put(np); - if (!rmem) - return -EINVAL; + ret = of_reserved_mem_region_to_resource(pdev->of_node, 0, &res); + if (ret) + return ret; - val = (rmem->base >> 16) & MT_TOP_MCU_EMI_BASE_MASK; + val = (res.start >> 16) & MT_TOP_MCU_EMI_BASE_MASK; if (is_mt7986(&dev->mt76)) { /* Set conninfra subsys PLL check */ @@ -318,8 +313,8 @@ static int mt798x_wmac_coninfra_setup(struct mt7915_dev *dev) MT_TOP_EFUSE_BASE_MASK, 0x11f20000 >> 16); } - mt76_wr(dev, MT_INFRA_BUS_EMI_START, rmem->base); - mt76_wr(dev, MT_INFRA_BUS_EMI_END, rmem->size); + mt76_wr(dev, MT_INFRA_BUS_EMI_START, res.start); + mt76_wr(dev, MT_INFRA_BUS_EMI_END, resource_size(&res)); mt76_rr(dev, MT_CONN_INFRA_EFUSE); diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/testmode.c b/sys/contrib/dev/mediatek/mt76/mt7915/testmode.c index d534fff5c952..618a5c2bdd29 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/testmode.c +++ b/sys/contrib/dev/mediatek/mt76/mt7915/testmode.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include "mt7915.h" @@ -409,7 +409,7 @@ mt7915_tm_init(struct mt7915_phy *phy, bool en) if (!test_bit(MT76_STATE_RUNNING, &phy->mt76->state)) return; - mt7915_mcu_set_sku_en(phy, !en); + mt7915_mcu_set_sku_en(phy); mt7915_tm_mode_ctrl(dev, en); mt7915_tm_reg_backup_restore(phy); diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/testmode.h b/sys/contrib/dev/mediatek/mt76/mt7915/testmode.h index 5573ac309363..bb1bc568751b 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7915/testmode.h +++ b/sys/contrib/dev/mediatek/mt76/mt7915/testmode.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7915_TESTMODE_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/Kconfig b/sys/contrib/dev/mediatek/mt76/mt7921/Kconfig index 7ed51e057857..37b5f46e76f4 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/Kconfig +++ b/sys/contrib/dev/mediatek/mt76/mt7921/Kconfig @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear config MT7921_COMMON tristate select MT792x_LIB diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/Makefile b/sys/contrib/dev/mediatek/mt76/mt7921/Makefile index 849be9e848e0..2ad3c1cc3779 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/Makefile +++ b/sys/contrib/dev/mediatek/mt76/mt7921/Makefile @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear obj-$(CONFIG_MT7921_COMMON) += mt7921-common.o obj-$(CONFIG_MT7921E) += mt7921e.o diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/debugfs.c b/sys/contrib/dev/mediatek/mt76/mt7921/debugfs.c index 616b66a3fde2..4333005b3ad9 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include "mt7921.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/init.c b/sys/contrib/dev/mediatek/mt76/mt7921/init.c index 2498f6d30f4f..dbbc9e945fe2 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/init.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/etherdevice.h> @@ -199,7 +199,9 @@ static int __mt7921_init_hardware(struct mt792x_dev *dev) if (ret) goto out; - mt76_eeprom_override(&dev->mphy); + ret = mt76_eeprom_override(&dev->mphy); + if (ret) + goto out; ret = mt7921_mcu_set_eeprom(dev); if (ret) @@ -355,7 +357,7 @@ int mt7921_register_device(struct mt792x_dev *dev) dev->mphy.hw->wiphy->available_antennas_rx = dev->mphy.chainmask; dev->mphy.hw->wiphy->available_antennas_tx = dev->mphy.chainmask; - queue_work(system_wq, &dev->init_work); + queue_work(system_percpu_wq, &dev->init_work); return 0; } diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/mac.c b/sys/contrib/dev/mediatek/mt76/mt7921/mac.c index 577b27d9faa0..36da611f9bfc 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/devcoredump.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/main.c b/sys/contrib/dev/mediatek/mt76/mt7921/main.c index 5881040ac195..5fae9a6e273c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/main.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/main.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/etherdevice.h> @@ -135,6 +135,8 @@ mt7921_init_he_caps(struct mt792x_phy *phy, enum nl80211_band band, if (is_mt7922(phy->mt76->dev)) { he_cap_elem->phy_cap_info[0] |= IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; + he_cap_elem->phy_cap_info[4] |= + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4; he_cap_elem->phy_cap_info[8] |= IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU; diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7921/mcu.c index 57b9de7a6659..c553423e178c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. */ #if defined(__FreeBSD__) @@ -659,10 +659,10 @@ int mt7921_run_firmware(struct mt792x_dev *dev) if (err) return err; - set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); err = mt7921_load_clc(dev, mt792x_ram_name(dev)); if (err) return err; + set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); return mt7921_mcu_fw_log_2_host(dev, 1); } diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7921/mcu.h index 2834c6c53e58..de676b83b89c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt7921/mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7921_MCU_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/mt7921.h b/sys/contrib/dev/mediatek/mt76/mt7921/mt7921.h index c88793fcec64..83fc7f49ff84 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/mt7921.h +++ b/sys/contrib/dev/mediatek/mt76/mt7921/mt7921.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7921_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/pci.c b/sys/contrib/dev/mediatek/mt76/mt7921/pci.c index bde91b0f0a59..30660837077f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/pci.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. * */ diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/pci_mac.c b/sys/contrib/dev/mediatek/mt76/mt7921/pci_mac.c index 881812ba03ff..5ec084432ae3 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/pci_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/pci_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2021 MediaTek Inc. */ #include "mt7921.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/pci_mcu.c b/sys/contrib/dev/mediatek/mt76/mt7921/pci_mcu.c index 4cf1f2f0f968..8439c849a7a6 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/pci_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/pci_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2021 MediaTek Inc. */ #include "mt7921.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/regs.h b/sys/contrib/dev/mediatek/mt76/mt7921/regs.h index 43427a3a48af..4d9eaf1e0692 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/regs.h +++ b/sys/contrib/dev/mediatek/mt76/mt7921/regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. */ #ifndef __MT7921_REGS_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/sdio.c b/sys/contrib/dev/mediatek/mt76/mt7921/sdio.c index d8d36b3c3068..3421e53dc948 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/sdio.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/sdio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2021 MediaTek Inc. * */ diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mac.c b/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mac.c index a9eb6252a904..416d49e53499 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2021 MediaTek Inc. */ #include <linux/iopoll.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mcu.c b/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mcu.c index 5e4501d7f1c0..14e66f3f5aad 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2021 MediaTek Inc. */ #include <linux/kernel.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/testmode.c b/sys/contrib/dev/mediatek/mt76/mt7921/testmode.c index e838d93477c1..e60ee992edf8 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/testmode.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/testmode.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include "mt7921.h" #include "mcu.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/usb.c b/sys/contrib/dev/mediatek/mt76/mt7921/usb.c index fe9751851ff7..17057e68bf21 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7921/usb.c +++ b/sys/contrib/dev/mediatek/mt76/mt7921/usb.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2022 MediaTek Inc. * * Author: Lorenzo Bianconi <lorenzo@kernel.org> @@ -21,6 +21,9 @@ static const struct usb_device_id mt7921u_device_table[] = { /* Netgear, Inc. [A8000,AXE3000] */ { USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9060, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM }, + /* Netgear, Inc. A7500 */ + { USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9065, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM }, /* TP-Link TXE50UH */ { USB_DEVICE_AND_INTERFACE_INFO(0x35bc, 0x0107, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM }, diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/Kconfig b/sys/contrib/dev/mediatek/mt76/mt7925/Kconfig index 5854e95e68a5..f4f7c93c2ea7 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/Kconfig +++ b/sys/contrib/dev/mediatek/mt76/mt7925/Kconfig @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear config MT7925_COMMON tristate select MT792x_LIB diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/Makefile b/sys/contrib/dev/mediatek/mt76/mt7925/Makefile index ade5e647c941..8f1078ce3231 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/Makefile +++ b/sys/contrib/dev/mediatek/mt76/mt7925/Makefile @@ -1,10 +1,10 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear obj-$(CONFIG_MT7925_COMMON) += mt7925-common.o obj-$(CONFIG_MT7925E) += mt7925e.o obj-$(CONFIG_MT7925U) += mt7925u.o -mt7925-common-y := mac.o mcu.o main.o init.o debugfs.o +mt7925-common-y := mac.o mcu.o regd.o main.o init.o debugfs.o mt7925-common-$(CONFIG_NL80211_TESTMODE) += testmode.o mt7925e-y := pci.o pci_mac.o pci_mcu.o mt7925u-y := usb.o diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/debugfs.c b/sys/contrib/dev/mediatek/mt76/mt7925/debugfs.c index 1e2fc6577e78..e2498659c884 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include "mt7925.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/init.c b/sys/contrib/dev/mediatek/mt76/mt7925/init.c index a5b893b39568..31592775feb3 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/init.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include <linux/etherdevice.h> @@ -7,6 +7,7 @@ #include <linux/thermal.h> #include <linux/firmware.h> #include "mt7925.h" +#include "regd.h" #include "mac.h" #include "mcu.h" @@ -62,151 +63,6 @@ static int mt7925_thermal_init(struct mt792x_phy *phy) } #endif -void mt7925_regd_be_ctrl(struct mt792x_dev *dev, u8 *alpha2) -{ - struct mt792x_phy *phy = &dev->phy; - struct mt7925_clc_rule_v2 *rule; - struct mt7925_clc *clc; - bool old = dev->has_eht, new = true; - u32 mtcl_conf = mt792x_acpi_get_mtcl_conf(&dev->phy, alpha2); - u8 *pos; - - if (mtcl_conf != MT792X_ACPI_MTCL_INVALID && - (((mtcl_conf >> 4) & 0x3) == 0)) { - new = false; - goto out; - } - - if (!phy->clc[MT792x_CLC_BE_CTRL]) - goto out; - - clc = (struct mt7925_clc *)phy->clc[MT792x_CLC_BE_CTRL]; - pos = clc->data; - - while (1) { - rule = (struct mt7925_clc_rule_v2 *)pos; - - if (rule->alpha2[0] == alpha2[0] && - rule->alpha2[1] == alpha2[1]) { - new = false; - break; - } - - /* Check the last one */ - if (rule->flag & BIT(0)) - break; - - pos += sizeof(*rule); - } - -out: - if (old == new) - return; - - dev->has_eht = new; - mt7925_set_stream_he_eht_caps(phy); -} - -static void -mt7925_regd_channel_update(struct wiphy *wiphy, struct mt792x_dev *dev) -{ -#define IS_UNII_INVALID(idx, sfreq, efreq, cfreq) \ - (!(dev->phy.clc_chan_conf & BIT(idx)) && (cfreq) >= (sfreq) && (cfreq) <= (efreq)) -#define MT7925_UNII_59G_IS_VALID 0x1 -#define MT7925_UNII_6G_IS_VALID 0x1e - struct ieee80211_supported_band *sband; - struct mt76_dev *mdev = &dev->mt76; - struct ieee80211_channel *ch; - u32 mtcl_conf = mt792x_acpi_get_mtcl_conf(&dev->phy, mdev->alpha2); - int i; - - if (mtcl_conf != MT792X_ACPI_MTCL_INVALID) { - if ((mtcl_conf & 0x3) == 0) - dev->phy.clc_chan_conf &= ~MT7925_UNII_59G_IS_VALID; - if (((mtcl_conf >> 2) & 0x3) == 0) - dev->phy.clc_chan_conf &= ~MT7925_UNII_6G_IS_VALID; - } - - sband = wiphy->bands[NL80211_BAND_5GHZ]; - if (!sband) - return; - - for (i = 0; i < sband->n_channels; i++) { - ch = &sband->channels[i]; - - /* UNII-4 */ - if (IS_UNII_INVALID(0, 5845, 5925, ch->center_freq)) - ch->flags |= IEEE80211_CHAN_DISABLED; - } - - sband = wiphy->bands[NL80211_BAND_6GHZ]; - if (!sband) - return; - - for (i = 0; i < sband->n_channels; i++) { - ch = &sband->channels[i]; - - /* UNII-5/6/7/8 */ - if (IS_UNII_INVALID(1, 5925, 6425, ch->center_freq) || - IS_UNII_INVALID(2, 6425, 6525, ch->center_freq) || - IS_UNII_INVALID(3, 6525, 6875, ch->center_freq) || - IS_UNII_INVALID(4, 6875, 7125, ch->center_freq)) - ch->flags |= IEEE80211_CHAN_DISABLED; - } -} - -void mt7925_regd_update(struct mt792x_dev *dev) -{ - struct mt76_dev *mdev = &dev->mt76; - struct ieee80211_hw *hw = mdev->hw; - struct wiphy *wiphy = hw->wiphy; - - if (!dev->regd_change) - return; - - mt7925_mcu_set_clc(dev, mdev->alpha2, dev->country_ie_env); - mt7925_regd_channel_update(wiphy, dev); - mt7925_mcu_set_channel_domain(hw->priv); - mt7925_set_tx_sar_pwr(hw, NULL); - dev->regd_change = false; -} -EXPORT_SYMBOL_GPL(mt7925_regd_update); - -static void -mt7925_regd_notifier(struct wiphy *wiphy, - struct regulatory_request *req) -{ - struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); - struct mt792x_dev *dev = mt792x_hw_dev(hw); - struct mt76_dev *mdev = &dev->mt76; - struct mt76_connac_pm *pm = &dev->pm; - - /* allow world regdom at the first boot only */ - if (!memcmp(req->alpha2, "00", 2) && - mdev->alpha2[0] && mdev->alpha2[1]) - return; - - /* do not need to update the same country twice */ - if (!memcmp(req->alpha2, mdev->alpha2, 2) && - dev->country_ie_env == req->country_ie_env) - return; - - memcpy(mdev->alpha2, req->alpha2, 2); - mdev->region = req->dfs_region; - dev->country_ie_env = req->country_ie_env; - dev->regd_change = true; - - if (pm->suspended) - return; - - dev->regd_in_progress = true; - mt792x_mutex_acquire(dev); - mt7925_regd_update(dev); - mt792x_mutex_release(dev); - dev->regd_in_progress = false; - wake_up(&dev->wait); -} - static void mt7925_mac_init_basic_rates(struct mt792x_dev *dev) { int i; @@ -237,8 +93,6 @@ int mt7925_mac_init(struct mt792x_dev *dev) mt7925_mac_init_basic_rates(dev); - memzero_explicit(&dev->mt76.alpha2, sizeof(dev->mt76.alpha2)); - return 0; } EXPORT_SYMBOL_GPL(mt7925_mac_init); @@ -251,7 +105,9 @@ static int __mt7925_init_hardware(struct mt792x_dev *dev) if (ret) goto out; - mt76_eeprom_override(&dev->mphy); + ret = mt76_eeprom_override(&dev->mphy); + if (ret) + goto out; ret = mt7925_mcu_set_eeprom(dev); if (ret) @@ -424,7 +280,7 @@ int mt7925_register_device(struct mt792x_dev *dev) dev->mphy.hw->wiphy->available_antennas_rx = dev->mphy.chainmask; dev->mphy.hw->wiphy->available_antennas_tx = dev->mphy.chainmask; - queue_work(system_wq, &dev->init_work); + queue_work(system_percpu_wq, &dev->init_work); return 0; } diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mac.c b/sys/contrib/dev/mediatek/mt76/mt7925/mac.c index 63995bf9c5d4..1b1a23198b9f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include <linux/devcoredump.h> @@ -6,6 +6,7 @@ #include <linux/timekeeping.h> #include "mt7925.h" #include "../dma.h" +#include "regd.h" #include "mac.h" #include "mcu.h" @@ -1304,7 +1305,6 @@ void mt7925_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&dev->mphy.mac_work); cancel_delayed_work_sync(&pm->ps_work); cancel_work_sync(&pm->wake_work); - dev->sar_inited = false; for (i = 0; i < 10; i++) { mutex_lock(&dev->mt76.mutex); @@ -1333,6 +1333,8 @@ void mt7925_mac_reset_work(struct work_struct *work) IEEE80211_IFACE_ITER_RESUME_ALL, mt7925_vif_connect_iter, NULL); mt76_connac_power_save_sched(&dev->mt76.phy, pm); + + mt7925_regd_change(&dev->phy, "00"); } void mt7925_coredump_work(struct work_struct *work) diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mac.h b/sys/contrib/dev/mediatek/mt76/mt7925/mac.h index b10a993326b9..83ea9021daea 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt7925/mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef __MT7925_MAC_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/main.c b/sys/contrib/dev/mediatek/mt76/mt7925/main.c index b0e053b15227..2d358a96640c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/main.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/main.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include <linux/etherdevice.h> @@ -8,6 +8,7 @@ #include <linux/ctype.h> #include <net/ipv6.h> #include "mt7925.h" +#include "regd.h" #include "mcu.h" #include "mac.h" @@ -138,10 +139,14 @@ mt7925_init_he_caps(struct mt792x_phy *phy, enum nl80211_band band, } if (band == NL80211_BAND_6GHZ) { + struct ieee80211_supported_band *sband = + &phy->mt76->sband_5g.sband; + struct ieee80211_sta_ht_cap *ht_cap = &sband->ht_cap; + u16 cap = IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS | IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS; - cap |= u16_encode_bits(IEEE80211_HT_MPDU_DENSITY_0_5, + cap |= u16_encode_bits(ht_cap->ampdu_density, IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START) | u16_encode_bits(IEEE80211_VHT_MAX_AMPDU_1024K, IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP) | @@ -240,6 +245,7 @@ int mt7925_init_mlo_caps(struct mt792x_phy *phy) { struct wiphy *wiphy = phy->mt76->hw->wiphy; static const u8 ext_capa_sta[] = { + [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT, [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF, }; static struct wiphy_iftype_ext_capab ext_capab[] = { @@ -310,7 +316,6 @@ void mt7925_set_stream_he_eht_caps(struct mt792x_phy *phy) int __mt7925_start(struct mt792x_phy *phy) { struct mt76_phy *mphy = phy->mt76; - struct mt792x_dev *dev = phy->dev; int err; err = mt7925_mcu_set_channel_domain(mphy); @@ -321,13 +326,6 @@ int __mt7925_start(struct mt792x_phy *phy) if (err) return err; - if (!dev->sar_inited) { - err = mt7925_set_tx_sar_pwr(mphy->hw, NULL); - if (err) - return err; - dev->sar_inited = true; - } - mt792x_mac_reset_counters(phy); set_bit(MT76_STATE_RUNNING, &mphy->state); @@ -437,6 +435,9 @@ mt7925_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif) goto out; vif->driver_flags |= IEEE80211_VIF_BEACON_FILTER; + if (phy->chip_cap & MT792x_CHIP_CAP_RSSI_NOTIFY_EVT_EN) + vif->driver_flags |= IEEE80211_VIF_SUPPORTS_CQM_RSSI; + out: mt792x_mutex_release(dev); @@ -987,56 +988,6 @@ int mt7925_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, } EXPORT_SYMBOL_GPL(mt7925_mac_sta_add); -static u16 -mt7925_mac_select_links(struct mt76_dev *mdev, struct ieee80211_vif *vif) -{ - unsigned long usable_links = ieee80211_vif_usable_links(vif); - struct { - u8 link_id; - enum nl80211_band band; - } data[IEEE80211_MLD_MAX_NUM_LINKS]; - u8 link_id, i, j, n_data = 0; - u16 sel_links = 0; - - if (!ieee80211_vif_is_mld(vif)) - return 0; - - if (vif->active_links == usable_links) - return vif->active_links; - - rcu_read_lock(); - for_each_set_bit(link_id, &usable_links, IEEE80211_MLD_MAX_NUM_LINKS) { - struct ieee80211_bss_conf *link_conf = - rcu_dereference(vif->link_conf[link_id]); - - if (WARN_ON_ONCE(!link_conf)) - continue; - - data[n_data].link_id = link_id; - data[n_data].band = link_conf->chanreq.oper.chan->band; - n_data++; - } - rcu_read_unlock(); - - for (i = 0; i < n_data; i++) { - if (!(BIT(data[i].link_id) & vif->active_links)) - continue; - - sel_links = BIT(data[i].link_id); - - for (j = 0; j < n_data; j++) { - if (data[i].band != data[j].band) { - sel_links |= BIT(data[j].link_id); - break; - } - } - - break; - } - - return sel_links; -} - static void mt7925_mac_set_links(struct mt76_dev *mdev, struct ieee80211_vif *vif) { @@ -1047,7 +998,7 @@ mt7925_mac_set_links(struct mt76_dev *mdev, struct ieee80211_vif *vif) struct cfg80211_chan_def *chandef = &link_conf->chanreq.oper; enum nl80211_band band = chandef->chan->band, secondary_band; - u16 sel_links = mt7925_mac_select_links(mdev, vif); + u16 sel_links = mt76_select_links(vif, 2); u8 secondary_link_id = __ffs(~BIT(mvif->deflink_id) & sel_links); if (!ieee80211_vif_is_mld(vif) || hweight16(sel_links) < 2) @@ -1369,20 +1320,6 @@ void mt7925_mlo_pm_work(struct work_struct *work) mt7925_mlo_pm_iter, dev); } -static bool is_valid_alpha2(const char *alpha2) -{ - if (!alpha2) - return false; - - if (alpha2[0] == '0' && alpha2[1] == '0') - return true; - - if (isalpha(alpha2[0]) && isalpha(alpha2[1])) - return true; - - return false; -} - void mt7925_scan_work(struct work_struct *work) { struct mt792x_phy *phy; @@ -1391,7 +1328,6 @@ void mt7925_scan_work(struct work_struct *work) scan_work.work); while (true) { - struct mt76_dev *mdev = &phy->dev->mt76; struct sk_buff *skb; struct tlv *tlv; int tlv_len; @@ -1422,15 +1358,7 @@ void mt7925_scan_work(struct work_struct *work) case UNI_EVENT_SCAN_DONE_CHNLINFO: evt = (struct mt7925_mcu_scan_chinfo_event *)tlv->data; - if (!is_valid_alpha2(evt->alpha2)) - break; - - mt7925_regd_be_ctrl(phy->dev, evt->alpha2); - - if (mdev->alpha2[0] != '0' && mdev->alpha2[1] != '0') - break; - - mt7925_mcu_set_clc(phy->dev, evt->alpha2, ENVIRON_INDOOR); + mt7925_regd_change(phy, evt->alpha2); break; case UNI_EVENT_SCAN_DONE_NLO: @@ -1731,13 +1659,7 @@ static int mt7925_set_sar_specs(struct ieee80211_hw *hw, int err; mt792x_mutex_acquire(dev); - err = mt7925_mcu_set_clc(dev, dev->mt76.alpha2, - dev->country_ie_env); - if (err < 0) - goto out; - err = mt7925_set_tx_sar_pwr(hw, sar); -out: mt792x_mutex_release(dev); return err; @@ -2021,6 +1943,9 @@ static void mt7925_link_info_changed(struct ieee80211_hw *hw, mt7925_mcu_set_eht_pp(mvif->phy->mt76, &mconf->mt76, link_conf, NULL); + if (changed & BSS_CHANGED_CQM) + mt7925_mcu_set_rssimonitor(dev, vif); + mt792x_mutex_release(dev); } diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7925/mcu.c index 77e494a4ece0..987251a1e5c3 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #if defined(__FreeBSD__) @@ -8,16 +8,13 @@ #include <linux/fs.h> #include <linux/firmware.h> #include "mt7925.h" +#include "regd.h" #include "mcu.h" #include "mac.h" #define MT_STA_BFER BIT(0) #define MT_STA_BFEE BIT(1) -static bool mt7925_disable_clc; -module_param_named(disable_clc, mt7925_disable_clc, bool, 0644); -MODULE_PARM_DESC(disable_clc, "disable CLC support"); - int mt7925_mcu_parse_response(struct mt76_dev *mdev, int cmd, struct sk_buff *skb, int seq) { @@ -455,6 +452,56 @@ mt7925_mcu_tx_done_event(struct mt792x_dev *dev, struct sk_buff *skb) } static void +mt7925_mcu_rssi_monitor_iter(void *priv, u8 *mac, + struct ieee80211_vif *vif) +{ + struct mt7925_uni_rssi_monitor_event *event = priv; + enum nl80211_cqm_rssi_threshold_event nl_event; + s32 rssi = le32_to_cpu(event->rssi); + + if (vif->type != NL80211_IFTYPE_STATION) + return; + + if (!(vif->driver_flags & IEEE80211_VIF_SUPPORTS_CQM_RSSI)) + return; + + if (rssi > vif->bss_conf.cqm_rssi_thold) + nl_event = NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH; + else + nl_event = NL80211_CQM_RSSI_THRESHOLD_EVENT_LOW; + + ieee80211_cqm_rssi_notify(vif, nl_event, rssi, GFP_KERNEL); +} + +static void +mt7925_mcu_rssi_monitor_event(struct mt792x_dev *dev, struct sk_buff *skb) +{ + struct tlv *tlv; + u32 tlv_len; + struct mt7925_uni_rssi_monitor_event *event; + + skb_pull(skb, sizeof(struct mt7925_mcu_rxd) + 4); + tlv = (struct tlv *)skb->data; + tlv_len = skb->len; + + while (tlv_len > 0 && le16_to_cpu(tlv->len) <= tlv_len) { + switch (le16_to_cpu(tlv->tag)) { + case UNI_EVENT_RSSI_MONITOR_INFO: + event = (struct mt7925_uni_rssi_monitor_event *)skb->data; + ieee80211_iterate_active_interfaces_atomic(dev->mt76.hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7925_mcu_rssi_monitor_iter, + event); + break; + default: + break; + } + tlv_len -= le16_to_cpu(tlv->len); + tlv = (struct tlv *)((char *)(tlv) + le16_to_cpu(tlv->len)); + } +} + +static void mt7925_mcu_uni_debug_msg_event(struct mt792x_dev *dev, struct sk_buff *skb) { struct mt7925_uni_debug_msg { @@ -550,6 +597,9 @@ mt7925_mcu_uni_rx_unsolicited_event(struct mt792x_dev *dev, case MCU_UNI_EVENT_BSS_BEACON_LOSS: mt7925_mcu_connection_loss_event(dev, skb); break; + case MCU_UNI_EVENT_RSSI_MONITOR: + mt7925_mcu_rssi_monitor_event(dev, skb); + break; case MCU_UNI_EVENT_COREDUMP: dev->fw_assert = true; mt76_connac_mcu_coredump_event(&dev->mt76, skb, &dev->coredump); @@ -697,8 +747,8 @@ static int mt7925_load_clc(struct mt792x_dev *dev, const char *fw_name) int ret, i, len, offset = 0; dev->phy.clc_chan_conf = 0xff; - if (mt7925_disable_clc || - mt76_is_usb(&dev->mt76)) + dev->regd_user = false; + if (!mt7925_regd_clc_supported(dev)) return 0; if (mt76_is_mmio(&dev->mt76)) { @@ -772,7 +822,7 @@ static int mt7925_load_clc(struct mt792x_dev *dev, const char *fw_name) } } - ret = mt7925_mcu_set_clc(dev, "00", ENVIRON_INDOOR); + ret = mt7925_regd_init(phy); out: release_firmware(fw); @@ -1017,10 +1067,10 @@ int mt7925_run_firmware(struct mt792x_dev *dev) if (err) return err; - set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); err = mt7925_load_clc(dev, mt792x_ram_name(dev)); if (err) return err; + set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); return mt7925_mcu_fw_log_2_host(dev, 1); } @@ -2635,6 +2685,25 @@ mt7925_mcu_bss_qos_tlv(struct sk_buff *skb, struct ieee80211_bss_conf *link_conf } static void +mt7925_mcu_bss_mbssid_tlv(struct sk_buff *skb, struct ieee80211_bss_conf *link_conf, + bool enable) +{ + struct bss_info_uni_mbssid *mbssid; + struct tlv *tlv; + + if (!enable && !link_conf->bssid_indicator) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_11V_MBSSID, + sizeof(*mbssid)); + + mbssid = (struct bss_info_uni_mbssid *)tlv; + mbssid->max_indicator = link_conf->bssid_indicator; + mbssid->mbss_idx = link_conf->bssid_index; + mbssid->tx_bss_omac_idx = 0; +} + +static void mt7925_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_bss_conf *link_conf, struct mt792x_phy *phy) { @@ -2800,8 +2869,10 @@ int mt7925_mcu_add_bss_info(struct mt792x_phy *phy, mt7925_mcu_bss_color_tlv(skb, link_conf, enable); } - if (enable) + if (enable) { mt7925_mcu_bss_rlm_tlv(skb, phy->mt76, link_conf, ctx); + mt7925_mcu_bss_mbssid_tlv(skb, link_conf, enable); + } return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_UNI_CMD(BSS_INFO_UPDATE), true); @@ -3376,6 +3447,9 @@ int mt7925_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2, struct mt792x_phy *phy = (struct mt792x_phy *)&dev->phy; int i, ret; + if (!ARRAY_SIZE(phy->clc)) + return -ESRCH; + /* submit all clc config */ for (i = 0; i < ARRAY_SIZE(phy->clc); i++) { if (i == MT792x_CLC_BE_CTRL) @@ -3716,6 +3790,8 @@ out: int mt7925_mcu_set_rate_txpower(struct mt76_phy *phy) { + struct mt76_dev *mdev = phy->dev; + struct mt792x_dev *dev = mt792x_hw_dev(mdev->hw); int err; if (phy->cap.has_2ghz) { @@ -3732,7 +3808,7 @@ int mt7925_mcu_set_rate_txpower(struct mt76_phy *phy) return err; } - if (phy->cap.has_6ghz) { + if (phy->cap.has_6ghz && dev->phy.clc_chan_conf) { err = mt7925_mcu_rate_txpower_band(phy, NL80211_BAND_6GHZ); if (err < 0) @@ -3809,3 +3885,32 @@ int mt7925_mcu_set_rxfilter(struct mt792x_dev *dev, u32 fif, return mt76_mcu_send_msg(&phy->dev->mt76, MCU_UNI_CMD(BAND_CONFIG), &req, sizeof(req), true); } + +int mt7925_mcu_set_rssimonitor(struct mt792x_dev *dev, struct ieee80211_vif *vif) +{ + struct mt792x_bss_conf *mconf = mt792x_link_conf_to_mconf(&vif->bss_conf); + struct { + struct { + u8 bss_idx; + u8 pad[3]; + } __packed hdr; + __le16 tag; + __le16 len; + u8 enable; + s8 cqm_rssi_high; + s8 cqm_rssi_low; + u8 rsv; + } req = { + .hdr = { + .bss_idx = mconf->mt76.idx, + }, + .tag = cpu_to_le16(UNI_CMD_RSSI_MONITOR_SET), + .len = cpu_to_le16(sizeof(req) - 4), + .enable = vif->cfg.assoc, + .cqm_rssi_high = (s8)(vif->bss_conf.cqm_rssi_thold + vif->bss_conf.cqm_rssi_hyst), + .cqm_rssi_low = (s8)(vif->bss_conf.cqm_rssi_thold - vif->bss_conf.cqm_rssi_hyst), + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(RSSI_MONITOR), &req, + sizeof(req), false); +} diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7925/mcu.h index a40764d89a1f..e09e0600534a 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt7925/mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef __MT7925_MCU_H @@ -152,6 +152,14 @@ enum { UNI_EVENT_SCAN_DONE_NLO = 3, }; +enum { + UNI_CMD_RSSI_MONITOR_SET = 0, +}; + +enum { + UNI_EVENT_RSSI_MONITOR_INFO = 0, +}; + enum connac3_mcu_cipher_type { CONNAC3_CIPHER_NONE = 0, CONNAC3_CIPHER_WEP40 = 1, diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mt7925.h b/sys/contrib/dev/mediatek/mt76/mt7925/mt7925.h index 1b165d0d8bd3..6b9bf1b89032 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/mt7925.h +++ b/sys/contrib/dev/mediatek/mt76/mt7925/mt7925.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef __MT7925_H @@ -103,6 +103,12 @@ struct mt7925_uni_beacon_loss_event { struct mt7925_beacon_loss_tlv beacon_loss; } __packed; +struct mt7925_uni_rssi_monitor_event { + __le16 tag; + __le16 len; + __le32 rssi; +} __packed; + #define to_rssi(field, rxv) ((FIELD_GET(field, rxv) - 220) / 2) #define to_rcpi(rssi) (2 * (rssi) + 220) @@ -257,8 +263,6 @@ int mt7925_mcu_chip_config(struct mt792x_dev *dev, const char *cmd); int mt7925_mcu_set_rxfilter(struct mt792x_dev *dev, u32 fif, u8 bit_op, u32 bit_map); -void mt7925_regd_be_ctrl(struct mt792x_dev *dev, u8 *alpha2); -void mt7925_regd_update(struct mt792x_dev *dev); int mt7925_mac_init(struct mt792x_dev *dev); int mt7925_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta); @@ -372,4 +376,5 @@ int mt7925_testmode_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif, int mt7925_testmode_dump(struct ieee80211_hw *hw, struct sk_buff *msg, struct netlink_callback *cb, void *data, int len); +int mt7925_mcu_set_rssimonitor(struct mt792x_dev *dev, struct ieee80211_vif *vif); #endif diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/pci.c b/sys/contrib/dev/mediatek/mt76/mt7925/pci.c index e34f99abd16c..021f431c9ded 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/pci.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #if defined(__FreeBSD__) @@ -12,6 +12,7 @@ #include "mt7925.h" #include "mac.h" #include "mcu.h" +#include "regd.h" #include "../dma.h" static const struct pci_device_id mt7925_pci_device_table[] = { @@ -444,9 +445,9 @@ static void mt7925_pci_remove(struct pci_dev *pdev) pci_free_irq_vectors(pdev); } -#if !defined(__FreeBSD__) || defined(CONFIG_PM_SLEEP) static int mt7925_pci_suspend(struct device *device) { +#if !defined(__FreeBSD__) || defined(CONFIG_PM_SLEEP) struct pci_dev *pdev = to_pci_dev(device); struct mt76_dev *mdev = pci_get_drvdata(pdev); struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76); @@ -532,10 +533,14 @@ restore_suspend: mt792x_reset(&dev->mt76); return err; +#else + return (-EOPNOTSUPP); +#endif } -static int mt7925_pci_resume(struct device *device) +static int _mt7925_pci_resume(struct device *device, bool restore) { +#if !defined(__FreeBSD__) || defined(CONFIG_PM_SLEEP) struct pci_dev *pdev = to_pci_dev(device); struct mt76_dev *mdev = pci_get_drvdata(pdev); struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76); @@ -574,6 +579,9 @@ static int mt7925_pci_resume(struct device *device) napi_schedule(&mdev->tx_napi); local_bh_enable(); + if (restore) + goto failed; + mt76_connac_mcu_set_hif_suspend(mdev, false, false); ret = wait_event_timeout(dev->wait, dev->hif_resumed, 3 * HZ); @@ -586,23 +594,42 @@ static int mt7925_pci_resume(struct device *device) if (!pm->ds_enable) mt7925_mcu_set_deep_sleep(dev, false); - mt7925_regd_update(dev); + mt7925_mcu_regd_update(dev, mdev->alpha2, dev->country_ie_env); failed: pm->suspended = false; - if (err < 0) + if (err < 0 || restore) mt792x_reset(&dev->mt76); return err; -} +#else + return (-EOPNOTSUPP); #endif +} static void mt7925_pci_shutdown(struct pci_dev *pdev) { mt7925_pci_remove(pdev); } -static DEFINE_SIMPLE_DEV_PM_OPS(mt7925_pm_ops, mt7925_pci_suspend, mt7925_pci_resume); +static int mt7925_pci_resume(struct device *device) +{ + return _mt7925_pci_resume(device, false); +} + +static int mt7925_pci_restore(struct device *device) +{ + return _mt7925_pci_resume(device, true); +} + +static const struct dev_pm_ops mt7925_pm_ops = { + .suspend = pm_sleep_ptr(mt7925_pci_suspend), + .resume = pm_sleep_ptr(mt7925_pci_resume), + .freeze = pm_sleep_ptr(mt7925_pci_suspend), + .thaw = pm_sleep_ptr(mt7925_pci_resume), + .poweroff = pm_sleep_ptr(mt7925_pci_suspend), + .restore = pm_sleep_ptr(mt7925_pci_restore), +}; static struct pci_driver mt7925_pci_driver = { .name = KBUILD_MODNAME, diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/pci_mac.c b/sys/contrib/dev/mediatek/mt76/mt7925/pci_mac.c index 4578d16bf456..3072850c2752 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/pci_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/pci_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include "mt7925.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/pci_mcu.c b/sys/contrib/dev/mediatek/mt76/mt7925/pci_mcu.c index f95bc5dcd830..6cceff88c656 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/pci_mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/pci_mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include "mt7925.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/regd.c b/sys/contrib/dev/mediatek/mt76/mt7925/regd.c new file mode 100644 index 000000000000..292087e882d1 --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/mt7925/regd.c @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* Copyright (C) 2025 MediaTek Inc. */ + +#include "mt7925.h" +#include "regd.h" +#include "mcu.h" + +static bool mt7925_disable_clc; +module_param_named(disable_clc, mt7925_disable_clc, bool, 0644); +MODULE_PARM_DESC(disable_clc, "disable CLC support"); + +bool mt7925_regd_clc_supported(struct mt792x_dev *dev) +{ + if (mt7925_disable_clc || + mt76_is_usb(&dev->mt76)) + return false; + + return true; +} + +void mt7925_regd_be_ctrl(struct mt792x_dev *dev, u8 *alpha2) +{ + struct mt792x_phy *phy = &dev->phy; + struct mt7925_clc_rule_v2 *rule; + struct mt7925_clc *clc; + bool old = dev->has_eht, new = true; + u32 mtcl_conf = mt792x_acpi_get_mtcl_conf(&dev->phy, alpha2); + u8 *pos; + + if (mtcl_conf != MT792X_ACPI_MTCL_INVALID && + (((mtcl_conf >> 4) & 0x3) == 0)) { + new = false; + goto out; + } + + if (!phy->clc[MT792x_CLC_BE_CTRL]) + goto out; + + clc = (struct mt7925_clc *)phy->clc[MT792x_CLC_BE_CTRL]; + pos = clc->data; + + while (1) { + rule = (struct mt7925_clc_rule_v2 *)pos; + + if (rule->alpha2[0] == alpha2[0] && + rule->alpha2[1] == alpha2[1]) { + new = false; + break; + } + + /* Check the last one */ + if (rule->flag & BIT(0)) + break; + + pos += sizeof(*rule); + } + +out: + if (old == new) + return; + + dev->has_eht = new; + mt7925_set_stream_he_eht_caps(phy); +} + +static void +mt7925_regd_channel_update(struct wiphy *wiphy, struct mt792x_dev *dev) +{ +#define IS_UNII_INVALID(idx, sfreq, efreq, cfreq) \ + (!(dev->phy.clc_chan_conf & BIT(idx)) && (cfreq) >= (sfreq) && (cfreq) <= (efreq)) +#define MT7925_UNII_59G_IS_VALID 0x1 +#define MT7925_UNII_6G_IS_VALID 0x1e + struct ieee80211_supported_band *sband; + struct mt76_dev *mdev = &dev->mt76; + struct ieee80211_channel *ch; + u32 mtcl_conf = mt792x_acpi_get_mtcl_conf(&dev->phy, mdev->alpha2); + int i; + + if (mtcl_conf != MT792X_ACPI_MTCL_INVALID) { + if ((mtcl_conf & 0x3) == 0) + dev->phy.clc_chan_conf &= ~MT7925_UNII_59G_IS_VALID; + if (((mtcl_conf >> 2) & 0x3) == 0) + dev->phy.clc_chan_conf &= ~MT7925_UNII_6G_IS_VALID; + } + + sband = wiphy->bands[NL80211_BAND_2GHZ]; + if (!sband) + return; + + for (i = 0; i < sband->n_channels; i++) { + ch = &sband->channels[i]; + + if (!dev->has_eht) + ch->flags |= IEEE80211_CHAN_NO_EHT; + } + + sband = wiphy->bands[NL80211_BAND_5GHZ]; + if (!sband) + return; + + for (i = 0; i < sband->n_channels; i++) { + ch = &sband->channels[i]; + + /* UNII-4 */ + if (IS_UNII_INVALID(0, 5845, 5925, ch->center_freq)) + ch->flags |= IEEE80211_CHAN_DISABLED; + + if (!dev->has_eht) + ch->flags |= IEEE80211_CHAN_NO_EHT; + } + + sband = wiphy->bands[NL80211_BAND_6GHZ]; + if (!sband) + return; + + for (i = 0; i < sband->n_channels; i++) { + ch = &sband->channels[i]; + + /* UNII-5/6/7/8 */ + if (IS_UNII_INVALID(1, 5925, 6425, ch->center_freq) || + IS_UNII_INVALID(2, 6425, 6525, ch->center_freq) || + IS_UNII_INVALID(3, 6525, 6875, ch->center_freq) || + IS_UNII_INVALID(4, 6875, 7125, ch->center_freq)) + ch->flags |= IEEE80211_CHAN_DISABLED; + + if (!dev->has_eht) + ch->flags |= IEEE80211_CHAN_NO_EHT; + } +} + +int mt7925_mcu_regd_update(struct mt792x_dev *dev, u8 *alpha2, + enum environment_cap country_ie_env) +{ + struct ieee80211_hw *hw = mt76_hw(dev); + struct wiphy *wiphy = hw->wiphy; + int ret = 0; + + dev->regd_in_progress = true; + + mt792x_mutex_acquire(dev); + if (!dev->regd_change) + goto err; + + ret = mt7925_mcu_set_clc(dev, alpha2, country_ie_env); + if (ret < 0) + goto err; + + mt7925_regd_be_ctrl(dev, alpha2); + mt7925_regd_channel_update(wiphy, dev); + + ret = mt7925_mcu_set_channel_domain(hw->priv); + if (ret < 0) + goto err; + + ret = mt7925_set_tx_sar_pwr(hw, NULL); + if (ret < 0) + goto err; + +err: + mt792x_mutex_release(dev); + dev->regd_change = false; + dev->regd_in_progress = false; + wake_up(&dev->wait); + + return ret; +} +EXPORT_SYMBOL_GPL(mt7925_mcu_regd_update); + +void mt7925_regd_notifier(struct wiphy *wiphy, struct regulatory_request *req) +{ + struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); + struct mt792x_dev *dev = mt792x_hw_dev(hw); + struct mt76_connac_pm *pm = &dev->pm; + struct mt76_dev *mdev = &dev->mt76; + + if (req->initiator == NL80211_REGDOM_SET_BY_USER && + !dev->regd_user) + dev->regd_user = true; + + /* allow world regdom at the first boot only */ + if (!memcmp(req->alpha2, "00", 2) && + mdev->alpha2[0] && mdev->alpha2[1]) + return; + + /* do not need to update the same country twice */ + if (!memcmp(req->alpha2, mdev->alpha2, 2) && + dev->country_ie_env == req->country_ie_env) + return; + + memcpy(mdev->alpha2, req->alpha2, 2); + mdev->region = req->dfs_region; + dev->country_ie_env = req->country_ie_env; + + dev->regd_change = true; + + if (pm->suspended) + /* postpone the mcu update to resume */ + return; + + mt7925_mcu_regd_update(dev, req->alpha2, + req->country_ie_env); + return; +} + +static bool +mt7925_regd_is_valid_alpha2(const char *alpha2) +{ + if (!alpha2) + return false; + + if (alpha2[0] == '0' && alpha2[1] == '0') + return true; + + if (isalpha(alpha2[0]) && isalpha(alpha2[1])) + return true; + + return false; +} + +int mt7925_regd_change(struct mt792x_phy *phy, char *alpha2) +{ + struct wiphy *wiphy = phy->mt76->hw->wiphy; + struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); + struct mt792x_dev *dev = mt792x_hw_dev(hw); + struct mt76_dev *mdev = &dev->mt76; + + if (dev->hw_full_reset) + return 0; + + if (!mt7925_regd_is_valid_alpha2(alpha2) || + !mt7925_regd_clc_supported(dev) || + dev->regd_user) + return -EINVAL; + + if (mdev->alpha2[0] != '0' && mdev->alpha2[1] != '0') + return 0; + + /* do not need to update the same country twice */ + if (!memcmp(alpha2, mdev->alpha2, 2)) + return 0; + + if (phy->chip_cap & MT792x_CHIP_CAP_11D_EN) { + return regulatory_hint(wiphy, alpha2); + } else { + return mt7925_mcu_set_clc(dev, alpha2, ENVIRON_INDOOR); + } +} +EXPORT_SYMBOL_GPL(mt7925_regd_change); + +int mt7925_regd_init(struct mt792x_phy *phy) +{ + struct wiphy *wiphy = phy->mt76->hw->wiphy; + struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); + struct mt792x_dev *dev = mt792x_hw_dev(hw); + struct mt76_dev *mdev = &dev->mt76; + + if (phy->chip_cap & MT792x_CHIP_CAP_11D_EN) { + wiphy->regulatory_flags |= REGULATORY_COUNTRY_IE_IGNORE | + REGULATORY_DISABLE_BEACON_HINTS; + } else { + memzero_explicit(&mdev->alpha2, sizeof(mdev->alpha2)); + } + + return 0; +} diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/regd.h b/sys/contrib/dev/mediatek/mt76/mt7925/regd.h new file mode 100644 index 000000000000..0767f078862e --- /dev/null +++ b/sys/contrib/dev/mediatek/mt76/mt7925/regd.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* Copyright (C) 2025 MediaTek Inc. */ + +#ifndef __MT7925_REGD_H +#define __MT7925_REGD_H + +#include "mt7925.h" + +int mt7925_mcu_regd_update(struct mt792x_dev *dev, u8 *alpha2, + enum environment_cap country_ie_env); + +void mt7925_regd_be_ctrl(struct mt792x_dev *dev, u8 *alpha2); +void mt7925_regd_notifier(struct wiphy *wiphy, struct regulatory_request *req); +bool mt7925_regd_clc_supported(struct mt792x_dev *dev); +int mt7925_regd_change(struct mt792x_phy *phy, char *alpha2); +int mt7925_regd_init(struct mt792x_phy *phy); + +#endif + diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/regs.h b/sys/contrib/dev/mediatek/mt76/mt7925/regs.h index 341987e47f67..24985bba1b90 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/regs.h +++ b/sys/contrib/dev/mediatek/mt76/mt7925/regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef __MT7925_REGS_H diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/testmode.c b/sys/contrib/dev/mediatek/mt76/mt7925/testmode.c index a3c97164ba21..3d40aacfc011 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/testmode.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/testmode.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear #include "mt7925.h" #include "mcu.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/usb.c b/sys/contrib/dev/mediatek/mt76/mt7925/usb.c index 4dfbc1b6cfdd..d9968f03856d 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7925/usb.c +++ b/sys/contrib/dev/mediatek/mt76/mt7925/usb.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include <linux/kernel.h> @@ -12,6 +12,9 @@ static const struct usb_device_id mt7925u_device_table[] = { { USB_DEVICE_AND_INTERFACE_INFO(0x0e8d, 0x7925, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)MT7925_FIRMWARE_WM }, + /* Netgear, Inc. A9000 */ + { USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9072, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)MT7925_FIRMWARE_WM }, { }, }; diff --git a/sys/contrib/dev/mediatek/mt76/mt792x.h b/sys/contrib/dev/mediatek/mt76/mt792x.h index 443d397d9961..8388638ed550 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x.h +++ b/sys/contrib/dev/mediatek/mt76/mt792x.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef __MT792X_H @@ -28,6 +28,7 @@ #define MT792x_CHIP_CAP_CLC_EVT_EN BIT(0) #define MT792x_CHIP_CAP_RSSI_NOTIFY_EVT_EN BIT(1) #define MT792x_CHIP_CAP_WF_RF_PIN_CTRL_EVT_EN BIT(3) +#define MT792x_CHIP_CAP_11D_EN BIT(4) #define MT792x_CHIP_CAP_MLO_EN BIT(8) #define MT792x_CHIP_CAP_MLO_EML_EN BIT(9) @@ -230,11 +231,11 @@ struct mt792x_dev { bool hw_init_done:1; bool fw_assert:1; bool has_eht:1; + bool regd_user:1; bool regd_in_progress:1; bool aspm_supported:1; bool hif_idle:1; bool hif_resumed:1; - bool sar_inited:1; bool regd_change:1; wait_queue_head_t wait; diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.c b/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.c index d1aebadd50aa..946dd7956e4a 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.c +++ b/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include <linux/acpi.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.h b/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.h index e45dcd7fbdb1..474033073831 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.h +++ b/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef __MT7921_ACPI_SAR_H diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_core.c b/sys/contrib/dev/mediatek/mt76/mt792x_core.c index 6ce282d7bd50..c33c141b859e 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_core.c +++ b/sys/contrib/dev/mediatek/mt76/mt792x_core.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include <linux/module.h> @@ -691,9 +691,12 @@ int mt792x_init_wiphy(struct ieee80211_hw *hw) ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS); ieee80211_hw_set(hw, SUPPORTS_VHT_EXT_NSS_BW); ieee80211_hw_set(hw, CONNECTION_MONITOR); - ieee80211_hw_set(hw, NO_VIRTUAL_MONITOR); - if (is_mt7921(&dev->mt76)) + ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); + ieee80211_hw_set(hw, SUPPORTS_ONLY_HE_MULTI_BSSID); + + if (is_mt7921(&dev->mt76)) { ieee80211_hw_set(hw, CHANCTX_STA_CSA); + } if (dev->pm.enable) ieee80211_hw_set(hw, CONNECTION_MONITOR); diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_debugfs.c b/sys/contrib/dev/mediatek/mt76/mt792x_debugfs.c index 9858d9a93851..65c37e0cef8f 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt792x_debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include "mt792x.h" diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_dma.c b/sys/contrib/dev/mediatek/mt76/mt792x_dma.c index d68f814b3581..2405ac51a7c0 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_dma.c +++ b/sys/contrib/dev/mediatek/mt76/mt792x_dma.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include <linux/module.h> @@ -184,13 +184,13 @@ mt792x_dma_reset(struct mt792x_dev *dev, bool force) /* reset hw queues */ for (i = 0; i < __MT_TXQ_MAX; i++) - mt76_queue_reset(dev, dev->mphy.q_tx[i]); + mt76_queue_reset(dev, dev->mphy.q_tx[i], true); for (i = 0; i < __MT_MCUQ_MAX; i++) - mt76_queue_reset(dev, dev->mt76.q_mcu[i]); + mt76_queue_reset(dev, dev->mt76.q_mcu[i], true); mt76_for_each_q_rx(&dev->mt76, i) - mt76_queue_reset(dev, &dev->mt76.q_rx[i]); + mt76_queue_reset(dev, &dev->mt76.q_rx[i], true); mt76_tx_status_check(&dev->mt76, true); diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_mac.c b/sys/contrib/dev/mediatek/mt76/mt792x_mac.c index d72cdb0215e6..830a2e0d6878 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt792x_mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #include <linux/module.h> diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_regs.h b/sys/contrib/dev/mediatek/mt76/mt792x_regs.h index 458cfd0260b1..acf627aed609 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_regs.h +++ b/sys/contrib/dev/mediatek/mt76/mt792x_regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef __MT792X_REGS_H diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_trace.c b/sys/contrib/dev/mediatek/mt76/mt792x_trace.c index b6f284fb929d..ffc77d3944bd 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_trace.c +++ b/sys/contrib/dev/mediatek/mt76/mt792x_trace.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2023 Lorenzo Bianconi <lorenzo@kernel.org> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_trace.h b/sys/contrib/dev/mediatek/mt76/mt792x_trace.h index 61f2aa260656..7b0e3f00b194 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_trace.h +++ b/sys/contrib/dev/mediatek/mt76/mt792x_trace.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2023 Lorenzo Bianconi <lorenzo@kernel.org> */ diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_usb.c b/sys/contrib/dev/mediatek/mt76/mt792x_usb.c index 76272a03b22e..552808458138 100644 --- a/sys/contrib/dev/mediatek/mt76/mt792x_usb.c +++ b/sys/contrib/dev/mediatek/mt76/mt792x_usb.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. * * Author: Lorenzo Bianconi <lorenzo@kernel.org> diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/Kconfig b/sys/contrib/dev/mediatek/mt76/mt7996/Kconfig index bb44d4a5e2dc..5503d03bf62c 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/Kconfig +++ b/sys/contrib/dev/mediatek/mt76/mt7996/Kconfig @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear config MT7996E tristate "MediaTek MT7996 (PCIe) support" select MT76_CONNAC_LIB @@ -12,3 +12,10 @@ config MT7996E and 2.4GHz IEEE 802.11be 4x4:4SS 4096-QAM, 320MHz channels. To compile this driver as a module, choose M here. + +config MT7996_NPU + bool "MT7996 (PCIe) NPU support" + depends on MT7996E + depends on NET_AIROHA_NPU=y || MT7996E=NET_AIROHA_NPU + select MT76_NPU + default n diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/Makefile b/sys/contrib/dev/mediatek/mt76/mt7996/Makefile index 07c8b555c1ac..69d2d4bb9e69 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/Makefile +++ b/sys/contrib/dev/mediatek/mt76/mt7996/Makefile @@ -1,8 +1,9 @@ -# SPDX-License-Identifier: ISC +# SPDX-License-Identifier: BSD-3-Clause-Clear obj-$(CONFIG_MT7996E) += mt7996e.o mt7996e-y := pci.o init.o dma.o eeprom.o main.o mcu.o mac.o \ debugfs.o mmio.o +mt7996e-$(CONFIG_MT7996_NPU) += npu.o mt7996e-$(CONFIG_DEV_COREDUMP) += coredump.o diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/coredump.c b/sys/contrib/dev/mediatek/mt76/mt7996/coredump.c index 433846d2319e..098fb8d5e323 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/coredump.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/coredump.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2023 MediaTek Inc. */ #if defined(__FreeBSD__) diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/coredump.h b/sys/contrib/dev/mediatek/mt76/mt7996/coredump.h index af2ba219b1b5..baa2f6f50832 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/coredump.h +++ b/sys/contrib/dev/mediatek/mt76/mt7996/coredump.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2023 MediaTek Inc. */ #ifndef _COREDUMP_H_ diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/debugfs.c b/sys/contrib/dev/mediatek/mt76/mt7996/debugfs.c index 0ab827f52fd7..76d623b2cafb 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/debugfs.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/debugfs.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -953,16 +953,34 @@ bool mt7996_debugfs_rx_log(struct mt7996_dev *dev, const void *data, int len) #ifdef CONFIG_MAC80211_DEBUGFS /** per-station debugfs **/ -static ssize_t mt7996_sta_fixed_rate_set(struct file *file, - const char __user *user_buf, - size_t count, loff_t *ppos) +static int +mt7996_queues_show(struct seq_file *s, void *data) +{ + struct ieee80211_sta *sta = s->private; + + mt7996_sta_hw_queue_read(s, sta); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mt7996_queues); + +void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, struct dentry *dir) +{ + debugfs_create_file("hw-queues", 0400, dir, sta, &mt7996_queues_fops); +} + +static ssize_t mt7996_link_sta_fixed_rate_set(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) { #define SHORT_PREAMBLE 0 #define LONG_PREAMBLE 1 - struct ieee80211_sta *sta = file->private_data; - struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct ieee80211_link_sta *link_sta = file->private_data; + struct mt7996_sta *msta = (struct mt7996_sta *)link_sta->sta->drv_priv; struct mt7996_dev *dev = msta->vif->deflink.phy->dev; - struct mt7996_sta_link *msta_link = &msta->deflink; + struct mt7996_sta_link *msta_link; struct ra_rate phy = {}; char buf[100]; int ret; @@ -981,12 +999,13 @@ static ssize_t mt7996_sta_fixed_rate_set(struct file *file, /* mode - cck: 0, ofdm: 1, ht: 2, gf: 3, vht: 4, he_su: 8, he_er: 9 EHT: 15 * bw - bw20: 0, bw40: 1, bw80: 2, bw160: 3, BW320: 4 - * nss - vht: 1~4, he: 1~4, eht: 1~4, others: ignore * mcs - cck: 0~4, ofdm: 0~7, ht: 0~32, vht: 0~9, he_su: 0~11, he_er: 0~2, eht: 0~13 + * nss - vht: 1~4, he: 1~4, eht: 1~4, others: ignore * gi - (ht/vht) lgi: 0, sgi: 1; (he) 0.8us: 0, 1.6us: 1, 3.2us: 2 * preamble - short: 1, long: 0 - * ldpc - off: 0, on: 1 * stbc - off: 0, on: 1 + * ldpc - off: 0, on: 1 + * spe - off: 0, on: 1 * ltf - 1xltf: 0, 2xltf: 1, 4xltf: 2 */ if (sscanf(buf, "%hhu %hhu %hhu %hhu %hu %hhu %hhu %hhu %hhu %hu", @@ -994,9 +1013,16 @@ static ssize_t mt7996_sta_fixed_rate_set(struct file *file, &phy.preamble, &phy.stbc, &phy.ldpc, &phy.spe, <f) != 10) { dev_warn(dev->mt76.dev, "format: Mode BW MCS NSS GI Preamble STBC LDPC SPE ltf\n"); - goto out; + return -EINVAL; } + mutex_lock(&dev->mt76.mutex); + + msta_link = mt76_dereference(msta->link[link_sta->link_id], &dev->mt76); + if (!msta_link) { + ret = -EINVAL; + goto out; + } phy.wlan_idx = cpu_to_le16(msta_link->wcid.idx); phy.gi = cpu_to_le16(gi); phy.ltf = cpu_to_le16(ltf); @@ -1005,36 +1031,26 @@ static ssize_t mt7996_sta_fixed_rate_set(struct file *file, ret = mt7996_mcu_set_fixed_rate_ctrl(dev, &phy, 0); if (ret) - return -EFAULT; + goto out; + ret = count; out: - return count; + mutex_unlock(&dev->mt76.mutex); + return ret; } static const struct file_operations fops_fixed_rate = { - .write = mt7996_sta_fixed_rate_set, + .write = mt7996_link_sta_fixed_rate_set, .open = simple_open, .owner = THIS_MODULE, .llseek = default_llseek, }; -static int -mt7996_queues_show(struct seq_file *s, void *data) -{ - struct ieee80211_sta *sta = s->private; - - mt7996_sta_hw_queue_read(s, sta); - - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(mt7996_queues); - -void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - struct ieee80211_sta *sta, struct dentry *dir) +void mt7996_link_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_link_sta *link_sta, + struct dentry *dir) { - debugfs_create_file("fixed_rate", 0600, dir, sta, &fops_fixed_rate); - debugfs_create_file("hw-queues", 0400, dir, sta, &mt7996_queues_fops); + debugfs_create_file("fixed_rate", 0600, dir, link_sta, &fops_fixed_rate); } #endif diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/dma.c b/sys/contrib/dev/mediatek/mt76/mt7996/dma.c index 304bb5a2318c..98ab62d76d24 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/dma.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/dma.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -20,12 +20,15 @@ int mt7996_init_tx_queues(struct mt7996_phy *phy, int idx, int n_desc, ring_base += MT_TXQ_ID(0) * MT_RING_SIZE; idx -= MT_TXQ_ID(0); - if (phy->mt76->band_idx == MT_BAND2) + if (wed == &dev->mt76.mmio.wed_hif2) flags = MT_WED_Q_TX(0); else flags = MT_WED_Q_TX(idx); } + if (mt76_npu_device_active(&dev->mt76)) + flags = MT_NPU_Q_TX(phy->mt76->band_idx); + return mt76_connac_init_tx_queues(phy->mt76, idx, n_desc, ring_base, wed, flags); } @@ -86,36 +89,74 @@ static void mt7996_dma_config(struct mt7996_dev *dev) break; } - if (dev->has_rro) { + if (mt7996_has_hwrro(dev)) { /* band0 */ RXQ_CONFIG(MT_RXQ_RRO_BAND0, WFDMA0, MT_INT_RX_DONE_RRO_BAND0, MT7996_RXQ_RRO_BAND0); - RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND0, WFDMA0, MT_INT_RX_DONE_MSDU_PG_BAND0, - MT7996_RXQ_MSDU_PG_BAND0); - RXQ_CONFIG(MT_RXQ_TXFREE_BAND0, WFDMA0, MT_INT_RX_TXFREE_MAIN, - MT7996_RXQ_TXFREE0); - /* band1 */ - RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND1, WFDMA0, MT_INT_RX_DONE_MSDU_PG_BAND1, - MT7996_RXQ_MSDU_PG_BAND1); - /* band2 */ - RXQ_CONFIG(MT_RXQ_RRO_BAND2, WFDMA0, MT_INT_RX_DONE_RRO_BAND2, - MT7996_RXQ_RRO_BAND2); - RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND2, WFDMA0, MT_INT_RX_DONE_MSDU_PG_BAND2, - MT7996_RXQ_MSDU_PG_BAND2); - RXQ_CONFIG(MT_RXQ_TXFREE_BAND2, WFDMA0, MT_INT_RX_TXFREE_TRI, - MT7996_RXQ_TXFREE2); - - RXQ_CONFIG(MT_RXQ_RRO_IND, WFDMA0, MT_INT_RX_DONE_RRO_IND, - MT7996_RXQ_RRO_IND); + if (dev->mt76.hwrro_mode == MT76_HWRRO_V3) + RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND0, WFDMA0, + MT_INT_RX_DONE_MSDU_PG_BAND0, + MT7996_RXQ_MSDU_PG_BAND0); + if (is_mt7996(&dev->mt76)) { + RXQ_CONFIG(MT_RXQ_TXFREE_BAND0, WFDMA0, + MT_INT_RX_TXFREE_MAIN, MT7996_RXQ_TXFREE0); + /* band1 */ + RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND1, WFDMA0, + MT_INT_RX_DONE_MSDU_PG_BAND1, + MT7996_RXQ_MSDU_PG_BAND1); + /* band2 */ + RXQ_CONFIG(MT_RXQ_RRO_BAND2, WFDMA0, + MT_INT_RX_DONE_RRO_BAND2, + MT7996_RXQ_RRO_BAND2); + RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND2, WFDMA0, + MT_INT_RX_DONE_MSDU_PG_BAND2, + MT7996_RXQ_MSDU_PG_BAND2); + RXQ_CONFIG(MT_RXQ_TXFREE_BAND2, WFDMA0, + MT_INT_RX_TXFREE_TRI, MT7996_RXQ_TXFREE2); + } else { + RXQ_CONFIG(MT_RXQ_RRO_BAND1, WFDMA0, + MT_INT_RX_DONE_RRO_BAND1, + MT7996_RXQ_RRO_BAND1); + } + + if (dev->mt76.hwrro_mode == MT76_HWRRO_V3) + RXQ_CONFIG(MT_RXQ_RRO_IND, WFDMA0, + MT_INT_RX_DONE_RRO_IND, + MT7996_RXQ_RRO_IND); + else + RXQ_CONFIG(MT_RXQ_RRO_RXDMAD_C, WFDMA0, + MT_INT_RX_DONE_RRO_RXDMAD_C, + MT7996_RXQ_RRO_RXDMAD_C); } /* data tx queue */ - TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, MT7996_TXQ_BAND0); if (is_mt7996(&dev->mt76)) { - TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, MT7996_TXQ_BAND1); - TXQ_CONFIG(2, WFDMA0, MT_INT_TX_DONE_BAND2, MT7996_TXQ_BAND2); + TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, MT7996_TXQ_BAND0); + if (dev->hif2) { + /* default bn1:ring19 bn2:ring21 */ + TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, + MT7996_TXQ_BAND1); + TXQ_CONFIG(2, WFDMA0, MT_INT_TX_DONE_BAND2, + MT7996_TXQ_BAND2); + } else { + /* single pcie bn0/1:ring18 bn2:ring19 */ + TXQ_CONFIG(2, WFDMA0, MT_INT_TX_DONE_BAND1, + MT7996_TXQ_BAND1); + } } else { - TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, MT7996_TXQ_BAND1); + if (dev->hif2) { + /* bn0:ring18 bn1:ring21 */ + TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, + MT7996_TXQ_BAND0); + TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND2, + MT7996_TXQ_BAND2); + } else { + /* single pcie bn0:ring18 bn1:ring19 */ + TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, + MT7996_TXQ_BAND0); + TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, + MT7996_TXQ_BAND1); + } } /* mcu tx queue */ @@ -169,11 +210,12 @@ static void __mt7996_dma_prefetch(struct mt7996_dev *dev, u32 ofs) /* Rx TxFreeDone From MAC Rings */ val = is_mt7996(&dev->mt76) ? 4 : 8; - if (is_mt7990(&dev->mt76) || (is_mt7996(&dev->mt76) && dev->has_rro)) + if ((is_mt7996(&dev->mt76) && mt7996_has_hwrro(dev)) || + is_mt7990(&dev->mt76)) mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_TXFREE_BAND0) + ofs, PREFETCH(val)); if (is_mt7990(&dev->mt76) && dev->hif2) mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_TXFREE_BAND1) + ofs, PREFETCH(val)); - else if (is_mt7996(&dev->mt76) && dev->has_rro) + else if (is_mt7996(&dev->mt76) && mt7996_has_hwrro(dev)) mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_TXFREE_BAND2) + ofs, PREFETCH(val)); /* Rx Data Rings */ @@ -182,7 +224,7 @@ static void __mt7996_dma_prefetch(struct mt7996_dev *dev, u32 ofs) mt76_wr(dev, MT_RXQ_EXT_CTRL(queue) + ofs, PREFETCH(0x10)); /* Rx RRO Rings */ - if (dev->has_rro) { + if (mt7996_has_hwrro(dev)) { mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_RRO_BAND0) + ofs, PREFETCH(0x10)); queue = is_mt7996(&dev->mt76) ? MT_RXQ_RRO_BAND2 : MT_RXQ_RRO_BAND1; mt76_wr(dev, MT_RXQ_EXT_CTRL(queue) + ofs, PREFETCH(0x10)); @@ -291,11 +333,14 @@ void mt7996_dma_start(struct mt7996_dev *dev, bool reset, bool wed_reset) if (mt7996_band_valid(dev, MT_BAND0)) irq_mask |= MT_INT_BAND0_RX_DONE; - if (mt7996_band_valid(dev, MT_BAND1)) + if (mt7996_band_valid(dev, MT_BAND1)) { irq_mask |= MT_INT_BAND1_RX_DONE; + if (is_mt7992(&dev->mt76) && dev->hif2) + irq_mask |= MT_INT_RX_TXFREE_BAND1_EXT; + } if (mt7996_band_valid(dev, MT_BAND2)) - irq_mask |= MT_INT_BAND2_RX_DONE; + irq_mask |= MT_INT_BAND2_RX_DONE | MT_INT_TX_RX_DONE_EXT; if (mtk_wed_device_active(wed) && wed_reset) { u32 wed_irq_mask = irq_mask; @@ -305,7 +350,7 @@ void mt7996_dma_start(struct mt7996_dev *dev, bool reset, bool wed_reset) mtk_wed_device_start(wed, wed_irq_mask); } - if (!mt7996_has_wa(dev)) + if (!mt7996_has_wa(dev) || mt76_npu_device_active(&dev->mt76)) irq_mask &= ~(MT_INT_RX(MT_RXQ_MAIN_WA) | MT_INT_RX(MT_RXQ_BAND1_WA)); irq_mask = reset ? MT_INT_MCU_CMD : irq_mask; @@ -381,13 +426,48 @@ static void mt7996_dma_enable(struct mt7996_dev *dev, bool reset) WF_WFDMA0_GLO_CFG_EXT1_TX_FCTRL_MODE); mt76_set(dev, MT_WFDMA_HOST_CONFIG, - MT_WFDMA_HOST_CONFIG_PDMA_BAND | - MT_WFDMA_HOST_CONFIG_BAND2_PCIE1); + MT_WFDMA_HOST_CONFIG_PDMA_BAND); + + mt76_clear(dev, MT_WFDMA_HOST_CONFIG, + MT_WFDMA_HOST_CONFIG_BAND0_PCIE1 | + MT_WFDMA_HOST_CONFIG_BAND1_PCIE1 | + MT_WFDMA_HOST_CONFIG_BAND2_PCIE1); + + if (is_mt7996(&dev->mt76)) + mt76_set(dev, MT_WFDMA_HOST_CONFIG, + MT_WFDMA_HOST_CONFIG_BAND2_PCIE1); + else + mt76_set(dev, MT_WFDMA_HOST_CONFIG, + MT_WFDMA_HOST_CONFIG_BAND1_PCIE1); /* AXI read outstanding number */ mt76_rmw(dev, MT_WFDMA_AXI_R2A_CTRL, MT_WFDMA_AXI_R2A_CTRL_OUTSTAND_MASK, 0x14); + if (dev->hif2->speed < PCIE_SPEED_5_0GT || + (dev->hif2->speed == PCIE_SPEED_5_0GT && + dev->hif2->width < PCIE_LNK_X2)) { + mt76_rmw(dev, WF_WFDMA0_GLO_CFG_EXT0 + hif1_ofs, + WF_WFDMA0_GLO_CFG_EXT0_OUTSTAND_MASK, + FIELD_PREP(WF_WFDMA0_GLO_CFG_EXT0_OUTSTAND_MASK, + 0x1)); + mt76_rmw(dev, MT_WFDMA_AXI_R2A_CTRL2, + MT_WFDMA_AXI_R2A_CTRL2_OUTSTAND_MASK, + FIELD_PREP(MT_WFDMA_AXI_R2A_CTRL2_OUTSTAND_MASK, + 0x1)); + } else if (dev->hif2->speed < PCIE_SPEED_8_0GT || + (dev->hif2->speed == PCIE_SPEED_8_0GT && + dev->hif2->width < PCIE_LNK_X2)) { + mt76_rmw(dev, WF_WFDMA0_GLO_CFG_EXT0 + hif1_ofs, + WF_WFDMA0_GLO_CFG_EXT0_OUTSTAND_MASK, + FIELD_PREP(WF_WFDMA0_GLO_CFG_EXT0_OUTSTAND_MASK, + 0x2)); + mt76_rmw(dev, MT_WFDMA_AXI_R2A_CTRL2, + MT_WFDMA_AXI_R2A_CTRL2_OUTSTAND_MASK, + FIELD_PREP(MT_WFDMA_AXI_R2A_CTRL2_OUTSTAND_MASK, + 0x2)); + } + /* WFDMA rx threshold */ mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_45_TH + hif1_ofs, 0xc000c); mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_67_TH + hif1_ofs, 0x10008); @@ -400,27 +480,61 @@ static void mt7996_dma_enable(struct mt7996_dev *dev, bool reset) * so, redirect pcie0 rx ring3 interrupt to pcie1 */ if (mtk_wed_device_active(&dev->mt76.mmio.wed) && - dev->has_rro) + mt7996_has_hwrro(dev)) { + u32 intr = is_mt7996(&dev->mt76) ? + MT_WFDMA0_RX_INT_SEL_RING6 : + MT_WFDMA0_RX_INT_SEL_RING9 | + MT_WFDMA0_RX_INT_SEL_RING5; + mt76_set(dev, MT_WFDMA0_RX_INT_PCIE_SEL + hif1_ofs, - MT_WFDMA0_RX_INT_SEL_RING6); - else + intr); + } else { mt76_set(dev, MT_WFDMA0_RX_INT_PCIE_SEL, MT_WFDMA0_RX_INT_SEL_RING3); + } } mt7996_dma_start(dev, reset, true); } -#ifdef CONFIG_NET_MEDIATEK_SOC_WED int mt7996_dma_rro_init(struct mt7996_dev *dev) { struct mt76_dev *mdev = &dev->mt76; u32 irq_mask; int ret; + if (dev->mt76.hwrro_mode == MT76_HWRRO_V3_1) { + /* rxdmad_c */ + mdev->q_rx[MT_RXQ_RRO_RXDMAD_C].flags = MT_WED_RRO_Q_RXDMAD_C; + if (mtk_wed_device_active(&mdev->mmio.wed)) + mdev->q_rx[MT_RXQ_RRO_RXDMAD_C].wed = &mdev->mmio.wed; + else if (!mt76_npu_device_active(&dev->mt76)) + mdev->q_rx[MT_RXQ_RRO_RXDMAD_C].flags |= MT_QFLAG_EMI_EN; + ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_RRO_RXDMAD_C], + MT_RXQ_ID(MT_RXQ_RRO_RXDMAD_C), + MT7996_RX_RING_SIZE, + MT7996_RX_BUF_SIZE, + MT_RXQ_RRO_AP_RING_BASE); + if (ret) + return ret; + + if (!mtk_wed_device_active(&mdev->mmio.wed)) { + /* We need to set cpu idx pointer before resetting the + * EMI queues. + */ + mdev->q_rx[MT_RXQ_RRO_RXDMAD_C].emi_cpu_idx = + &dev->wed_rro.emi_rings_cpu.ptr->ring[0].idx; + mt76_queue_reset(dev, &mdev->q_rx[MT_RXQ_RRO_RXDMAD_C], + true); + } + goto start_hw_rro; + } + /* ind cmd */ mdev->q_rx[MT_RXQ_RRO_IND].flags = MT_WED_RRO_Q_IND; - mdev->q_rx[MT_RXQ_RRO_IND].wed = &mdev->mmio.wed; + if (mtk_wed_device_active(&mdev->mmio.wed) && + mtk_wed_get_rx_capa(&mdev->mmio.wed)) + mdev->q_rx[MT_RXQ_RRO_IND].wed = &mdev->mmio.wed; ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_RRO_IND], MT_RXQ_ID(MT_RXQ_RRO_IND), MT7996_RX_RING_SIZE, @@ -431,7 +545,9 @@ int mt7996_dma_rro_init(struct mt7996_dev *dev) /* rx msdu page queue for band0 */ mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND0].flags = MT_WED_RRO_Q_MSDU_PG(0) | MT_QFLAG_WED_RRO_EN; - mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND0].wed = &mdev->mmio.wed; + if (mtk_wed_device_active(&mdev->mmio.wed) && + mtk_wed_get_rx_capa(&mdev->mmio.wed)) + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND0].wed = &mdev->mmio.wed; ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND0], MT_RXQ_ID(MT_RXQ_MSDU_PAGE_BAND0), MT7996_RX_RING_SIZE, @@ -440,11 +556,13 @@ int mt7996_dma_rro_init(struct mt7996_dev *dev) if (ret) return ret; - if (mt7996_band_valid(dev, MT_BAND1)) { + if (mt7996_band_valid(dev, MT_BAND1) && is_mt7996(&dev->mt76)) { /* rx msdu page queue for band1 */ mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND1].flags = MT_WED_RRO_Q_MSDU_PG(1) | MT_QFLAG_WED_RRO_EN; - mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND1].wed = &mdev->mmio.wed; + if (mtk_wed_device_active(&mdev->mmio.wed) && + mtk_wed_get_rx_capa(&mdev->mmio.wed)) + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND1].wed = &mdev->mmio.wed; ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND1], MT_RXQ_ID(MT_RXQ_MSDU_PAGE_BAND1), MT7996_RX_RING_SIZE, @@ -458,7 +576,9 @@ int mt7996_dma_rro_init(struct mt7996_dev *dev) /* rx msdu page queue for band2 */ mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND2].flags = MT_WED_RRO_Q_MSDU_PG(2) | MT_QFLAG_WED_RRO_EN; - mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND2].wed = &mdev->mmio.wed; + if (mtk_wed_device_active(&mdev->mmio.wed) && + mtk_wed_get_rx_capa(&mdev->mmio.wed)) + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND2].wed = &mdev->mmio.wed; ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND2], MT_RXQ_ID(MT_RXQ_MSDU_PAGE_BAND2), MT7996_RX_RING_SIZE, @@ -468,15 +588,44 @@ int mt7996_dma_rro_init(struct mt7996_dev *dev) return ret; } - irq_mask = mdev->mmio.irqmask | MT_INT_RRO_RX_DONE | - MT_INT_TX_DONE_BAND2; - mt76_wr(dev, MT_INT_MASK_CSR, irq_mask); - mtk_wed_device_start_hw_rro(&mdev->mmio.wed, irq_mask, false); - mt7996_irq_enable(dev, irq_mask); +start_hw_rro: + if (mtk_wed_device_active(&mdev->mmio.wed)) { + irq_mask = mdev->mmio.irqmask | + MT_INT_TX_DONE_BAND2; + + mt76_wr(dev, MT_INT_MASK_CSR, irq_mask); + mtk_wed_device_start_hw_rro(&mdev->mmio.wed, irq_mask, false); + mt7996_irq_enable(dev, irq_mask); + } else { + if (is_mt7996(&dev->mt76)) { + mt76_queue_rx_init(dev, MT_RXQ_MSDU_PAGE_BAND1, + mt76_dma_rx_poll); + mt76_queue_rx_init(dev, MT_RXQ_MSDU_PAGE_BAND2, + mt76_dma_rx_poll); + mt76_queue_rx_init(dev, MT_RXQ_RRO_BAND2, + mt76_dma_rx_poll); + } else { + mt76_queue_rx_init(dev, MT_RXQ_RRO_BAND1, + mt76_dma_rx_poll); + } + + mt76_queue_rx_init(dev, MT_RXQ_RRO_BAND0, mt76_dma_rx_poll); + if (dev->mt76.hwrro_mode == MT76_HWRRO_V3_1) { + mt76_queue_rx_init(dev, MT_RXQ_RRO_RXDMAD_C, + mt76_dma_rx_poll); + } else { + mt76_queue_rx_init(dev, MT_RXQ_RRO_IND, + mt76_dma_rx_poll); + mt76_queue_rx_init(dev, MT_RXQ_MSDU_PAGE_BAND0, + mt76_dma_rx_poll); + } + + if (!mt76_npu_device_active(&dev->mt76)) + mt7996_irq_enable(dev, MT_INT_RRO_RX_DONE); + } return 0; } -#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ int mt7996_dma_init(struct mt7996_dev *dev) { @@ -563,7 +712,9 @@ int mt7996_dma_init(struct mt7996_dev *dev) return ret; /* tx free notify event from WA for band0 */ - if (mtk_wed_device_active(wed) && !dev->has_rro) { + if (mtk_wed_device_active(wed) && + ((is_mt7996(&dev->mt76) && !mt7996_has_hwrro(dev)) || + (is_mt7992(&dev->mt76)))) { dev->mt76.q_rx[MT_RXQ_MAIN_WA].flags = MT_WED_Q_TXFREE; dev->mt76.q_rx[MT_RXQ_MAIN_WA].wed = wed; } @@ -618,7 +769,7 @@ int mt7996_dma_init(struct mt7996_dev *dev) /* tx free notify event from WA for mt7996 band2 * use pcie0's rx ring3, but, redirect pcie0 rx ring3 interrupt to pcie1 */ - if (mtk_wed_device_active(wed_hif2) && !dev->has_rro) { + if (mtk_wed_device_active(wed_hif2) && !mt7996_has_hwrro(dev)) { dev->mt76.q_rx[MT_RXQ_BAND2_WA].flags = MT_WED_Q_TXFREE; dev->mt76.q_rx[MT_RXQ_BAND2_WA].wed = wed_hif2; } @@ -633,6 +784,11 @@ int mt7996_dma_init(struct mt7996_dev *dev) } else if (mt7996_band_valid(dev, MT_BAND1)) { /* rx data queue for mt7992 band1 */ rx_base = MT_RXQ_RING_BASE(MT_RXQ_BAND1) + hif1_ofs; + if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed)) { + dev->mt76.q_rx[MT_RXQ_BAND1].flags = MT_WED_Q_RX(1); + dev->mt76.q_rx[MT_RXQ_BAND1].wed = wed; + } + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1], MT_RXQ_ID(MT_RXQ_BAND1), MT7996_RX_RING_SIZE, @@ -644,6 +800,12 @@ int mt7996_dma_init(struct mt7996_dev *dev) /* tx free notify event from WA for mt7992 band1 */ if (mt7996_has_wa(dev)) { rx_base = MT_RXQ_RING_BASE(MT_RXQ_BAND1_WA) + hif1_ofs; + if (mtk_wed_device_active(wed_hif2)) { + dev->mt76.q_rx[MT_RXQ_BAND1_WA].flags = + MT_WED_Q_TXFREE; + dev->mt76.q_rx[MT_RXQ_BAND1_WA].wed = wed_hif2; + } + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1_WA], MT_RXQ_ID(MT_RXQ_BAND1_WA), MT7996_RX_MCU_RING_SIZE, @@ -654,12 +816,12 @@ int mt7996_dma_init(struct mt7996_dev *dev) } } - if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed) && - dev->has_rro) { + if (mt7996_has_hwrro(dev)) { /* rx rro data queue for band0 */ dev->mt76.q_rx[MT_RXQ_RRO_BAND0].flags = MT_WED_RRO_Q_DATA(0) | MT_QFLAG_WED_RRO_EN; - dev->mt76.q_rx[MT_RXQ_RRO_BAND0].wed = wed; + if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed)) + dev->mt76.q_rx[MT_RXQ_RRO_BAND0].wed = wed; ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_RRO_BAND0], MT_RXQ_ID(MT_RXQ_RRO_BAND0), MT7996_RX_RING_SIZE, @@ -668,23 +830,44 @@ int mt7996_dma_init(struct mt7996_dev *dev) if (ret) return ret; - /* tx free notify event from WA for band0 */ - dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0].flags = MT_WED_Q_TXFREE; - dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0].wed = wed; + if (is_mt7992(&dev->mt76)) { + dev->mt76.q_rx[MT_RXQ_RRO_BAND1].flags = + MT_WED_RRO_Q_DATA(1) | MT_QFLAG_WED_RRO_EN; + if (mtk_wed_device_active(wed) && + mtk_wed_get_rx_capa(wed)) + dev->mt76.q_rx[MT_RXQ_RRO_BAND1].wed = wed; + ret = mt76_queue_alloc(dev, + &dev->mt76.q_rx[MT_RXQ_RRO_BAND1], + MT_RXQ_ID(MT_RXQ_RRO_BAND1), + MT7996_RX_RING_SIZE, + MT7996_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_RRO_BAND1) + hif1_ofs); + if (ret) + return ret; + } else { + if (mtk_wed_device_active(wed)) { + /* tx free notify event from WA for band0 */ + dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0].flags = MT_WED_Q_TXFREE; + dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0].wed = wed; + } - ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0], - MT_RXQ_ID(MT_RXQ_TXFREE_BAND0), - MT7996_RX_MCU_RING_SIZE, - MT7996_RX_BUF_SIZE, - MT_RXQ_RING_BASE(MT_RXQ_TXFREE_BAND0)); - if (ret) - return ret; + ret = mt76_queue_alloc(dev, + &dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0], + MT_RXQ_ID(MT_RXQ_TXFREE_BAND0), + MT7996_RX_MCU_RING_SIZE, + MT7996_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_TXFREE_BAND0)); + if (ret) + return ret; + } if (mt7996_band_valid(dev, MT_BAND2)) { /* rx rro data queue for band2 */ dev->mt76.q_rx[MT_RXQ_RRO_BAND2].flags = MT_WED_RRO_Q_DATA(1) | MT_QFLAG_WED_RRO_EN; - dev->mt76.q_rx[MT_RXQ_RRO_BAND2].wed = wed; + if (mtk_wed_device_active(wed) && + mtk_wed_get_rx_capa(wed)) + dev->mt76.q_rx[MT_RXQ_RRO_BAND2].wed = wed; ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_RRO_BAND2], MT_RXQ_ID(MT_RXQ_RRO_BAND2), MT7996_RX_RING_SIZE, @@ -712,6 +895,10 @@ int mt7996_dma_init(struct mt7996_dev *dev) if (ret < 0) return ret; + ret = mt7996_npu_rx_queues_init(dev); + if (ret) + return ret; + netif_napi_add_tx(dev->mt76.tx_napi_dev, &dev->mt76.tx_napi, mt7996_poll_tx); napi_enable(&dev->mt76.tx_napi); @@ -755,6 +942,10 @@ void mt7996_dma_reset(struct mt7996_dev *dev, bool force) mt76_tx_status_check(&dev->mt76, true); + if (mt7996_has_hwrro(dev) && + !mtk_wed_device_active(&dev->mt76.mmio.wed)) + mt7996_rro_msdu_page_map_free(dev); + /* reset wfsys */ if (force) mt7996_wfsys_reset(dev); @@ -765,6 +956,7 @@ void mt7996_dma_reset(struct mt7996_dev *dev, bool force) if (mtk_wed_device_active(&dev->mt76.mmio.wed)) mtk_wed_device_dma_reset(&dev->mt76.mmio.wed); + mt76_npu_disable_irqs(&dev->mt76); mt7996_dma_disable(dev, force); mt76_wed_dma_reset(&dev->mt76); @@ -778,21 +970,32 @@ void mt7996_dma_reset(struct mt7996_dev *dev, bool force) } for (i = 0; i < __MT_MCUQ_MAX; i++) - mt76_queue_reset(dev, dev->mt76.q_mcu[i]); + mt76_queue_reset(dev, dev->mt76.q_mcu[i], true); mt76_for_each_q_rx(&dev->mt76, i) { - if (mtk_wed_device_active(&dev->mt76.mmio.wed)) - if (mt76_queue_is_wed_rro(&dev->mt76.q_rx[i]) || - mt76_queue_is_wed_tx_free(&dev->mt76.q_rx[i])) - continue; + struct mt76_queue *q = &dev->mt76.q_rx[i]; - mt76_queue_reset(dev, &dev->mt76.q_rx[i]); + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) { + if (mt76_queue_is_wed_rro(q) || + mt76_queue_is_wed_tx_free(q)) { + if (force && mt76_queue_is_wed_rro_data(q)) + mt76_queue_reset(dev, q, false); + continue; + } + } + mt76_queue_reset(dev, q, true); } mt76_tx_status_check(&dev->mt76, true); - mt76_for_each_q_rx(&dev->mt76, i) + mt76_for_each_q_rx(&dev->mt76, i) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && force && + (mt76_queue_is_wed_rro_ind(&dev->mt76.q_rx[i]) || + mt76_queue_is_wed_rro_msdu_pg(&dev->mt76.q_rx[i]))) + continue; + mt76_queue_rx_reset(dev, i); + } mt7996_dma_enable(dev, !force); } diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.c b/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.c index 6354173427a5..3d165202a5fd 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -338,9 +338,8 @@ int mt7996_eeprom_init(struct mt7996_dev *dev) #elif defined(__FreeBSD__) memcpy(dev->mphy.macaddr, (u8 *)dev->mt76.eeprom.data + MT_EE_MAC_ADDR, ETH_ALEN); #endif - mt76_eeprom_override(&dev->mphy); - return 0; + return mt76_eeprom_override(&dev->mphy); } int mt7996_eeprom_get_target_power(struct mt7996_dev *dev, diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.h b/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.h index 7a771ca2434c..9e6f0e04caf9 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.h +++ b/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2022 MediaTek Inc. */ diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/init.c b/sys/contrib/dev/mediatek/mt76/mt7996/init.c index c0c827e1a80b..34f1df207f19 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/init.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/init.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -66,6 +66,33 @@ static const struct ieee80211_iface_combination if_comb = { .beacon_int_min_gcd = 100, }; +static const u8 if_types_ext_capa_ap[] = { + [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING, + [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT, + [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF, +}; + +static const struct wiphy_iftype_ext_capab iftypes_ext_capa[] = { + { + .iftype = NL80211_IFTYPE_AP, + .extended_capabilities = if_types_ext_capa_ap, + .extended_capabilities_mask = if_types_ext_capa_ap, + .extended_capabilities_len = sizeof(if_types_ext_capa_ap), + .eml_capabilities = IEEE80211_EML_CAP_EMLSR_SUPP, + .mld_capa_and_ops = + FIELD_PREP_CONST(IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS, + MT7996_MAX_RADIOS - 1), + }, { + .iftype = NL80211_IFTYPE_STATION, + .extended_capabilities = if_types_ext_capa_ap, + .extended_capabilities_mask = if_types_ext_capa_ap, + .extended_capabilities_len = sizeof(if_types_ext_capa_ap), + .mld_capa_and_ops = + FIELD_PREP_CONST(IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS, + MT7996_MAX_RADIOS - 1), + }, +}; + #if defined(CONFIG_HWMON) static ssize_t mt7996_thermal_temp_show(struct device *dev, struct device_attribute *attr, @@ -388,6 +415,7 @@ mt7996_init_wiphy_band(struct ieee80211_hw *hw, struct mt7996_phy *phy) phy->slottime = 9; phy->beacon_rate = -1; + phy->rxfilter = MT_WF_RFCR_DROP_OTHER_UC; if (phy->mt76->cap.has_2ghz) { phy->mt76->sband_2g.sband.ht_cap.cap |= @@ -452,7 +480,7 @@ mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed) hw->max_tx_aggregation_subframes = 512; hw->netdev_features = NETIF_F_RXCSUM; - if (mtk_wed_device_active(wed)) + if (mtk_wed_device_active(wed) || mt76_npu_device_active(mdev)) hw->netdev_features |= NETIF_F_HW_TC; hw->radiotap_timestamp.units_pos = @@ -468,8 +496,11 @@ mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed) wiphy->radio = dev->radios; wiphy->reg_notifier = mt7996_regd_notifier; - wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH; + wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH | + WIPHY_FLAG_SUPPORTS_MLO; wiphy->mbssid_max_interfaces = 16; + wiphy->iftype_ext_capab = iftypes_ext_capa; + wiphy->num_iftype_ext_capab = ARRAY_SIZE(iftypes_ext_capa); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BSS_COLOR); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_VHT_IBSS); @@ -584,19 +615,21 @@ void mt7996_mac_init(struct mt7996_dev *dev) } /* rro module init */ - if (is_mt7996(&dev->mt76)) - mt7996_mcu_set_rro(dev, UNI_RRO_SET_PLATFORM_TYPE, 2); - else + if (dev->hif2) mt7996_mcu_set_rro(dev, UNI_RRO_SET_PLATFORM_TYPE, - dev->hif2 ? 7 : 0); + is_mt7996(&dev->mt76) ? 2 : 7); + else + mt7996_mcu_set_rro(dev, UNI_RRO_SET_PLATFORM_TYPE, 0); - if (dev->has_rro) { + if (mt7996_has_hwrro(dev)) { u16 timeout; timeout = mt76_rr(dev, MT_HW_REV) == MT_HW_REV1 ? 512 : 128; mt7996_mcu_set_rro(dev, UNI_RRO_SET_FLUSH_TIMEOUT, timeout); - mt7996_mcu_set_rro(dev, UNI_RRO_SET_BYPASS_MODE, 1); - mt7996_mcu_set_rro(dev, UNI_RRO_SET_TXFREE_PATH, 0); + mt7996_mcu_set_rro(dev, UNI_RRO_SET_BYPASS_MODE, + is_mt7996(&dev->mt76) ? 1 : 2); + mt7996_mcu_set_rro(dev, UNI_RRO_SET_TXFREE_PATH, + !is_mt7996(&dev->mt76)); } else { mt7996_mcu_set_rro(dev, UNI_RRO_SET_BYPASS_MODE, 3); mt7996_mcu_set_rro(dev, UNI_RRO_SET_TXFREE_PATH, 1); @@ -643,7 +676,9 @@ static int mt7996_register_phy(struct mt7996_dev *dev, enum mt76_band_id band) if (!mt7996_band_valid(dev, band)) return 0; - if (is_mt7996(&dev->mt76) && band == MT_BAND2 && dev->hif2) { + if (dev->hif2 && + ((is_mt7996(&dev->mt76) && band == MT_BAND2) || + (is_mt7992(&dev->mt76) && band == MT_BAND1))) { hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); wed = &dev->mt76.mmio.wed_hif2; } @@ -684,17 +719,26 @@ static int mt7996_register_phy(struct mt7996_dev *dev, enum mt76_band_id band) if (band == MT_BAND2) mphy->macaddr[0] ^= BIT(6); } - mt76_eeprom_override(mphy); + ret = mt76_eeprom_override(mphy); + if (ret) + goto error; /* init wiphy according to mphy and phy */ mt7996_init_wiphy_band(mphy->hw, phy); - ret = mt7996_init_tx_queues(mphy->priv, - MT_TXQ_ID(band), - MT7996_TX_RING_SIZE, - MT_TXQ_RING_BASE(band) + hif1_ofs, - wed); - if (ret) - goto error; + + if (is_mt7996(&dev->mt76) && !dev->hif2 && band == MT_BAND1) { + int i; + + for (i = 0; i <= MT_TXQ_PSD; i++) + mphy->q_tx[i] = dev->mt76.phys[MT_BAND0]->q_tx[0]; + } else { + ret = mt7996_init_tx_queues(mphy->priv, MT_TXQ_ID(band), + MT7996_TX_RING_SIZE, + MT_TXQ_RING_BASE(band) + hif1_ofs, + wed); + if (ret) + goto error; + } ret = mt76_register_phy(mphy, true, mt76_rates, ARRAY_SIZE(mt76_rates)); @@ -702,10 +746,9 @@ static int mt7996_register_phy(struct mt7996_dev *dev, enum mt76_band_id band) goto error; if (wed == &dev->mt76.mmio.wed_hif2 && mtk_wed_device_active(wed)) { - u32 irq_mask = dev->mt76.mmio.irqmask | MT_INT_TX_DONE_BAND2; - - mt76_wr(dev, MT_INT1_MASK_CSR, irq_mask); - mtk_wed_device_start(&dev->mt76.mmio.wed_hif2, irq_mask); + mt76_wr(dev, MT_INT_PCIE1_MASK_CSR, MT_INT_TX_RX_DONE_EXT); + mtk_wed_device_start(&dev->mt76.mmio.wed_hif2, + MT_INT_TX_RX_DONE_EXT); } return 0; @@ -743,30 +786,152 @@ void mt7996_wfsys_reset(struct mt7996_dev *dev) msleep(20); } -static int mt7996_wed_rro_init(struct mt7996_dev *dev) +static void mt7996_rro_hw_init_v3(struct mt7996_dev *dev) { -#ifdef CONFIG_NET_MEDIATEK_SOC_WED struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + u32 session_id; + + if (dev->mt76.hwrro_mode == MT76_HWRRO_V3_1) + return; + +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed)) { + wed->wlan.ind_cmd.win_size = + ffs(MT7996_RRO_WINDOW_MAX_LEN) - 6; + if (is_mt7996(&dev->mt76)) + wed->wlan.ind_cmd.particular_sid = + MT7996_RRO_MAX_SESSION; + else + wed->wlan.ind_cmd.particular_sid = 1; + wed->wlan.ind_cmd.particular_se_phys = + dev->wed_rro.session.phy_addr; + wed->wlan.ind_cmd.se_group_nums = MT7996_RRO_ADDR_ELEM_LEN; + wed->wlan.ind_cmd.ack_sn_addr = MT_RRO_ACK_SN_CTRL; + } +#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ + + if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed)) { + mt76_wr(dev, MT_RRO_IND_CMD_SIGNATURE_BASE0, 0x15010e00); + mt76_set(dev, MT_RRO_IND_CMD_SIGNATURE_BASE1, + MT_RRO_IND_CMD_SIGNATURE_BASE1_EN); + } else { + mt76_wr(dev, MT_RRO_IND_CMD_SIGNATURE_BASE0, 0); + mt76_wr(dev, MT_RRO_IND_CMD_SIGNATURE_BASE1, 0); + } + + /* particular session configure */ + /* use max session idx + 1 as particular session id */ + mt76_wr(dev, MT_RRO_PARTICULAR_CFG0, dev->wed_rro.session.phy_addr); + + session_id = is_mt7996(&dev->mt76) ? MT7996_RRO_MAX_SESSION : 1; + mt76_wr(dev, MT_RRO_PARTICULAR_CFG1, + MT_RRO_PARTICULAR_CONFG_EN | + FIELD_PREP(MT_RRO_PARTICULAR_SID, session_id)); +} + +void mt7996_rro_hw_init(struct mt7996_dev *dev) +{ u32 reg = MT_RRO_ADDR_ELEM_SEG_ADDR0; + int i; + + if (!mt7996_has_hwrro(dev)) + return; + + INIT_LIST_HEAD(&dev->wed_rro.page_cache); + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.page_map); i++) + INIT_LIST_HEAD(&dev->wed_rro.page_map[i]); + + if (!is_mt7996(&dev->mt76)) { + reg = MT_RRO_MSDU_PG_SEG_ADDR0; + + if (dev->mt76.hwrro_mode == MT76_HWRRO_V3_1) { + mt76_clear(dev, MT_RRO_3_0_EMU_CONF, + MT_RRO_3_0_EMU_CONF_EN_MASK); + mt76_set(dev, MT_RRO_3_1_GLOBAL_CONFIG, + MT_RRO_3_1_GLOBAL_CONFIG_RXDMAD_SEL); + if (!mtk_wed_device_active(&dev->mt76.mmio.wed) && + !mt76_npu_device_active(&dev->mt76)) { + mt76_set(dev, MT_RRO_3_1_GLOBAL_CONFIG, + MT_RRO_3_1_GLOBAL_CONFIG_RX_DIDX_WR_EN | + MT_RRO_3_1_GLOBAL_CONFIG_RX_CIDX_RD_EN); + mt76_wr(dev, MT_RRO_RX_RING_AP_CIDX_ADDR, + dev->wed_rro.emi_rings_cpu.phy_addr >> 4); + mt76_wr(dev, MT_RRO_RX_RING_AP_DIDX_ADDR, + dev->wed_rro.emi_rings_dma.phy_addr >> 4); + } + } else { + /* set emul 3.0 function */ + mt76_wr(dev, MT_RRO_3_0_EMU_CONF, + MT_RRO_3_0_EMU_CONF_EN_MASK); + + mt76_wr(dev, MT_RRO_ADDR_ARRAY_BASE0, + dev->wed_rro.addr_elem[0].phy_addr); + } + + mt76_set(dev, MT_RRO_3_1_GLOBAL_CONFIG, + MT_RRO_3_1_GLOBAL_CONFIG_INTERLEAVE_EN); + + /* setup Msdu page address */ + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.msdu_pg); i++) { + mt76_wr(dev, reg, + dev->wed_rro.msdu_pg[i].phy_addr >> 4); + reg += 4; + } + } else { + /* TODO: remove line after WM has set */ + mt76_clear(dev, WF_RRO_AXI_MST_CFG, + WF_RRO_AXI_MST_CFG_DIDX_OK); + + /* setup BA bitmap cache address */ + mt76_wr(dev, MT_RRO_BA_BITMAP_BASE0, + dev->wed_rro.ba_bitmap[0].phy_addr); + mt76_wr(dev, MT_RRO_BA_BITMAP_BASE1, 0); + mt76_wr(dev, MT_RRO_BA_BITMAP_BASE_EXT0, + dev->wed_rro.ba_bitmap[1].phy_addr); + mt76_wr(dev, MT_RRO_BA_BITMAP_BASE_EXT1, 0); + + /* Setup Address element address */ + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.addr_elem); i++) { + mt76_wr(dev, reg, + dev->wed_rro.addr_elem[i].phy_addr >> 4); + reg += 4; + } + + /* Setup Address element address - separate address segment + * mode. + */ + mt76_wr(dev, MT_RRO_ADDR_ARRAY_BASE1, + MT_RRO_ADDR_ARRAY_ELEM_ADDR_SEG_MODE); + } + + mt7996_rro_hw_init_v3(dev); + + /* interrupt enable */ + mt76_wr(dev, MT_RRO_HOST_INT_ENA, + MT_RRO_HOST_INT_ENA_HOST_RRO_DONE_ENA); +} + +static int mt7996_wed_rro_init(struct mt7996_dev *dev) +{ + u32 val = FIELD_PREP(WED_RRO_ADDR_SIGNATURE_MASK, 0xff); struct mt7996_wed_rro_addr *addr; void *ptr; int i; - if (!dev->has_rro) + if (!mt7996_has_hwrro(dev)) return 0; - if (!mtk_wed_device_active(wed)) - return 0; - - for (i = 0; i < ARRAY_SIZE(dev->wed_rro.ba_bitmap); i++) { - ptr = dmam_alloc_coherent(dev->mt76.dma_dev, - MT7996_RRO_BA_BITMAP_CR_SIZE, - &dev->wed_rro.ba_bitmap[i].phy_addr, - GFP_KERNEL); - if (!ptr) - return -ENOMEM; + if (dev->mt76.hwrro_mode == MT76_HWRRO_V3) { + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.ba_bitmap); i++) { + ptr = dmam_alloc_coherent(dev->mt76.dma_dev, + MT7996_RRO_BA_BITMAP_CR_SIZE, + &dev->wed_rro.ba_bitmap[i].phy_addr, + GFP_KERNEL); + if (!ptr) + return -ENOMEM; - dev->wed_rro.ba_bitmap[i].ptr = ptr; + dev->wed_rro.ba_bitmap[i].ptr = ptr; + } } for (i = 0; i < ARRAY_SIZE(dev->wed_rro.addr_elem); i++) { @@ -785,12 +950,54 @@ static int mt7996_wed_rro_init(struct mt7996_dev *dev) addr = dev->wed_rro.addr_elem[i].ptr; for (j = 0; j < MT7996_RRO_WINDOW_MAX_SIZE; j++) { - addr->signature = 0xff; + addr->data = cpu_to_le32(val); addr++; } - wed->wlan.ind_cmd.addr_elem_phys[i] = - dev->wed_rro.addr_elem[i].phy_addr; +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && + mtk_wed_get_rx_capa(&dev->mt76.mmio.wed)) { + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + + wed->wlan.ind_cmd.addr_elem_phys[i] = + dev->wed_rro.addr_elem[i].phy_addr; + } +#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ + } + + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.msdu_pg); i++) { + ptr = dmam_alloc_coherent(dev->mt76.dma_dev, + MT7996_RRO_MSDU_PG_SIZE_PER_CR, + &dev->wed_rro.msdu_pg[i].phy_addr, + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + dev->wed_rro.msdu_pg[i].ptr = ptr; + + memset(dev->wed_rro.msdu_pg[i].ptr, 0, + MT7996_RRO_MSDU_PG_SIZE_PER_CR); + } + + if (!mtk_wed_device_active(&dev->mt76.mmio.wed) && + dev->mt76.hwrro_mode == MT76_HWRRO_V3_1) { + ptr = dmam_alloc_coherent(dev->mt76.dma_dev, + sizeof(*dev->wed_rro.emi_rings_cpu.ptr), + &dev->wed_rro.emi_rings_cpu.phy_addr, + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + dev->wed_rro.emi_rings_cpu.ptr = ptr; + + ptr = dmam_alloc_coherent(dev->mt76.dma_dev, + sizeof(*dev->wed_rro.emi_rings_dma.ptr), + &dev->wed_rro.emi_rings_dma.phy_addr, + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + dev->wed_rro.emi_rings_dma.ptr = ptr; } ptr = dmam_alloc_coherent(dev->mt76.dma_dev, @@ -803,69 +1010,20 @@ static int mt7996_wed_rro_init(struct mt7996_dev *dev) dev->wed_rro.session.ptr = ptr; addr = dev->wed_rro.session.ptr; for (i = 0; i < MT7996_RRO_WINDOW_MAX_LEN; i++) { - addr->signature = 0xff; + addr->data = cpu_to_le32(val); addr++; } - /* rro hw init */ - /* TODO: remove line after WM has set */ - mt76_clear(dev, WF_RRO_AXI_MST_CFG, WF_RRO_AXI_MST_CFG_DIDX_OK); - - /* setup BA bitmap cache address */ - mt76_wr(dev, MT_RRO_BA_BITMAP_BASE0, - dev->wed_rro.ba_bitmap[0].phy_addr); - mt76_wr(dev, MT_RRO_BA_BITMAP_BASE1, 0); - mt76_wr(dev, MT_RRO_BA_BITMAP_BASE_EXT0, - dev->wed_rro.ba_bitmap[1].phy_addr); - mt76_wr(dev, MT_RRO_BA_BITMAP_BASE_EXT1, 0); + mt7996_rro_hw_init(dev); - /* setup Address element address */ - for (i = 0; i < ARRAY_SIZE(dev->wed_rro.addr_elem); i++) { - mt76_wr(dev, reg, dev->wed_rro.addr_elem[i].phy_addr >> 4); - reg += 4; - } - - /* setup Address element address - separate address segment mode */ - mt76_wr(dev, MT_RRO_ADDR_ARRAY_BASE1, - MT_RRO_ADDR_ARRAY_ELEM_ADDR_SEG_MODE); - - wed->wlan.ind_cmd.win_size = ffs(MT7996_RRO_WINDOW_MAX_LEN) - 6; - wed->wlan.ind_cmd.particular_sid = MT7996_RRO_MAX_SESSION; - wed->wlan.ind_cmd.particular_se_phys = dev->wed_rro.session.phy_addr; - wed->wlan.ind_cmd.se_group_nums = MT7996_RRO_ADDR_ELEM_LEN; - wed->wlan.ind_cmd.ack_sn_addr = MT_RRO_ACK_SN_CTRL; - - mt76_wr(dev, MT_RRO_IND_CMD_SIGNATURE_BASE0, 0x15010e00); - mt76_set(dev, MT_RRO_IND_CMD_SIGNATURE_BASE1, - MT_RRO_IND_CMD_SIGNATURE_BASE1_EN); - - /* particular session configure */ - /* use max session idx + 1 as particular session id */ - mt76_wr(dev, MT_RRO_PARTICULAR_CFG0, dev->wed_rro.session.phy_addr); - mt76_wr(dev, MT_RRO_PARTICULAR_CFG1, - MT_RRO_PARTICULAR_CONFG_EN | - FIELD_PREP(MT_RRO_PARTICULAR_SID, MT7996_RRO_MAX_SESSION)); - - /* interrupt enable */ - mt76_wr(dev, MT_RRO_HOST_INT_ENA, - MT_RRO_HOST_INT_ENA_HOST_RRO_DONE_ENA); - - /* rro ind cmd queue init */ return mt7996_dma_rro_init(dev); -#else - return 0; -#endif } static void mt7996_wed_rro_free(struct mt7996_dev *dev) { -#ifdef CONFIG_NET_MEDIATEK_SOC_WED int i; - if (!dev->has_rro) - return; - - if (!mtk_wed_device_active(&dev->mt76.mmio.wed)) + if (!mt7996_has_hwrro(dev)) return; for (i = 0; i < ARRAY_SIZE(dev->wed_rro.ba_bitmap); i++) { @@ -889,6 +1047,28 @@ static void mt7996_wed_rro_free(struct mt7996_dev *dev) dev->wed_rro.addr_elem[i].phy_addr); } + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.msdu_pg); i++) { + if (!dev->wed_rro.msdu_pg[i].ptr) + continue; + + dmam_free_coherent(dev->mt76.dma_dev, + MT7996_RRO_MSDU_PG_SIZE_PER_CR, + dev->wed_rro.msdu_pg[i].ptr, + dev->wed_rro.msdu_pg[i].phy_addr); + } + + if (dev->wed_rro.emi_rings_cpu.ptr) + dmam_free_coherent(dev->mt76.dma_dev, + sizeof(*dev->wed_rro.emi_rings_cpu.ptr), + dev->wed_rro.emi_rings_cpu.ptr, + dev->wed_rro.emi_rings_cpu.phy_addr); + + if (dev->wed_rro.emi_rings_dma.ptr) + dmam_free_coherent(dev->mt76.dma_dev, + sizeof(*dev->wed_rro.emi_rings_dma.ptr), + dev->wed_rro.emi_rings_dma.ptr, + dev->wed_rro.emi_rings_dma.phy_addr); + if (!dev->wed_rro.session.ptr) return; @@ -897,12 +1077,11 @@ static void mt7996_wed_rro_free(struct mt7996_dev *dev) sizeof(struct mt7996_wed_rro_addr), dev->wed_rro.session.ptr, dev->wed_rro.session.phy_addr); -#endif } static void mt7996_wed_rro_work(struct work_struct *work) { -#ifdef CONFIG_NET_MEDIATEK_SOC_WED + u32 val = FIELD_PREP(WED_RRO_ADDR_SIGNATURE_MASK, 0xff); struct mt7996_dev *dev; LIST_HEAD(list); @@ -921,8 +1100,15 @@ static void mt7996_wed_rro_work(struct work_struct *work) list); list_del_init(&e->list); + if (mt76_npu_device_active(&dev->mt76)) + goto reset_session; + for (i = 0; i < MT7996_RRO_WINDOW_MAX_LEN; i++) { +#if defined(__linux__) void *ptr = dev->wed_rro.session.ptr; +#elif defined(__FreeBSD__) + u8 *ptr = dev->wed_rro.session.ptr; +#endif struct mt7996_wed_rro_addr *elem; u32 idx, elem_id = i; @@ -938,14 +1124,18 @@ static void mt7996_wed_rro_work(struct work_struct *work) (e->id % MT7996_RRO_BA_BITMAP_SESSION_SIZE) * MT7996_RRO_WINDOW_MAX_LEN; reset: +#if defined(__linux__) elem = ptr + elem_id * sizeof(*elem); - elem->signature = 0xff; +#elif defined(__FreeBSD__) + elem = (void *)(ptr + elem_id * sizeof(*elem)); +#endif + elem->data |= cpu_to_le32(val); } +reset_session: mt7996_mcu_wed_rro_reset_sessions(dev, e->id); out: kfree(e); } -#endif } static int mt7996_variant_type_init(struct mt7996_dev *dev) @@ -1340,7 +1530,6 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band, eht_cap->has_eht = true; eht_cap_elem->mac_cap_info[0] = - IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | IEEE80211_EHT_MAC_CAP0_OM_CONTROL | u8_encode_bits(IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_11454, IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK); @@ -1532,6 +1721,10 @@ int mt7996_register_device(struct mt7996_dev *dev) if (ret) return ret; + ret = mt7996_npu_hw_init(dev); + if (ret) + return ret; + ret = mt76_register_device(&dev->mt76, true, mt76_rates, ARRAY_SIZE(mt76_rates)); if (ret) @@ -1578,6 +1771,9 @@ void mt7996_unregister_device(struct mt7996_dev *dev) mt7996_mcu_exit(dev); mt7996_tx_token_put(dev); mt7996_dma_cleanup(dev); + if (mt7996_has_hwrro(dev) && + !mtk_wed_device_active(&dev->mt76.mmio.wed)) + mt7996_rro_msdu_page_map_free(dev); tasklet_disable(&dev->mt76.irq_tasklet); mt76_free_device(&dev->mt76); diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mac.c b/sys/contrib/dev/mediatek/mt76/mt7996/mac.c index 13ea5b55d619..b884f4475021 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/mac.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/mac.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -11,6 +11,7 @@ #include "mac.h" #include "mcu.h" #if defined(__FreeBSD__) +#include <linux/cache.h> #include <linux/delay.h> #endif @@ -232,7 +233,9 @@ static int mt7996_reverse_frag0_hdr_trans(struct sk_buff *skb, u16 hdr_gap) { struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; struct ethhdr *eth_hdr = (struct ethhdr *)(skb->data + hdr_gap); - struct mt7996_sta *msta = (struct mt7996_sta *)status->wcid; + struct mt7996_sta_link *msta_link = (void *)status->wcid; + struct mt7996_sta *msta = msta_link->sta; + struct ieee80211_bss_conf *link_conf; __le32 *rxd = (__le32 *)skb->data; struct ieee80211_sta *sta; struct ieee80211_vif *vif; @@ -249,8 +252,11 @@ static int mt7996_reverse_frag0_hdr_trans(struct sk_buff *skb, u16 hdr_gap) if (!msta || !msta->vif) return -EINVAL; - sta = container_of((void *)msta, struct ieee80211_sta, drv_priv); + sta = wcid_to_sta(status->wcid); vif = container_of((void *)msta->vif, struct ieee80211_vif, drv_priv); + link_conf = rcu_dereference(vif->link_conf[msta_link->wcid.link_id]); + if (!link_conf) + return -EINVAL; /* store the info from RXD and ethhdr to avoid being overridden */ frame_control = le32_get_bits(rxd[8], MT_RXD8_FRAME_CONTROL); @@ -263,7 +269,7 @@ static int mt7996_reverse_frag0_hdr_trans(struct sk_buff *skb, u16 hdr_gap) switch (frame_control & (IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS)) { case 0: - ether_addr_copy(hdr.addr3, vif->bss_conf.bssid); + ether_addr_copy(hdr.addr3, link_conf->bssid); break; case IEEE80211_FCTL_FROMDS: ether_addr_copy(hdr.addr3, eth_hdr->h_source); @@ -716,6 +722,7 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q, status->flag |= RX_FLAG_8023; mt7996_wed_check_ppe(dev, &dev->mt76.q_rx[q], msta, skb, *info); + mt76_npu_check_ppe(&dev->mt76, skb, *info); } if (rxv && !(status->flag & RX_FLAG_8023)) { @@ -792,6 +799,7 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK; __le16 fc = hdr->frame_control, sc = hdr->seq_ctrl; u16 seqno = le16_to_cpu(sc); + bool hw_bigtk = false; u8 fc_type, fc_stype; u32 val; @@ -800,6 +808,9 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) { if (is_mt7990(&dev->mt76)) txwi[6] |= cpu_to_le32(FIELD_PREP(MT_TXD6_TID_ADDBA, tid)); + else + txwi[7] |= cpu_to_le32(MT_TXD7_MAC_TXD); + tid = MT_TX_ADDBA; } else if (ieee80211_is_mgmt(hdr->frame_control)) { tid = MT_TX_NORMAL; @@ -814,7 +825,11 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, info->flags & IEEE80211_TX_CTL_USE_MINRATE) val |= MT_TXD1_FIXED_RATE; - if (key && multicast && ieee80211_is_robust_mgmt_frame(skb)) { + if (is_mt7990(&dev->mt76) && ieee80211_is_beacon(fc) && + (wcid->hw_key_idx2 == 6 || wcid->hw_key_idx2 == 7)) + hw_bigtk = true; + + if ((key && multicast && ieee80211_is_robust_mgmt_frame(skb)) || hw_bigtk) { val |= MT_TXD1_BIP; txwi[3] &= ~cpu_to_le32(MT_TXD3_PROTECT_FRAME); } @@ -963,8 +978,9 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, val |= MT_TXD5_TX_STATUS_HOST; txwi[5] = cpu_to_le32(val); - val = MT_TXD6_DAS; - if (q_idx >= MT_LMAC_ALTX0 && q_idx <= MT_LMAC_BCN0) + val = MT_TXD6_DAS | MT_TXD6_VTA; + if ((q_idx >= MT_LMAC_ALTX0 && q_idx <= MT_LMAC_BCN0) || + skb->protocol == cpu_to_be16(ETH_P_PAE)) val |= MT_TXD6_DIS_MAT; if (is_mt7996(&dev->mt76)) @@ -1028,15 +1044,20 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, struct ieee80211_sta *sta, struct mt76_tx_info *tx_info) { + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)tx_info->skb->data; struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_info->skb); struct ieee80211_key_conf *key = info->control.hw_key; struct ieee80211_vif *vif = info->control.vif; - struct mt76_connac_txp_common *txp; + struct mt7996_vif *mvif = vif ? (struct mt7996_vif *)vif->drv_priv : NULL; + struct mt7996_sta *msta = sta ? (struct mt7996_sta *)sta->drv_priv : NULL; + struct mt76_vif_link *mlink = NULL; struct mt76_txwi_cache *t; int id, i, pid, nbuf = tx_info->nbuf - 1; bool is_8023 = info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP; + __le32 *ptr = (__le32 *)txwi_ptr; u8 *txwi = (u8 *)txwi_ptr; + u8 link_id; if (unlikely(tx_info->skb->len <= ETH_HLEN)) return -EINVAL; @@ -1044,6 +1065,30 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, if (!wcid) wcid = &dev->mt76.global_wcid; + if ((is_8023 || ieee80211_is_data_qos(hdr->frame_control)) && sta->mlo && + likely(tx_info->skb->protocol != cpu_to_be16(ETH_P_PAE))) { + u8 tid = tx_info->skb->priority & IEEE80211_QOS_CTL_TID_MASK; + + link_id = (tid % 2) ? msta->seclink_id : msta->deflink_id; + } else { + link_id = u32_get_bits(info->control.flags, + IEEE80211_TX_CTRL_MLO_LINK); + } + + if (link_id != wcid->link_id && link_id != IEEE80211_LINK_UNSPECIFIED) { + if (msta) { + struct mt7996_sta_link *msta_link = + rcu_dereference(msta->link[link_id]); + + if (msta_link) + wcid = &msta_link->wcid; + } else if (mvif) { + mlink = rcu_dereference(mvif->mt76.link[link_id]); + if (mlink && mlink->wcid) + wcid = mlink->wcid; + } + } + t = (struct mt76_txwi_cache *)(txwi + mdev->drv->txwi_size); t->skb = tx_info->skb; @@ -1051,6 +1096,41 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, if (id < 0) return id; + /* Since the rules of HW MLD address translation are not fully + * compatible with 802.11 EAPOL frame, we do the translation by + * software + */ + if (tx_info->skb->protocol == cpu_to_be16(ETH_P_PAE) && sta->mlo) { + struct ieee80211_hdr *hdr = (void *)tx_info->skb->data; + struct ieee80211_bss_conf *link_conf; + struct ieee80211_link_sta *link_sta; + + link_conf = rcu_dereference(vif->link_conf[wcid->link_id]); + if (!link_conf) + return -EINVAL; + + link_sta = rcu_dereference(sta->link[wcid->link_id]); + if (!link_sta) + return -EINVAL; + + dma_sync_single_for_cpu(mdev->dma_dev, tx_info->buf[1].addr, + tx_info->buf[1].len, DMA_TO_DEVICE); + + memcpy(hdr->addr1, link_sta->addr, ETH_ALEN); + memcpy(hdr->addr2, link_conf->addr, ETH_ALEN); + if (ieee80211_has_a4(hdr->frame_control)) { + memcpy(hdr->addr3, sta->addr, ETH_ALEN); + memcpy(hdr->addr4, vif->addr, ETH_ALEN); + } else if (ieee80211_has_tods(hdr->frame_control)) { + memcpy(hdr->addr3, sta->addr, ETH_ALEN); + } else if (ieee80211_has_fromds(hdr->frame_control)) { + memcpy(hdr->addr3, vif->addr, ETH_ALEN); + } + + dma_sync_single_for_device(mdev->dma_dev, tx_info->buf[1].addr, + tx_info->buf[1].len, DMA_TO_DEVICE); + } + pid = mt76_tx_status_skb_add(mdev, wcid, tx_info->skb); memset(txwi_ptr, 0, MT_TXD_SIZE); /* Transmit non qos data by 802.11 header and need to fill txd by host*/ @@ -1058,46 +1138,73 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, mt7996_mac_write_txwi(dev, txwi_ptr, tx_info->skb, wcid, key, pid, qid, 0); - txp = (struct mt76_connac_txp_common *)(txwi + MT_TXD_SIZE); - for (i = 0; i < nbuf; i++) { - u16 len; + /* MT7996 and MT7992 require driver to provide the MAC TXP for AddBA + * req + */ + if (le32_to_cpu(ptr[7]) & MT_TXD7_MAC_TXD) { + u32 val; + + ptr = (__le32 *)(txwi + MT_TXD_SIZE); + memset((void *)ptr, 0, sizeof(struct mt76_connac_fw_txp)); + + val = FIELD_PREP(MT_TXP0_TOKEN_ID0, id) | + MT_TXP0_TOKEN_ID0_VALID_MASK; + ptr[0] = cpu_to_le32(val); - len = FIELD_PREP(MT_TXP_BUF_LEN, tx_info->buf[i + 1].len); + val = FIELD_PREP(MT_TXP1_TID_ADDBA, + tx_info->skb->priority & + IEEE80211_QOS_CTL_TID_MASK); + ptr[1] = cpu_to_le32(val); + ptr[2] = cpu_to_le32(tx_info->buf[1].addr & 0xFFFFFFFF); + + val = FIELD_PREP(MT_TXP_BUF_LEN, tx_info->buf[1].len) | + MT_TXP3_ML0_MASK; #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - len |= FIELD_PREP(MT_TXP_DMA_ADDR_H, - tx_info->buf[i + 1].addr >> 32); + val |= FIELD_PREP(MT_TXP3_DMA_ADDR_H, + tx_info->buf[1].addr >> 32); #endif + ptr[3] = cpu_to_le32(val); + } else { + struct mt76_connac_txp_common *txp; - txp->fw.buf[i] = cpu_to_le32(tx_info->buf[i + 1].addr); - txp->fw.len[i] = cpu_to_le16(len); - } - txp->fw.nbuf = nbuf; + txp = (struct mt76_connac_txp_common *)(txwi + MT_TXD_SIZE); + for (i = 0; i < nbuf; i++) { + u16 len; - txp->fw.flags = cpu_to_le16(MT_CT_INFO_FROM_HOST); + len = FIELD_PREP(MT_TXP_BUF_LEN, tx_info->buf[i + 1].len); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + len |= FIELD_PREP(MT_TXP_DMA_ADDR_H, + tx_info->buf[i + 1].addr >> 32); +#endif - if (!is_8023 || pid >= MT_PACKET_ID_FIRST) - txp->fw.flags |= cpu_to_le16(MT_CT_INFO_APPLY_TXD); + txp->fw.buf[i] = cpu_to_le32(tx_info->buf[i + 1].addr); + txp->fw.len[i] = cpu_to_le16(len); + } + txp->fw.nbuf = nbuf; - if (!key) - txp->fw.flags |= cpu_to_le16(MT_CT_INFO_NONE_CIPHER_FRAME); + txp->fw.flags = cpu_to_le16(MT_CT_INFO_FROM_HOST); - if (!is_8023 && mt7996_tx_use_mgmt(dev, tx_info->skb)) - txp->fw.flags |= cpu_to_le16(MT_CT_INFO_MGMT_FRAME); + if (!is_8023 || pid >= MT_PACKET_ID_FIRST) + txp->fw.flags |= cpu_to_le16(MT_CT_INFO_APPLY_TXD); - if (vif) { - struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; - struct mt76_vif_link *mlink = NULL; + if (!key) + txp->fw.flags |= cpu_to_le16(MT_CT_INFO_NONE_CIPHER_FRAME); - if (wcid->offchannel) - mlink = rcu_dereference(mvif->mt76.offchannel_link); - if (!mlink) - mlink = rcu_dereference(mvif->mt76.link[wcid->link_id]); + if (!is_8023 && mt7996_tx_use_mgmt(dev, tx_info->skb)) + txp->fw.flags |= cpu_to_le16(MT_CT_INFO_MGMT_FRAME); - txp->fw.bss_idx = mlink ? mlink->idx : mvif->deflink.mt76.idx; - } + if (mvif) { + if (wcid->offchannel) + mlink = rcu_dereference(mvif->mt76.offchannel_link); + if (!mlink) + mlink = rcu_dereference(mvif->mt76.link[wcid->link_id]); - txp->fw.token = cpu_to_le16(id); - txp->fw.rept_wds_wcid = cpu_to_le16(sta ? wcid->idx : 0xfff); + txp->fw.bss_idx = mlink ? mlink->idx : mvif->deflink.mt76.idx; + } + + txp->fw.token = cpu_to_le16(id); + txp->fw.rept_wds_wcid = cpu_to_le16(sta ? wcid->idx : 0xfff); + } tx_info->skb = NULL; @@ -1190,8 +1297,14 @@ mt7996_txwi_free(struct mt7996_dev *dev, struct mt76_txwi_cache *t, txwi = (__le32 *)mt76_get_txwi_ptr(mdev, t); if (link_sta) { wcid_idx = wcid->idx; - if (likely(t->skb->protocol != cpu_to_be16(ETH_P_PAE))) - mt7996_tx_check_aggr(link_sta, wcid, t->skb); + if (likely(t->skb->protocol != cpu_to_be16(ETH_P_PAE))) { + struct mt7996_sta *msta; + + /* AMPDU state is stored in the primary link */ + msta = (void *)link_sta->sta->drv_priv; + mt7996_tx_check_aggr(link_sta, &msta->deflink.wcid, + t->skb); + } } else { wcid_idx = le32_get_bits(txwi[9], MT_TXD9_WLAN_IDX); } @@ -1253,6 +1366,9 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len) info = le32_to_cpu(*cur_info); if (info & MT_TXFREE_INFO_PAIR) { struct ieee80211_sta *sta; + unsigned long valid_links; + struct mt7996_sta *msta; + unsigned int id; u16 idx; idx = FIELD_GET(MT_TXFREE_INFO_WLAN_ID, info); @@ -1267,7 +1383,21 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len) if (!link_sta) goto next; - mt76_wcid_add_poll(&dev->mt76, wcid); + msta = (struct mt7996_sta *)sta->drv_priv; + valid_links = sta->valid_links ?: BIT(0); + + /* For MLD STA, add all link's wcid to sta_poll_list */ + for_each_set_bit(id, &valid_links, + IEEE80211_MLD_MAX_NUM_LINKS) { + struct mt7996_sta_link *msta_link; + + msta_link = rcu_dereference(msta->link[id]); + if (!msta_link) + continue; + + mt76_wcid_add_poll(&dev->mt76, + &msta_link->wcid); + } next: /* ver 7 has a new DW with pair = 1, skip it */ if (ver == 7 && ((void *)(cur_info + 1) < end) && @@ -1589,6 +1719,366 @@ void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, } } +static struct mt7996_msdu_page * +mt7996_msdu_page_get_from_cache(struct mt7996_dev *dev) +{ + struct mt7996_msdu_page *p = NULL; + + spin_lock(&dev->wed_rro.lock); + + if (!list_empty(&dev->wed_rro.page_cache)) { + p = list_first_entry(&dev->wed_rro.page_cache, + struct mt7996_msdu_page, list); + list_del(&p->list); + } + + spin_unlock(&dev->wed_rro.lock); + + return p; +} + +static struct mt7996_msdu_page *mt7996_msdu_page_get(struct mt7996_dev *dev) +{ + struct mt7996_msdu_page *p; + + p = mt7996_msdu_page_get_from_cache(dev); + if (!p) { + p = kzalloc(L1_CACHE_ALIGN(sizeof(*p)), GFP_ATOMIC); + if (p) + INIT_LIST_HEAD(&p->list); + } + + return p; +} + +static void mt7996_msdu_page_put_to_cache(struct mt7996_dev *dev, + struct mt7996_msdu_page *p) +{ + if (p->buf) { + mt76_put_page_pool_buf(p->buf, false); + p->buf = NULL; + } + + spin_lock(&dev->wed_rro.lock); + list_add(&p->list, &dev->wed_rro.page_cache); + spin_unlock(&dev->wed_rro.lock); +} + +static void mt7996_msdu_page_free_cache(struct mt7996_dev *dev) +{ + while (true) { + struct mt7996_msdu_page *p; + + p = mt7996_msdu_page_get_from_cache(dev); + if (!p) + break; + + if (p->buf) + mt76_put_page_pool_buf(p->buf, false); + + kfree(p); + } +} + +static u32 mt7996_msdu_page_hash_from_addr(dma_addr_t dma_addr) +{ + u32 val = 0; + int i = 0; + + while (dma_addr) { + val += (u32)((dma_addr & 0xff) + i) % MT7996_RRO_MSDU_PG_HASH_SIZE; + dma_addr >>= 8; + i += 13; + } + + return val % MT7996_RRO_MSDU_PG_HASH_SIZE; +} + +static struct mt7996_msdu_page * +mt7996_rro_msdu_page_get(struct mt7996_dev *dev, dma_addr_t dma_addr) +{ + u32 hash = mt7996_msdu_page_hash_from_addr(dma_addr); + struct mt7996_msdu_page *p, *tmp, *addr = NULL; + + spin_lock(&dev->wed_rro.lock); + + list_for_each_entry_safe(p, tmp, &dev->wed_rro.page_map[hash], + list) { + if (p->dma_addr == dma_addr) { + list_del(&p->list); + addr = p; + break; + } + } + + spin_unlock(&dev->wed_rro.lock); + + return addr; +} + +static void mt7996_rx_token_put(struct mt7996_dev *dev) +{ + int i; + + for (i = 0; i < dev->mt76.rx_token_size; i++) { + struct mt76_txwi_cache *t; + + t = mt76_rx_token_release(&dev->mt76, i); + if (!t || !t->ptr) + continue; + + mt76_put_page_pool_buf(t->ptr, false); + t->dma_addr = 0; + t->ptr = NULL; + + mt76_put_rxwi(&dev->mt76, t); + } +} + +void mt7996_rro_msdu_page_map_free(struct mt7996_dev *dev) +{ + struct mt7996_msdu_page *p, *tmp; + int i; + + local_bh_disable(); + + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.page_map); i++) { + list_for_each_entry_safe(p, tmp, &dev->wed_rro.page_map[i], + list) { + list_del_init(&p->list); + if (p->buf) + mt76_put_page_pool_buf(p->buf, false); + kfree(p); + } + } + mt7996_msdu_page_free_cache(dev); + + local_bh_enable(); + + mt7996_rx_token_put(dev); +} + +int mt7996_rro_msdu_page_add(struct mt76_dev *mdev, struct mt76_queue *q, + dma_addr_t dma_addr, void *data) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + struct mt7996_msdu_page_info *pinfo = data; + struct mt7996_msdu_page *p; + u32 hash; + + pinfo->data |= cpu_to_le32(FIELD_PREP(MSDU_PAGE_INFO_OWNER_MASK, 1)); + p = mt7996_msdu_page_get(dev); + if (!p) + return -ENOMEM; + + p->buf = data; + p->dma_addr = dma_addr; + p->q = q; + + hash = mt7996_msdu_page_hash_from_addr(dma_addr); + + spin_lock(&dev->wed_rro.lock); + list_add_tail(&p->list, &dev->wed_rro.page_map[hash]); + spin_unlock(&dev->wed_rro.lock); + + return 0; +} + +static struct mt7996_wed_rro_addr * +mt7996_rro_addr_elem_get(struct mt7996_dev *dev, u16 session_id, u16 seq_num) +{ + u32 idx = 0; +#if defined(__linux__) + void *addr; +#elif defined(__FreeBSD__) + u8 *addr; +#endif + + if (session_id == MT7996_RRO_MAX_SESSION) { + addr = dev->wed_rro.session.ptr; + } else { + idx = session_id / MT7996_RRO_BA_BITMAP_SESSION_SIZE; + addr = dev->wed_rro.addr_elem[idx].ptr; + + idx = session_id % MT7996_RRO_BA_BITMAP_SESSION_SIZE; + idx = idx * MT7996_RRO_WINDOW_MAX_LEN; + } + idx += seq_num % MT7996_RRO_WINDOW_MAX_LEN; + + return (void *)(addr + idx * sizeof(struct mt7996_wed_rro_addr)); +} + +#define MT996_RRO_SN_MASK GENMASK(11, 0) + +void mt7996_rro_rx_process(struct mt76_dev *mdev, void *data) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + struct mt76_wed_rro_ind *cmd = (struct mt76_wed_rro_ind *)data; + u32 cmd_data0 = le32_to_cpu(cmd->data0); + u32 cmd_data1 = le32_to_cpu(cmd->data1); + u8 ind_reason = FIELD_GET(RRO_IND_DATA0_IND_REASON_MASK, cmd_data0); + u16 start_seq = FIELD_GET(RRO_IND_DATA0_START_SEQ_MASK, cmd_data0); + u16 seq_id = FIELD_GET(RRO_IND_DATA0_SEQ_ID_MASK, cmd_data0); + u16 ind_count = FIELD_GET(RRO_IND_DATA1_IND_COUNT_MASK, cmd_data1); + struct mt7996_msdu_page_info *pinfo = NULL; + struct mt7996_msdu_page *p = NULL; + int i, seq_num = 0; + + for (i = 0; i < ind_count; i++) { + struct mt7996_wed_rro_addr *e; + struct mt76_rx_status *status; + struct mt7996_rro_hif *rxd; + int j, len, qid, data_len; + struct mt76_txwi_cache *t; + dma_addr_t dma_addr = 0; + u16 rx_token_id, count; + struct mt76_queue *q; + struct sk_buff *skb; + u32 info = 0, data; + u8 signature; + void *buf; + bool ls; + + seq_num = FIELD_GET(MT996_RRO_SN_MASK, start_seq + i); + e = mt7996_rro_addr_elem_get(dev, seq_id, seq_num); + data = le32_to_cpu(e->data); + signature = FIELD_GET(WED_RRO_ADDR_SIGNATURE_MASK, data); + if (signature != (seq_num / MT7996_RRO_WINDOW_MAX_LEN)) { + u32 val = FIELD_PREP(WED_RRO_ADDR_SIGNATURE_MASK, + 0xff); + + e->data |= cpu_to_le32(val); + goto update_ack_seq_num; + } + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + dma_addr = FIELD_GET(WED_RRO_ADDR_HEAD_HIGH_MASK, data); + dma_addr <<= 32; +#endif + dma_addr |= le32_to_cpu(e->head_low); + + count = FIELD_GET(WED_RRO_ADDR_COUNT_MASK, data); + for (j = 0; j < count; j++) { + if (!p) { + p = mt7996_rro_msdu_page_get(dev, dma_addr); + if (!p) + continue; + + dma_sync_single_for_cpu(mdev->dma_dev, p->dma_addr, + SKB_WITH_OVERHEAD(p->q->buf_size), + page_pool_get_dma_dir(p->q->page_pool)); + pinfo = (struct mt7996_msdu_page_info *)p->buf; + } + + rxd = &pinfo->rxd[j % MT7996_MAX_HIF_RXD_IN_PG]; + len = FIELD_GET(RRO_HIF_DATA1_SDL_MASK, + le32_to_cpu(rxd->data1)); + + rx_token_id = FIELD_GET(RRO_HIF_DATA4_RX_TOKEN_ID_MASK, + le32_to_cpu(rxd->data4)); + t = mt76_rx_token_release(mdev, rx_token_id); + if (!t) + goto next_page; + + qid = t->qid; + buf = t->ptr; + q = &mdev->q_rx[qid]; + dma_sync_single_for_cpu(mdev->dma_dev, t->dma_addr, + SKB_WITH_OVERHEAD(q->buf_size), + page_pool_get_dma_dir(q->page_pool)); + + t->dma_addr = 0; + t->ptr = NULL; + mt76_put_rxwi(mdev, t); + if (!buf) + goto next_page; + + if (q->rx_head) + data_len = q->buf_size; + else + data_len = SKB_WITH_OVERHEAD(q->buf_size); + + if (data_len < len + q->buf_offset) { + dev_kfree_skb(q->rx_head); + mt76_put_page_pool_buf(buf, false); + q->rx_head = NULL; + goto next_page; + } + + ls = FIELD_GET(RRO_HIF_DATA1_LS_MASK, + le32_to_cpu(rxd->data1)); + if (q->rx_head) { + /* TODO: Take into account non-linear skb. */ + mt76_put_page_pool_buf(buf, false); + if (ls) { + dev_kfree_skb(q->rx_head); + q->rx_head = NULL; + } + goto next_page; + } + + if (ls && !mt7996_rx_check(mdev, buf, len)) + goto next_page; + + skb = build_skb(buf, q->buf_size); + if (!skb) + goto next_page; + + skb_reserve(skb, q->buf_offset); + skb_mark_for_recycle(skb); + __skb_put(skb, len); + + if (ind_reason == 1 || ind_reason == 2) { + dev_kfree_skb(skb); + goto next_page; + } + + if (!ls) { + q->rx_head = skb; + goto next_page; + } + + status = (struct mt76_rx_status *)skb->cb; + if (seq_id != MT7996_RRO_MAX_SESSION) + status->aggr = true; + + mt7996_queue_rx_skb(mdev, qid, skb, &info); +next_page: + if ((j + 1) % MT7996_MAX_HIF_RXD_IN_PG == 0) { +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + dma_addr = + FIELD_GET(MSDU_PAGE_INFO_PG_HIGH_MASK, + le32_to_cpu(pinfo->data)); + dma_addr <<= 32; + dma_addr |= le32_to_cpu(pinfo->pg_low); +#else + dma_addr = le32_to_cpu(pinfo->pg_low); +#endif + mt7996_msdu_page_put_to_cache(dev, p); + p = NULL; + } + } + +update_ack_seq_num: + if ((i + 1) % 4 == 0) + mt76_wr(dev, MT_RRO_ACK_SN_CTRL, + FIELD_PREP(MT_RRO_ACK_SN_CTRL_SESSION_MASK, + seq_id) | + FIELD_PREP(MT_RRO_ACK_SN_CTRL_SN_MASK, + seq_num)); + if (p) { + mt7996_msdu_page_put_to_cache(dev, p); + p = NULL; + } + } + + /* Update ack_seq_num for remaining addr_elem */ + if (i % 4) + mt76_wr(dev, MT_RRO_ACK_SN_CTRL, + FIELD_PREP(MT_RRO_ACK_SN_CTRL_SESSION_MASK, seq_id) | + FIELD_PREP(MT_RRO_ACK_SN_CTRL_SN_MASK, seq_num)); +} + void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy) { struct mt7996_dev *dev = phy->dev; @@ -1737,7 +2227,8 @@ mt7996_update_vif_beacon(void *priv, u8 *mac, struct ieee80211_vif *vif) if (!link || link->phy != phy) continue; - mt7996_mcu_add_beacon(dev->mt76.hw, vif, link_conf); + mt7996_mcu_add_beacon(dev->mt76.hw, vif, link_conf, + link_conf->enable_beacon); } } @@ -1781,13 +2272,10 @@ void mt7996_tx_token_put(struct mt7996_dev *dev) static int mt7996_mac_restart(struct mt7996_dev *dev) { - struct mt7996_phy *phy2, *phy3; struct mt76_dev *mdev = &dev->mt76; + struct mt7996_phy *phy; int i, ret; - phy2 = mt7996_phy2(dev); - phy3 = mt7996_phy3(dev); - if (dev->hif2) { mt76_wr(dev, MT_INT1_MASK_CSR, 0x0); mt76_wr(dev, MT_INT1_SOURCE_CSR, ~0); @@ -1799,20 +2287,14 @@ mt7996_mac_restart(struct mt7996_dev *dev) mt76_wr(dev, MT_PCIE1_MAC_INT_ENABLE, 0x0); } - set_bit(MT76_RESET, &dev->mphy.state); set_bit(MT76_MCU_RESET, &dev->mphy.state); + mt7996_for_each_phy(dev, phy) + set_bit(MT76_RESET, &phy->mt76->state); wake_up(&dev->mt76.mcu.wait); - if (phy2) - set_bit(MT76_RESET, &phy2->mt76->state); - if (phy3) - set_bit(MT76_RESET, &phy3->mt76->state); /* lock/unlock all queues to ensure that no tx is pending */ - mt76_txq_schedule_all(&dev->mphy); - if (phy2) - mt76_txq_schedule_all(phy2->mt76); - if (phy3) - mt76_txq_schedule_all(phy3->mt76); + mt7996_for_each_phy(dev, phy) + mt76_txq_schedule_all(phy->mt76); /* disable all tx/rx napi */ mt76_worker_disable(&dev->mt76.tx_worker); @@ -1864,42 +2346,57 @@ mt7996_mac_restart(struct mt7996_dev *dev) if (ret) goto out; + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && + mt7996_has_hwrro(dev)) { + u32 wed_irq_mask = dev->mt76.mmio.irqmask | + MT_INT_TX_DONE_BAND2; + + mt7996_rro_hw_init(dev); + mt76_for_each_q_rx(&dev->mt76, i) { + if (mt76_queue_is_wed_rro_ind(&dev->mt76.q_rx[i]) || + mt76_queue_is_wed_rro_msdu_pg(&dev->mt76.q_rx[i])) + mt76_queue_rx_reset(dev, i); + } + + mt76_wr(dev, MT_INT_MASK_CSR, wed_irq_mask); + mtk_wed_device_start_hw_rro(&dev->mt76.mmio.wed, wed_irq_mask, + false); + mt7996_irq_enable(dev, wed_irq_mask); + mt7996_irq_disable(dev, 0); + } + + if (mtk_wed_device_active(&dev->mt76.mmio.wed_hif2)) { + mt76_wr(dev, MT_INT_PCIE1_MASK_CSR, + MT_INT_TX_RX_DONE_EXT); + mtk_wed_device_start(&dev->mt76.mmio.wed_hif2, + MT_INT_TX_RX_DONE_EXT); + } + /* set the necessary init items */ ret = mt7996_mcu_set_eeprom(dev); if (ret) goto out; mt7996_mac_init(dev); - mt7996_init_txpower(&dev->phy); - mt7996_init_txpower(phy2); - mt7996_init_txpower(phy3); + mt7996_for_each_phy(dev, phy) + mt7996_init_txpower(phy); ret = mt7996_txbf_init(dev); + if (ret) + goto out; - if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state)) { - ret = mt7996_run(&dev->phy); - if (ret) - goto out; - } - - if (phy2 && test_bit(MT76_STATE_RUNNING, &phy2->mt76->state)) { - ret = mt7996_run(phy2); - if (ret) - goto out; - } + mt7996_for_each_phy(dev, phy) { + if (!test_bit(MT76_STATE_RUNNING, &phy->mt76->state)) + continue; - if (phy3 && test_bit(MT76_STATE_RUNNING, &phy3->mt76->state)) { - ret = mt7996_run(phy3); + ret = mt7996_run(phy); if (ret) goto out; } out: /* reset done */ - clear_bit(MT76_RESET, &dev->mphy.state); - if (phy2) - clear_bit(MT76_RESET, &phy2->mt76->state); - if (phy3) - clear_bit(MT76_RESET, &phy3->mt76->state); + mt7996_for_each_phy(dev, phy) + clear_bit(MT76_RESET, &phy->mt76->state); napi_enable(&dev->mt76.tx_napi); local_bh_disable(); @@ -1911,74 +2408,123 @@ out: } static void +mt7996_mac_reset_sta_iter(void *data, struct ieee80211_sta *sta) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_dev *dev = data; + int i; + + for (i = 0; i < ARRAY_SIZE(msta->link); i++) { + struct mt7996_sta_link *msta_link = NULL; + + msta_link = rcu_replace_pointer(msta->link[i], msta_link, + lockdep_is_held(&dev->mt76.mutex)); + if (!msta_link) + continue; + + mt7996_mac_sta_deinit_link(dev, msta_link); + + if (msta->deflink_id == i) { + msta->deflink_id = IEEE80211_LINK_UNSPECIFIED; + continue; + } + + kfree_rcu(msta_link, rcu_head); + } +} + +static void +mt7996_mac_reset_vif_iter(void *data, u8 *mac, struct ieee80211_vif *vif) +{ + struct mt76_vif_link *mlink = (struct mt76_vif_link *)vif->drv_priv; + struct mt76_vif_data *mvif = mlink->mvif; + struct mt7996_dev *dev = data; + int i; + + rcu_read_lock(); + for (i = 0; i < ARRAY_SIZE(mvif->link); i++) { + + mlink = mt76_dereference(mvif->link[i], &dev->mt76); + if (!mlink || mlink == (struct mt76_vif_link *)vif->drv_priv) + continue; + + rcu_assign_pointer(mvif->link[i], NULL); + kfree_rcu(mlink, rcu_head); + } + rcu_read_unlock(); +} + +static void mt7996_mac_full_reset(struct mt7996_dev *dev) { - struct mt7996_phy *phy2, *phy3; + struct ieee80211_hw *hw = mt76_hw(dev); + struct mt7996_phy *phy; + LIST_HEAD(list); int i; - phy2 = mt7996_phy2(dev); - phy3 = mt7996_phy3(dev); dev->recovery.hw_full_reset = true; wake_up(&dev->mt76.mcu.wait); - ieee80211_stop_queues(mt76_hw(dev)); - if (phy2) - ieee80211_stop_queues(phy2->mt76->hw); - if (phy3) - ieee80211_stop_queues(phy3->mt76->hw); + ieee80211_stop_queues(hw); cancel_work_sync(&dev->wed_rro.work); - cancel_delayed_work_sync(&dev->mphy.mac_work); - if (phy2) - cancel_delayed_work_sync(&phy2->mt76->mac_work); - if (phy3) - cancel_delayed_work_sync(&phy3->mt76->mac_work); + mt7996_for_each_phy(dev, phy) + cancel_delayed_work_sync(&phy->mt76->mac_work); + + mt76_abort_scan(&dev->mt76); mutex_lock(&dev->mt76.mutex); for (i = 0; i < 10; i++) { if (!mt7996_mac_restart(dev)) break; } - mutex_unlock(&dev->mt76.mutex); if (i == 10) dev_err(dev->mt76.dev, "chip full reset failed\n"); - ieee80211_restart_hw(mt76_hw(dev)); - if (phy2) - ieee80211_restart_hw(phy2->mt76->hw); - if (phy3) - ieee80211_restart_hw(phy3->mt76->hw); + mt7996_for_each_phy(dev, phy) + phy->omac_mask = 0; - ieee80211_wake_queues(mt76_hw(dev)); - if (phy2) - ieee80211_wake_queues(phy2->mt76->hw); - if (phy3) - ieee80211_wake_queues(phy3->mt76->hw); + ieee80211_iterate_stations_atomic(hw, mt7996_mac_reset_sta_iter, dev); + ieee80211_iterate_active_interfaces_atomic(hw, + IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER, + mt7996_mac_reset_vif_iter, dev); + mt76_reset_device(&dev->mt76); + + INIT_LIST_HEAD(&dev->sta_rc_list); + INIT_LIST_HEAD(&dev->twt_list); + + spin_lock_bh(&dev->wed_rro.lock); + list_splice_init(&dev->wed_rro.poll_list, &list); + spin_unlock_bh(&dev->wed_rro.lock); + + while (!list_empty(&list)) { + struct mt7996_wed_rro_session_id *e; + + e = list_first_entry(&list, struct mt7996_wed_rro_session_id, + list); + list_del_init(&e->list); + kfree(e); + } + i = mt76_wcid_alloc(dev->mt76.wcid_mask, MT7996_WTBL_STA); + dev->mt76.global_wcid.idx = i; dev->recovery.hw_full_reset = false; - ieee80211_queue_delayed_work(mt76_hw(dev), - &dev->mphy.mac_work, - MT7996_WATCHDOG_TIME); - if (phy2) - ieee80211_queue_delayed_work(phy2->mt76->hw, - &phy2->mt76->mac_work, - MT7996_WATCHDOG_TIME); - if (phy3) - ieee80211_queue_delayed_work(phy3->mt76->hw, - &phy3->mt76->mac_work, - MT7996_WATCHDOG_TIME); + + mutex_unlock(&dev->mt76.mutex); + + ieee80211_restart_hw(mt76_hw(dev)); } void mt7996_mac_reset_work(struct work_struct *work) { - struct mt7996_phy *phy2, *phy3; + struct ieee80211_hw *hw; struct mt7996_dev *dev; + struct mt7996_phy *phy; int i; dev = container_of(work, struct mt7996_dev, reset_work); - phy2 = mt7996_phy2(dev); - phy3 = mt7996_phy3(dev); + hw = mt76_hw(dev); /* chip full reset */ if (dev->recovery.restart) { @@ -2009,7 +2555,7 @@ void mt7996_mac_reset_work(struct work_struct *work) return; dev_info(dev->mt76.dev,"\n%s L1 SER recovery start.", - wiphy_name(dev->mt76.hw->wiphy)); + wiphy_name(hw->wiphy)); if (mtk_wed_device_active(&dev->mt76.mmio.wed_hif2)) mtk_wed_device_stop(&dev->mt76.mmio.wed_hif2); @@ -2018,25 +2564,19 @@ void mt7996_mac_reset_work(struct work_struct *work) mtk_wed_device_stop(&dev->mt76.mmio.wed); ieee80211_stop_queues(mt76_hw(dev)); - if (phy2) - ieee80211_stop_queues(phy2->mt76->hw); - if (phy3) - ieee80211_stop_queues(phy3->mt76->hw); set_bit(MT76_RESET, &dev->mphy.state); set_bit(MT76_MCU_RESET, &dev->mphy.state); + mt76_abort_scan(&dev->mt76); wake_up(&dev->mt76.mcu.wait); cancel_work_sync(&dev->wed_rro.work); - cancel_delayed_work_sync(&dev->mphy.mac_work); - if (phy2) { - set_bit(MT76_RESET, &phy2->mt76->state); - cancel_delayed_work_sync(&phy2->mt76->mac_work); - } - if (phy3) { - set_bit(MT76_RESET, &phy3->mt76->state); - cancel_delayed_work_sync(&phy3->mt76->mac_work); + mt7996_for_each_phy(dev, phy) { + mt76_abort_roc(phy->mt76); + set_bit(MT76_RESET, &phy->mt76->state); + cancel_delayed_work_sync(&phy->mt76->mac_work); } + mt76_worker_disable(&dev->mt76.tx_worker); mt76_for_each_q_rx(&dev->mt76, i) { if (mtk_wed_device_active(&dev->mt76.mmio.wed) && @@ -2049,6 +2589,8 @@ void mt7996_mac_reset_work(struct work_struct *work) mutex_lock(&dev->mt76.mutex); + mt7996_npu_hw_stop(dev); + mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_STOPPED); if (mt7996_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) { @@ -2064,18 +2606,17 @@ void mt7996_mac_reset_work(struct work_struct *work) mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_RESET_DONE); mt7996_wait_reset_state(dev, MT_MCU_CMD_NORMAL_STATE); - /* enable DMA Tx/Tx and interrupt */ + /* enable DMA Rx/Tx and interrupt */ mt7996_dma_start(dev, false, false); + if (!is_mt7996(&dev->mt76) && dev->mt76.hwrro_mode == MT76_HWRRO_V3) + mt76_wr(dev, MT_RRO_3_0_EMU_CONF, MT_RRO_3_0_EMU_CONF_EN_MASK); + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) { - u32 wed_irq_mask = MT_INT_RRO_RX_DONE | MT_INT_TX_DONE_BAND2 | + u32 wed_irq_mask = MT_INT_TX_DONE_BAND2 | dev->mt76.mmio.irqmask; - if (mtk_wed_get_rx_capa(&dev->mt76.mmio.wed)) - wed_irq_mask &= ~MT_INT_RX_DONE_RRO_IND; - mt76_wr(dev, MT_INT_MASK_CSR, wed_irq_mask); - mtk_wed_device_start_hw_rro(&dev->mt76.mmio.wed, wed_irq_mask, true); mt7996_irq_enable(dev, wed_irq_mask); @@ -2089,11 +2630,8 @@ void mt7996_mac_reset_work(struct work_struct *work) } clear_bit(MT76_MCU_RESET, &dev->mphy.state); - clear_bit(MT76_RESET, &dev->mphy.state); - if (phy2) - clear_bit(MT76_RESET, &phy2->mt76->state); - if (phy3) - clear_bit(MT76_RESET, &phy3->mt76->state); + mt7996_for_each_phy(dev, phy) + clear_bit(MT76_RESET, &phy->mt76->state); mt76_for_each_q_rx(&dev->mt76, i) { if (mtk_wed_device_active(&dev->mt76.mmio.wed) && @@ -2115,25 +2653,15 @@ void mt7996_mac_reset_work(struct work_struct *work) napi_schedule(&dev->mt76.tx_napi); local_bh_enable(); - ieee80211_wake_queues(mt76_hw(dev)); - if (phy2) - ieee80211_wake_queues(phy2->mt76->hw); - if (phy3) - ieee80211_wake_queues(phy3->mt76->hw); + ieee80211_wake_queues(hw); + mt7996_update_beacons(dev); mutex_unlock(&dev->mt76.mutex); - mt7996_update_beacons(dev); + mt7996_npu_hw_init(dev); - ieee80211_queue_delayed_work(mt76_hw(dev), &dev->mphy.mac_work, - MT7996_WATCHDOG_TIME); - if (phy2) - ieee80211_queue_delayed_work(phy2->mt76->hw, - &phy2->mt76->mac_work, - MT7996_WATCHDOG_TIME); - if (phy3) - ieee80211_queue_delayed_work(phy3->mt76->hw, - &phy3->mt76->mac_work, + mt7996_for_each_phy(dev, phy) + ieee80211_queue_delayed_work(hw, &phy->mt76->mac_work, MT7996_WATCHDOG_TIME); dev_info(dev->mt76.dev,"\n%s L1 SER recovery completed.", wiphy_name(dev->mt76.hw->wiphy)); @@ -2382,6 +2910,8 @@ void mt7996_mac_sta_rc_work(struct work_struct *work) LIST_HEAD(list); u32 changed; + mutex_lock(&dev->mt76.mutex); + spin_lock_bh(&dev->mt76.sta_poll_lock); list_splice_init(&dev->sta_rc_list, &list); @@ -2414,6 +2944,8 @@ void mt7996_mac_sta_rc_work(struct work_struct *work) } spin_unlock_bh(&dev->mt76.sta_poll_lock); + + mutex_unlock(&dev->mt76.mutex); } void mt7996_mac_work(struct work_struct *work) diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mac.h b/sys/contrib/dev/mediatek/mt76/mt7996/mac.h index e629324a5617..4eca37b013fc 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/mac.h +++ b/sys/contrib/dev/mediatek/mt76/mt7996/mac.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2022 MediaTek Inc. */ diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/main.c b/sys/contrib/dev/mediatek/mt76/mt7996/main.c index 84f731b387d2..beed795edb24 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/main.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/main.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -90,9 +90,11 @@ static void mt7996_stop(struct ieee80211_hw *hw, bool suspend) { } -static inline int get_free_idx(u32 mask, u8 start, u8 end) +static inline int get_free_idx(u64 mask, u8 start, u8 end) { - return ffs(~mask & GENMASK(end, start)); + if (~mask & GENMASK_ULL(end, start)) + return __ffs64(~mask & GENMASK_ULL(end, start)) + 1; + return 0; } static int get_omac_idx(enum nl80211_iftype type, u64 mask) @@ -138,6 +140,28 @@ static int get_omac_idx(enum nl80211_iftype type, u64 mask) return -1; } +static int get_own_mld_idx(u64 mask, bool group_mld) +{ + u8 start = group_mld ? 0 : 16; + u8 end = group_mld ? 15 : 63; + int idx; + + idx = get_free_idx(mask, start, end); + if (idx) + return idx - 1; + + /* If the 16-63 range is not available, perform another lookup in the + * range 0-15 + */ + if (!group_mld) { + idx = get_free_idx(mask, 0, 15); + if (idx) + return idx - 1; + } + + return -EINVAL; +} + static void mt7996_init_bitrate_mask(struct ieee80211_vif *vif, struct mt7996_vif_link *mlink) { @@ -160,112 +184,106 @@ mt7996_init_bitrate_mask(struct ieee80211_vif *vif, struct mt7996_vif_link *mlin static int mt7996_set_hw_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, struct ieee80211_vif *vif, struct ieee80211_sta *sta, - struct ieee80211_key_conf *key) + unsigned int link_id, struct ieee80211_key_conf *key) { struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct ieee80211_bss_conf *link_conf; + struct mt7996_sta_link *msta_link; + struct mt7996_vif_link *link; int idx = key->keyidx; - unsigned int link_id; - unsigned long links; + u8 *wcid_keyidx; + bool is_bigtk; + int err; - if (key->link_id >= 0) - links = BIT(key->link_id); - else if (sta && sta->valid_links) - links = sta->valid_links; - else if (vif->valid_links) - links = vif->valid_links; - else - links = BIT(0); + link = mt7996_vif_link(dev, vif, link_id); + if (!link) + return 0; - for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) { - struct mt7996_sta_link *msta_link; - struct mt7996_vif_link *link; - u8 *wcid_keyidx; - int err; + if (!mt7996_vif_link_phy(link)) + return 0; - link = mt7996_vif_link(dev, vif, link_id); - if (!link) - continue; + if (sta) { + struct mt7996_sta *msta; - if (sta) { - struct mt7996_sta *msta; + msta = (struct mt7996_sta *)sta->drv_priv; + msta_link = mt76_dereference(msta->link[link_id], + &dev->mt76); + if (!msta_link) + return 0; - msta = (struct mt7996_sta *)sta->drv_priv; - msta_link = mt76_dereference(msta->link[link_id], - &dev->mt76); - if (!msta_link) - continue; + if (!msta_link->wcid.sta) + return -EOPNOTSUPP; + } else { + msta_link = &link->msta_link; + } + wcid_keyidx = &msta_link->wcid.hw_key_idx; - if (!msta_link->wcid.sta) - return -EOPNOTSUPP; - } else { - msta_link = &link->msta_link; - } - wcid_keyidx = &msta_link->wcid.hw_key_idx; - - switch (key->cipher) { - case WLAN_CIPHER_SUITE_AES_CMAC: - case WLAN_CIPHER_SUITE_BIP_CMAC_256: - case WLAN_CIPHER_SUITE_BIP_GMAC_128: - case WLAN_CIPHER_SUITE_BIP_GMAC_256: - if (key->keyidx == 6 || key->keyidx == 7) { - wcid_keyidx = &msta_link->wcid.hw_key_idx2; - key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE; - } - break; - default: - break; + is_bigtk = key->keyidx == 6 || key->keyidx == 7; + switch (key->cipher) { + case WLAN_CIPHER_SUITE_AES_CMAC: + case WLAN_CIPHER_SUITE_BIP_CMAC_256: + case WLAN_CIPHER_SUITE_BIP_GMAC_128: + case WLAN_CIPHER_SUITE_BIP_GMAC_256: + if (is_bigtk) { + wcid_keyidx = &msta_link->wcid.hw_key_idx2; + key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE; } + break; + default: + break; + } - if (cmd == SET_KEY && !sta && !link->mt76.cipher) { - struct ieee80211_bss_conf *link_conf; + link_conf = link_conf_dereference_protected(vif, link_id); + if (!link_conf) + link_conf = &vif->bss_conf; - link_conf = link_conf_dereference_protected(vif, - link_id); - if (!link_conf) - link_conf = &vif->bss_conf; + if (cmd == SET_KEY && !sta && !link->mt76.cipher) { + link->mt76.cipher = + mt76_connac_mcu_get_cipher(key->cipher); + mt7996_mcu_add_bss_info(link->phy, vif, link_conf, + &link->mt76, msta_link, true); + } - link->mt76.cipher = - mt76_connac_mcu_get_cipher(key->cipher); - mt7996_mcu_add_bss_info(link->phy, vif, link_conf, - &link->mt76, msta_link, true); - } + if (cmd == SET_KEY) + *wcid_keyidx = idx; + else if (idx == *wcid_keyidx) + *wcid_keyidx = -1; - if (cmd == SET_KEY) { - *wcid_keyidx = idx; - } else { - if (idx == *wcid_keyidx) - *wcid_keyidx = -1; - continue; - } + /* only do remove key for BIGTK */ + if (cmd != SET_KEY && !is_bigtk) + return 0; - mt76_wcid_key_setup(&dev->mt76, &msta_link->wcid, key); + mt76_wcid_key_setup(&dev->mt76, &msta_link->wcid, key); - if (key->keyidx == 6 || key->keyidx == 7) { - err = mt7996_mcu_bcn_prot_enable(dev, link, - msta_link, key); - if (err) - return err; - } + err = mt7996_mcu_add_key(&dev->mt76, link, key, + MCU_WMWA_UNI_CMD(STA_REC_UPDATE), + &msta_link->wcid, cmd); - err = mt7996_mcu_add_key(&dev->mt76, vif, key, - MCU_WMWA_UNI_CMD(STA_REC_UPDATE), - &msta_link->wcid, cmd); - if (err) - return err; + /* remove and add beacon in order to enable beacon protection */ + if (cmd == SET_KEY && is_bigtk && link_conf->enable_beacon) { + mt7996_mcu_add_beacon(hw, vif, link_conf, false); + mt7996_mcu_add_beacon(hw, vif, link_conf, true); } - return 0; + return err; } +struct mt7996_key_iter_data { + enum set_key_cmd cmd; + unsigned int link_id; +}; + static void mt7996_key_iter(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct ieee80211_key_conf *key, void *data) { + struct mt7996_key_iter_data *it = data; + if (sta) return; - WARN_ON(mt7996_set_hw_key(hw, SET_KEY, vif, NULL, key)); + WARN_ON(mt7996_set_hw_key(hw, it->cmd, vif, NULL, it->link_id, key)); } int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, @@ -278,8 +296,12 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, struct mt7996_phy *phy = mphy->priv; struct mt7996_dev *dev = phy->dev; u8 band_idx = phy->mt76->band_idx; + struct mt7996_key_iter_data it = { + .cmd = SET_KEY, + .link_id = link_conf->link_id, + }; struct mt76_txq *mtxq; - int idx, ret; + int mld_idx, idx, ret; mlink->idx = __ffs64(~dev->mt76.vif_mask); if (mlink->idx >= mt7996_max_interface_num(dev)) @@ -289,6 +311,11 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, if (idx < 0) return -ENOSPC; + mld_idx = get_own_mld_idx(dev->mld_idx_mask, false); + if (mld_idx < 0) + return -ENOSPC; + + link->mld_idx = mld_idx; link->phy = phy; mlink->omac_idx = idx; mlink->band_idx = band_idx; @@ -301,6 +328,7 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, return ret; dev->mt76.vif_mask |= BIT_ULL(mlink->idx); + dev->mld_idx_mask |= BIT_ULL(link->mld_idx); phy->omac_mask |= BIT_ULL(mlink->omac_idx); idx = MT7996_WTBL_RESERVED - mlink->idx; @@ -308,6 +336,7 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, INIT_LIST_HEAD(&msta_link->rc_list); msta_link->wcid.idx = idx; msta_link->wcid.link_id = link_conf->link_id; + msta_link->wcid.link_valid = ieee80211_vif_is_mld(vif); msta_link->wcid.tx_info |= MT_WCID_TX_INFO_SET; mt76_wcid_init(&msta_link->wcid, band_idx); @@ -339,9 +368,10 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, CONN_STATE_PORT_SECURE, true); rcu_assign_pointer(dev->mt76.wcid[idx], &msta_link->wcid); - ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, NULL); + ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); - if (mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED) + if (!mlink->wcid->offchannel && + mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED) mvif->mt76.deflink_id = link_conf->link_id; return 0; @@ -356,8 +386,15 @@ void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif, struct mt7996_sta_link *msta_link = &link->msta_link; struct mt7996_phy *phy = mphy->priv; struct mt7996_dev *dev = phy->dev; + struct mt7996_key_iter_data it = { + .cmd = SET_KEY, + .link_id = link_conf->link_id, + }; int idx = msta_link->wcid.idx; + if (!mlink->wcid->offchannel) + ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); + mt7996_mcu_add_sta(dev, link_conf, NULL, link, NULL, CONN_STATE_DISCONNECT, false); mt7996_mcu_add_bss_info(phy, vif, link_conf, mlink, msta_link, false); @@ -366,7 +403,8 @@ void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif, rcu_assign_pointer(dev->mt76.wcid[idx], NULL); - if (mvif->mt76.deflink_id == link_conf->link_id) { + if (!mlink->wcid->offchannel && + mvif->mt76.deflink_id == link_conf->link_id) { struct ieee80211_bss_conf *iter; unsigned int link_id; @@ -380,6 +418,7 @@ void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif, } dev->mt76.vif_mask &= ~BIT_ULL(mlink->idx); + dev->mld_idx_mask &= ~BIT_ULL(link->mld_idx); phy->omac_mask &= ~BIT_ULL(mlink->omac_idx); spin_lock_bh(&dev->mt76.sta_poll_lock); @@ -551,8 +590,9 @@ static int mt7996_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, struct ieee80211_key_conf *key) { struct mt7996_dev *dev = mt7996_hw_dev(hw); - struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; - int err; + unsigned int link_id; + unsigned long links; + int err = 0; /* The hardware does not support per-STA RX GTK, fallback * to software mode for these. @@ -586,11 +626,22 @@ static int mt7996_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, return -EOPNOTSUPP; } - if (!mt7996_vif_link_phy(&mvif->deflink)) - return 0; /* defer until after link add */ - mutex_lock(&dev->mt76.mutex); - err = mt7996_set_hw_key(hw, cmd, vif, sta, key); + + if (key->link_id >= 0) + links = BIT(key->link_id); + else if (sta && sta->valid_links) + links = sta->valid_links; + else if (vif->valid_links) + links = vif->valid_links; + else + links = BIT(0); + + for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) { + err = mt7996_set_hw_key(hw, cmd, vif, sta, link_id, key); + if (err) + break; + } mutex_unlock(&dev->mt76.mutex); return err; @@ -606,8 +657,8 @@ mt7996_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, unsigned int link_id, u16 queue, const struct ieee80211_tx_queue_params *params) { - struct mt7996_dev *dev = mt7996_hw_dev(hw); - struct mt7996_vif_link *mlink = mt7996_vif_link(dev, vif, link_id); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_vif_link_info *link_info = &mvif->link_info[link_id]; static const u8 mq_to_aci[] = { [IEEE80211_AC_VO] = 3, [IEEE80211_AC_VI] = 2, @@ -616,7 +667,7 @@ mt7996_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, }; /* firmware uses access class index */ - mlink->queue_params[mq_to_aci[queue]] = *params; + link_info->queue_params[mq_to_aci[queue]] = *params; /* no need to update right away, we'll get BSS_CHANGED_QOS */ return 0; @@ -850,7 +901,7 @@ mt7996_link_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif, link->mt76.beacon_rates_idx = mt7996_get_rates_table(phy, info, true, false); - mt7996_mcu_add_beacon(hw, vif, info); + mt7996_mcu_add_beacon(hw, vif, info, info->enable_beacon); } if (changed & (BSS_CHANGED_UNSOL_BCAST_PROBE_RESP | @@ -878,7 +929,7 @@ mt7996_channel_switch_beacon(struct ieee80211_hw *hw, struct mt7996_dev *dev = mt7996_hw_dev(hw); mutex_lock(&dev->mt76.mutex); - mt7996_mcu_add_beacon(hw, vif, &vif->bss_conf); + mt7996_mcu_add_beacon(hw, vif, &vif->bss_conf, vif->bss_conf.enable_beacon); mutex_unlock(&dev->mt76.mutex); } @@ -903,6 +954,7 @@ mt7996_mac_sta_init_link(struct mt7996_dev *dev, msta_link = &msta->deflink; msta->deflink_id = link_id; + msta->seclink_id = msta->deflink_id; for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { struct mt76_txq *mtxq; @@ -917,6 +969,11 @@ mt7996_mac_sta_init_link(struct mt7996_dev *dev, msta_link = kzalloc(sizeof(*msta_link), GFP_KERNEL); if (!msta_link) return -ENOMEM; + + if (msta->seclink_id == msta->deflink_id && + (sta->valid_links & ~BIT(msta->deflink_id))) + msta->seclink_id = __ffs(sta->valid_links & + ~BIT(msta->deflink_id)); } INIT_LIST_HEAD(&msta_link->rc_list); @@ -925,6 +982,8 @@ mt7996_mac_sta_init_link(struct mt7996_dev *dev, msta_link->wcid.sta = 1; msta_link->wcid.idx = idx; msta_link->wcid.link_id = link_id; + msta_link->wcid.link_valid = !!sta->valid_links; + msta_link->wcid.def_wcid = &msta->deflink.wcid; ewma_avg_signal_init(&msta_link->avg_ack_signal); ewma_signal_init(&msta_link->wcid.rssi); @@ -941,18 +1000,9 @@ mt7996_mac_sta_init_link(struct mt7996_dev *dev, return 0; } -static void -mt7996_mac_sta_deinit_link(struct mt7996_dev *dev, - struct mt7996_sta_link *msta_link) +void mt7996_mac_sta_deinit_link(struct mt7996_dev *dev, + struct mt7996_sta_link *msta_link) { - int i; - - for (i = 0; i < ARRAY_SIZE(msta_link->wcid.aggr); i++) - mt76_rx_aggr_stop(&dev->mt76, &msta_link->wcid, i); - - mt7996_mac_wtbl_update(dev, msta_link->wcid.idx, - MT_WTBL_UPDATE_ADM_COUNT_CLEAR); - spin_lock_bh(&dev->mt76.sta_poll_lock); if (!list_empty(&msta_link->wcid.poll_list)) list_del_init(&msta_link->wcid.poll_list); @@ -982,6 +1032,9 @@ mt7996_mac_sta_remove_links(struct mt7996_dev *dev, struct ieee80211_vif *vif, if (!msta_link) continue; + mt7996_mac_wtbl_update(dev, msta_link->wcid.idx, + MT_WTBL_UPDATE_ADM_COUNT_CLEAR); + mt7996_mac_sta_deinit_link(dev, msta_link); link = mt7996_vif_link(dev, vif, link_id); if (!link) @@ -995,6 +1048,8 @@ mt7996_mac_sta_remove_links(struct mt7996_dev *dev, struct ieee80211_vif *vif, if (msta->deflink_id == link_id) { msta->deflink_id = IEEE80211_LINK_UNSPECIFIED; continue; + } else if (msta->seclink_id == link_id) { + msta->seclink_id = IEEE80211_LINK_UNSPECIFIED; } kfree_rcu(msta_link, rcu_head); @@ -1036,16 +1091,17 @@ mt7996_mac_sta_add_links(struct mt7996_dev *dev, struct ieee80211_vif *vif, goto error_unlink; } - err = mt7996_mac_sta_init_link(dev, link_conf, link_sta, link, - link_id); - if (err) - goto error_unlink; - mphy = mt76_vif_link_phy(&link->mt76); if (!mphy) { err = -EINVAL; goto error_unlink; } + + err = mt7996_mac_sta_init_link(dev, link_conf, link_sta, link, + link_id); + if (err) + goto error_unlink; + mphy->num_sta++; } @@ -1089,6 +1145,7 @@ mt7996_mac_sta_add(struct mt7996_dev *dev, struct ieee80211_vif *vif, mutex_lock(&dev->mt76.mutex); msta->deflink_id = IEEE80211_LINK_UNSPECIFIED; + msta->seclink_id = IEEE80211_LINK_UNSPECIFIED; msta->vif = mvif; err = mt7996_mac_sta_add_links(dev, vif, sta, links); @@ -1105,12 +1162,15 @@ mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif, unsigned long links = sta->valid_links; struct ieee80211_link_sta *link_sta; unsigned int link_id; + int err = 0; + + mutex_lock(&dev->mt76.mutex); for_each_sta_active_link(vif, sta, link_sta, link_id) { struct ieee80211_bss_conf *link_conf; struct mt7996_sta_link *msta_link; struct mt7996_vif_link *link; - int i, err; + int i; link_conf = link_conf_dereference_protected(vif, link_id); if (!link_conf) @@ -1130,12 +1190,12 @@ mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif, link, msta_link, CONN_STATE_CONNECT, true); if (err) - return err; + goto unlock; err = mt7996_mcu_add_rate_ctrl(dev, msta_link->sta, vif, link_id, false); if (err) - return err; + goto unlock; msta_link->wcid.tx_info |= MT_WCID_TX_INFO_SET; break; @@ -1144,28 +1204,30 @@ mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif, link, msta_link, CONN_STATE_PORT_SECURE, false); if (err) - return err; + goto unlock; break; case MT76_STA_EVENT_DISASSOC: for (i = 0; i < ARRAY_SIZE(msta_link->twt.flow); i++) mt7996_mac_twt_teardown_flow(dev, link, msta_link, i); - if (sta->mlo && links == BIT(link_id)) /* last link */ - mt7996_mcu_teardown_mld_sta(dev, link, - msta_link); - else + if (!sta->mlo) mt7996_mcu_add_sta(dev, link_conf, link_sta, link, msta_link, CONN_STATE_DISCONNECT, false); + else if (sta->mlo && links == BIT(link_id)) /* last link */ + mt7996_mcu_teardown_mld_sta(dev, link, + msta_link); msta_link->wcid.sta_disabled = 1; msta_link->wcid.sta = 0; links = links & ~BIT(link_id); break; } } +unlock: + mutex_unlock(&dev->mt76.mutex); - return 0; + return err; } static void @@ -1179,6 +1241,24 @@ mt7996_mac_sta_remove(struct mt7996_dev *dev, struct ieee80211_vif *vif, mutex_unlock(&dev->mt76.mutex); } +static void +mt7996_set_active_links(struct ieee80211_vif *vif) +{ + u16 active_links; + + if (vif->type != NL80211_IFTYPE_STATION) + return; + + if (!ieee80211_vif_is_mld(vif)) + return; + + active_links = mt76_select_links(vif, MT7996_MAX_RADIOS); + if (hweight16(active_links) < 2) + return; + + ieee80211_set_active_links_async(vif, active_links); +} + static int mt7996_sta_state(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, enum ieee80211_sta_state old_state, @@ -1196,16 +1276,18 @@ mt7996_sta_state(struct ieee80211_hw *hw, struct ieee80211_vif *vif, mt7996_mac_sta_remove(dev, vif, sta); if (old_state == IEEE80211_STA_AUTH && - new_state == IEEE80211_STA_ASSOC) + new_state == IEEE80211_STA_ASSOC) { + mt7996_set_active_links(vif); ev = MT76_STA_EVENT_ASSOC; - else if (old_state == IEEE80211_STA_ASSOC && - new_state == IEEE80211_STA_AUTHORIZED) + } else if (old_state == IEEE80211_STA_ASSOC && + new_state == IEEE80211_STA_AUTHORIZED) { ev = MT76_STA_EVENT_AUTHORIZE; - else if (old_state == IEEE80211_STA_ASSOC && - new_state == IEEE80211_STA_AUTH) + } else if (old_state == IEEE80211_STA_ASSOC && + new_state == IEEE80211_STA_AUTH) { ev = MT76_STA_EVENT_DISASSOC; - else + } else { return 0; + } return mt7996_mac_sta_event(dev, vif, sta, ev); } @@ -1214,29 +1296,60 @@ static void mt7996_tx(struct ieee80211_hw *hw, struct ieee80211_tx_control *control, struct sk_buff *skb) { + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct ieee80211_sta *sta = control->sta; + struct mt7996_sta *msta = sta ? (void *)sta->drv_priv : NULL; struct mt76_phy *mphy = hw->priv; struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); struct ieee80211_vif *vif = info->control.vif; + struct mt7996_vif *mvif = vif ? (void *)vif->drv_priv : NULL; struct mt76_wcid *wcid = &dev->mt76.global_wcid; u8 link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK); rcu_read_lock(); - if (vif) { - struct mt7996_vif *mvif = (void *)vif->drv_priv; - struct mt76_vif_link *mlink = &mvif->deflink.mt76; + /* Use primary link_id if the value from mac80211 is set to + * IEEE80211_LINK_UNSPECIFIED. + */ + if (link_id == IEEE80211_LINK_UNSPECIFIED) { + if (msta) + link_id = msta->deflink_id; + else if (mvif) + link_id = mvif->mt76.deflink_id; + } + + if (vif && ieee80211_vif_is_mld(vif)) { + struct ieee80211_bss_conf *link_conf; + + if (msta) { + struct ieee80211_link_sta *link_sta; - if (link_id < IEEE80211_LINK_UNSPECIFIED) - mlink = rcu_dereference(mvif->mt76.link[link_id]); + link_sta = rcu_dereference(sta->link[link_id]); + if (!link_sta) + link_sta = rcu_dereference(sta->link[msta->deflink_id]); - if (!mlink) { - ieee80211_free_txskb(hw, skb); - goto unlock; + if (link_sta) { + memcpy(hdr->addr1, link_sta->addr, ETH_ALEN); + if (ether_addr_equal(sta->addr, hdr->addr3)) + memcpy(hdr->addr3, link_sta->addr, ETH_ALEN); + } + } + + link_conf = rcu_dereference(vif->link_conf[link_id]); + if (link_conf) { + memcpy(hdr->addr2, link_conf->addr, ETH_ALEN); + if (ether_addr_equal(vif->addr, hdr->addr3)) + memcpy(hdr->addr3, link_conf->addr, ETH_ALEN); } + } - if (mlink->wcid) + if (mvif) { + struct mt76_vif_link *mlink; + + mlink = rcu_dereference(mvif->mt76.link[link_id]); + if (mlink && mlink->wcid) wcid = mlink->wcid; if (mvif->mt76.roc_phy && @@ -1244,7 +1357,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, mphy = mvif->mt76.roc_phy; if (mphy->roc_link) wcid = mphy->roc_link->wcid; - } else { + } else if (mlink) { mphy = mt76_vif_link_phy(mlink); } } @@ -1254,8 +1367,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, goto unlock; } - if (control->sta && link_id < IEEE80211_LINK_UNSPECIFIED) { - struct mt7996_sta *msta = (void *)control->sta->drv_priv; + if (msta) { struct mt7996_sta_link *msta_link; msta_link = rcu_dereference(msta->link[link_id]); @@ -1292,16 +1404,13 @@ static int mt7996_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_ampdu_params *params) { - enum ieee80211_ampdu_mlme_action action = params->action; struct mt7996_dev *dev = mt7996_hw_dev(hw); struct ieee80211_sta *sta = params->sta; struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; struct ieee80211_txq *txq = sta->txq[params->tid]; - struct ieee80211_link_sta *link_sta; u16 tid = params->tid; u16 ssn = params->ssn; struct mt76_txq *mtxq; - unsigned int link_id; int ret = 0; if (!txq) @@ -1311,59 +1420,42 @@ mt7996_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif, mutex_lock(&dev->mt76.mutex); - for_each_sta_active_link(vif, sta, link_sta, link_id) { - struct mt7996_sta_link *msta_link; - struct mt7996_vif_link *link; - - msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); - if (!msta_link) - continue; - - link = mt7996_vif_link(dev, vif, link_id); - if (!link) - continue; - - switch (action) { - case IEEE80211_AMPDU_RX_START: - mt76_rx_aggr_start(&dev->mt76, &msta_link->wcid, tid, - ssn, params->buf_size); - ret = mt7996_mcu_add_rx_ba(dev, params, link, true); - break; - case IEEE80211_AMPDU_RX_STOP: - mt76_rx_aggr_stop(&dev->mt76, &msta_link->wcid, tid); - ret = mt7996_mcu_add_rx_ba(dev, params, link, false); - break; - case IEEE80211_AMPDU_TX_OPERATIONAL: - mtxq->aggr = true; - mtxq->send_bar = false; - ret = mt7996_mcu_add_tx_ba(dev, params, link, - msta_link, true); - break; - case IEEE80211_AMPDU_TX_STOP_FLUSH: - case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT: - mtxq->aggr = false; - clear_bit(tid, &msta_link->wcid.ampdu_state); - ret = mt7996_mcu_add_tx_ba(dev, params, link, - msta_link, false); - break; - case IEEE80211_AMPDU_TX_START: - set_bit(tid, &msta_link->wcid.ampdu_state); - ret = IEEE80211_AMPDU_TX_START_IMMEDIATE; - break; - case IEEE80211_AMPDU_TX_STOP_CONT: - mtxq->aggr = false; - clear_bit(tid, &msta_link->wcid.ampdu_state); - ret = mt7996_mcu_add_tx_ba(dev, params, link, - msta_link, false); - break; - } - - if (ret) - break; - } - - if (action == IEEE80211_AMPDU_TX_STOP_CONT) + switch (params->action) { + case IEEE80211_AMPDU_RX_START: + /* Since packets belonging to the same TID can be split over + * multiple links, store the AMPDU state for reordering in the + * primary link + */ + mt76_rx_aggr_start(&dev->mt76, &msta->deflink.wcid, tid, + ssn, params->buf_size); + ret = mt7996_mcu_add_rx_ba(dev, params, vif, true); + break; + case IEEE80211_AMPDU_RX_STOP: + mt76_rx_aggr_stop(&dev->mt76, &msta->deflink.wcid, tid); + ret = mt7996_mcu_add_rx_ba(dev, params, vif, false); + break; + case IEEE80211_AMPDU_TX_OPERATIONAL: + mtxq->aggr = true; + mtxq->send_bar = false; + ret = mt7996_mcu_add_tx_ba(dev, params, vif, true); + break; + case IEEE80211_AMPDU_TX_STOP_FLUSH: + case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT: + mtxq->aggr = false; + clear_bit(tid, &msta->deflink.wcid.ampdu_state); + ret = mt7996_mcu_add_tx_ba(dev, params, vif, false); + break; + case IEEE80211_AMPDU_TX_START: + set_bit(tid, &msta->deflink.wcid.ampdu_state); + ret = IEEE80211_AMPDU_TX_START_IMMEDIATE; + break; + case IEEE80211_AMPDU_TX_STOP_CONT: + mtxq->aggr = false; + clear_bit(tid, &msta->deflink.wcid.ampdu_state); + ret = mt7996_mcu_add_tx_ba(dev, params, vif, false); ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid); + break; + } mutex_unlock(&dev->mt76.mutex); @@ -1617,19 +1709,13 @@ static void mt7996_sta_statistics(struct ieee80211_hw *hw, } } -static void mt7996_link_rate_ctrl_update(void *data, struct ieee80211_sta *sta) +static void mt7996_link_rate_ctrl_update(void *data, + struct mt7996_sta_link *msta_link) { - struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_sta *msta = msta_link->sta; struct mt7996_dev *dev = msta->vif->deflink.phy->dev; - struct mt7996_sta_link *msta_link; u32 *changed = data; - rcu_read_lock(); - - msta_link = rcu_dereference(msta->link[msta->deflink_id]); - if (!msta_link) - goto out; - spin_lock_bh(&dev->mt76.sta_poll_lock); msta_link->changed |= *changed; @@ -1637,8 +1723,6 @@ static void mt7996_link_rate_ctrl_update(void *data, struct ieee80211_sta *sta) list_add_tail(&msta_link->rc_list, &dev->sta_rc_list); spin_unlock_bh(&dev->mt76.sta_poll_lock); -out: - rcu_read_unlock(); } static void mt7996_link_sta_rc_update(struct ieee80211_hw *hw, @@ -1646,11 +1730,32 @@ static void mt7996_link_sta_rc_update(struct ieee80211_hw *hw, struct ieee80211_link_sta *link_sta, u32 changed) { - struct mt7996_dev *dev = mt7996_hw_dev(hw); struct ieee80211_sta *sta = link_sta->sta; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_sta_link *msta_link; - mt7996_link_rate_ctrl_update(&changed, sta); - ieee80211_queue_work(hw, &dev->rc_work); + rcu_read_lock(); + + msta_link = rcu_dereference(msta->link[link_sta->link_id]); + if (msta_link) { + struct mt7996_dev *dev = mt7996_hw_dev(hw); + + mt7996_link_rate_ctrl_update(&changed, msta_link); + ieee80211_queue_work(hw, &dev->rc_work); + } + + rcu_read_unlock(); +} + +static void mt7996_sta_rate_ctrl_update(void *data, struct ieee80211_sta *sta) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_sta_link *msta_link; + u32 *changed = data; + + msta_link = rcu_dereference(msta->link[msta->deflink_id]); + if (msta_link) + mt7996_link_rate_ctrl_update(&changed, msta_link); } static int @@ -1671,7 +1776,7 @@ mt7996_set_bitrate_mask(struct ieee80211_hw *hw, struct ieee80211_vif *vif, * - multiple rates: if it's not in range format i.e 0-{7,8,9} for VHT * then multiple MCS setting (MCS 4,5,6) is not supported. */ - ieee80211_iterate_stations_atomic(hw, mt7996_link_rate_ctrl_update, + ieee80211_iterate_stations_atomic(hw, mt7996_sta_rate_ctrl_update, &changed); ieee80211_queue_work(hw, &dev->rc_work); @@ -2059,7 +2164,6 @@ out: return ret; } -#ifdef CONFIG_NET_MEDIATEK_SOC_WED static int mt7996_net_fill_forward_path(struct ieee80211_hw *hw, struct ieee80211_vif *vif, @@ -2067,17 +2171,14 @@ mt7996_net_fill_forward_path(struct ieee80211_hw *hw, struct net_device_path_ctx *ctx, struct net_device_path *path) { - struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; struct mt7996_dev *dev = mt7996_hw_dev(hw); struct mtk_wed_device *wed = &dev->mt76.mmio.wed; struct mt7996_sta_link *msta_link; struct mt7996_vif_link *link; - struct mt76_vif_link *mlink; - struct mt7996_phy *phy; - mlink = rcu_dereference(mvif->mt76.link[msta->deflink_id]); - if (!mlink) + link = mt7996_vif_link(dev, vif, msta->deflink_id); + if (!link) return -EIO; msta_link = rcu_dereference(msta->link[msta->deflink_id]); @@ -2087,38 +2188,91 @@ mt7996_net_fill_forward_path(struct ieee80211_hw *hw, if (!msta_link->wcid.sta || msta_link->wcid.idx > MT7996_WTBL_STA) return -EIO; - link = (struct mt7996_vif_link *)mlink; - phy = mt7996_vif_link_phy(link); - if (!phy) - return -ENODEV; - - if (phy != &dev->phy && phy->mt76->band_idx == MT_BAND2) + if (dev->hif2 && + ((is_mt7996(&dev->mt76) && msta_link->wcid.phy_idx == MT_BAND2) || + (is_mt7992(&dev->mt76) && msta_link->wcid.phy_idx == MT_BAND1))) wed = &dev->mt76.mmio.wed_hif2; - if (!mtk_wed_device_active(wed)) + if (!mtk_wed_device_active(wed) && + !mt76_npu_device_active(&dev->mt76)) return -ENODEV; path->type = DEV_PATH_MTK_WDMA; path->dev = ctx->dev; - path->mtk_wdma.wdma_idx = wed->wdma_idx; - path->mtk_wdma.bss = mlink->idx; +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + if (mtk_wed_device_active(wed)) + path->mtk_wdma.wdma_idx = wed->wdma_idx; + else +#endif + path->mtk_wdma.wdma_idx = link->mt76.band_idx; + path->mtk_wdma.bss = link->mt76.idx; path->mtk_wdma.queue = 0; path->mtk_wdma.wcid = msta_link->wcid.idx; - path->mtk_wdma.amsdu = mtk_wed_is_amsdu_supported(wed); + if (ieee80211_hw_check(hw, SUPPORTS_AMSDU_IN_AMPDU) && + mtk_wed_is_amsdu_supported(wed)) + path->mtk_wdma.amsdu = msta_link->wcid.amsdu; + else + path->mtk_wdma.amsdu = 0; ctx->dev = NULL; return 0; } -#endif - static int mt7996_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif, u16 old_links, u16 new_links, struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]) { - return 0; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + int ret = 0; + + mutex_lock(&dev->mt76.mutex); + + if (!old_links) { + int idx; + + idx = get_own_mld_idx(dev->mld_idx_mask, true); + if (idx < 0) { + ret = -ENOSPC; + goto out; + } + mvif->mld_group_idx = idx; + dev->mld_idx_mask |= BIT_ULL(mvif->mld_group_idx); + + idx = get_free_idx(dev->mld_remap_idx_mask, 0, 15) - 1; + if (idx < 0) { + ret = -ENOSPC; + goto out; + } + mvif->mld_remap_idx = idx; + dev->mld_remap_idx_mask |= BIT_ULL(mvif->mld_remap_idx); + } + + if (new_links) + goto out; + + dev->mld_idx_mask &= ~BIT_ULL(mvif->mld_group_idx); + dev->mld_remap_idx_mask &= ~BIT_ULL(mvif->mld_remap_idx); + +out: + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + +static void +mt7996_reconfig_complete(struct ieee80211_hw *hw, + enum ieee80211_reconfig_type reconfig_type) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy; + + ieee80211_wake_queues(hw); + mt7996_for_each_phy(dev, phy) + ieee80211_queue_delayed_work(hw, &phy->mt76->mac_work, + MT7996_WATCHDOG_TIME); } const struct ieee80211_ops mt7996_ops = { @@ -2171,12 +2325,16 @@ const struct ieee80211_ops mt7996_ops = { .twt_teardown_request = mt7996_twt_teardown_request, #ifdef CONFIG_MAC80211_DEBUGFS .sta_add_debugfs = mt7996_sta_add_debugfs, + .link_sta_add_debugfs = mt7996_link_sta_add_debugfs, #endif .set_radar_background = mt7996_set_radar_background, -#ifdef CONFIG_NET_MEDIATEK_SOC_WED .net_fill_forward_path = mt7996_net_fill_forward_path, +#ifdef CONFIG_NET_MEDIATEK_SOC_WED .net_setup_tc = mt76_wed_net_setup_tc, +#elif defined(CONFIG_MT7996_NPU) + .net_setup_tc = mt76_npu_net_setup_tc, #endif .change_vif_links = mt7996_change_vif_links, .change_sta_links = mt7996_mac_sta_change_links, + .reconfig_complete = mt7996_reconfig_complete, }; diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7996/mcu.c index 443cd21a32b8..1e21c05bae28 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/mcu.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/mcu.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -246,6 +246,30 @@ mt7996_mcu_parse_response(struct mt76_dev *mdev, int cmd, return ret; } +static void +mt7996_mcu_set_timeout(struct mt76_dev *mdev, int cmd) +{ + mdev->mcu.timeout = 5 * HZ; + + if (!(cmd & __MCU_CMD_FIELD_UNI)) + return; + + switch (FIELD_GET(__MCU_CMD_FIELD_ID, cmd)) { + case MCU_UNI_CMD_THERMAL: + case MCU_UNI_CMD_TWT: + case MCU_UNI_CMD_GET_MIB_INFO: + case MCU_UNI_CMD_STA_REC_UPDATE: + case MCU_UNI_CMD_BSS_INFO_UPDATE: + mdev->mcu.timeout = 2 * HZ; + return; + case MCU_UNI_CMD_EFUSE_CTRL: + mdev->mcu.timeout = 20 * HZ; + return; + default: + break; + } +} + static int mt7996_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb, int cmd, int *wait_seq) @@ -259,7 +283,7 @@ mt7996_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb, u32 val; u8 seq; - mdev->mcu.timeout = 20 * HZ; + mt7996_mcu_set_timeout(mdev, cmd); seq = ++dev->mt76.mcu.msg_seq & 0xf; if (!seq) @@ -298,6 +322,9 @@ mt7996_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb, else uni_txd->option = MCU_CMD_UNI_EXT_ACK; + if (mcu_cmd == MCU_UNI_CMD_SDO) + uni_txd->option &= ~MCU_CMD_ACK; + if ((cmd & __MCU_CMD_FIELD_WA) && (cmd & __MCU_CMD_FIELD_WM)) uni_txd->s2d_index = MCU_S2D_H2CN; else if (cmd & __MCU_CMD_FIELD_WA) @@ -679,7 +706,7 @@ mt7996_mcu_wed_rro_event(struct mt7996_dev *dev, struct sk_buff *skb) { struct mt7996_mcu_wed_rro_event *event = (void *)skb->data; - if (!dev->has_rro) + if (!mt7996_has_hwrro(dev)) return; skb_pull(skb, sizeof(struct mt7996_mcu_rxd) + 4); @@ -918,17 +945,28 @@ mt7996_mcu_bss_txcmd_tlv(struct sk_buff *skb, bool en) } static void -mt7996_mcu_bss_mld_tlv(struct sk_buff *skb, struct mt76_vif_link *mlink) +mt7996_mcu_bss_mld_tlv(struct sk_buff *skb, + struct ieee80211_bss_conf *link_conf, + struct mt7996_vif_link *link) { + struct ieee80211_vif *vif = link_conf->vif; + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; struct bss_mld_tlv *mld; struct tlv *tlv; tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_MLD, sizeof(*mld)); - mld = (struct bss_mld_tlv *)tlv; - mld->group_mld_id = 0xff; - mld->own_mld_id = mlink->idx; - mld->remap_idx = 0xff; + mld->own_mld_id = link->mld_idx; + mld->link_id = link_conf->link_id; + + if (ieee80211_vif_is_mld(vif)) { + mld->group_mld_id = mvif->mld_group_idx; + mld->remap_idx = mvif->mld_remap_idx; + memcpy(mld->mac_addr, vif->addr, ETH_ALEN); + } else { + mld->group_mld_id = 0xff; + mld->remap_idx = 0xff; + } } static void @@ -1018,7 +1056,6 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, struct mt76_connac_bss_basic_tlv *bss; u32 type = CONNECTION_INFRA_AP; u16 sta_wlan_idx = wlan_idx; - struct ieee80211_sta *sta; struct tlv *tlv; int idx; @@ -1029,14 +1066,18 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, break; case NL80211_IFTYPE_STATION: if (enable) { + struct ieee80211_sta *sta; + rcu_read_lock(); - sta = ieee80211_find_sta(vif, vif->bss_conf.bssid); - /* TODO: enable BSS_INFO_UAPSD & BSS_INFO_PM */ + sta = ieee80211_find_sta(vif, link_conf->bssid); if (sta) { - struct mt76_wcid *wcid; + struct mt7996_sta *msta = (void *)sta->drv_priv; + struct mt7996_sta_link *msta_link; + int link_id = link_conf->link_id; - wcid = (struct mt76_wcid *)sta->drv_priv; - sta_wlan_idx = wcid->idx; + msta_link = rcu_dereference(msta->link[link_id]); + if (msta_link) + sta_wlan_idx = msta_link->wcid.idx; } rcu_read_unlock(); } @@ -1053,8 +1094,6 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_BASIC, sizeof(*bss)); bss = (struct mt76_connac_bss_basic_tlv *)tlv; - bss->bcn_interval = cpu_to_le16(link_conf->beacon_int); - bss->dtim_period = link_conf->dtim_period; bss->bmc_tx_wlan_idx = cpu_to_le16(wlan_idx); bss->sta_idx = cpu_to_le16(sta_wlan_idx); bss->conn_type = cpu_to_le32(type); @@ -1074,10 +1113,10 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, memcpy(bss->bssid, link_conf->bssid, ETH_ALEN); bss->bcn_interval = cpu_to_le16(link_conf->beacon_int); - bss->dtim_period = vif->bss_conf.dtim_period; + bss->dtim_period = link_conf->dtim_period; bss->phymode = mt76_connac_get_phy_mode(phy, vif, chandef->chan->band, NULL); - bss->phymode_ext = mt76_connac_get_phy_mode_ext(phy, &vif->bss_conf, + bss->phymode_ext = mt76_connac_get_phy_mode_ext(phy, link_conf, chandef->chan->band); return 0; @@ -1127,6 +1166,8 @@ int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, struct ieee80211_vif *vif, goto out; if (enable) { + struct mt7996_vif_link *link; + mt7996_mcu_bss_rfch_tlv(skb, phy); mt7996_mcu_bss_bmc_tlv(skb, mlink, phy); mt7996_mcu_bss_ra_tlv(skb, phy); @@ -1137,7 +1178,8 @@ int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, struct ieee80211_vif *vif, mt7996_mcu_bss_he_tlv(skb, vif, link_conf, phy); /* this tag is necessary no matter if the vif is MLD */ - mt7996_mcu_bss_mld_tlv(skb, mlink); + link = container_of(mlink, struct mt7996_vif_link, mt76); + mt7996_mcu_bss_mld_tlv(skb, link_conf, link); } mt7996_mcu_bss_mbssid_tlv(skb, link_conf, enable); @@ -1168,9 +1210,8 @@ int mt7996_mcu_set_timing(struct mt7996_phy *phy, struct ieee80211_vif *vif, static int mt7996_mcu_sta_ba(struct mt7996_dev *dev, struct mt76_vif_link *mvif, struct ieee80211_ampdu_params *params, - bool enable, bool tx) + struct mt76_wcid *wcid, bool enable, bool tx) { - struct mt76_wcid *wcid = (struct mt76_wcid *)params->sta->drv_priv; struct sta_rec_ba_uni *ba; struct sk_buff *skb; struct tlv *tlv; @@ -1189,7 +1230,7 @@ mt7996_mcu_sta_ba(struct mt7996_dev *dev, struct mt76_vif_link *mvif, ba->ba_en = enable << params->tid; ba->amsdu = params->amsdu; ba->tid = params->tid; - ba->ba_rdd_rro = !tx && enable && dev->has_rro; + ba->ba_rdd_rro = !tx && enable && mt7996_has_hwrro(dev); return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); @@ -1198,20 +1239,67 @@ mt7996_mcu_sta_ba(struct mt7996_dev *dev, struct mt76_vif_link *mvif, /** starec & wtbl **/ int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev, struct ieee80211_ampdu_params *params, - struct mt7996_vif_link *link, - struct mt7996_sta_link *msta_link, bool enable) + struct ieee80211_vif *vif, bool enable) { - if (enable && !params->amsdu) - msta_link->wcid.amsdu = false; + struct ieee80211_sta *sta = params->sta; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct ieee80211_link_sta *link_sta; + unsigned int link_id; + int ret = 0; + + for_each_sta_active_link(vif, sta, link_sta, link_id) { + struct mt7996_sta_link *msta_link; + struct mt7996_vif_link *link; + + msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + if (!msta_link) + continue; + + link = mt7996_vif_link(dev, vif, link_id); + if (!link) + continue; + + if (enable && !params->amsdu) + msta_link->wcid.amsdu = false; + + ret = mt7996_mcu_sta_ba(dev, &link->mt76, params, + &msta_link->wcid, enable, true); + if (ret) + break; + } - return mt7996_mcu_sta_ba(dev, &link->mt76, params, enable, true); + return ret; } int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev, struct ieee80211_ampdu_params *params, - struct mt7996_vif_link *link, bool enable) + struct ieee80211_vif *vif, bool enable) { - return mt7996_mcu_sta_ba(dev, &link->mt76, params, enable, false); + struct ieee80211_sta *sta = params->sta; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct ieee80211_link_sta *link_sta; + unsigned int link_id; + int ret = 0; + + for_each_sta_active_link(vif, sta, link_sta, link_id) { + struct mt7996_sta_link *msta_link; + struct mt7996_vif_link *link; + + msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + if (!msta_link) + continue; + + link = mt7996_vif_link(dev, vif, link_id); + if (!link) + continue; + + ret = mt7996_mcu_sta_ba(dev, &link->mt76, params, + &msta_link->wcid, enable, false); + if (ret) + break; + } + + return ret; } static void @@ -1757,8 +1845,8 @@ mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb, bf->ibf_nrow = tx_ant; if (link_sta->eht_cap.has_eht || link_sta->he_cap.has_he) - bf->ibf_timeout = is_mt7996(&dev->mt76) ? MT7996_IBF_TIMEOUT : - MT7992_IBF_TIMEOUT; + bf->ibf_timeout = is_mt7992(&dev->mt76) ? MT7992_IBF_TIMEOUT : + MT7996_IBF_TIMEOUT; else if (!ebf && link_sta->bandwidth <= IEEE80211_STA_RX_BW_40 && !bf->ncol) bf->ibf_timeout = MT7996_IBF_TIMEOUT_LEGACY; else @@ -1772,19 +1860,6 @@ mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb, bf->mem_20m = bf->nrow < BF_MAT_ORDER ? matrix[bf->nrow][bf->ncol] : 0; } - - switch (link_sta->bandwidth) { - case IEEE80211_STA_RX_BW_160: - case IEEE80211_STA_RX_BW_80: - bf->mem_total = bf->mem_20m * 2; - break; - case IEEE80211_STA_RX_BW_40: - bf->mem_total = bf->mem_20m; - break; - case IEEE80211_STA_RX_BW_20: - default: - break; - } } static void @@ -2284,13 +2359,10 @@ error_unlock: } static int -mt7996_mcu_add_group(struct mt7996_dev *dev, struct ieee80211_vif *vif, - struct ieee80211_sta *sta) +mt7996_mcu_add_group(struct mt7996_dev *dev, struct mt7996_vif_link *link, + struct mt76_wcid *wcid) { #define MT_STA_BSS_GROUP 1 - struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; - struct mt7996_sta_link *msta_link; - struct mt7996_sta *msta; struct { u8 __rsv1[4]; @@ -2305,13 +2377,10 @@ mt7996_mcu_add_group(struct mt7996_dev *dev, struct ieee80211_vif *vif, .tag = cpu_to_le16(UNI_VOW_DRR_CTRL), .len = cpu_to_le16(sizeof(req) - 4), .action = cpu_to_le32(MT_STA_BSS_GROUP), - .val = cpu_to_le32(mvif->deflink.mt76.idx % 16), + .val = cpu_to_le32(link->mt76.idx % 16), + .wlan_idx = cpu_to_le16(wcid->idx), }; - msta = sta ? (struct mt7996_sta *)sta->drv_priv : NULL; - msta_link = msta ? &msta->deflink : &mvif->deflink.msta_link; - req.wlan_idx = cpu_to_le16(msta_link->wcid.idx); - return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(VOW), &req, sizeof(req), true); } @@ -2344,8 +2413,8 @@ mt7996_mcu_sta_mld_setup_tlv(struct mt7996_dev *dev, struct sk_buff *skb, mld_setup->primary_id = cpu_to_le16(msta_link->wcid.idx); if (nlinks > 1) { - link_id = __ffs(sta->valid_links & ~BIT(msta->deflink_id)); - msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + msta_link = mt76_dereference(msta->link[msta->seclink_id], + &dev->mt76); if (!msta_link) return; } @@ -2451,7 +2520,7 @@ int mt7996_mcu_add_sta(struct mt7996_dev *dev, } } - ret = mt7996_mcu_add_group(dev, link_conf->vif, sta); + ret = mt7996_mcu_add_group(dev, link, wcid); if (ret) { dev_kfree_skb(skb); return ret; @@ -2480,157 +2549,94 @@ int mt7996_mcu_teardown_mld_sta(struct mt7996_dev *dev, } static int -mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid, +mt7996_mcu_sta_key_tlv(struct mt76_dev *dev, struct mt76_wcid *wcid, struct sk_buff *skb, struct ieee80211_key_conf *key, enum set_key_cmd cmd) { struct sta_rec_sec_uni *sec; + struct sec_key_uni *sec_key; struct tlv *tlv; + u8 cipher; tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_KEY_V2, sizeof(*sec)); sec = (struct sta_rec_sec_uni *)tlv; - sec->add = cmd; - - if (cmd == SET_KEY) { - struct sec_key_uni *sec_key; - u8 cipher; + /* due to connac3 FW design, we only do remove key for BIGTK; even for + * removal, the field should be filled with SET_KEY + */ + sec->add = SET_KEY; + sec->n_cipher = 1; + sec_key = &sec->key[0]; + sec_key->wlan_idx = cpu_to_le16(wcid->idx); + sec_key->key_id = key->keyidx; - cipher = mt76_connac_mcu_get_cipher(key->cipher); - if (cipher == MCU_CIPHER_NONE) - return -EOPNOTSUPP; + if (cmd != SET_KEY) + return 0; - sec_key = &sec->key[0]; - sec_key->wlan_idx = cpu_to_le16(wcid->idx); - sec_key->mgmt_prot = 0; - sec_key->cipher_id = cipher; - sec_key->cipher_len = sizeof(*sec_key); - sec_key->key_id = key->keyidx; - sec_key->key_len = key->keylen; - sec_key->need_resp = 0; - memcpy(sec_key->key, key->key, key->keylen); - - if (cipher == MCU_CIPHER_TKIP) { - /* Rx/Tx MIC keys are swapped */ - memcpy(sec_key->key + 16, key->key + 24, 8); - memcpy(sec_key->key + 24, key->key + 16, 8); - } + cipher = mt76_connac_mcu_get_cipher(key->cipher); + if (cipher == MCU_CIPHER_NONE) + return -EOPNOTSUPP; - sec->n_cipher = 1; - } else { - sec->n_cipher = 0; + sec_key->mgmt_prot = 0; + sec_key->cipher_id = cipher; + sec_key->cipher_len = sizeof(*sec_key); + sec_key->key_len = key->keylen; + sec_key->need_resp = 0; + memcpy(sec_key->key, key->key, key->keylen); + + if (cipher == MCU_CIPHER_TKIP) { + /* Rx/Tx MIC keys are swapped */ + memcpy(sec_key->key + 16, key->key + 24, 8); + memcpy(sec_key->key + 24, key->key + 16, 8); + return 0; } - return 0; -} - -int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, - struct ieee80211_key_conf *key, int mcu_cmd, - struct mt76_wcid *wcid, enum set_key_cmd cmd) -{ - struct mt76_vif_link *mvif = (struct mt76_vif_link *)vif->drv_priv; - struct sk_buff *skb; - int ret; - - skb = __mt76_connac_mcu_alloc_sta_req(dev, mvif, wcid, - MT7996_STA_UPDATE_MAX_SIZE); - if (IS_ERR(skb)) - return PTR_ERR(skb); - - ret = mt7996_mcu_sta_key_tlv(wcid, skb, key, cmd); - if (ret) - return ret; - - return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); -} - -static int mt7996_mcu_get_pn(struct mt7996_dev *dev, - struct mt7996_vif_link *link, - struct mt7996_sta_link *msta_link, u8 *pn) -{ -#define TSC_TYPE_BIGTK_PN 2 - struct sta_rec_pn_info *pn_info; - struct sk_buff *skb, *rskb; - struct tlv *tlv; - int ret; - - skb = mt76_connac_mcu_alloc_sta_req(&dev->mt76, &link->mt76, - &msta_link->wcid); - if (IS_ERR(skb)) - return PTR_ERR(skb); - - tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_PN_INFO, sizeof(*pn_info)); - pn_info = (struct sta_rec_pn_info *)tlv; - - pn_info->tsc_type = TSC_TYPE_BIGTK_PN; - ret = mt76_mcu_skb_send_and_get_msg(&dev->mt76, skb, - MCU_WM_UNI_CMD_QUERY(STA_REC_UPDATE), - true, &rskb); - if (ret) - return ret; + if (sec_key->key_id != 6 && sec_key->key_id != 7) + return 0; - skb_pull(rskb, 4); + switch (key->cipher) { + case WLAN_CIPHER_SUITE_AES_CMAC: + sec_key->cipher_id = MCU_CIPHER_BCN_PROT_CMAC_128; + break; + case WLAN_CIPHER_SUITE_BIP_GMAC_128: + sec_key->cipher_id = MCU_CIPHER_BCN_PROT_GMAC_128; + break; + case WLAN_CIPHER_SUITE_BIP_GMAC_256: + sec_key->cipher_id = MCU_CIPHER_BCN_PROT_GMAC_256; + break; + case WLAN_CIPHER_SUITE_BIP_CMAC_256: + if (!is_mt7990(dev)) + return -EOPNOTSUPP; + sec_key->cipher_id = MCU_CIPHER_BCN_PROT_CMAC_256; + break; + default: + return -EOPNOTSUPP; + } - pn_info = (struct sta_rec_pn_info *)rskb->data; - if (le16_to_cpu(pn_info->tag) == STA_REC_PN_INFO) - memcpy(pn, pn_info->pn, 6); + sec_key->bcn_mode = is_mt7990(dev) ? BP_HW_MODE : BP_SW_MODE; - dev_kfree_skb(rskb); return 0; } -int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, - struct mt7996_vif_link *link, - struct mt7996_sta_link *msta_link, - struct ieee80211_key_conf *key) +int mt7996_mcu_add_key(struct mt76_dev *dev, struct mt7996_vif_link *link, + struct ieee80211_key_conf *key, int mcu_cmd, + struct mt76_wcid *wcid, enum set_key_cmd cmd) { - struct mt7996_mcu_bcn_prot_tlv *bcn_prot; struct sk_buff *skb; - struct tlv *tlv; - u8 pn[6] = {}; - int len = sizeof(struct bss_req_hdr) + - sizeof(struct mt7996_mcu_bcn_prot_tlv); int ret; - skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &link->mt76, len); + skb = __mt76_connac_mcu_alloc_sta_req(dev, (struct mt76_vif_link *)link, + wcid, MT7996_STA_UPDATE_MAX_SIZE); if (IS_ERR(skb)) return PTR_ERR(skb); - tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_BCN_PROT, sizeof(*bcn_prot)); - - bcn_prot = (struct mt7996_mcu_bcn_prot_tlv *)tlv; - - ret = mt7996_mcu_get_pn(dev, link, msta_link, pn); + ret = mt7996_mcu_sta_key_tlv(dev, wcid, skb, key, cmd); if (ret) { dev_kfree_skb(skb); return ret; } - switch (key->cipher) { - case WLAN_CIPHER_SUITE_AES_CMAC: - bcn_prot->cipher_id = MCU_CIPHER_BCN_PROT_CMAC_128; - break; - case WLAN_CIPHER_SUITE_BIP_GMAC_128: - bcn_prot->cipher_id = MCU_CIPHER_BCN_PROT_GMAC_128; - break; - case WLAN_CIPHER_SUITE_BIP_GMAC_256: - bcn_prot->cipher_id = MCU_CIPHER_BCN_PROT_GMAC_256; - break; - case WLAN_CIPHER_SUITE_BIP_CMAC_256: - default: - dev_err(dev->mt76.dev, "Not supported Bigtk Cipher\n"); - dev_kfree_skb(skb); - return -EOPNOTSUPP; - } - - pn[0]++; - memcpy(bcn_prot->pn, pn, 6); - bcn_prot->enable = BP_SW_MODE; - memcpy(bcn_prot->key, key->key, WLAN_MAX_KEY_LEN); - bcn_prot->key_id = key->keyidx; - - return mt76_mcu_skb_send_msg(&dev->mt76, skb, - MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); + return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); } int mt7996_mcu_add_dev_info(struct mt7996_phy *phy, struct ieee80211_vif *vif, @@ -2748,12 +2754,18 @@ mt7996_mcu_beacon_mbss(struct sk_buff *rskb, struct sk_buff *skb, static void mt7996_mcu_beacon_cont(struct mt7996_dev *dev, struct ieee80211_bss_conf *link_conf, + struct mt7996_vif_link *link, struct sk_buff *rskb, struct sk_buff *skb, struct bss_bcn_content_tlv *bcn, struct ieee80211_mutable_offsets *offs) { - struct mt76_wcid *wcid = &dev->mt76.global_wcid; - u8 *buf; + u8 *buf, keyidx = link->msta_link.wcid.hw_key_idx2; + struct mt76_wcid *wcid; + + if (is_mt7990(&dev->mt76) && (keyidx == 6 || keyidx == 7)) + wcid = &link->msta_link.wcid; + else + wcid = &dev->mt76.global_wcid; bcn->pkt_len = cpu_to_le16(MT_TXD_SIZE + skb->len); bcn->tim_ie_pos = cpu_to_le16(offs->tim_offset); @@ -2775,7 +2787,7 @@ mt7996_mcu_beacon_cont(struct mt7996_dev *dev, } int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - struct ieee80211_bss_conf *link_conf) + struct ieee80211_bss_conf *link_conf, bool enabled) { struct mt7996_dev *dev = mt7996_hw_dev(hw); struct mt7996_vif_link *link = mt7996_vif_conf_link(dev, vif, link_conf); @@ -2786,7 +2798,6 @@ int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct tlv *tlv; struct bss_bcn_content_tlv *bcn; int len, extra_len = 0; - bool enabled = link_conf->enable_beacon; if (link_conf->nontransmitted) return 0; @@ -2829,7 +2840,7 @@ int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif, info = IEEE80211_SKB_CB(skb); info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, mlink->band_idx); - mt7996_mcu_beacon_cont(dev, link_conf, rskb, skb, bcn, &offs); + mt7996_mcu_beacon_cont(dev, link_conf, link, rskb, skb, bcn, &offs); if (link_conf->bssid_indicator) mt7996_mcu_beacon_mbss(rskb, skb, bcn, &offs); mt7996_mcu_beacon_cntdwn(rskb, skb, &offs, link_conf->csa_active); @@ -3451,6 +3462,9 @@ int mt7996_mcu_set_tx(struct mt7996_dev *dev, struct ieee80211_vif *vif, #define WMM_PARAM_SET (WMM_AIFS_SET | WMM_CW_MIN_SET | \ WMM_CW_MAX_SET | WMM_TXOP_SET) struct mt7996_vif_link *link = mt7996_vif_conf_link(dev, vif, link_conf); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + unsigned int link_id = link_conf->link_id; + struct mt7996_vif_link_info *link_info = &mvif->link_info[link_id]; struct { u8 bss_idx; u8 __rsv[3]; @@ -3468,7 +3482,7 @@ int mt7996_mcu_set_tx(struct mt7996_dev *dev, struct ieee80211_vif *vif, skb_put_data(skb, &hdr, sizeof(hdr)); for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) { - struct ieee80211_tx_queue_params *q = &link->queue_params[ac]; + struct ieee80211_tx_queue_params *q = &link_info->queue_params[ac]; struct edca *e; struct tlv *tlv; diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7996/mcu.h index 130ea95626d5..e0b83ac9f5e2 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/mcu.h +++ b/sys/contrib/dev/mediatek/mt76/mt7996/mcu.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2022 MediaTek Inc. */ @@ -351,17 +351,6 @@ enum { BP_HW_MODE, }; -struct mt7996_mcu_bcn_prot_tlv { - __le16 tag; - __le16 len; - u8 pn[6]; - u8 enable; - u8 cipher_id; - u8 key[WLAN_MAX_KEY_LEN]; - u8 key_id; - u8 __rsv[3]; -} __packed; - struct bss_ra_tlv { __le16 tag; __le16 len; @@ -481,7 +470,8 @@ struct bss_mld_tlv { u8 own_mld_id; u8 mac_addr[ETH_ALEN]; u8 remap_idx; - u8 __rsv[3]; + u8 link_id; + u8 __rsv[2]; } __packed; struct sta_rec_ht_uni { @@ -530,6 +520,9 @@ struct sec_key_uni { u8 key_len; u8 need_resp; u8 key[32]; + u8 pn[6]; + u8 bcn_mode; + u8 _rsv; } __packed; struct sta_rec_sec_uni { diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mmio.c b/sys/contrib/dev/mediatek/mt76/mt7996/mmio.c index 05a3d9019167..59a2a5f7534a 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/mmio.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/mmio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -474,14 +474,15 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, #ifdef CONFIG_NET_MEDIATEK_SOC_WED struct mtk_wed_device *wed = &dev->mt76.mmio.wed; struct pci_dev *pci_dev = pdev_ptr; - u32 hif1_ofs = 0; + u32 hif1_ofs; if (!wed_enable) return 0; - dev->has_rro = true; + dev->mt76.hwrro_mode = is_mt7996(&dev->mt76) ? MT76_HWRRO_V3 + : MT76_HWRRO_V3_1; - hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); + hif1_ofs = dev->hif2 ? MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0) : 0; if (hif2) wed = &dev->mt76.mmio.wed_hif2; @@ -505,11 +506,18 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.wpdma_tx = wed->wlan.phy_base + hif1_ofs + MT_TXQ_RING_BASE(0) + MT7996_TXQ_BAND2 * MT_RING_SIZE; - if (dev->has_rro) { - wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_TXFREE2 * MT_RING_SIZE; - wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_EXT) - 1; + if (mt7996_has_hwrro(dev)) { + if (is_mt7996(&dev->mt76)) { + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_EXT) - 1; + wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_TXFREE2 * MT_RING_SIZE; + } else { + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_BAND1_EXT) - 1; + wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_MCU_WA_EXT * MT_RING_SIZE; + } } else { wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + MT_RXQ_RING_BASE(0) + @@ -518,14 +526,14 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, } wed->wlan.wpdma_rx_glo = wed->wlan.phy_base + hif1_ofs + MT_WFDMA0_GLO_CFG; - wed->wlan.wpdma_rx = wed->wlan.phy_base + hif1_ofs + - MT_RXQ_RING_BASE(MT7996_RXQ_BAND0) + - MT7996_RXQ_BAND0 * MT_RING_SIZE; + wed->wlan.wpdma_rx[0] = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(MT7996_RXQ_BAND2) + + MT7996_RXQ_BAND2 * MT_RING_SIZE; wed->wlan.id = MT7996_DEVICE_ID_2; wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND2) - 1; } else { - wed->wlan.hw_rro = dev->has_rro; /* default on */ + wed->wlan.hw_rro = mt7996_has_hwrro(dev); wed->wlan.wpdma_int = wed->wlan.phy_base + MT_INT_SOURCE_CSR; wed->wlan.wpdma_mask = wed->wlan.phy_base + MT_INT_MASK_CSR; wed->wlan.wpdma_tx = wed->wlan.phy_base + MT_TXQ_RING_BASE(0) + @@ -533,16 +541,26 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.wpdma_rx_glo = wed->wlan.phy_base + MT_WFDMA0_GLO_CFG; - wed->wlan.wpdma_rx = wed->wlan.phy_base + - MT_RXQ_RING_BASE(MT7996_RXQ_BAND0) + - MT7996_RXQ_BAND0 * MT_RING_SIZE; + wed->wlan.wpdma_rx[0] = wed->wlan.phy_base + + MT_RXQ_RING_BASE(MT7996_RXQ_BAND0) + + MT7996_RXQ_BAND0 * MT_RING_SIZE; wed->wlan.wpdma_rx_rro[0] = wed->wlan.phy_base + MT_RXQ_RING_BASE(MT7996_RXQ_RRO_BAND0) + MT7996_RXQ_RRO_BAND0 * MT_RING_SIZE; - wed->wlan.wpdma_rx_rro[1] = wed->wlan.phy_base + hif1_ofs + - MT_RXQ_RING_BASE(MT7996_RXQ_RRO_BAND2) + - MT7996_RXQ_RRO_BAND2 * MT_RING_SIZE; + if (is_mt7996(&dev->mt76)) { + wed->wlan.wpdma_rx_rro[1] = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(MT7996_RXQ_RRO_BAND2) + + MT7996_RXQ_RRO_BAND2 * MT_RING_SIZE; + } else { + wed->wlan.wpdma_rx_rro[1] = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(MT7996_RXQ_RRO_BAND1) + + MT7996_RXQ_RRO_BAND1 * MT_RING_SIZE; + wed->wlan.wpdma_rx[1] = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(MT7996_RXQ_BAND1) + + MT7996_RXQ_BAND1 * MT_RING_SIZE; + } + wed->wlan.wpdma_rx_pg = wed->wlan.phy_base + MT_RXQ_RING_BASE(MT7996_RXQ_MSDU_PG_BAND0) + MT7996_RXQ_MSDU_PG_BAND0 * MT_RING_SIZE; @@ -552,10 +570,14 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.rx_size = SKB_WITH_OVERHEAD(MT_RX_BUF_SIZE); wed->wlan.rx_tbit[0] = ffs(MT_INT_RX_DONE_BAND0) - 1; - wed->wlan.rx_tbit[1] = ffs(MT_INT_RX_DONE_BAND2) - 1; - wed->wlan.rro_rx_tbit[0] = ffs(MT_INT_RX_DONE_RRO_BAND0) - 1; - wed->wlan.rro_rx_tbit[1] = ffs(MT_INT_RX_DONE_RRO_BAND2) - 1; + if (is_mt7996(&dev->mt76)) { + wed->wlan.rx_tbit[1] = ffs(MT_INT_RX_DONE_BAND2) - 1; + wed->wlan.rro_rx_tbit[1] = ffs(MT_INT_RX_DONE_RRO_BAND2) - 1; + } else { + wed->wlan.rx_tbit[1] = ffs(MT_INT_RX_DONE_BAND1) - 1; + wed->wlan.rro_rx_tbit[1] = ffs(MT_INT_RX_DONE_RRO_BAND1) - 1; + } wed->wlan.rx_pg_tbit[0] = ffs(MT_INT_RX_DONE_MSDU_PG_BAND0) - 1; wed->wlan.rx_pg_tbit[1] = ffs(MT_INT_RX_DONE_MSDU_PG_BAND1) - 1; @@ -563,20 +585,32 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND0) - 1; wed->wlan.tx_tbit[1] = ffs(MT_INT_TX_DONE_BAND1) - 1; - if (dev->has_rro) { - wed->wlan.wpdma_txfree = wed->wlan.phy_base + MT_RXQ_RING_BASE(0) + - MT7996_RXQ_TXFREE0 * MT_RING_SIZE; - wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_MAIN) - 1; + if (is_mt7996(&dev->mt76)) { + if (mt7996_has_hwrro(dev)) { + wed->wlan.wpdma_txfree = wed->wlan.phy_base + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_TXFREE0 * MT_RING_SIZE; + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_MAIN) - 1; + } else { + wed->wlan.wpdma_txfree = wed->wlan.phy_base + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_MCU_WA_MAIN * MT_RING_SIZE; + wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_MAIN) - 1; + } } else { wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_MAIN) - 1; wed->wlan.wpdma_txfree = wed->wlan.phy_base + MT_RXQ_RING_BASE(0) + MT7996_RXQ_MCU_WA_MAIN * MT_RING_SIZE; } dev->mt76.rx_token_size = MT7996_TOKEN_SIZE + wed->wlan.rx_npkt; + + if (dev->hif2 && is_mt7992(&dev->mt76)) + wed->wlan.id = 0x7992; } wed->wlan.nbuf = MT7996_HW_TOKEN_SIZE; wed->wlan.token_start = MT7996_TOKEN_SIZE - wed->wlan.nbuf; + wed->wlan.hif2 = hif2; wed->wlan.amsdu_max_subframes = 8; wed->wlan.amsdu_max_len = 1536; @@ -591,8 +625,10 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.reset_complete = mt76_wed_reset_complete; } - if (mtk_wed_device_attach(wed)) + if (mtk_wed_device_attach(wed)) { + dev->mt76.hwrro_mode = MT76_HWRRO_OFF; return 0; + } *irq = wed->irq; dev->mt76.dma_dev = wed->dev; @@ -686,9 +722,18 @@ void mt7996_dual_hif_set_irq_mask(struct mt7996_dev *dev, bool write_reg, static void mt7996_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q) { - struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + if (q == MT_RXQ_NPU0 || q == MT_RXQ_NPU1) { + struct airoha_npu *npu; - mt7996_irq_enable(dev, MT_INT_RX(q)); + npu = rcu_dereference(mdev->mmio.npu); + if (npu) + airoha_npu_wlan_enable_irq(npu, q - MT_RXQ_NPU0); + } else { + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, + mt76); + + mt7996_irq_enable(dev, MT_INT_RX(q)); + } } /* TODO: support 2/4/6/8 MSI-X vectors */ @@ -705,12 +750,18 @@ static void mt7996_irq_tasklet(struct tasklet_struct *t) dev->mt76.mmio.irqmask); if (intr1 & MT_INT_RX_TXFREE_EXT) napi_schedule(&dev->mt76.napi[MT_RXQ_TXFREE_BAND2]); + + if (intr1 & MT_INT_RX_DONE_BAND2_EXT) + napi_schedule(&dev->mt76.napi[MT_RXQ_BAND2]); + + if (intr1 & MT_INT_RX_TXFREE_BAND1_EXT) + napi_schedule(&dev->mt76.napi[MT_RXQ_BAND1_WA]); } if (mtk_wed_device_active(wed)) { mtk_wed_device_irq_set_mask(wed, 0); intr = mtk_wed_device_irq_get(wed, dev->mt76.mmio.irqmask); - intr |= (intr1 & ~MT_INT_RX_TXFREE_EXT); + intr |= (intr1 & ~MT_INT_TX_RX_DONE_EXT); } else { mt76_wr(dev, MT_INT_MASK_CSR, 0); if (dev->hif2) @@ -796,6 +847,8 @@ struct mt7996_dev *mt7996_mmio_probe(struct device *pdev, .rx_skb = mt7996_queue_rx_skb, .rx_check = mt7996_rx_check, .rx_poll_complete = mt7996_rx_poll_complete, + .rx_rro_ind_process = mt7996_rro_rx_process, + .rx_rro_add_msdu_page = mt7996_rro_msdu_page_add, .update_survey = mt7996_update_channel, .set_channel = mt7996_set_channel, .vif_link_add = mt7996_vif_link_add, diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mt7996.h b/sys/contrib/dev/mediatek/mt76/mt7996/mt7996.h index 81cc1ca22cc6..b4b07d1514a4 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/mt7996.h +++ b/sys/contrib/dev/mediatek/mt76/mt7996/mt7996.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2022 MediaTek Inc. */ @@ -115,6 +115,8 @@ #define MT7996_CRIT_TEMP 110 #define MT7996_MAX_TEMP 120 +#define MT7996_MAX_HIF_RXD_IN_PG 5 +#define MT7996_RRO_MSDU_PG_HASH_SIZE 127 #define MT7996_RRO_MAX_SESSION 1024 #define MT7996_RRO_WINDOW_MAX_LEN 1024 #define MT7996_RRO_ADDR_ELEM_LEN 128 @@ -131,6 +133,10 @@ #define MT7996_RX_MSDU_PAGE_SIZE (128 + \ SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) +/* RRO 3.1 */ +#define MT7996_RRO_MSDU_PG_CR_CNT 8 +#define MT7996_RRO_MSDU_PG_SIZE_PER_CR 0x10000 + struct mt7996_vif; struct mt7996_sta; struct mt7996_dfs_pulse; @@ -181,7 +187,7 @@ enum mt7996_rxq_id { MT7996_RXQ_BAND1 = 5, /* for mt7992 */ MT7996_RXQ_BAND2 = 5, MT7996_RXQ_RRO_BAND0 = 8, - MT7996_RXQ_RRO_BAND1 = 8,/* unused */ + MT7996_RXQ_RRO_BAND1 = 9, MT7996_RXQ_RRO_BAND2 = 6, MT7996_RXQ_MSDU_PG_BAND0 = 10, MT7996_RXQ_MSDU_PG_BAND1 = 11, @@ -190,6 +196,7 @@ enum mt7996_rxq_id { MT7996_RXQ_TXFREE1 = 9, MT7996_RXQ_TXFREE2 = 7, MT7996_RXQ_RRO_IND = 0, + MT7996_RXQ_RRO_RXDMAD_C = 0, MT7990_RXQ_TXFREE0 = 6, MT7990_RXQ_TXFREE1 = 7, }; @@ -239,6 +246,7 @@ struct mt7996_sta { struct mt7996_sta_link deflink; /* must be first */ struct mt7996_sta_link __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS]; u8 deflink_id; + u8 seclink_id; struct mt7996_vif *vif; }; @@ -249,13 +257,23 @@ struct mt7996_vif_link { struct mt7996_sta_link msta_link; struct mt7996_phy *phy; - struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS]; struct cfg80211_bitrate_mask bitrate_mask; + + u8 mld_idx; +}; + +struct mt7996_vif_link_info { + struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS]; }; struct mt7996_vif { struct mt7996_vif_link deflink; /* must be first */ struct mt76_vif_data mt76; + + struct mt7996_vif_link_info link_info[IEEE80211_MLD_MAX_NUM_LINKS]; + + u8 mld_group_idx; + u8 mld_remap_idx; }; /* crash-dump */ @@ -273,15 +291,17 @@ struct mt7996_hif { struct device *dev; void __iomem *regs; int irq; + + enum pci_bus_speed speed; + enum pcie_link_width width; }; +#define WED_RRO_ADDR_SIGNATURE_MASK GENMASK(31, 24) +#define WED_RRO_ADDR_COUNT_MASK GENMASK(14, 4) +#define WED_RRO_ADDR_HEAD_HIGH_MASK GENMASK(3, 0) struct mt7996_wed_rro_addr { - u32 head_low; - u32 head_high : 4; - u32 count: 11; - u32 oor: 1; - u32 rsv : 8; - u32 signature : 8; + __le32 head_low; + __le32 data; }; struct mt7996_wed_rro_session_id { @@ -289,6 +309,44 @@ struct mt7996_wed_rro_session_id { u16 id; }; +struct mt7996_msdu_page { + struct list_head list; + + struct mt76_queue *q; + dma_addr_t dma_addr; + void *buf; +}; + +/* data1 */ +#define RRO_HIF_DATA1_LS_MASK BIT(30) +#define RRO_HIF_DATA1_SDL_MASK GENMASK(29, 16) +/* data4 */ +#define RRO_HIF_DATA4_RX_TOKEN_ID_MASK GENMASK(15, 0) +struct mt7996_rro_hif { + __le32 data0; + __le32 data1; + __le32 data2; + __le32 data3; + __le32 data4; + __le32 data5; +}; + +#define MSDU_PAGE_INFO_OWNER_MASK BIT(31) +#define MSDU_PAGE_INFO_PG_HIGH_MASK GENMASK(3, 0) +struct mt7996_msdu_page_info { + struct mt7996_rro_hif rxd[MT7996_MAX_HIF_RXD_IN_PG]; + __le32 pg_low; + __le32 data; +}; + +#define MT7996_MAX_RRO_RRS_RING 4 +struct mt7996_rro_queue_regs_emi { + struct { + __le16 idx; + __le16 rsv; + } ring[MT7996_MAX_RRO_RRS_RING]; +}; + struct mt7996_phy { struct mt76_phy *mt76; struct mt7996_dev *dev; @@ -340,6 +398,9 @@ struct mt7996_dev { u32 q_int_mask[MT7996_MAX_QUEUE]; u32 q_wfdma_mask; + u64 mld_idx_mask; + u64 mld_remap_idx_mask; + const struct mt76_bus_ops *bus_ops; struct mt7996_phy phy; @@ -380,7 +441,6 @@ struct mt7996_dev { bool flash_mode:1; bool has_eht:1; - bool has_rro:1; struct { struct { @@ -395,10 +455,25 @@ struct mt7996_dev { void *ptr; dma_addr_t phy_addr; } session; + struct { + void *ptr; + dma_addr_t phy_addr; + } msdu_pg[MT7996_RRO_MSDU_PG_CR_CNT]; + struct { + struct mt7996_rro_queue_regs_emi *ptr; + dma_addr_t phy_addr; + } emi_rings_cpu; + struct { + struct mt7996_rro_queue_regs_emi *ptr; + dma_addr_t phy_addr; + } emi_rings_dma; struct work_struct work; struct list_head poll_list; spinlock_t lock; + + struct list_head page_cache; + struct list_head page_map[MT7996_RRO_MSDU_PG_HASH_SIZE]; } wed_rro; bool ibf; @@ -555,6 +630,7 @@ extern struct pci_driver mt7996_hif_driver; struct mt7996_dev *mt7996_mmio_probe(struct device *pdev, void __iomem *mem_base, u32 device_id); +void mt7996_rro_hw_init(struct mt7996_dev *dev); void mt7996_wfsys_reset(struct mt7996_dev *dev); irqreturn_t mt7996_irq_handler(int irq, void *dev_instance); u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif_link *link); @@ -607,16 +683,15 @@ int mt7996_mcu_teardown_mld_sta(struct mt7996_dev *dev, struct mt7996_sta_link *msta_link); int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev, struct ieee80211_ampdu_params *params, - struct mt7996_vif_link *link, - struct mt7996_sta_link *msta_link, bool enable); + struct ieee80211_vif *vif, bool enable); int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev, struct ieee80211_ampdu_params *params, - struct mt7996_vif_link *link, bool enable); + struct ieee80211_vif *vif, bool enable); int mt7996_mcu_update_bss_color(struct mt7996_dev *dev, struct mt76_vif_link *mlink, struct cfg80211_he_bss_color *he_bss_color); int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - struct ieee80211_bss_conf *link_conf); + struct ieee80211_bss_conf *link_conf, bool enabled); int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev, struct ieee80211_bss_conf *link_conf, struct mt7996_vif_link *link, u32 changed); @@ -672,6 +747,11 @@ int mt7996_mcu_get_all_sta_info(struct mt7996_phy *phy, u16 tag); int mt7996_mcu_wed_rro_reset_sessions(struct mt7996_dev *dev, u16 id); int mt7996_mcu_set_sniffer_mode(struct mt7996_phy *phy, bool enabled); +static inline bool mt7996_has_hwrro(struct mt7996_dev *dev) +{ + return dev->mt76.hwrro_mode != MT76_HWRRO_OFF; +} + static inline u8 mt7996_max_interface_num(struct mt7996_dev *dev) { return min(MT7996_MAX_INTERFACES * (1 + mt7996_band_valid(dev, MT_BAND1) + @@ -710,7 +790,7 @@ void mt7996_memcpy_fromio(struct mt7996_dev *dev, void *buf, u32 offset, static inline u16 mt7996_rx_chainmask(struct mt7996_phy *phy) { - int max_nss = hweight8(phy->mt76->hw->wiphy->available_antennas_tx); + int max_nss = hweight16(phy->orig_antenna_mask); int cur_nss = hweight8(phy->mt76->antenna_mask); u16 tx_chainmask = phy->mt76->chainmask; @@ -746,6 +826,8 @@ void mt7996_mac_twt_teardown_flow(struct mt7996_dev *dev, struct mt7996_vif_link *link, struct mt7996_sta_link *msta_link, u8 flowid); +void mt7996_mac_sta_deinit_link(struct mt7996_dev *dev, + struct mt7996_sta_link *msta_link); void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw, struct ieee80211_sta *sta, struct ieee80211_twt_setup *twt); @@ -756,6 +838,10 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, void mt7996_tx_token_put(struct mt7996_dev *dev); void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, struct sk_buff *skb, u32 *info); +void mt7996_rro_msdu_page_map_free(struct mt7996_dev *dev); +int mt7996_rro_msdu_page_add(struct mt76_dev *mdev, struct mt76_queue *q, + dma_addr_t dma_addr, void *data); +void mt7996_rro_rx_process(struct mt76_dev *mdev, void *data); bool mt7996_rx_check(struct mt76_dev *mdev, void *data, int len); void mt7996_stats_work(struct work_struct *work); int mt76_dfs_start_rdd(struct mt7996_dev *dev, bool force); @@ -766,7 +852,7 @@ void mt7996_update_channel(struct mt76_phy *mphy); int mt7996_init_debugfs(struct mt7996_dev *dev); void mt7996_debugfs_rx_fw_monitor(struct mt7996_dev *dev, const void *data, int len); bool mt7996_debugfs_rx_log(struct mt7996_dev *dev, const void *data, int len); -int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, +int mt7996_mcu_add_key(struct mt76_dev *dev, struct mt7996_vif_link *link, struct ieee80211_key_conf *key, int mcu_cmd, struct mt76_wcid *wcid, enum set_key_cmd cmd); int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, @@ -781,6 +867,9 @@ int mt7996_mcu_cp_support(struct mt7996_dev *dev, u8 mode); #ifdef CONFIG_MAC80211_DEBUGFS void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct dentry *dir); +void mt7996_link_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_link_sta *link_sta, + struct dentry *dir); #endif int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, bool hif2, int *irq); @@ -790,8 +879,27 @@ u32 mt7996_wed_init_buf(void *ptr, dma_addr_t phys, int token_id); int mt7996_mtk_init_debugfs(struct mt7996_phy *phy, struct dentry *dir); #endif -#ifdef CONFIG_NET_MEDIATEK_SOC_WED int mt7996_dma_rro_init(struct mt7996_dev *dev); -#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ + +#ifdef CONFIG_MT7996_NPU +int mt7996_npu_hw_init(struct mt7996_dev *dev); +int mt7996_npu_hw_stop(struct mt7996_dev *dev); +int mt7996_npu_rx_queues_init(struct mt7996_dev *dev); +#else +static inline int mt7996_npu_hw_init(struct mt7996_dev *dev) +{ + return 0; +} + +static inline int mt7996_npu_hw_stop(struct mt7996_dev *dev) +{ + return 0; +} + +static inline int mt7996_npu_rx_queues_init(struct mt7996_dev *dev) +{ + return 0; +} +#endif /* CONFIG_MT7996_NPU */ #endif diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/pci.c b/sys/contrib/dev/mediatek/mt76/mt7996/pci.c index ed32ff3089b9..241599dcbc72 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/pci.c +++ b/sys/contrib/dev/mediatek/mt76/mt7996/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2022 MediaTek Inc. */ @@ -105,6 +105,7 @@ static int mt7996_pci_hif2_probe(struct pci_dev *pdev) hif->dev = &pdev->dev; hif->regs = pcim_iomap_table(pdev)[0]; hif->irq = pdev->irq; + pcie_bandwidth_available(pdev, NULL, &hif->speed, &hif->width); spin_lock_bh(&hif_lock); list_add(&hif->list, &hif_list); spin_unlock_bh(&hif_lock); @@ -155,6 +156,10 @@ static int mt7996_pci_probe(struct pci_dev *pdev, mdev = &dev->mt76; mt7996_wfsys_reset(dev); hif2 = mt7996_pci_init_hif2(pdev); + dev->hif2 = hif2; + + mt76_npu_init(mdev, pci_resource_start(pdev, 0), + pdev->bus && pci_domain_nr(pdev->bus) ? 3 : 2); ret = mt7996_mmio_wed_init(dev, pdev, false, &irq); if (ret < 0) @@ -174,12 +179,11 @@ static int mt7996_pci_probe(struct pci_dev *pdev, goto free_wed_or_irq_vector; mt76_wr(dev, MT_INT_MASK_CSR, 0); - /* master switch of PCIe tnterrupt enable */ + /* master switch of PCIe interrupt enable */ mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); if (hif2) { hif2_dev = container_of(hif2->dev, struct pci_dev, dev); - dev->hif2 = hif2; ret = mt7996_mmio_wed_init(dev, hif2_dev, true, &hif2_irq); if (ret < 0) diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/regs.h b/sys/contrib/dev/mediatek/mt76/mt7996/regs.h index e942c0058731..e48e0e575b64 100644 --- a/sys/contrib/dev/mediatek/mt76/mt7996/regs.h +++ b/sys/contrib/dev/mediatek/mt76/mt7996/regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2022 MediaTek Inc. */ @@ -88,6 +88,8 @@ enum offs_rev { #define MT_RRO_BA_BITMAP_BASE1 MT_RRO_TOP(0xC) #define WF_RRO_AXI_MST_CFG MT_RRO_TOP(0xB8) #define WF_RRO_AXI_MST_CFG_DIDX_OK BIT(12) + +#define MT_RRO_ADDR_ARRAY_BASE0 MT_RRO_TOP(0x30) #define MT_RRO_ADDR_ARRAY_BASE1 MT_RRO_TOP(0x34) #define MT_RRO_ADDR_ARRAY_ELEM_ADDR_SEG_MODE BIT(31) @@ -108,6 +110,19 @@ enum offs_rev { #define MT_RRO_ADDR_ELEM_SEG_ADDR0 MT_RRO_TOP(0x400) +#define MT_RRO_3_0_EMU_CONF MT_RRO_TOP(0x600) +#define MT_RRO_3_0_EMU_CONF_EN_MASK BIT(11) + +#define MT_RRO_3_1_GLOBAL_CONFIG MT_RRO_TOP(0x604) +#define MT_RRO_3_1_GLOBAL_CONFIG_RXDMAD_SEL BIT(6) +#define MT_RRO_3_1_GLOBAL_CONFIG_RX_CIDX_RD_EN BIT(3) +#define MT_RRO_3_1_GLOBAL_CONFIG_RX_DIDX_WR_EN BIT(2) +#define MT_RRO_3_1_GLOBAL_CONFIG_INTERLEAVE_EN BIT(0) + +#define MT_RRO_MSDU_PG_SEG_ADDR0 MT_RRO_TOP(0x620) +#define MT_RRO_RX_RING_AP_CIDX_ADDR MT_RRO_TOP(0x6f0) +#define MT_RRO_RX_RING_AP_DIDX_ADDR MT_RRO_TOP(0x6f4) + #define MT_RRO_ACK_SN_CTRL MT_RRO_TOP(0x50) #define MT_RRO_ACK_SN_CTRL_SN_MASK GENMASK(27, 16) #define MT_RRO_ACK_SN_CTRL_SESSION_MASK GENMASK(11, 0) @@ -412,7 +427,9 @@ enum offs_rev { #define MT_WFDMA0_RX_INT_PCIE_SEL MT_WFDMA0(0x154) #define MT_WFDMA0_RX_INT_SEL_RING3 BIT(3) +#define MT_WFDMA0_RX_INT_SEL_RING5 BIT(5) #define MT_WFDMA0_RX_INT_SEL_RING6 BIT(6) +#define MT_WFDMA0_RX_INT_SEL_RING9 BIT(9) #define MT_WFDMA0_MCU_HOST_INT_ENA MT_WFDMA0(0x1f4) @@ -430,6 +447,7 @@ enum offs_rev { #define MT_WFDMA0_PAUSE_RX_Q_RRO_TH MT_WFDMA0(0x27c) #define WF_WFDMA0_GLO_CFG_EXT0 MT_WFDMA0(0x2b0) +#define WF_WFDMA0_GLO_CFG_EXT0_OUTSTAND_MASK GENMASK(27, 24) #define WF_WFDMA0_GLO_CFG_EXT0_RX_WB_RXD BIT(18) #define WF_WFDMA0_GLO_CFG_EXT0_WED_MERGE_MODE BIT(14) @@ -451,6 +469,8 @@ enum offs_rev { #define MT_WFDMA_HOST_CONFIG MT_WFDMA_EXT_CSR(0x30) #define MT_WFDMA_HOST_CONFIG_PDMA_BAND BIT(0) +#define MT_WFDMA_HOST_CONFIG_BAND0_PCIE1 BIT(20) +#define MT_WFDMA_HOST_CONFIG_BAND1_PCIE1 BIT(21) #define MT_WFDMA_HOST_CONFIG_BAND2_PCIE1 BIT(22) #define MT_WFDMA_EXT_CSR_HIF_MISC MT_WFDMA_EXT_CSR(0x44) @@ -459,6 +479,9 @@ enum offs_rev { #define MT_WFDMA_AXI_R2A_CTRL MT_WFDMA_EXT_CSR(0x500) #define MT_WFDMA_AXI_R2A_CTRL_OUTSTAND_MASK GENMASK(4, 0) +#define MT_WFDMA_AXI_R2A_CTRL2 MT_WFDMA_EXT_CSR(0x508) +#define MT_WFDMA_AXI_R2A_CTRL2_OUTSTAND_MASK GENMASK(31, 28) + #define MT_PCIE_RECOG_ID 0xd7090 #define MT_PCIE_RECOG_ID_MASK GENMASK(30, 0) #define MT_PCIE_RECOG_ID_SEM BIT(31) @@ -492,6 +515,8 @@ enum offs_rev { #define MT_RXQ_RING_BASE(q) (MT_Q_BASE(__RXQ(q)) + 0x500) #define MT_RXQ_RRO_IND_RING_BASE MT_RRO_TOP(0x40) +#define MT_RXQ_RRO_AP_RING_BASE MT_RRO_TOP(0x650) + #define MT_MCUQ_EXT_CTRL(q) (MT_Q_BASE(q) + 0x600 + \ MT_MCUQ_ID(q) * 0x4) #define MT_RXQ_EXT_CTRL(q) (MT_Q_BASE(__RXQ(q)) + 0x680 + \ @@ -514,7 +539,9 @@ enum offs_rev { #define MT_INT_RX_DONE_WA_EXT BIT(3) /* for mt7992 */ #define MT_INT_RX_DONE_WA_TRI BIT(3) #define MT_INT_RX_TXFREE_MAIN BIT(17) +#define MT_INT_RX_TXFREE_BAND1 BIT(15) #define MT_INT_RX_TXFREE_TRI BIT(15) +#define MT_INT_RX_TXFREE_BAND1_EXT BIT(19) /* for mt7992 two PCIE*/ #define MT_INT_RX_TXFREE_BAND0_MT7990 BIT(14) #define MT_INT_RX_TXFREE_BAND1_MT7990 BIT(15) #define MT_INT_RX_DONE_BAND2_EXT BIT(23) @@ -522,9 +549,10 @@ enum offs_rev { #define MT_INT_MCU_CMD BIT(29) #define MT_INT_RX_DONE_RRO_BAND0 BIT(16) -#define MT_INT_RX_DONE_RRO_BAND1 BIT(16) +#define MT_INT_RX_DONE_RRO_BAND1 BIT(17) #define MT_INT_RX_DONE_RRO_BAND2 BIT(14) #define MT_INT_RX_DONE_RRO_IND BIT(11) +#define MT_INT_RX_DONE_RRO_RXDMAD_C BIT(11) #define MT_INT_RX_DONE_MSDU_PG_BAND0 BIT(18) #define MT_INT_RX_DONE_MSDU_PG_BAND1 BIT(19) #define MT_INT_RX_DONE_MSDU_PG_BAND2 BIT(23) @@ -552,6 +580,8 @@ enum offs_rev { #define MT_INT_RRO_RX_DONE (MT_INT_RX(MT_RXQ_RRO_BAND0) | \ MT_INT_RX(MT_RXQ_RRO_BAND1) | \ MT_INT_RX(MT_RXQ_RRO_BAND2) | \ + MT_INT_RX(MT_RXQ_RRO_IND) | \ + MT_INT_RX(MT_RXQ_RRO_RXDMAD_C) | \ MT_INT_RX(MT_RXQ_MSDU_PAGE_BAND0) | \ MT_INT_RX(MT_RXQ_MSDU_PAGE_BAND1) | \ MT_INT_RX(MT_RXQ_MSDU_PAGE_BAND2)) diff --git a/sys/contrib/dev/mediatek/mt76/pci.c b/sys/contrib/dev/mediatek/mt76/pci.c index b5031ca7f73f..833923ab2483 100644 --- a/sys/contrib/dev/mediatek/mt76/pci.c +++ b/sys/contrib/dev/mediatek/mt76/pci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2019 Lorenzo Bianconi <lorenzo@kernel.org> */ diff --git a/sys/contrib/dev/mediatek/mt76/scan.c b/sys/contrib/dev/mediatek/mt76/scan.c index 9b20ccbeb8cf..ff9176cdee3d 100644 --- a/sys/contrib/dev/mediatek/mt76/scan.c +++ b/sys/contrib/dev/mediatek/mt76/scan.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2024 Felix Fietkau <nbd@nbd.name> */ @@ -16,11 +16,13 @@ static void mt76_scan_complete(struct mt76_dev *dev, bool abort) clear_bit(MT76_SCANNING, &phy->state); - if (dev->scan.chan && phy->main_chandef.chan) + if (dev->scan.chan && phy->main_chandef.chan && + !test_bit(MT76_MCU_RESET, &dev->phy.state)) mt76_set_channel(phy, &phy->main_chandef, false); mt76_put_vif_phy_link(phy, dev->scan.vif, dev->scan.mlink); memset(&dev->scan, 0, sizeof(dev->scan)); - ieee80211_scan_completed(phy->hw, &info); + if (!test_bit(MT76_MCU_RESET, &dev->phy.state)) + ieee80211_scan_completed(phy->hw, &info); } void mt76_abort_scan(struct mt76_dev *dev) @@ -28,6 +30,7 @@ void mt76_abort_scan(struct mt76_dev *dev) cancel_delayed_work_sync(&dev->scan_work); mt76_scan_complete(dev, true); } +EXPORT_SYMBOL_GPL(mt76_abort_scan); static void mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) @@ -112,9 +115,6 @@ void mt76_scan_work(struct work_struct *work) local_bh_enable(); out: - if (!duration) - return; - if (dev->scan.chan) duration = max_t(int, duration, msecs_to_jiffies(req->duration + @@ -139,7 +139,8 @@ int mt76_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif, mutex_lock(&dev->mutex); - if (dev->scan.req || phy->roc_vif) { + if (dev->scan.req || phy->roc_vif || + test_bit(MT76_MCU_RESET, &dev->phy.state)) { ret = -EBUSY; goto out; } diff --git a/sys/contrib/dev/mediatek/mt76/sdio.c b/sys/contrib/dev/mediatek/mt76/sdio.c index 8e9576747052..8bae77c761be 100644 --- a/sys/contrib/dev/mediatek/mt76/sdio.c +++ b/sys/contrib/dev/mediatek/mt76/sdio.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. * * This file is written based on mt76/usb.c. diff --git a/sys/contrib/dev/mediatek/mt76/sdio.h b/sys/contrib/dev/mediatek/mt76/sdio.h index 27d5d2077eba..41b89f3de86b 100644 --- a/sys/contrib/dev/mediatek/mt76/sdio.h +++ b/sys/contrib/dev/mediatek/mt76/sdio.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* Copyright (C) 2020 MediaTek Inc. * * Author: Sean Wang <sean.wang@mediatek.com> diff --git a/sys/contrib/dev/mediatek/mt76/sdio_txrx.c b/sys/contrib/dev/mediatek/mt76/sdio_txrx.c index f882d21c9f63..3f314e8e1e69 100644 --- a/sys/contrib/dev/mediatek/mt76/sdio_txrx.c +++ b/sys/contrib/dev/mediatek/mt76/sdio_txrx.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 MediaTek Inc. * * Author: Felix Fietkau <nbd@nbd.name> diff --git a/sys/contrib/dev/mediatek/mt76/testmode.c b/sys/contrib/dev/mediatek/mt76/testmode.c index ca4feccf38ca..6ee160bda882 100644 --- a/sys/contrib/dev/mediatek/mt76/testmode.c +++ b/sys/contrib/dev/mediatek/mt76/testmode.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ #include <linux/random.h> diff --git a/sys/contrib/dev/mediatek/mt76/testmode.h b/sys/contrib/dev/mediatek/mt76/testmode.h index 0590c35c7126..bed1ba40ba94 100644 --- a/sys/contrib/dev/mediatek/mt76/testmode.h +++ b/sys/contrib/dev/mediatek/mt76/testmode.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/trace.c b/sys/contrib/dev/mediatek/mt76/trace.c index f199fcd2a63d..f17cc01017f3 100644 --- a/sys/contrib/dev/mediatek/mt76/trace.c +++ b/sys/contrib/dev/mediatek/mt76/trace.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/trace.h b/sys/contrib/dev/mediatek/mt76/trace.h index 109a07f9733a..794b957ac79d 100644 --- a/sys/contrib/dev/mediatek/mt76/trace.h +++ b/sys/contrib/dev/mediatek/mt76/trace.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/tx.c b/sys/contrib/dev/mediatek/mt76/tx.c index 8ab5840fee57..9ec6d0b53a84 100644 --- a/sys/contrib/dev/mediatek/mt76/tx.c +++ b/sys/contrib/dev/mediatek/mt76/tx.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ @@ -618,7 +618,8 @@ mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && !ieee80211_is_data(hdr->frame_control) && (!ieee80211_is_bufferable_mmpdu(skb) || - ieee80211_is_deauth(hdr->frame_control))) + ieee80211_is_deauth(hdr->frame_control) || + head == &wcid->tx_offchannel)) qid = MT_TXQ_PSD; q = phy->q_tx[qid]; @@ -846,8 +847,10 @@ int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi) spin_lock_bh(&dev->token_lock); - token = idr_alloc(&dev->token, *ptxwi, 0, dev->token_size, GFP_ATOMIC); - if (token >= 0) + token = idr_alloc(&dev->token, *ptxwi, dev->token_start, + dev->token_start + dev->token_size, + GFP_ATOMIC); + if (token >= dev->token_start) dev->token_count++; #ifdef CONFIG_NET_MEDIATEK_SOC_WED diff --git a/sys/contrib/dev/mediatek/mt76/usb.c b/sys/contrib/dev/mediatek/mt76/usb.c index 11f9d2808f15..67c976bb9c22 100644 --- a/sys/contrib/dev/mediatek/mt76/usb.c +++ b/sys/contrib/dev/mediatek/mt76/usb.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/usb_trace.c b/sys/contrib/dev/mediatek/mt76/usb_trace.c index 9942bdd6177b..a04585b4b778 100644 --- a/sys/contrib/dev/mediatek/mt76/usb_trace.c +++ b/sys/contrib/dev/mediatek/mt76/usb_trace.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/usb_trace.h b/sys/contrib/dev/mediatek/mt76/usb_trace.h index 7b261ddb2ac6..93bb69c65a4f 100644 --- a/sys/contrib/dev/mediatek/mt76/usb_trace.h +++ b/sys/contrib/dev/mediatek/mt76/usb_trace.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: ISC */ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> */ diff --git a/sys/contrib/dev/mediatek/mt76/util.c b/sys/contrib/dev/mediatek/mt76/util.c index 37cfa133010f..4ca478fb1acd 100644 --- a/sys/contrib/dev/mediatek/mt76/util.c +++ b/sys/contrib/dev/mediatek/mt76/util.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ diff --git a/sys/contrib/dev/mediatek/mt76/util.h b/sys/contrib/dev/mediatek/mt76/util.h index 73a784fe2707..596fcf18c3c2 100644 --- a/sys/contrib/dev/mediatek/mt76/util.h +++ b/sys/contrib/dev/mediatek/mt76/util.h @@ -1,132 +1,118 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* - * Copyright (c) 2020-2025 Bjoern A. Zeeb <bz@FreeBSD.org> - * - * SPDX-License-Identifier: BSD-2-Clause + * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> */ -#ifndef _MT76_UTIL_H -#define _MT76_UTIL_H +#ifndef __MT76_UTIL_H +#define __MT76_UTIL_H +#include <linux/skbuff.h> +#include <linux/bitops.h> +#include <linux/bitfield.h> +#include <net/mac80211.h> +#if defined(__FreeBSD__) #include <linux/kthread.h> +#endif struct mt76_worker { - void(*fn)(struct mt76_worker *); struct task_struct *task; + void (*fn)(struct mt76_worker *); unsigned long state; }; -enum mt76_worker_state { +enum { MT76_WORKER_SCHEDULED, MT76_WORKER_RUNNING, }; -#if 0 -bool __mt76_poll(struct mt76_dev *, u32, u32, u32, int); -bool __mt76_poll_msec(struct mt76_dev *, u32, u32, u32, int); -int mt76_get_min_avg_rssi(struct mt76_dev *, bool); -#endif -int mt76_wcid_alloc(u32 *, int); -int __mt76_worker_fn(void *); +#define MT76_INCR(_var, _size) \ + (_var = (((_var) + 1) % (_size))) + +int mt76_wcid_alloc(u32 *mask, int size); -/* wcid_phy_mask is [32] */ static inline void -mt76_wcid_mask_set(u32 *mask, u16 bit) +mt76_wcid_mask_set(u32 *mask, int idx) { + mask[idx / 32] |= BIT(idx % 32); +} - mask[bit / 32] |= BIT(bit % 32); +static inline void +mt76_wcid_mask_clear(u32 *mask, int idx) +{ + mask[idx / 32] &= ~BIT(idx % 32); } static inline void -mt76_wcid_mask_clear(u32 *mask, u16 bit) +mt76_skb_set_moredata(struct sk_buff *skb, bool enable) { + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; - mask[bit / 32] &= ~BIT(bit % 32); + if (enable) + hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_MOREDATA); + else + hdr->frame_control &= ~cpu_to_le16(IEEE80211_FCTL_MOREDATA); } -/* See, e.g., __mt76_worker_fn for some details. */ +int __mt76_worker_fn(void *ptr); + static inline int -mt76_worker_setup(struct ieee80211_hw *hw __unused, struct mt76_worker *w, - void (*wfunc)(struct mt76_worker *), const char *name) +mt76_worker_setup(struct ieee80211_hw *hw, struct mt76_worker *w, + void (*fn)(struct mt76_worker *), + const char *name) { - int error; - - if (wfunc) - w->fn = wfunc; + const char *dev_name = wiphy_name(hw->wiphy); + int ret; + if (fn) + w->fn = fn; w->task = kthread_run(__mt76_worker_fn, w, - "mt76-%s", name); + "mt76-%s %s", name, dev_name); - if (!IS_ERR(w->task)) - return (0); + if (IS_ERR(w->task)) { + ret = PTR_ERR(w->task); + w->task = NULL; + return ret; + } - error = PTR_ERR(w->task); - w->task = NULL; - return (error); + return 0; } -static inline void -mt76_worker_schedule(struct mt76_worker *w) +static inline void mt76_worker_schedule(struct mt76_worker *w) { - - if (w->task == NULL) + if (!w->task) return; - if (!test_and_set_bit(MT76_WORKER_SCHEDULED, &w->state) || + if (!test_and_set_bit(MT76_WORKER_SCHEDULED, &w->state) && !test_bit(MT76_WORKER_RUNNING, &w->state)) wake_up_process(w->task); } -static inline void -mt76_worker_enable(struct mt76_worker *w) +static inline void mt76_worker_disable(struct mt76_worker *w) { - - if (w->task == NULL) + if (!w->task) return; - kthread_unpark(w->task); - mt76_worker_schedule(w); + kthread_park(w->task); + WRITE_ONCE(w->state, 0); } -static inline void -mt76_worker_disable(struct mt76_worker *w) +static inline void mt76_worker_enable(struct mt76_worker *w) { - - if (w->task == NULL) + if (!w->task) return; - kthread_park(w->task); - WRITE_ONCE(w->state, 0); + kthread_unpark(w->task); + mt76_worker_schedule(w); } -static inline void -mt76_worker_teardown(struct mt76_worker *w) +static inline void mt76_worker_teardown(struct mt76_worker *w) { - - if (w->task == NULL) + if (!w->task) return; kthread_stop(w->task); w->task = NULL; } -static inline void -mt76_skb_set_moredata(struct sk_buff *skb, bool moredata) -{ - /* - * This would be net80211::IEEE80211_FC1_MORE_DATA - * Implement it as mostly LinuxKPI 802.11 to avoid - * further header pollution and possible conflicts. - */ - struct ieee80211_hdr *hdr; - uint16_t val; - - hdr = (struct ieee80211_hdr *)skb->data; - val = cpu_to_le16(IEEE80211_FC1_MORE_DATA << 8); - if (!moredata) - hdr->frame_control &= ~val; - else - hdr->frame_control |= val; -} - -#endif /* _MT76_UTIL_H */ +#endif diff --git a/sys/contrib/dev/mediatek/mt76/wed.c b/sys/contrib/dev/mediatek/mt76/wed.c index 63f69e152b1c..ed657d952de2 100644 --- a/sys/contrib/dev/mediatek/mt76/wed.c +++ b/sys/contrib/dev/mediatek/mt76/wed.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: ISC +// SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (C) 2023 Lorenzo Bianconi <lorenzo@kernel.org> */ @@ -8,7 +8,7 @@ void mt76_wed_release_rx_buf(struct mtk_wed_device *wed) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt76_dev *dev = mt76_wed_to_dev(wed); int i; for (i = 0; i < dev->rx_token_size; i++) { @@ -31,8 +31,8 @@ EXPORT_SYMBOL_GPL(mt76_wed_release_rx_buf); #ifdef CONFIG_NET_MEDIATEK_SOC_WED u32 mt76_wed_init_rx_buf(struct mtk_wed_device *wed, int size) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); struct mtk_wed_bm_desc *desc = wed->rx_buf_ring.desc; + struct mt76_dev *dev = mt76_wed_to_dev(wed); struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN]; struct mt76_txwi_cache *t = NULL; int i; @@ -80,7 +80,7 @@ EXPORT_SYMBOL_GPL(mt76_wed_init_rx_buf); int mt76_wed_offload_enable(struct mtk_wed_device *wed) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt76_dev *dev = mt76_wed_to_dev(wed); spin_lock_bh(&dev->token_lock); dev->token_size = wed->wlan.token_start; @@ -118,7 +118,7 @@ int mt76_wed_dma_setup(struct mt76_dev *dev, struct mt76_queue *q, bool reset) case MT76_WED_Q_TXFREE: /* WED txfree queue needs ring to be initialized before setup */ q->flags = 0; - mt76_dma_queue_reset(dev, q); + mt76_dma_queue_reset(dev, q, true); mt76_dma_rx_fill(dev, q, false); ret = mtk_wed_device_txfree_ring_setup(q->wed, q->regs); @@ -133,21 +133,21 @@ int mt76_wed_dma_setup(struct mt76_dev *dev, struct mt76_queue *q, bool reset) break; case MT76_WED_RRO_Q_DATA: q->flags &= ~MT_QFLAG_WED; - __mt76_dma_queue_reset(dev, q, false); + mt76_dma_queue_reset(dev, q, false); mtk_wed_device_rro_rx_ring_setup(q->wed, ring, q->regs); q->head = q->ndesc - 1; q->queued = q->head; break; case MT76_WED_RRO_Q_MSDU_PG: q->flags &= ~MT_QFLAG_WED; - __mt76_dma_queue_reset(dev, q, false); + mt76_dma_queue_reset(dev, q, false); mtk_wed_device_msdu_pg_rx_ring_setup(q->wed, ring, q->regs); q->head = q->ndesc - 1; q->queued = q->head; break; case MT76_WED_RRO_Q_IND: q->flags &= ~MT_QFLAG_WED; - mt76_dma_queue_reset(dev, q); + mt76_dma_queue_reset(dev, q, true); mt76_dma_rx_fill(dev, q, false); mtk_wed_device_ind_rx_ring_setup(q->wed, q->regs); break; @@ -164,7 +164,7 @@ EXPORT_SYMBOL_GPL(mt76_wed_dma_setup); void mt76_wed_offload_disable(struct mtk_wed_device *wed) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt76_dev *dev = mt76_wed_to_dev(wed); spin_lock_bh(&dev->token_lock); dev->token_size = dev->drv->token_size; @@ -174,7 +174,7 @@ EXPORT_SYMBOL_GPL(mt76_wed_offload_disable); void mt76_wed_reset_complete(struct mtk_wed_device *wed) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt76_dev *dev = mt76_wed_to_dev(wed); complete(&dev->mmio.wed_reset_complete); } diff --git a/sys/contrib/dev/rtw88/bf.c b/sys/contrib/dev/rtw88/bf.c index 16c6a1d972e5..8969a1e0cb47 100644 --- a/sys/contrib/dev/rtw88/bf.c +++ b/sys/contrib/dev/rtw88/bf.c @@ -129,8 +129,11 @@ void rtw_bf_init_bfer_entry_mu(struct rtw_dev *rtwdev, void rtw_bf_cfg_sounding(struct rtw_dev *rtwdev, struct rtw_vif *vif, enum rtw_trx_desc_rate rate) { + u8 csi_rsc = CSI_RSC_FOLLOW_RX_PACKET_BW; u32 psf_ctl = 0; - u8 csi_rsc = 0x1; + + if (rtwdev->chip->id == RTW_CHIP_TYPE_8822C) + csi_rsc = CSI_RSC_PRIMARY_20M_BW; psf_ctl = rtw_read32(rtwdev, REG_BBPSF_CTRL) | BIT_WMAC_USE_NDPARATE | @@ -392,6 +395,9 @@ void rtw_bf_cfg_csi_rate(struct rtw_dev *rtwdev, u8 rssi, u8 cur_rate, csi_cfg = rtw_read32(rtwdev, REG_BBPSF_CTRL) & ~BIT_MASK_CSI_RATE; cur_rrsr = rtw_read16(rtwdev, REG_RRSR); + if (rtwdev->chip->id == RTW_CHIP_TYPE_8822C) + csi_cfg |= BIT_CSI_FORCE_RATE; + if (rssi >= 40) { if (cur_rate != DESC_RATE54M) { cur_rrsr |= BIT(DESC_RATE54M); diff --git a/sys/contrib/dev/rtw88/bf.h b/sys/contrib/dev/rtw88/bf.h index 7b40c2c03856..a5d3010e6be6 100644 --- a/sys/contrib/dev/rtw88/bf.h +++ b/sys/contrib/dev/rtw88/bf.h @@ -33,6 +33,7 @@ #define BIT_SHIFT_R_MU_RL 12 #define BIT_SHIFT_WMAC_TXMU_ACKPOLICY 4 #define BIT_SHIFT_CSI_RATE 24 +#define BIT_CSI_FORCE_RATE BIT(15) #define BIT_MASK_R_MU_RL (R_MU_RL << BIT_SHIFT_R_MU_RL) #define BIT_MASK_R_MU_TABLE_VALID 0x3f @@ -48,6 +49,12 @@ #define RTW_SND_CTRL_REMOVE 0x98 #define RTW_SND_CTRL_SOUNDING 0x9B +enum csi_rsc { + CSI_RSC_PRIMARY_20M_BW = 0, + CSI_RSC_FOLLOW_RX_PACKET_BW = 1, + CSI_RSC_DUPLICATE_MODE = 2, +}; + enum csi_seg_len { HAL_CSI_SEG_4K = 0, HAL_CSI_SEG_8K = 1, diff --git a/sys/contrib/dev/rtw88/rtw8822bu.c b/sys/contrib/dev/rtw88/rtw8822bu.c index efda9887cc41..2886f470df71 100644 --- a/sys/contrib/dev/rtw88/rtw8822bu.c +++ b/sys/contrib/dev/rtw88/rtw8822bu.c @@ -79,6 +79,8 @@ static const struct usb_device_id rtw_8822bu_id_table[] = { .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, /* D-Link DWA-T185 rev. A1 */ { USB_DEVICE_AND_INTERFACE_INFO(0x0411, 0x03d1, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, /* BUFFALO WI-U2-866DM */ + { USB_DEVICE_AND_INTERFACE_INFO(0x0411, 0x03d0, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, /* BUFFALO WI-U3-866DHP */ {}, }; MODULE_DEVICE_TABLE(usb, rtw_8822bu_id_table); diff --git a/sys/contrib/dev/rtw88/rtw8822cu.c b/sys/contrib/dev/rtw88/rtw8822cu.c index 90fcbb8ec629..a5f8311c022b 100644 --- a/sys/contrib/dev/rtw88/rtw8822cu.c +++ b/sys/contrib/dev/rtw88/rtw8822cu.c @@ -21,6 +21,8 @@ static const struct usb_device_id rtw_8822cu_id_table[] = { .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, { USB_DEVICE_AND_INTERFACE_INFO(0x13b1, 0x0043, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, /* Alpha - Alpha */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3329, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, /* D-Link AC13U rev. A1 */ {}, }; MODULE_DEVICE_TABLE(usb, rtw_8822cu_id_table); diff --git a/sys/contrib/dev/rtw88/sdio.c b/sys/contrib/dev/rtw88/sdio.c index 99d7c629eac6..e35de52d8eb4 100644 --- a/sys/contrib/dev/rtw88/sdio.c +++ b/sys/contrib/dev/rtw88/sdio.c @@ -144,8 +144,10 @@ static u32 rtw_sdio_to_io_address(struct rtw_dev *rtwdev, u32 addr, static bool rtw_sdio_use_direct_io(struct rtw_dev *rtwdev, u32 addr) { + bool might_indirect_under_power_off = rtwdev->chip->id == RTW_CHIP_TYPE_8822C; + if (!test_bit(RTW_FLAG_POWERON, rtwdev->flags) && - !rtw_sdio_is_bus_addr(addr)) + !rtw_sdio_is_bus_addr(addr) && might_indirect_under_power_off) return false; return !rtw_sdio_is_sdio30_supported(rtwdev) || diff --git a/sys/contrib/openzfs/module/nvpair/nvpair.c b/sys/contrib/openzfs/module/nvpair/nvpair.c index eb8c14b4a783..cb3a024ec95c 100644 --- a/sys/contrib/openzfs/module/nvpair/nvpair.c +++ b/sys/contrib/openzfs/module/nvpair/nvpair.c @@ -3246,7 +3246,8 @@ nvs_xdr_nvl_fini(nvstream_t *nvs) * xdrproc_t-compatible callbacks for xdr_array() */ -#if defined(_KERNEL) && defined(__linux__) /* Linux kernel */ +#if (defined(__FreeBSD_version) && __FreeBSD_version >= 1600010) || \ + defined(_KERNEL) && defined(__linux__) /* Linux kernel */ #define NVS_BUILD_XDRPROC_T(type) \ static bool_t \ diff --git a/sys/crypto/openssl/aarch64/vpsm4_ex-armv8.S b/sys/crypto/openssl/aarch64/vpsm4_ex-armv8.S index 5627d6d1c6b4..b8c728e68683 100644 --- a/sys/crypto/openssl/aarch64/vpsm4_ex-armv8.S +++ b/sys/crypto/openssl/aarch64/vpsm4_ex-armv8.S @@ -1,5 +1,5 @@ /* Do not modify. This file is auto-generated from vpsm4_ex-armv8.pl. */ -// Copyright 2022-2025 The OpenSSL Project Authors. All Rights Reserved. +// Copyright 2022-2026 The OpenSSL Project Authors. All Rights Reserved. // // Licensed under the Apache License 2.0 (the "License"). You may not use // this file except in compliance with the License. You can obtain a copy @@ -35,13 +35,25 @@ _vpsm4_ex_consts: .Lshuffles: .quad 0x0B0A090807060504,0x030201000F0E0D0C .Lxts_magic: +#ifndef __AARCH64EB__ .quad 0x0101010101010187,0x0101010101010101 +#else +.quad 0x0101010101010101,0x0101010101010187 +#endif .Lsbox_magic: +#ifndef __AARCH64EB__ .quad 0x0b0e0104070a0d00,0x0306090c0f020508 .quad 0x62185a2042387a00,0x22581a6002783a40 .quad 0x15df62a89e54e923,0xc10bb67c4a803df7 .quad 0xb9aa6b78c1d21300,0x1407c6d56c7fbead .quad 0x6404462679195b3b,0xe383c1a1fe9edcbc +#else +.quad 0x0306090c0f020508,0x0b0e0104070a0d00 +.quad 0x22581a6002783a40,0x62185a2042387a00 +.quad 0xc10bb67c4a803df7,0x15df62a89e54e923 +.quad 0x1407c6d56c7fbead,0xb9aa6b78c1d21300 +.quad 0xe383c1a1fe9edcbc,0x6404462679195b3b +#endif .quad 0x0f0f0f0f0f0f0f0f,0x0f0f0f0f0f0f0f0f .size _vpsm4_ex_consts,.-_vpsm4_ex_consts diff --git a/sys/crypto/openssl/arm_arch.h b/sys/crypto/openssl/arm_arch.h index acd8aee4d519..d570d1eba6c1 100644 --- a/sys/crypto/openssl/arm_arch.h +++ b/sys/crypto/openssl/arm_arch.h @@ -8,87 +8,80 @@ */ #ifndef OSSL_CRYPTO_ARM_ARCH_H -# define OSSL_CRYPTO_ARM_ARCH_H - -# if !defined(__ARM_ARCH__) -# if defined(__CC_ARM) -# define __ARM_ARCH__ __TARGET_ARCH_ARM -# if defined(__BIG_ENDIAN) -# define __ARMEB__ -# else -# define __ARMEL__ -# endif -# elif defined(__GNUC__) -# if defined(__aarch64__) -# define __ARM_ARCH__ 8 - /* - * Why doesn't gcc define __ARM_ARCH__? Instead it defines - * bunch of below macros. See all_architectures[] table in - * gcc/config/arm/arm.c. On a side note it defines - * __ARMEL__/__ARMEB__ for little-/big-endian. - */ -# elif defined(__ARM_ARCH) -# define __ARM_ARCH__ __ARM_ARCH -# elif defined(__ARM_ARCH_8A__) -# define __ARM_ARCH__ 8 -# elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ - defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__) || \ - defined(__ARM_ARCH_7EM__) -# define __ARM_ARCH__ 7 -# elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \ - defined(__ARM_ARCH_6K__)|| defined(__ARM_ARCH_6M__) || \ - defined(__ARM_ARCH_6Z__)|| defined(__ARM_ARCH_6ZK__) || \ - defined(__ARM_ARCH_6T2__) -# define __ARM_ARCH__ 6 -# elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) || \ - defined(__ARM_ARCH_5E__)|| defined(__ARM_ARCH_5TE__) || \ - defined(__ARM_ARCH_5TEJ__) -# define __ARM_ARCH__ 5 -# elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__) -# define __ARM_ARCH__ 4 -# else -# error "unsupported ARM architecture" -# endif -# elif defined(__ARM_ARCH) -# define __ARM_ARCH__ __ARM_ARCH -# endif -# endif - -# if !defined(__ARM_MAX_ARCH__) -# define __ARM_MAX_ARCH__ __ARM_ARCH__ -# endif - -# if __ARM_MAX_ARCH__<__ARM_ARCH__ -# error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__" -# elif __ARM_MAX_ARCH__!=__ARM_ARCH__ -# if __ARM_ARCH__<7 && __ARM_MAX_ARCH__>=7 && defined(__ARMEB__) -# error "can't build universal big-endian binary" -# endif -# endif - -# ifndef __ASSEMBLER__ +#define OSSL_CRYPTO_ARM_ARCH_H + +#if !defined(__ARM_ARCH__) +#if defined(__CC_ARM) +#define __ARM_ARCH__ __TARGET_ARCH_ARM +#if defined(__BIG_ENDIAN) +#define __ARMEB__ +#else +#define __ARMEL__ +#endif +#elif defined(__GNUC__) +#if defined(__aarch64__) +#define __ARM_ARCH__ 8 +/* + * Why doesn't gcc define __ARM_ARCH__? Instead it defines + * bunch of below macros. See all_architectures[] table in + * gcc/config/arm/arm.c. On a side note it defines + * __ARMEL__/__ARMEB__ for little-/big-endian. + */ +#elif defined(__ARM_ARCH) +#define __ARM_ARCH__ __ARM_ARCH +#elif defined(__ARM_ARCH_8A__) +#define __ARM_ARCH__ 8 +#elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) +#define __ARM_ARCH__ 7 +#elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6M__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) +#define __ARM_ARCH__ 6 +#elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) || defined(__ARM_ARCH_5E__) || defined(__ARM_ARCH_5TE__) || defined(__ARM_ARCH_5TEJ__) +#define __ARM_ARCH__ 5 +#elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__) +#define __ARM_ARCH__ 4 +#else +#error "unsupported ARM architecture" +#endif +#elif defined(__ARM_ARCH) +#define __ARM_ARCH__ __ARM_ARCH +#endif +#endif + +#if !defined(__ARM_MAX_ARCH__) +#define __ARM_MAX_ARCH__ __ARM_ARCH__ +#endif + +#if __ARM_MAX_ARCH__ < __ARM_ARCH__ +#error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__" +#elif __ARM_MAX_ARCH__ != __ARM_ARCH__ +#if __ARM_ARCH__ < 7 && __ARM_MAX_ARCH__ >= 7 && defined(__ARMEB__) +#error "can't build universal big-endian binary" +#endif +#endif + +#ifndef __ASSEMBLER__ extern unsigned int OPENSSL_armcap_P; extern unsigned int OPENSSL_arm_midr; extern unsigned int OPENSSL_armv8_rsa_neonized; -# endif - -# define ARMV7_NEON (1<<0) -# define ARMV7_TICK (1<<1) -# define ARMV8_AES (1<<2) -# define ARMV8_SHA1 (1<<3) -# define ARMV8_SHA256 (1<<4) -# define ARMV8_PMULL (1<<5) -# define ARMV8_SHA512 (1<<6) -# define ARMV8_CPUID (1<<7) -# define ARMV8_RNG (1<<8) -# define ARMV8_SM3 (1<<9) -# define ARMV8_SM4 (1<<10) -# define ARMV8_SHA3 (1<<11) -# define ARMV8_UNROLL8_EOR3 (1<<12) -# define ARMV8_SVE (1<<13) -# define ARMV8_SVE2 (1<<14) -# define ARMV8_HAVE_SHA3_AND_WORTH_USING (1<<15) -# define ARMV8_UNROLL12_EOR3 (1<<16) +#endif + +#define ARMV7_NEON (1 << 0) +#define ARMV7_TICK (1 << 1) +#define ARMV8_AES (1 << 2) +#define ARMV8_SHA1 (1 << 3) +#define ARMV8_SHA256 (1 << 4) +#define ARMV8_PMULL (1 << 5) +#define ARMV8_SHA512 (1 << 6) +#define ARMV8_CPUID (1 << 7) +#define ARMV8_RNG (1 << 8) +#define ARMV8_SM3 (1 << 9) +#define ARMV8_SM4 (1 << 10) +#define ARMV8_SHA3 (1 << 11) +#define ARMV8_UNROLL8_EOR3 (1 << 12) +#define ARMV8_SVE (1 << 13) +#define ARMV8_SVE2 (1 << 14) +#define ARMV8_HAVE_SHA3_AND_WORTH_USING (1 << 15) +#define ARMV8_UNROLL12_EOR3 (1 << 16) /* * MIDR_EL1 system register @@ -100,120 +93,116 @@ extern unsigned int OPENSSL_armv8_rsa_neonized; * */ -# define ARM_CPU_IMP_ARM 0x41 -# define HISI_CPU_IMP 0x48 -# define ARM_CPU_IMP_APPLE 0x61 -# define ARM_CPU_IMP_MICROSOFT 0x6D -# define ARM_CPU_IMP_AMPERE 0xC0 - -# define ARM_CPU_PART_CORTEX_A72 0xD08 -# define ARM_CPU_PART_N1 0xD0C -# define ARM_CPU_PART_V1 0xD40 -# define ARM_CPU_PART_N2 0xD49 -# define HISI_CPU_PART_KP920 0xD01 -# define ARM_CPU_PART_V2 0xD4F - -# define APPLE_CPU_PART_M1_ICESTORM 0x022 -# define APPLE_CPU_PART_M1_FIRESTORM 0x023 -# define APPLE_CPU_PART_M1_ICESTORM_PRO 0x024 -# define APPLE_CPU_PART_M1_FIRESTORM_PRO 0x025 -# define APPLE_CPU_PART_M1_ICESTORM_MAX 0x028 -# define APPLE_CPU_PART_M1_FIRESTORM_MAX 0x029 -# define APPLE_CPU_PART_M2_BLIZZARD 0x032 -# define APPLE_CPU_PART_M2_AVALANCHE 0x033 -# define APPLE_CPU_PART_M2_BLIZZARD_PRO 0x034 -# define APPLE_CPU_PART_M2_AVALANCHE_PRO 0x035 -# define APPLE_CPU_PART_M2_BLIZZARD_MAX 0x038 -# define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 - -# define MICROSOFT_CPU_PART_COBALT_100 0xD49 - -# define MIDR_PARTNUM_SHIFT 4 -# define MIDR_PARTNUM_MASK (0xfffU << MIDR_PARTNUM_SHIFT) -# define MIDR_PARTNUM(midr) \ - (((midr) & MIDR_PARTNUM_MASK) >> MIDR_PARTNUM_SHIFT) - -# define MIDR_IMPLEMENTER_SHIFT 24 -# define MIDR_IMPLEMENTER_MASK (0xffU << MIDR_IMPLEMENTER_SHIFT) -# define MIDR_IMPLEMENTER(midr) \ - (((midr) & MIDR_IMPLEMENTER_MASK) >> MIDR_IMPLEMENTER_SHIFT) - -# define MIDR_ARCHITECTURE_SHIFT 16 -# define MIDR_ARCHITECTURE_MASK (0xfU << MIDR_ARCHITECTURE_SHIFT) -# define MIDR_ARCHITECTURE(midr) \ - (((midr) & MIDR_ARCHITECTURE_MASK) >> MIDR_ARCHITECTURE_SHIFT) - -# define MIDR_CPU_MODEL_MASK \ - (MIDR_IMPLEMENTER_MASK | \ - MIDR_PARTNUM_MASK | \ - MIDR_ARCHITECTURE_MASK) - -# define MIDR_CPU_MODEL(imp, partnum) \ - (((imp) << MIDR_IMPLEMENTER_SHIFT) | \ - (0xfU << MIDR_ARCHITECTURE_SHIFT) | \ - ((partnum) << MIDR_PARTNUM_SHIFT)) - -# define MIDR_IS_CPU_MODEL(midr, imp, partnum) \ - (((midr) & MIDR_CPU_MODEL_MASK) == MIDR_CPU_MODEL(imp, partnum)) +#define ARM_CPU_IMP_ARM 0x41 +#define HISI_CPU_IMP 0x48 +#define ARM_CPU_IMP_APPLE 0x61 +#define ARM_CPU_IMP_MICROSOFT 0x6D +#define ARM_CPU_IMP_AMPERE 0xC0 + +#define ARM_CPU_PART_CORTEX_A72 0xD08 +#define ARM_CPU_PART_N1 0xD0C +#define ARM_CPU_PART_V1 0xD40 +#define ARM_CPU_PART_N2 0xD49 +#define HISI_CPU_PART_KP920 0xD01 +#define ARM_CPU_PART_V2 0xD4F + +#define APPLE_CPU_PART_M1_ICESTORM 0x022 +#define APPLE_CPU_PART_M1_FIRESTORM 0x023 +#define APPLE_CPU_PART_M1_ICESTORM_PRO 0x024 +#define APPLE_CPU_PART_M1_FIRESTORM_PRO 0x025 +#define APPLE_CPU_PART_M1_ICESTORM_MAX 0x028 +#define APPLE_CPU_PART_M1_FIRESTORM_MAX 0x029 +#define APPLE_CPU_PART_M2_BLIZZARD 0x032 +#define APPLE_CPU_PART_M2_AVALANCHE 0x033 +#define APPLE_CPU_PART_M2_BLIZZARD_PRO 0x034 +#define APPLE_CPU_PART_M2_AVALANCHE_PRO 0x035 +#define APPLE_CPU_PART_M2_BLIZZARD_MAX 0x038 +#define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 + +#define MICROSOFT_CPU_PART_COBALT_100 0xD49 + +#define MIDR_PARTNUM_SHIFT 4 +#define MIDR_PARTNUM_MASK (0xfffU << MIDR_PARTNUM_SHIFT) +#define MIDR_PARTNUM(midr) \ + (((midr) & MIDR_PARTNUM_MASK) >> MIDR_PARTNUM_SHIFT) + +#define MIDR_IMPLEMENTER_SHIFT 24 +#define MIDR_IMPLEMENTER_MASK (0xffU << MIDR_IMPLEMENTER_SHIFT) +#define MIDR_IMPLEMENTER(midr) \ + (((midr) & MIDR_IMPLEMENTER_MASK) >> MIDR_IMPLEMENTER_SHIFT) + +#define MIDR_ARCHITECTURE_SHIFT 16 +#define MIDR_ARCHITECTURE_MASK (0xfU << MIDR_ARCHITECTURE_SHIFT) +#define MIDR_ARCHITECTURE(midr) \ + (((midr) & MIDR_ARCHITECTURE_MASK) >> MIDR_ARCHITECTURE_SHIFT) + +#define MIDR_CPU_MODEL_MASK \ + (MIDR_IMPLEMENTER_MASK | MIDR_PARTNUM_MASK | MIDR_ARCHITECTURE_MASK) + +#define MIDR_CPU_MODEL(imp, partnum) \ + (((imp) << MIDR_IMPLEMENTER_SHIFT) | (0xfU << MIDR_ARCHITECTURE_SHIFT) | ((partnum) << MIDR_PARTNUM_SHIFT)) + +#define MIDR_IS_CPU_MODEL(midr, imp, partnum) \ + (((midr) & MIDR_CPU_MODEL_MASK) == MIDR_CPU_MODEL(imp, partnum)) #if defined(__ASSEMBLER__) - /* - * Support macros for - * - Armv8.3-A Pointer Authentication and - * - Armv8.5-A Branch Target Identification - * features which require emitting a .note.gnu.property section with the - * appropriate architecture-dependent feature bits set. - * Read more: "ELF for the Arm® 64-bit Architecture" - */ - -# if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 -# define GNU_PROPERTY_AARCH64_BTI (1 << 0) /* Has Branch Target Identification */ -# define AARCH64_VALID_CALL_TARGET hint #34 /* BTI 'c' */ -# else -# define GNU_PROPERTY_AARCH64_BTI 0 /* No Branch Target Identification */ -# define AARCH64_VALID_CALL_TARGET -# endif - -# if defined(__ARM_FEATURE_PAC_DEFAULT) && \ - (__ARM_FEATURE_PAC_DEFAULT & 1) == 1 /* Signed with A-key */ -# define GNU_PROPERTY_AARCH64_POINTER_AUTH \ - (1 << 1) /* Has Pointer Authentication */ -# define AARCH64_SIGN_LINK_REGISTER hint #25 /* PACIASP */ -# define AARCH64_VALIDATE_LINK_REGISTER hint #29 /* AUTIASP */ -# elif defined(__ARM_FEATURE_PAC_DEFAULT) && \ - (__ARM_FEATURE_PAC_DEFAULT & 2) == 2 /* Signed with B-key */ -# define GNU_PROPERTY_AARCH64_POINTER_AUTH \ - (1 << 1) /* Has Pointer Authentication */ -# define AARCH64_SIGN_LINK_REGISTER hint #27 /* PACIBSP */ -# define AARCH64_VALIDATE_LINK_REGISTER hint #31 /* AUTIBSP */ -# else -# define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 /* No Pointer Authentication */ -# if GNU_PROPERTY_AARCH64_BTI != 0 -# define AARCH64_SIGN_LINK_REGISTER AARCH64_VALID_CALL_TARGET -# else -# define AARCH64_SIGN_LINK_REGISTER -# endif -# define AARCH64_VALIDATE_LINK_REGISTER -# endif - -# if GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_BTI != 0 - .pushsection .note.gnu.property, "a"; - .balign 8; - .long 4; - .long 0x10; - .long 0x5; - .asciz "GNU"; - .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ - .long 4; - .long (GNU_PROPERTY_AARCH64_POINTER_AUTH | GNU_PROPERTY_AARCH64_BTI); - .long 0; - .popsection; -# endif - -# endif /* defined __ASSEMBLER__ */ - -# define IS_CPU_SUPPORT_UNROLL8_EOR3() \ - (OPENSSL_armcap_P & ARMV8_UNROLL8_EOR3) +/* + * Support macros for + * - Armv8.3-A Pointer Authentication and + * - Armv8.5-A Branch Target Identification + * features which require emitting a .note.gnu.property section with the + * appropriate architecture-dependent feature bits set. + * Read more: "ELF for the Arm® 64-bit Architecture" + */ + +#if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 +#define GNU_PROPERTY_AARCH64_BTI (1 << 0) /* Has Branch Target Identification */ +#define AARCH64_VALID_CALL_TARGET hint #34 /* BTI 'c' */ +#else +#define GNU_PROPERTY_AARCH64_BTI 0 /* No Branch Target Identification */ +#define AARCH64_VALID_CALL_TARGET +#endif + +#if defined(__ARM_FEATURE_PAC_DEFAULT) && (__ARM_FEATURE_PAC_DEFAULT & 1) == 1 /* Signed with A-key */ +#define GNU_PROPERTY_AARCH64_POINTER_AUTH \ + (1 << 1) /* Has Pointer Authentication */ +#define AARCH64_SIGN_LINK_REGISTER hint #25 /* PACIASP */ +#define AARCH64_VALIDATE_LINK_REGISTER hint #29 /* AUTIASP */ +#elif defined(__ARM_FEATURE_PAC_DEFAULT) && (__ARM_FEATURE_PAC_DEFAULT & 2) == 2 /* Signed with B-key */ +#define GNU_PROPERTY_AARCH64_POINTER_AUTH \ + (1 << 1) /* Has Pointer Authentication */ +#define AARCH64_SIGN_LINK_REGISTER hint #27 /* PACIBSP */ +#define AARCH64_VALIDATE_LINK_REGISTER hint #31 /* AUTIBSP */ +#else +#define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 /* No Pointer Authentication */ +#if GNU_PROPERTY_AARCH64_BTI != 0 +#define AARCH64_SIGN_LINK_REGISTER AARCH64_VALID_CALL_TARGET +#else +#define AARCH64_SIGN_LINK_REGISTER +#endif +#define AARCH64_VALIDATE_LINK_REGISTER +#endif + +#if GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_BTI != 0 +/* clang-format off */ +.pushsection .note.gnu.property, "a"; +/* clang-format on */ +.balign 8; +.long 4; +.long 0x10; +.long 0x5; +.asciz "GNU"; +.long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ +.long 4; +.long(GNU_PROPERTY_AARCH64_POINTER_AUTH | GNU_PROPERTY_AARCH64_BTI); +.long 0; +.popsection; +#endif + +#endif /* defined __ASSEMBLER__ */ + +#define IS_CPU_SUPPORT_UNROLL8_EOR3() \ + (OPENSSL_armcap_P & ARMV8_UNROLL8_EOR3) #endif diff --git a/sys/crypto/openssl/powerpc/aes-gcm-ppc.S b/sys/crypto/openssl/powerpc/aes-gcm-ppc.S index 23a8feb24745..51cfac7e45fc 100644 --- a/sys/crypto/openssl/powerpc/aes-gcm-ppc.S +++ b/sys/crypto/openssl/powerpc/aes-gcm-ppc.S @@ -1,531 +1,587 @@ /* Do not modify. This file is auto-generated from aes-gcm-ppc.pl. */ -.machine "any" +.machine "any" .text - - - - -.macro .Loop_aes_middle4x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - .long 0x11EF9D08 - .long 0x12109D08 - .long 0x12319D08 - .long 0x12529D08 - - .long 0x11EFA508 - .long 0x1210A508 - .long 0x1231A508 - .long 0x1252A508 - - .long 0x11EFAD08 - .long 0x1210AD08 - .long 0x1231AD08 - .long 0x1252AD08 - - .long 0x11EFB508 - .long 0x1210B508 - .long 0x1231B508 - .long 0x1252B508 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - .long 0x11EF9D08 - .long 0x12109D08 - .long 0x12319D08 - .long 0x12529D08 - - .long 0x11EFA508 - .long 0x1210A508 - .long 0x1231A508 - .long 0x1252A508 - - .long 0x11EFAD08 - .long 0x1210AD08 - .long 0x1231AD08 - .long 0x1252AD08 - - .long 0x11EFB508 - .long 0x1210B508 - .long 0x1231B508 - .long 0x1252B508 - - xxlor 23+32, 9, 9 - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 -.endm - - - - - -.macro .Loop_aes_middle8x - xxlor 23+32, 1, 1 - xxlor 24+32, 2, 2 - xxlor 25+32, 3, 3 - xxlor 26+32, 4, 4 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - .long 0x11EFCD08 - .long 0x1210CD08 - .long 0x1231CD08 - .long 0x1252CD08 - .long 0x1273CD08 - .long 0x1294CD08 - .long 0x12B5CD08 - .long 0x12D6CD08 - - .long 0x11EFD508 - .long 0x1210D508 - .long 0x1231D508 - .long 0x1252D508 - .long 0x1273D508 - .long 0x1294D508 - .long 0x12B5D508 - .long 0x12D6D508 - - xxlor 23+32, 5, 5 - xxlor 24+32, 6, 6 - xxlor 25+32, 7, 7 - xxlor 26+32, 8, 8 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - .long 0x11EFCD08 - .long 0x1210CD08 - .long 0x1231CD08 - .long 0x1252CD08 - .long 0x1273CD08 - .long 0x1294CD08 - .long 0x12B5CD08 - .long 0x12D6CD08 - - .long 0x11EFD508 - .long 0x1210D508 - .long 0x1231D508 - .long 0x1252D508 - .long 0x1273D508 - .long 0x1294D508 - .long 0x12B5D508 - .long 0x12D6D508 - - xxlor 23+32, 9, 9 - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 -.endm - - - - -ppc_aes_gcm_ghash: - vxor 15, 15, 0 - - xxlxor 29, 29, 29 - - .long 0x12EC7CC8 - .long 0x130984C8 - .long 0x13268CC8 - .long 0x134394C8 - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D7CC8 - .long 0x132A84C8 - .long 0x13478CC8 - .long 0x136494C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E7CC8 - .long 0x132B84C8 - .long 0x13488CC8 - .long 0x136594C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 - - blr - - - - - -.macro ppc_aes_gcm_ghash2_4x - - vxor 15, 15, 0 - - xxlxor 29, 29, 29 - - .long 0x12EC7CC8 - .long 0x130984C8 - .long 0x13268CC8 - .long 0x134394C8 +.macro SAVE_REGS + mflr 0 + std 0, 16(1) + stdu 1,-512(1) + + std 14, 112(1) + std 15, 120(1) + std 16, 128(1) + std 17, 136(1) + std 18, 144(1) + std 19, 152(1) + std 20, 160(1) + std 21, 168(1) + std 22, 176(1) + std 23, 184(1) + std 24, 192(1) + + stxv 32+20, 256(1) + stxv 32+21, 256+16(1) + stxv 32+22, 256+32(1) + stxv 32+23, 256+48(1) + stxv 32+24, 256+64(1) + stxv 32+25, 256+80(1) + stxv 32+26, 256+96(1) + stxv 32+27, 256+112(1) + stxv 32+28, 256+128(1) + stxv 32+29, 256+144(1) + stxv 32+30, 256+160(1) + stxv 32+31, 256+176(1) +.endm # SAVE_REGS + +.macro RESTORE_REGS + lxv 32+20, 256(1) + lxv 32+21, 256+16(1) + lxv 32+22, 256+32(1) + lxv 32+23, 256+48(1) + lxv 32+24, 256+64(1) + lxv 32+25, 256+80(1) + lxv 32+26, 256+96(1) + lxv 32+27, 256+112(1) + lxv 32+28, 256+128(1) + lxv 32+29, 256+144(1) + lxv 32+30, 256+160(1) + lxv 32+31, 256+176(1) + + ld 14, 112(1) + ld 15, 120(1) + ld 16, 128(1) + ld 17, 136(1) + ld 18, 144(1) + ld 19, 152(1) + ld 20, 160(1) + ld 21, 168(1) + ld 22, 176(1) + ld 23, 184(1) + ld 24, 192(1) + + addi 1, 1, 512 + ld 0, 16(1) + mtlr 0 +.endm # RESTORE_REGS + +# 4x loops +.macro AES_CIPHER_4x r + vcipher 15, 15, \r + vcipher 16, 16, \r + vcipher 17, 17, \r + vcipher 18, 18, \r +.endm + +# 8x loops +.macro AES_CIPHER_8x r + vcipher 15, 15, \r + vcipher 16, 16, \r + vcipher 17, 17, \r + vcipher 18, 18, \r + vcipher 19, 19, \r + vcipher 20, 20, \r + vcipher 21, 21, \r + vcipher 22, 22, \r +.endm + +.macro LOOP_8AES_STATE + AES_CIPHER_8x 23 + AES_CIPHER_8x 24 + AES_CIPHER_8x 25 + AES_CIPHER_8x 26 + AES_CIPHER_8x 27 + AES_CIPHER_8x 28 + AES_CIPHER_8x 29 + AES_CIPHER_8x 1 +.endm + +# +# PPC_GFMUL128_8x: Compute hash values of 8 blocks based on Karatsuba method. +# +# S1 should xor with the previous digest +# +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# vs10: vpermxor vector +# Scratch: v23 - v29 +# +.macro PPC_GFMUL128_8x + + vpmsumd 23, 12, 15 # H4.L * X.L + vpmsumd 24, 9, 16 + vpmsumd 25, 6, 17 + vpmsumd 26, 3, 18 vxor 23, 23, 24 vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D7CC8 - .long 0x132A84C8 - .long 0x13478CC8 - .long 0x136494C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - - vxor 24, 24, 27 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 + vxor 23, 23, 26 # L - .long 0x130E7CC8 - .long 0x132B84C8 - .long 0x13488CC8 - .long 0x136594C8 + vpmsumd 27, 13, 15 # H4.L * X.H + H4.H * X.L + vpmsumd 28, 10, 16 # H3.L * X1.H + H3.H * X1.L + vpmsumd 25, 7, 17 + vpmsumd 26, 4, 18 + vxor 24, 27, 28 vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 27, 23, 27 - - - .long 0x1309A4C8 - .long 0x1326ACC8 - .long 0x1343B4C8 - vxor 19, 19, 27 - .long 0x12EC9CC8 + vxor 24, 24, 26 # M + + vpmsumd 26, 14, 15 # H4.H * X.H + vpmsumd 27, 11, 16 + vpmsumd 28, 8, 17 + vpmsumd 29, 5, 18 + + vxor 26, 26, 27 + vxor 26, 26, 28 + vxor 26, 26, 29 + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + vxor 1, 1, 1 + vsldoi 25, 24, 1, 8 # mL + vsldoi 1, 1, 24, 8 # mH + vxor 23, 23, 25 # mL + L + + # This performs swap and xor like, + # vsldoi 23, 23, 23, 8 # swap + # vxor 23, 23, 28 + xxlor 32+29, 10, 10 + vpermxor 23, 23, 28, 29 + + vxor 24, 26, 1 # H + + # sum hash and reduction with H Poly + # + # vsldoi 25, 23, 23, 8 # swap + # vpmsumd 23, 23, 2 + # vxor 27, 25, 24 + # + vpermxor 27, 23, 24, 29 + vpmsumd 23, 23, 2 + vxor 0, 23, 27 # Digest of 4 blocks + + vxor 19, 19, 0 + + # Compute digest for the next 4 blocks + vpmsumd 24, 9, 20 + vpmsumd 25, 6, 21 + vpmsumd 26, 3, 22 + vpmsumd 23, 12, 19 # H4.L * X.L vxor 23, 23, 24 vxor 23, 23, 25 - vxor 23, 23, 26 + vxor 23, 23, 26 # L - .long 0x130D9CC8 - .long 0x132AA4C8 - .long 0x1347ACC8 - .long 0x1364B4C8 + vpmsumd 27, 13, 19 # H4.L * X.H + H4.H * X.L + vpmsumd 28, 10, 20 # H3.L * X1.H + H3.H * X1.L + vpmsumd 25, 7, 21 + vpmsumd 26, 4, 22 + vxor 24, 27, 28 vxor 24, 24, 25 - vxor 24, 24, 26 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - - vxor 24, 24, 27 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E9CC8 - .long 0x132BA4C8 - .long 0x1348ACC8 - .long 0x1365B4C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 - -.endm - - - - -.macro ppc_update_hash_1x - vxor 28, 28, 0 - - vxor 19, 19, 19 - - .long 0x12C3E4C8 - .long 0x12E4E4C8 - .long 0x1305E4C8 - - .long 0x137614C8 - - vsldoi 25, 23, 19, 8 - vsldoi 26, 19, 23, 8 - vxor 22, 22, 25 - vxor 24, 24, 26 - - vsldoi 22, 22, 22, 8 - vxor 22, 22, 27 - - vsldoi 20, 22, 22, 8 - .long 0x12D614C8 - vxor 20, 20, 24 - vxor 22, 22, 20 - - vor 0,22,22 - -.endm - - - - - - - - - - - - - -.global ppc_aes_gcm_encrypt -.align 5 -ppc_aes_gcm_encrypt: -_ppc_aes_gcm_encrypt: - - stdu 1,-512(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - std 0, 528(1) - - - lxvb16x 32, 0, 8 - + vxor 24, 24, 26 # M + + vpmsumd 26, 14, 19 # H4.H * X.H + vpmsumd 27, 11, 20 + vpmsumd 28, 8, 21 + vpmsumd 29, 5, 22 + + vxor 26, 26, 27 + vxor 26, 26, 28 + vxor 26, 26, 29 + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + vxor 1, 1, 1 + vsldoi 25, 24, 1, 8 # mL + vsldoi 1, 1, 24, 8 # mH + vxor 23, 23, 25 # mL + L + + # This performs swap and xor like, + # vsldoi 23, 23, 23, 8 # swap + # vxor 23, 23, 28 + xxlor 32+29, 10, 10 + vpermxor 23, 23, 28, 29 + + vxor 24, 26, 1 # H + + # sum hash and reduction with H Poly + # + # vsldoi 25, 23, 23, 8 # swap + # vpmsumd 23, 23, 2 + # vxor 27, 25, 24 + # + vpermxor 27, 23, 24, 29 + vpmsumd 23, 23, 2 + vxor 0, 23, 27 # Digest of 8 blocks +.endm + +# +# Compute update single ghash +# vs10: vpermxor vector +# scratch: v1, v22..v27 +# +.macro PPC_GHASH1x H S1 + + vxor 1, 1, 1 + + vpmsumd 22, 3, \S1 # L + vpmsumd 23, 4, \S1 # M + vpmsumd 24, 5, \S1 # H + + vpmsumd 27, 22, 2 # reduction + + vsldoi 25, 23, 1, 8 # mL + vsldoi 26, 1, 23, 8 # mH + vxor 22, 22, 25 # LL + LL + vxor 24, 24, 26 # HH + HH + + xxlor 32+25, 10, 10 + vpermxor 22, 22, 27, 25 + + # vsldoi 23, 22, 22, 8 # swap + # vpmsumd 22, 22, 2 # reduction + # vxor 23, 23, 24 + vpermxor 23, 22, 24, 25 + vpmsumd 22, 22, 2 # reduction + + vxor \H, 22, 23 +.endm + +# +# LOAD_HASH_TABLE +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# +.macro LOAD_HASH_TABLE + # Load Xi + lxvb16x 32, 0, 8 # load Xi + + vxor 1, 1, 1 li 10, 32 - lxvd2x 2+32, 10, 8 - li 10, 48 - lxvd2x 3+32, 10, 8 - li 10, 64 - lxvd2x 4+32, 10, 8 - li 10, 80 - lxvd2x 5+32, 10, 8 + lxvd2x 2+32, 10, 8 # H Poli - li 10, 96 - lxvd2x 6+32, 10, 8 + # load Hash - h^4, h^3, h^2, h + li 10, 64 + lxvd2x 4+32, 10, 8 # H + vsldoi 3, 1, 4, 8 # l + vsldoi 5, 4, 1, 8 # h li 10, 112 - lxvd2x 7+32, 10, 8 - li 10, 128 - lxvd2x 8+32, 10, 8 - - li 10, 144 - lxvd2x 9+32, 10, 8 + lxvd2x 7+32, 10, 8 # H^2 + vsldoi 6, 1, 7, 8 # l + vsldoi 8, 7, 1, 8 # h li 10, 160 - lxvd2x 10+32, 10, 8 - li 10, 176 - lxvd2x 11+32, 10, 8 - - li 10, 192 - lxvd2x 12+32, 10, 8 + lxvd2x 10+32, 10, 8 # H^3 + vsldoi 9, 1, 10, 8 # l + vsldoi 11, 10, 1, 8 # h li 10, 208 - lxvd2x 13+32, 10, 8 - li 10, 224 - lxvd2x 14+32, 10, 8 - - - lxvb16x 30+32, 0, 7 - - mr 12, 5 - li 11, 0 - - - vxor 31, 31, 31 - vspltisb 22,1 - vsldoi 31, 31, 22,1 - - - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - - lwz 9,240(6) + lxvd2x 13+32, 10, 8 # H^4 + vsldoi 12, 1, 13, 8 # l + vsldoi 14, 13, 1, 8 # h +.endm + +.macro PROCESS_8X_AES_STATES + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 +.endm + +.macro COMPUTE_STATES + xxlor 32+15, 9, 9 # last state + vadduwm 15, 15, 31 # state + counter + vadduwm 16, 15, 31 + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 +.endm + +################################################################################ +# Compute AES and ghash one block at a time. +# r23: AES rounds +# v30: current IV +# vs0: roundkey 0 +# +################################################################################ +.align 4 +aes_gcm_crypt_1x: +.localentry aes_gcm_crypt_1x,0 + + cmpdi 5, 16 + bge __More_1x + blr +__More_1x: + li 10, 16 + divdu 12, 5, 10 + + xxlxor 32+15, 32+30, 0 + + # Pre-load 8 AES rounds to scratch vectors. + lxv 32+16, 16(6) # round key 1 + lxv 32+17, 32(6) # round key 2 + lxv 32+18, 48(6) # round key 3 + lxv 32+19, 64(6) # round key 4 + lxv 32+20, 80(6) # round key 5 + lxv 32+21, 96(6) # round key 6 + lxv 32+28, 112(6) # round key 7 + lxv 32+29, 128(6) # round key 8 + + lwz 23, 240(6) # n rounds + addi 22, 23, -9 # remaining AES rounds + cmpdi 12, 0 + bgt __Loop_1x + blr + +__Loop_1x: + mtctr 22 + addi 10, 6, 144 + vcipher 15, 15, 16 + vcipher 15, 15, 17 + vcipher 15, 15, 18 + vcipher 15, 15, 19 + vcipher 15, 15, 20 + vcipher 15, 15, 21 + vcipher 15, 15, 28 + vcipher 15, 15, 29 + +__Loop_aes_1state: + lxv 32+1, 0(10) + vcipher 15, 15, 1 + addi 10, 10, 16 + bdnz __Loop_aes_1state + lxv 32+1, 0(10) # last round key + lxvb16x 11, 0, 14 # load input block + vcipherlast 15, 15, 1 - xxlor 32+29, 0, 0 - vxor 15, 30, 29 + xxlxor 32+15, 32+15, 11 + stxvb16x 32+15, 0, 9 # store output + addi 14, 14, 16 + addi 9, 9, 16 - cmpdi 9, 10 - beq .Loop_aes_gcm_8x + cmpdi 24, 0 # decrypt? + bne __Encrypt_1x + xxlor 15+32, 11, 11 +__Encrypt_1x: + vxor 15, 15, 0 + PPC_GHASH1x 0, 15 + addi 5, 5, -16 + addi 11, 11, 16 - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) + vadduwm 30, 30, 31 # IV + counter + xxlxor 32+15, 32+30, 0 + addi 12, 12, -1 + cmpdi 12, 0 + bgt __Loop_1x + + stxvb16x 32+0, 0, 8 # update Xi + blr +.size aes_gcm_crypt_1x,.-aes_gcm_crypt_1x + +################################################################################ +# Process a normal partial block when we come here. +# Compute partial mask, Load and store partial block to stack. +# Compute AES state. +# Compute ghash. +# +################################################################################ +.align 4 +__Process_partial: +.localentry __Process_partial,0 + + # create partial mask + vspltisb 16, -1 + li 12, 16 + sub 12, 12, 5 + sldi 12, 12, 3 + mtvsrdd 32+17, 0, 12 + vslo 16, 16, 17 # partial block mask + + lxvb16x 11, 0, 14 # load partial block + xxland 11, 11, 32+16 + + # AES crypt partial + xxlxor 32+15, 32+30, 0 + lwz 23, 240(6) # n rounds + addi 22, 23, -1 # loop - 1 + mtctr 22 + addi 10, 6, 16 + +__Loop_aes_pstate: + lxv 32+1, 0(10) + vcipher 15, 15, 1 + addi 10, 10, 16 + bdnz __Loop_aes_pstate + lxv 32+1, 0(10) # last round key + vcipherlast 15, 15, 1 - cmpdi 9, 12 - beq .Loop_aes_gcm_8x + xxlxor 32+15, 32+15, 11 + vand 15, 15, 16 + # AES crypt output v15 + # Write partial + li 10, 224 + stxvb16x 15+32, 10, 1 # write v15 to stack + addi 10, 1, 223 + addi 12, 9, -1 + mtctr 5 # partial block len +__Write_partial: + lbzu 22, 1(10) + stbu 22, 1(12) + bdnz __Write_partial + + cmpdi 24, 0 # decrypt? + bne __Encrypt_partial + xxlor 32+15, 11, 11 # decrypt using the input block +__Encrypt_partial: + vxor 15, 15, 0 # ^ previous hash + PPC_GHASH1x 0, 15 + li 5, 0 # done last byte + stxvb16x 32+0, 0, 8 # Update X1 + blr +.size __Process_partial,.-__Process_partial + +################################################################################ +# ppc_aes_gcm_encrypt (const void *inp, void *out, size_t len, +# const char *rk, unsigned char iv[16], void *Xip); +# +# r3 - inp +# r4 - out +# r5 - len +# r6 - AES round keys +# r7 - iv +# r8 - Xi, HPoli, hash keys +# +# rounds is at offset 240 in rk +# Xi is at 0 in gcm_table (Xip). +# +################################################################################ +.global ppc_aes_gcm_encrypt +.align 5 +ppc_aes_gcm_encrypt: +.localentry ppc_aes_gcm_encrypt,0 - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq .Loop_aes_gcm_8x + SAVE_REGS + LOAD_HASH_TABLE - b aes_gcm_out + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 -.align 5 -.Loop_aes_gcm_8x: mr 14, 3 mr 9, 4 + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 + + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + li 11, 0 + lxv 0, 0(6) # round key 0 + + # + # Process different blocks + # + cmpdi 5, 128 + blt __Process_more_enc + + # load 9 round keys + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 23, 240(6) # n rounds + +__Process_encrypt: +# +# Process 8x AES/GCM blocks +# +__Process_8x_enc: + # 8x blocks li 10, 128 - divdu 10, 5, 10 - cmpdi 10, 0 - beq .Loop_last_block - - .long 0x13DEF8C0 - vxor 16, 30, 29 - .long 0x13DEF8C0 - vxor 17, 30, 29 - .long 0x13DEF8C0 - vxor 18, 30, 29 - .long 0x13DEF8C0 - vxor 19, 30, 29 - .long 0x13DEF8C0 - vxor 20, 30, 29 - .long 0x13DEF8C0 - vxor 21, 30, 29 - .long 0x13DEF8C0 - vxor 22, 30, 29 - - mtctr 10 + divdu 12, 5, 10 # n 128 bytes-blocks + + addi 12, 12, -1 # loop - 1 + + vmr 15, 30 # first state: IV + vadduwm 16, 15, 31 # state + counter + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + # vxor state, state, w # addroundkey + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 li 15, 16 li 16, 32 @@ -535,523 +591,185 @@ _ppc_aes_gcm_encrypt: li 20, 96 li 21, 112 - lwz 10, 240(6) - -.Loop_8x_block: - - lxvb16x 15, 0, 14 - lxvb16x 16, 15, 14 - lxvb16x 17, 16, 14 - lxvb16x 18, 17, 14 - lxvb16x 19, 18, 14 - lxvb16x 20, 19, 14 - lxvb16x 21, 20, 14 - lxvb16x 22, 21, 14 - addi 14, 14, 128 - -.Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_ghash - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_ghash - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_ghash - b aes_gcm_out - -Do_next_ghash: - - - - .long 0x11EFBD09 - .long 0x1210BD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 - - .long 0x1231BD09 - .long 0x1252BD09 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 - - .long 0x1273BD09 - .long 0x1294BD09 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 - - .long 0x12B5BD09 - .long 0x12D6BD09 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 - - addi 9, 9, 128 - - - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - .long 0x13DEF8C0 - vor 29,30,30 - vxor 15, 30, 27 - .long 0x13DEF8C0 - vxor 16, 30, 27 - .long 0x13DEF8C0 - vxor 17, 30, 27 - .long 0x13DEF8C0 - vxor 18, 30, 27 - .long 0x13DEF8C0 - vxor 19, 30, 27 - .long 0x13DEF8C0 - vxor 20, 30, 27 - .long 0x13DEF8C0 - vxor 21, 30, 27 - .long 0x13DEF8C0 - vxor 22, 30, 27 - - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz .Loop_8x_block - - vor 30,29,29 - -.Loop_last_block: - cmpdi 12, 0 - beq aes_gcm_out - - - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10, 240(6) - - cmpdi 12, 16 - blt Final_block - -.macro .Loop_aes_middle_1x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - .long 0x11EF9D08 - .long 0x11EFA508 - .long 0x11EFAD08 - .long 0x11EFB508 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - .long 0x11EF9D08 - .long 0x11EFA508 - .long 0x11EFAD08 - .long 0x11EFB508 - - xxlor 19+32, 9, 9 - .long 0x11EF9D08 -.endm - -Next_rem_block: - lxvb16x 15, 0, 14 - -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 + # + # Pre-compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + # + addi 22, 23, -9 # process 8 keys + mtctr 22 # AES key loop + addi 10, 6, 144 - cmpdi 10, 10 - beq Do_next_1x + LOOP_8AES_STATE # process 8 AES keys +__PreLoop_aes_state: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __PreLoop_aes_state + lxv 32+1, 0(10) # last round key (v1) - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_1x - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x - -Do_next_1x: - .long 0x11EFBD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - addi 14, 14, 16 - addi 9, 9, 16 + cmpdi 12, 0 # Only one loop (8 block) + beq __Finish_ghash - vor 28,15,15 - ppc_update_hash_1x +# +# Loop 8x blocks and compute ghash +# +__Loop_8x_block_enc: + PROCESS_8X_AES_STATES - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - .long 0x13DEF8C0 - vxor 15, 30, 19 + # Compute ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x + + COMPUTE_STATES + + addi 5, 5, -128 + addi 11, 11, 128 + + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # Compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + LOOP_8AES_STATE # process 8 AES keys + mtctr 22 # AES key loop + addi 10, 6, 144 + +__LastLoop_aes_state: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __LastLoop_aes_state - bdnz Next_rem_block + lxv 32+1, 0(10) # last round key (v1) + addi 12, 12, -1 cmpdi 12, 0 - beq aes_gcm_out - -Final_block: -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x + bne __Loop_8x_block_enc + # + # Remainng blocks + # +__Finish_ghash: + PROCESS_8X_AES_STATES - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_final_1x - -Do_final_1x: - .long 0x11EFBD09 - - lxvb16x 15, 0, 14 - xxlxor 47, 47, 15 - - - li 15, 16 - sub 15, 15, 12 + # Compute ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x - vspltisb 16,-1 - vspltisb 17,0 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 + # Update IV and Xi + xxlor 30+32, 9, 9 # last ctr + vadduwm 30, 30, 31 # increase ctr + stxvb16x 32+0, 0, 8 # update Xi - addi 10, 1, 192 - lxvb16x 16, 15, 10 - xxland 47, 47, 16 + addi 5, 5, -128 + addi 11, 11, 128 - vor 28,15,15 - ppc_update_hash_1x + # + # Done 8x blocks + # + cmpdi 5, 0 + beq aes_gcm_out - bl Write_partial_block +__Process_more_enc: + li 24, 1 # encrypt + bl aes_gcm_crypt_1x + cmpdi 5, 0 + beq aes_gcm_out + bl __Process_partial b aes_gcm_out +.size ppc_aes_gcm_encrypt,.-ppc_aes_gcm_encrypt - - - - - -Write_partial_block: - li 10, 192 - stxvb16x 15+32, 10, 1 - - - addi 10, 9, -1 - addi 16, 1, 191 - - mtctr 12 - li 15, 0 - -Write_last_byte: - lbzu 14, 1(16) - stbu 14, 1(10) - bdnz Write_last_byte - blr - -aes_gcm_out: - - stxvb16x 32, 0, 8 - add 3, 11, 12 - - li 9, 256 - lvx 20, 9, 1 - addi 9, 9, 16 - lvx 21, 9, 1 - addi 9, 9, 16 - lvx 22, 9, 1 - addi 9, 9, 16 - lvx 23, 9, 1 - addi 9, 9, 16 - lvx 24, 9, 1 - addi 9, 9, 16 - lvx 25, 9, 1 - addi 9, 9, 16 - lvx 26, 9, 1 - addi 9, 9, 16 - lvx 27, 9, 1 - addi 9, 9, 16 - lvx 28, 9, 1 - addi 9, 9, 16 - lvx 29, 9, 1 - addi 9, 9, 16 - lvx 30, 9, 1 - addi 9, 9, 16 - lvx 31, 9, 1 - - ld 0, 528(1) - ld 14,112(1) - ld 15,120(1) - ld 16,128(1) - ld 17,136(1) - ld 18,144(1) - ld 19,152(1) - ld 20,160(1) - ld 21,168(1) - - mtlr 0 - addi 1, 1, 512 - blr - - - - -.global ppc_aes_gcm_decrypt -.align 5 +################################################################################ +# ppc_aes_gcm_decrypt (const void *inp, void *out, size_t len, +# const char *rk, unsigned char iv[16], void *Xip); +# 8x Decrypt +# +################################################################################ +.global ppc_aes_gcm_decrypt +.align 5 ppc_aes_gcm_decrypt: -_ppc_aes_gcm_decrypt: - - stdu 1,-512(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - std 0, 528(1) - - - lxvb16x 32, 0, 8 - - - li 10, 32 - lxvd2x 2+32, 10, 8 - li 10, 48 - lxvd2x 3+32, 10, 8 - li 10, 64 - lxvd2x 4+32, 10, 8 - li 10, 80 - lxvd2x 5+32, 10, 8 - - li 10, 96 - lxvd2x 6+32, 10, 8 - li 10, 112 - lxvd2x 7+32, 10, 8 - li 10, 128 - lxvd2x 8+32, 10, 8 - - li 10, 144 - lxvd2x 9+32, 10, 8 - li 10, 160 - lxvd2x 10+32, 10, 8 - li 10, 176 - lxvd2x 11+32, 10, 8 - - li 10, 192 - lxvd2x 12+32, 10, 8 - li 10, 208 - lxvd2x 13+32, 10, 8 - li 10, 224 - lxvd2x 14+32, 10, 8 - - - lxvb16x 30+32, 0, 7 - - mr 12, 5 - li 11, 0 - +.localentry ppc_aes_gcm_decrypt, 0 - vxor 31, 31, 31 - vspltisb 22,1 - vsldoi 31, 31, 22,1 - - - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - - lwz 9,240(6) - - - - xxlor 32+29, 0, 0 - vxor 15, 30, 29 - - cmpdi 9, 10 - beq .Loop_aes_gcm_8x_dec - - - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) - - cmpdi 9, 12 - beq .Loop_aes_gcm_8x_dec + SAVE_REGS + LOAD_HASH_TABLE + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq .Loop_aes_gcm_8x_dec - - b aes_gcm_out - -.align 5 -.Loop_aes_gcm_8x_dec: mr 14, 3 mr 9, 4 + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 + + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + li 11, 0 + lxv 0, 0(6) # round key 0 + + # + # Process different blocks + # + cmpdi 5, 128 + blt __Process_more_dec + + # load 9 round keys + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 23, 240(6) # n rounds + +__Process_decrypt: +# +# Process 8x AES/GCM blocks +# +__Process_8x_dec: + # 8x blocks li 10, 128 - divdu 10, 5, 10 - cmpdi 10, 0 - beq .Loop_last_block_dec - - .long 0x13DEF8C0 - vxor 16, 30, 29 - .long 0x13DEF8C0 - vxor 17, 30, 29 - .long 0x13DEF8C0 - vxor 18, 30, 29 - .long 0x13DEF8C0 - vxor 19, 30, 29 - .long 0x13DEF8C0 - vxor 20, 30, 29 - .long 0x13DEF8C0 - vxor 21, 30, 29 - .long 0x13DEF8C0 - vxor 22, 30, 29 - - mtctr 10 + divdu 12, 5, 10 # n 128 bytes-blocks + + addi 12, 12, -1 # loop - 1 + + vmr 15, 30 # first state: IV + vadduwm 16, 15, 31 # state + counter + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + # vxor state, state, w # addroundkey + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 li 15, 16 li 16, 32 @@ -1061,279 +779,215 @@ _ppc_aes_gcm_decrypt: li 20, 96 li 21, 112 - lwz 10, 240(6) + # + # Pre-compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + # + addi 22, 23, -9 # process 8 keys + mtctr 22 # AES key loop + addi 10, 6, 144 -.Loop_8x_block_dec: + LOOP_8AES_STATE # process 8 AES keys - lxvb16x 15, 0, 14 - lxvb16x 16, 15, 14 - lxvb16x 17, 16, 14 - lxvb16x 18, 17, 14 - lxvb16x 19, 18, 14 - lxvb16x 20, 19, 14 - lxvb16x 21, 20, 14 - lxvb16x 22, 21, 14 +__PreLoop_aes_state_dec: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __PreLoop_aes_state_dec + lxv 32+1, 0(10) # last round key (v1) + + cmpdi 12, 0 # Only one loop (8 block) + beq __Finish_ghash_dec + +# +# Loop 8x blocks and compute ghash +# +__Loop_8x_block_dec: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block addi 14, 14, 128 -.Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_last_aes_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_last_aes_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_last_aes_dec - b aes_gcm_out - -Do_last_aes_dec: - - - - .long 0x11EFBD09 - .long 0x1210BD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 - - .long 0x1231BD09 - .long 0x1252BD09 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 - - .long 0x1273BD09 - .long 0x1294BD09 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 - - .long 0x12B5BD09 - .long 0x12D6BD09 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output addi 9, 9, 128 - xxlor 15+32, 15, 15 - xxlor 16+32, 16, 16 - xxlor 17+32, 17, 17 - xxlor 18+32, 18, 18 - xxlor 19+32, 19, 19 - xxlor 20+32, 20, 20 - xxlor 21+32, 21, 21 - xxlor 22+32, 22, 22 - - - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - .long 0x13DEF8C0 - vor 29,30,30 - vxor 15, 30, 27 - .long 0x13DEF8C0 - vxor 16, 30, 27 - .long 0x13DEF8C0 - vxor 17, 30, 27 - .long 0x13DEF8C0 - vxor 18, 30, 27 - .long 0x13DEF8C0 - vxor 19, 30, 27 - .long 0x13DEF8C0 - vxor 20, 30, 27 - .long 0x13DEF8C0 - vxor 21, 30, 27 - .long 0x13DEF8C0 - vxor 22, 30, 27 - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz .Loop_8x_block_dec - - vor 30,29,29 - -.Loop_last_block_dec: - cmpdi 12, 0 - beq aes_gcm_out - - - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10,240(6) - - cmpdi 12, 16 - blt Final_block_dec - -Next_rem_block_dec: - lxvb16x 15, 0, 14 - -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_1x_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_1x_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x_dec - -Do_next_1x_dec: - .long 0x11EFBD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - addi 14, 14, 16 - addi 9, 9, 16 - - xxlor 28+32, 15, 15 - ppc_update_hash_1x + vmr 15, 23 + vmr 16, 24 + vmr 17, 25 + vmr 18, 26 + vmr 19, 27 + vmr 20, 28 + vmr 21, 29 + vmr 22, 30 - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - .long 0x13DEF8C0 - vxor 15, 30, 19 + # ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x + + xxlor 32+15, 9, 9 # last state + vadduwm 15, 15, 31 # state + counter + vadduwm 16, 15, 31 + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + xxlor 32+27, 0, 0 # restore roundkey 0 + vxor 15, 15, 27 # IV + round key - add round key 0 + vxor 16, 16, 27 + vxor 17, 17, 27 + vxor 18, 18, 27 + vxor 19, 19, 27 + vxor 20, 20, 27 + vxor 21, 21, 27 + vxor 22, 22, 27 - bdnz Next_rem_block_dec + addi 5, 5, -128 + addi 11, 11, 128 + + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + LOOP_8AES_STATE # process 8 AES keys + mtctr 22 # AES key loop + addi 10, 6, 144 +__LastLoop_aes_state_dec: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __LastLoop_aes_state_dec + lxv 32+1, 0(10) # last round key (v1) + addi 12, 12, -1 cmpdi 12, 0 - beq aes_gcm_out - -Final_block_dec: -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x_dec - + bne __Loop_8x_block_dec + +__Finish_ghash_dec: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 - xxlor 24+32, 13, 13 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 - .long 0x11EFBD08 - .long 0x11EFC508 + vxor 15, 23, 0 + vmr 16, 24 + vmr 17, 25 + vmr 18, 26 + vmr 19, 27 + vmr 20, 28 + vmr 21, 29 + vmr 22, 30 - xxlor 23+32, 14, 14 + #vxor 15, 15, 0 + PPC_GFMUL128_8x - cmpdi 10, 14 - beq Do_final_1x_dec + xxlor 30+32, 9, 9 # last ctr + vadduwm 30, 30, 31 # increase ctr + stxvb16x 32+0, 0, 8 # update Xi -Do_final_1x_dec: - .long 0x11EFBD09 + addi 5, 5, -128 + addi 11, 11, 128 - lxvb16x 15, 0, 14 - xxlxor 47, 47, 15 + # + # Done 8x blocks + # + cmpdi 5, 0 + beq aes_gcm_out - li 15, 16 - sub 15, 15, 12 +__Process_more_dec: + li 24, 0 # decrypt + bl aes_gcm_crypt_1x + cmpdi 5, 0 + beq aes_gcm_out - vspltisb 16,-1 - vspltisb 17,0 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 - - addi 10, 1, 192 - lxvb16x 16, 15, 10 - xxland 47, 47, 16 + bl __Process_partial + b aes_gcm_out +.size ppc_aes_gcm_decrypt,.-ppc_aes_gcm_decrypt - xxlor 28+32, 15, 15 - ppc_update_hash_1x +aes_gcm_out: +.localentry aes_gcm_out,0 + mr 3, 11 # return count - bl Write_partial_block + RESTORE_REGS + blr +.size aes_gcm_out,.-aes_gcm_out - b aes_gcm_out +.rodata +.align 4 +# for vector permute and xor +permx: +.long 0x4c5d6e7f, 0x08192a3b, 0xc4d5e6f7, 0x8091a2b3 diff --git a/sys/crypto/openssl/powerpc64/aes-gcm-ppc.S b/sys/crypto/openssl/powerpc64/aes-gcm-ppc.S index 2ff143c42ab7..51cfac7e45fc 100644 --- a/sys/crypto/openssl/powerpc64/aes-gcm-ppc.S +++ b/sys/crypto/openssl/powerpc64/aes-gcm-ppc.S @@ -1,532 +1,587 @@ /* Do not modify. This file is auto-generated from aes-gcm-ppc.pl. */ -.machine "any" -.abiversion 2 +.machine "any" .text - - - - -.macro .Loop_aes_middle4x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - .long 0x11EF9D08 - .long 0x12109D08 - .long 0x12319D08 - .long 0x12529D08 - - .long 0x11EFA508 - .long 0x1210A508 - .long 0x1231A508 - .long 0x1252A508 - - .long 0x11EFAD08 - .long 0x1210AD08 - .long 0x1231AD08 - .long 0x1252AD08 - - .long 0x11EFB508 - .long 0x1210B508 - .long 0x1231B508 - .long 0x1252B508 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - .long 0x11EF9D08 - .long 0x12109D08 - .long 0x12319D08 - .long 0x12529D08 - - .long 0x11EFA508 - .long 0x1210A508 - .long 0x1231A508 - .long 0x1252A508 - - .long 0x11EFAD08 - .long 0x1210AD08 - .long 0x1231AD08 - .long 0x1252AD08 - - .long 0x11EFB508 - .long 0x1210B508 - .long 0x1231B508 - .long 0x1252B508 - - xxlor 23+32, 9, 9 - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 -.endm - - - - - -.macro .Loop_aes_middle8x - xxlor 23+32, 1, 1 - xxlor 24+32, 2, 2 - xxlor 25+32, 3, 3 - xxlor 26+32, 4, 4 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - .long 0x11EFCD08 - .long 0x1210CD08 - .long 0x1231CD08 - .long 0x1252CD08 - .long 0x1273CD08 - .long 0x1294CD08 - .long 0x12B5CD08 - .long 0x12D6CD08 - - .long 0x11EFD508 - .long 0x1210D508 - .long 0x1231D508 - .long 0x1252D508 - .long 0x1273D508 - .long 0x1294D508 - .long 0x12B5D508 - .long 0x12D6D508 - - xxlor 23+32, 5, 5 - xxlor 24+32, 6, 6 - xxlor 25+32, 7, 7 - xxlor 26+32, 8, 8 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - .long 0x11EFCD08 - .long 0x1210CD08 - .long 0x1231CD08 - .long 0x1252CD08 - .long 0x1273CD08 - .long 0x1294CD08 - .long 0x12B5CD08 - .long 0x12D6CD08 - - .long 0x11EFD508 - .long 0x1210D508 - .long 0x1231D508 - .long 0x1252D508 - .long 0x1273D508 - .long 0x1294D508 - .long 0x12B5D508 - .long 0x12D6D508 - - xxlor 23+32, 9, 9 - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 -.endm - - - - -ppc_aes_gcm_ghash: - vxor 15, 15, 0 - - xxlxor 29, 29, 29 - - .long 0x12EC7CC8 - .long 0x130984C8 - .long 0x13268CC8 - .long 0x134394C8 - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D7CC8 - .long 0x132A84C8 - .long 0x13478CC8 - .long 0x136494C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E7CC8 - .long 0x132B84C8 - .long 0x13488CC8 - .long 0x136594C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 - - blr - - - - - -.macro ppc_aes_gcm_ghash2_4x - - vxor 15, 15, 0 - - xxlxor 29, 29, 29 - - .long 0x12EC7CC8 - .long 0x130984C8 - .long 0x13268CC8 - .long 0x134394C8 +.macro SAVE_REGS + mflr 0 + std 0, 16(1) + stdu 1,-512(1) + + std 14, 112(1) + std 15, 120(1) + std 16, 128(1) + std 17, 136(1) + std 18, 144(1) + std 19, 152(1) + std 20, 160(1) + std 21, 168(1) + std 22, 176(1) + std 23, 184(1) + std 24, 192(1) + + stxv 32+20, 256(1) + stxv 32+21, 256+16(1) + stxv 32+22, 256+32(1) + stxv 32+23, 256+48(1) + stxv 32+24, 256+64(1) + stxv 32+25, 256+80(1) + stxv 32+26, 256+96(1) + stxv 32+27, 256+112(1) + stxv 32+28, 256+128(1) + stxv 32+29, 256+144(1) + stxv 32+30, 256+160(1) + stxv 32+31, 256+176(1) +.endm # SAVE_REGS + +.macro RESTORE_REGS + lxv 32+20, 256(1) + lxv 32+21, 256+16(1) + lxv 32+22, 256+32(1) + lxv 32+23, 256+48(1) + lxv 32+24, 256+64(1) + lxv 32+25, 256+80(1) + lxv 32+26, 256+96(1) + lxv 32+27, 256+112(1) + lxv 32+28, 256+128(1) + lxv 32+29, 256+144(1) + lxv 32+30, 256+160(1) + lxv 32+31, 256+176(1) + + ld 14, 112(1) + ld 15, 120(1) + ld 16, 128(1) + ld 17, 136(1) + ld 18, 144(1) + ld 19, 152(1) + ld 20, 160(1) + ld 21, 168(1) + ld 22, 176(1) + ld 23, 184(1) + ld 24, 192(1) + + addi 1, 1, 512 + ld 0, 16(1) + mtlr 0 +.endm # RESTORE_REGS + +# 4x loops +.macro AES_CIPHER_4x r + vcipher 15, 15, \r + vcipher 16, 16, \r + vcipher 17, 17, \r + vcipher 18, 18, \r +.endm + +# 8x loops +.macro AES_CIPHER_8x r + vcipher 15, 15, \r + vcipher 16, 16, \r + vcipher 17, 17, \r + vcipher 18, 18, \r + vcipher 19, 19, \r + vcipher 20, 20, \r + vcipher 21, 21, \r + vcipher 22, 22, \r +.endm + +.macro LOOP_8AES_STATE + AES_CIPHER_8x 23 + AES_CIPHER_8x 24 + AES_CIPHER_8x 25 + AES_CIPHER_8x 26 + AES_CIPHER_8x 27 + AES_CIPHER_8x 28 + AES_CIPHER_8x 29 + AES_CIPHER_8x 1 +.endm + +# +# PPC_GFMUL128_8x: Compute hash values of 8 blocks based on Karatsuba method. +# +# S1 should xor with the previous digest +# +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# vs10: vpermxor vector +# Scratch: v23 - v29 +# +.macro PPC_GFMUL128_8x + + vpmsumd 23, 12, 15 # H4.L * X.L + vpmsumd 24, 9, 16 + vpmsumd 25, 6, 17 + vpmsumd 26, 3, 18 vxor 23, 23, 24 vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D7CC8 - .long 0x132A84C8 - .long 0x13478CC8 - .long 0x136494C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - - vxor 24, 24, 27 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 + vxor 23, 23, 26 # L - .long 0x130E7CC8 - .long 0x132B84C8 - .long 0x13488CC8 - .long 0x136594C8 + vpmsumd 27, 13, 15 # H4.L * X.H + H4.H * X.L + vpmsumd 28, 10, 16 # H3.L * X1.H + H3.H * X1.L + vpmsumd 25, 7, 17 + vpmsumd 26, 4, 18 + vxor 24, 27, 28 vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 27, 23, 27 - - - .long 0x1309A4C8 - .long 0x1326ACC8 - .long 0x1343B4C8 - vxor 19, 19, 27 - .long 0x12EC9CC8 + vxor 24, 24, 26 # M + + vpmsumd 26, 14, 15 # H4.H * X.H + vpmsumd 27, 11, 16 + vpmsumd 28, 8, 17 + vpmsumd 29, 5, 18 + + vxor 26, 26, 27 + vxor 26, 26, 28 + vxor 26, 26, 29 + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + vxor 1, 1, 1 + vsldoi 25, 24, 1, 8 # mL + vsldoi 1, 1, 24, 8 # mH + vxor 23, 23, 25 # mL + L + + # This performs swap and xor like, + # vsldoi 23, 23, 23, 8 # swap + # vxor 23, 23, 28 + xxlor 32+29, 10, 10 + vpermxor 23, 23, 28, 29 + + vxor 24, 26, 1 # H + + # sum hash and reduction with H Poly + # + # vsldoi 25, 23, 23, 8 # swap + # vpmsumd 23, 23, 2 + # vxor 27, 25, 24 + # + vpermxor 27, 23, 24, 29 + vpmsumd 23, 23, 2 + vxor 0, 23, 27 # Digest of 4 blocks + + vxor 19, 19, 0 + + # Compute digest for the next 4 blocks + vpmsumd 24, 9, 20 + vpmsumd 25, 6, 21 + vpmsumd 26, 3, 22 + vpmsumd 23, 12, 19 # H4.L * X.L vxor 23, 23, 24 vxor 23, 23, 25 - vxor 23, 23, 26 + vxor 23, 23, 26 # L - .long 0x130D9CC8 - .long 0x132AA4C8 - .long 0x1347ACC8 - .long 0x1364B4C8 + vpmsumd 27, 13, 19 # H4.L * X.H + H4.H * X.L + vpmsumd 28, 10, 20 # H3.L * X1.H + H3.H * X1.L + vpmsumd 25, 7, 21 + vpmsumd 26, 4, 22 + vxor 24, 27, 28 vxor 24, 24, 25 - vxor 24, 24, 26 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - - vxor 24, 24, 27 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E9CC8 - .long 0x132BA4C8 - .long 0x1348ACC8 - .long 0x1365B4C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 - -.endm - - - - -.macro ppc_update_hash_1x - vxor 28, 28, 0 - - vxor 19, 19, 19 - - .long 0x12C3E4C8 - .long 0x12E4E4C8 - .long 0x1305E4C8 - - .long 0x137614C8 - - vsldoi 25, 23, 19, 8 - vsldoi 26, 19, 23, 8 - vxor 22, 22, 25 - vxor 24, 24, 26 - - vsldoi 22, 22, 22, 8 - vxor 22, 22, 27 - - vsldoi 20, 22, 22, 8 - .long 0x12D614C8 - vxor 20, 20, 24 - vxor 22, 22, 20 - - vor 0,22,22 - -.endm - - - - - - - - - - - - - -.global ppc_aes_gcm_encrypt -.align 5 -ppc_aes_gcm_encrypt: -_ppc_aes_gcm_encrypt: - - stdu 1,-512(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - std 0, 528(1) - - - lxvb16x 32, 0, 8 - + vxor 24, 24, 26 # M + + vpmsumd 26, 14, 19 # H4.H * X.H + vpmsumd 27, 11, 20 + vpmsumd 28, 8, 21 + vpmsumd 29, 5, 22 + + vxor 26, 26, 27 + vxor 26, 26, 28 + vxor 26, 26, 29 + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + vxor 1, 1, 1 + vsldoi 25, 24, 1, 8 # mL + vsldoi 1, 1, 24, 8 # mH + vxor 23, 23, 25 # mL + L + + # This performs swap and xor like, + # vsldoi 23, 23, 23, 8 # swap + # vxor 23, 23, 28 + xxlor 32+29, 10, 10 + vpermxor 23, 23, 28, 29 + + vxor 24, 26, 1 # H + + # sum hash and reduction with H Poly + # + # vsldoi 25, 23, 23, 8 # swap + # vpmsumd 23, 23, 2 + # vxor 27, 25, 24 + # + vpermxor 27, 23, 24, 29 + vpmsumd 23, 23, 2 + vxor 0, 23, 27 # Digest of 8 blocks +.endm + +# +# Compute update single ghash +# vs10: vpermxor vector +# scratch: v1, v22..v27 +# +.macro PPC_GHASH1x H S1 + + vxor 1, 1, 1 + + vpmsumd 22, 3, \S1 # L + vpmsumd 23, 4, \S1 # M + vpmsumd 24, 5, \S1 # H + + vpmsumd 27, 22, 2 # reduction + + vsldoi 25, 23, 1, 8 # mL + vsldoi 26, 1, 23, 8 # mH + vxor 22, 22, 25 # LL + LL + vxor 24, 24, 26 # HH + HH + + xxlor 32+25, 10, 10 + vpermxor 22, 22, 27, 25 + + # vsldoi 23, 22, 22, 8 # swap + # vpmsumd 22, 22, 2 # reduction + # vxor 23, 23, 24 + vpermxor 23, 22, 24, 25 + vpmsumd 22, 22, 2 # reduction + + vxor \H, 22, 23 +.endm + +# +# LOAD_HASH_TABLE +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# +.macro LOAD_HASH_TABLE + # Load Xi + lxvb16x 32, 0, 8 # load Xi + + vxor 1, 1, 1 li 10, 32 - lxvd2x 2+32, 10, 8 - li 10, 48 - lxvd2x 3+32, 10, 8 - li 10, 64 - lxvd2x 4+32, 10, 8 - li 10, 80 - lxvd2x 5+32, 10, 8 + lxvd2x 2+32, 10, 8 # H Poli - li 10, 96 - lxvd2x 6+32, 10, 8 + # load Hash - h^4, h^3, h^2, h + li 10, 64 + lxvd2x 4+32, 10, 8 # H + vsldoi 3, 1, 4, 8 # l + vsldoi 5, 4, 1, 8 # h li 10, 112 - lxvd2x 7+32, 10, 8 - li 10, 128 - lxvd2x 8+32, 10, 8 - - li 10, 144 - lxvd2x 9+32, 10, 8 + lxvd2x 7+32, 10, 8 # H^2 + vsldoi 6, 1, 7, 8 # l + vsldoi 8, 7, 1, 8 # h li 10, 160 - lxvd2x 10+32, 10, 8 - li 10, 176 - lxvd2x 11+32, 10, 8 - - li 10, 192 - lxvd2x 12+32, 10, 8 + lxvd2x 10+32, 10, 8 # H^3 + vsldoi 9, 1, 10, 8 # l + vsldoi 11, 10, 1, 8 # h li 10, 208 - lxvd2x 13+32, 10, 8 - li 10, 224 - lxvd2x 14+32, 10, 8 - - - lxvb16x 30+32, 0, 7 - - mr 12, 5 - li 11, 0 - - - vxor 31, 31, 31 - vspltisb 22,1 - vsldoi 31, 31, 22,1 - - - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - - lwz 9,240(6) + lxvd2x 13+32, 10, 8 # H^4 + vsldoi 12, 1, 13, 8 # l + vsldoi 14, 13, 1, 8 # h +.endm + +.macro PROCESS_8X_AES_STATES + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 +.endm + +.macro COMPUTE_STATES + xxlor 32+15, 9, 9 # last state + vadduwm 15, 15, 31 # state + counter + vadduwm 16, 15, 31 + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 +.endm + +################################################################################ +# Compute AES and ghash one block at a time. +# r23: AES rounds +# v30: current IV +# vs0: roundkey 0 +# +################################################################################ +.align 4 +aes_gcm_crypt_1x: +.localentry aes_gcm_crypt_1x,0 + + cmpdi 5, 16 + bge __More_1x + blr +__More_1x: + li 10, 16 + divdu 12, 5, 10 + + xxlxor 32+15, 32+30, 0 + + # Pre-load 8 AES rounds to scratch vectors. + lxv 32+16, 16(6) # round key 1 + lxv 32+17, 32(6) # round key 2 + lxv 32+18, 48(6) # round key 3 + lxv 32+19, 64(6) # round key 4 + lxv 32+20, 80(6) # round key 5 + lxv 32+21, 96(6) # round key 6 + lxv 32+28, 112(6) # round key 7 + lxv 32+29, 128(6) # round key 8 + + lwz 23, 240(6) # n rounds + addi 22, 23, -9 # remaining AES rounds + cmpdi 12, 0 + bgt __Loop_1x + blr + +__Loop_1x: + mtctr 22 + addi 10, 6, 144 + vcipher 15, 15, 16 + vcipher 15, 15, 17 + vcipher 15, 15, 18 + vcipher 15, 15, 19 + vcipher 15, 15, 20 + vcipher 15, 15, 21 + vcipher 15, 15, 28 + vcipher 15, 15, 29 + +__Loop_aes_1state: + lxv 32+1, 0(10) + vcipher 15, 15, 1 + addi 10, 10, 16 + bdnz __Loop_aes_1state + lxv 32+1, 0(10) # last round key + lxvb16x 11, 0, 14 # load input block + vcipherlast 15, 15, 1 - xxlor 32+29, 0, 0 - vxor 15, 30, 29 + xxlxor 32+15, 32+15, 11 + stxvb16x 32+15, 0, 9 # store output + addi 14, 14, 16 + addi 9, 9, 16 - cmpdi 9, 10 - beq .Loop_aes_gcm_8x + cmpdi 24, 0 # decrypt? + bne __Encrypt_1x + xxlor 15+32, 11, 11 +__Encrypt_1x: + vxor 15, 15, 0 + PPC_GHASH1x 0, 15 + addi 5, 5, -16 + addi 11, 11, 16 - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) + vadduwm 30, 30, 31 # IV + counter + xxlxor 32+15, 32+30, 0 + addi 12, 12, -1 + cmpdi 12, 0 + bgt __Loop_1x + + stxvb16x 32+0, 0, 8 # update Xi + blr +.size aes_gcm_crypt_1x,.-aes_gcm_crypt_1x + +################################################################################ +# Process a normal partial block when we come here. +# Compute partial mask, Load and store partial block to stack. +# Compute AES state. +# Compute ghash. +# +################################################################################ +.align 4 +__Process_partial: +.localentry __Process_partial,0 + + # create partial mask + vspltisb 16, -1 + li 12, 16 + sub 12, 12, 5 + sldi 12, 12, 3 + mtvsrdd 32+17, 0, 12 + vslo 16, 16, 17 # partial block mask + + lxvb16x 11, 0, 14 # load partial block + xxland 11, 11, 32+16 + + # AES crypt partial + xxlxor 32+15, 32+30, 0 + lwz 23, 240(6) # n rounds + addi 22, 23, -1 # loop - 1 + mtctr 22 + addi 10, 6, 16 + +__Loop_aes_pstate: + lxv 32+1, 0(10) + vcipher 15, 15, 1 + addi 10, 10, 16 + bdnz __Loop_aes_pstate + lxv 32+1, 0(10) # last round key + vcipherlast 15, 15, 1 - cmpdi 9, 12 - beq .Loop_aes_gcm_8x + xxlxor 32+15, 32+15, 11 + vand 15, 15, 16 + # AES crypt output v15 + # Write partial + li 10, 224 + stxvb16x 15+32, 10, 1 # write v15 to stack + addi 10, 1, 223 + addi 12, 9, -1 + mtctr 5 # partial block len +__Write_partial: + lbzu 22, 1(10) + stbu 22, 1(12) + bdnz __Write_partial + + cmpdi 24, 0 # decrypt? + bne __Encrypt_partial + xxlor 32+15, 11, 11 # decrypt using the input block +__Encrypt_partial: + vxor 15, 15, 0 # ^ previous hash + PPC_GHASH1x 0, 15 + li 5, 0 # done last byte + stxvb16x 32+0, 0, 8 # Update X1 + blr +.size __Process_partial,.-__Process_partial + +################################################################################ +# ppc_aes_gcm_encrypt (const void *inp, void *out, size_t len, +# const char *rk, unsigned char iv[16], void *Xip); +# +# r3 - inp +# r4 - out +# r5 - len +# r6 - AES round keys +# r7 - iv +# r8 - Xi, HPoli, hash keys +# +# rounds is at offset 240 in rk +# Xi is at 0 in gcm_table (Xip). +# +################################################################################ +.global ppc_aes_gcm_encrypt +.align 5 +ppc_aes_gcm_encrypt: +.localentry ppc_aes_gcm_encrypt,0 - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq .Loop_aes_gcm_8x + SAVE_REGS + LOAD_HASH_TABLE - b aes_gcm_out + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 -.align 5 -.Loop_aes_gcm_8x: mr 14, 3 mr 9, 4 + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 + + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + li 11, 0 + lxv 0, 0(6) # round key 0 + + # + # Process different blocks + # + cmpdi 5, 128 + blt __Process_more_enc + + # load 9 round keys + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 23, 240(6) # n rounds + +__Process_encrypt: +# +# Process 8x AES/GCM blocks +# +__Process_8x_enc: + # 8x blocks li 10, 128 - divdu 10, 5, 10 - cmpdi 10, 0 - beq .Loop_last_block - - .long 0x13DEF8C0 - vxor 16, 30, 29 - .long 0x13DEF8C0 - vxor 17, 30, 29 - .long 0x13DEF8C0 - vxor 18, 30, 29 - .long 0x13DEF8C0 - vxor 19, 30, 29 - .long 0x13DEF8C0 - vxor 20, 30, 29 - .long 0x13DEF8C0 - vxor 21, 30, 29 - .long 0x13DEF8C0 - vxor 22, 30, 29 - - mtctr 10 + divdu 12, 5, 10 # n 128 bytes-blocks + + addi 12, 12, -1 # loop - 1 + + vmr 15, 30 # first state: IV + vadduwm 16, 15, 31 # state + counter + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + # vxor state, state, w # addroundkey + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 li 15, 16 li 16, 32 @@ -536,523 +591,185 @@ _ppc_aes_gcm_encrypt: li 20, 96 li 21, 112 - lwz 10, 240(6) - -.Loop_8x_block: - - lxvb16x 15, 0, 14 - lxvb16x 16, 15, 14 - lxvb16x 17, 16, 14 - lxvb16x 18, 17, 14 - lxvb16x 19, 18, 14 - lxvb16x 20, 19, 14 - lxvb16x 21, 20, 14 - lxvb16x 22, 21, 14 - addi 14, 14, 128 - -.Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_ghash - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_ghash - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_ghash - b aes_gcm_out - -Do_next_ghash: - - - - .long 0x11EFBD09 - .long 0x1210BD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 - - .long 0x1231BD09 - .long 0x1252BD09 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 - - .long 0x1273BD09 - .long 0x1294BD09 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 - - .long 0x12B5BD09 - .long 0x12D6BD09 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 - - addi 9, 9, 128 - - - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - .long 0x13DEF8C0 - vor 29,30,30 - vxor 15, 30, 27 - .long 0x13DEF8C0 - vxor 16, 30, 27 - .long 0x13DEF8C0 - vxor 17, 30, 27 - .long 0x13DEF8C0 - vxor 18, 30, 27 - .long 0x13DEF8C0 - vxor 19, 30, 27 - .long 0x13DEF8C0 - vxor 20, 30, 27 - .long 0x13DEF8C0 - vxor 21, 30, 27 - .long 0x13DEF8C0 - vxor 22, 30, 27 - - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz .Loop_8x_block - - vor 30,29,29 - -.Loop_last_block: - cmpdi 12, 0 - beq aes_gcm_out - - - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10, 240(6) - - cmpdi 12, 16 - blt Final_block - -.macro .Loop_aes_middle_1x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - .long 0x11EF9D08 - .long 0x11EFA508 - .long 0x11EFAD08 - .long 0x11EFB508 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - .long 0x11EF9D08 - .long 0x11EFA508 - .long 0x11EFAD08 - .long 0x11EFB508 - - xxlor 19+32, 9, 9 - .long 0x11EF9D08 -.endm - -Next_rem_block: - lxvb16x 15, 0, 14 - -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 + # + # Pre-compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + # + addi 22, 23, -9 # process 8 keys + mtctr 22 # AES key loop + addi 10, 6, 144 - cmpdi 10, 10 - beq Do_next_1x + LOOP_8AES_STATE # process 8 AES keys +__PreLoop_aes_state: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __PreLoop_aes_state + lxv 32+1, 0(10) # last round key (v1) - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_1x - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x - -Do_next_1x: - .long 0x11EFBD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - addi 14, 14, 16 - addi 9, 9, 16 + cmpdi 12, 0 # Only one loop (8 block) + beq __Finish_ghash - vor 28,15,15 - ppc_update_hash_1x +# +# Loop 8x blocks and compute ghash +# +__Loop_8x_block_enc: + PROCESS_8X_AES_STATES - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - .long 0x13DEF8C0 - vxor 15, 30, 19 + # Compute ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x + + COMPUTE_STATES + + addi 5, 5, -128 + addi 11, 11, 128 + + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # Compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + LOOP_8AES_STATE # process 8 AES keys + mtctr 22 # AES key loop + addi 10, 6, 144 + +__LastLoop_aes_state: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __LastLoop_aes_state - bdnz Next_rem_block + lxv 32+1, 0(10) # last round key (v1) + addi 12, 12, -1 cmpdi 12, 0 - beq aes_gcm_out - -Final_block: -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x + bne __Loop_8x_block_enc + # + # Remainng blocks + # +__Finish_ghash: + PROCESS_8X_AES_STATES - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_final_1x - -Do_final_1x: - .long 0x11EFBD09 - - lxvb16x 15, 0, 14 - xxlxor 47, 47, 15 - - - li 15, 16 - sub 15, 15, 12 + # Compute ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x - vspltisb 16,-1 - vspltisb 17,0 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 + # Update IV and Xi + xxlor 30+32, 9, 9 # last ctr + vadduwm 30, 30, 31 # increase ctr + stxvb16x 32+0, 0, 8 # update Xi - addi 10, 1, 192 - lxvb16x 16, 15, 10 - xxland 47, 47, 16 + addi 5, 5, -128 + addi 11, 11, 128 - vor 28,15,15 - ppc_update_hash_1x + # + # Done 8x blocks + # + cmpdi 5, 0 + beq aes_gcm_out - bl Write_partial_block +__Process_more_enc: + li 24, 1 # encrypt + bl aes_gcm_crypt_1x + cmpdi 5, 0 + beq aes_gcm_out + bl __Process_partial b aes_gcm_out +.size ppc_aes_gcm_encrypt,.-ppc_aes_gcm_encrypt - - - - - -Write_partial_block: - li 10, 192 - stxvb16x 15+32, 10, 1 - - - addi 10, 9, -1 - addi 16, 1, 191 - - mtctr 12 - li 15, 0 - -Write_last_byte: - lbzu 14, 1(16) - stbu 14, 1(10) - bdnz Write_last_byte - blr - -aes_gcm_out: - - stxvb16x 32, 0, 8 - add 3, 11, 12 - - li 9, 256 - lvx 20, 9, 1 - addi 9, 9, 16 - lvx 21, 9, 1 - addi 9, 9, 16 - lvx 22, 9, 1 - addi 9, 9, 16 - lvx 23, 9, 1 - addi 9, 9, 16 - lvx 24, 9, 1 - addi 9, 9, 16 - lvx 25, 9, 1 - addi 9, 9, 16 - lvx 26, 9, 1 - addi 9, 9, 16 - lvx 27, 9, 1 - addi 9, 9, 16 - lvx 28, 9, 1 - addi 9, 9, 16 - lvx 29, 9, 1 - addi 9, 9, 16 - lvx 30, 9, 1 - addi 9, 9, 16 - lvx 31, 9, 1 - - ld 0, 528(1) - ld 14,112(1) - ld 15,120(1) - ld 16,128(1) - ld 17,136(1) - ld 18,144(1) - ld 19,152(1) - ld 20,160(1) - ld 21,168(1) - - mtlr 0 - addi 1, 1, 512 - blr - - - - -.global ppc_aes_gcm_decrypt -.align 5 +################################################################################ +# ppc_aes_gcm_decrypt (const void *inp, void *out, size_t len, +# const char *rk, unsigned char iv[16], void *Xip); +# 8x Decrypt +# +################################################################################ +.global ppc_aes_gcm_decrypt +.align 5 ppc_aes_gcm_decrypt: -_ppc_aes_gcm_decrypt: - - stdu 1,-512(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - std 0, 528(1) - - - lxvb16x 32, 0, 8 - - - li 10, 32 - lxvd2x 2+32, 10, 8 - li 10, 48 - lxvd2x 3+32, 10, 8 - li 10, 64 - lxvd2x 4+32, 10, 8 - li 10, 80 - lxvd2x 5+32, 10, 8 - - li 10, 96 - lxvd2x 6+32, 10, 8 - li 10, 112 - lxvd2x 7+32, 10, 8 - li 10, 128 - lxvd2x 8+32, 10, 8 - - li 10, 144 - lxvd2x 9+32, 10, 8 - li 10, 160 - lxvd2x 10+32, 10, 8 - li 10, 176 - lxvd2x 11+32, 10, 8 - - li 10, 192 - lxvd2x 12+32, 10, 8 - li 10, 208 - lxvd2x 13+32, 10, 8 - li 10, 224 - lxvd2x 14+32, 10, 8 - - - lxvb16x 30+32, 0, 7 - - mr 12, 5 - li 11, 0 - +.localentry ppc_aes_gcm_decrypt, 0 - vxor 31, 31, 31 - vspltisb 22,1 - vsldoi 31, 31, 22,1 - - - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - - lwz 9,240(6) - - - - xxlor 32+29, 0, 0 - vxor 15, 30, 29 - - cmpdi 9, 10 - beq .Loop_aes_gcm_8x_dec - - - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) - - cmpdi 9, 12 - beq .Loop_aes_gcm_8x_dec + SAVE_REGS + LOAD_HASH_TABLE + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq .Loop_aes_gcm_8x_dec - - b aes_gcm_out - -.align 5 -.Loop_aes_gcm_8x_dec: mr 14, 3 mr 9, 4 + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 + + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + li 11, 0 + lxv 0, 0(6) # round key 0 + + # + # Process different blocks + # + cmpdi 5, 128 + blt __Process_more_dec + + # load 9 round keys + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 23, 240(6) # n rounds + +__Process_decrypt: +# +# Process 8x AES/GCM blocks +# +__Process_8x_dec: + # 8x blocks li 10, 128 - divdu 10, 5, 10 - cmpdi 10, 0 - beq .Loop_last_block_dec - - .long 0x13DEF8C0 - vxor 16, 30, 29 - .long 0x13DEF8C0 - vxor 17, 30, 29 - .long 0x13DEF8C0 - vxor 18, 30, 29 - .long 0x13DEF8C0 - vxor 19, 30, 29 - .long 0x13DEF8C0 - vxor 20, 30, 29 - .long 0x13DEF8C0 - vxor 21, 30, 29 - .long 0x13DEF8C0 - vxor 22, 30, 29 - - mtctr 10 + divdu 12, 5, 10 # n 128 bytes-blocks + + addi 12, 12, -1 # loop - 1 + + vmr 15, 30 # first state: IV + vadduwm 16, 15, 31 # state + counter + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + # vxor state, state, w # addroundkey + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 li 15, 16 li 16, 32 @@ -1062,279 +779,215 @@ _ppc_aes_gcm_decrypt: li 20, 96 li 21, 112 - lwz 10, 240(6) + # + # Pre-compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + # + addi 22, 23, -9 # process 8 keys + mtctr 22 # AES key loop + addi 10, 6, 144 -.Loop_8x_block_dec: + LOOP_8AES_STATE # process 8 AES keys - lxvb16x 15, 0, 14 - lxvb16x 16, 15, 14 - lxvb16x 17, 16, 14 - lxvb16x 18, 17, 14 - lxvb16x 19, 18, 14 - lxvb16x 20, 19, 14 - lxvb16x 21, 20, 14 - lxvb16x 22, 21, 14 +__PreLoop_aes_state_dec: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __PreLoop_aes_state_dec + lxv 32+1, 0(10) # last round key (v1) + + cmpdi 12, 0 # Only one loop (8 block) + beq __Finish_ghash_dec + +# +# Loop 8x blocks and compute ghash +# +__Loop_8x_block_dec: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block addi 14, 14, 128 -.Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_last_aes_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_last_aes_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_last_aes_dec - b aes_gcm_out - -Do_last_aes_dec: - - - - .long 0x11EFBD09 - .long 0x1210BD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 - - .long 0x1231BD09 - .long 0x1252BD09 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 - - .long 0x1273BD09 - .long 0x1294BD09 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 - - .long 0x12B5BD09 - .long 0x12D6BD09 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output addi 9, 9, 128 - xxlor 15+32, 15, 15 - xxlor 16+32, 16, 16 - xxlor 17+32, 17, 17 - xxlor 18+32, 18, 18 - xxlor 19+32, 19, 19 - xxlor 20+32, 20, 20 - xxlor 21+32, 21, 21 - xxlor 22+32, 22, 22 - - - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - .long 0x13DEF8C0 - vor 29,30,30 - vxor 15, 30, 27 - .long 0x13DEF8C0 - vxor 16, 30, 27 - .long 0x13DEF8C0 - vxor 17, 30, 27 - .long 0x13DEF8C0 - vxor 18, 30, 27 - .long 0x13DEF8C0 - vxor 19, 30, 27 - .long 0x13DEF8C0 - vxor 20, 30, 27 - .long 0x13DEF8C0 - vxor 21, 30, 27 - .long 0x13DEF8C0 - vxor 22, 30, 27 - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz .Loop_8x_block_dec - - vor 30,29,29 - -.Loop_last_block_dec: - cmpdi 12, 0 - beq aes_gcm_out - - - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10,240(6) - - cmpdi 12, 16 - blt Final_block_dec - -Next_rem_block_dec: - lxvb16x 15, 0, 14 - -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_1x_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_1x_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x_dec - -Do_next_1x_dec: - .long 0x11EFBD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - addi 14, 14, 16 - addi 9, 9, 16 - - xxlor 28+32, 15, 15 - ppc_update_hash_1x + vmr 15, 23 + vmr 16, 24 + vmr 17, 25 + vmr 18, 26 + vmr 19, 27 + vmr 20, 28 + vmr 21, 29 + vmr 22, 30 - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - .long 0x13DEF8C0 - vxor 15, 30, 19 + # ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x + + xxlor 32+15, 9, 9 # last state + vadduwm 15, 15, 31 # state + counter + vadduwm 16, 15, 31 + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + xxlor 32+27, 0, 0 # restore roundkey 0 + vxor 15, 15, 27 # IV + round key - add round key 0 + vxor 16, 16, 27 + vxor 17, 17, 27 + vxor 18, 18, 27 + vxor 19, 19, 27 + vxor 20, 20, 27 + vxor 21, 21, 27 + vxor 22, 22, 27 - bdnz Next_rem_block_dec + addi 5, 5, -128 + addi 11, 11, 128 + + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + LOOP_8AES_STATE # process 8 AES keys + mtctr 22 # AES key loop + addi 10, 6, 144 +__LastLoop_aes_state_dec: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __LastLoop_aes_state_dec + lxv 32+1, 0(10) # last round key (v1) + addi 12, 12, -1 cmpdi 12, 0 - beq aes_gcm_out - -Final_block_dec: -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x_dec - + bne __Loop_8x_block_dec + +__Finish_ghash_dec: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 - xxlor 24+32, 13, 13 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 - .long 0x11EFBD08 - .long 0x11EFC508 + vxor 15, 23, 0 + vmr 16, 24 + vmr 17, 25 + vmr 18, 26 + vmr 19, 27 + vmr 20, 28 + vmr 21, 29 + vmr 22, 30 - xxlor 23+32, 14, 14 + #vxor 15, 15, 0 + PPC_GFMUL128_8x - cmpdi 10, 14 - beq Do_final_1x_dec + xxlor 30+32, 9, 9 # last ctr + vadduwm 30, 30, 31 # increase ctr + stxvb16x 32+0, 0, 8 # update Xi -Do_final_1x_dec: - .long 0x11EFBD09 + addi 5, 5, -128 + addi 11, 11, 128 - lxvb16x 15, 0, 14 - xxlxor 47, 47, 15 + # + # Done 8x blocks + # + cmpdi 5, 0 + beq aes_gcm_out - li 15, 16 - sub 15, 15, 12 +__Process_more_dec: + li 24, 0 # decrypt + bl aes_gcm_crypt_1x + cmpdi 5, 0 + beq aes_gcm_out - vspltisb 16,-1 - vspltisb 17,0 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 - - addi 10, 1, 192 - lxvb16x 16, 15, 10 - xxland 47, 47, 16 + bl __Process_partial + b aes_gcm_out +.size ppc_aes_gcm_decrypt,.-ppc_aes_gcm_decrypt - xxlor 28+32, 15, 15 - ppc_update_hash_1x +aes_gcm_out: +.localentry aes_gcm_out,0 + mr 3, 11 # return count - bl Write_partial_block + RESTORE_REGS + blr +.size aes_gcm_out,.-aes_gcm_out - b aes_gcm_out +.rodata +.align 4 +# for vector permute and xor +permx: +.long 0x4c5d6e7f, 0x08192a3b, 0xc4d5e6f7, 0x8091a2b3 diff --git a/sys/crypto/openssl/powerpc64le/aes-gcm-ppc.S b/sys/crypto/openssl/powerpc64le/aes-gcm-ppc.S index 2ff143c42ab7..51cfac7e45fc 100644 --- a/sys/crypto/openssl/powerpc64le/aes-gcm-ppc.S +++ b/sys/crypto/openssl/powerpc64le/aes-gcm-ppc.S @@ -1,532 +1,587 @@ /* Do not modify. This file is auto-generated from aes-gcm-ppc.pl. */ -.machine "any" -.abiversion 2 +.machine "any" .text - - - - -.macro .Loop_aes_middle4x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - .long 0x11EF9D08 - .long 0x12109D08 - .long 0x12319D08 - .long 0x12529D08 - - .long 0x11EFA508 - .long 0x1210A508 - .long 0x1231A508 - .long 0x1252A508 - - .long 0x11EFAD08 - .long 0x1210AD08 - .long 0x1231AD08 - .long 0x1252AD08 - - .long 0x11EFB508 - .long 0x1210B508 - .long 0x1231B508 - .long 0x1252B508 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - .long 0x11EF9D08 - .long 0x12109D08 - .long 0x12319D08 - .long 0x12529D08 - - .long 0x11EFA508 - .long 0x1210A508 - .long 0x1231A508 - .long 0x1252A508 - - .long 0x11EFAD08 - .long 0x1210AD08 - .long 0x1231AD08 - .long 0x1252AD08 - - .long 0x11EFB508 - .long 0x1210B508 - .long 0x1231B508 - .long 0x1252B508 - - xxlor 23+32, 9, 9 - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 -.endm - - - - - -.macro .Loop_aes_middle8x - xxlor 23+32, 1, 1 - xxlor 24+32, 2, 2 - xxlor 25+32, 3, 3 - xxlor 26+32, 4, 4 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - .long 0x11EFCD08 - .long 0x1210CD08 - .long 0x1231CD08 - .long 0x1252CD08 - .long 0x1273CD08 - .long 0x1294CD08 - .long 0x12B5CD08 - .long 0x12D6CD08 - - .long 0x11EFD508 - .long 0x1210D508 - .long 0x1231D508 - .long 0x1252D508 - .long 0x1273D508 - .long 0x1294D508 - .long 0x12B5D508 - .long 0x12D6D508 - - xxlor 23+32, 5, 5 - xxlor 24+32, 6, 6 - xxlor 25+32, 7, 7 - xxlor 26+32, 8, 8 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - .long 0x11EFCD08 - .long 0x1210CD08 - .long 0x1231CD08 - .long 0x1252CD08 - .long 0x1273CD08 - .long 0x1294CD08 - .long 0x12B5CD08 - .long 0x12D6CD08 - - .long 0x11EFD508 - .long 0x1210D508 - .long 0x1231D508 - .long 0x1252D508 - .long 0x1273D508 - .long 0x1294D508 - .long 0x12B5D508 - .long 0x12D6D508 - - xxlor 23+32, 9, 9 - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 -.endm - - - - -ppc_aes_gcm_ghash: - vxor 15, 15, 0 - - xxlxor 29, 29, 29 - - .long 0x12EC7CC8 - .long 0x130984C8 - .long 0x13268CC8 - .long 0x134394C8 - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D7CC8 - .long 0x132A84C8 - .long 0x13478CC8 - .long 0x136494C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E7CC8 - .long 0x132B84C8 - .long 0x13488CC8 - .long 0x136594C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 - - blr - - - - - -.macro ppc_aes_gcm_ghash2_4x - - vxor 15, 15, 0 - - xxlxor 29, 29, 29 - - .long 0x12EC7CC8 - .long 0x130984C8 - .long 0x13268CC8 - .long 0x134394C8 +.macro SAVE_REGS + mflr 0 + std 0, 16(1) + stdu 1,-512(1) + + std 14, 112(1) + std 15, 120(1) + std 16, 128(1) + std 17, 136(1) + std 18, 144(1) + std 19, 152(1) + std 20, 160(1) + std 21, 168(1) + std 22, 176(1) + std 23, 184(1) + std 24, 192(1) + + stxv 32+20, 256(1) + stxv 32+21, 256+16(1) + stxv 32+22, 256+32(1) + stxv 32+23, 256+48(1) + stxv 32+24, 256+64(1) + stxv 32+25, 256+80(1) + stxv 32+26, 256+96(1) + stxv 32+27, 256+112(1) + stxv 32+28, 256+128(1) + stxv 32+29, 256+144(1) + stxv 32+30, 256+160(1) + stxv 32+31, 256+176(1) +.endm # SAVE_REGS + +.macro RESTORE_REGS + lxv 32+20, 256(1) + lxv 32+21, 256+16(1) + lxv 32+22, 256+32(1) + lxv 32+23, 256+48(1) + lxv 32+24, 256+64(1) + lxv 32+25, 256+80(1) + lxv 32+26, 256+96(1) + lxv 32+27, 256+112(1) + lxv 32+28, 256+128(1) + lxv 32+29, 256+144(1) + lxv 32+30, 256+160(1) + lxv 32+31, 256+176(1) + + ld 14, 112(1) + ld 15, 120(1) + ld 16, 128(1) + ld 17, 136(1) + ld 18, 144(1) + ld 19, 152(1) + ld 20, 160(1) + ld 21, 168(1) + ld 22, 176(1) + ld 23, 184(1) + ld 24, 192(1) + + addi 1, 1, 512 + ld 0, 16(1) + mtlr 0 +.endm # RESTORE_REGS + +# 4x loops +.macro AES_CIPHER_4x r + vcipher 15, 15, \r + vcipher 16, 16, \r + vcipher 17, 17, \r + vcipher 18, 18, \r +.endm + +# 8x loops +.macro AES_CIPHER_8x r + vcipher 15, 15, \r + vcipher 16, 16, \r + vcipher 17, 17, \r + vcipher 18, 18, \r + vcipher 19, 19, \r + vcipher 20, 20, \r + vcipher 21, 21, \r + vcipher 22, 22, \r +.endm + +.macro LOOP_8AES_STATE + AES_CIPHER_8x 23 + AES_CIPHER_8x 24 + AES_CIPHER_8x 25 + AES_CIPHER_8x 26 + AES_CIPHER_8x 27 + AES_CIPHER_8x 28 + AES_CIPHER_8x 29 + AES_CIPHER_8x 1 +.endm + +# +# PPC_GFMUL128_8x: Compute hash values of 8 blocks based on Karatsuba method. +# +# S1 should xor with the previous digest +# +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# vs10: vpermxor vector +# Scratch: v23 - v29 +# +.macro PPC_GFMUL128_8x + + vpmsumd 23, 12, 15 # H4.L * X.L + vpmsumd 24, 9, 16 + vpmsumd 25, 6, 17 + vpmsumd 26, 3, 18 vxor 23, 23, 24 vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D7CC8 - .long 0x132A84C8 - .long 0x13478CC8 - .long 0x136494C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - - vxor 24, 24, 27 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 + vxor 23, 23, 26 # L - .long 0x130E7CC8 - .long 0x132B84C8 - .long 0x13488CC8 - .long 0x136594C8 + vpmsumd 27, 13, 15 # H4.L * X.H + H4.H * X.L + vpmsumd 28, 10, 16 # H3.L * X1.H + H3.H * X1.L + vpmsumd 25, 7, 17 + vpmsumd 26, 4, 18 + vxor 24, 27, 28 vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 27, 23, 27 - - - .long 0x1309A4C8 - .long 0x1326ACC8 - .long 0x1343B4C8 - vxor 19, 19, 27 - .long 0x12EC9CC8 + vxor 24, 24, 26 # M + + vpmsumd 26, 14, 15 # H4.H * X.H + vpmsumd 27, 11, 16 + vpmsumd 28, 8, 17 + vpmsumd 29, 5, 18 + + vxor 26, 26, 27 + vxor 26, 26, 28 + vxor 26, 26, 29 + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + vxor 1, 1, 1 + vsldoi 25, 24, 1, 8 # mL + vsldoi 1, 1, 24, 8 # mH + vxor 23, 23, 25 # mL + L + + # This performs swap and xor like, + # vsldoi 23, 23, 23, 8 # swap + # vxor 23, 23, 28 + xxlor 32+29, 10, 10 + vpermxor 23, 23, 28, 29 + + vxor 24, 26, 1 # H + + # sum hash and reduction with H Poly + # + # vsldoi 25, 23, 23, 8 # swap + # vpmsumd 23, 23, 2 + # vxor 27, 25, 24 + # + vpermxor 27, 23, 24, 29 + vpmsumd 23, 23, 2 + vxor 0, 23, 27 # Digest of 4 blocks + + vxor 19, 19, 0 + + # Compute digest for the next 4 blocks + vpmsumd 24, 9, 20 + vpmsumd 25, 6, 21 + vpmsumd 26, 3, 22 + vpmsumd 23, 12, 19 # H4.L * X.L vxor 23, 23, 24 vxor 23, 23, 25 - vxor 23, 23, 26 + vxor 23, 23, 26 # L - .long 0x130D9CC8 - .long 0x132AA4C8 - .long 0x1347ACC8 - .long 0x1364B4C8 + vpmsumd 27, 13, 19 # H4.L * X.H + H4.H * X.L + vpmsumd 28, 10, 20 # H3.L * X1.H + H3.H * X1.L + vpmsumd 25, 7, 21 + vpmsumd 26, 4, 22 + vxor 24, 27, 28 vxor 24, 24, 25 - vxor 24, 24, 26 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - - vxor 24, 24, 27 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E9CC8 - .long 0x132BA4C8 - .long 0x1348ACC8 - .long 0x1365B4C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 - -.endm - - - - -.macro ppc_update_hash_1x - vxor 28, 28, 0 - - vxor 19, 19, 19 - - .long 0x12C3E4C8 - .long 0x12E4E4C8 - .long 0x1305E4C8 - - .long 0x137614C8 - - vsldoi 25, 23, 19, 8 - vsldoi 26, 19, 23, 8 - vxor 22, 22, 25 - vxor 24, 24, 26 - - vsldoi 22, 22, 22, 8 - vxor 22, 22, 27 - - vsldoi 20, 22, 22, 8 - .long 0x12D614C8 - vxor 20, 20, 24 - vxor 22, 22, 20 - - vor 0,22,22 - -.endm - - - - - - - - - - - - - -.global ppc_aes_gcm_encrypt -.align 5 -ppc_aes_gcm_encrypt: -_ppc_aes_gcm_encrypt: - - stdu 1,-512(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - std 0, 528(1) - - - lxvb16x 32, 0, 8 - + vxor 24, 24, 26 # M + + vpmsumd 26, 14, 19 # H4.H * X.H + vpmsumd 27, 11, 20 + vpmsumd 28, 8, 21 + vpmsumd 29, 5, 22 + + vxor 26, 26, 27 + vxor 26, 26, 28 + vxor 26, 26, 29 + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + vxor 1, 1, 1 + vsldoi 25, 24, 1, 8 # mL + vsldoi 1, 1, 24, 8 # mH + vxor 23, 23, 25 # mL + L + + # This performs swap and xor like, + # vsldoi 23, 23, 23, 8 # swap + # vxor 23, 23, 28 + xxlor 32+29, 10, 10 + vpermxor 23, 23, 28, 29 + + vxor 24, 26, 1 # H + + # sum hash and reduction with H Poly + # + # vsldoi 25, 23, 23, 8 # swap + # vpmsumd 23, 23, 2 + # vxor 27, 25, 24 + # + vpermxor 27, 23, 24, 29 + vpmsumd 23, 23, 2 + vxor 0, 23, 27 # Digest of 8 blocks +.endm + +# +# Compute update single ghash +# vs10: vpermxor vector +# scratch: v1, v22..v27 +# +.macro PPC_GHASH1x H S1 + + vxor 1, 1, 1 + + vpmsumd 22, 3, \S1 # L + vpmsumd 23, 4, \S1 # M + vpmsumd 24, 5, \S1 # H + + vpmsumd 27, 22, 2 # reduction + + vsldoi 25, 23, 1, 8 # mL + vsldoi 26, 1, 23, 8 # mH + vxor 22, 22, 25 # LL + LL + vxor 24, 24, 26 # HH + HH + + xxlor 32+25, 10, 10 + vpermxor 22, 22, 27, 25 + + # vsldoi 23, 22, 22, 8 # swap + # vpmsumd 22, 22, 2 # reduction + # vxor 23, 23, 24 + vpermxor 23, 22, 24, 25 + vpmsumd 22, 22, 2 # reduction + + vxor \H, 22, 23 +.endm + +# +# LOAD_HASH_TABLE +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# +.macro LOAD_HASH_TABLE + # Load Xi + lxvb16x 32, 0, 8 # load Xi + + vxor 1, 1, 1 li 10, 32 - lxvd2x 2+32, 10, 8 - li 10, 48 - lxvd2x 3+32, 10, 8 - li 10, 64 - lxvd2x 4+32, 10, 8 - li 10, 80 - lxvd2x 5+32, 10, 8 + lxvd2x 2+32, 10, 8 # H Poli - li 10, 96 - lxvd2x 6+32, 10, 8 + # load Hash - h^4, h^3, h^2, h + li 10, 64 + lxvd2x 4+32, 10, 8 # H + vsldoi 3, 1, 4, 8 # l + vsldoi 5, 4, 1, 8 # h li 10, 112 - lxvd2x 7+32, 10, 8 - li 10, 128 - lxvd2x 8+32, 10, 8 - - li 10, 144 - lxvd2x 9+32, 10, 8 + lxvd2x 7+32, 10, 8 # H^2 + vsldoi 6, 1, 7, 8 # l + vsldoi 8, 7, 1, 8 # h li 10, 160 - lxvd2x 10+32, 10, 8 - li 10, 176 - lxvd2x 11+32, 10, 8 - - li 10, 192 - lxvd2x 12+32, 10, 8 + lxvd2x 10+32, 10, 8 # H^3 + vsldoi 9, 1, 10, 8 # l + vsldoi 11, 10, 1, 8 # h li 10, 208 - lxvd2x 13+32, 10, 8 - li 10, 224 - lxvd2x 14+32, 10, 8 - - - lxvb16x 30+32, 0, 7 - - mr 12, 5 - li 11, 0 - - - vxor 31, 31, 31 - vspltisb 22,1 - vsldoi 31, 31, 22,1 - - - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - - lwz 9,240(6) + lxvd2x 13+32, 10, 8 # H^4 + vsldoi 12, 1, 13, 8 # l + vsldoi 14, 13, 1, 8 # h +.endm + +.macro PROCESS_8X_AES_STATES + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 +.endm + +.macro COMPUTE_STATES + xxlor 32+15, 9, 9 # last state + vadduwm 15, 15, 31 # state + counter + vadduwm 16, 15, 31 + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 +.endm + +################################################################################ +# Compute AES and ghash one block at a time. +# r23: AES rounds +# v30: current IV +# vs0: roundkey 0 +# +################################################################################ +.align 4 +aes_gcm_crypt_1x: +.localentry aes_gcm_crypt_1x,0 + + cmpdi 5, 16 + bge __More_1x + blr +__More_1x: + li 10, 16 + divdu 12, 5, 10 + + xxlxor 32+15, 32+30, 0 + + # Pre-load 8 AES rounds to scratch vectors. + lxv 32+16, 16(6) # round key 1 + lxv 32+17, 32(6) # round key 2 + lxv 32+18, 48(6) # round key 3 + lxv 32+19, 64(6) # round key 4 + lxv 32+20, 80(6) # round key 5 + lxv 32+21, 96(6) # round key 6 + lxv 32+28, 112(6) # round key 7 + lxv 32+29, 128(6) # round key 8 + + lwz 23, 240(6) # n rounds + addi 22, 23, -9 # remaining AES rounds + cmpdi 12, 0 + bgt __Loop_1x + blr + +__Loop_1x: + mtctr 22 + addi 10, 6, 144 + vcipher 15, 15, 16 + vcipher 15, 15, 17 + vcipher 15, 15, 18 + vcipher 15, 15, 19 + vcipher 15, 15, 20 + vcipher 15, 15, 21 + vcipher 15, 15, 28 + vcipher 15, 15, 29 + +__Loop_aes_1state: + lxv 32+1, 0(10) + vcipher 15, 15, 1 + addi 10, 10, 16 + bdnz __Loop_aes_1state + lxv 32+1, 0(10) # last round key + lxvb16x 11, 0, 14 # load input block + vcipherlast 15, 15, 1 - xxlor 32+29, 0, 0 - vxor 15, 30, 29 + xxlxor 32+15, 32+15, 11 + stxvb16x 32+15, 0, 9 # store output + addi 14, 14, 16 + addi 9, 9, 16 - cmpdi 9, 10 - beq .Loop_aes_gcm_8x + cmpdi 24, 0 # decrypt? + bne __Encrypt_1x + xxlor 15+32, 11, 11 +__Encrypt_1x: + vxor 15, 15, 0 + PPC_GHASH1x 0, 15 + addi 5, 5, -16 + addi 11, 11, 16 - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) + vadduwm 30, 30, 31 # IV + counter + xxlxor 32+15, 32+30, 0 + addi 12, 12, -1 + cmpdi 12, 0 + bgt __Loop_1x + + stxvb16x 32+0, 0, 8 # update Xi + blr +.size aes_gcm_crypt_1x,.-aes_gcm_crypt_1x + +################################################################################ +# Process a normal partial block when we come here. +# Compute partial mask, Load and store partial block to stack. +# Compute AES state. +# Compute ghash. +# +################################################################################ +.align 4 +__Process_partial: +.localentry __Process_partial,0 + + # create partial mask + vspltisb 16, -1 + li 12, 16 + sub 12, 12, 5 + sldi 12, 12, 3 + mtvsrdd 32+17, 0, 12 + vslo 16, 16, 17 # partial block mask + + lxvb16x 11, 0, 14 # load partial block + xxland 11, 11, 32+16 + + # AES crypt partial + xxlxor 32+15, 32+30, 0 + lwz 23, 240(6) # n rounds + addi 22, 23, -1 # loop - 1 + mtctr 22 + addi 10, 6, 16 + +__Loop_aes_pstate: + lxv 32+1, 0(10) + vcipher 15, 15, 1 + addi 10, 10, 16 + bdnz __Loop_aes_pstate + lxv 32+1, 0(10) # last round key + vcipherlast 15, 15, 1 - cmpdi 9, 12 - beq .Loop_aes_gcm_8x + xxlxor 32+15, 32+15, 11 + vand 15, 15, 16 + # AES crypt output v15 + # Write partial + li 10, 224 + stxvb16x 15+32, 10, 1 # write v15 to stack + addi 10, 1, 223 + addi 12, 9, -1 + mtctr 5 # partial block len +__Write_partial: + lbzu 22, 1(10) + stbu 22, 1(12) + bdnz __Write_partial + + cmpdi 24, 0 # decrypt? + bne __Encrypt_partial + xxlor 32+15, 11, 11 # decrypt using the input block +__Encrypt_partial: + vxor 15, 15, 0 # ^ previous hash + PPC_GHASH1x 0, 15 + li 5, 0 # done last byte + stxvb16x 32+0, 0, 8 # Update X1 + blr +.size __Process_partial,.-__Process_partial + +################################################################################ +# ppc_aes_gcm_encrypt (const void *inp, void *out, size_t len, +# const char *rk, unsigned char iv[16], void *Xip); +# +# r3 - inp +# r4 - out +# r5 - len +# r6 - AES round keys +# r7 - iv +# r8 - Xi, HPoli, hash keys +# +# rounds is at offset 240 in rk +# Xi is at 0 in gcm_table (Xip). +# +################################################################################ +.global ppc_aes_gcm_encrypt +.align 5 +ppc_aes_gcm_encrypt: +.localentry ppc_aes_gcm_encrypt,0 - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq .Loop_aes_gcm_8x + SAVE_REGS + LOAD_HASH_TABLE - b aes_gcm_out + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 -.align 5 -.Loop_aes_gcm_8x: mr 14, 3 mr 9, 4 + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 + + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + li 11, 0 + lxv 0, 0(6) # round key 0 + + # + # Process different blocks + # + cmpdi 5, 128 + blt __Process_more_enc + + # load 9 round keys + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 23, 240(6) # n rounds + +__Process_encrypt: +# +# Process 8x AES/GCM blocks +# +__Process_8x_enc: + # 8x blocks li 10, 128 - divdu 10, 5, 10 - cmpdi 10, 0 - beq .Loop_last_block - - .long 0x13DEF8C0 - vxor 16, 30, 29 - .long 0x13DEF8C0 - vxor 17, 30, 29 - .long 0x13DEF8C0 - vxor 18, 30, 29 - .long 0x13DEF8C0 - vxor 19, 30, 29 - .long 0x13DEF8C0 - vxor 20, 30, 29 - .long 0x13DEF8C0 - vxor 21, 30, 29 - .long 0x13DEF8C0 - vxor 22, 30, 29 - - mtctr 10 + divdu 12, 5, 10 # n 128 bytes-blocks + + addi 12, 12, -1 # loop - 1 + + vmr 15, 30 # first state: IV + vadduwm 16, 15, 31 # state + counter + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + # vxor state, state, w # addroundkey + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 li 15, 16 li 16, 32 @@ -536,523 +591,185 @@ _ppc_aes_gcm_encrypt: li 20, 96 li 21, 112 - lwz 10, 240(6) - -.Loop_8x_block: - - lxvb16x 15, 0, 14 - lxvb16x 16, 15, 14 - lxvb16x 17, 16, 14 - lxvb16x 18, 17, 14 - lxvb16x 19, 18, 14 - lxvb16x 20, 19, 14 - lxvb16x 21, 20, 14 - lxvb16x 22, 21, 14 - addi 14, 14, 128 - -.Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_ghash - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_ghash - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_ghash - b aes_gcm_out - -Do_next_ghash: - - - - .long 0x11EFBD09 - .long 0x1210BD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 - - .long 0x1231BD09 - .long 0x1252BD09 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 - - .long 0x1273BD09 - .long 0x1294BD09 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 - - .long 0x12B5BD09 - .long 0x12D6BD09 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 - - addi 9, 9, 128 - - - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - .long 0x13DEF8C0 - vor 29,30,30 - vxor 15, 30, 27 - .long 0x13DEF8C0 - vxor 16, 30, 27 - .long 0x13DEF8C0 - vxor 17, 30, 27 - .long 0x13DEF8C0 - vxor 18, 30, 27 - .long 0x13DEF8C0 - vxor 19, 30, 27 - .long 0x13DEF8C0 - vxor 20, 30, 27 - .long 0x13DEF8C0 - vxor 21, 30, 27 - .long 0x13DEF8C0 - vxor 22, 30, 27 - - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz .Loop_8x_block - - vor 30,29,29 - -.Loop_last_block: - cmpdi 12, 0 - beq aes_gcm_out - - - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10, 240(6) - - cmpdi 12, 16 - blt Final_block - -.macro .Loop_aes_middle_1x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - .long 0x11EF9D08 - .long 0x11EFA508 - .long 0x11EFAD08 - .long 0x11EFB508 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - .long 0x11EF9D08 - .long 0x11EFA508 - .long 0x11EFAD08 - .long 0x11EFB508 - - xxlor 19+32, 9, 9 - .long 0x11EF9D08 -.endm - -Next_rem_block: - lxvb16x 15, 0, 14 - -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 + # + # Pre-compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + # + addi 22, 23, -9 # process 8 keys + mtctr 22 # AES key loop + addi 10, 6, 144 - cmpdi 10, 10 - beq Do_next_1x + LOOP_8AES_STATE # process 8 AES keys +__PreLoop_aes_state: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __PreLoop_aes_state + lxv 32+1, 0(10) # last round key (v1) - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_1x - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x - -Do_next_1x: - .long 0x11EFBD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - addi 14, 14, 16 - addi 9, 9, 16 + cmpdi 12, 0 # Only one loop (8 block) + beq __Finish_ghash - vor 28,15,15 - ppc_update_hash_1x +# +# Loop 8x blocks and compute ghash +# +__Loop_8x_block_enc: + PROCESS_8X_AES_STATES - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - .long 0x13DEF8C0 - vxor 15, 30, 19 + # Compute ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x + + COMPUTE_STATES + + addi 5, 5, -128 + addi 11, 11, 128 + + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # Compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + LOOP_8AES_STATE # process 8 AES keys + mtctr 22 # AES key loop + addi 10, 6, 144 + +__LastLoop_aes_state: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __LastLoop_aes_state - bdnz Next_rem_block + lxv 32+1, 0(10) # last round key (v1) + addi 12, 12, -1 cmpdi 12, 0 - beq aes_gcm_out - -Final_block: -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x + bne __Loop_8x_block_enc + # + # Remainng blocks + # +__Finish_ghash: + PROCESS_8X_AES_STATES - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_final_1x - -Do_final_1x: - .long 0x11EFBD09 - - lxvb16x 15, 0, 14 - xxlxor 47, 47, 15 - - - li 15, 16 - sub 15, 15, 12 + # Compute ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x - vspltisb 16,-1 - vspltisb 17,0 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 + # Update IV and Xi + xxlor 30+32, 9, 9 # last ctr + vadduwm 30, 30, 31 # increase ctr + stxvb16x 32+0, 0, 8 # update Xi - addi 10, 1, 192 - lxvb16x 16, 15, 10 - xxland 47, 47, 16 + addi 5, 5, -128 + addi 11, 11, 128 - vor 28,15,15 - ppc_update_hash_1x + # + # Done 8x blocks + # + cmpdi 5, 0 + beq aes_gcm_out - bl Write_partial_block +__Process_more_enc: + li 24, 1 # encrypt + bl aes_gcm_crypt_1x + cmpdi 5, 0 + beq aes_gcm_out + bl __Process_partial b aes_gcm_out +.size ppc_aes_gcm_encrypt,.-ppc_aes_gcm_encrypt - - - - - -Write_partial_block: - li 10, 192 - stxvb16x 15+32, 10, 1 - - - addi 10, 9, -1 - addi 16, 1, 191 - - mtctr 12 - li 15, 0 - -Write_last_byte: - lbzu 14, 1(16) - stbu 14, 1(10) - bdnz Write_last_byte - blr - -aes_gcm_out: - - stxvb16x 32, 0, 8 - add 3, 11, 12 - - li 9, 256 - lvx 20, 9, 1 - addi 9, 9, 16 - lvx 21, 9, 1 - addi 9, 9, 16 - lvx 22, 9, 1 - addi 9, 9, 16 - lvx 23, 9, 1 - addi 9, 9, 16 - lvx 24, 9, 1 - addi 9, 9, 16 - lvx 25, 9, 1 - addi 9, 9, 16 - lvx 26, 9, 1 - addi 9, 9, 16 - lvx 27, 9, 1 - addi 9, 9, 16 - lvx 28, 9, 1 - addi 9, 9, 16 - lvx 29, 9, 1 - addi 9, 9, 16 - lvx 30, 9, 1 - addi 9, 9, 16 - lvx 31, 9, 1 - - ld 0, 528(1) - ld 14,112(1) - ld 15,120(1) - ld 16,128(1) - ld 17,136(1) - ld 18,144(1) - ld 19,152(1) - ld 20,160(1) - ld 21,168(1) - - mtlr 0 - addi 1, 1, 512 - blr - - - - -.global ppc_aes_gcm_decrypt -.align 5 +################################################################################ +# ppc_aes_gcm_decrypt (const void *inp, void *out, size_t len, +# const char *rk, unsigned char iv[16], void *Xip); +# 8x Decrypt +# +################################################################################ +.global ppc_aes_gcm_decrypt +.align 5 ppc_aes_gcm_decrypt: -_ppc_aes_gcm_decrypt: - - stdu 1,-512(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - std 0, 528(1) - - - lxvb16x 32, 0, 8 - - - li 10, 32 - lxvd2x 2+32, 10, 8 - li 10, 48 - lxvd2x 3+32, 10, 8 - li 10, 64 - lxvd2x 4+32, 10, 8 - li 10, 80 - lxvd2x 5+32, 10, 8 - - li 10, 96 - lxvd2x 6+32, 10, 8 - li 10, 112 - lxvd2x 7+32, 10, 8 - li 10, 128 - lxvd2x 8+32, 10, 8 - - li 10, 144 - lxvd2x 9+32, 10, 8 - li 10, 160 - lxvd2x 10+32, 10, 8 - li 10, 176 - lxvd2x 11+32, 10, 8 - - li 10, 192 - lxvd2x 12+32, 10, 8 - li 10, 208 - lxvd2x 13+32, 10, 8 - li 10, 224 - lxvd2x 14+32, 10, 8 - - - lxvb16x 30+32, 0, 7 - - mr 12, 5 - li 11, 0 - +.localentry ppc_aes_gcm_decrypt, 0 - vxor 31, 31, 31 - vspltisb 22,1 - vsldoi 31, 31, 22,1 - - - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - - lwz 9,240(6) - - - - xxlor 32+29, 0, 0 - vxor 15, 30, 29 - - cmpdi 9, 10 - beq .Loop_aes_gcm_8x_dec - - - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) - - cmpdi 9, 12 - beq .Loop_aes_gcm_8x_dec + SAVE_REGS + LOAD_HASH_TABLE + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq .Loop_aes_gcm_8x_dec - - b aes_gcm_out - -.align 5 -.Loop_aes_gcm_8x_dec: mr 14, 3 mr 9, 4 + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 + + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + li 11, 0 + lxv 0, 0(6) # round key 0 + + # + # Process different blocks + # + cmpdi 5, 128 + blt __Process_more_dec + + # load 9 round keys + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 23, 240(6) # n rounds + +__Process_decrypt: +# +# Process 8x AES/GCM blocks +# +__Process_8x_dec: + # 8x blocks li 10, 128 - divdu 10, 5, 10 - cmpdi 10, 0 - beq .Loop_last_block_dec - - .long 0x13DEF8C0 - vxor 16, 30, 29 - .long 0x13DEF8C0 - vxor 17, 30, 29 - .long 0x13DEF8C0 - vxor 18, 30, 29 - .long 0x13DEF8C0 - vxor 19, 30, 29 - .long 0x13DEF8C0 - vxor 20, 30, 29 - .long 0x13DEF8C0 - vxor 21, 30, 29 - .long 0x13DEF8C0 - vxor 22, 30, 29 - - mtctr 10 + divdu 12, 5, 10 # n 128 bytes-blocks + + addi 12, 12, -1 # loop - 1 + + vmr 15, 30 # first state: IV + vadduwm 16, 15, 31 # state + counter + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + # vxor state, state, w # addroundkey + xxlxor 32+15, 32+15, 0 # IV + round key - add round key 0 + xxlxor 32+16, 32+16, 0 + xxlxor 32+17, 32+17, 0 + xxlxor 32+18, 32+18, 0 + xxlxor 32+19, 32+19, 0 + xxlxor 32+20, 32+20, 0 + xxlxor 32+21, 32+21, 0 + xxlxor 32+22, 32+22, 0 li 15, 16 li 16, 32 @@ -1062,279 +779,215 @@ _ppc_aes_gcm_decrypt: li 20, 96 li 21, 112 - lwz 10, 240(6) + # + # Pre-compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + # + addi 22, 23, -9 # process 8 keys + mtctr 22 # AES key loop + addi 10, 6, 144 -.Loop_8x_block_dec: + LOOP_8AES_STATE # process 8 AES keys - lxvb16x 15, 0, 14 - lxvb16x 16, 15, 14 - lxvb16x 17, 16, 14 - lxvb16x 18, 17, 14 - lxvb16x 19, 18, 14 - lxvb16x 20, 19, 14 - lxvb16x 21, 20, 14 - lxvb16x 22, 21, 14 +__PreLoop_aes_state_dec: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __PreLoop_aes_state_dec + lxv 32+1, 0(10) # last round key (v1) + + cmpdi 12, 0 # Only one loop (8 block) + beq __Finish_ghash_dec + +# +# Loop 8x blocks and compute ghash +# +__Loop_8x_block_dec: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block addi 14, 14, 128 -.Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_last_aes_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_last_aes_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_last_aes_dec - b aes_gcm_out - -Do_last_aes_dec: - - - - .long 0x11EFBD09 - .long 0x1210BD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 - - .long 0x1231BD09 - .long 0x1252BD09 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 - - .long 0x1273BD09 - .long 0x1294BD09 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 - - .long 0x12B5BD09 - .long 0x12D6BD09 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output addi 9, 9, 128 - xxlor 15+32, 15, 15 - xxlor 16+32, 16, 16 - xxlor 17+32, 17, 17 - xxlor 18+32, 18, 18 - xxlor 19+32, 19, 19 - xxlor 20+32, 20, 20 - xxlor 21+32, 21, 21 - xxlor 22+32, 22, 22 - - - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - .long 0x13DEF8C0 - vor 29,30,30 - vxor 15, 30, 27 - .long 0x13DEF8C0 - vxor 16, 30, 27 - .long 0x13DEF8C0 - vxor 17, 30, 27 - .long 0x13DEF8C0 - vxor 18, 30, 27 - .long 0x13DEF8C0 - vxor 19, 30, 27 - .long 0x13DEF8C0 - vxor 20, 30, 27 - .long 0x13DEF8C0 - vxor 21, 30, 27 - .long 0x13DEF8C0 - vxor 22, 30, 27 - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz .Loop_8x_block_dec - - vor 30,29,29 - -.Loop_last_block_dec: - cmpdi 12, 0 - beq aes_gcm_out - - - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10,240(6) - - cmpdi 12, 16 - blt Final_block_dec - -Next_rem_block_dec: - lxvb16x 15, 0, 14 - -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_1x_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_1x_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x_dec - -Do_next_1x_dec: - .long 0x11EFBD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - addi 14, 14, 16 - addi 9, 9, 16 - - xxlor 28+32, 15, 15 - ppc_update_hash_1x + vmr 15, 23 + vmr 16, 24 + vmr 17, 25 + vmr 18, 26 + vmr 19, 27 + vmr 20, 28 + vmr 21, 29 + vmr 22, 30 - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - .long 0x13DEF8C0 - vxor 15, 30, 19 + # ghash here + vxor 15, 15, 0 + PPC_GFMUL128_8x + + xxlor 32+15, 9, 9 # last state + vadduwm 15, 15, 31 # state + counter + vadduwm 16, 15, 31 + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + xxlor 32+27, 0, 0 # restore roundkey 0 + vxor 15, 15, 27 # IV + round key - add round key 0 + vxor 16, 16, 27 + vxor 17, 17, 27 + vxor 18, 18, 27 + vxor 19, 19, 27 + vxor 20, 20, 27 + vxor 21, 21, 27 + vxor 22, 22, 27 - bdnz Next_rem_block_dec + addi 5, 5, -128 + addi 11, 11, 128 + + lxv 32+23, 16(6) # round key 1 + lxv 32+24, 32(6) # round key 2 + lxv 32+25, 48(6) # round key 3 + lxv 32+26, 64(6) # round key 4 + lxv 32+27, 80(6) # round key 5 + lxv 32+28, 96(6) # round key 6 + lxv 32+29, 112(6) # round key 7 + lxv 32+1, 128(6) # round key 8 + + LOOP_8AES_STATE # process 8 AES keys + mtctr 22 # AES key loop + addi 10, 6, 144 +__LastLoop_aes_state_dec: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x 1 + addi 10, 10, 16 + bdnz __LastLoop_aes_state_dec + lxv 32+1, 0(10) # last round key (v1) + addi 12, 12, -1 cmpdi 12, 0 - beq aes_gcm_out - -Final_block_dec: -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x_dec - + bne __Loop_8x_block_dec + +__Finish_ghash_dec: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 - xxlor 24+32, 13, 13 + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 - .long 0x11EFBD08 - .long 0x11EFC508 + vxor 15, 23, 0 + vmr 16, 24 + vmr 17, 25 + vmr 18, 26 + vmr 19, 27 + vmr 20, 28 + vmr 21, 29 + vmr 22, 30 - xxlor 23+32, 14, 14 + #vxor 15, 15, 0 + PPC_GFMUL128_8x - cmpdi 10, 14 - beq Do_final_1x_dec + xxlor 30+32, 9, 9 # last ctr + vadduwm 30, 30, 31 # increase ctr + stxvb16x 32+0, 0, 8 # update Xi -Do_final_1x_dec: - .long 0x11EFBD09 + addi 5, 5, -128 + addi 11, 11, 128 - lxvb16x 15, 0, 14 - xxlxor 47, 47, 15 + # + # Done 8x blocks + # + cmpdi 5, 0 + beq aes_gcm_out - li 15, 16 - sub 15, 15, 12 +__Process_more_dec: + li 24, 0 # decrypt + bl aes_gcm_crypt_1x + cmpdi 5, 0 + beq aes_gcm_out - vspltisb 16,-1 - vspltisb 17,0 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 - - addi 10, 1, 192 - lxvb16x 16, 15, 10 - xxland 47, 47, 16 + bl __Process_partial + b aes_gcm_out +.size ppc_aes_gcm_decrypt,.-ppc_aes_gcm_decrypt - xxlor 28+32, 15, 15 - ppc_update_hash_1x +aes_gcm_out: +.localentry aes_gcm_out,0 + mr 3, 11 # return count - bl Write_partial_block + RESTORE_REGS + blr +.size aes_gcm_out,.-aes_gcm_out - b aes_gcm_out +.rodata +.align 4 +# for vector permute and xor +permx: +.long 0x4c5d6e7f, 0x08192a3b, 0xc4d5e6f7, 0x8091a2b3 diff --git a/sys/dev/acpi_support/acpi_ibm.c b/sys/dev/acpi_support/acpi_ibm.c index f895d48bb6d0..1221384e7d8a 100644 --- a/sys/dev/acpi_support/acpi_ibm.c +++ b/sys/dev/acpi_support/acpi_ibm.c @@ -1447,8 +1447,13 @@ acpi_ibm_eventhandler(struct acpi_ibm_softc *sc, int arg) ACPI_SERIAL_BEGIN(ibm); switch (arg) { + /* + * XXX "Suspend-to-RAM" here is as opposed to suspend-to-disk, but it is + * fine if our suspend sleep state transition request puts us in s2idle + * instead of suspend-to-RAM. + */ case IBM_EVENT_SUSPEND_TO_RAM: - power_pm_suspend(POWER_SLEEP_STATE_SUSPEND); + power_pm_suspend(POWER_SSTATE_TRANSITION_SUSPEND); break; case IBM_EVENT_BLUETOOTH: diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c index c76bc8477537..53099db8ae84 100644 --- a/sys/dev/acpica/acpi.c +++ b/sys/dev/acpica/acpi.c @@ -293,6 +293,17 @@ static char acpi_remove_interface[256]; TUNABLE_STR("hw.acpi.remove_interface", acpi_remove_interface, sizeof(acpi_remove_interface)); +/* + * Automatically apply the Darwin OSI on Apple Mac hardware to obtain + * access to full ACPI hardware support on supported platforms. + * + * This flag automatically overrides any values set by + * `hw.acpi.acpi_install_interface` and unset by + * `hw.acpi.acpi_remove_interface`. + */ +static int acpi_apple_darwin_osi = 1; +TUNABLE_INT("hw.acpi.apple_darwin_osi", &acpi_apple_darwin_osi); + /* Allow users to dump Debug objects without ACPI debugger. */ static int acpi_debug_objects; TUNABLE_INT("debug.acpi.enable_debug_objects", &acpi_debug_objects); @@ -490,7 +501,6 @@ acpi_attach(device_t dev) ACPI_STATUS status; int error, state; UINT32 flags; - UINT8 TypeA, TypeB; char *env; enum power_stype stype; @@ -588,61 +598,6 @@ acpi_attach(device_t dev) goto out; } - /* - * Setup our sysctl tree. - * - * XXX: This doesn't check to make sure that none of these fail. - */ - sysctl_ctx_init(&sc->acpi_sysctl_ctx); - sc->acpi_sysctl_tree = SYSCTL_ADD_NODE(&sc->acpi_sysctl_ctx, - SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, device_get_name(dev), - CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); - SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "supported_sleep_state", - CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, - 0, 0, acpi_supported_sleep_state_sysctl, "A", - "List supported ACPI sleep states."); - SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "power_button_state", - CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, - &sc->acpi_power_button_stype, 0, acpi_stype_sysctl, "A", - "Power button ACPI sleep state."); - SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "sleep_button_state", - CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, - &sc->acpi_sleep_button_stype, 0, acpi_stype_sysctl, "A", - "Sleep button ACPI sleep state."); - SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "lid_switch_state", - CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, - &sc->acpi_lid_switch_stype, 0, acpi_stype_sysctl, "A", - "Lid ACPI sleep state. Set to s2idle or s2mem if you want to suspend " - "your laptop when you close the lid."); - SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "suspend_state", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, - NULL, 0, acpi_suspend_state_sysctl, "A", - "Current ACPI suspend state. This sysctl is deprecated; you probably " - "want to use kern.power.suspend instead."); - SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "standby_state", - CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, - &sc->acpi_standby_sx, 0, acpi_sleep_state_sysctl, "A", - "ACPI Sx state to use when going standby (usually S1 or S2)."); - SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "sleep_delay", CTLFLAG_RW, &sc->acpi_sleep_delay, 0, - "sleep delay in seconds"); - SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "s4bios", CTLFLAG_RW, &sc->acpi_s4bios, 0, - "Use S4BIOS when hibernating."); - SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "verbose", CTLFLAG_RW, &sc->acpi_verbose, 0, "verbose mode"); - SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "disable_on_reboot", CTLFLAG_RW, - &sc->acpi_do_disable, 0, "Disable ACPI when rebooting/halting system"); - SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "handle_reboot", CTLFLAG_RW, - &sc->acpi_handle_reboot, 0, "Use ACPI Reset Register to reboot"); - #if defined(__amd64__) || defined(__i386__) /* * Enable workaround for incorrect ISA IRQ polarity by default on @@ -650,10 +605,6 @@ acpi_attach(device_t dev) */ if (cpu_vendor_id == CPU_VENDOR_INTEL) acpi_override_isa_irq_polarity = 1; - SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), - OID_AUTO, "override_isa_irq_polarity", CTLFLAG_RDTUN, - &acpi_override_isa_irq_polarity, 0, - "Force active-hi polarity for edge-triggered ISA IRQs"); #endif /* @@ -673,11 +624,9 @@ acpi_attach(device_t dev) if (AcpiGbl_FADT.Flags & ACPI_FADT_RESET_REGISTER) sc->acpi_handle_reboot = 1; -#if !ACPI_REDUCED_HARDWARE /* Only enable S4BIOS by default if the FACS says it is available. */ if (AcpiGbl_FACS != NULL && AcpiGbl_FACS->Flags & ACPI_FACS_S4_BIOS_PRESENT) - sc->acpi_s4bios = 1; -#endif + sc->acpi_s4bios = true; /* * Probe all supported ACPI sleep states. Awake (S0) is always supported, @@ -688,13 +637,14 @@ acpi_attach(device_t dev) #if defined(__i386__) || defined(__amd64__) acpi_supported_stypes[POWER_STYPE_SUSPEND_TO_IDLE] = true; #endif - for (state = ACPI_STATE_S1; state <= ACPI_STATE_S5; state++) - if (ACPI_SUCCESS(AcpiEvaluateObject(ACPI_ROOT_OBJECT, - __DECONST(char *, AcpiGbl_SleepStateNames[state]), NULL, NULL)) && - ACPI_SUCCESS(AcpiGetSleepTypeData(state, &TypeA, &TypeB))) { + for (state = ACPI_STATE_S1; state <= ACPI_STATE_S5; state++) { + UINT8 TypeA, TypeB; + + if (ACPI_SUCCESS(AcpiGetSleepTypeData(state, &TypeA, &TypeB))) { acpi_supported_sstates[state] = true; acpi_supported_stypes[acpi_sstate_to_stype(state)] = true; } + } /* * Dispatch the default sleep type to devices. The lid switch is set @@ -761,6 +711,67 @@ acpi_attach(device_t dev) if ((error = acpi_machdep_init(dev))) goto out; + /* + * Setup our sysctl tree. + * + * XXX: This doesn't check to make sure that none of these fail. + */ + sysctl_ctx_init(&sc->acpi_sysctl_ctx); + sc->acpi_sysctl_tree = SYSCTL_ADD_NODE(&sc->acpi_sysctl_ctx, + SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, device_get_name(dev), + CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); + SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "supported_sleep_state", + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, + 0, 0, acpi_supported_sleep_state_sysctl, "A", + "List supported ACPI sleep states."); + SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "power_button_state", + CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, + &sc->acpi_power_button_stype, 0, acpi_stype_sysctl, "A", + "Power button ACPI sleep state."); + SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "sleep_button_state", + CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, + &sc->acpi_sleep_button_stype, 0, acpi_stype_sysctl, "A", + "Sleep button ACPI sleep state."); + SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "lid_switch_state", + CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, + &sc->acpi_lid_switch_stype, 0, acpi_stype_sysctl, "A", + "Lid ACPI sleep state. Set to s2idle or s2mem if you want to suspend " + "your laptop when you close the lid."); + SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "suspend_state", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, + NULL, 0, acpi_suspend_state_sysctl, "A", + "Current ACPI suspend state. This sysctl is deprecated; you probably " + "want to use kern.power.suspend instead."); + SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "standby_state", + CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, + &sc->acpi_standby_sx, 0, acpi_sleep_state_sysctl, "A", + "ACPI Sx state to use when going standby (usually S1 or S2)."); + SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "sleep_delay", CTLFLAG_RW, &sc->acpi_sleep_delay, 0, + "sleep delay in seconds"); + SYSCTL_ADD_BOOL(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "s4bios", CTLFLAG_RW, &sc->acpi_s4bios, 0, + "On hibernate, have the firmware save/restore the machine state (S4BIOS)."); + SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "verbose", CTLFLAG_RW, &sc->acpi_verbose, 0, "verbose mode"); + SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "disable_on_reboot", CTLFLAG_RW, + &sc->acpi_do_disable, 0, "Disable ACPI when rebooting/halting system"); + SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "handle_reboot", CTLFLAG_RW, + &sc->acpi_handle_reboot, 0, "Use ACPI Reset Register to reboot"); +#if defined(__amd64__) || defined(__i386__) + SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), + OID_AUTO, "override_isa_irq_polarity", CTLFLAG_RDTUN, + &acpi_override_isa_irq_polarity, 0, + "Force active-hi polarity for edge-triggered ISA IRQs"); +#endif + /* Register ACPI again to pass the correct argument of pm_func. */ power_pm_register(POWER_PM_TYPE_ACPI, acpi_pm_func, sc, acpi_supported_stypes); @@ -3636,6 +3647,7 @@ acpi_EnterSleepState(struct acpi_softc *sc, enum power_stype stype) device_printf(sc->acpi_dev, "device_suspend failed\n"); goto backout; } + EVENTHANDLER_INVOKE(acpi_post_dev_suspend, stype); slp_state |= ACPI_SS_DEV_SUSPEND; if (stype != POWER_STYPE_SUSPEND_TO_IDLE) { @@ -3685,6 +3697,7 @@ backout: slp_state &= ~ACPI_SS_GPE_SET; } if ((slp_state & ACPI_SS_DEV_SUSPEND) != 0) { + EVENTHANDLER_INVOKE(acpi_pre_dev_resume, stype); DEVICE_RESUME(root_bus); slp_state &= ~ACPI_SS_DEV_SUSPEND; } @@ -4049,7 +4062,7 @@ acpi_system_eventhandler_sleep(void *arg, enum power_stype stype) ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, stype); /* Check if button action is disabled or unknown. */ - if (stype == ACPI_STATE_UNKNOWN) + if (stype == POWER_STYPE_UNKNOWN) return; /* @@ -4618,6 +4631,7 @@ static struct debugtag dbg_layer[] = { {"ACPI_FAN", ACPI_FAN}, {"ACPI_POWERRES", ACPI_POWERRES}, {"ACPI_PROCESSOR", ACPI_PROCESSOR}, + {"ACPI_SPMC", ACPI_SPMC}, {"ACPI_THERMAL", ACPI_THERMAL}, {"ACPI_TIMER", ACPI_TIMER}, {"ACPI_ALL_DRIVERS", ACPI_ALL_DRIVERS}, @@ -4902,6 +4916,65 @@ acpi_reset_interfaces(device_t dev) } acpi_free_interfaces(&list); } + + /* + * Apple Mac hardware quirk: install Darwin OSI. + * + * On Apple hardware, install the Darwin OSI and remove the Windows OSI + * to match Linux behavior. + * + * This is required for dual-GPU MacBook Pro systems + * (Intel iGPU + AMD/NVIDIA dGPU) where the iGPU is hidden when the + * firmware doesn't see Darwin OSI, but it also unlocks additional ACPI + * support on non-MacBook Pro Apple platforms. + * + * Apple's ACPI firmware checks _OSI("Darwin") and sets OSYS=10000 + * for macOS. Many device methods use OSDW() which checks OSYS==10000 + * for macOS-specific behavior including GPU visibility and power + * management. + * + * Linux enables Darwin OSI by default on Apple hardware and disables + * all Windows OSI strings (drivers/acpi/osi.c). Users can override + * this behavior with acpi_osi=!Darwin to get Windows-like behavior, + * in general, but this logic makes that process unnecessary. + * + * Detect Apple via SMBIOS and enable Darwin while disabling Windows + * vendor strings. This makes both GPUs visible on dual-GPU MacBook Pro + * systems (Intel iGPU + AMD dGPU) and unlocks full platform + * ACPI support. + */ + if (acpi_apple_darwin_osi) { + char *vendor = kern_getenv("smbios.system.maker"); + if (vendor != NULL) { + if (strcmp(vendor, "Apple Inc.") == 0 || + strcmp(vendor, "Apple Computer, Inc.") == 0) { + /* Disable all other OSI vendor strings. */ + status = AcpiUpdateInterfaces( + ACPI_DISABLE_ALL_VENDOR_STRINGS); + if (ACPI_SUCCESS(status)) { + /* Install Darwin OSI */ + status = AcpiInstallInterface("Darwin"); + } + if (bootverbose) { + if (ACPI_SUCCESS(status)) { + device_printf(dev, + "disabled non-Darwin OSI & " + "installed Darwin OSI\n"); + } else { + device_printf(dev, + "could not install " + "Darwin OSI: %s\n", + AcpiFormatException(status)); + } + } + } else if (bootverbose) { + device_printf(dev, + "Not installing Darwin OSI on unsupported platform: %s\n", + vendor); + } + freeenv(vendor); + } + } } static int diff --git a/sys/dev/acpica/acpi_spmc.c b/sys/dev/acpica/acpi_spmc.c new file mode 100644 index 000000000000..e8ed14fdcb1b --- /dev/null +++ b/sys/dev/acpica/acpi_spmc.c @@ -0,0 +1,638 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024-2025 The FreeBSD Foundation + * + * This software was developed by Aymeric Wibo <obiwac@freebsd.org> + * under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/param.h> +#include <sys/bus.h> +#include <sys/eventhandler.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/module.h> +#include <sys/uuid.h> + +#include <machine/_inttypes.h> + +#include <contrib/dev/acpica/include/acpi.h> +#include <contrib/dev/acpica/include/accommon.h> + +#include <dev/acpica/acpivar.h> + +/* Hooks for the ACPI CA debugging infrastructure */ +#define _COMPONENT ACPI_SPMC +ACPI_MODULE_NAME("SPMC") + +static SYSCTL_NODE(_debug_acpi, OID_AUTO, spmc, CTLFLAG_RD | CTLFLAG_MPSAFE, + NULL, "SPMC debugging"); + +static char *spmc_ids[] = { + "PNP0D80", + NULL +}; + +enum intel_dsm_index { + DSM_ENUM_FUNCTIONS = 0, + DSM_GET_DEVICE_CONSTRAINTS = 1, + DSM_GET_CRASH_DUMP_DEVICE = 2, + DSM_DISPLAY_OFF_NOTIF = 3, + DSM_DISPLAY_ON_NOTIF = 4, + DSM_ENTRY_NOTIF = 5, + DSM_EXIT_NOTIF = 6, + /* Only for Microsoft DSM set. */ + DSM_MODERN_ENTRY_NOTIF = 7, + DSM_MODERN_EXIT_NOTIF = 8, +}; + +enum amd_dsm_index { + AMD_DSM_ENUM_FUNCTIONS = 0, + AMD_DSM_GET_DEVICE_CONSTRAINTS = 1, + AMD_DSM_ENTRY_NOTIF = 2, + AMD_DSM_EXIT_NOTIF = 3, + AMD_DSM_DISPLAY_OFF_NOTIF = 4, + AMD_DSM_DISPLAY_ON_NOTIF = 5, +}; + +enum dsm_set_flags { + DSM_SET_INTEL = 1 << 0, + DSM_SET_MS = 1 << 1, + DSM_SET_AMD = 1 << 2, +}; + +struct dsm_set { + enum dsm_set_flags flag; + const char *name; + int revision; + struct uuid uuid; + uint64_t dsms_expected; +}; + +static struct dsm_set intel_dsm_set = { + .flag = DSM_SET_INTEL, + .name = "Intel", + /* + * XXX Linux uses 1 for the revision on Intel DSMs, but doesn't explain + * why. The commit that introduces this links to a document mentioning + * revision 0, so default this to 0. + * + * The debug.acpi.spmc.intel_dsm_revision sysctl may be used to configure + * this just in case. + */ + .revision = 0, + .uuid = { /* c4eb40a0-6cd2-11e2-bcfd-0800200c9a66 */ + 0xc4eb40a0, 0x6cd2, 0x11e2, 0xbc, 0xfd, + {0x08, 0x00, 0x20, 0x0c, 0x9a, 0x66}, + }, + .dsms_expected = DSM_GET_DEVICE_CONSTRAINTS | DSM_DISPLAY_OFF_NOTIF | + DSM_DISPLAY_ON_NOTIF | DSM_ENTRY_NOTIF | DSM_EXIT_NOTIF, +}; + +SYSCTL_INT(_debug_acpi_spmc, OID_AUTO, intel_dsm_revision, CTLFLAG_RW, + &intel_dsm_set.revision, 0, + "Revision to use when evaluating Intel SPMC DSMs"); + +static struct dsm_set ms_dsm_set = { + .flag = DSM_SET_MS, + .name = "Microsoft", + .revision = 0, + .uuid = { /* 11e00d56-ce64-47ce-837b-1f898f9aa461 */ + 0x11e00d56, 0xce64, 0x47ce, 0x83, 0x7b, + {0x1f, 0x89, 0x8f, 0x9a, 0xa4, 0x61}, + }, + .dsms_expected = DSM_DISPLAY_OFF_NOTIF | DSM_DISPLAY_ON_NOTIF | + DSM_ENTRY_NOTIF | DSM_EXIT_NOTIF | DSM_MODERN_ENTRY_NOTIF | + DSM_MODERN_EXIT_NOTIF, +}; + +static struct dsm_set amd_dsm_set = { + .flag = DSM_SET_AMD, + .name = "AMD", + /* + * XXX Linux uses 0 for the revision on AMD DSMs, but at least on the + * Framework 13 AMD 7040 series, the enum functions DSM only returns a + * function mask that covers all the DSMs we need to call when called + * with revision 2. + * + * The debug.acpi.spmc.amd_dsm_revision sysctl may be used to configure + * this just in case. + */ + .revision = 2, + .uuid = { /* e3f32452-febc-43ce-9039-932122d37721 */ + 0xe3f32452, 0xfebc, 0x43ce, 0x90, 0x39, + {0x93, 0x21, 0x22, 0xd3, 0x77, 0x21}, + }, + .dsms_expected = AMD_DSM_GET_DEVICE_CONSTRAINTS | AMD_DSM_ENTRY_NOTIF | + AMD_DSM_EXIT_NOTIF | AMD_DSM_DISPLAY_OFF_NOTIF | + AMD_DSM_DISPLAY_ON_NOTIF, +}; + +SYSCTL_INT(_debug_acpi_spmc, OID_AUTO, amd_dsm_revision, CTLFLAG_RW, + &amd_dsm_set.revision, 0, "Revision to use when evaluating AMD SPMC DSMs"); + +union dsm_index { + int i; + enum intel_dsm_index regular; + enum amd_dsm_index amd; +}; + +struct acpi_spmc_constraint { + bool enabled; + char *name; + int min_d_state; + ACPI_HANDLE handle; + + /* Unused, spec only. */ + uint64_t lpi_uid; + uint64_t min_dev_specific_state; + + /* Unused, AMD only. */ + uint64_t function_states; +}; + +struct acpi_spmc_softc { + device_t dev; + ACPI_HANDLE handle; + ACPI_OBJECT *obj; + enum dsm_set_flags dsm_sets; + + struct eventhandler_entry *eh_suspend; + struct eventhandler_entry *eh_resume; + + bool constraints_populated; + size_t constraint_count; + struct acpi_spmc_constraint *constraints; +}; + +static void acpi_spmc_check_dsm_set(struct acpi_spmc_softc *sc, + ACPI_HANDLE handle, struct dsm_set *dsm_set); +static int acpi_spmc_get_constraints(device_t dev); +static void acpi_spmc_free_constraints(struct acpi_spmc_softc *sc); + +static void acpi_spmc_suspend(device_t dev, enum power_stype stype); +static void acpi_spmc_resume(device_t dev, enum power_stype stype); + +static int +acpi_spmc_probe(device_t dev) +{ + char *name; + ACPI_HANDLE handle; + struct acpi_spmc_softc *sc; + + /* Check that this is an enabled device. */ + if (acpi_get_type(dev) != ACPI_TYPE_DEVICE || acpi_disabled("spmc")) + return (ENXIO); + + if (ACPI_ID_PROBE(device_get_parent(dev), dev, spmc_ids, &name) > 0) + return (ENXIO); + + handle = acpi_get_handle(dev); + if (handle == NULL) + return (ENXIO); + + sc = device_get_softc(dev); + + /* Check which sets of DSM's are supported. */ + sc->dsm_sets = 0; + + acpi_spmc_check_dsm_set(sc, handle, &intel_dsm_set); + acpi_spmc_check_dsm_set(sc, handle, &ms_dsm_set); + acpi_spmc_check_dsm_set(sc, handle, &amd_dsm_set); + + if (sc->dsm_sets == 0) + return (ENXIO); + + device_set_descf(dev, "Low Power S0 Idle (DSM sets 0x%x)", + sc->dsm_sets); + + return (0); +} + +static int +acpi_spmc_attach(device_t dev) +{ + struct acpi_spmc_softc *sc = device_get_softc(dev); + + sc->dev = dev; + + sc->handle = acpi_get_handle(dev); + if (sc->handle == NULL) + return (ENXIO); + + sc->constraints_populated = false; + sc->constraint_count = 0; + sc->constraints = NULL; + + /* Get device constraints. We can only call this once so do this now. */ + acpi_spmc_get_constraints(dev); + + sc->eh_suspend = EVENTHANDLER_REGISTER(acpi_post_dev_suspend, + acpi_spmc_suspend, dev, 0); + sc->eh_resume = EVENTHANDLER_REGISTER(acpi_pre_dev_resume, + acpi_spmc_resume, dev, 0); + + return (0); +} + +static int +acpi_spmc_detach(device_t dev) +{ + struct acpi_spmc_softc *sc = device_get_softc(dev); + + EVENTHANDLER_DEREGISTER(acpi_post_dev_suspend, sc->eh_suspend); + EVENTHANDLER_DEREGISTER(acpi_pre_dev_resume, sc->eh_resume); + + acpi_spmc_free_constraints(device_get_softc(dev)); + return (0); +} + +static void +acpi_spmc_check_dsm_set(struct acpi_spmc_softc *sc, ACPI_HANDLE handle, + struct dsm_set *dsm_set) +{ + const uint64_t dsms_supported = acpi_DSMQuery(handle, + (uint8_t *)&dsm_set->uuid, dsm_set->revision); + + /* + * Check if DSM set supported at all. We do this by checking the + * existence of "enum functions". + */ + if ((dsms_supported & 1) == 0) + return; + if ((dsms_supported & dsm_set->dsms_expected) + != dsm_set->dsms_expected) { + device_printf(sc->dev, "DSM set %s does not support expected " + "DSMs (%#" PRIx64 " vs %#" PRIx64 "). " + "Some methods may fail.\n", + dsm_set->name, dsms_supported, dsm_set->dsms_expected); + } + sc->dsm_sets |= dsm_set->flag; +} + +static void +acpi_spmc_free_constraints(struct acpi_spmc_softc *sc) +{ + if (sc->constraints == NULL) + return; + + for (size_t i = 0; i < sc->constraint_count; i++) { + if (sc->constraints[i].name != NULL) + free(sc->constraints[i].name, M_TEMP); + } + + free(sc->constraints, M_TEMP); + sc->constraints = NULL; +} + +static int +acpi_spmc_get_constraints_spec(struct acpi_spmc_softc *sc, ACPI_OBJECT *object) +{ + struct acpi_spmc_constraint *constraint; + int revision; + ACPI_OBJECT *constraint_obj; + ACPI_OBJECT *name_obj; + ACPI_OBJECT *detail; + ACPI_OBJECT *constraint_package; + + KASSERT(sc->constraints_populated == false, + ("constraints already populated")); + + sc->constraint_count = object->Package.Count; + sc->constraints = malloc(sc->constraint_count * sizeof *sc->constraints, + M_TEMP, M_WAITOK | M_ZERO); + + /* + * The value of sc->constraint_count can change during the loop, so + * iterate until object->Package.Count so we actually go over all + * elements in the package. + */ + for (size_t i = 0; i < object->Package.Count; i++) { + constraint_obj = &object->Package.Elements[i]; + constraint = &sc->constraints[i]; + + constraint->enabled = + constraint_obj->Package.Elements[1].Integer.Value; + + name_obj = &constraint_obj->Package.Elements[0]; + constraint->name = strdup(name_obj->String.Pointer, M_TEMP); + if (constraint->name == NULL) { + acpi_spmc_free_constraints(sc); + return (ENOMEM); + } + + /* + * The first element in the device constraint detail package is + * the revision, and should always be zero. + */ + revision = constraint_obj->Package.Elements[0].Integer.Value; + if (revision != 0) { + device_printf(sc->dev, "Unknown revision %d for " + "device constraint detail package\n", revision); + sc->constraint_count--; + continue; + } + + detail = &constraint_obj->Package.Elements[2]; + constraint_package = &detail->Package.Elements[1]; + + constraint->lpi_uid = + constraint_package->Package.Elements[0].Integer.Value; + constraint->min_d_state = + constraint_package->Package.Elements[1].Integer.Value; + constraint->min_dev_specific_state = + constraint_package->Package.Elements[2].Integer.Value; + } + + sc->constraints_populated = true; + return (0); +} + +static int +acpi_spmc_get_constraints_amd(struct acpi_spmc_softc *sc, ACPI_OBJECT *object) +{ + size_t constraint_count; + ACPI_OBJECT *constraint_obj; + ACPI_OBJECT *constraints; + struct acpi_spmc_constraint *constraint; + ACPI_OBJECT *name_obj; + + KASSERT(sc->constraints_populated == false, + ("constraints already populated")); + + /* + * First element in the package is unknown. + * Second element is the number of device constraints. + * Third element is the list of device constraints itself. + */ + constraint_count = object->Package.Elements[1].Integer.Value; + constraints = &object->Package.Elements[2]; + + if (constraints->Package.Count != constraint_count) { + device_printf(sc->dev, "constraint count mismatch (%d to %zu)\n", + constraints->Package.Count, constraint_count); + return (ENXIO); + } + + sc->constraint_count = constraint_count; + sc->constraints = malloc(constraint_count * sizeof *sc->constraints, + M_TEMP, M_WAITOK | M_ZERO); + + for (size_t i = 0; i < constraint_count; i++) { + /* Parse the constraint package. */ + constraint_obj = &constraints->Package.Elements[i]; + if (constraint_obj->Package.Count != 4) { + device_printf(sc->dev, "constraint %zu has %d elements\n", + i, constraint_obj->Package.Count); + acpi_spmc_free_constraints(sc); + return (ENXIO); + } + + constraint = &sc->constraints[i]; + constraint->enabled = + constraint_obj->Package.Elements[0].Integer.Value; + + name_obj = &constraint_obj->Package.Elements[1]; + constraint->name = strdup(name_obj->String.Pointer, M_TEMP); + if (constraint->name == NULL) { + acpi_spmc_free_constraints(sc); + return (ENOMEM); + } + + constraint->function_states = + constraint_obj->Package.Elements[2].Integer.Value; + constraint->min_d_state = + constraint_obj->Package.Elements[3].Integer.Value; + } + + sc->constraints_populated = true; + return (0); +} + +static int +acpi_spmc_get_constraints(device_t dev) +{ + struct acpi_spmc_softc *sc; + union dsm_index dsm_index; + struct dsm_set *dsm_set; + ACPI_STATUS status; + ACPI_BUFFER result; + ACPI_OBJECT *object; + bool is_amd; + int rv; + struct acpi_spmc_constraint *constraint; + + sc = device_get_softc(dev); + if (sc->constraints_populated) + return (0); + + /* The Microsoft DSM set doesn't have this DSM. */ + is_amd = (sc->dsm_sets & DSM_SET_AMD) != 0; + if (is_amd) { + dsm_set = &amd_dsm_set; + dsm_index.amd = AMD_DSM_GET_DEVICE_CONSTRAINTS; + } else { + dsm_set = &intel_dsm_set; + dsm_index.regular = DSM_GET_DEVICE_CONSTRAINTS; + } + + /* XXX It seems like this DSM fails if called more than once. */ + status = acpi_EvaluateDSMTyped(sc->handle, (uint8_t *)&dsm_set->uuid, + dsm_set->revision, dsm_index.i, NULL, &result, + ACPI_TYPE_PACKAGE); + if (ACPI_FAILURE(status)) { + device_printf(dev, "%s failed to call %s DSM %d (rev %d)\n", + __func__, dsm_set->name, dsm_index.i, dsm_set->revision); + return (ENXIO); + } + + object = (ACPI_OBJECT *)result.Pointer; + if (is_amd) + rv = acpi_spmc_get_constraints_amd(sc, object); + else + rv = acpi_spmc_get_constraints_spec(sc, object); + AcpiOsFree(object); + if (rv != 0) + return (rv); + + /* Get handles for each constraint device. */ + for (size_t i = 0; i < sc->constraint_count; i++) { + constraint = &sc->constraints[i]; + + status = acpi_GetHandleInScope(sc->handle, + __DECONST(char *, constraint->name), &constraint->handle); + if (ACPI_FAILURE(status)) { + device_printf(dev, "failed to get handle for %s\n", + constraint->name); + constraint->handle = NULL; + } + } + return (0); +} + +static void +acpi_spmc_check_constraints(struct acpi_spmc_softc *sc) +{ + bool violation = false; + + KASSERT(sc->constraints_populated, ("constraints not populated")); + for (size_t i = 0; i < sc->constraint_count; i++) { + struct acpi_spmc_constraint *constraint = &sc->constraints[i]; + + if (!constraint->enabled) + continue; + if (constraint->handle == NULL) + continue; + + ACPI_STATUS status = acpi_GetHandleInScope(sc->handle, + __DECONST(char *, constraint->name), &constraint->handle); + if (ACPI_FAILURE(status)) { + device_printf(sc->dev, "failed to get handle for %s\n", + constraint->name); + constraint->handle = NULL; + } + if (constraint->handle == NULL) + continue; + +#ifdef notyet + int d_state; + if (ACPI_FAILURE(acpi_pwr_get_state(constraint->handle, &d_state))) + continue; + if (d_state < constraint->min_d_state) { + device_printf(sc->dev, "constraint for device %s" + " violated (minimum D-state required was %s, actual" + " D-state is %s), might fail to enter LPI state\n", + constraint->name, + acpi_d_state_to_str(constraint->min_d_state), + acpi_d_state_to_str(d_state)); + violation = true; + } +#endif + } + if (!violation) + device_printf(sc->dev, + "all device power constraints respected!\n"); +} + +static void +acpi_spmc_run_dsm(device_t dev, struct dsm_set *dsm_set, int index) +{ + struct acpi_spmc_softc *sc; + ACPI_STATUS status; + ACPI_BUFFER result; + + sc = device_get_softc(dev); + + status = acpi_EvaluateDSMTyped(sc->handle, (uint8_t *)&dsm_set->uuid, + dsm_set->revision, index, NULL, &result, ACPI_TYPE_ANY); + + if (ACPI_FAILURE(status)) { + device_printf(dev, "%s failed to call %s DSM %d (rev %d)\n", + __func__, dsm_set->name, index, dsm_set->revision); + return; + } + + AcpiOsFree(result.Pointer); +} + +/* + * Try running the DSMs from all the DSM sets we have, as them failing costs us + * nothing, and it seems like on AMD platforms, both the AMD entry and Microsoft + * "modern" DSM's are required for it to enter modern standby. + * + * This is what Linux does too. + */ +static void +acpi_spmc_display_off_notif(device_t dev) +{ + struct acpi_spmc_softc *sc = device_get_softc(dev); + + if ((sc->dsm_sets & DSM_SET_INTEL) != 0) + acpi_spmc_run_dsm(dev, &intel_dsm_set, DSM_DISPLAY_OFF_NOTIF); + if ((sc->dsm_sets & DSM_SET_MS) != 0) + acpi_spmc_run_dsm(dev, &ms_dsm_set, DSM_DISPLAY_OFF_NOTIF); + if ((sc->dsm_sets & DSM_SET_AMD) != 0) + acpi_spmc_run_dsm(dev, &amd_dsm_set, AMD_DSM_DISPLAY_OFF_NOTIF); +} + +static void +acpi_spmc_display_on_notif(device_t dev) +{ + struct acpi_spmc_softc *sc = device_get_softc(dev); + + if ((sc->dsm_sets & DSM_SET_INTEL) != 0) + acpi_spmc_run_dsm(dev, &intel_dsm_set, DSM_DISPLAY_ON_NOTIF); + if ((sc->dsm_sets & DSM_SET_MS) != 0) + acpi_spmc_run_dsm(dev, &ms_dsm_set, DSM_DISPLAY_ON_NOTIF); + if ((sc->dsm_sets & DSM_SET_AMD) != 0) + acpi_spmc_run_dsm(dev, &amd_dsm_set, AMD_DSM_DISPLAY_ON_NOTIF); +} + +static void +acpi_spmc_entry_notif(device_t dev) +{ + struct acpi_spmc_softc *sc = device_get_softc(dev); + + acpi_spmc_check_constraints(sc); + + if ((sc->dsm_sets & DSM_SET_AMD) != 0) + acpi_spmc_run_dsm(dev, &amd_dsm_set, AMD_DSM_ENTRY_NOTIF); + if ((sc->dsm_sets & DSM_SET_MS) != 0) { + acpi_spmc_run_dsm(dev, &ms_dsm_set, DSM_MODERN_ENTRY_NOTIF); + acpi_spmc_run_dsm(dev, &ms_dsm_set, DSM_ENTRY_NOTIF); + } + if ((sc->dsm_sets & DSM_SET_INTEL) != 0) + acpi_spmc_run_dsm(dev, &intel_dsm_set, DSM_ENTRY_NOTIF); +} + +static void +acpi_spmc_exit_notif(device_t dev) +{ + struct acpi_spmc_softc *sc = device_get_softc(dev); + + if ((sc->dsm_sets & DSM_SET_INTEL) != 0) + acpi_spmc_run_dsm(dev, &intel_dsm_set, DSM_EXIT_NOTIF); + if ((sc->dsm_sets & DSM_SET_AMD) != 0) + acpi_spmc_run_dsm(dev, &amd_dsm_set, AMD_DSM_EXIT_NOTIF); + if ((sc->dsm_sets & DSM_SET_MS) != 0) { + acpi_spmc_run_dsm(dev, &ms_dsm_set, DSM_EXIT_NOTIF); + acpi_spmc_run_dsm(dev, &ms_dsm_set, DSM_MODERN_EXIT_NOTIF); + } +} + +static void +acpi_spmc_suspend(device_t dev, enum power_stype stype) +{ + if (stype != POWER_STYPE_SUSPEND_TO_IDLE) + return; + + acpi_spmc_display_off_notif(dev); + acpi_spmc_entry_notif(dev); +} + +static void +acpi_spmc_resume(device_t dev, enum power_stype stype) +{ + if (stype != POWER_STYPE_SUSPEND_TO_IDLE) + return; + + acpi_spmc_exit_notif(dev); + acpi_spmc_display_on_notif(dev); +} + +static device_method_t acpi_spmc_methods[] = { + DEVMETHOD(device_probe, acpi_spmc_probe), + DEVMETHOD(device_attach, acpi_spmc_attach), + DEVMETHOD(device_detach, acpi_spmc_detach), + DEVMETHOD_END +}; + +static driver_t acpi_spmc_driver = { + "acpi_spmc", + acpi_spmc_methods, + sizeof(struct acpi_spmc_softc), +}; + +DRIVER_MODULE_ORDERED(acpi_spmc, acpi, acpi_spmc_driver, NULL, NULL, SI_ORDER_ANY); +MODULE_DEPEND(acpi_spmc, acpi, 1, 1, 1); diff --git a/sys/dev/acpica/acpivar.h b/sys/dev/acpica/acpivar.h index 6db55b10570d..51881c580d4b 100644 --- a/sys/dev/acpica/acpivar.h +++ b/sys/dev/acpica/acpivar.h @@ -64,7 +64,7 @@ struct acpi_softc { enum power_stype acpi_lid_switch_stype; int acpi_standby_sx; - int acpi_s4bios; + bool acpi_s4bios; int acpi_sleep_delay; int acpi_do_disable; @@ -191,6 +191,7 @@ extern struct mtx acpi_mutex; #define ACPI_THERMAL 0x01000000 #define ACPI_TIMER 0x02000000 #define ACPI_OEM 0x04000000 +#define ACPI_SPMC 0x08000000 /* * Constants for different interrupt models used with acpi_SetIntrModel(). @@ -481,12 +482,14 @@ UINT32 acpi_event_sleep_button_wake(void *context); #define ACPI_EVENT_PRI_DEFAULT 10000 #define ACPI_EVENT_PRI_LAST 20000 -typedef void (*acpi_event_handler_t)(void *, int); +typedef void (*acpi_event_handler_t)(void *, enum power_stype); EVENTHANDLER_DECLARE(acpi_sleep_event, acpi_event_handler_t); EVENTHANDLER_DECLARE(acpi_wakeup_event, acpi_event_handler_t); EVENTHANDLER_DECLARE(acpi_acad_event, acpi_event_handler_t); EVENTHANDLER_DECLARE(acpi_video_event, acpi_event_handler_t); +EVENTHANDLER_DECLARE(acpi_post_dev_suspend, acpi_event_handler_t); +EVENTHANDLER_DECLARE(acpi_pre_dev_resume, acpi_event_handler_t); /* Device power control. */ ACPI_STATUS acpi_pwr_wake_enable(ACPI_HANDLE consumer, int enable); diff --git a/sys/dev/aic7xxx/ahc_pci.c b/sys/dev/aic7xxx/ahc_pci.c index 8109a6714814..7f8fbe140237 100644 --- a/sys/dev/aic7xxx/ahc_pci.c +++ b/sys/dev/aic7xxx/ahc_pci.c @@ -105,7 +105,7 @@ ahc_pci_attach(device_t dev) error = aic_dma_tag_create(ahc, /*parent*/bus_get_dma_tag(dev), /*alignment*/1, /*boundary*/0, (ahc->flags & AHC_39BIT_ADDRESSING) - ? 0x7FFFFFFFFFLL + ? (bus_addr_t)0x7FFFFFFFFFLL : BUS_SPACE_MAXADDR_32BIT, /*highaddr*/BUS_SPACE_MAXADDR, /*filter*/NULL, /*filterarg*/NULL, diff --git a/sys/dev/ath/ath_hal/ar5211/ar5211_reset.c b/sys/dev/ath/ath_hal/ar5211/ar5211_reset.c index fa5293777f02..c5a86ef78edb 100644 --- a/sys/dev/ath/ath_hal/ar5211/ar5211_reset.c +++ b/sys/dev/ath/ath_hal/ar5211/ar5211_reset.c @@ -1800,7 +1800,7 @@ ar5211GetInterpolatedValue(uint16_t target, /* * Look for value being within 0.1 of the search values * however, NDIS can't do float calculations, so multiply everything - * up by EEP_SCALE so can do integer arithmatic + * up by EEP_SCALE so can do integer arithmetic * * INPUT value -value to search for * INPUT pList -ptr to the list to search diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h index 0ba7b5723b91..64482a656e9d 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt.h +++ b/sys/dev/bnxt/bnxt_en/bnxt.h @@ -455,6 +455,7 @@ struct bnxt_link_info { uint16_t req_link_speed; uint8_t module_status; struct hwrm_port_phy_qcfg_output phy_qcfg_resp; + uint8_t active_lanes; }; enum bnxt_phy_type { @@ -1167,6 +1168,7 @@ struct bnxt_softc { struct iflib_dma_info def_nq_ring_mem; struct task def_cp_task; int db_size; + int db_offset; int legacy_db_size; struct bnxt_doorbell_ops db_ops; @@ -1249,6 +1251,10 @@ struct bnxt_softc { #define BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO BIT_ULL(47) #define BNXT_FW_CAP_ENABLE_RDMA_SRIOV BIT_ULL(48) #define BNXT_FW_CAP_RSS_TCAM BIT_ULL(49) + + #define BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS BIT_ULL(61) + #define BNXT_SW_RES_LMT(bp) ((bp)->fw_cap & BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS) + uint32_t lpi_tmr_lo; uint32_t lpi_tmr_hi; /* copied from flags and flags2 in hwrm_port_phy_qcaps_output */ @@ -1264,6 +1270,7 @@ struct bnxt_softc { #define BNXT_PHY_FL_NO_PAUSE (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_PAUSE_UNSUPPORTED << 8) #define BNXT_PHY_FL_NO_PFC (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_PFC_UNSUPPORTED << 8) #define BNXT_PHY_FL_BANK_SEL (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_BANK_ADDR_SUPPORTED << 8) +#define BNXT_PHY_FL_SPEEDS2 (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_SPEEDS2_SUPPORTED << 8) struct bnxt_aux_dev *aux_dev; struct net_device *net_dev; struct mtx en_ops_lock; @@ -1333,6 +1340,7 @@ struct bnxt_softc { unsigned long fw_reset_timestamp; struct bnxt_fw_health *fw_health; + char board_partno[64]; }; struct bnxt_filter_info { diff --git a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c index 9e7f4614d9f9..2a79b418fe62 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c @@ -1218,6 +1218,9 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc) flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_DBR_PACING_SUPPORTED; + if (flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_MAX_RESOURCE_LIMITS_SUPPORTED) + softc->fw_cap |= BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS; + if (flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_GENERIC_STATS_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_GENERIC_STATS; func->fw_fid = le16toh(resp->fid); @@ -1309,6 +1312,7 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc) goto end; softc->legacy_db_size = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024; + softc->db_offset = le16toh(resp->legacy_l2_db_size_kb) * 1024; if (BNXT_CHIP_P5(softc)) { if (BNXT_PF(softc)) @@ -1316,6 +1320,7 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc) else min_db_offset = DB_VF_OFFSET_P5; softc->legacy_db_size = min_db_offset; + softc->db_offset = min_db_offset; } softc->db_size = roundup2(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * @@ -2912,10 +2917,14 @@ bnxt_hwrm_port_phy_qcfg(struct bnxt_softc *softc) } link_info->duplex_setting = resp->duplex_cfg; - if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) + if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) { link_info->link_speed = le16toh(resp->link_speed); - else + if (softc->phy_flags & BNXT_PHY_FL_SPEEDS2) + link_info->active_lanes = resp->active_lanes; + } else { link_info->link_speed = 0; + link_info->active_lanes = 0; + } link_info->force_link_speed = le16toh(resp->force_link_speed); link_info->auto_link_speeds = le16toh(resp->auto_link_speed); link_info->support_speeds = le16toh(resp->support_speeds); diff --git a/sys/dev/bnxt/bnxt_en/bnxt_txrx.c b/sys/dev/bnxt/bnxt_en/bnxt_txrx.c index 2e10de6f0174..3e867454de8a 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_txrx.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_txrx.c @@ -154,12 +154,22 @@ bnxt_isc_txd_encap(void *sc, if_pkt_info_t pi) lflags |= TX_BD_LONG_LFLAGS_LSO | TX_BD_LONG_LFLAGS_T_IPID; } - else if(pi->ipi_csum_flags & CSUM_OFFLOAD) { - lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM | - TX_BD_LONG_LFLAGS_IP_CHKSUM; - } - else if(pi->ipi_csum_flags & CSUM_IP) { - lflags |= TX_BD_LONG_LFLAGS_IP_CHKSUM; + else { + if (pi->ipi_csum_flags & CSUM_IP) { + lflags |= TX_BD_LONG_LFLAGS_IP_CHKSUM; + } + switch (pi->ipi_ipproto) { + case IPPROTO_TCP: + if (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) { + lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; + } + break; + case IPPROTO_UDP: + if (pi->ipi_csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) { + lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; + } + break; + } } tbdh->lflags = htole16(lflags); } diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c index 3c1f62cb4da3..c6d862a36a9a 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c @@ -125,7 +125,7 @@ static void bnxt_fill_msix_vecs(struct bnxt_softc *bp, struct bnxt_msix_entry *e ent[i].vector = bp->irq_tbl[idx + i].vector; ent[i].ring_idx = idx + i; if (BNXT_CHIP_P5_PLUS(bp)) - ent[i].db_offset = DB_PF_OFFSET_P5; + ent[i].db_offset = bp->db_offset; else ent[i].db_offset = (idx + i) * 0x80; @@ -449,6 +449,7 @@ static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_soft edev->pdev = bp->pdev; edev->softc = bp; edev->l2_db_size = bp->db_size; + edev->l2_db_offset = bp->db_offset; mtx_init(&bp->en_ops_lock, "Ethernet ops lock", NULL, MTX_DEF); if (bp->flags & BNXT_FLAG_ROCEV1_CAP) @@ -457,9 +458,12 @@ static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_soft edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP; if (bp->is_asym_q) edev->flags |= BNXT_EN_FLAG_ASYM_Q; + if (BNXT_SW_RES_LMT(bp)) + edev->flags |= BNXT_EN_FLAG_SW_RES_LMT; edev->hwrm_bar = bp->hwrm_bar; edev->port_partition_type = bp->port_partition_type; edev->ulp_version = BNXT_ULP_VERSION; + memcpy(edev->board_part_number, bp->board_partno, BNXT_VPD_PN_FLD_LEN - 1); } int bnxt_rdma_aux_device_del(struct bnxt_softc *softc) diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.h b/sys/dev/bnxt/bnxt_en/bnxt_ulp.h index 0108293046d7..53bb51b07135 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.h +++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.h @@ -90,10 +90,15 @@ struct bnxt_en_dev { #define BNXT_EN_FLAG_ULP_STOPPED 0x8 #define BNXT_EN_FLAG_ASYM_Q 0x10 #define BNXT_EN_FLAG_MULTI_HOST 0x20 + #define BNXT_EN_FLAG_SW_RES_LMT 0x400 #define BNXT_EN_ASYM_Q(edev) ((edev)->flags & BNXT_EN_FLAG_ASYM_Q) #define BNXT_EN_MH(edev) ((edev)->flags & BNXT_EN_FLAG_MULTI_HOST) +#define BNXT_EN_SW_RES_LMT(edev) ((edev)->flags & BNXT_EN_FLAG_SW_RES_LMT) const struct bnxt_en_ops *en_ops; struct bnxt_ulp ulp_tbl[BNXT_MAX_ULP]; + int l2_db_offset; /* Doorbell BAR offset + * of non-cacheable. + */ int l2_db_size; /* Doorbell BAR size in * bytes mapped by L2 * driver. @@ -121,6 +126,9 @@ struct bnxt_en_dev { struct bnxt_dbr *en_dbr; struct bnxt_bar_info hwrm_bar; u32 espeed; + uint8_t lanes; + #define BNXT_VPD_PN_FLD_LEN 32 + char board_part_number[BNXT_VPD_PN_FLD_LEN]; }; struct bnxt_en_ops { diff --git a/sys/dev/bnxt/bnxt_en/if_bnxt.c b/sys/dev/bnxt/bnxt_en/if_bnxt.c index fa37d04e0884..dea6fd68181e 100644 --- a/sys/dev/bnxt/bnxt_en/if_bnxt.c +++ b/sys/dev/bnxt/bnxt_en/if_bnxt.c @@ -1198,8 +1198,13 @@ static int bnxt_alloc_ctx_mem(struct bnxt_softc *softc) max_srqs = ctxm->max_entries; if (softc->flags & BNXT_FLAG_ROCE_CAP) { pg_lvl = 2; - extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps); - extra_srqs = min_t(u32, 8192, max_srqs - srqs); + if (BNXT_SW_RES_LMT(softc)) { + extra_qps = max_qps - l2_qps - qp1_qps; + extra_srqs = max_srqs - srqs; + } else { + extra_qps = min_t(uint32_t, 65536, max_qps - l2_qps - qp1_qps); + extra_srqs = min_t(uint32_t, 8192, max_srqs - srqs); + } } ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; @@ -2669,6 +2674,13 @@ bnxt_attach_pre(if_ctx_t ctx) softc->state_bv = bit_alloc(BNXT_STATE_MAX, M_DEVBUF, M_WAITOK|M_ZERO); + if (BNXT_PF(softc)) { + const char *part_num; + + if (pci_get_vpd_readonly(softc->dev, "PN", &part_num) == 0) + snprintf(softc->board_partno, sizeof(softc->board_partno), "%s", part_num); + } + return (rc); failed: @@ -3280,11 +3292,10 @@ bnxt_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; - if (link_info->link_up) - ifmr->ifm_status |= IFM_ACTIVE; - else - ifmr->ifm_status &= ~IFM_ACTIVE; + if (!link_info->link_up) + return; + ifmr->ifm_status |= IFM_ACTIVE; if (link_info->duplex == HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_FULL) ifmr->ifm_active |= IFM_FDX; else @@ -4790,9 +4801,11 @@ bnxt_report_link(struct bnxt_softc *softc) const char *duplex = NULL, *flow_ctrl = NULL; const char *signal_mode = ""; - if(softc->edev) + if(softc->edev) { softc->edev->espeed = bnxt_fw_to_ethtool_speed(link_info->link_speed); + softc->edev->lanes = link_info->active_lanes; + } if (link_info->link_up == link_info->last_link_up) { if (!link_info->link_up) diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h index 8f48609e7f6f..c7ca19f29046 100644 --- a/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h +++ b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h @@ -34,7 +34,7 @@ #include <asm/types.h> #include <linux/types.h> -#define BNXT_RE_ABI_VERSION 6 +#define BNXT_RE_ABI_VERSION 7 enum { BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED = 0x01, @@ -43,12 +43,14 @@ enum { BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED = 0x08, BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x10, BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED = 0x20, - BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40 + BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40, + BNXT_RE_COMP_MASK_UCNTX_CMASK_HAVE_MODE = 0x80, }; enum { BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01, - BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02 + BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02, + BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT = 0x03 }; struct bnxt_re_uctx_req { @@ -66,7 +68,7 @@ struct bnxt_re_uctx_resp { __u32 max_cqd; __u32 chip_id0; __u32 chip_id1; - __u32 modes; + __u32 mode; __aligned_u64 comp_mask; } __attribute__((packed)); @@ -134,6 +136,8 @@ struct bnxt_re_qp_req { __u64 qpsva; __u64 qprva; __u64 qp_handle; + __u64 comp_mask; + __u32 sq_slots; } __attribute__((packed)); struct bnxt_re_qp_resp { diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re.h b/sys/dev/bnxt/bnxt_re/bnxt_re.h index fe7a27f4e216..0afc8566c020 100644 --- a/sys/dev/bnxt/bnxt_re/bnxt_re.h +++ b/sys/dev/bnxt/bnxt_re/bnxt_re.h @@ -535,6 +535,7 @@ struct bnxt_re_dev { bool is_virtfn; u32 num_vfs; u32 espeed; + u8 lanes; /* * For storing the speed of slave interfaces. * Same as espeed when bond is not configured @@ -716,7 +717,7 @@ void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 removal_type, void bnxt_re_destroy_lag(struct bnxt_re_dev **rdev); int bnxt_re_add_device(struct bnxt_re_dev **rdev, struct ifnet *netdev, - u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested, + u8 qp_mode, u8 op_type, u32 num_msix_requested, struct auxiliary_device *aux_dev); void bnxt_re_create_base_interface(bool primary); int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, @@ -1069,6 +1070,15 @@ static inline void bnxt_re_set_def_do_pacing(struct bnxt_re_dev *rdev) rdev->qplib_res.pacing_data->do_pacing = rdev->dbr_def_do_pacing; } +static inline bool bnxt_re_is_var_size_supported(struct bnxt_re_dev *rdev, + struct bnxt_re_ucontext *uctx) +{ + if (uctx) + return uctx->cmask & BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED; + else + return rdev->chip_ctx->modes.wqe_mode; +} + static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev) { rdev->qplib_res.pacing_data->dev_err_state = diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.c b/sys/dev/bnxt/bnxt_re/ib_verbs.c index 0383a16757aa..32899abab9ff 100644 --- a/sys/dev/bnxt/bnxt_re/ib_verbs.c +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.c @@ -241,46 +241,99 @@ int bnxt_re_modify_device(struct ib_device *ibdev, return 0; } -static void __to_ib_speed_width(u32 espeed, u8 *speed, u8 *width) +static void __to_ib_speed_width(u32 espeed, u8 lanes, u8 *speed, u8 *width) { - switch (espeed) { - case SPEED_1000: - *speed = IB_SPEED_SDR; + if (!lanes) { + switch (espeed) { + case SPEED_1000: + *speed = IB_SPEED_SDR; + *width = IB_WIDTH_1X; + break; + case SPEED_10000: + *speed = IB_SPEED_QDR; + *width = IB_WIDTH_1X; + break; + case SPEED_20000: + *speed = IB_SPEED_DDR; + *width = IB_WIDTH_4X; + break; + case SPEED_25000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_1X; + break; + case SPEED_40000: + *speed = IB_SPEED_QDR; + *width = IB_WIDTH_4X; + break; + case SPEED_50000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_2X; + break; + case SPEED_100000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_4X; + break; + case SPEED_200000: + *speed = IB_SPEED_HDR; + *width = IB_WIDTH_4X; + break; + case SPEED_400000: + *speed = IB_SPEED_NDR; + *width = IB_WIDTH_4X; + break; + default: + *speed = IB_SPEED_SDR; + *width = IB_WIDTH_1X; + break; + } + return; + } + + switch (lanes) { + case 1: *width = IB_WIDTH_1X; break; - case SPEED_10000: - *speed = IB_SPEED_QDR; - *width = IB_WIDTH_1X; + case 2: + *width = IB_WIDTH_2X; break; - case SPEED_20000: - *speed = IB_SPEED_DDR; + case 4: *width = IB_WIDTH_4X; break; - case SPEED_25000: - *speed = IB_SPEED_EDR; + case 8: + *width = IB_WIDTH_8X; + break; + case 12: + *width = IB_WIDTH_12X; + break; + default: *width = IB_WIDTH_1X; + } + + switch (espeed / lanes) { + case SPEED_2500: + *speed = IB_SPEED_SDR; break; - case SPEED_40000: - *speed = IB_SPEED_QDR; - *width = IB_WIDTH_4X; + case SPEED_5000: + *speed = IB_SPEED_DDR; break; - case SPEED_50000: - *speed = IB_SPEED_EDR; - *width = IB_WIDTH_2X; + case SPEED_10000: + *speed = IB_SPEED_FDR10; break; - case SPEED_100000: + case SPEED_14000: + *speed = IB_SPEED_FDR; + break; + case SPEED_25000: *speed = IB_SPEED_EDR; - *width = IB_WIDTH_4X; break; - case SPEED_200000: + case SPEED_50000: *speed = IB_SPEED_HDR; - *width = IB_WIDTH_4X; + break; + case SPEED_100000: + *speed = IB_SPEED_NDR; break; default: *speed = IB_SPEED_SDR; - *width = IB_WIDTH_1X; - break; - } + } } /* Port */ @@ -318,9 +371,10 @@ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, port_attr->subnet_timeout = 0; port_attr->init_type_reply = 0; rdev->espeed = rdev->en_dev->espeed; + rdev->lanes = rdev->en_dev->lanes; if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) - __to_ib_speed_width(rdev->espeed, &active_speed, + __to_ib_speed_width(rdev->espeed, rdev->lanes, &active_speed, &active_width); port_attr->active_speed = active_speed; @@ -1613,15 +1667,18 @@ static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, align = sizeof(struct sq_send_hdr); ilsize = ALIGN(init_attr->cap.max_inline_data, align); - sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge); - if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges)) - return -EINVAL; - /* For Cu/Wh and gen p5 backward compatibility mode - * wqe size is fixed to 128 bytes + /* For gen p4 and gen p5 fixed wqe compatibility mode + * wqe size is fixed to 128 bytes - ie 6 SGEs */ - if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) && - qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) - sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges); + if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) { + sq->wqe_size = _get_swqe_sz(BNXT_STATIC_MAX_SGE); + sq->max_sge = BNXT_STATIC_MAX_SGE; + } else { + sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge); + if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges)) + return -EINVAL; + } + if (init_attr->cap.max_inline_data) { qplqp->max_inline_data = sq->wqe_size - @@ -1666,23 +1723,28 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, return rc; bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); + bytes = PAGE_ALIGN(bytes); /* Consider mapping PSN search memory only for RC QPs. */ if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ? sizeof(struct sq_psn_search_ext) : sizeof(struct sq_psn_search); - if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) psn_sz = sizeof(struct sq_msn_search); - psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? - qplib_qp->sq.max_wqe : - ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / - sizeof(struct bnxt_qplib_sge)); - if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { + psn_nume = ureq.sq_slots; + } else { + psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + qplib_qp->sq.max_wqe : + ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / + sizeof(struct bnxt_qplib_sge)); + } + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) psn_nume = roundup_pow_of_two(psn_nume); bytes += (psn_nume * psn_sz); + bytes = PAGE_ALIGN(bytes); } - bytes = PAGE_ALIGN(bytes); umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes, IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(umem)) { @@ -1857,6 +1919,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd, /* Shadow QP SQ depth should be same as QP1 RQ depth */ qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6); qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; + qp->qplib_qp.sq.max_sw_wqe = qp1_qp->rq.max_wqe; qp->qplib_qp.sq.max_sge = 2; /* Q full delta can be 1 since it is internal QP */ qp->qplib_qp.sq.q_full_delta = 1; @@ -1868,6 +1931,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd, qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */ qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; + qp->qplib_qp.rq.max_sw_wqe = qp1_qp->rq.max_wqe; qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE; qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT; @@ -1940,6 +2004,7 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, entries = init_attr->cap.max_recv_wr + 1; entries = bnxt_re_init_depth(entries, cntx); rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); + rq->max_sw_wqe = rq->max_wqe; rq->q_full_delta = 0; rq->sginfo.pgsize = PAGE_SIZE; rq->sginfo.pgshft = PAGE_SHIFT; @@ -1964,10 +2029,11 @@ static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr, - void *cntx) + void *cntx, struct ib_udata *udata) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; + struct bnxt_re_qp_req ureq; struct bnxt_re_dev *rdev; struct bnxt_qplib_q *sq; int diff = 0; @@ -1979,35 +2045,53 @@ static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, sq = &qplqp->sq; dev_attr = rdev->dev_attr; - sq->max_sge = init_attr->cap.max_send_sge; - if (sq->max_sge > dev_attr->max_qp_sges) { - sq->max_sge = dev_attr->max_qp_sges; - init_attr->cap.max_send_sge = sq->max_sge; + if (udata) { + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + return -EINVAL; } - rc = bnxt_re_setup_swqe_size(qp, init_attr); - if (rc) - return rc; - /* - * Change the SQ depth if user has requested minimum using - * configfs. Only supported for kernel consumers. Setting - * min_tx_depth to 4096 to handle iser SQ full condition - * in most of the newer OS distros - */ + + sq->max_sge = init_attr->cap.max_send_sge; entries = init_attr->cap.max_send_wr; - if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) { + if (cntx && udata && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) { + sq->max_wqe = ureq.sq_slots; + sq->max_sw_wqe = ureq.sq_slots; + sq->wqe_size = sizeof(struct sq_sge); + } else { + if (sq->max_sge > dev_attr->max_qp_sges) { + sq->max_sge = dev_attr->max_qp_sges; + init_attr->cap.max_send_sge = sq->max_sge; + } + rc = bnxt_re_setup_swqe_size(qp, init_attr); + if (rc) + return rc; /* - * If users specify any value greater than 1 use min_tx_depth - * provided by user for comparison. Else, compare it with the - * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly. + * Change the SQ depth if user has requested minimum using + * configfs. Only supported for kernel consumers. Setting + * min_tx_depth to 4096 to handle iser SQ full condition + * in most of the newer OS distros */ - if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth) - entries = rdev->min_tx_depth; - else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH) - entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH; - } - diff = bnxt_re_get_diff(cntx, rdev->chip_ctx); - entries = bnxt_re_init_depth(entries + diff + 1, cntx); - sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); + + if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) { + /* + * If users specify any value greater than 1 use min_tx_depth + * provided by user for comparison. Else, compare it with the + * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly. + */ + if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth) + entries = rdev->min_tx_depth; + else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH) + entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH; + } + diff = bnxt_re_get_diff(cntx, rdev->chip_ctx); + entries = bnxt_re_init_depth(entries + diff + 1, cntx); + sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); + if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) + sq->max_sw_wqe = bnxt_qplib_get_depth(sq, qplqp->wqe_mode, true); + else + sq->max_sw_wqe = sq->max_wqe; + } sq->q_full_delta = diff + 1; /* * Reserving one slot for Phantom WQE. Application can @@ -2073,11 +2157,6 @@ out: return qptype; } -static int bnxt_re_init_qp_wqe_mode(struct bnxt_re_dev *rdev) -{ - return rdev->chip_ctx->modes.wqe_mode; -} - static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) @@ -2111,7 +2190,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, goto out; } qplqp->type = (u8)qptype; - qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev); + qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, cntx); ether_addr_copy(qplqp->smac, rdev->dev_addr); if (init_attr->qp_type == IB_QPT_RC) { @@ -2158,7 +2237,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, bnxt_re_adjust_gsi_rq_attr(qp); /* Setup SQ */ - rc = bnxt_re_init_sq_attr(qp, init_attr, cntx); + rc = bnxt_re_init_sq_attr(qp, init_attr, cntx, udata); if (rc) goto out; if (init_attr->qp_type == IB_QPT_GSI) @@ -2794,6 +2873,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, if (entries > dev_attr->max_qp_wqes) entries = dev_attr->max_qp_wqes; qp->qplib_qp.rq.max_wqe = entries; + qp->qplib_qp.rq.max_sw_wqe = qp->qplib_qp.rq.max_wqe; qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - qp_attr->cap.max_recv_wr; qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; @@ -5294,11 +5374,9 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, } genp5 = _is_chip_gen_p5_p7(cctx); - if (BNXT_RE_ABI_VERSION > 5) { - resp.modes = genp5 ? cctx->modes.wqe_mode : 0; - if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) - resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; - } + resp.mode = genp5 ? cctx->modes.wqe_mode : 0; + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) + resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; resp.pg_size = PAGE_SIZE; resp.cqe_sz = sizeof(struct cq_base); @@ -5331,6 +5409,12 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5)) dev_warn(rdev_to_dev(rdev), "Rsvd wqe in use! Try the updated library.\n"); + if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) { + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_CMASK_HAVE_MODE; + resp.mode = rdev->chip_ctx->modes.wqe_mode; + if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE) + resp.comp_mask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED; + } } else { dev_warn(rdev_to_dev(rdev), "Enabled roundup logic. Update the library!\n"); diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.h b/sys/dev/bnxt/bnxt_re/ib_verbs.h index cb9f7974e92d..66d15dd2b767 100644 --- a/sys/dev/bnxt/bnxt_re/ib_verbs.h +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.h @@ -49,10 +49,22 @@ struct bnxt_re_dev; #define SPEED_1000 1000 #endif +#ifndef SPEED_2500 +#define SPEED_2500 2500 +#endif + +#ifndef SPEED_5000 +#define SPEED_5000 5000 +#endif + #ifndef SPEED_10000 #define SPEED_10000 10000 #endif +#ifndef SPEED_14000 +#define SPEED_14000 14000 +#endif + #ifndef SPEED_20000 #define SPEED_20000 20000 #endif @@ -77,10 +89,18 @@ struct bnxt_re_dev; #define SPEED_200000 200000 #endif +#ifndef SPEED_400000 +#define SPEED_400000 400000 +#endif + #ifndef IB_SPEED_HDR #define IB_SPEED_HDR 64 #endif +#ifndef IB_SPEED_NDR +#define IB_SPEED_NDR 128 +#endif + #define RDMA_NETWORK_IPV4 1 #define RDMA_NETWORK_IPV6 2 @@ -488,6 +508,11 @@ static inline int bnxt_re_init_pow2_flag(struct bnxt_re_uctx_req *req, return 0; } +enum { + BNXT_RE_UCNTX_CAP_POW2_DISABLED = 0x1ULL, + BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED = 0x2ULL, +}; + static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx) { return uctx ? (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED) ? diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c index 3d26d21f3fc7..01c2710bc3ea 100644 --- a/sys/dev/bnxt/bnxt_re/main.c +++ b/sys/dev/bnxt/bnxt_re/main.c @@ -1101,7 +1101,6 @@ static int bnxt_re_handle_start(struct auxiliary_device *adev) rc = bnxt_re_add_device(&rdev, real_dev, en_info->gsi_mode, BNXT_RE_POST_RECOVERY_INIT, - en_info->wqe_mode, en_info->num_msix_requested, adev); if (rc) { /* Add device failed. Unregister the device. @@ -1411,12 +1410,14 @@ static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev) dev_info(rdev_to_dev(rdev), "Couldn't get DB bar size, Low latency framework is disabled\n"); /* set register offsets for both UC and WC */ - if (_is_chip_p7(cctx)) - res->dpi_tbl.ucreg.offset = offset; - else + if (_is_chip_p7(cctx)) { + res->dpi_tbl.ucreg.offset = en_dev->l2_db_offset; + res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; + } else { res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET : BNXT_QPLIB_DBR_PF_DB_OFFSET; - res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; + res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; + } /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size * is equal to the DB-Bar actual size. This indicates that L2 @@ -1433,15 +1434,15 @@ static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev) return; } -static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode) +static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev) { struct bnxt_qplib_chip_ctx *cctx; struct bnxt_en_dev *en_dev; en_dev = rdev->en_dev; cctx = rdev->chip_ctx; - cctx->modes.wqe_mode = _is_chip_gen_p5_p7(rdev->chip_ctx) ? - mode : BNXT_QPLIB_WQE_MODE_STATIC; + cctx->modes.wqe_mode = _is_chip_p7(rdev->chip_ctx) ? + BNXT_QPLIB_WQE_MODE_VARIABLE : BNXT_QPLIB_WQE_MODE_STATIC; cctx->modes.te_bypass = false; if (bnxt_re_hwrm_qcaps(rdev)) dev_err(rdev_to_dev(rdev), @@ -1490,7 +1491,7 @@ static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) kfree(chip_ctx); } -static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode) +static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev) { struct bnxt_qplib_chip_ctx *chip_ctx; struct bnxt_en_dev *en_dev; @@ -1525,7 +1526,7 @@ static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode) rc = -ENOMEM; goto fail; } - bnxt_re_set_drv_mode(rdev, wqe_mode); + bnxt_re_set_drv_mode(rdev); bnxt_re_set_db_offset(rdev); rc = bnxt_qplib_map_db_bar(&rdev->qplib_res); @@ -2029,11 +2030,30 @@ static ssize_t show_hca(struct device *device, struct device_attribute *attr, return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); } +static ssize_t show_board_id(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); + char buffer[BNXT_VPD_PN_FLD_LEN] = {}; + + if (!rdev->is_virtfn) + memcpy(buffer, rdev->en_dev->board_part_number, + BNXT_VPD_PN_FLD_LEN - 1); + else + scnprintf(buffer, BNXT_VPD_PN_FLD_LEN, + "0x%x-VF", rdev->en_dev->pdev->device); + + return scnprintf(buf, PAGE_SIZE, "%s\n", buffer); +} + static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); +static DEVICE_ATTR(board_id, 0444, show_board_id, NULL); + static struct device_attribute *bnxt_re_attributes[] = { &dev_attr_hw_rev, - &dev_attr_hca_type + &dev_attr_hca_type, + &dev_attr_board_id }; int ib_register_device_compat(struct bnxt_re_dev *rdev) @@ -3530,7 +3550,7 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type) } } -static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode) +static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type) { struct bnxt_re_ring_attr rattr = {}; struct bnxt_qplib_creq_ctx *creq; @@ -3545,7 +3565,7 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode) } set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); - rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode); + rc = bnxt_re_setup_chip_ctx(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc); bnxt_re_unregister_netdev(rdev); @@ -3592,19 +3612,24 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode) goto release_rtnl; } + set_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags); + if (!rdev->chip_ctx) goto release_rtnl; - /* Program the NQ ID for DBQ notification */ - if (rdev->chip_ctx->modes.dbr_pacing_v0 || - bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || - bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { - rc = bnxt_re_initialize_dbr_pacing(rdev); - if (!rc) - rdev->dbr_pacing = true; - else - rdev->dbr_pacing = false; - dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", - __func__, rc); + + if (!(_is_chip_p7(rdev->chip_ctx))) { + /* Program the NQ ID for DBQ notification */ + if (rdev->chip_ctx->modes.dbr_pacing_v0 || + bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || + bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + rc = bnxt_re_initialize_dbr_pacing(rdev); + if (!rc) + rdev->dbr_pacing = true; + else + rdev->dbr_pacing = false; + dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", + __func__, rc); + } } vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector; @@ -3811,6 +3836,7 @@ static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev, void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev) { rdev->espeed = rdev->en_dev->espeed; + rdev->lanes = rdev->en_dev->lanes; return; } @@ -3852,7 +3878,7 @@ void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type, int bnxt_re_add_device(struct bnxt_re_dev **rdev, struct ifnet *netdev, - u8 qp_mode, u8 op_type, u8 wqe_mode, + u8 qp_mode, u8 op_type, u32 num_msix_requested, struct auxiliary_device *aux_dev) { @@ -3925,7 +3951,7 @@ int bnxt_re_add_device(struct bnxt_re_dev **rdev, rtnl_lock(); en_info->rdev = *rdev; rtnl_unlock(); - rc = bnxt_re_dev_init(*rdev, op_type, wqe_mode); + rc = bnxt_re_dev_init(*rdev, op_type); if (rc) { ref_error: bnxt_re_dev_unreg(*rdev); @@ -4374,7 +4400,6 @@ static int bnxt_re_probe(struct auxiliary_device *adev, rc = bnxt_re_add_device(&rdev, en_dev->net, BNXT_RE_GSI_MODE_ALL, BNXT_RE_COMPLETE_INIT, - BNXT_QPLIB_WQE_MODE_STATIC, BNXT_RE_MSIX_FROM_MOD_PARAM, adev); if (rc) { mutex_unlock(&bnxt_re_mutex); diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.c b/sys/dev/bnxt/bnxt_re/qplib_fp.c index 3f1b02406f7f..19708302198b 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_fp.c +++ b/sys/dev/bnxt/bnxt_re/qplib_fp.c @@ -388,10 +388,14 @@ static void bnxt_qplib_service_nq(unsigned long data) struct bnxt_qplib_srq *srq; struct nq_srq_event *nqsrqe = (struct nq_srq_event *)nqe; + u8 toggle; q_handle = le32_to_cpu(nqsrqe->srq_handle_low); q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32; srq = (struct bnxt_qplib_srq *)q_handle; + toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) + >> NQ_CN_TOGGLE_SFT; + srq->dbinfo.toggle = toggle; bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); if (!nq->srqn_handler(nq, @@ -838,15 +842,15 @@ static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que) int rc = 0; int indx; - que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL); + que->swq = kcalloc(que->max_sw_wqe, sizeof(*que->swq), GFP_KERNEL); if (!que->swq) { rc = -ENOMEM; goto out; } que->swq_start = 0; - que->swq_last = que->max_wqe - 1; - for (indx = 0; indx < que->max_wqe; indx++) + que->swq_last = que->max_sw_wqe - 1; + for (indx = 0; indx < que->max_sw_wqe; indx++) que->swq[indx].next_idx = indx + 1; que->swq[que->swq_last].next_idx = 0; /* Make it circular */ que->swq_last = 0; @@ -875,19 +879,23 @@ static u32 bnxt_qplib_get_stride(void) return sizeof(struct sq_sge); } -static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que) +u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que, u8 wqe_mode, bool is_sq) { - u8 stride; + u32 slots; - stride = bnxt_qplib_get_stride(); - return (que->wqe_size * que->max_wqe) / stride; + /* Queue depth is the number of slots. */ + slots = (que->wqe_size * que->max_wqe) / sizeof(struct sq_sge); + /* For variable WQE mode, need to align the slots to 256 */ + if (wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE && is_sq) + slots = ALIGN(slots, BNXT_VAR_MAX_SLOT_ALIGN); + return slots; } static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode) { /* For Variable mode supply number of 16B slots */ return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? - que->max_wqe : bnxt_qplib_get_depth(que); + que->max_wqe : bnxt_qplib_get_depth(que, wqe_mode, true); } static u32 _set_sq_max_slot(u8 wqe_mode) @@ -925,7 +933,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) hwq_attr.res = res; hwq_attr.sginfo = &sq->sginfo; hwq_attr.stride = bnxt_qplib_get_stride(); - hwq_attr.depth = bnxt_qplib_get_depth(sq); + hwq_attr.depth = bnxt_qplib_get_depth(sq, qp->wqe_mode, false); hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); if (rc) @@ -949,7 +957,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) hwq_attr.res = res; hwq_attr.sginfo = &rq->sginfo; hwq_attr.stride = bnxt_qplib_get_stride(); - hwq_attr.depth = bnxt_qplib_get_depth(rq); + hwq_attr.depth = bnxt_qplib_get_depth(rq, qp->wqe_mode, false); hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); if (rc) @@ -1075,8 +1083,11 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) u32 sqsz; qp->cctx = res->cctx; - if (res->dattr) + if (res->dattr) { qp->dev_cap_flags = res->dattr->dev_cap_flags; + qp->is_host_msn_tbl = _is_host_msn_table(res->dattr->dev_cap_ext_flags2); + } + /* General */ req.type = qp->type; req.dpi = cpu_to_le32(qp->dpi->dpi); @@ -1087,7 +1098,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) psn_sz = _is_chip_gen_p5_p7(qp->cctx) ? sizeof(struct sq_psn_search_ext) : sizeof(struct sq_psn_search); - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + if (qp->is_host_msn_tbl) { psn_sz = sizeof(struct sq_msn_search); qp->msn = 0; } @@ -1098,12 +1109,12 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) hwq_attr.res = res; hwq_attr.sginfo = &sq->sginfo; hwq_attr.stride = bnxt_qplib_get_stride(); - hwq_attr.depth = bnxt_qplib_get_depth(sq); + hwq_attr.depth = bnxt_qplib_get_depth(sq, qp->wqe_mode, true); hwq_attr.aux_stride = psn_sz; hwq_attr.aux_depth = (psn_sz) ? _set_sq_size(sq, qp->wqe_mode) : 0; /* Update msn tbl size */ - if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { + if (qp->is_host_msn_tbl && psn_sz) { if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)); else @@ -1131,8 +1142,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) req.sq_pg_size_sq_lvl = pg_sz_lvl; req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) & CMDQ_CREATE_QP_SQ_FWO_MASK) | - ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ? - BNXT_MSN_TBLE_SGE : sq->max_sge & + (sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK)); req.scq_cid = cpu_to_le32(qp->scq->id); @@ -1141,7 +1151,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) hwq_attr.res = res; hwq_attr.sginfo = &rq->sginfo; hwq_attr.stride = bnxt_qplib_get_stride(); - hwq_attr.depth = bnxt_qplib_get_depth(rq); + hwq_attr.depth = bnxt_qplib_get_depth(rq, qp->wqe_mode, false); hwq_attr.aux_stride = 0; hwq_attr.aux_depth = 0; hwq_attr.type = HWQ_TYPE_QUEUE; @@ -1764,7 +1774,7 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, return; /* Handle MSN differently on cap flags */ - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + if (qp->is_host_msn_tbl) { bnxt_qplib_fill_msn_search(qp, wqe, swq); return; } @@ -1897,7 +1907,7 @@ static u16 _translate_q_full_delta(struct bnxt_qplib_q *que, u16 wqe_bytes) } static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq, - struct bnxt_qplib_swq *swq, bool hw_retx) + struct bnxt_qplib_swq *swq, bool is_host_msn_tbl) { struct bnxt_qplib_hwq *sq_hwq; u32 pg_num, pg_indx; @@ -1909,8 +1919,11 @@ static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib return; tail = swq->slot_idx / sq->dbinfo.max_slot; - if (hw_retx) + if (is_host_msn_tbl) { + /* For HW retx use qp msn index */ + tail = qp->msn; tail %= qp->msn_tbl_sz; + } pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride); pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride); buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride); @@ -1935,6 +1948,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swq *swq; bool sch_handler = false; u16 slots_needed; + bool msn_update; void *base_hdr; void *ext_hdr; __le32 temp32; @@ -1976,7 +1990,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, sw_prod = sq_hwq->prod; swq = bnxt_qplib_get_swqe(sq, &wqe_idx); swq->slot_idx = sw_prod; - bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); + bnxt_qplib_pull_psn_buff(qp, sq, swq, qp->is_host_msn_tbl); swq->wr_id = wqe->wr_id; swq->type = wqe->type; @@ -2010,6 +2024,9 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, wqe->num_sge, &sw_prod); if (data_len < 0) goto queue_err; + /* Make sure we update MSN table only for wired wqes */ + msn_update = true; + /* Specifics */ switch (wqe->type) { case BNXT_QPLIB_SWQE_TYPE_SEND: @@ -2064,6 +2081,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, ext_sqe->avid = cpu_to_le32(wqe->send.avid & SQ_SEND_AVID_MASK); sq->psn = (sq->psn + 1) & BTH_PSN_MASK; + msn_update = false; } else { sqe->length = cpu_to_le32(data_len); if (qp->mtu) @@ -2157,6 +2175,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, "\tflags = 0x%x\n" "\tinv_l_key = 0x%x\n", sqe->wqe_type, sqe->flags, sqe->inv_l_key); + msn_update = false; break; } case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: @@ -2207,6 +2226,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, sqe->zero_based_page_size_log, sqe->l_key, *(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log, ext_sqe->pblptr, ext_sqe->va); + msn_update = false; break; } case BNXT_QPLIB_SWQE_TYPE_BIND_MW: @@ -2236,6 +2256,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, sqe->wqe_type, sqe->flags, sqe->access_cntl, sqe->mw_type_zero_based, sqe->parent_l_key, sqe->l_key, sqe->va, ext_sqe->length_lo); + msn_update = false; break; } default: @@ -2243,8 +2264,10 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, rc = -EINVAL; goto done; } - swq->next_psn = sq->psn & BTH_PSN_MASK; - bnxt_qplib_fill_psn_search(qp, wqe, swq); + if (!qp->is_host_msn_tbl || msn_update) { + swq->next_psn = sq->psn & BTH_PSN_MASK; + bnxt_qplib_fill_psn_search(qp, wqe, swq); + } queue_err: bnxt_qplib_swq_mod_start(sq, wqe_idx); @@ -2859,6 +2882,32 @@ out: return rc; } +static int bnxt_qplib_get_cqe_sq_cons(struct bnxt_qplib_q *sq, u32 cqe_slot) +{ + struct bnxt_qplib_hwq *sq_hwq; + struct bnxt_qplib_swq *swq; + int cqe_sq_cons = -1; + u32 start, last; + + sq_hwq = &sq->hwq; + + start = sq->swq_start; + last = sq->swq_last; + + while (last != start) { + swq = &sq->swq[last]; + if (swq->slot_idx == cqe_slot) { + cqe_sq_cons = swq->next_idx; + dev_err(&sq_hwq->pdev->dev, "%s: Found cons wqe = %d slot = %d\n", + __func__, cqe_sq_cons, cqe_slot); + break; + } + + last = swq->next_idx; + } + return cqe_sq_cons; +} + static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, struct cq_req *hwcqe, struct bnxt_qplib_cqe **pcqe, int *budget, @@ -2867,8 +2916,9 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, struct bnxt_qplib_qp *qp; struct bnxt_qplib_q *sq; struct bnxt_qplib_cqe *cqe; - u32 cqe_sq_cons; + u32 cqe_sq_cons, slot_num; struct bnxt_qplib_swq *swq; + int cqe_cons; int rc = 0; qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); @@ -2880,13 +2930,26 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, } sq = &qp->sq; - cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe; + cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_sw_wqe; if (qp->sq.flushed) { dev_dbg(&cq->hwq.pdev->dev, "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); goto done; } + if (__is_err_cqe_for_var_wqe(qp, hwcqe->status)) { + slot_num = le16_to_cpu(hwcqe->sq_cons_idx); + cqe_cons = bnxt_qplib_get_cqe_sq_cons(sq, slot_num); + if (cqe_cons < 0) { + dev_dbg(&cq->hwq.pdev->dev, "%s: Wrong SQ cons cqe_slot_indx = %d\n", + __func__, slot_num); + goto done; + } + cqe_sq_cons = cqe_cons; + dev_dbg(&cq->hwq.pdev->dev, "%s: cqe_sq_cons = %d swq_last = %d swq_start = %d\n", + __func__, cqe_sq_cons, sq->swq_last, sq->swq_start); + } + /* Require to walk the sq's swq to fabricate CQEs for all previously * signaled SWQEs due to CQE aggregation from the current sq cons * to the cqe_sq_cons @@ -3329,7 +3392,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, if (cqe_cons == 0xFFFF) goto do_rq; - cqe_cons %= sq->max_wqe; + cqe_cons %= sq->max_sw_wqe; if (qp->sq.flushed) { dev_dbg(&cq->hwq.pdev->dev, "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.h b/sys/dev/bnxt/bnxt_re/qplib_fp.h index 527c377f0aa5..542a26782c62 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_fp.h +++ b/sys/dev/bnxt/bnxt_re/qplib_fp.h @@ -300,6 +300,7 @@ struct bnxt_qplib_q { struct bnxt_qplib_sg_info sginfo; struct bnxt_qplib_hwq hwq; u32 max_wqe; + u32 max_sw_wqe; u16 max_sge; u16 wqe_size; u16 q_full_delta; @@ -390,6 +391,7 @@ struct bnxt_qplib_qp { u32 msn_tbl_sz; /* get devflags in PI code */ u16 dev_cap_flags; + bool is_host_msn_tbl; }; @@ -634,5 +636,16 @@ static inline uint64_t bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, ui SQ_MSN_SEARCH_START_PSN_MASK)); } +static inline bool __is_var_wqe(struct bnxt_qplib_qp *qp) +{ + return (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE); +} + +static inline bool __is_err_cqe_for_var_wqe(struct bnxt_qplib_qp *qp, u8 status) +{ + return (status != CQ_REQ_STATUS_OK) && __is_var_wqe(qp); +} + void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res); +u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que, u8 wqe_mode, bool is_sq); #endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.c b/sys/dev/bnxt/bnxt_re/qplib_res.c index f527af031176..9051f4c9f2b7 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_res.c +++ b/sys/dev/bnxt/bnxt_re/qplib_res.c @@ -1139,7 +1139,8 @@ int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res) ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION; ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id); - ucreg->offset = 65536; + if (_is_chip_gen_p5(res->cctx)) + ucreg->offset = 65536; ucreg->len = ucreg->offset + PAGE_SIZE; diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.h b/sys/dev/bnxt/bnxt_re/qplib_res.h index 6468207a49aa..59a8a43ecef6 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_res.h +++ b/sys/dev/bnxt/bnxt_re/qplib_res.h @@ -616,6 +616,12 @@ static inline bool _is_hw_retx_supported(u16 dev_cap_flags) /* Disable HW_RETX */ #define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) +static inline bool _is_host_msn_table(u16 dev_cap_ext_flags2) +{ + return (dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK) == + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE; +} + static inline bool _is_cqe_v2_supported(u16 dev_cap_flags) { return dev_cap_flags & @@ -650,7 +656,7 @@ static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info, #define BNXT_QPLIB_INIT_DBHDR(xid, type, indx, toggle) \ (((u64)(((xid) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | \ (type) | BNXT_QPLIB_DBR_VALID) << 32) | (indx) | \ - ((toggle) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) + (((u32)(toggle)) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) static inline void bnxt_qplib_write_db(struct bnxt_qplib_db_info *info, u64 key, void __iomem *db, @@ -724,7 +730,7 @@ static inline void bnxt_qplib_armen_db(struct bnxt_qplib_db_info *info, u64 key = 0; u8 toggle = 0; - if (type == DBC_DBC_TYPE_CQ_ARMENA) + if (type == DBC_DBC_TYPE_CQ_ARMENA || type == DBC_DBC_TYPE_SRQ_ARMENA) toggle = info->toggle; /* Index always at 0 */ key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, 0, toggle); @@ -746,7 +752,7 @@ static inline void bnxt_qplib_srq_arm_db(struct bnxt_qplib_db_info *info) u64 key = 0; /* Index always at 0 */ - key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, 0); + key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, info->toggle); bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key); } @@ -837,4 +843,11 @@ static inline void bnxt_qplib_max_res_supported(struct bnxt_qplib_chip_ctx *cctx break; } } + +static inline u32 bnxt_re_cap_fw_res(u32 fw_val, u32 drv_cap, bool sw_max_en) +{ + if (sw_max_en) + return fw_val; + return min_t(u32, fw_val, drv_cap); +} #endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.c b/sys/dev/bnxt/bnxt_re/qplib_sp.c index c414718a816f..f876573ce69f 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_sp.c +++ b/sys/dev/bnxt/bnxt_re/qplib_sp.c @@ -40,6 +40,7 @@ #include "qplib_res.h" #include "qplib_rcfw.h" #include "qplib_sp.h" +#include "bnxt_ulp.h" const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }}; @@ -79,6 +80,7 @@ static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, char *fw_ver) int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) { + struct bnxt_qplib_max_res dev_res = {}; struct creq_query_func_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct creq_query_func_resp_sb *sb; @@ -86,10 +88,10 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) struct bnxt_qplib_dev_attr *attr; struct bnxt_qplib_chip_ctx *cctx; struct cmdq_query_func req = {}; + bool sw_max_en; u8 *tqm_alloc; int i, rc = 0; u32 temp; - u8 chip_gen = BNXT_RE_DEFAULT; cctx = rcfw->res->cctx; attr = rcfw->res->dattr; @@ -110,10 +112,11 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto bail; + bnxt_qplib_max_res_supported(cctx, rcfw->res, &dev_res, false); + sw_max_en = BNXT_EN_SW_RES_LMT(rcfw->res->en_dev); /* Extract the context from the side buffer */ - chip_gen = _get_chip_gen_p5_type(cctx); - attr->max_qp = le32_to_cpu(sb->max_qp); - attr->max_qp = min_t(u32, attr->max_qp, BNXT_RE_MAX_QP_SUPPORTED(chip_gen)); + attr->max_qp = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_qp), + dev_res.max_qp, sw_max_en); /* max_qp value reported by FW does not include the QP1 */ attr->max_qp += 1; attr->max_qp_rd_atom = @@ -126,11 +129,6 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) * one extra entry while creating the qp */ attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr) - 1; - /* Adjust for max_qp_wqes for variable wqe */ - if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) { - attr->max_qp_wqes = (BNXT_MAX_SQ_SIZE) / - (BNXT_MAX_VAR_WQE_SIZE / BNXT_SGE_SIZE) - 1; - } if (!_is_chip_gen_p5_p7(cctx)) { /* * 128 WQEs needs to be reserved for the HW (8916). Prevent @@ -138,33 +136,36 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) */ attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS; } - attr->max_qp_sges = sb->max_sge; - if (_is_chip_gen_p5_p7(cctx) && - cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) - attr->max_qp_sges = sb->max_sge_var_wqe; - attr->max_cq = le32_to_cpu(sb->max_cq); - attr->max_cq = min_t(u32, attr->max_cq, BNXT_RE_MAX_CQ_SUPPORTED(chip_gen)); + + /* Adjust for max_qp_wqes for variable wqe */ + if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) + attr->max_qp_wqes = BNXT_VAR_MAX_WQE - 1; + + attr->max_qp_sges = cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE ? + min_t(u32, sb->max_sge_var_wqe, BNXT_VAR_MAX_SGE) : sb->max_sge; + attr->max_cq = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_cq), + dev_res.max_cq, sw_max_en); attr->max_cq_wqes = le32_to_cpu(sb->max_cqe); attr->max_cq_wqes = min_t(u32, BNXT_QPLIB_MAX_CQ_WQES, attr->max_cq_wqes); attr->max_cq_sges = attr->max_qp_sges; - attr->max_mr = le32_to_cpu(sb->max_mr); - attr->max_mr = min_t(u32, attr->max_mr, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen)); - attr->max_mw = le32_to_cpu(sb->max_mw); - attr->max_mw = min_t(u32, attr->max_mw, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen)); + attr->max_mr = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_mr), + dev_res.max_mr, sw_max_en); + attr->max_mw = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_mw), + dev_res.max_mr, sw_max_en); attr->max_mr_size = le64_to_cpu(sb->max_mr_size); attr->max_pd = BNXT_QPLIB_MAX_PD; attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp); - attr->max_ah = le32_to_cpu(sb->max_ah); - attr->max_ah = min_t(u32, attr->max_ah, BNXT_RE_MAX_AH_SUPPORTED(chip_gen)); + attr->max_ah = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_ah), + dev_res.max_ah, sw_max_en); attr->max_fmr = le32_to_cpu(sb->max_fmr); attr->max_map_per_fmr = sb->max_map_per_fmr; - attr->max_srq = le16_to_cpu(sb->max_srq); - attr->max_srq = min_t(u32, attr->max_srq, BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen)); + attr->max_srq = bnxt_re_cap_fw_res(le16_to_cpu(sb->max_srq), + dev_res.max_srq, sw_max_en); attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; attr->max_srq_sges = sb->max_srq_sge; attr->max_pkey = 1; @@ -185,6 +186,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) attr->page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(12); bnxt_qplib_query_version(rcfw, attr->fw_ver); + attr->dev_cap_ext_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2); for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) { temp = le32_to_cpu(sb->tqm_alloc_reqs[i]); diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.h b/sys/dev/bnxt/bnxt_re/qplib_sp.h index e306db3b9d8e..5a5485dc5250 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_sp.h +++ b/sys/dev/bnxt/bnxt_re/qplib_sp.h @@ -32,6 +32,7 @@ #define __BNXT_QPLIB_SP_H__ #include <rdma/ib_verbs.h> +#include "bnxt_re-abi.h" #define BNXT_QPLIB_RESERVED_QP_WRS 128 @@ -71,6 +72,7 @@ struct bnxt_qplib_dev_attr { u32 l2_db_size; u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; u8 is_atomic; + u16 dev_cap_ext_flags2; u16 dev_cap_flags; u64 page_size_cap; u32 max_dpi; @@ -394,6 +396,13 @@ bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, enum ib_qp_state next #define BNXT_MAX_VAR_WQE_SIZE 512 #define BNXT_SGE_SIZE 16 +#define BNXT_VAR_MAX_WQE 4352 +#define BNXT_VAR_MAX_SLOT_ALIGN 256 +#define BNXT_VAR_MAX_SGE 13 +#define BNXT_RE_MAX_RQ_WQES 65536 + +#define BNXT_STATIC_MAX_SGE 6 + /* PF defines */ #define BNXT_RE_MAX_QP_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 3237c6649713..e35bb9f64951 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -900,6 +900,7 @@ static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); static int sysctl_cpus(SYSCTL_HANDLER_ARGS); static int sysctl_reset(SYSCTL_HANDLER_ARGS); +static int sysctl_tcb_cache(SYSCTL_HANDLER_ARGS); #ifdef TCP_OFFLOAD static int sysctl_tls(SYSCTL_HANDLER_ARGS); static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS); @@ -8119,6 +8120,12 @@ t4_sysctls(struct adapter *sc) sysctl_wcwr_stats, "A", "write combined work requests"); } + if (chip_id(sc) >= CHELSIO_T7) { + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcb_cache", + CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tcb_cache, "I", + "1 = enabled (default), 0 = disabled (for debug only)"); + } + #ifdef KERN_TLS if (is_ktls(sc)) { /* @@ -12204,6 +12211,40 @@ sysctl_reset(SYSCTL_HANDLER_ARGS) return (0); } +static int +sysctl_tcb_cache(SYSCTL_HANDLER_ARGS) +{ + struct adapter *sc = arg1; + u_int val, v; + int rc; + + mtx_lock(&sc->reg_lock); + if (hw_off_limits(sc)) { + rc = ENXIO; + goto done; + } + t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1); + mtx_unlock(&sc->reg_lock); + + val = v & F_GLFL ? 0 : 1; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc != 0 || req->newptr == NULL) + return (rc); + if (val == 0) + v |= F_GLFL; + else + v &= ~F_GLFL; + + mtx_lock(&sc->reg_lock); + if (hw_off_limits(sc)) + rc = ENXIO; + else + t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1); +done: + mtx_unlock(&sc->reg_lock); + return (rc); +} + #ifdef TCP_OFFLOAD static int sysctl_tls(SYSCTL_HANDLER_ARGS) diff --git a/sys/dev/cxgbe/t4_netmap.c b/sys/dev/cxgbe/t4_netmap.c index 0135bec6e2c1..a858867239c6 100644 --- a/sys/dev/cxgbe/t4_netmap.c +++ b/sys/dev/cxgbe/t4_netmap.c @@ -606,10 +606,8 @@ cxgbe_netmap_split_rss(struct adapter *sc, struct vi_info *vi, (nm_state == NM_OFF && nm_kring_pending_on(kring))) { MPASS(nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID); nactive[j]++; - if (dq[j] == -1) { + if (dq[j] == -1) dq[j] = nm_rxq->iq_abs_id; - break; - } } } diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index b6d44792dce4..af18b3019760 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -828,12 +828,17 @@ t4_tweak_chip_settings(struct adapter *sc) t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v); if (sc->debug_flags & DF_DISABLE_TCB_CACHE) { - m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN | - V_WRTHRTHRESH(M_WRTHRTHRESH); t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1); - v &= ~m; - v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN | - V_WRTHRTHRESH(16); + if (chip_id(sc) >= CHELSIO_T7) { + v |= F_GLFL; + } else { + m = V_RDTHRESHOLD(M_RDTHRESHOLD) | + F_WRTHRTHRESHEN | + V_WRTHRTHRESH(M_WRTHRTHRESH); + v &= ~m; + v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN | + V_WRTHRTHRESH(16); + } t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1); } } diff --git a/sys/dev/dpaa2/dpaa2_ni.c b/sys/dev/dpaa2/dpaa2_ni.c index eda5bab78bde..c72e68b8a62f 100644 --- a/sys/dev/dpaa2/dpaa2_ni.c +++ b/sys/dev/dpaa2/dpaa2_ni.c @@ -220,6 +220,9 @@ MALLOC_DEFINE(M_DPAA2_TXB, "dpaa2_txb", "DPAA2 DMA-mapped buffer (Tx)"); #define RXH_L4_B_2_3 (1 << 7) /* dst port in case of TCP/UDP/SCTP */ #define RXH_DISCARD (1 << 31) +/* Transmit checksum offload */ +#define DPAA2_CSUM_TX_OFFLOAD (CSUM_IP | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6) + /* Default Rx hash options, set during attaching. */ #define DPAA2_RXH_DEFAULT (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3) @@ -559,6 +562,7 @@ dpaa2_ni_attach(device_t dev) if_settransmitfn(ifp, dpaa2_ni_transmit); if_setqflushfn(ifp, dpaa2_ni_qflush); + if_sethwassist(sc->ifp, DPAA2_CSUM_TX_OFFLOAD); if_setcapabilities(ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU); if_setcapenable(ifp, if_getcapabilities(ifp)); @@ -2528,6 +2532,7 @@ dpaa2_ni_transmit(if_t ifp, struct mbuf *m) ch = sc->channels[chidx]; error = buf_ring_enqueue(ch->xmit_br, m); if (__predict_false(error != 0)) { + if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); m_freem(m); } else { taskqueue_enqueue(ch->cleanup_tq, &ch->cleanup_task); @@ -2600,8 +2605,10 @@ dpaa2_ni_ioctl(if_t ifp, u_long c, caddr_t data) changed = if_getcapenable(ifp) ^ ifr->ifr_reqcap; if ((changed & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) != 0) if_togglecapenable(ifp, IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); - if ((changed & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) != 0) - if_togglecapenable(ifp, IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); + if ((changed & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) != 0) { + if_togglecapenable(ifp, IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); + if_togglehwassist(ifp, DPAA2_CSUM_TX_OFFLOAD); + } rc = dpaa2_ni_setup_if_caps(sc); if (rc) { @@ -2937,6 +2944,8 @@ dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, bus_dma_segment_t segs[DPAA2_TX_SEGLIMIT]; int rc, nsegs; int error; + int len; + bool mcast; mtx_assert(&tx->lock, MA_NOTOWNED); mtx_lock(&tx->lock); @@ -2951,6 +2960,8 @@ dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, buf->m = m; sgt = buf->sgt; } + len = m->m_pkthdr.len; + mcast = (m->m_flags & M_MCAST) != 0; #if defined(INVARIANTS) struct dpaa2_ni_tx_ring *btx = (struct dpaa2_ni_tx_ring *)buf->opt; @@ -2968,6 +2979,7 @@ dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, if (md == NULL) { device_printf(dev, "%s: m_collapse() failed\n", __func__); fq->chan->tx_dropped++; + if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); goto err; } @@ -2978,6 +2990,7 @@ dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, device_printf(dev, "%s: bus_dmamap_load_mbuf_sg() " "failed: error=%d\n", __func__, error); fq->chan->tx_dropped++; + if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); goto err; } } @@ -2987,6 +3000,7 @@ dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, device_printf(dev, "%s: failed to build frame descriptor: " "error=%d\n", __func__, error); fq->chan->tx_dropped++; + if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); goto err_unload; } @@ -3004,8 +3018,13 @@ dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, if (rc != 1) { fq->chan->tx_dropped++; + if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); goto err_unload; } else { + if (mcast) + if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1); + if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); + if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, len); fq->chan->tx_frames++; } return; @@ -3184,6 +3203,7 @@ dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, struct dpaa2_fd *f m->m_pkthdr.rcvif = sc->ifp; m->m_pkthdr.flowid = fq->fqid; M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); + if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); if (ctx->head == NULL) { KASSERT(ctx->tail == NULL, ("%s: tail already given?", __func__)); diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c index 7d7655a7ae6f..4d7bc794a0b2 100644 --- a/sys/dev/e1000/if_em.c +++ b/sys/dev/e1000/if_em.c @@ -2106,11 +2106,11 @@ em_if_set_promisc(if_ctx_t ctx, int flags) if (flags & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); - em_if_vlan_filter_disable(sc); /* Turn this on if you want to see bad packets */ if (em_debug_sbp) reg_rctl |= E1000_RCTL_SBP; E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); + em_if_vlan_filter_disable(sc); } else { if (flags & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; diff --git a/sys/dev/hpt27xx/hpt27xx_osm_bsd.c b/sys/dev/hpt27xx/hpt27xx_osm_bsd.c index e086a1554940..638efa9e2ae2 100644 --- a/sys/dev/hpt27xx/hpt27xx_osm_bsd.c +++ b/sys/dev/hpt27xx/hpt27xx_osm_bsd.c @@ -1271,13 +1271,9 @@ static driver_t hpt_pci_driver = { #error "no TARGETNAME found" #endif -/* use this to make TARGETNAME be expanded */ -#define __DRIVER_MODULE(p1, p2, p3, p4, p5) DRIVER_MODULE(p1, p2, p3, p4, p5) -#define __MODULE_VERSION(p1, p2) MODULE_VERSION(p1, p2) -#define __MODULE_DEPEND(p1, p2, p3, p4, p5) MODULE_DEPEND(p1, p2, p3, p4, p5) -__DRIVER_MODULE(TARGETNAME, pci, hpt_pci_driver, 0, 0); -__MODULE_VERSION(TARGETNAME, 1); -__MODULE_DEPEND(TARGETNAME, cam, 1, 1, 1); +DRIVER_MODULE(TARGETNAME, pci, hpt_pci_driver, NULL, NULL); +MODULE_VERSION(TARGETNAME, 1); +MODULE_DEPEND(TARGETNAME, cam, 1, 1, 1); static int hpt_open(struct cdev *dev, int flags, int devtype, struct thread *td) { diff --git a/sys/dev/hptmv/entry.c b/sys/dev/hptmv/entry.c index f3d58f285b39..a82ffb66bdc8 100644 --- a/sys/dev/hptmv/entry.c +++ b/sys/dev/hptmv/entry.c @@ -95,8 +95,7 @@ static driver_t hpt_pci_driver = { sizeof(IAL_ADAPTER_T) }; -#define __DRIVER_MODULE(p1, p2, p3, p4, p5) DRIVER_MODULE(p1, p2, p3, p4, p5) -__DRIVER_MODULE(PROC_DIR_NAME, pci, hpt_pci_driver, 0, 0); +DRIVER_MODULE(PROC_DIR_NAME, pci, hpt_pci_driver, NULL, NULL); MODULE_DEPEND(PROC_DIR_NAME, cam, 1, 1, 1); #define ccb_ccb_ptr spriv_ptr0 diff --git a/sys/dev/hptnr/hptnr_osm_bsd.c b/sys/dev/hptnr/hptnr_osm_bsd.c index 7426873964fb..fa0f78a7e01a 100644 --- a/sys/dev/hptnr/hptnr_osm_bsd.c +++ b/sys/dev/hptnr/hptnr_osm_bsd.c @@ -1561,13 +1561,9 @@ static driver_t hpt_pci_driver = { #error "no TARGETNAME found" #endif -/* use this to make TARGETNAME be expanded */ -#define __DRIVER_MODULE(p1, p2, p3, p4, p5) DRIVER_MODULE(p1, p2, p3, p4, p5) -#define __MODULE_VERSION(p1, p2) MODULE_VERSION(p1, p2) -#define __MODULE_DEPEND(p1, p2, p3, p4, p5) MODULE_DEPEND(p1, p2, p3, p4, p5) -__DRIVER_MODULE(TARGETNAME, pci, hpt_pci_driver, 0, 0); -__MODULE_VERSION(TARGETNAME, 1); -__MODULE_DEPEND(TARGETNAME, cam, 1, 1, 1); +DRIVER_MODULE(TARGETNAME, pci, hpt_pci_driver, NULL, NULL); +MODULE_VERSION(TARGETNAME, 1); +MODULE_DEPEND(TARGETNAME, cam, 1, 1, 1); static int hpt_open(struct cdev *dev, int flags, int devtype, struct thread *td) { diff --git a/sys/dev/hptrr/hptrr_osm_bsd.c b/sys/dev/hptrr/hptrr_osm_bsd.c index 78a051d54cf8..f4ae6732ea5d 100644 --- a/sys/dev/hptrr/hptrr_osm_bsd.c +++ b/sys/dev/hptrr/hptrr_osm_bsd.c @@ -1204,13 +1204,9 @@ static driver_t hpt_pci_driver = { #error "no TARGETNAME found" #endif -/* use this to make TARGETNAME be expanded */ -#define __DRIVER_MODULE(p1, p2, p3, p4, p5) DRIVER_MODULE(p1, p2, p3, p4, p5) -#define __MODULE_VERSION(p1, p2) MODULE_VERSION(p1, p2) -#define __MODULE_DEPEND(p1, p2, p3, p4, p5) MODULE_DEPEND(p1, p2, p3, p4, p5) -__DRIVER_MODULE(TARGETNAME, pci, hpt_pci_driver, 0, 0); -__MODULE_VERSION(TARGETNAME, 1); -__MODULE_DEPEND(TARGETNAME, cam, 1, 1, 1); +DRIVER_MODULE(TARGETNAME, pci, hpt_pci_driver, NULL, NULL); +MODULE_VERSION(TARGETNAME, 1); +MODULE_DEPEND(TARGETNAME, cam, 1, 1, 1); static int hpt_open(struct cdev *dev, int flags, int devtype, struct thread *td) { diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index d2fb17549990..801b75b39595 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -53,43 +53,14 @@ DPCPU_DEFINE_STATIC(uint32_t, nmi_counter); /* AMD K8 PMCs */ struct amd_descr { - struct pmc_descr pm_descr; /* "base class" */ - uint32_t pm_evsel; /* address of EVSEL register */ - uint32_t pm_perfctr; /* address of PERFCTR register */ + struct pmc_descr pm_descr; /* "base class" */ + uint32_t pm_evsel; /* address of EVSEL register */ + uint32_t pm_perfctr; /* address of PERFCTR register */ + enum sub_class pm_subclass; /* register subclass */ }; -/* Counter hardware. */ -#define PMCDESC(evsel, perfctr) \ - { \ - .pm_descr = { \ - .pd_name = "", \ - .pd_class = PMC_CLASS_K8, \ - .pd_caps = AMD_PMC_CAPS, \ - .pd_width = 48 \ - }, \ - .pm_evsel = (evsel), \ - .pm_perfctr = (perfctr) \ - } - -static struct amd_descr amd_pmcdesc[AMD_NPMCS] = -{ - PMCDESC(AMD_PMC_EVSEL_0, AMD_PMC_PERFCTR_0), - PMCDESC(AMD_PMC_EVSEL_1, AMD_PMC_PERFCTR_1), - PMCDESC(AMD_PMC_EVSEL_2, AMD_PMC_PERFCTR_2), - PMCDESC(AMD_PMC_EVSEL_3, AMD_PMC_PERFCTR_3), - PMCDESC(AMD_PMC_EVSEL_4, AMD_PMC_PERFCTR_4), - PMCDESC(AMD_PMC_EVSEL_5, AMD_PMC_PERFCTR_5), - PMCDESC(AMD_PMC_EVSEL_EP_L3_0, AMD_PMC_PERFCTR_EP_L3_0), - PMCDESC(AMD_PMC_EVSEL_EP_L3_1, AMD_PMC_PERFCTR_EP_L3_1), - PMCDESC(AMD_PMC_EVSEL_EP_L3_2, AMD_PMC_PERFCTR_EP_L3_2), - PMCDESC(AMD_PMC_EVSEL_EP_L3_3, AMD_PMC_PERFCTR_EP_L3_3), - PMCDESC(AMD_PMC_EVSEL_EP_L3_4, AMD_PMC_PERFCTR_EP_L3_4), - PMCDESC(AMD_PMC_EVSEL_EP_L3_5, AMD_PMC_PERFCTR_EP_L3_5), - PMCDESC(AMD_PMC_EVSEL_EP_DF_0, AMD_PMC_PERFCTR_EP_DF_0), - PMCDESC(AMD_PMC_EVSEL_EP_DF_1, AMD_PMC_PERFCTR_EP_DF_1), - PMCDESC(AMD_PMC_EVSEL_EP_DF_2, AMD_PMC_PERFCTR_EP_DF_2), - PMCDESC(AMD_PMC_EVSEL_EP_DF_3, AMD_PMC_PERFCTR_EP_DF_3) -}; +static int amd_npmcs; +static struct amd_descr amd_pmcdesc[AMD_NPMCS_MAX]; struct amd_event_code_map { enum pmc_event pe_ev; /* enum value */ @@ -203,7 +174,7 @@ const int amd_event_codes_size = nitems(amd_event_codes); * Per-processor information */ struct amd_cpu { - struct pmc_hw pc_amdpmcs[AMD_NPMCS]; + struct pmc_hw pc_amdpmcs[AMD_NPMCS_MAX]; }; static struct amd_cpu **amd_pcpu; @@ -219,7 +190,7 @@ amd_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] illegal row-index %d", __LINE__, ri)); KASSERT(amd_pcpu[cpu], ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); @@ -264,7 +235,7 @@ amd_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] illegal row-index %d", __LINE__, ri)); pd = &amd_pmcdesc[ri]; @@ -293,7 +264,7 @@ amd_config_pmc(int cpu, int ri, struct pmc *pm) KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; @@ -362,7 +333,7 @@ amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, enum pmc_event pe; int i; - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] illegal row index %d", __LINE__, ri)); pd = &amd_pmcdesc[ri].pm_descr; @@ -379,14 +350,7 @@ amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, PMCDBG2(MDP, ALL, 1,"amd-allocate ri=%d caps=0x%x", ri, caps); /* Validate sub-class. */ - if ((ri >= 0 && ri < 6) && a->pm_md.pm_amd.pm_amd_sub_class != - PMC_AMD_SUB_CLASS_CORE) - return (EINVAL); - if ((ri >= 6 && ri < 12) && a->pm_md.pm_amd.pm_amd_sub_class != - PMC_AMD_SUB_CLASS_L3_CACHE) - return (EINVAL); - if ((ri >= 12 && ri < 16) && a->pm_md.pm_amd.pm_amd_sub_class != - PMC_AMD_SUB_CLASS_DATA_FABRIC) + if (amd_pmcdesc[ri].pm_subclass != a->pm_md.pm_amd.pm_amd_sub_class) return (EINVAL); if (strlen(pmc_cpuid) != 0) { @@ -455,7 +419,7 @@ amd_release_pmc(int cpu, int ri, struct pmc *pmc __unused) KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; @@ -477,13 +441,18 @@ amd_start_pmc(int cpu __diagused, int ri, struct pmc *pm) KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] illegal row-index %d", __LINE__, ri)); pd = &amd_pmcdesc[ri]; PMCDBG2(MDP, STA, 1, "amd-start cpu=%d ri=%d", cpu, ri); + /* + * Triggered by DF counters because all DF MSRs are shared. We need to + * change the code to honor the per-package flag in the JSON event + * definitions. + */ KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, ri, cpu, pd->pm_descr.pd_name)); @@ -509,7 +478,7 @@ amd_stop_pmc(int cpu __diagused, int ri, struct pmc *pm) KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] illegal row-index %d", __LINE__, ri)); pd = &amd_pmcdesc[ri]; @@ -578,7 +547,10 @@ amd_intr(struct trapframe *tf) * a single interrupt. Check all the valid pmcs for * overflow. */ - for (i = 0; i < AMD_CORE_NPMCS; i++) { + for (i = 0; i < amd_npmcs; i++) { + if (amd_pmcdesc[i].pm_subclass != PMC_AMD_SUB_CLASS_CORE) + break; + if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { continue; @@ -654,7 +626,7 @@ amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU %d", __LINE__, cpu)); - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] row-index %d out of range", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; @@ -680,7 +652,7 @@ amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) static int amd_get_msr(int ri, uint32_t *msr) { - KASSERT(ri >= 0 && ri < AMD_NPMCS, + KASSERT(ri >= 0 && ri < amd_npmcs, ("[amd,%d] ri %d out of range", __LINE__, ri)); *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; @@ -715,7 +687,7 @@ amd_pcpu_init(struct pmc_mdep *md, int cpu) KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__)); - for (n = 0, phw = pac->pc_amdpmcs; n < AMD_NPMCS; n++, phw++) { + for (n = 0, phw = pac->pc_amdpmcs; n < amd_npmcs; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; @@ -733,7 +705,6 @@ amd_pcpu_fini(struct pmc_mdep *md, int cpu) { struct amd_cpu *pac; struct pmc_cpu *pc; - uint32_t evsel; int first_ri, i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), @@ -742,15 +713,6 @@ amd_pcpu_fini(struct pmc_mdep *md, int cpu) PMCDBG1(MDP, INI, 1, "amd-cleanup cpu=%d", cpu); /* - * First, turn off all PMCs on this CPU. - */ - for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ - evsel = rdmsr(AMD_PMC_EVSEL_0 + i); - evsel &= ~AMD_PMC_ENABLE; - wrmsr(AMD_PMC_EVSEL_0 + i, evsel); - } - - /* * Next, free up allocated space. */ if ((pac = amd_pcpu[cpu]) == NULL) @@ -759,7 +721,7 @@ amd_pcpu_fini(struct pmc_mdep *md, int cpu) amd_pcpu[cpu] = NULL; #ifdef HWPMC_DEBUG - for (i = 0; i < AMD_NPMCS; i++) { + for (i = 0; i < AMD_NPMCS_K8; i++) { KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL, ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i), @@ -775,7 +737,7 @@ amd_pcpu_fini(struct pmc_mdep *md, int cpu) /* * Reset pointers in the MI 'per-cpu' state. */ - for (i = 0; i < AMD_NPMCS; i++) + for (i = 0; i < amd_npmcs; i++) pc->pc_hwpmcs[i + first_ri] = NULL; free(pac, M_PMC); @@ -793,6 +755,8 @@ pmc_amd_initialize(void) enum pmc_cputype cputype; int error, i, ncpus; int family, model, stepping; + int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs; + struct amd_descr *d; /* * The presence of hardware performance counters on the AMD @@ -824,6 +788,80 @@ pmc_amd_initialize(void) } /* + * From PPR for AMD Family 1Ah, a new cpuid leaf specifies the maximum + * number of PMCs of each type. If we do not have that leaf, we use + * the prior default values that are only valid if we have the feature + * bit enabled in CPU. + */ + if ((amd_feature2 & AMDID2_PCXC) != 0) { + amd_core_npmcs = AMD_PMC_CORE_DEFAULT; + } else { + amd_core_npmcs = AMD_NPMCS_K8; + } + amd_l3_npmcs = AMD_PMC_L3_DEFAULT; + amd_df_npmcs = AMD_PMC_DF_DEFAULT; + + if (cpu_exthigh >= CPUID_EXTPERFMON) { + u_int regs[4]; + do_cpuid(CPUID_EXTPERFMON, regs); + if (regs[1] != 0) { + amd_core_npmcs = EXTPERFMON_CORE_PMCS(regs[1]); + amd_df_npmcs = EXTPERFMON_DF_PMCS(regs[1]); + } + } + + /* Enable the newer core counters */ + for (i = 0; i < amd_core_npmcs; i++) { + d = &amd_pmcdesc[i]; + snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, + "K8-%d", i); + d->pm_descr.pd_class = PMC_CLASS_K8; + d->pm_descr.pd_caps = AMD_PMC_CAPS; + d->pm_descr.pd_width = 48; + if ((amd_feature2 & AMDID2_PCXC) != 0) { + d->pm_evsel = AMD_PMC_CORE_BASE + 2 * i; + d->pm_perfctr = AMD_PMC_CORE_BASE + 2 * i + 1; + } else { + d->pm_evsel = AMD_PMC_EVSEL_0 + i; + d->pm_perfctr = AMD_PMC_PERFCTR_0 + i; + } + d->pm_subclass = PMC_AMD_SUB_CLASS_CORE; + } + amd_npmcs = amd_core_npmcs; + + if ((amd_feature2 & AMDID2_PTSCEL2I) != 0) { + /* Enable the LLC/L3 counters */ + for (i = 0; i < amd_l3_npmcs; i++) { + d = &amd_pmcdesc[amd_npmcs + i]; + snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, + "K8-L3-%d", i); + d->pm_descr.pd_class = PMC_CLASS_K8; + d->pm_descr.pd_caps = AMD_PMC_CAPS; + d->pm_descr.pd_width = 48; + d->pm_evsel = AMD_PMC_L3_BASE + 2 * i; + d->pm_perfctr = AMD_PMC_L3_BASE + 2 * i + 1; + d->pm_subclass = PMC_AMD_SUB_CLASS_L3_CACHE; + } + amd_npmcs += amd_l3_npmcs; + } + + if ((amd_feature2 & AMDID2_PNXC) != 0) { + /* Enable the data fabric counters */ + for (i = 0; i < amd_df_npmcs; i++) { + d = &amd_pmcdesc[amd_npmcs + i]; + snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, + "K8-DF-%d", i); + d->pm_descr.pd_class = PMC_CLASS_K8; + d->pm_descr.pd_caps = AMD_PMC_CAPS; + d->pm_descr.pd_width = 48; + d->pm_evsel = AMD_PMC_DF_BASE + 2 * i; + d->pm_perfctr = AMD_PMC_DF_BASE + 2 * i + 1; + d->pm_subclass = PMC_AMD_SUB_CLASS_DATA_FABRIC; + } + amd_npmcs += amd_df_npmcs; + } + + /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ @@ -848,16 +886,10 @@ pmc_amd_initialize(void) pcd->pcd_caps = AMD_PMC_CAPS; pcd->pcd_class = PMC_CLASS_K8; - pcd->pcd_num = AMD_NPMCS; + pcd->pcd_num = amd_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 48; - /* fill in the correct pmc name and class */ - for (i = 0; i < AMD_NPMCS; i++) { - snprintf(amd_pmcdesc[i].pm_descr.pd_name, PMC_NAME_MAX, "K8-%d", - i); - } - pcd->pcd_allocate_pmc = amd_allocate_pmc; pcd->pcd_config_pmc = amd_config_pmc; pcd->pcd_describe = amd_describe; @@ -876,7 +908,7 @@ pmc_amd_initialize(void) pmc_mdep->pmd_switch_in = amd_switch_in; pmc_mdep->pmd_switch_out = amd_switch_out; - pmc_mdep->pmd_npmc += AMD_NPMCS; + pmc_mdep->pmd_npmc += amd_npmcs; PMCDBG0(MDP, INI, 0, "amd-initialize"); diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h index d72fbc4878b6..be484a1111a2 100644 --- a/sys/dev/hwpmc/hwpmc_amd.h +++ b/sys/dev/hwpmc/hwpmc_amd.h @@ -31,8 +31,12 @@ #ifndef _DEV_HWPMC_AMD_H_ #define _DEV_HWPMC_AMD_H_ 1 -/* AMD K8 PMCs */ +/* CPUIDs */ +#define CPUID_EXTPERFMON 0x80000022 +#define EXTPERFMON_CORE_PMCS(x) ((x) & 0x0F) +#define EXTPERFMON_DF_PMCS(x) (((x) >> 10) & 0x3F) +/* AMD K8 PMCs */ #define AMD_PMC_EVSEL_0 0xC0010000 #define AMD_PMC_EVSEL_1 0xC0010001 #define AMD_PMC_EVSEL_2 0xC0010002 @@ -42,40 +46,35 @@ #define AMD_PMC_PERFCTR_1 0xC0010005 #define AMD_PMC_PERFCTR_2 0xC0010006 #define AMD_PMC_PERFCTR_3 0xC0010007 + +/* + * For older AMD processors we have hard coded the original four core counters. + * For newer processors we use the cpuid bits to setup the counter table. The + * counts below are the default number of registers assuming that you do not + * have CPUID leaf 0x80000022. The maximum number of counters is computed + * based on the available bits in the CPUID leaf and reserved MSR space. + * + * Refer to the PPRs for AMD Family 1Ah. + */ + /* CORE */ -#define AMD_PMC_EVSEL_4 0xC0010208 -#define AMD_PMC_EVSEL_5 0xC001020A +#define AMD_PMC_CORE_BASE 0xC0010200 +#define AMD_PMC_CORE_DEFAULT 6 +#define AMD_PMC_CORE_MAX 16 -#define AMD_PMC_PERFCTR_4 0xC0010209 -#define AMD_PMC_PERFCTR_5 0xC001020B /* L3 */ -#define AMD_PMC_EVSEL_EP_L3_0 0xC0010230 -#define AMD_PMC_EVSEL_EP_L3_1 0xC0010232 -#define AMD_PMC_EVSEL_EP_L3_2 0xC0010234 -#define AMD_PMC_EVSEL_EP_L3_3 0xC0010236 -#define AMD_PMC_EVSEL_EP_L3_4 0xC0010238 -#define AMD_PMC_EVSEL_EP_L3_5 0xC001023A - -#define AMD_PMC_PERFCTR_EP_L3_0 0xC0010231 -#define AMD_PMC_PERFCTR_EP_L3_1 0xC0010233 -#define AMD_PMC_PERFCTR_EP_L3_2 0xC0010235 -#define AMD_PMC_PERFCTR_EP_L3_3 0xC0010237 -#define AMD_PMC_PERFCTR_EP_L3_4 0xC0010239 -#define AMD_PMC_PERFCTR_EP_L3_5 0xC001023B -/* DF */ -#define AMD_PMC_EVSEL_EP_DF_0 0xC0010240 -#define AMD_PMC_EVSEL_EP_DF_1 0xC0010242 -#define AMD_PMC_EVSEL_EP_DF_2 0xC0010244 -#define AMD_PMC_EVSEL_EP_DF_3 0xC0010246 +#define AMD_PMC_L3_BASE 0xC0010230 +#define AMD_PMC_L3_DEFAULT 6 +#define AMD_PMC_L3_MAX 6 -#define AMD_PMC_PERFCTR_EP_DF_0 0xC0010241 -#define AMD_PMC_PERFCTR_EP_DF_1 0xC0010243 -#define AMD_PMC_PERFCTR_EP_DF_2 0xC0010245 -#define AMD_PMC_PERFCTR_EP_DF_3 0xC0010247 - -#define AMD_NPMCS 16 -#define AMD_CORE_NPMCS 6 +/* DF */ +#define AMD_PMC_DF_BASE 0xC0010240 +#define AMD_PMC_DF_DEFAULT 4 +#define AMD_PMC_DF_MAX 64 +#define AMD_NPMCS_K8 4 +#define AMD_NPMCS_MAX (AMD_PMC_CORE_MAX + AMD_PMC_L3_MAX + \ + AMD_PMC_DF_MAX) #define AMD_PMC_COUNTERMASK 0xFF000000 #define AMD_PMC_TO_COUNTER(x) (((x) << 24) & AMD_PMC_COUNTERMASK) @@ -118,7 +117,7 @@ #define AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(V) (-(V)) #define AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(P) (-(P)) -enum sub_class{ +enum sub_class { PMC_AMD_SUB_CLASS_CORE, PMC_AMD_SUB_CLASS_L3_CACHE, PMC_AMD_SUB_CLASS_DATA_FABRIC diff --git a/sys/dev/hwpmc/hwpmc_intel.c b/sys/dev/hwpmc/hwpmc_intel.c index 6d8b5c5de6b6..e1788a9ea409 100644 --- a/sys/dev/hwpmc/hwpmc_intel.c +++ b/sys/dev/hwpmc/hwpmc_intel.c @@ -247,6 +247,11 @@ pmc_intel_initialize(void) cputype = PMC_CPU_INTEL_ATOM_TREMONT; nclasses = 3; break; + case 0xAD: + case 0xAE: + cputype = PMC_CPU_INTEL_GRANITE_RAPIDS; + nclasses = 3; + break; case 0xBE: cputype = PMC_CPU_INTEL_ALDERLAKEN; nclasses = 3; diff --git a/sys/dev/ice/if_ice_iflib.c b/sys/dev/ice/if_ice_iflib.c index 1469d2916465..6ea539d52e02 100644 --- a/sys/dev/ice/if_ice_iflib.c +++ b/sys/dev/ice/if_ice_iflib.c @@ -1907,15 +1907,13 @@ ice_if_promisc_set(if_ctx_t ctx, int flags) ice_set_default_promisc_mask(promisc_mask); - if (multi_enable) - return (EOPNOTSUPP); - if (promisc_enable) { status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx, promisc_mask, 0); if (status && status != ICE_ERR_ALREADY_EXISTS) { device_printf(dev, - "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n", + "Failed to enable promiscuous mode for " + "PF VSI, err %s aq_err %s\n", ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); return (EIO); @@ -1925,11 +1923,28 @@ ice_if_promisc_set(if_ctx_t ctx, int flags) promisc_mask, 0); if (status) { device_printf(dev, - "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n", + "Failed to disable promiscuous mode for" + " PF VSI, err %s aq_err %s\n", ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); return (EIO); } + + if (multi_enable) { + ice_clear_bit(ICE_PROMISC_UCAST_TX, promisc_mask); + ice_clear_bit(ICE_PROMISC_UCAST_RX, promisc_mask); + status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx, + promisc_mask, 0); + if (status && status != ICE_ERR_ALREADY_EXISTS) { + device_printf(dev, + "Failed to enable allmulti mode " + "for PF VSI, err %s aq_err %s\n", + ice_status_str(status), + ice_aq_str( + hw->adminq.sq_last_status)); + return (EIO); + } + } } return (0); diff --git a/sys/dev/ichiic/ig4_pci.c b/sys/dev/ichiic/ig4_pci.c index 3a49e220e335..49036c6dabc9 100644 --- a/sys/dev/ichiic/ig4_pci.c +++ b/sys/dev/ichiic/ig4_pci.c @@ -192,6 +192,10 @@ static int ig4iic_pci_detach(device_t dev); #define PCI_CHIP_ARROWLAKE_U_I2C_3 0x777b8086 #define PCI_CHIP_ARROWLAKE_U_I2C_4 0x77508086 #define PCI_CHIP_ARROWLAKE_U_I2C_5 0x77518086 +#define PCI_CHIP_LUNARLAKE_M_I2C_0 0xa8788086 +#define PCI_CHIP_LUNARLAKE_M_I2C_1 0xa8798086 +#define PCI_CHIP_LUNARLAKE_M_I2C_2 0xa87a8086 +#define PCI_CHIP_LUNARLAKE_M_I2C_3 0xa87b8086 struct ig4iic_pci_device { uint32_t devid; @@ -328,6 +332,10 @@ static struct ig4iic_pci_device ig4iic_pci_devices[] = { { PCI_CHIP_ARROWLAKE_U_I2C_3, "Intel Arrow Lake-H/U I2C Controller-3", IG4_TIGERLAKE}, { PCI_CHIP_ARROWLAKE_U_I2C_4, "Intel Arrow Lake-H/U I2C Controller-4", IG4_TIGERLAKE}, { PCI_CHIP_ARROWLAKE_U_I2C_5, "Intel Arrow Lake-H/U I2C Controller-5", IG4_TIGERLAKE}, + { PCI_CHIP_LUNARLAKE_M_I2C_0, "Intel Lunar Lake-M I2C Controller-0", IG4_TIGERLAKE}, + { PCI_CHIP_LUNARLAKE_M_I2C_1, "Intel Lunar Lake-M I2C Controller-1", IG4_TIGERLAKE}, + { PCI_CHIP_LUNARLAKE_M_I2C_2, "Intel Lunar Lake-M I2C Controller-2", IG4_TIGERLAKE}, + { PCI_CHIP_LUNARLAKE_M_I2C_3, "Intel Lunar Lake-M I2C Controller-3", IG4_TIGERLAKE}, }; static int diff --git a/sys/dev/mpr/mpr_sas_lsi.c b/sys/dev/mpr/mpr_sas_lsi.c index 6529b38cc5f1..f88dd6e2532f 100644 --- a/sys/dev/mpr/mpr_sas_lsi.c +++ b/sys/dev/mpr/mpr_sas_lsi.c @@ -1516,7 +1516,6 @@ mprsas_SSU_to_SATA_devices(struct mpr_softc *sc, int howto) /*start*/FALSE, /*load/eject*/0, /*immediate*/FALSE, - /*power_condition*/SSS_PC_START_VALID, MPR_SENSE_LEN, /*timeout*/10000); xpt_action(ccb); diff --git a/sys/dev/mps/mps_sas_lsi.c b/sys/dev/mps/mps_sas_lsi.c index c48c8e15d4a4..bcde5d69a021 100644 --- a/sys/dev/mps/mps_sas_lsi.c +++ b/sys/dev/mps/mps_sas_lsi.c @@ -1152,7 +1152,6 @@ mpssas_SSU_to_SATA_devices(struct mps_softc *sc, int howto) /*start*/FALSE, /*load/eject*/0, /*immediate*/FALSE, - /*power_condition*/SSS_PC_START_VALID, MPS_SENSE_LEN, /*timeout*/10000); xpt_action(ccb); diff --git a/sys/dev/rge/if_rge.c b/sys/dev/rge/if_rge.c index fde8e9df31fe..443acbbd77c3 100644 --- a/sys/dev/rge/if_rge.c +++ b/sys/dev/rge/if_rge.c @@ -423,8 +423,12 @@ rge_attach(device_t dev) // device_printf(dev, "RTL8126_2\n"); break; case 0x68800000: - sc->rge_type = MAC_R25D; -// device_printf(dev, "RTL8125D\n"); + sc->rge_type = MAC_R25D_1; +// device_printf(dev, "RTL8125D_1\n"); + break; + case 0x68900000: + sc->rge_type = MAC_R25D_2; +// device_printf(dev, "RTL8125D_2\n"); break; case 0x6c900000: sc->rge_type = MAC_R27; @@ -1159,7 +1163,7 @@ rge_init_locked(struct rge_softc *sc) rxconf = RGE_RXCFG_CONFIG; else if (sc->rge_type == MAC_R25B) rxconf = RGE_RXCFG_CONFIG_8125B; - else if (sc->rge_type == MAC_R25D) + else if (RGE_TYPE_R25D(sc)) rxconf = RGE_RXCFG_CONFIG_8125D; else rxconf = RGE_RXCFG_CONFIG_8126; @@ -1173,7 +1177,7 @@ rge_init_locked(struct rge_softc *sc) /* Disable L1 timeout. */ val = rge_read_csi(sc, 0x890) & ~0x00000001; rge_write_csi(sc, 0x890, val); - } else if (sc->rge_type != MAC_R25D) + } else if (!RGE_TYPE_R25D(sc)) RGE_WRITE_2(sc, 0x0382, 0x221b); RGE_WRITE_1(sc, RGE_RSS_CTRL, 0); @@ -1199,7 +1203,7 @@ rge_init_locked(struct rge_softc *sc) val = rge_read_mac_ocp(sc, 0xe614); val &= (sc->rge_type == MAC_R27) ? ~0x0f00 : ~0x0700; - if (sc->rge_type == MAC_R25 || sc->rge_type == MAC_R25D) + if (sc->rge_type == MAC_R25 || RGE_TYPE_R25D(sc)) rge_write_mac_ocp(sc, 0xe614, val | 0x0300); else if (sc->rge_type == MAC_R25B) rge_write_mac_ocp(sc, 0xe614, val | 0x0200); @@ -1235,7 +1239,7 @@ rge_init_locked(struct rge_softc *sc) val = rge_read_mac_ocp(sc, 0xea1c) & ~0x0003; rge_write_mac_ocp(sc, 0xea1c, val | 0x0001); - if (sc->rge_type == MAC_R25D) + if (RGE_TYPE_R25D(sc)) rge_write_mac_ocp(sc, 0xe0c0, 0x4403); else rge_write_mac_ocp(sc, 0xe0c0, 0x4000); @@ -1251,7 +1255,7 @@ rge_init_locked(struct rge_softc *sc) if (sc->rge_type == MAC_R25) RGE_SETBIT_1(sc, RGE_MCUCMD, 0x01); - if (sc->rge_type != MAC_R25D) { + if (!RGE_TYPE_R25D(sc)) { /* Disable EEE plus. */ RGE_MAC_CLRBIT(sc, 0xe080, 0x0002); } @@ -1306,7 +1310,7 @@ rge_init_locked(struct rge_softc *sc) val = rge_read_csi(sc, 0x98) & ~0x0000ff00; rge_write_csi(sc, 0x98, val); - if (sc->rge_type == MAC_R25D) { + if (RGE_TYPE_R25D(sc)) { val = rge_read_mac_ocp(sc, 0xe092) & ~0x00ff; rge_write_mac_ocp(sc, 0xe092, val | 0x0008); } else diff --git a/sys/dev/rge/if_rge_hw.c b/sys/dev/rge/if_rge_hw.c index ccd6580b2cfc..35a0e93dd193 100644 --- a/sys/dev/rge/if_rge_hw.c +++ b/sys/dev/rge/if_rge_hw.c @@ -78,7 +78,8 @@ static void rge_phy_config_mac_r26_1(struct rge_softc *sc); static void rge_phy_config_mac_r26_2(struct rge_softc *sc); static void rge_phy_config_mac_r25(struct rge_softc *sc); static void rge_phy_config_mac_r25b(struct rge_softc *sc); -static void rge_phy_config_mac_r25d(struct rge_softc *sc); +static void rge_phy_config_mac_r25d_1(struct rge_softc *); +static void rge_phy_config_mac_r25d_2(struct rge_softc *); static void rge_phy_config_mcu(struct rge_softc *sc, uint16_t rcodever); static void rge_hw_init(struct rge_softc *sc); static void rge_disable_phy_ocp_pwrsave(struct rge_softc *sc); @@ -230,7 +231,7 @@ rge_mac_config_mcu(struct rge_softc *sc, enum rge_mac_type type) rge_write_mac_ocp(sc, rtl8125b_mac_bps[i].reg, rtl8125b_mac_bps[i].val); } - } else if (type == MAC_R25D) { + } else if (type == MAC_R25D_1) { for (npages = 0; npages < 3; npages++) { rge_switch_mcu_ram_page(sc, npages); @@ -268,6 +269,35 @@ rge_mac_config_mcu(struct rge_softc *sc, enum rge_mac_type type) rge_write_mac_ocp(sc, 0xfc26, 0x8000); rge_write_mac_ocp(sc, 0xfc28, 0x14a2); rge_write_mac_ocp(sc, 0xfc48, 0x0001); + } else if (type == MAC_R25D_2) { + for (npages = 0; npages < 3; npages++) { + rge_switch_mcu_ram_page(sc, npages); + + for (i = 0; i < nitems(rtl8125d_2_mac_bps); i++) { + if (npages == 0) + rge_write_mac_ocp(sc, + rtl8125d_2_mac_bps[i].reg, + rtl8125d_2_mac_bps[i].val); + else + rge_write_mac_ocp(sc, + rtl8125d_2_mac_bps[i].reg, 0); + } + + for (reg = 0xf884; reg <= 0xf9f6; reg += 2) + rge_write_mac_ocp(sc, reg, 0); + + rge_write_mac_ocp(sc, 0xf9f8, + (npages == 2) ? 0x6938 : 0); + rge_write_mac_ocp(sc, 0xf9fa, + (npages == 2) ? 0x0a19: 0); + rge_write_mac_ocp(sc, 0xf9fc, + (npages == 2) ? 0x030e: 0); + rge_write_mac_ocp(sc, 0xf9fe, + (npages == 2) ? 0x0b2f: 0); + } + rge_write_mac_ocp(sc, 0xfc26, 0x8000); + rge_write_mac_ocp(sc, 0xfc28, 0x2382); + rge_write_mac_ocp(sc, 0xfc48, 0x0001); } } @@ -300,7 +330,7 @@ rge_mac_config_ext_mcu(struct rge_softc *sc, enum rge_mac_type type) } rge_write_mac_ocp(sc, 0xfc26, 0x8000); rge_write_mac_ocp(sc, 0xfc2c, 0x2360); - rge_write_mac_ocp(sc, 0xfc2E, 0x14a4); + rge_write_mac_ocp(sc, 0xfc2e, 0x14a4); rge_write_mac_ocp(sc, 0xfc30, 0x415e); rge_write_mac_ocp(sc, 0xfc32, 0x41e4); rge_write_mac_ocp(sc, 0xfc34, 0x4280); @@ -508,8 +538,11 @@ rge_phy_config(struct rge_softc *sc) case MAC_R25B: rge_phy_config_mac_r25b(sc); break; - case MAC_R25D: - rge_phy_config_mac_r25d(sc); + case MAC_R25D_1: + rge_phy_config_mac_r25d_1(sc); + break; + case MAC_R25D_2: + rge_phy_config_mac_r25d_2(sc); break; case MAC_R26_1: rge_phy_config_mac_r26_1(sc); @@ -531,7 +564,7 @@ rge_phy_config(struct rge_softc *sc) if (sc->rge_type == MAC_R25) { RGE_MAC_CLRBIT(sc, 0xeb62, 0x0006); RGE_PHY_CLRBIT(sc, 0xa432, 0x0010); - } else if (sc->rge_type == MAC_R25B || sc->rge_type == MAC_R25D) + } else if (sc->rge_type == MAC_R25B || RGE_TYPE_R25D(sc)) RGE_PHY_SETBIT(sc, 0xa432, 0x0010); RGE_PHY_CLRBIT(sc, 0xa5d0, (sc->rge_type == MAC_R27) ? 0x000e : 0x0006); @@ -1363,12 +1396,12 @@ rge_phy_config_mac_r25b(struct rge_softc *sc) } static void -rge_phy_config_mac_r25d(struct rge_softc *sc) +rge_phy_config_mac_r25d_1(struct rge_softc *sc) { uint16_t val; int i; - rge_phy_config_mcu(sc, RGE_MAC_R25D_RCODE_VER); + rge_phy_config_mcu(sc, RGE_MAC_R25D_1_RCODE_VER); RGE_PHY_SETBIT(sc, 0xa442, 0x0800); @@ -1556,6 +1589,67 @@ rge_phy_config_mac_r25d(struct rge_softc *sc) } static void +rge_phy_config_mac_r25d_2(struct rge_softc *sc) +{ + uint16_t val; + + rge_phy_config_mcu(sc, RGE_MAC_R25D_2_RCODE_VER); + + RGE_PHY_SETBIT(sc, 0xa442, 0x0800); + + rge_patch_phy_mcu(sc, 1); + val = rge_read_phy_ocp(sc, 0xbcd8) & ~0xc000; + rge_write_phy_ocp(sc, 0xbcd8, val | 0x4000); + RGE_PHY_SETBIT(sc, 0xbcd8, 0xc000); + val = rge_read_phy_ocp(sc, 0xbcd8) & ~0xc000; + rge_write_phy_ocp(sc, 0xbcd8, val | 0x4000); + val = rge_read_phy_ocp(sc, 0xbc80) & ~0x001f; + rge_write_phy_ocp(sc, 0xbc80, val | 0x0004); + RGE_PHY_SETBIT(sc, 0xbc82, 0xe000); + RGE_PHY_SETBIT(sc, 0xbc82, 0x1c00); + val = rge_read_phy_ocp(sc, 0xbc80) & ~0x001f; + rge_write_phy_ocp(sc, 0xbc80, val | 0x0005); + val = rge_read_phy_ocp(sc, 0xbc82) & ~0x00e0; + rge_write_phy_ocp(sc, 0xbc82, val | 0x0040); + RGE_PHY_SETBIT(sc, 0xbc82, 0x001c); + RGE_PHY_CLRBIT(sc, 0xbcd8, 0xc000); + val = rge_read_phy_ocp(sc, 0xbcd8) & ~0xc000; + rge_write_phy_ocp(sc, 0xbcd8, val | 0x8000); + RGE_PHY_CLRBIT(sc, 0xbcd8, 0xc000); + rge_patch_phy_mcu(sc, 0); + + val = rge_read_phy_ocp(sc, 0xac7e) & ~0x01fc; + rge_write_phy_ocp(sc, 0xac7e, val | 0x00b4); + rge_write_phy_ocp(sc, 0xb87c, 0x8105); + val = rge_read_phy_ocp(sc, 0xb87e) & ~0xff00; + rge_write_phy_ocp(sc, 0xb87e, val | 0x7a00); + rge_write_phy_ocp(sc, 0xb87c, 0x8117); + val = rge_read_phy_ocp(sc, 0xb87e) & ~0xff00; + rge_write_phy_ocp(sc, 0xb87e, val | 0x3a00); + rge_write_phy_ocp(sc, 0xb87c, 0x8103); + val = rge_read_phy_ocp(sc, 0xb87e) & ~0xff00; + rge_write_phy_ocp(sc, 0xb87e, val | 0x7400); + rge_write_phy_ocp(sc, 0xb87c, 0x8115); + val = rge_read_phy_ocp(sc, 0xb87e) & ~0xff00; + rge_write_phy_ocp(sc, 0xb87e, val | 0x3400); + rge_write_phy_ocp(sc, 0xb87c, 0x8feb); + val = rge_read_phy_ocp(sc, 0xb87e) & ~0xff00; + rge_write_phy_ocp(sc, 0xb87e, val | 0x0500); + rge_write_phy_ocp(sc, 0xb87c, 0x8fea); + val = rge_read_phy_ocp(sc, 0xb87e) & ~0xff00; + rge_write_phy_ocp(sc, 0xb87e, val | 0x0700); + rge_write_phy_ocp(sc, 0xb87c, 0x80d6); + val = rge_read_phy_ocp(sc, 0xb87e) & ~0xff00; + rge_write_phy_ocp(sc, 0xb87e, val | 0xef00); + RGE_PHY_CLRBIT(sc, 0xa5d4, 0x0020); + RGE_PHY_CLRBIT(sc, 0xa654, 0x0800); + RGE_PHY_CLRBIT(sc, 0xa448, 0x0400); + RGE_PHY_CLRBIT(sc, 0xa586, 0x0400); + RGE_PHY_SETBIT(sc, 0xa430, 0x1001); + RGE_PHY_SETBIT(sc, 0xa442, 0x0080); +} + +static void rge_phy_config_mcu(struct rge_softc *sc, uint16_t rcodever) { if (sc->rge_rcodever != rcodever) { @@ -1586,22 +1680,37 @@ rge_phy_config_mcu(struct rge_softc *sc, uint16_t rcodever) for (i = 0; i < nitems(mac_r25b_mcu); i++) rge_write_phy_ocp(sc, mac_r25b_mcu[i].reg, mac_r25b_mcu[i].val); - } else if (sc->rge_type == MAC_R25D) { + } else if (sc->rge_type == MAC_R25D_1) { for (i = 0; i < 2403; i++) rge_write_phy_ocp(sc, - mac_r25d_mcu[i].reg, mac_r25d_mcu[i].val); + mac_r25d_1_mcu[i].reg, + mac_r25d_1_mcu[i].val); rge_patch_phy_mcu(sc, 0); rge_patch_phy_mcu(sc, 1); for (; i < 2528; i++) rge_write_phy_ocp(sc, - mac_r25d_mcu[i].reg, mac_r25d_mcu[i].val); + mac_r25d_1_mcu[i].reg, + mac_r25d_1_mcu[i].val); + rge_patch_phy_mcu(sc, 0); + + rge_patch_phy_mcu(sc, 1); + for (; i < nitems(mac_r25d_1_mcu); i++) + rge_write_phy_ocp(sc, + mac_r25d_1_mcu[i].reg, + mac_r25d_1_mcu[i].val); + } else if (sc->rge_type == MAC_R25D_2) { + for (i = 0; i < 1269; i++) + rge_write_phy_ocp(sc, + mac_r25d_2_mcu[i].reg, + mac_r25d_2_mcu[i].val); rge_patch_phy_mcu(sc, 0); rge_patch_phy_mcu(sc, 1); - for (; i < nitems(mac_r25d_mcu); i++) + for (; i < nitems(mac_r25d_2_mcu); i++) rge_write_phy_ocp(sc, - mac_r25d_mcu[i].reg, mac_r25d_mcu[i].val); + mac_r25d_2_mcu[i].reg, + mac_r25d_2_mcu[i].val); } else if (sc->rge_type == MAC_R26_1) { for (i = 0; i < 6989; i++) rge_write_phy_ocp(sc, diff --git a/sys/dev/rge/if_rge_microcode.h b/sys/dev/rge/if_rge_microcode.h index f2803b869e3f..15f07099f80e 100644 --- a/sys/dev/rge/if_rge_microcode.h +++ b/sys/dev/rge/if_rge_microcode.h @@ -166,6 +166,31 @@ static const struct rge_hw_regaddr_array rtl8125b_mac_bps[] = { { 0xfc30, 0x4a20 }, { 0xfc32, 0x47a0 }, { 0xfc48, 0x003f } }; +static const struct rge_hw_regaddr_array rtl8125d_2_mac_bps[] = { + { 0xf800, 0xe010 }, { 0xf802, 0xe014 }, { 0xf804, 0xe016 }, + { 0xf806, 0xe018 }, { 0xf808, 0xe01a }, { 0xf80a, 0xe01c }, + { 0xf80c, 0xe01e }, { 0xf80e, 0xe020 }, { 0xf810, 0xe022 }, + { 0xf812, 0xe024 }, { 0xf814, 0xe026 }, { 0xf816, 0xe028 }, + { 0xf818, 0xe02a }, { 0xf81a, 0xe02c }, { 0xf81c, 0xe02e }, + { 0xf81e, 0xe030 }, { 0xf820, 0xc104 }, { 0xf822, 0xc202 }, + { 0xf824, 0xba00 }, { 0xf826, 0x2384 }, { 0xf828, 0xd116 }, + { 0xf82a, 0xc602 }, { 0xf82c, 0xbe00 }, { 0xf82e, 0x0000 }, + { 0xf830, 0xc602 }, { 0xf832, 0xbe00 }, { 0xf834, 0x0000 }, + { 0xf836, 0xc602 }, { 0xf838, 0xbe00 }, { 0xf83a, 0x0000 }, + { 0xf83c, 0xc602 }, { 0xf83e, 0xbe00 }, { 0xf840, 0x0000 }, + { 0xf842, 0xc602 }, { 0xf844, 0xbe00 }, { 0xf846, 0x0000 }, + { 0xf848, 0xc602 }, { 0xf84a, 0xbe00 }, { 0xf84c, 0x0000 }, + { 0xf84e, 0xc602 }, { 0xf850, 0xbe00 }, { 0xf852, 0x0000 }, + { 0xf854, 0xc602 }, { 0xf856, 0xbe00 }, { 0xf858, 0x0000 }, + { 0xf85a, 0xc602 }, { 0xf85c, 0xbe00 }, { 0xf85e, 0x0000 }, + { 0xf860, 0xc602 }, { 0xf862, 0xbe00 }, { 0xf864, 0x0000 }, + { 0xf866, 0xc602 }, { 0xf868, 0xbe00 }, { 0xf86a, 0x0000 }, + { 0xf86c, 0xc602 }, { 0xf86e, 0xbe00 }, { 0xf870, 0x0000 }, + { 0xf872, 0xc602 }, { 0xf874, 0xbe00 }, { 0xf876, 0x0000 }, + { 0xf878, 0xc602 }, { 0xf87a, 0xbe00 }, { 0xf87c, 0x0000 }, + { 0xf87e, 0xc602 }, { 0xf880, 0xbe00 }, { 0xf882, 0x0000 } +}; + struct rge_mac_bps { int count; const uint16_t *regs; @@ -489,6 +514,615 @@ static const struct rge_hw_regaddr_array mac_r27_ephy[] = { { 0x941c, 0x0190 }, { 0x981c, 0x0140 }, { 0x9c1c, 0x0140 } }; +static const struct rge_hw_regaddr_array mac_r25d_2_mcu[] = { + { 0xa436, 0x8023 }, { 0xa438, 0x3801 }, { 0xa436, 0xb82e }, + { 0xa438, 0x0001 }, { 0xb820, 0x0090 }, { 0xa436, 0xa016 }, + { 0xa438, 0x0000 }, { 0xa436, 0xa012 }, { 0xa438, 0x0000 }, + { 0xa436, 0xa014 }, { 0xa438, 0x1800 }, { 0xa438, 0x8010 }, + { 0xa438, 0x1800 }, { 0xa438, 0x808e }, { 0xa438, 0x1800 }, + { 0xa438, 0x80d6 }, { 0xa438, 0x1800 }, { 0xa438, 0x81e2 }, + { 0xa438, 0x1800 }, { 0xa438, 0x81e2 }, { 0xa438, 0x1800 }, + { 0xa438, 0x81e2 }, { 0xa438, 0x1800 }, { 0xa438, 0x81e2 }, + { 0xa438, 0x1800 }, { 0xa438, 0x81e2 }, { 0xa438, 0xd500 }, + { 0xa438, 0xc48d }, { 0xa438, 0xd504 }, { 0xa438, 0x8d03 }, + { 0xa438, 0xd701 }, { 0xa438, 0x4045 }, { 0xa438, 0xad02 }, + { 0xa438, 0xd504 }, { 0xa438, 0xd706 }, { 0xa438, 0x2529 }, + { 0xa438, 0x8021 }, { 0xa438, 0xd718 }, { 0xa438, 0x607b }, + { 0xa438, 0x40da }, { 0xa438, 0xf019 }, { 0xa438, 0x459a }, + { 0xa438, 0xf03f }, { 0xa438, 0xd718 }, { 0xa438, 0x62bb }, + { 0xa438, 0xbb01 }, { 0xa438, 0xd75e }, { 0xa438, 0x6231 }, + { 0xa438, 0x0cf0 }, { 0xa438, 0x0c10 }, { 0xa438, 0xd501 }, + { 0xa438, 0xce01 }, { 0xa438, 0xd70c }, { 0xa438, 0x6147 }, + { 0xa438, 0x8480 }, { 0xa438, 0x8440 }, { 0xa438, 0x8420 }, + { 0xa438, 0xa410 }, { 0xa438, 0xce00 }, { 0xa438, 0xd505 }, + { 0xa438, 0x0c0f }, { 0xa438, 0x0808 }, { 0xa438, 0xf002 }, + { 0xa438, 0xa4f0 }, { 0xa438, 0xf03c }, { 0xa438, 0xbb02 }, + { 0xa438, 0xd75e }, { 0xa438, 0x6231 }, { 0xa438, 0x0cf0 }, + { 0xa438, 0x0c20 }, { 0xa438, 0xd501 }, { 0xa438, 0xce01 }, + { 0xa438, 0xd70c }, { 0xa438, 0x6147 }, { 0xa438, 0x8480 }, + { 0xa438, 0x8440 }, { 0xa438, 0xa420 }, { 0xa438, 0x8410 }, + { 0xa438, 0xce00 }, { 0xa438, 0xd505 }, { 0xa438, 0x0c0f }, + { 0xa438, 0x0804 }, { 0xa438, 0xf002 }, { 0xa438, 0xa4f0 }, + { 0xa438, 0xf028 }, { 0xa438, 0xbb04 }, { 0xa438, 0xd75e }, + { 0xa438, 0x6231 }, { 0xa438, 0x0cf0 }, { 0xa438, 0x0c40 }, + { 0xa438, 0xd501 }, { 0xa438, 0xce01 }, { 0xa438, 0xd70c }, + { 0xa438, 0x6147 }, { 0xa438, 0x8480 }, { 0xa438, 0xa440 }, + { 0xa438, 0x8420 }, { 0xa438, 0x8410 }, { 0xa438, 0xce00 }, + { 0xa438, 0xd505 }, { 0xa438, 0x0c0f }, { 0xa438, 0x0802 }, + { 0xa438, 0xf002 }, { 0xa438, 0xa4f0 }, { 0xa438, 0xf014 }, + { 0xa438, 0xbb08 }, { 0xa438, 0xd75e }, { 0xa438, 0x6231 }, + { 0xa438, 0x0cf0 }, { 0xa438, 0x0c80 }, { 0xa438, 0xd501 }, + { 0xa438, 0xce01 }, { 0xa438, 0xd70c }, { 0xa438, 0x6147 }, + { 0xa438, 0xa480 }, { 0xa438, 0x8440 }, { 0xa438, 0x8420 }, + { 0xa438, 0x8410 }, { 0xa438, 0xce00 }, { 0xa438, 0xd505 }, + { 0xa438, 0x0c0f }, { 0xa438, 0x0801 }, { 0xa438, 0xf002 }, + { 0xa438, 0xa4f0 }, { 0xa438, 0xce00 }, { 0xa438, 0xd500 }, + { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, { 0xa438, 0x1000 }, + { 0xa438, 0x1829 }, { 0xa438, 0xd73e }, { 0xa438, 0x6074 }, + { 0xa438, 0xd718 }, { 0xa438, 0x5f2d }, { 0xa438, 0x1000 }, + { 0xa438, 0x81b7 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, + { 0xa438, 0x1000 }, { 0xa438, 0x1829 }, { 0xa438, 0xd73e }, + { 0xa438, 0x7f74 }, { 0xa438, 0x1000 }, { 0xa438, 0x81ce }, + { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, { 0xa438, 0x1000 }, + { 0xa438, 0x1829 }, { 0xa438, 0xd718 }, { 0xa438, 0x5f6d }, + { 0xa438, 0x1800 }, { 0xa438, 0x1660 }, { 0xa438, 0xd75e }, + { 0xa438, 0x68b1 }, { 0xa438, 0xd504 }, { 0xa438, 0xd71e }, + { 0xa438, 0x667b }, { 0xa438, 0x645a }, { 0xa438, 0x6239 }, + { 0xa438, 0x0cf0 }, { 0xa438, 0x0c10 }, { 0xa438, 0xd505 }, + { 0xa438, 0x0c0f }, { 0xa438, 0x0808 }, { 0xa438, 0xd501 }, + { 0xa438, 0xce01 }, { 0xa438, 0xd70c }, { 0xa438, 0x60c7 }, + { 0xa438, 0x8480 }, { 0xa438, 0x8440 }, { 0xa438, 0x8420 }, + { 0xa438, 0xa410 }, { 0xa438, 0xf032 }, { 0xa438, 0xa4f0 }, + { 0xa438, 0xf030 }, { 0xa438, 0x0cf0 }, { 0xa438, 0x0c20 }, + { 0xa438, 0xd505 }, { 0xa438, 0x0c0f }, { 0xa438, 0x0804 }, + { 0xa438, 0xd501 }, { 0xa438, 0xce01 }, { 0xa438, 0xd70c }, + { 0xa438, 0x60c7 }, { 0xa438, 0x8480 }, { 0xa438, 0x8440 }, + { 0xa438, 0xa420 }, { 0xa438, 0x8410 }, { 0xa438, 0xf022 }, + { 0xa438, 0xa4f0 }, { 0xa438, 0xf020 }, { 0xa438, 0x0cf0 }, + { 0xa438, 0x0c40 }, { 0xa438, 0xd505 }, { 0xa438, 0x0c0f }, + { 0xa438, 0x0802 }, { 0xa438, 0xd501 }, { 0xa438, 0xce01 }, + { 0xa438, 0xd70c }, { 0xa438, 0x60c7 }, { 0xa438, 0x8480 }, + { 0xa438, 0xa440 }, { 0xa438, 0x8420 }, { 0xa438, 0x8410 }, + { 0xa438, 0xf012 }, { 0xa438, 0xa4f0 }, { 0xa438, 0xf010 }, + { 0xa438, 0x0cf0 }, { 0xa438, 0x0c80 }, { 0xa438, 0xd505 }, + { 0xa438, 0x0c0f }, { 0xa438, 0x0801 }, { 0xa438, 0xd501 }, + { 0xa438, 0xce01 }, { 0xa438, 0xd70c }, { 0xa438, 0x60c7 }, + { 0xa438, 0xa480 }, { 0xa438, 0x8440 }, { 0xa438, 0x8420 }, + { 0xa438, 0x8410 }, { 0xa438, 0xf002 }, { 0xa438, 0xa4f0 }, + { 0xa438, 0x1800 }, { 0xa438, 0x168c }, { 0xa438, 0xd500 }, + { 0xa438, 0xd706 }, { 0xa438, 0x2529 }, { 0xa438, 0x80e0 }, + { 0xa438, 0xd718 }, { 0xa438, 0x607b }, { 0xa438, 0x40da }, + { 0xa438, 0xf00f }, { 0xa438, 0x431a }, { 0xa438, 0xf021 }, + { 0xa438, 0xd718 }, { 0xa438, 0x617b }, { 0xa438, 0x1000 }, + { 0xa438, 0x1a8a }, { 0xa438, 0x1000 }, { 0xa438, 0x1b1a }, + { 0xa438, 0xd718 }, { 0xa438, 0x608e }, { 0xa438, 0xd73e }, + { 0xa438, 0x5f34 }, { 0xa438, 0xf020 }, { 0xa438, 0xf053 }, + { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, { 0xa438, 0x1000 }, + { 0xa438, 0x1b1a }, { 0xa438, 0xd718 }, { 0xa438, 0x608e }, + { 0xa438, 0xd73e }, { 0xa438, 0x5f34 }, { 0xa438, 0xf023 }, + { 0xa438, 0xf067 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, + { 0xa438, 0x1000 }, { 0xa438, 0x1b1a }, { 0xa438, 0xd718 }, + { 0xa438, 0x608e }, { 0xa438, 0xd73e }, { 0xa438, 0x5f34 }, + { 0xa438, 0xf026 }, { 0xa438, 0xf07b }, { 0xa438, 0x1000 }, + { 0xa438, 0x1a8a }, { 0xa438, 0x1000 }, { 0xa438, 0x1b1a }, + { 0xa438, 0xd718 }, { 0xa438, 0x608e }, { 0xa438, 0xd73e }, + { 0xa438, 0x5f34 }, { 0xa438, 0xf029 }, { 0xa438, 0xf08f }, + { 0xa438, 0x1000 }, { 0xa438, 0x81b7 }, { 0xa438, 0x1000 }, + { 0xa438, 0x1a8a }, { 0xa438, 0xd73e }, { 0xa438, 0x7fb4 }, + { 0xa438, 0x1000 }, { 0xa438, 0x81ce }, { 0xa438, 0x1000 }, + { 0xa438, 0x1a8a }, { 0xa438, 0xd718 }, { 0xa438, 0x5fae }, + { 0xa438, 0xf028 }, { 0xa438, 0x1000 }, { 0xa438, 0x81b7 }, + { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, { 0xa438, 0xd73e }, + { 0xa438, 0x7fb4 }, { 0xa438, 0x1000 }, { 0xa438, 0x81ce }, + { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, { 0xa438, 0xd718 }, + { 0xa438, 0x5fae }, { 0xa438, 0xf039 }, { 0xa438, 0x1000 }, + { 0xa438, 0x81b7 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, + { 0xa438, 0xd73e }, { 0xa438, 0x7fb4 }, { 0xa438, 0x1000 }, + { 0xa438, 0x81ce }, { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, + { 0xa438, 0xd718 }, { 0xa438, 0x5fae }, { 0xa438, 0xf04a }, + { 0xa438, 0x1000 }, { 0xa438, 0x81b7 }, { 0xa438, 0x1000 }, + { 0xa438, 0x1a8a }, { 0xa438, 0xd73e }, { 0xa438, 0x7fb4 }, + { 0xa438, 0x1000 }, { 0xa438, 0x81ce }, { 0xa438, 0x1000 }, + { 0xa438, 0x1a8a }, { 0xa438, 0xd718 }, { 0xa438, 0x5fae }, + { 0xa438, 0xf05b }, { 0xa438, 0xd719 }, { 0xa438, 0x4119 }, + { 0xa438, 0xd504 }, { 0xa438, 0xac01 }, { 0xa438, 0xae01 }, + { 0xa438, 0xd500 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a78 }, + { 0xa438, 0xf00a }, { 0xa438, 0xd719 }, { 0xa438, 0x4118 }, + { 0xa438, 0xd504 }, { 0xa438, 0xac11 }, { 0xa438, 0xd501 }, + { 0xa438, 0xce01 }, { 0xa438, 0xa410 }, { 0xa438, 0xce00 }, + { 0xa438, 0xd500 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, + { 0xa438, 0xd718 }, { 0xa438, 0x5fb0 }, { 0xa438, 0xd505 }, + { 0xa438, 0xd719 }, { 0xa438, 0x4079 }, { 0xa438, 0xa80f }, + { 0xa438, 0xf05d }, { 0xa438, 0x4b98 }, { 0xa438, 0xa808 }, + { 0xa438, 0xf05a }, { 0xa438, 0xd719 }, { 0xa438, 0x4119 }, + { 0xa438, 0xd504 }, { 0xa438, 0xac02 }, { 0xa438, 0xae01 }, + { 0xa438, 0xd500 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a78 }, + { 0xa438, 0xf00a }, { 0xa438, 0xd719 }, { 0xa438, 0x4118 }, + { 0xa438, 0xd504 }, { 0xa438, 0xac22 }, { 0xa438, 0xd501 }, + { 0xa438, 0xce01 }, { 0xa438, 0xa420 }, { 0xa438, 0xce00 }, + { 0xa438, 0xd500 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, + { 0xa438, 0xd718 }, { 0xa438, 0x5fb0 }, { 0xa438, 0xd505 }, + { 0xa438, 0xd719 }, { 0xa438, 0x4079 }, { 0xa438, 0xa80f }, + { 0xa438, 0xf03f }, { 0xa438, 0x47d8 }, { 0xa438, 0xa804 }, + { 0xa438, 0xf03c }, { 0xa438, 0xd719 }, { 0xa438, 0x4119 }, + { 0xa438, 0xd504 }, { 0xa438, 0xac04 }, { 0xa438, 0xae01 }, + { 0xa438, 0xd500 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a78 }, + { 0xa438, 0xf00a }, { 0xa438, 0xd719 }, { 0xa438, 0x4118 }, + { 0xa438, 0xd504 }, { 0xa438, 0xac44 }, { 0xa438, 0xd501 }, + { 0xa438, 0xce01 }, { 0xa438, 0xa440 }, { 0xa438, 0xce00 }, + { 0xa438, 0xd500 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, + { 0xa438, 0xd718 }, { 0xa438, 0x5fb0 }, { 0xa438, 0xd505 }, + { 0xa438, 0xd719 }, { 0xa438, 0x4079 }, { 0xa438, 0xa80f }, + { 0xa438, 0xf021 }, { 0xa438, 0x4418 }, { 0xa438, 0xa802 }, + { 0xa438, 0xf01e }, { 0xa438, 0xd719 }, { 0xa438, 0x4119 }, + { 0xa438, 0xd504 }, { 0xa438, 0xac08 }, { 0xa438, 0xae01 }, + { 0xa438, 0xd500 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a78 }, + { 0xa438, 0xf00a }, { 0xa438, 0xd719 }, { 0xa438, 0x4118 }, + { 0xa438, 0xd504 }, { 0xa438, 0xac88 }, { 0xa438, 0xd501 }, + { 0xa438, 0xce01 }, { 0xa438, 0xa480 }, { 0xa438, 0xce00 }, + { 0xa438, 0xd500 }, { 0xa438, 0x1000 }, { 0xa438, 0x1a8a }, + { 0xa438, 0xd718 }, { 0xa438, 0x5fb0 }, { 0xa438, 0xd505 }, + { 0xa438, 0xd719 }, { 0xa438, 0x4079 }, { 0xa438, 0xa80f }, + { 0xa438, 0xf003 }, { 0xa438, 0x4058 }, { 0xa438, 0xa801 }, + { 0xa438, 0x1800 }, { 0xa438, 0x1736 }, { 0xa438, 0xd73e }, + { 0xa438, 0xd505 }, { 0xa438, 0x3088 }, { 0xa438, 0x81c0 }, + { 0xa438, 0x61d3 }, { 0xa438, 0x6172 }, { 0xa438, 0x6111 }, + { 0xa438, 0x60b0 }, { 0xa438, 0xf00d }, { 0xa438, 0x3298 }, + { 0xa438, 0x81cb }, { 0xa438, 0xf00a }, { 0xa438, 0xa808 }, + { 0xa438, 0xf008 }, { 0xa438, 0xa804 }, { 0xa438, 0xf006 }, + { 0xa438, 0xa802 }, { 0xa438, 0xf004 }, { 0xa438, 0xa801 }, + { 0xa438, 0xf002 }, { 0xa438, 0xa80f }, { 0xa438, 0xd500 }, + { 0xa438, 0x0800 }, { 0xa438, 0xd505 }, { 0xa438, 0xd75e }, + { 0xa438, 0x6211 }, { 0xa438, 0xd71e }, { 0xa438, 0x619b }, + { 0xa438, 0x611a }, { 0xa438, 0x6099 }, { 0xa438, 0x0c0f }, + { 0xa438, 0x0808 }, { 0xa438, 0xf009 }, { 0xa438, 0x0c0f }, + { 0xa438, 0x0804 }, { 0xa438, 0xf006 }, { 0xa438, 0x0c0f }, + { 0xa438, 0x0802 }, { 0xa438, 0xf003 }, { 0xa438, 0x0c0f }, + { 0xa438, 0x0801 }, { 0xa438, 0xd500 }, { 0xa438, 0x0800 }, + { 0xa436, 0xa026 }, { 0xa438, 0xffff }, { 0xa436, 0xa024 }, + { 0xa438, 0xffff }, { 0xa436, 0xa022 }, { 0xa438, 0xffff }, + { 0xa436, 0xa020 }, { 0xa438, 0xffff }, { 0xa436, 0xa006 }, + { 0xa438, 0xffff }, { 0xa436, 0xa004 }, { 0xa438, 0x16ab }, + { 0xa436, 0xa002 }, { 0xa438, 0x1663 }, { 0xa436, 0xa000 }, + { 0xa438, 0x1608 }, { 0xa436, 0xa008 }, { 0xa438, 0x0700 }, + { 0xa436, 0xa016 }, { 0xa438, 0x0000 }, { 0xa436, 0xa012 }, + { 0xa438, 0x07f8 }, { 0xa436, 0xa014 }, { 0xa438, 0xcc01 }, + { 0xa438, 0x20f6 }, { 0xa438, 0x0000 }, { 0xa438, 0x0000 }, + { 0xa438, 0x0000 }, { 0xa438, 0x0000 }, { 0xa438, 0x0000 }, + { 0xa438, 0x0000 }, { 0xa436, 0xa152 }, { 0xa438, 0x021c }, + { 0xa436, 0xa154 }, { 0xa438, 0x2100 }, { 0xa436, 0xa156 }, + { 0xa438, 0x3fff }, { 0xa436, 0xa158 }, { 0xa438, 0x3fff }, + { 0xa436, 0xa15a }, { 0xa438, 0x3fff }, { 0xa436, 0xa15c }, + { 0xa438, 0x3fff }, { 0xa436, 0xa15e }, { 0xa438, 0x3fff }, + { 0xa436, 0xa160 }, { 0xa438, 0x3fff }, { 0xa436, 0xa150 }, + { 0xa438, 0x0003 }, { 0xa436, 0xa016 }, { 0xa438, 0x0010 }, + { 0xa436, 0xa012 }, { 0xa438, 0x0000 }, { 0xa436, 0xa014 }, + { 0xa438, 0x1800 }, { 0xa438, 0x8010 }, { 0xa438, 0x1800 }, + { 0xa438, 0x8014 }, { 0xa438, 0x1800 }, { 0xa438, 0x803d }, + { 0xa438, 0x1800 }, { 0xa438, 0x804a }, { 0xa438, 0x1800 }, + { 0xa438, 0x804e }, { 0xa438, 0x1800 }, { 0xa438, 0x8052 }, + { 0xa438, 0x1800 }, { 0xa438, 0x8092 }, { 0xa438, 0x1800 }, + { 0xa438, 0x80a0 }, { 0xa438, 0xc2ff }, { 0xa438, 0x9a40 }, + { 0xa438, 0x1800 }, { 0xa438, 0x0042 }, { 0xa438, 0x1000 }, + { 0xa438, 0x02e5 }, { 0xa438, 0xba20 }, { 0xa438, 0x1000 }, + { 0xa438, 0x02b4 }, { 0xa438, 0xd701 }, { 0xa438, 0x4103 }, + { 0xa438, 0xd700 }, { 0xa438, 0x5f6c }, { 0xa438, 0x1000 }, + { 0xa438, 0x8024 }, { 0xa438, 0x9a20 }, { 0xa438, 0x1800 }, + { 0xa438, 0x0073 }, { 0xa438, 0x1800 }, { 0xa438, 0x0084 }, + { 0xa438, 0xd701 }, { 0xa438, 0x4061 }, { 0xa438, 0xba0f }, + { 0xa438, 0xf004 }, { 0xa438, 0x4060 }, { 0xa438, 0x1000 }, + { 0xa438, 0x802d }, { 0xa438, 0xba10 }, { 0xa438, 0x0800 }, + { 0xa438, 0xd700 }, { 0xa438, 0x60bb }, { 0xa438, 0x611c }, + { 0xa438, 0x0c0f }, { 0xa438, 0x1a01 }, { 0xa438, 0xf00a }, + { 0xa438, 0x60fc }, { 0xa438, 0x0c0f }, { 0xa438, 0x1a02 }, + { 0xa438, 0xf006 }, { 0xa438, 0x0c0f }, { 0xa438, 0x1a04 }, + { 0xa438, 0xf003 }, { 0xa438, 0x0c0f }, { 0xa438, 0x1a08 }, + { 0xa438, 0x0800 }, { 0xa438, 0x0c0f }, { 0xa438, 0x0504 }, + { 0xa438, 0xad02 }, { 0xa438, 0xd73e }, { 0xa438, 0x40f6 }, + { 0xa438, 0x1000 }, { 0xa438, 0x02c0 }, { 0xa438, 0xd700 }, + { 0xa438, 0x5fac }, { 0xa438, 0x1000 }, { 0xa438, 0x8024 }, + { 0xa438, 0x1800 }, { 0xa438, 0x0139 }, { 0xa438, 0x9a3f }, + { 0xa438, 0x8bf0 }, { 0xa438, 0x1800 }, { 0xa438, 0x02df }, + { 0xa438, 0x9a3f }, { 0xa438, 0x9910 }, { 0xa438, 0x1800 }, + { 0xa438, 0x02d7 }, { 0xa438, 0xad02 }, { 0xa438, 0x8d01 }, + { 0xa438, 0x9a7f }, { 0xa438, 0x9910 }, { 0xa438, 0x9860 }, + { 0xa438, 0xcb00 }, { 0xa438, 0xd501 }, { 0xa438, 0xce01 }, + { 0xa438, 0x85f0 }, { 0xa438, 0xd500 }, { 0xa438, 0x0c0f }, + { 0xa438, 0x0505 }, { 0xa438, 0xb820 }, { 0xa438, 0xc000 }, + { 0xa438, 0xc100 }, { 0xa438, 0xc628 }, { 0xa438, 0xc700 }, + { 0xa438, 0xc801 }, { 0xa438, 0xc91e }, { 0xa438, 0xc001 }, + { 0xa438, 0x4019 }, { 0xa438, 0xc6f8 }, { 0xa438, 0xc702 }, + { 0xa438, 0xc809 }, { 0xa438, 0xc940 }, { 0xa438, 0xc002 }, + { 0xa438, 0x4019 }, { 0xa438, 0x1000 }, { 0xa438, 0x02cc }, + { 0xa438, 0xd700 }, { 0xa438, 0x5fa7 }, { 0xa438, 0xc010 }, + { 0xa438, 0x1000 }, { 0xa438, 0x02cc }, { 0xa438, 0xd700 }, + { 0xa438, 0x5fa0 }, { 0xa438, 0xc020 }, { 0xa438, 0x1000 }, + { 0xa438, 0x02cc }, { 0xa438, 0xd700 }, { 0xa438, 0x5fa1 }, + { 0xa438, 0x0c0f }, { 0xa438, 0x0506 }, { 0xa438, 0xb840 }, + { 0xa438, 0xc6ca }, { 0xa438, 0xc701 }, { 0xa438, 0xc809 }, + { 0xa438, 0xc900 }, { 0xa438, 0xc001 }, { 0xa438, 0x4019 }, + { 0xa438, 0xc6b8 }, { 0xa438, 0xc700 }, { 0xa438, 0xc800 }, + { 0xa438, 0xc900 }, { 0xa438, 0xc008 }, { 0xa438, 0x4019 }, + { 0xa438, 0x1000 }, { 0xa438, 0x02cc }, { 0xa438, 0xd700 }, + { 0xa438, 0x5fa5 }, { 0xa438, 0x8580 }, { 0xa438, 0x8d02 }, + { 0xa438, 0x1800 }, { 0xa438, 0x018f }, { 0xa438, 0x1000 }, + { 0xa438, 0x02cc }, { 0xa438, 0xd700 }, { 0xa438, 0x6124 }, + { 0xa438, 0xd73e }, { 0xa438, 0x5f75 }, { 0xa438, 0xd700 }, + { 0xa438, 0x5f2c }, { 0xa438, 0x1000 }, { 0xa438, 0x8024 }, + { 0xa438, 0x9a20 }, { 0xa438, 0xfff5 }, { 0xa438, 0x1800 }, + { 0xa438, 0x00b8 }, { 0xa438, 0x0c0f }, { 0xa438, 0x0503 }, + { 0xa438, 0xad02 }, { 0xa438, 0x68c8 }, { 0xa438, 0x1000 }, + { 0xa438, 0x02c0 }, { 0xa438, 0xd700 }, { 0xa438, 0x6848 }, + { 0xa438, 0x604d }, { 0xa438, 0xfffb }, { 0xa438, 0xd73e }, + { 0xa438, 0x6082 }, { 0xa438, 0x1000 }, { 0xa438, 0x02a1 }, + { 0xa438, 0x8a0f }, { 0xa438, 0x1000 }, { 0xa438, 0x02c0 }, + { 0xa438, 0xd700 }, { 0xa438, 0x5fae }, { 0xa438, 0x1000 }, + { 0xa438, 0x02de }, { 0xa438, 0x1000 }, { 0xa438, 0x02c0 }, + { 0xa438, 0xd700 }, { 0xa438, 0x5faf }, { 0xa438, 0x8d01 }, + { 0xa438, 0x8b0f }, { 0xa438, 0x1000 }, { 0xa438, 0x02c0 }, + { 0xa438, 0xd700 }, { 0xa438, 0x2a58 }, { 0xa438, 0x80c5 }, + { 0xa438, 0x2a5b }, { 0xa438, 0x80cd }, { 0xa438, 0x2b53 }, + { 0xa438, 0x80d9 }, { 0xa438, 0xfff7 }, { 0xa438, 0x1000 }, + { 0xa438, 0x022a }, { 0xa438, 0x1000 }, { 0xa438, 0x02e5 }, + { 0xa438, 0xba40 }, { 0xa438, 0x1000 }, { 0xa438, 0x02fd }, + { 0xa438, 0xf018 }, { 0xa438, 0x1000 }, { 0xa438, 0x022a }, + { 0xa438, 0x1000 }, { 0xa438, 0x02e5 }, { 0xa438, 0xba40 }, + { 0xa438, 0x1000 }, { 0xa438, 0x02c0 }, { 0xa438, 0xd700 }, + { 0xa438, 0x5faa }, { 0xa438, 0x1000 }, { 0xa438, 0x02fd }, + { 0xa438, 0xf00c }, { 0xa438, 0x1000 }, { 0xa438, 0x022a }, + { 0xa438, 0x1000 }, { 0xa438, 0x02fd }, { 0xa438, 0x1000 }, + { 0xa438, 0x02c0 }, { 0xa438, 0xd700 }, { 0xa438, 0x5fab }, + { 0xa438, 0x1000 }, { 0xa438, 0x02e5 }, { 0xa438, 0xba40 }, + { 0xa438, 0x1000 }, { 0xa438, 0x02c0 }, { 0xa438, 0xd700 }, + { 0xa438, 0x6088 }, { 0xa438, 0xfffc }, { 0xa438, 0x1800 }, + { 0xa438, 0x0120 }, { 0xa438, 0x1800 }, { 0xa438, 0x0122 }, + { 0xa436, 0xa08e }, { 0xa438, 0x00db }, { 0xa436, 0xa08c }, + { 0xa438, 0x00b4 }, { 0xa436, 0xa08a }, { 0xa438, 0x015a }, + { 0xa436, 0xa088 }, { 0xa438, 0x02d6 }, { 0xa436, 0xa086 }, + { 0xa438, 0x02de }, { 0xa436, 0xa084 }, { 0xa438, 0x0137 }, + { 0xa436, 0xa082 }, { 0xa438, 0x0071 }, { 0xa436, 0xa080 }, + { 0xa438, 0x0041 }, { 0xa436, 0xa090 }, { 0xa438, 0x00ff }, + { 0xa436, 0xa016 }, { 0xa438, 0x0020 }, { 0xa436, 0xa012 }, + { 0xa438, 0x0000 }, { 0xa436, 0xa014 }, { 0xa438, 0x1800 }, + { 0xa438, 0x8010 }, { 0xa438, 0x1800 }, { 0xa438, 0x801d }, + { 0xa438, 0x1800 }, { 0xa438, 0x808a }, { 0xa438, 0x1800 }, + { 0xa438, 0x80a5 }, { 0xa438, 0x1800 }, { 0xa438, 0x80b8 }, + { 0xa438, 0x1800 }, { 0xa438, 0x8108 }, { 0xa438, 0x1800 }, + { 0xa438, 0x810f }, { 0xa438, 0x1800 }, { 0xa438, 0x811b }, + { 0xa438, 0x8980 }, { 0xa438, 0xd702 }, { 0xa438, 0x6126 }, + { 0xa438, 0xd704 }, { 0xa438, 0x4063 }, { 0xa438, 0xd702 }, + { 0xa438, 0x6060 }, { 0xa438, 0xd702 }, { 0xa438, 0x6077 }, + { 0xa438, 0x1800 }, { 0xa438, 0x0c29 }, { 0xa438, 0x1800 }, + { 0xa438, 0x0c2b }, { 0xa438, 0x1000 }, { 0xa438, 0x115a }, + { 0xa438, 0xd71f }, { 0xa438, 0x5fb4 }, { 0xa438, 0xd702 }, + { 0xa438, 0x6c46 }, { 0xa438, 0xd704 }, { 0xa438, 0x4063 }, + { 0xa438, 0xd702 }, { 0xa438, 0x6060 }, { 0xa438, 0xd702 }, + { 0xa438, 0x6b97 }, { 0xa438, 0xa340 }, { 0xa438, 0x0c06 }, + { 0xa438, 0x0102 }, { 0xa438, 0xce01 }, { 0xa438, 0x1000 }, + { 0xa438, 0x117a }, { 0xa438, 0xa240 }, { 0xa438, 0xa902 }, + { 0xa438, 0xa204 }, { 0xa438, 0xa280 }, { 0xa438, 0xa364 }, + { 0xa438, 0xab02 }, { 0xa438, 0x8380 }, { 0xa438, 0xa00a }, + { 0xa438, 0xcd8d }, { 0xa438, 0x1000 }, { 0xa438, 0x115a }, + { 0xa438, 0xd706 }, { 0xa438, 0x5fb5 }, { 0xa438, 0xb920 }, + { 0xa438, 0x1000 }, { 0xa438, 0x115a }, { 0xa438, 0xd71f }, + { 0xa438, 0x7fb4 }, { 0xa438, 0x9920 }, { 0xa438, 0x1000 }, + { 0xa438, 0x115a }, { 0xa438, 0xd71f }, { 0xa438, 0x6065 }, + { 0xa438, 0x7c74 }, { 0xa438, 0xfffb }, { 0xa438, 0xb820 }, + { 0xa438, 0x1000 }, { 0xa438, 0x115a }, { 0xa438, 0xd71f }, + { 0xa438, 0x7fa5 }, { 0xa438, 0x9820 }, { 0xa438, 0xa410 }, + { 0xa438, 0x8902 }, { 0xa438, 0xa120 }, { 0xa438, 0xa380 }, + { 0xa438, 0xce02 }, { 0xa438, 0x1000 }, { 0xa438, 0x117a }, + { 0xa438, 0x8280 }, { 0xa438, 0xa324 }, { 0xa438, 0xab02 }, + { 0xa438, 0xa00a }, { 0xa438, 0x8118 }, { 0xa438, 0x863f }, + { 0xa438, 0x87fb }, { 0xa438, 0xcd8e }, { 0xa438, 0xd193 }, + { 0xa438, 0xd047 }, { 0xa438, 0x1000 }, { 0xa438, 0x115a }, + { 0xa438, 0x1000 }, { 0xa438, 0x115f }, { 0xa438, 0xd700 }, + { 0xa438, 0x5f7b }, { 0xa438, 0xa280 }, { 0xa438, 0x1000 }, + { 0xa438, 0x115a }, { 0xa438, 0x1000 }, { 0xa438, 0x115f }, + { 0xa438, 0xd706 }, { 0xa438, 0x5f78 }, { 0xa438, 0xa210 }, + { 0xa438, 0xd700 }, { 0xa438, 0x6083 }, { 0xa438, 0xd101 }, + { 0xa438, 0xd047 }, { 0xa438, 0xf003 }, { 0xa438, 0xd160 }, + { 0xa438, 0xd04b }, { 0xa438, 0x1000 }, { 0xa438, 0x115a }, + { 0xa438, 0x1000 }, { 0xa438, 0x115f }, { 0xa438, 0xd700 }, + { 0xa438, 0x5f7b }, { 0xa438, 0x1000 }, { 0xa438, 0x115a }, + { 0xa438, 0x1000 }, { 0xa438, 0x115f }, { 0xa438, 0xd706 }, + { 0xa438, 0x5f79 }, { 0xa438, 0x8120 }, { 0xa438, 0xbb20 }, + { 0xa438, 0x1800 }, { 0xa438, 0x0c8b }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0x8f80 }, { 0xa438, 0x9503 }, + { 0xa438, 0x1800 }, { 0xa438, 0x0c3c }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0xa608 }, { 0xa438, 0x9503 }, + { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, { 0xa438, 0x8f80 }, + { 0xa438, 0x9503 }, { 0xa438, 0xd704 }, { 0xa438, 0x6192 }, + { 0xa438, 0xd702 }, { 0xa438, 0x4116 }, { 0xa438, 0xce04 }, + { 0xa438, 0x1000 }, { 0xa438, 0x117a }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0x8f40 }, { 0xa438, 0x9503 }, + { 0xa438, 0x1800 }, { 0xa438, 0x0b3d }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0xaf40 }, { 0xa438, 0x9503 }, + { 0xa438, 0x1800 }, { 0xa438, 0x0b48 }, { 0xa438, 0xd704 }, + { 0xa438, 0x6192 }, { 0xa438, 0xd702 }, { 0xa438, 0x4116 }, + { 0xa438, 0xce04 }, { 0xa438, 0x1000 }, { 0xa438, 0x117a }, + { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, { 0xa438, 0x8f40 }, + { 0xa438, 0x9503 }, { 0xa438, 0x1800 }, { 0xa438, 0x1269 }, + { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, { 0xa438, 0xaf40 }, + { 0xa438, 0x9503 }, { 0xa438, 0x1800 }, { 0xa438, 0x1274 }, + { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, { 0xa438, 0xa608 }, + { 0xa438, 0xc700 }, { 0xa438, 0x9503 }, { 0xa438, 0xce54 }, + { 0xa438, 0x1000 }, { 0xa438, 0x117a }, { 0xa438, 0xa290 }, + { 0xa438, 0xa304 }, { 0xa438, 0xab02 }, { 0xa438, 0xd700 }, + { 0xa438, 0x6050 }, { 0xa438, 0xab04 }, { 0xa438, 0x0c38 }, + { 0xa438, 0x0608 }, { 0xa438, 0xaa0b }, { 0xa438, 0xd702 }, + { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, { 0xa438, 0x8d01 }, + { 0xa438, 0xae40 }, { 0xa438, 0x4044 }, { 0xa438, 0x8e20 }, + { 0xa438, 0x9503 }, { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, + { 0xa438, 0x8c20 }, { 0xa438, 0x9503 }, { 0xa438, 0xd700 }, + { 0xa438, 0x6078 }, { 0xa438, 0xd700 }, { 0xa438, 0x609a }, + { 0xa438, 0xd109 }, { 0xa438, 0xd074 }, { 0xa438, 0xf003 }, + { 0xa438, 0xd109 }, { 0xa438, 0xd075 }, { 0xa438, 0x1000 }, + { 0xa438, 0x115a }, { 0xa438, 0xd704 }, { 0xa438, 0x6252 }, + { 0xa438, 0xd702 }, { 0xa438, 0x4116 }, { 0xa438, 0xce54 }, + { 0xa438, 0x1000 }, { 0xa438, 0x117a }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0x8f40 }, { 0xa438, 0x9503 }, + { 0xa438, 0xa00a }, { 0xa438, 0xd704 }, { 0xa438, 0x41e7 }, + { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, { 0xa438, 0xa570 }, + { 0xa438, 0x9503 }, { 0xa438, 0xf00a }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0xaf40 }, { 0xa438, 0x9503 }, + { 0xa438, 0x800a }, { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, + { 0xa438, 0x8570 }, { 0xa438, 0x9503 }, { 0xa438, 0xd704 }, + { 0xa438, 0x60f3 }, { 0xa438, 0xd71f }, { 0xa438, 0x60ee }, + { 0xa438, 0xd700 }, { 0xa438, 0x5bbe }, { 0xa438, 0x1800 }, + { 0xa438, 0x0e71 }, { 0xa438, 0x1800 }, { 0xa438, 0x0e7c }, + { 0xa438, 0x1800 }, { 0xa438, 0x0e7e }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0xaf80 }, { 0xa438, 0x9503 }, + { 0xa438, 0xcd62 }, { 0xa438, 0x1800 }, { 0xa438, 0x0bd2 }, + { 0xa438, 0x800a }, { 0xa438, 0x8530 }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0x8d10 }, { 0xa438, 0x9503 }, + { 0xa438, 0xd700 }, { 0xa438, 0x6050 }, { 0xa438, 0xaa20 }, + { 0xa438, 0x8306 }, { 0xa438, 0x1800 }, { 0xa438, 0x0cb6 }, + { 0xa438, 0xd105 }, { 0xa438, 0xd040 }, { 0xa438, 0x1000 }, + { 0xa438, 0x0d8f }, { 0xa438, 0xd700 }, { 0xa438, 0x5fbb }, + { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, { 0xa438, 0x8608 }, + { 0xa438, 0x9503 }, { 0xa438, 0x1000 }, { 0xa438, 0x0d8f }, + { 0xa438, 0xd704 }, { 0xa438, 0x7fb6 }, { 0xa438, 0x0c03 }, + { 0xa438, 0x1502 }, { 0xa438, 0x87f0 }, { 0xa438, 0x9503 }, + { 0xa438, 0xce88 }, { 0xa438, 0x1000 }, { 0xa438, 0x117a }, + { 0xa438, 0x0c03 }, { 0xa438, 0x1502 }, { 0xa438, 0xa608 }, + { 0xa438, 0x9503 }, { 0xa438, 0xd73e }, { 0xa438, 0x60a5 }, + { 0xa438, 0xd705 }, { 0xa438, 0x4071 }, { 0xa438, 0x1800 }, + { 0xa438, 0x0d65 }, { 0xa438, 0x1800 }, { 0xa438, 0x0d6f }, + { 0xa436, 0xa10e }, { 0xa438, 0x0d58 }, { 0xa436, 0xa10c }, + { 0xa438, 0x0cb5 }, { 0xa436, 0xa10a }, { 0xa438, 0x0bd1 }, + { 0xa436, 0xa108 }, { 0xa438, 0x0e37 }, { 0xa436, 0xa106 }, + { 0xa438, 0x1267 }, { 0xa436, 0xa104 }, { 0xa438, 0x0b3b }, + { 0xa436, 0xa102 }, { 0xa438, 0x0c38 }, { 0xa436, 0xa100 }, + { 0xa438, 0x0c24 }, { 0xa436, 0xa110 }, { 0xa438, 0x00ff }, + { 0xa436, 0xa016 }, { 0xa438, 0x0020 }, { 0xa436, 0xa012 }, + { 0xa438, 0x1ff8 }, { 0xa436, 0xa014 }, { 0xa438, 0x0000 }, + { 0xa438, 0x0000 }, { 0xa438, 0x0000 }, { 0xa438, 0x0000 }, + { 0xa438, 0x0000 }, { 0xa438, 0x0000 }, { 0xa438, 0x0000 }, + { 0xa438, 0x0000 }, { 0xa436, 0xa164 }, { 0xa438, 0x0ceb }, + { 0xa436, 0xa166 }, { 0xa438, 0x0e73 }, { 0xa436, 0xa168 }, + { 0xa438, 0x0deb }, { 0xa436, 0xa16a }, { 0xa438, 0x3fff }, + { 0xa436, 0xa16c }, { 0xa438, 0x3fff }, { 0xa436, 0xa16e }, + { 0xa438, 0x3fff }, { 0xa436, 0xa170 }, { 0xa438, 0x3fff }, + { 0xa436, 0xa172 }, { 0xa438, 0x3fff }, { 0xa436, 0xa162 }, + { 0xa438, 0x0007 }, { 0xa436, 0xb87c }, { 0xa438, 0x85bf }, + { 0xa436, 0xb87e }, { 0xa438, 0xaf85 }, { 0xa438, 0xd7af }, + { 0xa438, 0x85fb }, { 0xa438, 0xaf86 }, { 0xa438, 0x10af }, + { 0xa438, 0x8638 }, { 0xa438, 0xaf86 }, { 0xa438, 0x47af }, + { 0xa438, 0x8647 }, { 0xa438, 0xaf86 }, { 0xa438, 0x47af }, + { 0xa438, 0x8647 }, { 0xa438, 0xbf85 }, { 0xa438, 0xf802 }, + { 0xa438, 0x627f }, { 0xa438, 0xbf61 }, { 0xa438, 0xc702 }, + { 0xa438, 0x627f }, { 0xa438, 0xae0c }, { 0xa438, 0xbf85 }, + { 0xa438, 0xf802 }, { 0xa438, 0x6276 }, { 0xa438, 0xbf61 }, + { 0xa438, 0xc702 }, { 0xa438, 0x6276 }, { 0xa438, 0xee85 }, + { 0xa438, 0x4200 }, { 0xa438, 0xaf1b }, { 0xa438, 0x2333 }, + { 0xa438, 0xa484 }, { 0xa438, 0xbf86 }, { 0xa438, 0x0a02 }, + { 0xa438, 0x627f }, { 0xa438, 0xbf86 }, { 0xa438, 0x0d02 }, + { 0xa438, 0x627f }, { 0xa438, 0xaf1b }, { 0xa438, 0x8422 }, + { 0xa438, 0xa484 }, { 0xa438, 0x66ac }, { 0xa438, 0x0ef8 }, + { 0xa438, 0xfbef }, { 0xa438, 0x79fb }, { 0xa438, 0xe080 }, + { 0xa438, 0x16ad }, { 0xa438, 0x230f }, { 0xa438, 0xee85 }, + { 0xa438, 0x4200 }, { 0xa438, 0x1f44 }, { 0xa438, 0xbf86 }, + { 0xa438, 0x30d7 }, { 0xa438, 0x0008 }, { 0xa438, 0x0264 }, + { 0xa438, 0xa3ff }, { 0xa438, 0xef97 }, { 0xa438, 0xfffc }, + { 0xa438, 0x0485 }, { 0xa438, 0xf861 }, { 0xa438, 0xc786 }, + { 0xa438, 0x0a86 }, { 0xa438, 0x0de1 }, { 0xa438, 0x8feb }, + { 0xa438, 0xe583 }, { 0xa438, 0x20e1 }, { 0xa438, 0x8fea }, + { 0xa438, 0xe583 }, { 0xa438, 0x21af }, { 0xa438, 0x41a7 }, + { 0xa436, 0xb85e }, { 0xa438, 0x1b05 }, { 0xa436, 0xb860 }, + { 0xa438, 0x1b78 }, { 0xa436, 0xb862 }, { 0xa438, 0x1a08 }, + { 0xa436, 0xb864 }, { 0xa438, 0x419f }, { 0xa436, 0xb886 }, + { 0xa438, 0xffff }, { 0xa436, 0xb888 }, { 0xa438, 0xffff }, + { 0xa436, 0xb88a }, { 0xa438, 0xffff }, { 0xa436, 0xb88c }, + { 0xa438, 0xffff }, { 0xa436, 0xb838 }, { 0xa438, 0x000f }, + { 0xb820, 0x0010 }, { 0xa436, 0x0000 }, { 0xa438, 0x0000 }, + { 0xb82e, 0x0000 }, { 0xa436, 0x8023 }, { 0xa438, 0x0000 }, + { 0xa436, 0x801e }, { 0xa438, 0x0013 }, { 0xb820, 0x0000 }, + { 0xa436, 0xacca }, { 0xa438, 0x0104 }, { 0xa436, 0xaccc }, + { 0xa438, 0x8000 }, { 0xa436, 0xacce }, { 0xa438, 0xffff }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0fff }, { 0xa436, 0xacce }, + { 0xa438, 0xfd47 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0fff }, + { 0xa436, 0xacce }, { 0xa438, 0xffff }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0xe56f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x01c0 }, { 0xa436, 0xacce }, + { 0xa438, 0xffff }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0xed97 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x01c8 }, { 0xa436, 0xacce }, { 0xa438, 0xffff }, + { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, { 0xa436, 0xacce }, + { 0xa438, 0xf5bf }, { 0xa436, 0xacd0 }, { 0xa438, 0x01d0 }, + { 0xa436, 0xacce }, { 0xa438, 0xfb07 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0xfb0f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x01d8 }, { 0xa436, 0xacce }, + { 0xa438, 0xa087 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0180 }, + { 0xa436, 0xacce }, { 0xa438, 0xa00f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0108 }, { 0xa436, 0xacce }, { 0xa438, 0xa807 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0100 }, { 0xa436, 0xacce }, + { 0xa438, 0xa88f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0188 }, + { 0xa436, 0xacce }, { 0xa438, 0xb027 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0120 }, { 0xa436, 0xacce }, { 0xa438, 0xb02f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0128 }, { 0xa436, 0xacce }, + { 0xa438, 0xb847 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0140 }, + { 0xa436, 0xacce }, { 0xa438, 0xb84f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0148 }, { 0xa436, 0xacce }, { 0xa438, 0xfb17 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, { 0xa436, 0xacce }, + { 0xa438, 0xfb1f }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0xa017 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0110 }, { 0xa436, 0xacce }, { 0xa438, 0xa01f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0118 }, { 0xa436, 0xacce }, + { 0xa438, 0xa837 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0130 }, + { 0xa436, 0xacce }, { 0xa438, 0xa83f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0138 }, { 0xa436, 0xacce }, { 0xa438, 0xb097 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0190 }, { 0xa436, 0xacce }, + { 0xa438, 0xb05f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0158 }, + { 0xa436, 0xacce }, { 0xa438, 0xb857 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0150 }, { 0xa436, 0xacce }, { 0xa438, 0xb89f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0198 }, { 0xa436, 0xacce }, + { 0xa438, 0xfb27 }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0xfb2f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0x8087 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0180 }, { 0xa436, 0xacce }, + { 0xa438, 0x800f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0108 }, + { 0xa436, 0xacce }, { 0xa438, 0x8807 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0100 }, { 0xa436, 0xacce }, { 0xa438, 0x888f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0188 }, { 0xa436, 0xacce }, + { 0xa438, 0x9027 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0120 }, + { 0xa436, 0xacce }, { 0xa438, 0x902f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0128 }, { 0xa436, 0xacce }, { 0xa438, 0x9847 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0140 }, { 0xa436, 0xacce }, + { 0xa438, 0x984f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0148 }, + { 0xa436, 0xacce }, { 0xa438, 0xa0a7 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x01a0 }, { 0xa436, 0xacce }, { 0xa438, 0xa8af }, + { 0xa436, 0xacd0 }, { 0xa438, 0x01a8 }, { 0xa436, 0xacce }, + { 0xa438, 0xa067 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0161 }, + { 0xa436, 0xacce }, { 0xa438, 0xa86f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0169 }, { 0xa436, 0xacce }, { 0xa438, 0xfb37 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, { 0xa436, 0xacce }, + { 0xa438, 0xfb3f }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0x8017 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0110 }, { 0xa436, 0xacce }, { 0xa438, 0x801f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0118 }, { 0xa436, 0xacce }, + { 0xa438, 0x8837 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0130 }, + { 0xa436, 0xacce }, { 0xa438, 0x883f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0138 }, { 0xa436, 0xacce }, { 0xa438, 0x9097 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0190 }, { 0xa436, 0xacce }, + { 0xa438, 0x905f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0158 }, + { 0xa436, 0xacce }, { 0xa438, 0x9857 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0150 }, { 0xa436, 0xacce }, { 0xa438, 0x989f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0198 }, { 0xa436, 0xacce }, + { 0xa438, 0xb0b7 }, { 0xa436, 0xacd0 }, { 0xa438, 0x01b0 }, + { 0xa436, 0xacce }, { 0xa438, 0xb8bf }, { 0xa436, 0xacd0 }, + { 0xa438, 0x01b8 }, { 0xa436, 0xacce }, { 0xa438, 0xb077 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0171 }, { 0xa436, 0xacce }, + { 0xa438, 0xb87f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0179 }, + { 0xa436, 0xacce }, { 0xa438, 0xfb47 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0xfb4f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, { 0xa436, 0xacce }, + { 0xa438, 0x6087 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0180 }, + { 0xa436, 0xacce }, { 0xa438, 0x600f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0108 }, { 0xa436, 0xacce }, { 0xa438, 0x6807 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0100 }, { 0xa436, 0xacce }, + { 0xa438, 0x688f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0188 }, + { 0xa436, 0xacce }, { 0xa438, 0x7027 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0120 }, { 0xa436, 0xacce }, { 0xa438, 0x702f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0128 }, { 0xa436, 0xacce }, + { 0xa438, 0x7847 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0140 }, + { 0xa436, 0xacce }, { 0xa438, 0x784f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0148 }, { 0xa436, 0xacce }, { 0xa438, 0x80a7 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x01a0 }, { 0xa436, 0xacce }, + { 0xa438, 0x88af }, { 0xa436, 0xacd0 }, { 0xa438, 0x01a8 }, + { 0xa436, 0xacce }, { 0xa438, 0x8067 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0161 }, { 0xa436, 0xacce }, { 0xa438, 0x886f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0169 }, { 0xa436, 0xacce }, + { 0xa438, 0xfb57 }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0xfb5f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0x6017 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0110 }, { 0xa436, 0xacce }, + { 0xa438, 0x601f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0118 }, + { 0xa436, 0xacce }, { 0xa438, 0x6837 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0130 }, { 0xa436, 0xacce }, { 0xa438, 0x683f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0138 }, { 0xa436, 0xacce }, + { 0xa438, 0x7097 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0190 }, + { 0xa436, 0xacce }, { 0xa438, 0x705f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0158 }, { 0xa436, 0xacce }, { 0xa438, 0x7857 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0150 }, { 0xa436, 0xacce }, + { 0xa438, 0x789f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0198 }, + { 0xa436, 0xacce }, { 0xa438, 0x90b7 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x01b0 }, { 0xa436, 0xacce }, { 0xa438, 0x98bf }, + { 0xa436, 0xacd0 }, { 0xa438, 0x01b8 }, { 0xa436, 0xacce }, + { 0xa438, 0x9077 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0171 }, + { 0xa436, 0xacce }, { 0xa438, 0x987f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0179 }, { 0xa436, 0xacce }, { 0xa438, 0xfb67 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, { 0xa436, 0xacce }, + { 0xa438, 0xfb6f }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0x4087 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0180 }, { 0xa436, 0xacce }, { 0xa438, 0x400f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0108 }, { 0xa436, 0xacce }, + { 0xa438, 0x4807 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0100 }, + { 0xa436, 0xacce }, { 0xa438, 0x488f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0188 }, { 0xa436, 0xacce }, { 0xa438, 0x5027 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0120 }, { 0xa436, 0xacce }, + { 0xa438, 0x502f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0128 }, + { 0xa436, 0xacce }, { 0xa438, 0x5847 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0140 }, { 0xa436, 0xacce }, { 0xa438, 0x584f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0148 }, { 0xa436, 0xacce }, + { 0xa438, 0x60a7 }, { 0xa436, 0xacd0 }, { 0xa438, 0x01a0 }, + { 0xa436, 0xacce }, { 0xa438, 0x68af }, { 0xa436, 0xacd0 }, + { 0xa438, 0x01a8 }, { 0xa436, 0xacce }, { 0xa438, 0x6067 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0161 }, { 0xa436, 0xacce }, + { 0xa438, 0x686f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0169 }, + { 0xa436, 0xacce }, { 0xa438, 0xfb77 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0xfb7f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, { 0xa436, 0xacce }, + { 0xa438, 0x4017 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0110 }, + { 0xa436, 0xacce }, { 0xa438, 0x401f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0118 }, { 0xa436, 0xacce }, { 0xa438, 0x4837 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0130 }, { 0xa436, 0xacce }, + { 0xa438, 0x483f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0138 }, + { 0xa436, 0xacce }, { 0xa438, 0x5097 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0190 }, { 0xa436, 0xacce }, { 0xa438, 0x505f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0158 }, { 0xa436, 0xacce }, + { 0xa438, 0x5857 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0150 }, + { 0xa436, 0xacce }, { 0xa438, 0x589f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0198 }, { 0xa436, 0xacce }, { 0xa438, 0x70b7 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x01b0 }, { 0xa436, 0xacce }, + { 0xa438, 0x78bf }, { 0xa436, 0xacd0 }, { 0xa438, 0x01b8 }, + { 0xa436, 0xacce }, { 0xa438, 0x7077 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0171 }, { 0xa436, 0xacce }, { 0xa438, 0x787f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0179 }, { 0xa436, 0xacce }, + { 0xa438, 0xfb87 }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0xfb8f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0x40a7 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x01a0 }, { 0xa436, 0xacce }, + { 0xa438, 0x48af }, { 0xa436, 0xacd0 }, { 0xa438, 0x01a8 }, + { 0xa436, 0xacce }, { 0xa438, 0x4067 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0161 }, { 0xa436, 0xacce }, { 0xa438, 0x486f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0169 }, { 0xa436, 0xacce }, + { 0xa438, 0xfb97 }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0xfb9f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0x50b7 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x01b0 }, { 0xa436, 0xacce }, + { 0xa438, 0x58bf }, { 0xa436, 0xacd0 }, { 0xa438, 0x01b8 }, + { 0xa436, 0xacce }, { 0xa438, 0x5077 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0171 }, { 0xa436, 0xacce }, { 0xa438, 0x587f }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0179 }, { 0xa436, 0xacce }, + { 0xa438, 0xfba7 }, { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, + { 0xa436, 0xacce }, { 0xa438, 0xfbaf }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0x2067 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x0161 }, { 0xa436, 0xacce }, + { 0xa438, 0x286f }, { 0xa436, 0xacd0 }, { 0xa438, 0x0169 }, + { 0xa436, 0xacce }, { 0xa438, 0xfbb7 }, { 0xa436, 0xacd0 }, + { 0xa438, 0x07ff }, { 0xa436, 0xacce }, { 0xa438, 0xfbbf }, + { 0xa436, 0xacd0 }, { 0xa438, 0x07ff }, { 0xa436, 0xacce }, + { 0xa438, 0x3077 }, { 0xa436, 0xacd0 }, { 0xa438, 0x0171 }, + { 0xa436, 0xacce }, { 0xa438, 0x387f }, { 0xa436, 0xacd0 }, + { 0xa438, 0x0179 }, { 0xa436, 0xacce }, { 0xa438, 0xfff9 }, + { 0xa436, 0xacd0 }, { 0xa438, 0x17ff }, { 0xa436, 0xacce }, + { 0xa438, 0xfff9 }, { 0xa436, 0xacd0 }, { 0xa438, 0x17ff }, + { 0xa436, 0xacca }, { 0xa438, 0x0004 }, { 0xa436, 0xacc6 }, + { 0xa438, 0x0008 }, { 0xa436, 0xacc8 }, { 0xa438, 0xc000 }, + { 0xa436, 0xacc8 }, { 0xa438, 0x0000 }, { 0xb820, 0x0000 } +}; + static const struct rge_hw_regaddr_array mac_r26_1_mcu[] = { { 0xa436, 0x8023 }, { 0xa438, 0x4700 }, { 0xa436, 0xb82e }, { 0xa438, 0x0001 }, { 0xb820, 0x0090 }, { 0xa436, 0xa016 }, @@ -9614,7 +10248,7 @@ static const struct { MAC_R25_MCU }, mac_r25b_mcu[] = { MAC_R25B_MCU -}, mac_r25d_mcu[] = { +}, mac_r25d_1_mcu[] = { MAC_R25D_MCU }; diff --git a/sys/dev/rge/if_rgereg.h b/sys/dev/rge/if_rgereg.h index 7b34e3cefbe6..7da8ff42f2b4 100644 --- a/sys/dev/rge/if_rgereg.h +++ b/sys/dev/rge/if_rgereg.h @@ -365,13 +365,17 @@ struct rge_hw_mac_stats { #define RGE_STATS_ALIGNMENT 64 /* Ram version */ -#define RGE_MAC_R25D_RCODE_VER 0x0027 +#define RGE_MAC_R25D_1_RCODE_VER 0x0027 +#define RGE_MAC_R25D_2_RCODE_VER 0x0031 #define RGE_MAC_R26_1_RCODE_VER 0x0033 #define RGE_MAC_R26_2_RCODE_VER 0x0060 #define RGE_MAC_R27_RCODE_VER 0x0036 #define RGE_MAC_R25_RCODE_VER 0x0b33 #define RGE_MAC_R25B_RCODE_VER 0x0b99 +#define RGE_TYPE_R25D(sc) \ + ((sc)->rge_type == MAC_R25D_1 || (sc)->rge_type == MAC_R25D_2) + #define RGE_TYPE_R26(sc) \ ((sc)->rge_type == MAC_R26_1 || (sc)->rge_type == MAC_R26_2) diff --git a/sys/dev/rge/if_rgevar.h b/sys/dev/rge/if_rgevar.h index 504d56cba423..924133da45e3 100644 --- a/sys/dev/rge/if_rgevar.h +++ b/sys/dev/rge/if_rgevar.h @@ -30,7 +30,8 @@ enum rge_mac_type { MAC_UNKNOWN = 1, MAC_R25, MAC_R25B, - MAC_R25D, + MAC_R25D_1, + MAC_R25D_2, MAC_R26_1, MAC_R26_2, MAC_R27 diff --git a/sys/dev/smartpqi/smartpqi_cam.c b/sys/dev/smartpqi/smartpqi_cam.c index 93043a296c5d..690b38c9f855 100644 --- a/sys/dev/smartpqi/smartpqi_cam.c +++ b/sys/dev/smartpqi/smartpqi_cam.c @@ -54,7 +54,7 @@ update_sim_properties(struct cam_sim *sim, struct ccb_pathinq *cpi) cpi->initiator_id = 255; strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN-1); cpi->sim_vid[sizeof(cpi->sim_vid)-1] = '\0'; - strncpy(cpi->hba_vid, "Microsemi", HBA_IDLEN-1); + strncpy(cpi->hba_vid, "Microchip", HBA_IDLEN-1); cpi->hba_vid[sizeof(cpi->hba_vid)-1] = '\0'; strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN-1); cpi->dev_name[sizeof(cpi->dev_name)-1] = '\0'; diff --git a/sys/dev/smartpqi/smartpqi_controllers.h b/sys/dev/smartpqi/smartpqi_controllers.h index 6356159fd5f7..d5276f01a7e1 100644 --- a/sys/dev/smartpqi/smartpqi_controllers.h +++ b/sys/dev/smartpqi/smartpqi_controllers.h @@ -122,6 +122,7 @@ struct pqi_ident {0x9005, 0x028f, 0x1f51, 0x1044, PQI_HWIF_SRCV, "SmartHBA P6500-8i"}, {0x9005, 0x028f, 0x1f3f, 0x0610, PQI_HWIF_SRCV, "3SNIC SSSRAID 3S610"}, {0x9005, 0x028f, 0x207d, 0x4840, PQI_HWIF_SRCV, "HRDT TrustHBA H3100s-8i"}, + {0x9005, 0x028f, 0x207d, 0x4940, PQI_HWIF_SRCV, "HRDT TrustRAID D3102s-8i"}, /* (SRCx MSCC FVB 24x12G based) */ {0x9005, 0x028f, 0x103c, 0x1001, PQI_HWIF_SRCV, "MSCC FVB"}, @@ -151,11 +152,13 @@ struct pqi_ident {0x9005, 0x028f, 0x1cf2, 0x0B27, PQI_HWIF_SRCV, "ZTE SmartROC3100 SDPSA/B-18i 4G"}, {0x9005, 0x028f, 0x1cf2, 0x0B45, PQI_HWIF_SRCV, "ZTE SmartROC3100 SDPSA/B_L-18i 2G"}, {0x9005, 0x028f, 0x1cf2, 0x5445, PQI_HWIF_SRCV, "ZTE SmartROC3100 RM241-18i 2G"}, + {0x9005, 0x028f, 0x1cf2, 0x544D, PQI_HWIF_SRCV, "ZTE SmartROC3100 RM241B-18i 2G"}, {0x9005, 0x028f, 0x1cf2, 0x5446, PQI_HWIF_SRCV, "ZTE SmartROC3100 RM242-18i 4G"}, + {0x9005, 0x028f, 0x1cf2, 0x544E, PQI_HWIF_SRCV, "ZTE SmartROC3100 RM242B-18i 4G"}, + {0x9005, 0x028f, 0x1cf2, 0x5451, PQI_HWIF_SRCV, "ZTE SmartROC3100 RS231-8i 2G"}, + {0x9005, 0x028f, 0x1cf2, 0x5452, PQI_HWIF_SRCV, "ZTE SmartROC3100 RS232-8i 4G"}, {0x9005, 0x028f, 0x1cf2, 0x5449, PQI_HWIF_SRCV, "ZTE SmartROC3100 RS241-18i 2G"}, {0x9005, 0x028f, 0x1cf2, 0x544A, PQI_HWIF_SRCV, "ZTE SmartROC3100 RS242-18i 4G"}, - {0x9005, 0x028f, 0x1cf2, 0x544D, PQI_HWIF_SRCV, "ZTE SmartROC3100 RM241B-18i 2G"}, - {0x9005, 0x028f, 0x1cf2, 0x544E, PQI_HWIF_SRCV, "ZTE SmartROC3100 RM242B-18i 4G"}, {0x9005, 0x028f, 0x1bd4, 0x006F, PQI_HWIF_SRCV, "RS0804M5R16i"}, {0x9005, 0x028f, 0x1ff9, 0x006F, PQI_HWIF_SRCV, "RS0804M5R16i"}, {0x9005, 0x028f, 0x1f51, 0x1010, PQI_HWIF_SRCV, "SmartRAID P7504N-16i"}, @@ -178,6 +181,7 @@ struct pqi_ident {0x9005, 0x028f, 0x1458, 0x1000, PQI_HWIF_SRCV, "GIGABYTE SmartHBA CLN1832"}, {0x9005, 0x028f, 0x1cf2, 0x0B29, PQI_HWIF_SRCV, "ZTE SmartIOC2100 SDPSA/B_I-18i"}, {0x9005, 0x028f, 0x1cf2, 0x5447, PQI_HWIF_SRCV, "ZTE SmartIOC2100 RM243-18i"}, + {0x9005, 0x028f, 0x1cf2, 0x5453, PQI_HWIF_SRCV, "ZTE SmartIOC2100 RS233-8i"}, {0x9005, 0x028f, 0x1cf2, 0x544B, PQI_HWIF_SRCV, "ZTE SmartIOC2100 RS243-18i"}, {0x9005, 0x028f, 0x1cf2, 0x544F, PQI_HWIF_SRCV, "ZTE SmartIOC2100 RM243B-18i"}, {0x9005, 0x028f, 0x1bd4, 0x0071, PQI_HWIF_SRCV, "RS0800M5H16i"}, diff --git a/sys/dev/smartpqi/smartpqi_defines.h b/sys/dev/smartpqi/smartpqi_defines.h index fe2edf7a74a9..0277abd3e318 100644 --- a/sys/dev/smartpqi/smartpqi_defines.h +++ b/sys/dev/smartpqi/smartpqi_defines.h @@ -911,9 +911,14 @@ static inline uint64_t CALC_PERCENT_TOTAL(uint64_t val, uint64_t total) #define CCISS_GETDRIVVER _IOWR(SMARTPQI_IOCTL_BASE, 0, driver_info) #define CCISS_GETPCIINFO _IOWR(SMARTPQI_IOCTL_BASE, 1, pqi_pci_info_t) #define SMARTPQI_PASS_THRU _IOWR(SMARTPQI_IOCTL_BASE, 2, IOCTL_Command_struct) +#define SMARTPQI_BIG_PASS_THRU _IOWR(SMARTPQI_IOCTL_BASE, 3, BIG_IOCTL_Command_struct) #define CCISS_PASSTHRU _IOWR('C', 210, IOCTL_Command_struct) #define CCISS_REGNEWD _IO(CCISS_IOC_MAGIC, 14) +#if !defined(SMARTPQI_BIG_PASSTHRU_SUPPORTED) +#define SMARTPQI_BIG_PASSTHRU_SUPPORTED _IO(SMARTPQI_IOCTL_BASE, 4) +#endif + /*IOCTL pci_info structure */ typedef struct pqi_pci_info { @@ -939,12 +944,12 @@ typedef uint8_t *passthru_buf_type_t; #define PQISRC_DRIVER_MAJOR __FreeBSD__ #if __FreeBSD__ <= 14 -#define PQISRC_DRIVER_MINOR 4660 +#define PQISRC_DRIVER_MINOR 4690 #else -#define PQISRC_DRIVER_MINOR 0 +#define PQISRC_DRIVER_MINOR 2 #endif #define PQISRC_DRIVER_RELEASE 0 -#define PQISRC_DRIVER_REVISION 2002 +#define PQISRC_DRIVER_REVISION 2008 #define STR(s) # s #define PQISRC_VERSION(a, b, c, d) STR(a.b.c-d) diff --git a/sys/dev/smartpqi/smartpqi_event.c b/sys/dev/smartpqi/smartpqi_event.c index 761bb5588ff9..77a70f9fb031 100644 --- a/sys/dev/smartpqi/smartpqi_event.c +++ b/sys/dev/smartpqi/smartpqi_event.c @@ -417,7 +417,7 @@ pqisrc_report_event_config(pqisrc_softstate_t *softs) softs->event_config.num_event_descriptors = MIN(event_config_p->num_event_descriptors, PQI_MAX_EVENT_DESCRIPTORS) ; - for (i = 0; i < softs->event_config.num_event_descriptors; i++) { + for (i=0; i < softs->event_config.num_event_descriptors; i++) { softs->event_config.descriptors[i].event_type = event_config_p->descriptors[i].event_type; } @@ -477,7 +477,7 @@ pqisrc_set_event_config(pqisrc_softstate_t *softs) event_config_p->num_event_descriptors = softs->event_config.num_event_descriptors; - for (i = 0; i < softs->event_config.num_event_descriptors; i++) { + for (i=0; i < softs->event_config.num_event_descriptors; i++) { event_config_p->descriptors[i].event_type = softs->event_config.descriptors[i].event_type; if( pqisrc_event_type_to_event_index(event_config_p->descriptors[i].event_type) != -1) diff --git a/sys/dev/smartpqi/smartpqi_features.c b/sys/dev/smartpqi/smartpqi_features.c index 2a53dbe654b1..ae6114ef066b 100644 --- a/sys/dev/smartpqi/smartpqi_features.c +++ b/sys/dev/smartpqi/smartpqi_features.c @@ -1,5 +1,5 @@ /*- - * Copyright 2016-2023 Microchip Technology, Inc. and/or its subsidiaries. + * Copyright 2016-2025 Microchip Technology, Inc. and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/sys/dev/smartpqi/smartpqi_helper.c b/sys/dev/smartpqi/smartpqi_helper.c index 0a4bf6ec21cb..9454fa8b63ab 100644 --- a/sys/dev/smartpqi/smartpqi_helper.c +++ b/sys/dev/smartpqi/smartpqi_helper.c @@ -333,6 +333,8 @@ check_struct_sizes(void) 64 bit and 32 bit system*/ ASSERT(sizeof(IOCTL_Command_struct)== 86 || sizeof(IOCTL_Command_struct)== 82); + ASSERT(sizeof(BIG_IOCTL_Command_struct)== 88 || + sizeof(BIG_IOCTL_Command_struct)== 84); ASSERT(sizeof(struct bmic_host_wellness_driver_version)== 44); ASSERT(sizeof(struct bmic_host_wellness_time)== 20); ASSERT(sizeof(struct pqi_dev_adminq_cap)== 8); @@ -386,7 +388,7 @@ void check_device_pending_commands_to_complete(pqisrc_softstate_t *softs, pqi_scsi_dev_t *device) { uint32_t tag = softs->max_outstanding_io, active_requests; - uint64_t timeout = 0, delay_in_usec = 1000; /* In micro Seconds */ + uint64_t timeout = 0, delay_in_usec = 1000; /* In microseconds */ rcb_t* rcb; DBG_FUNC("IN\n"); diff --git a/sys/dev/smartpqi/smartpqi_helper.h b/sys/dev/smartpqi/smartpqi_helper.h index 507caeb9b0b6..362eb0d05379 100644 --- a/sys/dev/smartpqi/smartpqi_helper.h +++ b/sys/dev/smartpqi/smartpqi_helper.h @@ -1,5 +1,5 @@ /*- - * Copyright 2016-2023 Microchip Technology, Inc. and/or its subsidiaries. + * Copyright 2016-2025 Microchip Technology, Inc. and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/sys/dev/smartpqi/smartpqi_includes.h b/sys/dev/smartpqi/smartpqi_includes.h index a8682e8a848d..30846b1a1d6c 100644 --- a/sys/dev/smartpqi/smartpqi_includes.h +++ b/sys/dev/smartpqi/smartpqi_includes.h @@ -1,5 +1,5 @@ /*- - * Copyright 2016-2023 Microchip Technology, Inc. and/or its subsidiaries. + * Copyright 2016-2025 Microchip Technology, Inc. and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/sys/dev/smartpqi/smartpqi_ioctl.c b/sys/dev/smartpqi/smartpqi_ioctl.c index 0517c0513cce..7fd68b1a3263 100644 --- a/sys/dev/smartpqi/smartpqi_ioctl.c +++ b/sys/dev/smartpqi/smartpqi_ioctl.c @@ -169,6 +169,13 @@ smartpqi_ioctl(struct cdev *cdev, u_long cmd, caddr_t udata, pqi_status = pqisrc_passthru_ioctl(softs, udata, 0); bsd_status = pqi_status_to_bsd_ioctl_status(pqi_status); break; + case SMARTPQI_BIG_PASS_THRU: + pqi_status = pqisrc_big_passthru_ioctl(softs, udata, 0); + bsd_status = pqi_status_to_bsd_ioctl_status(pqi_status); + break; + case SMARTPQI_BIG_PASSTHRU_SUPPORTED: + bsd_status = BSD_SUCCESS; + break; case CCISS_REGNEWD: pqi_status = pqisrc_scan_devices(softs); bsd_status = pqi_status_to_bsd_ioctl_status(pqi_status); @@ -422,3 +429,195 @@ out: DBG_FUNC("Failed OUT\n"); return PQI_STATUS_FAILURE; } + +/* + * Function used to send big passthru commands to adapter + * to support management tools. For eg. ssacli, sscon. + */ +int +pqisrc_big_passthru_ioctl(struct pqisrc_softstate *softs, void *arg, int mode) +{ + int ret; + char *drv_buf = NULL; + uint32_t tag = 0; + BIG_IOCTL_Command_struct *iocommand = (BIG_IOCTL_Command_struct *)arg; + dma_mem_t ioctl_dma_buf; + pqisrc_raid_req_t request; + raid_path_error_info_elem_t error_info; + ib_queue_t *ib_q = &softs->op_raid_ib_q[PQI_DEFAULT_IB_QUEUE]; + ob_queue_t const *ob_q = &softs->op_ob_q[PQI_DEFAULT_IB_QUEUE]; + rcb_t *rcb = NULL; + + memset(&request, 0, sizeof(request)); + memset(&error_info, 0, sizeof(error_info)); + + DBG_FUNC("IN\n"); + + if (pqisrc_ctrl_offline(softs)) + return PQI_STATUS_FAILURE; + + if (!arg) + return PQI_STATUS_FAILURE; + + if (iocommand->buf_size < 1 && + iocommand->Request.Type.Direction != PQIIOCTL_NONE) + return PQI_STATUS_FAILURE; + if (iocommand->Request.CDBLen > sizeof(request.cmd.cdb)) + return PQI_STATUS_FAILURE; + + switch (iocommand->Request.Type.Direction) { + case PQIIOCTL_NONE: + case PQIIOCTL_WRITE: + case PQIIOCTL_READ: + case PQIIOCTL_BIDIRECTIONAL: + break; + default: + return PQI_STATUS_FAILURE; + } + + if (iocommand->buf_size > 0) { + memset(&ioctl_dma_buf, 0, sizeof(struct dma_mem)); + os_strlcpy(ioctl_dma_buf.tag, "Ioctl_PassthruCmd_Buffer", sizeof(ioctl_dma_buf.tag)); + ioctl_dma_buf.size = iocommand->buf_size; + ioctl_dma_buf.align = PQISRC_DEFAULT_DMA_ALIGN; + /* allocate memory */ + ret = os_dma_mem_alloc(softs, &ioctl_dma_buf); + if (ret) { + DBG_ERR("Failed to Allocate dma mem for Ioctl PassthruCmd Buffer : %d\n", ret); + goto out; + } + + DBG_IO("ioctl_dma_buf.dma_addr = %p\n",(void*)ioctl_dma_buf.dma_addr); + DBG_IO("ioctl_dma_buf.virt_addr = %p\n",(void*)ioctl_dma_buf.virt_addr); + + drv_buf = (char *)ioctl_dma_buf.virt_addr; + if (iocommand->Request.Type.Direction & PQIIOCTL_WRITE) { + ret = os_copy_from_user(softs, (void *)drv_buf, (void *)iocommand->buf, iocommand->buf_size, mode); + if (ret != 0) { + goto free_mem; + } + } + } + + request.header.iu_type = PQI_IU_TYPE_RAID_PATH_IO_REQUEST; + request.header.iu_length = offsetof(pqisrc_raid_req_t, sg_descriptors[1]) - + PQI_REQUEST_HEADER_LENGTH; + memcpy(request.lun_number, iocommand->LUN_info.LunAddrBytes, + sizeof(request.lun_number)); + memcpy(request.cmd.cdb, iocommand->Request.CDB, iocommand->Request.CDBLen); + request.additional_cdb_bytes_usage = PQI_ADDITIONAL_CDB_BYTES_0; + + switch (iocommand->Request.Type.Direction) { + case PQIIOCTL_NONE: + request.data_direction = SOP_DATA_DIR_NONE; + break; + case PQIIOCTL_WRITE: + request.data_direction = SOP_DATA_DIR_FROM_DEVICE; + break; + case PQIIOCTL_READ: + request.data_direction = SOP_DATA_DIR_TO_DEVICE; + break; + case PQIIOCTL_BIDIRECTIONAL: + request.data_direction = SOP_DATA_DIR_BIDIRECTIONAL; + break; + } + + request.task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE; + if (iocommand->buf_size > 0) { + request.buffer_length = iocommand->buf_size; + request.sg_descriptors[0].addr = ioctl_dma_buf.dma_addr; + request.sg_descriptors[0].len = iocommand->buf_size; + request.sg_descriptors[0].flags = SG_FLAG_LAST; + } + tag = pqisrc_get_tag(&softs->taglist); + if (INVALID_ELEM == tag) { + DBG_ERR("Tag not available\n"); + goto free_mem; + } + request.request_id = tag; + request.response_queue_id = ob_q->q_id; + request.error_index = request.request_id; + if (softs->timeout_in_passthrough) { + request.timeout_in_sec = iocommand->Request.Timeout; + } + + rcb = &softs->rcb[tag]; + rcb->success_cmp_callback = pqisrc_process_internal_raid_response_success; + rcb->error_cmp_callback = pqisrc_process_internal_raid_response_error; + rcb->tag = tag; + rcb->req_pending = true; + /* Submit Command */ + ret = pqisrc_submit_cmnd(softs, ib_q, &request); + if (ret != PQI_STATUS_SUCCESS) { + DBG_ERR("Unable to submit command\n"); + goto err_out; + } + + ret = pqisrc_wait_on_condition(softs, rcb, PQISRC_PASSTHROUGH_CMD_TIMEOUT); + if (ret != PQI_STATUS_SUCCESS) { + DBG_ERR("Passthru IOCTL cmd timed out !!\n"); + goto err_out; + } + + memset(&iocommand->error_info, 0, sizeof(iocommand->error_info)); + + + if (rcb->status) { + size_t sense_data_length; + + memcpy(&error_info, rcb->error_info, sizeof(error_info)); + iocommand->error_info.ScsiStatus = error_info.status; + sense_data_length = error_info.sense_data_len; + + if (!sense_data_length) + sense_data_length = error_info.resp_data_len; + + if (sense_data_length && + (sense_data_length > sizeof(error_info.data))) + sense_data_length = sizeof(error_info.data); + + if (sense_data_length) { + if (sense_data_length > + sizeof(iocommand->error_info.SenseInfo)) + sense_data_length = + sizeof(iocommand->error_info.SenseInfo); + memcpy (iocommand->error_info.SenseInfo, + error_info.data, sense_data_length); + iocommand->error_info.SenseLen = sense_data_length; + } + + if (error_info.data_out_result == PQI_RAID_DATA_IN_OUT_UNDERFLOW) { + rcb->status = PQI_STATUS_SUCCESS; + } + } + + if (rcb->status == PQI_STATUS_SUCCESS && iocommand->buf_size > 0 && + (iocommand->Request.Type.Direction & PQIIOCTL_READ)) { + + ret = os_copy_to_user(softs, (void*)iocommand->buf, (void*)drv_buf, iocommand->buf_size, mode); + if (ret != 0) { + DBG_ERR("Failed to copy the response\n"); + goto err_out; + } + } + + os_reset_rcb(rcb); + pqisrc_put_tag(&softs->taglist, request.request_id); + if (iocommand->buf_size > 0) + os_dma_mem_free(softs,&ioctl_dma_buf); + + DBG_FUNC("OUT\n"); + return PQI_STATUS_SUCCESS; + +err_out: + os_reset_rcb(rcb); + pqisrc_put_tag(&softs->taglist, request.request_id); + +free_mem: + if (iocommand->buf_size > 0) + os_dma_mem_free(softs, &ioctl_dma_buf); + +out: + DBG_FUNC("Failed OUT\n"); + return PQI_STATUS_FAILURE; +}
\ No newline at end of file diff --git a/sys/dev/smartpqi/smartpqi_ioctl.h b/sys/dev/smartpqi/smartpqi_ioctl.h index 633465aeb8cd..ef4678ad5a3b 100644 --- a/sys/dev/smartpqi/smartpqi_ioctl.h +++ b/sys/dev/smartpqi/smartpqi_ioctl.h @@ -1,5 +1,5 @@ /*- - * Copyright 2016-2023 Microchip Technology, Inc. and/or its subsidiaries. + * Copyright 2016-2025 Microchip Technology, Inc. and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -137,5 +137,12 @@ typedef struct pqi_ioctl_passthruCmd_struct { }OS_ATTRIBUTE_PACKED IOCTL_Command_struct; +typedef struct _BIG_IOCTL_Command_struct { + LUNAddr_struct LUN_info; + RequestBlock_struct Request; + ErrorInfo_struct error_info; + DWORD buf_size; /* size in bytes of the buf */ + passthru_buf_type_t buf; +}OS_ATTRIBUTE_PACKED BIG_IOCTL_Command_struct; #endif /* _PQI_IOCTL_H_ */ diff --git a/sys/dev/smartpqi/smartpqi_main.c b/sys/dev/smartpqi/smartpqi_main.c index 6274ecc957d4..dea7809a1b8e 100644 --- a/sys/dev/smartpqi/smartpqi_main.c +++ b/sys/dev/smartpqi/smartpqi_main.c @@ -25,7 +25,7 @@ /* - * Driver for the Microsemi Smart storage controllers + * Driver for the Microchip Smart storage controllers */ #include "smartpqi_includes.h" diff --git a/sys/dev/smartpqi/smartpqi_mem.c b/sys/dev/smartpqi/smartpqi_mem.c index 239d619968d2..08f806681dce 100644 --- a/sys/dev/smartpqi/smartpqi_mem.c +++ b/sys/dev/smartpqi/smartpqi_mem.c @@ -1,5 +1,5 @@ /*- - * Copyright 2016-2023 Microchip Technology, Inc. and/or its subsidiaries. + * Copyright 2016-2025 Microchip Technology, Inc. and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/sys/dev/smartpqi/smartpqi_prototypes.h b/sys/dev/smartpqi/smartpqi_prototypes.h index fdf69e38fa59..7abce9c3789d 100644 --- a/sys/dev/smartpqi/smartpqi_prototypes.h +++ b/sys/dev/smartpqi/smartpqi_prototypes.h @@ -251,6 +251,7 @@ int pqisrc_process_task_management_response(pqisrc_softstate_t *, /* smartpqi_ioctl.c*/ int pqisrc_passthru_ioctl(struct pqisrc_softstate *, void *, int); +int pqisrc_big_passthru_ioctl(struct pqisrc_softstate *, void *, int); /* Functions Prototypes */ /* smartpqi_mem.c */ diff --git a/sys/dev/smartpqi/smartpqi_queue.c b/sys/dev/smartpqi/smartpqi_queue.c index 22bc2db572f8..e8a467531aa4 100644 --- a/sys/dev/smartpqi/smartpqi_queue.c +++ b/sys/dev/smartpqi/smartpqi_queue.c @@ -688,7 +688,7 @@ pqisrc_create_op_obq(pqisrc_softstate_t *softs, int i = 0; DBG_WARN("Error Status Descriptors\n"); for (i = 0; i < 4; i++) - DBG_WARN(" %x ",admin_resp.resp_type.create_op_oq.status_desc[i]); + DBG_WARN(" %x\n",admin_resp.resp_type.create_op_oq.status_desc[i]); } DBG_FUNC("OUT ret : %d\n", ret); @@ -731,7 +731,7 @@ pqisrc_create_op_ibq(pqisrc_softstate_t *softs, int i = 0; DBG_WARN("Error Status Decsriptors\n"); for (i = 0; i < 4; i++) - DBG_WARN(" %x ",admin_resp.resp_type.create_op_iq.status_desc[i]); + DBG_WARN(" %x\n",admin_resp.resp_type.create_op_iq.status_desc[i]); } DBG_FUNC("OUT ret : %d\n", ret); diff --git a/sys/dev/smartpqi/smartpqi_request.c b/sys/dev/smartpqi/smartpqi_request.c index e6ba41a814b2..c5f8ac3c41ba 100644 --- a/sys/dev/smartpqi/smartpqi_request.c +++ b/sys/dev/smartpqi/smartpqi_request.c @@ -1180,7 +1180,7 @@ fill_lba_for_scsi_rw(pqisrc_softstate_t *softs, uint8_t *cdb, aio_req_locator_t /* determine whether writes to certain types of RAID are supported. */ -static inline boolean_t +static boolean_t pqisrc_is_supported_write(pqisrc_softstate_t const *softs, pqi_scsi_dev_t const *device) { diff --git a/sys/dev/smartpqi/smartpqi_tag.c b/sys/dev/smartpqi/smartpqi_tag.c index 214b1bd3ce7d..d9680a65f265 100644 --- a/sys/dev/smartpqi/smartpqi_tag.c +++ b/sys/dev/smartpqi/smartpqi_tag.c @@ -1,5 +1,5 @@ /*- - * Copyright 2016-2023 Microchip Technology, Inc. and/or its subsidiaries. + * Copyright 2016-2025 Microchip Technology, Inc. and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/sys/dev/sym/sym_hipd.c b/sys/dev/sym/sym_hipd.c index b4e5c1075fb4..1399eadb21fb 100644 --- a/sys/dev/sym/sym_hipd.c +++ b/sys/dev/sym/sym_hipd.c @@ -58,7 +58,6 @@ */ #include <sys/cdefs.h> -#define SYM_DRIVER_NAME "sym-1.6.5-20000902" /* #define SYM_DEBUG_GENERIC_SUPPORT */ @@ -114,17 +113,15 @@ typedef u_int32_t u32; #include <dev/sym/sym_fw.h> /* - * IA32 architecture does not reorder STORES and prevents - * LOADS from passing STORES. It is called `program order' - * by Intel and allows device drivers to deal with memory - * ordering by only ensuring that the code is not reordered - * by the compiler when ordering is required. - * Other architectures implement a weaker ordering that - * requires memory barriers (and also IO barriers when they - * make sense) to be used. + * With uncacheable memory, x86 does not reorder STORES and prevents LOADS + * from passing STORES. For ensuring this program order, we still need to + * employ compiler barriers, though, when the ordering of LOADS and STORES + * matters. + * Other architectures may implement weaker ordering guarantees and, thus, + * require memory barriers (and also IO barriers) to be used. */ #if defined __i386__ || defined __amd64__ -#define MEMORY_BARRIER() do { ; } while(0) +#define MEMORY_BARRIER() __compiler_membar() #elif defined __powerpc__ #define MEMORY_BARRIER() __asm__ volatile("eieio; sync" : : : "memory") #elif defined __arm__ @@ -594,10 +591,10 @@ static m_addr_t ___dma_getp(m_pool_s *mp) goto out_err; if (bus_dmamem_alloc(mp->dmat, &vaddr, - BUS_DMA_COHERENT | BUS_DMA_WAITOK, &vbp->dmamap)) + BUS_DMA_COHERENT | BUS_DMA_NOCACHE | BUS_DMA_WAITOK, &vbp->dmamap)) goto out_err; - bus_dmamap_load(mp->dmat, vbp->dmamap, vaddr, - MEMO_CLUSTER_SIZE, getbaddrcb, &baddr, BUS_DMA_NOWAIT); + bus_dmamap_load(mp->dmat, vbp->dmamap, vaddr, MEMO_CLUSTER_SIZE, + getbaddrcb, &baddr, BUS_DMA_NOWAIT); if (baddr) { int hc = VTOB_HASH_CODE(vaddr); vbp->vaddr = (m_addr_t) vaddr; @@ -1484,7 +1481,7 @@ struct sym_hcb { u32 scr_ram_seg; /* - * Chip and controller indentification. + * Chip and controller identification. */ device_t device; @@ -1534,7 +1531,6 @@ struct sym_hcb { struct resource *io_res; struct resource *mmio_res; struct resource *ram_res; - int ram_id; void *intr; /* @@ -1558,8 +1554,6 @@ struct sym_hcb { * BUS addresses of the chip */ vm_offset_t mmio_ba; /* MMIO BUS address */ - int mmio_ws; /* MMIO Window size */ - vm_offset_t ram_ba; /* RAM BUS address */ int ram_ws; /* RAM window size */ @@ -2012,8 +2006,8 @@ static void sym_fw_bind_script (hcb_p np, u32 *start, int len) * command. */ if (opcode == 0) { - printf ("%s: ERROR0 IN SCRIPT at %d.\n", - sym_name(np), (int) (cur-start)); + device_printf(np->device, "ERROR0 IN SCRIPT at %d.\n", + (int)(cur-start)); MDELAY (10000); ++cur; continue; @@ -2056,8 +2050,9 @@ static void sym_fw_bind_script (hcb_p np, u32 *start, int len) tmp1 = cur[1]; tmp2 = cur[2]; if ((tmp1 ^ tmp2) & 3) { - printf ("%s: ERROR1 IN SCRIPT at %d.\n", - sym_name(np), (int) (cur-start)); + device_printf(np->device, + "ERROR1 IN SCRIPT at %d.\n", + (int)(cur-start)); MDELAY (10000); } /* @@ -2248,10 +2243,11 @@ static void sym_update_dflags(hcb_p np, u_char *flags, struct ccb_trans_settings *cts); static const struct sym_pci_chip *sym_find_pci_chip (device_t dev); -static int sym_pci_probe (device_t dev); -static int sym_pci_attach (device_t dev); -static void sym_pci_free (hcb_p np); +static device_probe_t sym_pci_probe; +static device_attach_t sym_pci_attach; +static device_detach_t sym_pci_detach; + static int sym_cam_attach (hcb_p np); static void sym_cam_free (hcb_p np); @@ -2426,8 +2422,8 @@ static void sym_print_targets_flag(hcb_p np, int mask, char *msg) continue; if (np->target[i].usrflags & mask) { if (!cnt++) - printf("%s: %s disabled for targets", - sym_name(np), msg); + device_printf(np->device, + "%s disabled for targets", msg); printf(" %d", i); } } @@ -2750,41 +2746,42 @@ static int sym_prepare_setting(hcb_p np, struct sym_nvram *nvram) * Let user know about the settings. */ i = nvram->type; - printf("%s: %s NVRAM, ID %d, Fast-%d, %s, %s\n", sym_name(np), - i == SYM_SYMBIOS_NVRAM ? "Symbios" : - (i == SYM_TEKRAM_NVRAM ? "Tekram" : "No"), - np->myaddr, - (np->features & FE_ULTRA3) ? 80 : - (np->features & FE_ULTRA2) ? 40 : - (np->features & FE_ULTRA) ? 20 : 10, - sym_scsi_bus_mode(np->scsi_mode), - (np->rv_scntl0 & 0xa) ? "parity checking" : "NO parity"); + device_printf(np->device, "%s NVRAM, ID %d, Fast-%d, %s, %s\n", + i == SYM_SYMBIOS_NVRAM ? "Symbios" : + (i == SYM_TEKRAM_NVRAM ? "Tekram" : "No"), + np->myaddr, + (np->features & FE_ULTRA3) ? 80 : + (np->features & FE_ULTRA2) ? 40 : + (np->features & FE_ULTRA) ? 20 : 10, + sym_scsi_bus_mode(np->scsi_mode), + (np->rv_scntl0 & 0xa) ? "parity checking" : "NO parity"); /* * Tell him more on demand. */ if (sym_verbose) { - printf("%s: %s IRQ line driver%s\n", - sym_name(np), - np->rv_dcntl & IRQM ? "totem pole" : "open drain", - np->ram_ba ? ", using on-chip SRAM" : ""); - printf("%s: using %s firmware.\n", sym_name(np), np->fw_name); + device_printf(np->device, "%s IRQ line driver%s\n", + np->rv_dcntl & IRQM ? "totem pole" : "open drain", + np->ram_ba ? ", using on-chip SRAM" : ""); + device_printf(np->device, "using %s firmware.\n", np->fw_name); if (np->features & FE_NOPM) - printf("%s: handling phase mismatch from SCRIPTS.\n", - sym_name(np)); + device_printf(np->device, + "handling phase mismatch from SCRIPTS.\n"); } /* * And still more. */ if (sym_verbose > 1) { - printf ("%s: initial SCNTL3/DMODE/DCNTL/CTEST3/4/5 = " - "(hex) %02x/%02x/%02x/%02x/%02x/%02x\n", - sym_name(np), np->sv_scntl3, np->sv_dmode, np->sv_dcntl, - np->sv_ctest3, np->sv_ctest4, np->sv_ctest5); + device_printf(np->device, + "initial SCNTL3/DMODE/DCNTL/CTEST3/4/5 = " + "(hex) %02x/%02x/%02x/%02x/%02x/%02x\n", + np->sv_scntl3, np->sv_dmode, np->sv_dcntl, np->sv_ctest3, + np->sv_ctest4, np->sv_ctest5); - printf ("%s: final SCNTL3/DMODE/DCNTL/CTEST3/4/5 = " - "(hex) %02x/%02x/%02x/%02x/%02x/%02x\n", - sym_name(np), np->rv_scntl3, np->rv_dmode, np->rv_dcntl, - np->rv_ctest3, np->rv_ctest4, np->rv_ctest5); + device_printf(np->device, + "final SCNTL3/DMODE/DCNTL/CTEST3/4/5 = " + "(hex) %02x/%02x/%02x/%02x/%02x/%02x\n", + np->rv_scntl3, np->rv_dmode, np->rv_dcntl, + np->rv_ctest3, np->rv_ctest4, np->rv_ctest5); } /* * Let user be aware of targets that have some disable flags set. @@ -2911,7 +2908,7 @@ static void sym_put_start_queue(hcb_p np, ccb_p cp) np->squeueput = qidx; if (DEBUG_FLAGS & DEBUG_QUEUE) - printf ("%s: queuepos=%d.\n", sym_name (np), np->squeueput); + device_printf(np->device, "queuepos=%d.\n", np->squeueput); /* * Script processor may be waiting for reselect. @@ -2965,8 +2962,8 @@ static void sym_soft_reset (hcb_p np) } } if (!i) - printf("%s: unable to abort current chip operation.\n", - sym_name(np)); + device_printf(np->device, + "unable to abort current chip operation.\n"); sym_chip_reset (np); } @@ -3016,13 +3013,12 @@ static int sym_reset_scsi_bus(hcb_p np, int enab_int) term &= 0x3ffff; if (term != (2<<7)) { - printf("%s: suspicious SCSI data while resetting the BUS.\n", - sym_name(np)); - printf("%s: %sdp0,d7-0,rst,req,ack,bsy,sel,atn,msg,c/d,i/o = " - "0x%lx, expecting 0x%lx\n", - sym_name(np), - (np->features & FE_WIDE) ? "dp1,d15-8," : "", - (u_long)term, (u_long)(2<<7)); + device_printf(np->device, + "suspicious SCSI data while resetting the BUS.\n"); + device_printf(np->device, + "%sdp0,d7-0,rst,req,ack,bsy,sel,atn,msg,c/d,i/o = " + "0x%lx, expecting 0x%lx\n", (np->features & FE_WIDE) ? + "dp1,d15-8," : "", (u_long)term, (u_long)(2 << 7)); if (SYM_SETUP_SCSI_BUS_CHECK == 1) retv = 1; } @@ -3062,10 +3058,9 @@ static int sym_wakeup_done (hcb_p np) MEMORY_BARRIER(); sym_complete_ok (np, cp); ++n; - } - else - printf ("%s: bad DSA (%x) in done queue.\n", - sym_name(np), (u_int) dsa); + } else + device_printf(np->device, + "bad DSA (%x) in done queue.\n", (u_int)dsa); } np->dqueueget = i; @@ -3286,8 +3281,8 @@ static void sym_init (hcb_p np, int reason) */ if (np->ram_ba) { if (sym_verbose > 1) - printf ("%s: Downloading SCSI SCRIPTS.\n", - sym_name(np)); + device_printf(np->device, + "Downloading SCSI SCRIPTS.\n"); if (np->ram_ws == 8192) { OUTRAM_OFF(4096, np->scriptb0, np->scriptb_sz); OUTL (nc_mmws, np->scr_ram_seg); @@ -3710,11 +3705,11 @@ static void sym_log_hard_error(hcb_p np, u_short sist, u_char dstat) if (((script_ofs & 3) == 0) && (unsigned)script_ofs < script_size) { - printf ("%s: script cmd = %08x\n", sym_name(np), - scr_to_cpu((int) *(u32 *)(script_base + script_ofs))); + device_printf(np->device, "script cmd = %08x\n", + scr_to_cpu((int) *(u32 *)(script_base + script_ofs))); } - printf ("%s: regdump:", sym_name(np)); + device_printf(np->device, "regdump:"); for (i = 0; i < 24; i++) printf (" %02x", (unsigned)INB_OFF(i)); printf (".\n"); @@ -3727,8 +3722,8 @@ static void sym_log_hard_error(hcb_p np, u_short sist, u_char dstat) pci_sts = pci_read_config(np->device, PCIR_STATUS, 2); if (pci_sts & 0xf900) { pci_write_config(np->device, PCIR_STATUS, pci_sts, 2); - printf("%s: PCI STATUS = 0x%04x\n", - sym_name(np), pci_sts & 0xf900); + device_printf(np->device, "PCI STATUS = 0x%04x\n", + pci_sts & 0xf900); } } } @@ -3933,9 +3928,9 @@ unknown_int: * We just miss the cause of the interrupt. :( * Print a message. The timeout will do the real work. */ - printf( "%s: unknown interrupt(s) ignored, " - "ISTAT=0x%x DSTAT=0x%x SIST=0x%x\n", - sym_name(np), istat, dstat, sist); + device_printf(np->device, + "unknown interrupt(s) ignored, ISTAT=0x%x DSTAT=0x%x SIST=0x%x\n", + istat, dstat, sist); } static void sym_intr(void *arg) @@ -4050,7 +4045,7 @@ static void sym_int_sto (hcb_p np) */ static void sym_int_udc (hcb_p np) { - printf ("%s: unexpected disconnect\n", sym_name(np)); + device_printf(np->device, "unexpected disconnect\n"); sym_recover_scsi_int(np, HS_UNEXPECTED); } @@ -4117,8 +4112,9 @@ static void sym_int_par (hcb_p np, u_short sist) int phase = cmd & 7; ccb_p cp = sym_ccb_from_dsa(np, dsa); - printf("%s: SCSI parity error detected: SCR1=%d DBC=%x SBCL=%x\n", - sym_name(np), hsts, dbc, sbcl); + device_printf(np->device, + "SCSI parity error detected: SCR1=%d DBC=%x SBCL=%x\n", hsts, dbc, + sbcl); /* * Check that the chip is connected to the SCSI BUS. @@ -4305,14 +4301,14 @@ static void sym_int_ma (hcb_p np) } if (!vdsp) { - printf ("%s: interrupted SCRIPT address not found.\n", - sym_name (np)); + device_printf(np->device, + "interrupted SCRIPT address not found.\n"); goto reset_all; } if (!cp) { - printf ("%s: SCSI phase error fixup: CCB already dequeued.\n", - sym_name (np)); + device_printf(np->device, + "SCSI phase error fixup: CCB already dequeued.\n"); goto reset_all; } @@ -6757,15 +6753,15 @@ restart_test: dstat = INB (nc_dstat); #if 1 /* Band aiding for broken hardwares that fail PCI parity */ if ((dstat & MDPE) && (np->rv_ctest4 & MPEE)) { - printf ("%s: PCI DATA PARITY ERROR DETECTED - " - "DISABLING MASTER DATA PARITY CHECKING.\n", - sym_name(np)); + device_printf(np->device, "PCI DATA PARITY ERROR DETECTED - " + "DISABLING MASTER DATA PARITY CHECKING.\n"); np->rv_ctest4 &= ~MPEE; goto restart_test; } #endif if (dstat & (MDPE|BF|IID)) { - printf ("CACHE TEST FAILED: DMA error (dstat=0x%02x).", dstat); + device_printf(np->device, + "CACHE TEST FAILED: DMA error (dstat=0x%02x).\n", dstat); return (0x80); } /* @@ -6783,28 +6779,32 @@ restart_test: * Check termination position. */ if (pc != SCRIPTB0_BA (np, snoopend)+8) { - printf ("CACHE TEST FAILED: script execution failed.\n"); - printf ("start=%08lx, pc=%08lx, end=%08lx\n", - (u_long) SCRIPTB0_BA (np, snooptest), (u_long) pc, - (u_long) SCRIPTB0_BA (np, snoopend) +8); + device_printf(np->device, + "CACHE TEST FAILED: script execution failed.\n"); + device_printf(np->device, "start=%08lx, pc=%08lx, end=%08lx\n", + (u_long)SCRIPTB0_BA(np, snooptest), (u_long)pc, + (u_long)SCRIPTB0_BA(np, snoopend) + 8); return (0x40); } /* * Show results. */ if (host_wr != sym_rd) { - printf ("CACHE TEST FAILED: host wrote %d, chip read %d.\n", - (int) host_wr, (int) sym_rd); + device_printf(np->device, + "CACHE TEST FAILED: host wrote %d, chip read %d.\n", + (int)host_wr, (int)sym_rd); err |= 1; } if (host_rd != sym_wr) { - printf ("CACHE TEST FAILED: chip wrote %d, host read %d.\n", - (int) sym_wr, (int) host_rd); + device_printf(np->device, + "CACHE TEST FAILED: chip wrote %d, host read %d.\n", + (int)sym_wr, (int)host_rd); err |= 2; } if (sym_bk != sym_wr) { - printf ("CACHE TEST FAILED: chip wrote %d, read back %d.\n", - (int) sym_wr, (int) sym_bk); + device_printf(np->device, + "CACHE TEST FAILED: chip wrote %d, read back %d.\n", + (int)sym_wr, (int)sym_bk); err |= 4; } @@ -6843,7 +6843,7 @@ static void sym_selectclock(hcb_p np, u_char scntl3) } if (sym_verbose >= 2) - printf ("%s: enabling clock multiplier\n", sym_name(np)); + device_printf(np->device, "enabling clock multiplier\n"); OUTB(nc_stest1, DBLEN); /* Enable clock multiplier */ /* @@ -6855,8 +6855,8 @@ static void sym_selectclock(hcb_p np, u_char scntl3) while (!(INB(nc_stest4) & LCKFRQ) && --i > 0) UDELAY (20); if (!i) - printf("%s: the chip cannot lock the frequency\n", - sym_name(np)); + device_printf(np->device, + "the chip cannot lock the frequency\n"); } else UDELAY (20); OUTB(nc_stest3, HSC); /* Halt the scsi clock */ @@ -6911,8 +6911,8 @@ static unsigned getfreq (hcb_p np, int gen) f = ms ? ((1 << gen) * 4340) / ms : 0; if (sym_verbose >= 2) - printf ("%s: Delay (GEN=%d): %u msec, %u KHz\n", - sym_name(np), gen, ms, f); + device_printf(np->device, "Delay (GEN=%d): %u msec, %u KHz\n", + gen, ms, f); return f; } @@ -6954,7 +6954,7 @@ static void sym_getclock (hcb_p np, int mult) */ if (mult > 1 && (stest1 & (DBLEN+DBLSEL)) == DBLEN+DBLSEL) { if (sym_verbose >= 2) - printf ("%s: clock multiplier found\n", sym_name(np)); + device_printf(np->device, "clock multiplier found\n"); np->multiplier = mult; } @@ -6968,7 +6968,7 @@ static void sym_getclock (hcb_p np, int mult) f1 = sym_getfreq (np); if (sym_verbose) - printf ("%s: chip clock is %uKHz\n", sym_name(np), f1); + device_printf(np->device, "chip clock is %uKHz\n", f1); if (f1 < 45000) f1 = 40000; else if (f1 < 55000) f1 = 50000; @@ -6976,8 +6976,8 @@ static void sym_getclock (hcb_p np, int mult) if (f1 < 80000 && mult > 1) { if (sym_verbose >= 2) - printf ("%s: clock multiplier assumed\n", - sym_name(np)); + device_printf(np->device, + "clock multiplier assumed\n"); np->multiplier = mult; } } else { @@ -7146,7 +7146,7 @@ static void sym_complete_error (hcb_p np, ccb_p cp) sense_returned; else csio->sense_resid = 0; - bcopy(cp->sns_bbuf, &csio->sense_data, + memcpy(&csio->sense_data, cp->sns_bbuf, MIN(csio->sense_len, sense_returned)); #if 0 /* @@ -7631,7 +7631,7 @@ static int sym_setup_cdb(hcb_p np, struct ccb_scsiio *csio, ccb_p cp) /* CDB is a pointer */ if (!(ccb_h->flags & CAM_CDB_PHYS)) { /* CDB pointer is virtual */ - bcopy(csio->cdb_io.cdb_ptr, cp->cdb_buf, cmd_len); + memcpy(cp->cdb_buf, csio->cdb_io.cdb_ptr, cmd_len); cmd_ba = CCB_BA (cp, cdb_buf[0]); } else { /* CDB pointer is physical */ @@ -7644,7 +7644,7 @@ static int sym_setup_cdb(hcb_p np, struct ccb_scsiio *csio, ccb_p cp) } } else { /* CDB is in the CAM ccb (buffer) */ - bcopy(csio->cdb_io.cdb_bytes, cp->cdb_buf, cmd_len); + memcpy(cp->cdb_buf, csio->cdb_io.cdb_bytes, cmd_len); cmd_ba = CCB_BA (cp, cdb_buf[0]); } @@ -7858,9 +7858,9 @@ sym_fast_scatter_sg_physical(hcb_p np, ccb_p cp, data->addr = cpu_to_scr(psegs2->ds_addr); data->size = cpu_to_scr(psegs2->ds_len); if (DEBUG_FLAGS & DEBUG_SCATTER) { - printf ("%s scatter: paddr=%lx len=%ld\n", - sym_name(np), (long) psegs2->ds_addr, - (long) psegs2->ds_len); + device_printf(np->device, + "scatter: paddr=%lx len=%ld\n", + (long)psegs2->ds_addr, (long)psegs2->ds_len); } if (psegs2 != psegs) { --data; @@ -7895,8 +7895,8 @@ sym_scatter_sg_physical(hcb_p np, ccb_p cp, bus_dma_segment_t *psegs, int nsegs) pn = ps; k = pe - pn; if (DEBUG_FLAGS & DEBUG_SCATTER) { - printf ("%s scatter: paddr=%lx len=%ld\n", - sym_name(np), pn, k); + device_printf(np->device, + "scatter: paddr=%lx len=%ld\n", pn, k); } cp->phys.data[s].addr = cpu_to_scr(pn); cp->phys.data[s].size = cpu_to_scr(k); @@ -8232,6 +8232,7 @@ sym_update_dflags(hcb_p np, u_char *flags, struct ccb_trans_settings *cts) static device_method_t sym_pci_methods[] = { DEVMETHOD(device_probe, sym_pci_probe), DEVMETHOD(device_attach, sym_pci_attach), + DEVMETHOD(device_detach, sym_pci_detach), DEVMETHOD_END }; @@ -8521,16 +8522,15 @@ sym_pci_attach(device_t dev) * Alloc/get/map/retrieve the corresponding resources. */ if (np->features & (FE_RAM|FE_RAM8K)) { - int regs_id = SYM_PCI_RAM; + i = SYM_PCI_RAM; if (np->features & FE_64BIT) - regs_id = SYM_PCI_RAM64; - np->ram_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - ®s_id, RF_ACTIVE); + i = SYM_PCI_RAM64; + np->ram_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &i, + RF_ACTIVE); if (!np->ram_res) { device_printf(dev,"failed to allocate RAM resources\n"); goto attach_failed; } - np->ram_id = regs_id; np->ram_ba = rman_get_start(np->ram_res); } @@ -8631,8 +8631,8 @@ sym_pci_attach(device_t dev) /* * Copy scripts to controller instance. */ - bcopy(fw->a_base, np->scripta0, np->scripta_sz); - bcopy(fw->b_base, np->scriptb0, np->scriptb_sz); + memcpy(np->scripta0, fw->a_base, np->scripta_sz); + memcpy(np->scriptb0, fw->b_base, np->scriptb_sz); /* * Setup variable parts in scripts and compute @@ -8735,21 +8735,25 @@ sym_pci_attach(device_t dev) */ attach_failed: if (np) - sym_pci_free(np); + sym_pci_detach(dev); return ENXIO; } /* - * Free everything that have been allocated for this device. + * Detach a device by freeing everything that has been allocated for it. */ -static void sym_pci_free(hcb_p np) +static int +sym_pci_detach(device_t dev) { + hcb_p np; SYM_QUEHEAD *qp; ccb_p cp; tcb_p tp; lcb_p lp; int target, lun; + np = device_get_softc(dev); + /* * First free CAM resources. */ @@ -8761,16 +8765,15 @@ static void sym_pci_free(hcb_p np) */ if (np->ram_res) bus_release_resource(np->device, SYS_RES_MEMORY, - np->ram_id, np->ram_res); + rman_get_rid(np->ram_res), np->ram_res); if (np->mmio_res) bus_release_resource(np->device, SYS_RES_MEMORY, - SYM_PCI_MMIO, np->mmio_res); + rman_get_rid(np->mmio_res), np->mmio_res); if (np->io_res) bus_release_resource(np->device, SYS_RES_IOPORT, - SYM_PCI_IO, np->io_res); + rman_get_rid(np->io_res), np->io_res); if (np->irq_res) - bus_release_resource(np->device, SYS_RES_IRQ, - 0, np->irq_res); + bus_release_resource(np->device, SYS_RES_IRQ, 0, np->irq_res); if (np->scriptb0) sym_mfree_dma(np->scriptb0, np->scriptb_sz, "SCRIPTB0"); @@ -8824,6 +8827,8 @@ static void sym_pci_free(hcb_p np) SYM_LOCK_DESTROY(); device_set_softc(np->device, NULL); sym_mfree_dma(np, sizeof(*np), "HCB"); + + return (0); } /* @@ -8897,11 +8902,6 @@ static int sym_cam_attach(hcb_p np) return 1; fail: - if (sim) - cam_sim_free(sim, FALSE); - if (devq) - cam_simq_free(devq); - SYM_UNLOCK(); sym_cam_free(np); @@ -8924,15 +8924,16 @@ static void sym_cam_free(hcb_p np) SYM_LOCK(); + if (np->path) { + xpt_async(AC_LOST_DEVICE, np->path, NULL); + xpt_free_path(np->path); + np->path = NULL; + } if (np->sim) { xpt_bus_deregister(cam_sim_path(np->sim)); cam_sim_free(np->sim, /*free_devq*/ TRUE); np->sim = NULL; } - if (np->path) { - xpt_free_path(np->path); - np->path = NULL; - } SYM_UNLOCK(); } @@ -9057,14 +9058,14 @@ static void sym_display_Symbios_nvram(hcb_p np, Symbios_nvram *nvram) int i; /* display Symbios nvram host data */ - printf("%s: HOST ID=%d%s%s%s%s%s%s\n", - sym_name(np), nvram->host_id & 0x0f, - (nvram->flags & SYMBIOS_SCAM_ENABLE) ? " SCAM" :"", - (nvram->flags & SYMBIOS_PARITY_ENABLE) ? " PARITY" :"", - (nvram->flags & SYMBIOS_VERBOSE_MSGS) ? " VERBOSE" :"", - (nvram->flags & SYMBIOS_CHS_MAPPING) ? " CHS_ALT" :"", - (nvram->flags2 & SYMBIOS_AVOID_BUS_RESET)?" NO_RESET" :"", - (nvram->flags1 & SYMBIOS_SCAN_HI_LO) ? " HI_LO" :""); + device_printf(np->device, "HOST ID=%d%s%s%s%s%s%s\n", + nvram->host_id & 0x0f, + (nvram->flags & SYMBIOS_SCAM_ENABLE) ? " SCAM" : "", + (nvram->flags & SYMBIOS_PARITY_ENABLE) ? " PARITY" : "", + (nvram->flags & SYMBIOS_VERBOSE_MSGS) ? " VERBOSE" : "", + (nvram->flags & SYMBIOS_CHS_MAPPING) ? " CHS_ALT" : "", + (nvram->flags2 & SYMBIOS_AVOID_BUS_RESET) ? " NO_RESET" : "", + (nvram->flags1 & SYMBIOS_SCAN_HI_LO) ? " HI_LO" : ""); /* display Symbios nvram drive data */ for (i = 0 ; i < 15 ; i++) { @@ -9102,17 +9103,18 @@ static void sym_display_Tekram_nvram(hcb_p np, Tekram_nvram *nvram) case 2: rem = " REMOVABLE=all"; break; } - printf("%s: HOST ID=%d%s%s%s%s%s%s%s%s%s BOOT DELAY=%d tags=%d\n", - sym_name(np), nvram->host_id & 0x0f, - (nvram->flags1 & SYMBIOS_SCAM_ENABLE) ? " SCAM" :"", - (nvram->flags & TEKRAM_MORE_THAN_2_DRIVES) ? " >2DRIVES" :"", - (nvram->flags & TEKRAM_DRIVES_SUP_1GB) ? " >1GB" :"", - (nvram->flags & TEKRAM_RESET_ON_POWER_ON) ? " RESET" :"", - (nvram->flags & TEKRAM_ACTIVE_NEGATION) ? " ACT_NEG" :"", - (nvram->flags & TEKRAM_IMMEDIATE_SEEK) ? " IMM_SEEK" :"", - (nvram->flags & TEKRAM_SCAN_LUNS) ? " SCAN_LUNS" :"", - (nvram->flags1 & TEKRAM_F2_F6_ENABLED) ? " F2_F6" :"", - rem, boot_delay, tags); + device_printf(np->device, + "HOST ID=%d%s%s%s%s%s%s%s%s%s BOOT DELAY=%d tags=%d\n", + nvram->host_id & 0x0f, + (nvram->flags1 & SYMBIOS_SCAM_ENABLE) ? " SCAM" : "", + (nvram->flags & TEKRAM_MORE_THAN_2_DRIVES) ? " >2DRIVES" : "", + (nvram->flags & TEKRAM_DRIVES_SUP_1GB) ? " >1GB" : "", + (nvram->flags & TEKRAM_RESET_ON_POWER_ON) ? " RESET" : "", + (nvram->flags & TEKRAM_ACTIVE_NEGATION) ? " ACT_NEG" : "", + (nvram->flags & TEKRAM_IMMEDIATE_SEEK) ? " IMM_SEEK" : "", + (nvram->flags & TEKRAM_SCAN_LUNS) ? " SCAN_LUNS" : "", + (nvram->flags1 & TEKRAM_F2_F6_ENABLED) ? " F2_F6" : "", + rem, boot_delay, tags); /* display Tekram nvram drive data */ for (i = 0; i <= 15; i++) { diff --git a/sys/dev/syscons/syscons.c b/sys/dev/syscons/syscons.c index e9d664f51a1f..1a2969d6f2d8 100644 --- a/sys/dev/syscons/syscons.c +++ b/sys/dev/syscons/syscons.c @@ -3987,10 +3987,10 @@ next_code: break; case SUSP: - power_pm_suspend(POWER_SLEEP_STATE_SUSPEND); + power_pm_suspend(POWER_SSTATE_TRANSITION_SUSPEND); break; case STBY: - power_pm_suspend(POWER_SLEEP_STATE_STANDBY); + power_pm_suspend(POWER_SSTATE_TRANSITION_STANDBY); break; case DBG: diff --git a/sys/dev/tpm/tpm20.c b/sys/dev/tpm/tpm20.c index 6c587818058d..c521ca9dda9d 100644 --- a/sys/dev/tpm/tpm20.c +++ b/sys/dev/tpm/tpm20.c @@ -45,6 +45,7 @@ static void tpm20_discard_buffer(void *arg); #if defined TPM_HARVEST || defined RANDOM_ENABLE_TPM static void tpm20_harvest(void *arg, int unused); #endif +static int tpm20_restart(device_t dev, bool clear); static int tpm20_save_state(device_t dev, bool suspend); static d_open_t tpm20_open; @@ -243,8 +244,30 @@ tpm20_release(struct tpm_sc *sc) } int +tpm20_resume(device_t dev) +{ + + tpm20_restart(dev, false); + +#if defined TPM_HARVEST || defined RANDOM_ENABLE_TPM + struct tpm_sc *sc; + + sc = device_get_softc(dev); + taskqueue_enqueue_timeout(taskqueue_thread, &sc->harvest_task, + hz * TPM_HARVEST_INTERVAL); +#endif + return (0); +} + +int tpm20_suspend(device_t dev) { +#if defined TPM_HARVEST || defined RANDOM_ENABLE_TPM + struct tpm_sc *sc; + + sc = device_get_softc(dev); + taskqueue_drain_timeout(taskqueue_thread, &sc->harvest_task); +#endif return (tpm20_save_state(dev, true)); } @@ -308,6 +331,43 @@ tpm20_harvest(void *arg, int unused) #endif /* TPM_HARVEST */ static int +tpm20_restart(device_t dev, bool clear) +{ + struct tpm_sc *sc; + uint8_t startup_cmd[] = { + 0x80, 0x01, /* TPM_ST_NO_SESSIONS tag*/ + 0x00, 0x00, 0x00, 0x0C, /* cmd length */ + 0x00, 0x00, 0x01, 0x44, /* cmd TPM_CC_Startup */ + 0x00, 0x01 /* TPM_SU_STATE */ + }; + + sc = device_get_softc(dev); + + /* + * Inform the TPM whether we are resetting or resuming. + */ + if (clear) + startup_cmd[11] = 0; /* TPM_SU_CLEAR */ + + if (sc == NULL || sc->buf == NULL) + return (0); + + sx_xlock(&sc->dev_lock); + + MPASS(sc->pending_data_length == 0); + memcpy(sc->buf, startup_cmd, sizeof(startup_cmd)); + + /* XXX Ignoring both TPM_TRANSMIT return and tpm's response */ + TPM_TRANSMIT(sc->dev, sizeof(startup_cmd)); + sc->pending_data_length = 0; + sc->total_length = 0; + + sx_xunlock(&sc->dev_lock); + + return (0); +} + +static int tpm20_save_state(device_t dev, bool suspend) { struct tpm_sc *sc; @@ -331,8 +391,13 @@ tpm20_save_state(device_t dev, bool suspend) sx_xlock(&sc->dev_lock); + MPASS(sc->pending_data_length == 0); memcpy(sc->buf, save_cmd, sizeof(save_cmd)); + + /* XXX Ignoring both TPM_TRANSMIT return and tpm's response */ TPM_TRANSMIT(sc->dev, sizeof(save_cmd)); + sc->pending_data_length = 0; + sc->total_length = 0; sx_xunlock(&sc->dev_lock); diff --git a/sys/dev/tpm/tpm20.h b/sys/dev/tpm/tpm20.h index b2cfcd4f25bd..96b2920283b6 100644 --- a/sys/dev/tpm/tpm20.h +++ b/sys/dev/tpm/tpm20.h @@ -136,6 +136,7 @@ struct tpm_sc { }; int tpm20_suspend(device_t dev); +int tpm20_resume(device_t dev); int tpm20_shutdown(device_t dev); int32_t tpm20_get_timeout(uint32_t command); int tpm20_init(struct tpm_sc *sc); diff --git a/sys/dev/tpm/tpm_crb.c b/sys/dev/tpm/tpm_crb.c index 28b4f21eccfb..18414a6c799b 100644 --- a/sys/dev/tpm/tpm_crb.c +++ b/sys/dev/tpm/tpm_crb.c @@ -75,8 +75,52 @@ #define TPM_CRB_INT_ENABLE_BIT BIT(31) +struct tpmcrb_sc; +/* Attach */ +typedef bool (sm_attach_t)(struct tpmcrb_sc *, void *, size_t); +/* State change notification (timeout == 0 for 'no timeout') */ +typedef bool (sm_statechange_t)(struct tpmcrb_sc *, int); + +struct tpmcrb_sm_cfg { + sm_attach_t *sm_attach; + sm_statechange_t *sm_statechange; + sm_statechange_t *sm_cmdready; +}; + +static sm_attach_t pluton_attach; +static sm_statechange_t pluton_doorbell; + +static const struct tpmcrb_sm_cfg_map { + int acpi_sm; + const char *desc; + const struct tpmcrb_sm_cfg sm_cfg; +} tpmcrb_sm_cfg_map[] = { + { + .acpi_sm = TPM2_START_METHOD_CRB, + .desc = "Trusted Platform Module 2.0, CRB mode", + .sm_cfg = { NULL }, /* No notifications required */ + }, + { + .acpi_sm = ACPI_TPM2_COMMAND_BUFFER_WITH_PLUTON, + .desc = "Trusted Platform Module 2.0, CRB mode (Pluton)", + .sm_cfg = { + .sm_attach = &pluton_attach, + .sm_statechange = &pluton_doorbell, + .sm_cmdready = &pluton_doorbell, + }, + }, +}; + struct tpmcrb_sc { struct tpm_sc base; + const struct tpmcrb_sm_cfg *sm_cfg; + union { + /* StartMethod data */ + struct { + uint64_t start_reg; + uint64_t reply_reg; + } pluton; + }; bus_size_t cmd_off; bus_size_t rsp_off; size_t cmd_buf_size; @@ -99,23 +143,46 @@ static bool tpmcrb_cancel_cmd(struct tpm_sc *sc); char *tpmcrb_ids[] = {"MSFT0101", NULL}; +static const struct tpmcrb_sm_cfg_map * +tpmcrb_acpi_startmethod_cfg(int method) +{ + const struct tpmcrb_sm_cfg_map *entry; + + for (size_t i = 0; i < nitems(tpmcrb_sm_cfg_map); i++) { + entry = &tpmcrb_sm_cfg_map[i]; + + if (method == entry->acpi_sm) + return (entry); + } + + return (NULL); +} + static int tpmcrb_acpi_probe(device_t dev) { - int err; + int err, smethod; + const struct tpmcrb_sm_cfg_map *sm_cfg_map; ACPI_TABLE_TPM23 *tbl; ACPI_STATUS status; + err = ACPI_ID_PROBE(device_get_parent(dev), dev, tpmcrb_ids, NULL); if (err > 0) return (err); /*Find TPM2 Header*/ status = AcpiGetTable(ACPI_SIG_TPM2, 1, (ACPI_TABLE_HEADER **) &tbl); - if(ACPI_FAILURE(status) || - tbl->StartMethod != TPM2_START_METHOD_CRB) + if (ACPI_FAILURE(status)) + return (ENXIO); + + smethod = tbl->StartMethod; + AcpiPutTable((ACPI_TABLE_HEADER *)tbl); + + sm_cfg_map = tpmcrb_acpi_startmethod_cfg(smethod); + if (sm_cfg_map == NULL) return (ENXIO); - device_set_desc(dev, "Trusted Platform Module 2.0, CRB mode"); - return (err); + device_set_desc(dev, sm_cfg_map->desc); + return (0); } static ACPI_STATUS @@ -141,6 +208,40 @@ tpmcrb_fix_buff_offsets(ACPI_RESOURCE *res, void *arg) return (AE_OK); } +static bool +tpmcrb_attach_startmethod(struct tpmcrb_sc *crb_sc) +{ + const struct tpmcrb_sm_cfg_map *sm_cfg_map; + const struct tpmcrb_sm_cfg *sm_cfg; + ACPI_TABLE_TPM23 *tbl; + void *smdata; + ACPI_STATUS status; + bool ret; + + /* + * Grab what we need from the StartMethod. + */ + status = AcpiGetTable(ACPI_SIG_TPM2, 1, (ACPI_TABLE_HEADER **)(void **)&tbl); + if (ACPI_FAILURE(status)) + return (false); + + sm_cfg_map = tpmcrb_acpi_startmethod_cfg(tbl->StartMethod); + MPASS(sm_cfg_map != NULL); + sm_cfg = &sm_cfg_map->sm_cfg; + + crb_sc->sm_cfg = sm_cfg; + smdata = tbl + 1; + if (sm_cfg->sm_attach != NULL) { + ret = (*sm_cfg->sm_attach)(crb_sc, smdata, + tbl->Header.Length - sizeof(*tbl)); + } else { + ret = true; + } + + AcpiPutTable((ACPI_TABLE_HEADER *)tbl); + return (ret); +} + static int tpmcrb_attach(device_t dev) { @@ -166,6 +267,11 @@ tpmcrb_attach(device_t dev) return (ENXIO); } + if (!tpmcrb_attach_startmethod(crb_sc)) { + tpmcrb_detach(dev); + return (ENXIO); + } + if(!tpmcrb_request_locality(sc, 0)) { tpmcrb_detach(dev); return (ENXIO); @@ -302,6 +408,30 @@ tpmcrb_cancel_cmd(struct tpm_sc *sc) } static bool +tpmcrb_notify_cmdready(struct tpmcrb_sc *crb_sc, int timeout) +{ + sm_statechange_t *cmdready_fn; + + cmdready_fn = crb_sc->sm_cfg->sm_cmdready; + if (cmdready_fn == NULL) + return (true); + + return ((*cmdready_fn)(crb_sc, timeout)); +} + +static bool +tpmcrb_notify_state_changing(struct tpmcrb_sc *crb_sc, int timeout) +{ + sm_statechange_t *statechange_fn; + + statechange_fn = crb_sc->sm_cfg->sm_statechange; + if (statechange_fn == NULL) + return (true); + + return ((*statechange_fn)(crb_sc, timeout)); +} + +static bool tpmcrb_state_idle(struct tpmcrb_sc *crb_sc, bool wait) { struct tpm_sc *sc; @@ -312,6 +442,9 @@ tpmcrb_state_idle(struct tpmcrb_sc *crb_sc, bool wait) sc = &crb_sc->base; OR4(sc, TPM_CRB_CTRL_REQ, TPM_CRB_CTRL_REQ_GO_IDLE); + if (!tpmcrb_notify_state_changing(crb_sc, timeout)) + return (false); + if (timeout > 0) { mask = TPM_CRB_CTRL_STS_IDLE_BIT; if (!tpm_wait_for_u32(sc, TPM_CRB_CTRL_STS, mask, mask, @@ -333,6 +466,9 @@ tpmcrb_state_ready(struct tpmcrb_sc *crb_sc, bool wait) sc = &crb_sc->base; OR4(sc, TPM_CRB_CTRL_REQ, TPM_CRB_CTRL_REQ_GO_READY); + if (!tpmcrb_notify_state_changing(crb_sc, timeout)) + return (false); + if (timeout > 0) { mask = TPM_CRB_CTRL_REQ_GO_READY; if (!tpm_wait_for_u32(sc, TPM_CRB_CTRL_STS, mask, !mask, @@ -406,6 +542,13 @@ tpmcrb_transmit(device_t dev, size_t length) TPM_WRITE_4(dev, TPM_CRB_CTRL_START, TPM_CRB_CTRL_START_CMD); TPM_WRITE_BARRIER(dev, TPM_CRB_CTRL_START, 4); + if (!tpmcrb_notify_cmdready(crb_sc, timeout)) { + device_printf(dev, + "Timeout while waiting for device to ready\n"); + if (!tpmcrb_cancel_cmd(sc)) + return (EIO); + } + mask = ~0; if (!tpm_wait_for_u32(sc, TPM_CRB_CTRL_START, mask, ~mask, timeout)) { device_printf(dev, @@ -429,6 +572,10 @@ tpmcrb_transmit(device_t dev, size_t length) bus_read_region_stream_1(sc->mem_res, crb_sc->rsp_off + TPM_HEADER_SIZE, &sc->buf[TPM_HEADER_SIZE], bytes_available - TPM_HEADER_SIZE); + /* + * No need to wait for the transition to idle on the way out, we can + * relinquish locality right away. + */ if (!tpmcrb_state_idle(crb_sc, false)) { device_printf(dev, "Failed to transition to idle state post-send\n"); @@ -442,6 +589,61 @@ tpmcrb_transmit(device_t dev, size_t length) return (0); } +/* StartMethod Implementation Details */ + +/** Pluton **/ +struct tpmcrb_startmethod_pluton { + uint64_t sm_startaddr; + uint64_t sm_replyaddr; +}; + +static bool +pluton_attach(struct tpmcrb_sc *crb_sc, void *smdataregion, size_t datasz) +{ + struct tpmcrb_startmethod_pluton *smdata; + struct tpm_sc *sc; + rman_res_t base_addr, end_addr; + + if (datasz < sizeof(*smdata)) + return (false); + + smdata = smdataregion; + sc = &crb_sc->base; + + base_addr = rman_get_start(sc->mem_res); + end_addr = rman_get_end(sc->mem_res); + /* Sanity check */ + if (smdata->sm_startaddr < base_addr || + smdata->sm_startaddr > end_addr || + smdata->sm_replyaddr < base_addr || + smdata->sm_replyaddr > end_addr) + return (false); + + crb_sc->pluton.start_reg = smdata->sm_startaddr - base_addr; + crb_sc->pluton.reply_reg = smdata->sm_replyaddr - base_addr; + return (true); +} + +static bool +pluton_doorbell(struct tpmcrb_sc *crb_sc, int timeout) +{ + struct tpm_sc *sc; + device_t dev; + + sc = &crb_sc->base; + dev = sc->dev; + TPM_WRITE_4(dev, crb_sc->pluton.start_reg, 1); + TPM_WRITE_BARRIER(dev, crb_sc->pluton.start_reg, 4); + + if (timeout > 0) { + if (!tpm_wait_for_u32(sc, crb_sc->pluton.reply_reg, ~0U, 1, + timeout)) + return (false); + } + + return (true); +} + /* ACPI Driver */ static device_method_t tpmcrb_methods[] = { DEVMETHOD(device_probe, tpmcrb_acpi_probe), @@ -449,6 +651,7 @@ static device_method_t tpmcrb_methods[] = { DEVMETHOD(device_detach, tpmcrb_detach), DEVMETHOD(device_shutdown, tpm20_shutdown), DEVMETHOD(device_suspend, tpm20_suspend), + DEVMETHOD(device_resume, tpm20_resume), DEVMETHOD(tpm_transmit, tpmcrb_transmit), {0, 0} }; diff --git a/sys/dev/tpm/tpm_tis_core.c b/sys/dev/tpm/tpm_tis_core.c index 4159de4daf3b..34ecfcc283b1 100644 --- a/sys/dev/tpm/tpm_tis_core.c +++ b/sys/dev/tpm/tpm_tis_core.c @@ -487,6 +487,7 @@ static device_method_t tpmtis_methods[] = { DEVMETHOD(device_detach, tpmtis_detach), DEVMETHOD(device_shutdown, tpm20_shutdown), DEVMETHOD(device_suspend, tpm20_suspend), + DEVMETHOD(device_resume, tpm20_resume), DEVMETHOD(tpm_transmit, tpmtis_transmit), DEVMETHOD_END }; diff --git a/sys/dev/uart/uart_tty.c b/sys/dev/uart/uart_tty.c index d15d1d0c6ac2..7f0f730b3b6e 100644 --- a/sys/dev/uart/uart_tty.c +++ b/sys/dev/uart/uart_tty.c @@ -447,8 +447,9 @@ uart_tty_detach(struct uart_softc *sc) tp = sc->sc_u.u_tty.tp; - tty_lock(tp); swi_remove(sc->sc_softih); + + tty_lock(tp); tty_rel_gone(tp); return (0); diff --git a/sys/dev/ufshci/ufshci_dev.c b/sys/dev/ufshci/ufshci_dev.c index 406130c537a7..3167945b53b6 100644 --- a/sys/dev/ufshci/ufshci_dev.c +++ b/sys/dev/ufshci/ufshci_dev.c @@ -273,40 +273,31 @@ ufshci_dev_init_unipro(struct ufshci_controller *ctrlr) uint32_t pa_granularity, peer_pa_granularity; uint32_t t_activate, pear_t_activate; - /* - * Unipro Version: - * - 7~15 = Above 2.0, 6 = 2.0, 5 = 1.8, 4 = 1.61, 3 = 1.6, 2 = 1.41, - * 1 = 1.40, 0 = Reserved - */ - if (ufshci_uic_send_dme_get(ctrlr, PA_LocalVerInfo, - &ctrlr->unipro_version)) - return (ENXIO); - if (ufshci_uic_send_dme_get(ctrlr, PA_RemoteVerInfo, - &ctrlr->ufs_dev.unipro_version)) - return (ENXIO); - - /* - * PA_Granularity: Granularity for PA_TActivate and PA_Hibern8Time - * - 1=1us, 2=4us, 3=8us, 4=16us, 5=32us, 6=100us - */ - if (ufshci_uic_send_dme_get(ctrlr, PA_Granularity, &pa_granularity)) - return (ENXIO); - if (ufshci_uic_send_dme_peer_get(ctrlr, PA_Granularity, - &peer_pa_granularity)) - return (ENXIO); + if (ctrlr->quirks & UFSHCI_QUIRK_LONG_PEER_PA_TACTIVATE) { + /* + * PA_Granularity: Granularity for PA_TActivate and + * PA_Hibern8Time + * - 1=1us, 2=4us, 3=8us, 4=16us, 5=32us, 6=100us + */ + if (ufshci_uic_send_dme_get(ctrlr, PA_Granularity, + &pa_granularity)) + return (ENXIO); + if (ufshci_uic_send_dme_peer_get(ctrlr, PA_Granularity, + &peer_pa_granularity)) + return (ENXIO); - /* - * PA_TActivate: Time to wait before activating a burst in order to - * wake-up peer M-RX - * UniPro automatically sets timing information such as PA_TActivate - * through the PACP_CAP_EXT1_ind command during Link Startup operation. - */ - if (ufshci_uic_send_dme_get(ctrlr, PA_TActivate, &t_activate)) - return (ENXIO); - if (ufshci_uic_send_dme_peer_get(ctrlr, PA_TActivate, &pear_t_activate)) - return (ENXIO); + /* + * PA_TActivate: Time to wait before activating a burst in order + * to wake-up peer M-RX UniPro automatically sets timing + * information such as PA_TActivate through the + * PACP_CAP_EXT1_ind command during Link Startup operation. + */ + if (ufshci_uic_send_dme_get(ctrlr, PA_TActivate, &t_activate)) + return (ENXIO); + if (ufshci_uic_send_dme_peer_get(ctrlr, PA_TActivate, + &pear_t_activate)) + return (ENXIO); - if (ctrlr->quirks & UFSHCI_QUIRK_LONG_PEER_PA_TACTIVATE) { /* * Intel Lake-field UFSHCI has a quirk. We need to add 200us to * the PEER's PA_TActivate. diff --git a/sys/dev/ufshci/ufshci_private.h b/sys/dev/ufshci/ufshci_private.h index b57142bf58fc..1634cf51c9fb 100644 --- a/sys/dev/ufshci/ufshci_private.h +++ b/sys/dev/ufshci/ufshci_private.h @@ -278,8 +278,6 @@ struct ufshci_device { struct ufshci_device_descriptor dev_desc; struct ufshci_geometry_descriptor geo_desc; - uint32_t unipro_version; - /* WriteBooster */ bool is_wb_enabled; bool is_wb_flush_enabled; @@ -398,7 +396,6 @@ struct ufshci_controller { /* UFS Interconnect Layer (UIC) */ struct mtx uic_cmd_lock; - uint32_t unipro_version; uint8_t hs_gear; uint32_t tx_lanes; uint32_t rx_lanes; diff --git a/sys/dev/ufshci/ufshci_sim.c b/sys/dev/ufshci/ufshci_sim.c index bee1fff858ff..404f3e99d1e2 100644 --- a/sys/dev/ufshci/ufshci_sim.c +++ b/sys/dev/ufshci/ufshci_sim.c @@ -494,7 +494,7 @@ ufshci_sim_send_ssu(struct ufshci_controller *ctrlr, bool start, return ENOMEM; } - scsi_start_stop(&ccb->csio, + scsi_start_stop_pc(&ccb->csio, /*retries*/ 4, /*cbfcnp*/ NULL, /*tag_action*/ MSG_SIMPLE_Q_TAG, diff --git a/sys/dev/usb/net/if_ure.c b/sys/dev/usb/net/if_ure.c index c3f7b622d687..3dea88eb10f1 100644 --- a/sys/dev/usb/net/if_ure.c +++ b/sys/dev/usb/net/if_ure.c @@ -1015,6 +1015,7 @@ ure_attach_post_sub(struct usb_ether *ue) if_sethwassist(ifp, CSUM_IP|CSUM_IP_UDP|CSUM_IP_TCP); #ifdef INET6 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM_IPV6, 0); + if_sethwassistbits(ifp, CSUM_IP6_UDP|CSUM_IP6_TCP, 0); #endif if_setcapenable(ifp, if_getcapabilities(ifp)); @@ -1463,6 +1464,7 @@ ure_ioctl(if_t ifp, u_long cmd, caddr_t data) if ((mask & IFCAP_TXCSUM) != 0 && (if_getcapabilities(ifp) & IFCAP_TXCSUM) != 0) { if_togglecapenable(ifp, IFCAP_TXCSUM); + if_togglehwassist(ifp, CSUM_IP|CSUM_IP_UDP|CSUM_IP_TCP); } if ((mask & IFCAP_RXCSUM) != 0 && (if_getcapabilities(ifp) & IFCAP_RXCSUM) != 0) { @@ -1471,6 +1473,7 @@ ure_ioctl(if_t ifp, u_long cmd, caddr_t data) if ((mask & IFCAP_TXCSUM_IPV6) != 0 && (if_getcapabilities(ifp) & IFCAP_TXCSUM_IPV6) != 0) { if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); + if_togglehwassist(ifp, CSUM_IP6_UDP|CSUM_IP6_TCP); } if ((mask & IFCAP_RXCSUM_IPV6) != 0 && (if_getcapabilities(ifp) & IFCAP_RXCSUM_IPV6) != 0) { diff --git a/sys/dev/usb/usb.h b/sys/dev/usb/usb.h index a6c3c8030c73..3e972f876c6a 100644 --- a/sys/dev/usb/usb.h +++ b/sys/dev/usb/usb.h @@ -541,7 +541,7 @@ struct usb_endpoint_descriptor { #define UE_XFERTYPE 0x03 #define UE_CONTROL 0x00 #define UE_ISOCHRONOUS 0x01 -#define UE_BULK 0x02 +#define UE_BULK 0x02 #define UE_INTERRUPT 0x03 #define UE_BULK_INTR 0xfe /* for internal use only! */ #define UE_TYPE_ANY 0xff /* for internal use only! */ diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c index 19775416f8d4..da89dea03f87 100644 --- a/sys/dev/virtio/network/if_vtnet.c +++ b/sys/dev/virtio/network/if_vtnet.c @@ -2059,8 +2059,16 @@ vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, } } - m->m_pkthdr.flowid = rxq->vtnrx_id; - M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); + if (sc->vtnet_act_vq_pairs == 1) { + /* + * When RSS is not needed (one active rx queue), let the upper + * layer know and react. + */ + M_HASHTYPE_CLEAR(m); + } else { + m->m_pkthdr.flowid = rxq->vtnrx_id; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); + } if (hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID)) { diff --git a/sys/dev/virtio/p9fs/virtio_p9fs.c b/sys/dev/virtio/p9fs/virtio_p9fs.c index aa84d3970698..2b276a60aa9a 100644 --- a/sys/dev/virtio/p9fs/virtio_p9fs.c +++ b/sys/dev/virtio/p9fs/virtio_p9fs.c @@ -84,6 +84,8 @@ static struct virtio_feature_desc virtio_9p_feature_desc[] = { { 0, NULL } }; +VIRTIO_SIMPLE_PNPINFO(virtio_p9fs, VIRTIO_ID_9P, "VirtIO 9P Transport"); + /* We don't currently allow canceling of virtio requests */ static int vt9p_cancel(void *handle, struct p9_req_t *req) @@ -257,13 +259,7 @@ vt9p_alloc_virtqueue(struct vt9p_softc *sc) static int vt9p_probe(device_t dev) { - - /* If the virtio device type is a 9P device, then we claim and attach it */ - if (virtio_get_device_type(dev) != VIRTIO_ID_9P) - return (ENXIO); - device_set_desc(dev, "VirtIO 9P Transport"); - - return (BUS_PROBE_DEFAULT); + return (VIRTIO_SIMPLE_PROBE(dev, virtio_p9fs)); } static void diff --git a/sys/dev/vmm/vmm_dev.c b/sys/dev/vmm/vmm_dev.c index fa51dc950459..09fd3a9048bd 100644 --- a/sys/dev/vmm/vmm_dev.c +++ b/sys/dev/vmm/vmm_dev.c @@ -876,8 +876,7 @@ vmmdev_destroy(struct vmmdev_softc *sc) free(dsc, M_VMMDEV); } - if (sc->vm != NULL) - vm_destroy(sc->vm); + vm_destroy(sc->vm); chgvmmcnt(sc->ucred->cr_ruidinfo, -1, 0); crfree(sc->ucred); @@ -990,9 +989,15 @@ vmmdev_create(const char *name, uint32_t flags, struct ucred *cred) return (EEXIST); } + if (!chgvmmcnt(cred->cr_ruidinfo, 1, vm_maxvmms)) { + sx_xunlock(&vmmdev_mtx); + return (ENOMEM); + } + error = vm_create(name, &vm); if (error != 0) { sx_xunlock(&vmmdev_mtx); + (void)chgvmmcnt(cred->cr_ruidinfo, -1, 0); return (error); } sc = vmmdev_alloc(vm, cred); @@ -1015,12 +1020,6 @@ vmmdev_create(const char *name, uint32_t flags, struct ucred *cred) vmmdev_destroy(sc); return (error); } - if (!chgvmmcnt(cred->cr_ruidinfo, 1, vm_maxvmms)) { - sx_xunlock(&vmmdev_mtx); - destroy_dev(cdev); - vmmdev_destroy(sc); - return (ENOMEM); - } sc->cdev = cdev; sx_xunlock(&vmmdev_mtx); return (0); diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index a6a5f0eeff9d..fd18e85092db 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -804,11 +804,11 @@ vt_machine_kbdevent(struct vt_device *vd, int c) return (1); case SPCLKEY | STBY: /* XXX Not present in kbdcontrol parser. */ /* Put machine into Stand-By mode. */ - power_pm_suspend(POWER_SLEEP_STATE_STANDBY); + power_pm_suspend(POWER_SSTATE_TRANSITION_STANDBY); return (1); case SPCLKEY | SUSP: /* kbdmap(5) keyword `susp`. */ /* Suspend machine. */ - power_pm_suspend(POWER_SLEEP_STATE_SUSPEND); + power_pm_suspend(POWER_SSTATE_TRANSITION_SUSPEND); return (1); } diff --git a/sys/dev/wg/if_wg.c b/sys/dev/wg/if_wg.c index 17aedee0e6b0..611314883643 100644 --- a/sys/dev/wg/if_wg.c +++ b/sys/dev/wg/if_wg.c @@ -3033,8 +3033,8 @@ wg_clone_create(struct if_clone *ifc, char *name, size_t len, if_attach(ifp); bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); #ifdef INET6 - ND_IFINFO(ifp)->flags &= ~ND6_IFF_AUTO_LINKLOCAL; - ND_IFINFO(ifp)->flags |= ND6_IFF_NO_DAD; + if_getinet6(ifp)->nd_flags &= ~ND6_IFF_AUTO_LINKLOCAL; + if_getinet6(ifp)->nd_flags |= ND6_IFF_NO_DAD; #endif sx_xlock(&wg_sx); LIST_INSERT_HEAD(&wg_list, sc, sc_entry); diff --git a/sys/fs/devfs/devfs_vnops.c b/sys/fs/devfs/devfs_vnops.c index 323f1e0fa961..d594b1584757 100644 --- a/sys/fs/devfs/devfs_vnops.c +++ b/sys/fs/devfs/devfs_vnops.c @@ -367,6 +367,9 @@ devfs_populate_vp(struct vnode *vp) ASSERT_VOP_LOCKED(vp, "devfs_populate_vp"); + if (VN_IS_DOOMED(vp)) + return (ENOENT); + dmp = VFSTODEVFS(vp->v_mount); if (!devfs_populate_needed(dmp)) { sx_xlock(&dmp->dm_lock); @@ -1128,8 +1131,25 @@ devfs_lookupx(struct vop_lookup_args *ap, int *dm_unlock) cdev = NULL; DEVFS_DMP_HOLD(dmp); sx_xunlock(&dmp->dm_lock); + dvplocked = VOP_ISLOCKED(dvp); + + /* + * Invoke the dev_clone handler. Unlock dvp around it + * to simplify the cloner operations. + * + * If dvp is reclaimed while we unlocked it, we return + * with ENOENT by some of the paths below. If cloner + * returned cdev, then devfs_populate_vp() notes the + * reclamation. Otherwise, note that either our devfs + * mount is being unmounted, then DEVFS_DMP_DROP() + * returns true, and we return ENOENT this way. Or, + * because de == NULL, the check for it after the loop + * returns ENOENT. + */ + VOP_UNLOCK(dvp); EVENTHANDLER_INVOKE(dev_clone, td->td_ucred, pname, strlen(pname), &cdev); + vn_lock(dvp, dvplocked | LK_RETRY); if (cdev == NULL) sx_xlock(&dmp->dm_lock); diff --git a/sys/fs/fuse/fuse_device.c b/sys/fs/fuse/fuse_device.c index 41387de3ce71..2c1e19953dfd 100644 --- a/sys/fs/fuse/fuse_device.c +++ b/sys/fs/fuse/fuse_device.c @@ -65,7 +65,6 @@ #include <sys/module.h> #include <sys/systm.h> #include <sys/errno.h> -#include <sys/param.h> #include <sys/kernel.h> #include <sys/conf.h> #include <sys/uio.h> @@ -177,6 +176,11 @@ fdata_dtor(void *arg) fuse_lck_mtx_unlock(fdata->ms_mtx); FUSE_UNLOCK(); + if (fdata->mp && fdata->dataflags & FSESS_AUTO_UNMOUNT) { + vfs_ref(fdata->mp); + dounmount(fdata->mp, MNT_FORCE, curthread); + } + fdata_trydestroy(fdata); } diff --git a/sys/fs/fuse/fuse_ipc.h b/sys/fs/fuse/fuse_ipc.h index d9d79f38c269..6fb04fd4ee25 100644 --- a/sys/fs/fuse/fuse_ipc.h +++ b/sys/fs/fuse/fuse_ipc.h @@ -240,10 +240,11 @@ struct fuse_data { #define FSESS_WARN_READLINK_EMBEDDED_NUL 0x1000000 /* corrupt READLINK output */ #define FSESS_WARN_DOT_LOOKUP 0x2000000 /* Inconsistent . LOOKUP response */ #define FSESS_WARN_INODE_MISMATCH 0x4000000 /* ino != nodeid */ +#define FSESS_SETXATTR_EXT 0x8000000 /* extended fuse_setxattr_in */ +#define FSESS_AUTO_UNMOUNT 0x10000000 /* perform unmount when server dies */ #define FSESS_MNTOPTS_MASK ( \ FSESS_DAEMON_CAN_SPY | FSESS_PUSH_SYMLINKS_IN | \ - FSESS_DEFAULT_PERMISSIONS | FSESS_INTR) -#define FSESS_SETXATTR_EXT 0x8000000 /* extended fuse_setxattr_in */ + FSESS_DEFAULT_PERMISSIONS | FSESS_INTR | FSESS_AUTO_UNMOUNT) extern int fuse_data_cache_mode; diff --git a/sys/fs/fuse/fuse_vfsops.c b/sys/fs/fuse/fuse_vfsops.c index 0ff79913128a..a5118aa7675f 100644 --- a/sys/fs/fuse/fuse_vfsops.c +++ b/sys/fs/fuse/fuse_vfsops.c @@ -337,6 +337,7 @@ fuse_vfsop_mount(struct mount *mp) FUSE_FLAGOPT(push_symlinks_in, FSESS_PUSH_SYMLINKS_IN); FUSE_FLAGOPT(default_permissions, FSESS_DEFAULT_PERMISSIONS); FUSE_FLAGOPT(intr, FSESS_INTR); + FUSE_FLAGOPT(auto_unmount, FSESS_AUTO_UNMOUNT); (void)vfs_scanopt(opts, "max_read=", "%u", &max_read); (void)vfs_scanopt(opts, "linux_errnos", "%d", &linux_errnos); diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 3f659432552c..821265cc2911 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -1544,6 +1544,7 @@ init386(int first) /* Initialize preload_kmdp */ preload_initkmdp(!metadata_missing); + sched_instance_select(); link_elf_ireloc(); vm86_initialize(); diff --git a/sys/i386/i386/swtch.S b/sys/i386/i386/swtch.S index 5c2e078b5446..ad7c1d16c788 100644 --- a/sys/i386/i386/swtch.S +++ b/sys/i386/i386/swtch.S @@ -30,28 +30,10 @@ * SUCH DAMAGE. */ -#include "opt_sched.h" - #include <machine/asmacros.h> #include "assym.inc" -#if defined(SMP) && defined(SCHED_ULE) -#define SETOP xchgl -#define BLOCK_SPIN(reg) \ - movl $blocked_lock,%eax ; \ - 100: ; \ - lock ; \ - cmpxchgl %eax,TD_LOCK(reg) ; \ - jne 101f ; \ - pause ; \ - jmp 100b ; \ - 101: -#else -#define SETOP movl -#define BLOCK_SPIN(reg) -#endif - /*****************************************************************************/ /* Scheduling */ /*****************************************************************************/ @@ -162,7 +144,7 @@ ENTRY(cpu_switch) /* Switchout td_lock */ movl %esi,%eax movl PCPU(CPUID),%esi - SETOP %eax,TD_LOCK(%edi) + xchgl %eax,TD_LOCK(%edi) /* Release bit from old pmap->pm_active */ movl PCPU(CURPMAP), %ebx @@ -181,7 +163,18 @@ ENTRY(cpu_switch) #endif btsl %esi, PM_ACTIVE(%ebx) /* set new */ sw1: - BLOCK_SPIN(%ecx) +#ifdef SMP + movl $blocked_lock,%eax +100: + + lock + cmpxchgl %eax,TD_LOCK(%ecx) + jne 101f + pause + jmp 100b +101: +#endif + /* * At this point, we have managed thread locks and are ready * to load up the rest of the next context. diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index 80b45070d896..698d3b26813c 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -91,7 +91,8 @@ get_pcb_user_save_td(struct thread *td) p = td->td_kstack + td->td_kstack_pages * PAGE_SIZE - roundup2(cpu_max_ext_state_size, XSAVE_AREA_ALIGN); - KASSERT((p % XSAVE_AREA_ALIGN) == 0, ("Unaligned pcb_user_save area")); + KASSERT(__is_aligned(p, XSAVE_AREA_ALIGN), + ("Unaligned pcb_user_save area")); return ((union savefpu *)p); } diff --git a/sys/i386/include/ifunc.h b/sys/i386/include/ifunc.h new file mode 100644 index 000000000000..1af46757b836 --- /dev/null +++ b/sys/i386/include/ifunc.h @@ -0,0 +1,5 @@ +/* + * This file is in the public domain. + */ + +#include <x86/ifunc.h> diff --git a/sys/kern/init_sysent.c b/sys/kern/init_sysent.c index cd305de1ed44..33586db22600 100644 --- a/sys/kern/init_sysent.c +++ b/sys/kern/init_sysent.c @@ -666,4 +666,6 @@ struct sysent sysent[] = { { .sy_narg = AS(jail_attach_jd_args), .sy_call = (sy_call_t *)sys_jail_attach_jd, .sy_auevent = AUE_JAIL_ATTACH, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 597 = jail_attach_jd */ { .sy_narg = AS(jail_remove_jd_args), .sy_call = (sy_call_t *)sys_jail_remove_jd, .sy_auevent = AUE_JAIL_REMOVE, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 598 = jail_remove_jd */ { .sy_narg = AS(kexec_load_args), .sy_call = (sy_call_t *)sys_kexec_load, .sy_auevent = AUE_NULL, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 599 = kexec_load */ + { .sy_narg = AS(pdrfork_args), .sy_call = (sy_call_t *)sys_pdrfork, .sy_auevent = AUE_PDRFORK, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 600 = pdrfork */ + { .sy_narg = AS(pdwait_args), .sy_call = (sy_call_t *)sys_pdwait, .sy_auevent = AUE_PDWAIT, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 601 = pdwait */ }; diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c index c4b1c8201ff2..18ea3a7bd29d 100644 --- a/sys/kern/kern_exit.c +++ b/sys/kern/kern_exit.c @@ -34,45 +34,45 @@ * SUCH DAMAGE. */ -#include <sys/cdefs.h> #include "opt_ddb.h" #include "opt_ktrace.h" -#include <sys/param.h> +#define EXTERR_CATEGORY EXTERR_CAT_PROCEXIT #include <sys/systm.h> -#include <sys/sysproto.h> +#include <sys/acct.h> /* for acct_process() function prototype */ #include <sys/capsicum.h> #include <sys/eventhandler.h> +#include <sys/exterrvar.h> +#include <sys/filedesc.h> +#include <sys/jail.h> #include <sys/kernel.h> #include <sys/ktr.h> -#include <sys/malloc.h> #include <sys/lock.h> +#include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> #include <sys/procdesc.h> -#include <sys/jail.h> -#include <sys/tty.h> -#include <sys/wait.h> -#include <sys/vmmeter.h> -#include <sys/vnode.h> +#include <sys/ptrace.h> #include <sys/racct.h> #include <sys/resourcevar.h> #include <sys/sbuf.h> -#include <sys/signalvar.h> #include <sys/sched.h> +#include <sys/sdt.h> +#include <sys/sem.h> +#include <sys/shm.h> +#include <sys/signalvar.h> #include <sys/sx.h> #include <sys/syscallsubr.h> #include <sys/sysctl.h> -#include <sys/syslog.h> -#include <sys/ptrace.h> -#include <sys/acct.h> /* for acct_process() function prototype */ -#include <sys/filedesc.h> -#include <sys/sdt.h> -#include <sys/shm.h> -#include <sys/sem.h> #include <sys/sysent.h> +#include <sys/syslog.h> +#include <sys/sysproto.h> #include <sys/timers.h> +#include <sys/tty.h> #include <sys/umtxvar.h> +#include <sys/vmmeter.h> +#include <sys/vnode.h> +#include <sys/wait.h> #ifdef KTRACE #include <sys/ktrace.h> #endif @@ -906,6 +906,33 @@ sys_wait6(struct thread *td, struct wait6_args *uap) return (error); } +int +sys_pdwait(struct thread *td, struct pdwait_args *uap) +{ + struct __wrusage wru, *wrup; + siginfo_t si, *sip; + int error, status; + + wrup = uap->wrusage != NULL ? &wru : NULL; + + if (uap->info != NULL) { + sip = &si; + bzero(sip, sizeof(*sip)); + } else { + sip = NULL; + } + + error = kern_pdwait(td, uap->fd, &status, uap->options, wrup, sip); + + if (uap->status != NULL && error == 0) + error = copyout(&status, uap->status, sizeof(status)); + if (uap->wrusage != NULL && error == 0) + error = copyout(&wru, uap->wrusage, sizeof(wru)); + if (uap->info != NULL && error == 0) + error = copyout(&si, uap->info, sizeof(si)); + return (error); +} + /* * Reap the remains of a zombie process and optionally return status and * rusage. Asserts and will release both the proctree_lock and the process @@ -924,9 +951,9 @@ proc_reap(struct thread *td, struct proc *p, int *status, int options) q = td->td_proc; - if (status) + if (status != NULL) *status = KW_EXITCODE(p->p_xexit, p->p_xsig); - if (options & WNOWAIT) { + if ((options & WNOWAIT) != 0) { /* * Only poll, returning the status. Caller does not wish to * release the proc struct just yet. @@ -979,10 +1006,10 @@ proc_reap(struct thread *td, struct proc *p, int *status, int options) leavepgrp(p); if (p->p_procdesc != NULL) procdesc_reap(p); + else + proc_id_clear(PROC_ID_PID, p->p_pid); sx_xunlock(&proctree_lock); - proc_id_clear(PROC_ID_PID, p->p_pid); - PROC_LOCK(p); knlist_detach(p->p_klist); p->p_klist = NULL; @@ -1042,13 +1069,75 @@ proc_reap(struct thread *td, struct proc *p, int *status, int options) atomic_add_int(&nprocs, -1); } +static void +wait_fill_siginfo(struct proc *p, siginfo_t *siginfo) +{ + PROC_LOCK_ASSERT(p, MA_OWNED); + + if (siginfo == NULL) + return; + + bzero(siginfo, sizeof(*siginfo)); + siginfo->si_errno = 0; + + /* + * SUSv4 requires that the si_signo value is always + * SIGCHLD. Obey it despite the rfork(2) interface allows to + * request other signal for child exit notification. + */ + siginfo->si_signo = SIGCHLD; + + /* + * This is still a rough estimate. We will fix the cases + * TRAPPED, STOPPED, and CONTINUED later. + */ + if (WCOREDUMP(p->p_xsig)) { + siginfo->si_code = CLD_DUMPED; + siginfo->si_status = WTERMSIG(p->p_xsig); + } else if (WIFSIGNALED(p->p_xsig)) { + siginfo->si_code = CLD_KILLED; + siginfo->si_status = WTERMSIG(p->p_xsig); + } else { + siginfo->si_code = CLD_EXITED; + siginfo->si_status = p->p_xexit; + } + + siginfo->si_pid = p->p_pid; + siginfo->si_uid = p->p_ucred->cr_uid; + + /* + * The si_addr field would be useful additional detail, but + * apparently the PC value may be lost when we reach this + * point. bzero() above sets siginfo->si_addr to NULL. + */ +} + +static void +wait_fill_wrusage(struct proc *p, struct __wrusage *wrusage) +{ + struct rusage *rup; + + PROC_LOCK_ASSERT(p, MA_OWNED); + + if (wrusage == NULL) + return; + + rup = &wrusage->wru_self; + *rup = p->p_ru; + PROC_STATLOCK(p); + calcru(p, &rup->ru_utime, &rup->ru_stime); + PROC_STATUNLOCK(p); + + rup = &wrusage->wru_children; + *rup = p->p_stats->p_cru; + calccru(p, &rup->ru_utime, &rup->ru_stime); +} + static int proc_to_reap(struct thread *td, struct proc *p, idtype_t idtype, id_t id, int *status, int options, struct __wrusage *wrusage, siginfo_t *siginfo, int check_only) { - struct rusage *rup; - sx_assert(&proctree_lock, SA_XLOCKED); PROC_LOCK(p); @@ -1114,7 +1203,7 @@ proc_to_reap(struct thread *td, struct proc *p, idtype_t idtype, id_t id, return (0); } - if (((options & WEXITED) == 0) && (p->p_state == PRS_ZOMBIE)) { + if ((options & WEXITED) == 0 && p->p_state == PRS_ZOMBIE) { PROC_UNLOCK(p); return (0); } @@ -1133,60 +1222,14 @@ proc_to_reap(struct thread *td, struct proc *p, idtype_t idtype, id_t id, return (0); } - if (siginfo != NULL) { - bzero(siginfo, sizeof(*siginfo)); - siginfo->si_errno = 0; - - /* - * SUSv4 requires that the si_signo value is always - * SIGCHLD. Obey it despite the rfork(2) interface - * allows to request other signal for child exit - * notification. - */ - siginfo->si_signo = SIGCHLD; - - /* - * This is still a rough estimate. We will fix the - * cases TRAPPED, STOPPED, and CONTINUED later. - */ - if (WCOREDUMP(p->p_xsig)) { - siginfo->si_code = CLD_DUMPED; - siginfo->si_status = WTERMSIG(p->p_xsig); - } else if (WIFSIGNALED(p->p_xsig)) { - siginfo->si_code = CLD_KILLED; - siginfo->si_status = WTERMSIG(p->p_xsig); - } else { - siginfo->si_code = CLD_EXITED; - siginfo->si_status = p->p_xexit; - } - - siginfo->si_pid = p->p_pid; - siginfo->si_uid = p->p_ucred->cr_uid; - - /* - * The si_addr field would be useful additional - * detail, but apparently the PC value may be lost - * when we reach this point. bzero() above sets - * siginfo->si_addr to NULL. - */ - } + wait_fill_siginfo(p, siginfo); /* * There should be no reason to limit resources usage info to * exited processes only. A snapshot about any resources used * by a stopped process may be exactly what is needed. */ - if (wrusage != NULL) { - rup = &wrusage->wru_self; - *rup = p->p_ru; - PROC_STATLOCK(p); - calcru(p, &rup->ru_utime, &rup->ru_stime); - PROC_STATUNLOCK(p); - - rup = &wrusage->wru_children; - *rup = p->p_stats->p_cru; - calccru(p, &rup->ru_utime, &rup->ru_stime); - } + wait_fill_wrusage(p, wrusage); if (p->p_state == PRS_ZOMBIE && !check_only) { proc_reap(td, p, status, options); @@ -1267,10 +1310,85 @@ report_alive_proc(struct thread *td, struct proc *p, siginfo_t *siginfo, } if (status != NULL) *status = cont ? SIGCONT : W_STOPCODE(p->p_xsig); - td->td_retval[0] = p->p_pid; PROC_UNLOCK(p); } +static int +wait6_checkopt(int options) +{ + /* If we don't know the option, just return. */ + if ((options & ~(WUNTRACED | WNOHANG | WCONTINUED | WNOWAIT | + WEXITED | WTRAPPED | WLINUXCLONE)) != 0) + return (EXTERROR(EINVAL, "Unknown options %#jx", options)); + if ((options & (WEXITED | WUNTRACED | WCONTINUED | WTRAPPED)) == 0) { + /* + * We will be unable to find any matching processes, + * because there are no known events to look for. + * Prefer to return error instead of blocking + * indefinitely. + */ + return (EXTERROR(EINVAL, + "Cannot match processes %#jx", options)); + } + return (0); +} + +/* + * Checks and reports status for alive process, according to the + * options. Returns true if the process fits one of the requested + * options and its status was updated in siginfo. + * + * If the process was reported (the function result is true), both the + * process and proctree locks are unlocked. + */ +static bool +wait6_check_alive(struct thread *td, int options, struct proc *p, int *status, + siginfo_t *siginfo) +{ + bool report; + + PROC_LOCK_ASSERT(p, MA_OWNED); + sx_assert(&proctree_lock, SA_XLOCKED); + + if ((options & WTRAPPED) != 0 && (p->p_flag & P_TRACED) != 0) { + PROC_SLOCK(p); + report = (p->p_flag & (P_STOPPED_TRACE | P_STOPPED_SIG)) && + p->p_suspcount == p->p_numthreads && + (p->p_flag & P_WAITED) == 0; + PROC_SUNLOCK(p); + if (report) { + CTR4(KTR_PTRACE, + "wait: returning trapped pid %d status %#x (xstat %d) xthread %d", + p->p_pid, W_STOPCODE(p->p_xsig), p->p_xsig, + p->p_xthread != NULL ? + p->p_xthread->td_tid : -1); + report_alive_proc(td, p, siginfo, status, + options, CLD_TRAPPED); + return (true); + } + } + + if ((options & WUNTRACED) != 0 && (p->p_flag & P_STOPPED_SIG) != 0) { + PROC_SLOCK(p); + report = p->p_suspcount == p->p_numthreads && + (p->p_flag & P_WAITED) == 0; + PROC_SUNLOCK(p); + if (report) { + report_alive_proc(td, p, siginfo, status, options, + CLD_STOPPED); + return (true); + } + } + + if ((options & WCONTINUED) != 0 && (p->p_flag & P_CONTINUED) != 0) { + report_alive_proc(td, p, siginfo, status, options, + CLD_CONTINUED); + return (true); + } + + return (false); +} + int kern_wait6(struct thread *td, idtype_t idtype, id_t id, int *status, int options, struct __wrusage *wrusage, siginfo_t *siginfo) @@ -1278,7 +1396,6 @@ kern_wait6(struct thread *td, idtype_t idtype, id_t id, int *status, struct proc *p, *q; pid_t pid; int error, nfound, ret; - bool report; AUDIT_ARG_VALUE((int)idtype); /* XXX - This is likely wrong! */ AUDIT_ARG_PID((pid_t)id); /* XXX - This may be wrong! */ @@ -1293,20 +1410,9 @@ kern_wait6(struct thread *td, idtype_t idtype, id_t id, int *status, idtype = P_PGID; } - /* If we don't know the option, just return. */ - if ((options & ~(WUNTRACED | WNOHANG | WCONTINUED | WNOWAIT | - WEXITED | WTRAPPED | WLINUXCLONE)) != 0) - return (EINVAL); - if ((options & (WEXITED | WUNTRACED | WCONTINUED | WTRAPPED)) == 0) { - /* - * We will be unable to find any matching processes, - * because there are no known events to look for. - * Prefer to return error instead of blocking - * indefinitely. - */ - return (EINVAL); - } - + error = wait6_checkopt(options); + if (error != 0) + return (error); loop: if (q->p_flag & P_STATCHILD) { PROC_LOCK(q); @@ -1342,44 +1448,11 @@ loop_locked: nfound++; PROC_LOCK_ASSERT(p, MA_OWNED); - if ((options & WTRAPPED) != 0 && - (p->p_flag & P_TRACED) != 0) { - PROC_SLOCK(p); - report = - ((p->p_flag & (P_STOPPED_TRACE | P_STOPPED_SIG)) && - p->p_suspcount == p->p_numthreads && - (p->p_flag & P_WAITED) == 0); - PROC_SUNLOCK(p); - if (report) { - CTR4(KTR_PTRACE, - "wait: returning trapped pid %d status %#x " - "(xstat %d) xthread %d", - p->p_pid, W_STOPCODE(p->p_xsig), p->p_xsig, - p->p_xthread != NULL ? - p->p_xthread->td_tid : -1); - report_alive_proc(td, p, siginfo, status, - options, CLD_TRAPPED); - return (0); - } - } - if ((options & WUNTRACED) != 0 && - (p->p_flag & P_STOPPED_SIG) != 0) { - PROC_SLOCK(p); - report = (p->p_suspcount == p->p_numthreads && - ((p->p_flag & P_WAITED) == 0)); - PROC_SUNLOCK(p); - if (report) { - report_alive_proc(td, p, siginfo, status, - options, CLD_STOPPED); - return (0); - } - } - if ((options & WCONTINUED) != 0 && - (p->p_flag & P_CONTINUED) != 0) { - report_alive_proc(td, p, siginfo, status, options, - CLD_CONTINUED); + if (wait6_check_alive(td, options, p, status, siginfo)) { + td->td_retval[0] = pid; return (0); } + PROC_UNLOCK(p); } @@ -1412,24 +1485,102 @@ loop_locked: sx_xunlock(&proctree_lock); return (ECHILD); } - if (options & WNOHANG) { + if ((options & WNOHANG) != 0) { sx_xunlock(&proctree_lock); td->td_retval[0] = 0; return (0); } PROC_LOCK(q); - if (q->p_flag & P_STATCHILD) { + if ((q->p_flag & P_STATCHILD) != 0) { q->p_flag &= ~P_STATCHILD; PROC_UNLOCK(q); goto loop_locked; } sx_xunlock(&proctree_lock); error = msleep(q, &q->p_mtx, PWAIT | PCATCH | PDROP, "wait", 0); - if (error) + if (error != 0) return (error); goto loop; } +int +kern_pdwait(struct thread *td, int fd, int *status, + int options, struct __wrusage *wrusage, siginfo_t *siginfo) +{ + struct proc *p; + struct file *fp; + struct procdesc *pd; + int error; + + AUDIT_ARG_FD(fd); + AUDIT_ARG_VALUE(options); + + error = wait6_checkopt(options); + if (error != 0) + return (error); + + error = fget(td, fd, &cap_pdwait_rights, &fp); + if (error != 0) + return (error); + if (fp->f_type != DTYPE_PROCDESC) { + error = EINVAL; + goto exit_unlocked; + } + pd = fp->f_data; + + for (;;) { + /* We own a reference on the procdesc file. */ + KASSERT((pd->pd_flags & PDF_CLOSED) == 0, + ("PDF_CLOSED proc %p procdesc %p pd flags %#x", + p, pd, pd->pd_flags)); + + sx_xlock(&proctree_lock); + p = pd->pd_proc; + if (p == NULL) { + error = ESRCH; + goto exit_tree_locked; + } + PROC_LOCK(p); + + error = p_canwait(td, p); + if (error != 0) + break; + if ((options & WEXITED) == 0 && p->p_state == PRS_ZOMBIE) { + error = ESRCH; + break; + } + + wait_fill_siginfo(p, siginfo); + wait_fill_wrusage(p, wrusage); + + if (p->p_state == PRS_ZOMBIE) { + proc_reap(td, p, status, options); + goto exit_unlocked; + } + + if (wait6_check_alive(td, options, p, status, siginfo)) + goto exit_unlocked; + + if ((options & WNOHANG) != 0) { + error = EWOULDBLOCK; + break; + } + + PROC_UNLOCK(p); + error = sx_sleep(&p->p_procdesc, &proctree_lock, + PWAIT | PCATCH | PDROP, "pdwait", 0); + if (error != 0) + goto exit_unlocked; + } + + PROC_UNLOCK(p); +exit_tree_locked: + sx_xunlock(&proctree_lock); +exit_unlocked: + fdrop(fp, td); + return (error); +} + void proc_add_orphan(struct proc *child, struct proc *parent) { diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 961d72c46d2c..6313bf3bb651 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -34,20 +34,22 @@ * SUCH DAMAGE. */ -#include <sys/cdefs.h> #include "opt_ktrace.h" #include "opt_kstack_pages.h" -#include <sys/param.h> +#define EXTERR_CATEGORY EXTERR_CAT_FORK #include <sys/systm.h> +#include <sys/acct.h> #include <sys/bitstring.h> -#include <sys/sysproto.h> #include <sys/eventhandler.h> +#include <sys/exterrvar.h> #include <sys/fcntl.h> #include <sys/filedesc.h> #include <sys/jail.h> #include <sys/kernel.h> #include <sys/kthread.h> +#include <sys/ktr.h> +#include <sys/ktrace.h> #include <sys/sysctl.h> #include <sys/lock.h> #include <sys/malloc.h> @@ -60,17 +62,15 @@ #include <sys/racct.h> #include <sys/resourcevar.h> #include <sys/sched.h> +#include <sys/sdt.h> +#include <sys/signalvar.h> +#include <sys/sx.h> #include <sys/syscall.h> +#include <sys/sysent.h> +#include <sys/sysproto.h> #include <sys/vmmeter.h> #include <sys/vnode.h> -#include <sys/acct.h> -#include <sys/ktr.h> -#include <sys/ktrace.h> #include <sys/unistd.h> -#include <sys/sdt.h> -#include <sys/sx.h> -#include <sys/sysent.h> -#include <sys/signalvar.h> #include <security/audit/audit.h> #include <security/mac/mac_framework.h> @@ -166,10 +166,11 @@ sys_rfork(struct thread *td, struct rfork_args *uap) /* Don't allow kernel-only flags. */ if ((uap->flags & RFKERNELONLY) != 0) - return (EINVAL); + return (EXTERROR(EINVAL, "Kernel-only flags %#jx", uap->flags)); /* RFSPAWN must not appear with others */ if ((uap->flags & RFSPAWN) != 0 && uap->flags != RFSPAWN) - return (EINVAL); + return (EXTERROR(EINVAL, "RFSPAWN must be the only flag %#jx", + uap->flags)); AUDIT_ARG_FFLAGS(uap->flags); bzero(&fr, sizeof(fr)); @@ -188,6 +189,48 @@ sys_rfork(struct thread *td, struct rfork_args *uap) return (error); } +int +sys_pdrfork(struct thread *td, struct pdrfork_args *uap) +{ + struct fork_req fr; + int error, fd, pid; + + bzero(&fr, sizeof(fr)); + fd = -1; + + AUDIT_ARG_FFLAGS(uap->pdflags); + AUDIT_ARG_CMD(uap->rfflags); + + if ((uap->rfflags & (RFSTOPPED | RFHIGHPID)) != 0) + return (EXTERROR(EINVAL, + "Kernel-only flags %#jx", uap->rfflags)); + + /* RFSPAWN must not appear with others */ + if ((uap->rfflags & RFSPAWN) != 0) { + if (uap->rfflags != RFSPAWN) + return (EXTERROR(EINVAL, + "RFSPAWN must be the only flag %#jx", + uap->rfflags)); + fr.fr_flags = RFFDG | RFPROC | RFPPWAIT | RFMEM | RFPROCDESC; + fr.fr_flags2 = FR2_DROPSIG_CAUGHT; + } else { + fr.fr_flags = uap->rfflags; + } + + fr.fr_pidp = &pid; + fr.fr_pd_fd = &fd; + fr.fr_pd_flags = uap->pdflags; + error = fork1(td, &fr); + if (error == 0) { + td->td_retval[0] = pid; + td->td_retval[1] = 0; + if ((fr.fr_flags & (RFPROC | RFPROCDESC)) == + (RFPROC | RFPROCDESC) || uap->rfflags == RFSPAWN) + error = copyout(&fd, uap->fdp, sizeof(fd)); + } + return (error); +} + int __exclusive_cache_line nprocs = 1; /* process 0 */ int lastpid = 0; SYSCTL_INT(_kern, OID_AUTO, lastpid, CTLFLAG_RD, &lastpid, 0, @@ -871,34 +914,32 @@ fork1(struct thread *td, struct fork_req *fr) else MPASS(fr->fr_procp == NULL); - /* Check for the undefined or unimplemented flags. */ if ((flags & ~(RFFLAGS | RFTSIGFLAGS(RFTSIGMASK))) != 0) - return (EINVAL); + return (EXTERROR(EINVAL, + "Undef or unimplemented flags %#jx", flags)); - /* Signal value requires RFTSIGZMB. */ if ((flags & RFTSIGFLAGS(RFTSIGMASK)) != 0 && (flags & RFTSIGZMB) == 0) - return (EINVAL); + return (EXTERROR(EINVAL, + "Signal value requires RFTSIGZMB", flags)); - /* Can't copy and clear. */ - if ((flags & (RFFDG|RFCFDG)) == (RFFDG|RFCFDG)) - return (EINVAL); + if ((flags & (RFFDG | RFCFDG)) == (RFFDG | RFCFDG)) + return (EXTERROR(EINVAL, "Can not copy and clear")); - /* Check the validity of the signal number. */ if ((flags & RFTSIGZMB) != 0 && (u_int)RFTSIGNUM(flags) > _SIG_MAXSIG) - return (EINVAL); + return (EXTERROR(EINVAL, "Invalid signal", RFTSIGNUM(flags))); if ((flags & RFPROCDESC) != 0) { - /* Can't not create a process yet get a process descriptor. */ if ((flags & RFPROC) == 0) - return (EINVAL); + return (EXTERROR(EINVAL, + "Can not not create a process yet get a process descriptor")); - /* Must provide a place to put a procdesc if creating one. */ if (fr->fr_pd_fd == NULL) - return (EINVAL); + return (EXTERROR(EINVAL, + "Must provide a place to put a procdesc if creating one")); - /* Check if we are using supported flags. */ if ((fr->fr_pd_flags & ~PD_ALLOWED_AT_FORK) != 0) - return (EINVAL); + return (EXTERROR(EINVAL, + "Invallid pdflags at fork %#jx", fr->fr_pd_flags)); } p1 = td->td_proc; diff --git a/sys/kern/kern_ktrace.c b/sys/kern/kern_ktrace.c index b58e69a3f38e..f3ee1c53fafd 100644 --- a/sys/kern/kern_ktrace.c +++ b/sys/kern/kern_ktrace.c @@ -838,8 +838,8 @@ ktrpsig(int sig, sig_t action, sigset_t *mask, int code) ktrace_exit(td); } -void -ktrcsw(int out, int user, const char *wmesg) +static void +ktrcsw_impl(int out, int user, const char *wmesg, const struct timespec *tv) { struct thread *td = curthread; struct ktr_request *req; @@ -854,6 +854,8 @@ ktrcsw(int out, int user, const char *wmesg) kc = &req->ktr_data.ktr_csw; kc->out = out; kc->user = user; + if (tv != NULL) + req->ktr_header.ktr_time = *tv; if (wmesg != NULL) strlcpy(kc->wmesg, wmesg, sizeof(kc->wmesg)); else @@ -863,6 +865,18 @@ ktrcsw(int out, int user, const char *wmesg) } void +ktrcsw(int out, int user, const char *wmesg) +{ + ktrcsw_impl(out, user, wmesg, NULL); +} + +void +ktrcsw_out(const struct timespec *tv, const char *wmesg) +{ + ktrcsw_impl(1, 0, wmesg, tv); +} + +void ktrstruct(const char *name, const void *data, size_t datalen) { struct ktr_request *req; diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 8e956324ee23..22628c78d8a5 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -133,6 +133,7 @@ _sleep(const void *ident, struct lock_object *lock, int priority, { struct thread *td __ktrace_used; struct lock_class *class; + struct timespec sw_out_tv __ktrace_used; uintptr_t lock_state; int catch, pri, rval, sleepq_flags; WITNESS_SAVE_DECL(lock_witness); @@ -141,7 +142,7 @@ _sleep(const void *ident, struct lock_object *lock, int priority, td = curthread; #ifdef KTRACE if (KTRPOINT(td, KTR_CSW)) - ktrcsw(1, 0, wmesg); + nanotime(&sw_out_tv); #endif WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, "Sleeping on \"%s\"", wmesg); @@ -222,8 +223,10 @@ _sleep(const void *ident, struct lock_object *lock, int priority, rval = 0; } #ifdef KTRACE - if (KTRPOINT(td, KTR_CSW)) + if (KTRPOINT(td, KTR_CSW)) { + ktrcsw_out(&sw_out_tv, wmesg); ktrcsw(0, 0, wmesg); + } #endif PICKUP_GIANT(); if (lock != NULL && lock != &Giant.lock_object && !(priority & PDROP)) { @@ -239,6 +242,7 @@ msleep_spin_sbt(const void *ident, struct mtx *mtx, const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags) { struct thread *td __ktrace_used; + struct timespec sw_out_tv __ktrace_used; int rval; WITNESS_SAVE_DECL(mtx); @@ -266,19 +270,9 @@ msleep_spin_sbt(const void *ident, struct mtx *mtx, const char *wmesg, if (sbt != 0) sleepq_set_timeout_sbt(ident, sbt, pr, flags); - /* - * Can't call ktrace with any spin locks held so it can lock the - * ktrace_mtx lock, and WITNESS_WARN considers it an error to hold - * any spin lock. Thus, we have to drop the sleepq spin lock while - * we handle those requests. This is safe since we have placed our - * thread on the sleep queue already. - */ #ifdef KTRACE - if (KTRPOINT(td, KTR_CSW)) { - sleepq_release(ident); - ktrcsw(1, 0, wmesg); - sleepq_lock(ident); - } + if (KTRPOINT(td, KTR_CSW)) + nanotime(&sw_out_tv); #endif #ifdef WITNESS sleepq_release(ident); @@ -293,8 +287,10 @@ msleep_spin_sbt(const void *ident, struct mtx *mtx, const char *wmesg, rval = 0; } #ifdef KTRACE - if (KTRPOINT(td, KTR_CSW)) + if (KTRPOINT(td, KTR_CSW)) { + ktrcsw_out(&sw_out_tv, wmesg); ktrcsw(0, 0, wmesg); + } #endif PICKUP_GIANT(); mtx_lock_spin(mtx); @@ -600,7 +596,7 @@ loadav(void *arg) loadav, NULL, C_DIRECT_EXEC | C_PREL(32)); } -static void +void ast_scheduler(struct thread *td, int tda __unused) { #ifdef KTRACE diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c index 25da134661e9..be0acb0a4a55 100644 --- a/sys/kern/kern_sysctl.c +++ b/sys/kern/kern_sysctl.c @@ -1619,8 +1619,8 @@ static SYSCTL_NODE(_sysctl, CTL_SYSCTL_OIDLABEL, oidlabel, CTLFLAG_RD | int sysctl_handle_bool(SYSCTL_HANDLER_ARGS) { - uint8_t temp; int error; + uint8_t temp; /* * Attempt to get a coherent snapshot by making a copy of the data. @@ -1630,16 +1630,40 @@ sysctl_handle_bool(SYSCTL_HANDLER_ARGS) else temp = arg2 ? 1 : 0; - error = SYSCTL_OUT(req, &temp, sizeof(temp)); + /* + * In order to support backwards-compatible conversion of integer knobs + * that are used as booleans to true boolean knobs, whose internal state + * is stored as a 'bool' and not an 'int', if exactly 4 bytes remain in + * the output buffer, we assume that the caller expected an 'int' + * instead of a 'uint8_t'. + */ + if (req->oldlen - req->oldidx == sizeof(int)) { + int temp_int = temp; + + error = SYSCTL_OUT(req, &temp_int, sizeof(temp_int)); + } else + error = SYSCTL_OUT(req, &temp, sizeof(temp)); if (error || !req->newptr) return (error); if (!arg1) error = EPERM; else { - error = SYSCTL_IN(req, &temp, sizeof(temp)); - if (!error) - *(bool *)arg1 = temp ? 1 : 0; + /* + * Conversely, if the input buffer has exactly 4 bytes to read, + * use them all to produce a bool. + */ + if (req->newlen - req->newidx == sizeof(int)) { + int temp_int; + + error = SYSCTL_IN(req, &temp_int, sizeof(temp_int)); + if (error == 0) + *(bool *)arg1 = temp_int != 0 ? 1 : 0; + } else { + error = SYSCTL_IN(req, &temp, sizeof(temp)); + if (error == 0) + *(bool *)arg1 = temp != 0 ? 1 : 0; + } } return (error); } diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c index 45edd186e6ce..4c77f444d961 100644 --- a/sys/kern/link_elf.c +++ b/sys/kern/link_elf.c @@ -2041,7 +2041,6 @@ link_elf_propagate_vnets(linker_file_t lf) } #endif -#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) || defined(__powerpc__) /* * Use this lookup routine when performing relocations early during boot. * The generic lookup routine depends on kobj, which is not initialized @@ -2104,4 +2103,3 @@ link_elf_late_ireloc(void) relocate_file1(ef, elf_lookup_ifunc, elf_reloc_late, true); } #endif -#endif diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index 504f9a2338ef..a47fe3ce72c3 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -34,12 +34,10 @@ * SUCH DAMAGE. */ -#include <sys/cdefs.h> #include "opt_hwpmc_hooks.h" #include "opt_hwt_hooks.h" #include "opt_sched.h" -#include <sys/param.h> #include <sys/systm.h> #include <sys/cpuset.h> #include <sys/kernel.h> @@ -68,12 +66,6 @@ #include <dev/hwt/hwt_hook.h> #endif -#ifdef KDTRACE_HOOKS -#include <sys/dtrace_bsd.h> -int __read_mostly dtrace_vtime_active; -dtrace_vtime_switch_func_t dtrace_vtime_switch_func; -#endif - /* * INVERSE_ESTCPU_WEIGHT is only suitable for statclock() frequencies in * the range 100-256 Hz (approximately). @@ -116,6 +108,8 @@ struct td_sched { #define TDF_BOUND TDF_SCHED1 /* Bound to one CPU. */ #define TDF_SLICEEND TDF_SCHED2 /* Thread time slice is over. */ +#define TDP_RESCHED TDP_SCHED1 /* Reschedule due to maybe_resched(). */ + /* flags kept in ts_flags */ #define TSF_AFFINITY 0x0001 /* Has a non-"full" CPU set. */ @@ -139,7 +133,6 @@ static void setup_runqs(void); static void schedcpu(void); static void schedcpu_thread(void); static void sched_priority(struct thread *td, u_char prio); -static void sched_setup(void *dummy); static void maybe_resched(struct thread *td); static void updatepri(struct thread *td); static void resetpriority(struct thread *td); @@ -155,13 +148,12 @@ static struct kproc_desc sched_kp = { schedcpu_thread, NULL }; -SYSINIT(schedcpu, SI_SUB_LAST, SI_ORDER_FIRST, kproc_start, - &sched_kp); -SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL); -static void sched_initticks(void *dummy); -SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, - NULL); +static void +sched_4bsd_schedcpu(void) +{ + kproc_start(&sched_kp); +} /* * Global run queue. @@ -198,7 +190,7 @@ setup_runqs(void) } static int -sysctl_kern_quantum(SYSCTL_HANDLER_ARGS) +sysctl_kern_4bsd_quantum(SYSCTL_HANDLER_ARGS) { int error, new_val, period; @@ -215,77 +207,58 @@ sysctl_kern_quantum(SYSCTL_HANDLER_ARGS) return (0); } -SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, - "Scheduler"); +SYSCTL_NODE(_kern_sched, OID_AUTO, 4bsd, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "4BSD Scheduler"); -SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, "4BSD", 0, - "Scheduler name"); -SYSCTL_PROC(_kern_sched, OID_AUTO, quantum, +SYSCTL_PROC(_kern_sched_4bsd, OID_AUTO, quantum, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, - sysctl_kern_quantum, "I", + sysctl_kern_4bsd_quantum, "I", "Quantum for timeshare threads in microseconds"); -SYSCTL_INT(_kern_sched, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0, +SYSCTL_INT(_kern_sched_4bsd, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0, "Quantum for timeshare threads in stathz ticks"); #ifdef SMP /* Enable forwarding of wakeups to all other cpus */ -static SYSCTL_NODE(_kern_sched, OID_AUTO, ipiwakeup, +static SYSCTL_NODE(_kern_sched_4bsd, OID_AUTO, ipiwakeup, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Kernel SMP"); static int runq_fuzz = 1; -SYSCTL_INT(_kern_sched, OID_AUTO, runq_fuzz, CTLFLAG_RW, &runq_fuzz, 0, ""); +SYSCTL_INT(_kern_sched_4bsd, OID_AUTO, runq_fuzz, CTLFLAG_RW, + &runq_fuzz, 0, ""); static int forward_wakeup_enabled = 1; -SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, enabled, CTLFLAG_RW, +SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, enabled, CTLFLAG_RW, &forward_wakeup_enabled, 0, "Forwarding of wakeup to idle CPUs"); static int forward_wakeups_requested = 0; -SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, requested, CTLFLAG_RD, +SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, requested, CTLFLAG_RD, &forward_wakeups_requested, 0, "Requests for Forwarding of wakeup to idle CPUs"); static int forward_wakeups_delivered = 0; -SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, delivered, CTLFLAG_RD, +SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, delivered, CTLFLAG_RD, &forward_wakeups_delivered, 0, "Completed Forwarding of wakeup to idle CPUs"); static int forward_wakeup_use_mask = 1; -SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, usemask, CTLFLAG_RW, +SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, usemask, CTLFLAG_RW, &forward_wakeup_use_mask, 0, "Use the mask of idle cpus"); static int forward_wakeup_use_loop = 0; -SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, useloop, CTLFLAG_RW, +SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, useloop, CTLFLAG_RW, &forward_wakeup_use_loop, 0, "Use a loop to find idle cpus"); #endif #if 0 static int sched_followon = 0; -SYSCTL_INT(_kern_sched, OID_AUTO, followon, CTLFLAG_RW, +SYSCTL_INT(_kern_sched_4bsd, OID_AUTO, followon, CTLFLAG_RW, &sched_followon, 0, "allow threads to share a quantum"); #endif -SDT_PROVIDER_DEFINE(sched); - -SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *", - "struct proc *", "uint8_t"); -SDT_PROBE_DEFINE3(sched, , , dequeue, "struct thread *", - "struct proc *", "void *"); -SDT_PROBE_DEFINE4(sched, , , enqueue, "struct thread *", - "struct proc *", "void *", "int"); -SDT_PROBE_DEFINE4(sched, , , lend__pri, "struct thread *", - "struct proc *", "uint8_t", "struct thread *"); -SDT_PROBE_DEFINE2(sched, , , load__change, "int", "int"); -SDT_PROBE_DEFINE2(sched, , , off__cpu, "struct thread *", - "struct proc *"); -SDT_PROBE_DEFINE(sched, , , on__cpu); -SDT_PROBE_DEFINE(sched, , , remain__cpu); -SDT_PROBE_DEFINE2(sched, , , surrender, "struct thread *", - "struct proc *"); - static __inline void sched_load_add(void) { @@ -303,6 +276,17 @@ sched_load_rem(void) KTR_COUNTER0(KTR_SCHED, "load", "global load", sched_tdcnt); SDT_PROBE2(sched, , , load__change, NOCPU, sched_tdcnt); } + +static void +maybe_resched_ast(struct thread *td, int tda) +{ + MPASS(td == curthread); /* We are AST */ + if ((td->td_pflags & TDP_RESCHED) != 0) { + td->td_pflags &= ~TDP_RESCHED; + ast_scheduler(td, tda); + } +} + /* * Arrange to reschedule if necessary, taking the priorities and * schedulers into account. @@ -310,10 +294,12 @@ sched_load_rem(void) static void maybe_resched(struct thread *td) { + struct thread *ctd; + ctd = curthread; THREAD_LOCK_ASSERT(td, MA_OWNED); - if (td->td_priority < curthread->td_priority) - ast_sched_locked(curthread, TDA_SCHED); + if (td->td_priority < ctd->td_priority) + ctd->td_pflags |= TDP_RESCHED; } /* @@ -322,7 +308,7 @@ maybe_resched(struct thread *td) * determines if the new thread should preempt the current thread. If so, * it sets td_owepreempt to request a preemption. */ -int +static int maybe_preempt(struct thread *td) { #ifdef PREEMPTION @@ -441,10 +427,7 @@ maybe_preempt(struct thread *td) #define loadfactor(loadav) (2 * (loadav)) #define decay_cpu(loadfac, cpu) (((loadfac) * (cpu)) / ((loadfac) + FSCALE)) -/* decay 95% of `ts_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */ -static fixpt_t ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */ -SYSCTL_UINT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, - "Decay factor used for updating %CPU"); +extern fixpt_t ccpu; /* * If `ccpu' is not equal to `exp(-1/20)' and you still want to use the @@ -640,22 +623,28 @@ resetpriority_thread(struct thread *td) sched_prio(td, td->td_user_pri); } -/* ARGSUSED */ static void -sched_setup(void *dummy) +sched_4bsd_setup(void) { + /* + * Decay 95% of `ts_pctcpu' in 60 seconds; see CCPU_SHIFT + * before changing. + */ + ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */ setup_runqs(); /* Account for thread0. */ sched_load_add(); + + ast_register(TDA_SCHED_PRIV, ASTR_UNCOND, 0, maybe_resched_ast); } /* * This routine determines time constants after stathz and hz are setup. */ static void -sched_initticks(void *dummy) +sched_4bsd_initticks(void) { realstathz = stathz ? stathz : hz; @@ -672,8 +661,8 @@ sched_initticks(void *dummy) * Called from: * proc0_init() */ -void -schedinit(void) +static void +sched_4bsd_init(void) { /* @@ -684,15 +673,15 @@ schedinit(void) mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN); } -void -schedinit_ap(void) +static void +sched_4bsd_init_ap(void) { /* Nothing needed. */ } -bool -sched_runnable(void) +static bool +sched_4bsd_runnable(void) { #ifdef SMP return (runq_not_empty(&runq) || @@ -702,18 +691,14 @@ sched_runnable(void) #endif } -int -sched_rr_interval(void) +static int +sched_4bsd_rr_interval(void) { /* Convert sched_slice from stathz to hz. */ return (imax(1, (sched_slice * hz + realstathz / 2) / realstathz)); } -SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions"); -SCHED_STAT_DEFINE(ithread_preemptions, - "Interrupt thread preemptions due to time-sharing"); - /* * We adjust the priority of the current process. The priority of a * process gets worse as it accumulates CPU time. The cpu usage @@ -773,8 +758,8 @@ sched_clock_tick(struct thread *td) stat->idlecalls = 0; } -void -sched_clock(struct thread *td, int cnt) +static void +sched_4bsd_clock(struct thread *td, int cnt) { for ( ; cnt > 0; cnt--) @@ -784,8 +769,8 @@ sched_clock(struct thread *td, int cnt) /* * Charge child's scheduling CPU usage to parent. */ -void -sched_exit(struct proc *p, struct thread *td) +static void +sched_4bsd_exit(struct proc *p, struct thread *td) { KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "proc exit", @@ -795,8 +780,8 @@ sched_exit(struct proc *p, struct thread *td) sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); } -void -sched_exit_thread(struct thread *td, struct thread *child) +static void +sched_4bsd_exit_thread(struct thread *td, struct thread *child) { KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "exit", @@ -811,14 +796,14 @@ sched_exit_thread(struct thread *td, struct thread *child) thread_unlock(child); } -void -sched_fork(struct thread *td, struct thread *childtd) +static void +sched_4bsd_fork(struct thread *td, struct thread *childtd) { sched_fork_thread(td, childtd); } -void -sched_fork_thread(struct thread *td, struct thread *childtd) +static void +sched_4bsd_fork_thread(struct thread *td, struct thread *childtd) { struct td_sched *ts, *tsc; @@ -836,8 +821,8 @@ sched_fork_thread(struct thread *td, struct thread *childtd) ts->ts_slice = 1; } -void -sched_nice(struct proc *p, int nice) +static void +sched_4bsd_nice(struct proc *p, int nice) { struct thread *td; @@ -851,8 +836,8 @@ sched_nice(struct proc *p, int nice) } } -void -sched_class(struct thread *td, int class) +static void +sched_4bsd_class(struct thread *td, int class) { THREAD_LOCK_ASSERT(td, MA_OWNED); td->td_pri_class = class; @@ -890,8 +875,8 @@ sched_priority(struct thread *td, u_char prio) * Update a thread's priority when it is lent another thread's * priority. */ -void -sched_lend_prio(struct thread *td, u_char prio) +static void +sched_4bsd_lend_prio(struct thread *td, u_char prio) { td->td_flags |= TDF_BORROWING; @@ -906,8 +891,8 @@ sched_lend_prio(struct thread *td, u_char prio) * important than prio the thread will keep a priority boost * of prio. */ -void -sched_unlend_prio(struct thread *td, u_char prio) +static void +sched_4bsd_unlend_prio(struct thread *td, u_char prio) { u_char base_pri; @@ -923,8 +908,8 @@ sched_unlend_prio(struct thread *td, u_char prio) sched_lend_prio(td, prio); } -void -sched_prio(struct thread *td, u_char prio) +static void +sched_4bsd_prio(struct thread *td, u_char prio) { u_char oldprio; @@ -950,8 +935,8 @@ sched_prio(struct thread *td, u_char prio) turnstile_adjust(td, oldprio); } -void -sched_ithread_prio(struct thread *td, u_char prio) +static void +sched_4bsd_ithread_prio(struct thread *td, u_char prio) { THREAD_LOCK_ASSERT(td, MA_OWNED); MPASS(td->td_pri_class == PRI_ITHD); @@ -959,8 +944,8 @@ sched_ithread_prio(struct thread *td, u_char prio) sched_prio(td, prio); } -void -sched_user_prio(struct thread *td, u_char prio) +static void +sched_4bsd_user_prio(struct thread *td, u_char prio) { THREAD_LOCK_ASSERT(td, MA_OWNED); @@ -970,8 +955,8 @@ sched_user_prio(struct thread *td, u_char prio) td->td_user_pri = prio; } -void -sched_lend_user_prio(struct thread *td, u_char prio) +static void +sched_4bsd_lend_user_prio(struct thread *td, u_char prio) { THREAD_LOCK_ASSERT(td, MA_OWNED); @@ -986,8 +971,8 @@ sched_lend_user_prio(struct thread *td, u_char prio) /* * Like the above but first check if there is anything to do. */ -void -sched_lend_user_prio_cond(struct thread *td, u_char prio) +static void +sched_4bsd_lend_user_prio_cond(struct thread *td, u_char prio) { if (td->td_lend_user_pri == prio) @@ -998,8 +983,8 @@ sched_lend_user_prio_cond(struct thread *td, u_char prio) thread_unlock(td); } -void -sched_sleep(struct thread *td, int pri) +static void +sched_4bsd_sleep(struct thread *td, int pri) { THREAD_LOCK_ASSERT(td, MA_OWNED); @@ -1009,8 +994,8 @@ sched_sleep(struct thread *td, int pri) sched_prio(td, pri); } -void -sched_switch(struct thread *td, int flags) +static void +sched_4bsd_sswitch(struct thread *td, int flags) { struct thread *newtd; struct mtx *tmtx; @@ -1142,8 +1127,8 @@ sched_switch(struct thread *td, int flags) mtx_unlock_spin(&sched_lock); } -void -sched_wakeup(struct thread *td, int srqflags) +static void +sched_4bsd_wakeup(struct thread *td, int srqflags) { struct td_sched *ts; @@ -1318,8 +1303,8 @@ sched_pickcpu(struct thread *td) } #endif -void -sched_add(struct thread *td, int flags) +static void +sched_4bsd_add(struct thread *td, int flags) #ifdef SMP { cpuset_t tidlemsk; @@ -1466,8 +1451,8 @@ sched_add(struct thread *td, int flags) } #endif /* SMP */ -void -sched_rem(struct thread *td) +static void +sched_4bsd_rem(struct thread *td) { struct td_sched *ts; @@ -1496,8 +1481,8 @@ sched_rem(struct thread *td) * Select threads to run. Note that running threads still consume a * slot. */ -struct thread * -sched_choose(void) +static struct thread * +sched_4bsd_choose(void) { struct thread *td; struct runq *rq; @@ -1541,8 +1526,8 @@ sched_choose(void) return (PCPU_GET(idlethread)); } -void -sched_preempt(struct thread *td) +static void +sched_4bsd_preempt(struct thread *td) { int flags; @@ -1558,8 +1543,8 @@ sched_preempt(struct thread *td) } } -void -sched_userret_slowpath(struct thread *td) +static void +sched_4bsd_userret_slowpath(struct thread *td) { thread_lock(td); @@ -1568,8 +1553,8 @@ sched_userret_slowpath(struct thread *td) thread_unlock(td); } -void -sched_bind(struct thread *td, int cpu) +static void +sched_4bsd_bind(struct thread *td, int cpu) { #ifdef SMP struct td_sched *ts = td_get_sched(td); @@ -1589,48 +1574,48 @@ sched_bind(struct thread *td, int cpu) #endif } -void -sched_unbind(struct thread* td) +static void +sched_4bsd_unbind(struct thread* td) { THREAD_LOCK_ASSERT(td, MA_OWNED); KASSERT(td == curthread, ("sched_unbind: can only bind curthread")); td->td_flags &= ~TDF_BOUND; } -int -sched_is_bound(struct thread *td) +static int +sched_4bsd_is_bound(struct thread *td) { THREAD_LOCK_ASSERT(td, MA_OWNED); return (td->td_flags & TDF_BOUND); } -void -sched_relinquish(struct thread *td) +static void +sched_4bsd_relinquish(struct thread *td) { thread_lock(td); mi_switch(SW_VOL | SWT_RELINQUISH); } -int -sched_load(void) +static int +sched_4bsd_load(void) { return (sched_tdcnt); } -int -sched_sizeof_proc(void) +static int +sched_4bsd_sizeof_proc(void) { return (sizeof(struct proc)); } -int -sched_sizeof_thread(void) +static int +sched_4bsd_sizeof_thread(void) { return (sizeof(struct thread) + sizeof(struct td_sched)); } -fixpt_t -sched_pctcpu(struct thread *td) +static fixpt_t +sched_4bsd_pctcpu(struct thread *td) { struct td_sched *ts; @@ -1639,42 +1624,8 @@ sched_pctcpu(struct thread *td) return (ts->ts_pctcpu); } -#ifdef RACCT -/* - * Calculates the contribution to the thread cpu usage for the latest - * (unfinished) second. - */ -fixpt_t -sched_pctcpu_delta(struct thread *td) -{ - struct td_sched *ts; - fixpt_t delta; - int realstathz; - - THREAD_LOCK_ASSERT(td, MA_OWNED); - ts = td_get_sched(td); - delta = 0; - realstathz = stathz ? stathz : hz; - if (ts->ts_cpticks != 0) { -#if (FSHIFT >= CCPU_SHIFT) - delta = (realstathz == 100) - ? ((fixpt_t) ts->ts_cpticks) << - (FSHIFT - CCPU_SHIFT) : - 100 * (((fixpt_t) ts->ts_cpticks) - << (FSHIFT - CCPU_SHIFT)) / realstathz; -#else - delta = ((FSCALE - ccpu) * - (ts->ts_cpticks * - FSCALE / realstathz)) >> FSHIFT; -#endif - } - - return (delta); -} -#endif - -u_int -sched_estcpu(struct thread *td) +static u_int +sched_4bsd_estcpu(struct thread *td) { return (td_get_sched(td)->ts_estcpu); @@ -1683,8 +1634,8 @@ sched_estcpu(struct thread *td) /* * The actual idle process. */ -void -sched_idletd(void *dummy) +static void +sched_4bsd_idletd(void *dummy) { struct pcpuidlestat *stat; @@ -1725,8 +1676,8 @@ sched_throw_tail(struct thread *td) /* * A CPU is entering for the first time. */ -void -sched_ap_entry(void) +static void +sched_4bsd_ap_entry(void) { /* @@ -1749,8 +1700,8 @@ sched_ap_entry(void) /* * A thread is exiting. */ -void -sched_throw(struct thread *td) +static void +sched_4bsd_throw(struct thread *td) { MPASS(td != NULL); @@ -1763,8 +1714,8 @@ sched_throw(struct thread *td) sched_throw_tail(td); } -void -sched_fork_exit(struct thread *td) +static void +sched_4bsd_fork_exit(struct thread *td) { /* @@ -1782,8 +1733,8 @@ sched_fork_exit(struct thread *td) SDT_PROBE0(sched, , , on__cpu); } -char * -sched_tdname(struct thread *td) +static char * +sched_4bsd_tdname(struct thread *td) { #ifdef KTR struct td_sched *ts; @@ -1798,19 +1749,19 @@ sched_tdname(struct thread *td) #endif } -#ifdef KTR -void -sched_clear_tdname(struct thread *td) +static void +sched_4bsd_clear_tdname(struct thread *td) { +#ifdef KTR struct td_sched *ts; ts = td_get_sched(td); ts->ts_name[0] = '\0'; -} #endif +} -void -sched_affinity(struct thread *td) +static void +sched_4bsd_affinity(struct thread *td) { #ifdef SMP struct td_sched *ts; @@ -1872,3 +1823,83 @@ sched_affinity(struct thread *td) } #endif } + +static bool +sched_4bsd_do_timer_accounting(void) +{ +#ifdef SMP + /* + * Don't do any accounting for the disabled HTT cores, since it + * will provide misleading numbers for the userland. + * + * No locking is necessary here, since even if we lose the race + * when hlt_cpus_mask changes it is not a big deal, really. + * + * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask + * and unlike other schedulers it actually schedules threads to + * those CPUs. + */ + return (!CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask)); +#else + return (true); +#endif +} + +static int +sched_4bsd_find_l2_neighbor(int cpu) +{ + return (-1); +} + +struct sched_instance sched_4bsd_instance = { +#define SLOT(name) .name = sched_4bsd_##name + SLOT(load), + SLOT(rr_interval), + SLOT(runnable), + SLOT(exit), + SLOT(fork), + SLOT(fork_exit), + SLOT(class), + SLOT(nice), + SLOT(ap_entry), + SLOT(exit_thread), + SLOT(estcpu), + SLOT(fork_thread), + SLOT(ithread_prio), + SLOT(lend_prio), + SLOT(lend_user_prio), + SLOT(lend_user_prio_cond), + SLOT(pctcpu), + SLOT(prio), + SLOT(sleep), + SLOT(sswitch), + SLOT(throw), + SLOT(unlend_prio), + SLOT(user_prio), + SLOT(userret_slowpath), + SLOT(add), + SLOT(choose), + SLOT(clock), + SLOT(idletd), + SLOT(preempt), + SLOT(relinquish), + SLOT(rem), + SLOT(wakeup), + SLOT(bind), + SLOT(unbind), + SLOT(is_bound), + SLOT(affinity), + SLOT(sizeof_proc), + SLOT(sizeof_thread), + SLOT(tdname), + SLOT(clear_tdname), + SLOT(do_timer_accounting), + SLOT(find_l2_neighbor), + SLOT(init), + SLOT(init_ap), + SLOT(setup), + SLOT(initticks), + SLOT(schedcpu), +#undef SLOT +}; +DECLARE_SCHEDULER(fourbsd_sched_selector, "4BSD", &sched_4bsd_instance); diff --git a/sys/kern/sched_shim.c b/sys/kern/sched_shim.c new file mode 100644 index 000000000000..2b1dcecda433 --- /dev/null +++ b/sys/kern/sched_shim.c @@ -0,0 +1,317 @@ +/* + * Copyright 2026 The FreeBSD Foundation + * + * SPDX-License-Identifier: BSD-2-Clause + * + * This software was developed by Konstantin Belousov <kib@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + */ + +#include "opt_sched.h" + +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/proc.h> +#include <sys/runq.h> +#include <sys/sbuf.h> +#include <sys/sched.h> +#include <sys/smp.h> +#include <sys/sysctl.h> +#include <machine/ifunc.h> + +const struct sched_instance *active_sched; + +#define __DEFINE_SHIM(__m, __r, __n, __p, __a) \ + __r \ + __n __p \ + { \ + return (active_sched->__m __a); \ + } +#define DEFINE_SHIM0(__m, __r, __n) \ + __DEFINE_SHIM(__m, __r, __n, (void), ()) +#define DEFINE_SHIM1(__m, __r, __n, __t1, __a1) \ + __DEFINE_SHIM(__m, __r, __n, (__t1 __a1), (__a1)) +#define DEFINE_SHIM2(__m, __r, __n, __t1, __a1, __t2, __a2) \ + __DEFINE_SHIM(__m, __r, __n, (__t1 __a1, __t2 __a2), (__a1, __a2)) + +DEFINE_SHIM0(load, int, sched_load) +DEFINE_SHIM0(rr_interval, int, sched_rr_interval) +DEFINE_SHIM0(runnable, bool, sched_runnable) +DEFINE_SHIM2(exit, void, sched_exit, struct proc *, p, + struct thread *, childtd) +DEFINE_SHIM2(fork, void, sched_fork, struct thread *, td, + struct thread *, childtd) +DEFINE_SHIM1(fork_exit, void, sched_fork_exit, struct thread *, td) +DEFINE_SHIM2(class, void, sched_class, struct thread *, td, int, class) +DEFINE_SHIM2(nice, void, sched_nice, struct proc *, p, int, nice) +DEFINE_SHIM0(ap_entry, void, sched_ap_entry) +DEFINE_SHIM2(exit_thread, void, sched_exit_thread, struct thread *, td, + struct thread *, child) +DEFINE_SHIM1(estcpu, u_int, sched_estcpu, struct thread *, td) +DEFINE_SHIM2(fork_thread, void, sched_fork_thread, struct thread *, td, + struct thread *, child) +DEFINE_SHIM2(ithread_prio, void, sched_ithread_prio, struct thread *, td, + u_char, prio) +DEFINE_SHIM2(lend_prio, void, sched_lend_prio, struct thread *, td, + u_char, prio) +DEFINE_SHIM2(lend_user_prio, void, sched_lend_user_prio, struct thread *, td, + u_char, pri) +DEFINE_SHIM2(lend_user_prio_cond, void, sched_lend_user_prio_cond, + struct thread *, td, u_char, pri) +DEFINE_SHIM1(pctcpu, fixpt_t, sched_pctcpu, struct thread *, td) +DEFINE_SHIM2(prio, void, sched_prio, struct thread *, td, u_char, prio) +DEFINE_SHIM2(sleep, void, sched_sleep, struct thread *, td, int, prio) +DEFINE_SHIM2(sswitch, void, sched_switch, struct thread *, td, int, flags) +DEFINE_SHIM1(throw, void, sched_throw, struct thread *, td) +DEFINE_SHIM2(unlend_prio, void, sched_unlend_prio, struct thread *, td, + u_char, prio) +DEFINE_SHIM2(user_prio, void, sched_user_prio, struct thread *, td, + u_char, prio) +DEFINE_SHIM1(userret_slowpath, void, sched_userret_slowpath, + struct thread *, td) +DEFINE_SHIM2(add, void, sched_add, struct thread *, td, int, flags) +DEFINE_SHIM0(choose, struct thread *, sched_choose) +DEFINE_SHIM2(clock, void, sched_clock, struct thread *, td, int, cnt) +DEFINE_SHIM1(idletd, void, sched_idletd, void *, dummy) +DEFINE_SHIM1(preempt, void, sched_preempt, struct thread *, td) +DEFINE_SHIM1(relinquish, void, sched_relinquish, struct thread *, td) +DEFINE_SHIM1(rem, void, sched_rem, struct thread *, td) +DEFINE_SHIM2(wakeup, void, sched_wakeup, struct thread *, td, int, srqflags) +DEFINE_SHIM2(bind, void, sched_bind, struct thread *, td, int, cpu) +DEFINE_SHIM1(unbind, void, sched_unbind, struct thread *, td) +DEFINE_SHIM1(is_bound, int, sched_is_bound, struct thread *, td) +DEFINE_SHIM1(affinity, void, sched_affinity, struct thread *, td) +DEFINE_SHIM0(sizeof_proc, int, sched_sizeof_proc) +DEFINE_SHIM0(sizeof_thread, int, sched_sizeof_thread) +DEFINE_SHIM1(tdname, char *, sched_tdname, struct thread *, td) +DEFINE_SHIM1(clear_tdname, void, sched_clear_tdname, struct thread *, td) +DEFINE_SHIM0(do_timer_accounting, bool, sched_do_timer_accounting) +DEFINE_SHIM1(find_l2_neighbor, int, sched_find_l2_neighbor, int, cpu) +DEFINE_SHIM0(init_ap, void, schedinit_ap) + + +SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions"); +SCHED_STAT_DEFINE(ithread_preemptions, + "Interrupt thread preemptions due to time-sharing"); + +SDT_PROVIDER_DEFINE(sched); + +SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *", + "struct proc *", "uint8_t"); +SDT_PROBE_DEFINE3(sched, , , dequeue, "struct thread *", + "struct proc *", "void *"); +SDT_PROBE_DEFINE4(sched, , , enqueue, "struct thread *", + "struct proc *", "void *", "int"); +SDT_PROBE_DEFINE4(sched, , , lend__pri, "struct thread *", + "struct proc *", "uint8_t", "struct thread *"); +SDT_PROBE_DEFINE2(sched, , , load__change, "int", "int"); +SDT_PROBE_DEFINE2(sched, , , off__cpu, "struct thread *", + "struct proc *"); +SDT_PROBE_DEFINE(sched, , , on__cpu); +SDT_PROBE_DEFINE(sched, , , remain__cpu); +SDT_PROBE_DEFINE2(sched, , , surrender, "struct thread *", + "struct proc *"); + +#ifdef KDTRACE_HOOKS +#include <sys/dtrace_bsd.h> +int __read_mostly dtrace_vtime_active; +dtrace_vtime_switch_func_t dtrace_vtime_switch_func; +#endif + +static char sched_name[32] = "ULE"; + +SET_DECLARE(sched_instance_set, struct sched_selection); + +void +sched_instance_select(void) +{ + struct sched_selection *s, **ss; + int i; + + TUNABLE_STR_FETCH("kern.sched.name", sched_name, sizeof(sched_name)); + SET_FOREACH(ss, sched_instance_set) { + s = *ss; + for (i = 0; s->name[i] == sched_name[i]; i++) { + if (s->name[i] == '\0') { + active_sched = s->instance; + return; + } + } + } + + /* + * No scheduler matching the configuration was found. If + * there is any scheduler compiled in, at all, use the first + * scheduler from the linker set. + */ + if (SET_BEGIN(sched_instance_set) < SET_LIMIT(sched_instance_set)) { + s = *SET_BEGIN(sched_instance_set); + active_sched = s->instance; + for (i = 0;; i++) { + sched_name[i] = s->name[i]; + if (s->name[i] == '\0') + break; + } + } +} + +void +schedinit(void) +{ + if (active_sched == NULL) + panic("Cannot find scheduler %s", sched_name); + active_sched->init(); +} + +struct cpu_group __read_mostly *cpu_top; /* CPU topology */ + +static void +sched_setup(void *dummy) +{ + cpu_top = smp_topo(); + active_sched->setup(); +} +SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL); + +static void +sched_initticks(void *dummy) +{ + active_sched->initticks(); +} +SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, + NULL); + +static void +sched_schedcpu(void) +{ + active_sched->schedcpu(); +} +SYSINIT(schedcpu, SI_SUB_LAST, SI_ORDER_FIRST, sched_schedcpu, NULL); + +SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "Scheduler"); + +SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, sched_name, 0, + "Scheduler name"); + +static int +sysctl_kern_sched_available(SYSCTL_HANDLER_ARGS) +{ + struct sched_selection *s, **ss; + struct sbuf *sb, sm; + int error; + bool first; + + sb = sbuf_new_for_sysctl(&sm, NULL, 0, req); + if (sb == NULL) + return (ENOMEM); + first = true; + SET_FOREACH(ss, sched_instance_set) { + s = *ss; + if (first) + first = false; + else + sbuf_cat(sb, ","); + sbuf_cat(sb, s->name); + } + error = sbuf_finish(sb); + sbuf_delete(sb); + return (error); +} + +SYSCTL_PROC(_kern_sched, OID_AUTO, available, + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, + NULL, 0, sysctl_kern_sched_available, "A", + "List of available schedulers"); + +fixpt_t ccpu; +SYSCTL_UINT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, + "Decay factor used for updating %CPU"); + +/* + * Build the CPU topology dump string. Is recursively called to collect + * the topology tree. + */ +static int +sysctl_kern_sched_topology_spec_internal(struct sbuf *sb, + struct cpu_group *cg, int indent) +{ + char cpusetbuf[CPUSETBUFSIZ]; + int i, first; + + if (cpu_top == NULL) { + sbuf_printf(sb, "%*s<group level=\"1\" cache-level=\"1\">\n", + indent, ""); + sbuf_printf(sb, "%*s</group>\n", indent, ""); + return (0); + } + + sbuf_printf(sb, "%*s<group level=\"%d\" cache-level=\"%d\">\n", indent, + "", 1 + indent / 2, cg->cg_level); + sbuf_printf(sb, "%*s <cpu count=\"%d\" mask=\"%s\">", indent, "", + cg->cg_count, cpusetobj_strprint(cpusetbuf, &cg->cg_mask)); + first = TRUE; + for (i = cg->cg_first; i <= cg->cg_last; i++) { + if (CPU_ISSET(i, &cg->cg_mask)) { + if (!first) + sbuf_cat(sb, ", "); + else + first = FALSE; + sbuf_printf(sb, "%d", i); + } + } + sbuf_cat(sb, "</cpu>\n"); + + if (cg->cg_flags != 0) { + sbuf_printf(sb, "%*s <flags>", indent, ""); + if ((cg->cg_flags & CG_FLAG_HTT) != 0) + sbuf_cat(sb, "<flag name=\"HTT\">HTT group</flag>"); + if ((cg->cg_flags & CG_FLAG_THREAD) != 0) + sbuf_cat(sb, "<flag name=\"THREAD\">THREAD group</flag>"); + if ((cg->cg_flags & CG_FLAG_SMT) != 0) + sbuf_cat(sb, "<flag name=\"SMT\">SMT group</flag>"); + if ((cg->cg_flags & CG_FLAG_NODE) != 0) + sbuf_cat(sb, "<flag name=\"NODE\">NUMA node</flag>"); + sbuf_cat(sb, "</flags>\n"); + } + + if (cg->cg_children > 0) { + sbuf_printf(sb, "%*s <children>\n", indent, ""); + for (i = 0; i < cg->cg_children; i++) + sysctl_kern_sched_topology_spec_internal(sb, + &cg->cg_child[i], indent + 2); + sbuf_printf(sb, "%*s </children>\n", indent, ""); + } + sbuf_printf(sb, "%*s</group>\n", indent, ""); + return (0); +} + +/* + * Sysctl handler for retrieving topology dump. It's a wrapper for + * the recursive sysctl_kern_smp_topology_spec_internal(). + */ +static int +sysctl_kern_sched_topology_spec(SYSCTL_HANDLER_ARGS) +{ + struct sbuf *topo; + int err; + + topo = sbuf_new_for_sysctl(NULL, NULL, 512, req); + if (topo == NULL) + return (ENOMEM); + + sbuf_cat(topo, "<groups>\n"); + err = sysctl_kern_sched_topology_spec_internal(topo, cpu_top, 1); + sbuf_cat(topo, "</groups>\n"); + + if (err == 0) + err = sbuf_finish(topo); + sbuf_delete(topo); + return (err); +} + +SYSCTL_PROC(_kern_sched, OID_AUTO, topology_spec, CTLTYPE_STRING | + CTLFLAG_MPSAFE | CTLFLAG_RD, NULL, 0, + sysctl_kern_sched_topology_spec, "A", + "XML dump of detected CPU topology"); diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index 409439ca34da..c6bfe15e768b 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -37,12 +37,10 @@ * Isilon Systems and a general lack of creativity on the part of the author. */ -#include <sys/cdefs.h> #include "opt_hwpmc_hooks.h" #include "opt_hwt_hooks.h" #include "opt_sched.h" -#include <sys/param.h> #include <sys/systm.h> #include <sys/kdb.h> #include <sys/kernel.h> @@ -74,12 +72,6 @@ #include <dev/hwt/hwt_hook.h> #endif -#ifdef KDTRACE_HOOKS -#include <sys/dtrace_bsd.h> -int __read_mostly dtrace_vtime_active; -dtrace_vtime_switch_func_t dtrace_vtime_switch_func; -#endif - #include <machine/cpu.h> #include <machine/smp.h> @@ -312,7 +304,6 @@ struct tdq { atomic_load_short(&(tdq)->tdq_switchcnt) + 1)) #ifdef SMP -struct cpu_group __read_mostly *cpu_top; /* CPU topology */ #define SCHED_AFFINITY_DEFAULT (max(1, hz / 1000)) /* @@ -406,36 +397,8 @@ static void sched_balance(void); static bool sched_balance_pair(struct tdq *, struct tdq *); static inline struct tdq *sched_setcpu(struct thread *, int, int); static inline void thread_unblock_switch(struct thread *, struct mtx *); -static int sysctl_kern_sched_topology_spec(SYSCTL_HANDLER_ARGS); -static int sysctl_kern_sched_topology_spec_internal(struct sbuf *sb, - struct cpu_group *cg, int indent); #endif -static void sched_setup(void *dummy); -SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL); - -static void sched_initticks(void *dummy); -SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, - NULL); - -SDT_PROVIDER_DEFINE(sched); - -SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *", - "struct proc *", "uint8_t"); -SDT_PROBE_DEFINE3(sched, , , dequeue, "struct thread *", - "struct proc *", "void *"); -SDT_PROBE_DEFINE4(sched, , , enqueue, "struct thread *", - "struct proc *", "void *", "int"); -SDT_PROBE_DEFINE4(sched, , , lend__pri, "struct thread *", - "struct proc *", "uint8_t", "struct thread *"); -SDT_PROBE_DEFINE2(sched, , , load__change, "int", "int"); -SDT_PROBE_DEFINE2(sched, , , off__cpu, "struct thread *", - "struct proc *"); -SDT_PROBE_DEFINE(sched, , , on__cpu); -SDT_PROBE_DEFINE(sched, , , remain__cpu); -SDT_PROBE_DEFINE2(sched, , , surrender, "struct thread *", - "struct proc *"); - /* * Print the threads waiting on a run-queue. */ @@ -1623,7 +1586,6 @@ sched_setup_smp(void) struct tdq *tdq; int i; - cpu_top = smp_topo(); CPU_FOREACH(i) { tdq = DPCPU_ID_PTR(i, tdq); tdq_setup(tdq, i); @@ -1642,7 +1604,7 @@ sched_setup_smp(void) * information. */ static void -sched_setup(void *dummy) +sched_ule_setup(void) { struct tdq *tdq; @@ -1667,7 +1629,7 @@ sched_setup(void *dummy) */ /* ARGSUSED */ static void -sched_initticks(void *dummy) +sched_ule_initticks(void) { int incr; @@ -1891,8 +1853,8 @@ sched_interact_fork(struct thread *td) /* * Called from proc0_init() to setup the scheduler fields. */ -void -schedinit(void) +static void +sched_ule_init(void) { struct td_sched *ts0; @@ -1916,8 +1878,8 @@ schedinit(void) * TDQ_SELF() relies on the below sched pcpu setting; it may be used only * after schedinit_ap(). */ -void -schedinit_ap(void) +static void +sched_ule_init_ap(void) { #ifdef SMP @@ -1931,8 +1893,8 @@ schedinit_ap(void) * priority they will switch when their slices run out, which will be * at most sched_slice stathz ticks. */ -int -sched_rr_interval(void) +static int +sched_ule_rr_interval(void) { /* Convert sched_slice from stathz to hz. */ @@ -2051,8 +2013,8 @@ sched_thread_priority(struct thread *td, u_char prio) * Update a thread's priority when it is lent another thread's * priority. */ -void -sched_lend_prio(struct thread *td, u_char prio) +static void +sched_ule_lend_prio(struct thread *td, u_char prio) { td->td_flags |= TDF_BORROWING; @@ -2067,8 +2029,8 @@ sched_lend_prio(struct thread *td, u_char prio) * important than prio, the thread will keep a priority boost * of prio. */ -void -sched_unlend_prio(struct thread *td, u_char prio) +static void +sched_ule_unlend_prio(struct thread *td, u_char prio) { u_char base_pri; @@ -2087,8 +2049,8 @@ sched_unlend_prio(struct thread *td, u_char prio) /* * Standard entry for setting the priority to an absolute value. */ -void -sched_prio(struct thread *td, u_char prio) +static void +sched_ule_prio(struct thread *td, u_char prio) { u_char oldprio; @@ -2117,8 +2079,8 @@ sched_prio(struct thread *td, u_char prio) /* * Set the base interrupt thread priority. */ -void -sched_ithread_prio(struct thread *td, u_char prio) +static void +sched_ule_ithread_prio(struct thread *td, u_char prio) { THREAD_LOCK_ASSERT(td, MA_OWNED); MPASS(td->td_pri_class == PRI_ITHD); @@ -2129,8 +2091,8 @@ sched_ithread_prio(struct thread *td, u_char prio) /* * Set the base user priority, does not effect current running priority. */ -void -sched_user_prio(struct thread *td, u_char prio) +static void +sched_ule_user_prio(struct thread *td, u_char prio) { td->td_base_user_pri = prio; @@ -2139,8 +2101,8 @@ sched_user_prio(struct thread *td, u_char prio) td->td_user_pri = prio; } -void -sched_lend_user_prio(struct thread *td, u_char prio) +static void +sched_ule_lend_user_prio(struct thread *td, u_char prio) { THREAD_LOCK_ASSERT(td, MA_OWNED); @@ -2155,8 +2117,8 @@ sched_lend_user_prio(struct thread *td, u_char prio) /* * Like the above but first check if there is anything to do. */ -void -sched_lend_user_prio_cond(struct thread *td, u_char prio) +static void +sched_ule_lend_user_prio_cond(struct thread *td, u_char prio) { if (td->td_lend_user_pri == prio) @@ -2327,8 +2289,8 @@ thread_unblock_switch(struct thread *td, struct mtx *mtx) * migrating a thread from one queue to another as running threads may * be assigned elsewhere via binding. */ -void -sched_switch(struct thread *td, int flags) +static void +sched_ule_sswitch(struct thread *td, int flags) { struct thread *newtd; struct tdq *tdq; @@ -2466,8 +2428,8 @@ sched_switch(struct thread *td, int flags) /* * Adjust thread priorities as a result of a nice request. */ -void -sched_nice(struct proc *p, int nice) +static void +sched_ule_nice(struct proc *p, int nice) { struct thread *td; @@ -2485,8 +2447,8 @@ sched_nice(struct proc *p, int nice) /* * Record the sleep time for the interactivity scorer. */ -void -sched_sleep(struct thread *td, int prio) +static void +sched_ule_sleep(struct thread *td, int prio) { THREAD_LOCK_ASSERT(td, MA_OWNED); @@ -2506,8 +2468,8 @@ sched_sleep(struct thread *td, int prio) * * Requires the thread lock on entry, drops on exit. */ -void -sched_wakeup(struct thread *td, int srqflags) +static void +sched_ule_wakeup(struct thread *td, int srqflags) { struct td_sched *ts; int slptick; @@ -2546,8 +2508,8 @@ sched_wakeup(struct thread *td, int srqflags) * Penalize the parent for creating a new child and initialize the child's * priority. */ -void -sched_fork(struct thread *td, struct thread *child) +static void +sched_ule_fork(struct thread *td, struct thread *child) { THREAD_LOCK_ASSERT(td, MA_OWNED); sched_pctcpu_update(td_get_sched(td), 1); @@ -2565,8 +2527,8 @@ sched_fork(struct thread *td, struct thread *child) /* * Fork a new thread, may be within the same process. */ -void -sched_fork_thread(struct thread *td, struct thread *child) +static void +sched_ule_fork_thread(struct thread *td, struct thread *child) { struct td_sched *ts; struct td_sched *ts2; @@ -2611,8 +2573,8 @@ sched_fork_thread(struct thread *td, struct thread *child) /* * Adjust the priority class of a thread. */ -void -sched_class(struct thread *td, int class) +static void +sched_ule_class(struct thread *td, int class) { THREAD_LOCK_ASSERT(td, MA_OWNED); @@ -2624,8 +2586,8 @@ sched_class(struct thread *td, int class) /* * Return some of the child's priority and interactivity to the parent. */ -void -sched_exit(struct proc *p, struct thread *child) +static void +sched_ule_exit(struct proc *p, struct thread *child) { struct thread *td; @@ -2642,8 +2604,8 @@ sched_exit(struct proc *p, struct thread *child) * jobs such as make. This has little effect on the make process itself but * causes new processes spawned by it to receive worse scores immediately. */ -void -sched_exit_thread(struct thread *td, struct thread *child) +static void +sched_ule_exit_thread(struct thread *td, struct thread *child) { KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "thread exit", @@ -2660,8 +2622,8 @@ sched_exit_thread(struct thread *td, struct thread *child) thread_unlock(td); } -void -sched_preempt(struct thread *td) +static void +sched_ule_preempt(struct thread *td) { struct tdq *tdq; int flags; @@ -2691,8 +2653,8 @@ sched_preempt(struct thread *td) * Fix priorities on return to user-space. Priorities may be elevated due * to static priorities in msleep() or similar. */ -void -sched_userret_slowpath(struct thread *td) +static void +sched_ule_userret_slowpath(struct thread *td) { thread_lock(td); @@ -2702,10 +2664,6 @@ sched_userret_slowpath(struct thread *td) thread_unlock(td); } -SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions"); -SCHED_STAT_DEFINE(ithread_preemptions, - "Interrupt thread preemptions due to time-sharing"); - /* * Return time slice for a given thread. For ithreads this is * sched_slice. For other threads it is tdq_slice(tdq). @@ -2722,8 +2680,8 @@ td_slice(struct thread *td, struct tdq *tdq) * Handle a stathz tick. This is really only relevant for timeshare * and interrupt threads. */ -void -sched_clock(struct thread *td, int cnt) +static void +sched_ule_clock(struct thread *td, int cnt) { struct tdq *tdq; struct td_sched *ts; @@ -2808,8 +2766,8 @@ sched_clock(struct thread *td, int cnt) } } -u_int -sched_estcpu(struct thread *td __unused) +static u_int +sched_ule_estcpu(struct thread *td __unused) { return (0); @@ -2819,8 +2777,8 @@ sched_estcpu(struct thread *td __unused) * Return whether the current CPU has runnable tasks. Used for in-kernel * cooperative idle threads. */ -bool -sched_runnable(void) +static bool +sched_ule_runnable(void) { struct tdq *tdq; @@ -2832,8 +2790,8 @@ sched_runnable(void) * Choose the highest priority thread to run. The thread is removed from * the run-queue while running however the load remains. */ -struct thread * -sched_choose(void) +static struct thread * +sched_ule_choose(void) { struct thread *td; struct tdq *tdq; @@ -2909,8 +2867,8 @@ tdq_add(struct tdq *tdq, struct thread *td, int flags) * * Requires the thread lock on entry, drops on exit. */ -void -sched_add(struct thread *td, int flags) +static void +sched_ule_add(struct thread *td, int flags) { struct tdq *tdq; #ifdef SMP @@ -2969,8 +2927,8 @@ sched_add(struct thread *td, int flags) * when we're stealing a thread from a remote queue. Otherwise all threads * exit by calling sched_exit_thread() and sched_throw() themselves. */ -void -sched_rem(struct thread *td) +static void +sched_ule_rem(struct thread *td) { struct tdq *tdq; @@ -2992,8 +2950,8 @@ sched_rem(struct thread *td) /* * Fetch cpu utilization information. Updates on demand. */ -fixpt_t -sched_pctcpu(struct thread *td) +static fixpt_t +sched_ule_pctcpu(struct thread *td) { struct td_sched *ts; u_int len; @@ -3014,8 +2972,8 @@ sched_pctcpu(struct thread *td) * Enforce affinity settings for a thread. Called after adjustments to * cpumask. */ -void -sched_affinity(struct thread *td) +static void +sched_ule_affinity(struct thread *td) { #ifdef SMP struct td_sched *ts; @@ -3045,8 +3003,8 @@ sched_affinity(struct thread *td) /* * Bind a thread to a target cpu. */ -void -sched_bind(struct thread *td, int cpu) +static void +sched_ule_bind(struct thread *td, int cpu) { struct td_sched *ts; @@ -3069,8 +3027,8 @@ sched_bind(struct thread *td, int cpu) /* * Release a bound thread. */ -void -sched_unbind(struct thread *td) +static void +sched_ule_unbind(struct thread *td) { struct td_sched *ts; @@ -3083,8 +3041,8 @@ sched_unbind(struct thread *td) sched_unpin(); } -int -sched_is_bound(struct thread *td) +static int +sched_ule_is_bound(struct thread *td) { THREAD_LOCK_ASSERT(td, MA_OWNED); return (td_get_sched(td)->ts_flags & TSF_BOUND); @@ -3093,8 +3051,8 @@ sched_is_bound(struct thread *td) /* * Basic yield call. */ -void -sched_relinquish(struct thread *td) +static void +sched_ule_relinquish(struct thread *td) { thread_lock(td); mi_switch(SW_VOL | SWT_RELINQUISH); @@ -3103,8 +3061,8 @@ sched_relinquish(struct thread *td) /* * Return the total system load. */ -int -sched_load(void) +static int +sched_ule_load(void) { #ifdef SMP int total; @@ -3119,14 +3077,14 @@ sched_load(void) #endif } -int -sched_sizeof_proc(void) +static int +sched_ule_sizeof_proc(void) { return (sizeof(struct proc)); } -int -sched_sizeof_thread(void) +static int +sched_ule_sizeof_thread(void) { return (sizeof(struct thread) + sizeof(struct td_sched)); } @@ -3141,8 +3099,8 @@ sched_sizeof_thread(void) /* * The actual idle process. */ -void -sched_idletd(void *dummy) +static void +sched_ule_idletd(void *dummy) { struct thread *td; struct tdq *tdq; @@ -3244,8 +3202,8 @@ sched_throw_grab(struct tdq *tdq) /* * A CPU is entering for the first time. */ -void -sched_ap_entry(void) +static void +sched_ule_ap_entry(void) { struct thread *newtd; struct tdq *tdq; @@ -3274,8 +3232,8 @@ sched_ap_entry(void) /* * A thread is exiting. */ -void -sched_throw(struct thread *td) +static void +sched_ule_throw(struct thread *td) { struct thread *newtd; struct tdq *tdq; @@ -3305,8 +3263,8 @@ sched_throw(struct thread *td) * This is called from fork_exit(). Just acquire the correct locks and * let fork do the rest of the work. */ -void -sched_fork_exit(struct thread *td) +static void +sched_ule_fork_exit(struct thread *td) { struct tdq *tdq; int cpuid; @@ -3331,8 +3289,8 @@ sched_fork_exit(struct thread *td) /* * Create on first use to catch odd startup conditions. */ -char * -sched_tdname(struct thread *td) +static char * +sched_ule_tdname(struct thread *td) { #ifdef KTR struct td_sched *ts; @@ -3347,99 +3305,148 @@ sched_tdname(struct thread *td) #endif } -#ifdef KTR -void -sched_clear_tdname(struct thread *td) +static void +sched_ule_clear_tdname(struct thread *td) { +#ifdef KTR struct td_sched *ts; ts = td_get_sched(td); ts->ts_name[0] = '\0'; -} #endif +} -#ifdef SMP +static void +sched_ule_schedcpu(void) +{ +} -/* - * Build the CPU topology dump string. Is recursively called to collect - * the topology tree. - */ +static bool +sched_ule_do_timer_accounting(void) +{ + return (true); +} + +#ifdef SMP static int -sysctl_kern_sched_topology_spec_internal(struct sbuf *sb, struct cpu_group *cg, - int indent) -{ - char cpusetbuf[CPUSETBUFSIZ]; - int i, first; - - sbuf_printf(sb, "%*s<group level=\"%d\" cache-level=\"%d\">\n", indent, - "", 1 + indent / 2, cg->cg_level); - sbuf_printf(sb, "%*s <cpu count=\"%d\" mask=\"%s\">", indent, "", - cg->cg_count, cpusetobj_strprint(cpusetbuf, &cg->cg_mask)); - first = TRUE; - for (i = cg->cg_first; i <= cg->cg_last; i++) { - if (CPU_ISSET(i, &cg->cg_mask)) { - if (!first) - sbuf_cat(sb, ", "); - else - first = FALSE; - sbuf_printf(sb, "%d", i); - } - } - sbuf_cat(sb, "</cpu>\n"); - - if (cg->cg_flags != 0) { - sbuf_printf(sb, "%*s <flags>", indent, ""); - if ((cg->cg_flags & CG_FLAG_HTT) != 0) - sbuf_cat(sb, "<flag name=\"HTT\">HTT group</flag>"); - if ((cg->cg_flags & CG_FLAG_THREAD) != 0) - sbuf_cat(sb, "<flag name=\"THREAD\">THREAD group</flag>"); - if ((cg->cg_flags & CG_FLAG_SMT) != 0) - sbuf_cat(sb, "<flag name=\"SMT\">SMT group</flag>"); - if ((cg->cg_flags & CG_FLAG_NODE) != 0) - sbuf_cat(sb, "<flag name=\"NODE\">NUMA node</flag>"); - sbuf_cat(sb, "</flags>\n"); - } +sched_ule_find_child_with_core(int cpu, struct cpu_group *grp) +{ + int i; - if (cg->cg_children > 0) { - sbuf_printf(sb, "%*s <children>\n", indent, ""); - for (i = 0; i < cg->cg_children; i++) - sysctl_kern_sched_topology_spec_internal(sb, - &cg->cg_child[i], indent+2); - sbuf_printf(sb, "%*s </children>\n", indent, ""); + if (grp->cg_children == 0) + return (-1); + + MPASS(grp->cg_child); + for (i = 0; i < grp->cg_children; i++) { + if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask)) + return (i); } - sbuf_printf(sb, "%*s</group>\n", indent, ""); - return (0); + + return (-1); } -/* - * Sysctl handler for retrieving topology dump. It's a wrapper for - * the recursive sysctl_kern_smp_topology_spec_internal(). - */ static int -sysctl_kern_sched_topology_spec(SYSCTL_HANDLER_ARGS) +sched_ule_find_l2_neighbor(int cpu) { - struct sbuf *topo; - int err; + struct cpu_group *grp; + int i; - KASSERT(cpu_top != NULL, ("cpu_top isn't initialized")); + grp = cpu_top; + if (grp == NULL) + return (-1); - topo = sbuf_new_for_sysctl(NULL, NULL, 512, req); - if (topo == NULL) - return (ENOMEM); + /* + * Find the smallest CPU group that contains the given core. + */ + i = 0; + while ((i = sched_ule_find_child_with_core(cpu, grp)) != -1) { + /* + * If the smallest group containing the given CPU has less + * than two members, we conclude the given CPU has no + * L2 neighbor. + */ + if (grp->cg_child[i].cg_count <= 1) + return (-1); + grp = &grp->cg_child[i]; + } - sbuf_cat(topo, "<groups>\n"); - err = sysctl_kern_sched_topology_spec_internal(topo, cpu_top, 1); - sbuf_cat(topo, "</groups>\n"); + /* Must share L2. */ + if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE) + return (-1); - if (err == 0) { - err = sbuf_finish(topo); + /* + * Select the first member of the set that isn't the reference + * CPU, which at this point is guaranteed to exist. + */ + for (i = 0; i < CPU_SETSIZE; i++) { + if (CPU_ISSET(i, &grp->cg_mask) && i != cpu) + return (i); } - sbuf_delete(topo); - return (err); -} + /* Should never be reached */ + return (-1); +} +#else +static int +sched_ule_find_l2_neighbor(int cpu) +{ + return (-1); +} #endif +struct sched_instance sched_ule_instance = { +#define SLOT(name) .name = sched_ule_##name + SLOT(load), + SLOT(rr_interval), + SLOT(runnable), + SLOT(exit), + SLOT(fork), + SLOT(fork_exit), + SLOT(class), + SLOT(nice), + SLOT(ap_entry), + SLOT(exit_thread), + SLOT(estcpu), + SLOT(fork_thread), + SLOT(ithread_prio), + SLOT(lend_prio), + SLOT(lend_user_prio), + SLOT(lend_user_prio_cond), + SLOT(pctcpu), + SLOT(prio), + SLOT(sleep), + SLOT(sswitch), + SLOT(throw), + SLOT(unlend_prio), + SLOT(user_prio), + SLOT(userret_slowpath), + SLOT(add), + SLOT(choose), + SLOT(clock), + SLOT(idletd), + SLOT(preempt), + SLOT(relinquish), + SLOT(rem), + SLOT(wakeup), + SLOT(bind), + SLOT(unbind), + SLOT(is_bound), + SLOT(affinity), + SLOT(sizeof_proc), + SLOT(sizeof_thread), + SLOT(tdname), + SLOT(clear_tdname), + SLOT(do_timer_accounting), + SLOT(find_l2_neighbor), + SLOT(init), + SLOT(init_ap), + SLOT(setup), + SLOT(initticks), + SLOT(schedcpu), +#undef SLOT +}; +DECLARE_SCHEDULER(ule_sched_selector, "ULE", &sched_ule_instance); + static int sysctl_kern_quantum(SYSCTL_HANDLER_ARGS) { @@ -3459,51 +3466,47 @@ sysctl_kern_quantum(SYSCTL_HANDLER_ARGS) return (0); } -SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, - "Scheduler"); -SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, "ULE", 0, - "Scheduler name"); -SYSCTL_PROC(_kern_sched, OID_AUTO, quantum, +SYSCTL_NODE(_kern_sched, OID_AUTO, ule, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "ULE Scheduler"); + +SYSCTL_PROC(_kern_sched_ule, OID_AUTO, quantum, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, sysctl_kern_quantum, "I", "Quantum for timeshare threads in microseconds"); -SYSCTL_INT(_kern_sched, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0, "Quantum for timeshare threads in stathz ticks"); -SYSCTL_UINT(_kern_sched, OID_AUTO, interact, CTLFLAG_RWTUN, &sched_interact, 0, +SYSCTL_UINT(_kern_sched_ule, OID_AUTO, interact, CTLFLAG_RWTUN, &sched_interact, 0, "Interactivity score threshold"); -SYSCTL_INT(_kern_sched, OID_AUTO, preempt_thresh, CTLFLAG_RWTUN, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, preempt_thresh, CTLFLAG_RWTUN, &preempt_thresh, 0, "Maximal (lowest) priority for preemption"); -SYSCTL_INT(_kern_sched, OID_AUTO, static_boost, CTLFLAG_RWTUN, &static_boost, 0, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, static_boost, CTLFLAG_RWTUN, + &static_boost, 0, "Assign static kernel priorities to sleeping threads"); -SYSCTL_INT(_kern_sched, OID_AUTO, idlespins, CTLFLAG_RWTUN, &sched_idlespins, 0, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, idlespins, CTLFLAG_RWTUN, + &sched_idlespins, 0, "Number of times idle thread will spin waiting for new work"); -SYSCTL_INT(_kern_sched, OID_AUTO, idlespinthresh, CTLFLAG_RW, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, idlespinthresh, CTLFLAG_RW, &sched_idlespinthresh, 0, "Threshold before we will permit idle thread spinning"); #ifdef SMP -SYSCTL_INT(_kern_sched, OID_AUTO, affinity, CTLFLAG_RW, &affinity, 0, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, affinity, CTLFLAG_RW, &affinity, 0, "Number of hz ticks to keep thread affinity for"); -SYSCTL_INT(_kern_sched, OID_AUTO, balance, CTLFLAG_RWTUN, &rebalance, 0, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, balance, CTLFLAG_RWTUN, &rebalance, 0, "Enables the long-term load balancer"); -SYSCTL_INT(_kern_sched, OID_AUTO, balance_interval, CTLFLAG_RW, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, balance_interval, CTLFLAG_RW, &balance_interval, 0, "Average period in stathz ticks to run the long-term balancer"); -SYSCTL_INT(_kern_sched, OID_AUTO, steal_idle, CTLFLAG_RWTUN, &steal_idle, 0, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, steal_idle, CTLFLAG_RWTUN, + &steal_idle, 0, "Attempts to steal work from other cores before idling"); -SYSCTL_INT(_kern_sched, OID_AUTO, steal_thresh, CTLFLAG_RWTUN, &steal_thresh, 0, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, steal_thresh, CTLFLAG_RWTUN, + &steal_thresh, 0, "Minimum load on remote CPU before we'll steal"); -SYSCTL_INT(_kern_sched, OID_AUTO, trysteal_limit, CTLFLAG_RWTUN, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, trysteal_limit, CTLFLAG_RWTUN, &trysteal_limit, 0, "Topological distance limit for stealing threads in sched_switch()"); -SYSCTL_INT(_kern_sched, OID_AUTO, always_steal, CTLFLAG_RWTUN, &always_steal, 0, +SYSCTL_INT(_kern_sched_ule, OID_AUTO, always_steal, CTLFLAG_RWTUN, + &always_steal, 0, "Always run the stealer from the idle thread"); -SYSCTL_PROC(_kern_sched, OID_AUTO, topology_spec, CTLTYPE_STRING | - CTLFLAG_MPSAFE | CTLFLAG_RD, NULL, 0, sysctl_kern_sched_topology_spec, "A", - "XML dump of detected CPU topology"); #endif - -/* ps compat. All cpu percentages from ULE are weighted. */ -static int ccpu = 0; -SYSCTL_INT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, - "Decay factor used for updating %CPU in 4BSD scheduler"); diff --git a/sys/kern/subr_capability.c b/sys/kern/subr_capability.c index 5ad5b0af1681..6e23525186ea 100644 --- a/sys/kern/subr_capability.c +++ b/sys/kern/subr_capability.c @@ -90,6 +90,7 @@ const cap_rights_t cap_mkfifoat_rights = CAP_RIGHTS_INITIALIZER(CAP_MKFIFOAT); const cap_rights_t cap_mknodat_rights = CAP_RIGHTS_INITIALIZER(CAP_MKNODAT); const cap_rights_t cap_pdgetpid_rights = CAP_RIGHTS_INITIALIZER(CAP_PDGETPID); const cap_rights_t cap_pdkill_rights = CAP_RIGHTS_INITIALIZER(CAP_PDKILL); +const cap_rights_t cap_pdwait_rights = CAP_RIGHTS_INITIALIZER(CAP_PDWAIT); const cap_rights_t cap_pread_rights = CAP_RIGHTS_INITIALIZER(CAP_PREAD); const cap_rights_t cap_pwrite_rights = CAP_RIGHTS_INITIALIZER(CAP_PWRITE); const cap_rights_t cap_read_rights = CAP_RIGHTS_INITIALIZER(CAP_READ); diff --git a/sys/kern/subr_devstat.c b/sys/kern/subr_devstat.c index c4d0223d484f..c62df0e210e1 100644 --- a/sys/kern/subr_devstat.c +++ b/sys/kern/subr_devstat.c @@ -43,6 +43,10 @@ #include <vm/vm.h> #include <vm/pmap.h> +#ifdef COMPAT_FREEBSD32 +#include <compat/freebsd32/freebsd32.h> +#endif + #include <machine/atomic.h> SDT_PROVIDER_DEFINE(io); @@ -398,25 +402,63 @@ sysctl_devstat(SYSCTL_HANDLER_ARGS) */ mygen = devstat_generation; - error = SYSCTL_OUT(req, &mygen, sizeof(mygen)); - - if (devstat_num_devs == 0) - return(0); +#ifdef COMPAT_FREEBSD32 + if ((req->flags & SCTL_MASK32) != 0) { + int32_t mygen32 = (int32_t)mygen; + error = SYSCTL_OUT(req, &mygen32, sizeof(mygen32)); + } else +#endif /* COMPAT_FREEBSD32 */ + error = SYSCTL_OUT(req, &mygen, sizeof(mygen)); if (error != 0) return (error); + if (devstat_num_devs == 0) + return(0); + mtx_lock(&devstat_mutex); nds = STAILQ_FIRST(&device_statq); if (mygen != devstat_generation) error = EBUSY; mtx_unlock(&devstat_mutex); - if (error != 0) return (error); while (nds != NULL) { - error = SYSCTL_OUT(req, nds, sizeof(struct devstat)); +#ifdef COMPAT_FREEBSD32 + if ((req->flags & SCTL_MASK32) != 0) { + struct devstat32 ds32; + unsigned int i; + + CP(*nds, ds32, sequence0); + CP(*nds, ds32, allocated); + CP(*nds, ds32, start_count); + CP(*nds, ds32, end_count); + BT_CP(*nds, ds32, busy_from); + PTROUT_CP(*nds, ds32, dev_links.stqe_next); + CP(*nds, ds32, device_number); + strcpy(ds32.device_name, nds->device_name); + CP(*nds, ds32, unit_number); + for (i = 0; i < DEVSTAT_N_TRANS_FLAGS; i++) { + FU64_CP(*nds, ds32, bytes[i]); + FU64_CP(*nds, ds32, operations[i]); + BT_CP(*nds, ds32, duration[i]); + } + BT_CP(*nds, ds32, busy_time); + BT_CP(*nds, ds32, creation_time); + CP(*nds, ds32, block_size); + for (i = 0; i < nitems(ds32.tag_types); i++) { + FU64_CP(*nds, ds32, tag_types[i]); + } + CP(*nds, ds32, flags); + CP(*nds, ds32, device_type); + CP(*nds, ds32, priority); + PTROUT_CP(*nds, ds32, id); + CP(*nds, ds32, sequence1); + error = SYSCTL_OUT(req, &ds32, sizeof(ds32)); + } else +#endif /* COMPAT_FREEBSD32 */ + error = SYSCTL_OUT(req, nds, sizeof(*nds)); if (error != 0) return (error); mtx_lock(&devstat_mutex); @@ -428,7 +470,7 @@ sysctl_devstat(SYSCTL_HANDLER_ARGS) if (error != 0) return (error); } - return(error); + return (error); } /* diff --git a/sys/kern/subr_firmware.c b/sys/kern/subr_firmware.c index d616339f2c07..88b2e71ba8df 100644 --- a/sys/kern/subr_firmware.c +++ b/sys/kern/subr_firmware.c @@ -281,7 +281,7 @@ try_binary_file(const char *imagename, uint32_t flags) int oflags; size_t resid; int error; - bool warn = flags & FIRMWARE_GET_NOWARN; + bool warn = (flags & FIRMWARE_GET_NOWARN) == 0; /* * XXX TODO: Loop over some path instead of a single element path. diff --git a/sys/kern/subr_module.c b/sys/kern/subr_module.c index d07990cebdc3..f8b6770e3189 100644 --- a/sys/kern/subr_module.c +++ b/sys/kern/subr_module.c @@ -433,16 +433,12 @@ preload_modinfo_type(struct sbuf *sbp, int type) sbuf_cat(sbp, "MODINFOMD_VBE_FB"); break; #endif -#ifdef MODINFOMD_FONT case MODINFOMD_FONT: sbuf_cat(sbp, "MODINFOMD_FONT"); break; -#endif -#ifdef MODINFOMD_SPLASH case MODINFOMD_SPLASH: sbuf_cat(sbp, "MODINFOMD_SPLASH"); break; -#endif #ifdef MODINFOMD_BOOT_HARTID case MODINFOMD_BOOT_HARTID: sbuf_cat(sbp, "MODINFOMD_BOOT_HARTID"); @@ -497,12 +493,8 @@ preload_modinfo_value(struct sbuf *sbp, uint32_t *bptr, int type, int len) #ifdef MODINFOMD_VBE_FB case MODINFO_METADATA | MODINFOMD_VBE_FB: #endif -#ifdef MODINFOMD_FONT case MODINFO_METADATA | MODINFOMD_FONT: -#endif -#ifdef MODINFOMD_SPLASH case MODINFO_METADATA | MODINFOMD_SPLASH: -#endif sbuf_print_vmoffset(sbp, *(vm_offset_t *)bptr); break; case MODINFO_METADATA | MODINFOMD_HOWTO: diff --git a/sys/kern/subr_power.c b/sys/kern/subr_power.c index f5a581e42bf3..13448cfa4d1e 100644 --- a/sys/kern/subr_power.c +++ b/sys/kern/subr_power.c @@ -176,25 +176,26 @@ power_pm_get_type(void) } void -power_pm_suspend(int state) +power_pm_suspend(enum power_sstate_transition trans) { enum power_stype stype; if (power_pm_fn == NULL) return; - switch (state) { - case POWER_SLEEP_STATE_STANDBY: + switch (trans) { + case POWER_SSTATE_TRANSITION_STANDBY: stype = power_standby_stype; break; - case POWER_SLEEP_STATE_SUSPEND: + case POWER_SSTATE_TRANSITION_SUSPEND: stype = power_suspend_stype; break; - case POWER_SLEEP_STATE_HIBERNATE: + case POWER_SSTATE_TRANSITION_HIBERNATE: stype = power_hibernate_stype; break; default: - printf("%s: unknown sleep state %d\n", __func__, state); + printf("%s: unknown sleep state transition %d\n", __func__, + trans); return; } diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 353a69435971..2256ba648e4d 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -50,9 +50,43 @@ #include "opt_sched.h" -#ifdef SMP MALLOC_DEFINE(M_TOPO, "toponodes", "SMP topology data"); +struct cpu_group * +smp_topo_alloc(u_int count) +{ + static struct cpu_group *group = NULL; + static u_int index; + u_int curr; + + if (group == NULL) { + group = mallocarray((mp_maxid + 1) * MAX_CACHE_LEVELS + 1, + sizeof(*group), M_DEVBUF, M_WAITOK | M_ZERO); + } + curr = index; + index += count; + return (&group[curr]); +} + +struct cpu_group * +smp_topo_none(void) +{ + struct cpu_group *top; + + top = smp_topo_alloc(1); + top->cg_parent = NULL; + top->cg_child = NULL; + top->cg_mask = all_cpus; + top->cg_count = mp_ncpus; + top->cg_children = 0; + top->cg_level = CG_SHARE_NONE; + top->cg_flags = 0; + + return (top); +} + +#ifdef SMP + volatile cpuset_t stopped_cpus; volatile cpuset_t started_cpus; volatile cpuset_t suspended_cpus; @@ -731,39 +765,6 @@ smp_topo(void) return (top); } -struct cpu_group * -smp_topo_alloc(u_int count) -{ - static struct cpu_group *group = NULL; - static u_int index; - u_int curr; - - if (group == NULL) { - group = mallocarray((mp_maxid + 1) * MAX_CACHE_LEVELS + 1, - sizeof(*group), M_DEVBUF, M_WAITOK | M_ZERO); - } - curr = index; - index += count; - return (&group[curr]); -} - -struct cpu_group * -smp_topo_none(void) -{ - struct cpu_group *top; - - top = smp_topo_alloc(1); - top->cg_parent = NULL; - top->cg_child = NULL; - top->cg_mask = all_cpus; - top->cg_count = mp_ncpus; - top->cg_children = 0; - top->cg_level = CG_SHARE_NONE; - top->cg_flags = 0; - - return (top); -} - static int smp_topo_addleaf(struct cpu_group *parent, struct cpu_group *child, int share, int count, int flags, int start) @@ -901,6 +902,18 @@ smp_rendezvous(void (*setup_func)(void *), arg); } +struct cpu_group * +smp_topo(void) +{ + static struct cpu_group *top = NULL; + + if (top != NULL) + return (top); + + top = smp_topo_none(); + return (top); +} + /* * Provide dummy SMP support for UP kernels. Modules that need to use SMP * APIs will still work using this dummy support. diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index abed76315c34..7437a7e238f0 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -168,7 +168,7 @@ #define WITNESS_RELATED_MASK (WITNESS_ANCESTOR_MASK | WITNESS_DESCENDANT_MASK) #define WITNESS_REVERSAL 0x10 /* A lock order reversal has been observed. */ #define WITNESS_RESERVED1 0x20 /* Unused flag, reserved. */ -#define WITNESS_RESERVED2 0x40 /* Unused flag, reserved. */ +#define WITNESS_ORDER_LISTS 0x40 /* Relationship set in order_lists[]. */ #define WITNESS_LOCK_ORDER_KNOWN 0x80 /* This lock order is known. */ /* Descendant to ancestor flags */ @@ -390,12 +390,16 @@ SYSCTL_INT(_debug_witness, OID_AUTO, kdb, CTLFLAG_RWTUN, &witness_kdb, 0, ""); #if defined(DDB) || defined(KDB) /* - * When DDB or KDB is enabled and witness_trace is 1, it will cause the system - * to print a stack trace: + * When DDB or KDB is enabled and witness_trace is > 0, it will cause the system + * to print a stack trace when: * - a lock hierarchy violation occurs * - locks are held when going to sleep. + * + * Additionally, if witness_trace is 2, it will cause the system to search + * for all locks which established the known lock ordering and print + * stack traces of where the lock ordering was first established. */ -int witness_trace = 1; +int witness_trace = 2; SYSCTL_INT(_debug_witness, OID_AUTO, trace, CTLFLAG_RWTUN, &witness_trace, 0, ""); #endif /* DDB || KDB */ @@ -854,6 +858,10 @@ witness_startup(void *mem) continue; w1->w_file = "order list"; itismychild(w, w1); + w_rmatrix[w->w_index][w1->w_index] |= + WITNESS_ORDER_LISTS; + w_rmatrix[w1->w_index][w->w_index] |= + WITNESS_ORDER_LISTS; w = w1; } } @@ -1075,6 +1083,240 @@ witness_ddb_display(int(*prnt)(const char *fmt, ...)) } #endif /* DDB */ +#define NUM_VERBOSE_STACKS 256 +#define MAX_LOCKCHAIN_RECURSION 32 + +/* + * Struct used by the verbose witness functionality. Only sb, generation, + * pairs, pair_count, check_generation, and alloc_flags communicate data + * between multiple functions. The rest are used to pre-allocate space for + * data which would otherwise end up on the stack. + */ +struct verbose_tracker { + struct witness t_w1, t_w2; + struct stack t_stack; + struct sbuf *sb; + int generation; + int alloc_flags; + int pairs[2 * NUM_VERBOSE_STACKS]; + int pair_count; + int recursion_list[MAX_LOCKCHAIN_RECURSION]; + int found[MAX_LOCKCHAIN_RECURSION + 1]; + int iter[MAX_LOCKCHAIN_RECURSION]; + bool check_generation; +}; + +static void +init_verbose_tracker(struct verbose_tracker *t, struct sbuf *sb, + int alloc_flags, bool check_generation) +{ + + KASSERT(t != NULL, + ("%s: NULL t argument", __func__)); + KASSERT(alloc_flags == M_WAITOK || alloc_flags == M_NOWAIT, + ("%s: Unexpected alloc_flags %d", __func__, alloc_flags)); + t->sb = sb; + t->check_generation = check_generation; + t->alloc_flags = alloc_flags; +} + +static void +reset_verbose_tracker(struct verbose_tracker *t, int generation) +{ + + KASSERT(t != NULL, + ("%s: NULL t argument", __func__)); + t->pair_count = 0; + t->generation = generation; +} + +static bool +has_verbose_lockpair(const struct verbose_tracker *t, int from, int to) +{ + int i; + + /* Look for value. */ + for (i = 0; i < (2 * t->pair_count); i += 2) + if (t->pairs[i] == from && t->pairs[i + 1] == to) + return (true); + return (false); +} + +static void +add_verbose_lockpair(struct verbose_tracker *t, int from, int to) +{ + + /* Check for duplicates. */ + if (has_verbose_lockpair(t, from, to)) + return; + + /* Add a new value. */ + if (t->pair_count < NUM_VERBOSE_STACKS) { + t->pairs[t->pair_count * 2] = from; + t->pairs[(t->pair_count * 2) + 1] = to; + t->pair_count++; + } +} + +static void +sbuf_print_verbose_witness_chains(struct verbose_tracker *t, int from, int to) +{ + struct witness *w1, *w2; + int i, recursion_count; + + recursion_count = 0; + + mtx_lock_spin(&w_mtx); + if (t->check_generation && t->generation != w_generation) { + mtx_unlock_spin(&w_mtx); + + /* + * The graph has changed. Break the recursion loop. + * The calling function should figure out what happened and + * restart. + */ + return; + } + +top: + t->found[recursion_count] = 0; + + /* + * Check for a direct dependence. If so, print that here. + * However, we keep scanning just in case there are other + * locking paths between these two locks. + */ + w1 = &w_data[from]; + w2 = &w_data[to]; + if (isitmychild(w1, w2)) { + t->t_w1 = *w1; + t->t_w2 = *w2; + mtx_unlock_spin(&w_mtx); + + sbuf_printf(t->sb, "\"%s\" -> \"%s\"", + t->t_w1.w_name, t->t_w2.w_name); + + /* Add the lockchain which got us here. */ + KASSERT(recursion_count >= 0 && + recursion_count <= MAX_LOCKCHAIN_RECURSION, + ("Invalid recursion_count: %d", recursion_count)); + for (i = recursion_count - 1; i >= 0; i--) { + mtx_lock_spin(&w_mtx); + if (t->check_generation && + t->generation != w_generation) { + mtx_unlock_spin(&w_mtx); + /* The graph has changed. */ + return; + } + /* + * Make a local copy, drop the lock, and add the lock + * to the sbuf. + */ + t->t_w1 = w_data[t->recursion_list[i]]; + mtx_unlock_spin(&w_mtx); + sbuf_printf(t->sb, " -> \"%s\"", t->t_w1.w_name); + } + + sbuf_putc(t->sb, '\n'); + add_verbose_lockpair(t, from, to); + t->found[recursion_count]++; + + mtx_lock_spin(&w_mtx); + if (t->check_generation && t->generation != w_generation) { + mtx_unlock_spin(&w_mtx); + return; + } + } + + /* + * Ensure we aren't recursing too many times. We do this check + * after looking for direct dependencies so we don't fail to + * catch at least those at the limits of our recursion. + */ + if (recursion_count >= MAX_LOCKCHAIN_RECURSION) + goto end; + + /* + * Record our 'to' lock on the recursion list. We will use this + * to build successful lock chains later. + */ + t->recursion_list[recursion_count] = to; + t->iter[recursion_count] = 1; + +loop: + /* Walk all parents of 'to' to see if any have a path to 'from'. */ + for (; t->iter[recursion_count] < w_max_used_index; + t->iter[recursion_count]++) { + if (t->iter[recursion_count] == to || + t->iter[recursion_count] == from) + continue; + if (isitmychild(&w_data[t->iter[recursion_count]], + &w_data[to])) { + /* Recurse to the parent. */ + to = t->iter[recursion_count]; + recursion_count++; + goto top; + } + } +end: + if (recursion_count != 0) { + recursion_count--; + to = t->recursion_list[recursion_count]; + if (t->found[recursion_count + 1] > 0) { + add_verbose_lockpair(t, t->iter[recursion_count], to); + t->found[recursion_count]++; + } + t->iter[recursion_count]++; + goto loop; + } + mtx_unlock_spin(&w_mtx); +} + +static void +sbuf_print_verbose_witness_stacks(struct verbose_tracker *t) +{ + struct witness_lock_order_data *data; + int i; + bool hardcoded; + + for (i = 0; i < (2 * t->pair_count); i += 2) { + mtx_lock_spin(&w_mtx); + if (t->check_generation && t->generation != w_generation) { + /* + * The graph has changed. Return to the calling + * function so it can restart. + */ + mtx_unlock_spin(&w_mtx); + break; + } + + /* + * Make a local copy of the data we need so we can drop + * the lock. + */ + t->t_w1 = w_data[t->pairs[i]]; + t->t_w2 = w_data[t->pairs[i + 1]]; + data = witness_lock_order_get(&t->t_w1, &t->t_w2); + if (data != NULL) + stack_copy(&data->wlod_stack, &t->t_stack); + hardcoded = (w_rmatrix[t->pairs[i]][t->pairs[i + 1]] & + WITNESS_ORDER_LISTS) == WITNESS_ORDER_LISTS; + mtx_unlock_spin(&w_mtx); + + sbuf_printf(t->sb, + "%slock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n", + hardcoded ? "hardcoded " : "", + t->t_w1.w_name, t->t_w1.w_class->lc_name, + t->t_w2.w_name, t->t_w2.w_class->lc_name); + if (data != NULL) + stack_sbuf_print_flags(t->sb, &t->t_stack, + t->alloc_flags, STACK_SBUF_FMT_LONG); + else + sbuf_printf(t->sb, "(No stack trace)\n"); + sbuf_putc(t->sb, '\n'); + } +} + int witness_defineorder(struct lock_object *lock1, struct lock_object *lock2) { @@ -1117,6 +1359,7 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file, struct witness *w, *w1; struct thread *td; int i, j; + bool print_lock_order; if (witness_cold || witness_watch < 1 || lock->lo_witness == NULL || KERNEL_PANICKED()) @@ -1279,7 +1522,8 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file, for (j = 0, lle = lock_list; lle != NULL; lle = lle->ll_next) { for (i = lle->ll_count - 1; i >= 0; i--, j++) { struct stack pstack; - bool pstackv, trace; + int trace; + bool pstackv; MPASS(j < LOCK_CHILDCOUNT * LOCK_NCHILDREN); lock1 = &lle->ll_children[i]; @@ -1396,6 +1640,7 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file, /* * Ok, yell about it. */ + print_lock_order = false; if ((lock->lo_flags & LO_SLEEPABLE) != 0 && (flags & LOP_NOSLEEP) == 0 && (lock1->li_flags & LI_SLEEPABLE) == 0) @@ -1405,8 +1650,10 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file, && lock == &Giant.lock_object) witness_output( "lock order reversal: (Giant after non-sleepable)\n"); - else + else { witness_output("lock order reversal:\n"); + print_lock_order = true; + } /* * Try to locate an earlier lock with @@ -1455,6 +1702,7 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file, if (trace) { char buf[64]; struct sbuf sb; + struct verbose_tracker *t; sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN); sbuf_set_drain(&sb, witness_output_drain, @@ -1466,6 +1714,37 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file, w->w_name, w1->w_name); stack_sbuf_print_flags(&sb, &pstack, M_NOWAIT, STACK_SBUF_FMT_LONG); + } else if (trace > 1 && print_lock_order && + (t = malloc(sizeof(struct verbose_tracker), + M_TEMP, M_NOWAIT | M_ZERO)) != NULL) { + /* + * We make a purposeful decision to + * ignore generation changes while + * printing. The two locks in + * question are in use, so won't be + * going away. There is a small + * chance that intermediate locks + * in a lock chain get destroyed + * while we are traversing the + * chain or printing them, but even + * then nothing "bad" should happen + * with the current code since the + * WITNESS objects are not actually + * freed and re-used. If that changes, + * we might need to reassess the + * decision to ignore generation. + */ + init_verbose_tracker(t, &sb, M_NOWAIT, + false); + reset_verbose_tracker(t, 0); + sbuf_printf(&sb, + "All lock orders from %s -> %s:\n", + w->w_name, w1->w_name); + sbuf_print_verbose_witness_chains(t, + w->w_index, w1->w_index); + sbuf_putc(&sb, '\n'); + sbuf_print_verbose_witness_stacks(t); + free(t, M_TEMP); } sbuf_printf(&sb, @@ -2645,16 +2924,14 @@ DB_SHOW_COMMAND_FLAGS(witness, db_witness_display, DB_CMD_MEMSAFE) #endif static void -sbuf_print_witness_badstacks(struct sbuf *sb, size_t *oldidx) +sbuf_print_witness_badstacks(struct sbuf *sb, size_t *oldidx, + bool check_generation) { struct witness_lock_order_data *data1, *data2, *tmp_data1, *tmp_data2; struct witness *tmp_w1, *tmp_w2, *w1, *w2; + struct verbose_tracker *t; int generation, i, j; - - tmp_data1 = NULL; - tmp_data2 = NULL; - tmp_w1 = NULL; - tmp_w2 = NULL; + bool w1_is_parent, w2_is_parent; /* Allocate and init temporary storage space. */ tmp_w1 = malloc(sizeof(struct witness), M_TEMP, M_WAITOK | M_ZERO); @@ -2665,16 +2942,19 @@ sbuf_print_witness_badstacks(struct sbuf *sb, size_t *oldidx) M_WAITOK | M_ZERO); stack_zero(&tmp_data1->wlod_stack); stack_zero(&tmp_data2->wlod_stack); + t = malloc(sizeof(struct verbose_tracker), M_TEMP, M_WAITOK | M_ZERO); + init_verbose_tracker(t, sb, M_WAITOK, check_generation); restart: mtx_lock_spin(&w_mtx); generation = w_generation; mtx_unlock_spin(&w_mtx); + reset_verbose_tracker(t, generation); sbuf_printf(sb, "Number of known direct relationships is %d\n", w_lohash.wloh_count); for (i = 1; i < w_max_used_index; i++) { mtx_lock_spin(&w_mtx); - if (generation != w_generation) { + if (check_generation && generation != w_generation) { mtx_unlock_spin(&w_mtx); /* The graph has changed, try again. */ @@ -2700,7 +2980,7 @@ restart: continue; mtx_lock_spin(&w_mtx); - if (generation != w_generation) { + if (check_generation && generation != w_generation) { mtx_unlock_spin(&w_mtx); /* The graph has changed, try again. */ @@ -2729,6 +3009,8 @@ restart: stack_copy(&data2->wlod_stack, &tmp_data2->wlod_stack); } + w1_is_parent = isitmydescendant(w1, w2); + w2_is_parent = isitmydescendant(w2, w1); mtx_unlock_spin(&w_mtx); if (blessed(tmp_w1, tmp_w2)) @@ -2738,26 +3020,49 @@ restart: "\nLock order reversal between \"%s\"(%s) and \"%s\"(%s)!\n", tmp_w1->w_name, tmp_w1->w_class->lc_name, tmp_w2->w_name, tmp_w2->w_class->lc_name); - if (data1) { + if (w1_is_parent || data1 != NULL) { sbuf_printf(sb, - "Lock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n", + "All lock orders from \"%s\"(%s) -> \"%s\"(%s):\n", tmp_w1->w_name, tmp_w1->w_class->lc_name, tmp_w2->w_name, tmp_w2->w_class->lc_name); - stack_sbuf_print(sb, &tmp_data1->wlod_stack); + if (w1_is_parent) + sbuf_print_verbose_witness_chains(t, i, + j); + if (data1 && !has_verbose_lockpair(t, i, j)) { + sbuf_printf(t->sb, + "** \"%s\" -> \"%s\"\n", + tmp_w1->w_name, tmp_w2->w_name); + add_verbose_lockpair(t, i, j); + } + sbuf_putc(sb, '\n'); + sbuf_print_verbose_witness_stacks(t); sbuf_putc(sb, '\n'); + reset_verbose_tracker(t, generation); } - if (data2 && data2 != data1) { + if (w2_is_parent || (data2 != NULL && data2 != data1)) { sbuf_printf(sb, - "Lock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n", + "All lock orders from \"%s\"(%s) -> \"%s\"(%s):\n", tmp_w2->w_name, tmp_w2->w_class->lc_name, tmp_w1->w_name, tmp_w1->w_class->lc_name); - stack_sbuf_print(sb, &tmp_data2->wlod_stack); + if (w2_is_parent) + sbuf_print_verbose_witness_chains(t, j, + i); + if (data2 && data2 != data1 && + !has_verbose_lockpair(t, j, i)) { + sbuf_printf(t->sb, + "** \"%s\" -> \"%s\"\n", + tmp_w2->w_name, tmp_w1->w_name); + add_verbose_lockpair(t, j, i); + } + sbuf_putc(sb, '\n'); + sbuf_print_verbose_witness_stacks(t); sbuf_putc(sb, '\n'); + reset_verbose_tracker(t, generation); } } } mtx_lock_spin(&w_mtx); - if (generation != w_generation) { + if (check_generation && generation != w_generation) { mtx_unlock_spin(&w_mtx); /* @@ -2775,6 +3080,7 @@ restart: free(tmp_data2, M_TEMP); free(tmp_w1, M_TEMP); free(tmp_w2, M_TEMP); + free(t, M_TEMP); } static int @@ -2796,7 +3102,7 @@ sysctl_debug_witness_badstacks(SYSCTL_HANDLER_ARGS) if (sb == NULL) return (ENOMEM); - sbuf_print_witness_badstacks(sb, &req->oldidx); + sbuf_print_witness_badstacks(sb, &req->oldidx, true); sbuf_finish(sb); error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); @@ -2820,7 +3126,7 @@ DB_SHOW_COMMAND_FLAGS(badstacks, db_witness_badstacks, DB_CMD_MEMSAFE) sbuf_new(&sb, buffer, sizeof(buffer), SBUF_FIXEDLEN); sbuf_set_drain(&sb, sbuf_db_printf_drain, NULL); - sbuf_print_witness_badstacks(&sb, &dummy); + sbuf_print_witness_badstacks(&sb, &dummy, false); sbuf_finish(&sb); } #endif diff --git a/sys/kern/sys_eventfd.c b/sys/kern/sys_eventfd.c index 04ed107c933d..47f1fcc316ec 100644 --- a/sys/kern/sys_eventfd.c +++ b/sys/kern/sys_eventfd.c @@ -40,6 +40,7 @@ #include <sys/mutex.h> #include <sys/poll.h> #include <sys/proc.h> +#include <sys/refcount.h> #include <sys/selinfo.h> #include <sys/stat.h> #include <sys/uio.h> @@ -102,6 +103,7 @@ struct eventfd { uint32_t efd_flags; struct selinfo efd_sel; struct mtx efd_lock; + unsigned int efd_refcount; }; int @@ -119,6 +121,7 @@ eventfd_create_file(struct thread *td, struct file *fp, uint32_t initval, efd->efd_count = initval; mtx_init(&efd->efd_lock, "eventfd", NULL, MTX_DEF); knlist_init_mtx(&efd->efd_sel.si_note, &efd->efd_lock); + refcount_init(&efd->efd_refcount, 1); fflags = FREAD | FWRITE; if ((flags & EFD_NONBLOCK) != 0) @@ -128,16 +131,60 @@ eventfd_create_file(struct thread *td, struct file *fp, uint32_t initval, return (0); } -static int -eventfd_close(struct file *fp, struct thread *td) +struct eventfd * +eventfd_get(struct file *fp) { struct eventfd *efd; + if (fp->f_data == NULL || fp->f_ops != &eventfdops) + return (NULL); + efd = fp->f_data; + refcount_acquire(&efd->efd_refcount); + + return (efd); +} + +void +eventfd_put(struct eventfd *efd) +{ + if (!refcount_release(&efd->efd_refcount)) + return; + seldrain(&efd->efd_sel); knlist_destroy(&efd->efd_sel.si_note); mtx_destroy(&efd->efd_lock); free(efd, M_EVENTFD); +} + +static void +eventfd_wakeup(struct eventfd *efd) +{ + KNOTE_LOCKED(&efd->efd_sel.si_note, 0); + selwakeup(&efd->efd_sel); + wakeup(&efd->efd_count); +} + +void +eventfd_signal(struct eventfd *efd) +{ + mtx_lock(&efd->efd_lock); + + if (efd->efd_count < UINT64_MAX) + efd->efd_count++; + + eventfd_wakeup(efd); + + mtx_unlock(&efd->efd_lock); +} + +static int +eventfd_close(struct file *fp, struct thread *td) +{ + struct eventfd *efd; + + efd = fp->f_data; + eventfd_put(efd); return (0); } @@ -218,9 +265,7 @@ retry: if (error == 0) { MPASS(UINT64_MAX - efd->efd_count > count); efd->efd_count += count; - KNOTE_LOCKED(&efd->efd_sel.si_note, 0); - selwakeup(&efd->efd_sel); - wakeup(&efd->efd_count); + eventfd_wakeup(efd); } mtx_unlock(&efd->efd_lock); diff --git a/sys/kern/sys_procdesc.c b/sys/kern/sys_procdesc.c index c5db21544b0f..ec3b37f96148 100644 --- a/sys/kern/sys_procdesc.c +++ b/sys/kern/sys_procdesc.c @@ -75,6 +75,7 @@ #include <sys/procdesc.h> #include <sys/resourcevar.h> #include <sys/stat.h> +#include <sys/syscallsubr.h> #include <sys/sysproto.h> #include <sys/sysctl.h> #include <sys/systm.h> @@ -270,6 +271,9 @@ procdesc_free(struct procdesc *pd) KASSERT((pd->pd_flags & PDF_CLOSED), ("procdesc_free: !PDF_CLOSED")); + if (pd->pd_pid != -1) + proc_id_clear(PROC_ID_PID, pd->pd_pid); + knlist_destroy(&pd->pd_selinfo.si_note); PROCDESC_LOCK_DESTROY(pd); free(pd, M_PROCDESC); @@ -318,6 +322,9 @@ procdesc_exit(struct proc *p) } KNOTE_LOCKED(&pd->pd_selinfo.si_note, NOTE_EXIT); PROCDESC_UNLOCK(pd); + + /* Wakeup all waiters for this procdesc' process exit. */ + wakeup(&p->p_procdesc); return (0); } @@ -389,6 +396,7 @@ procdesc_close(struct file *fp, struct thread *td) */ pd->pd_proc = NULL; p->p_procdesc = NULL; + pd->pd_pid = -1; procdesc_free(pd); /* diff --git a/sys/kern/syscalls.c b/sys/kern/syscalls.c index 06a4adc3d8cb..e2467c39fe6d 100644 --- a/sys/kern/syscalls.c +++ b/sys/kern/syscalls.c @@ -605,4 +605,6 @@ const char *syscallnames[] = { "jail_attach_jd", /* 597 = jail_attach_jd */ "jail_remove_jd", /* 598 = jail_remove_jd */ "kexec_load", /* 599 = kexec_load */ + "pdrfork", /* 600 = pdrfork */ + "pdwait", /* 601 = pdwait */ }; diff --git a/sys/kern/syscalls.master b/sys/kern/syscalls.master index ea6d2b5aa1ef..8a30e5931a0e 100644 --- a/sys/kern/syscalls.master +++ b/sys/kern/syscalls.master @@ -3402,4 +3402,23 @@ u_long flags ); } + +600 AUE_PDRFORK STD|CAPENABLED { + int pdrfork( + _Out_ int *fdp, + int pdflags, + int rfflags + ); + } + +601 AUE_PDWAIT STD|CAPENABLED { + int pdwait( + int fd, + _Out_opt_ int *status, + int options, + _Out_opt_ _Contains_long_ struct __wrusage *wrusage, + _Out_opt_ _Contains_long_ptr_ struct __siginfo *info + ); + } + ; vim: syntax=off diff --git a/sys/kern/systrace_args.c b/sys/kern/systrace_args.c index 5951cebbe74a..8f5a5cd5153d 100644 --- a/sys/kern/systrace_args.c +++ b/sys/kern/systrace_args.c @@ -3524,6 +3524,26 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) *n_args = 4; break; } + /* pdrfork */ + case 600: { + struct pdrfork_args *p = params; + uarg[a++] = (intptr_t)p->fdp; /* int * */ + iarg[a++] = p->pdflags; /* int */ + iarg[a++] = p->rfflags; /* int */ + *n_args = 3; + break; + } + /* pdwait */ + case 601: { + struct pdwait_args *p = params; + iarg[a++] = p->fd; /* int */ + uarg[a++] = (intptr_t)p->status; /* int * */ + iarg[a++] = p->options; /* int */ + uarg[a++] = (intptr_t)p->wrusage; /* struct __wrusage * */ + uarg[a++] = (intptr_t)p->info; /* struct __siginfo * */ + *n_args = 5; + break; + } default: *n_args = 0; break; @@ -9430,6 +9450,44 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) break; }; break; + /* pdrfork */ + case 600: + switch (ndx) { + case 0: + p = "userland int *"; + break; + case 1: + p = "int"; + break; + case 2: + p = "int"; + break; + default: + break; + }; + break; + /* pdwait */ + case 601: + switch (ndx) { + case 0: + p = "int"; + break; + case 1: + p = "userland int *"; + break; + case 2: + p = "int"; + break; + case 3: + p = "userland struct __wrusage *"; + break; + case 4: + p = "userland struct __siginfo *"; + break; + default: + break; + }; + break; default: break; }; @@ -11443,6 +11501,16 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) if (ndx == 0 || ndx == 1) p = "int"; break; + /* pdrfork */ + case 600: + if (ndx == 0 || ndx == 1) + p = "int"; + break; + /* pdwait */ + case 601: + if (ndx == 0 || ndx == 1) + p = "int"; + break; default: break; }; diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index efc8ebfcece2..73ac2c6efc4e 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -818,8 +818,6 @@ soalloc(struct vnet *vnet) * a feature to change class of an existing lock, so we use DUPOK. */ mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); - mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF); - mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF); so->so_rcv.sb_sel = &so->so_rdsel; so->so_snd.sb_sel = &so->so_wrsel; sx_init(&so->so_snd_sx, "so_snd_sx"); @@ -893,8 +891,6 @@ sodealloc(struct socket *so) &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); sx_destroy(&so->so_snd_sx); sx_destroy(&so->so_rcv_sx); - mtx_destroy(&so->so_snd_mtx); - mtx_destroy(&so->so_rcv_mtx); } crfree(so->so_cred); mtx_destroy(&so->so_lock); @@ -902,6 +898,41 @@ sodealloc(struct socket *so) } /* + * Shim to accomodate protocols that already do their own socket buffers + * management (marked with PR_SOCKBUF) with protocols that yet do not. + * + * Attach via socket(2) is different from attach via accept(2). In case of + * normal socket(2) syscall it is the pr_attach that calls soreserve(), even + * for protocols that don't yet do PR_SOCKBUF. In case of accepted connection + * it is our shim that calls soreserve() and the hiwat values are taken from + * the parent socket. + */ +static int +soattach(struct socket *so, int proto, struct thread *td, struct socket *head) +{ + int error; + + VNET_ASSERT(curvnet == so->so_vnet, + ("%s: %p != %p", __func__, curvnet, so->so_vnet)); + + if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) { + mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF); + mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF); + so->so_snd.sb_mtx = &so->so_snd_mtx; + so->so_rcv.sb_mtx = &so->so_rcv_mtx; + } + if (head == NULL || (error = soreserve(so, head->sol_sbsnd_hiwat, + head->sol_sbrcv_hiwat)) == 0) + error = so->so_proto->pr_attach(so, proto, td); + if (error != 0 && (so->so_proto->pr_flags & PR_SOCKBUF) == 0) { + mtx_destroy(&so->so_snd_mtx); + mtx_destroy(&so->so_rcv_mtx); + } + + return (error); +} + +/* * socreate returns a socket with a ref count of 1 and a file descriptor * reference. The socket should be closed with soclose(). */ @@ -956,16 +987,8 @@ socreate(int dom, struct socket **aso, int type, int proto, so_rdknl_assert_lock); knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, so_wrknl_assert_lock); - if ((prp->pr_flags & PR_SOCKBUF) == 0) { - so->so_snd.sb_mtx = &so->so_snd_mtx; - so->so_rcv.sb_mtx = &so->so_rcv_mtx; - } - /* - * Auto-sizing of socket buffers is managed by the protocols and - * the appropriate flags must be set in the pr_attach() method. - */ CURVNET_SET(so->so_vnet); - error = prp->pr_attach(so, proto, td); + error = soattach(so, proto, td, NULL); CURVNET_RESTORE(); if (error) { sodealloc(so); @@ -1192,13 +1215,6 @@ solisten_clone(struct socket *head) so_rdknl_assert_lock); knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, so_wrknl_assert_lock); - VNET_SO_ASSERT(head); - if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) { - sodealloc(so); - log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", - __func__, head->so_pcb); - return (NULL); - } so->so_rcv.sb_lowat = head->sol_sbrcv_lowat; so->so_snd.sb_lowat = head->sol_sbsnd_lowat; so->so_rcv.sb_timeo = head->sol_sbrcv_timeo; @@ -1206,10 +1222,6 @@ solisten_clone(struct socket *head) so->so_rcv.sb_flags = head->sol_sbrcv_flags & SB_AUTOSIZE; so->so_snd.sb_flags = head->sol_sbsnd_flags & (SB_AUTOSIZE | SB_AUTOLOWAT); - if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) { - so->so_snd.sb_mtx = &so->so_snd_mtx; - so->so_rcv.sb_mtx = &so->so_rcv_mtx; - } return (so); } @@ -1223,7 +1235,7 @@ sonewconn(struct socket *head, int connstatus) if ((so = solisten_clone(head)) == NULL) return (NULL); - if (so->so_proto->pr_attach(so, 0, NULL) != 0) { + if (soattach(so, 0, NULL, head) != 0) { sodealloc(so); log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n", __func__, head->so_pcb); @@ -1292,7 +1304,7 @@ solisten_enqueue(struct socket *so, int connstatus) #if defined(SCTP) || defined(SCTP_SUPPORT) /* - * Socket part of sctp_peeloff(). Detach a new socket from an + * Socket part of sctp_peeloff(). Create a new socket for an * association. The new socket is returned with a reference. * * XXXGL: reduce copy-paste with solisten_clone(). @@ -1304,6 +1316,8 @@ sopeeloff(struct socket *head) VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p", __func__, __LINE__, head)); + KASSERT(head->so_type == SOCK_SEQPACKET, + ("%s: unexpecte so_type: %d", __func__, head->so_type)); so = soalloc(head->so_vnet); if (so == NULL) { log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " @@ -1311,7 +1325,7 @@ sopeeloff(struct socket *head) __func__, head->so_pcb); return (NULL); } - so->so_type = head->so_type; + so->so_type = SOCK_STREAM; so->so_options = head->so_options; so->so_linger = head->so_linger; so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED; @@ -1325,14 +1339,7 @@ sopeeloff(struct socket *head) so_rdknl_assert_lock); knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, so_wrknl_assert_lock); - VNET_SO_ASSERT(head); - if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) { - sodealloc(so); - log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", - __func__, head->so_pcb); - return (NULL); - } - if (so->so_proto->pr_attach(so, 0, NULL)) { + if (soattach(so, 0, NULL, head)) { sodealloc(so); log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n", __func__, head->so_pcb); @@ -1344,10 +1351,6 @@ sopeeloff(struct socket *head) so->so_snd.sb_timeo = head->so_snd.sb_timeo; so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; - if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) { - so->so_snd.sb_mtx = &so->so_snd_mtx; - so->so_rcv.sb_mtx = &so->so_rcv_mtx; - } soref(so); @@ -1904,6 +1907,8 @@ sofree(struct socket *so) SOCK_SENDBUF_UNLOCK(so); SOCK_RECVBUF_UNLOCK(so); #endif + mtx_destroy(&so->so_snd_mtx); + mtx_destroy(&so->so_rcv_mtx); } seldrain(&so->so_rdsel); seldrain(&so->so_wrsel); diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c index 60736af5adf6..d56aac883d9c 100644 --- a/sys/kern/uipc_usrreq.c +++ b/sys/kern/uipc_usrreq.c @@ -479,7 +479,7 @@ uipc_attach(struct socket *so, int proto, struct thread *td) { u_long sendspace, recvspace; struct unpcb *unp; - int error; + int error, rcvmtxopts; bool locked; KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL")); @@ -494,6 +494,7 @@ uipc_attach(struct socket *so, int proto, struct thread *td) * limits to unpdg_recvspace. */ sendspace = recvspace = unpdg_recvspace; + rcvmtxopts = 0; break; case SOCK_STREAM: @@ -505,14 +506,7 @@ uipc_attach(struct socket *so, int proto, struct thread *td) sendspace = unpsp_sendspace; recvspace = unpsp_recvspace; common: - /* - * XXXGL: we need to initialize the mutex with MTX_DUPOK. - * Ideally, protocols that have PR_SOCKBUF should be - * responsible for mutex initialization officially, and then - * this uglyness with mtx_destroy(); mtx_init(); would go away. - */ - mtx_destroy(&so->so_rcv_mtx); - mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF | MTX_DUPOK); + rcvmtxopts = MTX_DUPOK; knlist_init(&so->so_wrsel.si_note, so, uipc_wrknl_lock, uipc_wrknl_unlock, uipc_wrknl_assert_lock); STAILQ_INIT(&so->so_rcv.uxst_mbq); @@ -520,6 +514,8 @@ common: default: panic("uipc_attach"); } + mtx_init(&so->so_rcv_mtx, "unix so_rcv", NULL, MTX_DEF | rcvmtxopts); + mtx_init(&so->so_snd_mtx, "unix so_snd", NULL, MTX_DEF); error = soreserve(so, sendspace, recvspace); if (error) return (error); @@ -888,6 +884,9 @@ uipc_detach(struct socket *so) MPASS(TAILQ_EMPTY(&so->so_rcv.uxdg_conns)); MPASS(STAILQ_EMPTY(&so->so_snd.uxdg_mb)); } + + mtx_destroy(&so->so_snd_mtx); + mtx_destroy(&so->so_rcv_mtx); } static int diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c index 13403acacc08..240bf5451a84 100644 --- a/sys/kern/vfs_mount.c +++ b/sys/kern/vfs_mount.c @@ -1733,6 +1733,8 @@ kern_unmount(struct thread *td, const char *path, int flags) int error; AUDIT_ARG_VALUE(flags); + if ((flags & (MNT_DEFERRED | MNT_RECURSE)) != 0) + return (EINVAL); if (jailed(td->td_ucred) || usermount == 0) { error = priv_check(td, PRIV_VFS_UNMOUNT); if (error) diff --git a/sys/libkern/ffs.c b/sys/libkern/ffs.c new file mode 100644 index 000000000000..e0308ef76bae --- /dev/null +++ b/sys/libkern/ffs.c @@ -0,0 +1,47 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +int ffs(int mask); + +/* + * Find First Set bit + */ +int +ffs(int mask) +{ + int bit; + + if (mask == 0) + return (0); + for (bit = 1; !(mask & 1); bit++) + mask = (unsigned int)mask >> 1; + return (bit); +} diff --git a/sys/libkern/ffsdi2.c b/sys/libkern/ffsdi2.c new file mode 100644 index 000000000000..33e255a26dc6 --- /dev/null +++ b/sys/libkern/ffsdi2.c @@ -0,0 +1,47 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <libkern/quad.h> + +/* + * Find First Set bit + */ +int +__ffsdi2(quad_t mask) +{ + int bit; + + if (mask == 0) + return (0); + for (bit = 1; !(mask & 1); bit++) + mask = (quad_t)mask >> 1; + return (bit); +} diff --git a/sys/libkern/quad.h b/sys/libkern/quad.h index 59b3da44dbaa..cbb94ab98828 100644 --- a/sys/libkern/quad.h +++ b/sys/libkern/quad.h @@ -102,6 +102,7 @@ quad_t __ashrdi3(quad_t, qshift_t); int __cmpdi2(quad_t a, quad_t b); quad_t __divdi3(quad_t a, quad_t b); quad_t __divmoddi4(quad_t a, quad_t b, quad_t *rem); +int __ffsdi2(quad_t a); quad_t __lshrdi3(quad_t, qshift_t); quad_t __moddi3(quad_t a, quad_t b); u_quad_t __qdivrem(u_quad_t u, u_quad_t v, u_quad_t *rem); diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 538cd7a1f37d..caf703bb318d 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -967,7 +967,7 @@ SUBDIR:= ${SUBDIR:N${reject}} afterinstall: .PHONY ${KLDXREF_CMD} ${DESTDIR}${KMODDIR} .if defined(NO_ROOT) && defined(METALOG) - echo ".${DISTBASE}${KMODDIR}/linker.hints type=file mode=0644 uname=root gname=wheel" | \ + echo ".${DISTBASE}${KMODDIR}/linker.hints type=file uname=root gname=wheel mode=0644" | \ cat -l >> ${METALOG} .endif .endif diff --git a/sys/modules/ath10k/Makefile b/sys/modules/ath10k/Makefile index 98df270b6791..d8196854b681 100644 --- a/sys/modules/ath10k/Makefile +++ b/sys/modules/ath10k/Makefile @@ -29,6 +29,7 @@ SRCS+= leds.c .endif CFLAGS+= -DKBUILD_MODNAME='"ath10k"' +CFLAGS+= -DLINUXKPI_VERSION=61900 CFLAGS+= -I${DEVATH10KDIR} CFLAGS+= -I${DEVATH10KDIR}/.. diff --git a/sys/modules/dtrace/Makefile b/sys/modules/dtrace/Makefile index 3a3769b8b4b5..019b5e951dcf 100644 --- a/sys/modules/dtrace/Makefile +++ b/sys/modules/dtrace/Makefile @@ -9,6 +9,7 @@ SUBDIR= dtaudit \ fbt \ profile \ prototype \ + sdt \ systrace .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" @@ -31,8 +32,5 @@ SUBDIR+= fasttrap SUBDIR+= systrace_freebsd32 .endif .endif -.if ${MACHINE_CPUARCH} != "powerpc" || ${MACHINE_ARCH} == "powerpc64" -SUBDIR+= sdt -.endif .include <bsd.subdir.mk> diff --git a/sys/modules/hyperv/pcib/Makefile b/sys/modules/hyperv/pcib/Makefile index be0a905aab80..aa2566461d03 100644 --- a/sys/modules/hyperv/pcib/Makefile +++ b/sys/modules/hyperv/pcib/Makefile @@ -4,6 +4,7 @@ KMOD= vmbus_pcib SRCS= vmbus_pcib.c SRCS+= bus_if.h device_if.h pci_if.h pcib_if.h vmbus_if.h +SRCS+= opt_acpi.h CFLAGS+= -I${SRCTOP}/sys/dev/hyperv/pcib diff --git a/sys/modules/linuxkpi/Makefile b/sys/modules/linuxkpi/Makefile index c465c76a7626..2090fe76d857 100644 --- a/sys/modules/linuxkpi/Makefile +++ b/sys/modules/linuxkpi/Makefile @@ -6,6 +6,7 @@ SRCS= linux_compat.c \ linux_devres.c \ linux_dmi.c \ linux_domain.c \ + linux_eventfd.c \ linux_firmware.c \ linux_folio.c \ linux_fpu.c \ @@ -24,6 +25,7 @@ SRCS= linux_compat.c \ linux_radix.c \ linux_rcu.c \ linux_schedule.c \ + linux_seq_buf.c \ linux_seq_file.c \ linux_shmemfs.c \ linux_shrinker.c \ diff --git a/sys/modules/mt76/Makefile.inc b/sys/modules/mt76/Makefile.inc index 44aa94c954a8..e4369564237e 100644 --- a/sys/modules/mt76/Makefile.inc +++ b/sys/modules/mt76/Makefile.inc @@ -32,7 +32,6 @@ CFLAGS+= CONFIG_NET_MEDIATEK_SOC_WED CFLAGS+= -I${COMMONDIR} CFLAGS+= ${LINUXKPI_INCLUDES} -CFLAGS+= -DLINUXKPI_VERSION=61700 - +CFLAGS+= -DLINUXKPI_VERSION=61900 # end diff --git a/sys/modules/mt76/mt7925/Makefile b/sys/modules/mt76/mt7925/Makefile index 58e23d06a9ad..dc6de5085d77 100644 --- a/sys/modules/mt76/mt7925/Makefile +++ b/sys/modules/mt76/mt7925/Makefile @@ -5,7 +5,7 @@ DEVDIR= ${SRCTOP}/sys/contrib/dev/mediatek/mt76/mt7925 KMOD= if_mt7925 # Common stuff. -SRCS= init.c main.c mac.c mcu.c +SRCS= init.c main.c mac.c mcu.c regd.c # PCI stuff. SRCS+= pci.c pci_mac.c pci_mcu.c diff --git a/sys/modules/rtw88/Makefile b/sys/modules/rtw88/Makefile index ee47df54bcf9..0ce6ad3f99bb 100644 --- a/sys/modules/rtw88/Makefile +++ b/sys/modules/rtw88/Makefile @@ -46,7 +46,7 @@ SRCS+= ${LINUXKPI_GENSRCS} SRCS+= opt_wlan.h opt_inet6.h opt_inet.h CFLAGS+= -DKBUILD_MODNAME='"rtw88"' -CFLAGS+= -DLINUXKPI_VERSION=61700 +CFLAGS+= -DLINUXKPI_VERSION=61900 CFLAGS+= -I${DEVRTW88DIR} CFLAGS+= ${LINUXKPI_INCLUDES} diff --git a/sys/net/bpf.c b/sys/net/bpf.c index 9f0b57728e88..228ac9867bd7 100644 --- a/sys/net/bpf.c +++ b/sys/net/bpf.c @@ -678,8 +678,8 @@ bpf_detachd(struct bpf_d *d, bool detached_ifp) BPFD_LOCK(d); CK_LIST_REMOVE(d, bd_next); writer = (d->bd_writer > 0); - d->bd_bif = NULL; if (detached_ifp) { + d->bd_bif = NULL; /* * Notify descriptor as it's detached, so that any * sleepers wake up and get ENXIO. diff --git a/sys/net/if_epair.c b/sys/net/if_epair.c index fbffa8f359a0..60849ca1d4f2 100644 --- a/sys/net/if_epair.c +++ b/sys/net/if_epair.c @@ -150,10 +150,6 @@ epair_clear_mbuf(struct mbuf *m) m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; } - /* Clear vlan information. */ - m->m_flags &= ~M_VLANTAG; - m->m_pkthdr.ether_vtag = 0; - m_tag_delete_nonpersistent(m); } @@ -493,6 +489,7 @@ epair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) ifp->if_capenable = ifr->ifr_reqcap | IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6; epair_caps_changed(ifp); + VLAN_CAPABILITIES(ifp); /* * If IFCAP_TXCSUM(_IPV6) has been changed, change it on the * other epair interface as well. @@ -501,17 +498,23 @@ epair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) * In that case this capability needs to be disabled on the * other epair interface to avoid sending packets in the bridge * that rely on this capability. + * Do the same for IFCAP_VLAN_HWTAGGING. If the sending epair + * end has this capability enabled, the other end has to have + * it enabled too. Otherwise, epair would have to add the VLAN + * tag in the Ethernet header. */ sc = ifp->if_softc; if ((ifp->if_capenable ^ sc->oifp->if_capenable) & - (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) { + (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | IFCAP_VLAN_HWTAGGING)) { sc->oifp->if_capenable &= - ~(IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); + ~(IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | + IFCAP_VLAN_HWTAGGING); sc->oifp->if_capenable |= ifp->if_capenable & - (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); + (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | + IFCAP_VLAN_HWTAGGING); epair_caps_changed(sc->oifp); + VLAN_CAPABILITIES(sc->oifp); } - VLAN_CAPABILITIES(ifp); error = 0; break; @@ -626,10 +629,11 @@ epair_setup_ifp(struct epair_softc *sc, char *name, int unit) ifp->if_dname = epairname; ifp->if_dunit = unit; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_TXCSUM | - IFCAP_TXCSUM_IPV6 | IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6; - ifp->if_capenable = IFCAP_VLAN_MTU | IFCAP_TXCSUM | - IFCAP_TXCSUM_IPV6 | IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6; + ifp->if_capabilities = + IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | + IFCAP_TXCSUM | IFCAP_RXCSUM | + IFCAP_TXCSUM_IPV6 | IFCAP_RXCSUM_IPV6; + ifp->if_capenable = ifp->if_capabilities; epair_caps_changed(ifp); ifp->if_transmit = epair_transmit; ifp->if_qflush = epair_qflush; diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c index da9264aa4a23..812a31595df9 100644 --- a/sys/net/if_ethersubr.c +++ b/sys/net/if_ethersubr.c @@ -479,7 +479,7 @@ ether_output_frame(struct ifnet *ifp, struct mbuf *m) #if defined(INET6) && defined(INET) /* draft-ietf-6man-ipv6only-flag */ /* Catch ETHERTYPE_IP, and ETHERTYPE_[REV]ARP if we are v6-only. */ - if ((ND_IFINFO(ifp)->flags & ND6_IFF_IPV6_ONLY_MASK) != 0) { + if ((ifp->if_inet6->nd_flags & ND6_IFF_IPV6_ONLY_MASK) != 0) { struct ether_header *eh; eh = mtod(m, struct ether_header *); @@ -545,7 +545,7 @@ ether_input_internal(struct ifnet *ifp, struct mbuf *m) #if defined(INET6) && defined(INET) /* draft-ietf-6man-ipv6only-flag */ /* Catch ETHERTYPE_IP, and ETHERTYPE_[REV]ARP if we are v6-only. */ - if ((ND_IFINFO(ifp)->flags & ND6_IFF_IPV6_ONLY_MASK) != 0) { + if ((ifp->if_inet6->nd_flags & ND6_IFF_IPV6_ONLY_MASK) != 0) { switch (etype) { case ETHERTYPE_IP: case ETHERTYPE_ARP: diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c index 5b52bfa80e3b..1e4d3d8d85ac 100644 --- a/sys/net/if_lagg.c +++ b/sys/net/if_lagg.c @@ -169,6 +169,11 @@ static void lagg_media_status(struct ifnet *, struct ifmediareq *); static struct lagg_port *lagg_link_active(struct lagg_softc *, struct lagg_port *); +/* No proto */ +static int lagg_none_start(struct lagg_softc *, struct mbuf *); +static struct mbuf *lagg_none_input(struct lagg_softc *, struct lagg_port *, + struct mbuf *); + /* Simple round robin */ static void lagg_rr_attach(struct lagg_softc *); static int lagg_rr_start(struct lagg_softc *, struct mbuf *); @@ -203,7 +208,6 @@ static struct mbuf *lagg_default_input(struct lagg_softc *, struct lagg_port *, /* lagg protocol table */ static const struct lagg_proto { - lagg_proto pr_num; void (*pr_attach)(struct lagg_softc *); void (*pr_detach)(struct lagg_softc *); int (*pr_start)(struct lagg_softc *, struct mbuf *); @@ -218,22 +222,20 @@ static const struct lagg_proto { void (*pr_request)(struct lagg_softc *, void *); void (*pr_portreq)(struct lagg_port *, void *); } lagg_protos[] = { - { - .pr_num = LAGG_PROTO_NONE + [LAGG_PROTO_NONE] = { + .pr_start = lagg_none_start, + .pr_input = lagg_none_input, }, - { - .pr_num = LAGG_PROTO_ROUNDROBIN, + [LAGG_PROTO_ROUNDROBIN] = { .pr_attach = lagg_rr_attach, .pr_start = lagg_rr_start, .pr_input = lagg_default_input, }, - { - .pr_num = LAGG_PROTO_FAILOVER, + [LAGG_PROTO_FAILOVER] = { .pr_start = lagg_fail_start, .pr_input = lagg_fail_input, }, - { - .pr_num = LAGG_PROTO_LOADBALANCE, + [LAGG_PROTO_LOADBALANCE] = { .pr_attach = lagg_lb_attach, .pr_detach = lagg_lb_detach, .pr_start = lagg_lb_start, @@ -241,8 +243,7 @@ static const struct lagg_proto { .pr_addport = lagg_lb_port_create, .pr_delport = lagg_lb_port_destroy, }, - { - .pr_num = LAGG_PROTO_LACP, + [LAGG_PROTO_LACP] = { .pr_attach = lagg_lacp_attach, .pr_detach = lagg_lacp_detach, .pr_start = lagg_lacp_start, @@ -256,8 +257,7 @@ static const struct lagg_proto { .pr_request = lacp_req, .pr_portreq = lacp_portreq, }, - { - .pr_num = LAGG_PROTO_BROADCAST, + [LAGG_PROTO_BROADCAST] = { .pr_start = lagg_bcast_start, .pr_input = lagg_default_input, }, @@ -2129,8 +2129,8 @@ lagg_transmit_ethernet(struct ifnet *ifp, struct mbuf *m) if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) MPASS(m->m_pkthdr.snd_tag->ifp == ifp); #endif - /* We need a Tx algorithm and at least one port */ - if (sc->sc_proto == LAGG_PROTO_NONE || sc->sc_count == 0) { + /* We need at least one port */ + if (sc->sc_count == 0) { m_freem(m); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); return (ENXIO); @@ -2151,8 +2151,8 @@ lagg_transmit_infiniband(struct ifnet *ifp, struct mbuf *m) if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) MPASS(m->m_pkthdr.snd_tag->ifp == ifp); #endif - /* We need a Tx algorithm and at least one port */ - if (sc->sc_proto == LAGG_PROTO_NONE || sc->sc_count == 0) { + /* We need at least one port */ + if (sc->sc_count == 0) { m_freem(m); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); return (ENXIO); @@ -2180,8 +2180,7 @@ lagg_input_ethernet(struct ifnet *ifp, struct mbuf *m) NET_EPOCH_ASSERT(); if ((scifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || - lp->lp_detaching != 0 || - sc->sc_proto == LAGG_PROTO_NONE) { + lp->lp_detaching != 0) { m_freem(m); return (NULL); } @@ -2215,8 +2214,7 @@ lagg_input_infiniband(struct ifnet *ifp, struct mbuf *m) NET_EPOCH_ASSERT(); if ((scifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || - lp->lp_detaching != 0 || - sc->sc_proto == LAGG_PROTO_NONE) { + lp->lp_detaching != 0) { m_freem(m); return (NULL); } @@ -2391,6 +2389,25 @@ lagg_enqueue(struct ifnet *ifp, struct mbuf *m) } /* + * No proto + */ +static int +lagg_none_start(struct lagg_softc *sc, struct mbuf *m) +{ + m_freem(m); + if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1); + /* No active ports available */ + return (ENETDOWN); +} + +static struct mbuf * +lagg_none_input(struct lagg_softc *sc, struct lagg_port *lp, struct mbuf *m) +{ + m_freem(m); + return (NULL); +} + +/* * Simple round robin aggregation */ static void diff --git a/sys/net/iflib.c b/sys/net/iflib.c index b0e4bb9470c9..8e2fd257ca74 100644 --- a/sys/net/iflib.c +++ b/sys/net/iflib.c @@ -29,7 +29,6 @@ #include "opt_inet.h" #include "opt_inet6.h" #include "opt_acpi.h" -#include "opt_sched.h" #include <sys/param.h> #include <sys/types.h> @@ -40,8 +39,10 @@ #include <sys/mutex.h> #include <sys/module.h> #include <sys/kobj.h> +#include <sys/proc.h> #include <sys/rman.h> #include <sys/sbuf.h> +#include <sys/sched.h> #include <sys/smp.h> #include <sys/socket.h> #include <sys/sockio.h> @@ -4813,83 +4814,6 @@ cpuid_advance(if_ctx_t ctx, unsigned int cpuid, unsigned int n) return (cpuid); } -#if defined(SMP) && defined(SCHED_ULE) -extern struct cpu_group *cpu_top; /* CPU topology */ - -static int -find_child_with_core(int cpu, struct cpu_group *grp) -{ - int i; - - if (grp->cg_children == 0) - return (-1); - - MPASS(grp->cg_child); - for (i = 0; i < grp->cg_children; i++) { - if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask)) - return (i); - } - - return (-1); -} - - -/* - * Find an L2 neighbor of the given CPU or return -1 if none found. This - * does not distinguish among multiple L2 neighbors if the given CPU has - * more than one (it will always return the same result in that case). - */ -static int -find_l2_neighbor(int cpu) -{ - struct cpu_group *grp; - int i; - - grp = cpu_top; - if (grp == NULL) - return (-1); - - /* - * Find the smallest CPU group that contains the given core. - */ - i = 0; - while ((i = find_child_with_core(cpu, grp)) != -1) { - /* - * If the smallest group containing the given CPU has less - * than two members, we conclude the given CPU has no - * L2 neighbor. - */ - if (grp->cg_child[i].cg_count <= 1) - return (-1); - grp = &grp->cg_child[i]; - } - - /* Must share L2. */ - if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE) - return (-1); - - /* - * Select the first member of the set that isn't the reference - * CPU, which at this point is guaranteed to exist. - */ - for (i = 0; i < CPU_SETSIZE; i++) { - if (CPU_ISSET(i, &grp->cg_mask) && i != cpu) - return (i); - } - - /* Should never be reached */ - return (-1); -} - -#else -static int -find_l2_neighbor(int cpu) -{ - - return (-1); -} -#endif - /* * CPU mapping behaviors * --------------------- @@ -4942,7 +4866,7 @@ get_cpuid_for_queue(if_ctx_t ctx, unsigned int base_cpuid, unsigned int qid, unsigned int rx_cpuid; rx_cpuid = cpuid_advance(ctx, base_cpuid, qid); - l2_neighbor = find_l2_neighbor(rx_cpuid); + l2_neighbor = sched_find_l2_neighbor(rx_cpuid); if (l2_neighbor != -1) { return (l2_neighbor); } diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h index eb17c4ff5ef0..68616a7de5e4 100644 --- a/sys/net/pfvar.h +++ b/sys/net/pfvar.h @@ -1442,6 +1442,8 @@ struct pf_test_ctx { int limiter_drop; u_short reason; struct pf_src_node *sns[PF_SN_MAX]; + struct pf_krule_slist *match_rules; + struct pf_krule_item *last_match_rule; struct pf_krule *nr; struct pf_krule *tr; struct pf_krule **rm; @@ -3128,10 +3130,7 @@ int pf_osfp_match(struct pf_osfp_enlist *, pf_osfp_t); #ifdef _KERNEL void pf_print_host(struct pf_addr *, u_int16_t, sa_family_t); -enum pf_test_status pf_step_into_anchor(struct pf_test_ctx *, struct pf_krule *, - struct pf_krule_slist *match_rules); -enum pf_test_status pf_match_rule(struct pf_test_ctx *, struct pf_kruleset *, - struct pf_krule_slist *); +enum pf_test_status pf_step_into_anchor(struct pf_test_ctx *, struct pf_krule *); void pf_step_into_keth_anchor(struct pf_keth_anchor_stackframe *, int *, struct pf_keth_ruleset **, struct pf_keth_rule **, struct pf_keth_rule **, diff --git a/sys/net80211/ieee80211_radiotap.h b/sys/net80211/ieee80211_radiotap.h index d44f81b68b8c..d729323fce3a 100644 --- a/sys/net80211/ieee80211_radiotap.h +++ b/sys/net80211/ieee80211_radiotap.h @@ -512,8 +512,11 @@ struct ieee80211_radiotap_lsig { uint16_t data1; uint16_t data2; } __packed; + +#define IEEE80211_RADIOTAP_LSIG_DATA1_RATE_KNOWN 0x0001 #define IEEE80211_RADIOTAP_LSIG_DATA1_LENGTH_KNOWN 0x0002 +#define IEEE80211_RADIOTAP_LSIG_DATA2_RATE 0x000F #define IEEE80211_RADIOTAP_LSIG_DATA2_LENGTH 0xFFF0 /* https://www.radiotap.org/fields/MCS.html */ @@ -615,6 +618,24 @@ struct ieee80211_radiotap_eht { #define IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_2_2_2 0x000001ff #define IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_2_2_2_KNOWN 0x00000200 +#define IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3 0x0007fc00 +#define IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3_KNOWN 0x00080000 +#define IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_2_2_3 0x1ff00000 +#define IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_2_2_3_KNOWN 0x20000000 + +#define IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4 0x000001ff +#define IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4_KNOWN 0x00000200 +#define IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_2_2_4 0x0007fc00 +#define IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_2_2_4_KNOWN 0x00080000 +#define IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5 0x1ff00000 +#define IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5_KNOWN 0x20000000 + +#define IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_2_2_5 0x000001ff +#define IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_2_2_5_KNOWN 0x00000200 +#define IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6 0x0007fc00 +#define IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6_KNOWN 0x00080000 +#define IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_2_2_6 0x1ff00000 +#define IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_2_2_6_KNOWN 0x20000000 #define IEEE80211_RADIOTAP_EHT_DATA7_NSS_S 0x0000f000 #define IEEE80211_RADIOTAP_EHT_DATA7_BEAMFORMED_S 0x00010000 diff --git a/sys/netgraph/bluetooth/include/ng_hci.h b/sys/netgraph/bluetooth/include/ng_hci.h index 54980e128bba..bbe9541c25c9 100644 --- a/sys/netgraph/bluetooth/include/ng_hci.h +++ b/sys/netgraph/bluetooth/include/ng_hci.h @@ -447,8 +447,8 @@ typedef struct { } __attribute__ ((packed)) bdaddr_t; typedef bdaddr_t * bdaddr_p; -/* Any BD_ADDR. Note: This is actually 7 bytes (count '\0' terminator) */ -#define NG_HCI_BDADDR_ANY (&(const bdaddr_t){"\000\000\000\000\000\000"}) +/* Any BD_ADDR. */ +#define NG_HCI_BDADDR_ANY (&(const bdaddr_t){ { 0, 0, 0, 0, 0, 0 } }) /* HCI status return parameter */ typedef struct { diff --git a/sys/netinet/icmp6.h b/sys/netinet/icmp6.h index 082ef5d29ce9..9ed39d118c16 100644 --- a/sys/netinet/icmp6.h +++ b/sys/netinet/icmp6.h @@ -717,8 +717,7 @@ int icmp6_ratelimit(const struct in6_addr *, const int, const int); #define icmp6_ifstat_inc(ifp, tag) \ do { \ if (ifp) \ - counter_u64_add(((struct in6_ifextra *) \ - ((ifp)->if_inet6))->icmp6_ifstat[ \ + counter_u64_add((ifp)->if_inet6->icmp6_ifstat[ \ offsetof(struct icmp6_ifstat, tag) / sizeof(uint64_t)], 1);\ } while (/*CONSTCOND*/ 0) diff --git a/sys/netinet/ip6.h b/sys/netinet/ip6.h index 2f61d594e59d..580283b09745 100644 --- a/sys/netinet/ip6.h +++ b/sys/netinet/ip6.h @@ -257,7 +257,17 @@ struct ip6_frag { #define IPV6_HLIMDEC 1 /* subtracted when forwarding */ #define IPV6_MMTU 1280 /* minimal MTU and reassembly. 1024 + 256 */ -#define IPV6_MAXPACKET 65535 /* ip6 max packet size without Jumbo payload*/ +/* + * XXX: IPV6_MAXPACKET is historically used as the maximum packet size. + * The maximum IPv6 packet size is: + * + * v6 header size (40) + payload load size (65535) + * + * practically this isn't encountered as it requires a link with an mtu of + * 65575 octets. IPV6_MAXPACKET is preserved at this value for compatibility. + */ +#define IPV6_MAXPACKET 65535 +#define IPV6_MAXPAYLOAD 65535 /* max size that can be carried in a payload */ #define IPV6_MAXOPTHDR 2048 /* max option header size, 256 64-bit words */ #endif /* not _NETINET_IP6_H_ */ diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c index f9efdc419882..196ae5292909 100644 --- a/sys/netinet/ip_mroute.c +++ b/sys/netinet/ip_mroute.c @@ -546,11 +546,6 @@ X_mrt_ioctl(u_long cmd, caddr_t data, int fibnum __unused) { int error; - /* - * Currently the only function calling this ioctl routine is rtioctl_fib(). - * Typically, only root can create the raw socket in order to execute - * this ioctl method, however the request might be coming from a prison - */ error = priv_check(curthread, PRIV_NETINET_MROUTE); if (error) return (error); @@ -606,12 +601,12 @@ get_vif_cnt(struct sioc_vif_req *req) return EINVAL; } - mtx_lock_spin(&V_viftable[vifi].v_spin); + mtx_lock(&V_viftable[vifi].v_mtx); req->icount = V_viftable[vifi].v_pkt_in; req->ocount = V_viftable[vifi].v_pkt_out; req->ibytes = V_viftable[vifi].v_bytes_in; req->obytes = V_viftable[vifi].v_bytes_out; - mtx_unlock_spin(&V_viftable[vifi].v_spin); + mtx_unlock(&V_viftable[vifi].v_mtx); MRW_RUNLOCK(); return 0; @@ -1004,8 +999,8 @@ add_vif(struct vifctl *vifcp) vifp->v_pkt_out = 0; vifp->v_bytes_in = 0; vifp->v_bytes_out = 0; - sprintf(vifp->v_spin_name, "BM[%d] spin", vifcp->vifc_vifi); - mtx_init(&vifp->v_spin, vifp->v_spin_name, NULL, MTX_SPIN); + sprintf(vifp->v_mtx_name, "BM[%d] mtx", vifcp->vifc_vifi); + mtx_init(&vifp->v_mtx, vifp->v_mtx_name, NULL, MTX_DEF); /* Adjust numvifs up if the vifi is higher than numvifs */ if (V_numvifs <= vifcp->vifc_vifi) @@ -1053,7 +1048,7 @@ del_vif_locked(vifi_t vifi, struct ifnet **ifp_multi_leave, struct ifnet **ifp_f } } - mtx_destroy(&vifp->v_spin); + mtx_destroy(&vifp->v_mtx); bzero((caddr_t)vifp, sizeof (*vifp)); @@ -1659,7 +1654,7 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif) } /* If I sourced this packet, it counts as output, else it was input. */ - mtx_lock_spin(&V_viftable[vifi].v_spin); + mtx_lock(&V_viftable[vifi].v_mtx); if (in_hosteq(ip->ip_src, V_viftable[vifi].v_lcl_addr)) { V_viftable[vifi].v_pkt_out++; V_viftable[vifi].v_bytes_out += plen; @@ -1667,7 +1662,7 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif) V_viftable[vifi].v_pkt_in++; V_viftable[vifi].v_bytes_in += plen; } - mtx_unlock_spin(&V_viftable[vifi].v_spin); + mtx_unlock(&V_viftable[vifi].v_mtx); rt->mfc_pkt_cnt++; rt->mfc_byte_cnt += plen; @@ -1704,14 +1699,14 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif) for (x = rt->mfc_bw_meter_leq; x != NULL; x = x->bm_mfc_next) { /* * Record that a packet is received. - * Spin lock has to be taken as callout context + * A lock has to be taken as callout context * (expire_bw_meter_leq) might modify these fields * as well */ - mtx_lock_spin(&x->bm_spin); + mtx_lock(&x->bm_mtx); x->bm_measured.b_packets++; x->bm_measured.b_bytes += plen; - mtx_unlock_spin(&x->bm_spin); + mtx_unlock(&x->bm_mtx); } } @@ -1894,13 +1889,14 @@ expire_bw_meter_leq(void *arg) /* Reset counters */ x->bm_start_time = now; - /* Spin lock has to be taken as ip_forward context + /* + * The lock has to be taken as ip_forward context * might modify these fields as well */ - mtx_lock_spin(&x->bm_spin); + mtx_lock(&x->bm_mtx); x->bm_measured.b_bytes = 0; x->bm_measured.b_packets = 0; - mtx_unlock_spin(&x->bm_spin); + mtx_unlock(&x->bm_mtx); callout_schedule(&x->bm_meter_callout, tvtohz(&x->bm_threshold.b_time)); @@ -1986,8 +1982,8 @@ add_bw_upcall(struct bw_upcall *req) x->bm_time_next = NULL; x->bm_mfc = mfc; x->arg = curvnet; - sprintf(x->bm_spin_name, "BM spin %p", x); - mtx_init(&x->bm_spin, x->bm_spin_name, NULL, MTX_SPIN); + sprintf(x->bm_mtx_name, "BM mtx %p", x); + mtx_init(&x->bm_mtx, x->bm_mtx_name, NULL, MTX_DEF); /* For LEQ case create periodic callout */ if (req->bu_flags & BW_UPCALL_LEQ) { @@ -2014,7 +2010,7 @@ free_bw_list(struct bw_meter *list) /* MRW_WLOCK must be held here */ if (x->bm_flags & BW_METER_LEQ) { callout_drain(&x->bm_meter_callout); - mtx_destroy(&x->bm_spin); + mtx_destroy(&x->bm_mtx); } list = list->bm_mfc_next; @@ -2115,7 +2111,7 @@ bw_meter_geq_receive_packet(struct bw_meter *x, int plen, struct timeval *nowp) /* * Processing for ">=" type of bw_meter entry. - * bm_spin does not have to be hold here as in GEQ + * bm_mtx does not have to be hold here as in GEQ * case this is the only context accessing bm_measured. */ if (BW_TIMEVALCMP(&delta, &x->bm_threshold.b_time, >)) { @@ -2834,12 +2830,6 @@ ip_mroute_modevent(module_t mod, int type, void *unused) if_detach_event_tag = EVENTHANDLER_REGISTER(ifnet_departure_event, if_detached_event, NULL, EVENTHANDLER_PRI_ANY); - if (if_detach_event_tag == NULL) { - printf("ip_mroute: unable to register " - "ifnet_departure_event handler\n"); - MRW_LOCK_DESTROY(); - return (EINVAL); - } if (!powerof2(mfchashsize)) { printf("WARNING: %s not a power of 2; using default\n", diff --git a/sys/netinet/ip_mroute.h b/sys/netinet/ip_mroute.h index 2c71dc10dfd9..ed98e59a7c77 100644 --- a/sys/netinet/ip_mroute.h +++ b/sys/netinet/ip_mroute.h @@ -262,9 +262,9 @@ struct vif { u_long v_bytes_in; /* # bytes in on interface */ u_long v_bytes_out; /* # bytes out on interface */ #ifdef _KERNEL -#define MROUTE_VIF_SYSCTL_LEN __offsetof(struct vif, v_spin) - struct mtx v_spin; /* Spin mutex for pkt stats */ - char v_spin_name[32]; +#define MROUTE_VIF_SYSCTL_LEN __offsetof(struct vif, v_mtx) + struct mtx v_mtx; /* mutex for pkt stats */ + char v_mtx_name[32]; #endif }; @@ -350,8 +350,8 @@ struct bw_meter { #ifdef _KERNEL struct callout bm_meter_callout; /* Periodic callout */ void* arg; /* custom argument */ - struct mtx bm_spin; /* meter spin lock */ - char bm_spin_name[32]; + struct mtx bm_mtx; /* meter lock */ + char bm_mtx_name[32]; #endif }; diff --git a/sys/netinet/sctp_output.c b/sys/netinet/sctp_output.c index e4bdb4291972..4f6fbc6be783 100644 --- a/sys/netinet/sctp_output.c +++ b/sys/netinet/sctp_output.c @@ -4596,9 +4596,9 @@ sctp_lowlevel_chunk_output(struct sctp_inpcb *inp, } } } else if (ifp != NULL) { - if ((ND_IFINFO(ifp)->linkmtu > 0) && - (stcb->asoc.smallest_mtu > ND_IFINFO(ifp)->linkmtu)) { - sctp_pathmtu_adjustment(stcb, ND_IFINFO(ifp)->linkmtu, false); + if ((ifp->if_inet6->nd_linkmtu > 0) && + (stcb->asoc.smallest_mtu > ifp->if_inet6->nd_linkmtu)) { + sctp_pathmtu_adjustment(stcb, ifp->if_inet6->nd_linkmtu, false); } } } diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c index 8c58be63cd5a..4b501b221bcb 100644 --- a/sys/netinet/tcp_syncache.c +++ b/sys/netinet/tcp_syncache.c @@ -772,12 +772,21 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m) NET_EPOCH_ASSERT(); /* - * Ok, create the full blown connection, and set things up - * as they would have been set up if we had created the - * connection when the SYN arrived. + * Creation of a socket via solisten_clone() bypasses call to pr_attach. + * That's why there is some pasted code from soattach() and from + * tcp_usr_attach() here. This should improve once TCP is PR_SOCKBUF. */ if ((so = solisten_clone(lso)) == NULL) goto allocfail; + mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF); + mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF); + so->so_snd.sb_mtx = &so->so_snd_mtx; + so->so_rcv.sb_mtx = &so->so_rcv_mtx; + error = soreserve(so, lso->sol_sbsnd_hiwat, lso->sol_sbrcv_hiwat); + if (error) { + sodealloc(so); + goto allocfail; + } #ifdef MAC mac_socketpeer_set_from_mbuf(m, so); #endif diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c index c05850f1477b..5b5f7b83623e 100644 --- a/sys/netinet6/icmp6.c +++ b/sys/netinet6/icmp6.c @@ -2080,7 +2080,8 @@ icmp6_reflect(struct mbuf *m, size_t off) if (m->m_pkthdr.rcvif != NULL) { /* XXX: This may not be the outgoing interface */ - hlim = ND_IFINFO(m->m_pkthdr.rcvif)->chlim; + hlim = + m->m_pkthdr.rcvif->if_inet6->nd_curhoplimit; } else hlim = V_ip6_defhlim; } diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c index 43452525e992..e1504400d55b 100644 --- a/sys/netinet6/in6.c +++ b/sys/netinet6/in6.c @@ -1090,7 +1090,7 @@ in6_update_ifa_internal(struct ifnet *ifp, struct in6_aliasreq *ifra, * an interface with ND6_IFF_IFDISABLED. */ if (in6if_do_dad(ifp) && - (hostIsNew || (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED))) + (hostIsNew || (ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED))) ia->ia6_flags |= IN6_IFF_TENTATIVE; /* notify other subsystems */ @@ -1386,11 +1386,11 @@ aifaddr_out: * Try to clear the flag when a new IPv6 address is added * onto an IFDISABLED interface and it succeeds. */ - if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) { + if (ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED) { struct in6_ndireq nd; memset(&nd, 0, sizeof(nd)); - nd.ndi.flags = ND_IFINFO(ifp)->flags; + nd.ndi.flags = ifp->if_inet6->nd_flags; nd.ndi.flags &= ~ND6_IFF_IFDISABLED; if (nd6_ioctl(SIOCSIFINFO_FLAGS, (caddr_t)&nd, ifp) < 0) log(LOG_NOTICE, "SIOCAIFADDR_IN6: " @@ -1712,7 +1712,7 @@ in6ifa_llaonifp(struct ifnet *ifp) struct sockaddr_in6 *sin6; struct ifaddr *ifa; - if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) + if (ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED) return (NULL); NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { @@ -2136,7 +2136,7 @@ in6if_do_dad(struct ifnet *ifp) return (0); if ((ifp->if_flags & IFF_MULTICAST) == 0) return (0); - if ((ND_IFINFO(ifp)->flags & + if ((ifp->if_inet6->nd_flags & (ND6_IFF_IFDISABLED | ND6_IFF_NO_DAD)) != 0) return (0); return (1); @@ -2607,35 +2607,33 @@ in6_ifarrival(void *arg __unused, struct ifnet *ifp) ifp->if_inet6 = NULL; return; } - ext = (struct in6_ifextra *)malloc(sizeof(*ext), M_IFADDR, M_WAITOK); - bzero(ext, sizeof(*ext)); - - ext->in6_ifstat = malloc(sizeof(counter_u64_t) * - sizeof(struct in6_ifstat) / sizeof(uint64_t), M_IFADDR, M_WAITOK); + ext = ifp->if_inet6 = malloc(sizeof(*ext), M_IFADDR, M_WAITOK | M_ZERO); COUNTER_ARRAY_ALLOC(ext->in6_ifstat, sizeof(struct in6_ifstat) / sizeof(uint64_t), M_WAITOK); - - ext->icmp6_ifstat = malloc(sizeof(counter_u64_t) * - sizeof(struct icmp6_ifstat) / sizeof(uint64_t), M_IFADDR, - M_WAITOK); COUNTER_ARRAY_ALLOC(ext->icmp6_ifstat, sizeof(struct icmp6_ifstat) / sizeof(uint64_t), M_WAITOK); + nd6_ifattach(ifp); + mld_domifattach(ifp); + scope6_ifattach(ifp); - ext->nd_ifinfo = nd6_ifattach(ifp); - ext->scope6_id = scope6_ifattach(ifp); ext->lltable = in6_lltattach(ifp); - - ext->mld_ifinfo = mld_domifattach(ifp); - - ifp->if_inet6 = ext; } EVENTHANDLER_DEFINE(ifnet_arrival_event, in6_ifarrival, NULL, EVENTHANDLER_PRI_ANY); uint32_t -in6_ifmtu(struct ifnet *ifp) -{ - return (IN6_LINKMTU(ifp)); +in6_ifmtu(const struct ifnet *ifp) +{ + const uint32_t + linkmtu = ifp->if_inet6->nd_linkmtu, + maxmtu = ifp->if_inet6->nd_maxmtu, + ifmtu = ifp->if_mtu; + + if (linkmtu > 0 && linkmtu < ifmtu) + return (linkmtu); + if (maxmtu > 0 && maxmtu < ifmtu) + return (maxmtu); + return (ifmtu); } /* diff --git a/sys/netinet6/in6_ifattach.c b/sys/netinet6/in6_ifattach.c index c3d256a8d51f..c38fe90632f8 100644 --- a/sys/netinet6/in6_ifattach.c +++ b/sys/netinet6/in6_ifattach.c @@ -465,7 +465,7 @@ in6_get_ifid(struct ifnet *ifp0, struct ifnet *altifp, NET_EPOCH_ASSERT(); /* first, try to get it from the interface itself, with stable algorithm, if configured */ - if ((ND_IFINFO(ifp0)->flags & ND6_IFF_STABLEADDR) && in6_get_stableifid(ifp0, in6, 64) == 0) { + if ((ifp0->if_inet6->nd_flags & ND6_IFF_STABLEADDR) && in6_get_stableifid(ifp0, in6, 64) == 0) { nd6log((LOG_DEBUG, "%s: got interface identifier from itself (stable private)\n", if_name(ifp0))); goto success; @@ -799,8 +799,8 @@ in6_ifattach(struct ifnet *ifp, struct ifnet *altifp) * linklocals for 6to4 interface, but there's no use and * it is rather harmful to have one. */ - ND_IFINFO(ifp)->flags &= ~ND6_IFF_AUTO_LINKLOCAL; - ND_IFINFO(ifp)->flags |= ND6_IFF_NO_DAD; + ifp->if_inet6->nd_flags &= ~ND6_IFF_AUTO_LINKLOCAL; + ifp->if_inet6->nd_flags |= ND6_IFF_NO_DAD; break; default: break; @@ -831,8 +831,8 @@ in6_ifattach(struct ifnet *ifp, struct ifnet *altifp) /* * assign a link-local address, if there's none. */ - if (!(ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) && - ND_IFINFO(ifp)->flags & ND6_IFF_AUTO_LINKLOCAL) { + if (!(ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED) && + ifp->if_inet6->nd_flags & ND6_IFF_AUTO_LINKLOCAL) { struct epoch_tracker et; NET_EPOCH_ENTER(et); @@ -900,6 +900,19 @@ in6_ifdetach(struct ifnet *ifp) } static void +in6_ifextra_free(epoch_context_t ctx) +{ + struct in6_ifextra *ext = + __containerof(ctx, struct in6_ifextra, epoch_ctx); + + COUNTER_ARRAY_FREE(ext->in6_ifstat, + sizeof(struct in6_ifstat) / sizeof(uint64_t)); + COUNTER_ARRAY_FREE(ext->icmp6_ifstat, + sizeof(struct icmp6_ifstat) / sizeof(uint64_t)); + free(ext, M_IFADDR); +} + +static void in6_ifdeparture(void *arg __unused, struct ifnet *ifp) { struct in6_ifextra *ext = ifp->if_inet6; @@ -916,17 +929,16 @@ in6_ifdeparture(void *arg __unused, struct ifnet *ifp) if (!VNET_IS_SHUTTING_DOWN(ifp->if_vnet)) #endif _in6_ifdetach(ifp, 1); + /* + * XXXGL: mld and nd bits are left in a consistent state after + * destructors, but I'm not sure if it safe to call lltable_free() here. + * Individual lle entries are epoch(9) protected, but the table itself + * isn't. + */ mld_domifdetach(ifp); - scope6_ifdetach(ext->scope6_id); - nd6_ifdetach(ifp, ext->nd_ifinfo); + nd6_ifdetach(ifp); lltable_free(ext->lltable); - COUNTER_ARRAY_FREE(ext->in6_ifstat, - sizeof(struct in6_ifstat) / sizeof(uint64_t)); - free(ext->in6_ifstat, M_IFADDR); - COUNTER_ARRAY_FREE(ext->icmp6_ifstat, - sizeof(struct icmp6_ifstat) / sizeof(uint64_t)); - free(ext->icmp6_ifstat, M_IFADDR); - free(ext, M_IFADDR); + NET_EPOCH_CALL(in6_ifextra_free, &ext->epoch_ctx); } EVENTHANDLER_DEFINE(ifnet_departure_event, in6_ifdeparture, NULL, EVENTHANDLER_PRI_ANY); diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c index f2eb00161690..9eb7a59fcf55 100644 --- a/sys/netinet6/in6_pcb.c +++ b/sys/netinet6/in6_pcb.c @@ -115,7 +115,7 @@ SYSCTL_DECL(_net_inet6); SYSCTL_DECL(_net_inet6_ip6); -VNET_DEFINE_STATIC(int, connect_in6addr_wild) = 1; +VNET_DEFINE_STATIC(int, connect_in6addr_wild) = 0; #define V_connect_in6addr_wild VNET(connect_in6addr_wild) SYSCTL_INT(_net_inet6_ip6, OID_AUTO, connect_in6addr_wild, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(connect_in6addr_wild), 0, diff --git a/sys/netinet6/in6_rmx.c b/sys/netinet6/in6_rmx.c index d1c121115b60..35c6cd5ba5f1 100644 --- a/sys/netinet6/in6_rmx.c +++ b/sys/netinet6/in6_rmx.c @@ -114,10 +114,8 @@ rib6_augment_nh(u_int fibnum, struct nhop_object *nh) * inherit interface MTU if not set or * check if MTU is too large. */ - if (nh->nh_mtu == 0) { - nh->nh_mtu = IN6_LINKMTU(nh->nh_ifp); - } else if (nh->nh_mtu > IN6_LINKMTU(nh->nh_ifp)) - nh->nh_mtu = IN6_LINKMTU(nh->nh_ifp); + if (nh->nh_mtu == 0 || nh->nh_mtu > in6_ifmtu(nh->nh_ifp)) + nh->nh_mtu = in6_ifmtu(nh->nh_ifp); /* Set nexthop type */ if (nhop_get_type(nh) == 0) { diff --git a/sys/netinet6/in6_src.c b/sys/netinet6/in6_src.c index 5171bc1d4ea6..d5e8e0f952c6 100644 --- a/sys/netinet6/in6_src.c +++ b/sys/netinet6/in6_src.c @@ -366,7 +366,7 @@ in6_selectsrc(uint32_t fibnum, struct sockaddr_in6 *dstsock, */ /* Rule 5: Prefer outgoing interface */ - if (!(ND_IFINFO(ifp)->flags & ND6_IFF_NO_PREFER_IFACE)) { + if (!(ifp->if_inet6->nd_flags & ND6_IFF_NO_PREFER_IFACE)) { if (ia_best->ia_ifp == ifp && ia->ia_ifp != ifp) NEXT(5); if (ia_best->ia_ifp != ifp && ia->ia_ifp == ifp) @@ -868,7 +868,7 @@ in6_selecthlim(struct inpcb *inp, struct ifnet *ifp) if (inp && inp->in6p_hops >= 0) return (inp->in6p_hops); else if (ifp) - return (ND_IFINFO(ifp)->chlim); + return (ifp->if_inet6->nd_curhoplimit); else if (inp && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) { struct nhop_object *nh; struct in6_addr dst; @@ -879,7 +879,7 @@ in6_selecthlim(struct inpcb *inp, struct ifnet *ifp) in6_splitscope(&inp->in6p_faddr, &dst, &scopeid); nh = fib6_lookup(fibnum, &dst, scopeid, 0, 0); if (nh != NULL) { - hlim = ND_IFINFO(nh->nh_ifp)->chlim; + hlim = nh->nh_ifp->if_inet6->nd_curhoplimit; return (hlim); } } diff --git a/sys/netinet6/in6_var.h b/sys/netinet6/in6_var.h index 8881885ecf86..057cd84b6ea7 100644 --- a/sys/netinet6/in6_var.h +++ b/sys/netinet6/in6_var.h @@ -93,25 +93,6 @@ struct in6_addrlifetime { u_int32_t ia6t_pltime; /* prefix lifetime */ }; -struct nd_ifinfo; -struct scope6_id; -struct lltable; -struct mld_ifsoftc; -struct in6_multi; - -struct in6_ifextra { - counter_u64_t *in6_ifstat; - counter_u64_t *icmp6_ifstat; - struct nd_ifinfo *nd_ifinfo; - struct scope6_id *scope6_id; - struct lltable *lltable; - struct mld_ifsoftc *mld_ifinfo; - u_int dad_failures; /* DAD failures when using RFC 7217 stable addresses */ -}; - -#define LLTABLE6(ifp) ((ifp)->if_inet6->lltable) -#define DAD_FAILURES(ifp) ((ifp)->if_inet6->dad_failures) - #ifdef _KERNEL SLIST_HEAD(in6_multi_head, in6_multi); @@ -507,6 +488,56 @@ struct in6_rrenumreq { #endif #ifdef _KERNEL +/* + * Structure pointed at by ifp->if_inet6. + */ +struct in6_ifextra { + counter_u64_t in6_ifstat[sizeof(struct in6_ifstat) / sizeof(uint64_t)]; + counter_u64_t icmp6_ifstat[sizeof(struct icmp6_ifstat) / + sizeof(uint64_t)]; + /* ND6 */ + uint32_t nd_linkmtu; + uint32_t nd_maxmtu; + uint32_t nd_basereachable; + uint32_t nd_reachable; + uint32_t nd_retrans; + uint32_t nd_flags; + int nd_recalc_timer; + u_int nd_dad_failures; + uint8_t nd_curhoplimit; + + struct mld_ifsoftc { + /* Timers and invervals measured in seconds. */ + LIST_ENTRY(mld_ifsoftc) mli_link; + struct ifnet *mli_ifp; /* interface this instance belongs to */ + uint32_t mli_version; /* MLDv1 Host Compatibility Mode */ + uint32_t mli_v1_timer; /* MLDv1 Querier Present timer */ + uint32_t mli_v2_timer; /* MLDv2 General Query timer */ + uint32_t mli_flags; /* MLD per-interface flags */ + uint32_t mli_rv; /* MLDv2 Robustness Variable */ + uint32_t mli_qi; /* MLDv2 Query Interval */ + uint32_t mli_qri; /* MLDv2 Query Response Interval */ + uint32_t mli_uri; /* MLDv2 Unsolicited Report Interval */ + struct mbufq mli_gq; /* queue of general query responses */ + } mld_ifsoftc; + + struct scope6_id { + /* + * 16 is correspondent to 4bit multicast scope field. i.e. from + * node-local to global with some reserved/unassigned types. + */ +#define IPV6_ADDR_SCOPES_COUNT 16 + uint32_t s6id_list[IPV6_ADDR_SCOPES_COUNT]; + } scope6_id; + + struct lltable *lltable; + + struct epoch_context epoch_ctx; +}; + +#define LLTABLE6(ifp) ((ifp)->if_inet6->lltable) +#define DAD_FAILURES(ifp) ((ifp)->if_inet6->nd_dad_failures) + VNET_DECLARE(struct in6_ifaddrhead, in6_ifaddrhead); VNET_DECLARE(struct in6_ifaddrlisthead *, in6_ifaddrhashtbl); VNET_DECLARE(u_long, in6_ifaddrhmask); @@ -863,7 +894,7 @@ void in6_purgeaddr(struct ifaddr *); void in6_purgeifaddr(struct in6_ifaddr *); int in6if_do_dad(struct ifnet *); void in6_savemkludge(struct in6_ifaddr *); -uint32_t in6_ifmtu(struct ifnet *); +uint32_t in6_ifmtu(const struct ifnet *); struct rib_head *in6_inithead(uint32_t fibnum); void in6_detachhead(struct rib_head *rh); int in6_if2idlen(struct ifnet *); diff --git a/sys/netinet6/ip6_forward.c b/sys/netinet6/ip6_forward.c index 9823366b0156..0ebb51fd80f0 100644 --- a/sys/netinet6/ip6_forward.c +++ b/sys/netinet6/ip6_forward.c @@ -384,11 +384,11 @@ again: pass: /* See if the size was changed by the packet filter. */ /* TODO: change to nh->nh_mtu */ - if (m->m_pkthdr.len > IN6_LINKMTU(nh->nh_ifp)) { + if (m->m_pkthdr.len > in6_ifmtu(nh->nh_ifp)) { in6_ifstat_inc(nh->nh_ifp, ifs6_in_toobig); if (mcopy) icmp6_error(mcopy, ICMP6_PACKET_TOO_BIG, 0, - IN6_LINKMTU(nh->nh_ifp)); + in6_ifmtu(nh->nh_ifp)); goto bad; } diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c index 29fa4741a509..a23f5d46d6a3 100644 --- a/sys/netinet6/ip6_input.c +++ b/sys/netinet6/ip6_input.c @@ -219,7 +219,7 @@ VNET_PCPUSTAT_SYSUNINIT(ip6stat); struct rmlock in6_ifaddr_lock; RM_SYSINIT(in6_ifaddr_lock, &in6_ifaddr_lock, "in6_ifaddr_lock"); -static int ip6_hopopts_input(u_int32_t *, u_int32_t *, struct mbuf **, int *); +static int ip6_hopopts_input(u_int32_t *, struct mbuf **, int *); /* * IP6 initialization: fill in IP6 protocol switch table. @@ -390,7 +390,6 @@ ip6_destroy(void *unused __unused) } /* IF_ADDR_UNLOCK(ifp); */ in6_ifdetach_destroy(ifp); - mld_domifdetach(ifp); } IFNET_RUNLOCK(); @@ -408,14 +407,14 @@ VNET_SYSUNINIT(inet6, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, ip6_destroy, NULL); #endif static int -ip6_input_hbh(struct mbuf **mp, uint32_t *plen, uint32_t *rtalert, int *off, +ip6_input_hbh(struct mbuf **mp, uint32_t *rtalert, int *off, int *nxt, int *ours) { struct mbuf *m; struct ip6_hdr *ip6; struct ip6_hbh *hbh; - if (ip6_hopopts_input(plen, rtalert, mp, off)) { + if (ip6_hopopts_input(rtalert, mp, off)) { #if 0 /*touches NULL pointer*/ in6_ifstat_inc((*mp)->m_pkthdr.rcvif, ifs6_in_discard); #endif @@ -427,16 +426,11 @@ ip6_input_hbh(struct mbuf **mp, uint32_t *plen, uint32_t *rtalert, int *off, ip6 = mtod(m, struct ip6_hdr *); /* - * if the payload length field is 0 and the next header field - * indicates Hop-by-Hop Options header, then a Jumbo Payload - * option MUST be included. + * If the payload length field is 0 and the next header field indicates + * Hop-by-Hop Options header, then a Jumbo Payload option MUST be + * included. We no not support Jumbo Payloads so report an error. */ - if (ip6->ip6_plen == 0 && *plen == 0) { - /* - * Note that if a valid jumbo payload option is - * contained, ip6_hopopts_input() must set a valid - * (non-zero) payload length to the variable plen. - */ + if (ip6->ip6_plen == 0) { IP6STAT_INC(ip6s_badoptions); in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_discard); in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_hdrerr); @@ -551,7 +545,7 @@ ip6_input(struct mbuf *m) * Drop the packet if IPv6 operation is disabled on the interface. */ rcvif = m->m_pkthdr.rcvif; - if ((ND_IFINFO(rcvif)->flags & ND6_IFF_IFDISABLED)) + if ((rcvif->if_inet6->nd_flags & ND6_IFF_IFDISABLED)) goto bad; #if defined(IPSEC) || defined(IPSEC_SUPPORT) @@ -775,6 +769,15 @@ passin: goto bad; } + plen = (uint32_t)ntohs(ip6->ip6_plen); + + /* + * We don't support Jumbograms, reject packets with plen == 0 as early + * as we can. + */ + if (plen == 0) + goto bad; + /* * Disambiguate address scope zones (if there is ambiguity). * We first make sure that the original source or destination address @@ -851,11 +854,9 @@ passin: /* * Process Hop-by-Hop options header if it's contained. * m may be modified in ip6_hopopts_input(). - * If a JumboPayload option is included, plen will also be modified. */ - plen = (u_int32_t)ntohs(ip6->ip6_plen); if (ip6->ip6_nxt == IPPROTO_HOPOPTS) { - if (ip6_input_hbh(&m, &plen, &rtalert, &off, &nxt, &ours) != 0) + if (ip6_input_hbh(&m, &rtalert, &off, &nxt, &ours) != 0) return; } else nxt = ip6->ip6_nxt; @@ -964,13 +965,12 @@ bad: /* * Hop-by-Hop options header processing. If a valid jumbo payload option is - * included, the real payload length will be stored in plenp. + * included report an error. * * rtalertp - XXX: should be stored more smart way */ static int -ip6_hopopts_input(u_int32_t *plenp, u_int32_t *rtalertp, - struct mbuf **mp, int *offp) +ip6_hopopts_input(u_int32_t *rtalertp, struct mbuf **mp, int *offp) { struct mbuf *m = *mp; int off = *offp, hbhlen; @@ -1000,7 +1000,7 @@ ip6_hopopts_input(u_int32_t *plenp, u_int32_t *rtalertp, off += hbhlen; hbhlen -= sizeof(struct ip6_hbh); if (ip6_process_hopopts(m, (u_int8_t *)hbh + sizeof(struct ip6_hbh), - hbhlen, rtalertp, plenp) < 0) { + hbhlen, rtalertp) < 0) { *mp = NULL; return (-1); } @@ -1022,13 +1022,11 @@ ip6_hopopts_input(u_int32_t *plenp, u_int32_t *rtalertp, */ int ip6_process_hopopts(struct mbuf *m, u_int8_t *opthead, int hbhlen, - u_int32_t *rtalertp, u_int32_t *plenp) + u_int32_t *rtalertp) { - struct ip6_hdr *ip6; int optlen = 0; u_int8_t *opt = opthead; u_int16_t rtalert_val; - u_int32_t jumboplen; const int erroff = sizeof(struct ip6_hdr) + sizeof(struct ip6_hbh); for (; hbhlen > 0; hbhlen -= optlen, opt += optlen) { @@ -1061,71 +1059,8 @@ ip6_process_hopopts(struct mbuf *m, u_int8_t *opthead, int hbhlen, *rtalertp = ntohs(rtalert_val); break; case IP6OPT_JUMBO: - /* XXX may need check for alignment */ - if (hbhlen < IP6OPT_JUMBO_LEN) { - IP6STAT_INC(ip6s_toosmall); - goto bad; - } - if (*(opt + 1) != IP6OPT_JUMBO_LEN - 2) { - /* XXX stat */ - icmp6_error(m, ICMP6_PARAM_PROB, - ICMP6_PARAMPROB_HEADER, - erroff + opt + 1 - opthead); - return (-1); - } - optlen = IP6OPT_JUMBO_LEN; - - /* - * IPv6 packets that have non 0 payload length - * must not contain a jumbo payload option. - */ - ip6 = mtod(m, struct ip6_hdr *); - if (ip6->ip6_plen) { - IP6STAT_INC(ip6s_badoptions); - icmp6_error(m, ICMP6_PARAM_PROB, - ICMP6_PARAMPROB_HEADER, - erroff + opt - opthead); - return (-1); - } - - /* - * We may see jumbolen in unaligned location, so - * we'd need to perform bcopy(). - */ - bcopy(opt + 2, &jumboplen, sizeof(jumboplen)); - jumboplen = (u_int32_t)htonl(jumboplen); - -#if 1 - /* - * if there are multiple jumbo payload options, - * *plenp will be non-zero and the packet will be - * rejected. - * the behavior may need some debate in ipngwg - - * multiple options does not make sense, however, - * there's no explicit mention in specification. - */ - if (*plenp != 0) { - IP6STAT_INC(ip6s_badoptions); - icmp6_error(m, ICMP6_PARAM_PROB, - ICMP6_PARAMPROB_HEADER, - erroff + opt + 2 - opthead); - return (-1); - } -#endif - - /* - * jumbo payload length must be larger than 65535. - */ - if (jumboplen <= IPV6_MAXPACKET) { - IP6STAT_INC(ip6s_badoptions); - icmp6_error(m, ICMP6_PARAM_PROB, - ICMP6_PARAMPROB_HEADER, - erroff + opt + 2 - opthead); - return (-1); - } - *plenp = jumboplen; - - break; + /* We do not support the Jumbo Payload option. */ + goto bad; default: /* unknown option */ if (hbhlen < IP6OPT_MINLEN) { IP6STAT_INC(ip6s_toosmall); diff --git a/sys/netinet6/ip6_mroute.c b/sys/netinet6/ip6_mroute.c index 4473d3931af8..5706c12fc5e4 100644 --- a/sys/netinet6/ip6_mroute.c +++ b/sys/netinet6/ip6_mroute.c @@ -89,6 +89,7 @@ #include <sys/mbuf.h> #include <sys/module.h> #include <sys/domain.h> +#include <sys/priv.h> #include <sys/protosw.h> #include <sys/sdt.h> #include <sys/signalvar.h> @@ -254,10 +255,6 @@ static void expire_upcalls(void *); #define UPCALL_EXPIRE 6 /* number of timeouts */ /* - * XXX TODO: maintain a count to if_allmulti() calls in struct ifnet. - */ - -/* * 'Interfaces' associated with decapsulator (so we can tell * packets that went through it from ones that get reflected * by a broken gateway). Different from IPv4 register_if, @@ -270,8 +267,6 @@ static void expire_upcalls(void *); */ static struct ifnet *multicast_register_if6; -#define ENCAP_HOPS 64 - /* * Private variables. */ @@ -296,26 +291,6 @@ VNET_DEFINE_STATIC(int, pim6); (g).s6_addr32[2] ^ (g).s6_addr32[3]) /* - * Find a route for a given origin IPv6 address and Multicast group address. - */ -#define MF6CFIND(o, g, rt) do { \ - struct mf6c *_rt = mf6ctable[MF6CHASH(o,g)]; \ - rt = NULL; \ - while (_rt) { \ - if (IN6_ARE_ADDR_EQUAL(&_rt->mf6c_origin.sin6_addr, &(o)) && \ - IN6_ARE_ADDR_EQUAL(&_rt->mf6c_mcastgrp.sin6_addr, &(g)) && \ - (_rt->mf6c_stall == NULL)) { \ - rt = _rt; \ - break; \ - } \ - _rt = _rt->mf6c_next; \ - } \ - if (rt == NULL) { \ - MRT6STAT_INC(mrt6s_mfc_misses); \ - } \ -} while (/*CONSTCOND*/ 0) - -/* * Macros to compute elapsed time efficiently * Borrowed from Van Jacobson's scheduling code * XXX: replace with timersub() ? @@ -345,7 +320,7 @@ VNET_DEFINE_STATIC(int, pim6); #ifdef UPCALL_TIMING #define UPCALL_MAX 50 static u_long upcall_data[UPCALL_MAX + 1]; -static void collate(); +static void collate(struct timeval *); #endif /* UPCALL_TIMING */ static int ip6_mrouter_init(struct socket *, int, int); @@ -359,16 +334,32 @@ static int get_sg_cnt(struct sioc_sg_req6 *); static struct callout expire_upcalls_ch; -int X_ip6_mforward(struct ip6_hdr *, struct ifnet *, struct mbuf *); -int X_ip6_mrouter_done(void); -int X_ip6_mrouter_set(struct socket *, struct sockopt *); -int X_ip6_mrouter_get(struct socket *, struct sockopt *); -int X_mrt6_ioctl(u_long, caddr_t); +static int X_ip6_mforward(struct ip6_hdr *, struct ifnet *, struct mbuf *); +static int X_ip6_mrouter_done(void); +static int X_ip6_mrouter_set(struct socket *, struct sockopt *); +static int X_ip6_mrouter_get(struct socket *, struct sockopt *); +static int X_mrt6_ioctl(u_long, caddr_t); + +static struct mf6c * +mf6c_find(const struct in6_addr *origin, const struct in6_addr *group) +{ + MFC6_LOCK_ASSERT(); + + for (struct mf6c *rt = mf6ctable[MF6CHASH(*origin, *group)]; rt != NULL; + rt = rt->mf6c_next) { + if (IN6_ARE_ADDR_EQUAL(&rt->mf6c_origin.sin6_addr, origin) && + IN6_ARE_ADDR_EQUAL(&rt->mf6c_mcastgrp.sin6_addr, group) && + rt->mf6c_stall == NULL) + return (rt); + } + MRT6STAT_INC(mrt6s_mfc_misses); + return (NULL); +} /* * Handle MRT setsockopt commands to modify the multicast routing tables. */ -int +static int X_ip6_mrouter_set(struct socket *so, struct sockopt *sopt) { int error = 0; @@ -436,7 +427,7 @@ X_ip6_mrouter_set(struct socket *so, struct sockopt *sopt) /* * Handle MRT getsockopt commands */ -int +static int X_ip6_mrouter_get(struct socket *so, struct sockopt *sopt) { int error = 0; @@ -455,27 +446,29 @@ X_ip6_mrouter_get(struct socket *so, struct sockopt *sopt) /* * Handle ioctl commands to obtain information from the cache */ -int +static int X_mrt6_ioctl(u_long cmd, caddr_t data) { - int ret; - - ret = EINVAL; + int error; + error = priv_check(curthread, PRIV_NETINET_MROUTE); + if (error) + return (error); + error = EINVAL; switch (cmd) { case SIOCGETSGCNT_IN6: - ret = get_sg_cnt((struct sioc_sg_req6 *)data); + error = get_sg_cnt((struct sioc_sg_req6 *)data); break; case SIOCGETMIFCNT_IN6: - ret = get_mif6_cnt((struct sioc_mif_req6 *)data); + error = get_mif6_cnt((struct sioc_mif_req6 *)data); break; default: break; } - return (ret); + return (error); } /* @@ -491,7 +484,7 @@ get_sg_cnt(struct sioc_sg_req6 *req) MFC6_LOCK(); - MF6CFIND(req->src.sin6_addr, req->grp.sin6_addr, rt); + rt = mf6c_find(&req->src.sin6_addr, &req->grp.sin6_addr); if (rt == NULL) { ret = ESRCH; } else { @@ -585,7 +578,7 @@ ip6_mrouter_init(struct socket *so, int v, int cmd) /* * Disable IPv6 multicast forwarding. */ -int +static int X_ip6_mrouter_done(void) { mifi_t mifi; @@ -814,9 +807,8 @@ add_m6fc(struct mf6cctl *mfccp) MFC6_LOCK(); - MF6CFIND(mfccp->mf6cc_origin.sin6_addr, - mfccp->mf6cc_mcastgrp.sin6_addr, rt); - + rt = mf6c_find(&mfccp->mf6cc_origin.sin6_addr, + &mfccp->mf6cc_mcastgrp.sin6_addr); /* If an entry already exists, just update the fields */ if (rt) { MRT6_DLOG(DEBUG_MFC, "no upcall o %s g %s p %x", @@ -916,8 +908,7 @@ add_m6fc(struct mf6cctl *mfccp) } if (rt == NULL) { /* no upcall, so make a new entry */ - rt = (struct mf6c *)malloc(sizeof(*rt), M_MRTABLE6, - M_NOWAIT); + rt = malloc(sizeof(*rt), M_MRTABLE6, M_NOWAIT); if (rt == NULL) { MFC6_UNLOCK(); return (ENOBUFS); @@ -1055,7 +1046,7 @@ socket_send(struct socket *s, struct mbuf *mm, struct sockaddr_in6 *src) * that if this function is called from somewhere else in the originating * context in the future. */ -int +static int X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m) { struct rtdetq *rte; @@ -1112,7 +1103,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m) /* * Determine forwarding mifs from the forwarding cache table */ - MF6CFIND(ip6->ip6_src, ip6->ip6_dst, rt); + rt = mf6c_find(&ip6->ip6_src, &ip6->ip6_dst); MRT6STAT_INC(mrt6s_mfc_lookups); /* Entry exists, so forward if necessary */ @@ -1134,7 +1125,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m) * Allocate mbufs early so that we don't do extra work if we * are just going to fail anyway. */ - rte = (struct rtdetq *)malloc(sizeof(*rte), M_MRTABLE6, M_NOWAIT); + rte = malloc(sizeof(*rte), M_MRTABLE6, M_NOWAIT); if (rte == NULL) { MFC6_UNLOCK(); return (ENOBUFS); @@ -1168,7 +1159,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m) struct omrt6msg *oim; #endif /* no upcall, so make a new entry */ - rt = (struct mf6c *)malloc(sizeof(*rt), M_MRTABLE6, M_NOWAIT); + rt = malloc(sizeof(*rt), M_MRTABLE6, M_NOWAIT); if (rt == NULL) { free(rte, M_MRTABLE6); m_freem(mb0); @@ -1582,7 +1573,7 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m) * Put the packet into the sending queue of the outgoing interface * if it would fit in the MTU of the interface. */ - linkmtu = IN6_LINKMTU(ifp); + linkmtu = in6_ifmtu(ifp); if (mb_copy->m_pkthdr.len <= linkmtu || linkmtu < IPV6_MMTU) { struct sockaddr_in6 dst6; diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c index d61bc983cc12..dca1bcf04371 100644 --- a/sys/netinet6/ip6_output.c +++ b/sys/netinet6/ip6_output.c @@ -142,7 +142,6 @@ static int ip6_setpktopt(int, u_char *, int, struct ip6_pktopts *, static int ip6_copyexthdr(struct mbuf **, caddr_t, int); static int ip6_insertfraghdr(struct mbuf *, struct mbuf *, int, struct ip6_frag **); -static int ip6_insert_jumboopt(struct ip6_exthdrs *, u_int32_t); static int ip6_splithdr(struct mbuf *, struct ip6_exthdrs *); static void ip6_getpmtu(struct route_in6 *, int, struct ifnet *, const struct in6_addr *, u_long *, u_int, u_int); @@ -542,21 +541,9 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, m->m_pkthdr.len += optlen; plen = m->m_pkthdr.len - sizeof(*ip6); - /* If this is a jumbo payload, insert a jumbo payload option. */ if (plen > IPV6_MAXPACKET) { - if (!hdrsplit) { - if ((error = ip6_splithdr(m, &exthdrs)) != 0) { - m = NULL; - goto freehdrs; - } - m = exthdrs.ip6e_ip6; - ip6 = mtod(m, struct ip6_hdr *); - hdrsplit = true; - } - if ((error = ip6_insert_jumboopt(&exthdrs, plen)) != 0) - goto freehdrs; - ip6->ip6_plen = 0; - optlen += 8; /* JUMBOOPTLEN */ + error = EMSGSIZE; + goto freehdrs; } else ip6->ip6_plen = htons(plen); nexthdrp = &ip6->ip6_nxt; @@ -982,7 +969,6 @@ nonh6lookup: if (exthdrs.ip6e_hbh) { struct ip6_hbh *hbh = mtod(exthdrs.ip6e_hbh, struct ip6_hbh *); u_int32_t dummy; /* XXX unused */ - u_int32_t plen = 0; /* XXX: ip6_process will check the value */ #ifdef DIAGNOSTIC if ((hbh->ip6h_len + 1) << 3 > exthdrs.ip6e_hbh->m_len) @@ -998,7 +984,7 @@ nonh6lookup: m->m_pkthdr.rcvif = ifp; if (ip6_process_hopopts(m, (u_int8_t *)(hbh + 1), ((hbh->ip6h_len + 1) << 3) - sizeof(struct ip6_hbh), - &dummy, &plen) < 0) { + &dummy) < 0) { /* m was already freed at this point. */ error = EINVAL;/* better error? */ goto done; @@ -1147,7 +1133,7 @@ passout: dontfrag = 1; else dontfrag = 0; - if (dontfrag && tlen > IN6_LINKMTU(ifp) && !tso) { /* Case 2-b. */ + if (dontfrag && tlen > in6_ifmtu(ifp) && !tso) { /* Case 2-b. */ /* * If the DONTFRAG option is specified, we cannot send the * packet when the data length is larger than the MTU of the @@ -1186,7 +1172,7 @@ passout: in6_ifstat_inc(ifp, ifs6_out_fragfail); goto bad; } else if (ip6->ip6_plen == 0) { - /* Jumbo payload cannot be fragmented. */ + /* We do not support jumbo payload. */ error = EMSGSIZE; in6_ifstat_inc(ifp, ifs6_out_fragfail); goto bad; @@ -1313,94 +1299,6 @@ ip6_copyexthdr(struct mbuf **mp, caddr_t hdr, int hlen) } /* - * Insert jumbo payload option. - */ -static int -ip6_insert_jumboopt(struct ip6_exthdrs *exthdrs, u_int32_t plen) -{ - struct mbuf *mopt; - u_char *optbuf; - u_int32_t v; - -#define JUMBOOPTLEN 8 /* length of jumbo payload option and padding */ - - /* - * If there is no hop-by-hop options header, allocate new one. - * If there is one but it doesn't have enough space to store the - * jumbo payload option, allocate a cluster to store the whole options. - * Otherwise, use it to store the options. - */ - if (exthdrs->ip6e_hbh == NULL) { - mopt = m_get(M_NOWAIT, MT_DATA); - if (mopt == NULL) - return (ENOBUFS); - mopt->m_len = JUMBOOPTLEN; - optbuf = mtod(mopt, u_char *); - optbuf[1] = 0; /* = ((JUMBOOPTLEN) >> 3) - 1 */ - exthdrs->ip6e_hbh = mopt; - } else { - struct ip6_hbh *hbh; - - mopt = exthdrs->ip6e_hbh; - if (M_TRAILINGSPACE(mopt) < JUMBOOPTLEN) { - /* - * XXX assumption: - * - exthdrs->ip6e_hbh is not referenced from places - * other than exthdrs. - * - exthdrs->ip6e_hbh is not an mbuf chain. - */ - int oldoptlen = mopt->m_len; - struct mbuf *n; - - /* - * XXX: give up if the whole (new) hbh header does - * not fit even in an mbuf cluster. - */ - if (oldoptlen + JUMBOOPTLEN > MCLBYTES) - return (ENOBUFS); - - /* - * As a consequence, we must always prepare a cluster - * at this point. - */ - n = m_getcl(M_NOWAIT, MT_DATA, 0); - if (n == NULL) - return (ENOBUFS); - n->m_len = oldoptlen + JUMBOOPTLEN; - bcopy(mtod(mopt, caddr_t), mtod(n, caddr_t), - oldoptlen); - optbuf = mtod(n, caddr_t) + oldoptlen; - m_freem(mopt); - mopt = exthdrs->ip6e_hbh = n; - } else { - optbuf = mtod(mopt, u_char *) + mopt->m_len; - mopt->m_len += JUMBOOPTLEN; - } - optbuf[0] = IP6OPT_PADN; - optbuf[1] = 1; - - /* - * Adjust the header length according to the pad and - * the jumbo payload option. - */ - hbh = mtod(mopt, struct ip6_hbh *); - hbh->ip6h_len += (JUMBOOPTLEN >> 3); - } - - /* fill in the option. */ - optbuf[2] = IP6OPT_JUMBO; - optbuf[3] = 4; - v = (u_int32_t)htonl(plen + JUMBOOPTLEN); - bcopy(&v, &optbuf[4], sizeof(u_int32_t)); - - /* finally, adjust the packet header length */ - exthdrs->ip6e_ip6->m_pkthdr.len += JUMBOOPTLEN; - - return (0); -#undef JUMBOOPTLEN -} - -/* * Insert fragment header and copy unfragmentable header portions. */ static int @@ -1561,7 +1459,7 @@ ip6_calcmtu(struct ifnet *ifp, const struct in6_addr *dst, u_long rt_mtu, } if (mtu == 0) - mtu = IN6_LINKMTU(ifp); + mtu = in6_ifmtu(ifp); *mtup = mtu; } @@ -2922,7 +2820,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, return (ENXIO); } if (ifp != NULL && (ifp->if_inet6 == NULL || - (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) != 0)) + (ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED) != 0)) return (ENETDOWN); if (ifp != NULL && diff --git a/sys/netinet6/ip6_var.h b/sys/netinet6/ip6_var.h index db1631736c4a..c1645f587483 100644 --- a/sys/netinet6/ip6_var.h +++ b/sys/netinet6/ip6_var.h @@ -393,8 +393,7 @@ int ip6_lasthdr(const struct mbuf *, int, int, int *); extern int (*ip6_mforward)(struct ip6_hdr *, struct ifnet *, struct mbuf *); -int ip6_process_hopopts(struct mbuf *, u_int8_t *, int, u_int32_t *, - u_int32_t *); +int ip6_process_hopopts(struct mbuf *, u_int8_t *, int, u_int32_t *); struct mbuf **ip6_savecontrol_v4(struct inpcb *, struct mbuf *, struct mbuf **, int *); void ip6_savecontrol(struct inpcb *, struct mbuf *, struct mbuf **); diff --git a/sys/netinet6/mld6.c b/sys/netinet6/mld6.c index 8e2bbf8adc01..f14d2c76ffda 100644 --- a/sys/netinet6/mld6.c +++ b/sys/netinet6/mld6.c @@ -99,7 +99,6 @@ #define KTR_MLD KTR_INET6 #endif -static void mli_delete_locked(struct ifnet *); static void mld_dispatch_packet(struct mbuf *); static void mld_dispatch_queue(struct mbufq *, int); static void mld_final_leave(struct in6_multi *, struct mld_ifsoftc *); @@ -465,21 +464,21 @@ mld_is_addr_reported(const struct in6_addr *addr) * Attach MLD when PF_INET6 is attached to an interface. Assumes that the * current VNET is set by the caller. */ -struct mld_ifsoftc * +void mld_domifattach(struct ifnet *ifp) { - struct mld_ifsoftc *mli; + struct mld_ifsoftc *mli = MLD_IFINFO(ifp); CTR3(KTR_MLD, "%s: called for ifp %p(%s)", __func__, ifp, if_name(ifp)); - mli = malloc(sizeof(struct mld_ifsoftc), M_MLD, M_WAITOK | M_ZERO); - mli->mli_ifp = ifp; - mli->mli_version = MLD_VERSION_2; - mli->mli_flags = 0; - mli->mli_rv = MLD_RV_INIT; - mli->mli_qi = MLD_QI_INIT; - mli->mli_qri = MLD_QRI_INIT; - mli->mli_uri = MLD_URI_INIT; + *mli = (struct mld_ifsoftc){ + .mli_ifp = ifp, + .mli_version = MLD_VERSION_2, + .mli_rv = MLD_RV_INIT, + .mli_qi = MLD_QI_INIT, + .mli_qri = MLD_QRI_INIT, + .mli_uri = MLD_URI_INIT, + }; mbufq_init(&mli->mli_gq, MLD_MAX_RESPONSE_PACKETS); if ((ifp->if_flags & IFF_MULTICAST) == 0) mli->mli_flags |= MLIF_SILENT; @@ -489,8 +488,6 @@ mld_domifattach(struct ifnet *ifp) MLD_LOCK(); LIST_INSERT_HEAD(&V_mli_head, mli, mli_link); MLD_UNLOCK(); - - return (mli); } /* @@ -552,44 +549,19 @@ mld_ifdetach(struct ifnet *ifp, struct in6_multi_head *inmh) /* * Hook for domifdetach. * Runs after link-layer cleanup; free MLD state. - * - * SMPng: Normally called with LLTABLE_LOCK held. */ void mld_domifdetach(struct ifnet *ifp) { + struct mld_ifsoftc *mli = MLD_IFINFO(ifp); CTR3(KTR_MLD, "%s: called for ifp %p(%s)", __func__, ifp, if_name(ifp)); MLD_LOCK(); - mli_delete_locked(ifp); + LIST_REMOVE(mli, mli_link); MLD_UNLOCK(); -} - -static void -mli_delete_locked(struct ifnet *ifp) -{ - struct mld_ifsoftc *mli, *tmli; - - CTR3(KTR_MLD, "%s: freeing mld_ifsoftc for ifp %p(%s)", - __func__, ifp, if_name(ifp)); - - MLD_LOCK_ASSERT(); - - LIST_FOREACH_SAFE(mli, &V_mli_head, mli_link, tmli) { - if (mli->mli_ifp == ifp) { - /* - * Free deferred General Query responses. - */ - mbufq_drain(&mli->mli_gq); - - LIST_REMOVE(mli, mli_link); - - free(mli, M_MLD); - return; - } - } + mbufq_drain(&mli->mli_gq); } /* diff --git a/sys/netinet6/mld6_var.h b/sys/netinet6/mld6_var.h index d75ac2450c10..a063771f4dc8 100644 --- a/sys/netinet6/mld6_var.h +++ b/sys/netinet6/mld6_var.h @@ -120,23 +120,6 @@ struct mld_ifinfo { }; #ifdef _KERNEL -/* - * Per-link MLD state. - */ -struct mld_ifsoftc { - LIST_ENTRY(mld_ifsoftc) mli_link; - struct ifnet *mli_ifp; /* interface this instance belongs to */ - uint32_t mli_version; /* MLDv1 Host Compatibility Mode */ - uint32_t mli_v1_timer; /* MLDv1 Querier Present timer (s) */ - uint32_t mli_v2_timer; /* MLDv2 General Query (interface) timer (s)*/ - uint32_t mli_flags; /* MLD per-interface flags */ - uint32_t mli_rv; /* MLDv2 Robustness Variable */ - uint32_t mli_qi; /* MLDv2 Query Interval (s) */ - uint32_t mli_qri; /* MLDv2 Query Response Interval (s) */ - uint32_t mli_uri; /* MLDv2 Unsolicited Report Interval (s) */ - struct mbufq mli_gq; /* queue of general query responses */ -}; - #define MLD_RANDOM_DELAY(X) (arc4random() % (X) + 1) #define MLD_MAX_STATE_CHANGES 24 /* Max pending changes per group */ @@ -155,12 +138,11 @@ struct mld_ifsoftc { /* * Per-link MLD context. */ -#define MLD_IFINFO(ifp) ((ifp)->if_inet6->mld_ifinfo) +#define MLD_IFINFO(ifp) (&(ifp)->if_inet6->mld_ifsoftc) struct in6_multi_head; int mld_change_state(struct in6_multi *, const int); -struct mld_ifsoftc * - mld_domifattach(struct ifnet *); +void mld_domifattach(struct ifnet *); void mld_domifdetach(struct ifnet *); void mld_ifdetach(struct ifnet *, struct in6_multi_head *); int mld_input(struct mbuf **, int, int); diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index 04ce9bf6dd55..969b32032a60 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -88,8 +88,6 @@ #define ND6_SLOWTIMER_INTERVAL (60 * 60) /* 1 hour */ #define ND6_RECALC_REACHTM_INTERVAL (60 * 120) /* 2 hours */ -MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery"); - VNET_DEFINE_STATIC(int, nd6_prune) = 1; #define V_nd6_prune VNET(nd6_prune) SYSCTL_INT(_net_inet6_icmp6, ICMPV6CTL_ND6_PRUNE, nd6_prune, @@ -150,7 +148,6 @@ int (*send_sendso_input_hook)(struct mbuf *, struct ifnet *, int, int); static bool nd6_is_new_addr_neighbor(const struct sockaddr_in6 *, struct ifnet *); -static void nd6_setmtu0(struct ifnet *, struct nd_ifinfo *); static void nd6_slowtimo(void *); static int regen_tmpaddr(struct in6_ifaddr *); static void nd6_free(struct llentry **, int); @@ -277,24 +274,30 @@ nd6_destroy(void) } #endif -struct nd_ifinfo * +void nd6_ifattach(struct ifnet *ifp) { - struct nd_ifinfo *nd; - - nd = malloc(sizeof(*nd), M_IP6NDP, M_WAITOK | M_ZERO); - nd->initialized = 1; + struct in6_ifextra *nd = ifp->if_inet6; - nd->chlim = IPV6_DEFHLIM; - nd->basereachable = REACHABLE_TIME; - nd->reachable = ND_COMPUTE_RTIME(nd->basereachable); - nd->retrans = RETRANS_TIMER; + nd->nd_linkmtu = 0; + nd->nd_maxmtu = ifp->if_mtu; + nd->nd_basereachable = REACHABLE_TIME; + nd->nd_reachable = ND_COMPUTE_RTIME(nd->nd_basereachable); + nd->nd_retrans = RETRANS_TIMER; + nd->nd_recalc_timer = 0; + nd->nd_dad_failures = 0; + nd->nd_curhoplimit = IPV6_DEFHLIM; - nd->flags = ND6_IFF_PERFORMNUD; + nd->nd_flags = ND6_IFF_PERFORMNUD; /* Set IPv6 disabled on all interfaces but loopback by default. */ - if ((ifp->if_flags & IFF_LOOPBACK) == 0) - nd->flags |= ND6_IFF_IFDISABLED; + if ((ifp->if_flags & IFF_LOOPBACK) == 0) { + nd->nd_flags |= ND6_IFF_IFDISABLED; + if (V_ip6_no_radr) + nd->nd_flags |= ND6_IFF_NO_RADR; + if (V_ip6_use_stableaddr) + nd->nd_flags |= ND6_IFF_STABLEADDR; + } /* A loopback interface always has ND6_IFF_AUTO_LINKLOCAL. * XXXHRS: Clear ND6_IFF_AUTO_LINKLOCAL on an IFT_BRIDGE interface by @@ -303,7 +306,7 @@ nd6_ifattach(struct ifnet *ifp) */ if ((V_ip6_auto_linklocal && ifp->if_type != IFT_BRIDGE && ifp->if_type != IFT_WIREGUARD) || (ifp->if_flags & IFF_LOOPBACK)) - nd->flags |= ND6_IFF_AUTO_LINKLOCAL; + nd->nd_flags |= ND6_IFF_AUTO_LINKLOCAL; /* * A loopback interface does not need to accept RTADV. * XXXHRS: Clear ND6_IFF_ACCEPT_RTADV on an IFT_BRIDGE interface by @@ -314,26 +317,14 @@ nd6_ifattach(struct ifnet *ifp) if (V_ip6_accept_rtadv && !(ifp->if_flags & IFF_LOOPBACK) && (ifp->if_type != IFT_BRIDGE)) { - nd->flags |= ND6_IFF_ACCEPT_RTADV; + nd->nd_flags |= ND6_IFF_ACCEPT_RTADV; /* If we globally accept rtadv, assume IPv6 on. */ - nd->flags &= ~ND6_IFF_IFDISABLED; + nd->nd_flags &= ~ND6_IFF_IFDISABLED; } - if (V_ip6_no_radr && !(ifp->if_flags & IFF_LOOPBACK)) - nd->flags |= ND6_IFF_NO_RADR; - - /* XXX: we cannot call nd6_setmtu since ifp is not fully initialized */ - nd6_setmtu0(ifp, nd); - - /* Configure default value for stable addresses algorithm, skip loopback interface */ - if (V_ip6_use_stableaddr && !(ifp->if_flags & IFF_LOOPBACK)) { - nd->flags |= ND6_IFF_STABLEADDR; - } - - return nd; } void -nd6_ifdetach(struct ifnet *ifp, struct nd_ifinfo *nd) +nd6_ifdetach(struct ifnet *ifp) { struct epoch_tracker et; struct ifaddr *ifa, *next; @@ -347,32 +338,25 @@ nd6_ifdetach(struct ifnet *ifp, struct nd_ifinfo *nd) nd6_dad_stop(ifa); } NET_EPOCH_EXIT(et); - - free(nd, M_IP6NDP); } /* * Reset ND level link MTU. This function is called when the physical MTU * changes, which means we might have to adjust the ND level MTU. + * XXX todo: do not maintain copy of ifp->if_mtu in if_inet6->nd_maxmtu. */ void nd6_setmtu(struct ifnet *ifp) { - /* XXXGL: ??? */ - if (ifp->if_inet6 == NULL) - return; - - nd6_setmtu0(ifp, ND_IFINFO(ifp)); -} + struct in6_ifextra *ndi = ifp->if_inet6; + uint32_t omaxmtu; -/* XXX todo: do not maintain copy of ifp->if_mtu in ndi->maxmtu */ -void -nd6_setmtu0(struct ifnet *ifp, struct nd_ifinfo *ndi) -{ - u_int32_t omaxmtu; + /* XXXGL: safety against IFT_PFSYNC & IFT_PFLOG */ + if (ndi == NULL) + return; - omaxmtu = ndi->maxmtu; - ndi->maxmtu = ifp->if_mtu; + omaxmtu = ndi->nd_maxmtu; + ndi->nd_maxmtu = ifp->if_mtu; /* * Decreasing the interface MTU under IPV6 minimum MTU may cause @@ -380,10 +364,10 @@ nd6_setmtu0(struct ifnet *ifp, struct nd_ifinfo *ndi) * explicitly. The check for omaxmtu is necessary to restrict the * log to the case of changing the MTU, not initializing it. */ - if (omaxmtu >= IPV6_MMTU && ndi->maxmtu < IPV6_MMTU) { - log(LOG_NOTICE, "nd6_setmtu0: " + if (omaxmtu >= IPV6_MMTU && ndi->nd_maxmtu < IPV6_MMTU) { + log(LOG_NOTICE, "%s: " "new link MTU on %s (%lu) is too small for IPv6\n", - if_name(ifp), (unsigned long)ndi->maxmtu); + __func__, if_name(ifp), (unsigned long)ndi->nd_maxmtu); } } @@ -714,12 +698,12 @@ nd6_llinfo_setstate(struct llentry *lle, int newstate) switch (newstate) { case ND6_LLINFO_INCOMPLETE: ifp = lle->lle_tbl->llt_ifp; - delay = (long)ND_IFINFO(ifp)->retrans * hz / 1000; + delay = (long)ifp->if_inet6->nd_retrans * hz / 1000; break; case ND6_LLINFO_REACHABLE: if (!ND6_LLINFO_PERMANENT(lle)) { ifp = lle->lle_tbl->llt_ifp; - delay = (long)ND_IFINFO(ifp)->reachable * hz; + delay = (long)ifp->if_inet6->nd_reachable * hz; } break; case ND6_LLINFO_STALE: @@ -756,7 +740,7 @@ nd6_llinfo_timer(void *arg) struct llentry *ln; struct in6_addr *dst, *pdst, *psrc, src; struct ifnet *ifp; - struct nd_ifinfo *ndi; + struct in6_ifextra *ndi; int do_switch, send_ns; long delay; @@ -790,7 +774,7 @@ nd6_llinfo_timer(void *arg) return; } NET_EPOCH_ENTER(et); - ndi = ND_IFINFO(ifp); + ndi = ifp->if_inet6; send_ns = 0; dst = &ln->r_l3addr.addr6; pdst = dst; @@ -892,7 +876,7 @@ nd6_llinfo_timer(void *arg) /* FALLTHROUGH */ case ND6_LLINFO_DELAY: - if (ndi && (ndi->flags & ND6_IFF_PERFORMNUD) != 0) { + if ((ndi->nd_flags & ND6_IFF_PERFORMNUD) != 0) { /* We need NUD */ ln->la_asked = 1; nd6_llinfo_setstate(ln, ND6_LLINFO_PROBE); @@ -916,7 +900,8 @@ done: if (ln != NULL) ND6_RUNLOCK(); if (send_ns != 0) { - nd6_llinfo_settimer_locked(ln, (long)ndi->retrans * hz / 1000); + nd6_llinfo_settimer_locked(ln, + (long)ndi->nd_retrans * hz / 1000); psrc = nd6_llinfo_get_holdsrc(ln, &src); LLE_FREE_LOCKED(ln); ln = NULL; @@ -1027,10 +1012,10 @@ nd6_timer(void *arg) * mark the address as tentative for future DAD. */ ifp = ia6->ia_ifp; - if ((ND_IFINFO(ifp)->flags & ND6_IFF_NO_DAD) == 0 && + if ((ifp->if_inet6->nd_flags & ND6_IFF_NO_DAD) == 0 && ((ifp->if_flags & IFF_UP) == 0 || (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || - (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) != 0)){ + (ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED))){ ia6->ia6_flags &= ~IN6_IFF_DUPLICATED; ia6->ia6_flags |= IN6_IFF_TENTATIVE; } @@ -1198,7 +1183,7 @@ nd6_purge(struct ifnet *ifp) if (V_nd6_defifindex == ifp->if_index) nd6_setdefaultiface(0); - if (ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV) { + if (ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV) { /* Refresh default router list. */ defrouter_select_fib(ifp->if_fib); } @@ -1324,7 +1309,7 @@ nd6_is_new_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp) * If the default router list is empty, all addresses are regarded * as on-link, and thus, as a neighbor. */ - if (ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV && + if (ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV && nd6_defrouter_list_empty() && V_nd6_defifindex == ifp->if_index) { return (1); @@ -1448,7 +1433,7 @@ nd6_free(struct llentry **lnp, int gc) KASSERT((ln->la_flags & LLE_CHILD) == 0, ("child lle")); ifp = lltable_get_ifp(ln->lle_tbl); - if ((ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV) != 0) + if ((ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV) != 0) dr = defrouter_lookup_locked(&ln->r_l3addr.addr6, ifp); else dr = NULL; @@ -1465,7 +1450,7 @@ nd6_free(struct llentry **lnp, int gc) /* cancel timer */ nd6_llinfo_settimer_locked(ln, -1); - if (ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV) { + if (ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV) { if (dr != NULL && dr->expire && ln->ln_state == ND6_LLINFO_STALE && gc) { /* @@ -1640,19 +1625,30 @@ nd6_subscription_cb(struct rib_head *rnh, struct rib_cmd_info *rc, void *arg) int nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) { + struct epoch_tracker et; struct in6_ndireq *ndi = (struct in6_ndireq *)data; struct in6_nbrinfo *nbi = (struct in6_nbrinfo *)data; struct in6_ndifreq *ndif = (struct in6_ndifreq *)data; - struct epoch_tracker et; + struct in6_ifextra *ext = ifp->if_inet6; int error = 0; - /* XXXGL: ??? */ - if (ifp->if_inet6 == NULL) + /* XXXGL: safety against IFT_PFSYNC & IFT_PFLOG */ + if (ext == NULL) return (EPFNOSUPPORT); #define ND ndi->ndi switch (cmd) { case SIOCGIFINFO_IN6: - ND = *ND_IFINFO(ifp); + ND = (struct nd_ifinfo){ + .linkmtu = ext->nd_linkmtu, + .maxmtu = ext->nd_maxmtu, + .basereachable = ext->nd_basereachable, + .reachable = ext->nd_reachable, + .retrans = ext->nd_retrans, + .flags = ext->nd_flags, + .recalctm = ext->nd_recalc_timer, + .chlim = ext->nd_curhoplimit, + .initialized = 1, + }; break; case SIOCSIFINFO_IN6: /* @@ -1662,32 +1658,32 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) /* 0 means 'unspecified' */ if (ND.linkmtu != 0) { if (ND.linkmtu < IPV6_MMTU || - ND.linkmtu > IN6_LINKMTU(ifp)) { + ND.linkmtu > in6_ifmtu(ifp)) { error = EINVAL; break; } - ND_IFINFO(ifp)->linkmtu = ND.linkmtu; + ext->nd_linkmtu = ND.linkmtu; } if (ND.basereachable != 0) { - int obasereachable = ND_IFINFO(ifp)->basereachable; + uint32_t obasereachable = ext->nd_basereachable; - ND_IFINFO(ifp)->basereachable = ND.basereachable; + ext->nd_basereachable = ND.basereachable; if (ND.basereachable != obasereachable) - ND_IFINFO(ifp)->reachable = + ext->nd_reachable = ND_COMPUTE_RTIME(ND.basereachable); } if (ND.retrans != 0) - ND_IFINFO(ifp)->retrans = ND.retrans; + ext->nd_retrans = ND.retrans; if (ND.chlim != 0) - ND_IFINFO(ifp)->chlim = ND.chlim; + ext->nd_curhoplimit = ND.chlim; /* FALLTHROUGH */ case SIOCSIFINFO_FLAGS: { struct ifaddr *ifa; struct in6_ifaddr *ia; - if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) && + if ((ext->nd_flags & ND6_IFF_IFDISABLED) && !(ND.flags & ND6_IFF_IFDISABLED)) { /* ifdisabled 1->0 transision */ @@ -1715,18 +1711,18 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) " with a link-local address marked" " duplicate.\n"); } else { - ND_IFINFO(ifp)->flags &= ~ND6_IFF_IFDISABLED; + ext->nd_flags &= ~ND6_IFF_IFDISABLED; if (ifp->if_flags & IFF_UP) in6_if_up(ifp); } - } else if (!(ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) && + } else if (!(ext->nd_flags & ND6_IFF_IFDISABLED) && (ND.flags & ND6_IFF_IFDISABLED)) { /* ifdisabled 0->1 transision */ /* Mark all IPv6 address as tentative. */ - ND_IFINFO(ifp)->flags |= ND6_IFF_IFDISABLED; + ext->nd_flags |= ND6_IFF_IFDISABLED; if (V_ip6_dad_count > 0 && - (ND_IFINFO(ifp)->flags & ND6_IFF_NO_DAD) == 0) { + (ext->nd_flags & ND6_IFF_NO_DAD) == 0) { NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { @@ -1741,11 +1737,11 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) } if (ND.flags & ND6_IFF_AUTO_LINKLOCAL) { - if (!(ND_IFINFO(ifp)->flags & ND6_IFF_AUTO_LINKLOCAL)) { + if (!(ext->nd_flags & ND6_IFF_AUTO_LINKLOCAL)) { /* auto_linklocal 0->1 transision */ /* If no link-local address on ifp, configure */ - ND_IFINFO(ifp)->flags |= ND6_IFF_AUTO_LINKLOCAL; + ext->nd_flags |= ND6_IFF_AUTO_LINKLOCAL; in6_ifattach(ifp, NULL); } else if (!(ND.flags & ND6_IFF_IFDISABLED) && ifp->if_flags & IFF_UP) { @@ -1771,7 +1767,7 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) in6_ifattach(ifp, NULL); } } - ND_IFINFO(ifp)->flags = ND.flags; + ext->nd_flags = ND.flags; break; } #undef ND @@ -2108,7 +2104,7 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr, * cases for safety. */ if ((do_update || is_newentry) && router && - ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV) { + ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV) { /* * guaranteed recursion */ @@ -2121,26 +2117,26 @@ nd6_slowtimo(void *arg) { struct epoch_tracker et; CURVNET_SET((struct vnet *) arg); - struct nd_ifinfo *nd6if; + struct in6_ifextra *nd6if; struct ifnet *ifp; callout_reset(&V_nd6_slowtimo_ch, ND6_SLOWTIMER_INTERVAL * hz, nd6_slowtimo, curvnet); NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { - if (ifp->if_inet6 == NULL) + if ((nd6if = ifp->if_inet6) == NULL) continue; - nd6if = ND_IFINFO(ifp); - if (nd6if->basereachable && /* already initialized */ - (nd6if->recalctm -= ND6_SLOWTIMER_INTERVAL) <= 0) { + if (nd6if->nd_basereachable && /* already initialized */ + (nd6if->nd_recalc_timer -= ND6_SLOWTIMER_INTERVAL) <= 0) { /* * Since reachable time rarely changes by router * advertisements, we SHOULD insure that a new random * value gets recomputed at least once every few hours. * (RFC 2461, 6.3.4) */ - nd6if->recalctm = V_nd6_recalc_reachtm_interval; - nd6if->reachable = ND_COMPUTE_RTIME(nd6if->basereachable); + nd6if->nd_recalc_timer = V_nd6_recalc_reachtm_interval; + nd6if->nd_reachable = + ND_COMPUTE_RTIME(nd6if->nd_basereachable); } } NET_EPOCH_EXIT(et); @@ -2248,7 +2244,7 @@ nd6_resolve(struct ifnet *ifp, int gw_flags, struct mbuf *m, dst6 = (const struct sockaddr_in6 *)sa_dst; /* discard the packet if IPv6 operation is disabled on the interface */ - if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED)) { + if ((ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED)) { m_freem(m); return (ENETDOWN); /* better error? */ } diff --git a/sys/netinet6/nd6.h b/sys/netinet6/nd6.h index 0f61e167f480..a22a0e24735b 100644 --- a/sys/netinet6/nd6.h +++ b/sys/netinet6/nd6.h @@ -62,22 +62,6 @@ struct llentry; #define ND6_IS_LLINFO_PROBREACH(n) ((n)->ln_state > ND6_LLINFO_INCOMPLETE) #define ND6_LLINFO_PERMANENT(n) (((n)->la_expire == 0) && ((n)->ln_state > ND6_LLINFO_INCOMPLETE)) -struct nd_ifinfo { - u_int32_t linkmtu; /* LinkMTU */ - u_int32_t maxmtu; /* Upper bound of LinkMTU */ - u_int32_t basereachable; /* BaseReachableTime */ - u_int32_t reachable; /* Reachable Time */ - u_int32_t retrans; /* Retrans Timer */ - u_int32_t flags; /* Flags */ - int recalctm; /* BaseReacable re-calculation timer */ - u_int8_t chlim; /* CurHopLimit */ - u_int8_t initialized; /* Flag to see the entry is initialized */ - /* the following 3 members are for privacy extension for addrconf */ - u_int8_t randomseed0[8]; /* upper 64 bits of MD5 digest */ - u_int8_t randomseed1[8]; /* lower 64 bits (usually the EUI64 IFID) */ - u_int8_t randomid[8]; /* current random ID */ -}; - #define ND6_IFF_PERFORMNUD 0x1 #define ND6_IFF_ACCEPT_RTADV 0x2 #define ND6_IFF_PREFER_SOURCE 0x4 /* Not used in FreeBSD. */ @@ -97,15 +81,6 @@ struct nd_ifinfo { #define ND6_IFF_IPV6_ONLY_MASK (ND6_IFF_IPV6_ONLY|ND6_IFF_IPV6_ONLY_MANUAL) #endif -#ifdef _KERNEL -#define ND_IFINFO(ifp) ((if_getinet6(ifp))->nd_ifinfo) -#define IN6_LINKMTU(ifp) \ - ((ND_IFINFO(ifp)->linkmtu && ND_IFINFO(ifp)->linkmtu < (ifp)->if_mtu) \ - ? ND_IFINFO(ifp)->linkmtu \ - : ((ND_IFINFO(ifp)->maxmtu && ND_IFINFO(ifp)->maxmtu < (ifp)->if_mtu) \ - ? ND_IFINFO(ifp)->maxmtu : (ifp)->if_mtu)) -#endif - struct in6_nbrinfo { char ifname[IFNAMSIZ]; /* if name, e.g. "en0" */ struct in6_addr addr; /* IPv6 address of the neighbor */ @@ -139,9 +114,29 @@ struct in6_prefix { /* struct sockaddr_in6 advrtr[] */ }; -struct in6_ndireq { +struct in6_ndireq { char ifname[IFNAMSIZ]; - struct nd_ifinfo ndi; + struct nd_ifinfo { + uint32_t linkmtu; /* LinkMTU */ + uint32_t maxmtu; /* Upper bound of LinkMTU */ + uint32_t basereachable; /* BaseReachableTime */ + uint32_t reachable; /* Reachable Time */ + uint32_t retrans; /* Retrans Timer */ + uint32_t flags; /* Flags */ + int recalctm; /* BaseReacable re-calculation timer */ + uint8_t chlim; /* CurHopLimit */ + /* + * The below members are not used. They came from KAME and + * are hanging around to preserve ABI compatibility of the + * SIOCGIFINFO_IN6 ioctl. + * The original comment documented the random* members as a + * privacy extension for addrconf. + */ + uint8_t initialized; /* compat: always 1 */ + uint8_t randomseed0[8]; /* upper 64 bits of MD5 digest */ + uint8_t randomseed1[8]; /* lower 64 bits (the EUI64 IFID?) */ + uint8_t randomid[8]; /* current random ID */ + } ndi; }; struct in6_ndifreq { @@ -236,10 +231,6 @@ struct nd_pfxrouter { struct nd_defrouter *router; }; -#ifdef MALLOC_DECLARE -MALLOC_DECLARE(M_IP6NDP); -#endif - /* nd6.c */ VNET_DECLARE(int, nd6_mmaxtries); VNET_DECLARE(struct nd_prhead, nd_prefix); @@ -332,8 +323,8 @@ void nd6_init(void); #ifdef VIMAGE void nd6_destroy(void); #endif -struct nd_ifinfo *nd6_ifattach(struct ifnet *); -void nd6_ifdetach(struct ifnet *, struct nd_ifinfo *); +void nd6_ifattach(struct ifnet *); +void nd6_ifdetach(struct ifnet *); int nd6_is_addr_neighbor(const struct sockaddr_in6 *, struct ifnet *); void nd6_option_init(void *, int, union nd_opts *); struct nd_opt_hdr *nd6_option(union nd_opts *); diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c index aa7cb3b41973..4da62575eaac 100644 --- a/sys/netinet6/nd6_nbr.c +++ b/sys/netinet6/nd6_nbr.c @@ -80,6 +80,8 @@ #define SDL(s) ((struct sockaddr_dl *)s) +MALLOC_DECLARE(M_IP6NDP); + struct dadq; static struct dadq *nd6_dad_find(struct ifaddr *, struct nd_opt_nonce *); static void nd6_dad_add(struct dadq *dp); @@ -173,7 +175,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len) goto bad; rflag = (V_ip6_forwarding) ? ND_NA_FLAG_ROUTER : 0; - if (ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV && V_ip6_norbit_raif) + if (ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV && V_ip6_norbit_raif) rflag = 0; if (IN6_IS_ADDR_UNSPECIFIED(&saddr6)) { @@ -910,7 +912,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) nd6_ifp = lltable_get_ifp(ln->lle_tbl); if (!defrouter_remove(&ln->r_l3addr.addr6, nd6_ifp) && - (ND_IFINFO(nd6_ifp)->flags & + (nd6_ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV) != 0) /* * Even if the neighbor is not in the default @@ -1281,13 +1283,13 @@ nd6_dad_start(struct ifaddr *ifa, int delay) */ if ((ia->ia6_flags & IN6_IFF_ANYCAST) != 0 || V_ip6_dad_count == 0 || - (ND_IFINFO(ifa->ifa_ifp)->flags & ND6_IFF_NO_DAD) != 0) { + (ifa->ifa_ifp->if_inet6->nd_flags & ND6_IFF_NO_DAD) != 0) { ia->ia6_flags &= ~IN6_IFF_TENTATIVE; return; } if ((ifa->ifa_ifp->if_flags & IFF_UP) == 0 || (ifa->ifa_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || - (ND_IFINFO(ifa->ifa_ifp)->flags & ND6_IFF_IFDISABLED) != 0) + (ifa->ifa_ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED) != 0) return; DADQ_WLOCK(); @@ -1377,7 +1379,7 @@ nd6_dad_timer(void *arg) KASSERT(ia != NULL, ("DAD entry %p with no address", dp)); NET_EPOCH_ENTER(et); - if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) { + if (ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED) { /* Do not need DAD for ifdisabled interface. */ log(LOG_ERR, "nd6_dad_timer: cancel DAD on %s because of " "ND6_IFF_IFDISABLED.\n", ifp->if_xname); @@ -1414,7 +1416,7 @@ nd6_dad_timer(void *arg) * We have more NS to go. Send NS packet for DAD. */ nd6_dad_starttimer(dp, - (long)ND_IFINFO(ifa->ifa_ifp)->retrans * hz / 1000); + (long)ifa->ifa_ifp->if_inet6->nd_retrans * hz / 1000); nd6_dad_ns_output(dp); goto done; } else { @@ -1446,7 +1448,7 @@ nd6_dad_timer(void *arg) dp->dad_count = dp->dad_ns_ocount + V_nd6_mmaxtries - 1; nd6_dad_starttimer(dp, - (long)ND_IFINFO(ifa->ifa_ifp)->retrans * hz / 1000); + (long)ifa->ifa_ifp->if_inet6->nd_retrans * hz / 1000); nd6_dad_ns_output(dp); goto done; } else { @@ -1458,9 +1460,9 @@ nd6_dad_timer(void *arg) * * Reset DAD failures counter if using stable addresses. */ - if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) == 0) { + if ((ifp->if_inet6->nd_flags & ND6_IFF_IFDISABLED) == 0) { ia->ia6_flags &= ~IN6_IFF_TENTATIVE; - if ((ND_IFINFO(ifp)->flags & ND6_IFF_STABLEADDR) && !(ia->ia6_flags & IN6_IFF_TEMPORARY)) + if ((ifp->if_inet6->nd_flags & ND6_IFF_STABLEADDR) && !(ia->ia6_flags & IN6_IFF_TEMPORARY)) atomic_store_int(&DAD_FAILURES(ifp), 0); } @@ -1509,7 +1511,7 @@ nd6_dad_duplicated(struct ifaddr *ifa, struct dadq *dp) * For RFC 7217 stable addresses, increment failure counter here if we still have retries. * More addresses will be generated as long as retries are not exhausted. */ - if ((ND_IFINFO(ifp)->flags & ND6_IFF_STABLEADDR) && !(ia->ia6_flags & IN6_IFF_TEMPORARY)) { + if ((ifp->if_inet6->nd_flags & ND6_IFF_STABLEADDR) && !(ia->ia6_flags & IN6_IFF_TEMPORARY)) { u_int dad_failures = atomic_load_int(&DAD_FAILURES(ifp)); if (dad_failures <= V_ip6_stableaddr_maxretries) { @@ -1547,7 +1549,7 @@ nd6_dad_duplicated(struct ifaddr *ifa, struct dadq *dp) in6 = ia->ia_addr.sin6_addr; if (in6_get_hw_ifid(ifp, &in6) == 0 && IN6_ARE_ADDR_EQUAL(&ia->ia_addr.sin6_addr, &in6)) { - ND_IFINFO(ifp)->flags |= ND6_IFF_IFDISABLED; + ifp->if_inet6->nd_flags |= ND6_IFF_IFDISABLED; log(LOG_ERR, "%s: possible hardware address " "duplication detected, disable IPv6\n", if_name(ifp)); diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c index 821d6b27bc43..0ca97125110c 100644 --- a/sys/netinet6/nd6_rtr.c +++ b/sys/netinet6/nd6_rtr.c @@ -75,6 +75,8 @@ #include <machine/atomic.h> +MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery"); + static struct nd_defrouter *defrtrlist_update(struct nd_defrouter *); static int prelist_update(struct nd_prefixctl *, struct nd_defrouter *, struct mbuf *, int); @@ -175,7 +177,7 @@ nd6_rs_input(struct mbuf *m, int off, int icmp6len) * Accept RS only when V_ip6_forwarding=1 and the interface has * no ND6_IFF_ACCEPT_RTADV. */ - if (!V_ip6_forwarding || ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV) + if (!V_ip6_forwarding || ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV) goto freeit; /* RFC 6980: Nodes MUST silently ignore fragments */ @@ -280,7 +282,7 @@ defrtr_ipv6_only_ifp(struct ifnet *ifp) ND6_RUNLOCK(); IF_ADDR_WLOCK(ifp); - ipv6_only_old = ND_IFINFO(ifp)->flags & ND6_IFF_IPV6_ONLY; + ipv6_only_old = ifp->if_inet6->nd_flags & ND6_IFF_IPV6_ONLY; IF_ADDR_WUNLOCK(ifp); /* If nothing changed, we have an early exit. */ @@ -317,9 +319,9 @@ defrtr_ipv6_only_ifp(struct ifnet *ifp) IF_ADDR_WLOCK(ifp); if (ipv6_only) - ND_IFINFO(ifp)->flags |= ND6_IFF_IPV6_ONLY; + ifp->if_inet6->nd_flags |= ND6_IFF_IPV6_ONLY; else - ND_IFINFO(ifp)->flags &= ~ND6_IFF_IPV6_ONLY; + ifp->if_inet6->nd_flags &= ~ND6_IFF_IPV6_ONLY; IF_ADDR_WUNLOCK(ifp); #ifdef notyet @@ -332,7 +334,7 @@ defrtr_ipv6_only_ipf_down(struct ifnet *ifp) { IF_ADDR_WLOCK(ifp); - ND_IFINFO(ifp)->flags &= ~ND6_IFF_IPV6_ONLY; + ifp->if_inet6->nd_flags &= ~ND6_IFF_IPV6_ONLY; IF_ADDR_WUNLOCK(ifp); } #endif /* EXPERIMENTAL */ @@ -364,7 +366,7 @@ void nd6_ra_input(struct mbuf *m, int off, int icmp6len) { struct ifnet *ifp; - struct nd_ifinfo *ndi; + struct in6_ifextra *ndi; struct ip6_hdr *ip6; struct nd_router_advert *nd_ra; struct in6_addr saddr6; @@ -378,8 +380,8 @@ nd6_ra_input(struct mbuf *m, int off, int icmp6len) * ND6_IFF_ACCEPT_RTADV is on the receiving interface. */ ifp = m->m_pkthdr.rcvif; - ndi = ND_IFINFO(ifp); - if (!(ndi->flags & ND6_IFF_ACCEPT_RTADV)) + ndi = ifp->if_inet6; + if (!(ndi->nd_flags & ND6_IFF_ACCEPT_RTADV)) goto freeit; /* RFC 6980: Nodes MUST silently ignore fragments */ @@ -441,7 +443,7 @@ nd6_ra_input(struct mbuf *m, int off, int icmp6len) * ND6_IFF_NO_RADR enabled on the receiving interface or * (ip6.forwarding == 1 && ip6.rfc6204w3 != 1). */ - if (ndi->flags & ND6_IFF_NO_RADR) + if (ndi->nd_flags & ND6_IFF_NO_RADR) dr0.rtlifetime = 0; else if (V_ip6_forwarding && !V_ip6_rfc6204w3) dr0.rtlifetime = 0; @@ -453,22 +455,24 @@ nd6_ra_input(struct mbuf *m, int off, int icmp6len) if (advreachable) { advreachable = ntohl(advreachable); if (advreachable <= MAX_REACHABLE_TIME && - ndi->basereachable != advreachable) { - ndi->basereachable = advreachable; - ndi->reachable = ND_COMPUTE_RTIME(ndi->basereachable); - ndi->recalctm = V_nd6_recalc_reachtm_interval; /* reset */ + ndi->nd_basereachable != advreachable) { + ndi->nd_basereachable = advreachable; + ndi->nd_reachable = + ND_COMPUTE_RTIME(ndi->nd_basereachable); + ndi->nd_recalc_timer = V_nd6_recalc_reachtm_interval; } } if (nd_ra->nd_ra_retransmit) - ndi->retrans = ntohl(nd_ra->nd_ra_retransmit); + ndi->nd_retrans = ntohl(nd_ra->nd_ra_retransmit); if (nd_ra->nd_ra_curhoplimit) { - if (ndi->chlim < nd_ra->nd_ra_curhoplimit) - ndi->chlim = nd_ra->nd_ra_curhoplimit; - else if (ndi->chlim != nd_ra->nd_ra_curhoplimit) { + if (ndi->nd_curhoplimit < nd_ra->nd_ra_curhoplimit) + ndi->nd_curhoplimit = nd_ra->nd_ra_curhoplimit; + else if (ndi->nd_curhoplimit != nd_ra->nd_ra_curhoplimit) { log(LOG_ERR, "RA with a lower CurHopLimit sent from " "%s on %s (current = %d, received = %d). " "Ignored.\n", ip6_sprintf(ip6bufs, &ip6->ip6_src), - if_name(ifp), ndi->chlim, nd_ra->nd_ra_curhoplimit); + if_name(ifp), ndi->nd_curhoplimit, + nd_ra->nd_ra_curhoplimit); } } dr = defrtrlist_update(&dr0); @@ -557,11 +561,11 @@ nd6_ra_input(struct mbuf *m, int off, int icmp6len) } /* upper bound */ - maxmtu = (ndi->maxmtu && ndi->maxmtu < ifp->if_mtu) - ? ndi->maxmtu : ifp->if_mtu; + maxmtu = (ndi->nd_maxmtu && ndi->nd_maxmtu < ifp->if_mtu) + ? ndi->nd_maxmtu : ifp->if_mtu; if (mtu <= maxmtu) { - if (ndi->linkmtu != mtu) { - ndi->linkmtu = mtu; + if (ndi->nd_linkmtu != mtu) { + ndi->nd_linkmtu = mtu; rt_updatemtu(ifp); } } else { @@ -751,7 +755,7 @@ defrouter_del(struct nd_defrouter *dr) * Flush all the routing table entries that use the router * as a next hop. */ - if (ND_IFINFO(dr->ifp)->flags & ND6_IFF_ACCEPT_RTADV) + if (dr->ifp->if_inet6->nd_flags & ND6_IFF_ACCEPT_RTADV) rt6_flush(&dr->rtaddr, dr->ifp); #ifdef EXPERIMENTAL @@ -1202,7 +1206,7 @@ in6_ifadd(struct nd_prefixctl *pr, int mcast) /* make ifaddr */ in6_prepare_ifra(&ifra, &pr->ndpr_prefix.sin6_addr, &mask); - if (ND_IFINFO(ifp)->flags & ND6_IFF_STABLEADDR) { + if (ifp->if_inet6->nd_flags & ND6_IFF_STABLEADDR) { memcpy(&newaddr, &pr->ndpr_prefix.sin6_addr, sizeof(pr->ndpr_prefix.sin6_addr)); if(!in6_get_stableifid(ifp, &newaddr, prefixlen)) @@ -1693,7 +1697,7 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, * if stable addresses (RFC 7217) are enabled, mark that a temporary address has been found * to avoid generating uneeded extra ones. */ - if (ND_IFINFO(ifp)->flags & ND6_IFF_STABLEADDR) + if (ifp->if_inet6->nd_flags & ND6_IFF_STABLEADDR) has_temporary = true; if (V_ip6_temp_valid_lifetime > @@ -1735,7 +1739,7 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, * between here and when a new address is generated, but this will cause that generation * to fail and no further retries should happen. */ - if (ND_IFINFO(ifp)->flags & ND6_IFF_STABLEADDR && + if (ifp->if_inet6->nd_flags & ND6_IFF_STABLEADDR && atomic_load_int(&DAD_FAILURES(ifp)) <= V_ip6_stableaddr_maxretries && ifa6->ia6_flags & (IN6_IFF_DUPLICATED | IN6_IFF_TEMPORARY)) continue; diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c index c90a1213bd66..7deb605c07a2 100644 --- a/sys/netinet6/raw_ip6.c +++ b/sys/netinet6/raw_ip6.c @@ -604,6 +604,9 @@ rip6_ctloutput(struct socket *so, struct sockopt *sopt) case MRT6_ADD_MFC: case MRT6_DEL_MFC: case MRT6_PIM: + error = priv_check(curthread, PRIV_NETINET_MROUTE); + if (error != 0) + return (error); if (inp->inp_ip_p != IPPROTO_ICMPV6) return (EOPNOTSUPP); error = ip6_mrouter_get ? ip6_mrouter_get(so, sopt) : @@ -627,6 +630,9 @@ rip6_ctloutput(struct socket *so, struct sockopt *sopt) case MRT6_ADD_MFC: case MRT6_DEL_MFC: case MRT6_PIM: + error = priv_check(curthread, PRIV_NETINET_MROUTE); + if (error != 0) + return (error); if (inp->inp_ip_p != IPPROTO_ICMPV6) return (EOPNOTSUPP); error = ip6_mrouter_set ? ip6_mrouter_set(so, sopt) : diff --git a/sys/netinet6/scope6.c b/sys/netinet6/scope6.c index 6862c75fb5e7..44a9d976d5fe 100644 --- a/sys/netinet6/scope6.c +++ b/sys/netinet6/scope6.c @@ -73,10 +73,11 @@ static struct mtx scope6_lock; VNET_DEFINE_STATIC(struct scope6_id, sid_default); #define V_sid_default VNET(sid_default) -#define SID(ifp) ((ifp)->if_inet6->scope6_id) +#define SID(ifp) (&(ifp)->if_inet6->scope6_id) static int scope6_get(struct ifnet *, struct scope6_id *); static int scope6_set(struct ifnet *, struct scope6_id *); +static int scope6_get_default(struct scope6_id *); void scope6_init(void) @@ -90,26 +91,18 @@ scope6_init(void) SCOPE6_LOCK_INIT(); } -struct scope6_id * +void scope6_ifattach(struct ifnet *ifp) { - struct scope6_id *sid; + struct scope6_id *sid = &ifp->if_inet6->scope6_id; - sid = malloc(sizeof(*sid), M_IFADDR, M_WAITOK | M_ZERO); /* * XXX: IPV6_ADDR_SCOPE_xxx macros are not standard. * Should we rather hardcode here? */ + bzero(sid, sizeof(*sid)); sid->s6id_list[IPV6_ADDR_SCOPE_INTFACELOCAL] = ifp->if_index; sid->s6id_list[IPV6_ADDR_SCOPE_LINKLOCAL] = ifp->if_index; - return (sid); -} - -void -scope6_ifdetach(struct scope6_id *sid) -{ - - free(sid, M_IFADDR); } int @@ -280,7 +273,7 @@ scope6_setdefault(struct ifnet *ifp) SCOPE6_UNLOCK(); } -int +static int scope6_get_default(struct scope6_id *idlist) { diff --git a/sys/netinet6/scope6_var.h b/sys/netinet6/scope6_var.h index f914d5981bb0..7832444e0658 100644 --- a/sys/netinet6/scope6_var.h +++ b/sys/netinet6/scope6_var.h @@ -37,21 +37,10 @@ #ifdef _KERNEL #include <net/vnet.h> -#define IPV6_ADDR_SCOPES_COUNT 16 -struct scope6_id { - /* - * 16 is correspondent to 4bit multicast scope field. - * i.e. from node-local to global with some reserved/unassigned types. - */ - uint32_t s6id_list[IPV6_ADDR_SCOPES_COUNT]; -}; - void scope6_init(void); -struct scope6_id *scope6_ifattach(struct ifnet *); -void scope6_ifdetach(struct scope6_id *); +void scope6_ifattach(struct ifnet *); int scope6_ioctl(u_long cmd, caddr_t data, struct ifnet *); void scope6_setdefault(struct ifnet *); -int scope6_get_default(struct scope6_id *); u_int32_t scope6_addr2default(struct in6_addr *); int sa6_embedscope(struct sockaddr_in6 *, int); int sa6_recoverscope(struct sockaddr_in6 *); diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c index ca7c95497510..1d1dcb75a1df 100644 --- a/sys/netinet6/udp6_usrreq.c +++ b/sys/netinet6/udp6_usrreq.c @@ -703,11 +703,6 @@ udp6_send(struct socket *so, int flags_arg, struct mbuf *m, sin6 = (struct sockaddr_in6 *)addr6; - /* - * In contrast to IPv4 we do not validate the max. packet length - * here due to IPv6 Jumbograms (RFC2675). - */ - scope_ambiguous = 0; if (sin6) { /* Protect *addr6 from overwrites. */ @@ -865,10 +860,21 @@ udp6_send(struct socket *so, int flags_arg, struct mbuf *m, fport = inp->inp_fport; } + + /* + * We do not support IPv6 Jumbograms (RFC2675), so validate the payload + * length fits in a normal gram. + */ ulen = m->m_pkthdr.len; plen = sizeof(struct udphdr) + ulen; hlen = sizeof(struct ip6_hdr); + if (plen > IPV6_MAXPAYLOAD) { + m_freem(control); + m_freem(m); + return (EMSGSIZE); + } + /* * Calculate data length and get a mbuf for UDP, IP6, and possible * link-layer headers. Immediate slide the data pointer back forward @@ -903,10 +909,10 @@ udp6_send(struct socket *so, int flags_arg, struct mbuf *m, * the entire UDPLite packet is covered by the checksum. */ cscov_partial = (cscov == 0) ? 0 : 1; - } else if (plen <= 0xffff) + } else { + MPASS(plen <= IPV6_MAXPAYLOAD); udp6->uh_ulen = htons((u_short)plen); - else - udp6->uh_ulen = 0; + } udp6->uh_sum = 0; ip6 = mtod(m, struct ip6_hdr *); diff --git a/sys/netlink/ktest_netlink_message_writer.c b/sys/netlink/ktest_netlink_message_writer.c index 805f52197f69..0e8b962bbd34 100644 --- a/sys/netlink/ktest_netlink_message_writer.c +++ b/sys/netlink/ktest_netlink_message_writer.c @@ -37,8 +37,6 @@ #define KTEST_CALLER #include <netlink/ktest_netlink_message_writer.h> -#ifdef INVARIANTS - struct test_nlbuf_attrs { uint32_t size; uint32_t expected_avail; @@ -98,16 +96,13 @@ test_nlbuf_writer_allocation(struct ktest_test_context *ctx) return (0); } -#endif static const struct ktest_test_info tests[] = { -#ifdef INVARIANTS { .name = "test_nlbuf_writer_allocation", .desc = "test different buffer sizes in the netlink writer", .func = &test_nlbuf_writer_allocation, .parse = &test_nlbuf_parser, }, -#endif }; KTEST_MODULE_DECLARE(ktest_netlink_message_writer, tests); diff --git a/sys/netlink/ktest_netlink_message_writer.h b/sys/netlink/ktest_netlink_message_writer.h index 447593e0e700..1abf0d48de95 100644 --- a/sys/netlink/ktest_netlink_message_writer.h +++ b/sys/netlink/ktest_netlink_message_writer.h @@ -28,7 +28,7 @@ #ifndef _NETLINK_KTEST_NETLINK_MESSAGE_WRITER_H_ #define _NETLINK_KTEST_NETLINK_MESSAGE_WRITER_H_ -#if defined(_KERNEL) && defined(INVARIANTS) +#if defined(_KERNEL) bool nlmsg_get_buf_wrapper(struct nl_writer *nw, size_t size, bool waitok); diff --git a/sys/netlink/netlink_domain.c b/sys/netlink/netlink_domain.c index 74b46114716e..e906e0d635af 100644 --- a/sys/netlink/netlink_domain.c +++ b/sys/netlink/netlink_domain.c @@ -330,14 +330,17 @@ nl_attach(struct socket *so, int proto, struct thread *td) so, is_linux ? "(linux) " : "", curproc->p_pid, nl_get_proto_name(proto)); - nlp = malloc(sizeof(struct nlpcb), M_PCB, M_WAITOK | M_ZERO); + mtx_init(&so->so_snd_mtx, "netlink so_snd", NULL, MTX_DEF); + mtx_init(&so->so_rcv_mtx, "netlink so_rcv", NULL, MTX_DEF); error = soreserve(so, nl_sendspace, nl_recvspace); if (error != 0) { - free(nlp, M_PCB); + mtx_destroy(&so->so_snd_mtx); + mtx_destroy(&so->so_rcv_mtx); return (error); } TAILQ_INIT(&so->so_rcv.nl_queue); TAILQ_INIT(&so->so_snd.nl_queue); + nlp = malloc(sizeof(struct nlpcb), M_PCB, M_WAITOK | M_ZERO); so->so_pcb = nlp; nlp->nl_socket = so; nlp->nl_proto = proto; @@ -517,6 +520,9 @@ nl_close(struct socket *so) nl_buf_free(nb); } + mtx_destroy(&so->so_snd_mtx); + mtx_destroy(&so->so_rcv_mtx); + NL_LOG(LOG_DEBUG3, "socket %p, detached", so); /* XXX: is delayed free needed? */ diff --git a/sys/netlink/route/iface.c b/sys/netlink/route/iface.c index e9c053015fad..3e30d74a3793 100644 --- a/sys/netlink/route/iface.c +++ b/sys/netlink/route/iface.c @@ -36,6 +36,7 @@ #include <sys/socket.h> #include <sys/sockio.h> #include <sys/syslog.h> +#include <sys/proc.h> #include <net/if.h> #include <net/if_dl.h> @@ -675,6 +676,8 @@ static int rtnl_handle_newlink(struct nlmsghdr *hdr, struct nlpcb *nlp, struct nl_pstate *npt) { struct nlattr_bmask bm; + struct thread *td = curthread; + struct ucred *cred; int error; struct nl_parsed_link attrs = {}; @@ -683,10 +686,16 @@ rtnl_handle_newlink(struct nlmsghdr *hdr, struct nlpcb *nlp, struct nl_pstate *n return (error); nl_get_attrs_bmask_nlmsg(hdr, &ifmsg_parser, &bm); + /* XXX: temporary patch until the D39180 review lands */ + cred = td->td_ucred; + td->td_ucred = nlp_get_cred(nlp); if (hdr->nlmsg_flags & NLM_F_CREATE) - return (create_link(hdr, &attrs, &bm, nlp, npt)); + error = create_link(hdr, &attrs, &bm, nlp, npt); else - return (modify_link(hdr, &attrs, &bm, nlp, npt)); + error = modify_link(hdr, &attrs, &bm, nlp, npt); + td->td_ucred = cred; + + return (error); } static void diff --git a/sys/netpfil/ipfilter/netinet/fil.c b/sys/netpfil/ipfilter/netinet/fil.c index f8b8ffec7bdd..24ac91c69fc8 100644 --- a/sys/netpfil/ipfilter/netinet/fil.c +++ b/sys/netpfil/ipfilter/netinet/fil.c @@ -3503,7 +3503,7 @@ ipf_group_add(ipf_main_softc_t *softc, char *group, void *head, u_32_t flags, fg->fg_head = head; fg->fg_start = NULL; fg->fg_next = *fgp; - bcopy(group, fg->fg_name, strlen(group) + 1); + bcopy(group, fg->fg_name, strnlen(group, FR_GROUPLEN) + 1); fg->fg_flags = gflags; fg->fg_ref = 1; fg->fg_set = &softc->ipf_groups[unit][set]; diff --git a/sys/netpfil/ipfw/ip_fw_dynamic.c b/sys/netpfil/ipfw/ip_fw_dynamic.c index 99fd72de5e0a..d2bf4f4fc899 100644 --- a/sys/netpfil/ipfw/ip_fw_dynamic.c +++ b/sys/netpfil/ipfw/ip_fw_dynamic.c @@ -2498,13 +2498,8 @@ dyn_send_keepalive_ipv4(struct ip_fw_chain *chain) uint32_t bucket; mbufq_init(&q, INT_MAX); - IPFW_UH_RLOCK(chain); - /* - * It is safe to not use hazard pointer and just do lockless - * access to the lists, because states entries can not be deleted - * while we hold IPFW_UH_RLOCK. - */ for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) { + DYN_BUCKET_LOCK(bucket); CK_SLIST_FOREACH(s, &V_dyn_ipv4[bucket], entry) { /* * Only established TCP connections that will @@ -2517,8 +2512,8 @@ dyn_send_keepalive_ipv4(struct ip_fw_chain *chain) continue; dyn_enqueue_keepalive_ipv4(&q, s); } + DYN_BUCKET_UNLOCK(bucket); } - IPFW_UH_RUNLOCK(chain); while ((m = mbufq_dequeue(&q)) != NULL) ip_output(m, NULL, NULL, 0, NULL, NULL); } @@ -2605,13 +2600,8 @@ dyn_send_keepalive_ipv6(struct ip_fw_chain *chain) uint32_t bucket; mbufq_init(&q, INT_MAX); - IPFW_UH_RLOCK(chain); - /* - * It is safe to not use hazard pointer and just do lockless - * access to the lists, because states entries can not be deleted - * while we hold IPFW_UH_RLOCK. - */ for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) { + DYN_BUCKET_LOCK(bucket); CK_SLIST_FOREACH(s, &V_dyn_ipv6[bucket], entry) { /* * Only established TCP connections that will @@ -2624,8 +2614,8 @@ dyn_send_keepalive_ipv6(struct ip_fw_chain *chain) continue; dyn_enqueue_keepalive_ipv6(&q, s); } + DYN_BUCKET_UNLOCK(bucket); } - IPFW_UH_RUNLOCK(chain); while ((m = mbufq_dequeue(&q)) != NULL) ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL); } diff --git a/sys/netpfil/ipfw/ip_fw_iface.c b/sys/netpfil/ipfw/ip_fw_iface.c index 7aa4a7bfbf7d..332a90f1844a 100644 --- a/sys/netpfil/ipfw/ip_fw_iface.c +++ b/sys/netpfil/ipfw/ip_fw_iface.c @@ -95,12 +95,12 @@ enum ifevent { ARRIVAL, DEPARTURE, RENAME }; static void ipfw_kifhandler(void *arg, struct ifnet *ifp, const char *old_name) { - enum ifevent *what = arg; + enum ifevent what = (uintptr_t)arg; struct ip_fw_chain *ch; struct ipfw_iface *iif; struct namedobj_instance *ii; - MPASS(*what != RENAME || old_name != NULL); + MPASS(what != RENAME || old_name != NULL); if (V_ipfw_vnet_ready == 0) return; @@ -114,9 +114,9 @@ ipfw_kifhandler(void *arg, struct ifnet *ifp, const char *old_name) return; } iif = (struct ipfw_iface*)ipfw_objhash_lookup_name(ii, 0, - *what == RENAME ? old_name : if_name(ifp)); + what == RENAME ? old_name : if_name(ifp)); if (iif != NULL) { - switch (*what) { + switch (what) { case ARRIVAL: handle_ifattach(ch, iif, ifp->if_index); break; @@ -246,13 +246,13 @@ vnet_ipfw_iface_init(struct ip_fw_chain *ch) { struct namedobj_instance *ii; + IPFW_UH_WLOCK_ASSERT(ch); + ii = ipfw_objhash_create(DEFAULT_IFACES, DEFAULT_OBJHASH_SIZE); - IPFW_UH_WLOCK(ch); if (ch->ifcfg == NULL) { ch->ifcfg = ii; ii = NULL; } - IPFW_UH_WUNLOCK(ch); if (ii != NULL) { /* Already initialized. Free namehash. */ @@ -296,9 +296,7 @@ vnet_ipfw_iface_destroy(struct ip_fw_chain *ch) /* * Notify the subsystem that we are interested in tracking - * interface @name. This function has to be called without - * holding any locks to permit allocating the necessary states - * for proper interface tracking. + * interface @name. * * Returns 0 on success. */ @@ -309,11 +307,11 @@ ipfw_iface_ref(struct ip_fw_chain *ch, char *name, struct namedobj_instance *ii; struct ipfw_iface *iif, *tmp; + IPFW_UH_WLOCK_ASSERT(ch); + if (strlen(name) >= sizeof(iif->ifname)) return (EINVAL); - IPFW_UH_WLOCK(ch); - ii = CHAIN_TO_II(ch); if (ii == NULL) { /* @@ -329,7 +327,6 @@ ipfw_iface_ref(struct ip_fw_chain *ch, char *name, if (iif != NULL) { iif->no.refcnt++; ic->iface = iif; - IPFW_UH_WUNLOCK(ch); return (0); } @@ -352,7 +349,6 @@ ipfw_iface_ref(struct ip_fw_chain *ch, char *name, /* Interface has been created since unlock. Ref and return */ tmp->no.refcnt++; ic->iface = tmp; - IPFW_UH_WUNLOCK(ch); free(iif, M_IPFW); return (0); } @@ -364,8 +360,6 @@ ipfw_iface_ref(struct ip_fw_chain *ch, char *name, ipfw_objhash_add(ii, &iif->no); ic->iface = iif; - IPFW_UH_WUNLOCK(ch); - return (0); } diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index 13e2f5bb77f2..b7c79437584e 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -342,14 +342,14 @@ static int pf_dummynet_route(struct pf_pdesc *, struct ifnet *, const struct sockaddr *, struct mbuf **); static int pf_test_eth_rule(int, struct pfi_kkif *, struct mbuf **); +static enum pf_test_status pf_match_rule(struct pf_test_ctx *, struct pf_kruleset *); static int pf_test_rule(struct pf_krule **, struct pf_kstate **, struct pf_pdesc *, struct pf_krule **, struct pf_kruleset **, u_short *, struct inpcb *, struct pf_krule_slist *); static int pf_create_state(struct pf_krule *, struct pf_test_ctx *, - struct pf_kstate **, u_int16_t, u_int16_t, - struct pf_krule_slist *match_rules); + struct pf_kstate **, u_int16_t, u_int16_t); static int pf_state_key_addr_setup(struct pf_pdesc *, struct pf_state_key_cmp *, int); static int pf_tcp_track_full(struct pf_kstate *, @@ -5108,8 +5108,7 @@ pf_tag_packet(struct pf_pdesc *pd, int tag) } while (0) enum pf_test_status -pf_step_into_anchor(struct pf_test_ctx *ctx, struct pf_krule *r, - struct pf_krule_slist *match_rules) +pf_step_into_anchor(struct pf_test_ctx *ctx, struct pf_krule *r) { enum pf_test_status rv; @@ -5127,7 +5126,7 @@ pf_step_into_anchor(struct pf_test_ctx *ctx, struct pf_krule *r, struct pf_kanchor *child; rv = PF_TEST_OK; RB_FOREACH(child, pf_kanchor_node, &r->anchor->children) { - rv = pf_match_rule(ctx, &child->ruleset, match_rules); + rv = pf_match_rule(ctx, &child->ruleset); if ((rv == PF_TEST_QUICK) || (rv == PF_TEST_FAIL)) { /* * we either hit a rule with quick action @@ -5138,7 +5137,7 @@ pf_step_into_anchor(struct pf_test_ctx *ctx, struct pf_krule *r, } } } else { - rv = pf_match_rule(ctx, &r->anchor->ruleset, match_rules); + rv = pf_match_rule(ctx, &r->anchor->ruleset); /* * Unless errors occured, stop iff any rule matched * within quick anchors. @@ -5987,10 +5986,9 @@ pf_rule_apply_nat(struct pf_test_ctx *ctx, struct pf_krule *r) } enum pf_test_status -pf_match_rule(struct pf_test_ctx *ctx, struct pf_kruleset *ruleset, - struct pf_krule_slist *match_rules) +pf_match_rule(struct pf_test_ctx *ctx, struct pf_kruleset *ruleset) { - struct pf_krule_item *ri, *rt; + struct pf_krule_item *ri; struct pf_krule *r; struct pf_krule *save_a; struct pf_kruleset *save_aruleset; @@ -6300,12 +6298,12 @@ pf_match_rule(struct pf_test_ctx *ctx, struct pf_kruleset *ruleset, } ri->r = r; - if (SLIST_EMPTY(match_rules)) { - SLIST_INSERT_HEAD(match_rules, ri, entry); + if (SLIST_EMPTY(ctx->match_rules)) { + SLIST_INSERT_HEAD(ctx->match_rules, ri, entry); } else { - SLIST_INSERT_AFTER(rt, ri, entry); + SLIST_INSERT_AFTER(ctx->last_match_rule, ri, entry); } - rt = ri; + ctx->last_match_rule = ri; pf_rule_to_actions(r, &pd->act); if (r->log) @@ -6337,7 +6335,7 @@ pf_match_rule(struct pf_test_ctx *ctx, struct pf_kruleset *ruleset, ctx->source = sr; } if (pd->act.log & PF_LOG_MATCHES) - pf_log_matches(pd, r, ctx->a, ruleset, match_rules); + pf_log_matches(pd, r, ctx->a, ruleset, ctx->match_rules); if (r->quick) { ctx->test_status = PF_TEST_QUICK; break; @@ -6354,7 +6352,7 @@ pf_match_rule(struct pf_test_ctx *ctx, struct pf_kruleset *ruleset, * Note: we don't need to restore if we are not going * to continue with ruleset evaluation. */ - if (pf_step_into_anchor(ctx, r, match_rules) != PF_TEST_OK) { + if (pf_step_into_anchor(ctx, r) != PF_TEST_OK) { break; } ctx->a = save_a; @@ -6391,6 +6389,7 @@ pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm, ctx.rsm = rsm; ctx.th = &pd->hdr.tcp; ctx.reason = *reason; + ctx.match_rules = match_rules; pf_addrcpy(&pd->nsaddr, pd->src, pd->af); pf_addrcpy(&pd->ndaddr, pd->dst, pd->af); @@ -6488,7 +6487,7 @@ pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm, ruleset = *ctx.rsm; } else { ruleset = &pf_main_ruleset; - rv = pf_match_rule(&ctx, ruleset, match_rules); + rv = pf_match_rule(&ctx, ruleset); if (rv == PF_TEST_FAIL || ctx.limiter_drop == 1) { REASON_SET(reason, ctx.reason); goto cleanup; @@ -6523,7 +6522,7 @@ pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm, PFLOG_PACKET(r->action, ctx.reason, r, ctx.a, ruleset, pd, 1, NULL); } if (pd->act.log & PF_LOG_MATCHES) - pf_log_matches(pd, r, ctx.a, ruleset, match_rules); + pf_log_matches(pd, r, ctx.a, ruleset, ctx.match_rules); if (pd->virtual_proto != PF_VPROTO_FRAGMENT && (r->action == PF_DROP) && ((r->rule_flag & PFRULE_RETURNRST) || @@ -6568,8 +6567,7 @@ pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm, (pd->flags & PFDESC_TCP_NORM)))) { bool nat64; - action = pf_create_state(r, &ctx, sm, bproto_sum, bip_sum, - match_rules); + action = pf_create_state(r, &ctx, sm, bproto_sum, bip_sum); ctx.sk = ctx.nk = NULL; if (action != PF_PASS) { pf_udp_mapping_release(ctx.udp_mapping); @@ -6659,8 +6657,7 @@ cleanup: static int pf_create_state(struct pf_krule *r, struct pf_test_ctx *ctx, - struct pf_kstate **sm, u_int16_t bproto_sum, u_int16_t bip_sum, - struct pf_krule_slist *match_rules) + struct pf_kstate **sm, u_int16_t bproto_sum, u_int16_t bip_sum) { struct pf_pdesc *pd = ctx->pd; struct pf_kstate *s = NULL; @@ -6729,7 +6726,7 @@ pf_create_state(struct pf_krule *r, struct pf_test_ctx *ctx, s->rule = r; s->nat_rule = ctx->nr; s->anchor = ctx->a; - s->match_rules = *match_rules; + s->match_rules = *ctx->match_rules; SLIST_INIT(&s->linkage); memcpy(&s->act, &pd->act, sizeof(struct pf_rule_actions)); @@ -11708,9 +11705,9 @@ pf_test(sa_family_t af, int dir, int pflags, struct ifnet *ifp, struct mbuf **m0 * it here, before we do any NAT. */ if (af == AF_INET6 && dir == PF_OUT && pflags & PFIL_FWD && - IN6_LINKMTU(ifp) < pf_max_frag_size(*m0)) { + in6_ifmtu(ifp) < pf_max_frag_size(*m0)) { PF_RULES_RUNLOCK(); - icmp6_error(*m0, ICMP6_PACKET_TOO_BIG, 0, IN6_LINKMTU(ifp)); + icmp6_error(*m0, ICMP6_PACKET_TOO_BIG, 0, in6_ifmtu(ifp)); *m0 = NULL; return (PF_DROP); } @@ -11961,11 +11958,10 @@ done: pf_is_loopback(af, pd.dst)) pd.m->m_flags |= M_SKIP_FIREWALL; - if (af == AF_INET && __predict_false(ip_divert_ptr != NULL) && - action == PF_PASS && r->divert.port && !PACKET_LOOPED(&pd)) { + if (action == PF_PASS && r->divert.port && !PACKET_LOOPED(&pd)) { mtag = m_tag_alloc(MTAG_PF_DIVERT, 0, sizeof(struct pf_divert_mtag), M_NOWAIT | M_ZERO); - if (mtag != NULL) { + if (__predict_true(mtag != NULL && ip_divert_ptr != NULL)) { ((struct pf_divert_mtag *)(mtag+1))->port = ntohs(r->divert.port); ((struct pf_divert_mtag *)(mtag+1))->idir = @@ -11994,20 +11990,22 @@ done: } ip_divert_ptr(*m0, dir == PF_IN); *m0 = NULL; - return (action); - } else { + } else if (mtag == NULL) { /* XXX: ipfw has the same behaviour! */ action = PF_DROP; REASON_SET(&reason, PFRES_MEMORY); pd.act.log = PF_LOG_FORCE; DPFPRINTF(PF_DEBUG_MISC, "pf: failed to allocate divert tag"); + } else { + action = PF_DROP; + REASON_SET(&reason, PFRES_MATCH); + pd.act.log = PF_LOG_FORCE; + DPFPRINTF(PF_DEBUG_MISC, + "pf: divert(4) is not loaded"); } } - /* XXX: Anybody working on it?! */ - if (af == AF_INET6 && r->divert.port) - printf("pf: divert(9) is not supported for IPv6\n"); /* this flag will need revising if the pkt is forwarded */ if (pd.pf_mtag) diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c index 62bbb6ddaf9b..a7042ffb02a9 100644 --- a/sys/powerpc/aim/mmu_oea64.c +++ b/sys/powerpc/aim/mmu_oea64.c @@ -125,7 +125,7 @@ uintptr_t moea64_get_unique_vsid(void); #define PV_LOCK_COUNT MAXCPU static struct mtx_padalign pv_lock[PV_LOCK_COUNT]; -#define PV_LOCK_SHIFT 21 +#define PV_LOCK_SHIFT HPT_SP_SHIFT #define pa_index(pa) ((pa) >> PV_LOCK_SHIFT) /* @@ -146,48 +146,6 @@ static struct mtx_padalign pv_lock[PV_LOCK_COUNT]; #define PV_PAGE_UNLOCK(m) PV_UNLOCK(VM_PAGE_TO_PHYS(m)) #define PV_PAGE_LOCKASSERT(m) PV_LOCKASSERT(VM_PAGE_TO_PHYS(m)) -/* Superpage PV lock */ - -#define PV_LOCK_SIZE (1 << PV_LOCK_SHIFT) - -static __always_inline void -moea64_sp_pv_lock(vm_paddr_t pa) -{ - vm_paddr_t pa_end; - - /* Note: breaking when pa_end is reached to avoid overflows */ - pa_end = pa + (HPT_SP_SIZE - PV_LOCK_SIZE); - for (;;) { - mtx_lock_flags(PV_LOCKPTR(pa), MTX_DUPOK); - if (pa == pa_end) - break; - pa += PV_LOCK_SIZE; - } -} - -static __always_inline void -moea64_sp_pv_unlock(vm_paddr_t pa) -{ - vm_paddr_t pa_end; - - /* Note: breaking when pa_end is reached to avoid overflows */ - pa_end = pa; - pa += HPT_SP_SIZE - PV_LOCK_SIZE; - for (;;) { - mtx_unlock_flags(PV_LOCKPTR(pa), MTX_DUPOK); - if (pa == pa_end) - break; - pa -= PV_LOCK_SIZE; - } -} - -#define SP_PV_LOCK_ALIGNED(pa) moea64_sp_pv_lock(pa) -#define SP_PV_UNLOCK_ALIGNED(pa) moea64_sp_pv_unlock(pa) -#define SP_PV_LOCK(pa) moea64_sp_pv_lock((pa) & ~HPT_SP_MASK) -#define SP_PV_UNLOCK(pa) moea64_sp_pv_unlock((pa) & ~HPT_SP_MASK) -#define SP_PV_PAGE_LOCK(m) SP_PV_LOCK(VM_PAGE_TO_PHYS(m)) -#define SP_PV_PAGE_UNLOCK(m) SP_PV_UNLOCK(VM_PAGE_TO_PHYS(m)) - struct ofw_map { cell_t om_va; cell_t om_len; @@ -206,9 +164,10 @@ extern void *slbtrap, *slbtrapend; static struct mem_region *regions; static struct mem_region *pregions; static struct numa_mem_region *numa_pregions; -static u_int phys_avail_count; static int regions_sz, pregions_sz, numapregions_sz; +u_int phys_avail_count; + extern void bs_remap_earlyboot(void); /* @@ -932,7 +891,7 @@ moea64_early_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend) int rm_pavail; /* Level 0 reservations consist of 4096 pages (16MB superpage). */ - vm_level_0_order = 12; + vm_level_0_order = VM_LEVEL_0_ORDER_HPT; #ifndef __powerpc64__ /* We don't have a direct map since there is no BAT */ @@ -3736,7 +3695,7 @@ moea64_sp_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, } } - SP_PV_LOCK_ALIGNED(spa); + PV_LOCK(spa); PMAP_LOCK(pmap); /* Note: moea64_remove_locked() also clears cached REF/CHG bits. */ @@ -3775,7 +3734,7 @@ moea64_sp_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, } PMAP_UNLOCK(pmap); - SP_PV_UNLOCK_ALIGNED(spa); + PV_UNLOCK(spa); sync = (sm->a.flags & PGA_EXECUTABLE) == 0; /* Note: moea64_pvo_cleanup() also clears page prot. flags. */ diff --git a/sys/powerpc/aim/mmu_oea64.h b/sys/powerpc/aim/mmu_oea64.h index 38b743159980..bc93cf4d521d 100644 --- a/sys/powerpc/aim/mmu_oea64.h +++ b/sys/powerpc/aim/mmu_oea64.h @@ -139,5 +139,6 @@ extern u_long moea64_pteg_count; extern u_long moea64_pteg_mask; extern int n_slbs; extern bool moea64_has_lp_4k_16m; +extern u_int phys_avail_count; #endif /* _POWERPC_AIM_MMU_OEA64_H */ diff --git a/sys/powerpc/aim/mmu_radix.c b/sys/powerpc/aim/mmu_radix.c index a12142fc2d7b..85008de83870 100644 --- a/sys/powerpc/aim/mmu_radix.c +++ b/sys/powerpc/aim/mmu_radix.c @@ -632,7 +632,6 @@ static unsigned int isa3_base_pid; */ static struct mem_region *regions, *pregions; static struct numa_mem_region *numa_pregions; -static u_int phys_avail_count; static int regions_sz, pregions_sz, numa_pregions_sz; static struct pate *isa3_parttab; static struct prte *isa3_proctab; diff --git a/sys/powerpc/aim/moea64_native.c b/sys/powerpc/aim/moea64_native.c index bf254e1f466c..7714187c2f4d 100644 --- a/sys/powerpc/aim/moea64_native.c +++ b/sys/powerpc/aim/moea64_native.c @@ -183,7 +183,8 @@ TLBIE(uint64_t vpn, uint64_t oldptehi) "memory"); __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); - goto done; + tlbie_lock = 0; + return; #endif } } @@ -197,20 +198,8 @@ TLBIE(uint64_t vpn, uint64_t oldptehi) (oldptehi & LPTE_KERNEL_VSID_BIT) == 0) vpn |= AP_16M; - /* - * Explicitly clobber r0. The tlbie instruction has two forms: an old - * one used by PowerISA 2.03 and prior, and a newer one used by PowerISA - * 2.06 (maybe 2.05?) and later. We need to support both, and it just - * so happens that since we use 4k pages we can simply zero out r0, and - * clobber it, and the assembler will interpret the single-operand form - * of tlbie as having RB set, and everything else as 0. The RS operand - * in the newer form is in the same position as the L(page size) bit of - * the old form, so a slong as RS is 0, we're good on both sides. - */ - __asm __volatile("li 0, 0 \n tlbie %0, 0" :: "r"(vpn) : "r0", "memory"); + __asm __volatile("tlbie %0, %1" :: "r"(vpn), "r"(0) : "memory"); __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); -done: - #else vpn_hi = (uint32_t)(vpn >> 32); vpn_lo = (uint32_t)vpn; @@ -565,7 +554,12 @@ moea64_bootstrap_native(vm_offset_t kernelstart, vm_offset_t kernelend) moea64_early_bootstrap(kernelstart, kernelend); switch (mfpvr() >> 16) { + case IBMPOWER8: + case IBMPOWER8E: + case IBMPOWER8NVL: case IBMPOWER9: + case IBMPOWER10: + case IBMPOWER11: moea64_need_lock = false; break; case IBMPOWER4: diff --git a/sys/powerpc/cpufreq/pcr.c b/sys/powerpc/cpufreq/pcr.c index 335a1d011a66..c58176997470 100644 --- a/sys/powerpc/cpufreq/pcr.c +++ b/sys/powerpc/cpufreq/pcr.c @@ -141,7 +141,7 @@ read_scom(register_t address) __asm __volatile ("mfspr %0,%1;" " mr %0+1, %0; srdi %0,%0,32" : "=r" (ret) : "K" (SPR_SCOMD)); - (void)mfspr(SPR_SCOMC); /* Complete transcation */ + (void)mfspr(SPR_SCOMC); /* Complete transaction */ mtmsr(msr); isync(); diff --git a/sys/powerpc/include/pte.h b/sys/powerpc/include/pte.h index ed926f80c879..2e8cdacbe165 100644 --- a/sys/powerpc/include/pte.h +++ b/sys/powerpc/include/pte.h @@ -145,10 +145,11 @@ typedef struct lpte lpte_t; #define LPTE_RO LPTE_BR /* HPT superpage definitions */ -#define HPT_SP_SHIFT (VM_LEVEL_0_ORDER + PAGE_SHIFT) +#define VM_LEVEL_0_ORDER_HPT 12 +#define HPT_SP_SHIFT (VM_LEVEL_0_ORDER_HPT + PAGE_SHIFT) #define HPT_SP_SIZE (1 << HPT_SP_SHIFT) #define HPT_SP_MASK (HPT_SP_SIZE - 1) -#define HPT_SP_PAGES (1 << VM_LEVEL_0_ORDER) +#define HPT_SP_PAGES (1 << VM_LEVEL_0_ORDER_HPT) /* POWER ISA 3.0 Radix Table Definitions */ #define RPTE_VALID 0x8000000000000000ULL diff --git a/sys/powerpc/powermac/platform_powermac.c b/sys/powerpc/powermac/platform_powermac.c index c63ef521ca8f..9f8a06d8dd3a 100644 --- a/sys/powerpc/powermac/platform_powermac.c +++ b/sys/powerpc/powermac/platform_powermac.c @@ -405,21 +405,88 @@ powermac_register_timebase(device_t dev, powermac_tb_disable_t cb) freeze_timebase = cb; } +/** + * @brief Implement a default platform AP/BSP SMP timebase synchronisation + * + * Some powermac platforms don't have a freeze/unfreeze method. + * Here just try our best to force synchronisation. + */ static void -powermac_smp_timebase_sync(platform_t plat, u_long tb, int ap) +powermac_smp_timebase_sync_fallback(platform_t plat, u_long tb, int ap) +{ + static volatile bool tb_ready = false; + static volatile int cpu_done; + + if (bootverbose) + printf("[%d] %s: called, AP tb=0x%jx tb=0x%jx\n", + ap, __func__, (uintmax_t)tb, (uintmax_t)mftb()); + + /* Do initial timebase sync */ + mttb(tb); + + if (ap) { + /* + * APs - wait until the BSP signals its ready to sync, + * then wait for all CPUs to be ready. + */ + critical_enter(); + while (!tb_ready) + atomic_thread_fence_seq_cst(); + atomic_add_int(&cpu_done, 1); + do { + atomic_thread_fence_seq_cst(); + } while (cpu_done < mp_ncpus); + mttb(tb); + critical_exit(); + } else { + /* + * BSP - signify that the timebase sync is about to start, + * then wait for other CPUs to be ready. + */ + critical_enter(); + /* Ensure cpu_done is zeroed so we can resync at runtime */ + atomic_store_int(&cpu_done, 0); + tb_ready = true; + atomic_add_int(&cpu_done, 1); + do { + atomic_thread_fence_seq_cst(); + } while (cpu_done < mp_ncpus); + mttb(tb); + /* Reset tb_ready so we can resync at runtime */ + tb_ready = false; + critical_exit(); + } + if (bootverbose) + printf("[%d] %s: finished; AP tb=0x%jx called tb=0x%jx\n", + ap, __func__, (uintmax_t)tb, (uintmax_t)mftb()); +} + +/** + * @brief Implement freeze/unfreeze AP/BSP SMP timebase synchronisation + * + * This implements SMP timebase synchronisation for hardware that + * implements freezing a shared timebase clock source. + * + * The BSP will freeze the timebase and signal the APs to program their + * local timebase with the shared timebase value. The BSP will then + * unfreeze the timebase clock, allowing all CPUs to march forward + * from the same base timebase value. + */ +static void +powermac_smp_timebase_sync_freeze(platform_t plat, u_long tb, int ap) { - static volatile bool tb_ready; + static volatile bool tb_ready = false; static volatile int cpu_done; + if (bootverbose) + printf("[%d] %s: called, AP tb=0x%jx tb=0x%jx\n", + ap, __func__, (uintmax_t)tb, (uintmax_t)mftb()); + /* - * XXX Temporary fallback for platforms we don't know how to freeze. - * * This needs to be replaced with a cpu-to-cpu software sync * protocol, because this is not a consistent way to sync timebase. */ mttb(tb); - if (freeze_timebase == dummy_timebase) - return; if (ap) { /* APs. Hold off until we get a stable timebase. */ @@ -428,25 +495,39 @@ powermac_smp_timebase_sync(platform_t plat, u_long tb, int ap) atomic_thread_fence_seq_cst(); mttb(tb); atomic_add_int(&cpu_done, 1); - while (cpu_done < mp_ncpus) + do { atomic_thread_fence_seq_cst(); + } while (cpu_done < mp_ncpus); critical_exit(); } else { /* BSP */ critical_enter(); /* Ensure cpu_done is zeroed so we can resync at runtime */ - atomic_set_int(&cpu_done, 0); + atomic_store_int(&cpu_done, 0); freeze_timebase(powermac_tb_dev, true); tb_ready = true; mttb(tb); atomic_add_int(&cpu_done, 1); - while (cpu_done < mp_ncpus) + do { atomic_thread_fence_seq_cst(); + } while (cpu_done < mp_ncpus); freeze_timebase(powermac_tb_dev, false); /* Reset tb_ready so we can resync at runtime */ tb_ready = false; critical_exit(); } + if (bootverbose) + printf("[%d] %s: finished; AP tb=0x%jx called tb=0x%jx\n", + ap, __func__, (uintmax_t)tb, (uintmax_t)mftb()); +} + +static void +powermac_smp_timebase_sync(platform_t plat, u_long tb, int ap) +{ + if (freeze_timebase == dummy_timebase) + powermac_smp_timebase_sync_fallback(plat, tb, ap); + else + powermac_smp_timebase_sync_freeze(plat, tb, ap); } /* Fallback freeze. In case no real handler is found in the device tree. */ diff --git a/sys/powerpc/powerpc/exec_machdep.c b/sys/powerpc/powerpc/exec_machdep.c index 18c9bddb84c5..00c04b4ddbaa 100644 --- a/sys/powerpc/powerpc/exec_machdep.c +++ b/sys/powerpc/powerpc/exec_machdep.c @@ -1082,8 +1082,8 @@ cpu_thread_alloc(struct thread *td) { struct pcb *pcb; - pcb = (struct pcb *)((td->td_kstack + td->td_kstack_pages * PAGE_SIZE - - sizeof(struct pcb)) & ~0x2fUL); + pcb = (struct pcb *)__align_down(td->td_kstack + td->td_kstack_pages * + PAGE_SIZE - sizeof(struct pcb), 0x40); td->td_pcb = pcb; td->td_frame = (struct trapframe *)pcb - 1; } diff --git a/sys/powerpc/powerpc/machdep.c b/sys/powerpc/powerpc/machdep.c index e9979712aa9c..04b3967ee110 100644 --- a/sys/powerpc/powerpc/machdep.c +++ b/sys/powerpc/powerpc/machdep.c @@ -83,6 +83,7 @@ #include <sys/reboot.h> #include <sys/reg.h> #include <sys/rwlock.h> +#include <sys/sched.h> #include <sys/signalvar.h> #include <sys/syscallsubr.h> #include <sys/sysctl.h> @@ -467,6 +468,7 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp, * Bring up MMU */ pmap_mmu_init(); + sched_instance_select(); link_elf_ireloc(); pmap_bootstrap(startkernel, endkernel); mtmsr(psl_kernset & ~PSL_EE); @@ -488,9 +490,8 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp, /* * Finish setting up thread0. */ - thread0.td_pcb = (struct pcb *) - ((thread0.td_kstack + thread0.td_kstack_pages * PAGE_SIZE - - sizeof(struct pcb)) & ~15UL); + thread0.td_pcb = (struct pcb *)__align_down(thread0.td_kstack + + thread0.td_kstack_pages * PAGE_SIZE - sizeof(struct pcb), 16); bzero((void *)thread0.td_pcb, sizeof(struct pcb)); pc->pc_curpcb = thread0.td_pcb; diff --git a/sys/powerpc/powerpc/sdt_machdep.c b/sys/powerpc/powerpc/sdt_machdep.c index 8a84016a9571..04a4cc170769 100644 --- a/sys/powerpc/powerpc/sdt_machdep.c +++ b/sys/powerpc/powerpc/sdt_machdep.c @@ -37,8 +37,8 @@ sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target) uint32_t instr; KASSERT(sdt_tracepoint_valid(patchpoint, target), - ("%s: invalid tracepoint %#lx -> %#lx", - __func__, patchpoint, target)); + ("%s: invalid tracepoint %#jx -> %#jx", + __func__, (uintmax_t)patchpoint, (uintmax_t)target)); instr = ((target - patchpoint) & 0x7fffffful) | 0x48000000; memcpy((void *)patchpoint, &instr, sizeof(instr)); diff --git a/sys/powerpc/powerpc/swtch32.S b/sys/powerpc/powerpc/swtch32.S index 7fc0641722aa..262e7035bb29 100644 --- a/sys/powerpc/powerpc/swtch32.S +++ b/sys/powerpc/powerpc/swtch32.S @@ -56,7 +56,6 @@ */ #include "assym.inc" -#include "opt_sched.h" #include <sys/syscall.h> @@ -125,7 +124,7 @@ ENTRY(cpu_switch) sync /* Make sure all of that finished */ cpu_switchin: -#if defined(SMP) && defined(SCHED_ULE) +#if defined(SMP) /* Wait for the new thread to become unblocked */ bl 1f 1: diff --git a/sys/powerpc/powerpc/swtch64.S b/sys/powerpc/powerpc/swtch64.S index ba37274d32bb..61af10aabaee 100644 --- a/sys/powerpc/powerpc/swtch64.S +++ b/sys/powerpc/powerpc/swtch64.S @@ -56,7 +56,6 @@ */ #include "assym.inc" -#include "opt_sched.h" #include <sys/syscall.h> @@ -187,7 +186,7 @@ save_tar: sync /* Make sure all of that finished */ cpu_switchin: -#if defined(SMP) && defined(SCHED_ULE) +#if defined(SMP) /* Wait for the new thread to become unblocked */ addis %r6,%r2,TOC_REF(blocked_lock)@ha ld %r6,TOC_REF(blocked_lock)@l(%r6) diff --git a/sys/powerpc/powerpc/vm_machdep.c b/sys/powerpc/powerpc/vm_machdep.c index d47beedb595e..00fdc301a7e7 100644 --- a/sys/powerpc/powerpc/vm_machdep.c +++ b/sys/powerpc/powerpc/vm_machdep.c @@ -123,8 +123,8 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) if (td1 == curthread) cpu_update_pcb(td1); - pcb = (struct pcb *)((td2->td_kstack + - td2->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb)) & ~0x2fUL); + pcb = (struct pcb *)__align_down(td2->td_kstack + + td2->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb), 0x40); td2->td_pcb = pcb; /* Copy the pcb */ diff --git a/sys/riscv/include/cpufunc.h b/sys/riscv/include/cpufunc.h index 75b22632c546..c39f17131eb7 100644 --- a/sys/riscv/include/cpufunc.h +++ b/sys/riscv/include/cpufunc.h @@ -119,6 +119,13 @@ sfence_vma_asid_page(uint64_t asid, uintptr_t addr) : "memory"); } +static __inline void +hfence_gvma(void) +{ + + __asm __volatile("hfence.gvma" ::: "memory"); +} + #define rdcycle() csr_read64(cycle) #define rdtime() csr_read64(time) #define rdinstret() csr_read64(instret) diff --git a/sys/riscv/riscv/elf_machdep.c b/sys/riscv/riscv/elf_machdep.c index 5bd4af4c15f8..b52050e56a14 100644 --- a/sys/riscv/riscv/elf_machdep.c +++ b/sys/riscv/riscv/elf_machdep.c @@ -271,10 +271,10 @@ reloctype_to_str(int type) } bool -elf_is_ifunc_reloc(Elf_Size r_info __unused) +elf_is_ifunc_reloc(Elf_Size r_info) { - return (false); + return (ELF_R_TYPE(r_info) == R_RISCV_IRELATIVE); } /* @@ -501,7 +501,12 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, (local ? 'l' : 'g'), reloctype_to_str(rtype), before32, *insn32p); break; - + case R_RISCV_IRELATIVE: + addr = relocbase + addend; + val = ((Elf64_Addr (*)(void))addr)(); + if (*where != val) + *where = val; + break; default: printf("kldload: unexpected relocation type %ld, " "symbol index %ld\n", rtype, symidx); diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c index 235cc651b87e..0214426eea18 100644 --- a/sys/riscv/riscv/machdep.c +++ b/sys/riscv/riscv/machdep.c @@ -624,6 +624,10 @@ initriscv(struct riscv_bootparams *rvbp) */ identify_cpu(0); + sched_instance_select(); + + link_elf_ireloc(); + /* Do basic tuning, hz etc */ init_param1(); diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c index 0deb8b93a6dc..3fc261a15c06 100644 --- a/sys/riscv/riscv/pmap.c +++ b/sys/riscv/riscv/pmap.c @@ -578,16 +578,13 @@ pmap_early_alloc_tables(vm_paddr_t *freemempos, int npages) } /* - * Construct the direct map -- a linear mapping of physical memory into + * Construct the Direct Map -- a linear mapping of physical memory into * the kernel address space. * * We walk the list of physical memory segments (of arbitrary size and - * address) mapping each appropriately using L2 and L1 superpages. - * Consequently, the DMAP address space will have unmapped regions - * corresponding to any holes between physical memory segments. - * - * The lowest usable physical address will always be mapped to - * DMAP_MIN_ADDRESS. + * alignment) mapping each appropriately. Consequently, the DMAP address + * space will have unmapped regions corresponding to the holes between + * physical memory segments. */ static vm_paddr_t pmap_bootstrap_dmap(pd_entry_t *l1, vm_paddr_t freemempos) @@ -595,9 +592,9 @@ pmap_bootstrap_dmap(pd_entry_t *l1, vm_paddr_t freemempos) vm_paddr_t physmap[PHYS_AVAIL_ENTRIES]; vm_offset_t va; vm_paddr_t min_pa, max_pa, pa, endpa; - pd_entry_t *l2; + pd_entry_t *l3, *l2; pt_entry_t memattr; - u_int l1slot, l2slot; + u_int l1slot, l2slot, l3slot; int physmap_idx; physmap_idx = physmem_avail(physmap, nitems(physmap)); @@ -614,17 +611,58 @@ pmap_bootstrap_dmap(pd_entry_t *l1, vm_paddr_t freemempos) memattr = pmap_memattr_bits(VM_MEMATTR_DEFAULT); - /* Walk the physmap table. */ - l2 = NULL; - l1slot = Ln_ENTRIES; /* sentinel value */ + /* + * Walk the physmap table, using the largest page sizes possible for each + * mapping. So, for each physmap entry, map as needed/able: + * - 4K/L3 page prefix + * - 2M/L2 superpage prefix + * - 1G/L1 superpages + * - 2M/L2 superpage suffix + * - 4K/L3 page suffix + */ + l3 = l2 = NULL; + l2slot = l1slot = Ln_ENTRIES; /* sentinel value */ for (int idx = 0; idx < physmap_idx; idx += 2) { - pa = rounddown(physmap[idx], L2_SIZE); + pa = rounddown(physmap[idx], L3_SIZE); endpa = physmap[idx + 1]; /* Virtual address for this range. */ va = PHYS_TO_DMAP(pa); - /* Any 1GB possible for this range? */ + /* Any 2MB possible for this range? */ + if (roundup(pa, L2_SIZE) + L2_SIZE > endpa) + goto l3end; + + /* Loop until the next 2MB boundary. */ + while ((pa & L2_OFFSET) != 0) { + if (l2 == NULL || pmap_l1_index(va) != l1slot) { + /* Need to alloc another page table. */ + l2 = pmap_early_alloc_tables(&freemempos, 1); + + /* Link it. */ + l1slot = pmap_l1_index(va); + pmap_store(&l1[l1slot], + L1_PDE((vm_paddr_t)l2, PTE_V)); + } + + if (l3 == NULL || pmap_l2_index(va) != l2slot) { + l3 = pmap_early_alloc_tables(&freemempos, 1); + + /* Link it to L2. */ + l2slot = pmap_l2_index(va); + pmap_store(&l2[l2slot], + L2_PDE((vm_paddr_t)l3, PTE_V)); + } + + /* map l3 pages */ + l3slot = pmap_l3_index(va); + pmap_store(&l3[l3slot], L3_PTE(pa, PTE_KERN | memattr)); + + pa += L3_SIZE; + va += L3_SIZE; + } + + /* Any 1GB possible for remaining range? */ if (roundup(pa, L1_SIZE) + L1_SIZE > endpa) goto l2end; @@ -659,7 +697,8 @@ pmap_bootstrap_dmap(pd_entry_t *l1, vm_paddr_t freemempos) } l2end: - while (pa < endpa) { + /* Map what we can with 2MB superpages. */ + while (pa + L2_SIZE - 1 < endpa) { if (l2 == NULL || pmap_l1_index(va) != l1slot) { /* Need to alloc another page table. */ l2 = pmap_early_alloc_tables(&freemempos, 1); @@ -677,6 +716,35 @@ l2end: pa += L2_SIZE; va += L2_SIZE; } + +l3end: + while (pa < endpa) { + if (l2 == NULL || pmap_l1_index(va) != l1slot) { + /* Need to alloc another page table. */ + l2 = pmap_early_alloc_tables(&freemempos, 1); + + /* Link it. */ + l1slot = pmap_l1_index(va); + pmap_store(&l1[l1slot], + L1_PDE((vm_paddr_t)l2, PTE_V)); + } + + if (l3 == NULL || pmap_l2_index(va) != l2slot) { + l3 = pmap_early_alloc_tables(&freemempos, 1); + + /* Link it to L2. */ + l2slot = pmap_l2_index(va); + pmap_store(&l2[l2slot], + L2_PDE((vm_paddr_t)l3, PTE_V)); + } + + /* map l3 pages */ + l3slot = pmap_l3_index(va); + pmap_store(&l3[l3slot], L3_PTE(pa, PTE_KERN | memattr)); + + pa += L3_SIZE; + va += L3_SIZE; + } } /* And finally, the limit on DMAP VA. */ diff --git a/sys/riscv/riscv/swtch.S b/sys/riscv/riscv/swtch.S index cfd19a2d76d6..fc9b493744b8 100644 --- a/sys/riscv/riscv/swtch.S +++ b/sys/riscv/riscv/swtch.S @@ -33,7 +33,6 @@ */ #include "assym.inc" -#include "opt_sched.h" #include <machine/param.h> #include <machine/asm.h> @@ -315,7 +314,7 @@ ENTRY(cpu_switch) /* Release the old thread */ sd s2, TD_LOCK(s0) -#if defined(SCHED_ULE) && defined(SMP) +#if defined(SMP) /* Spin if TD_LOCK points to a blocked_lock */ la s2, _C_LABEL(blocked_lock) 1: diff --git a/sys/riscv/vmm/vmm_riscv.c b/sys/riscv/vmm/vmm_riscv.c index cfd3decd8cfa..fe2ca5c07789 100644 --- a/sys/riscv/vmm/vmm_riscv.c +++ b/sys/riscv/vmm/vmm_riscv.c @@ -625,7 +625,7 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) * have been modified, it may be necessary to execute an HFENCE.GVMA * instruction (see Section 5.3.2) before or after writing hgatp. */ - __asm __volatile("hfence.gvma" ::: "memory"); + hfence_gvma(); csr_write(hgatp, pmap->pm_satp); if (has_sstc) diff --git a/sys/rpc/xdr.h b/sys/rpc/xdr.h index 4307b5101477..b3eafcb864b2 100644 --- a/sys/rpc/xdr.h +++ b/sys/rpc/xdr.h @@ -133,14 +133,7 @@ typedef struct XDR { * to be decoded. If this pointer is 0, then the type routines should * allocate dynamic storage of the appropriate size and return it. */ -#ifdef _KERNEL -typedef bool_t (*xdrproc_t)(XDR *, void *, ...); -#else -/* - * XXX can't actually prototype it, because some take three args!!! - */ -typedef bool_t (*xdrproc_t)(XDR *, ...); -#endif +typedef bool_t (*xdrproc_t)(XDR *, void *); /* * Operations defined on a XDR handle diff --git a/sys/security/audit/audit_bsm.c b/sys/security/audit/audit_bsm.c index 2cd1511b2296..6cd96ebe092f 100644 --- a/sys/security/audit/audit_bsm.c +++ b/sys/security/audit/audit_bsm.c @@ -1115,6 +1115,16 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau) } break; + case AUE_PDWAIT: + if (ARG_IS_VALID(kar, ARG_FFLAGS)) { + tok = au_to_arg32(1, "flags", ar->ar_arg_fflags); + kau_write(rec, tok); + } + if (ARG_IS_VALID(kar, ARG_FD)) { + tok = au_to_arg32(1, "fd", ar->ar_arg_fd); + kau_write(rec, tok); + } + case AUE_IOCTL: if (ARG_IS_VALID(kar, ARG_CMD)) { tok = au_to_arg32(2, "cmd", ar->ar_arg_cmd); @@ -1365,6 +1375,24 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau) kau_write(rec, tok); } break; + case AUE_PDRFORK: + if (ARG_IS_VALID(kar, ARG_PID)) { + tok = au_to_arg32(0, "child PID", ar->ar_arg_pid); + kau_write(rec, tok); + } + if (ARG_IS_VALID(kar, ARG_CMD)) { + tok = au_to_arg32(2, "fflags", ar->ar_arg_cmd); + kau_write(rec, tok); + } + if (ARG_IS_VALID(kar, ARG_FFLAGS)) { + tok = au_to_arg32(2, "flags", ar->ar_arg_fflags); + kau_write(rec, tok); + } + if (ARG_IS_VALID(kar, ARG_FD)) { + tok = au_to_arg32(1, "fd", ar->ar_arg_fd); + kau_write(rec, tok); + } + break; case AUE_PDGETPID: if (ARG_IS_VALID(kar, ARG_FD)) { tok = au_to_arg32(1, "fd", ar->ar_arg_fd); diff --git a/sys/sys/buf_ring.h b/sys/sys/buf_ring.h index 00870cbf3531..07a4fa52891e 100644 --- a/sys/sys/buf_ring.h +++ b/sys/sys/buf_ring.h @@ -266,7 +266,7 @@ buf_ring_advance_sc(struct buf_ring *br) * the compare and an atomic. */ static __inline void -buf_ring_putback_sc(struct buf_ring *br, void *new) +buf_ring_putback_sc(struct buf_ring *br, void *buf) { uint32_t cons_idx, mask; @@ -274,7 +274,7 @@ buf_ring_putback_sc(struct buf_ring *br, void *new) cons_idx = atomic_load_32(&br->br_cons_head) & mask; KASSERT(cons_idx != (atomic_load_32(&br->br_prod_tail) & mask), ("Buf-Ring has none in putback")) ; - br->br_ring[cons_idx] = new; + br->br_ring[cons_idx] = buf; } /* @@ -305,7 +305,7 @@ static __inline void * buf_ring_peek_clear_sc(struct buf_ring *br) { uint32_t cons_head, prod_tail, mask; - void *ret; + void *buf; #if defined(DEBUG_BUFRING) && defined(_KERNEL) if (!mtx_owned(br->br_lock)) @@ -319,7 +319,7 @@ buf_ring_peek_clear_sc(struct buf_ring *br) if (cons_head == prod_tail) return (NULL); - ret = br->br_ring[cons_head & mask]; + buf = br->br_ring[cons_head & mask]; #ifdef DEBUG_BUFRING /* * Single consumer, i.e. cons_head will not move while we are @@ -327,13 +327,12 @@ buf_ring_peek_clear_sc(struct buf_ring *br) */ br->br_ring[cons_head & mask] = NULL; #endif - return (ret); + return (buf); } static __inline int buf_ring_full(struct buf_ring *br) { - return (atomic_load_32(&br->br_prod_head) == atomic_load_32(&br->br_cons_tail) + br->br_cons_size - 1); } @@ -341,7 +340,6 @@ buf_ring_full(struct buf_ring *br) static __inline int buf_ring_empty(struct buf_ring *br) { - return (atomic_load_32(&br->br_cons_head) == atomic_load_32(&br->br_prod_tail)); } diff --git a/sys/sys/caprights.h b/sys/sys/caprights.h index 6a5a17eda5ee..904d9b4e843a 100644 --- a/sys/sys/caprights.h +++ b/sys/sys/caprights.h @@ -92,6 +92,7 @@ extern const cap_rights_t cap_mmap_rights; extern const cap_rights_t cap_no_rights; extern const cap_rights_t cap_pdgetpid_rights; extern const cap_rights_t cap_pdkill_rights; +extern const cap_rights_t cap_pdwait_rights; extern const cap_rights_t cap_pread_rights; extern const cap_rights_t cap_pwrite_rights; extern const cap_rights_t cap_read_rights; diff --git a/sys/sys/elf_common.h b/sys/sys/elf_common.h index efda38279848..3782bfc2df9c 100644 --- a/sys/sys/elf_common.h +++ b/sys/sys/elf_common.h @@ -1122,6 +1122,7 @@ typedef struct { #define R_ARM_PLT32 27 /* Add PC-relative PLT offset. */ #define R_ARM_GNU_VTENTRY 100 #define R_ARM_GNU_VTINHERIT 101 +#define R_ARM_IRELATIVE 160 #define R_ARM_RSBREL32 250 #define R_ARM_THM_RPC22 251 #define R_ARM_RREL32 252 diff --git a/sys/sys/eventfd.h b/sys/sys/eventfd.h index 0f64483753e5..1b390feed48e 100644 --- a/sys/sys/eventfd.h +++ b/sys/sys/eventfd.h @@ -38,8 +38,13 @@ typedef uint64_t eventfd_t; #ifdef _KERNEL +struct eventfd; + int eventfd_create_file(struct thread *td, struct file *fp, uint32_t initval, int flags); +struct eventfd *eventfd_get(struct file *fp); +void eventfd_put(struct eventfd *efd); +void eventfd_signal(struct eventfd *efd); #else diff --git a/sys/sys/exterr_cat.h b/sys/sys/exterr_cat.h index 24f07539fe35..015eb6a1ae76 100644 --- a/sys/sys/exterr_cat.h +++ b/sys/sys/exterr_cat.h @@ -37,6 +37,8 @@ #define EXTERR_CAT_GEOM 12 #define EXTERR_CAT_FUSE_VFS 13 #define EXTERR_CAT_FUSE_DEVICE 14 +#define EXTERR_CAT_FORK 15 +#define EXTERR_CAT_PROCEXIT 16 #endif diff --git a/sys/sys/ktrace.h b/sys/sys/ktrace.h index 822822a6ffe7..1ba5c84000b0 100644 --- a/sys/sys/ktrace.h +++ b/sys/sys/ktrace.h @@ -340,6 +340,7 @@ ktr_get_tracevp(struct proc *p, bool ref) void ktr_io_params_free(struct ktr_io_params *); void ktrnamei(const char *); void ktrcsw(int, int, const char *); +void ktrcsw_out(const struct timespec *, const char *); void ktrpsig(int, sig_t, sigset_t *, int); void ktrfault(vm_offset_t, int); void ktrfaultend(int); diff --git a/sys/sys/module.h b/sys/sys/module.h index f749455bdf63..b4a08e2fc26f 100644 --- a/sys/sys/module.h +++ b/sys/sys/module.h @@ -116,15 +116,17 @@ struct mod_pnp_match_info }; \ DATA_SET(modmetadata_set, MODULE_METADATA_CONCAT(uniquifier)) +#define MODULE_DEPEND_CONCAT(module, mdepend) _##module##_depend_on_##mdepend #define MODULE_DEPEND(module, mdepend, vmin, vpref, vmax) \ - static struct mod_depend _##module##_depend_on_##mdepend \ + static struct mod_depend MODULE_DEPEND_CONCAT(module, mdepend) \ __section(".data") = { \ vmin, \ vpref, \ vmax \ }; \ - MODULE_METADATA(_md_##module##_on_##mdepend, MDT_DEPEND, \ - &_##module##_depend_on_##mdepend, #mdepend) + MODULE_METADATA(MODULE_DEPEND_CONCAT(module, mdepend), \ + MDT_DEPEND, &MODULE_DEPEND_CONCAT(module, mdepend), \ + __XSTRING(mdepend)) /* * Every kernel has a 'kernel' module with the version set to diff --git a/sys/sys/param.h b/sys/sys/param.h index a5dd31519ea5..65a244311fee 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -74,7 +74,7 @@ * cannot include sys/param.h and should only be updated here. */ #undef __FreeBSD_version -#define __FreeBSD_version 1600009 +#define __FreeBSD_version 1600011 /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h index a9853e5f7d10..12b8ddcb156f 100644 --- a/sys/sys/pmc.h +++ b/sys/sys/pmc.h @@ -109,6 +109,7 @@ extern char pmc_cpuid[PMC_CPUID_LEN]; __PMC_CPU(INTEL_ATOM_TREMONT, 0x9F, "Intel Atom Tremont") \ __PMC_CPU(INTEL_EMERALD_RAPIDS, 0xA0, "Intel Emerald Rapids") \ __PMC_CPU(INTEL_ALDERLAKEN, 0xA1, "Intel AlderlakeN") \ + __PMC_CPU(INTEL_GRANITE_RAPIDS, 0xA2, "Intel Granite Rapids") \ __PMC_CPU(INTEL_XSCALE, 0x100, "Intel XScale") \ __PMC_CPU(PPC_7450, 0x300, "PowerPC MPC7450") \ __PMC_CPU(PPC_E500, 0x340, "PowerPC e500 Core") \ diff --git a/sys/sys/power.h b/sys/sys/power.h index 33ace400bfd2..d34af0ddb86a 100644 --- a/sys/sys/power.h +++ b/sys/sys/power.h @@ -45,22 +45,24 @@ #define POWER_CMD_SUSPEND 0x00 /* - * Sleep state. + * Sleep state transition requests. * * These are high-level sleep states that the system can enter. They map to * a specific generic sleep type (enum power_stype). */ -#define POWER_SLEEP_STATE_STANDBY 0x00 -#define POWER_SLEEP_STATE_SUSPEND 0x01 -#define POWER_SLEEP_STATE_HIBERNATE 0x02 +enum power_sstate_transition { + POWER_SSTATE_TRANSITION_STANDBY, + POWER_SSTATE_TRANSITION_SUSPEND, + POWER_SSTATE_TRANSITION_HIBERNATE, +}; /* * Sleep type. * * These are the specific generic methods of entering a sleep state. E.g. - * POWER_SLEEP_STATE_SUSPEND could be set to enter either suspend-to-RAM (which - * is S3 on ACPI systems), or suspend-to-idle (S0ix on ACPI systems). This - * would be done through the kern.power.suspend sysctl. + * POWER_SSTATE_TRANSITION_SUSPEND could be set to enter either suspend-to-RAM + * (which is S3 on ACPI systems), or suspend-to-idle (S0ix on ACPI systems). + * This would be done through the kern.power.suspend sysctl. */ enum power_stype { POWER_STYPE_AWAKE, @@ -94,7 +96,7 @@ extern int power_pm_register(u_int _pm_type, power_pm_fn_t _pm_fn, void *_pm_arg, bool _pm_supported[static POWER_STYPE_COUNT]); extern u_int power_pm_get_type(void); -extern void power_pm_suspend(int); +extern void power_pm_suspend(enum power_sstate_transition _trans); /* * System power API. diff --git a/sys/sys/proc.h b/sys/sys/proc.h index b27b67999e8b..7c4431ab0819 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -503,6 +503,7 @@ enum { TDA_SIGSUSPEND, TDA_MOD3, /* .. and after */ TDA_MOD4, + TDA_SCHED_PRIV, TDA_MAX, }; #define TDAI(tda) (1U << (tda)) @@ -1173,7 +1174,6 @@ void kern_proc_vmmap_resident(struct vm_map *map, struct vm_map_entry *entry, void kern_yield(int); void killjobc(void); int leavepgrp(struct proc *p); -int maybe_preempt(struct thread *td); void maybe_yield(void); void mi_switch(int flags); int p_candebug(struct thread *td, struct proc *p); diff --git a/sys/sys/procdesc.h b/sys/sys/procdesc.h index 81102dffa6ff..b477903f8053 100644 --- a/sys/sys/procdesc.h +++ b/sys/sys/procdesc.h @@ -122,9 +122,15 @@ struct rusage; * Process descriptor system calls. */ __BEGIN_DECLS +struct __wrusage; +struct __siginfo; + pid_t pdfork(int *, int); +pid_t pdrfork(int *, int, int); int pdkill(int, int); int pdgetpid(int, pid_t *); +int pdwait(int, int *, int, struct __wrusage *, struct __siginfo *); +pid_t pdrfork_thread(int *, int, int, void *, int (*)(void *), void *); __END_DECLS #endif /* _KERNEL */ diff --git a/sys/sys/sched.h b/sys/sys/sched.h index 052b8bb85f2f..c7b7b849947c 100644 --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -68,6 +68,8 @@ #ifdef SCHED_STATS #include <sys/pcpu.h> #endif +#include <sys/linker_set.h> +#include <sys/sdt.h> struct proc; struct thread; @@ -114,11 +116,6 @@ void sched_throw(struct thread *td); void sched_unlend_prio(struct thread *td, u_char prio); void sched_user_prio(struct thread *td, u_char prio); void sched_userret_slowpath(struct thread *td); -#ifdef RACCT -#ifdef SCHED_4BSD -fixpt_t sched_pctcpu_delta(struct thread *td); -#endif -#endif static inline void sched_userret(struct thread *td) @@ -174,9 +171,7 @@ int sched_sizeof_thread(void); * functions. */ char *sched_tdname(struct thread *td); -#ifdef KTR void sched_clear_tdname(struct thread *td); -#endif static __inline void sched_pin(void) @@ -193,6 +188,8 @@ sched_unpin(void) curthread->td_pinned--; } +void ast_scheduler(struct thread *td, int tda); + /* sched_add arguments (formerly setrunqueue) */ #define SRQ_BORING 0x0000 /* No special circumstances. */ #define SRQ_YIELDING 0x0001 /* We are yielding (from mi_switch). */ @@ -221,6 +218,10 @@ SYSINIT(name, SI_SUB_LAST, SI_ORDER_MIDDLE, name ## _add_proc, NULL); #define SCHED_STAT_DEFINE(name, descr) \ DPCPU_DEFINE(unsigned long, name); \ SCHED_STAT_DEFINE_VAR(name, &DPCPU_NAME(name), descr) + +#define SCHED_STAT_DECLARE(name) \ + DPCPU_DECLARE(unsigned long, name); + /* * Sched stats are always incremented in critical sections so no atomic * is necessary to increment them. @@ -229,9 +230,29 @@ SYSINIT(name, SI_SUB_LAST, SI_ORDER_MIDDLE, name ## _add_proc, NULL); #else #define SCHED_STAT_DEFINE_VAR(name, descr, ptr) #define SCHED_STAT_DEFINE(name, descr) +#define SCHED_STAT_DECLARE(name) #define SCHED_STAT_INC(var) (void)0 #endif +SCHED_STAT_DECLARE(ithread_demotions); +SCHED_STAT_DECLARE(ithread_preemptions); + +SDT_PROBE_DECLARE(sched, , , change__pri); +SDT_PROBE_DECLARE(sched, , , dequeue); +SDT_PROBE_DECLARE(sched, , , enqueue); +SDT_PROBE_DECLARE(sched, , , lend__pri); +SDT_PROBE_DECLARE(sched, , , load__change); +SDT_PROBE_DECLARE(sched, , , off__cpu); +SDT_PROBE_DECLARE(sched, , , on__cpu); +SDT_PROBE_DECLARE(sched, , , remain__cpu); +SDT_PROBE_DECLARE(sched, , , surrender); + +#ifdef KDTRACE_HOOKS +#include <sys/dtrace_bsd.h> +extern int dtrace_vtime_active; +extern dtrace_vtime_switch_func_t dtrace_vtime_switch_func; +#endif + /* * Fixup scheduler state for proc0 and thread0 */ @@ -241,6 +262,81 @@ void schedinit(void); * Fixup scheduler state for secondary APs */ void schedinit_ap(void); + +bool sched_do_timer_accounting(void); + +/* + * Find an L2 neighbor of the given CPU or return -1 if none found. This + * does not distinguish among multiple L2 neighbors if the given CPU has + * more than one (it will always return the same result in that case). + */ +int sched_find_l2_neighbor(int cpu); + +struct sched_instance { + int (*load)(void); + int (*rr_interval)(void); + bool (*runnable)(void); + void (*exit)(struct proc *p, struct thread *childtd); + void (*fork)(struct thread *td, struct thread *childtd); + void (*fork_exit)(struct thread *td); + void (*class)(struct thread *td, int class); + void (*nice)(struct proc *p, int nice); + void (*ap_entry)(void); + void (*exit_thread)(struct thread *td, struct thread *child); + u_int (*estcpu)(struct thread *td); + void (*fork_thread)(struct thread *td, struct thread *child); + void (*ithread_prio)(struct thread *td, u_char prio); + void (*lend_prio)(struct thread *td, u_char prio); + void (*lend_user_prio)(struct thread *td, u_char pri); + void (*lend_user_prio_cond)(struct thread *td, u_char pri); + fixpt_t (*pctcpu)(struct thread *td); + void (*prio)(struct thread *td, u_char prio); + void (*sleep)(struct thread *td, int prio); + void (*sswitch)(struct thread *td, int flags); + void (*throw)(struct thread *td); + void (*unlend_prio)(struct thread *td, u_char prio); + void (*user_prio)(struct thread *td, u_char prio); + void (*userret_slowpath)(struct thread *td); + void (*add)(struct thread *td, int flags); + struct thread *(*choose)(void); + void (*clock)(struct thread *td, int cnt); + void (*idletd)(void *); + void (*preempt)(struct thread *td); + void (*relinquish)(struct thread *td); + void (*rem)(struct thread *td); + void (*wakeup)(struct thread *td, int srqflags); + void (*bind)(struct thread *td, int cpu); + void (*unbind)(struct thread *td); + int (*is_bound)(struct thread *td); + void (*affinity)(struct thread *td); + int (*sizeof_proc)(void); + int (*sizeof_thread)(void); + char *(*tdname)(struct thread *td); + void (*clear_tdname)(struct thread *td); + bool (*do_timer_accounting)(void); + int (*find_l2_neighbor)(int cpuid); + void (*init)(void); + void (*init_ap)(void); + void (*setup)(void); + void (*initticks)(void); + void (*schedcpu)(void); +}; + +extern const struct sched_instance *active_sched; + +struct sched_selection { + const char *name; + const struct sched_instance *instance; +}; +#define DECLARE_SCHEDULER(xsel_name, xsched_name, xsched_instance) \ + static struct sched_selection xsel_name = { \ + .name = xsched_name, \ + .instance = xsched_instance, \ + }; \ + DATA_SET(sched_instance_set, xsel_name); + +void sched_instance_select(void); + #endif /* _KERNEL */ /* POSIX 1003.1b Process Scheduling */ diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h index f705be915684..0be4e9ba6240 100644 --- a/sys/sys/sdt.h +++ b/sys/sys/sdt.h @@ -194,8 +194,12 @@ SET_DECLARE(sdt_argtypes_set, struct sdt_argtype); #define _SDT_ASM_PROBE_CONSTRAINT "i" #endif #ifndef _SDT_ASM_PROBE_OPERAND +#if !defined(__clang__) && __GNUC_PREREQ__(15, 0) +#define _SDT_ASM_PROBE_OPERAND "cc" +#else #define _SDT_ASM_PROBE_OPERAND "c" #endif +#endif /* * The asm below generates records corresponding to the structure's layout, so diff --git a/sys/sys/smp.h b/sys/sys/smp.h index fdb69b13c0d4..493dc91043bd 100644 --- a/sys/sys/smp.h +++ b/sys/sys/smp.h @@ -89,6 +89,8 @@ struct cpu_group { typedef struct cpu_group *cpu_group_t; +extern cpu_group_t cpu_top; + /* * Defines common resources for CPUs in the group. The highest level * resource should be used when multiple are shared. @@ -147,9 +149,6 @@ int topo_analyze(struct topo_node *topo_root, int all, #define TOPO_FOREACH(i, root) \ for (i = root; i != NULL; i = topo_next_node(root, i)) -struct cpu_group *smp_topo(void); -struct cpu_group *smp_topo_alloc(u_int count); -struct cpu_group *smp_topo_none(void); struct cpu_group *smp_topo_1level(int l1share, int l1count, int l1flags); struct cpu_group *smp_topo_2level(int l2share, int l2count, int l1share, int l1count, int l1flags); @@ -166,6 +165,10 @@ extern cpuset_t hlt_cpus_mask; /* XXX 'mask' is detail in old impl */ extern cpuset_t logical_cpus_mask; #endif /* SMP */ +struct cpu_group *smp_topo(void); +struct cpu_group *smp_topo_alloc(u_int count); +struct cpu_group *smp_topo_none(void); + extern u_int mp_maxid; extern int mp_maxcpus; extern int mp_ncores; diff --git a/sys/sys/syscall.h b/sys/sys/syscall.h index 43f46f063e3e..814437732df3 100644 --- a/sys/sys/syscall.h +++ b/sys/sys/syscall.h @@ -538,4 +538,6 @@ #define SYS_jail_attach_jd 597 #define SYS_jail_remove_jd 598 #define SYS_kexec_load 599 -#define SYS_MAXSYSCALL 600 +#define SYS_pdrfork 600 +#define SYS_pdwait 601 +#define SYS_MAXSYSCALL 602 diff --git a/sys/sys/syscall.mk b/sys/sys/syscall.mk index ce29c050885e..e9d54983b5c4 100644 --- a/sys/sys/syscall.mk +++ b/sys/sys/syscall.mk @@ -441,4 +441,6 @@ MIASM = \ setgroups.o \ jail_attach_jd.o \ jail_remove_jd.o \ - kexec_load.o + kexec_load.o \ + pdrfork.o \ + pdwait.o diff --git a/sys/sys/syscallsubr.h b/sys/sys/syscallsubr.h index 4ddd2eba25c8..e2bbbc188553 100644 --- a/sys/sys/syscallsubr.h +++ b/sys/sys/syscallsubr.h @@ -286,6 +286,8 @@ int kern_posix_fallocate(struct thread *td, int fd, off_t offset, off_t len); int kern_fspacectl(struct thread *td, int fd, int cmd, const struct spacectl_range *, int flags, struct spacectl_range *); +int kern_pdwait(struct thread *td, int fd, int *status, + int options, struct __wrusage *wrusage, siginfo_t *sip); int kern_procctl(struct thread *td, enum idtype idtype, id_t id, int com, void *data); int kern_pread(struct thread *td, int fd, void *buf, size_t nbyte, diff --git a/sys/sys/sysproto.h b/sys/sys/sysproto.h index 5f5524a4519b..0496077bb555 100644 --- a/sys/sys/sysproto.h +++ b/sys/sys/sysproto.h @@ -1913,6 +1913,18 @@ struct kexec_load_args { char segments_l_[PADL_(struct kexec_segment *)]; struct kexec_segment * segments; char segments_r_[PADR_(struct kexec_segment *)]; char flags_l_[PADL_(u_long)]; u_long flags; char flags_r_[PADR_(u_long)]; }; +struct pdrfork_args { + char fdp_l_[PADL_(int *)]; int * fdp; char fdp_r_[PADR_(int *)]; + char pdflags_l_[PADL_(int)]; int pdflags; char pdflags_r_[PADR_(int)]; + char rfflags_l_[PADL_(int)]; int rfflags; char rfflags_r_[PADR_(int)]; +}; +struct pdwait_args { + char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)]; + char status_l_[PADL_(int *)]; int * status; char status_r_[PADR_(int *)]; + char options_l_[PADL_(int)]; int options; char options_r_[PADR_(int)]; + char wrusage_l_[PADL_(struct __wrusage *)]; struct __wrusage * wrusage; char wrusage_r_[PADR_(struct __wrusage *)]; + char info_l_[PADL_(struct __siginfo *)]; struct __siginfo * info; char info_r_[PADR_(struct __siginfo *)]; +}; int sys__exit(struct thread *, struct _exit_args *); int sys_fork(struct thread *, struct fork_args *); int sys_read(struct thread *, struct read_args *); @@ -2320,6 +2332,8 @@ int sys_setgroups(struct thread *, struct setgroups_args *); int sys_jail_attach_jd(struct thread *, struct jail_attach_jd_args *); int sys_jail_remove_jd(struct thread *, struct jail_remove_jd_args *); int sys_kexec_load(struct thread *, struct kexec_load_args *); +int sys_pdrfork(struct thread *, struct pdrfork_args *); +int sys_pdwait(struct thread *, struct pdwait_args *); #ifdef COMPAT_43 @@ -3319,6 +3333,8 @@ int freebsd14_setgroups(struct thread *, struct freebsd14_setgroups_args *); #define SYS_AUE_jail_attach_jd AUE_JAIL_ATTACH #define SYS_AUE_jail_remove_jd AUE_JAIL_REMOVE #define SYS_AUE_kexec_load AUE_NULL +#define SYS_AUE_pdrfork AUE_PDRFORK +#define SYS_AUE_pdwait AUE_PDWAIT #undef PAD_ #undef PADL_ diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c index ba16ae551093..1737020436c6 100644 --- a/sys/vm/vm_phys.c +++ b/sys/vm/vm_phys.c @@ -1213,7 +1213,7 @@ vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end) rw_wlock(&vm_phys_fictitious_reg_lock); seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); - if (seg->start != start || seg->end != end) { + if (seg == NULL || seg->start != start || seg->end != end) { rw_wunlock(&vm_phys_fictitious_reg_lock); panic( "Unregistering not registered fictitious range [%#jx:%#jx]", diff --git a/sys/x86/cpufreq/hwpstate_amd.c b/sys/x86/cpufreq/hwpstate_amd.c index 5676d8ce7dae..d8ad090a6a58 100644 --- a/sys/x86/cpufreq/hwpstate_amd.c +++ b/sys/x86/cpufreq/hwpstate_amd.c @@ -103,21 +103,23 @@ #define AMD_1AH_CUR_FID(msr) ((msr) & 0xFFF) -#define AMD_CPPC_CAPS_1_HIGH_PERF_BITS 0xff000000 +#define AMD_CPPC_CAPS_1_HIGHEST_PERF_BITS 0xff000000 #define AMD_CPPC_CAPS_1_NOMINAL_PERF_BITS 0x00ff0000 -#define AMD_CPPC_CAPS_1_LOW_NONLIN_PERF_BITS 0x0000ff00 -#define AMD_CPPC_CAPS_1_LOW_PERF_BITS 0x000000ff +#define AMD_CPPC_CAPS_1_EFFICIENT_PERF_BITS 0x0000ff00 +#define AMD_CPPC_CAPS_1_LOWEST_PERF_BITS 0x000000ff -#define AMD_CPPC_REQUEST_ENERGY_PERF_BITS 0xff000000 +#define AMD_CPPC_REQUEST_EPP_BITS 0xff000000 #define AMD_CPPC_REQUEST_DES_PERF_BITS 0x00ff0000 #define AMD_CPPC_REQUEST_MIN_PERF_BITS 0x0000ff00 #define AMD_CPPC_REQUEST_MAX_PERF_BITS 0x000000ff #define HWP_AMD_CLASSNAME "hwpstate_amd" -#define BITS_VALUE(bits, num) (((num) & (bits)) >> (ffsll((bits)) - 1)) -#define BITS_WITH_VALUE(bits, val) ((uintmax_t)(val) << (ffsll((bits)) - 1)) -#define SET_BITS_VALUE(var, bits, val) \ +#define BITS_VALUE(bits, val) \ + (((val) & (bits)) >> (ffsll((bits)) - 1)) +#define BITS_WITH_VALUE(bits, val) \ + (((uintmax_t)(val) << (ffsll((bits)) - 1)) & (bits)) +#define SET_BITS_VALUE(var, bits, val) \ ((var) = ((var) & ~(bits)) | BITS_WITH_VALUE((bits), (val))) #define HWPSTATE_DEBUG(dev, msg...) \ @@ -179,6 +181,11 @@ SYSCTL_BOOL(_machdep, OID_AUTO, hwpstate_pkg_ctrl, CTLFLAG_RDTUN, &hwpstate_pkg_ctrl_enable, 0, "Set 1 (default) to enable package-level control, 0 to disable"); +static bool hwpstate_amd_cppc_enable = true; +SYSCTL_BOOL(_machdep, OID_AUTO, hwpstate_amd_cppc_enable, CTLFLAG_RDTUN, + &hwpstate_amd_cppc_enable, 0, + "Set 1 (default) to enable AMD CPPC, 0 to disable"); + static device_method_t hwpstate_methods[] = { /* Device interface */ DEVMETHOD(device_identify, hwpstate_identify), @@ -252,13 +259,13 @@ amdhwp_dump_sysctl_handler(SYSCTL_HANDLER_ARGS) data = request.caps; sbuf_printf(sb, "\tHighest Performance: %03ju\n", - BITS_VALUE(AMD_CPPC_CAPS_1_HIGH_PERF_BITS, data)); + BITS_VALUE(AMD_CPPC_CAPS_1_HIGHEST_PERF_BITS, data)); sbuf_printf(sb, "\tGuaranteed Performance: %03ju\n", BITS_VALUE(AMD_CPPC_CAPS_1_NOMINAL_PERF_BITS, data)); sbuf_printf(sb, "\tEfficient Performance: %03ju\n", - BITS_VALUE(AMD_CPPC_CAPS_1_LOW_NONLIN_PERF_BITS, data)); + BITS_VALUE(AMD_CPPC_CAPS_1_EFFICIENT_PERF_BITS, data)); sbuf_printf(sb, "\tLowest Performance: %03ju\n", - BITS_VALUE(AMD_CPPC_CAPS_1_LOW_PERF_BITS, data)); + BITS_VALUE(AMD_CPPC_CAPS_1_LOWEST_PERF_BITS, data)); sbuf_putc(sb, '\n'); data = request.req; @@ -292,9 +299,9 @@ sysctl_epp_select_per_core(device_t hwp_device, uint32_t val) struct hwpstate_softc *sc; sc = device_get_softc(hwp_device); - if (BITS_VALUE(AMD_CPPC_REQUEST_ENERGY_PERF_BITS, sc->req) == val) + if (BITS_VALUE(AMD_CPPC_REQUEST_EPP_BITS, sc->req) == val) return; - SET_BITS_VALUE(sc->req, AMD_CPPC_REQUEST_ENERGY_PERF_BITS, val); + SET_BITS_VALUE(sc->req, AMD_CPPC_REQUEST_EPP_BITS, val); x86_msr_op(MSR_AMD_CPPC_REQUEST, MSR_OP_RENDEZVOUS_ONE | MSR_OP_WRITE | MSR_OP_CPUID(cpu_get_pcpu(hwp_device)->pc_cpuid), @@ -308,7 +315,7 @@ sysctl_epp_select(SYSCTL_HANDLER_ARGS) devclass_t dc; struct hwpstate_softc *sc; const uint32_t max_energy_perf = - BITS_VALUE(AMD_CPPC_REQUEST_ENERGY_PERF_BITS, (uint64_t)-1); + BITS_VALUE(AMD_CPPC_REQUEST_EPP_BITS, (uint64_t)-1); uint32_t val; int ret = 0; int cpu; @@ -319,7 +326,7 @@ sysctl_epp_select(SYSCTL_HANDLER_ARGS) if (!(sc->flags & PSTATE_CPPC)) return (ENODEV); - val = BITS_VALUE(AMD_CPPC_REQUEST_ENERGY_PERF_BITS, sc->req) * 100 / + val = BITS_VALUE(AMD_CPPC_REQUEST_EPP_BITS, sc->req) * 100 / max_energy_perf; ret = sysctl_handle_int(oidp, &val, 0, req); if (ret != 0 || req->newptr == NULL) @@ -624,11 +631,11 @@ amd_set_autonomous_hwp_cb(void *args) * is the balanced mode. For consistency, we set the same value in AMD's * CPPC driver. */ - SET_BITS_VALUE(sc->req, AMD_CPPC_REQUEST_ENERGY_PERF_BITS, 0x80); + SET_BITS_VALUE(sc->req, AMD_CPPC_REQUEST_EPP_BITS, 0x80); SET_BITS_VALUE(sc->req, AMD_CPPC_REQUEST_MIN_PERF_BITS, - BITS_VALUE(AMD_CPPC_CAPS_1_LOW_PERF_BITS, caps)); + BITS_VALUE(AMD_CPPC_CAPS_1_LOWEST_PERF_BITS, caps)); SET_BITS_VALUE(sc->req, AMD_CPPC_REQUEST_MAX_PERF_BITS, - BITS_VALUE(AMD_CPPC_CAPS_1_HIGH_PERF_BITS, caps)); + BITS_VALUE(AMD_CPPC_CAPS_1_HIGHEST_PERF_BITS, caps)); /* enable autonomous mode by setting desired performance to 0 */ SET_BITS_VALUE(sc->req, AMD_CPPC_REQUEST_DES_PERF_BITS, 0); @@ -667,7 +674,8 @@ hwpstate_probe(device_t dev) sc = device_get_softc(dev); - if (amd_extended_feature_extensions & AMDFEID_CPPC) { + if (hwpstate_amd_cppc_enable && + (amd_extended_feature_extensions & AMDFEID_CPPC)) { sc->flags |= PSTATE_CPPC; device_set_desc(dev, "AMD Collaborative Processor Performance Control (CPPC)"); diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h index 701b982e6afb..215fe0562465 100644 --- a/sys/x86/include/x86_var.h +++ b/sys/x86/include/x86_var.h @@ -171,6 +171,8 @@ uint64_t rdtsc_ordered(void); /* * Where and which execution mode + * + * All modes cause execution on the target CPU(s) with interrupts disabled. */ #define MSR_OP_LOCAL 0x10000000 #define MSR_OP_SCHED_ALL 0x20000000 diff --git a/sys/x86/x86/cpu_machdep.c b/sys/x86/x86/cpu_machdep.c index 5b4abfe71642..5f8965bd5614 100644 --- a/sys/x86/x86/cpu_machdep.c +++ b/sys/x86/x86/cpu_machdep.c @@ -47,7 +47,6 @@ #include "opt_kstack_pages.h" #include "opt_maxmem.h" #include "opt_platform.h" -#include "opt_sched.h" #ifdef __i386__ #include "opt_apic.h" #endif @@ -146,6 +145,8 @@ x86_msr_op_one(void *argp) v = rdmsr(a->msr); *a->res = v; break; + default: + __assert_unreachable(); } } @@ -159,22 +160,21 @@ x86_msr_op(u_int msr, u_int op, uint64_t arg1, uint64_t *res) struct thread *td; struct msr_op_arg a; cpuset_t set; + register_t flags; u_int exmode; int bound_cpu, cpu, i, is_bound; - a.op = op & MSR_OP_OP_MASK; - MPASS(a.op == MSR_OP_ANDNOT || a.op == MSR_OP_OR || - a.op == MSR_OP_WRITE || a.op == MSR_OP_READ); exmode = op & MSR_OP_EXMODE_MASK; - MPASS(exmode == MSR_OP_LOCAL || exmode == MSR_OP_SCHED_ALL || - exmode == MSR_OP_SCHED_ONE || exmode == MSR_OP_RENDEZVOUS_ALL || - exmode == MSR_OP_RENDEZVOUS_ONE); + a.op = op & MSR_OP_OP_MASK; a.msr = msr; a.arg1 = arg1; a.res = res; + switch (exmode) { case MSR_OP_LOCAL: + flags = intr_disable(); x86_msr_op_one(&a); + intr_restore(flags); break; case MSR_OP_SCHED_ALL: td = curthread; @@ -218,6 +218,8 @@ x86_msr_op(u_int msr, u_int op, uint64_t arg1, uint64_t *res) smp_rendezvous_cpus(set, smp_no_rendezvous_barrier, x86_msr_op_one, smp_no_rendezvous_barrier, &a); break; + default: + __assert_unreachable(); } } @@ -543,9 +545,7 @@ cpu_idle_enter(int *statep, int newstate) * is visible before calling cpu_idle_wakeup(). */ atomic_store_int(statep, newstate); -#if defined(SCHED_ULE) && defined(SMP) atomic_thread_fence_seq_cst(); -#endif /* * Since we may be in a critical section from cpu_idle(), if diff --git a/sys/x86/x86/local_apic.c b/sys/x86/x86/local_apic.c index b444142d5481..54026f83dc15 100644 --- a/sys/x86/x86/local_apic.c +++ b/sys/x86/x86/local_apic.c @@ -129,6 +129,7 @@ struct lvt { u_int lvt_mode:16; u_int lvt_vector:8; u_int lvt_reg; + const char *lvt_desc; }; struct lapic { @@ -158,6 +159,7 @@ static struct lvt lvts[] = { .lvt_mode = APIC_LVT_DM_EXTINT, .lvt_vector = 0, .lvt_reg = LAPIC_LVT_LINT0, + .lvt_desc = "LINT0", }, /* LINT1: NMI */ [APIC_LVT_LINT1] = { @@ -168,6 +170,7 @@ static struct lvt lvts[] = { .lvt_mode = APIC_LVT_DM_NMI, .lvt_vector = 0, .lvt_reg = LAPIC_LVT_LINT1, + .lvt_desc = "LINT1", }, [APIC_LVT_TIMER] = { .lvt_edgetrigger = 1, @@ -177,6 +180,7 @@ static struct lvt lvts[] = { .lvt_mode = APIC_LVT_DM_FIXED, .lvt_vector = APIC_TIMER_INT, .lvt_reg = LAPIC_LVT_TIMER, + .lvt_desc = "TIMER", }, [APIC_LVT_ERROR] = { .lvt_edgetrigger = 1, @@ -186,6 +190,7 @@ static struct lvt lvts[] = { .lvt_mode = APIC_LVT_DM_FIXED, .lvt_vector = APIC_ERROR_INT, .lvt_reg = LAPIC_LVT_ERROR, + .lvt_desc = "ERROR", }, [APIC_LVT_PMC] = { .lvt_edgetrigger = 1, @@ -195,6 +200,7 @@ static struct lvt lvts[] = { .lvt_mode = APIC_LVT_DM_NMI, .lvt_vector = 0, .lvt_reg = LAPIC_LVT_PCINT, + .lvt_desc = "PMC", }, [APIC_LVT_THERMAL] = { .lvt_edgetrigger = 1, @@ -204,6 +210,7 @@ static struct lvt lvts[] = { .lvt_mode = APIC_LVT_DM_FIXED, .lvt_vector = APIC_THERMAL_INT, .lvt_reg = LAPIC_LVT_THERMAL, + .lvt_desc = "THERM", }, [APIC_LVT_CMCI] = { .lvt_edgetrigger = 1, @@ -213,6 +220,7 @@ static struct lvt lvts[] = { .lvt_mode = APIC_LVT_DM_FIXED, .lvt_vector = APIC_CMC_INT, .lvt_reg = LAPIC_LVT_CMCI, + .lvt_desc = "CMCI", }, }; @@ -226,6 +234,7 @@ static struct lvt elvts[] = { .lvt_mode = APIC_LVT_DM_FIXED, .lvt_vector = 0, .lvt_reg = LAPIC_EXT_LVT0, + .lvt_desc = "ELVT0", }, [APIC_ELVT_MCA] = { .lvt_edgetrigger = 1, @@ -235,6 +244,7 @@ static struct lvt elvts[] = { .lvt_mode = APIC_LVT_DM_FIXED, .lvt_vector = APIC_CMC_INT, .lvt_reg = LAPIC_EXT_LVT1, + .lvt_desc = "MCA", }, [APIC_ELVT_DEI] = { .lvt_edgetrigger = 1, @@ -244,6 +254,7 @@ static struct lvt elvts[] = { .lvt_mode = APIC_LVT_DM_FIXED, .lvt_vector = 0, .lvt_reg = LAPIC_EXT_LVT2, + .lvt_desc = "ELVT2", }, [APIC_ELVT_SBI] = { .lvt_edgetrigger = 1, @@ -253,6 +264,7 @@ static struct lvt elvts[] = { .lvt_mode = APIC_LVT_DM_FIXED, .lvt_vector = 0, .lvt_reg = LAPIC_EXT_LVT3, + .lvt_desc = "ELVT3", }, }; @@ -1431,21 +1443,8 @@ lapic_handle_timer(struct trapframe *frame) kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED); trap_check_kstack(); -#if defined(SMP) && !defined(SCHED_ULE) - /* - * Don't do any accounting for the disabled HTT cores, since it - * will provide misleading numbers for the userland. - * - * No locking is necessary here, since even if we lose the race - * when hlt_cpus_mask changes it is not a big deal, really. - * - * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask - * and unlike other schedulers it actually schedules threads to - * those CPUs. - */ - if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask)) + if (!sched_do_timer_accounting()) return; -#endif /* Look up our local APIC structure for the tick counters. */ la = &lapics[PCPU_GET(apic_id)]; @@ -1890,18 +1889,34 @@ dump_mask(const char *prefix, uint32_t v, int base) /* Show info from the lapic regs for this CPU. */ DB_SHOW_COMMAND_FLAGS(lapic, db_show_lapic, DB_CMD_MEMSAFE) { - uint32_t v; + const struct lvt *l; + int elvt_count, lvts_count, i; + uint32_t v, vr; db_printf("lapic ID = %d\n", lapic_id()); v = lapic_read32(LAPIC_VERSION); - db_printf("version = %d.%d\n", (v & APIC_VER_VERSION) >> 4, - v & 0xf); + db_printf("version = %d.%d (%#x) \n", (v & APIC_VER_VERSION) >> 4, + v & 0xf, v); db_printf("max LVT = %d\n", lapic_maxlvt(v)); - v = lapic_read32(LAPIC_SVR); - db_printf("SVR = %02x (%s)\n", v & APIC_SVR_VECTOR, - v & APIC_SVR_ENABLE ? "enabled" : "disabled"); + vr = lapic_read32(LAPIC_SVR); + db_printf("SVR = %02x (%s)\n", vr & APIC_SVR_VECTOR, + vr & APIC_SVR_ENABLE ? "enabled" : "disabled"); db_printf("TPR = %02x\n", lapic_read32(LAPIC_TPR)); + lvts_count = min(nitems(lvts), lapic_maxlvt(v) + 1); + for (i = 0; i < lvts_count; i++) { + l = &lvts[i]; + db_printf("LVT%d (reg %#x %-5s) = %#010x\n", i, l->lvt_reg, + l->lvt_desc, lapic_read32(l->lvt_reg)); + } + + elvt_count = amd_read_elvt_count(); + for (i = 0; i < elvt_count; i++) { + l = &elvts[i]; + db_printf("ELVT%d (reg %#x %-5s) = %#010x\n", i, l->lvt_reg, + l->lvt_desc, lapic_read32(l->lvt_reg)); + } + #define dump_field(prefix, regn, index) \ dump_mask(__XSTRING(prefix ## index), \ lapic_read32(LAPIC_ ## regn ## index), \ diff --git a/sys/xdr/xdr.c b/sys/xdr/xdr.c index 81d238ebf19f..f983a474abdd 100644 --- a/sys/xdr/xdr.c +++ b/sys/xdr/xdr.c @@ -620,6 +620,13 @@ xdr_string(XDR *xdrs, char **cpp, u_int maxsize) if (sp == NULL) { return(TRUE); /* already free */ } + /* + * XXX: buggy software may call this without a third + * argument via xdr_free(). Ignore maxsize since it may + * be invalid. Otherwise, if it's very small, we might + * fail to free the string. + */ + maxsize = RPC_MAXDATASIZE; /* FALLTHROUGH */ case XDR_ENCODE: size = strlen(sp); |