diff options
Diffstat (limited to 'sys/arm64/arm64/gic_v3.c')
-rw-r--r-- | sys/arm64/arm64/gic_v3.c | 315 |
1 files changed, 229 insertions, 86 deletions
diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c index 27f41c58fe92..b57dd9be48aa 100644 --- a/sys/arm64/arm64/gic_v3.c +++ b/sys/arm64/arm64/gic_v3.c @@ -32,9 +32,6 @@ #include "opt_acpi.h" #include "opt_platform.h" -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - #include <sys/param.h> #include <sys/systm.h> #include <sys/bitstring.h> @@ -77,9 +74,11 @@ __FBSDID("$FreeBSD$"); #include "gic_v3_reg.h" #include "gic_v3_var.h" +static bus_print_child_t gic_v3_print_child; static bus_get_domain_t gic_v3_get_domain; static bus_read_ivar_t gic_v3_read_ivar; static bus_write_ivar_t gic_v3_write_ivar; +static bus_alloc_resource_t gic_v3_alloc_resource; static pic_disable_intr_t gic_v3_disable_intr; static pic_enable_intr_t gic_v3_enable_intr; @@ -119,9 +118,12 @@ static device_method_t gic_v3_methods[] = { DEVMETHOD(device_detach, gic_v3_detach), /* Bus interface */ + DEVMETHOD(bus_print_child, gic_v3_print_child), DEVMETHOD(bus_get_domain, gic_v3_get_domain), DEVMETHOD(bus_read_ivar, gic_v3_read_ivar), DEVMETHOD(bus_write_ivar, gic_v3_write_ivar), + DEVMETHOD(bus_alloc_resource, gic_v3_alloc_resource), + DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, gic_v3_disable_intr), @@ -221,7 +223,8 @@ gic_r_read_4(device_t dev, bus_size_t offset) struct resource *rdist; sc = device_get_softc(dev); - rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res; + rdist = sc->gic_redists.pcpu[PCPU_GET(cpuid)].res; + offset += sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset; return (bus_read_4(rdist, offset)); } @@ -232,7 +235,8 @@ gic_r_read_8(device_t dev, bus_size_t offset) struct resource *rdist; sc = device_get_softc(dev); - rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res; + rdist = sc->gic_redists.pcpu[PCPU_GET(cpuid)].res; + offset += sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset; return (bus_read_8(rdist, offset)); } @@ -243,7 +247,8 @@ gic_r_write_4(device_t dev, bus_size_t offset, uint32_t val) struct resource *rdist; sc = device_get_softc(dev); - rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res; + rdist = sc->gic_redists.pcpu[PCPU_GET(cpuid)].res; + offset += sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset; bus_write_4(rdist, offset, val); } @@ -254,7 +259,8 @@ gic_r_write_8(device_t dev, bus_size_t offset, uint64_t val) struct resource *rdist; sc = device_get_softc(dev); - rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res; + rdist = sc->gic_redists.pcpu[PCPU_GET(cpuid)].res; + offset += sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset; bus_write_8(rdist, offset, val); } @@ -377,6 +383,13 @@ gic_v3_attach(device_t dev) mtx_init(&sc->gic_mbi_mtx, "GICv3 mbi lock", NULL, MTX_DEF); if (sc->gic_mbi_start > 0) { + if (!sc->gic_mbi_end) { + /* + * This is to address SPI based msi ranges, where + * SPI range is not specified in ACPI + */ + sc->gic_mbi_end = sc->gic_nirqs - 1; + } gic_v3_reserve_msi_range(dev, sc->gic_mbi_start, sc->gic_mbi_end - sc->gic_mbi_start); @@ -415,7 +428,6 @@ int gic_v3_detach(device_t dev) { struct gic_v3_softc *sc; - size_t i; int rid; sc = device_get_softc(dev); @@ -430,9 +442,9 @@ gic_v3_detach(device_t dev) for (rid = 0; rid < (sc->gic_redists.nregions + 1); rid++) bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->gic_res[rid]); - for (i = 0; i <= mp_maxid; i++) - free(sc->gic_redists.pcpu[i], M_GIC_V3); + free(sc->gic_redists.pcpu, M_GIC_V3); + free(sc->ranges, M_GIC_V3); free(sc->gic_res, M_GIC_V3); free(sc->gic_redists.regions, M_GIC_V3); @@ -440,6 +452,21 @@ gic_v3_detach(device_t dev) } static int +gic_v3_print_child(device_t bus, device_t child) +{ + struct resource_list *rl; + int retval = 0; + + rl = BUS_GET_RESOURCE_LIST(bus, child); + KASSERT(rl != NULL, ("%s: No resource list", __func__)); + retval += bus_print_child_header(bus, child); + retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx"); + retval += bus_print_child_footer(bus, child); + + return (retval); +} + +static int gic_v3_get_domain(device_t dev, device_t child, int *domain) { struct gic_v3_devinfo *di; @@ -456,6 +483,7 @@ static int gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct gic_v3_softc *sc; + struct gic_v3_devinfo *di; sc = device_get_softc(dev); @@ -464,7 +492,7 @@ gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) *result = (intr_nirq - sc->gic_nirqs) / sc->gic_nchildren; return (0); case GICV3_IVAR_REDIST: - *result = (uintptr_t)sc->gic_redists.pcpu[PCPU_GET(cpuid)]; + *result = (uintptr_t)&sc->gic_redists.pcpu[PCPU_GET(cpuid)]; return (0); case GIC_IVAR_HW_REV: KASSERT( @@ -481,6 +509,12 @@ gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) ("gic_v3_read_ivar: Invalid bus type %u", sc->gic_bus)); *result = sc->gic_bus; return (0); + case GIC_IVAR_VGIC: + di = device_get_ivars(child); + if (di == NULL) + return (EINVAL); + *result = di->is_vgic; + return (0); } return (ENOENT); @@ -500,6 +534,59 @@ gic_v3_write_ivar(device_t dev, device_t child, int which, uintptr_t value) return (ENOENT); } +static struct resource * +gic_v3_alloc_resource(device_t bus, device_t child, int type, int *rid, + rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) +{ + struct gic_v3_softc *sc; + struct resource_list_entry *rle; + struct resource_list *rl; + int j; + + /* We only allocate memory */ + if (type != SYS_RES_MEMORY) + return (NULL); + + sc = device_get_softc(bus); + + if (RMAN_IS_DEFAULT_RANGE(start, end)) { + rl = BUS_GET_RESOURCE_LIST(bus, child); + if (rl == NULL) + return (NULL); + + /* Find defaults for this rid */ + rle = resource_list_find(rl, type, *rid); + if (rle == NULL) + return (NULL); + + start = rle->start; + end = rle->end; + count = rle->count; + } + + /* Remap through ranges property */ + for (j = 0; j < sc->nranges; j++) { + if (start >= sc->ranges[j].bus && end < + sc->ranges[j].bus + sc->ranges[j].size) { + start -= sc->ranges[j].bus; + start += sc->ranges[j].host; + end -= sc->ranges[j].bus; + end += sc->ranges[j].host; + break; + } + } + if (j == sc->nranges && sc->nranges != 0) { + if (bootverbose) + device_printf(bus, "Could not map resource " + "%#jx-%#jx\n", (uintmax_t)start, (uintmax_t)end); + + return (NULL); + } + + return (bus_generic_alloc_resource(bus, child, type, rid, start, end, + count, flags)); +} + int arm_gic_v3_intr(void *arg) { @@ -543,7 +630,7 @@ arm_gic_v3_intr(void *arg) /* Call EOI for all IPI before dispatch. */ gic_icc_write(EOIR1, (uint64_t)active_irq); #ifdef SMP - intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf); + intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq]); #else device_printf(sc->dev, "SGI %ju on UP system detected\n", (uintmax_t)(active_irq - GIC_FIRST_SGI)); @@ -747,15 +834,66 @@ gic_v3_map_intr(device_t dev, struct intr_map_data *data, return (error); } +struct gic_v3_setup_periph_args { + device_t dev; + struct intr_irqsrc *isrc; +}; + +static void +gic_v3_setup_intr_periph(void *argp) +{ + struct gic_v3_setup_periph_args *args = argp; + struct intr_irqsrc *isrc = args->isrc; + struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; + device_t dev = args->dev; + u_int irq = gi->gi_irq; + struct gic_v3_softc *sc = device_get_softc(dev); + uint32_t reg; + + MPASS(irq <= GIC_LAST_SPI); + + /* + * We need the lock for both SGIs and PPIs for an atomic CPU_SET() at a + * minimum, but we also need it below for SPIs. + */ + mtx_lock_spin(&sc->gic_mtx); + + if (isrc->isrc_flags & INTR_ISRCF_PPI) + CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); + + if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_SPI) { + /* Set the trigger and polarity */ + if (irq <= GIC_LAST_PPI) + reg = gic_r_read(sc, 4, + GICR_SGI_BASE_SIZE + GICD_ICFGR(irq)); + else + reg = gic_d_read(sc, 4, GICD_ICFGR(irq)); + if (gi->gi_trig == INTR_TRIGGER_LEVEL) + reg &= ~(2 << ((irq % 16) * 2)); + else + reg |= 2 << ((irq % 16) * 2); + + if (irq <= GIC_LAST_PPI) { + gic_r_write(sc, 4, + GICR_SGI_BASE_SIZE + GICD_ICFGR(irq), reg); + gic_v3_wait_for_rwp(sc, REDIST); + } else { + gic_d_write(sc, 4, GICD_ICFGR(irq), reg); + gic_v3_wait_for_rwp(sc, DIST); + } + } + + mtx_unlock_spin(&sc->gic_mtx); +} + static int gic_v3_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { - struct gic_v3_softc *sc = device_get_softc(dev); struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; + struct gic_v3_setup_periph_args pargs; enum intr_trigger trig; enum intr_polarity pol; - uint32_t reg; u_int irq; int error; @@ -784,41 +922,18 @@ gic_v3_setup_intr(device_t dev, struct intr_irqsrc *isrc, gi->gi_trig = trig; } - /* - * XXX - In case that per CPU interrupt is going to be enabled in time - * when SMP is already started, we need some IPI call which - * enables it on others CPUs. Further, it's more complicated as - * pic_enable_source() and pic_disable_source() should act on - * per CPU basis only. Thus, it should be solved here somehow. - */ - if (isrc->isrc_flags & INTR_ISRCF_PPI) - CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); - - if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_SPI) { - mtx_lock_spin(&sc->gic_mtx); - - /* Set the trigger and polarity */ - if (irq <= GIC_LAST_PPI) - reg = gic_r_read(sc, 4, - GICR_SGI_BASE_SIZE + GICD_ICFGR(irq)); - else - reg = gic_d_read(sc, 4, GICD_ICFGR(irq)); - if (trig == INTR_TRIGGER_LEVEL) - reg &= ~(2 << ((irq % 16) * 2)); - else - reg |= 2 << ((irq % 16) * 2); - - if (irq <= GIC_LAST_PPI) { - gic_r_write(sc, 4, - GICR_SGI_BASE_SIZE + GICD_ICFGR(irq), reg); - gic_v3_wait_for_rwp(sc, REDIST); - } else { - gic_d_write(sc, 4, GICD_ICFGR(irq), reg); - gic_v3_wait_for_rwp(sc, DIST); - } - - mtx_unlock_spin(&sc->gic_mtx); + pargs.dev = dev; + pargs.isrc = isrc; + if (isrc->isrc_flags & INTR_ISRCF_PPI) { + /* + * If APs haven't been fired up yet, smp_rendezvous() will just + * execute it on the single CPU and gic_v3_init_secondary() will + * clean up afterwards. + */ + smp_rendezvous(NULL, gic_v3_setup_intr_periph, NULL, &pargs); + } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { + gic_v3_setup_intr_periph(&pargs); gic_v3_bind_intr(dev, isrc); } @@ -864,22 +979,49 @@ gic_v3_disable_intr(device_t dev, struct intr_irqsrc *isrc) } static void +gic_v3_enable_intr_periph(void *argp) +{ + struct gic_v3_setup_periph_args *args = argp; + struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)args->isrc; + device_t dev = args->dev; + struct gic_v3_softc *sc = device_get_softc(dev); + u_int irq = gi->gi_irq; + + /* SGIs and PPIs in corresponding Re-Distributor */ + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq), + GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, REDIST); +} + +static void gic_v3_enable_intr(device_t dev, struct intr_irqsrc *isrc) { + struct gic_v3_setup_periph_args pargs; struct gic_v3_softc *sc; struct gic_v3_irqsrc *gi; u_int irq; - sc = device_get_softc(dev); gi = (struct gic_v3_irqsrc *)isrc; irq = gi->gi_irq; + pargs.isrc = isrc; + pargs.dev = dev; if (irq <= GIC_LAST_PPI) { - /* SGIs and PPIs in corresponding Re-Distributor */ - gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq), - GICD_I_MASK(irq)); - gic_v3_wait_for_rwp(sc, REDIST); - } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { + /* + * SGIs only need configured on the current AP. We'll setup and + * enable IPIs as APs come online. + */ + if (irq <= GIC_LAST_SGI) + gic_v3_enable_intr_periph(&pargs); + else + smp_rendezvous(NULL, gic_v3_enable_intr_periph, NULL, + &pargs); + return; + } + + sc = device_get_softc(dev); + + if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, DIST); @@ -922,8 +1064,6 @@ gic_v3_bind_intr(device_t dev, struct intr_irqsrc *isrc) int cpu; gi = (struct gic_v3_irqsrc *)isrc; - if (gi->gi_irq <= GIC_LAST_PPI) - return (EINVAL); KASSERT(gi->gi_irq >= GIC_FIRST_SPI && gi->gi_irq <= GIC_LAST_SPI, ("%s: Attempting to bind an invalid IRQ", __func__)); @@ -951,6 +1091,7 @@ gic_v3_bind_intr(device_t dev, struct intr_irqsrc *isrc) static void gic_v3_init_secondary(device_t dev) { + struct gic_v3_setup_periph_args pargs; device_t child; struct gic_v3_softc *sc; gic_v3_initseq_t *init_func; @@ -972,18 +1113,25 @@ gic_v3_init_secondary(device_t dev) } } + pargs.dev = dev; + /* Unmask attached SGI interrupts. */ for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) { isrc = GIC_INTR_ISRC(sc, irq); - if (intr_isrc_init_on_cpu(isrc, cpu)) - gic_v3_enable_intr(dev, isrc); + if (intr_isrc_init_on_cpu(isrc, cpu)) { + pargs.isrc = isrc; + gic_v3_enable_intr_periph(&pargs); + } } /* Unmask attached PPI interrupts. */ for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) { isrc = GIC_INTR_ISRC(sc, irq); - if (intr_isrc_init_on_cpu(isrc, cpu)) - gic_v3_enable_intr(dev, isrc); + if (intr_isrc_init_on_cpu(isrc, cpu)) { + pargs.isrc = isrc; + gic_v3_setup_intr_periph(&pargs); + gic_v3_enable_intr_periph(&pargs); + } } for (i = 0; i < sc->gic_nchildren; i++) { @@ -1066,6 +1214,7 @@ static void gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist) { struct resource *res; + bus_size_t offset; u_int cpuid; size_t us_left = 1000000; @@ -1074,16 +1223,18 @@ gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist) switch (xdist) { case DIST: res = sc->gic_dist; + offset = 0; break; case REDIST: - res = &sc->gic_redists.pcpu[cpuid]->res; + res = sc->gic_redists.pcpu[cpuid].res; + offset = sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset; break; default: KASSERT(0, ("%s: Attempt to wait for unknown RWP", __func__)); return; } - while ((bus_read_4(res, GICD_CTLR) & GICD_CTLR_RWP) != 0) { + while ((bus_read_4(res, offset + GICD_CTLR) & GICD_CTLR_RWP) != 0) { DELAY(1); if (us_left-- == 0) panic("GICD Register write pending for too long"); @@ -1212,24 +1363,16 @@ gic_v3_dist_init(struct gic_v3_softc *sc) static int gic_v3_redist_alloc(struct gic_v3_softc *sc) { - u_int cpuid; - - /* Allocate struct resource for all CPU's Re-Distributor registers */ - for (cpuid = 0; cpuid <= mp_maxid; cpuid++) - if (CPU_ISSET(cpuid, &all_cpus) != 0) - sc->gic_redists.pcpu[cpuid] = - malloc(sizeof(*sc->gic_redists.pcpu[0]), - M_GIC_V3, M_WAITOK); - else - sc->gic_redists.pcpu[cpuid] = NULL; + sc->gic_redists.pcpu = mallocarray(mp_maxid + 1, + sizeof(sc->gic_redists.pcpu[0]), M_GIC_V3, M_WAITOK); return (0); } static int gic_v3_redist_find(struct gic_v3_softc *sc) { - struct resource r_res; - bus_space_handle_t r_bsh; + struct resource *r_res; + bus_size_t offset; uint64_t aff; uint64_t typer; uint32_t pidr2; @@ -1250,10 +1393,9 @@ gic_v3_redist_find(struct gic_v3_softc *sc) /* Iterate through Re-Distributor regions */ for (i = 0; i < sc->gic_redists.nregions; i++) { /* Take a copy of the region's resource */ - r_res = *sc->gic_redists.regions[i]; - r_bsh = rman_get_bushandle(&r_res); + r_res = sc->gic_redists.regions[i]; - pidr2 = bus_read_4(&r_res, GICR_PIDR2); + pidr2 = bus_read_4(r_res, GICR_PIDR2); switch (GICR_PIDR2_ARCH(pidr2)) { case GICR_PIDR2_ARCH_GICv3: /* fall through */ case GICR_PIDR2_ARCH_GICv4: @@ -1264,14 +1406,16 @@ gic_v3_redist_find(struct gic_v3_softc *sc) return (ENODEV); } + offset = 0; do { - typer = bus_read_8(&r_res, GICR_TYPER); + typer = bus_read_8(r_res, offset + GICR_TYPER); if ((typer >> GICR_TYPER_AFF_SHIFT) == aff) { - KASSERT(sc->gic_redists.pcpu[cpuid] != NULL, + KASSERT(cpuid <= mp_maxid, ("Invalid pointer to per-CPU redistributor")); /* Copy res contents to its final destination */ - sc->gic_redists.pcpu[cpuid]->res = r_res; - sc->gic_redists.pcpu[cpuid]->lpi_enabled = false; + sc->gic_redists.pcpu[cpuid].res = r_res; + sc->gic_redists.pcpu[cpuid].offset = offset; + sc->gic_redists.pcpu[cpuid].lpi_enabled = false; if (bootverbose) { device_printf(sc->dev, "CPU%u Re-Distributor has been found\n", @@ -1280,14 +1424,13 @@ gic_v3_redist_find(struct gic_v3_softc *sc) return (0); } - r_bsh += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); + offset += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); if ((typer & GICR_TYPER_VLPIS) != 0) { - r_bsh += + offset += (GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE); } - - rman_set_bushandle(&r_res, r_bsh); - } while ((typer & GICR_TYPER_LAST) == 0); + } while (offset < rman_get_size(r_res) && + (typer & GICR_TYPER_LAST) == 0); } device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); |