aboutsummaryrefslogtreecommitdiff
path: root/sys/arm64/arm64/gic_v3.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/arm64/arm64/gic_v3.c')
-rw-r--r--sys/arm64/arm64/gic_v3.c315
1 files changed, 229 insertions, 86 deletions
diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c
index 27f41c58fe92..b57dd9be48aa 100644
--- a/sys/arm64/arm64/gic_v3.c
+++ b/sys/arm64/arm64/gic_v3.c
@@ -32,9 +32,6 @@
#include "opt_acpi.h"
#include "opt_platform.h"
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bitstring.h>
@@ -77,9 +74,11 @@ __FBSDID("$FreeBSD$");
#include "gic_v3_reg.h"
#include "gic_v3_var.h"
+static bus_print_child_t gic_v3_print_child;
static bus_get_domain_t gic_v3_get_domain;
static bus_read_ivar_t gic_v3_read_ivar;
static bus_write_ivar_t gic_v3_write_ivar;
+static bus_alloc_resource_t gic_v3_alloc_resource;
static pic_disable_intr_t gic_v3_disable_intr;
static pic_enable_intr_t gic_v3_enable_intr;
@@ -119,9 +118,12 @@ static device_method_t gic_v3_methods[] = {
DEVMETHOD(device_detach, gic_v3_detach),
/* Bus interface */
+ DEVMETHOD(bus_print_child, gic_v3_print_child),
DEVMETHOD(bus_get_domain, gic_v3_get_domain),
DEVMETHOD(bus_read_ivar, gic_v3_read_ivar),
DEVMETHOD(bus_write_ivar, gic_v3_write_ivar),
+ DEVMETHOD(bus_alloc_resource, gic_v3_alloc_resource),
+ DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
/* Interrupt controller interface */
DEVMETHOD(pic_disable_intr, gic_v3_disable_intr),
@@ -221,7 +223,8 @@ gic_r_read_4(device_t dev, bus_size_t offset)
struct resource *rdist;
sc = device_get_softc(dev);
- rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res;
+ rdist = sc->gic_redists.pcpu[PCPU_GET(cpuid)].res;
+ offset += sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset;
return (bus_read_4(rdist, offset));
}
@@ -232,7 +235,8 @@ gic_r_read_8(device_t dev, bus_size_t offset)
struct resource *rdist;
sc = device_get_softc(dev);
- rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res;
+ rdist = sc->gic_redists.pcpu[PCPU_GET(cpuid)].res;
+ offset += sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset;
return (bus_read_8(rdist, offset));
}
@@ -243,7 +247,8 @@ gic_r_write_4(device_t dev, bus_size_t offset, uint32_t val)
struct resource *rdist;
sc = device_get_softc(dev);
- rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res;
+ rdist = sc->gic_redists.pcpu[PCPU_GET(cpuid)].res;
+ offset += sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset;
bus_write_4(rdist, offset, val);
}
@@ -254,7 +259,8 @@ gic_r_write_8(device_t dev, bus_size_t offset, uint64_t val)
struct resource *rdist;
sc = device_get_softc(dev);
- rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res;
+ rdist = sc->gic_redists.pcpu[PCPU_GET(cpuid)].res;
+ offset += sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset;
bus_write_8(rdist, offset, val);
}
@@ -377,6 +383,13 @@ gic_v3_attach(device_t dev)
mtx_init(&sc->gic_mbi_mtx, "GICv3 mbi lock", NULL, MTX_DEF);
if (sc->gic_mbi_start > 0) {
+ if (!sc->gic_mbi_end) {
+ /*
+ * This is to address SPI based msi ranges, where
+ * SPI range is not specified in ACPI
+ */
+ sc->gic_mbi_end = sc->gic_nirqs - 1;
+ }
gic_v3_reserve_msi_range(dev, sc->gic_mbi_start,
sc->gic_mbi_end - sc->gic_mbi_start);
@@ -415,7 +428,6 @@ int
gic_v3_detach(device_t dev)
{
struct gic_v3_softc *sc;
- size_t i;
int rid;
sc = device_get_softc(dev);
@@ -430,9 +442,9 @@ gic_v3_detach(device_t dev)
for (rid = 0; rid < (sc->gic_redists.nregions + 1); rid++)
bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->gic_res[rid]);
- for (i = 0; i <= mp_maxid; i++)
- free(sc->gic_redists.pcpu[i], M_GIC_V3);
+ free(sc->gic_redists.pcpu, M_GIC_V3);
+ free(sc->ranges, M_GIC_V3);
free(sc->gic_res, M_GIC_V3);
free(sc->gic_redists.regions, M_GIC_V3);
@@ -440,6 +452,21 @@ gic_v3_detach(device_t dev)
}
static int
+gic_v3_print_child(device_t bus, device_t child)
+{
+ struct resource_list *rl;
+ int retval = 0;
+
+ rl = BUS_GET_RESOURCE_LIST(bus, child);
+ KASSERT(rl != NULL, ("%s: No resource list", __func__));
+ retval += bus_print_child_header(bus, child);
+ retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
+ retval += bus_print_child_footer(bus, child);
+
+ return (retval);
+}
+
+static int
gic_v3_get_domain(device_t dev, device_t child, int *domain)
{
struct gic_v3_devinfo *di;
@@ -456,6 +483,7 @@ static int
gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
{
struct gic_v3_softc *sc;
+ struct gic_v3_devinfo *di;
sc = device_get_softc(dev);
@@ -464,7 +492,7 @@ gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
*result = (intr_nirq - sc->gic_nirqs) / sc->gic_nchildren;
return (0);
case GICV3_IVAR_REDIST:
- *result = (uintptr_t)sc->gic_redists.pcpu[PCPU_GET(cpuid)];
+ *result = (uintptr_t)&sc->gic_redists.pcpu[PCPU_GET(cpuid)];
return (0);
case GIC_IVAR_HW_REV:
KASSERT(
@@ -481,6 +509,12 @@ gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
("gic_v3_read_ivar: Invalid bus type %u", sc->gic_bus));
*result = sc->gic_bus;
return (0);
+ case GIC_IVAR_VGIC:
+ di = device_get_ivars(child);
+ if (di == NULL)
+ return (EINVAL);
+ *result = di->is_vgic;
+ return (0);
}
return (ENOENT);
@@ -500,6 +534,59 @@ gic_v3_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
return (ENOENT);
}
+static struct resource *
+gic_v3_alloc_resource(device_t bus, device_t child, int type, int *rid,
+ rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
+{
+ struct gic_v3_softc *sc;
+ struct resource_list_entry *rle;
+ struct resource_list *rl;
+ int j;
+
+ /* We only allocate memory */
+ if (type != SYS_RES_MEMORY)
+ return (NULL);
+
+ sc = device_get_softc(bus);
+
+ if (RMAN_IS_DEFAULT_RANGE(start, end)) {
+ rl = BUS_GET_RESOURCE_LIST(bus, child);
+ if (rl == NULL)
+ return (NULL);
+
+ /* Find defaults for this rid */
+ rle = resource_list_find(rl, type, *rid);
+ if (rle == NULL)
+ return (NULL);
+
+ start = rle->start;
+ end = rle->end;
+ count = rle->count;
+ }
+
+ /* Remap through ranges property */
+ for (j = 0; j < sc->nranges; j++) {
+ if (start >= sc->ranges[j].bus && end <
+ sc->ranges[j].bus + sc->ranges[j].size) {
+ start -= sc->ranges[j].bus;
+ start += sc->ranges[j].host;
+ end -= sc->ranges[j].bus;
+ end += sc->ranges[j].host;
+ break;
+ }
+ }
+ if (j == sc->nranges && sc->nranges != 0) {
+ if (bootverbose)
+ device_printf(bus, "Could not map resource "
+ "%#jx-%#jx\n", (uintmax_t)start, (uintmax_t)end);
+
+ return (NULL);
+ }
+
+ return (bus_generic_alloc_resource(bus, child, type, rid, start, end,
+ count, flags));
+}
+
int
arm_gic_v3_intr(void *arg)
{
@@ -543,7 +630,7 @@ arm_gic_v3_intr(void *arg)
/* Call EOI for all IPI before dispatch. */
gic_icc_write(EOIR1, (uint64_t)active_irq);
#ifdef SMP
- intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf);
+ intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq]);
#else
device_printf(sc->dev, "SGI %ju on UP system detected\n",
(uintmax_t)(active_irq - GIC_FIRST_SGI));
@@ -747,15 +834,66 @@ gic_v3_map_intr(device_t dev, struct intr_map_data *data,
return (error);
}
+struct gic_v3_setup_periph_args {
+ device_t dev;
+ struct intr_irqsrc *isrc;
+};
+
+static void
+gic_v3_setup_intr_periph(void *argp)
+{
+ struct gic_v3_setup_periph_args *args = argp;
+ struct intr_irqsrc *isrc = args->isrc;
+ struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc;
+ device_t dev = args->dev;
+ u_int irq = gi->gi_irq;
+ struct gic_v3_softc *sc = device_get_softc(dev);
+ uint32_t reg;
+
+ MPASS(irq <= GIC_LAST_SPI);
+
+ /*
+ * We need the lock for both SGIs and PPIs for an atomic CPU_SET() at a
+ * minimum, but we also need it below for SPIs.
+ */
+ mtx_lock_spin(&sc->gic_mtx);
+
+ if (isrc->isrc_flags & INTR_ISRCF_PPI)
+ CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
+
+ if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_SPI) {
+ /* Set the trigger and polarity */
+ if (irq <= GIC_LAST_PPI)
+ reg = gic_r_read(sc, 4,
+ GICR_SGI_BASE_SIZE + GICD_ICFGR(irq));
+ else
+ reg = gic_d_read(sc, 4, GICD_ICFGR(irq));
+ if (gi->gi_trig == INTR_TRIGGER_LEVEL)
+ reg &= ~(2 << ((irq % 16) * 2));
+ else
+ reg |= 2 << ((irq % 16) * 2);
+
+ if (irq <= GIC_LAST_PPI) {
+ gic_r_write(sc, 4,
+ GICR_SGI_BASE_SIZE + GICD_ICFGR(irq), reg);
+ gic_v3_wait_for_rwp(sc, REDIST);
+ } else {
+ gic_d_write(sc, 4, GICD_ICFGR(irq), reg);
+ gic_v3_wait_for_rwp(sc, DIST);
+ }
+ }
+
+ mtx_unlock_spin(&sc->gic_mtx);
+}
+
static int
gic_v3_setup_intr(device_t dev, struct intr_irqsrc *isrc,
struct resource *res, struct intr_map_data *data)
{
- struct gic_v3_softc *sc = device_get_softc(dev);
struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc;
+ struct gic_v3_setup_periph_args pargs;
enum intr_trigger trig;
enum intr_polarity pol;
- uint32_t reg;
u_int irq;
int error;
@@ -784,41 +922,18 @@ gic_v3_setup_intr(device_t dev, struct intr_irqsrc *isrc,
gi->gi_trig = trig;
}
- /*
- * XXX - In case that per CPU interrupt is going to be enabled in time
- * when SMP is already started, we need some IPI call which
- * enables it on others CPUs. Further, it's more complicated as
- * pic_enable_source() and pic_disable_source() should act on
- * per CPU basis only. Thus, it should be solved here somehow.
- */
- if (isrc->isrc_flags & INTR_ISRCF_PPI)
- CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
-
- if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_SPI) {
- mtx_lock_spin(&sc->gic_mtx);
-
- /* Set the trigger and polarity */
- if (irq <= GIC_LAST_PPI)
- reg = gic_r_read(sc, 4,
- GICR_SGI_BASE_SIZE + GICD_ICFGR(irq));
- else
- reg = gic_d_read(sc, 4, GICD_ICFGR(irq));
- if (trig == INTR_TRIGGER_LEVEL)
- reg &= ~(2 << ((irq % 16) * 2));
- else
- reg |= 2 << ((irq % 16) * 2);
-
- if (irq <= GIC_LAST_PPI) {
- gic_r_write(sc, 4,
- GICR_SGI_BASE_SIZE + GICD_ICFGR(irq), reg);
- gic_v3_wait_for_rwp(sc, REDIST);
- } else {
- gic_d_write(sc, 4, GICD_ICFGR(irq), reg);
- gic_v3_wait_for_rwp(sc, DIST);
- }
-
- mtx_unlock_spin(&sc->gic_mtx);
+ pargs.dev = dev;
+ pargs.isrc = isrc;
+ if (isrc->isrc_flags & INTR_ISRCF_PPI) {
+ /*
+ * If APs haven't been fired up yet, smp_rendezvous() will just
+ * execute it on the single CPU and gic_v3_init_secondary() will
+ * clean up afterwards.
+ */
+ smp_rendezvous(NULL, gic_v3_setup_intr_periph, NULL, &pargs);
+ } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) {
+ gic_v3_setup_intr_periph(&pargs);
gic_v3_bind_intr(dev, isrc);
}
@@ -864,22 +979,49 @@ gic_v3_disable_intr(device_t dev, struct intr_irqsrc *isrc)
}
static void
+gic_v3_enable_intr_periph(void *argp)
+{
+ struct gic_v3_setup_periph_args *args = argp;
+ struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)args->isrc;
+ device_t dev = args->dev;
+ struct gic_v3_softc *sc = device_get_softc(dev);
+ u_int irq = gi->gi_irq;
+
+ /* SGIs and PPIs in corresponding Re-Distributor */
+ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq),
+ GICD_I_MASK(irq));
+ gic_v3_wait_for_rwp(sc, REDIST);
+}
+
+static void
gic_v3_enable_intr(device_t dev, struct intr_irqsrc *isrc)
{
+ struct gic_v3_setup_periph_args pargs;
struct gic_v3_softc *sc;
struct gic_v3_irqsrc *gi;
u_int irq;
- sc = device_get_softc(dev);
gi = (struct gic_v3_irqsrc *)isrc;
irq = gi->gi_irq;
+ pargs.isrc = isrc;
+ pargs.dev = dev;
if (irq <= GIC_LAST_PPI) {
- /* SGIs and PPIs in corresponding Re-Distributor */
- gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq),
- GICD_I_MASK(irq));
- gic_v3_wait_for_rwp(sc, REDIST);
- } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) {
+ /*
+ * SGIs only need configured on the current AP. We'll setup and
+ * enable IPIs as APs come online.
+ */
+ if (irq <= GIC_LAST_SGI)
+ gic_v3_enable_intr_periph(&pargs);
+ else
+ smp_rendezvous(NULL, gic_v3_enable_intr_periph, NULL,
+ &pargs);
+ return;
+ }
+
+ sc = device_get_softc(dev);
+
+ if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) {
/* SPIs in distributor */
gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq));
gic_v3_wait_for_rwp(sc, DIST);
@@ -922,8 +1064,6 @@ gic_v3_bind_intr(device_t dev, struct intr_irqsrc *isrc)
int cpu;
gi = (struct gic_v3_irqsrc *)isrc;
- if (gi->gi_irq <= GIC_LAST_PPI)
- return (EINVAL);
KASSERT(gi->gi_irq >= GIC_FIRST_SPI && gi->gi_irq <= GIC_LAST_SPI,
("%s: Attempting to bind an invalid IRQ", __func__));
@@ -951,6 +1091,7 @@ gic_v3_bind_intr(device_t dev, struct intr_irqsrc *isrc)
static void
gic_v3_init_secondary(device_t dev)
{
+ struct gic_v3_setup_periph_args pargs;
device_t child;
struct gic_v3_softc *sc;
gic_v3_initseq_t *init_func;
@@ -972,18 +1113,25 @@ gic_v3_init_secondary(device_t dev)
}
}
+ pargs.dev = dev;
+
/* Unmask attached SGI interrupts. */
for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) {
isrc = GIC_INTR_ISRC(sc, irq);
- if (intr_isrc_init_on_cpu(isrc, cpu))
- gic_v3_enable_intr(dev, isrc);
+ if (intr_isrc_init_on_cpu(isrc, cpu)) {
+ pargs.isrc = isrc;
+ gic_v3_enable_intr_periph(&pargs);
+ }
}
/* Unmask attached PPI interrupts. */
for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) {
isrc = GIC_INTR_ISRC(sc, irq);
- if (intr_isrc_init_on_cpu(isrc, cpu))
- gic_v3_enable_intr(dev, isrc);
+ if (intr_isrc_init_on_cpu(isrc, cpu)) {
+ pargs.isrc = isrc;
+ gic_v3_setup_intr_periph(&pargs);
+ gic_v3_enable_intr_periph(&pargs);
+ }
}
for (i = 0; i < sc->gic_nchildren; i++) {
@@ -1066,6 +1214,7 @@ static void
gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist)
{
struct resource *res;
+ bus_size_t offset;
u_int cpuid;
size_t us_left = 1000000;
@@ -1074,16 +1223,18 @@ gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist)
switch (xdist) {
case DIST:
res = sc->gic_dist;
+ offset = 0;
break;
case REDIST:
- res = &sc->gic_redists.pcpu[cpuid]->res;
+ res = sc->gic_redists.pcpu[cpuid].res;
+ offset = sc->gic_redists.pcpu[PCPU_GET(cpuid)].offset;
break;
default:
KASSERT(0, ("%s: Attempt to wait for unknown RWP", __func__));
return;
}
- while ((bus_read_4(res, GICD_CTLR) & GICD_CTLR_RWP) != 0) {
+ while ((bus_read_4(res, offset + GICD_CTLR) & GICD_CTLR_RWP) != 0) {
DELAY(1);
if (us_left-- == 0)
panic("GICD Register write pending for too long");
@@ -1212,24 +1363,16 @@ gic_v3_dist_init(struct gic_v3_softc *sc)
static int
gic_v3_redist_alloc(struct gic_v3_softc *sc)
{
- u_int cpuid;
-
- /* Allocate struct resource for all CPU's Re-Distributor registers */
- for (cpuid = 0; cpuid <= mp_maxid; cpuid++)
- if (CPU_ISSET(cpuid, &all_cpus) != 0)
- sc->gic_redists.pcpu[cpuid] =
- malloc(sizeof(*sc->gic_redists.pcpu[0]),
- M_GIC_V3, M_WAITOK);
- else
- sc->gic_redists.pcpu[cpuid] = NULL;
+ sc->gic_redists.pcpu = mallocarray(mp_maxid + 1,
+ sizeof(sc->gic_redists.pcpu[0]), M_GIC_V3, M_WAITOK);
return (0);
}
static int
gic_v3_redist_find(struct gic_v3_softc *sc)
{
- struct resource r_res;
- bus_space_handle_t r_bsh;
+ struct resource *r_res;
+ bus_size_t offset;
uint64_t aff;
uint64_t typer;
uint32_t pidr2;
@@ -1250,10 +1393,9 @@ gic_v3_redist_find(struct gic_v3_softc *sc)
/* Iterate through Re-Distributor regions */
for (i = 0; i < sc->gic_redists.nregions; i++) {
/* Take a copy of the region's resource */
- r_res = *sc->gic_redists.regions[i];
- r_bsh = rman_get_bushandle(&r_res);
+ r_res = sc->gic_redists.regions[i];
- pidr2 = bus_read_4(&r_res, GICR_PIDR2);
+ pidr2 = bus_read_4(r_res, GICR_PIDR2);
switch (GICR_PIDR2_ARCH(pidr2)) {
case GICR_PIDR2_ARCH_GICv3: /* fall through */
case GICR_PIDR2_ARCH_GICv4:
@@ -1264,14 +1406,16 @@ gic_v3_redist_find(struct gic_v3_softc *sc)
return (ENODEV);
}
+ offset = 0;
do {
- typer = bus_read_8(&r_res, GICR_TYPER);
+ typer = bus_read_8(r_res, offset + GICR_TYPER);
if ((typer >> GICR_TYPER_AFF_SHIFT) == aff) {
- KASSERT(sc->gic_redists.pcpu[cpuid] != NULL,
+ KASSERT(cpuid <= mp_maxid,
("Invalid pointer to per-CPU redistributor"));
/* Copy res contents to its final destination */
- sc->gic_redists.pcpu[cpuid]->res = r_res;
- sc->gic_redists.pcpu[cpuid]->lpi_enabled = false;
+ sc->gic_redists.pcpu[cpuid].res = r_res;
+ sc->gic_redists.pcpu[cpuid].offset = offset;
+ sc->gic_redists.pcpu[cpuid].lpi_enabled = false;
if (bootverbose) {
device_printf(sc->dev,
"CPU%u Re-Distributor has been found\n",
@@ -1280,14 +1424,13 @@ gic_v3_redist_find(struct gic_v3_softc *sc)
return (0);
}
- r_bsh += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE);
+ offset += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE);
if ((typer & GICR_TYPER_VLPIS) != 0) {
- r_bsh +=
+ offset +=
(GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE);
}
-
- rman_set_bushandle(&r_res, r_bsh);
- } while ((typer & GICR_TYPER_LAST) == 0);
+ } while (offset < rman_get_size(r_res) &&
+ (typer & GICR_TYPER_LAST) == 0);
}
device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid);