aboutsummaryrefslogtreecommitdiff
path: root/sys/arm64/vmm
diff options
context:
space:
mode:
Diffstat (limited to 'sys/arm64/vmm')
-rw-r--r--sys/arm64/vmm/arm64.h41
-rw-r--r--sys/arm64/vmm/hyp.h1
-rw-r--r--sys/arm64/vmm/io/vtimer.c94
-rw-r--r--sys/arm64/vmm/io/vtimer.h2
-rw-r--r--sys/arm64/vmm/vmm.c77
-rw-r--r--sys/arm64/vmm/vmm_arm64.c15
-rw-r--r--sys/arm64/vmm/vmm_hyp.c103
-rw-r--r--sys/arm64/vmm/vmm_reset.c7
8 files changed, 223 insertions, 117 deletions
diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h
index 6a0c4c78e568..f530dab05331 100644
--- a/sys/arm64/vmm/arm64.h
+++ b/sys/arm64/vmm/arm64.h
@@ -78,14 +78,16 @@ struct hypctx {
uint64_t pmcr_el0; /* Performance Monitors Control Register */
uint64_t pmccntr_el0;
uint64_t pmccfiltr_el0;
+ uint64_t pmuserenr_el0;
+ uint64_t pmselr_el0;
+ uint64_t pmxevcntr_el0;
uint64_t pmcntenset_el0;
uint64_t pmintenset_el1;
uint64_t pmovsset_el0;
- uint64_t pmselr_el0;
- uint64_t pmuserenr_el0;
uint64_t pmevcntr_el0[31];
uint64_t pmevtyper_el0[31];
+ uint64_t dbgclaimset_el1;
uint64_t dbgbcr_el1[16]; /* Debug Breakpoint Control Registers */
uint64_t dbgbvr_el1[16]; /* Debug Breakpoint Value Registers */
uint64_t dbgwcr_el1[16]; /* Debug Watchpoint Control Registers */
@@ -117,6 +119,7 @@ struct hypctx {
struct vgic_v3_regs vgic_v3_regs;
struct vgic_v3_cpu *vgic_cpu;
bool has_exception;
+ bool dbg_oslock;
};
struct hyp {
@@ -125,42 +128,14 @@ struct hyp {
uint64_t vmid_generation;
uint64_t vttbr_el2;
uint64_t el2_addr; /* The address of this in el2 space */
+ uint64_t feats; /* Which features are enabled */
+#define HYP_FEAT_HCX (0x1ul << 0)
+#define HYP_FEAT_ECV_POFF (0x1ul << 1)
bool vgic_attached;
struct vgic_v3 *vgic;
struct hypctx *ctx[];
};
-#define DEFINE_VMMOPS_IFUNC(ret_type, opname, args) \
- ret_type vmmops_##opname args;
-
-DEFINE_VMMOPS_IFUNC(int, modinit, (int ipinum))
-DEFINE_VMMOPS_IFUNC(int, modcleanup, (void))
-DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap))
-DEFINE_VMMOPS_IFUNC(int, gla2gpa, (void *vcpui, struct vm_guest_paging *paging,
- uint64_t gla, int prot, uint64_t *gpa, int *is_fault))
-DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap,
- struct vm_eventinfo *info))
-DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi))
-DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu,
- int vcpu_id))
-DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vcpui))
-DEFINE_VMMOPS_IFUNC(int, exception, (void *vcpui, uint64_t esr, uint64_t far))
-DEFINE_VMMOPS_IFUNC(int, getreg, (void *vcpui, int num, uint64_t *retval))
-DEFINE_VMMOPS_IFUNC(int, setreg, (void *vcpui, int num, uint64_t val))
-DEFINE_VMMOPS_IFUNC(int, getcap, (void *vcpui, int num, int *retval))
-DEFINE_VMMOPS_IFUNC(int, setcap, (void *vcpui, int num, int val))
-DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min,
- vm_offset_t max))
-DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace))
-#ifdef notyet
-#ifdef BHYVE_SNAPSHOT
-DEFINE_VMMOPS_IFUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta *meta))
-DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vcpui,
- struct vm_snapshot_meta *meta))
-DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vcpui, uint64_t now))
-#endif
-#endif
-
uint64_t vmm_call_hyp(uint64_t, ...);
#if 0
diff --git a/sys/arm64/vmm/hyp.h b/sys/arm64/vmm/hyp.h
index 0b2977c73960..0c8d0fb28b18 100644
--- a/sys/arm64/vmm/hyp.h
+++ b/sys/arm64/vmm/hyp.h
@@ -80,7 +80,6 @@
#define HYP_ENTER_GUEST 0x00000002
#define HYP_READ_REGISTER 0x00000003
#define HYP_REG_ICH_VTR 0x1
-#define HYP_REG_CNTHCTL 0x2
#define HYP_CLEAN_S2_TLBI 0x00000004
#define HYP_DC_CIVAC 0x00000005
#define HYP_EL2_TLBI 0x00000006
diff --git a/sys/arm64/vmm/io/vtimer.c b/sys/arm64/vmm/io/vtimer.c
index f59d7ebc1ad4..da0f0d96c431 100644
--- a/sys/arm64/vmm/io/vtimer.c
+++ b/sys/arm64/vmm/io/vtimer.c
@@ -36,6 +36,7 @@
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/rman.h>
+#include <sys/sysctl.h>
#include <sys/time.h>
#include <sys/timeet.h>
#include <sys/timetc.h>
@@ -55,11 +56,18 @@
#define timer_enabled(ctl) \
(!((ctl) & CNTP_CTL_IMASK) && ((ctl) & CNTP_CTL_ENABLE))
-static uint64_t cnthctl_el2_reg;
static uint32_t tmr_frq;
#define timer_condition_met(ctl) ((ctl) & CNTP_CTL_ISTATUS)
+SYSCTL_DECL(_hw_vmm);
+SYSCTL_NODE(_hw_vmm, OID_AUTO, vtimer, CTLFLAG_RW, NULL, NULL);
+
+static bool allow_ecv_phys = false;
+SYSCTL_BOOL(_hw_vmm_vtimer, OID_AUTO, allow_ecv_phys, CTLFLAG_RW,
+ &allow_ecv_phys, 0,
+ "Enable hardware access to the physical timer if FEAT_ECV_POFF is supported");
+
static void vtimer_schedule_irq(struct hypctx *hypctx, bool phys);
static int
@@ -111,9 +119,8 @@ out:
}
int
-vtimer_init(uint64_t cnthctl_el2)
+vtimer_init(void)
{
- cnthctl_el2_reg = cnthctl_el2;
/*
* The guest *MUST* use the same timer frequency as the host. The
* register CNTFRQ_EL0 is accessible to the guest and a different value
@@ -128,8 +135,12 @@ void
vtimer_vminit(struct hyp *hyp)
{
uint64_t now;
+ bool ecv_poff;
- hyp->vtimer.cnthctl_el2 = cnthctl_el2_reg;
+ ecv_poff = false;
+
+ if (allow_ecv_phys && (hyp->feats & HYP_FEAT_ECV_POFF) != 0)
+ ecv_poff = true;
/*
* Configure the Counter-timer Hypervisor Control Register for the VM.
@@ -137,35 +148,58 @@ vtimer_vminit(struct hyp *hyp)
if (in_vhe()) {
/*
* CNTHCTL_E2H_EL0PCTEN: trap EL0 access to CNTP{CT,CTSS}_EL0
- * CNTHCTL_E2H_EL1VCTEN: don't trap EL0 access to
- * CNTV{CT,CTSS}_EL0
+ * CNTHCTL_E2H_EL0VCTEN: don't trap EL0 access to
+ * CNTV{CT,CTXX}_EL0
* CNTHCTL_E2H_EL0VTEN: don't trap EL0 access to
* CNTV_{CTL,CVAL,TVAL}_EL0
* CNTHCTL_E2H_EL0PTEN: trap EL0 access to
* CNTP_{CTL,CVAL,TVAL}_EL0
- * CNTHCTL_E2H_EL1PCEN: trap EL1 access to
- CNTP_{CTL,CVAL,TVAL}_EL0
* CNTHCTL_E2H_EL1PCTEN: trap access to CNTPCT_EL0
+ * CNTHCTL_E2H_EL1PTEN: trap access to
+ * CNTP_{CTL,CVAL,TVAL}_EL0
+ * CNTHCTL_E2H_EL1VCTEN: don't trap EL0 access to
+ * CNTV{CT,CTSS}_EL0
+ * CNTHCTL_E2H_EL1PCEN: trap EL1 access to
+ * CNTP_{CTL,CVAL,TVAL}_EL0
*
* TODO: Don't trap when FEAT_ECV is present
*/
- hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL0PCTEN;
- hyp->vtimer.cnthctl_el2 |= CNTHCTL_E2H_EL0VCTEN;
- hyp->vtimer.cnthctl_el2 |= CNTHCTL_E2H_EL0VTEN;
- hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL0PTEN;
-
- hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL1PTEN;
- hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL1PCTEN;
+ hyp->vtimer.cnthctl_el2 =
+ CNTHCTL_E2H_EL0VCTEN_NOTRAP |
+ CNTHCTL_E2H_EL0VTEN_NOTRAP;
+ if (ecv_poff) {
+ hyp->vtimer.cnthctl_el2 |=
+ CNTHCTL_E2H_EL0PCTEN_NOTRAP |
+ CNTHCTL_E2H_EL0PTEN_NOTRAP |
+ CNTHCTL_E2H_EL1PCTEN_NOTRAP |
+ CNTHCTL_E2H_EL1PTEN_NOTRAP;
+ } else {
+ hyp->vtimer.cnthctl_el2 |=
+ CNTHCTL_E2H_EL0PCTEN_TRAP |
+ CNTHCTL_E2H_EL0PTEN_TRAP |
+ CNTHCTL_E2H_EL1PCTEN_TRAP |
+ CNTHCTL_E2H_EL1PTEN_TRAP;
+ }
} else {
/*
* CNTHCTL_EL1PCEN: trap access to CNTP_{CTL, CVAL, TVAL}_EL0
* from EL1
* CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0
*/
- hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCEN;
- hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCTEN;
+ if (ecv_poff) {
+ hyp->vtimer.cnthctl_el2 =
+ CNTHCTL_EL1PCTEN_NOTRAP |
+ CNTHCTL_EL1PCEN_NOTRAP;
+ } else {
+ hyp->vtimer.cnthctl_el2 =
+ CNTHCTL_EL1PCTEN_TRAP |
+ CNTHCTL_EL1PCEN_TRAP;
+ }
}
+ if (ecv_poff)
+ hyp->vtimer.cnthctl_el2 |= CNTHCTL_ECV_EN;
+
now = READ_SPECIALREG(cntpct_el0);
hyp->vtimer.cntvoff_el2 = now;
@@ -231,15 +265,10 @@ vtimer_cleanup(void)
{
}
-void
-vtimer_sync_hwstate(struct hypctx *hypctx)
+static void
+vtime_sync_timer(struct hypctx *hypctx, struct vtimer_timer *timer,
+ uint64_t cntpct_el0)
{
- struct vtimer_timer *timer;
- uint64_t cntpct_el0;
-
- timer = &hypctx->vtimer_cpu.virt_timer;
- cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
- hypctx->hyp->vtimer.cntvoff_el2;
if (!timer_enabled(timer->cntx_ctl_el0)) {
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
timer->irqid, false);
@@ -253,6 +282,21 @@ vtimer_sync_hwstate(struct hypctx *hypctx)
}
}
+void
+vtimer_sync_hwstate(struct hypctx *hypctx)
+{
+ uint64_t cntpct_el0;
+
+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
+ hypctx->hyp->vtimer.cntvoff_el2;
+ vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.virt_timer, cntpct_el0);
+ /* If FEAT_ECV_POFF is in use then we need to sync the physical timer */
+ if ((hypctx->hyp->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0) {
+ vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.phys_timer,
+ cntpct_el0);
+ }
+}
+
static void
vtimer_inject_irq_callout_phys(void *context)
{
diff --git a/sys/arm64/vmm/io/vtimer.h b/sys/arm64/vmm/io/vtimer.h
index 71a20344d05e..92ce025968d2 100644
--- a/sys/arm64/vmm/io/vtimer.h
+++ b/sys/arm64/vmm/io/vtimer.h
@@ -66,7 +66,7 @@ struct vtimer_cpu {
uint32_t cntkctl_el1;
};
-int vtimer_init(uint64_t cnthctl_el2);
+int vtimer_init(void);
void vtimer_vminit(struct hyp *);
void vtimer_cpuinit(struct hypctx *);
void vtimer_cpucleanup(struct hypctx *);
diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c
index 3082d2941221..aeda689f3b1a 100644
--- a/sys/arm64/vmm/vmm.c
+++ b/sys/arm64/vmm/vmm.c
@@ -88,7 +88,6 @@ struct vcpu {
struct vfpstate *guestfpu; /* (a,i) guest fpu state */
};
-#define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
#define vcpu_lock_destroy(v) mtx_destroy(&((v)->mtx))
#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx))
@@ -126,7 +125,6 @@ struct vm {
bool dying; /* (o) is dying */
volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */
volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */
- struct vmspace *vmspace; /* (o) guest's address space */
struct vm_mem mem; /* (i) guest memory */
char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */
struct vcpu **vcpu; /* (i) guest vcpus */
@@ -274,6 +272,7 @@ vcpu_cleanup(struct vcpu *vcpu, bool destroy)
vmm_stat_free(vcpu->stats);
fpu_save_area_free(vcpu->guestfpu);
vcpu_lock_destroy(vcpu);
+ free(vcpu, M_VMM);
}
}
@@ -407,7 +406,7 @@ vm_init(struct vm *vm, bool create)
{
int i;
- vm->cookie = vmmops_init(vm, vmspace_pmap(vm->vmspace));
+ vm->cookie = vmmops_init(vm, vmspace_pmap(vm_vmspace(vm)));
MPASS(vm->cookie != NULL);
CPU_ZERO(&vm->active_cpus);
@@ -485,7 +484,7 @@ int
vm_create(const char *name, struct vm **retvm)
{
struct vm *vm;
- struct vmspace *vmspace;
+ int error;
/*
* If vmm.ko could not be successfully initialized then don't attempt
@@ -497,14 +496,13 @@ vm_create(const char *name, struct vm **retvm)
if (name == NULL || strlen(name) >= VM_MAX_NAMELEN)
return (EINVAL);
- vmspace = vmmops_vmspace_alloc(0, 1ul << 39);
- if (vmspace == NULL)
- return (ENOMEM);
-
vm = malloc(sizeof(struct vm), M_VMM, M_WAITOK | M_ZERO);
+ error = vm_mem_init(&vm->mem, 0, 1ul << 39);
+ if (error != 0) {
+ free(vm, M_VMM);
+ return (error);
+ }
strcpy(vm->name, name);
- vm->vmspace = vmspace;
- vm_mem_init(&vm->mem);
sx_init(&vm->vcpus_init_lock, "vm vcpus");
vm->sockets = 1;
@@ -558,7 +556,7 @@ vm_cleanup(struct vm *vm, bool destroy)
if (destroy) {
vm_xlock_memsegs(vm);
- pmap = vmspace_pmap(vm->vmspace);
+ pmap = vmspace_pmap(vm_vmspace(vm));
sched_pin();
PCPU_SET(curvmpmap, NULL);
sched_unpin();
@@ -582,11 +580,6 @@ vm_cleanup(struct vm *vm, bool destroy)
if (destroy) {
vm_mem_destroy(vm);
- vmmops_vmspace_free(vm->vmspace);
- vm->vmspace = NULL;
-
- for (i = 0; i < vm->maxcpus; i++)
- free(vm->vcpu[i], M_VMM);
free(vm->vcpu, M_VMM);
sx_destroy(&vm->vcpus_init_lock);
}
@@ -651,6 +644,33 @@ vmm_reg_wi(struct vcpu *vcpu, uint64_t wval, void *arg)
return (0);
}
+static int
+vmm_write_oslar_el1(struct vcpu *vcpu, uint64_t wval, void *arg)
+{
+ struct hypctx *hypctx;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ /* All other fields are RES0 & we don't do anything with this */
+ /* TODO: Disable access to other debug state when locked */
+ hypctx->dbg_oslock = (wval & OSLAR_OSLK) == OSLAR_OSLK;
+ return (0);
+}
+
+static int
+vmm_read_oslsr_el1(struct vcpu *vcpu, uint64_t *rval, void *arg)
+{
+ struct hypctx *hypctx;
+ uint64_t val;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ val = OSLSR_OSLM_1;
+ if (hypctx->dbg_oslock)
+ val |= OSLSR_OSLK;
+ *rval = val;
+
+ return (0);
+}
+
static const struct vmm_special_reg vmm_special_regs[] = {
#define SPECIAL_REG(_reg, _read, _write) \
{ \
@@ -707,6 +727,13 @@ static const struct vmm_special_reg vmm_special_regs[] = {
SPECIAL_REG(CNTP_TVAL_EL0, vtimer_phys_tval_read,
vtimer_phys_tval_write),
SPECIAL_REG(CNTPCT_EL0, vtimer_phys_cnt_read, vtimer_phys_cnt_write),
+
+ /* Debug registers */
+ SPECIAL_REG(DBGPRCR_EL1, vmm_reg_raz, vmm_reg_wi),
+ SPECIAL_REG(OSDLR_EL1, vmm_reg_raz, vmm_reg_wi),
+ /* TODO: Exceptions on invalid access */
+ SPECIAL_REG(OSLAR_EL1, vmm_reg_raz, vmm_write_oslar_el1),
+ SPECIAL_REG(OSLSR_EL1, vmm_read_oslsr_el1, vmm_reg_wi),
#undef SPECIAL_REG
};
@@ -1056,12 +1083,6 @@ vcpu_notify_event(struct vcpu *vcpu)
vcpu_unlock(vcpu);
}
-struct vmspace *
-vm_vmspace(struct vm *vm)
-{
- return (vm->vmspace);
-}
-
struct vm_mem *
vm_mem(struct vm *vm)
{
@@ -1342,8 +1363,14 @@ vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
static int
vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
{
+ struct vm *vm;
+
+ vm = vcpu->vm;
vcpu_lock(vcpu);
while (1) {
+ if (vm->suspend)
+ break;
+
if (vgic_has_pending_irq(vcpu->cookie))
break;
@@ -1376,7 +1403,7 @@ vm_handle_paging(struct vcpu *vcpu, bool *retu)
vme = &vcpu->exitinfo;
- pmap = vmspace_pmap(vcpu->vm->vmspace);
+ pmap = vmspace_pmap(vm_vmspace(vcpu->vm));
addr = vme->u.paging.gpa;
esr = vme->u.paging.esr;
@@ -1393,7 +1420,7 @@ vm_handle_paging(struct vcpu *vcpu, bool *retu)
panic("%s: Invalid exception (esr = %lx)", __func__, esr);
}
- map = &vm->vmspace->vm_map;
+ map = &vm_vmspace(vm)->vm_map;
rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL, NULL);
if (rv != KERN_SUCCESS)
return (EFAULT);
@@ -1467,7 +1494,7 @@ vm_run(struct vcpu *vcpu)
if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
return (EINVAL);
- pmap = vmspace_pmap(vm->vmspace);
+ pmap = vmspace_pmap(vm_vmspace(vm));
vme = &vcpu->exitinfo;
evinfo.rptr = NULL;
evinfo.sptr = &vm->suspend;
diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c
index e293c99a6646..618f4afaf8ee 100644
--- a/sys/arm64/vmm/vmm_arm64.c
+++ b/sys/arm64/vmm/vmm_arm64.c
@@ -238,7 +238,6 @@ vmmops_modinit(int ipinum)
vm_offset_t next_hyp_va;
vm_paddr_t vmm_base;
uint64_t id_aa64mmfr0_el1, pa_range_bits, pa_range_field;
- uint64_t cnthctl_el2;
int cpu, i;
bool rv __diagused;
@@ -444,10 +443,9 @@ vmmops_modinit(int ipinum)
vmem_add(el2_mem_alloc, next_hyp_va,
HYP_VM_MAX_ADDRESS - next_hyp_va, M_WAITOK);
}
- cnthctl_el2 = vmm_read_reg(HYP_REG_CNTHCTL);
vgic_init();
- vtimer_init(cnthctl_el2);
+ vtimer_init();
return (0);
}
@@ -517,6 +515,7 @@ vmmops_init(struct vm *vm, pmap_t pmap)
{
struct hyp *hyp;
vm_size_t size;
+ uint64_t idreg;
size = el2_hyp_size(vm);
hyp = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO);
@@ -524,6 +523,16 @@ vmmops_init(struct vm *vm, pmap_t pmap)
hyp->vm = vm;
hyp->vgic_attached = false;
+ if (get_kernel_reg(ID_AA64MMFR0_EL1, &idreg)) {
+ if (ID_AA64MMFR0_ECV_VAL(idreg) >= ID_AA64MMFR0_ECV_POFF)
+ hyp->feats |= HYP_FEAT_ECV_POFF;
+ }
+
+ if (get_kernel_reg(ID_AA64MMFR1_EL1, &idreg)) {
+ if (ID_AA64MMFR1_HCX_VAL(idreg) >= ID_AA64MMFR1_HCX_IMPL)
+ hyp->feats |= HYP_FEAT_HCX;
+ }
+
vtimer_vminit(hyp);
vgic_vminit(hyp);
diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c
index d61885c15871..b8c6d2ab7a9a 100644
--- a/sys/arm64/vmm/vmm_hyp.c
+++ b/sys/arm64/vmm/vmm_hyp.c
@@ -42,11 +42,11 @@ struct hypctx;
uint64_t VMM_HYP_FUNC(do_call_guest)(struct hypctx *);
static void
-vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
+vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest,
+ bool ecv_poff)
{
uint64_t dfr0;
- /* Store the guest VFP registers */
if (guest) {
/* Store the timer registers */
hypctx->vtimer_cpu.cntkctl_el1 =
@@ -55,7 +55,20 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
READ_SPECIALREG(EL0_REG(CNTV_CVAL));
hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 =
READ_SPECIALREG(EL0_REG(CNTV_CTL));
+ }
+ if (guest_or_nonvhe(guest) && ecv_poff) {
+ /*
+ * If we have ECV then the guest could modify these registers.
+ * If VHE is enabled then the kernel will see a different view
+ * of the registers, so doesn't need to handle them.
+ */
+ hypctx->vtimer_cpu.phys_timer.cntx_cval_el0 =
+ READ_SPECIALREG(EL0_REG(CNTP_CVAL));
+ hypctx->vtimer_cpu.phys_timer.cntx_ctl_el0 =
+ READ_SPECIALREG(EL0_REG(CNTP_CTL));
+ }
+ if (guest) {
/* Store the GICv3 registers */
hypctx->vgic_v3_regs.ich_eisr_el2 =
READ_SPECIALREG(ich_eisr_el2);
@@ -108,6 +121,8 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
}
}
+ hypctx->dbgclaimset_el1 = READ_SPECIALREG(dbgclaimset_el1);
+
dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
#define STORE_DBG_BRP(x) \
@@ -167,10 +182,13 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
hypctx->pmcr_el0 = READ_SPECIALREG(pmcr_el0);
hypctx->pmccntr_el0 = READ_SPECIALREG(pmccntr_el0);
hypctx->pmccfiltr_el0 = READ_SPECIALREG(pmccfiltr_el0);
+ hypctx->pmuserenr_el0 = READ_SPECIALREG(pmuserenr_el0);
+ hypctx->pmselr_el0 = READ_SPECIALREG(pmselr_el0);
+ hypctx->pmxevcntr_el0 = READ_SPECIALREG(pmxevcntr_el0);
hypctx->pmcntenset_el0 = READ_SPECIALREG(pmcntenset_el0);
hypctx->pmintenset_el1 = READ_SPECIALREG(pmintenset_el1);
hypctx->pmovsset_el0 = READ_SPECIALREG(pmovsset_el0);
- hypctx->pmuserenr_el0 = READ_SPECIALREG(pmuserenr_el0);
+
switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
#define STORE_PMU(x) \
case (x + 1): \
@@ -259,29 +277,20 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
hypctx->hcr_el2 = READ_SPECIALREG(hcr_el2);
hypctx->vpidr_el2 = READ_SPECIALREG(vpidr_el2);
hypctx->vmpidr_el2 = READ_SPECIALREG(vmpidr_el2);
-
-#ifndef VMM_VHE
- /* hcrx_el2 depends on feat_hcx */
- uint64_t mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
- if (ID_AA64MMFR1_HCX_VAL(mmfr1) >> ID_AA64MMFR1_HCX_SHIFT) {
- hypctx->hcrx_el2 = READ_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2));
- }
-#endif
}
static void
-vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
+vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest,
+ bool ecv_poff)
{
uint64_t dfr0;
/* Restore the special registers */
WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2);
- if (guest_or_nonvhe(guest)) {
- uint64_t mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
- if (ID_AA64MMFR1_HCX_VAL(mmfr1) >> ID_AA64MMFR1_HCX_SHIFT) {
- WRITE_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2), hypctx->hcrx_el2);
- }
+ if (guest) {
+ if ((hyp->feats & HYP_FEAT_HCX) != 0)
+ WRITE_SPECIALREG(HCRX_EL2_REG, hypctx->hcrx_el2);
}
isb();
@@ -333,12 +342,15 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
WRITE_SPECIALREG(pmcr_el0, hypctx->pmcr_el0);
WRITE_SPECIALREG(pmccntr_el0, hypctx->pmccntr_el0);
WRITE_SPECIALREG(pmccfiltr_el0, hypctx->pmccfiltr_el0);
+ WRITE_SPECIALREG(pmuserenr_el0, hypctx->pmuserenr_el0);
+ WRITE_SPECIALREG(pmselr_el0, hypctx->pmselr_el0);
+ WRITE_SPECIALREG(pmxevcntr_el0, hypctx->pmxevcntr_el0);
/* Clear all events/interrupts then enable them */
- WRITE_SPECIALREG(pmcntenclr_el0, 0xfffffffful);
+ WRITE_SPECIALREG(pmcntenclr_el0, ~0ul);
WRITE_SPECIALREG(pmcntenset_el0, hypctx->pmcntenset_el0);
- WRITE_SPECIALREG(pmintenclr_el1, 0xfffffffful);
+ WRITE_SPECIALREG(pmintenclr_el1, ~0ul);
WRITE_SPECIALREG(pmintenset_el1, hypctx->pmintenset_el1);
- WRITE_SPECIALREG(pmovsclr_el0, 0xfffffffful);
+ WRITE_SPECIALREG(pmovsclr_el0, ~0ul);
WRITE_SPECIALREG(pmovsset_el0, hypctx->pmovsset_el0);
switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
@@ -384,6 +396,9 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
#undef LOAD_PMU
}
+ WRITE_SPECIALREG(dbgclaimclr_el1, ~0ul);
+ WRITE_SPECIALREG(dbgclaimclr_el1, hypctx->dbgclaimset_el1);
+
dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
#define LOAD_DBG_BRP(x) \
@@ -450,6 +465,29 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2);
WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2);
+ if (ecv_poff) {
+ /*
+ * Load the same offset as the virtual timer
+ * to keep in sync.
+ */
+ WRITE_SPECIALREG(CNTPOFF_EL2_REG,
+ hyp->vtimer.cntvoff_el2);
+ isb();
+ }
+ }
+ if (guest_or_nonvhe(guest) && ecv_poff) {
+ /*
+ * If we have ECV then the guest could modify these registers.
+ * If VHE is enabled then the kernel will see a different view
+ * of the registers, so doesn't need to handle them.
+ */
+ WRITE_SPECIALREG(EL0_REG(CNTP_CVAL),
+ hypctx->vtimer_cpu.phys_timer.cntx_cval_el0);
+ WRITE_SPECIALREG(EL0_REG(CNTP_CTL),
+ hypctx->vtimer_cpu.phys_timer.cntx_ctl_el0);
+ }
+
+ if (guest) {
/* Load the GICv3 registers */
WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
WRITE_SPECIALREG(ich_vmcr_el2,
@@ -502,11 +540,19 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
struct hypctx host_hypctx;
uint64_t cntvoff_el2;
uint64_t ich_hcr_el2, ich_vmcr_el2, cnthctl_el2, cntkctl_el1;
+#ifndef VMM_VHE
+ uint64_t hcrx_el2;
+#endif
uint64_t ret;
uint64_t s1e1r, hpfar_el2;
- bool hpfar_valid;
+ bool ecv_poff, hpfar_valid;
- vmm_hyp_reg_store(&host_hypctx, NULL, false);
+ ecv_poff = (hyp->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0;
+ vmm_hyp_reg_store(&host_hypctx, NULL, false, ecv_poff);
+#ifndef VMM_VHE
+ if ((hyp->feats & HYP_FEAT_HCX) != 0)
+ hcrx_el2 = READ_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2));
+#endif
/* Save the host special registers */
cnthctl_el2 = READ_SPECIALREG(cnthctl_el2);
@@ -516,7 +562,7 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
- vmm_hyp_reg_restore(hypctx, hyp, true);
+ vmm_hyp_reg_restore(hypctx, hyp, true, ecv_poff);
/* Load the common hypervisor registers */
WRITE_SPECIALREG(vttbr_el2, hyp->vttbr_el2);
@@ -532,7 +578,7 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
/* Store the exit info */
hypctx->exit_info.far_el2 = READ_SPECIALREG(far_el2);
- vmm_hyp_reg_store(hypctx, hyp, true);
+ vmm_hyp_reg_store(hypctx, hyp, true, ecv_poff);
hpfar_valid = true;
if (ret == EXCP_TYPE_EL1_SYNC) {
@@ -582,7 +628,12 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
}
}
- vmm_hyp_reg_restore(&host_hypctx, NULL, false);
+ vmm_hyp_reg_restore(&host_hypctx, NULL, false, ecv_poff);
+
+#ifndef VMM_VHE
+ if ((hyp->feats & HYP_FEAT_HCX) != 0)
+ WRITE_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2), hcrx_el2);
+#endif
/* Restore the host special registers */
WRITE_SPECIALREG(ich_hcr_el2, ich_hcr_el2);
@@ -613,8 +664,6 @@ VMM_HYP_FUNC(read_reg)(uint64_t reg)
switch (reg) {
case HYP_REG_ICH_VTR:
return (READ_SPECIALREG(ich_vtr_el2));
- case HYP_REG_CNTHCTL:
- return (READ_SPECIALREG(cnthctl_el2));
}
return (0);
diff --git a/sys/arm64/vmm/vmm_reset.c b/sys/arm64/vmm/vmm_reset.c
index 79d022cf33e8..1240c3ed16ec 100644
--- a/sys/arm64/vmm/vmm_reset.c
+++ b/sys/arm64/vmm/vmm_reset.c
@@ -100,10 +100,12 @@ reset_vm_el01_regs(void *vcpu)
el2ctx->pmcr_el0 |= PMCR_LC;
set_arch_unknown(el2ctx->pmccntr_el0);
set_arch_unknown(el2ctx->pmccfiltr_el0);
+ set_arch_unknown(el2ctx->pmuserenr_el0);
+ set_arch_unknown(el2ctx->pmselr_el0);
+ set_arch_unknown(el2ctx->pmxevcntr_el0);
set_arch_unknown(el2ctx->pmcntenset_el0);
set_arch_unknown(el2ctx->pmintenset_el1);
set_arch_unknown(el2ctx->pmovsset_el0);
- set_arch_unknown(el2ctx->pmuserenr_el0);
memset(el2ctx->pmevcntr_el0, 0, sizeof(el2ctx->pmevcntr_el0));
memset(el2ctx->pmevtyper_el0, 0, sizeof(el2ctx->pmevtyper_el0));
}
@@ -143,7 +145,8 @@ reset_vm_el2_regs(void *vcpu)
/* Set the Extended Hypervisor Configuration Register */
el2ctx->hcrx_el2 = 0;
/* TODO: Trap all extensions we don't support */
- el2ctx->mdcr_el2 = 0;
+ el2ctx->mdcr_el2 = MDCR_EL2_TDOSA | MDCR_EL2_TDRA | MDCR_EL2_TPMS |
+ MDCR_EL2_TTRF;
/* PMCR_EL0.N is read from MDCR_EL2.HPMN */
el2ctx->mdcr_el2 |= (el2ctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT;
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-13 13:40:16 +0000