1144 lines
24 KiB
C
1144 lines
24 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* Copyright (C) 2012-2015 - ARM Ltd
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
|
*/
|
|
|
|
#include <hyp/adjust_pc.h>
|
|
|
|
#include <linux/compiler.h>
|
|
#include <linux/irqchip/arm-gic-v3.h>
|
|
#include <linux/kvm_host.h>
|
|
|
|
#include <asm/kvm_emulate.h>
|
|
#include <asm/kvm_hyp.h>
|
|
#include <asm/kvm_mmu.h>
|
|
|
|
#define vtr_to_max_lr_idx(v) ((v) & 0xf)
|
|
#define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1)
|
|
#define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5))
|
|
|
|
static u64 __gic_v3_get_lr(unsigned int lr)
|
|
{
|
|
switch (lr & 0xf) {
|
|
case 0:
|
|
return read_gicreg(ICH_LR0_EL2);
|
|
case 1:
|
|
return read_gicreg(ICH_LR1_EL2);
|
|
case 2:
|
|
return read_gicreg(ICH_LR2_EL2);
|
|
case 3:
|
|
return read_gicreg(ICH_LR3_EL2);
|
|
case 4:
|
|
return read_gicreg(ICH_LR4_EL2);
|
|
case 5:
|
|
return read_gicreg(ICH_LR5_EL2);
|
|
case 6:
|
|
return read_gicreg(ICH_LR6_EL2);
|
|
case 7:
|
|
return read_gicreg(ICH_LR7_EL2);
|
|
case 8:
|
|
return read_gicreg(ICH_LR8_EL2);
|
|
case 9:
|
|
return read_gicreg(ICH_LR9_EL2);
|
|
case 10:
|
|
return read_gicreg(ICH_LR10_EL2);
|
|
case 11:
|
|
return read_gicreg(ICH_LR11_EL2);
|
|
case 12:
|
|
return read_gicreg(ICH_LR12_EL2);
|
|
case 13:
|
|
return read_gicreg(ICH_LR13_EL2);
|
|
case 14:
|
|
return read_gicreg(ICH_LR14_EL2);
|
|
case 15:
|
|
return read_gicreg(ICH_LR15_EL2);
|
|
}
|
|
|
|
unreachable();
|
|
}
|
|
|
|
static void __gic_v3_set_lr(u64 val, int lr)
|
|
{
|
|
switch (lr & 0xf) {
|
|
case 0:
|
|
write_gicreg(val, ICH_LR0_EL2);
|
|
break;
|
|
case 1:
|
|
write_gicreg(val, ICH_LR1_EL2);
|
|
break;
|
|
case 2:
|
|
write_gicreg(val, ICH_LR2_EL2);
|
|
break;
|
|
case 3:
|
|
write_gicreg(val, ICH_LR3_EL2);
|
|
break;
|
|
case 4:
|
|
write_gicreg(val, ICH_LR4_EL2);
|
|
break;
|
|
case 5:
|
|
write_gicreg(val, ICH_LR5_EL2);
|
|
break;
|
|
case 6:
|
|
write_gicreg(val, ICH_LR6_EL2);
|
|
break;
|
|
case 7:
|
|
write_gicreg(val, ICH_LR7_EL2);
|
|
break;
|
|
case 8:
|
|
write_gicreg(val, ICH_LR8_EL2);
|
|
break;
|
|
case 9:
|
|
write_gicreg(val, ICH_LR9_EL2);
|
|
break;
|
|
case 10:
|
|
write_gicreg(val, ICH_LR10_EL2);
|
|
break;
|
|
case 11:
|
|
write_gicreg(val, ICH_LR11_EL2);
|
|
break;
|
|
case 12:
|
|
write_gicreg(val, ICH_LR12_EL2);
|
|
break;
|
|
case 13:
|
|
write_gicreg(val, ICH_LR13_EL2);
|
|
break;
|
|
case 14:
|
|
write_gicreg(val, ICH_LR14_EL2);
|
|
break;
|
|
case 15:
|
|
write_gicreg(val, ICH_LR15_EL2);
|
|
break;
|
|
}
|
|
}
|
|
|
|
static void __vgic_v3_write_ap0rn(u32 val, int n)
|
|
{
|
|
switch (n) {
|
|
case 0:
|
|
write_gicreg(val, ICH_AP0R0_EL2);
|
|
break;
|
|
case 1:
|
|
write_gicreg(val, ICH_AP0R1_EL2);
|
|
break;
|
|
case 2:
|
|
write_gicreg(val, ICH_AP0R2_EL2);
|
|
break;
|
|
case 3:
|
|
write_gicreg(val, ICH_AP0R3_EL2);
|
|
break;
|
|
}
|
|
}
|
|
|
|
static void __vgic_v3_write_ap1rn(u32 val, int n)
|
|
{
|
|
switch (n) {
|
|
case 0:
|
|
write_gicreg(val, ICH_AP1R0_EL2);
|
|
break;
|
|
case 1:
|
|
write_gicreg(val, ICH_AP1R1_EL2);
|
|
break;
|
|
case 2:
|
|
write_gicreg(val, ICH_AP1R2_EL2);
|
|
break;
|
|
case 3:
|
|
write_gicreg(val, ICH_AP1R3_EL2);
|
|
break;
|
|
}
|
|
}
|
|
|
|
static u32 __vgic_v3_read_ap0rn(int n)
|
|
{
|
|
u32 val;
|
|
|
|
switch (n) {
|
|
case 0:
|
|
val = read_gicreg(ICH_AP0R0_EL2);
|
|
break;
|
|
case 1:
|
|
val = read_gicreg(ICH_AP0R1_EL2);
|
|
break;
|
|
case 2:
|
|
val = read_gicreg(ICH_AP0R2_EL2);
|
|
break;
|
|
case 3:
|
|
val = read_gicreg(ICH_AP0R3_EL2);
|
|
break;
|
|
default:
|
|
unreachable();
|
|
}
|
|
|
|
return val;
|
|
}
|
|
|
|
static u32 __vgic_v3_read_ap1rn(int n)
|
|
{
|
|
u32 val;
|
|
|
|
switch (n) {
|
|
case 0:
|
|
val = read_gicreg(ICH_AP1R0_EL2);
|
|
break;
|
|
case 1:
|
|
val = read_gicreg(ICH_AP1R1_EL2);
|
|
break;
|
|
case 2:
|
|
val = read_gicreg(ICH_AP1R2_EL2);
|
|
break;
|
|
case 3:
|
|
val = read_gicreg(ICH_AP1R3_EL2);
|
|
break;
|
|
default:
|
|
unreachable();
|
|
}
|
|
|
|
return val;
|
|
}
|
|
|
|
void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
|
|
{
|
|
u64 used_lrs = cpu_if->used_lrs;
|
|
|
|
/*
|
|
* Make sure stores to the GIC via the memory mapped interface
|
|
* are now visible to the system register interface when reading the
|
|
* LRs, and when reading back the VMCR on non-VHE systems.
|
|
*/
|
|
if (used_lrs || !has_vhe()) {
|
|
if (!cpu_if->vgic_sre) {
|
|
dsb(sy);
|
|
isb();
|
|
}
|
|
}
|
|
|
|
if (used_lrs || cpu_if->its_vpe.its_vm) {
|
|
int i;
|
|
u32 elrsr;
|
|
|
|
elrsr = read_gicreg(ICH_ELRSR_EL2);
|
|
|
|
write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2);
|
|
|
|
for (i = 0; i < used_lrs; i++) {
|
|
if (elrsr & (1 << i))
|
|
cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
|
|
else
|
|
cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
|
|
|
|
__gic_v3_set_lr(0, i);
|
|
}
|
|
}
|
|
}
|
|
|
|
void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
|
|
{
|
|
u64 used_lrs = cpu_if->used_lrs;
|
|
int i;
|
|
|
|
if (used_lrs || cpu_if->its_vpe.its_vm) {
|
|
write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
|
|
|
|
for (i = 0; i < used_lrs; i++)
|
|
__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
|
|
}
|
|
|
|
/*
|
|
* Ensure that writes to the LRs, and on non-VHE systems ensure that
|
|
* the write to the VMCR in __vgic_v3_activate_traps(), will have
|
|
* reached the (re)distributors. This ensure the guest will read the
|
|
* correct values from the memory-mapped interface.
|
|
*/
|
|
if (used_lrs || !has_vhe()) {
|
|
if (!cpu_if->vgic_sre) {
|
|
isb();
|
|
dsb(sy);
|
|
}
|
|
}
|
|
}
|
|
|
|
void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
|
|
{
|
|
/*
|
|
* VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
|
|
* Group0 interrupt (as generated in GICv2 mode) to be
|
|
* delivered as a FIQ to the guest, with potentially fatal
|
|
* consequences. So we must make sure that ICC_SRE_EL1 has
|
|
* been actually programmed with the value we want before
|
|
* starting to mess with the rest of the GIC, and VMCR_EL2 in
|
|
* particular. This logic must be called before
|
|
* __vgic_v3_restore_state().
|
|
*/
|
|
if (!cpu_if->vgic_sre) {
|
|
write_gicreg(0, ICC_SRE_EL1);
|
|
isb();
|
|
write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
|
|
|
|
|
|
if (has_vhe()) {
|
|
/*
|
|
* Ensure that the write to the VMCR will have reached
|
|
* the (re)distributors. This ensure the guest will
|
|
* read the correct values from the memory-mapped
|
|
* interface.
|
|
*/
|
|
isb();
|
|
dsb(sy);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Prevent the guest from touching the GIC system registers if
|
|
* SRE isn't enabled for GICv3 emulation.
|
|
*/
|
|
write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
|
|
ICC_SRE_EL2);
|
|
|
|
/*
|
|
* If we need to trap system registers, we must write
|
|
* ICH_HCR_EL2 anyway, even if no interrupts are being
|
|
* injected,
|
|
*/
|
|
if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
|
|
cpu_if->its_vpe.its_vm)
|
|
write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
|
|
}
|
|
|
|
void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
|
|
{
|
|
u64 val;
|
|
|
|
if (!cpu_if->vgic_sre) {
|
|
cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
|
|
}
|
|
|
|
val = read_gicreg(ICC_SRE_EL2);
|
|
write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
|
|
|
|
if (!cpu_if->vgic_sre) {
|
|
/* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
|
|
isb();
|
|
write_gicreg(1, ICC_SRE_EL1);
|
|
}
|
|
|
|
/*
|
|
* If we were trapping system registers, we enabled the VGIC even if
|
|
* no interrupts were being injected, and we disable it again here.
|
|
*/
|
|
if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
|
|
cpu_if->its_vpe.its_vm)
|
|
write_gicreg(0, ICH_HCR_EL2);
|
|
}
|
|
|
|
void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
|
|
{
|
|
u64 val;
|
|
u32 nr_pre_bits;
|
|
|
|
val = read_gicreg(ICH_VTR_EL2);
|
|
nr_pre_bits = vtr_to_nr_pre_bits(val);
|
|
|
|
switch (nr_pre_bits) {
|
|
case 7:
|
|
cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
|
|
cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
|
|
fallthrough;
|
|
case 6:
|
|
cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
|
|
fallthrough;
|
|
default:
|
|
cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
|
|
}
|
|
|
|
switch (nr_pre_bits) {
|
|
case 7:
|
|
cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
|
|
cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
|
|
fallthrough;
|
|
case 6:
|
|
cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
|
|
fallthrough;
|
|
default:
|
|
cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
|
|
}
|
|
}
|
|
|
|
void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
|
|
{
|
|
u64 val;
|
|
u32 nr_pre_bits;
|
|
|
|
val = read_gicreg(ICH_VTR_EL2);
|
|
nr_pre_bits = vtr_to_nr_pre_bits(val);
|
|
|
|
switch (nr_pre_bits) {
|
|
case 7:
|
|
__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
|
|
__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
|
|
fallthrough;
|
|
case 6:
|
|
__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
|
|
fallthrough;
|
|
default:
|
|
__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
|
|
}
|
|
|
|
switch (nr_pre_bits) {
|
|
case 7:
|
|
__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
|
|
__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
|
|
fallthrough;
|
|
case 6:
|
|
__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
|
|
fallthrough;
|
|
default:
|
|
__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
|
|
}
|
|
}
|
|
|
|
void __vgic_v3_init_lrs(void)
|
|
{
|
|
int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
|
|
int i;
|
|
|
|
for (i = 0; i <= max_lr_idx; i++)
|
|
__gic_v3_set_lr(0, i);
|
|
}
|
|
|
|
/*
|
|
* Return the GIC CPU configuration:
|
|
* - [31:0] ICH_VTR_EL2
|
|
* - [62:32] RES0
|
|
* - [63] MMIO (GICv2) capable
|
|
*/
|
|
u64 __vgic_v3_get_gic_config(void)
|
|
{
|
|
u64 val, sre = read_gicreg(ICC_SRE_EL1);
|
|
unsigned long flags = 0;
|
|
|
|
/*
|
|
* To check whether we have a MMIO-based (GICv2 compatible)
|
|
* CPU interface, we need to disable the system register
|
|
* view. To do that safely, we have to prevent any interrupt
|
|
* from firing (which would be deadly).
|
|
*
|
|
* Note that this only makes sense on VHE, as interrupts are
|
|
* already masked for nVHE as part of the exception entry to
|
|
* EL2.
|
|
*/
|
|
if (has_vhe())
|
|
flags = local_daif_save();
|
|
|
|
/*
|
|
* Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
|
|
* that to be able to set ICC_SRE_EL1.SRE to 0, all the
|
|
* interrupt overrides must be set. You've got to love this.
|
|
*/
|
|
sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);
|
|
isb();
|
|
write_gicreg(0, ICC_SRE_EL1);
|
|
isb();
|
|
|
|
val = read_gicreg(ICC_SRE_EL1);
|
|
|
|
write_gicreg(sre, ICC_SRE_EL1);
|
|
isb();
|
|
sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
|
|
isb();
|
|
|
|
if (has_vhe())
|
|
local_daif_restore(flags);
|
|
|
|
val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
|
|
val |= read_gicreg(ICH_VTR_EL2);
|
|
|
|
return val;
|
|
}
|
|
|
|
u64 __vgic_v3_read_vmcr(void)
|
|
{
|
|
return read_gicreg(ICH_VMCR_EL2);
|
|
}
|
|
|
|
void __vgic_v3_write_vmcr(u32 vmcr)
|
|
{
|
|
write_gicreg(vmcr, ICH_VMCR_EL2);
|
|
}
|
|
|
|
static int __vgic_v3_bpr_min(void)
|
|
{
|
|
/* See Pseudocode for VPriorityGroup */
|
|
return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
|
|
}
|
|
|
|
static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
|
|
{
|
|
u32 esr = kvm_vcpu_get_esr(vcpu);
|
|
u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
|
|
|
|
return crm != 8;
|
|
}
|
|
|
|
#define GICv3_IDLE_PRIORITY 0xff
|
|
|
|
static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
|
|
u64 *lr_val)
|
|
{
|
|
unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
|
|
u8 priority = GICv3_IDLE_PRIORITY;
|
|
int i, lr = -1;
|
|
|
|
for (i = 0; i < used_lrs; i++) {
|
|
u64 val = __gic_v3_get_lr(i);
|
|
u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
|
|
|
|
/* Not pending in the state? */
|
|
if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
|
|
continue;
|
|
|
|
/* Group-0 interrupt, but Group-0 disabled? */
|
|
if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
|
|
continue;
|
|
|
|
/* Group-1 interrupt, but Group-1 disabled? */
|
|
if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
|
|
continue;
|
|
|
|
/* Not the highest priority? */
|
|
if (lr_prio >= priority)
|
|
continue;
|
|
|
|
/* This is a candidate */
|
|
priority = lr_prio;
|
|
*lr_val = val;
|
|
lr = i;
|
|
}
|
|
|
|
if (lr == -1)
|
|
*lr_val = ICC_IAR1_EL1_SPURIOUS;
|
|
|
|
return lr;
|
|
}
|
|
|
|
static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
|
|
u64 *lr_val)
|
|
{
|
|
unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
|
|
int i;
|
|
|
|
for (i = 0; i < used_lrs; i++) {
|
|
u64 val = __gic_v3_get_lr(i);
|
|
|
|
if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
|
|
(val & ICH_LR_ACTIVE_BIT)) {
|
|
*lr_val = val;
|
|
return i;
|
|
}
|
|
}
|
|
|
|
*lr_val = ICC_IAR1_EL1_SPURIOUS;
|
|
return -1;
|
|
}
|
|
|
|
static int __vgic_v3_get_highest_active_priority(void)
|
|
{
|
|
u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
|
|
u32 hap = 0;
|
|
int i;
|
|
|
|
for (i = 0; i < nr_apr_regs; i++) {
|
|
u32 val;
|
|
|
|
/*
|
|
* The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
|
|
* contain the active priority levels for this VCPU
|
|
* for the maximum number of supported priority
|
|
* levels, and we return the full priority level only
|
|
* if the BPR is programmed to its minimum, otherwise
|
|
* we return a combination of the priority level and
|
|
* subpriority, as determined by the setting of the
|
|
* BPR, but without the full subpriority.
|
|
*/
|
|
val = __vgic_v3_read_ap0rn(i);
|
|
val |= __vgic_v3_read_ap1rn(i);
|
|
if (!val) {
|
|
hap += 32;
|
|
continue;
|
|
}
|
|
|
|
return (hap + __ffs(val)) << __vgic_v3_bpr_min();
|
|
}
|
|
|
|
return GICv3_IDLE_PRIORITY;
|
|
}
|
|
|
|
static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
|
|
{
|
|
return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
|
|
}
|
|
|
|
static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
|
|
{
|
|
unsigned int bpr;
|
|
|
|
if (vmcr & ICH_VMCR_CBPR_MASK) {
|
|
bpr = __vgic_v3_get_bpr0(vmcr);
|
|
if (bpr < 7)
|
|
bpr++;
|
|
} else {
|
|
bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
|
|
}
|
|
|
|
return bpr;
|
|
}
|
|
|
|
/*
|
|
* Convert a priority to a preemption level, taking the relevant BPR
|
|
* into account by zeroing the sub-priority bits.
|
|
*/
|
|
static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
|
|
{
|
|
unsigned int bpr;
|
|
|
|
if (!grp)
|
|
bpr = __vgic_v3_get_bpr0(vmcr) + 1;
|
|
else
|
|
bpr = __vgic_v3_get_bpr1(vmcr);
|
|
|
|
return pri & (GENMASK(7, 0) << bpr);
|
|
}
|
|
|
|
/*
|
|
* The priority value is independent of any of the BPR values, so we
|
|
* normalize it using the minimal BPR value. This guarantees that no
|
|
* matter what the guest does with its BPR, we can always set/get the
|
|
* same value of a priority.
|
|
*/
|
|
static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
|
|
{
|
|
u8 pre, ap;
|
|
u32 val;
|
|
int apr;
|
|
|
|
pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
|
|
ap = pre >> __vgic_v3_bpr_min();
|
|
apr = ap / 32;
|
|
|
|
if (!grp) {
|
|
val = __vgic_v3_read_ap0rn(apr);
|
|
__vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
|
|
} else {
|
|
val = __vgic_v3_read_ap1rn(apr);
|
|
__vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
|
|
}
|
|
}
|
|
|
|
static int __vgic_v3_clear_highest_active_priority(void)
|
|
{
|
|
u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
|
|
u32 hap = 0;
|
|
int i;
|
|
|
|
for (i = 0; i < nr_apr_regs; i++) {
|
|
u32 ap0, ap1;
|
|
int c0, c1;
|
|
|
|
ap0 = __vgic_v3_read_ap0rn(i);
|
|
ap1 = __vgic_v3_read_ap1rn(i);
|
|
if (!ap0 && !ap1) {
|
|
hap += 32;
|
|
continue;
|
|
}
|
|
|
|
c0 = ap0 ? __ffs(ap0) : 32;
|
|
c1 = ap1 ? __ffs(ap1) : 32;
|
|
|
|
/* Always clear the LSB, which is the highest priority */
|
|
if (c0 < c1) {
|
|
ap0 &= ~BIT(c0);
|
|
__vgic_v3_write_ap0rn(ap0, i);
|
|
hap += c0;
|
|
} else {
|
|
ap1 &= ~BIT(c1);
|
|
__vgic_v3_write_ap1rn(ap1, i);
|
|
hap += c1;
|
|
}
|
|
|
|
/* Rescale to 8 bits of priority */
|
|
return hap << __vgic_v3_bpr_min();
|
|
}
|
|
|
|
return GICv3_IDLE_PRIORITY;
|
|
}
|
|
|
|
static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u64 lr_val;
|
|
u8 lr_prio, pmr;
|
|
int lr, grp;
|
|
|
|
grp = __vgic_v3_get_group(vcpu);
|
|
|
|
lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
|
|
if (lr < 0)
|
|
goto spurious;
|
|
|
|
if (grp != !!(lr_val & ICH_LR_GROUP))
|
|
goto spurious;
|
|
|
|
pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
|
|
lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
|
|
if (pmr <= lr_prio)
|
|
goto spurious;
|
|
|
|
if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
|
|
goto spurious;
|
|
|
|
lr_val &= ~ICH_LR_STATE;
|
|
lr_val |= ICH_LR_ACTIVE_BIT;
|
|
__gic_v3_set_lr(lr_val, lr);
|
|
__vgic_v3_set_active_priority(lr_prio, vmcr, grp);
|
|
vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
|
|
return;
|
|
|
|
spurious:
|
|
vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
|
|
}
|
|
|
|
static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
|
|
{
|
|
lr_val &= ~ICH_LR_ACTIVE_BIT;
|
|
if (lr_val & ICH_LR_HW) {
|
|
u32 pid;
|
|
|
|
pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
|
|
gic_write_dir(pid);
|
|
}
|
|
|
|
__gic_v3_set_lr(lr_val, lr);
|
|
}
|
|
|
|
static void __vgic_v3_bump_eoicount(void)
|
|
{
|
|
u32 hcr;
|
|
|
|
hcr = read_gicreg(ICH_HCR_EL2);
|
|
hcr += 1 << ICH_HCR_EOIcount_SHIFT;
|
|
write_gicreg(hcr, ICH_HCR_EL2);
|
|
}
|
|
|
|
static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u32 vid = vcpu_get_reg(vcpu, rt);
|
|
u64 lr_val;
|
|
int lr;
|
|
|
|
/* EOImode == 0, nothing to be done here */
|
|
if (!(vmcr & ICH_VMCR_EOIM_MASK))
|
|
return;
|
|
|
|
/* No deactivate to be performed on an LPI */
|
|
if (vid >= VGIC_MIN_LPI)
|
|
return;
|
|
|
|
lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
|
|
if (lr == -1) {
|
|
__vgic_v3_bump_eoicount();
|
|
return;
|
|
}
|
|
|
|
__vgic_v3_clear_active_lr(lr, lr_val);
|
|
}
|
|
|
|
static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u32 vid = vcpu_get_reg(vcpu, rt);
|
|
u64 lr_val;
|
|
u8 lr_prio, act_prio;
|
|
int lr, grp;
|
|
|
|
grp = __vgic_v3_get_group(vcpu);
|
|
|
|
/* Drop priority in any case */
|
|
act_prio = __vgic_v3_clear_highest_active_priority();
|
|
|
|
lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
|
|
if (lr == -1) {
|
|
/* Do not bump EOIcount for LPIs that aren't in the LRs */
|
|
if (!(vid >= VGIC_MIN_LPI))
|
|
__vgic_v3_bump_eoicount();
|
|
return;
|
|
}
|
|
|
|
/* EOImode == 1 and not an LPI, nothing to be done here */
|
|
if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
|
|
return;
|
|
|
|
lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
|
|
|
|
/* If priorities or group do not match, the guest has fscked-up. */
|
|
if (grp != !!(lr_val & ICH_LR_GROUP) ||
|
|
__vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
|
|
return;
|
|
|
|
/* Let's now perform the deactivation */
|
|
__vgic_v3_clear_active_lr(lr, lr_val);
|
|
}
|
|
|
|
static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
|
|
}
|
|
|
|
static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
|
|
}
|
|
|
|
static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u64 val = vcpu_get_reg(vcpu, rt);
|
|
|
|
if (val & 1)
|
|
vmcr |= ICH_VMCR_ENG0_MASK;
|
|
else
|
|
vmcr &= ~ICH_VMCR_ENG0_MASK;
|
|
|
|
__vgic_v3_write_vmcr(vmcr);
|
|
}
|
|
|
|
static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u64 val = vcpu_get_reg(vcpu, rt);
|
|
|
|
if (val & 1)
|
|
vmcr |= ICH_VMCR_ENG1_MASK;
|
|
else
|
|
vmcr &= ~ICH_VMCR_ENG1_MASK;
|
|
|
|
__vgic_v3_write_vmcr(vmcr);
|
|
}
|
|
|
|
static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
|
|
}
|
|
|
|
static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
|
|
}
|
|
|
|
static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u64 val = vcpu_get_reg(vcpu, rt);
|
|
u8 bpr_min = __vgic_v3_bpr_min() - 1;
|
|
|
|
/* Enforce BPR limiting */
|
|
if (val < bpr_min)
|
|
val = bpr_min;
|
|
|
|
val <<= ICH_VMCR_BPR0_SHIFT;
|
|
val &= ICH_VMCR_BPR0_MASK;
|
|
vmcr &= ~ICH_VMCR_BPR0_MASK;
|
|
vmcr |= val;
|
|
|
|
__vgic_v3_write_vmcr(vmcr);
|
|
}
|
|
|
|
static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u64 val = vcpu_get_reg(vcpu, rt);
|
|
u8 bpr_min = __vgic_v3_bpr_min();
|
|
|
|
if (vmcr & ICH_VMCR_CBPR_MASK)
|
|
return;
|
|
|
|
/* Enforce BPR limiting */
|
|
if (val < bpr_min)
|
|
val = bpr_min;
|
|
|
|
val <<= ICH_VMCR_BPR1_SHIFT;
|
|
val &= ICH_VMCR_BPR1_MASK;
|
|
vmcr &= ~ICH_VMCR_BPR1_MASK;
|
|
vmcr |= val;
|
|
|
|
__vgic_v3_write_vmcr(vmcr);
|
|
}
|
|
|
|
static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
|
|
{
|
|
u32 val;
|
|
|
|
if (!__vgic_v3_get_group(vcpu))
|
|
val = __vgic_v3_read_ap0rn(n);
|
|
else
|
|
val = __vgic_v3_read_ap1rn(n);
|
|
|
|
vcpu_set_reg(vcpu, rt, val);
|
|
}
|
|
|
|
static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
|
|
{
|
|
u32 val = vcpu_get_reg(vcpu, rt);
|
|
|
|
if (!__vgic_v3_get_group(vcpu))
|
|
__vgic_v3_write_ap0rn(val, n);
|
|
else
|
|
__vgic_v3_write_ap1rn(val, n);
|
|
}
|
|
|
|
static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
|
|
u32 vmcr, int rt)
|
|
{
|
|
__vgic_v3_read_apxrn(vcpu, rt, 0);
|
|
}
|
|
|
|
static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
|
|
u32 vmcr, int rt)
|
|
{
|
|
__vgic_v3_read_apxrn(vcpu, rt, 1);
|
|
}
|
|
|
|
static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
__vgic_v3_read_apxrn(vcpu, rt, 2);
|
|
}
|
|
|
|
static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
__vgic_v3_read_apxrn(vcpu, rt, 3);
|
|
}
|
|
|
|
static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
__vgic_v3_write_apxrn(vcpu, rt, 0);
|
|
}
|
|
|
|
static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
__vgic_v3_write_apxrn(vcpu, rt, 1);
|
|
}
|
|
|
|
static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
__vgic_v3_write_apxrn(vcpu, rt, 2);
|
|
}
|
|
|
|
static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
__vgic_v3_write_apxrn(vcpu, rt, 3);
|
|
}
|
|
|
|
static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u64 lr_val;
|
|
int lr, lr_grp, grp;
|
|
|
|
grp = __vgic_v3_get_group(vcpu);
|
|
|
|
lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
|
|
if (lr == -1)
|
|
goto spurious;
|
|
|
|
lr_grp = !!(lr_val & ICH_LR_GROUP);
|
|
if (lr_grp != grp)
|
|
lr_val = ICC_IAR1_EL1_SPURIOUS;
|
|
|
|
spurious:
|
|
vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
|
|
}
|
|
|
|
static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
vmcr &= ICH_VMCR_PMR_MASK;
|
|
vmcr >>= ICH_VMCR_PMR_SHIFT;
|
|
vcpu_set_reg(vcpu, rt, vmcr);
|
|
}
|
|
|
|
static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u32 val = vcpu_get_reg(vcpu, rt);
|
|
|
|
val <<= ICH_VMCR_PMR_SHIFT;
|
|
val &= ICH_VMCR_PMR_MASK;
|
|
vmcr &= ~ICH_VMCR_PMR_MASK;
|
|
vmcr |= val;
|
|
|
|
write_gicreg(vmcr, ICH_VMCR_EL2);
|
|
}
|
|
|
|
static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u32 val = __vgic_v3_get_highest_active_priority();
|
|
vcpu_set_reg(vcpu, rt, val);
|
|
}
|
|
|
|
static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u32 vtr, val;
|
|
|
|
vtr = read_gicreg(ICH_VTR_EL2);
|
|
/* PRIbits */
|
|
val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
|
|
/* IDbits */
|
|
val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
|
|
/* SEIS */
|
|
if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK)
|
|
val |= BIT(ICC_CTLR_EL1_SEIS_SHIFT);
|
|
/* A3V */
|
|
val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
|
|
/* EOImode */
|
|
val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
|
|
/* CBPR */
|
|
val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
|
|
|
|
vcpu_set_reg(vcpu, rt, val);
|
|
}
|
|
|
|
static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
|
|
{
|
|
u32 val = vcpu_get_reg(vcpu, rt);
|
|
|
|
if (val & ICC_CTLR_EL1_CBPR_MASK)
|
|
vmcr |= ICH_VMCR_CBPR_MASK;
|
|
else
|
|
vmcr &= ~ICH_VMCR_CBPR_MASK;
|
|
|
|
if (val & ICC_CTLR_EL1_EOImode_MASK)
|
|
vmcr |= ICH_VMCR_EOIM_MASK;
|
|
else
|
|
vmcr &= ~ICH_VMCR_EOIM_MASK;
|
|
|
|
write_gicreg(vmcr, ICH_VMCR_EL2);
|
|
}
|
|
|
|
int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
|
|
{
|
|
int rt;
|
|
u32 esr;
|
|
u32 vmcr;
|
|
void (*fn)(struct kvm_vcpu *, u32, int);
|
|
bool is_read;
|
|
u32 sysreg;
|
|
|
|
esr = kvm_vcpu_get_esr(vcpu);
|
|
if (vcpu_mode_is_32bit(vcpu)) {
|
|
if (!kvm_condition_valid(vcpu)) {
|
|
__kvm_skip_instr(vcpu);
|
|
return 1;
|
|
}
|
|
|
|
sysreg = esr_cp15_to_sysreg(esr);
|
|
} else {
|
|
sysreg = esr_sys64_to_sysreg(esr);
|
|
}
|
|
|
|
is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
|
|
|
|
switch (sysreg) {
|
|
case SYS_ICC_IAR0_EL1:
|
|
case SYS_ICC_IAR1_EL1:
|
|
if (unlikely(!is_read))
|
|
return 0;
|
|
fn = __vgic_v3_read_iar;
|
|
break;
|
|
case SYS_ICC_EOIR0_EL1:
|
|
case SYS_ICC_EOIR1_EL1:
|
|
if (unlikely(is_read))
|
|
return 0;
|
|
fn = __vgic_v3_write_eoir;
|
|
break;
|
|
case SYS_ICC_IGRPEN1_EL1:
|
|
if (is_read)
|
|
fn = __vgic_v3_read_igrpen1;
|
|
else
|
|
fn = __vgic_v3_write_igrpen1;
|
|
break;
|
|
case SYS_ICC_BPR1_EL1:
|
|
if (is_read)
|
|
fn = __vgic_v3_read_bpr1;
|
|
else
|
|
fn = __vgic_v3_write_bpr1;
|
|
break;
|
|
case SYS_ICC_AP0Rn_EL1(0):
|
|
case SYS_ICC_AP1Rn_EL1(0):
|
|
if (is_read)
|
|
fn = __vgic_v3_read_apxr0;
|
|
else
|
|
fn = __vgic_v3_write_apxr0;
|
|
break;
|
|
case SYS_ICC_AP0Rn_EL1(1):
|
|
case SYS_ICC_AP1Rn_EL1(1):
|
|
if (is_read)
|
|
fn = __vgic_v3_read_apxr1;
|
|
else
|
|
fn = __vgic_v3_write_apxr1;
|
|
break;
|
|
case SYS_ICC_AP0Rn_EL1(2):
|
|
case SYS_ICC_AP1Rn_EL1(2):
|
|
if (is_read)
|
|
fn = __vgic_v3_read_apxr2;
|
|
else
|
|
fn = __vgic_v3_write_apxr2;
|
|
break;
|
|
case SYS_ICC_AP0Rn_EL1(3):
|
|
case SYS_ICC_AP1Rn_EL1(3):
|
|
if (is_read)
|
|
fn = __vgic_v3_read_apxr3;
|
|
else
|
|
fn = __vgic_v3_write_apxr3;
|
|
break;
|
|
case SYS_ICC_HPPIR0_EL1:
|
|
case SYS_ICC_HPPIR1_EL1:
|
|
if (unlikely(!is_read))
|
|
return 0;
|
|
fn = __vgic_v3_read_hppir;
|
|
break;
|
|
case SYS_ICC_IGRPEN0_EL1:
|
|
if (is_read)
|
|
fn = __vgic_v3_read_igrpen0;
|
|
else
|
|
fn = __vgic_v3_write_igrpen0;
|
|
break;
|
|
case SYS_ICC_BPR0_EL1:
|
|
if (is_read)
|
|
fn = __vgic_v3_read_bpr0;
|
|
else
|
|
fn = __vgic_v3_write_bpr0;
|
|
break;
|
|
case SYS_ICC_DIR_EL1:
|
|
if (unlikely(is_read))
|
|
return 0;
|
|
fn = __vgic_v3_write_dir;
|
|
break;
|
|
case SYS_ICC_RPR_EL1:
|
|
if (unlikely(!is_read))
|
|
return 0;
|
|
fn = __vgic_v3_read_rpr;
|
|
break;
|
|
case SYS_ICC_CTLR_EL1:
|
|
if (is_read)
|
|
fn = __vgic_v3_read_ctlr;
|
|
else
|
|
fn = __vgic_v3_write_ctlr;
|
|
break;
|
|
case SYS_ICC_PMR_EL1:
|
|
if (is_read)
|
|
fn = __vgic_v3_read_pmr;
|
|
else
|
|
fn = __vgic_v3_write_pmr;
|
|
break;
|
|
default:
|
|
return 0;
|
|
}
|
|
|
|
vmcr = __vgic_v3_read_vmcr();
|
|
rt = kvm_vcpu_sys_get_rt(vcpu);
|
|
fn(vcpu, vmcr, rt);
|
|
|
|
__kvm_skip_instr(vcpu);
|
|
|
|
return 1;
|
|
}
|