diff options
author | Paolo Bonzini <pbonzini@redhat.com> | 2020-06-01 04:26:27 -0400 |
---|---|---|
committer | Paolo Bonzini <pbonzini@redhat.com> | 2020-06-01 04:26:27 -0400 |
commit | 380609445cff6ee7b6c7e43bbdc953985b45a6ed (patch) | |
tree | 2526fa1599b19804e8cd9823415919ad757f07e7 /virt | |
parent | 09d952c971a58c897eb7ea7d41516e7d953bb65d (diff) | |
parent | 8f7f4fe756bd5cfef73cf8234445081385bdbf7d (diff) |
Merge tag 'kvmarm-5.8' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD
KVM/arm64 updates for Linux 5.8:
- Move the arch-specific code into arch/arm64/kvm
- Start the post-32bit cleanup
- Cherry-pick a few non-invasive pre-NV patches
Diffstat (limited to 'virt')
32 files changed, 3 insertions, 19576 deletions
diff --git a/virt/kvm/arm/aarch32.c b/virt/kvm/arm/aarch32.c deleted file mode 100644 index 0a356aa91aa1..000000000000 --- a/virt/kvm/arm/aarch32.c +++ /dev/null @@ -1,204 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * (not much of an) Emulation layer for 32bit guests. - * - * Copyright (C) 2012,2013 - ARM Ltd - * Author: Marc Zyngier <marc.zyngier@arm.com> - * - * based on arch/arm/kvm/emulate.c - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall <c.dall@virtualopensystems.com> - */ - -#include <linux/bits.h> -#include <linux/kvm_host.h> -#include <asm/kvm_emulate.h> -#include <asm/kvm_hyp.h> - -#define DFSR_FSC_EXTABT_LPAE 0x10 -#define DFSR_FSC_EXTABT_nLPAE 0x08 -#define DFSR_LPAE BIT(9) - -/* - * Table taken from ARMv8 ARM DDI0487B-B, table G1-10. - */ -static const u8 return_offsets[8][2] = { - [0] = { 0, 0 }, /* Reset, unused */ - [1] = { 4, 2 }, /* Undefined */ - [2] = { 0, 0 }, /* SVC, unused */ - [3] = { 4, 4 }, /* Prefetch abort */ - [4] = { 8, 8 }, /* Data abort */ - [5] = { 0, 0 }, /* HVC, unused */ - [6] = { 4, 4 }, /* IRQ, unused */ - [7] = { 4, 4 }, /* FIQ, unused */ -}; - -/* - * When an exception is taken, most CPSR fields are left unchanged in the - * handler. However, some are explicitly overridden (e.g. M[4:0]). - * - * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with - * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was - * obsoleted by the ARMv7 virtualization extensions and is RES0. - * - * For the SPSR layout seen from AArch32, see: - * - ARM DDI 0406C.d, page B1-1148 - * - ARM DDI 0487E.a, page G8-6264 - * - * For the SPSR_ELx layout for AArch32 seen from AArch64, see: - * - ARM DDI 0487E.a, page C5-426 - * - * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from - * MSB to LSB. - */ -static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode) -{ - u32 sctlr = vcpu_cp15(vcpu, c1_SCTLR); - unsigned long old, new; - - old = *vcpu_cpsr(vcpu); - new = 0; - - new |= (old & PSR_AA32_N_BIT); - new |= (old & PSR_AA32_Z_BIT); - new |= (old & PSR_AA32_C_BIT); - new |= (old & PSR_AA32_V_BIT); - new |= (old & PSR_AA32_Q_BIT); - - // CPSR.IT[7:0] are set to zero upon any exception - // See ARM DDI 0487E.a, section G1.12.3 - // See ARM DDI 0406C.d, section B1.8.3 - - new |= (old & PSR_AA32_DIT_BIT); - - // CPSR.SSBS is set to SCTLR.DSSBS upon any exception - // See ARM DDI 0487E.a, page G8-6244 - if (sctlr & BIT(31)) - new |= PSR_AA32_SSBS_BIT; - - // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0 - // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented - // See ARM DDI 0487E.a, page G8-6246 - new |= (old & PSR_AA32_PAN_BIT); - if (!(sctlr & BIT(23))) - new |= PSR_AA32_PAN_BIT; - - // SS does not exist in AArch32, so ignore - - // CPSR.IL is set to zero upon any exception - // See ARM DDI 0487E.a, page G1-5527 - - new |= (old & PSR_AA32_GE_MASK); - - // CPSR.IT[7:0] are set to zero upon any exception - // See prior comment above - - // CPSR.E is set to SCTLR.EE upon any exception - // See ARM DDI 0487E.a, page G8-6245 - // See ARM DDI 0406C.d, page B4-1701 - if (sctlr & BIT(25)) - new |= PSR_AA32_E_BIT; - - // CPSR.A is unchanged upon an exception to Undefined, Supervisor - // CPSR.A is set upon an exception to other modes - // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 - // See ARM DDI 0406C.d, page B1-1182 - new |= (old & PSR_AA32_A_BIT); - if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC) - new |= PSR_AA32_A_BIT; - - // CPSR.I is set upon any exception - // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 - // See ARM DDI 0406C.d, page B1-1182 - new |= PSR_AA32_I_BIT; - - // CPSR.F is set upon an exception to FIQ - // CPSR.F is unchanged upon an exception to other modes - // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 - // See ARM DDI 0406C.d, page B1-1182 - new |= (old & PSR_AA32_F_BIT); - if (mode == PSR_AA32_MODE_FIQ) - new |= PSR_AA32_F_BIT; - - // CPSR.T is set to SCTLR.TE upon any exception - // See ARM DDI 0487E.a, page G8-5514 - // See ARM DDI 0406C.d, page B1-1181 - if (sctlr & BIT(30)) - new |= PSR_AA32_T_BIT; - - new |= mode; - - return new; -} - -static void prepare_fault32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) -{ - unsigned long spsr = *vcpu_cpsr(vcpu); - bool is_thumb = (spsr & PSR_AA32_T_BIT); - u32 return_offset = return_offsets[vect_offset >> 2][is_thumb]; - u32 sctlr = vcpu_cp15(vcpu, c1_SCTLR); - - *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode); - - /* Note: These now point to the banked copies */ - vcpu_write_spsr(vcpu, host_spsr_to_spsr32(spsr)); - *vcpu_reg32(vcpu, 14) = *vcpu_pc(vcpu) + return_offset; - - /* Branch to exception vector */ - if (sctlr & (1 << 13)) - vect_offset += 0xffff0000; - else /* always have security exceptions */ - vect_offset += vcpu_cp15(vcpu, c12_VBAR); - - *vcpu_pc(vcpu) = vect_offset; -} - -void kvm_inject_undef32(struct kvm_vcpu *vcpu) -{ - prepare_fault32(vcpu, PSR_AA32_MODE_UND, 4); -} - -/* - * Modelled after TakeDataAbortException() and TakePrefetchAbortException - * pseudocode. - */ -static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, - unsigned long addr) -{ - u32 vect_offset; - u32 *far, *fsr; - bool is_lpae; - - if (is_pabt) { - vect_offset = 12; - far = &vcpu_cp15(vcpu, c6_IFAR); - fsr = &vcpu_cp15(vcpu, c5_IFSR); - } else { /* !iabt */ - vect_offset = 16; - far = &vcpu_cp15(vcpu, c6_DFAR); - fsr = &vcpu_cp15(vcpu, c5_DFSR); - } - - prepare_fault32(vcpu, PSR_AA32_MODE_ABT, vect_offset); - - *far = addr; - - /* Give the guest an IMPLEMENTATION DEFINED exception */ - is_lpae = (vcpu_cp15(vcpu, c2_TTBCR) >> 31); - if (is_lpae) { - *fsr = DFSR_LPAE | DFSR_FSC_EXTABT_LPAE; - } else { - /* no need to shuffle FS[4] into DFSR[10] as its 0 */ - *fsr = DFSR_FSC_EXTABT_nLPAE; - } -} - -void kvm_inject_dabt32(struct kvm_vcpu *vcpu, unsigned long addr) -{ - inject_abt32(vcpu, false, addr); -} - -void kvm_inject_pabt32(struct kvm_vcpu *vcpu, unsigned long addr) -{ - inject_abt32(vcpu, true, addr); -} diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c deleted file mode 100644 index d5024416e722..000000000000 --- a/virt/kvm/arm/arch_timer.c +++ /dev/null @@ -1,1181 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012 ARM Ltd. - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <linux/cpu.h> -#include <linux/kvm.h> -#include <linux/kvm_host.h> -#include <linux/interrupt.h> -#include <linux/irq.h> -#include <linux/uaccess.h> - -#include <clocksource/arm_arch_timer.h> -#include <asm/arch_timer.h> -#include <asm/kvm_emulate.h> -#include <asm/kvm_hyp.h> - -#include <kvm/arm_vgic.h> -#include <kvm/arm_arch_timer.h> - -#include "trace.h" - -static struct timecounter *timecounter; -static unsigned int host_vtimer_irq; -static unsigned int host_ptimer_irq; -static u32 host_vtimer_irq_flags; -static u32 host_ptimer_irq_flags; - -static DEFINE_STATIC_KEY_FALSE(has_gic_active_state); - -static const struct kvm_irq_level default_ptimer_irq = { - .irq = 30, - .level = 1, -}; - -static const struct kvm_irq_level default_vtimer_irq = { - .irq = 27, - .level = 1, -}; - -static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx); -static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, - struct arch_timer_context *timer_ctx); -static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx); -static void kvm_arm_timer_write(struct kvm_vcpu *vcpu, - struct arch_timer_context *timer, - enum kvm_arch_timer_regs treg, - u64 val); -static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu, - struct arch_timer_context *timer, - enum kvm_arch_timer_regs treg); - -u64 kvm_phys_timer_read(void) -{ - return timecounter->cc->read(timecounter->cc); -} - -static void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map) -{ - if (has_vhe()) { - map->direct_vtimer = vcpu_vtimer(vcpu); - map->direct_ptimer = vcpu_ptimer(vcpu); - map->emul_ptimer = NULL; - } else { - map->direct_vtimer = vcpu_vtimer(vcpu); - map->direct_ptimer = NULL; - map->emul_ptimer = vcpu_ptimer(vcpu); - } - - trace_kvm_get_timer_map(vcpu->vcpu_id, map); -} - -static inline bool userspace_irqchip(struct kvm *kvm) -{ - return static_branch_unlikely(&userspace_irqchip_in_use) && - unlikely(!irqchip_in_kernel(kvm)); -} - -static void soft_timer_start(struct hrtimer *hrt, u64 ns) -{ - hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns), - HRTIMER_MODE_ABS_HARD); -} - -static void soft_timer_cancel(struct hrtimer *hrt) -{ - hrtimer_cancel(hrt); -} - -static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id) -{ - struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id; - struct arch_timer_context *ctx; - struct timer_map map; - - /* - * We may see a timer interrupt after vcpu_put() has been called which - * sets the CPU's vcpu pointer to NULL, because even though the timer - * has been disabled in timer_save_state(), the hardware interrupt - * signal may not have been retired from the interrupt controller yet. - */ - if (!vcpu) - return IRQ_HANDLED; - - get_timer_map(vcpu, &map); - - if (irq == host_vtimer_irq) - ctx = map.direct_vtimer; - else - ctx = map.direct_ptimer; - - if (kvm_timer_should_fire(ctx)) - kvm_timer_update_irq(vcpu, true, ctx); - - if (userspace_irqchip(vcpu->kvm) && - !static_branch_unlikely(&has_gic_active_state)) - disable_percpu_irq(host_vtimer_irq); - - return IRQ_HANDLED; -} - -static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx) -{ - u64 cval, now; - - cval = timer_ctx->cnt_cval; - now = kvm_phys_timer_read() - timer_ctx->cntvoff; - - if (now < cval) { - u64 ns; - - ns = cyclecounter_cyc2ns(timecounter->cc, - cval - now, - timecounter->mask, - &timecounter->frac); - return ns; - } - - return 0; -} - -static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) -{ - WARN_ON(timer_ctx && timer_ctx->loaded); - return timer_ctx && - !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) && - (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE); -} - -/* - * Returns the earliest expiration time in ns among guest timers. - * Note that it will return 0 if none of timers can fire. - */ -static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu) -{ - u64 min_delta = ULLONG_MAX; - int i; - - for (i = 0; i < NR_KVM_TIMERS; i++) { - struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i]; - - WARN(ctx->loaded, "timer %d loaded\n", i); - if (kvm_timer_irq_can_fire(ctx)) - min_delta = min(min_delta, kvm_timer_compute_delta(ctx)); - } - - /* If none of timers can fire, then return 0 */ - if (min_delta == ULLONG_MAX) - return 0; - - return min_delta; -} - -static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt) -{ - struct arch_timer_cpu *timer; - struct kvm_vcpu *vcpu; - u64 ns; - - timer = container_of(hrt, struct arch_timer_cpu, bg_timer); - vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu); - - /* - * Check that the timer has really expired from the guest's - * PoV (NTP on the host may have forced it to expire - * early). If we should have slept longer, restart it. - */ - ns = kvm_timer_earliest_exp(vcpu); - if (unlikely(ns)) { - hrtimer_forward_now(hrt, ns_to_ktime(ns)); - return HRTIMER_RESTART; - } - - kvm_vcpu_wake_up(vcpu); - return HRTIMER_NORESTART; -} - -static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt) -{ - struct arch_timer_context *ctx; - struct kvm_vcpu *vcpu; - u64 ns; - - ctx = container_of(hrt, struct arch_timer_context, hrtimer); - vcpu = ctx->vcpu; - - trace_kvm_timer_hrtimer_expire(ctx); - - /* - * Check that the timer has really expired from the guest's - * PoV (NTP on the host may have forced it to expire - * early). If not ready, schedule for a later time. - */ - ns = kvm_timer_compute_delta(ctx); - if (unlikely(ns)) { - hrtimer_forward_now(hrt, ns_to_ktime(ns)); - return HRTIMER_RESTART; - } - - kvm_timer_update_irq(vcpu, true, ctx); - return HRTIMER_NORESTART; -} - -static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) -{ - enum kvm_arch_timers index; - u64 cval, now; - - if (!timer_ctx) - return false; - - index = arch_timer_ctx_index(timer_ctx); - - if (timer_ctx->loaded) { - u32 cnt_ctl = 0; - - switch (index) { - case TIMER_VTIMER: - cnt_ctl = read_sysreg_el0(SYS_CNTV_CTL); - break; - case TIMER_PTIMER: - cnt_ctl = read_sysreg_el0(SYS_CNTP_CTL); - break; - case NR_KVM_TIMERS: - /* GCC is braindead */ - cnt_ctl = 0; - break; - } - - return (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) && - (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) && - !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK); - } - - if (!kvm_timer_irq_can_fire(timer_ctx)) - return false; - - cval = timer_ctx->cnt_cval; - now = kvm_phys_timer_read() - timer_ctx->cntvoff; - - return cval <= now; -} - -bool kvm_timer_is_pending(struct kvm_vcpu *vcpu) -{ - struct timer_map map; - - get_timer_map(vcpu, &map); - - return kvm_timer_should_fire(map.direct_vtimer) || - kvm_timer_should_fire(map.direct_ptimer) || - kvm_timer_should_fire(map.emul_ptimer); -} - -/* - * Reflect the timer output level into the kvm_run structure - */ -void kvm_timer_update_run(struct kvm_vcpu *vcpu) -{ - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); - struct kvm_sync_regs *regs = &vcpu->run->s.regs; - - /* Populate the device bitmap with the timer states */ - regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER | - KVM_ARM_DEV_EL1_PTIMER); - if (kvm_timer_should_fire(vtimer)) - regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER; - if (kvm_timer_should_fire(ptimer)) - regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER; -} - -static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, - struct arch_timer_context *timer_ctx) -{ - int ret; - - timer_ctx->irq.level = new_level; - trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq, - timer_ctx->irq.level); - - if (!userspace_irqchip(vcpu->kvm)) { - ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, - timer_ctx->irq.irq, - timer_ctx->irq.level, - timer_ctx); - WARN_ON(ret); - } -} - -/* Only called for a fully emulated timer */ -static void timer_emulate(struct arch_timer_context *ctx) -{ - bool should_fire = kvm_timer_should_fire(ctx); - - trace_kvm_timer_emulate(ctx, should_fire); - - if (should_fire != ctx->irq.level) { - kvm_timer_update_irq(ctx->vcpu, should_fire, ctx); - return; - } - - /* - * If the timer can fire now, we don't need to have a soft timer - * scheduled for the future. If the timer cannot fire at all, - * then we also don't need a soft timer. - */ - if (!kvm_timer_irq_can_fire(ctx)) { - soft_timer_cancel(&ctx->hrtimer); - return; - } - - soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx)); -} - -static void timer_save_state(struct arch_timer_context *ctx) -{ - struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu); - enum kvm_arch_timers index = arch_timer_ctx_index(ctx); - unsigned long flags; - - if (!timer->enabled) - return; - - local_irq_save(flags); - - if (!ctx->loaded) - goto out; - - switch (index) { - case TIMER_VTIMER: - ctx->cnt_ctl = read_sysreg_el0(SYS_CNTV_CTL); - ctx->cnt_cval = read_sysreg_el0(SYS_CNTV_CVAL); - - /* Disable the timer */ - write_sysreg_el0(0, SYS_CNTV_CTL); - isb(); - - break; - case TIMER_PTIMER: - ctx->cnt_ctl = read_sysreg_el0(SYS_CNTP_CTL); - ctx->cnt_cval = read_sysreg_el0(SYS_CNTP_CVAL); - - /* Disable the timer */ - write_sysreg_el0(0, SYS_CNTP_CTL); - isb(); - - break; - case NR_KVM_TIMERS: - BUG(); - } - - trace_kvm_timer_save_state(ctx); - - ctx->loaded = false; -out: - local_irq_restore(flags); -} - -/* - * Schedule the background timer before calling kvm_vcpu_block, so that this - * thread is removed from its waitqueue and made runnable when there's a timer - * interrupt to handle. - */ -static void kvm_timer_blocking(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = vcpu_timer(vcpu); - struct timer_map map; - - get_timer_map(vcpu, &map); - - /* - * If no timers are capable of raising interrupts (disabled or - * masked), then there's no more work for us to do. - */ - if (!kvm_timer_irq_can_fire(map.direct_vtimer) && - !kvm_timer_irq_can_fire(map.direct_ptimer) && - !kvm_timer_irq_can_fire(map.emul_ptimer)) - return; - - /* - * At least one guest time will expire. Schedule a background timer. - * Set the earliest expiration time among the guest timers. - */ - soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu)); -} - -static void kvm_timer_unblocking(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = vcpu_timer(vcpu); - - soft_timer_cancel(&timer->bg_timer); -} - -static void timer_restore_state(struct arch_timer_context *ctx) -{ - struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu); - enum kvm_arch_timers index = arch_timer_ctx_index(ctx); - unsigned long flags; - - if (!timer->enabled) - return; - - local_irq_save(flags); - - if (ctx->loaded) - goto out; - - switch (index) { - case TIMER_VTIMER: - write_sysreg_el0(ctx->cnt_cval, SYS_CNTV_CVAL); - isb(); - write_sysreg_el0(ctx->cnt_ctl, SYS_CNTV_CTL); - break; - case TIMER_PTIMER: - write_sysreg_el0(ctx->cnt_cval, SYS_CNTP_CVAL); - isb(); - write_sysreg_el0(ctx->cnt_ctl, SYS_CNTP_CTL); - break; - case NR_KVM_TIMERS: - BUG(); - } - - trace_kvm_timer_restore_state(ctx); - - ctx->loaded = true; -out: - local_irq_restore(flags); -} - -static void set_cntvoff(u64 cntvoff) -{ - u32 low = lower_32_bits(cntvoff); - u32 high = upper_32_bits(cntvoff); - - /* - * Since kvm_call_hyp doesn't fully support the ARM PCS especially on - * 32-bit systems, but rather passes register by register shifted one - * place (we put the function address in r0/x0), we cannot simply pass - * a 64-bit value as an argument, but have to split the value in two - * 32-bit halves. - */ - kvm_call_hyp(__kvm_timer_set_cntvoff, low, high); -} - -static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active) -{ - int r; - r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active); - WARN_ON(r); -} - -static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) -{ - struct kvm_vcpu *vcpu = ctx->vcpu; - bool phys_active = false; - - /* - * Update the timer output so that it is likely to match the - * state we're about to restore. If the timer expires between - * this point and the register restoration, we'll take the - * interrupt anyway. - */ - kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx); - - if (irqchip_in_kernel(vcpu->kvm)) - phys_active = kvm_vgic_map_is_active(vcpu, ctx->irq.irq); - - phys_active |= ctx->irq.level; - - set_timer_irq_phys_active(ctx, phys_active); -} - -static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu) -{ - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - - /* - * Update the timer output so that it is likely to match the - * state we're about to restore. If the timer expires between - * this point and the register restoration, we'll take the - * interrupt anyway. - */ - kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer); - - /* - * When using a userspace irqchip with the architected timers and a - * host interrupt controller that doesn't support an active state, we - * must still prevent continuously exiting from the guest, and - * therefore mask the physical interrupt by disabling it on the host - * interrupt controller when the virtual level is high, such that the - * guest can make forward progress. Once we detect the output level - * being de-asserted, we unmask the interrupt again so that we exit - * from the guest when the timer fires. - */ - if (vtimer->irq.level) - disable_percpu_irq(host_vtimer_irq); - else - enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); -} - -void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = vcpu_timer(vcpu); - struct timer_map map; - - if (unlikely(!timer->enabled)) - return; - - get_timer_map(vcpu, &map); - - if (static_branch_likely(&has_gic_active_state)) { - kvm_timer_vcpu_load_gic(map.direct_vtimer); - if (map.direct_ptimer) - kvm_timer_vcpu_load_gic(map.direct_ptimer); - } else { - kvm_timer_vcpu_load_nogic(vcpu); - } - - set_cntvoff(map.direct_vtimer->cntvoff); - - kvm_timer_unblocking(vcpu); - - timer_restore_state(map.direct_vtimer); - if (map.direct_ptimer) - timer_restore_state(map.direct_ptimer); - - if (map.emul_ptimer) - timer_emulate(map.emul_ptimer); -} - -bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu) -{ - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); - struct kvm_sync_regs *sregs = &vcpu->run->s.regs; - bool vlevel, plevel; - - if (likely(irqchip_in_kernel(vcpu->kvm))) - return false; - - vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER; - plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER; - - return kvm_timer_should_fire(vtimer) != vlevel || - kvm_timer_should_fire(ptimer) != plevel; -} - -void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = vcpu_timer(vcpu); - struct timer_map map; - struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); - - if (unlikely(!timer->enabled)) - return; - - get_timer_map(vcpu, &map); - - timer_save_state(map.direct_vtimer); - if (map.direct_ptimer) - timer_save_state(map.direct_ptimer); - - /* - * Cancel soft timer emulation, because the only case where we - * need it after a vcpu_put is in the context of a sleeping VCPU, and - * in that case we already factor in the deadline for the physical - * timer when scheduling the bg_timer. - * - * In any case, we re-schedule the hrtimer for the physical timer when - * coming back to the VCPU thread in kvm_timer_vcpu_load(). - */ - if (map.emul_ptimer) - soft_timer_cancel(&map.emul_ptimer->hrtimer); - - if (rcuwait_active(wait)) - kvm_timer_blocking(vcpu); - - /* - * The kernel may decide to run userspace after calling vcpu_put, so - * we reset cntvoff to 0 to ensure a consistent read between user - * accesses to the virtual counter and kernel access to the physical - * counter of non-VHE case. For VHE, the virtual counter uses a fixed - * virtual offset of zero, so no need to zero CNTVOFF_EL2 register. - */ - set_cntvoff(0); -} - -/* - * With a userspace irqchip we have to check if the guest de-asserted the - * timer and if so, unmask the timer irq signal on the host interrupt - * controller to ensure that we see future timer signals. - */ -static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu) -{ - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - - if (!kvm_timer_should_fire(vtimer)) { - kvm_timer_update_irq(vcpu, false, vtimer); - if (static_branch_likely(&has_gic_active_state)) - set_timer_irq_phys_active(vtimer, false); - else - enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); - } -} - -void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = vcpu_timer(vcpu); - - if (unlikely(!timer->enabled)) - return; - - if (unlikely(!irqchip_in_kernel(vcpu->kvm))) - unmask_vtimer_irq_user(vcpu); -} - -int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = vcpu_timer(vcpu); - struct timer_map map; - - get_timer_map(vcpu, &map); - - /* - * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8 - * and to 0 for ARMv7. We provide an implementation that always - * resets the timer to be disabled and unmasked and is compliant with - * the ARMv7 architecture. - */ - vcpu_vtimer(vcpu)->cnt_ctl = 0; - vcpu_ptimer(vcpu)->cnt_ctl = 0; - - if (timer->enabled) { - kvm_timer_update_irq(vcpu, false, vcpu_vtimer(vcpu)); - kvm_timer_update_irq(vcpu, false, vcpu_ptimer(vcpu)); - - if (irqchip_in_kernel(vcpu->kvm)) { - kvm_vgic_reset_mapped_irq(vcpu, map.direct_vtimer->irq.irq); - if (map.direct_ptimer) - kvm_vgic_reset_mapped_irq(vcpu, map.direct_ptimer->irq.irq); - } - } - - if (map.emul_ptimer) - soft_timer_cancel(&map.emul_ptimer->hrtimer); - - return 0; -} - -/* Make the updates of cntvoff for all vtimer contexts atomic */ -static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff) -{ - int i; - struct kvm *kvm = vcpu->kvm; - struct kvm_vcpu *tmp; - - mutex_lock(&kvm->lock); - kvm_for_each_vcpu(i, tmp, kvm) - vcpu_vtimer(tmp)->cntvoff = cntvoff; - - /* - * When called from the vcpu create path, the CPU being created is not - * included in the loop above, so we just set it here as well. - */ - vcpu_vtimer(vcpu)->cntvoff = cntvoff; - mutex_unlock(&kvm->lock); -} - -void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = vcpu_timer(vcpu); - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); - - /* Synchronize cntvoff across all vtimers of a VM. */ - update_vtimer_cntvoff(vcpu, kvm_phys_timer_read()); - ptimer->cntvoff = 0; - - hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); - timer->bg_timer.function = kvm_bg_ti |