From 2da3ffa6e840b9f8fc65a71326c43b716992861d Mon Sep 17 00:00:00 2001 From: Andrew Scull Date: Thu, 18 Jun 2020 10:36:16 +0100 Subject: arm64: kvm: Remove kern_hyp_va from get_vcpu_ptr get_vcpu_ptr is an assembly accessor for the percpu value kvm_host_data->host_ctxt.__hyp_running_vcpu. kern_hyp_va only applies to nVHE however __hyp_running_vcpu is always assigned a pointer that has already had kern_hyp_va applied in __kvm_vcpu_run_nvhe. kern_hyp_va is currently idempotent as it just masks and inserts the tag, but this could change in future and the second application is unnecessary. Signed-off-by: Andrew Scull Signed-off-by: Marc Zyngier Reviewed-by: James Morse Link: https://lore.kernel.org/r/20200618093616.164413-1-ascull@google.com --- arch/arm64/include/asm/kvm_asm.h | 1 - 1 file changed, 1 deletion(-) (limited to 'arch/arm64/include/asm/kvm_asm.h') diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 352aaebf4198..ac71d0939f2e 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -143,7 +143,6 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; .macro get_vcpu_ptr vcpu, ctxt get_host_ctxt \ctxt, \vcpu ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU] - kern_hyp_va \vcpu .endm #endif -- cgit v1.2.3 From f50b6f6ae131b6ee7d5dd738961eda0c00b7f027 Mon Sep 17 00:00:00 2001 From: Andrew Scull Date: Thu, 25 Jun 2020 14:14:10 +0100 Subject: KVM: arm64: Handle calls to prefixed hyp functions Once hyp functions are moved to a hyp object, they will have prefixed symbols. This change declares and gets the address of the prefixed version for calls to the hyp functions. To aid migration, the hyp functions that have not yet moved have their prefixed versions aliased to their non-prefixed version. This begins with all the hyp functions being listed and will reduce to none of them once the migration is complete. Signed-off-by: Andrew Scull [David: Extracted kvm_call_hyp nVHE branches into own helper macros, added comments around symbol aliases.] Signed-off-by: David Brazdil Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200625131420.71444-6-dbrazdil@google.com --- arch/arm64/include/asm/kvm_asm.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'arch/arm64/include/asm/kvm_asm.h') diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 352aaebf4198..6a682d66a640 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -42,6 +42,24 @@ #include +/* + * Translate name of a symbol defined in nVHE hyp to the name seen + * by kernel proper. All nVHE symbols are prefixed by the build system + * to avoid clashes with the VHE variants. + */ +#define kvm_nvhe_sym(sym) __kvm_nvhe_##sym + +#define DECLARE_KVM_VHE_SYM(sym) extern char sym[] +#define DECLARE_KVM_NVHE_SYM(sym) extern char kvm_nvhe_sym(sym)[] + +/* + * Define a pair of symbols sharing the same name but one defined in + * VHE and the other in nVHE hyp implementations. + */ +#define DECLARE_KVM_HYP_SYM(sym) \ + DECLARE_KVM_VHE_SYM(sym); \ + DECLARE_KVM_NVHE_SYM(sym) + /* Translate a kernel address of @sym into its equivalent linear mapping */ #define kvm_ksym_ref(sym) \ ({ \ @@ -50,6 +68,7 @@ val = lm_alias(&sym); \ val; \ }) +#define kvm_ksym_ref_nvhe(sym) kvm_ksym_ref(kvm_nvhe_sym(sym)) struct kvm; struct kvm_vcpu; -- cgit v1.2.3 From b877e9849d41e7d2100d2933e0a3971d0ddec011 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Thu, 25 Jun 2020 14:14:11 +0100 Subject: KVM: arm64: Build hyp-entry.S separately for VHE/nVHE hyp-entry.S contains implementation of KVM hyp vectors. This code is mostly shared between VHE/nVHE, therefore compile it under both VHE and nVHE build rules. nVHE-specific host HVC handler is hidden behind __KVM_NVHE_HYPERVISOR__. Adjust code which selects which KVM hyp vecs to install to choose the correct VHE/nVHE symbol. Signed-off-by: David Brazdil Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200625131420.71444-7-dbrazdil@google.com --- arch/arm64/include/asm/kvm_asm.h | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'arch/arm64/include/asm/kvm_asm.h') diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 6a682d66a640..6026cbd204ae 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -60,12 +60,17 @@ DECLARE_KVM_VHE_SYM(sym); \ DECLARE_KVM_NVHE_SYM(sym) -/* Translate a kernel address of @sym into its equivalent linear mapping */ -#define kvm_ksym_ref(sym) \ +#define CHOOSE_VHE_SYM(sym) sym +#define CHOOSE_NVHE_SYM(sym) kvm_nvhe_sym(sym) +#define CHOOSE_HYP_SYM(sym) (has_vhe() ? CHOOSE_VHE_SYM(sym) \ + : CHOOSE_NVHE_SYM(sym)) + +/* Translate a kernel address @ptr into its equivalent linear mapping */ +#define kvm_ksym_ref(ptr) \ ({ \ - void *val = &sym; \ + void *val = (ptr); \ if (!is_kernel_in_hyp_mode()) \ - val = lm_alias(&sym); \ + val = lm_alias((ptr)); \ val; \ }) #define kvm_ksym_ref_nvhe(sym) kvm_ksym_ref(kvm_nvhe_sym(sym)) @@ -76,7 +81,14 @@ struct kvm_vcpu; extern char __kvm_hyp_init[]; extern char __kvm_hyp_init_end[]; -extern char __kvm_hyp_vector[]; +DECLARE_KVM_HYP_SYM(__kvm_hyp_vector); +#define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector) + +#ifdef CONFIG_KVM_INDIRECT_VECTORS +extern atomic_t arm64_el2_vector_last_slot; +DECLARE_KVM_HYP_SYM(__bp_harden_hyp_vecs); +#define __bp_harden_hyp_vecs CHOOSE_HYP_SYM(__bp_harden_hyp_vecs) +#endif extern void __kvm_flush_vm_context(void); extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); -- cgit v1.2.3 From 208243c752a7eeef4236f7b7d67e806ee356e3f8 Mon Sep 17 00:00:00 2001 From: Andrew Scull Date: Thu, 25 Jun 2020 14:14:12 +0100 Subject: KVM: arm64: Move hyp-init.S to nVHE hyp-init.S contains the identity mapped initialisation code for the non-VHE code that runs at EL2. It is only used for non-VHE. Adjust code that calls into this to use the prefixed symbol name. Signed-off-by: Andrew Scull Signed-off-by: David Brazdil Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200625131420.71444-8-dbrazdil@google.com --- arch/arm64/include/asm/kvm_asm.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'arch/arm64/include/asm/kvm_asm.h') diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 6026cbd204ae..3476abb046e3 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -78,10 +78,9 @@ struct kvm; struct kvm_vcpu; -extern char __kvm_hyp_init[]; -extern char __kvm_hyp_init_end[]; - +DECLARE_KVM_NVHE_SYM(__kvm_hyp_init); DECLARE_KVM_HYP_SYM(__kvm_hyp_vector); +#define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init) #define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector) #ifdef CONFIG_KVM_INDIRECT_VECTORS -- cgit v1.2.3 From 09cf57eba304246141367b95c89801fd2047ac96 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Thu, 25 Jun 2020 14:14:14 +0100 Subject: KVM: arm64: Split hyp/switch.c to VHE/nVHE switch.c implements context-switching for KVM, with large parts shared between VHE/nVHE. These common routines are moved to a header file, VHE-specific code is moved to vhe/switch.c and nVHE-specific code is moved to nvhe/switch.c. Previously __kvm_vcpu_run needed a different symbol name for VHE/nVHE. This is cleaned up and the caller in arm.c simplified. Signed-off-by: David Brazdil Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200625131420.71444-10-dbrazdil@google.com --- arch/arm64/include/asm/kvm_asm.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'arch/arm64/include/asm/kvm_asm.h') diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 3476abb046e3..5a91aaae78d2 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -96,9 +96,7 @@ extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu); extern void __kvm_timer_set_cntvoff(u64 cntvoff); -extern int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu); - -extern int __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu); +extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); extern void __kvm_enable_ssbs(void); -- cgit v1.2.3 From a0e50aa3f4a8a5c9fb791b8f32d624e2a1a735bd Mon Sep 17 00:00:00 2001 From: Christoffer Dall Date: Fri, 4 Jan 2019 21:09:05 +0100 Subject: KVM: arm64: Factor out stage 2 page table data from struct kvm As we are about to reuse our stage 2 page table manipulation code for shadow stage 2 page tables in the context of nested virtualization, we are going to manage multiple stage 2 page tables for a single VM. This requires some pretty invasive changes to our data structures, which moves the vmid and pgd pointers into a separate structure and change pretty much all of our mmu code to operate on this structure instead. The new structure is called struct kvm_s2_mmu. There is no intended functional change by this patch alone. Reviewed-by: James Morse Reviewed-by: Alexandru Elisei [Designed data structure layout in collaboration] Signed-off-by: Christoffer Dall Co-developed-by: Marc Zyngier [maz: Moved the last_vcpu_ran down to the S2 MMU structure as well] Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/arm64/include/asm/kvm_asm.h') diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 5a91aaae78d2..18d39b30d9c9 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -77,6 +77,7 @@ struct kvm; struct kvm_vcpu; +struct kvm_s2_mmu; DECLARE_KVM_NVHE_SYM(__kvm_hyp_init); DECLARE_KVM_HYP_SYM(__kvm_hyp_vector); @@ -90,9 +91,9 @@ DECLARE_KVM_HYP_SYM(__bp_harden_hyp_vecs); #endif extern void __kvm_flush_vm_context(void); -extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); -extern void __kvm_tlb_flush_vmid(struct kvm *kvm); -extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu); +extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa); +extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu); +extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu); extern void __kvm_timer_set_cntvoff(u64 cntvoff); -- cgit v1.2.3 From efaa5b93afde088411b67a0dde34e67aedc6e72f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 2 Jan 2019 12:34:25 +0000 Subject: KVM: arm64: Use TTL hint in when invalidating stage-2 translations Since we often have a precise idea of the level we're dealing with when invalidating TLBs, we can provide it to as a hint to our invalidation helper. Reviewed-by: James Morse Reviewed-by: Alexandru Elisei Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/arm64/include/asm/kvm_asm.h') diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 18d39b30d9c9..61f654a4ae00 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -91,7 +91,8 @@ DECLARE_KVM_HYP_SYM(__bp_harden_hyp_vecs); #endif extern void __kvm_flush_vm_context(void); -extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa); +extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa, + int level); extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu); extern void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu); -- cgit v1.2.3 From 6de7dd31ded078b31a3812a46d8874f871b4e82f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 7 Jul 2020 15:34:06 +0100 Subject: KVM: arm64: Don't use has_vhe() for CHOOSE_HYP_SYM() The recently introduced CHOOSE_HYP_SYM() macro picks one symbol or another, depending on whether the kernel run as a VHE hypervisor or not. For that, it uses the has_vhe() helper, which is itself implemented as a final capability. Unfortunately, __copy_hyp_vect_bpi now indirectly uses CHOOSE_HYP_SYM to get the __bp_harden_hyp_vecs symbol, using has_vhe() in the process. At this stage, the capability isn't final and things explode: [ 0.000000] ACPI: SRAT not present [ 0.000000] percpu: Embedded 34 pages/cpu s101264 r8192 d29808 u139264 [ 0.000000] Detected PIPT I-cache on CPU0 [ 0.000000] ------------[ cut here ]------------ [ 0.000000] kernel BUG at arch/arm64/include/asm/cpufeature.h:459! [ 0.000000] Internal error: Oops - BUG: 0 [#1] PREEMPT SMP [ 0.000000] Modules linked in: [ 0.000000] CPU: 0 PID: 0 Comm: swapper Not tainted 5.8.0-rc4-00080-gd630681366e5 #1388 [ 0.000000] pstate: 80000085 (Nzcv daIf -PAN -UAO BTYPE=--) [ 0.000000] pc : check_branch_predictor+0x3a4/0x408 [ 0.000000] lr : check_branch_predictor+0x2a4/0x408 [ 0.000000] sp : ffff800011693e90 [ 0.000000] x29: ffff800011693e90 x28: ffff8000116a1530 [ 0.000000] x27: ffff8000112c1008 x26: ffff800010ca6ff8 [ 0.000000] x25: ffff8000112c1000 x24: ffff8000116a1320 [ 0.000000] x23: 0000000000000000 x22: ffff8000112c1000 [ 0.000000] x21: ffff800010177120 x20: ffff8000116ae108 [ 0.000000] x19: 0000000000000000 x18: ffff800011965c90 [ 0.000000] x17: 0000000000022000 x16: 0000000000000003 [ 0.000000] x15: 00000000ffffffff x14: ffff8000118c3a38 [ 0.000000] x13: 0000000000000021 x12: 0000000000000022 [ 0.000000] x11: d37a6f4de9bd37a7 x10: 000000000000001d [ 0.000000] x9 : 0000000000000000 x8 : ffff800011f8dad8 [ 0.000000] x7 : ffff800011965ad0 x6 : 0000000000000003 [ 0.000000] x5 : 0000000000000000 x4 : 0000000000000000 [ 0.000000] x3 : 0000000000000100 x2 : 0000000000000004 [ 0.000000] x1 : ffff8000116ae148 x0 : 0000000000000000 [ 0.000000] Call trace: [ 0.000000] check_branch_predictor+0x3a4/0x408 [ 0.000000] update_cpu_capabilities+0x84/0x138 [ 0.000000] init_cpu_features+0x2c0/0x2d8 [ 0.000000] cpuinfo_store_boot_cpu+0x54/0x64 [ 0.000000] smp_prepare_boot_cpu+0x2c/0x60 [ 0.000000] start_kernel+0x16c/0x574 [ 0.000000] Code: 17ffffc7 91010281 14000198 17ffffca (d4210000) This is addressed using a two-fold process: - Replace has_vhe() with is_kernel_in_hyp_mode(), which tests whether we are running at EL2. - Make CHOOSE_HYP_SYM() return an *undefined* symbol when compiled in the nVHE hypervisor, as we really should never use this helper in the nVHE-specific code. With this in place, we're back to a bootable kernel again. Fixes: b877e9849d41 ("KVM: arm64: Build hyp-entry.S separately for VHE/nVHE") Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'arch/arm64/include/asm/kvm_asm.h') diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 5a91aaae78d2..1a66815ea01b 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -62,8 +62,26 @@ #define CHOOSE_VHE_SYM(sym) sym #define CHOOSE_NVHE_SYM(sym) kvm_nvhe_sym(sym) -#define CHOOSE_HYP_SYM(sym) (has_vhe() ? CHOOSE_VHE_SYM(sym) \ + +#ifndef __KVM_NVHE_HYPERVISOR__ +/* + * BIG FAT WARNINGS: + * + * - Don't be tempted to change the following is_kernel_in_hyp_mode() + * to has_vhe(). has_vhe() is implemented as a *final* capability, + * while this is used early at boot time, when the capabilities are + * not final yet.... + * + * - Don't let the nVHE hypervisor have access to this, as it will + * pick the *wrong* symbol (yes, it runs at EL2...). + */ +#define CHOOSE_HYP_SYM(sym) (is_kernel_in_hyp_mode() ? CHOOSE_VHE_SYM(sym) \ : CHOOSE_NVHE_SYM(sym)) +#else +/* The nVHE hypervisor shouldn't even try to access anything */ +extern void *__nvhe_undefined_symbol; +#define CHOOSE_HYP_SYM(sym) __nvhe_undefined_symbol +#endif /* Translate a kernel address @ptr into its equivalent linear mapping */ #define kvm_ksym_ref(ptr) \ -- cgit v1.2.3