KVM: SVM: Flush Hyper-V TLB when required

[ Upstream commit e5c972c1fadacc858b6a564d056f177275238040 ]

The Hyper-V "EnlightenedNptTlb" enlightenment is always enabled when KVM
is running on top of Hyper-V and Hyper-V exposes support for it (which
is always). On AMD CPUs this enlightenment results in ASID invalidations
not flushing TLB entries derived from the NPT. To force the underlying
(L0) hypervisor to rebuild its shadow page tables, an explicit hypercall
is needed.

The original KVM implementation of Hyper-V's "EnlightenedNptTlb" on SVM
only added remote TLB flush hooks. This worked out fine for a while, as
sufficient remote TLB flushes where being issued in KVM to mask the
problem. Since v5.17, changes in the TDP code reduced the number of
flushes and the out-of-sync TLB prevents guests from booting
successfully.

Split svm_flush_tlb_current() into separate callbacks for the 3 cases
(guest/all/current), and issue the required Hyper-V hypercall when a
Hyper-V TLB flush is needed. The most important case where the TLB flush
was missing is when loading a new PGD, which is followed by what is now
svm_flush_tlb_current().

Cc: stable@vger.kernel.org # v5.17+
Fixes: 1e0c7d40758b ("KVM: SVM: hyper-v: Remote TLB flush for SVM")
Link: https://lore.kernel.org/lkml/43980946-7bbf-dcef-7e40-af904c456250@linux.microsoft.com/
Suggested-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Jeremi Piotrowski <jpiotrowski@linux.microsoft.com>
Reviewed-by: Vitaly Kuznetsov <vkuznets@redhat.com>
Message-Id: <20230324145233.4585-1-jpiotrowski@linux.microsoft.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/arch/x86/kvm/kvm_onhyperv.h b/arch/x86/kvm/kvm_onhyperv.h
index 287e98ef9df3d820244133f4aa73547abe613f1d..6272dabec02da8ae81aaf39fe5165d5b3c225f7b 100644 (file)
--- a/arch/x86/kvm/kvm_onhyperv.h
+++ b/arch/x86/kvm/kvm_onhyperv.h
@@ -12,6 +12,11 @@ int hv_remote_flush_tlb_with_range(struct kvm *kvm,
 int hv_remote_flush_tlb(struct kvm *kvm);
 void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp);
 #else /* !CONFIG_HYPERV */
 int hv_remote_flush_tlb(struct kvm *kvm);
 void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp);
 #else /* !CONFIG_HYPERV */

+static inline int hv_remote_flush_tlb(struct kvm *kvm)
+{
+       return -EOPNOTSUPP;
+}
+

 static inline void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp)
 {
 }
 static inline void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp)
 {
 }

diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 3629dd979667c5be6a59bc256aaec83b590391b3..9599931c7d57231585e93661bdbc2482d847c9b1 100644 (file)
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -3719,7 +3719,7 @@ static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
        svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
 }
 
        svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
 }
 

-static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
+static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)

 {
        struct vcpu_svm *svm = to_svm(vcpu);
 
 {
        struct vcpu_svm *svm = to_svm(vcpu);
 


@@ -3736,6 +3736,37 @@ static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
                svm->current_vmcb->asid_generation--;
 }
 
                svm->current_vmcb->asid_generation--;
 }
 

+static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
+{
+       hpa_t root_tdp = vcpu->arch.mmu->root.hpa;
+
+       /*
+        * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly
+        * flush the NPT mappings via hypercall as flushing the ASID only
+        * affects virtual to physical mappings, it does not invalidate guest
+        * physical to host physical mappings.
+        */
+       if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp))
+               hyperv_flush_guest_mapping(root_tdp);
+
+       svm_flush_tlb_asid(vcpu);
+}
+
+static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
+{
+       /*
+        * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB
+        * flushes should be routed to hv_remote_flush_tlb() without requesting
+        * a "regular" remote flush.  Reaching this point means either there's
+        * a KVM bug or a prior hv_remote_flush_tlb() call failed, both of
+        * which might be fatal to the guest.  Yell, but try to recover.
+        */
+       if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu)))
+               hv_remote_flush_tlb(vcpu->kvm);
+
+       svm_flush_tlb_asid(vcpu);
+}
+

 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
 {
        struct vcpu_svm *svm = to_svm(vcpu);
 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
 {
        struct vcpu_svm *svm = to_svm(vcpu);


@@ -4733,10 +4764,10 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
        .set_rflags = svm_set_rflags,
        .get_if_flag = svm_get_if_flag,
 
        .set_rflags = svm_set_rflags,
        .get_if_flag = svm_get_if_flag,
 

-       .flush_tlb_all = svm_flush_tlb_current,
+       .flush_tlb_all = svm_flush_tlb_all,

        .flush_tlb_current = svm_flush_tlb_current,
        .flush_tlb_gva = svm_flush_tlb_gva,
        .flush_tlb_current = svm_flush_tlb_current,
        .flush_tlb_gva = svm_flush_tlb_gva,

-       .flush_tlb_guest = svm_flush_tlb_current,
+       .flush_tlb_guest = svm_flush_tlb_asid,

 
        .vcpu_pre_run = svm_vcpu_pre_run,
        .vcpu_run = svm_vcpu_run,
 
        .vcpu_pre_run = svm_vcpu_pre_run,
        .vcpu_run = svm_vcpu_run,

diff --git a/arch/x86/kvm/svm/svm_onhyperv.h b/arch/x86/kvm/svm/svm_onhyperv.h
index 780b6c09cfe4b4687541e1ae5465129f1d9d7e6b..9a6a34149919c4a0b68f7ecd3a04dd10372a66c3 100644 (file)
--- a/arch/x86/kvm/svm/svm_onhyperv.h
+++ b/arch/x86/kvm/svm/svm_onhyperv.h
@@ -6,6 +6,8 @@
 #ifndef __ARCH_X86_KVM_SVM_ONHYPERV_H__
 #define __ARCH_X86_KVM_SVM_ONHYPERV_H__
 
 #ifndef __ARCH_X86_KVM_SVM_ONHYPERV_H__
 #define __ARCH_X86_KVM_SVM_ONHYPERV_H__
 

+#include <asm/mshyperv.h>
+

 #if IS_ENABLED(CONFIG_HYPERV)
 
 #include "kvm_onhyperv.h"
 #if IS_ENABLED(CONFIG_HYPERV)
 
 #include "kvm_onhyperv.h"


@@ -15,6 +17,14 @@ static struct kvm_x86_ops svm_x86_ops;
 
 int svm_hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu);
 
 
 int svm_hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu);
 

+static inline bool svm_hv_is_enlightened_tlb_enabled(struct kvm_vcpu *vcpu)
+{
+       struct hv_vmcb_enlightenments *hve = &to_svm(vcpu)->vmcb->control.hv_enlightenments;
+
+       return ms_hyperv.nested_features & HV_X64_NESTED_ENLIGHTENED_TLB &&
+              !!hve->hv_enlightenments_control.enlightened_npt_tlb;
+}
+

 static inline void svm_hv_init_vmcb(struct vmcb *vmcb)
 {
        struct hv_vmcb_enlightenments *hve = &vmcb->control.hv_enlightenments;
 static inline void svm_hv_init_vmcb(struct vmcb *vmcb)
 {
        struct hv_vmcb_enlightenments *hve = &vmcb->control.hv_enlightenments;


@@ -80,6 +90,11 @@ static inline void svm_hv_update_vp_id(struct vmcb *vmcb, struct kvm_vcpu *vcpu)
 }
 #else
 
 }
 #else
 

+static inline bool svm_hv_is_enlightened_tlb_enabled(struct kvm_vcpu *vcpu)
+{
+       return false;
+}
+

 static inline void svm_hv_init_vmcb(struct vmcb *vmcb)
 {
 }
 static inline void svm_hv_init_vmcb(struct vmcb *vmcb)
 {
 }