KVM: x86/mmu: Zap only TDP MMU leafs in zap range and mmu_notifier unmap

Re-introduce zapping only leaf SPTEs in kvm_zap_gfn_range() and
kvm_tdp_mmu_unmap_gfn_range(), this time without losing a pending TLB
flush when processing multiple roots (including nested TDP shadow roots).
Dropping the TLB flush resulted in random crashes when running Hyper-V
Server 2019 in a guest with KSM enabled in the host (or any source of
mmu_notifier invalidations, KSM is just the easiest to force).

This effectively revert commits 873dd122172f8cce329113cfb0dfe3d2344d80c0
and fcb93eb6d09dd302cbef22bd95a5858af75e4156, and thus restores commit
cf3e26427c08ad9015956293ab389004ac6a338e, plus this delta on top:

bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end,
        struct kvm_mmu_page *root;

        for_each_tdp_mmu_root_yield_safe(kvm, root, as_id)
-               flush = tdp_mmu_zap_leafs(kvm, root, start, end, can_yield, false);
+               flush = tdp_mmu_zap_leafs(kvm, root, start, end, can_yield, flush);

        return flush;
 }

Cc: Ben Gardon <bgardon@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Tested-by: Vitaly Kuznetsov <vkuznets@redhat.com>
Message-Id: <20220325230348.2587437-1-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 857ba93b5c9293e587063c75941954178b40a85c..333aaf6d68061b3ba3691637ac7605bd55be8d6a 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -5849,8 +5849,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
 
        if (is_tdp_mmu_enabled(kvm)) {
                for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
-                       flush = kvm_tdp_mmu_zap_gfn_range(kvm, i, gfn_start,
-                                                         gfn_end, flush);
+                       flush = kvm_tdp_mmu_zap_leafs(kvm, i, gfn_start,
+                                                     gfn_end, true, flush);
        }
 
        if (flush)

diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 4be517a9f22fb45b49c9ec2c4be6410440767b25..d71d177ae6b8716e98a2a74f91c5e5a0d85231b1 100644 (file)
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -909,10 +909,8 @@ bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
 }
 
 /*
- * Tears down the mappings for the range of gfns, [start, end), and frees the
- * non-root pages mapping GFNs strictly within that range. Returns true if
- * SPTEs have been cleared and a TLB flush is needed before releasing the
- * MMU lock.
+ * Zap leafs SPTEs for the range of gfns, [start, end). Returns true if SPTEs
+ * have been cleared and a TLB flush is needed before releasing the MMU lock.
  *
  * If can_yield is true, will release the MMU lock and reschedule if the
  * scheduler needs the CPU or there is contention on the MMU lock. If this
@@ -920,42 +918,25 @@ bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
  * the caller must ensure it does not supply too large a GFN range, or the
  * operation can cause a soft lockup.
  */
-static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
-                         gfn_t start, gfn_t end, bool can_yield, bool flush)
+static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root,
+                             gfn_t start, gfn_t end, bool can_yield, bool flush)
 {
-       bool zap_all = (start == 0 && end >= tdp_mmu_max_gfn_host());
        struct tdp_iter iter;
 
-       /*
-        * No need to try to step down in the iterator when zapping all SPTEs,
-        * zapping the top-level non-leaf SPTEs will recurse on their children.
-        */
-       int min_level = zap_all ? root->role.level : PG_LEVEL_4K;
-
        end = min(end, tdp_mmu_max_gfn_host());
 
        lockdep_assert_held_write(&kvm->mmu_lock);
 
        rcu_read_lock();
 
-       for_each_tdp_pte_min_level(iter, root, min_level, start, end) {
+       for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end) {
                if (can_yield &&
                    tdp_mmu_iter_cond_resched(kvm, &iter, flush, false)) {
                        flush = false;
                        continue;
                }
 
-               if (!is_shadow_present_pte(iter.old_spte))
-                       continue;
-
-               /*
-                * If this is a non-last-level SPTE that covers a larger range
-                * than should be zapped, continue, and zap the mappings at a
-                * lower level, except when zapping all SPTEs.
-                */
-               if (!zap_all &&
-                   (iter.gfn < start ||
-                    iter.gfn + KVM_PAGES_PER_HPAGE(iter.level) > end) &&
+               if (!is_shadow_present_pte(iter.old_spte) ||
                    !is_last_spte(iter.old_spte, iter.level))
                        continue;
 
@@ -963,17 +944,13 @@ static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
                flush = true;
        }
 
-       /*
-        * Need to flush before releasing RCU.  TODO: do it only if intermediate
-        * page tables were zapped; there is no need to flush under RCU protection
-        * if no 'struct kvm_mmu_page' is freed.
-        */
-       if (flush)
-               kvm_flush_remote_tlbs_with_address(kvm, start, end - start);
-
        rcu_read_unlock();
 
-       return false;
+       /*
+        * Because this flow zaps _only_ leaf SPTEs, the caller doesn't need
+        * to provide RCU protection as no 'struct kvm_mmu_page' will be freed.
+        */
+       return flush;
 }
 
 /*
@@ -982,13 +959,13 @@ static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
  * SPTEs have been cleared and a TLB flush is needed before releasing the
  * MMU lock.
  */
-bool __kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, int as_id, gfn_t start,
-                                gfn_t end, bool can_yield, bool flush)
+bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end,
+                          bool can_yield, bool flush)
 {
        struct kvm_mmu_page *root;
 
        for_each_tdp_mmu_root_yield_safe(kvm, root, as_id)
-               flush = zap_gfn_range(kvm, root, start, end, can_yield, flush);
+               flush = tdp_mmu_zap_leafs(kvm, root, start, end, can_yield, flush);
 
        return flush;
 }
@@ -1236,8 +1213,8 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
                                 bool flush)
 {
-       return __kvm_tdp_mmu_zap_gfn_range(kvm, range->slot->as_id, range->start,
-                                          range->end, range->may_block, flush);
+       return kvm_tdp_mmu_zap_leafs(kvm, range->slot->as_id, range->start,
+                                    range->end, range->may_block, flush);
 }
 
 typedef bool (*tdp_handler_t)(struct kvm *kvm, struct tdp_iter *iter,

diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h
index 647926541e384188d0c6e9fc8ba9b43da24714bb..c163f7cc23ca54aa93d466447d9aaf7ab176bf41 100644 (file)
--- a/arch/x86/kvm/mmu/tdp_mmu.h
+++ b/arch/x86/kvm/mmu/tdp_mmu.h
@@ -15,14 +15,8 @@ __must_check static inline bool kvm_tdp_mmu_get_root(struct kvm_mmu_page *root)
 void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
                          bool shared);
 
-bool __kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, int as_id, gfn_t start,
+bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start,
                                 gfn_t end, bool can_yield, bool flush);
-static inline bool kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, int as_id,
-                                            gfn_t start, gfn_t end, bool flush)
-{
-       return __kvm_tdp_mmu_zap_gfn_range(kvm, as_id, start, end, true, flush);
-}
-
 bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp);
 void kvm_tdp_mmu_zap_all(struct kvm *kvm);
 void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm);