KVM: x86/mmu: Plumb "struct kvm" all the way to pte_list_remove()

Plumb "struct kvm" all the way to pte_list_remove() to allow the usage of
KVM_BUG() and/or KVM_BUG_ON().  This will allow killing only the offending
VM instead of doing BUG() if the kernel is built with
CONFIG_BUG_ON_DATA_CORRUPTION=n, i.e. does NOT want to BUG() if KVM's data
structures (rmaps) appear to be corrupted.

Signed-off-by: Mingwei Zhang <mizhang@google.com>
[sean: tweak changelog]
Link: https://lore.kernel.org/r/20230729004722.1056172-12-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index baa3123..f518dd5 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -961,7 +961,8 @@ static int pte_list_add(struct kvm_mmu_memory_cache *cache, u64 *spte,
        return count;
 }
 
-static void pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
+static void pte_list_desc_remove_entry(struct kvm *kvm,
+                                      struct kvm_rmap_head *rmap_head,
                                       struct pte_list_desc *desc, int i)
 {
        struct pte_list_desc *head_desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
@@ -997,7 +998,8 @@ static void pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
        mmu_free_pte_list_desc(head_desc);
 }
 
-static void pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
+static void pte_list_remove(struct kvm *kvm, u64 *spte,
+                           struct kvm_rmap_head *rmap_head)
 {
        struct pte_list_desc *desc;
        int i;
@@ -1016,7 +1018,8 @@ static void pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
                while (desc) {
                        for (i = 0; i < desc->spte_count; ++i) {
                                if (desc->sptes[i] == spte) {
-                                       pte_list_desc_remove_entry(rmap_head, desc, i);
+                                       pte_list_desc_remove_entry(kvm, rmap_head,
+                                                                  desc, i);
                                        return;
                                }
                        }
@@ -1031,7 +1034,7 @@ static void kvm_zap_one_rmap_spte(struct kvm *kvm,
                                  struct kvm_rmap_head *rmap_head, u64 *sptep)
 {
        mmu_spte_clear_track_bits(kvm, sptep);
-       pte_list_remove(sptep, rmap_head);
+       pte_list_remove(kvm, sptep, rmap_head);
 }
 
 /* Return true if at least one SPTE was zapped, false otherwise */
@@ -1106,7 +1109,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
        slot = __gfn_to_memslot(slots, gfn);
        rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
 
-       pte_list_remove(spte, rmap_head);
+       pte_list_remove(kvm, spte, rmap_head);
 }
 
 /*
@@ -1753,16 +1756,16 @@ static void mmu_page_add_parent_pte(struct kvm_mmu_memory_cache *cache,
        pte_list_add(cache, parent_pte, &sp->parent_ptes);
 }
 
-static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
+static void mmu_page_remove_parent_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
                                       u64 *parent_pte)
 {
-       pte_list_remove(parent_pte, &sp->parent_ptes);
+       pte_list_remove(kvm, parent_pte, &sp->parent_ptes);
 }
 
-static void drop_parent_pte(struct kvm_mmu_page *sp,
+static void drop_parent_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
                            u64 *parent_pte)
 {
-       mmu_page_remove_parent_pte(sp, parent_pte);
+       mmu_page_remove_parent_pte(kvm, sp, parent_pte);
        mmu_spte_clear_no_track(parent_pte);
 }
 
@@ -2477,7 +2480,7 @@ static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                if (child->role.access == direct_access)
                        return;
 
-               drop_parent_pte(child, sptep);
+               drop_parent_pte(vcpu->kvm, child, sptep);
                kvm_flush_remote_tlbs_sptep(vcpu->kvm, sptep);
        }
 }
@@ -2495,7 +2498,7 @@ static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
                        drop_spte(kvm, spte);
                } else {
                        child = spte_to_child_sp(pte);
-                       drop_parent_pte(child, spte);
+                       drop_parent_pte(kvm, child, spte);
 
                        /*
                         * Recursively zap nested TDP SPs, parentless SPs are
@@ -2526,13 +2529,13 @@ static int kvm_mmu_page_unlink_children(struct kvm *kvm,
        return zapped;
 }
 
-static void kvm_mmu_unlink_parents(struct kvm_mmu_page *sp)
+static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
 {
        u64 *sptep;
        struct rmap_iterator iter;
 
        while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
-               drop_parent_pte(sp, sptep);
+               drop_parent_pte(kvm, sp, sptep);
 }
 
 static int mmu_zap_unsync_children(struct kvm *kvm,
@@ -2571,7 +2574,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
        ++kvm->stat.mmu_shadow_zapped;
        *nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list);
        *nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list);
-       kvm_mmu_unlink_parents(sp);
+       kvm_mmu_unlink_parents(kvm, sp);
 
        /* Zapping children means active_mmu_pages has become unstable. */
        list_unstable = *nr_zapped;
@@ -2929,7 +2932,7 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
                        u64 pte = *sptep;
 
                        child = spte_to_child_sp(pte);
-                       drop_parent_pte(child, sptep);
+                       drop_parent_pte(vcpu->kvm, child, sptep);
                        flush = true;
                } else if (pfn != spte_to_pfn(*sptep)) {
                        drop_spte(vcpu->kvm, sptep);