KVM: arm64: Refcount hyp stage-1 pgtable pages

To prepare the ground for allowing hyp stage-1 mappings to be removed at
run-time, update the KVM page-table code to maintain a correct refcount
using the ->{get,put}_page() function callbacks.

Signed-off-by: Quentin Perret <qperret@google.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20211215161232.1480836-3-qperret@google.com

diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index f8ceebe..e50e915 100644 (file)
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -383,21 +383,6 @@ enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte)
        return prot;
 }
 
-static bool hyp_pte_needs_update(kvm_pte_t old, kvm_pte_t new)
-{
-       /*
-        * Tolerate KVM recreating the exact same mapping, or changing software
-        * bits if the existing mapping was valid.
-        */
-       if (old == new)
-               return false;
-
-       if (!kvm_pte_valid(old))
-               return true;
-
-       return !WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW);
-}
-
 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
                                    kvm_pte_t *ptep, struct hyp_map_data *data)
 {
@@ -407,11 +392,16 @@ static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
        if (!kvm_block_mapping_supported(addr, end, phys, level))
                return false;
 
+       data->phys += granule;
        new = kvm_init_valid_leaf_pte(phys, data->attr, level);
-       if (hyp_pte_needs_update(old, new))
-               smp_store_release(ptep, new);
+       if (old == new)
+               return true;
+       if (!kvm_pte_valid(old))
+               data->mm_ops->get_page(ptep);
+       else if (WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))
+               return false;
 
-       data->phys += granule;
+       smp_store_release(ptep, new);
        return true;
 }
 
@@ -433,6 +423,7 @@ static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
                return -ENOMEM;
 
        kvm_set_table_pte(ptep, childp, mm_ops);
+       mm_ops->get_page(ptep);
        return 0;
 }
 
@@ -482,8 +473,16 @@ static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
                           enum kvm_pgtable_walk_flags flag, void * const arg)
 {
        struct kvm_pgtable_mm_ops *mm_ops = arg;
+       kvm_pte_t pte = *ptep;
+
+       if (!kvm_pte_valid(pte))
+               return 0;
+
+       mm_ops->put_page(ptep);
+
+       if (kvm_pte_table(pte, level))
+               mm_ops->put_page(kvm_pte_follow(pte, mm_ops));
 
-       mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops));
        return 0;
 }
 
@@ -491,7 +490,7 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)
 {
        struct kvm_pgtable_walker walker = {
                .cb     = hyp_free_walker,
-               .flags  = KVM_PGTABLE_WALK_TABLE_POST,
+               .flags  = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
                .arg    = pgt->mm_ops,
        };