KVM: arm64: Fixup hyp stage-1 refcount

In nVHE-protected mode, the hyp stage-1 page-table refcount is broken
due to the lack of refcount support in the early allocator. Fix-up the
refcount in the finalize walker, once the 'hyp_vmemmap' is up and running.

Acked-by: Will Deacon <will@kernel.org>
Signed-off-by: Quentin Perret <qperret@google.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20211215161232.1480836-4-qperret@google.com

diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index 578f717..875b517 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -165,6 +165,7 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level,
                                         enum kvm_pgtable_walk_flags flag,
                                         void * const arg)
 {
+       struct kvm_pgtable_mm_ops *mm_ops = arg;
        enum kvm_pgtable_prot prot;
        enum pkvm_page_state state;
        kvm_pte_t pte = *ptep;
@@ -173,6 +174,15 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level,
        if (!kvm_pte_valid(pte))
                return 0;
 
+       /*
+        * Fix-up the refcount for the page-table pages as the early allocator
+        * was unable to access the hyp_vmemmap and so the buddy allocator has
+        * initialised the refcount to '1'.
+        */
+       mm_ops->get_page(ptep);
+       if (flag != KVM_PGTABLE_WALK_LEAF)
+               return 0;
+
        if (level != (KVM_PGTABLE_MAX_LEVELS - 1))
                return -EINVAL;
 
@@ -205,7 +215,8 @@ static int finalize_host_mappings(void)
 {
        struct kvm_pgtable_walker walker = {
                .cb     = finalize_host_mappings_walker,
-               .flags  = KVM_PGTABLE_WALK_LEAF,
+               .flags  = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
+               .arg    = pkvm_pgtable.mm_ops,
        };
        int i, ret;
 
@@ -240,10 +251,6 @@ void __noreturn __pkvm_init_finalise(void)
        if (ret)
                goto out;
 
-       ret = finalize_host_mappings();
-       if (ret)
-               goto out;
-
        pkvm_pgtable_mm_ops = (struct kvm_pgtable_mm_ops) {
                .zalloc_page = hyp_zalloc_hyp_page,
                .phys_to_virt = hyp_phys_to_virt,
@@ -253,6 +260,10 @@ void __noreturn __pkvm_init_finalise(void)
        };
        pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;
 
+       ret = finalize_host_mappings();
+       if (ret)
+               goto out;
+
 out:
        /*
         * We tail-called to here from handle___pkvm_init() and will not return,