KVM: x86/mmu: Allocate pae_root and lm_root pages in dedicated helper

Move the on-demand allocation of the pae_root and lm_root pages, used by
nested NPT for 32-bit L1s, into a separate helper.  This will allow a
future patch to hold mmu_lock while allocating the non-special roots so
that make_mmu_pages_available() can be checked once at the start of root
allocation, and thus avoid having to deal with failure in the middle of
root allocation.

Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-Id: <20210305011101.3597423-6-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 973dcc1..804de7d 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -3314,38 +3314,10 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
         * the shadow page table may be a PAE or a long mode page table.
         */
        pm_mask = PT_PRESENT_MASK;
-       if (mmu->shadow_root_level == PT64_ROOT_4LEVEL)
+       if (mmu->shadow_root_level == PT64_ROOT_4LEVEL) {
                pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
 
-       /*
-        * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP
-        * tables are allocated and initialized at root creation as there is no
-        * equivalent level in the guest's NPT to shadow.  Allocate the tables
-        * on demand, as running a 32-bit L1 VMM is very rare.  Unlike 32-bit
-        * NPT, the PDP table doesn't need to be in low mem.  Preallocate the
-        * pages so that the PAE roots aren't leaked on failure.
-        */
-       if (mmu->shadow_root_level == PT64_ROOT_4LEVEL &&
-           (!mmu->pae_root || !mmu->lm_root)) {
-               u64 *lm_root, *pae_root;
-
-               if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->lm_root))
-                       return -EIO;
-
-               pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
-               if (!pae_root)
-                       return -ENOMEM;
-
-               lm_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
-               if (!lm_root) {
-                       free_page((unsigned long)pae_root);
-                       return -ENOMEM;
-               }
-
-               mmu->pae_root = pae_root;
-               mmu->lm_root = lm_root;
-
-               lm_root[0] = __pa(mmu->pae_root) | pm_mask;
+               mmu->lm_root[0] = __pa(mmu->pae_root) | pm_mask;
        }
 
        for (i = 0; i < 4; ++i) {
@@ -3379,6 +3351,55 @@ set_root_pgd:
        return 0;
 }
 
+static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
+{
+       struct kvm_mmu *mmu = vcpu->arch.mmu;
+       u64 *lm_root, *pae_root;
+
+       /*
+        * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP
+        * tables are allocated and initialized at root creation as there is no
+        * equivalent level in the guest's NPT to shadow.  Allocate the tables
+        * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
+        */
+       if (mmu->direct_map || mmu->root_level >= PT64_ROOT_4LEVEL ||
+           mmu->shadow_root_level < PT64_ROOT_4LEVEL)
+               return 0;
+
+       /*
+        * This mess only works with 4-level paging and needs to be updated to
+        * work with 5-level paging.
+        */
+       if (WARN_ON_ONCE(mmu->shadow_root_level != PT64_ROOT_4LEVEL))
+               return -EIO;
+
+       if (mmu->pae_root && mmu->lm_root)
+               return 0;
+
+       /*
+        * The special roots should always be allocated in concert.  Yell and
+        * bail if KVM ends up in a state where only one of the roots is valid.
+        */
+       if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->lm_root))
+               return -EIO;
+
+       /* Unlike 32-bit NPT, the PDP table doesn't need to be in low mem. */
+       pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
+       if (!pae_root)
+               return -ENOMEM;
+
+       lm_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
+       if (!lm_root) {
+               free_page((unsigned long)pae_root);
+               return -ENOMEM;
+       }
+
+       mmu->pae_root = pae_root;
+       mmu->lm_root = lm_root;
+
+       return 0;
+}
+
 static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
 {
        if (vcpu->arch.mmu->direct_map)
@@ -4814,6 +4835,9 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
        r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->direct_map);
        if (r)
                goto out;
+       r = mmu_alloc_special_roots(vcpu);
+       if (r)
+               goto out;
        r = mmu_alloc_roots(vcpu);
        kvm_mmu_sync_roots(vcpu);
        if (r)