KVM: x86/mmu: Avoid indirect call for get_cr3

Most of the time, calls to get_guest_pgd result in calling
kvm_read_cr3 (the exception is only nested TDP).  Hardcode
the default instead of using the get_cr3 function, avoiding
a retpoline if they are enabled.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Mathias Krause <minipli@grsecurity.net>
Link: https://lore.kernel.org/r/20230322013731.102955-2-minipli@grsecurity.net
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index b212496..4c874d4 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -256,6 +256,20 @@ static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
        return regs;
 }
 
+static unsigned long get_guest_cr3(struct kvm_vcpu *vcpu)
+{
+       return kvm_read_cr3(vcpu);
+}
+
+static inline unsigned long kvm_mmu_get_guest_pgd(struct kvm_vcpu *vcpu,
+                                                 struct kvm_mmu *mmu)
+{
+       if (IS_ENABLED(CONFIG_RETPOLINE) && mmu->get_guest_pgd == get_guest_cr3)
+               return kvm_read_cr3(vcpu);
+
+       return mmu->get_guest_pgd(vcpu);
+}
+
 static inline bool kvm_available_flush_tlb_with_range(void)
 {
        return kvm_x86_ops.tlb_remote_flush_with_range;
@@ -3801,7 +3815,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
        int quadrant, i, r;
        hpa_t root;
 
-       root_pgd = mmu->get_guest_pgd(vcpu);
+       root_pgd = kvm_mmu_get_guest_pgd(vcpu, mmu);
        root_gfn = root_pgd >> PAGE_SHIFT;
 
        if (mmu_check_root(vcpu, root_gfn))
@@ -4251,7 +4265,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
        arch.token = alloc_apf_token(vcpu);
        arch.gfn = gfn;
        arch.direct_map = vcpu->arch.mmu->root_role.direct;
-       arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu);
+       arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);
 
        return kvm_setup_async_pf(vcpu, cr2_or_gpa,
                                  kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
@@ -4270,7 +4284,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
                return;
 
        if (!vcpu->arch.mmu->root_role.direct &&
-             work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))
+             work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu))
                return;
 
        kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true, NULL);
@@ -4673,11 +4687,6 @@ void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd)
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd);
 
-static unsigned long get_cr3(struct kvm_vcpu *vcpu)
-{
-       return kvm_read_cr3(vcpu);
-}
-
 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
                           unsigned int access)
 {
@@ -5223,7 +5232,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
        context->root_role.word = root_role.word;
        context->page_fault = kvm_tdp_page_fault;
        context->sync_spte = NULL;
-       context->get_guest_pgd = get_cr3;
+       context->get_guest_pgd = get_guest_cr3;
        context->get_pdptr = kvm_pdptr_read;
        context->inject_page_fault = kvm_inject_page_fault;
 
@@ -5372,7 +5381,7 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
 
        kvm_init_shadow_mmu(vcpu, cpu_role);
 
-       context->get_guest_pgd     = get_cr3;
+       context->get_guest_pgd     = get_guest_cr3;
        context->get_pdptr         = kvm_pdptr_read;
        context->inject_page_fault = kvm_inject_page_fault;
 }
@@ -5386,7 +5395,7 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
                return;
 
        g_context->cpu_role.as_u64   = new_mode.as_u64;
-       g_context->get_guest_pgd     = get_cr3;
+       g_context->get_guest_pgd     = get_guest_cr3;
        g_context->get_pdptr         = kvm_pdptr_read;
        g_context->inject_page_fault = kvm_inject_page_fault;
 

diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 03a9577..0662e02 100644 (file)
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -324,7 +324,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
        trace_kvm_mmu_pagetable_walk(addr, access);
 retry_walk:
        walker->level = mmu->cpu_role.base.level;
-       pte           = mmu->get_guest_pgd(vcpu);
+       pte           = kvm_mmu_get_guest_pgd(vcpu, mmu);
        have_ad       = PT_HAVE_ACCESSED_DIRTY(mmu);
 
 #if PTTYPE == 64