From b94742c958f0b97d304d4aecb4603a20ee9a2df3 Mon Sep 17 00:00:00 2001 From: Junaid Shahid Date: Wed, 27 Jun 2018 14:59:20 -0700 Subject: [PATCH] kvm: x86: Add multi-entry LRU cache for previous CR3s Adds support for storing multiple previous CR3/root_hpa pairs maintained as an LRU cache, so that the lockless CR3 switch path can be used when switching back to any of them. Signed-off-by: Junaid Shahid Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 10 ++-- arch/x86/kvm/mmu.c | 111 ++++++++++++++++++++++++++++------------ arch/x86/kvm/vmx.c | 12 +++-- 3 files changed, 92 insertions(+), 41 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index dcdc915..4f19836 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -335,6 +335,8 @@ struct kvm_mmu_root_info { #define KVM_MMU_ROOT_INFO_INVALID \ ((struct kvm_mmu_root_info) { .cr3 = INVALID_PAGE, .hpa = INVALID_PAGE }) +#define KVM_MMU_NUM_PREV_ROOTS 3 + /* * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the @@ -363,7 +365,7 @@ struct kvm_mmu { u8 shadow_root_level; u8 ept_ad; bool direct_map; - struct kvm_mmu_root_info prev_root; + struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; /* * Bitmap; bit set = permission fault @@ -1295,9 +1297,9 @@ static inline int __kvm_irq_line_state(unsigned long *irq_state, return !!(*irq_state); } -#define KVM_MMU_ROOT_CURRENT BIT(0) -#define KVM_MMU_ROOT_PREVIOUS BIT(1) -#define KVM_MMU_ROOTS_ALL (~0UL) +#define KVM_MMU_ROOT_CURRENT BIT(0) +#define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i) +#define KVM_MMU_ROOTS_ALL (~0UL) int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level); void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id); diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index f84c194..687bfb0 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -3445,18 +3445,26 @@ void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, ulong roots_to_free) LIST_HEAD(invalid_list); struct kvm_mmu *mmu = &vcpu->arch.mmu; bool free_active_root = roots_to_free & KVM_MMU_ROOT_CURRENT; - bool free_prev_root = roots_to_free & KVM_MMU_ROOT_PREVIOUS; + + BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG); /* Before acquiring the MMU lock, see if we need to do any real work. */ - if (!(free_active_root && VALID_PAGE(mmu->root_hpa)) && - !(free_prev_root && VALID_PAGE(mmu->prev_root.hpa))) - return; + if (!(free_active_root && VALID_PAGE(mmu->root_hpa))) { + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) && + VALID_PAGE(mmu->prev_roots[i].hpa)) + break; + + if (i == KVM_MMU_NUM_PREV_ROOTS) + return; + } spin_lock(&vcpu->kvm->mmu_lock); - if (free_prev_root) - mmu_free_root_page(vcpu->kvm, &mmu->prev_root.hpa, - &invalid_list); + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) + mmu_free_root_page(vcpu->kvm, &mmu->prev_roots[i].hpa, + &invalid_list); if (free_active_root) { if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && @@ -4064,6 +4072,38 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu, context->nx = false; } +/* + * Find out if a previously cached root matching the new CR3/role is available. + * The current root is also inserted into the cache. + * If a matching root was found, it is assigned to kvm_mmu->root_hpa and true is + * returned. + * Otherwise, the LRU root from the cache is assigned to kvm_mmu->root_hpa and + * false is returned. This root should now be freed by the caller. + */ +static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_cr3, + union kvm_mmu_page_role new_role) +{ + uint i; + struct kvm_mmu_root_info root; + struct kvm_mmu *mmu = &vcpu->arch.mmu; + + root.cr3 = mmu->get_cr3(vcpu); + root.hpa = mmu->root_hpa; + + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { + swap(root, mmu->prev_roots[i]); + + if (new_cr3 == root.cr3 && VALID_PAGE(root.hpa) && + page_header(root.hpa) != NULL && + new_role.word == page_header(root.hpa)->role.word) + break; + } + + mmu->root_hpa = root.hpa; + + return i < KVM_MMU_NUM_PREV_ROOTS; +} + static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, union kvm_mmu_page_role new_role, bool skip_tlb_flush) @@ -4077,18 +4117,10 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, */ if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && mmu->root_level >= PT64_ROOT_4LEVEL) { - gpa_t prev_cr3 = mmu->prev_root.cr3; - if (mmu_check_root(vcpu, new_cr3 >> PAGE_SHIFT)) return false; - swap(mmu->root_hpa, mmu->prev_root.hpa); - mmu->prev_root.cr3 = mmu->get_cr3(vcpu); - - if (new_cr3 == prev_cr3 && - VALID_PAGE(mmu->root_hpa) && - page_header(mmu->root_hpa) != NULL && - new_role.word == page_header(mmu->root_hpa)->role.word) { + if (cached_root_available(vcpu, new_cr3, new_role)) { /* * It is possible that the cached previous root page is * obsolete because of a change in the MMU @@ -4854,8 +4886,12 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) void kvm_init_mmu(struct kvm_vcpu *vcpu, bool reset_roots) { if (reset_roots) { + uint i; + vcpu->arch.mmu.root_hpa = INVALID_PAGE; - vcpu->arch.mmu.prev_root = KVM_MMU_ROOT_INFO_INVALID; + + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + vcpu->arch.mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; } if (mmu_is_nested(vcpu)) @@ -5225,6 +5261,7 @@ EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) { struct kvm_mmu *mmu = &vcpu->arch.mmu; + int i; /* INVLPG on a * non-canonical address is a NOP according to the SDM. */ if (is_noncanonical_address(gva, vcpu)) @@ -5235,16 +5272,17 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) /* * INVLPG is required to invalidate any global mappings for the VA, * irrespective of PCID. Since it would take us roughly similar amount - * of work to determine whether the prev_root mapping of the VA is - * marked global, or to just sync it blindly, so we might as well just - * always sync it. + * of work to determine whether any of the prev_root mappings of the VA + * is marked global, or to just sync it blindly, so we might as well + * just always sync it. * - * Mappings not reachable via the current cr3 or the prev_root.cr3 will - * be synced when switching to that cr3, so nothing needs to be done - * here for them. + * Mappings not reachable via the current cr3 or the prev_roots will be + * synced when switching to that cr3, so nothing needs to be done here + * for them. */ - if (VALID_PAGE(mmu->prev_root.hpa)) - mmu->invlpg(vcpu, gva, mmu->prev_root.hpa); + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + if (VALID_PAGE(mmu->prev_roots[i].hpa)) + mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); kvm_x86_ops->tlb_flush_gva(vcpu, gva); ++vcpu->stat.invlpg; @@ -5255,16 +5293,19 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) { struct kvm_mmu *mmu = &vcpu->arch.mmu; bool tlb_flush = false; + uint i; if (pcid == kvm_get_active_pcid(vcpu)) { mmu->invlpg(vcpu, gva, mmu->root_hpa); tlb_flush = true; } - if (VALID_PAGE(mmu->prev_root.hpa) && - pcid == kvm_get_pcid(vcpu, mmu->prev_root.cr3)) { - mmu->invlpg(vcpu, gva, mmu->prev_root.hpa); - tlb_flush = true; + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { + if (VALID_PAGE(mmu->prev_roots[i].hpa) && + pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].cr3)) { + mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); + tlb_flush = true; + } } if (tlb_flush) @@ -5273,9 +5314,9 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) ++vcpu->stat.invlpg; /* - * Mappings not reachable via the current cr3 or the prev_root.cr3 will - * be synced when switching to that cr3, so nothing needs to be done - * here for them. + * Mappings not reachable via the current cr3 or the prev_roots will be + * synced when switching to that cr3, so nothing needs to be done here + * for them. */ } EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva); @@ -5321,12 +5362,16 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) int kvm_mmu_create(struct kvm_vcpu *vcpu) { + uint i; + vcpu->arch.walk_mmu = &vcpu->arch.mmu; vcpu->arch.mmu.root_hpa = INVALID_PAGE; - vcpu->arch.mmu.prev_root = KVM_MMU_ROOT_INFO_INVALID; vcpu->arch.mmu.translate_gpa = translate_gpa; vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa; + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + vcpu->arch.mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; + return alloc_mmu_pages(vcpu); } diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 5aea5af..97e4d71 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -8788,6 +8788,8 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) bool pcid_enabled; gva_t gva; struct x86_exception e; + unsigned i; + unsigned long roots_to_free = 0; struct { u64 pcid; u64 gla; @@ -8846,12 +8848,14 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); } - if (kvm_get_pcid(vcpu, vcpu->arch.mmu.prev_root.cr3) - == operand.pcid) - kvm_mmu_free_roots(vcpu, KVM_MMU_ROOT_PREVIOUS); + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + if (kvm_get_pcid(vcpu, vcpu->arch.mmu.prev_roots[i].cr3) + == operand.pcid) + roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); + kvm_mmu_free_roots(vcpu, roots_to_free); /* - * If neither the current cr3 nor the prev_root.cr3 use the + * If neither the current cr3 nor any of the prev_roots use the * given PCID, then nothing needs to be done here because a * resync will happen anyway before switching to any other CR3. */ -- 2.7.4