KVM: VMX: Dynamically enable/disable PML based on memslot dirty logging

Currently, if enable_pml=1 PML remains enabled for the entire lifetime
of the VM irrespective of whether dirty logging is enable or disabled.
When dirty logging is disabled, all the pages of the VM are manually
marked dirty, so that PML is effectively non-operational.  Setting
the dirty bits is an expensive operation which can cause severe MMU
lock contention in a performance sensitive path when dirty logging is
disabled after a failed or canceled live migration.

Manually setting dirty bits also fails to prevent PML activity if some
code path clears dirty bits, which can incur unnecessary VM-Exits.

In order to avoid this extra overhead, dynamically enable/disable PML
when dirty logging gets turned on/off for the first/last memslot.

Signed-off-by: Makarand Sonare <makarandsonare@google.com>
Co-developed-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-Id: <20210213005015.1651772-12-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 90affdb2cbbcd9f15c914a38134cd2910a3963cb..323641097f63aa697bcc6aefb621252cce61849c 100644 (file)
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -93,6 +93,7 @@ KVM_X86_OP(check_intercept)
 KVM_X86_OP(handle_exit_irqoff)
 KVM_X86_OP_NULL(request_immediate_exit)
 KVM_X86_OP(sched_in)
+KVM_X86_OP_NULL(update_cpu_dirty_logging)
 KVM_X86_OP_NULL(pre_block)
 KVM_X86_OP_NULL(post_block)
 KVM_X86_OP_NULL(vcpu_blocking)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 5cf382ec48b07eec16c6aa7131310bb2e176f795..ffcfa84c969d030d288c09915babde5f09f2db24 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -89,6 +89,8 @@
        KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP)
 #define KVM_REQ_APF_READY              KVM_ARCH_REQ(28)
 #define KVM_REQ_MSR_FILTER_CHANGED     KVM_ARCH_REQ(29)
+#define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \
+       KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 
 #define CR0_RESERVED_BITS                                               \
        (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
@@ -1007,6 +1009,7 @@ struct kvm_arch {
        u32 bsp_vcpu_id;
 
        u64 disabled_quirks;
+       int cpu_dirty_logging_count;
 
        enum kvm_irqchip_mode irqchip_mode;
        u8 nr_reserved_ioapic_pins;
@@ -1275,6 +1278,7 @@ struct kvm_x86_ops {
         * value indicates CPU dirty logging is unsupported or disabled.
         */
        int cpu_dirty_log_size;
+       void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu);
 
        /* pmu operations of sub-arch */
        const struct kvm_pmu_ops *pmu_ops;

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 486c7ac6d8132edf15b52b20964a902e18e49aaa..bcca0b80e0d040be4ce9a4f9a2ee05d77e0c8257 100644 (file)
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -4490,6 +4490,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
                vmx_set_virtual_apic_mode(vcpu);
        }
 
+       if (vmx->nested.update_vmcs01_cpu_dirty_logging) {
+               vmx->nested.update_vmcs01_cpu_dirty_logging = false;
+               vmx_update_cpu_dirty_logging(vcpu);
+       }
+
        /* Unpin physical memory we referred to in vmcs02 */
        if (vmx->nested.apic_access_page) {
                kvm_release_page_clean(vmx->nested.apic_access_page);

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index eb207ffb7873ade991d24c57f0293111f9cfd8e8..50810d4714628c2f3978847e9068c6106529797a 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -4277,7 +4277,12 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
        */
        exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
 
-       if (!enable_pml)
+       /*
+        * PML is enabled/disabled when dirty logging of memsmlots changes, but
+        * it needs to be set here when dirty logging is already active, e.g.
+        * if this vCPU was created after dirty logging was enabled.
+        */
+       if (!vcpu->kvm->arch.cpu_dirty_logging_count)
                exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
 
        if (cpu_has_vmx_xsaves()) {
@@ -7491,6 +7496,26 @@ static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
                shrink_ple_window(vcpu);
 }
 
+void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
+{
+       struct vcpu_vmx *vmx = to_vmx(vcpu);
+
+       if (is_guest_mode(vcpu)) {
+               vmx->nested.update_vmcs01_cpu_dirty_logging = true;
+               return;
+       }
+
+       /*
+        * Note, cpu_dirty_logging_count can be changed concurrent with this
+        * code, but in that case another update request will be made and so
+        * the guest will never run with a stale PML value.
+        */
+       if (vcpu->kvm->arch.cpu_dirty_logging_count)
+               secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML);
+       else
+               secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML);
+}
+
 static int vmx_pre_block(struct kvm_vcpu *vcpu)
 {
        if (pi_pre_block(vcpu))
@@ -7698,6 +7723,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
        .sched_in = vmx_sched_in,
 
        .cpu_dirty_log_size = PML_ENTITY_NUM,
+       .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging,
 
        .pre_block = vmx_pre_block,
        .post_block = vmx_post_block,

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index 12c53d05a902becd08fa8f07904cba623c32cbd0..89da5e1251f18c115c8436aca484b3656151a0cf 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -165,6 +165,7 @@ struct nested_vmx {
 
        bool change_vmcs01_virtual_apic_mode;
        bool reload_vmcs01_apic_access_page;
+       bool update_vmcs01_cpu_dirty_logging;
 
        /*
         * Enlightened VMCS has been enabled. It does not mean that L1 has to
@@ -393,6 +394,7 @@ int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr);
 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);
 void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu,
        u32 msr, int type, bool value);
+void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu);
 
 static inline u8 vmx_get_rvi(void)
 {

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c0d22f19aed0755eabdfc959a40e21d44226e626..b9a8c8af9713ecefa884b15775a49ed71e2e0ccb 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -8987,6 +8987,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                        kvm_check_async_pf_completion(vcpu);
                if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
                        static_call(kvm_x86_msr_filter_changed)(vcpu);
+
+               if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))
+                       static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);
        }
 
        if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
@@ -10755,14 +10758,38 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
        return 0;
 }
 
+
+static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable)
+{
+       struct kvm_arch *ka = &kvm->arch;
+
+       if (!kvm_x86_ops.cpu_dirty_log_size)
+               return;
+
+       if ((enable && ++ka->cpu_dirty_logging_count == 1) ||
+           (!enable && --ka->cpu_dirty_logging_count == 0))
+               kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING);
+
+       WARN_ON_ONCE(ka->cpu_dirty_logging_count < 0);
+}
+
 static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
                                     struct kvm_memory_slot *old,
                                     struct kvm_memory_slot *new,
                                     enum kvm_mr_change change)
 {
+       bool log_dirty_pages = new->flags & KVM_MEM_LOG_DIRTY_PAGES;
+
+       /*
+        * Update CPU dirty logging if dirty logging is being toggled.  This
+        * applies to all operations.
+        */
+       if ((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES)
+               kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages);
+
        /*
-        * Nothing to do for RO slots (which can't be dirtied and can't be made
-        * writable) or CREATE/MOVE/DELETE of a slot.  See comments below.
+        * Nothing more to do for RO slots (which can't be dirtied and can't be
+        * made writable) or CREATE/MOVE/DELETE of a slot.  See comments below.
         */
        if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY))
                return;
@@ -10792,7 +10819,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
         * MOVE/DELETE: The old mappings will already have been cleaned up by
         *              kvm_arch_flush_shadow_memslot()
         */
-       if (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
+       if (!log_dirty_pages)
                kvm_mmu_zap_collapsible_sptes(kvm, new);
 
        /*
@@ -10823,7 +10850,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
         * initial-all-set state.  Otherwise, depending on whether pml
         * is enabled the D-bit or the W-bit will be cleared.
         */
-       if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
+       if (log_dirty_pages) {
                if (kvm_x86_ops.cpu_dirty_log_size) {
                        if (!kvm_dirty_log_manual_protect_and_init_set(kvm))
                                kvm_mmu_slot_leaf_clear_dirty(kvm, new);