Merge branch kvm-arm64/tlbi-range into kvmarm-master/next

* kvm-arm64/tlbi-range:
  : .
  : FEAT_TLBIRANGE support, courtesy of Raghavendra Rao Ananta.
  : From the cover letter:
  :
  : "In certain code paths, KVM/ARM currently invalidates the entire VM's
  : page-tables instead of just invalidating a necessary range. For example,
  : when collapsing a table PTE to a block PTE, instead of iterating over
  : each PTE and flushing them, KVM uses 'vmalls12e1is' TLBI operation to
  : flush all the entries. This is inefficient since the guest would have
  : to refill the TLBs again, even for the addresses that aren't covered
  : by the table entry. The performance impact would scale poorly if many
  : addresses in the VM is going through this remapping.
  :
  : For architectures that implement FEAT_TLBIRANGE, KVM can replace such
  : inefficient paths by performing the invalidations only on the range of
  : addresses that are in scope. This series tries to achieve the same in
  : the areas of stage-2 map, unmap and write-protecting the pages."
  : .
  KVM: arm64: Use TLBI range-based instructions for unmap
  KVM: arm64: Invalidate the table entries upon a range
  KVM: arm64: Flush only the memslot after write-protect
  KVM: arm64: Implement kvm_arch_flush_remote_tlbs_range()
  KVM: arm64: Define kvm_tlb_flush_vmid_range()
  KVM: arm64: Implement __kvm_tlb_flush_vmid_range()
  arm64: tlb: Implement __flush_s2_tlb_range_op()
  arm64: tlb: Refactor the core flush algorithm of __flush_tlb_range
  KVM: Move kvm_arch_flush_remote_tlbs_memslot() to common code
  KVM: Allow range-based TLB invalidation from common code
  KVM: Remove CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
  KVM: arm64: Use kvm_arch_flush_remote_tlbs()
  KVM: Declare kvm_arch_flush_remote_tlbs() globally
  KVM: Rename kvm_arch_flush_remote_tlb() to kvm_arch_flush_remote_tlbs()

Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --combined arch/arm64/include/asm/kvm_host.h
index 616dca4,f623b98..967ee7e
--- 1/arch/arm64/include/asm/kvm_host.h
--- 2/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@@ -380,7 -380,6 +380,7 @@@ enum vcpu_sysreg 
        CPTR_EL2,       /* Architectural Feature Trap Register (EL2) */
        HSTR_EL2,       /* Hypervisor System Trap Register */
        HACR_EL2,       /* Hypervisor Auxiliary Control Register */
 +      HCRX_EL2,       /* Extended Hypervisor Configuration Register */
        TTBR0_EL2,      /* Translation Table Base Register 0 (EL2) */
        TTBR1_EL2,      /* Translation Table Base Register 1 (EL2) */
        TCR_EL2,        /* Translation Control Register (EL2) */
@@@ -401,11 -400,6 +401,11 @@@
        TPIDR_EL2,      /* EL2 Software Thread ID Register */
        CNTHCTL_EL2,    /* Counter-timer Hypervisor Control register */
        SP_EL2,         /* EL2 Stack Pointer */
 +      HFGRTR_EL2,
 +      HFGWTR_EL2,
 +      HFGITR_EL2,
 +      HDFGRTR_EL2,
 +      HDFGWTR_EL2,
        CNTHP_CTL_EL2,
        CNTHP_CVAL_EL2,
        CNTHV_CTL_EL2,
@@@ -573,7 -567,8 +573,7 @@@ struct kvm_vcpu_arch 
        /* Cache some mmu pages needed inside spinlock regions */
        struct kvm_mmu_memory_cache mmu_page_cache;
  
 -      /* Target CPU and feature flags */
 -      int target;
 +      /* feature flags */
        DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
  
        /* Virtual SError ESR to restore when HCR_EL2.VSE is set */
@@@ -674,8 -669,6 +674,8 @@@
  #define VCPU_SVE_FINALIZED    __vcpu_single_flag(cflags, BIT(1))
  /* PTRAUTH exposed to guest */
  #define GUEST_HAS_PTRAUTH     __vcpu_single_flag(cflags, BIT(2))
 +/* KVM_ARM_VCPU_INIT completed */
 +#define VCPU_INITIALIZED      __vcpu_single_flag(cflags, BIT(3))
  
  /* Exception pending */
  #define PENDING_EXCEPTION     __vcpu_single_flag(iflags, BIT(0))
@@@ -906,6 -899,7 +906,6 @@@ struct kvm_vcpu_stat 
        u64 exits;
  };
  
 -void kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
  unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
  int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
  int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
@@@ -989,7 -983,6 +989,7 @@@ int kvm_handle_cp10_id(struct kvm_vcpu 
  void kvm_reset_sys_regs(struct kvm_vcpu *vcpu);
  
  int __init kvm_sys_reg_table_init(void);
 +int __init populate_nv_trap_config(void);
  
  bool lock_all_vcpus(struct kvm *kvm);
  void unlock_all_vcpus(struct kvm *kvm);
@@@ -1120,6 -1113,10 +1120,10 @@@ int __init kvm_set_ipa_limit(void)
  #define __KVM_HAVE_ARCH_VM_ALLOC
  struct kvm *kvm_arch_alloc_vm(void);
  
+ #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS
+ 
+ #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
+ 
  static inline bool kvm_vm_is_protected(struct kvm *kvm)
  {
        return false;

diff --combined arch/arm64/kvm/Kconfig
index f4f0b19,6b730fc..83c1e09
--- 1/arch/arm64/kvm/Kconfig
--- 2/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@@ -25,7 -25,6 +25,6 @@@ menuconfig KV
        select MMU_NOTIFIER
        select PREEMPT_NOTIFIERS
        select HAVE_KVM_CPU_RELAX_INTERCEPT
-       select HAVE_KVM_ARCH_TLB_FLUSH_ALL
        select KVM_MMIO
        select KVM_GENERIC_DIRTYLOG_READ_PROTECT
        select KVM_XFER_TO_GUEST_WORK
@@@ -43,7 -42,6 +42,7 @@@
        select SCHED_INFO
        select GUEST_PERF_EVENTS if PERF_EVENTS
        select INTERVAL_TREE
 +      select XARRAY_MULTI
        help
          Support hosting virtualized guest machines.
  

diff --combined arch/arm64/kvm/arm.c
index 183786f,fd2af63..1cad736
--- 1/arch/arm64/kvm/arm.c
--- 2/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@@ -36,7 -36,6 +36,7 @@@
  #include <asm/kvm_arm.h>
  #include <asm/kvm_asm.h>
  #include <asm/kvm_mmu.h>
 +#include <asm/kvm_nested.h>
  #include <asm/kvm_pkvm.h>
  #include <asm/kvm_emulate.h>
  #include <asm/sections.h>
@@@ -366,7 -365,7 +366,7 @@@ int kvm_arch_vcpu_create(struct kvm_vcp
  #endif
  
        /* Force users to call KVM_ARM_VCPU_INIT */
 -      vcpu->arch.target = -1;
 +      vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
        bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
  
        vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
@@@ -575,7 -574,7 +575,7 @@@ unsigned long kvm_arch_vcpu_get_ip(stru
  
  static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
  {
 -      return vcpu->arch.target >= 0;
 +      return vcpu_get_flag(vcpu, VCPU_INITIALIZED);
  }
  
  /*
@@@ -819,9 -818,6 +819,9 @@@ static bool vcpu_mode_is_bad_32bit(stru
        if (likely(!vcpu_mode_is_32bit(vcpu)))
                return false;
  
 +      if (vcpu_has_nv(vcpu))
 +              return true;
 +
        return !kvm_supports_32bit_el0();
  }
  
@@@ -1062,7 -1058,7 +1062,7 @@@ int kvm_arch_vcpu_ioctl_run(struct kvm_
                         * invalid. The VMM can try and fix it by issuing  a
                         * KVM_ARM_VCPU_INIT if it really wants to.
                         */
 -                      vcpu->arch.target = -1;
 +                      vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
                        ret = ARM_EXCEPTION_IL;
                }
  
@@@ -1223,7 -1219,8 +1223,7 @@@ static bool kvm_vcpu_init_changed(struc
  {
        unsigned long features = init->features[0];
  
 -      return !bitmap_equal(vcpu->arch.features, &features, KVM_VCPU_MAX_FEATURES) ||
 -                      vcpu->arch.target != init->target;
 +      return !bitmap_equal(vcpu->arch.features, &features, KVM_VCPU_MAX_FEATURES);
  }
  
  static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
@@@ -1239,18 -1236,20 +1239,18 @@@
            !bitmap_equal(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES))
                goto out_unlock;
  
 -      vcpu->arch.target = init->target;
        bitmap_copy(vcpu->arch.features, &features, KVM_VCPU_MAX_FEATURES);
  
        /* Now we know what it is, we can reset it. */
        ret = kvm_reset_vcpu(vcpu);
        if (ret) {
 -              vcpu->arch.target = -1;
                bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
                goto out_unlock;
        }
  
        bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
        set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
 -
 +      vcpu_set_flag(vcpu, VCPU_INITIALIZED);
  out_unlock:
        mutex_unlock(&kvm->arch.config_lock);
        return ret;
@@@ -1261,15 -1260,14 +1261,15 @@@ static int kvm_vcpu_set_target(struct k
  {
        int ret;
  
 -      if (init->target != kvm_target_cpu())
 +      if (init->target != KVM_ARM_TARGET_GENERIC_V8 &&
 +          init->target != kvm_target_cpu())
                return -EINVAL;
  
        ret = kvm_vcpu_init_check_features(vcpu, init);
        if (ret)
                return ret;
  
 -      if (vcpu->arch.target == -1)
 +      if (!kvm_vcpu_initialized(vcpu))
                return __kvm_vcpu_set_target(vcpu, init);
  
        if (kvm_vcpu_init_changed(vcpu, init))
@@@ -1534,12 -1532,6 +1534,6 @@@ void kvm_arch_sync_dirty_log(struct kv
  
  }
  
- void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
-                                       const struct kvm_memory_slot *memslot)
- {
-       kvm_flush_remote_tlbs(kvm);
- }
- 
  static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
                                        struct kvm_arm_device_addr *dev_addr)
  {
@@@ -1597,9 -1589,9 +1591,9 @@@ int kvm_arch_vm_ioctl(struct file *filp
                return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
        }
        case KVM_ARM_PREFERRED_TARGET: {
 -              struct kvm_vcpu_init init;
 -
 -              kvm_vcpu_preferred_target(&init);
 +              struct kvm_vcpu_init init = {
 +                      .target = KVM_ARM_TARGET_GENERIC_V8,
 +              };
  
                if (copy_to_user(argp, &init, sizeof(init)))
                        return -EFAULT;