arch/x86/kvm/lapic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /*
   4  * Local APIC virtualization
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright (C) 2007 Novell
   8  * Copyright (C) 2007 Intel
   9  * Copyright 2009 Red Hat, Inc. and/or its affiliates.
  10  *
  11  * Authors:
  12  *   Dor Laor <dor.laor@qumranet.com>
  13  *   Gregory Haskins <ghaskins@novell.com>
  14  *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
  15  *
  16  * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
  17  */
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/kvm_host.h>
  21 #include <linux/kvm.h>
  22 #include <linux/mm.h>
  23 #include <linux/highmem.h>
  24 #include <linux/smp.h>
  25 #include <linux/hrtimer.h>
  26 #include <linux/io.h>
  27 #include <linux/export.h>
  28 #include <linux/math64.h>
  29 #include <linux/slab.h>
  30 #include <asm/processor.h>
  31 #include <asm/mce.h>
  32 #include <asm/msr.h>
  33 #include <asm/page.h>
  34 #include <asm/current.h>
  35 #include <asm/apicdef.h>
  36 #include <asm/delay.h>
  37 #include <linux/atomic.h>
  38 #include <linux/jump_label.h>
  39 #include "kvm_cache_regs.h"
  40 #include "irq.h"
  41 #include "ioapic.h"
  42 #include "trace.h"
  43 #include "x86.h"
  44 #include "cpuid.h"
  45 #include "hyperv.h"
  46 #include "smm.h"
  47
  48 #ifndef CONFIG_X86_64
  49 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
  50 #else
  51 #define mod_64(x, y) ((x) % (y))
  52 #endif
  53
  54 #define PRId64 "d"
  55 #define PRIx64 "llx"
  56 #define PRIu64 "u"
  57 #define PRIo64 "o"
  58
  59 /* 14 is the version for Xeon and Pentium 8.4.8*/
  60 #define APIC_VERSION                    0x14UL
  61 #define LAPIC_MMIO_LENGTH               (1 << 12)
  62 /* followed define is not in apicdef.h */
  63 #define MAX_APIC_VECTOR                 256
  64 #define APIC_VECTORS_PER_REG            32
  65
  66 static bool lapic_timer_advance_dynamic __read_mostly;
  67 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN  100     /* clock cycles */
  68 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX  10000   /* clock cycles */
  69 #define LAPIC_TIMER_ADVANCE_NS_INIT     1000
  70 #define LAPIC_TIMER_ADVANCE_NS_MAX     5000
  71 /* step-by-step approximation to mitigate fluctuation */
  72 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
  73 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data);
  74 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data);
  75
  76 static inline void __kvm_lapic_set_reg(char *regs, int reg_off, u32 val)
  77 {
  78         *((u32 *) (regs + reg_off)) = val;
  79 }
  80
  81 static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val)
  82 {
  83         __kvm_lapic_set_reg(apic->regs, reg_off, val);
  84 }
  85
  86 static __always_inline u64 __kvm_lapic_get_reg64(char *regs, int reg)
  87 {
  88         BUILD_BUG_ON(reg != APIC_ICR);
  89         return *((u64 *) (regs + reg));
  90 }
  91
  92 static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg)
  93 {
  94         return __kvm_lapic_get_reg64(apic->regs, reg);
  95 }
  96
  97 static __always_inline void __kvm_lapic_set_reg64(char *regs, int reg, u64 val)
  98 {
  99         BUILD_BUG_ON(reg != APIC_ICR);
 100         *((u64 *) (regs + reg)) = val;
 101 }
 102
 103 static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic,
 104                                                 int reg, u64 val)
 105 {
 106         __kvm_lapic_set_reg64(apic->regs, reg, val);
 107 }
 108
 109 static inline int apic_test_vector(int vec, void *bitmap)
 110 {
 111         return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
 112 }
 113
 114 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
 115 {
 116         struct kvm_lapic *apic = vcpu->arch.apic;
 117
 118         return apic_test_vector(vector, apic->regs + APIC_ISR) ||
 119                 apic_test_vector(vector, apic->regs + APIC_IRR);
 120 }
 121
 122 static inline int __apic_test_and_set_vector(int vec, void *bitmap)
 123 {
 124         return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
 125 }
 126
 127 static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
 128 {
 129         return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
 130 }
 131
 132 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
 133 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);
 134
 135 static inline int apic_enabled(struct kvm_lapic *apic)
 136 {
 137         return kvm_apic_sw_enabled(apic) &&     kvm_apic_hw_enabled(apic);
 138 }
 139
 140 #define LVT_MASK        \
 141         (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
 142
 143 #define LINT_MASK       \
 144         (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
 145          APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
 146
 147 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
 148 {
 149         return apic->vcpu->vcpu_id;
 150 }
 151
 152 static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
 153 {
 154         return pi_inject_timer && kvm_vcpu_apicv_active(vcpu) &&
 155                 (kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm));
 156 }
 157
 158 bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu)
 159 {
 160         return kvm_x86_ops.set_hv_timer
 161                && !(kvm_mwait_in_guest(vcpu->kvm) ||
 162                     kvm_can_post_timer_interrupt(vcpu));
 163 }
 164
 165 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
 166 {
 167         return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
 168 }
 169
 170 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
 171 {
 172         return ((id >> 4) << 16) | (1 << (id & 0xf));
 173 }
 174
 175 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
 176                 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
 177         switch (map->logical_mode) {
 178         case KVM_APIC_MODE_SW_DISABLED:
 179                 /* Arbitrarily use the flat map so that @cluster isn't NULL. */
 180                 *cluster = map->xapic_flat_map;
 181                 *mask = 0;
 182                 return true;
 183         case KVM_APIC_MODE_X2APIC: {
 184                 u32 offset = (dest_id >> 16) * 16;
 185                 u32 max_apic_id = map->max_apic_id;
 186
 187                 if (offset <= max_apic_id) {
 188                         u8 cluster_size = min(max_apic_id - offset + 1, 16U);
 189
 190                         offset = array_index_nospec(offset, map->max_apic_id + 1);
 191                         *cluster = &map->phys_map[offset];
 192                         *mask = dest_id & (0xffff >> (16 - cluster_size));
 193                 } else {
 194                         *mask = 0;
 195                 }
 196
 197                 return true;
 198                 }
 199         case KVM_APIC_MODE_XAPIC_FLAT:
 200                 *cluster = map->xapic_flat_map;
 201                 *mask = dest_id & 0xff;
 202                 return true;
 203         case KVM_APIC_MODE_XAPIC_CLUSTER:
 204                 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
 205                 *mask = dest_id & 0xf;
 206                 return true;
 207         case KVM_APIC_MODE_MAP_DISABLED:
 208                 return false;
 209         default:
 210                 WARN_ON_ONCE(1);
 211                 return false;
 212         }
 213 }
 214
 215 static void kvm_apic_map_free(struct rcu_head *rcu)
 216 {
 217         struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
 218
 219         kvfree(map);
 220 }
 221
 222 static int kvm_recalculate_phys_map(struct kvm_apic_map *new,
 223                                     struct kvm_vcpu *vcpu,
 224                                     bool *xapic_id_mismatch)
 225 {
 226         struct kvm_lapic *apic = vcpu->arch.apic;
 227         u32 x2apic_id = kvm_x2apic_id(apic);
 228         u32 xapic_id = kvm_xapic_id(apic);
 229         u32 physical_id;
 230
 231         /*
 232          * For simplicity, KVM always allocates enough space for all possible
 233          * xAPIC IDs.  Yell, but don't kill the VM, as KVM can continue on
 234          * without the optimized map.
 235          */
 236         if (WARN_ON_ONCE(xapic_id > new->max_apic_id))
 237                 return -EINVAL;
 238
 239         /*
 240          * Bail if a vCPU was added and/or enabled its APIC between allocating
 241          * the map and doing the actual calculations for the map.  Note, KVM
 242          * hardcodes the x2APIC ID to vcpu_id, i.e. there's no TOCTOU bug if
 243          * the compiler decides to reload x2apic_id after this check.
 244          */
 245         if (x2apic_id > new->max_apic_id)
 246                 return -E2BIG;
 247
 248         /*
 249          * Deliberately truncate the vCPU ID when detecting a mismatched APIC
 250          * ID to avoid false positives if the vCPU ID, i.e. x2APIC ID, is a
 251          * 32-bit value.  Any unwanted aliasing due to truncation results will
 252          * be detected below.
 253          */
 254         if (!apic_x2apic_mode(apic) && xapic_id != (u8)vcpu->vcpu_id)
 255                 *xapic_id_mismatch = true;
 256
 257         /*
 258          * Apply KVM's hotplug hack if userspace has enable 32-bit APIC IDs.
 259          * Allow sending events to vCPUs by their x2APIC ID even if the target
 260          * vCPU is in legacy xAPIC mode, and silently ignore aliased xAPIC IDs
 261          * (the x2APIC ID is truncated to 8 bits, causing IDs > 0xff to wrap
 262          * and collide).
 263          *
 264          * Honor the architectural (and KVM's non-optimized) behavior if
 265          * userspace has not enabled 32-bit x2APIC IDs.  Each APIC is supposed
 266          * to process messages independently.  If multiple vCPUs have the same
 267          * effective APIC ID, e.g. due to the x2APIC wrap or because the guest
 268          * manually modified its xAPIC IDs, events targeting that ID are
 269          * supposed to be recognized by all vCPUs with said ID.
 270          */
 271         if (vcpu->kvm->arch.x2apic_format) {
 272                 /* See also kvm_apic_match_physical_addr(). */
 273                 if (apic_x2apic_mode(apic) || x2apic_id > 0xff)
 274                         new->phys_map[x2apic_id] = apic;
 275
 276                 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
 277                         new->phys_map[xapic_id] = apic;
 278         } else {
 279                 /*
 280                  * Disable the optimized map if the physical APIC ID is already
 281                  * mapped, i.e. is aliased to multiple vCPUs.  The optimized
 282                  * map requires a strict 1:1 mapping between IDs and vCPUs.
 283                  */
 284                 if (apic_x2apic_mode(apic))
 285                         physical_id = x2apic_id;
 286                 else
 287                         physical_id = xapic_id;
 288
 289                 if (new->phys_map[physical_id])
 290                         return -EINVAL;
 291
 292                 new->phys_map[physical_id] = apic;
 293         }
 294
 295         return 0;
 296 }
 297
 298 static void kvm_recalculate_logical_map(struct kvm_apic_map *new,
 299                                         struct kvm_vcpu *vcpu)
 300 {
 301         struct kvm_lapic *apic = vcpu->arch.apic;
 302         enum kvm_apic_logical_mode logical_mode;
 303         struct kvm_lapic **cluster;
 304         u16 mask;
 305         u32 ldr;
 306
 307         if (new->logical_mode == KVM_APIC_MODE_MAP_DISABLED)
 308                 return;
 309
 310         if (!kvm_apic_sw_enabled(apic))
 311                 return;
 312
 313         ldr = kvm_lapic_get_reg(apic, APIC_LDR);
 314         if (!ldr)
 315                 return;
 316
 317         if (apic_x2apic_mode(apic)) {
 318                 logical_mode = KVM_APIC_MODE_X2APIC;
 319         } else {
 320                 ldr = GET_APIC_LOGICAL_ID(ldr);
 321                 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
 322                         logical_mode = KVM_APIC_MODE_XAPIC_FLAT;
 323                 else
 324                         logical_mode = KVM_APIC_MODE_XAPIC_CLUSTER;
 325         }
 326
 327         /*
 328          * To optimize logical mode delivery, all software-enabled APICs must
 329          * be configured for the same mode.
 330          */
 331         if (new->logical_mode == KVM_APIC_MODE_SW_DISABLED) {
 332                 new->logical_mode = logical_mode;
 333         } else if (new->logical_mode != logical_mode) {
 334                 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED;
 335                 return;
 336         }
 337
 338         /*
 339          * In x2APIC mode, the LDR is read-only and derived directly from the
 340          * x2APIC ID, thus is guaranteed to be addressable.  KVM reuses
 341          * kvm_apic_map.phys_map to optimize logical mode x2APIC interrupts by
 342          * reversing the LDR calculation to get cluster of APICs, i.e. no
 343          * additional work is required.
 344          */
 345         if (apic_x2apic_mode(apic)) {
 346                 WARN_ON_ONCE(ldr != kvm_apic_calc_x2apic_ldr(kvm_x2apic_id(apic)));
 347                 return;
 348         }
 349
 350         if (WARN_ON_ONCE(!kvm_apic_map_get_logical_dest(new, ldr,
 351                                                         &cluster, &mask))) {
 352                 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED;
 353                 return;
 354         }
 355
 356         if (!mask)
 357                 return;
 358
 359         ldr = ffs(mask) - 1;
 360         if (!is_power_of_2(mask) || cluster[ldr])
 361                 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED;
 362         else
 363                 cluster[ldr] = apic;
 364 }
 365
 366 /*
 367  * CLEAN -> DIRTY and UPDATE_IN_PROGRESS -> DIRTY changes happen without a lock.
 368  *
 369  * DIRTY -> UPDATE_IN_PROGRESS and UPDATE_IN_PROGRESS -> CLEAN happen with
 370  * apic_map_lock_held.
 371  */
 372 enum {
 373         CLEAN,
 374         UPDATE_IN_PROGRESS,
 375         DIRTY
 376 };
 377
 378 void kvm_recalculate_apic_map(struct kvm *kvm)
 379 {
 380         struct kvm_apic_map *new, *old = NULL;
 381         struct kvm_vcpu *vcpu;
 382         unsigned long i;
 383         u32 max_id = 255; /* enough space for any xAPIC ID */
 384         bool xapic_id_mismatch = false;
 385
 386         /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map.  */
 387         if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN)
 388                 return;
 389
 390         WARN_ONCE(!irqchip_in_kernel(kvm),
 391                   "Dirty APIC map without an in-kernel local APIC");
 392
 393         mutex_lock(&kvm->arch.apic_map_lock);
 394         /*
 395          * Read kvm->arch.apic_map_dirty before kvm->arch.apic_map
 396          * (if clean) or the APIC registers (if dirty).
 397          */
 398         if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty,
 399                                    DIRTY, UPDATE_IN_PROGRESS) == CLEAN) {
 400                 /* Someone else has updated the map. */
 401                 mutex_unlock(&kvm->arch.apic_map_lock);
 402                 return;
 403         }
 404
 405         kvm_for_each_vcpu(i, vcpu, kvm)
 406                 if (kvm_apic_present(vcpu))
 407                         max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
 408
 409         new = kvzalloc(sizeof(struct kvm_apic_map) +
 410                            sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
 411                            GFP_KERNEL_ACCOUNT);
 412
 413         if (!new)
 414                 goto out;
 415
 416         new->max_apic_id = max_id;
 417         new->logical_mode = KVM_APIC_MODE_SW_DISABLED;
 418
 419         kvm_for_each_vcpu(i, vcpu, kvm) {
 420                 if (!kvm_apic_present(vcpu))
 421                         continue;
 422
 423                 if (kvm_recalculate_phys_map(new, vcpu, &xapic_id_mismatch)) {
 424                         kvfree(new);
 425                         new = NULL;
 426                         goto out;
 427                 }
 428
 429                 kvm_recalculate_logical_map(new, vcpu);
 430         }
 431 out:
 432         /*
 433          * The optimized map is effectively KVM's internal version of APICv,
 434          * and all unwanted aliasing that results in disabling the optimized
 435          * map also applies to APICv.
 436          */
 437         if (!new)
 438                 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED);
 439         else
 440                 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED);
 441
 442         if (!new || new->logical_mode == KVM_APIC_MODE_MAP_DISABLED)
 443                 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED);
 444         else
 445                 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED);
 446
 447         if (xapic_id_mismatch)
 448                 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED);
 449         else
 450                 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED);
 451
 452         old = rcu_dereference_protected(kvm->arch.apic_map,
 453                         lockdep_is_held(&kvm->arch.apic_map_lock));
 454         rcu_assign_pointer(kvm->arch.apic_map, new);
 455         /*
 456          * Write kvm->arch.apic_map before clearing apic->apic_map_dirty.
 457          * If another update has come in, leave it DIRTY.
 458          */
 459         atomic_cmpxchg_release(&kvm->arch.apic_map_dirty,
 460                                UPDATE_IN_PROGRESS, CLEAN);
 461         mutex_unlock(&kvm->arch.apic_map_lock);
 462
 463         if (old)
 464                 call_rcu(&old->rcu, kvm_apic_map_free);
 465
 466         kvm_make_scan_ioapic_request(kvm);
 467 }
 468
 469 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
 470 {
 471         bool enabled = val & APIC_SPIV_APIC_ENABLED;
 472
 473         kvm_lapic_set_reg(apic, APIC_SPIV, val);
 474
 475         if (enabled != apic->sw_enabled) {
 476                 apic->sw_enabled = enabled;
 477                 if (enabled)
 478                         static_branch_slow_dec_deferred(&apic_sw_disabled);
 479                 else
 480                         static_branch_inc(&apic_sw_disabled.key);
 481
 482                 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 483         }
 484
 485         /* Check if there are APF page ready requests pending */
 486         if (enabled)
 487                 kvm_make_request(KVM_REQ_APF_READY, apic->vcpu);
 488 }
 489
 490 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
 491 {
 492         kvm_lapic_set_reg(apic, APIC_ID, id << 24);
 493         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 494 }
 495
 496 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
 497 {
 498         kvm_lapic_set_reg(apic, APIC_LDR, id);
 499         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 500 }
 501
 502 static inline void kvm_apic_set_dfr(struct kvm_lapic *apic, u32 val)
 503 {
 504         kvm_lapic_set_reg(apic, APIC_DFR, val);
 505         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 506 }
 507
 508 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
 509 {
 510         u32 ldr = kvm_apic_calc_x2apic_ldr(id);
 511
 512         WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
 513
 514         kvm_lapic_set_reg(apic, APIC_ID, id);
 515         kvm_lapic_set_reg(apic, APIC_LDR, ldr);
 516         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 517 }
 518
 519 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
 520 {
 521         return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
 522 }
 523
 524 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
 525 {
 526         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
 527 }
 528
 529 static inline int apic_lvtt_period(struct kvm_lapic *apic)
 530 {
 531         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
 532 }
 533
 534 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
 535 {
 536         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
 537 }
 538
 539 static inline int apic_lvt_nmi_mode(u32 lvt_val)
 540 {
 541         return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
 542 }
 543
 544 static inline bool kvm_lapic_lvt_supported(struct kvm_lapic *apic, int lvt_index)
 545 {
 546         return apic->nr_lvt_entries > lvt_index;
 547 }
 548
 549 static inline int kvm_apic_calc_nr_lvt_entries(struct kvm_vcpu *vcpu)
 550 {
 551         return KVM_APIC_MAX_NR_LVT_ENTRIES - !(vcpu->arch.mcg_cap & MCG_CMCI_P);
 552 }
 553
 554 void kvm_apic_set_version(struct kvm_vcpu *vcpu)
 555 {
 556         struct kvm_lapic *apic = vcpu->arch.apic;
 557         u32 v = 0;
 558
 559         if (!lapic_in_kernel(vcpu))
 560                 return;
 561
 562         v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16);
 563
 564         /*
 565          * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
 566          * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
 567          * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
 568          * version first and level-triggered interrupts never get EOIed in
 569          * IOAPIC.
 570          */
 571         if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) &&
 572             !ioapic_in_kernel(vcpu->kvm))
 573                 v |= APIC_LVR_DIRECTED_EOI;
 574         kvm_lapic_set_reg(apic, APIC_LVR, v);
 575 }
 576
 577 void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu)
 578 {
 579         int nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu);
 580         struct kvm_lapic *apic = vcpu->arch.apic;
 581         int i;
 582
 583         if (!lapic_in_kernel(vcpu) || nr_lvt_entries == apic->nr_lvt_entries)
 584                 return;
 585
 586         /* Initialize/mask any "new" LVT entries. */
 587         for (i = apic->nr_lvt_entries; i < nr_lvt_entries; i++)
 588                 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED);
 589
 590         apic->nr_lvt_entries = nr_lvt_entries;
 591
 592         /* The number of LVT entries is reflected in the version register. */
 593         kvm_apic_set_version(vcpu);
 594 }
 595
 596 static const unsigned int apic_lvt_mask[KVM_APIC_MAX_NR_LVT_ENTRIES] = {
 597         [LVT_TIMER] = LVT_MASK,      /* timer mode mask added at runtime */
 598         [LVT_THERMAL_MONITOR] = LVT_MASK | APIC_MODE_MASK,
 599         [LVT_PERFORMANCE_COUNTER] = LVT_MASK | APIC_MODE_MASK,
 600         [LVT_LINT0] = LINT_MASK,
 601         [LVT_LINT1] = LINT_MASK,
 602         [LVT_ERROR] = LVT_MASK,
 603         [LVT_CMCI] = LVT_MASK | APIC_MODE_MASK
 604 };
 605
 606 static int find_highest_vector(void *bitmap)
 607 {
 608         int vec;
 609         u32 *reg;
 610
 611         for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
 612              vec >= 0; vec -= APIC_VECTORS_PER_REG) {
 613                 reg = bitmap + REG_POS(vec);
 614                 if (*reg)
 615                         return __fls(*reg) + vec;
 616         }
 617
 618         return -1;
 619 }
 620
 621 static u8 count_vectors(void *bitmap)
 622 {
 623         int vec;
 624         u32 *reg;
 625         u8 count = 0;
 626
 627         for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
 628                 reg = bitmap + REG_POS(vec);
 629                 count += hweight32(*reg);
 630         }
 631
 632         return count;
 633 }
 634
 635 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
 636 {
 637         u32 i, vec;
 638         u32 pir_val, irr_val, prev_irr_val;
 639         int max_updated_irr;
 640
 641         max_updated_irr = -1;
 642         *max_irr = -1;
 643
 644         for (i = vec = 0; i <= 7; i++, vec += 32) {
 645                 pir_val = READ_ONCE(pir[i]);
 646                 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
 647                 if (pir_val) {
 648                         prev_irr_val = irr_val;
 649                         irr_val |= xchg(&pir[i], 0);
 650                         *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
 651                         if (prev_irr_val != irr_val) {
 652                                 max_updated_irr =
 653                                         __fls(irr_val ^ prev_irr_val) + vec;
 654                         }
 655                 }
 656                 if (irr_val)
 657                         *max_irr = __fls(irr_val) + vec;
 658         }
 659
 660         return ((max_updated_irr != -1) &&
 661                 (max_updated_irr == *max_irr));
 662 }
 663 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
 664
 665 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
 666 {
 667         struct kvm_lapic *apic = vcpu->arch.apic;
 668
 669         return __kvm_apic_update_irr(pir, apic->regs, max_irr);
 670 }
 671 EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
 672
 673 static inline int apic_search_irr(struct kvm_lapic *apic)
 674 {
 675         return find_highest_vector(apic->regs + APIC_IRR);
 676 }
 677
 678 static inline int apic_find_highest_irr(struct kvm_lapic *apic)
 679 {
 680         int result;
 681
 682         /*
 683          * Note that irr_pending is just a hint. It will be always
 684          * true with virtual interrupt delivery enabled.
 685          */
 686         if (!apic->irr_pending)
 687                 return -1;
 688
 689         result = apic_search_irr(apic);
 690         ASSERT(result == -1 || result >= 16);
 691
 692         return result;
 693 }
 694
 695 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
 696 {
 697         if (unlikely(apic->apicv_active)) {
 698                 /* need to update RVI */
 699                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 700                 static_call_cond(kvm_x86_hwapic_irr_update)(apic->vcpu,
 701                                                             apic_find_highest_irr(apic));
 702         } else {
 703                 apic->irr_pending = false;
 704                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 705                 if (apic_search_irr(apic) != -1)
 706                         apic->irr_pending = true;
 707         }
 708 }
 709
 710 void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec)
 711 {
 712         apic_clear_irr(vec, vcpu->arch.apic);
 713 }
 714 EXPORT_SYMBOL_GPL(kvm_apic_clear_irr);
 715
 716 static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
 717 {
 718         if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
 719                 return;
 720
 721         /*
 722          * With APIC virtualization enabled, all caching is disabled
 723          * because the processor can modify ISR under the hood.  Instead
 724          * just set SVI.
 725          */
 726         if (unlikely(apic->apicv_active))
 727                 static_call_cond(kvm_x86_hwapic_isr_update)(vec);
 728         else {
 729                 ++apic->isr_count;
 730                 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
 731                 /*
 732                  * ISR (in service register) bit is set when injecting an interrupt.
 733                  * The highest vector is injected. Thus the latest bit set matches
 734                  * the highest bit in ISR.
 735                  */
 736                 apic->highest_isr_cache = vec;
 737         }
 738 }
 739
 740 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
 741 {
 742         int result;
 743
 744         /*
 745          * Note that isr_count is always 1, and highest_isr_cache
 746          * is always -1, with APIC virtualization enabled.
 747          */
 748         if (!apic->isr_count)
 749                 return -1;
 750         if (likely(apic->highest_isr_cache != -1))
 751                 return apic->highest_isr_cache;
 752
 753         result = find_highest_vector(apic->regs + APIC_ISR);
 754         ASSERT(result == -1 || result >= 16);
 755
 756         return result;
 757 }
 758
 759 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
 760 {
 761         if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
 762                 return;
 763
 764         /*
 765          * We do get here for APIC virtualization enabled if the guest
 766          * uses the Hyper-V APIC enlightenment.  In this case we may need
 767          * to trigger a new interrupt delivery by writing the SVI field;
 768          * on the other hand isr_count and highest_isr_cache are unused
 769          * and must be left alone.
 770          */
 771         if (unlikely(apic->apicv_active))
 772                 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic));
 773         else {
 774                 --apic->isr_count;
 775                 BUG_ON(apic->isr_count < 0);
 776                 apic->highest_isr_cache = -1;
 777         }
 778 }
 779
 780 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 781 {
 782         /* This may race with setting of irr in __apic_accept_irq() and
 783          * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
 784          * will cause vmexit immediately and the value will be recalculated
 785          * on the next vmentry.
 786          */
 787         return apic_find_highest_irr(vcpu->arch.apic);
 788 }
 789 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
 790
 791 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
 792                              int vector, int level, int trig_mode,
 793                              struct dest_map *dest_map);
 794
 795 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
 796                      struct dest_map *dest_map)
 797 {
 798         struct kvm_lapic *apic = vcpu->arch.apic;
 799
 800         return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
 801                         irq->level, irq->trig_mode, dest_map);
 802 }
 803
 804 static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map,
 805                          struct kvm_lapic_irq *irq, u32 min)
 806 {
 807         int i, count = 0;
 808         struct kvm_vcpu *vcpu;
 809
 810         if (min > map->max_apic_id)
 811                 return 0;
 812
 813         for_each_set_bit(i, ipi_bitmap,
 814                 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
 815                 if (map->phys_map[min + i]) {
 816                         vcpu = map->phys_map[min + i]->vcpu;
 817                         count += kvm_apic_set_irq(vcpu, irq, NULL);
 818                 }
 819         }
 820
 821         return count;
 822 }
 823
 824 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
 825                     unsigned long ipi_bitmap_high, u32 min,
 826                     unsigned long icr, int op_64_bit)
 827 {
 828         struct kvm_apic_map *map;
 829         struct kvm_lapic_irq irq = {0};
 830         int cluster_size = op_64_bit ? 64 : 32;
 831         int count;
 832
 833         if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK))
 834                 return -KVM_EINVAL;
 835
 836         irq.vector = icr & APIC_VECTOR_MASK;
 837         irq.delivery_mode = icr & APIC_MODE_MASK;
 838         irq.level = (icr & APIC_INT_ASSERT) != 0;
 839         irq.trig_mode = icr & APIC_INT_LEVELTRIG;
 840
 841         rcu_read_lock();
 842         map = rcu_dereference(kvm->arch.apic_map);
 843
 844         count = -EOPNOTSUPP;
 845         if (likely(map)) {
 846                 count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min);
 847                 min += cluster_size;
 848                 count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min);
 849         }
 850
 851         rcu_read_unlock();
 852         return count;
 853 }
 854
 855 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
 856 {
 857
 858         return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
 859                                       sizeof(val));
 860 }
 861
 862 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
 863 {
 864
 865         return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
 866                                       sizeof(*val));
 867 }
 868
 869 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
 870 {
 871         return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
 872 }
 873
 874 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
 875 {
 876         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0)
 877                 return;
 878
 879         __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 880 }
 881
 882 static bool pv_eoi_test_and_clr_pending(struct kvm_vcpu *vcpu)
 883 {
 884         u8 val;
 885
 886         if (pv_eoi_get_user(vcpu, &val) < 0)
 887                 return false;
 888
 889         val &= KVM_PV_EOI_ENABLED;
 890
 891         if (val && pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0)
 892                 return false;
 893
 894         /*
 895          * Clear pending bit in any case: it will be set again on vmentry.
 896          * While this might not be ideal from performance point of view,
 897          * this makes sure pv eoi is only enabled when we know it's safe.
 898          */
 899         __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 900
 901         return val;
 902 }
 903
 904 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
 905 {
 906         int highest_irr;
 907         if (kvm_x86_ops.sync_pir_to_irr)
 908                 highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu);
 909         else
 910                 highest_irr = apic_find_highest_irr(apic);
 911         if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
 912                 return -1;
 913         return highest_irr;
 914 }
 915
 916 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
 917 {
 918         u32 tpr, isrv, ppr, old_ppr;
 919         int isr;
 920
 921         old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
 922         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
 923         isr = apic_find_highest_isr(apic);
 924         isrv = (isr != -1) ? isr : 0;
 925
 926         if ((tpr & 0xf0) >= (isrv & 0xf0))
 927                 ppr = tpr & 0xff;
 928         else
 929                 ppr = isrv & 0xf0;
 930
 931         *new_ppr = ppr;
 932         if (old_ppr != ppr)
 933                 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
 934
 935         return ppr < old_ppr;
 936 }
 937
 938 static void apic_update_ppr(struct kvm_lapic *apic)
 939 {
 940         u32 ppr;
 941
 942         if (__apic_update_ppr(apic, &ppr) &&
 943             apic_has_interrupt_for_ppr(apic, ppr) != -1)
 944                 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
 945 }
 946
 947 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
 948 {
 949         apic_update_ppr(vcpu->arch.apic);
 950 }
 951 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
 952
 953 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
 954 {
 955         kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
 956         apic_update_ppr(apic);
 957 }
 958
 959 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
 960 {
 961         return mda == (apic_x2apic_mode(apic) ?
 962                         X2APIC_BROADCAST : APIC_BROADCAST);
 963 }
 964
 965 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
 966 {
 967         if (kvm_apic_broadcast(apic, mda))
 968                 return true;
 969
 970         /*
 971          * Hotplug hack: Accept interrupts for vCPUs in xAPIC mode as if they
 972          * were in x2APIC mode if the target APIC ID can't be encoded as an
 973          * xAPIC ID.  This allows unique addressing of hotplugged vCPUs (which
 974          * start in xAPIC mode) with an APIC ID that is unaddressable in xAPIC
 975          * mode.  Match the x2APIC ID if and only if the target APIC ID can't
 976          * be encoded in xAPIC to avoid spurious matches against a vCPU that
 977          * changed its (addressable) xAPIC ID (which is writable).
 978          */
 979         if (apic_x2apic_mode(apic) || mda > 0xff)
 980                 return mda == kvm_x2apic_id(apic);
 981
 982         return mda == kvm_xapic_id(apic);
 983 }
 984
 985 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
 986 {
 987         u32 logical_id;
 988
 989         if (kvm_apic_broadcast(apic, mda))
 990                 return true;
 991
 992         logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
 993
 994         if (apic_x2apic_mode(apic))
 995                 return ((logical_id >> 16) == (mda >> 16))
 996                        && (logical_id & mda & 0xffff) != 0;
 997
 998         logical_id = GET_APIC_LOGICAL_ID(logical_id);
 999
1000         switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
1001         case APIC_DFR_FLAT:
1002                 return (logical_id & mda) != 0;
1003         case APIC_DFR_CLUSTER:
1004                 return ((logical_id >> 4) == (mda >> 4))
1005                        && (logical_id & mda & 0xf) != 0;
1006         default:
1007                 return false;
1008         }
1009 }
1010
1011 /* The KVM local APIC implementation has two quirks:
1012  *
1013  *  - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
1014  *    in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
1015  *    KVM doesn't do that aliasing.
1016  *
1017  *  - in-kernel IOAPIC messages have to be delivered directly to
1018  *    x2APIC, because the kernel does not support interrupt remapping.
1019  *    In order to support broadcast without interrupt remapping, x2APIC
1020  *    rewrites the destination of non-IPI messages from APIC_BROADCAST
1021  *    to X2APIC_BROADCAST.
1022  *
1023  * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
1024  * important when userspace wants to use x2APIC-format MSIs, because
1025  * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
1026  */
1027 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
1028                 struct kvm_lapic *source, struct kvm_lapic *target)
1029 {
1030         bool ipi = source != NULL;
1031
1032         if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
1033             !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
1034                 return X2APIC_BROADCAST;
1035
1036         return dest_id;
1037 }
1038
1039 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
1040                            int shorthand, unsigned int dest, int dest_mode)
1041 {
1042         struct kvm_lapic *target = vcpu->arch.apic;
1043         u32 mda = kvm_apic_mda(vcpu, dest, source, target);
1044
1045         ASSERT(target);
1046         switch (shorthand) {
1047         case APIC_DEST_NOSHORT:
1048                 if (dest_mode == APIC_DEST_PHYSICAL)
1049                         return kvm_apic_match_physical_addr(target, mda);
1050                 else
1051                         return kvm_apic_match_logical_addr(target, mda);
1052         case APIC_DEST_SELF:
1053                 return target == source;
1054         case APIC_DEST_ALLINC:
1055                 return true;
1056         case APIC_DEST_ALLBUT:
1057                 return target != source;
1058         default:
1059                 return false;
1060         }
1061 }
1062 EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
1063
1064 int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
1065                        const unsigned long *bitmap, u32 bitmap_size)
1066 {
1067         u32 mod;
1068         int i, idx = -1;
1069
1070         mod = vector % dest_vcpus;
1071
1072         for (i = 0; i <= mod; i++) {
1073                 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
1074                 BUG_ON(idx == bitmap_size);
1075         }
1076
1077         return idx;
1078 }
1079
1080 static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
1081 {
1082         if (!kvm->arch.disabled_lapic_found) {
1083                 kvm->arch.disabled_lapic_found = true;
1084                 pr_info("Disabled LAPIC found during irq injection\n");
1085         }
1086 }
1087
1088 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
1089                 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
1090 {
1091         if (kvm->arch.x2apic_broadcast_quirk_disabled) {
1092                 if ((irq->dest_id == APIC_BROADCAST &&
1093                      map->logical_mode != KVM_APIC_MODE_X2APIC))
1094                         return true;
1095                 if (irq->dest_id == X2APIC_BROADCAST)
1096                         return true;
1097         } else {
1098                 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
1099                 if (irq->dest_id == (x2apic_ipi ?
1100                                      X2APIC_BROADCAST : APIC_BROADCAST))
1101                         return true;
1102         }
1103
1104         return false;
1105 }
1106
1107 /* Return true if the interrupt can be handled by using *bitmap as index mask
1108  * for valid destinations in *dst array.
1109  * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
1110  * Note: we may have zero kvm_lapic destinations when we return true, which
1111  * means that the interrupt should be dropped.  In this case, *bitmap would be
1112  * zero and *dst undefined.
1113  */
1114 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
1115                 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
1116                 struct kvm_apic_map *map, struct kvm_lapic ***dst,
1117                 unsigned long *bitmap)
1118 {
1119         int i, lowest;
1120
1121         if (irq->shorthand == APIC_DEST_SELF && src) {
1122                 *dst = src;
1123                 *bitmap = 1;
1124                 return true;
1125         } else if (irq->shorthand)
1126                 return false;
1127
1128         if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
1129                 return false;
1130
1131         if (irq->dest_mode == APIC_DEST_PHYSICAL) {
1132                 if (irq->dest_id > map->max_apic_id) {
1133                         *bitmap = 0;
1134                 } else {
1135                         u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
1136                         *dst = &map->phys_map[dest_id];
1137                         *bitmap = 1;
1138                 }
1139                 return true;
1140         }
1141
1142         *bitmap = 0;
1143         if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
1144                                 (u16 *)bitmap))
1145                 return false;
1146
1147         if (!kvm_lowest_prio_delivery(irq))
1148                 return true;
1149
1150         if (!kvm_vector_hashing_enabled()) {
1151                 lowest = -1;
1152                 for_each_set_bit(i, bitmap, 16) {
1153                         if (!(*dst)[i])
1154                                 continue;
1155                         if (lowest < 0)
1156                                 lowest = i;
1157                         else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
1158                                                 (*dst)[lowest]->vcpu) < 0)
1159                                 lowest = i;
1160                 }
1161         } else {
1162                 if (!*bitmap)
1163                         return true;
1164
1165                 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
1166                                 bitmap, 16);
1167
1168                 if (!(*dst)[lowest]) {
1169                         kvm_apic_disabled_lapic_found(kvm);
1170                         *bitmap = 0;
1171                         return true;
1172                 }
1173         }
1174
1175         *bitmap = (lowest >= 0) ? 1 << lowest : 0;
1176
1177         return true;
1178 }
1179
1180 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
1181                 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
1182 {
1183         struct kvm_apic_map *map;
1184         unsigned long bitmap;
1185         struct kvm_lapic **dst = NULL;
1186         int i;
1187         bool ret;
1188
1189         *r = -1;
1190
1191         if (irq->shorthand == APIC_DEST_SELF) {
1192                 if (KVM_BUG_ON(!src, kvm)) {
1193                         *r = 0;
1194                         return true;
1195                 }
1196                 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
1197                 return true;
1198         }
1199
1200         rcu_read_lock();
1201         map = rcu_dereference(kvm->arch.apic_map);
1202
1203         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
1204         if (ret) {
1205                 *r = 0;
1206                 for_each_set_bit(i, &bitmap, 16) {
1207                         if (!dst[i])
1208                                 continue;
1209                         *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
1210                 }
1211         }
1212
1213         rcu_read_unlock();
1214         return ret;
1215 }
1216
1217 /*
1218  * This routine tries to handle interrupts in posted mode, here is how
1219  * it deals with different cases:
1220  * - For single-destination interrupts, handle it in posted mode
1221  * - Else if vector hashing is enabled and it is a lowest-priority
1222  *   interrupt, handle it in posted mode and use the following mechanism
1223  *   to find the destination vCPU.
1224  *      1. For lowest-priority interrupts, store all the possible
1225  *         destination vCPUs in an array.
1226  *      2. Use "guest vector % max number of destination vCPUs" to find
1227  *         the right destination vCPU in the array for the lowest-priority
1228  *         interrupt.
1229  * - Otherwise, use remapped mode to inject the interrupt.
1230  */
1231 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
1232                         struct kvm_vcpu **dest_vcpu)
1233 {
1234         struct kvm_apic_map *map;
1235         unsigned long bitmap;
1236         struct kvm_lapic **dst = NULL;
1237         bool ret = false;
1238
1239         if (irq->shorthand)
1240                 return false;
1241
1242         rcu_read_lock();
1243         map = rcu_dereference(kvm->arch.apic_map);
1244
1245         if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
1246                         hweight16(bitmap) == 1) {
1247                 unsigned long i = find_first_bit(&bitmap, 16);
1248
1249                 if (dst[i]) {
1250                         *dest_vcpu = dst[i]->vcpu;
1251                         ret = true;
1252                 }
1253         }
1254
1255         rcu_read_unlock();
1256         return ret;
1257 }
1258
1259 /*
1260  * Add a pending IRQ into lapic.
1261  * Return 1 if successfully added and 0 if discarded.
1262  */
1263 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
1264                              int vector, int level, int trig_mode,
1265                              struct dest_map *dest_map)
1266 {
1267         int result = 0;
1268         struct kvm_vcpu *vcpu = apic->vcpu;
1269
1270         trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
1271                                   trig_mode, vector);
1272         switch (delivery_mode) {
1273         case APIC_DM_LOWEST:
1274                 vcpu->arch.apic_arb_prio++;
1275                 fallthrough;
1276         case APIC_DM_FIXED:
1277                 if (unlikely(trig_mode && !level))
1278                         break;
1279
1280                 /* FIXME add logic for vcpu on reset */
1281                 if (unlikely(!apic_enabled(apic)))
1282                         break;
1283
1284                 result = 1;
1285
1286                 if (dest_map) {
1287                         __set_bit(vcpu->vcpu_id, dest_map->map);
1288                         dest_map->vectors[vcpu->vcpu_id] = vector;
1289                 }
1290
1291                 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
1292                         if (trig_mode)
1293                                 kvm_lapic_set_vector(vector,
1294                                                      apic->regs + APIC_TMR);
1295                         else
1296                                 kvm_lapic_clear_vector(vector,
1297                                                        apic->regs + APIC_TMR);
1298                 }
1299
1300                 static_call(kvm_x86_deliver_interrupt)(apic, delivery_mode,
1301                                                        trig_mode, vector);
1302                 break;
1303
1304         case APIC_DM_REMRD:
1305                 result = 1;
1306                 vcpu->arch.pv.pv_unhalted = 1;
1307                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1308                 kvm_vcpu_kick(vcpu);
1309                 break;
1310
1311         case APIC_DM_SMI:
1312                 if (!kvm_inject_smi(vcpu)) {
1313                         kvm_vcpu_kick(vcpu);
1314                         result = 1;
1315                 }
1316                 break;
1317
1318         case APIC_DM_NMI:
1319                 result = 1;
1320                 kvm_inject_nmi(vcpu);
1321                 kvm_vcpu_kick(vcpu);
1322                 break;
1323
1324         case APIC_DM_INIT:
1325                 if (!trig_mode || level) {
1326                         result = 1;
1327                         /* assumes that there are only KVM_APIC_INIT/SIPI */
1328                         apic->pending_events = (1UL << KVM_APIC_INIT);
1329                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1330                         kvm_vcpu_kick(vcpu);
1331                 }
1332                 break;
1333
1334         case APIC_DM_STARTUP:
1335                 result = 1;
1336                 apic->sipi_vector = vector;
1337                 /* make sure sipi_vector is visible for the receiver */
1338                 smp_wmb();
1339                 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1340                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1341                 kvm_vcpu_kick(vcpu);
1342                 break;
1343
1344         case APIC_DM_EXTINT:
1345                 /*
1346                  * Should only be called by kvm_apic_local_deliver() with LVT0,
1347                  * before NMI watchdog was enabled. Already handled by
1348                  * kvm_apic_accept_pic_intr().
1349                  */
1350                 break;
1351
1352         default:
1353                 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1354                        delivery_mode);
1355                 break;
1356         }
1357         return result;
1358 }
1359
1360 /*
1361  * This routine identifies the destination vcpus mask meant to receive the
1362  * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find
1363  * out the destination vcpus array and set the bitmap or it traverses to
1364  * each available vcpu to identify the same.
1365  */
1366 void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq,
1367                               unsigned long *vcpu_bitmap)
1368 {
1369         struct kvm_lapic **dest_vcpu = NULL;
1370         struct kvm_lapic *src = NULL;
1371         struct kvm_apic_map *map;
1372         struct kvm_vcpu *vcpu;
1373         unsigned long bitmap, i;
1374         int vcpu_idx;
1375         bool ret;
1376
1377         rcu_read_lock();
1378         map = rcu_dereference(kvm->arch.apic_map);
1379
1380         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu,
1381                                           &bitmap);
1382         if (ret) {
1383                 for_each_set_bit(i, &bitmap, 16) {
1384                         if (!dest_vcpu[i])
1385                                 continue;
1386                         vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx;
1387                         __set_bit(vcpu_idx, vcpu_bitmap);
1388                 }
1389         } else {
1390                 kvm_for_each_vcpu(i, vcpu, kvm) {
1391                         if (!kvm_apic_present(vcpu))
1392                                 continue;
1393                         if (!kvm_apic_match_dest(vcpu, NULL,
1394                                                  irq->shorthand,
1395                                                  irq->dest_id,
1396                                                  irq->dest_mode))
1397                                 continue;
1398                         __set_bit(i, vcpu_bitmap);
1399                 }
1400         }
1401         rcu_read_unlock();
1402 }
1403
1404 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1405 {
1406         return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1407 }
1408
1409 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1410 {
1411         return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1412 }
1413
1414 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1415 {
1416         int trigger_mode;
1417
1418         /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1419         if (!kvm_ioapic_handles_vector(apic, vector))
1420                 return;
1421
1422         /* Request a KVM exit to inform the userspace IOAPIC. */
1423         if (irqchip_split(apic->vcpu->kvm)) {
1424                 apic->vcpu->arch.pending_ioapic_eoi = vector;
1425                 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1426                 return;
1427         }
1428
1429         if (apic_test_vector(vector, apic->regs + APIC_TMR))
1430                 trigger_mode = IOAPIC_LEVEL_TRIG;
1431         else
1432                 trigger_mode = IOAPIC_EDGE_TRIG;
1433
1434         kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1435 }
1436
1437 static int apic_set_eoi(struct kvm_lapic *apic)
1438 {
1439         int vector = apic_find_highest_isr(apic);
1440
1441         trace_kvm_eoi(apic, vector);
1442
1443         /*
1444          * Not every write EOI will has corresponding ISR,
1445          * one example is when Kernel check timer on setup_IO_APIC
1446          */
1447         if (vector == -1)
1448                 return vector;
1449
1450         apic_clear_isr(vector, apic);
1451         apic_update_ppr(apic);
1452
1453         if (to_hv_vcpu(apic->vcpu) &&
1454             test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap))
1455                 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1456
1457         kvm_ioapic_send_eoi(apic, vector);
1458         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1459         return vector;
1460 }
1461
1462 /*
1463  * this interface assumes a trap-like exit, which has already finished
1464  * desired side effect including vISR and vPPR update.
1465  */
1466 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1467 {
1468         struct kvm_lapic *apic = vcpu->arch.apic;
1469
1470         trace_kvm_eoi(apic, vector);
1471
1472         kvm_ioapic_send_eoi(apic, vector);
1473         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1474 }
1475 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1476
1477 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
1478 {
1479         struct kvm_lapic_irq irq;
1480
1481         /* KVM has no delay and should always clear the BUSY/PENDING flag. */
1482         WARN_ON_ONCE(icr_low & APIC_ICR_BUSY);
1483
1484         irq.vector = icr_low & APIC_VECTOR_MASK;
1485         irq.delivery_mode = icr_low & APIC_MODE_MASK;
1486         irq.dest_mode = icr_low & APIC_DEST_MASK;
1487         irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1488         irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1489         irq.shorthand = icr_low & APIC_SHORT_MASK;
1490         irq.msi_redir_hint = false;
1491         if (apic_x2apic_mode(apic))
1492                 irq.dest_id = icr_high;
1493         else
1494                 irq.dest_id = GET_XAPIC_DEST_FIELD(icr_high);
1495
1496         trace_kvm_apic_ipi(icr_low, irq.dest_id);
1497
1498         kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
1499 }
1500 EXPORT_SYMBOL_GPL(kvm_apic_send_ipi);
1501
1502 static u32 apic_get_tmcct(struct kvm_lapic *apic)
1503 {
1504         ktime_t remaining, now;
1505         s64 ns;
1506
1507         ASSERT(apic != NULL);
1508
1509         /* if initial count is 0, current count should also be 0 */
1510         if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1511                 apic->lapic_timer.period == 0)
1512                 return 0;
1513
1514         now = ktime_get();
1515         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1516         if (ktime_to_ns(remaining) < 0)
1517                 remaining = 0;
1518
1519         ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1520         return div64_u64(ns, (APIC_BUS_CYCLE_NS * apic->divide_count));
1521 }
1522
1523 static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1524 {
1525         struct kvm_vcpu *vcpu = apic->vcpu;
1526         struct kvm_run *run = vcpu->run;
1527
1528         kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1529         run->tpr_access.rip = kvm_rip_read(vcpu);
1530         run->tpr_access.is_write = write;
1531 }
1532
1533 static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1534 {
1535         if (apic->vcpu->arch.tpr_access_reporting)
1536                 __report_tpr_access(apic, write);
1537 }
1538
1539 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1540 {
1541         u32 val = 0;
1542
1543         if (offset >= LAPIC_MMIO_LENGTH)
1544                 return 0;
1545
1546         switch (offset) {
1547         case APIC_ARBPRI:
1548                 break;
1549
1550         case APIC_TMCCT:        /* Timer CCR */
1551                 if (apic_lvtt_tscdeadline(apic))
1552                         return 0;
1553
1554                 val = apic_get_tmcct(apic);
1555                 break;
1556         case APIC_PROCPRI:
1557                 apic_update_ppr(apic);
1558                 val = kvm_lapic_get_reg(apic, offset);
1559                 break;
1560         case APIC_TASKPRI:
1561                 report_tpr_access(apic, false);
1562                 fallthrough;
1563         default:
1564                 val = kvm_lapic_get_reg(apic, offset);
1565                 break;
1566         }
1567
1568         return val;
1569 }
1570
1571 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1572 {
1573         return container_of(dev, struct kvm_lapic, dev);
1574 }
1575
1576 #define APIC_REG_MASK(reg)      (1ull << ((reg) >> 4))
1577 #define APIC_REGS_MASK(first, count) \
1578         (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1579
1580 u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic)
1581 {
1582         /* Leave bits '0' for reserved and write-only registers. */
1583         u64 valid_reg_mask =
1584                 APIC_REG_MASK(APIC_ID) |
1585                 APIC_REG_MASK(APIC_LVR) |
1586                 APIC_REG_MASK(APIC_TASKPRI) |
1587                 APIC_REG_MASK(APIC_PROCPRI) |
1588                 APIC_REG_MASK(APIC_LDR) |
1589                 APIC_REG_MASK(APIC_SPIV) |
1590                 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
1591                 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
1592                 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
1593                 APIC_REG_MASK(APIC_ESR) |
1594                 APIC_REG_MASK(APIC_ICR) |
1595                 APIC_REG_MASK(APIC_LVTT) |
1596                 APIC_REG_MASK(APIC_LVTTHMR) |
1597                 APIC_REG_MASK(APIC_LVTPC) |
1598                 APIC_REG_MASK(APIC_LVT0) |
1599                 APIC_REG_MASK(APIC_LVT1) |
1600                 APIC_REG_MASK(APIC_LVTERR) |
1601                 APIC_REG_MASK(APIC_TMICT) |
1602                 APIC_REG_MASK(APIC_TMCCT) |
1603                 APIC_REG_MASK(APIC_TDCR);
1604
1605         if (kvm_lapic_lvt_supported(apic, LVT_CMCI))
1606                 valid_reg_mask |= APIC_REG_MASK(APIC_LVTCMCI);
1607
1608         /* ARBPRI, DFR, and ICR2 are not valid in x2APIC mode. */
1609         if (!apic_x2apic_mode(apic))
1610                 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI) |
1611                                   APIC_REG_MASK(APIC_DFR) |
1612                                   APIC_REG_MASK(APIC_ICR2);
1613
1614         return valid_reg_mask;
1615 }
1616 EXPORT_SYMBOL_GPL(kvm_lapic_readable_reg_mask);
1617
1618 static int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
1619                               void *data)
1620 {
1621         unsigned char alignment = offset & 0xf;
1622         u32 result;
1623
1624         /*
1625          * WARN if KVM reads ICR in x2APIC mode, as it's an 8-byte register in
1626          * x2APIC and needs to be manually handled by the caller.
1627          */
1628         WARN_ON_ONCE(apic_x2apic_mode(apic) && offset == APIC_ICR);
1629
1630         if (alignment + len > 4)
1631                 return 1;
1632
1633         if (offset > 0x3f0 ||
1634             !(kvm_lapic_readable_reg_mask(apic) & APIC_REG_MASK(offset)))
1635                 return 1;
1636
1637         result = __apic_read(apic, offset & ~0xf);
1638
1639         trace_kvm_apic_read(offset, result);
1640
1641         switch (len) {
1642         case 1:
1643         case 2:
1644         case 4:
1645                 memcpy(data, (char *)&result + alignment, len);
1646                 break;
1647         default:
1648                 printk(KERN_ERR "Local APIC read with len = %x, "
1649                        "should be 1,2, or 4 instead\n", len);
1650                 break;
1651         }
1652         return 0;
1653 }
1654
1655 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1656 {
1657         return addr >= apic->base_address &&
1658                 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1659 }
1660
1661 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1662                            gpa_t address, int len, void *data)
1663 {
1664         struct kvm_lapic *apic = to_lapic(this);
1665         u32 offset = address - apic->base_address;
1666
1667         if (!apic_mmio_in_range(apic, address))
1668                 return -EOPNOTSUPP;
1669
1670         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1671                 if (!kvm_check_has_quirk(vcpu->kvm,
1672                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1673                         return -EOPNOTSUPP;
1674
1675                 memset(data, 0xff, len);
1676                 return 0;
1677         }
1678
1679         kvm_lapic_reg_read(apic, offset, len, data);
1680
1681         return 0;
1682 }
1683
1684 static void update_divide_count(struct kvm_lapic *apic)
1685 {
1686         u32 tmp1, tmp2, tdcr;
1687
1688         tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
1689         tmp1 = tdcr & 0xf;
1690         tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1691         apic->divide_count = 0x1 << (tmp2 & 0x7);
1692 }
1693
1694 static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1695 {
1696         /*
1697          * Do not allow the guest to program periodic timers with small
1698          * interval, since the hrtimers are not throttled by the host
1699          * scheduler.
1700          */
1701         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1702                 s64 min_period = min_timer_period_us * 1000LL;
1703
1704                 if (apic->lapic_timer.period < min_period) {
1705                         pr_info_ratelimited(
1706                             "vcpu %i: requested %lld ns "
1707                             "lapic timer period limited to %lld ns\n",
1708                             apic->vcpu->vcpu_id,
1709                             apic->lapic_timer.period, min_period);
1710                         apic->lapic_timer.period = min_period;
1711                 }
1712         }
1713 }
1714
1715 static void cancel_hv_timer(struct kvm_lapic *apic);
1716
1717 static void cancel_apic_timer(struct kvm_lapic *apic)
1718 {
1719         hrtimer_cancel(&apic->lapic_timer.timer);
1720         preempt_disable();
1721         if (apic->lapic_timer.hv_timer_in_use)
1722                 cancel_hv_timer(apic);
1723         preempt_enable();
1724         atomic_set(&apic->lapic_timer.pending, 0);
1725 }
1726
1727 static void apic_update_lvtt(struct kvm_lapic *apic)
1728 {
1729         u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1730                         apic->lapic_timer.timer_mode_mask;
1731
1732         if (apic->lapic_timer.timer_mode != timer_mode) {
1733                 if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1734                                 APIC_LVT_TIMER_TSCDEADLINE)) {
1735                         cancel_apic_timer(apic);
1736                         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
1737                         apic->lapic_timer.period = 0;
1738                         apic->lapic_timer.tscdeadline = 0;
1739                 }
1740                 apic->lapic_timer.timer_mode = timer_mode;
1741                 limit_periodic_timer_frequency(apic);
1742         }
1743 }
1744
1745 /*
1746  * On APICv, this test will cause a busy wait
1747  * during a higher-priority task.
1748  */
1749
1750 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1751 {
1752         struct kvm_lapic *apic = vcpu->arch.apic;
1753         u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1754
1755         if (kvm_apic_hw_enabled(apic)) {
1756                 int vec = reg & APIC_VECTOR_MASK;
1757                 void *bitmap = apic->regs + APIC_ISR;
1758
1759                 if (apic->apicv_active)
1760                         bitmap = apic->regs + APIC_IRR;
1761
1762                 if (apic_test_vector(vec, bitmap))
1763                         return true;
1764         }
1765         return false;
1766 }
1767
1768 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
1769 {
1770         u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
1771
1772         /*
1773          * If the guest TSC is running at a different ratio than the host, then
1774          * convert the delay to nanoseconds to achieve an accurate delay.  Note
1775          * that __delay() uses delay_tsc whenever the hardware has TSC, thus
1776          * always for VMX enabled hardware.
1777          */
1778         if (vcpu->arch.tsc_scaling_ratio == kvm_caps.default_tsc_scaling_ratio) {
1779                 __delay(min(guest_cycles,
1780                         nsec_to_cycles(vcpu, timer_advance_ns)));
1781         } else {
1782                 u64 delay_ns = guest_cycles * 1000000ULL;
1783                 do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
1784                 ndelay(min_t(u32, delay_ns, timer_advance_ns));
1785         }
1786 }
1787
1788 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
1789                                               s64 advance_expire_delta)
1790 {
1791         struct kvm_lapic *apic = vcpu->arch.apic;
1792         u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
1793         u64 ns;
1794
1795         /* Do not adjust for tiny fluctuations or large random spikes. */
1796         if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX ||
1797             abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN)
1798                 return;
1799
1800         /* too early */
1801         if (advance_expire_delta < 0) {
1802                 ns = -advance_expire_delta * 1000000ULL;
1803                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1804                 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1805         } else {
1806         /* too late */
1807                 ns = advance_expire_delta * 1000000ULL;
1808                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1809                 timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1810         }
1811
1812         if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX))
1813                 timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
1814         apic->lapic_timer.timer_advance_ns = timer_advance_ns;
1815 }
1816
1817 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1818 {
1819         struct kvm_lapic *apic = vcpu->arch.apic;
1820         u64 guest_tsc, tsc_deadline;
1821
1822         tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1823         apic->lapic_timer.expired_tscdeadline = 0;
1824         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1825         trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
1826
1827         if (lapic_timer_advance_dynamic) {
1828                 adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline);
1829                 /*
1830                  * If the timer fired early, reread the TSC to account for the
1831                  * overhead of the above adjustment to avoid waiting longer
1832                  * than is necessary.
1833                  */
1834                 if (guest_tsc < tsc_deadline)
1835                         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1836         }
1837
1838         if (guest_tsc < tsc_deadline)
1839                 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
1840 }
1841
1842 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1843 {
1844         if (lapic_in_kernel(vcpu) &&
1845             vcpu->arch.apic->lapic_timer.expired_tscdeadline &&
1846             vcpu->arch.apic->lapic_timer.timer_advance_ns &&
1847             lapic_timer_int_injected(vcpu))
1848                 __kvm_wait_lapic_expire(vcpu);
1849 }
1850 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
1851
1852 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
1853 {
1854         struct kvm_timer *ktimer = &apic->lapic_timer;
1855
1856         kvm_apic_local_deliver(apic, APIC_LVTT);
1857         if (apic_lvtt_tscdeadline(apic)) {
1858                 ktimer->tscdeadline = 0;
1859         } else if (apic_lvtt_oneshot(apic)) {
1860                 ktimer->tscdeadline = 0;
1861                 ktimer->target_expiration = 0;
1862         }
1863 }
1864
1865 static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn)
1866 {
1867         struct kvm_vcpu *vcpu = apic->vcpu;
1868         struct kvm_timer *ktimer = &apic->lapic_timer;
1869
1870         if (atomic_read(&apic->lapic_timer.pending))
1871                 return;
1872
1873         if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
1874                 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1875
1876         if (!from_timer_fn && apic->apicv_active) {
1877                 WARN_ON(kvm_get_running_vcpu() != vcpu);
1878                 kvm_apic_inject_pending_timer_irqs(apic);
1879                 return;
1880         }
1881
1882         if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
1883                 /*
1884                  * Ensure the guest's timer has truly expired before posting an
1885                  * interrupt.  Open code the relevant checks to avoid querying
1886                  * lapic_timer_int_injected(), which will be false since the
1887                  * interrupt isn't yet injected.  Waiting until after injecting
1888                  * is not an option since that won't help a posted interrupt.
1889                  */
1890                 if (vcpu->arch.apic->lapic_timer.expired_tscdeadline &&
1891                     vcpu->arch.apic->lapic_timer.timer_advance_ns)
1892                         __kvm_wait_lapic_expire(vcpu);
1893                 kvm_apic_inject_pending_timer_irqs(apic);
1894                 return;
1895         }
1896
1897         atomic_inc(&apic->lapic_timer.pending);
1898         kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1899         if (from_timer_fn)
1900                 kvm_vcpu_kick(vcpu);
1901 }
1902
1903 static void start_sw_tscdeadline(struct kvm_lapic *apic)
1904 {
1905         struct kvm_timer *ktimer = &apic->lapic_timer;
1906         u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
1907         u64 ns = 0;
1908         ktime_t expire;
1909         struct kvm_vcpu *vcpu = apic->vcpu;
1910         unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1911         unsigned long flags;
1912         ktime_t now;
1913
1914         if (unlikely(!tscdeadline || !this_tsc_khz))
1915                 return;
1916
1917         local_irq_save(flags);
1918
1919         now = ktime_get();
1920         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1921
1922         ns = (tscdeadline - guest_tsc) * 1000000ULL;
1923         do_div(ns, this_tsc_khz);
1924
1925         if (likely(tscdeadline > guest_tsc) &&
1926             likely(ns > apic->lapic_timer.timer_advance_ns)) {
1927                 expire = ktime_add_ns(now, ns);
1928                 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
1929                 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
1930         } else
1931                 apic_timer_expired(apic, false);
1932
1933         local_irq_restore(flags);
1934 }
1935
1936 static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict)
1937 {
1938         return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count;
1939 }
1940
1941 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1942 {
1943         ktime_t now, remaining;
1944         u64 ns_remaining_old, ns_remaining_new;
1945
1946         apic->lapic_timer.period =
1947                         tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
1948         limit_periodic_timer_frequency(apic);
1949
1950         now = ktime_get();
1951         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1952         if (ktime_to_ns(remaining) < 0)
1953                 remaining = 0;
1954
1955         ns_remaining_old = ktime_to_ns(remaining);
1956         ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1957                                            apic->divide_count, old_divisor);
1958
1959         apic->lapic_timer.tscdeadline +=
1960                 nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1961                 nsec_to_cycles(apic->vcpu, ns_remaining_old);
1962         apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1963 }
1964
1965 static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg)
1966 {
1967         ktime_t now;
1968         u64 tscl = rdtsc();
1969         s64 deadline;
1970
1971         now = ktime_get();
1972         apic->lapic_timer.period =
1973                         tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
1974
1975         if (!apic->lapic_timer.period) {
1976                 apic->lapic_timer.tscdeadline = 0;
1977                 return false;
1978         }
1979
1980         limit_periodic_timer_frequency(apic);
1981         deadline = apic->lapic_timer.period;
1982
1983         if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
1984                 if (unlikely(count_reg != APIC_TMICT)) {
1985                         deadline = tmict_to_ns(apic,
1986                                      kvm_lapic_get_reg(apic, count_reg));
1987                         if (unlikely(deadline <= 0)) {
1988                                 if (apic_lvtt_period(apic))
1989                                         deadline = apic->lapic_timer.period;
1990                                 else
1991                                         deadline = 0;
1992                         }
1993                         else if (unlikely(deadline > apic->lapic_timer.period)) {
1994                                 pr_info_ratelimited(
1995                                     "vcpu %i: requested lapic timer restore with "
1996                                     "starting count register %#x=%u (%lld ns) > initial count (%lld ns). "
1997                                     "Using initial count to start timer.\n",
1998                                     apic->vcpu->vcpu_id,
1999                                     count_reg,
2000                                     kvm_lapic_get_reg(apic, count_reg),
2001                                     deadline, apic->lapic_timer.period);
2002                                 kvm_lapic_set_reg(apic, count_reg, 0);
2003                                 deadline = apic->lapic_timer.period;
2004                         }
2005                 }
2006         }
2007
2008         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
2009                 nsec_to_cycles(apic->vcpu, deadline);
2010         apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline);
2011
2012         return true;
2013 }
2014
2015 static void advance_periodic_target_expiration(struct kvm_lapic *apic)
2016 {
2017         ktime_t now = ktime_get();
2018         u64 tscl = rdtsc();
2019         ktime_t delta;
2020
2021         /*
2022          * Synchronize both deadlines to the same time source or
2023          * differences in the periods (caused by differences in the
2024          * underlying clocks or numerical approximation errors) will
2025          * cause the two to drift apart over time as the errors
2026          * accumulate.
2027          */
2028         apic->lapic_timer.target_expiration =
2029                 ktime_add_ns(apic->lapic_timer.target_expiration,
2030                                 apic->lapic_timer.period);
2031         delta = ktime_sub(apic->lapic_timer.target_expiration, now);
2032         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
2033                 nsec_to_cycles(apic->vcpu, delta);
2034 }
2035
2036 static void start_sw_period(struct kvm_lapic *apic)
2037 {
2038         if (!apic->lapic_timer.period)
2039                 return;
2040
2041         if (ktime_after(ktime_get(),
2042                         apic->lapic_timer.target_expiration)) {
2043                 apic_timer_expired(apic, false);
2044
2045                 if (apic_lvtt_oneshot(apic))
2046                         return;
2047
2048                 advance_periodic_target_expiration(apic);
2049         }
2050
2051         hrtimer_start(&apic->lapic_timer.timer,
2052                 apic->lapic_timer.target_expiration,
2053                 HRTIMER_MODE_ABS_HARD);
2054 }
2055
2056 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
2057 {
2058         if (!lapic_in_kernel(vcpu))
2059                 return false;
2060
2061         return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
2062 }
2063
2064 static void cancel_hv_timer(struct kvm_lapic *apic)
2065 {
2066         WARN_ON(preemptible());
2067         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
2068         static_call(kvm_x86_cancel_hv_timer)(apic->vcpu);
2069         apic->lapic_timer.hv_timer_in_use = false;
2070 }
2071
2072 static bool start_hv_timer(struct kvm_lapic *apic)
2073 {
2074         struct kvm_timer *ktimer = &apic->lapic_timer;
2075         struct kvm_vcpu *vcpu = apic->vcpu;
2076         bool expired;
2077
2078         WARN_ON(preemptible());
2079         if (!kvm_can_use_hv_timer(vcpu))
2080                 return false;
2081
2082         if (!ktimer->tscdeadline)
2083                 return false;
2084
2085         if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
2086                 return false;
2087
2088         ktimer->hv_timer_in_use = true;
2089         hrtimer_cancel(&ktimer->timer);
2090
2091         /*
2092          * To simplify handling the periodic timer, leave the hv timer running
2093          * even if the deadline timer has expired, i.e. rely on the resulting
2094          * VM-Exit to recompute the periodic timer's target expiration.
2095          */
2096         if (!apic_lvtt_period(apic)) {
2097                 /*
2098                  * Cancel the hv timer if the sw timer fired while the hv timer
2099                  * was being programmed, or if the hv timer itself expired.
2100                  */
2101                 if (atomic_read(&ktimer->pending)) {
2102                         cancel_hv_timer(apic);
2103                 } else if (expired) {
2104                         apic_timer_expired(apic, false);
2105                         cancel_hv_timer(apic);
2106                 }
2107         }
2108
2109         trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
2110
2111         return true;
2112 }
2113
2114 static void start_sw_timer(struct kvm_lapic *apic)
2115 {
2116         struct kvm_timer *ktimer = &apic->lapic_timer;
2117
2118         WARN_ON(preemptible());
2119         if (apic->lapic_timer.hv_timer_in_use)
2120                 cancel_hv_timer(apic);
2121         if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
2122                 return;
2123
2124         if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
2125                 start_sw_period(apic);
2126         else if (apic_lvtt_tscdeadline(apic))
2127                 start_sw_tscdeadline(apic);
2128         trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
2129 }
2130
2131 static void restart_apic_timer(struct kvm_lapic *apic)
2132 {
2133         preempt_disable();
2134
2135         if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
2136                 goto out;
2137
2138         if (!start_hv_timer(apic))
2139                 start_sw_timer(apic);
2140 out:
2141         preempt_enable();
2142 }
2143
2144 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
2145 {
2146         struct kvm_lapic *apic = vcpu->arch.apic;
2147
2148         preempt_disable();
2149         /* If the preempt notifier has already run, it also called apic_timer_expired */
2150         if (!apic->lapic_timer.hv_timer_in_use)
2151                 goto out;
2152         WARN_ON(kvm_vcpu_is_blocking(vcpu));
2153         apic_timer_expired(apic, false);
2154         cancel_hv_timer(apic);
2155
2156         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
2157                 advance_periodic_target_expiration(apic);
2158                 restart_apic_timer(apic);
2159         }
2160 out:
2161         preempt_enable();
2162 }
2163 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
2164
2165 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
2166 {
2167         restart_apic_timer(vcpu->arch.apic);
2168 }
2169
2170 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
2171 {
2172         struct kvm_lapic *apic = vcpu->arch.apic;
2173
2174         preempt_disable();
2175         /* Possibly the TSC deadline timer is not enabled yet */
2176         if (apic->lapic_timer.hv_timer_in_use)
2177                 start_sw_timer(apic);
2178         preempt_enable();
2179 }
2180
2181 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
2182 {
2183         struct kvm_lapic *apic = vcpu->arch.apic;
2184
2185         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
2186         restart_apic_timer(apic);
2187 }
2188
2189 static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg)
2190 {
2191         atomic_set(&apic->lapic_timer.pending, 0);
2192
2193         if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
2194             && !set_target_expiration(apic, count_reg))
2195                 return;
2196
2197         restart_apic_timer(apic);
2198 }
2199
2200 static void start_apic_timer(struct kvm_lapic *apic)
2201 {
2202         __start_apic_timer(apic, APIC_TMICT);
2203 }
2204
2205 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
2206 {
2207         bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
2208
2209         if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
2210                 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
2211                 if (lvt0_in_nmi_mode) {
2212                         atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
2213                 } else
2214                         atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
2215         }
2216 }
2217
2218 static int get_lvt_index(u32 reg)
2219 {
2220         if (reg == APIC_LVTCMCI)
2221                 return LVT_CMCI;
2222         if (reg < APIC_LVTT || reg > APIC_LVTERR)
2223                 return -1;
2224         return array_index_nospec(
2225                         (reg - APIC_LVTT) >> 4, KVM_APIC_MAX_NR_LVT_ENTRIES);
2226 }
2227
2228 static int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
2229 {
2230         int ret = 0;
2231
2232         trace_kvm_apic_write(reg, val);
2233
2234         switch (reg) {
2235         case APIC_ID:           /* Local APIC ID */
2236                 if (!apic_x2apic_mode(apic)) {
2237                         kvm_apic_set_xapic_id(apic, val >> 24);
2238                 } else {
2239                         ret = 1;
2240                 }
2241                 break;
2242
2243         case APIC_TASKPRI:
2244                 report_tpr_access(apic, true);
2245                 apic_set_tpr(apic, val & 0xff);
2246                 break;
2247
2248         case APIC_EOI:
2249                 apic_set_eoi(apic);
2250                 break;
2251
2252         case APIC_LDR:
2253                 if (!apic_x2apic_mode(apic))
2254                         kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
2255                 else
2256                         ret = 1;
2257                 break;
2258
2259         case APIC_DFR:
2260                 if (!apic_x2apic_mode(apic))
2261                         kvm_apic_set_dfr(apic, val | 0x0FFFFFFF);
2262                 else
2263                         ret = 1;
2264                 break;
2265
2266         case APIC_SPIV: {
2267                 u32 mask = 0x3ff;
2268                 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
2269                         mask |= APIC_SPIV_DIRECTED_EOI;
2270                 apic_set_spiv(apic, val & mask);
2271                 if (!(val & APIC_SPIV_APIC_ENABLED)) {
2272                         int i;
2273
2274                         for (i = 0; i < apic->nr_lvt_entries; i++) {
2275                                 kvm_lapic_set_reg(apic, APIC_LVTx(i),
2276                                         kvm_lapic_get_reg(apic, APIC_LVTx(i)) | APIC_LVT_MASKED);
2277                         }
2278                         apic_update_lvtt(apic);
2279                         atomic_set(&apic->lapic_timer.pending, 0);
2280
2281                 }
2282                 break;
2283         }
2284         case APIC_ICR:
2285                 WARN_ON_ONCE(apic_x2apic_mode(apic));
2286
2287                 /* No delay here, so we always clear the pending bit */
2288                 val &= ~APIC_ICR_BUSY;
2289                 kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
2290                 kvm_lapic_set_reg(apic, APIC_ICR, val);
2291                 break;
2292         case APIC_ICR2:
2293                 if (apic_x2apic_mode(apic))
2294                         ret = 1;
2295                 else
2296                         kvm_lapic_set_reg(apic, APIC_ICR2, val & 0xff000000);
2297                 break;
2298
2299         case APIC_LVT0:
2300                 apic_manage_nmi_watchdog(apic, val);
2301                 fallthrough;
2302         case APIC_LVTTHMR:
2303         case APIC_LVTPC:
2304         case APIC_LVT1:
2305         case APIC_LVTERR:
2306         case APIC_LVTCMCI: {
2307                 u32 index = get_lvt_index(reg);
2308                 if (!kvm_lapic_lvt_supported(apic, index)) {
2309                         ret = 1;
2310                         break;
2311                 }
2312                 if (!kvm_apic_sw_enabled(apic))
2313                         val |= APIC_LVT_MASKED;
2314                 val &= apic_lvt_mask[index];
2315                 kvm_lapic_set_reg(apic, reg, val);
2316                 break;
2317         }
2318
2319         case APIC_LVTT:
2320                 if (!kvm_apic_sw_enabled(apic))
2321                         val |= APIC_LVT_MASKED;
2322                 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
2323                 kvm_lapic_set_reg(apic, APIC_LVTT, val);
2324                 apic_update_lvtt(apic);
2325                 break;
2326
2327         case APIC_TMICT:
2328                 if (apic_lvtt_tscdeadline(apic))
2329                         break;
2330
2331                 cancel_apic_timer(apic);
2332                 kvm_lapic_set_reg(apic, APIC_TMICT, val);
2333                 start_apic_timer(apic);
2334                 break;
2335
2336         case APIC_TDCR: {
2337                 uint32_t old_divisor = apic->divide_count;
2338
2339                 kvm_lapic_set_reg(apic, APIC_TDCR, val & 0xb);
2340                 update_divide_count(apic);
2341                 if (apic->divide_count != old_divisor &&
2342                                 apic->lapic_timer.period) {
2343                         hrtimer_cancel(&apic->lapic_timer.timer);
2344                         update_target_expiration(apic, old_divisor);
2345                         restart_apic_timer(apic);
2346                 }
2347                 break;
2348         }
2349         case APIC_ESR:
2350                 if (apic_x2apic_mode(apic) && val != 0)
2351                         ret = 1;
2352                 break;
2353
2354         case APIC_SELF_IPI:
2355                 /*
2356                  * Self-IPI exists only when x2APIC is enabled.  Bits 7:0 hold
2357                  * the vector, everything else is reserved.
2358                  */
2359                 if (!apic_x2apic_mode(apic) || (val & ~APIC_VECTOR_MASK))
2360                         ret = 1;
2361                 else
2362                         kvm_apic_send_ipi(apic, APIC_DEST_SELF | val, 0);
2363                 break;
2364         default:
2365                 ret = 1;
2366                 break;
2367         }
2368
2369         /*
2370          * Recalculate APIC maps if necessary, e.g. if the software enable bit
2371          * was toggled, the APIC ID changed, etc...   The maps are marked dirty
2372          * on relevant changes, i.e. this is a nop for most writes.
2373          */
2374         kvm_recalculate_apic_map(apic->vcpu->kvm);
2375
2376         return ret;
2377 }
2378
2379 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
2380                             gpa_t address, int len, const void *data)
2381 {
2382         struct kvm_lapic *apic = to_lapic(this);
2383         unsigned int offset = address - apic->base_address;
2384         u32 val;
2385
2386         if (!apic_mmio_in_range(apic, address))
2387                 return -EOPNOTSUPP;
2388
2389         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
2390                 if (!kvm_check_has_quirk(vcpu->kvm,
2391                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
2392                         return -EOPNOTSUPP;
2393
2394                 return 0;
2395         }
2396
2397         /*
2398          * APIC register must be aligned on 128-bits boundary.
2399          * 32/64/128 bits registers must be accessed thru 32 bits.
2400          * Refer SDM 8.4.1
2401          */
2402         if (len != 4 || (offset & 0xf))
2403                 return 0;
2404
2405         val = *(u32*)data;
2406
2407         kvm_lapic_reg_write(apic, offset & 0xff0, val);
2408
2409         return 0;
2410 }
2411
2412 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
2413 {
2414         kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
2415 }
2416 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
2417
2418 /* emulate APIC access in a trap manner */
2419 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
2420 {
2421         struct kvm_lapic *apic = vcpu->arch.apic;
2422         u64 val;
2423
2424         /*
2425          * ICR is a single 64-bit register when x2APIC is enabled.  For legacy
2426          * xAPIC, ICR writes need to go down the common (slightly slower) path
2427          * to get the upper half from ICR2.
2428          */
2429         if (apic_x2apic_mode(apic) && offset == APIC_ICR) {
2430                 val = kvm_lapic_get_reg64(apic, APIC_ICR);
2431                 kvm_apic_send_ipi(apic, (u32)val, (u32)(val >> 32));
2432                 trace_kvm_apic_write(APIC_ICR, val);
2433         } else {
2434                 /* TODO: optimize to just emulate side effect w/o one more write */
2435                 val = kvm_lapic_get_reg(apic, offset);
2436                 kvm_lapic_reg_write(apic, offset, (u32)val);
2437         }
2438 }
2439 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
2440
2441 void kvm_free_lapic(struct kvm_vcpu *vcpu)
2442 {
2443         struct kvm_lapic *apic = vcpu->arch.apic;
2444
2445         if (!vcpu->arch.apic)
2446                 return;
2447
2448         hrtimer_cancel(&apic->lapic_timer.timer);
2449
2450         if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2451                 static_branch_slow_dec_deferred(&apic_hw_disabled);
2452
2453         if (!apic->sw_enabled)
2454                 static_branch_slow_dec_deferred(&apic_sw_disabled);
2455
2456         if (apic->regs)
2457                 free_page((unsigned long)apic->regs);
2458
2459         kfree(apic);
2460 }
2461
2462 /*
2463  *----------------------------------------------------------------------
2464  * LAPIC interface
2465  *----------------------------------------------------------------------
2466  */
2467 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
2468 {
2469         struct kvm_lapic *apic = vcpu->arch.apic;
2470
2471         if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic))
2472                 return 0;
2473
2474         return apic->lapic_timer.tscdeadline;
2475 }
2476
2477 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
2478 {
2479         struct kvm_lapic *apic = vcpu->arch.apic;
2480
2481         if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic))
2482                 return;
2483
2484         hrtimer_cancel(&apic->lapic_timer.timer);
2485         apic->lapic_timer.tscdeadline = data;
2486         start_apic_timer(apic);
2487 }
2488
2489 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
2490 {
2491         apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4);
2492 }
2493
2494 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
2495 {
2496         u64 tpr;
2497
2498         tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
2499
2500         return (tpr & 0xf0) >> 4;
2501 }
2502
2503 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
2504 {
2505         u64 old_value = vcpu->arch.apic_base;
2506         struct kvm_lapic *apic = vcpu->arch.apic;
2507
2508         vcpu->arch.apic_base = value;
2509
2510         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
2511                 kvm_update_cpuid_runtime(vcpu);
2512
2513         if (!apic)
2514                 return;
2515
2516         /* update jump label if enable bit changes */
2517         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2518                 if (value & MSR_IA32_APICBASE_ENABLE) {
2519                         kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2520                         static_branch_slow_dec_deferred(&apic_hw_disabled);
2521                         /* Check if there are APF page ready requests pending */
2522                         kvm_make_request(KVM_REQ_APF_READY, vcpu);
2523                 } else {
2524                         static_branch_inc(&apic_hw_disabled.key);
2525                         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
2526                 }
2527         }
2528
2529         if ((old_value ^ value) & X2APIC_ENABLE) {
2530                 if (value & X2APIC_ENABLE)
2531                         kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2532                 else if (value & MSR_IA32_APICBASE_ENABLE)
2533                         kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2534         }
2535
2536         if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) {
2537                 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
2538                 static_call_cond(kvm_x86_set_virtual_apic_mode)(vcpu);
2539         }
2540
2541         apic->base_address = apic->vcpu->arch.apic_base &
2542                              MSR_IA32_APICBASE_BASE;
2543
2544         if ((value & MSR_IA32_APICBASE_ENABLE) &&
2545              apic->base_address != APIC_DEFAULT_PHYS_BASE) {
2546                 kvm_set_apicv_inhibit(apic->vcpu->kvm,
2547                                       APICV_INHIBIT_REASON_APIC_BASE_MODIFIED);
2548         }
2549 }
2550
2551 void kvm_apic_update_apicv(struct kvm_vcpu *vcpu)
2552 {
2553         struct kvm_lapic *apic = vcpu->arch.apic;
2554
2555         if (apic->apicv_active) {
2556                 /* irr_pending is always true when apicv is activated. */
2557                 apic->irr_pending = true;
2558                 apic->isr_count = 1;
2559         } else {
2560                 /*
2561                  * Don't clear irr_pending, searching the IRR can race with
2562                  * updates from the CPU as APICv is still active from hardware's
2563                  * perspective.  The flag will be cleared as appropriate when
2564                  * KVM injects the interrupt.
2565                  */
2566                 apic->isr_count = count_vectors(apic->regs + APIC_ISR);
2567         }
2568         apic->highest_isr_cache = -1;
2569 }
2570
2571 int kvm_alloc_apic_access_page(struct kvm *kvm)
2572 {
2573         struct page *page;
2574         void __user *hva;
2575         int ret = 0;
2576
2577         mutex_lock(&kvm->slots_lock);
2578         if (kvm->arch.apic_access_memslot_enabled ||
2579             kvm->arch.apic_access_memslot_inhibited)
2580                 goto out;
2581
2582         hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
2583                                       APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
2584         if (IS_ERR(hva)) {
2585                 ret = PTR_ERR(hva);
2586                 goto out;
2587         }
2588
2589         page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
2590         if (is_error_page(page)) {
2591                 ret = -EFAULT;
2592                 goto out;
2593         }
2594
2595         /*
2596          * Do not pin the page in memory, so that memory hot-unplug
2597          * is able to migrate it.
2598          */
2599         put_page(page);
2600         kvm->arch.apic_access_memslot_enabled = true;
2601 out:
2602         mutex_unlock(&kvm->slots_lock);
2603         return ret;
2604 }
2605 EXPORT_SYMBOL_GPL(kvm_alloc_apic_access_page);
2606
2607 void kvm_inhibit_apic_access_page(struct kvm_vcpu *vcpu)
2608 {
2609         struct kvm *kvm = vcpu->kvm;
2610
2611         if (!kvm->arch.apic_access_memslot_enabled)
2612                 return;
2613
2614         kvm_vcpu_srcu_read_unlock(vcpu);
2615
2616         mutex_lock(&kvm->slots_lock);
2617
2618         if (kvm->arch.apic_access_memslot_enabled) {
2619                 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0);
2620                 /*
2621                  * Clear "enabled" after the memslot is deleted so that a
2622                  * different vCPU doesn't get a false negative when checking
2623                  * the flag out of slots_lock.  No additional memory barrier is
2624                  * needed as modifying memslots requires waiting other vCPUs to
2625                  * drop SRCU (see above), and false positives are ok as the
2626                  * flag is rechecked after acquiring slots_lock.
2627                  */
2628                 kvm->arch.apic_access_memslot_enabled = false;
2629
2630                 /*
2631                  * Mark the memslot as inhibited to prevent reallocating the
2632                  * memslot during vCPU creation, e.g. if a vCPU is hotplugged.
2633                  */
2634                 kvm->arch.apic_access_memslot_inhibited = true;
2635         }
2636
2637         mutex_unlock(&kvm->slots_lock);
2638
2639         kvm_vcpu_srcu_read_lock(vcpu);
2640 }
2641
2642 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
2643 {
2644         struct kvm_lapic *apic = vcpu->arch.apic;
2645         u64 msr_val;
2646         int i;
2647
2648         if (!init_event) {
2649                 msr_val = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE;
2650                 if (kvm_vcpu_is_reset_bsp(vcpu))
2651                         msr_val |= MSR_IA32_APICBASE_BSP;
2652                 kvm_lapic_set_base(vcpu, msr_val);
2653         }
2654
2655         if (!apic)
2656                 return;
2657
2658         /* Stop the timer in case it's a reset to an active apic */
2659         hrtimer_cancel(&apic->lapic_timer.timer);
2660
2661         /* The xAPIC ID is set at RESET even if the APIC was already enabled. */
2662         if (!init_event)
2663                 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2664         kvm_apic_set_version(apic->vcpu);
2665
2666         for (i = 0; i < apic->nr_lvt_entries; i++)
2667                 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED);
2668         apic_update_lvtt(apic);
2669         if (kvm_vcpu_is_reset_bsp(vcpu) &&
2670             kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2671                 kvm_lapic_set_reg(apic, APIC_LVT0,
2672                              SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2673         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2674
2675         kvm_apic_set_dfr(apic, 0xffffffffU);
2676         apic_set_spiv(apic, 0xff);
2677         kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2678         if (!apic_x2apic_mode(apic))
2679                 kvm_apic_set_ldr(apic, 0);
2680         kvm_lapic_set_reg(apic, APIC_ESR, 0);
2681         if (!apic_x2apic_mode(apic)) {
2682                 kvm_lapic_set_reg(apic, APIC_ICR, 0);
2683                 kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2684         } else {
2685                 kvm_lapic_set_reg64(apic, APIC_ICR, 0);
2686         }
2687         kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2688         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
2689         for (i = 0; i < 8; i++) {
2690                 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2691                 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2692                 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
2693         }
2694         kvm_apic_update_apicv(vcpu);
2695         update_divide_count(apic);
2696         atomic_set(&apic->lapic_timer.pending, 0);
2697
2698         vcpu->arch.pv_eoi.msr_val = 0;
2699         apic_update_ppr(apic);
2700         if (apic->apicv_active) {
2701                 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu);
2702                 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, -1);
2703                 static_call_cond(kvm_x86_hwapic_isr_update)(-1);
2704         }
2705
2706         vcpu->arch.apic_arb_prio = 0;
2707         vcpu->arch.apic_attention = 0;
2708
2709         kvm_recalculate_apic_map(vcpu->kvm);
2710 }
2711
2712 /*
2713  *----------------------------------------------------------------------
2714  * timer interface
2715  *----------------------------------------------------------------------
2716  */
2717
2718 static bool lapic_is_periodic(struct kvm_lapic *apic)
2719 {
2720         return apic_lvtt_period(apic);
2721 }
2722
2723 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2724 {
2725         struct kvm_lapic *apic = vcpu->arch.apic;
2726
2727         if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2728                 return atomic_read(&apic->lapic_timer.pending);
2729
2730         return 0;
2731 }
2732
2733 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2734 {
2735         u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2736         int vector, mode, trig_mode;
2737
2738         if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2739                 vector = reg & APIC_VECTOR_MASK;
2740                 mode = reg & APIC_MODE_MASK;
2741                 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2742                 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2743                                         NULL);
2744         }
2745         return 0;
2746 }
2747
2748 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2749 {
2750         struct kvm_lapic *apic = vcpu->arch.apic;
2751
2752         if (apic)
2753                 kvm_apic_local_deliver(apic, APIC_LVT0);
2754 }
2755
2756 static const struct kvm_io_device_ops apic_mmio_ops = {
2757         .read     = apic_mmio_read,
2758         .write    = apic_mmio_write,
2759 };
2760
2761 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2762 {
2763         struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2764         struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2765
2766         apic_timer_expired(apic, true);
2767
2768         if (lapic_is_periodic(apic)) {
2769                 advance_periodic_target_expiration(apic);
2770                 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2771                 return HRTIMER_RESTART;
2772         } else
2773                 return HRTIMER_NORESTART;
2774 }
2775
2776 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
2777 {
2778         struct kvm_lapic *apic;
2779
2780         ASSERT(vcpu != NULL);
2781
2782         apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
2783         if (!apic)
2784                 goto nomem;
2785
2786         vcpu->arch.apic = apic;
2787
2788         apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
2789         if (!apic->regs) {
2790                 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2791                        vcpu->vcpu_id);
2792                 goto nomem_free_apic;
2793         }
2794         apic->vcpu = vcpu;
2795
2796         apic->nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu);
2797
2798         hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2799                      HRTIMER_MODE_ABS_HARD);
2800         apic->lapic_timer.timer.function = apic_timer_fn;
2801         if (timer_advance_ns == -1) {
2802                 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
2803                 lapic_timer_advance_dynamic = true;
2804         } else {
2805                 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2806                 lapic_timer_advance_dynamic = false;
2807         }
2808
2809         /*
2810          * Stuff the APIC ENABLE bit in lieu of temporarily incrementing
2811          * apic_hw_disabled; the full RESET value is set by kvm_lapic_reset().
2812          */
2813         vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2814         static_branch_inc(&apic_sw_disabled.key); /* sw disabled at reset */
2815         kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
2816
2817         return 0;
2818 nomem_free_apic:
2819         kfree(apic);
2820         vcpu->arch.apic = NULL;
2821 nomem:
2822         return -ENOMEM;
2823 }
2824
2825 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2826 {
2827         struct kvm_lapic *apic = vcpu->arch.apic;
2828         u32 ppr;
2829
2830         if (!kvm_apic_present(vcpu))
2831                 return -1;
2832
2833         __apic_update_ppr(apic, &ppr);
2834         return apic_has_interrupt_for_ppr(apic, ppr);
2835 }
2836 EXPORT_SYMBOL_GPL(kvm_apic_has_interrupt);
2837
2838 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2839 {
2840         u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
2841
2842         if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2843                 return 1;
2844         if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2845             GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2846                 return 1;
2847         return 0;
2848 }
2849
2850 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2851 {
2852         struct kvm_lapic *apic = vcpu->arch.apic;
2853
2854         if (atomic_read(&apic->lapic_timer.pending) > 0) {
2855                 kvm_apic_inject_pending_timer_irqs(apic);
2856                 atomic_set(&apic->lapic_timer.pending, 0);
2857         }
2858 }
2859
2860 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2861 {
2862         int vector = kvm_apic_has_interrupt(vcpu);
2863         struct kvm_lapic *apic = vcpu->arch.apic;
2864         u32 ppr;
2865
2866         if (vector == -1)
2867                 return -1;
2868
2869         /*
2870          * We get here even with APIC virtualization enabled, if doing
2871          * nested virtualization and L1 runs with the "acknowledge interrupt
2872          * on exit" mode.  Then we cannot inject the interrupt via RVI,
2873          * because the process would deliver it through the IDT.
2874          */
2875
2876         apic_clear_irr(vector, apic);
2877         if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) {
2878                 /*
2879                  * For auto-EOI interrupts, there might be another pending
2880                  * interrupt above PPR, so check whether to raise another
2881                  * KVM_REQ_EVENT.
2882                  */
2883                 apic_update_ppr(apic);
2884         } else {
2885                 /*
2886                  * For normal interrupts, PPR has been raised and there cannot
2887                  * be a higher-priority pending interrupt---except if there was
2888                  * a concurrent interrupt injection, but that would have
2889                  * triggered KVM_REQ_EVENT already.
2890                  */
2891                 apic_set_isr(vector, apic);
2892                 __apic_update_ppr(apic, &ppr);
2893         }
2894
2895         return vector;
2896 }
2897
2898 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2899                 struct kvm_lapic_state *s, bool set)
2900 {
2901         if (apic_x2apic_mode(vcpu->arch.apic)) {
2902                 u32 *id = (u32 *)(s->regs + APIC_ID);
2903                 u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2904                 u64 icr;
2905
2906                 if (vcpu->kvm->arch.x2apic_format) {
2907                         if (*id != vcpu->vcpu_id)
2908                                 return -EINVAL;
2909                 } else {
2910                         if (set)
2911                                 *id >>= 24;
2912                         else
2913                                 *id <<= 24;
2914                 }
2915
2916                 /*
2917                  * In x2APIC mode, the LDR is fixed and based on the id.  And
2918                  * ICR is internally a single 64-bit register, but needs to be
2919                  * split to ICR+ICR2 in userspace for backwards compatibility.
2920                  */
2921                 if (set) {
2922                         *ldr = kvm_apic_calc_x2apic_ldr(*id);
2923
2924                         icr = __kvm_lapic_get_reg(s->regs, APIC_ICR) |
2925                               (u64)__kvm_lapic_get_reg(s->regs, APIC_ICR2) << 32;
2926                         __kvm_lapic_set_reg64(s->regs, APIC_ICR, icr);
2927                 } else {
2928                         icr = __kvm_lapic_get_reg64(s->regs, APIC_ICR);
2929                         __kvm_lapic_set_reg(s->regs, APIC_ICR2, icr >> 32);
2930                 }
2931         }
2932
2933         return 0;
2934 }
2935
2936 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2937 {
2938         memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2939
2940         /*
2941          * Get calculated timer current count for remaining timer period (if
2942          * any) and store it in the returned register set.
2943          */
2944         __kvm_lapic_set_reg(s->regs, APIC_TMCCT,
2945                             __apic_read(vcpu->arch.apic, APIC_TMCCT));
2946
2947         return kvm_apic_state_fixup(vcpu, s, false);
2948 }
2949
2950 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2951 {
2952         struct kvm_lapic *apic = vcpu->arch.apic;
2953         int r;
2954
2955         kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
2956         /* set SPIV separately to get count of SW disabled APICs right */
2957         apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
2958
2959         r = kvm_apic_state_fixup(vcpu, s, true);
2960         if (r) {
2961                 kvm_recalculate_apic_map(vcpu->kvm);
2962                 return r;
2963         }
2964         memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
2965
2966         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
2967         kvm_recalculate_apic_map(vcpu->kvm);
2968         kvm_apic_set_version(vcpu);
2969
2970         apic_update_ppr(apic);
2971         cancel_apic_timer(apic);
2972         apic->lapic_timer.expired_tscdeadline = 0;
2973         apic_update_lvtt(apic);
2974         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2975         update_divide_count(apic);
2976         __start_apic_timer(apic, APIC_TMCCT);
2977         kvm_lapic_set_reg(apic, APIC_TMCCT, 0);
2978         kvm_apic_update_apicv(vcpu);
2979         if (apic->apicv_active) {
2980                 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu);
2981                 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, apic_find_highest_irr(apic));
2982                 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic));
2983         }
2984         kvm_make_request(KVM_REQ_EVENT, vcpu);
2985         if (ioapic_in_kernel(vcpu->kvm))
2986                 kvm_rtc_eoi_tracking_restore_one(vcpu);
2987
2988         vcpu->arch.apic_arb_prio = 0;
2989
2990         return 0;
2991 }
2992
2993 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
2994 {
2995         struct hrtimer *timer;
2996
2997         if (!lapic_in_kernel(vcpu) ||
2998                 kvm_can_post_timer_interrupt(vcpu))
2999                 return;
3000
3001         timer = &vcpu->arch.apic->lapic_timer.timer;
3002         if (hrtimer_cancel(timer))
3003                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
3004 }
3005
3006 /*
3007  * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
3008  *
3009  * Detect whether guest triggered PV EOI since the
3010  * last entry. If yes, set EOI on guests's behalf.
3011  * Clear PV EOI in guest memory in any case.
3012  */
3013 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
3014                                         struct kvm_lapic *apic)
3015 {
3016         int vector;
3017         /*
3018          * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
3019          * and KVM_PV_EOI_ENABLED in guest memory as follows:
3020          *
3021          * KVM_APIC_PV_EOI_PENDING is unset:
3022          *      -> host disabled PV EOI.
3023          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
3024          *      -> host enabled PV EOI, guest did not execute EOI yet.
3025          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
3026          *      -> host enabled PV EOI, guest executed EOI.
3027          */
3028         BUG_ON(!pv_eoi_enabled(vcpu));
3029
3030         if (pv_eoi_test_and_clr_pending(vcpu))
3031                 return;
3032         vector = apic_set_eoi(apic);
3033         trace_kvm_pv_eoi(apic, vector);
3034 }
3035
3036 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
3037 {
3038         u32 data;
3039
3040         if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
3041                 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
3042
3043         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
3044                 return;
3045
3046         if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
3047                                   sizeof(u32)))
3048                 return;
3049
3050         apic_set_tpr(vcpu->arch.apic, data & 0xff);
3051 }
3052
3053 /*
3054  * apic_sync_pv_eoi_to_guest - called before vmentry
3055  *
3056  * Detect whether it's safe to enable PV EOI and
3057  * if yes do so.
3058  */
3059 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
3060                                         struct kvm_lapic *apic)
3061 {
3062         if (!pv_eoi_enabled(vcpu) ||
3063             /* IRR set or many bits in ISR: could be nested. */
3064             apic->irr_pending ||
3065             /* Cache not set: could be safe but we don't bother. */
3066             apic->highest_isr_cache == -1 ||
3067             /* Need EOI to update ioapic. */
3068             kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
3069                 /*
3070                  * PV EOI was disabled by apic_sync_pv_eoi_from_guest
3071                  * so we need not do anything here.
3072                  */
3073                 return;
3074         }
3075
3076         pv_eoi_set_pending(apic->vcpu);
3077 }
3078
3079 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
3080 {
3081         u32 data, tpr;
3082         int max_irr, max_isr;
3083         struct kvm_lapic *apic = vcpu->arch.apic;
3084
3085         apic_sync_pv_eoi_to_guest(vcpu, apic);
3086
3087         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
3088                 return;
3089
3090         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
3091         max_irr = apic_find_highest_irr(apic);
3092         if (max_irr < 0)
3093                 max_irr = 0;
3094         max_isr = apic_find_highest_isr(apic);
3095         if (max_isr < 0)
3096                 max_isr = 0;
3097         data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
3098
3099         kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
3100                                 sizeof(u32));
3101 }
3102
3103 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
3104 {
3105         if (vapic_addr) {
3106                 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
3107                                         &vcpu->arch.apic->vapic_cache,
3108                                         vapic_addr, sizeof(u32)))
3109                         return -EINVAL;
3110                 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
3111         } else {
3112                 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
3113         }
3114
3115         vcpu->arch.apic->vapic_addr = vapic_addr;
3116         return 0;
3117 }
3118
3119 int kvm_x2apic_icr_write(struct kvm_lapic *apic, u64 data)
3120 {
3121         data &= ~APIC_ICR_BUSY;
3122
3123         kvm_apic_send_ipi(apic, (u32)data, (u32)(data >> 32));
3124         kvm_lapic_set_reg64(apic, APIC_ICR, data);
3125         trace_kvm_apic_write(APIC_ICR, data);
3126         return 0;
3127 }
3128
3129 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data)
3130 {
3131         u32 low;
3132
3133         if (reg == APIC_ICR) {
3134                 *data = kvm_lapic_get_reg64(apic, APIC_ICR);
3135                 return 0;
3136         }
3137
3138         if (kvm_lapic_reg_read(apic, reg, 4, &low))
3139                 return 1;
3140
3141         *data = low;
3142
3143         return 0;
3144 }
3145
3146 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data)
3147 {
3148         /*
3149          * ICR is a 64-bit register in x2APIC mode (and Hyper-V PV vAPIC) and
3150          * can be written as such, all other registers remain accessible only
3151          * through 32-bit reads/writes.
3152          */
3153         if (reg == APIC_ICR)
3154                 return kvm_x2apic_icr_write(apic, data);
3155
3156         /* Bits 63:32 are reserved in all other registers. */
3157         if (data >> 32)
3158                 return 1;
3159
3160         return kvm_lapic_reg_write(apic, reg, (u32)data);
3161 }
3162
3163 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
3164 {
3165         struct kvm_lapic *apic = vcpu->arch.apic;
3166         u32 reg = (msr - APIC_BASE_MSR) << 4;
3167
3168         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
3169                 return 1;
3170
3171         return kvm_lapic_msr_write(apic, reg, data);
3172 }
3173
3174 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
3175 {
3176         struct kvm_lapic *apic = vcpu->arch.apic;
3177         u32 reg = (msr - APIC_BASE_MSR) << 4;
3178
3179         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
3180                 return 1;
3181
3182         return kvm_lapic_msr_read(apic, reg, data);
3183 }
3184
3185 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
3186 {
3187         if (!lapic_in_kernel(vcpu))
3188                 return 1;
3189
3190         return kvm_lapic_msr_write(vcpu->arch.apic, reg, data);
3191 }
3192
3193 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
3194 {
3195         if (!lapic_in_kernel(vcpu))
3196                 return 1;
3197
3198         return kvm_lapic_msr_read(vcpu->arch.apic, reg, data);
3199 }
3200
3201 int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
3202 {
3203         u64 addr = data & ~KVM_MSR_ENABLED;
3204         struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
3205         unsigned long new_len;
3206         int ret;
3207
3208         if (!IS_ALIGNED(addr, 4))
3209                 return 1;
3210
3211         if (data & KVM_MSR_ENABLED) {
3212                 if (addr == ghc->gpa && len <= ghc->len)
3213                         new_len = ghc->len;
3214                 else
3215                         new_len = len;
3216
3217                 ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
3218                 if (ret)
3219                         return ret;
3220         }
3221
3222         vcpu->arch.pv_eoi.msr_val = data;
3223
3224         return 0;
3225 }
3226
3227 int kvm_apic_accept_events(struct kvm_vcpu *vcpu)
3228 {
3229         struct kvm_lapic *apic = vcpu->arch.apic;
3230         u8 sipi_vector;
3231         int r;
3232
3233         if (!kvm_apic_has_pending_init_or_sipi(vcpu))
3234                 return 0;
3235
3236         if (is_guest_mode(vcpu)) {
3237                 r = kvm_check_nested_events(vcpu);
3238                 if (r < 0)
3239                         return r == -EBUSY ? 0 : r;
3240                 /*
3241                  * Continue processing INIT/SIPI even if a nested VM-Exit
3242                  * occurred, e.g. pending SIPIs should be dropped if INIT+SIPI
3243                  * are blocked as a result of transitioning to VMX root mode.
3244                  */
3245         }
3246
3247         /*
3248          * INITs are blocked while CPU is in specific states (SMM, VMX root
3249          * mode, SVM with GIF=0), while SIPIs are dropped if the CPU isn't in
3250          * wait-for-SIPI (WFS).
3251          */
3252         if (!kvm_apic_init_sipi_allowed(vcpu)) {
3253                 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
3254                 clear_bit(KVM_APIC_SIPI, &apic->pending_events);
3255                 return 0;
3256         }
3257
3258         if (test_and_clear_bit(KVM_APIC_INIT, &apic->pending_events)) {
3259                 kvm_vcpu_reset(vcpu, true);
3260                 if (kvm_vcpu_is_bsp(apic->vcpu))
3261                         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
3262                 else
3263                         vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
3264         }
3265         if (test_and_clear_bit(KVM_APIC_SIPI, &apic->pending_events)) {
3266                 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
3267                         /* evaluate pending_events before reading the vector */
3268                         smp_rmb();
3269                         sipi_vector = apic->sipi_vector;
3270                         static_call(kvm_x86_vcpu_deliver_sipi_vector)(vcpu, sipi_vector);
3271                         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
3272                 }
3273         }
3274         return 0;
3275 }
3276
3277 void kvm_lapic_exit(void)
3278 {
3279         static_key_deferred_flush(&apic_hw_disabled);
3280         WARN_ON(static_branch_unlikely(&apic_hw_disabled.key));
3281         static_key_deferred_flush(&apic_sw_disabled);
3282         WARN_ON(static_branch_unlikely(&apic_sw_disabled.key));
3283 }