arch/x86/kvm/vmx.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.  See
  15  * the COPYING file in the top-level directory.
  16  *
  17  */
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "cpuid.h"
  22 #include "lapic.h"
  23
  24 #include <linux/kvm_host.h>
  25 #include <linux/module.h>
  26 #include <linux/kernel.h>
  27 #include <linux/mm.h>
  28 #include <linux/highmem.h>
  29 #include <linux/sched.h>
  30 #include <linux/moduleparam.h>
  31 #include <linux/mod_devicetable.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34 #include <linux/tboot.h>
  35 #include <linux/hrtimer.h>
  36 #include <linux/frame.h>
  37 #include <linux/nospec.h>
  38 #include "kvm_cache_regs.h"
  39 #include "x86.h"
  40
  41 #include <asm/cpu.h>
  42 #include <asm/io.h>
  43 #include <asm/desc.h>
  44 #include <asm/vmx.h>
  45 #include <asm/virtext.h>
  46 #include <asm/mce.h>
  47 #include <asm/fpu/internal.h>
  48 #include <asm/perf_event.h>
  49 #include <asm/debugreg.h>
  50 #include <asm/kexec.h>
  51 #include <asm/apic.h>
  52 #include <asm/irq_remapping.h>
  53 #include <asm/mmu_context.h>
  54 #include <asm/microcode.h>
  55 #include <asm/spec-ctrl.h>
  56
  57 #include "trace.h"
  58 #include "pmu.h"
  59
  60 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  61 #define __ex_clear(x, reg) \
  62         ____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
  63
  64 MODULE_AUTHOR("Qumranet");
  65 MODULE_LICENSE("GPL");
  66
  67 static const struct x86_cpu_id vmx_cpu_id[] = {
  68         X86_FEATURE_MATCH(X86_FEATURE_VMX),
  69         {}
  70 };
  71 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
  72
  73 static bool __read_mostly nosmt;
  74 module_param(nosmt, bool, S_IRUGO);
  75
  76 static bool __read_mostly enable_vpid = 1;
  77 module_param_named(vpid, enable_vpid, bool, 0444);
  78
  79 static bool __read_mostly flexpriority_enabled = 1;
  80 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  81
  82 static bool __read_mostly enable_ept = 1;
  83 module_param_named(ept, enable_ept, bool, S_IRUGO);
  84
  85 static bool __read_mostly enable_unrestricted_guest = 1;
  86 module_param_named(unrestricted_guest,
  87                         enable_unrestricted_guest, bool, S_IRUGO);
  88
  89 static bool __read_mostly enable_ept_ad_bits = 1;
  90 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
  91
  92 static bool __read_mostly emulate_invalid_guest_state = true;
  93 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
  94
  95 static bool __read_mostly fasteoi = 1;
  96 module_param(fasteoi, bool, S_IRUGO);
  97
  98 static bool __read_mostly enable_apicv = 1;
  99 module_param(enable_apicv, bool, S_IRUGO);
 100
 101 static bool __read_mostly enable_shadow_vmcs = 1;
 102 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
 103 /*
 104  * If nested=1, nested virtualization is supported, i.e., guests may use
 105  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 106  * use VMX instructions.
 107  */
 108 static bool __read_mostly nested = 0;
 109 module_param(nested, bool, S_IRUGO);
 110
 111 static u64 __read_mostly host_xss;
 112
 113 static bool __read_mostly enable_pml = 1;
 114 module_param_named(pml, enable_pml, bool, S_IRUGO);
 115
 116 #define MSR_TYPE_R      1
 117 #define MSR_TYPE_W      2
 118 #define MSR_TYPE_RW     3
 119
 120 #define MSR_BITMAP_MODE_X2APIC          1
 121 #define MSR_BITMAP_MODE_X2APIC_APICV    2
 122 #define MSR_BITMAP_MODE_LM              4
 123
 124 #define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL
 125
 126 /* Guest_tsc -> host_tsc conversion requires 64-bit division.  */
 127 static int __read_mostly cpu_preemption_timer_multi;
 128 static bool __read_mostly enable_preemption_timer = 1;
 129 #ifdef CONFIG_X86_64
 130 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
 131 #endif
 132
 133 #define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
 134 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST (X86_CR0_WP | X86_CR0_NE)
 135 #define KVM_VM_CR0_ALWAYS_ON                                            \
 136         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
 137 #define KVM_CR4_GUEST_OWNED_BITS                                      \
 138         (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
 139          | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_TSD)
 140
 141 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
 142 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
 143
 144 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
 145
 146 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
 147
 148 /*
 149  * Hyper-V requires all of these, so mark them as supported even though
 150  * they are just treated the same as all-context.
 151  */
 152 #define VMX_VPID_EXTENT_SUPPORTED_MASK          \
 153         (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |  \
 154         VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |    \
 155         VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |    \
 156         VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
 157
 158 /*
 159  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 160  * ple_gap:    upper bound on the amount of time between two successive
 161  *             executions of PAUSE in a loop. Also indicate if ple enabled.
 162  *             According to test, this time is usually smaller than 128 cycles.
 163  * ple_window: upper bound on the amount of time a guest is allowed to execute
 164  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 165  *             less than 2^12 cycles
 166  * Time is measured based on a counter that runs at the same rate as the TSC,
 167  * refer SDM volume 3b section 21.6.13 & 22.1.3.
 168  */
 169 #define KVM_VMX_DEFAULT_PLE_GAP           128
 170 #define KVM_VMX_DEFAULT_PLE_WINDOW        4096
 171 #define KVM_VMX_DEFAULT_PLE_WINDOW_GROW   2
 172 #define KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK 0
 173 #define KVM_VMX_DEFAULT_PLE_WINDOW_MAX    \
 174                 INT_MAX / KVM_VMX_DEFAULT_PLE_WINDOW_GROW
 175
 176 static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
 177 module_param(ple_gap, int, S_IRUGO);
 178
 179 static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
 180 module_param(ple_window, int, S_IRUGO);
 181
 182 /* Default doubles per-vcpu window every exit. */
 183 static int ple_window_grow = KVM_VMX_DEFAULT_PLE_WINDOW_GROW;
 184 module_param(ple_window_grow, int, S_IRUGO);
 185
 186 /* Default resets per-vcpu window every exit to ple_window. */
 187 static int ple_window_shrink = KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK;
 188 module_param(ple_window_shrink, int, S_IRUGO);
 189
 190 /* Default is to compute the maximum so we can never overflow. */
 191 static int ple_window_actual_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 192 static int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 193 module_param(ple_window_max, int, S_IRUGO);
 194
 195 extern const ulong vmx_return;
 196
 197 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
 198
 199 /* These MUST be in sync with vmentry_l1d_param order. */
 200 enum vmx_l1d_flush_state {
 201         VMENTER_L1D_FLUSH_NEVER,
 202         VMENTER_L1D_FLUSH_COND,
 203         VMENTER_L1D_FLUSH_ALWAYS,
 204 };
 205
 206 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush = VMENTER_L1D_FLUSH_COND;
 207
 208 static const struct {
 209         const char *option;
 210         enum vmx_l1d_flush_state cmd;
 211 } vmentry_l1d_param[] = {
 212         {"never",       VMENTER_L1D_FLUSH_NEVER},
 213         {"cond",        VMENTER_L1D_FLUSH_COND},
 214         {"always",      VMENTER_L1D_FLUSH_ALWAYS},
 215 };
 216
 217 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
 218 {
 219         unsigned int i;
 220
 221         if (!s)
 222                 return -EINVAL;
 223
 224         for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) {
 225                 if (!strcmp(s, vmentry_l1d_param[i].option)) {
 226                         vmentry_l1d_flush = vmentry_l1d_param[i].cmd;
 227                         return 0;
 228                 }
 229         }
 230
 231         return -EINVAL;
 232 }
 233
 234 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
 235 {
 236         return sprintf(s, "%s\n", vmentry_l1d_param[vmentry_l1d_flush].option);
 237 }
 238
 239 static const struct kernel_param_ops vmentry_l1d_flush_ops = {
 240         .set = vmentry_l1d_flush_set,
 241         .get = vmentry_l1d_flush_get,
 242 };
 243 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, &vmentry_l1d_flush, S_IRUGO);
 244
 245 #define NR_AUTOLOAD_MSRS 8
 246
 247 struct vmcs {
 248         u32 revision_id;
 249         u32 abort;
 250         char data[0];
 251 };
 252
 253 /*
 254  * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
 255  * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
 256  * loaded on this CPU (so we can clear them if the CPU goes down).
 257  */
 258 struct loaded_vmcs {
 259         struct vmcs *vmcs;
 260         struct vmcs *shadow_vmcs;
 261         int cpu;
 262         bool launched;
 263         bool nmi_known_unmasked;
 264         unsigned long vmcs_host_cr3;    /* May not match real cr3 */
 265         unsigned long vmcs_host_cr4;    /* May not match real cr4 */
 266         /* Support for vnmi-less CPUs */
 267         int soft_vnmi_blocked;
 268         ktime_t entry_time;
 269         s64 vnmi_blocked_time;
 270         unsigned long *msr_bitmap;
 271         struct list_head loaded_vmcss_on_cpu_link;
 272 };
 273
 274 struct shared_msr_entry {
 275         unsigned index;
 276         u64 data;
 277         u64 mask;
 278 };
 279
 280 /*
 281  * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
 282  * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
 283  * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
 284  * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
 285  * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
 286  * More than one of these structures may exist, if L1 runs multiple L2 guests.
 287  * nested_vmx_run() will use the data here to build the vmcs02: a VMCS for the
 288  * underlying hardware which will be used to run L2.
 289  * This structure is packed to ensure that its layout is identical across
 290  * machines (necessary for live migration).
 291  * If there are changes in this struct, VMCS12_REVISION must be changed.
 292  */
 293 typedef u64 natural_width;
 294 struct __packed vmcs12 {
 295         /* According to the Intel spec, a VMCS region must start with the
 296          * following two fields. Then follow implementation-specific data.
 297          */
 298         u32 revision_id;
 299         u32 abort;
 300
 301         u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
 302         u32 padding[7]; /* room for future expansion */
 303
 304         u64 io_bitmap_a;
 305         u64 io_bitmap_b;
 306         u64 msr_bitmap;
 307         u64 vm_exit_msr_store_addr;
 308         u64 vm_exit_msr_load_addr;
 309         u64 vm_entry_msr_load_addr;
 310         u64 tsc_offset;
 311         u64 virtual_apic_page_addr;
 312         u64 apic_access_addr;
 313         u64 posted_intr_desc_addr;
 314         u64 vm_function_control;
 315         u64 ept_pointer;
 316         u64 eoi_exit_bitmap0;
 317         u64 eoi_exit_bitmap1;
 318         u64 eoi_exit_bitmap2;
 319         u64 eoi_exit_bitmap3;
 320         u64 eptp_list_address;
 321         u64 xss_exit_bitmap;
 322         u64 guest_physical_address;
 323         u64 vmcs_link_pointer;
 324         u64 pml_address;
 325         u64 guest_ia32_debugctl;
 326         u64 guest_ia32_pat;
 327         u64 guest_ia32_efer;
 328         u64 guest_ia32_perf_global_ctrl;
 329         u64 guest_pdptr0;
 330         u64 guest_pdptr1;
 331         u64 guest_pdptr2;
 332         u64 guest_pdptr3;
 333         u64 guest_bndcfgs;
 334         u64 host_ia32_pat;
 335         u64 host_ia32_efer;
 336         u64 host_ia32_perf_global_ctrl;
 337         u64 padding64[8]; /* room for future expansion */
 338         /*
 339          * To allow migration of L1 (complete with its L2 guests) between
 340          * machines of different natural widths (32 or 64 bit), we cannot have
 341          * unsigned long fields with no explict size. We use u64 (aliased
 342          * natural_width) instead. Luckily, x86 is little-endian.
 343          */
 344         natural_width cr0_guest_host_mask;
 345         natural_width cr4_guest_host_mask;
 346         natural_width cr0_read_shadow;
 347         natural_width cr4_read_shadow;
 348         natural_width cr3_target_value0;
 349         natural_width cr3_target_value1;
 350         natural_width cr3_target_value2;
 351         natural_width cr3_target_value3;
 352         natural_width exit_qualification;
 353         natural_width guest_linear_address;
 354         natural_width guest_cr0;
 355         natural_width guest_cr3;
 356         natural_width guest_cr4;
 357         natural_width guest_es_base;
 358         natural_width guest_cs_base;
 359         natural_width guest_ss_base;
 360         natural_width guest_ds_base;
 361         natural_width guest_fs_base;
 362         natural_width guest_gs_base;
 363         natural_width guest_ldtr_base;
 364         natural_width guest_tr_base;
 365         natural_width guest_gdtr_base;
 366         natural_width guest_idtr_base;
 367         natural_width guest_dr7;
 368         natural_width guest_rsp;
 369         natural_width guest_rip;
 370         natural_width guest_rflags;
 371         natural_width guest_pending_dbg_exceptions;
 372         natural_width guest_sysenter_esp;
 373         natural_width guest_sysenter_eip;
 374         natural_width host_cr0;
 375         natural_width host_cr3;
 376         natural_width host_cr4;
 377         natural_width host_fs_base;
 378         natural_width host_gs_base;
 379         natural_width host_tr_base;
 380         natural_width host_gdtr_base;
 381         natural_width host_idtr_base;
 382         natural_width host_ia32_sysenter_esp;
 383         natural_width host_ia32_sysenter_eip;
 384         natural_width host_rsp;
 385         natural_width host_rip;
 386         natural_width paddingl[8]; /* room for future expansion */
 387         u32 pin_based_vm_exec_control;
 388         u32 cpu_based_vm_exec_control;
 389         u32 exception_bitmap;
 390         u32 page_fault_error_code_mask;
 391         u32 page_fault_error_code_match;
 392         u32 cr3_target_count;
 393         u32 vm_exit_controls;
 394         u32 vm_exit_msr_store_count;
 395         u32 vm_exit_msr_load_count;
 396         u32 vm_entry_controls;
 397         u32 vm_entry_msr_load_count;
 398         u32 vm_entry_intr_info_field;
 399         u32 vm_entry_exception_error_code;
 400         u32 vm_entry_instruction_len;
 401         u32 tpr_threshold;
 402         u32 secondary_vm_exec_control;
 403         u32 vm_instruction_error;
 404         u32 vm_exit_reason;
 405         u32 vm_exit_intr_info;
 406         u32 vm_exit_intr_error_code;
 407         u32 idt_vectoring_info_field;
 408         u32 idt_vectoring_error_code;
 409         u32 vm_exit_instruction_len;
 410         u32 vmx_instruction_info;
 411         u32 guest_es_limit;
 412         u32 guest_cs_limit;
 413         u32 guest_ss_limit;
 414         u32 guest_ds_limit;
 415         u32 guest_fs_limit;
 416         u32 guest_gs_limit;
 417         u32 guest_ldtr_limit;
 418         u32 guest_tr_limit;
 419         u32 guest_gdtr_limit;
 420         u32 guest_idtr_limit;
 421         u32 guest_es_ar_bytes;
 422         u32 guest_cs_ar_bytes;
 423         u32 guest_ss_ar_bytes;
 424         u32 guest_ds_ar_bytes;
 425         u32 guest_fs_ar_bytes;
 426         u32 guest_gs_ar_bytes;
 427         u32 guest_ldtr_ar_bytes;
 428         u32 guest_tr_ar_bytes;
 429         u32 guest_interruptibility_info;
 430         u32 guest_activity_state;
 431         u32 guest_sysenter_cs;
 432         u32 host_ia32_sysenter_cs;
 433         u32 vmx_preemption_timer_value;
 434         u32 padding32[7]; /* room for future expansion */
 435         u16 virtual_processor_id;
 436         u16 posted_intr_nv;
 437         u16 guest_es_selector;
 438         u16 guest_cs_selector;
 439         u16 guest_ss_selector;
 440         u16 guest_ds_selector;
 441         u16 guest_fs_selector;
 442         u16 guest_gs_selector;
 443         u16 guest_ldtr_selector;
 444         u16 guest_tr_selector;
 445         u16 guest_intr_status;
 446         u16 guest_pml_index;
 447         u16 host_es_selector;
 448         u16 host_cs_selector;
 449         u16 host_ss_selector;
 450         u16 host_ds_selector;
 451         u16 host_fs_selector;
 452         u16 host_gs_selector;
 453         u16 host_tr_selector;
 454 };
 455
 456 /*
 457  * VMCS12_REVISION is an arbitrary id that should be changed if the content or
 458  * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
 459  * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
 460  */
 461 #define VMCS12_REVISION 0x11e57ed0
 462
 463 /*
 464  * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
 465  * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
 466  * current implementation, 4K are reserved to avoid future complications.
 467  */
 468 #define VMCS12_SIZE 0x1000
 469
 470 /*
 471  * The nested_vmx structure is part of vcpu_vmx, and holds information we need
 472  * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
 473  */
 474 struct nested_vmx {
 475         /* Has the level1 guest done vmxon? */
 476         bool vmxon;
 477         gpa_t vmxon_ptr;
 478         bool pml_full;
 479
 480         /* The guest-physical address of the current VMCS L1 keeps for L2 */
 481         gpa_t current_vmptr;
 482         /*
 483          * Cache of the guest's VMCS, existing outside of guest memory.
 484          * Loaded from guest memory during VMPTRLD. Flushed to guest
 485          * memory during VMCLEAR and VMPTRLD.
 486          */
 487         struct vmcs12 *cached_vmcs12;
 488         /*
 489          * Indicates if the shadow vmcs must be updated with the
 490          * data hold by vmcs12
 491          */
 492         bool sync_shadow_vmcs;
 493
 494         bool change_vmcs01_virtual_x2apic_mode;
 495         /* L2 must run next, and mustn't decide to exit to L1. */
 496         bool nested_run_pending;
 497
 498         struct loaded_vmcs vmcs02;
 499
 500         /*
 501          * Guest pages referred to in the vmcs02 with host-physical
 502          * pointers, so we must keep them pinned while L2 runs.
 503          */
 504         struct page *apic_access_page;
 505         struct page *virtual_apic_page;
 506         struct page *pi_desc_page;
 507         struct pi_desc *pi_desc;
 508         bool pi_pending;
 509         u16 posted_intr_nv;
 510
 511         struct hrtimer preemption_timer;
 512         bool preemption_timer_expired;
 513
 514         /* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */
 515         u64 vmcs01_debugctl;
 516
 517         u16 vpid02;
 518         u16 last_vpid;
 519
 520         /*
 521          * We only store the "true" versions of the VMX capability MSRs. We
 522          * generate the "non-true" versions by setting the must-be-1 bits
 523          * according to the SDM.
 524          */
 525         u32 nested_vmx_procbased_ctls_low;
 526         u32 nested_vmx_procbased_ctls_high;
 527         u32 nested_vmx_secondary_ctls_low;
 528         u32 nested_vmx_secondary_ctls_high;
 529         u32 nested_vmx_pinbased_ctls_low;
 530         u32 nested_vmx_pinbased_ctls_high;
 531         u32 nested_vmx_exit_ctls_low;
 532         u32 nested_vmx_exit_ctls_high;
 533         u32 nested_vmx_entry_ctls_low;
 534         u32 nested_vmx_entry_ctls_high;
 535         u32 nested_vmx_misc_low;
 536         u32 nested_vmx_misc_high;
 537         u32 nested_vmx_ept_caps;
 538         u32 nested_vmx_vpid_caps;
 539         u64 nested_vmx_basic;
 540         u64 nested_vmx_cr0_fixed0;
 541         u64 nested_vmx_cr0_fixed1;
 542         u64 nested_vmx_cr4_fixed0;
 543         u64 nested_vmx_cr4_fixed1;
 544         u64 nested_vmx_vmcs_enum;
 545         u64 nested_vmx_vmfunc_controls;
 546 };
 547
 548 #define POSTED_INTR_ON  0
 549 #define POSTED_INTR_SN  1
 550
 551 /* Posted-Interrupt Descriptor */
 552 struct pi_desc {
 553         u32 pir[8];     /* Posted interrupt requested */
 554         union {
 555                 struct {
 556                                 /* bit 256 - Outstanding Notification */
 557                         u16     on      : 1,
 558                                 /* bit 257 - Suppress Notification */
 559                                 sn      : 1,
 560                                 /* bit 271:258 - Reserved */
 561                                 rsvd_1  : 14;
 562                                 /* bit 279:272 - Notification Vector */
 563                         u8      nv;
 564                                 /* bit 287:280 - Reserved */
 565                         u8      rsvd_2;
 566                                 /* bit 319:288 - Notification Destination */
 567                         u32     ndst;
 568                 };
 569                 u64 control;
 570         };
 571         u32 rsvd[6];
 572 } __aligned(64);
 573
 574 static bool pi_test_and_set_on(struct pi_desc *pi_desc)
 575 {
 576         return test_and_set_bit(POSTED_INTR_ON,
 577                         (unsigned long *)&pi_desc->control);
 578 }
 579
 580 static bool pi_test_and_clear_on(struct pi_desc *pi_desc)
 581 {
 582         return test_and_clear_bit(POSTED_INTR_ON,
 583                         (unsigned long *)&pi_desc->control);
 584 }
 585
 586 static int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
 587 {
 588         return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
 589 }
 590
 591 static inline void pi_clear_sn(struct pi_desc *pi_desc)
 592 {
 593         return clear_bit(POSTED_INTR_SN,
 594                         (unsigned long *)&pi_desc->control);
 595 }
 596
 597 static inline void pi_set_sn(struct pi_desc *pi_desc)
 598 {
 599         return set_bit(POSTED_INTR_SN,
 600                         (unsigned long *)&pi_desc->control);
 601 }
 602
 603 static inline void pi_clear_on(struct pi_desc *pi_desc)
 604 {
 605         clear_bit(POSTED_INTR_ON,
 606                   (unsigned long *)&pi_desc->control);
 607 }
 608
 609 static inline int pi_test_on(struct pi_desc *pi_desc)
 610 {
 611         return test_bit(POSTED_INTR_ON,
 612                         (unsigned long *)&pi_desc->control);
 613 }
 614
 615 static inline int pi_test_sn(struct pi_desc *pi_desc)
 616 {
 617         return test_bit(POSTED_INTR_SN,
 618                         (unsigned long *)&pi_desc->control);
 619 }
 620
 621 struct vcpu_vmx {
 622         struct kvm_vcpu       vcpu;
 623         unsigned long         host_rsp;
 624         u8                    fail;
 625         u8                    msr_bitmap_mode;
 626         u32                   exit_intr_info;
 627         u32                   idt_vectoring_info;
 628         ulong                 rflags;
 629         struct shared_msr_entry *guest_msrs;
 630         int                   nmsrs;
 631         int                   save_nmsrs;
 632         unsigned long         host_idt_base;
 633 #ifdef CONFIG_X86_64
 634         u64                   msr_host_kernel_gs_base;
 635         u64                   msr_guest_kernel_gs_base;
 636 #endif
 637
 638         u64                   arch_capabilities;
 639         u64                   spec_ctrl;
 640
 641         u32 vm_entry_controls_shadow;
 642         u32 vm_exit_controls_shadow;
 643         u32 secondary_exec_control;
 644
 645         /*
 646          * loaded_vmcs points to the VMCS currently used in this vcpu. For a
 647          * non-nested (L1) guest, it always points to vmcs01. For a nested
 648          * guest (L2), it points to a different VMCS.
 649          */
 650         struct loaded_vmcs    vmcs01;
 651         struct loaded_vmcs   *loaded_vmcs;
 652         bool                  __launched; /* temporary, used in vmx_vcpu_run */
 653         struct msr_autoload {
 654                 unsigned nr;
 655                 struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
 656                 struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
 657         } msr_autoload;
 658         struct {
 659                 int           loaded;
 660                 u16           fs_sel, gs_sel, ldt_sel;
 661 #ifdef CONFIG_X86_64
 662                 u16           ds_sel, es_sel;
 663 #endif
 664                 int           gs_ldt_reload_needed;
 665                 int           fs_reload_needed;
 666                 u64           msr_host_bndcfgs;
 667         } host_state;
 668         struct {
 669                 int vm86_active;
 670                 ulong save_rflags;
 671                 struct kvm_segment segs[8];
 672         } rmode;
 673         struct {
 674                 u32 bitmask; /* 4 bits per segment (1 bit per field) */
 675                 struct kvm_save_segment {
 676                         u16 selector;
 677                         unsigned long base;
 678                         u32 limit;
 679                         u32 ar;
 680                 } seg[8];
 681         } segment_cache;
 682         int vpid;
 683         bool emulation_required;
 684
 685         u32 exit_reason;
 686
 687         /* Posted interrupt descriptor */
 688         struct pi_desc pi_desc;
 689
 690         /* Support for a guest hypervisor (nested VMX) */
 691         struct nested_vmx nested;
 692
 693         /* Dynamic PLE window. */
 694         int ple_window;
 695         bool ple_window_dirty;
 696
 697         /* Support for PML */
 698 #define PML_ENTITY_NUM          512
 699         struct page *pml_pg;
 700
 701         /* apic deadline value in host tsc */
 702         u64 hv_deadline_tsc;
 703
 704         u64 current_tsc_ratio;
 705
 706         u32 host_pkru;
 707
 708         /*
 709          * Only bits masked by msr_ia32_feature_control_valid_bits can be set in
 710          * msr_ia32_feature_control. FEATURE_CONTROL_LOCKED is always included
 711          * in msr_ia32_feature_control_valid_bits.
 712          */
 713         u64 msr_ia32_feature_control;
 714         u64 msr_ia32_feature_control_valid_bits;
 715 };
 716
 717 enum segment_cache_field {
 718         SEG_FIELD_SEL = 0,
 719         SEG_FIELD_BASE = 1,
 720         SEG_FIELD_LIMIT = 2,
 721         SEG_FIELD_AR = 3,
 722
 723         SEG_FIELD_NR = 4
 724 };
 725
 726 static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
 727 {
 728         return container_of(vcpu, struct vcpu_vmx, vcpu);
 729 }
 730
 731 static struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
 732 {
 733         return &(to_vmx(vcpu)->pi_desc);
 734 }
 735
 736 #define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
 737 #define FIELD(number, name)     [number] = VMCS12_OFFSET(name)
 738 #define FIELD64(number, name)   [number] = VMCS12_OFFSET(name), \
 739                                 [number##_HIGH] = VMCS12_OFFSET(name)+4
 740
 741
 742 static unsigned long shadow_read_only_fields[] = {
 743         /*
 744          * We do NOT shadow fields that are modified when L0
 745          * traps and emulates any vmx instruction (e.g. VMPTRLD,
 746          * VMXON...) executed by L1.
 747          * For example, VM_INSTRUCTION_ERROR is read
 748          * by L1 if a vmx instruction fails (part of the error path).
 749          * Note the code assumes this logic. If for some reason
 750          * we start shadowing these fields then we need to
 751          * force a shadow sync when L0 emulates vmx instructions
 752          * (e.g. force a sync if VM_INSTRUCTION_ERROR is modified
 753          * by nested_vmx_failValid)
 754          */
 755         VM_EXIT_REASON,
 756         VM_EXIT_INTR_INFO,
 757         VM_EXIT_INSTRUCTION_LEN,
 758         IDT_VECTORING_INFO_FIELD,
 759         IDT_VECTORING_ERROR_CODE,
 760         VM_EXIT_INTR_ERROR_CODE,
 761         EXIT_QUALIFICATION,
 762         GUEST_LINEAR_ADDRESS,
 763         GUEST_PHYSICAL_ADDRESS
 764 };
 765 static int max_shadow_read_only_fields =
 766         ARRAY_SIZE(shadow_read_only_fields);
 767
 768 static unsigned long shadow_read_write_fields[] = {
 769         TPR_THRESHOLD,
 770         GUEST_RIP,
 771         GUEST_RSP,
 772         GUEST_CR0,
 773         GUEST_CR3,
 774         GUEST_CR4,
 775         GUEST_INTERRUPTIBILITY_INFO,
 776         GUEST_RFLAGS,
 777         GUEST_CS_SELECTOR,
 778         GUEST_CS_AR_BYTES,
 779         GUEST_CS_LIMIT,
 780         GUEST_CS_BASE,
 781         GUEST_ES_BASE,
 782         GUEST_BNDCFGS,
 783         CR0_GUEST_HOST_MASK,
 784         CR0_READ_SHADOW,
 785         CR4_READ_SHADOW,
 786         TSC_OFFSET,
 787         EXCEPTION_BITMAP,
 788         CPU_BASED_VM_EXEC_CONTROL,
 789         VM_ENTRY_EXCEPTION_ERROR_CODE,
 790         VM_ENTRY_INTR_INFO_FIELD,
 791         VM_ENTRY_INSTRUCTION_LEN,
 792         VM_ENTRY_EXCEPTION_ERROR_CODE,
 793         HOST_FS_BASE,
 794         HOST_GS_BASE,
 795         HOST_FS_SELECTOR,
 796         HOST_GS_SELECTOR
 797 };
 798 static int max_shadow_read_write_fields =
 799         ARRAY_SIZE(shadow_read_write_fields);
 800
 801 static const unsigned short vmcs_field_to_offset_table[] = {
 802         FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
 803         FIELD(POSTED_INTR_NV, posted_intr_nv),
 804         FIELD(GUEST_ES_SELECTOR, guest_es_selector),
 805         FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
 806         FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
 807         FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
 808         FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
 809         FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
 810         FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
 811         FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
 812         FIELD(GUEST_INTR_STATUS, guest_intr_status),
 813         FIELD(GUEST_PML_INDEX, guest_pml_index),
 814         FIELD(HOST_ES_SELECTOR, host_es_selector),
 815         FIELD(HOST_CS_SELECTOR, host_cs_selector),
 816         FIELD(HOST_SS_SELECTOR, host_ss_selector),
 817         FIELD(HOST_DS_SELECTOR, host_ds_selector),
 818         FIELD(HOST_FS_SELECTOR, host_fs_selector),
 819         FIELD(HOST_GS_SELECTOR, host_gs_selector),
 820         FIELD(HOST_TR_SELECTOR, host_tr_selector),
 821         FIELD64(IO_BITMAP_A, io_bitmap_a),
 822         FIELD64(IO_BITMAP_B, io_bitmap_b),
 823         FIELD64(MSR_BITMAP, msr_bitmap),
 824         FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
 825         FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
 826         FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
 827         FIELD64(TSC_OFFSET, tsc_offset),
 828         FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
 829         FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
 830         FIELD64(POSTED_INTR_DESC_ADDR, posted_intr_desc_addr),
 831         FIELD64(VM_FUNCTION_CONTROL, vm_function_control),
 832         FIELD64(EPT_POINTER, ept_pointer),
 833         FIELD64(EOI_EXIT_BITMAP0, eoi_exit_bitmap0),
 834         FIELD64(EOI_EXIT_BITMAP1, eoi_exit_bitmap1),
 835         FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
 836         FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
 837         FIELD64(EPTP_LIST_ADDRESS, eptp_list_address),
 838         FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
 839         FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
 840         FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
 841         FIELD64(PML_ADDRESS, pml_address),
 842         FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
 843         FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
 844         FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
 845         FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
 846         FIELD64(GUEST_PDPTR0, guest_pdptr0),
 847         FIELD64(GUEST_PDPTR1, guest_pdptr1),
 848         FIELD64(GUEST_PDPTR2, guest_pdptr2),
 849         FIELD64(GUEST_PDPTR3, guest_pdptr3),
 850         FIELD64(GUEST_BNDCFGS, guest_bndcfgs),
 851         FIELD64(HOST_IA32_PAT, host_ia32_pat),
 852         FIELD64(HOST_IA32_EFER, host_ia32_efer),
 853         FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
 854         FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
 855         FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
 856         FIELD(EXCEPTION_BITMAP, exception_bitmap),
 857         FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
 858         FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
 859         FIELD(CR3_TARGET_COUNT, cr3_target_count),
 860         FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
 861         FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
 862         FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
 863         FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
 864         FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
 865         FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
 866         FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
 867         FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
 868         FIELD(TPR_THRESHOLD, tpr_threshold),
 869         FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
 870         FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
 871         FIELD(VM_EXIT_REASON, vm_exit_reason),
 872         FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
 873         FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
 874         FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
 875         FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
 876         FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
 877         FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
 878         FIELD(GUEST_ES_LIMIT, guest_es_limit),
 879         FIELD(GUEST_CS_LIMIT, guest_cs_limit),
 880         FIELD(GUEST_SS_LIMIT, guest_ss_limit),
 881         FIELD(GUEST_DS_LIMIT, guest_ds_limit),
 882         FIELD(GUEST_FS_LIMIT, guest_fs_limit),
 883         FIELD(GUEST_GS_LIMIT, guest_gs_limit),
 884         FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
 885         FIELD(GUEST_TR_LIMIT, guest_tr_limit),
 886         FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
 887         FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
 888         FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
 889         FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
 890         FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
 891         FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
 892         FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
 893         FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
 894         FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
 895         FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
 896         FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
 897         FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
 898         FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
 899         FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
 900         FIELD(VMX_PREEMPTION_TIMER_VALUE, vmx_preemption_timer_value),
 901         FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
 902         FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
 903         FIELD(CR0_READ_SHADOW, cr0_read_shadow),
 904         FIELD(CR4_READ_SHADOW, cr4_read_shadow),
 905         FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
 906         FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
 907         FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
 908         FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
 909         FIELD(EXIT_QUALIFICATION, exit_qualification),
 910         FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
 911         FIELD(GUEST_CR0, guest_cr0),
 912         FIELD(GUEST_CR3, guest_cr3),
 913         FIELD(GUEST_CR4, guest_cr4),
 914         FIELD(GUEST_ES_BASE, guest_es_base),
 915         FIELD(GUEST_CS_BASE, guest_cs_base),
 916         FIELD(GUEST_SS_BASE, guest_ss_base),
 917         FIELD(GUEST_DS_BASE, guest_ds_base),
 918         FIELD(GUEST_FS_BASE, guest_fs_base),
 919         FIELD(GUEST_GS_BASE, guest_gs_base),
 920         FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
 921         FIELD(GUEST_TR_BASE, guest_tr_base),
 922         FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
 923         FIELD(GUEST_IDTR_BASE, guest_idtr_base),
 924         FIELD(GUEST_DR7, guest_dr7),
 925         FIELD(GUEST_RSP, guest_rsp),
 926         FIELD(GUEST_RIP, guest_rip),
 927         FIELD(GUEST_RFLAGS, guest_rflags),
 928         FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
 929         FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
 930         FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
 931         FIELD(HOST_CR0, host_cr0),
 932         FIELD(HOST_CR3, host_cr3),
 933         FIELD(HOST_CR4, host_cr4),
 934         FIELD(HOST_FS_BASE, host_fs_base),
 935         FIELD(HOST_GS_BASE, host_gs_base),
 936         FIELD(HOST_TR_BASE, host_tr_base),
 937         FIELD(HOST_GDTR_BASE, host_gdtr_base),
 938         FIELD(HOST_IDTR_BASE, host_idtr_base),
 939         FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
 940         FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
 941         FIELD(HOST_RSP, host_rsp),
 942         FIELD(HOST_RIP, host_rip),
 943 };
 944
 945 static inline short vmcs_field_to_offset(unsigned long field)
 946 {
 947         const size_t size = ARRAY_SIZE(vmcs_field_to_offset_table);
 948         unsigned short offset;
 949
 950         BUILD_BUG_ON(size > SHRT_MAX);
 951         if (field >= size)
 952                 return -ENOENT;
 953
 954         field = array_index_nospec(field, size);
 955         offset = vmcs_field_to_offset_table[field];
 956         if (offset == 0)
 957                 return -ENOENT;
 958         return offset;
 959 }
 960
 961 static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
 962 {
 963         return to_vmx(vcpu)->nested.cached_vmcs12;
 964 }
 965
 966 static bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu);
 967 static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu);
 968 static u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa);
 969 static bool vmx_xsaves_supported(void);
 970 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
 971 static void vmx_set_segment(struct kvm_vcpu *vcpu,
 972                             struct kvm_segment *var, int seg);
 973 static void vmx_get_segment(struct kvm_vcpu *vcpu,
 974                             struct kvm_segment *var, int seg);
 975 static bool guest_state_valid(struct kvm_vcpu *vcpu);
 976 static u32 vmx_segment_access_rights(struct kvm_segment *var);
 977 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx);
 978 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
 979 static int alloc_identity_pagetable(struct kvm *kvm);
 980 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
 981 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
 982 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
 983                                             u16 error_code);
 984 static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);
 985 static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
 986                                                           u32 msr, int type);
 987
 988 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 989 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
 990 /*
 991  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
 992  * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
 993  */
 994 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
 995
 996 /*
 997  * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
 998  * can find which vCPU should be waken up.
 999  */
1000 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
1001 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
1002
1003 enum {
1004         VMX_IO_BITMAP_A,
1005         VMX_IO_BITMAP_B,
1006         VMX_VMREAD_BITMAP,
1007         VMX_VMWRITE_BITMAP,
1008         VMX_BITMAP_NR
1009 };
1010
1011 static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
1012
1013 #define vmx_io_bitmap_a                      (vmx_bitmap[VMX_IO_BITMAP_A])
1014 #define vmx_io_bitmap_b                      (vmx_bitmap[VMX_IO_BITMAP_B])
1015 #define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
1016 #define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
1017
1018 static bool cpu_has_load_ia32_efer;
1019 static bool cpu_has_load_perf_global_ctrl;
1020
1021 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
1022 static DEFINE_SPINLOCK(vmx_vpid_lock);
1023
1024 static struct vmcs_config {
1025         int size;
1026         int order;
1027         u32 basic_cap;
1028         u32 revision_id;
1029         u32 pin_based_exec_ctrl;
1030         u32 cpu_based_exec_ctrl;
1031         u32 cpu_based_2nd_exec_ctrl;
1032         u32 vmexit_ctrl;
1033         u32 vmentry_ctrl;
1034 } vmcs_config;
1035
1036 static struct vmx_capability {
1037         u32 ept;
1038         u32 vpid;
1039 } vmx_capability;
1040
1041 #define VMX_SEGMENT_FIELD(seg)                                  \
1042         [VCPU_SREG_##seg] = {                                   \
1043                 .selector = GUEST_##seg##_SELECTOR,             \
1044                 .base = GUEST_##seg##_BASE,                     \
1045                 .limit = GUEST_##seg##_LIMIT,                   \
1046                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
1047         }
1048
1049 static const struct kvm_vmx_segment_field {
1050         unsigned selector;
1051         unsigned base;
1052         unsigned limit;
1053         unsigned ar_bytes;
1054 } kvm_vmx_segment_fields[] = {
1055         VMX_SEGMENT_FIELD(CS),
1056         VMX_SEGMENT_FIELD(DS),
1057         VMX_SEGMENT_FIELD(ES),
1058         VMX_SEGMENT_FIELD(FS),
1059         VMX_SEGMENT_FIELD(GS),
1060         VMX_SEGMENT_FIELD(SS),
1061         VMX_SEGMENT_FIELD(TR),
1062         VMX_SEGMENT_FIELD(LDTR),
1063 };
1064
1065 static u64 host_efer;
1066
1067 static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
1068
1069 /*
1070  * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
1071  * away by decrementing the array size.
1072  */
1073 static const u32 vmx_msr_index[] = {
1074 #ifdef CONFIG_X86_64
1075         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
1076 #endif
1077         MSR_EFER, MSR_TSC_AUX, MSR_STAR,
1078 };
1079
1080 static inline bool is_exception_n(u32 intr_info, u8 vector)
1081 {
1082         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
1083                              INTR_INFO_VALID_MASK)) ==
1084                 (INTR_TYPE_HARD_EXCEPTION | vector | INTR_INFO_VALID_MASK);
1085 }
1086
1087 static inline bool is_debug(u32 intr_info)
1088 {
1089         return is_exception_n(intr_info, DB_VECTOR);
1090 }
1091
1092 static inline bool is_breakpoint(u32 intr_info)
1093 {
1094         return is_exception_n(intr_info, BP_VECTOR);
1095 }
1096
1097 static inline bool is_page_fault(u32 intr_info)
1098 {
1099         return is_exception_n(intr_info, PF_VECTOR);
1100 }
1101
1102 static inline bool is_no_device(u32 intr_info)
1103 {
1104         return is_exception_n(intr_info, NM_VECTOR);
1105 }
1106
1107 static inline bool is_invalid_opcode(u32 intr_info)
1108 {
1109         return is_exception_n(intr_info, UD_VECTOR);
1110 }
1111
1112 static inline bool is_external_interrupt(u32 intr_info)
1113 {
1114         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1115                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
1116 }
1117
1118 static inline bool is_machine_check(u32 intr_info)
1119 {
1120         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
1121                              INTR_INFO_VALID_MASK)) ==
1122                 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
1123 }
1124
1125 /* Undocumented: icebp/int1 */
1126 static inline bool is_icebp(u32 intr_info)
1127 {
1128         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1129                 == (INTR_TYPE_PRIV_SW_EXCEPTION | INTR_INFO_VALID_MASK);
1130 }
1131
1132 static inline bool cpu_has_vmx_msr_bitmap(void)
1133 {
1134         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
1135 }
1136
1137 static inline bool cpu_has_vmx_tpr_shadow(void)
1138 {
1139         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
1140 }
1141
1142 static inline bool cpu_need_tpr_shadow(struct kvm_vcpu *vcpu)
1143 {
1144         return cpu_has_vmx_tpr_shadow() && lapic_in_kernel(vcpu);
1145 }
1146
1147 static inline bool cpu_has_secondary_exec_ctrls(void)
1148 {
1149         return vmcs_config.cpu_based_exec_ctrl &
1150                 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1151 }
1152
1153 static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
1154 {
1155         return vmcs_config.cpu_based_2nd_exec_ctrl &
1156                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
1157 }
1158
1159 static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
1160 {
1161         return vmcs_config.cpu_based_2nd_exec_ctrl &
1162                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
1163 }
1164
1165 static inline bool cpu_has_vmx_apic_register_virt(void)
1166 {
1167         return vmcs_config.cpu_based_2nd_exec_ctrl &
1168                 SECONDARY_EXEC_APIC_REGISTER_VIRT;
1169 }
1170
1171 static inline bool cpu_has_vmx_virtual_intr_delivery(void)
1172 {
1173         return vmcs_config.cpu_based_2nd_exec_ctrl &
1174                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
1175 }
1176
1177 /*
1178  * Comment's format: document - errata name - stepping - processor name.
1179  * Refer from
1180  * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
1181  */
1182 static u32 vmx_preemption_cpu_tfms[] = {
1183 /* 323344.pdf - BA86   - D0 - Xeon 7500 Series */
1184 0x000206E6,
1185 /* 323056.pdf - AAX65  - C2 - Xeon L3406 */
1186 /* 322814.pdf - AAT59  - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
1187 /* 322911.pdf - AAU65  - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
1188 0x00020652,
1189 /* 322911.pdf - AAU65  - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
1190 0x00020655,
1191 /* 322373.pdf - AAO95  - B1 - Xeon 3400 Series */
1192 /* 322166.pdf - AAN92  - B1 - i7-800 and i5-700 Desktop */
1193 /*
1194  * 320767.pdf - AAP86  - B1 -
1195  * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
1196  */
1197 0x000106E5,
1198 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
1199 0x000106A0,
1200 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
1201 0x000106A1,
1202 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
1203 0x000106A4,
1204  /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
1205  /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
1206  /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
1207 0x000106A5,
1208 };
1209
1210 static inline bool cpu_has_broken_vmx_preemption_timer(void)
1211 {
1212         u32 eax = cpuid_eax(0x00000001), i;
1213
1214         /* Clear the reserved bits */
1215         eax &= ~(0x3U << 14 | 0xfU << 28);
1216         for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
1217                 if (eax == vmx_preemption_cpu_tfms[i])
1218                         return true;
1219
1220         return false;
1221 }
1222
1223 static inline bool cpu_has_vmx_preemption_timer(void)
1224 {
1225         return vmcs_config.pin_based_exec_ctrl &
1226                 PIN_BASED_VMX_PREEMPTION_TIMER;
1227 }
1228
1229 static inline bool cpu_has_vmx_posted_intr(void)
1230 {
1231         return IS_ENABLED(CONFIG_X86_LOCAL_APIC) &&
1232                 vmcs_config.pin_based_exec_ctrl & PIN_BASED_POSTED_INTR;
1233 }
1234
1235 static inline bool cpu_has_vmx_apicv(void)
1236 {
1237         return cpu_has_vmx_apic_register_virt() &&
1238                 cpu_has_vmx_virtual_intr_delivery() &&
1239                 cpu_has_vmx_posted_intr();
1240 }
1241
1242 static inline bool cpu_has_vmx_flexpriority(void)
1243 {
1244         return cpu_has_vmx_tpr_shadow() &&
1245                 cpu_has_vmx_virtualize_apic_accesses();
1246 }
1247
1248 static inline bool cpu_has_vmx_ept_execute_only(void)
1249 {
1250         return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
1251 }
1252
1253 static inline bool cpu_has_vmx_ept_2m_page(void)
1254 {
1255         return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
1256 }
1257
1258 static inline bool cpu_has_vmx_ept_1g_page(void)
1259 {
1260         return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
1261 }
1262
1263 static inline bool cpu_has_vmx_ept_4levels(void)
1264 {
1265         return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
1266 }
1267
1268 static inline bool cpu_has_vmx_ept_mt_wb(void)
1269 {
1270         return vmx_capability.ept & VMX_EPTP_WB_BIT;
1271 }
1272
1273 static inline bool cpu_has_vmx_ept_5levels(void)
1274 {
1275         return vmx_capability.ept & VMX_EPT_PAGE_WALK_5_BIT;
1276 }
1277
1278 static inline bool cpu_has_vmx_ept_ad_bits(void)
1279 {
1280         return vmx_capability.ept & VMX_EPT_AD_BIT;
1281 }
1282
1283 static inline bool cpu_has_vmx_invept_context(void)
1284 {
1285         return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
1286 }
1287
1288 static inline bool cpu_has_vmx_invept_global(void)
1289 {
1290         return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
1291 }
1292
1293 static inline bool cpu_has_vmx_invvpid_single(void)
1294 {
1295         return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
1296 }
1297
1298 static inline bool cpu_has_vmx_invvpid_global(void)
1299 {
1300         return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
1301 }
1302
1303 static inline bool cpu_has_vmx_invvpid(void)
1304 {
1305         return vmx_capability.vpid & VMX_VPID_INVVPID_BIT;
1306 }
1307
1308 static inline bool cpu_has_vmx_ept(void)
1309 {
1310         return vmcs_config.cpu_based_2nd_exec_ctrl &
1311                 SECONDARY_EXEC_ENABLE_EPT;
1312 }
1313
1314 static inline bool cpu_has_vmx_unrestricted_guest(void)
1315 {
1316         return vmcs_config.cpu_based_2nd_exec_ctrl &
1317                 SECONDARY_EXEC_UNRESTRICTED_GUEST;
1318 }
1319
1320 static inline bool cpu_has_vmx_ple(void)
1321 {
1322         return vmcs_config.cpu_based_2nd_exec_ctrl &
1323                 SECONDARY_EXEC_PAUSE_LOOP_EXITING;
1324 }
1325
1326 static inline bool cpu_has_vmx_basic_inout(void)
1327 {
1328         return  (((u64)vmcs_config.basic_cap << 32) & VMX_BASIC_INOUT);
1329 }
1330
1331 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
1332 {
1333         return flexpriority_enabled && lapic_in_kernel(vcpu);
1334 }
1335
1336 static inline bool cpu_has_vmx_vpid(void)
1337 {
1338         return vmcs_config.cpu_based_2nd_exec_ctrl &
1339                 SECONDARY_EXEC_ENABLE_VPID;
1340 }
1341
1342 static inline bool cpu_has_vmx_rdtscp(void)
1343 {
1344         return vmcs_config.cpu_based_2nd_exec_ctrl &
1345                 SECONDARY_EXEC_RDTSCP;
1346 }
1347
1348 static inline bool cpu_has_vmx_invpcid(void)
1349 {
1350         return vmcs_config.cpu_based_2nd_exec_ctrl &
1351                 SECONDARY_EXEC_ENABLE_INVPCID;
1352 }
1353
1354 static inline bool cpu_has_virtual_nmis(void)
1355 {
1356         return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
1357 }
1358
1359 static inline bool cpu_has_vmx_wbinvd_exit(void)
1360 {
1361         return vmcs_config.cpu_based_2nd_exec_ctrl &
1362                 SECONDARY_EXEC_WBINVD_EXITING;
1363 }
1364
1365 static inline bool cpu_has_vmx_shadow_vmcs(void)
1366 {
1367         u64 vmx_msr;
1368         rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
1369         /* check if the cpu supports writing r/o exit information fields */
1370         if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS))
1371                 return false;
1372
1373         return vmcs_config.cpu_based_2nd_exec_ctrl &
1374                 SECONDARY_EXEC_SHADOW_VMCS;
1375 }
1376
1377 static inline bool cpu_has_vmx_pml(void)
1378 {
1379         return vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_ENABLE_PML;
1380 }
1381
1382 static inline bool cpu_has_vmx_tsc_scaling(void)
1383 {
1384         return vmcs_config.cpu_based_2nd_exec_ctrl &
1385                 SECONDARY_EXEC_TSC_SCALING;
1386 }
1387
1388 static inline bool cpu_has_vmx_vmfunc(void)
1389 {
1390         return vmcs_config.cpu_based_2nd_exec_ctrl &
1391                 SECONDARY_EXEC_ENABLE_VMFUNC;
1392 }
1393
1394 static inline bool report_flexpriority(void)
1395 {
1396         return flexpriority_enabled;
1397 }
1398
1399 static inline unsigned nested_cpu_vmx_misc_cr3_count(struct kvm_vcpu *vcpu)
1400 {
1401         return vmx_misc_cr3_count(to_vmx(vcpu)->nested.nested_vmx_misc_low);
1402 }
1403
1404 static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
1405 {
1406         return vmcs12->cpu_based_vm_exec_control & bit;
1407 }
1408
1409 static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
1410 {
1411         return (vmcs12->cpu_based_vm_exec_control &
1412                         CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
1413                 (vmcs12->secondary_vm_exec_control & bit);
1414 }
1415
1416 static inline bool nested_cpu_has_preemption_timer(struct vmcs12 *vmcs12)
1417 {
1418         return vmcs12->pin_based_vm_exec_control &
1419                 PIN_BASED_VMX_PREEMPTION_TIMER;
1420 }
1421
1422 static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12)
1423 {
1424         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT);
1425 }
1426
1427 static inline bool nested_cpu_has_xsaves(struct vmcs12 *vmcs12)
1428 {
1429         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
1430 }
1431
1432 static inline bool nested_cpu_has_pml(struct vmcs12 *vmcs12)
1433 {
1434         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_PML);
1435 }
1436
1437 static inline bool nested_cpu_has_virt_x2apic_mode(struct vmcs12 *vmcs12)
1438 {
1439         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
1440 }
1441
1442 static inline bool nested_cpu_has_vpid(struct vmcs12 *vmcs12)
1443 {
1444         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VPID);
1445 }
1446
1447 static inline bool nested_cpu_has_apic_reg_virt(struct vmcs12 *vmcs12)
1448 {
1449         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_APIC_REGISTER_VIRT);
1450 }
1451
1452 static inline bool nested_cpu_has_vid(struct vmcs12 *vmcs12)
1453 {
1454         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
1455 }
1456
1457 static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12)
1458 {
1459         return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR;
1460 }
1461
1462 static inline bool nested_cpu_has_vmfunc(struct vmcs12 *vmcs12)
1463 {
1464         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VMFUNC);
1465 }
1466
1467 static inline bool nested_cpu_has_eptp_switching(struct vmcs12 *vmcs12)
1468 {
1469         return nested_cpu_has_vmfunc(vmcs12) &&
1470                 (vmcs12->vm_function_control &
1471                  VMX_VMFUNC_EPTP_SWITCHING);
1472 }
1473
1474 static inline bool is_nmi(u32 intr_info)
1475 {
1476         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1477                 == (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK);
1478 }
1479
1480 static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
1481                               u32 exit_intr_info,
1482                               unsigned long exit_qualification);
1483 static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
1484                         struct vmcs12 *vmcs12,
1485                         u32 reason, unsigned long qualification);
1486
1487 static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
1488 {
1489         int i;
1490
1491         for (i = 0; i < vmx->nmsrs; ++i)
1492                 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
1493                         return i;
1494         return -1;
1495 }
1496
1497 static inline void __invvpid(int ext, u16 vpid, gva_t gva)
1498 {
1499     struct {
1500         u64 vpid : 16;
1501         u64 rsvd : 48;
1502         u64 gva;
1503     } operand = { vpid, 0, gva };
1504
1505     asm volatile (__ex(ASM_VMX_INVVPID)
1506                   /* CF==1 or ZF==1 --> rc = -1 */
1507                   "; ja 1f ; ud2 ; 1:"
1508                   : : "a"(&operand), "c"(ext) : "cc", "memory");
1509 }
1510
1511 static inline void __invept(int ext, u64 eptp, gpa_t gpa)
1512 {
1513         struct {
1514                 u64 eptp, gpa;
1515         } operand = {eptp, gpa};
1516
1517         asm volatile (__ex(ASM_VMX_INVEPT)
1518                         /* CF==1 or ZF==1 --> rc = -1 */
1519                         "; ja 1f ; ud2 ; 1:\n"
1520                         : : "a" (&operand), "c" (ext) : "cc", "memory");
1521 }
1522
1523 static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
1524 {
1525         int i;
1526
1527         i = __find_msr_index(vmx, msr);
1528         if (i >= 0)
1529                 return &vmx->guest_msrs[i];
1530         return NULL;
1531 }
1532
1533 static void vmcs_clear(struct vmcs *vmcs)
1534 {
1535         u64 phys_addr = __pa(vmcs);
1536         u8 error;
1537
1538         asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
1539                       : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1540                       : "cc", "memory");
1541         if (error)
1542                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
1543                        vmcs, phys_addr);
1544 }
1545
1546 static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
1547 {
1548         vmcs_clear(loaded_vmcs->vmcs);
1549         if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
1550                 vmcs_clear(loaded_vmcs->shadow_vmcs);
1551         loaded_vmcs->cpu = -1;
1552         loaded_vmcs->launched = 0;
1553 }
1554
1555 static void vmcs_load(struct vmcs *vmcs)
1556 {
1557         u64 phys_addr = __pa(vmcs);
1558         u8 error;
1559
1560         asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
1561                         : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1562                         : "cc", "memory");
1563         if (error)
1564                 printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
1565                        vmcs, phys_addr);
1566 }
1567
1568 #ifdef CONFIG_KEXEC_CORE
1569 /*
1570  * This bitmap is used to indicate whether the vmclear
1571  * operation is enabled on all cpus. All disabled by
1572  * default.
1573  */
1574 static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;
1575
1576 static inline void crash_enable_local_vmclear(int cpu)
1577 {
1578         cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
1579 }
1580
1581 static inline void crash_disable_local_vmclear(int cpu)
1582 {
1583         cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
1584 }
1585
1586 static inline int crash_local_vmclear_enabled(int cpu)
1587 {
1588         return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
1589 }
1590
1591 static void crash_vmclear_local_loaded_vmcss(void)
1592 {
1593         int cpu = raw_smp_processor_id();
1594         struct loaded_vmcs *v;
1595
1596         if (!crash_local_vmclear_enabled(cpu))
1597                 return;
1598
1599         list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
1600                             loaded_vmcss_on_cpu_link)
1601                 vmcs_clear(v->vmcs);
1602 }
1603 #else
1604 static inline void crash_enable_local_vmclear(int cpu) { }
1605 static inline void crash_disable_local_vmclear(int cpu) { }
1606 #endif /* CONFIG_KEXEC_CORE */
1607
1608 static void __loaded_vmcs_clear(void *arg)
1609 {
1610         struct loaded_vmcs *loaded_vmcs = arg;
1611         int cpu = raw_smp_processor_id();
1612
1613         if (loaded_vmcs->cpu != cpu)
1614                 return; /* vcpu migration can race with cpu offline */
1615         if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
1616                 per_cpu(current_vmcs, cpu) = NULL;
1617         crash_disable_local_vmclear(cpu);
1618         list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
1619
1620         /*
1621          * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
1622          * is before setting loaded_vmcs->vcpu to -1 which is done in
1623          * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
1624          * then adds the vmcs into percpu list before it is deleted.
1625          */
1626         smp_wmb();
1627
1628         loaded_vmcs_init(loaded_vmcs);
1629         crash_enable_local_vmclear(cpu);
1630 }
1631
1632 static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
1633 {
1634         int cpu = loaded_vmcs->cpu;
1635
1636         if (cpu != -1)
1637                 smp_call_function_single(cpu,
1638                          __loaded_vmcs_clear, loaded_vmcs, 1);
1639 }
1640
1641 static inline void vpid_sync_vcpu_single(int vpid)
1642 {
1643         if (vpid == 0)
1644                 return;
1645
1646         if (cpu_has_vmx_invvpid_single())
1647                 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
1648 }
1649
1650 static inline void vpid_sync_vcpu_global(void)
1651 {
1652         if (cpu_has_vmx_invvpid_global())
1653                 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
1654 }
1655
1656 static inline void vpid_sync_context(int vpid)
1657 {
1658         if (cpu_has_vmx_invvpid_single())
1659                 vpid_sync_vcpu_single(vpid);
1660         else
1661                 vpid_sync_vcpu_global();
1662 }
1663
1664 static inline void ept_sync_global(void)
1665 {
1666         if (cpu_has_vmx_invept_global())
1667                 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
1668 }
1669
1670 static inline void ept_sync_context(u64 eptp)
1671 {
1672         if (enable_ept) {
1673                 if (cpu_has_vmx_invept_context())
1674                         __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
1675                 else
1676                         ept_sync_global();
1677         }
1678 }
1679
1680 static __always_inline void vmcs_check16(unsigned long field)
1681 {
1682         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
1683                          "16-bit accessor invalid for 64-bit field");
1684         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
1685                          "16-bit accessor invalid for 64-bit high field");
1686         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
1687                          "16-bit accessor invalid for 32-bit high field");
1688         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
1689                          "16-bit accessor invalid for natural width field");
1690 }
1691
1692 static __always_inline void vmcs_check32(unsigned long field)
1693 {
1694         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
1695                          "32-bit accessor invalid for 16-bit field");
1696         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
1697                          "32-bit accessor invalid for natural width field");
1698 }
1699
1700 static __always_inline void vmcs_check64(unsigned long field)
1701 {
1702         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
1703                          "64-bit accessor invalid for 16-bit field");
1704         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
1705                          "64-bit accessor invalid for 64-bit high field");
1706         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
1707                          "64-bit accessor invalid for 32-bit field");
1708         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
1709                          "64-bit accessor invalid for natural width field");
1710 }
1711
1712 static __always_inline void vmcs_checkl(unsigned long field)
1713 {
1714         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
1715                          "Natural width accessor invalid for 16-bit field");
1716         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
1717                          "Natural width accessor invalid for 64-bit field");
1718         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
1719                          "Natural width accessor invalid for 64-bit high field");
1720         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
1721                          "Natural width accessor invalid for 32-bit field");
1722 }
1723
1724 static __always_inline unsigned long __vmcs_readl(unsigned long field)
1725 {
1726         unsigned long value;
1727
1728         asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
1729                       : "=a"(value) : "d"(field) : "cc");
1730         return value;
1731 }
1732
1733 static __always_inline u16 vmcs_read16(unsigned long field)
1734 {
1735         vmcs_check16(field);
1736         return __vmcs_readl(field);
1737 }
1738
1739 static __always_inline u32 vmcs_read32(unsigned long field)
1740 {
1741         vmcs_check32(field);
1742         return __vmcs_readl(field);
1743 }
1744
1745 static __always_inline u64 vmcs_read64(unsigned long field)
1746 {
1747         vmcs_check64(field);
1748 #ifdef CONFIG_X86_64
1749         return __vmcs_readl(field);
1750 #else
1751         return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
1752 #endif
1753 }
1754
1755 static __always_inline unsigned long vmcs_readl(unsigned long field)
1756 {
1757         vmcs_checkl(field);
1758         return __vmcs_readl(field);
1759 }
1760
1761 static noinline void vmwrite_error(unsigned long field, unsigned long value)
1762 {
1763         printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
1764                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
1765         dump_stack();
1766 }
1767
1768 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
1769 {
1770         u8 error;
1771
1772         asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
1773                        : "=q"(error) : "a"(value), "d"(field) : "cc");
1774         if (unlikely(error))
1775                 vmwrite_error(field, value);
1776 }
1777
1778 static __always_inline void vmcs_write16(unsigned long field, u16 value)
1779 {
1780         vmcs_check16(field);
1781         __vmcs_writel(field, value);
1782 }
1783
1784 static __always_inline void vmcs_write32(unsigned long field, u32 value)
1785 {
1786         vmcs_check32(field);
1787         __vmcs_writel(field, value);
1788 }
1789
1790 static __always_inline void vmcs_write64(unsigned long field, u64 value)
1791 {
1792         vmcs_check64(field);
1793         __vmcs_writel(field, value);
1794 #ifndef CONFIG_X86_64
1795         asm volatile ("");
1796         __vmcs_writel(field+1, value >> 32);
1797 #endif
1798 }
1799
1800 static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
1801 {
1802         vmcs_checkl(field);
1803         __vmcs_writel(field, value);
1804 }
1805
1806 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
1807 {
1808         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
1809                          "vmcs_clear_bits does not support 64-bit fields");
1810         __vmcs_writel(field, __vmcs_readl(field) & ~mask);
1811 }
1812
1813 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
1814 {
1815         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
1816                          "vmcs_set_bits does not support 64-bit fields");
1817         __vmcs_writel(field, __vmcs_readl(field) | mask);
1818 }
1819
1820 static inline void vm_entry_controls_reset_shadow(struct vcpu_vmx *vmx)
1821 {
1822         vmx->vm_entry_controls_shadow = vmcs_read32(VM_ENTRY_CONTROLS);
1823 }
1824
1825 static inline void vm_entry_controls_init(struct vcpu_vmx *vmx, u32 val)
1826 {
1827         vmcs_write32(VM_ENTRY_CONTROLS, val);
1828         vmx->vm_entry_controls_shadow = val;
1829 }
1830
1831 static inline void vm_entry_controls_set(struct vcpu_vmx *vmx, u32 val)
1832 {
1833         if (vmx->vm_entry_controls_shadow != val)
1834                 vm_entry_controls_init(vmx, val);
1835 }
1836
1837 static inline u32 vm_entry_controls_get(struct vcpu_vmx *vmx)
1838 {
1839         return vmx->vm_entry_controls_shadow;
1840 }
1841
1842
1843 static inline void vm_entry_controls_setbit(struct vcpu_vmx *vmx, u32 val)
1844 {
1845         vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) | val);
1846 }
1847
1848 static inline void vm_entry_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
1849 {
1850         vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) & ~val);
1851 }
1852
1853 static inline void vm_exit_controls_reset_shadow(struct vcpu_vmx *vmx)
1854 {
1855         vmx->vm_exit_controls_shadow = vmcs_read32(VM_EXIT_CONTROLS);
1856 }
1857
1858 static inline void vm_exit_controls_init(struct vcpu_vmx *vmx, u32 val)
1859 {
1860         vmcs_write32(VM_EXIT_CONTROLS, val);
1861         vmx->vm_exit_controls_shadow = val;
1862 }
1863
1864 static inline void vm_exit_controls_set(struct vcpu_vmx *vmx, u32 val)
1865 {
1866         if (vmx->vm_exit_controls_shadow != val)
1867                 vm_exit_controls_init(vmx, val);
1868 }
1869
1870 static inline u32 vm_exit_controls_get(struct vcpu_vmx *vmx)
1871 {
1872         return vmx->vm_exit_controls_shadow;
1873 }
1874
1875
1876 static inline void vm_exit_controls_setbit(struct vcpu_vmx *vmx, u32 val)
1877 {
1878         vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) | val);
1879 }
1880
1881 static inline void vm_exit_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
1882 {
1883         vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) & ~val);
1884 }
1885
1886 static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
1887 {
1888         vmx->segment_cache.bitmask = 0;
1889 }
1890
1891 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
1892                                        unsigned field)
1893 {
1894         bool ret;
1895         u32 mask = 1 << (seg * SEG_FIELD_NR + field);
1896
1897         if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
1898                 vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
1899                 vmx->segment_cache.bitmask = 0;
1900         }
1901         ret = vmx->segment_cache.bitmask & mask;
1902         vmx->segment_cache.bitmask |= mask;
1903         return ret;
1904 }
1905
1906 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
1907 {
1908         u16 *p = &vmx->segment_cache.seg[seg].selector;
1909
1910         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
1911                 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
1912         return *p;
1913 }
1914
1915 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
1916 {
1917         ulong *p = &vmx->segment_cache.seg[seg].base;
1918
1919         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
1920                 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
1921         return *p;
1922 }
1923
1924 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
1925 {
1926         u32 *p = &vmx->segment_cache.seg[seg].limit;
1927
1928         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
1929                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
1930         return *p;
1931 }
1932
1933 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
1934 {
1935         u32 *p = &vmx->segment_cache.seg[seg].ar;
1936
1937         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
1938                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
1939         return *p;
1940 }
1941
1942 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
1943 {
1944         u32 eb;
1945
1946         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
1947              (1u << DB_VECTOR) | (1u << AC_VECTOR);
1948         if ((vcpu->guest_debug &
1949              (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
1950             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
1951                 eb |= 1u << BP_VECTOR;
1952         if (to_vmx(vcpu)->rmode.vm86_active)
1953                 eb = ~0;
1954         if (enable_ept)
1955                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
1956
1957         /* When we are running a nested L2 guest and L1 specified for it a
1958          * certain exception bitmap, we must trap the same exceptions and pass
1959          * them to L1. When running L2, we will only handle the exceptions
1960          * specified above if L1 did not want them.
1961          */
1962         if (is_guest_mode(vcpu))
1963                 eb |= get_vmcs12(vcpu)->exception_bitmap;
1964
1965         vmcs_write32(EXCEPTION_BITMAP, eb);
1966 }
1967
1968 /*
1969  * Check if MSR is intercepted for currently loaded MSR bitmap.
1970  */
1971 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
1972 {
1973         unsigned long *msr_bitmap;
1974         int f = sizeof(unsigned long);
1975
1976         if (!cpu_has_vmx_msr_bitmap())
1977                 return true;
1978
1979         msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap;
1980
1981         if (msr <= 0x1fff) {
1982                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
1983         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
1984                 msr &= 0x1fff;
1985                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
1986         }
1987
1988         return true;
1989 }
1990
1991 /*
1992  * Check if MSR is intercepted for L01 MSR bitmap.
1993  */
1994 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
1995 {
1996         unsigned long *msr_bitmap;
1997         int f = sizeof(unsigned long);
1998
1999         if (!cpu_has_vmx_msr_bitmap())
2000                 return true;
2001
2002         msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
2003
2004         if (msr <= 0x1fff) {
2005                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
2006         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2007                 msr &= 0x1fff;
2008                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
2009         }
2010
2011         return true;
2012 }
2013
2014 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
2015                 unsigned long entry, unsigned long exit)
2016 {
2017         vm_entry_controls_clearbit(vmx, entry);
2018         vm_exit_controls_clearbit(vmx, exit);
2019 }
2020
2021 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
2022 {
2023         unsigned i;
2024         struct msr_autoload *m = &vmx->msr_autoload;
2025
2026         switch (msr) {
2027         case MSR_EFER:
2028                 if (cpu_has_load_ia32_efer) {
2029                         clear_atomic_switch_msr_special(vmx,
2030                                         VM_ENTRY_LOAD_IA32_EFER,
2031                                         VM_EXIT_LOAD_IA32_EFER);
2032                         return;
2033                 }
2034                 break;
2035         case MSR_CORE_PERF_GLOBAL_CTRL:
2036                 if (cpu_has_load_perf_global_ctrl) {
2037                         clear_atomic_switch_msr_special(vmx,
2038                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
2039                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
2040                         return;
2041                 }
2042                 break;
2043         }
2044
2045         for (i = 0; i < m->nr; ++i)
2046                 if (m->guest[i].index == msr)
2047                         break;
2048
2049         if (i == m->nr)
2050                 return;
2051         --m->nr;
2052         m->guest[i] = m->guest[m->nr];
2053         m->host[i] = m->host[m->nr];
2054         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
2055         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
2056 }
2057
2058 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
2059                 unsigned long entry, unsigned long exit,
2060                 unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
2061                 u64 guest_val, u64 host_val)
2062 {
2063         vmcs_write64(guest_val_vmcs, guest_val);
2064         vmcs_write64(host_val_vmcs, host_val);
2065         vm_entry_controls_setbit(vmx, entry);
2066         vm_exit_controls_setbit(vmx, exit);
2067 }
2068
2069 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
2070                                   u64 guest_val, u64 host_val)
2071 {
2072         unsigned i;
2073         struct msr_autoload *m = &vmx->msr_autoload;
2074
2075         switch (msr) {
2076         case MSR_EFER:
2077                 if (cpu_has_load_ia32_efer) {
2078                         add_atomic_switch_msr_special(vmx,
2079                                         VM_ENTRY_LOAD_IA32_EFER,
2080                                         VM_EXIT_LOAD_IA32_EFER,
2081                                         GUEST_IA32_EFER,
2082                                         HOST_IA32_EFER,
2083                                         guest_val, host_val);
2084                         return;
2085                 }
2086                 break;
2087         case MSR_CORE_PERF_GLOBAL_CTRL:
2088                 if (cpu_has_load_perf_global_ctrl) {
2089                         add_atomic_switch_msr_special(vmx,
2090                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
2091                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
2092                                         GUEST_IA32_PERF_GLOBAL_CTRL,
2093                                         HOST_IA32_PERF_GLOBAL_CTRL,
2094                                         guest_val, host_val);
2095                         return;
2096                 }
2097                 break;
2098         case MSR_IA32_PEBS_ENABLE:
2099                 /* PEBS needs a quiescent period after being disabled (to write
2100                  * a record).  Disabling PEBS through VMX MSR swapping doesn't
2101                  * provide that period, so a CPU could write host's record into
2102                  * guest's memory.
2103                  */
2104                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
2105         }
2106
2107         for (i = 0; i < m->nr; ++i)
2108                 if (m->guest[i].index == msr)
2109                         break;
2110
2111         if (i == NR_AUTOLOAD_MSRS) {
2112                 printk_once(KERN_WARNING "Not enough msr switch entries. "
2113                                 "Can't add msr %x\n", msr);
2114                 return;
2115         } else if (i == m->nr) {
2116                 ++m->nr;
2117                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
2118                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
2119         }
2120
2121         m->guest[i].index = msr;
2122         m->guest[i].value = guest_val;
2123         m->host[i].index = msr;
2124         m->host[i].value = host_val;
2125 }
2126
2127 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
2128 {
2129         u64 guest_efer = vmx->vcpu.arch.efer;
2130         u64 ignore_bits = 0;
2131
2132         if (!enable_ept) {
2133                 /*
2134                  * NX is needed to handle CR0.WP=1, CR4.SMEP=1.  Testing
2135                  * host CPUID is more efficient than testing guest CPUID
2136                  * or CR4.  Host SMEP is anyway a requirement for guest SMEP.
2137                  */
2138                 if (boot_cpu_has(X86_FEATURE_SMEP))
2139                         guest_efer |= EFER_NX;
2140                 else if (!(guest_efer & EFER_NX))
2141                         ignore_bits |= EFER_NX;
2142         }
2143
2144         /*
2145          * LMA and LME handled by hardware; SCE meaningless outside long mode.
2146          */
2147         ignore_bits |= EFER_SCE;
2148 #ifdef CONFIG_X86_64
2149         ignore_bits |= EFER_LMA | EFER_LME;
2150         /* SCE is meaningful only in long mode on Intel */
2151         if (guest_efer & EFER_LMA)
2152                 ignore_bits &= ~(u64)EFER_SCE;
2153 #endif
2154
2155         clear_atomic_switch_msr(vmx, MSR_EFER);
2156
2157         /*
2158          * On EPT, we can't emulate NX, so we must switch EFER atomically.
2159          * On CPUs that support "load IA32_EFER", always switch EFER
2160          * atomically, since it's faster than switching it manually.
2161          */
2162         if (cpu_has_load_ia32_efer ||
2163             (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
2164                 if (!(guest_efer & EFER_LMA))
2165                         guest_efer &= ~EFER_LME;
2166                 if (guest_efer != host_efer)
2167                         add_atomic_switch_msr(vmx, MSR_EFER,
2168                                               guest_efer, host_efer);
2169                 return false;
2170         } else {
2171                 guest_efer &= ~ignore_bits;
2172                 guest_efer |= host_efer & ignore_bits;
2173
2174                 vmx->guest_msrs[efer_offset].data = guest_efer;
2175                 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
2176
2177                 return true;
2178         }
2179 }
2180
2181 #ifdef CONFIG_X86_32
2182 /*
2183  * On 32-bit kernels, VM exits still load the FS and GS bases from the
2184  * VMCS rather than the segment table.  KVM uses this helper to figure
2185  * out the current bases to poke them into the VMCS before entry.
2186  */
2187 static unsigned long segment_base(u16 selector)
2188 {
2189         struct desc_struct *table;
2190         unsigned long v;
2191
2192         if (!(selector & ~SEGMENT_RPL_MASK))
2193                 return 0;
2194
2195         table = get_current_gdt_ro();
2196
2197         if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2198                 u16 ldt_selector = kvm_read_ldt();
2199
2200                 if (!(ldt_selector & ~SEGMENT_RPL_MASK))
2201                         return 0;
2202
2203                 table = (struct desc_struct *)segment_base(ldt_selector);
2204         }
2205         v = get_desc_base(&table[selector >> 3]);
2206         return v;
2207 }
2208 #endif
2209
2210 static void vmx_save_host_state(struct kvm_vcpu *vcpu)
2211 {
2212         struct vcpu_vmx *vmx = to_vmx(vcpu);
2213         int i;
2214
2215         if (vmx->host_state.loaded)
2216                 return;
2217
2218         vmx->host_state.loaded = 1;
2219         /*
2220          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
2221          * allow segment selectors with cpl > 0 or ti == 1.
2222          */
2223         vmx->host_state.ldt_sel = kvm_read_ldt();
2224         vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
2225         savesegment(fs, vmx->host_state.fs_sel);
2226         if (!(vmx->host_state.fs_sel & 7)) {
2227                 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
2228                 vmx->host_state.fs_reload_needed = 0;
2229         } else {
2230                 vmcs_write16(HOST_FS_SELECTOR, 0);
2231                 vmx->host_state.fs_reload_needed = 1;
2232         }
2233         savesegment(gs, vmx->host_state.gs_sel);
2234         if (!(vmx->host_state.gs_sel & 7))
2235                 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
2236         else {
2237                 vmcs_write16(HOST_GS_SELECTOR, 0);
2238                 vmx->host_state.gs_ldt_reload_needed = 1;
2239         }
2240
2241 #ifdef CONFIG_X86_64
2242         savesegment(ds, vmx->host_state.ds_sel);
2243         savesegment(es, vmx->host_state.es_sel);
2244 #endif
2245
2246 #ifdef CONFIG_X86_64
2247         vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
2248         vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
2249 #else
2250         vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
2251         vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
2252 #endif
2253
2254 #ifdef CONFIG_X86_64
2255         rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
2256         if (is_long_mode(&vmx->vcpu))
2257                 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
2258 #endif
2259         if (boot_cpu_has(X86_FEATURE_MPX))
2260                 rdmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
2261         for (i = 0; i < vmx->save_nmsrs; ++i)
2262                 kvm_set_shared_msr(vmx->guest_msrs[i].index,
2263                                    vmx->guest_msrs[i].data,
2264                                    vmx->guest_msrs[i].mask);
2265 }
2266
2267 static void __vmx_load_host_state(struct vcpu_vmx *vmx)
2268 {
2269         if (!vmx->host_state.loaded)
2270                 return;
2271
2272         ++vmx->vcpu.stat.host_state_reload;
2273         vmx->host_state.loaded = 0;
2274 #ifdef CONFIG_X86_64
2275         if (is_long_mode(&vmx->vcpu))
2276                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
2277 #endif
2278         if (vmx->host_state.gs_ldt_reload_needed) {
2279                 kvm_load_ldt(vmx->host_state.ldt_sel);
2280 #ifdef CONFIG_X86_64
2281                 load_gs_index(vmx->host_state.gs_sel);
2282 #else
2283                 loadsegment(gs, vmx->host_state.gs_sel);
2284 #endif
2285         }
2286         if (vmx->host_state.fs_reload_needed)
2287                 loadsegment(fs, vmx->host_state.fs_sel);
2288 #ifdef CONFIG_X86_64
2289         if (unlikely(vmx->host_state.ds_sel | vmx->host_state.es_sel)) {
2290                 loadsegment(ds, vmx->host_state.ds_sel);
2291                 loadsegment(es, vmx->host_state.es_sel);
2292         }
2293 #endif
2294         invalidate_tss_limit();
2295 #ifdef CONFIG_X86_64
2296         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
2297 #endif
2298         if (vmx->host_state.msr_host_bndcfgs)
2299                 wrmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
2300         load_fixmap_gdt(raw_smp_processor_id());
2301 }
2302
2303 static void vmx_load_host_state(struct vcpu_vmx *vmx)
2304 {
2305         preempt_disable();
2306         __vmx_load_host_state(vmx);
2307         preempt_enable();
2308 }
2309
2310 static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
2311 {
2312         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
2313         struct pi_desc old, new;
2314         unsigned int dest;
2315
2316         /*
2317          * In case of hot-plug or hot-unplug, we may have to undo
2318          * vmx_vcpu_pi_put even if there is no assigned device.  And we
2319          * always keep PI.NDST up to date for simplicity: it makes the
2320          * code easier, and CPU migration is not a fast path.
2321          */
2322         if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu)
2323                 return;
2324
2325         /*
2326          * First handle the simple case where no cmpxchg is necessary; just
2327          * allow posting non-urgent interrupts.
2328          *
2329          * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change
2330          * PI.NDST: pi_post_block will do it for us and the wakeup_handler
2331          * expects the VCPU to be on the blocked_vcpu_list that matches
2332          * PI.NDST.
2333          */
2334         if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR ||
2335             vcpu->cpu == cpu) {
2336                 pi_clear_sn(pi_desc);
2337                 return;
2338         }
2339
2340         /* The full case.  */
2341         do {
2342                 old.control = new.control = pi_desc->control;
2343
2344                 dest = cpu_physical_id(cpu);
2345
2346                 if (x2apic_enabled())
2347                         new.ndst = dest;
2348                 else
2349                         new.ndst = (dest << 8) & 0xFF00;
2350
2351                 new.sn = 0;
2352         } while (cmpxchg64(&pi_desc->control, old.control,
2353                            new.control) != old.control);
2354 }
2355
2356 static void decache_tsc_multiplier(struct vcpu_vmx *vmx)
2357 {
2358         vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio;
2359         vmcs_write64(TSC_MULTIPLIER, vmx->current_tsc_ratio);
2360 }
2361
2362 /*
2363  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
2364  * vcpu mutex is already taken.
2365  */
2366 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2367 {
2368         struct vcpu_vmx *vmx = to_vmx(vcpu);
2369         bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
2370
2371         if (!already_loaded) {
2372                 loaded_vmcs_clear(vmx->loaded_vmcs);
2373                 local_irq_disable();
2374                 crash_disable_local_vmclear(cpu);
2375
2376                 /*
2377                  * Read loaded_vmcs->cpu should be before fetching
2378                  * loaded_vmcs->loaded_vmcss_on_cpu_link.
2379                  * See the comments in __loaded_vmcs_clear().
2380                  */
2381                 smp_rmb();
2382
2383                 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
2384                          &per_cpu(loaded_vmcss_on_cpu, cpu));
2385                 crash_enable_local_vmclear(cpu);
2386                 local_irq_enable();
2387         }
2388
2389         if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
2390                 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
2391                 vmcs_load(vmx->loaded_vmcs->vmcs);
2392                 indirect_branch_prediction_barrier();
2393         }
2394
2395         if (!already_loaded) {
2396                 void *gdt = get_current_gdt_ro();
2397                 unsigned long sysenter_esp;
2398
2399                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2400
2401                 /*
2402                  * Linux uses per-cpu TSS and GDT, so set these when switching
2403                  * processors.  See 22.2.4.
2404                  */
2405                 vmcs_writel(HOST_TR_BASE,
2406                             (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
2407                 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt);   /* 22.2.4 */
2408
2409                 /*
2410                  * VM exits change the host TR limit to 0x67 after a VM
2411                  * exit.  This is okay, since 0x67 covers everything except
2412                  * the IO bitmap and have have code to handle the IO bitmap
2413                  * being lost after a VM exit.
2414                  */
2415                 BUILD_BUG_ON(IO_BITMAP_OFFSET - 1 != 0x67);
2416
2417                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
2418                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
2419
2420                 vmx->loaded_vmcs->cpu = cpu;
2421         }
2422
2423         /* Setup TSC multiplier */
2424         if (kvm_has_tsc_control &&
2425             vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
2426                 decache_tsc_multiplier(vmx);
2427
2428         vmx_vcpu_pi_load(vcpu, cpu);
2429         vmx->host_pkru = read_pkru();
2430 }
2431
2432 static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
2433 {
2434         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
2435
2436         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
2437                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
2438                 !kvm_vcpu_apicv_active(vcpu))
2439                 return;
2440
2441         /* Set SN when the vCPU is preempted */
2442         if (vcpu->preempted)
2443                 pi_set_sn(pi_desc);
2444 }
2445
2446 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
2447 {
2448         vmx_vcpu_pi_put(vcpu);
2449
2450         __vmx_load_host_state(to_vmx(vcpu));
2451 }
2452
2453 static bool emulation_required(struct kvm_vcpu *vcpu)
2454 {
2455         return emulate_invalid_guest_state && !guest_state_valid(vcpu);
2456 }
2457
2458 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
2459
2460 /*
2461  * Return the cr0 value that a nested guest would read. This is a combination
2462  * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
2463  * its hypervisor (cr0_read_shadow).
2464  */
2465 static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
2466 {
2467         return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
2468                 (fields->cr0_read_shadow & fields->cr0_guest_host_mask);
2469 }
2470 static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
2471 {
2472         return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
2473                 (fields->cr4_read_shadow & fields->cr4_guest_host_mask);
2474 }
2475
2476 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
2477 {
2478         unsigned long rflags, save_rflags;
2479
2480         if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
2481                 __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
2482                 rflags = vmcs_readl(GUEST_RFLAGS);
2483                 if (to_vmx(vcpu)->rmode.vm86_active) {
2484                         rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
2485                         save_rflags = to_vmx(vcpu)->rmode.save_rflags;
2486                         rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
2487                 }
2488                 to_vmx(vcpu)->rflags = rflags;
2489         }
2490         return to_vmx(vcpu)->rflags;
2491 }
2492
2493 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2494 {
2495         unsigned long old_rflags = vmx_get_rflags(vcpu);
2496
2497         __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
2498         to_vmx(vcpu)->rflags = rflags;
2499         if (to_vmx(vcpu)->rmode.vm86_active) {
2500                 to_vmx(vcpu)->rmode.save_rflags = rflags;
2501                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
2502         }
2503         vmcs_writel(GUEST_RFLAGS, rflags);
2504
2505         if ((old_rflags ^ to_vmx(vcpu)->rflags) & X86_EFLAGS_VM)
2506                 to_vmx(vcpu)->emulation_required = emulation_required(vcpu);
2507 }
2508
2509 static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
2510 {
2511         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
2512         int ret = 0;
2513
2514         if (interruptibility & GUEST_INTR_STATE_STI)
2515                 ret |= KVM_X86_SHADOW_INT_STI;
2516         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
2517                 ret |= KVM_X86_SHADOW_INT_MOV_SS;
2518
2519         return ret;
2520 }
2521
2522 static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
2523 {
2524         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
2525         u32 interruptibility = interruptibility_old;
2526
2527         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
2528
2529         if (mask & KVM_X86_SHADOW_INT_MOV_SS)
2530                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
2531         else if (mask & KVM_X86_SHADOW_INT_STI)
2532                 interruptibility |= GUEST_INTR_STATE_STI;
2533
2534         if ((interruptibility != interruptibility_old))
2535                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
2536 }
2537
2538 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
2539 {
2540         unsigned long rip;
2541
2542         rip = kvm_rip_read(vcpu);
2543         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
2544         kvm_rip_write(vcpu, rip);
2545
2546         /* skipping an emulated instruction also counts */
2547         vmx_set_interrupt_shadow(vcpu, 0);
2548 }
2549
2550 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu,
2551                                                unsigned long exit_qual)
2552 {
2553         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
2554         unsigned int nr = vcpu->arch.exception.nr;
2555         u32 intr_info = nr | INTR_INFO_VALID_MASK;
2556
2557         if (vcpu->arch.exception.has_error_code) {
2558                 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code;
2559                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
2560         }
2561
2562         if (kvm_exception_is_soft(nr))
2563                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
2564         else
2565                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
2566
2567         if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
2568             vmx_get_nmi_mask(vcpu))
2569                 intr_info |= INTR_INFO_UNBLOCK_NMI;
2570
2571         nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
2572 }
2573
2574 /*
2575  * KVM wants to inject page-faults which it got to the guest. This function
2576  * checks whether in a nested guest, we need to inject them to L1 or L2.
2577  */
2578 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual)
2579 {
2580         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
2581         unsigned int nr = vcpu->arch.exception.nr;
2582
2583         if (nr == PF_VECTOR) {
2584                 if (vcpu->arch.exception.nested_apf) {
2585                         *exit_qual = vcpu->arch.apf.nested_apf_token;
2586                         return 1;
2587                 }
2588                 /*
2589                  * FIXME: we must not write CR2 when L1 intercepts an L2 #PF exception.
2590                  * The fix is to add the ancillary datum (CR2 or DR6) to structs
2591                  * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6
2592                  * can be written only when inject_pending_event runs.  This should be
2593                  * conditional on a new capability---if the capability is disabled,
2594                  * kvm_multiple_exception would write the ancillary information to
2595                  * CR2 or DR6, for backwards ABI-compatibility.
2596                  */
2597                 if (nested_vmx_is_page_fault_vmexit(vmcs12,
2598                                                     vcpu->arch.exception.error_code)) {
2599                         *exit_qual = vcpu->arch.cr2;
2600                         return 1;
2601                 }
2602         } else {
2603                 if (vmcs12->exception_bitmap & (1u << nr)) {
2604                         if (nr == DB_VECTOR)
2605                                 *exit_qual = vcpu->arch.dr6;
2606                         else
2607                                 *exit_qual = 0;
2608                         return 1;
2609                 }
2610         }
2611
2612         return 0;
2613 }
2614
2615 static void vmx_queue_exception(struct kvm_vcpu *vcpu)
2616 {
2617         struct vcpu_vmx *vmx = to_vmx(vcpu);
2618         unsigned nr = vcpu->arch.exception.nr;
2619         bool has_error_code = vcpu->arch.exception.has_error_code;
2620         u32 error_code = vcpu->arch.exception.error_code;
2621         u32 intr_info = nr | INTR_INFO_VALID_MASK;
2622
2623         if (has_error_code) {
2624                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
2625                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
2626         }
2627
2628         if (vmx->rmode.vm86_active) {
2629                 int inc_eip = 0;
2630                 if (kvm_exception_is_soft(nr))
2631                         inc_eip = vcpu->arch.event_exit_inst_len;
2632                 if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
2633                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2634                 return;
2635         }
2636
2637         WARN_ON_ONCE(vmx->emulation_required);
2638
2639         if (kvm_exception_is_soft(nr)) {
2640                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2641                              vmx->vcpu.arch.event_exit_inst_len);
2642                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
2643         } else
2644                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
2645
2646         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
2647 }
2648
2649 static bool vmx_rdtscp_supported(void)
2650 {
2651         return cpu_has_vmx_rdtscp();
2652 }
2653
2654 static bool vmx_invpcid_supported(void)
2655 {
2656         return cpu_has_vmx_invpcid() && enable_ept;
2657 }
2658
2659 /*
2660  * Swap MSR entry in host/guest MSR entry array.
2661  */
2662 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
2663 {
2664         struct shared_msr_entry tmp;
2665
2666         tmp = vmx->guest_msrs[to];
2667         vmx->guest_msrs[to] = vmx->guest_msrs[from];
2668         vmx->guest_msrs[from] = tmp;
2669 }
2670
2671 /*
2672  * Set up the vmcs to automatically save and restore system
2673  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
2674  * mode, as fiddling with msrs is very expensive.
2675  */
2676 static void setup_msrs(struct vcpu_vmx *vmx)
2677 {
2678         int save_nmsrs, index;
2679
2680         save_nmsrs = 0;
2681 #ifdef CONFIG_X86_64
2682         if (is_long_mode(&vmx->vcpu)) {
2683                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
2684                 if (index >= 0)
2685                         move_msr_up(vmx, index, save_nmsrs++);
2686                 index = __find_msr_index(vmx, MSR_LSTAR);
2687                 if (index >= 0)
2688                         move_msr_up(vmx, index, save_nmsrs++);
2689                 index = __find_msr_index(vmx, MSR_CSTAR);
2690                 if (index >= 0)
2691                         move_msr_up(vmx, index, save_nmsrs++);
2692                 index = __find_msr_index(vmx, MSR_TSC_AUX);
2693                 if (index >= 0 && guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP))
2694                         move_msr_up(vmx, index, save_nmsrs++);
2695                 /*
2696                  * MSR_STAR is only needed on long mode guests, and only
2697                  * if efer.sce is enabled.
2698                  */
2699                 index = __find_msr_index(vmx, MSR_STAR);
2700                 if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
2701                         move_msr_up(vmx, index, save_nmsrs++);
2702         }
2703 #endif
2704         index = __find_msr_index(vmx, MSR_EFER);
2705         if (index >= 0 && update_transition_efer(vmx, index))
2706                 move_msr_up(vmx, index, save_nmsrs++);
2707
2708         vmx->save_nmsrs = save_nmsrs;
2709
2710         if (cpu_has_vmx_msr_bitmap())
2711                 vmx_update_msr_bitmap(&vmx->vcpu);
2712 }
2713
2714 /*
2715  * reads and returns guest's timestamp counter "register"
2716  * guest_tsc = (host_tsc * tsc multiplier) >> 48 + tsc_offset
2717  * -- Intel TSC Scaling for Virtualization White Paper, sec 1.3
2718  */
2719 static u64 guest_read_tsc(struct kvm_vcpu *vcpu)
2720 {
2721         u64 host_tsc, tsc_offset;
2722
2723         host_tsc = rdtsc();
2724         tsc_offset = vmcs_read64(TSC_OFFSET);
2725         return kvm_scale_tsc(vcpu, host_tsc) + tsc_offset;
2726 }
2727
2728 /*
2729  * writes 'offset' into guest's timestamp counter offset register
2730  */
2731 static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
2732 {
2733         if (is_guest_mode(vcpu)) {
2734                 /*
2735                  * We're here if L1 chose not to trap WRMSR to TSC. According
2736                  * to the spec, this should set L1's TSC; The offset that L1
2737                  * set for L2 remains unchanged, and still needs to be added
2738                  * to the newly set TSC to get L2's TSC.
2739                  */
2740                 struct vmcs12 *vmcs12;
2741                 /* recalculate vmcs02.TSC_OFFSET: */
2742                 vmcs12 = get_vmcs12(vcpu);
2743                 vmcs_write64(TSC_OFFSET, offset +
2744                         (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ?
2745                          vmcs12->tsc_offset : 0));
2746         } else {
2747                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
2748                                            vmcs_read64(TSC_OFFSET), offset);
2749                 vmcs_write64(TSC_OFFSET, offset);
2750         }
2751 }
2752
2753 /*
2754  * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
2755  * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
2756  * all guests if the "nested" module option is off, and can also be disabled
2757  * for a single guest by disabling its VMX cpuid bit.
2758  */
2759 static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
2760 {
2761         return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
2762 }
2763
2764 /*
2765  * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
2766  * returned for the various VMX controls MSRs when nested VMX is enabled.
2767  * The same values should also be used to verify that vmcs12 control fields are
2768  * valid during nested entry from L1 to L2.
2769  * Each of these control msrs has a low and high 32-bit half: A low bit is on
2770  * if the corresponding bit in the (32-bit) control field *must* be on, and a
2771  * bit in the high half is on if the corresponding bit in the control field
2772  * may be on. See also vmx_control_verify().
2773  */
2774 static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
2775 {
2776         /*
2777          * Note that as a general rule, the high half of the MSRs (bits in
2778          * the control fields which may be 1) should be initialized by the
2779          * intersection of the underlying hardware's MSR (i.e., features which
2780          * can be supported) and the list of features we want to expose -
2781          * because they are known to be properly supported in our code.
2782          * Also, usually, the low half of the MSRs (bits which must be 1) can
2783          * be set to 0, meaning that L1 may turn off any of these bits. The
2784          * reason is that if one of these bits is necessary, it will appear
2785          * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
2786          * fields of vmcs01 and vmcs02, will turn these bits off - and
2787          * nested_vmx_exit_reflected() will not pass related exits to L1.
2788          * These rules have exceptions below.
2789          */
2790
2791         /* pin-based controls */
2792         rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
2793                 vmx->nested.nested_vmx_pinbased_ctls_low,
2794                 vmx->nested.nested_vmx_pinbased_ctls_high);
2795         vmx->nested.nested_vmx_pinbased_ctls_low |=
2796                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
2797         vmx->nested.nested_vmx_pinbased_ctls_high &=
2798                 PIN_BASED_EXT_INTR_MASK |
2799                 PIN_BASED_NMI_EXITING |
2800                 PIN_BASED_VIRTUAL_NMIS;
2801         vmx->nested.nested_vmx_pinbased_ctls_high |=
2802                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
2803                 PIN_BASED_VMX_PREEMPTION_TIMER;
2804         if (kvm_vcpu_apicv_active(&vmx->vcpu))
2805                 vmx->nested.nested_vmx_pinbased_ctls_high |=
2806                         PIN_BASED_POSTED_INTR;
2807
2808         /* exit controls */
2809         rdmsr(MSR_IA32_VMX_EXIT_CTLS,
2810                 vmx->nested.nested_vmx_exit_ctls_low,
2811                 vmx->nested.nested_vmx_exit_ctls_high);
2812         vmx->nested.nested_vmx_exit_ctls_low =
2813                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
2814
2815         vmx->nested.nested_vmx_exit_ctls_high &=
2816 #ifdef CONFIG_X86_64
2817                 VM_EXIT_HOST_ADDR_SPACE_SIZE |
2818 #endif
2819                 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
2820         vmx->nested.nested_vmx_exit_ctls_high |=
2821                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
2822                 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
2823                 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
2824
2825         if (kvm_mpx_supported())
2826                 vmx->nested.nested_vmx_exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
2827
2828         /* We support free control of debug control saving. */
2829         vmx->nested.nested_vmx_exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
2830
2831         /* entry controls */
2832         rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
2833                 vmx->nested.nested_vmx_entry_ctls_low,
2834                 vmx->nested.nested_vmx_entry_ctls_high);
2835         vmx->nested.nested_vmx_entry_ctls_low =
2836                 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
2837         vmx->nested.nested_vmx_entry_ctls_high &=
2838 #ifdef CONFIG_X86_64
2839                 VM_ENTRY_IA32E_MODE |
2840 #endif
2841                 VM_ENTRY_LOAD_IA32_PAT;
2842         vmx->nested.nested_vmx_entry_ctls_high |=
2843                 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
2844         if (kvm_mpx_supported())
2845                 vmx->nested.nested_vmx_entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
2846
2847         /* We support free control of debug control loading. */
2848         vmx->nested.nested_vmx_entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
2849
2850         /* cpu-based controls */
2851         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
2852                 vmx->nested.nested_vmx_procbased_ctls_low,
2853                 vmx->nested.nested_vmx_procbased_ctls_high);
2854         vmx->nested.nested_vmx_procbased_ctls_low =
2855                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
2856         vmx->nested.nested_vmx_procbased_ctls_high &=
2857                 CPU_BASED_VIRTUAL_INTR_PENDING |
2858                 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING |
2859                 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
2860                 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
2861                 CPU_BASED_CR3_STORE_EXITING |
2862 #ifdef CONFIG_X86_64
2863                 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
2864 #endif
2865                 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
2866                 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
2867                 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
2868                 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
2869                 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2870         /*
2871          * We can allow some features even when not supported by the
2872          * hardware. For example, L1 can specify an MSR bitmap - and we
2873          * can use it to avoid exits to L1 - even when L0 runs L2
2874          * without MSR bitmaps.
2875          */
2876         vmx->nested.nested_vmx_procbased_ctls_high |=
2877                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
2878                 CPU_BASED_USE_MSR_BITMAPS;
2879
2880         /* We support free control of CR3 access interception. */
2881         vmx->nested.nested_vmx_procbased_ctls_low &=
2882                 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
2883
2884         /*
2885          * secondary cpu-based controls.  Do not include those that
2886          * depend on CPUID bits, they are added later by vmx_cpuid_update.
2887          */
2888         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
2889                 vmx->nested.nested_vmx_secondary_ctls_low,
2890                 vmx->nested.nested_vmx_secondary_ctls_high);
2891         vmx->nested.nested_vmx_secondary_ctls_low = 0;
2892         vmx->nested.nested_vmx_secondary_ctls_high &=
2893                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2894                 SECONDARY_EXEC_DESC |
2895                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2896                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2897                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2898                 SECONDARY_EXEC_WBINVD_EXITING;
2899
2900         if (enable_ept) {
2901                 /* nested EPT: emulate EPT also to L1 */
2902                 vmx->nested.nested_vmx_secondary_ctls_high |=
2903                         SECONDARY_EXEC_ENABLE_EPT;
2904                 vmx->nested.nested_vmx_ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
2905                          VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT;
2906                 if (cpu_has_vmx_ept_execute_only())
2907                         vmx->nested.nested_vmx_ept_caps |=
2908                                 VMX_EPT_EXECUTE_ONLY_BIT;
2909                 vmx->nested.nested_vmx_ept_caps &= vmx_capability.ept;
2910                 vmx->nested.nested_vmx_ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
2911                         VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
2912                         VMX_EPT_1GB_PAGE_BIT;
2913                 if (enable_ept_ad_bits) {
2914                         vmx->nested.nested_vmx_secondary_ctls_high |=
2915                                 SECONDARY_EXEC_ENABLE_PML;
2916                         vmx->nested.nested_vmx_ept_caps |= VMX_EPT_AD_BIT;
2917                 }
2918         } else
2919                 vmx->nested.nested_vmx_ept_caps = 0;
2920
2921         if (cpu_has_vmx_vmfunc()) {
2922                 vmx->nested.nested_vmx_secondary_ctls_high |=
2923                         SECONDARY_EXEC_ENABLE_VMFUNC;
2924                 /*
2925                  * Advertise EPTP switching unconditionally
2926                  * since we emulate it
2927                  */
2928                 if (enable_ept)
2929                         vmx->nested.nested_vmx_vmfunc_controls =
2930                                 VMX_VMFUNC_EPTP_SWITCHING;
2931         }
2932
2933         /*
2934          * Old versions of KVM use the single-context version without
2935          * checking for support, so declare that it is supported even
2936          * though it is treated as global context.  The alternative is
2937          * not failing the single-context invvpid, and it is worse.
2938          */
2939         if (enable_vpid) {
2940                 vmx->nested.nested_vmx_secondary_ctls_high |=
2941                         SECONDARY_EXEC_ENABLE_VPID;
2942                 vmx->nested.nested_vmx_vpid_caps = VMX_VPID_INVVPID_BIT |
2943                         VMX_VPID_EXTENT_SUPPORTED_MASK;
2944         } else
2945                 vmx->nested.nested_vmx_vpid_caps = 0;
2946
2947         if (enable_unrestricted_guest)
2948                 vmx->nested.nested_vmx_secondary_ctls_high |=
2949                         SECONDARY_EXEC_UNRESTRICTED_GUEST;
2950
2951         /* miscellaneous data */
2952         rdmsr(MSR_IA32_VMX_MISC,
2953                 vmx->nested.nested_vmx_misc_low,
2954                 vmx->nested.nested_vmx_misc_high);
2955         vmx->nested.nested_vmx_misc_low &= VMX_MISC_SAVE_EFER_LMA;
2956         vmx->nested.nested_vmx_misc_low |=
2957                 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
2958                 VMX_MISC_ACTIVITY_HLT;
2959         vmx->nested.nested_vmx_misc_high = 0;
2960
2961         /*
2962          * This MSR reports some information about VMX support. We
2963          * should return information about the VMX we emulate for the
2964          * guest, and the VMCS structure we give it - not about the
2965          * VMX support of the underlying hardware.
2966          */
2967         vmx->nested.nested_vmx_basic =
2968                 VMCS12_REVISION |
2969                 VMX_BASIC_TRUE_CTLS |
2970                 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
2971                 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
2972
2973         if (cpu_has_vmx_basic_inout())
2974                 vmx->nested.nested_vmx_basic |= VMX_BASIC_INOUT;
2975
2976         /*
2977          * These MSRs specify bits which the guest must keep fixed on
2978          * while L1 is in VMXON mode (in L1's root mode, or running an L2).
2979          * We picked the standard core2 setting.
2980          */
2981 #define VMXON_CR0_ALWAYSON     (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
2982 #define VMXON_CR4_ALWAYSON     X86_CR4_VMXE
2983         vmx->nested.nested_vmx_cr0_fixed0 = VMXON_CR0_ALWAYSON;
2984         vmx->nested.nested_vmx_cr4_fixed0 = VMXON_CR4_ALWAYSON;
2985
2986         /* These MSRs specify bits which the guest must keep fixed off. */
2987         rdmsrl(MSR_IA32_VMX_CR0_FIXED1, vmx->nested.nested_vmx_cr0_fixed1);
2988         rdmsrl(MSR_IA32_VMX_CR4_FIXED1, vmx->nested.nested_vmx_cr4_fixed1);
2989
2990         /* highest index: VMX_PREEMPTION_TIMER_VALUE */
2991         vmx->nested.nested_vmx_vmcs_enum = 0x2e;
2992 }
2993
2994 /*
2995  * if fixed0[i] == 1: val[i] must be 1
2996  * if fixed1[i] == 0: val[i] must be 0
2997  */
2998 static inline bool fixed_bits_valid(u64 val, u64 fixed0, u64 fixed1)
2999 {
3000         return ((val & fixed1) | fixed0) == val;
3001 }
3002
3003 static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
3004 {
3005         return fixed_bits_valid(control, low, high);
3006 }
3007
3008 static inline u64 vmx_control_msr(u32 low, u32 high)
3009 {
3010         return low | ((u64)high << 32);
3011 }
3012
3013 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
3014 {
3015         superset &= mask;
3016         subset &= mask;
3017
3018         return (superset | subset) == superset;
3019 }
3020
3021 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
3022 {
3023         const u64 feature_and_reserved =
3024                 /* feature (except bit 48; see below) */
3025                 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
3026                 /* reserved */
3027                 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
3028         u64 vmx_basic = vmx->nested.nested_vmx_basic;
3029
3030         if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
3031                 return -EINVAL;
3032
3033         /*
3034          * KVM does not emulate a version of VMX that constrains physical
3035          * addresses of VMX structures (e.g. VMCS) to 32-bits.
3036          */
3037         if (data & BIT_ULL(48))
3038                 return -EINVAL;
3039
3040         if (vmx_basic_vmcs_revision_id(vmx_basic) !=
3041             vmx_basic_vmcs_revision_id(data))
3042                 return -EINVAL;
3043
3044         if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
3045                 return -EINVAL;
3046
3047         vmx->nested.nested_vmx_basic = data;
3048         return 0;
3049 }
3050
3051 static int
3052 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
3053 {
3054         u64 supported;
3055         u32 *lowp, *highp;
3056
3057         switch (msr_index) {
3058         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
3059                 lowp = &vmx->nested.nested_vmx_pinbased_ctls_low;
3060                 highp = &vmx->nested.nested_vmx_pinbased_ctls_high;
3061                 break;
3062         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
3063                 lowp = &vmx->nested.nested_vmx_procbased_ctls_low;
3064                 highp = &vmx->nested.nested_vmx_procbased_ctls_high;
3065                 break;
3066         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
3067                 lowp = &vmx->nested.nested_vmx_exit_ctls_low;
3068                 highp = &vmx->nested.nested_vmx_exit_ctls_high;
3069                 break;
3070         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
3071                 lowp = &vmx->nested.nested_vmx_entry_ctls_low;
3072                 highp = &vmx->nested.nested_vmx_entry_ctls_high;
3073                 break;
3074         case MSR_IA32_VMX_PROCBASED_CTLS2:
3075                 lowp = &vmx->nested.nested_vmx_secondary_ctls_low;
3076                 highp = &vmx->nested.nested_vmx_secondary_ctls_high;
3077                 break;
3078         default:
3079                 BUG();
3080         }
3081
3082         supported = vmx_control_msr(*lowp, *highp);
3083
3084         /* Check must-be-1 bits are still 1. */
3085         if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
3086                 return -EINVAL;
3087
3088         /* Check must-be-0 bits are still 0. */
3089         if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
3090                 return -EINVAL;
3091
3092         *lowp = data;
3093         *highp = data >> 32;
3094         return 0;
3095 }
3096
3097 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
3098 {
3099         const u64 feature_and_reserved_bits =
3100                 /* feature */
3101                 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
3102                 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
3103                 /* reserved */
3104                 GENMASK_ULL(13, 9) | BIT_ULL(31);
3105         u64 vmx_misc;
3106
3107         vmx_misc = vmx_control_msr(vmx->nested.nested_vmx_misc_low,
3108                                    vmx->nested.nested_vmx_misc_high);
3109
3110         if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
3111                 return -EINVAL;
3112
3113         if ((vmx->nested.nested_vmx_pinbased_ctls_high &
3114              PIN_BASED_VMX_PREEMPTION_TIMER) &&
3115             vmx_misc_preemption_timer_rate(data) !=
3116             vmx_misc_preemption_timer_rate(vmx_misc))
3117                 return -EINVAL;
3118
3119         if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
3120                 return -EINVAL;
3121
3122         if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
3123                 return -EINVAL;
3124
3125         if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
3126                 return -EINVAL;
3127
3128         vmx->nested.nested_vmx_misc_low = data;
3129         vmx->nested.nested_vmx_misc_high = data >> 32;
3130         return 0;
3131 }
3132
3133 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
3134 {
3135         u64 vmx_ept_vpid_cap;
3136
3137         vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.nested_vmx_ept_caps,
3138                                            vmx->nested.nested_vmx_vpid_caps);
3139
3140         /* Every bit is either reserved or a feature bit. */
3141         if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
3142                 return -EINVAL;
3143
3144         vmx->nested.nested_vmx_ept_caps = data;
3145         vmx->nested.nested_vmx_vpid_caps = data >> 32;
3146         return 0;
3147 }
3148
3149 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
3150 {
3151         u64 *msr;
3152
3153         switch (msr_index) {
3154         case MSR_IA32_VMX_CR0_FIXED0:
3155                 msr = &vmx->nested.nested_vmx_cr0_fixed0;
3156                 break;
3157         case MSR_IA32_VMX_CR4_FIXED0:
3158                 msr = &vmx->nested.nested_vmx_cr4_fixed0;
3159                 break;
3160         default:
3161                 BUG();
3162         }
3163
3164         /*
3165          * 1 bits (which indicates bits which "must-be-1" during VMX operation)
3166          * must be 1 in the restored value.
3167          */
3168         if (!is_bitwise_subset(data, *msr, -1ULL))
3169                 return -EINVAL;
3170
3171         *msr = data;
3172         return 0;
3173 }
3174
3175 /*
3176  * Called when userspace is restoring VMX MSRs.
3177  *
3178  * Returns 0 on success, non-0 otherwise.
3179  */
3180 static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
3181 {
3182         struct vcpu_vmx *vmx = to_vmx(vcpu);
3183
3184         switch (msr_index) {
3185         case MSR_IA32_VMX_BASIC:
3186                 return vmx_restore_vmx_basic(vmx, data);
3187         case MSR_IA32_VMX_PINBASED_CTLS:
3188         case MSR_IA32_VMX_PROCBASED_CTLS:
3189         case MSR_IA32_VMX_EXIT_CTLS:
3190         case MSR_IA32_VMX_ENTRY_CTLS:
3191                 /*
3192                  * The "non-true" VMX capability MSRs are generated from the
3193                  * "true" MSRs, so we do not support restoring them directly.
3194                  *
3195                  * If userspace wants to emulate VMX_BASIC[55]=0, userspace
3196                  * should restore the "true" MSRs with the must-be-1 bits
3197                  * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
3198                  * DEFAULT SETTINGS".
3199                  */
3200                 return -EINVAL;
3201         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
3202         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
3203         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
3204         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
3205         case MSR_IA32_VMX_PROCBASED_CTLS2:
3206                 return vmx_restore_control_msr(vmx, msr_index, data);
3207         case MSR_IA32_VMX_MISC:
3208                 return vmx_restore_vmx_misc(vmx, data);
3209         case MSR_IA32_VMX_CR0_FIXED0:
3210         case MSR_IA32_VMX_CR4_FIXED0:
3211                 return vmx_restore_fixed0_msr(vmx, msr_index, data);
3212         case MSR_IA32_VMX_CR0_FIXED1:
3213         case MSR_IA32_VMX_CR4_FIXED1:
3214                 /*
3215                  * These MSRs are generated based on the vCPU's CPUID, so we
3216                  * do not support restoring them directly.
3217                  */
3218                 return -EINVAL;
3219         case MSR_IA32_VMX_EPT_VPID_CAP:
3220                 return vmx_restore_vmx_ept_vpid_cap(vmx, data);
3221         case MSR_IA32_VMX_VMCS_ENUM:
3222                 vmx->nested.nested_vmx_vmcs_enum = data;
3223                 return 0;
3224         default:
3225                 /*
3226                  * The rest of the VMX capability MSRs do not support restore.
3227                  */
3228                 return -EINVAL;
3229         }
3230 }
3231
3232 /* Returns 0 on success, non-0 otherwise. */
3233 static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
3234 {
3235         struct vcpu_vmx *vmx = to_vmx(vcpu);
3236
3237         switch (msr_index) {
3238         case MSR_IA32_VMX_BASIC:
3239                 *pdata = vmx->nested.nested_vmx_basic;
3240                 break;
3241         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
3242         case MSR_IA32_VMX_PINBASED_CTLS:
3243                 *pdata = vmx_control_msr(
3244                         vmx->nested.nested_vmx_pinbased_ctls_low,
3245                         vmx->nested.nested_vmx_pinbased_ctls_high);
3246                 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
3247                         *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
3248                 break;
3249         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
3250         case MSR_IA32_VMX_PROCBASED_CTLS:
3251                 *pdata = vmx_control_msr(
3252                         vmx->nested.nested_vmx_procbased_ctls_low,
3253                         vmx->nested.nested_vmx_procbased_ctls_high);
3254                 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
3255                         *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
3256                 break;
3257         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
3258         case MSR_IA32_VMX_EXIT_CTLS:
3259                 *pdata = vmx_control_msr(
3260                         vmx->nested.nested_vmx_exit_ctls_low,
3261                         vmx->nested.nested_vmx_exit_ctls_high);
3262                 if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
3263                         *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
3264                 break;
3265         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
3266         case MSR_IA32_VMX_ENTRY_CTLS:
3267                 *pdata = vmx_control_msr(
3268                         vmx->nested.nested_vmx_entry_ctls_low,
3269                         vmx->nested.nested_vmx_entry_ctls_high);
3270                 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
3271                         *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
3272                 break;
3273         case MSR_IA32_VMX_MISC:
3274                 *pdata = vmx_control_msr(
3275                         vmx->nested.nested_vmx_misc_low,
3276                         vmx->nested.nested_vmx_misc_high);
3277                 break;
3278         case MSR_IA32_VMX_CR0_FIXED0:
3279                 *pdata = vmx->nested.nested_vmx_cr0_fixed0;
3280                 break;
3281         case MSR_IA32_VMX_CR0_FIXED1:
3282                 *pdata = vmx->nested.nested_vmx_cr0_fixed1;
3283                 break;
3284         case MSR_IA32_VMX_CR4_FIXED0:
3285                 *pdata = vmx->nested.nested_vmx_cr4_fixed0;
3286                 break;
3287         case MSR_IA32_VMX_CR4_FIXED1:
3288                 *pdata = vmx->nested.nested_vmx_cr4_fixed1;
3289                 break;
3290         case MSR_IA32_VMX_VMCS_ENUM:
3291                 *pdata = vmx->nested.nested_vmx_vmcs_enum;
3292                 break;
3293         case MSR_IA32_VMX_PROCBASED_CTLS2:
3294                 *pdata = vmx_control_msr(
3295                         vmx->nested.nested_vmx_secondary_ctls_low,
3296                         vmx->nested.nested_vmx_secondary_ctls_high);
3297                 break;
3298         case MSR_IA32_VMX_EPT_VPID_CAP:
3299                 *pdata = vmx->nested.nested_vmx_ept_caps |
3300                         ((u64)vmx->nested.nested_vmx_vpid_caps << 32);
3301                 break;
3302         case MSR_IA32_VMX_VMFUNC:
3303                 *pdata = vmx->nested.nested_vmx_vmfunc_controls;
3304                 break;
3305         default:
3306                 return 1;
3307         }
3308
3309         return 0;
3310 }
3311
3312 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
3313                                                  uint64_t val)
3314 {
3315         uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
3316
3317         return !(val & ~valid_bits);
3318 }
3319
3320 /*
3321  * Reads an msr value (of 'msr_index') into 'pdata'.
3322  * Returns 0 on success, non-0 otherwise.
3323  * Assumes vcpu_load() was already called.
3324  */
3325 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3326 {
3327         struct shared_msr_entry *msr;
3328
3329         switch (msr_info->index) {
3330 #ifdef CONFIG_X86_64
3331         case MSR_FS_BASE:
3332                 msr_info->data = vmcs_readl(GUEST_FS_BASE);
3333                 break;
3334         case MSR_GS_BASE:
3335                 msr_info->data = vmcs_readl(GUEST_GS_BASE);
3336                 break;
3337         case MSR_KERNEL_GS_BASE:
3338                 vmx_load_host_state(to_vmx(vcpu));
3339                 msr_info->data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
3340                 break;
3341 #endif
3342         case MSR_EFER:
3343                 return kvm_get_msr_common(vcpu, msr_info);
3344         case MSR_IA32_TSC:
3345                 msr_info->data = guest_read_tsc(vcpu);
3346                 break;
3347         case MSR_IA32_SPEC_CTRL:
3348                 if (!msr_info->host_initiated &&
3349                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
3350                         return 1;
3351
3352                 msr_info->data = to_vmx(vcpu)->spec_ctrl;
3353                 break;
3354         case MSR_IA32_ARCH_CAPABILITIES:
3355                 if (!msr_info->host_initiated &&
3356                     !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
3357                         return 1;
3358                 msr_info->data = to_vmx(vcpu)->arch_capabilities;
3359                 break;
3360         case MSR_IA32_SYSENTER_CS:
3361                 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
3362                 break;
3363         case MSR_IA32_SYSENTER_EIP:
3364                 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
3365                 break;
3366         case MSR_IA32_SYSENTER_ESP:
3367                 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
3368                 break;
3369         case MSR_IA32_BNDCFGS:
3370                 if (!kvm_mpx_supported() ||
3371                     (!msr_info->host_initiated &&
3372                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
3373                         return 1;
3374                 msr_info->data = vmcs_read64(GUEST_BNDCFGS);
3375                 break;
3376         case MSR_IA32_MCG_EXT_CTL:
3377                 if (!msr_info->host_initiated &&
3378                     !(to_vmx(vcpu)->msr_ia32_feature_control &
3379                       FEATURE_CONTROL_LMCE))
3380                         return 1;
3381                 msr_info->data = vcpu->arch.mcg_ext_ctl;
3382                 break;
3383         case MSR_IA32_FEATURE_CONTROL:
3384                 msr_info->data = to_vmx(vcpu)->msr_ia32_feature_control;
3385                 break;
3386         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
3387                 if (!nested_vmx_allowed(vcpu))
3388                         return 1;
3389                 return vmx_get_vmx_msr(vcpu, msr_info->index, &msr_info->data);
3390         case MSR_IA32_XSS:
3391                 if (!vmx_xsaves_supported())
3392                         return 1;
3393                 msr_info->data = vcpu->arch.ia32_xss;
3394                 break;
3395         case MSR_TSC_AUX:
3396                 if (!msr_info->host_initiated &&
3397                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
3398                         return 1;
3399                 /* Otherwise falls through */
3400         default:
3401                 msr = find_msr_entry(to_vmx(vcpu), msr_info->index);
3402                 if (msr) {
3403                         msr_info->data = msr->data;
3404                         break;
3405                 }
3406                 return kvm_get_msr_common(vcpu, msr_info);
3407         }
3408
3409         return 0;
3410 }
3411
3412 static void vmx_leave_nested(struct kvm_vcpu *vcpu);
3413
3414 /*
3415  * Writes msr value into into the appropriate "register".
3416  * Returns 0 on success, non-0 otherwise.
3417  * Assumes vcpu_load() was already called.
3418  */
3419 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3420 {
3421         struct vcpu_vmx *vmx = to_vmx(vcpu);
3422         struct shared_msr_entry *msr;
3423         int ret = 0;
3424         u32 msr_index = msr_info->index;
3425         u64 data = msr_info->data;
3426
3427         switch (msr_index) {
3428         case MSR_EFER:
3429                 ret = kvm_set_msr_common(vcpu, msr_info);
3430                 break;
3431 #ifdef CONFIG_X86_64
3432         case MSR_FS_BASE:
3433                 vmx_segment_cache_clear(vmx);
3434                 vmcs_writel(GUEST_FS_BASE, data);
3435                 break;
3436         case MSR_GS_BASE:
3437                 vmx_segment_cache_clear(vmx);
3438                 vmcs_writel(GUEST_GS_BASE, data);
3439                 break;
3440         case MSR_KERNEL_GS_BASE:
3441                 vmx_load_host_state(vmx);
3442                 vmx->msr_guest_kernel_gs_base = data;
3443                 break;
3444 #endif
3445         case MSR_IA32_SYSENTER_CS:
3446                 vmcs_write32(GUEST_SYSENTER_CS, data);
3447                 break;
3448         case MSR_IA32_SYSENTER_EIP:
3449                 vmcs_writel(GUEST_SYSENTER_EIP, data);
3450                 break;
3451         case MSR_IA32_SYSENTER_ESP:
3452                 vmcs_writel(GUEST_SYSENTER_ESP, data);
3453                 break;
3454         case MSR_IA32_BNDCFGS:
3455                 if (!kvm_mpx_supported() ||
3456                     (!msr_info->host_initiated &&
3457                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
3458                         return 1;
3459                 if (is_noncanonical_address(data & PAGE_MASK, vcpu) ||
3460                     (data & MSR_IA32_BNDCFGS_RSVD))
3461                         return 1;
3462                 vmcs_write64(GUEST_BNDCFGS, data);
3463                 break;
3464         case MSR_IA32_TSC:
3465                 kvm_write_tsc(vcpu, msr_info);
3466                 break;
3467         case MSR_IA32_SPEC_CTRL:
3468                 if (!msr_info->host_initiated &&
3469                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
3470                         return 1;
3471
3472                 /* The STIBP bit doesn't fault even if it's not advertised */
3473                 if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD))
3474                         return 1;
3475
3476                 vmx->spec_ctrl = data;
3477
3478                 if (!data)
3479                         break;
3480
3481                 /*
3482                  * For non-nested:
3483                  * When it's written (to non-zero) for the first time, pass
3484                  * it through.
3485                  *
3486                  * For nested:
3487                  * The handling of the MSR bitmap for L2 guests is done in
3488                  * nested_vmx_merge_msr_bitmap. We should not touch the
3489                  * vmcs02.msr_bitmap here since it gets completely overwritten
3490                  * in the merging. We update the vmcs01 here for L1 as well
3491                  * since it will end up touching the MSR anyway now.
3492                  */
3493                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap,
3494                                               MSR_IA32_SPEC_CTRL,
3495                                               MSR_TYPE_RW);
3496                 break;
3497         case MSR_IA32_PRED_CMD:
3498                 if (!msr_info->host_initiated &&
3499                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
3500                         return 1;
3501
3502                 if (data & ~PRED_CMD_IBPB)
3503                         return 1;
3504
3505                 if (!data)
3506                         break;
3507
3508                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
3509
3510                 /*
3511                  * For non-nested:
3512                  * When it's written (to non-zero) for the first time, pass
3513                  * it through.
3514                  *
3515                  * For nested:
3516                  * The handling of the MSR bitmap for L2 guests is done in
3517                  * nested_vmx_merge_msr_bitmap. We should not touch the
3518                  * vmcs02.msr_bitmap here since it gets completely overwritten
3519                  * in the merging.
3520                  */
3521                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, MSR_IA32_PRED_CMD,
3522                                               MSR_TYPE_W);
3523                 break;
3524         case MSR_IA32_ARCH_CAPABILITIES:
3525                 if (!msr_info->host_initiated)
3526                         return 1;
3527                 vmx->arch_capabilities = data;
3528                 break;
3529         case MSR_IA32_CR_PAT:
3530                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
3531                         if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
3532                                 return 1;
3533                         vmcs_write64(GUEST_IA32_PAT, data);
3534                         vcpu->arch.pat = data;
3535                         break;
3536                 }
3537                 ret = kvm_set_msr_common(vcpu, msr_info);
3538                 break;
3539         case MSR_IA32_TSC_ADJUST:
3540                 ret = kvm_set_msr_common(vcpu, msr_info);
3541                 break;
3542         case MSR_IA32_MCG_EXT_CTL:
3543                 if ((!msr_info->host_initiated &&
3544                      !(to_vmx(vcpu)->msr_ia32_feature_control &
3545                        FEATURE_CONTROL_LMCE)) ||
3546                     (data & ~MCG_EXT_CTL_LMCE_EN))
3547                         return 1;
3548                 vcpu->arch.mcg_ext_ctl = data;
3549                 break;
3550         case MSR_IA32_FEATURE_CONTROL:
3551                 if (!vmx_feature_control_msr_valid(vcpu, data) ||
3552                     (to_vmx(vcpu)->msr_ia32_feature_control &
3553                      FEATURE_CONTROL_LOCKED && !msr_info->host_initiated))
3554                         return 1;
3555                 vmx->msr_ia32_feature_control = data;
3556                 if (msr_info->host_initiated && data == 0)
3557                         vmx_leave_nested(vcpu);
3558                 break;
3559         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
3560                 if (!msr_info->host_initiated)
3561                         return 1; /* they are read-only */
3562                 if (!nested_vmx_allowed(vcpu))
3563                         return 1;
3564                 return vmx_set_vmx_msr(vcpu, msr_index, data);
3565         case MSR_IA32_XSS:
3566                 if (!vmx_xsaves_supported())
3567                         return 1;
3568                 /*
3569                  * The only supported bit as of Skylake is bit 8, but
3570                  * it is not supported on KVM.
3571                  */
3572                 if (data != 0)
3573                         return 1;
3574                 vcpu->arch.ia32_xss = data;
3575                 if (vcpu->arch.ia32_xss != host_xss)
3576                         add_atomic_switch_msr(vmx, MSR_IA32_XSS,
3577                                 vcpu->arch.ia32_xss, host_xss);
3578                 else
3579                         clear_atomic_switch_msr(vmx, MSR_IA32_XSS);
3580                 break;
3581         case MSR_TSC_AUX:
3582                 if (!msr_info->host_initiated &&
3583                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
3584                         return 1;
3585                 /* Check reserved bit, higher 32 bits should be zero */
3586                 if ((data >> 32) != 0)
3587                         return 1;
3588                 /* Otherwise falls through */
3589         default:
3590                 msr = find_msr_entry(vmx, msr_index);
3591                 if (msr) {
3592                         u64 old_msr_data = msr->data;
3593                         msr->data = data;
3594                         if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
3595                                 preempt_disable();
3596                                 ret = kvm_set_shared_msr(msr->index, msr->data,
3597                                                          msr->mask);
3598                                 preempt_enable();
3599                                 if (ret)
3600                                         msr->data = old_msr_data;
3601                         }
3602                         break;
3603                 }
3604                 ret = kvm_set_msr_common(vcpu, msr_info);
3605         }
3606
3607         return ret;
3608 }
3609
3610 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
3611 {
3612         __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
3613         switch (reg) {
3614         case VCPU_REGS_RSP:
3615                 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
3616                 break;
3617         case VCPU_REGS_RIP:
3618                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
3619                 break;
3620         case VCPU_EXREG_PDPTR:
3621                 if (enable_ept)
3622                         ept_save_pdptrs(vcpu);
3623                 break;
3624         default:
3625                 break;
3626         }
3627 }
3628
3629 static __init int cpu_has_kvm_support(void)
3630 {
3631         return cpu_has_vmx();
3632 }
3633
3634 static __init int vmx_disabled_by_bios(void)
3635 {
3636         u64 msr;
3637
3638         rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
3639         if (msr & FEATURE_CONTROL_LOCKED) {
3640                 /* launched w/ TXT and VMX disabled */
3641                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
3642                         && tboot_enabled())
3643                         return 1;
3644                 /* launched w/o TXT and VMX only enabled w/ TXT */
3645                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
3646                         && (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
3647                         && !tboot_enabled()) {
3648                         printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
3649                                 "activate TXT before enabling KVM\n");
3650                         return 1;
3651                 }
3652                 /* launched w/o TXT and VMX disabled */
3653                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
3654                         && !tboot_enabled())
3655                         return 1;
3656         }
3657
3658         return 0;
3659 }
3660
3661 static void kvm_cpu_vmxon(u64 addr)
3662 {
3663         cr4_set_bits(X86_CR4_VMXE);
3664         intel_pt_handle_vmx(1);
3665
3666         asm volatile (ASM_VMX_VMXON_RAX
3667                         : : "a"(&addr), "m"(addr)
3668                         : "memory", "cc");
3669 }
3670
3671 static int hardware_enable(void)
3672 {
3673         int cpu = raw_smp_processor_id();
3674         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
3675         u64 old, test_bits;
3676
3677         if (cr4_read_shadow() & X86_CR4_VMXE)
3678                 return -EBUSY;
3679
3680         INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
3681         INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
3682         spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
3683
3684         /*
3685          * Now we can enable the vmclear operation in kdump
3686          * since the loaded_vmcss_on_cpu list on this cpu
3687          * has been initialized.
3688          *
3689          * Though the cpu is not in VMX operation now, there
3690          * is no problem to enable the vmclear operation
3691          * for the loaded_vmcss_on_cpu list is empty!
3692          */
3693         crash_enable_local_vmclear(cpu);
3694
3695         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
3696
3697         test_bits = FEATURE_CONTROL_LOCKED;
3698         test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
3699         if (tboot_enabled())
3700                 test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;
3701
3702         if ((old & test_bits) != test_bits) {
3703                 /* enable and lock */
3704                 wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
3705         }
3706         kvm_cpu_vmxon(phys_addr);
3707         ept_sync_global();
3708
3709         return 0;
3710 }
3711
3712 static void vmclear_local_loaded_vmcss(void)
3713 {
3714         int cpu = raw_smp_processor_id();
3715         struct loaded_vmcs *v, *n;
3716
3717         list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
3718                                  loaded_vmcss_on_cpu_link)
3719                 __loaded_vmcs_clear(v);
3720 }
3721
3722
3723 /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
3724  * tricks.
3725  */
3726 static void kvm_cpu_vmxoff(void)
3727 {
3728         asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
3729
3730         intel_pt_handle_vmx(0);
3731         cr4_clear_bits(X86_CR4_VMXE);
3732 }
3733
3734 static void hardware_disable(void)
3735 {
3736         vmclear_local_loaded_vmcss();
3737         kvm_cpu_vmxoff();
3738 }
3739
3740 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
3741                                       u32 msr, u32 *result)
3742 {
3743         u32 vmx_msr_low, vmx_msr_high;
3744         u32 ctl = ctl_min | ctl_opt;
3745
3746         rdmsr(msr, vmx_msr_low, vmx_msr_high);
3747
3748         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
3749         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
3750
3751         /* Ensure minimum (required) set of control bits are supported. */
3752         if (ctl_min & ~ctl)
3753                 return -EIO;
3754
3755         *result = ctl;
3756         return 0;
3757 }
3758
3759 static __init bool allow_1_setting(u32 msr, u32 ctl)
3760 {
3761         u32 vmx_msr_low, vmx_msr_high;
3762
3763         rdmsr(msr, vmx_msr_low, vmx_msr_high);
3764         return vmx_msr_high & ctl;
3765 }
3766
3767 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
3768 {
3769         u32 vmx_msr_low, vmx_msr_high;
3770         u32 min, opt, min2, opt2;
3771         u32 _pin_based_exec_control = 0;
3772         u32 _cpu_based_exec_control = 0;
3773         u32 _cpu_based_2nd_exec_control = 0;
3774         u32 _vmexit_control = 0;
3775         u32 _vmentry_control = 0;
3776
3777         min = CPU_BASED_HLT_EXITING |
3778 #ifdef CONFIG_X86_64
3779               CPU_BASED_CR8_LOAD_EXITING |
3780               CPU_BASED_CR8_STORE_EXITING |
3781 #endif
3782               CPU_BASED_CR3_LOAD_EXITING |
3783               CPU_BASED_CR3_STORE_EXITING |
3784               CPU_BASED_USE_IO_BITMAPS |
3785               CPU_BASED_MOV_DR_EXITING |
3786               CPU_BASED_USE_TSC_OFFSETING |
3787               CPU_BASED_INVLPG_EXITING |
3788               CPU_BASED_RDPMC_EXITING;
3789
3790         if (!kvm_mwait_in_guest())
3791                 min |= CPU_BASED_MWAIT_EXITING |
3792                         CPU_BASED_MONITOR_EXITING;
3793
3794         opt = CPU_BASED_TPR_SHADOW |
3795               CPU_BASED_USE_MSR_BITMAPS |
3796               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
3797         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
3798                                 &_cpu_based_exec_control) < 0)
3799                 return -EIO;
3800 #ifdef CONFIG_X86_64
3801         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
3802                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
3803                                            ~CPU_BASED_CR8_STORE_EXITING;
3804 #endif
3805         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
3806                 min2 = 0;
3807                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
3808                         SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
3809                         SECONDARY_EXEC_WBINVD_EXITING |
3810                         SECONDARY_EXEC_ENABLE_VPID |
3811                         SECONDARY_EXEC_ENABLE_EPT |
3812                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
3813                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
3814                         SECONDARY_EXEC_RDTSCP |
3815                         SECONDARY_EXEC_ENABLE_INVPCID |
3816                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
3817                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
3818                         SECONDARY_EXEC_SHADOW_VMCS |
3819                         SECONDARY_EXEC_XSAVES |
3820                         SECONDARY_EXEC_RDSEED |
3821                         SECONDARY_EXEC_RDRAND |
3822                         SECONDARY_EXEC_ENABLE_PML |
3823                         SECONDARY_EXEC_TSC_SCALING |
3824                         SECONDARY_EXEC_ENABLE_VMFUNC;
3825                 if (adjust_vmx_controls(min2, opt2,
3826                                         MSR_IA32_VMX_PROCBASED_CTLS2,
3827                                         &_cpu_based_2nd_exec_control) < 0)
3828                         return -EIO;
3829         }
3830 #ifndef CONFIG_X86_64
3831         if (!(_cpu_based_2nd_exec_control &
3832                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
3833                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
3834 #endif
3835
3836         if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
3837                 _cpu_based_2nd_exec_control &= ~(
3838                                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
3839                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
3840                                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
3841
3842         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
3843                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
3844                    enabled */
3845                 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
3846                                              CPU_BASED_CR3_STORE_EXITING |
3847                                              CPU_BASED_INVLPG_EXITING);
3848                 rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
3849                       vmx_capability.ept, vmx_capability.vpid);
3850         }
3851
3852         min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
3853 #ifdef CONFIG_X86_64
3854         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
3855 #endif
3856         opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT |
3857                 VM_EXIT_CLEAR_BNDCFGS;
3858         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
3859                                 &_vmexit_control) < 0)
3860                 return -EIO;
3861
3862         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
3863         opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
3864                  PIN_BASED_VMX_PREEMPTION_TIMER;
3865         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
3866                                 &_pin_based_exec_control) < 0)
3867                 return -EIO;
3868
3869         if (cpu_has_broken_vmx_preemption_timer())
3870                 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
3871         if (!(_cpu_based_2nd_exec_control &
3872                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
3873                 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
3874
3875         min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
3876         opt = VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS;
3877         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
3878                                 &_vmentry_control) < 0)
3879                 return -EIO;
3880
3881         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
3882
3883         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
3884         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
3885                 return -EIO;
3886
3887 #ifdef CONFIG_X86_64
3888         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
3889         if (vmx_msr_high & (1u<<16))
3890                 return -EIO;
3891 #endif
3892
3893         /* Require Write-Back (WB) memory type for VMCS accesses. */
3894         if (((vmx_msr_high >> 18) & 15) != 6)
3895                 return -EIO;
3896
3897         vmcs_conf->size = vmx_msr_high & 0x1fff;
3898         vmcs_conf->order = get_order(vmcs_conf->size);
3899         vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
3900         vmcs_conf->revision_id = vmx_msr_low;
3901
3902         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
3903         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
3904         vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
3905         vmcs_conf->vmexit_ctrl         = _vmexit_control;
3906         vmcs_conf->vmentry_ctrl        = _vmentry_control;
3907
3908         cpu_has_load_ia32_efer =
3909                 allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
3910                                 VM_ENTRY_LOAD_IA32_EFER)
3911                 && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
3912                                    VM_EXIT_LOAD_IA32_EFER);
3913
3914         cpu_has_load_perf_global_ctrl =
3915                 allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
3916                                 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
3917                 && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
3918                                    VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
3919
3920         /*
3921          * Some cpus support VM_ENTRY_(LOAD|SAVE)_IA32_PERF_GLOBAL_CTRL
3922          * but due to errata below it can't be used. Workaround is to use
3923          * msr load mechanism to switch IA32_PERF_GLOBAL_CTRL.
3924          *
3925          * VM Exit May Incorrectly Clear IA32_PERF_GLOBAL_CTRL [34:32]
3926          *
3927          * AAK155             (model 26)
3928          * AAP115             (model 30)
3929          * AAT100             (model 37)
3930          * BC86,AAY89,BD102   (model 44)
3931          * BA97               (model 46)
3932          *
3933          */
3934         if (cpu_has_load_perf_global_ctrl && boot_cpu_data.x86 == 0x6) {
3935                 switch (boot_cpu_data.x86_model) {
3936                 case 26:
3937                 case 30:
3938                 case 37:
3939                 case 44:
3940                 case 46:
3941                         cpu_has_load_perf_global_ctrl = false;
3942                         printk_once(KERN_WARNING"kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
3943                                         "does not work properly. Using workaround\n");
3944                         break;
3945                 default:
3946                         break;
3947                 }
3948         }
3949
3950         if (boot_cpu_has(X86_FEATURE_XSAVES))
3951                 rdmsrl(MSR_IA32_XSS, host_xss);
3952
3953         return 0;
3954 }
3955
3956 static struct vmcs *alloc_vmcs_cpu(int cpu)
3957 {
3958         int node = cpu_to_node(cpu);
3959         struct page *pages;
3960         struct vmcs *vmcs;
3961
3962         pages = __alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);
3963         if (!pages)
3964                 return NULL;
3965         vmcs = page_address(pages);
3966         memset(vmcs, 0, vmcs_config.size);
3967         vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
3968         return vmcs;
3969 }
3970
3971 static void free_vmcs(struct vmcs *vmcs)
3972 {
3973         free_pages((unsigned long)vmcs, vmcs_config.order);
3974 }
3975
3976 /*
3977  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
3978  */
3979 static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
3980 {
3981         if (!loaded_vmcs->vmcs)
3982                 return;
3983         loaded_vmcs_clear(loaded_vmcs);
3984         free_vmcs(loaded_vmcs->vmcs);
3985         loaded_vmcs->vmcs = NULL;
3986         if (loaded_vmcs->msr_bitmap)
3987                 free_page((unsigned long)loaded_vmcs->msr_bitmap);
3988         WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
3989 }
3990
3991 static struct vmcs *alloc_vmcs(void)
3992 {
3993         return alloc_vmcs_cpu(raw_smp_processor_id());
3994 }
3995
3996 static int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
3997 {
3998         loaded_vmcs->vmcs = alloc_vmcs();
3999         if (!loaded_vmcs->vmcs)
4000                 return -ENOMEM;
4001
4002         loaded_vmcs->shadow_vmcs = NULL;
4003         loaded_vmcs_init(loaded_vmcs);
4004
4005         if (cpu_has_vmx_msr_bitmap()) {
4006                 loaded_vmcs->msr_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
4007                 if (!loaded_vmcs->msr_bitmap)
4008                         goto out_vmcs;
4009                 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
4010         }
4011         return 0;
4012
4013 out_vmcs:
4014         free_loaded_vmcs(loaded_vmcs);
4015         return -ENOMEM;
4016 }
4017
4018 static void free_kvm_area(void)
4019 {
4020         int cpu;
4021
4022         for_each_possible_cpu(cpu) {
4023                 free_vmcs(per_cpu(vmxarea, cpu));
4024                 per_cpu(vmxarea, cpu) = NULL;
4025         }
4026 }
4027
4028 enum vmcs_field_type {
4029         VMCS_FIELD_TYPE_U16 = 0,
4030         VMCS_FIELD_TYPE_U64 = 1,
4031         VMCS_FIELD_TYPE_U32 = 2,
4032         VMCS_FIELD_TYPE_NATURAL_WIDTH = 3
4033 };
4034
4035 static inline int vmcs_field_type(unsigned long field)
4036 {
4037         if (0x1 & field)        /* the *_HIGH fields are all 32 bit */
4038                 return VMCS_FIELD_TYPE_U32;
4039         return (field >> 13) & 0x3 ;
4040 }
4041
4042 static inline int vmcs_field_readonly(unsigned long field)
4043 {
4044         return (((field >> 10) & 0x3) == 1);
4045 }
4046
4047 static void init_vmcs_shadow_fields(void)
4048 {
4049         int i, j;
4050
4051         /* No checks for read only fields yet */
4052
4053         for (i = j = 0; i < max_shadow_read_write_fields; i++) {
4054                 switch (shadow_read_write_fields[i]) {
4055                 case GUEST_BNDCFGS:
4056                         if (!kvm_mpx_supported())
4057                                 continue;
4058                         break;
4059                 default:
4060                         break;
4061                 }
4062
4063                 if (j < i)
4064                         shadow_read_write_fields[j] =
4065                                 shadow_read_write_fields[i];
4066                 j++;
4067         }
4068         max_shadow_read_write_fields = j;
4069
4070         /* shadowed fields guest access without vmexit */
4071         for (i = 0; i < max_shadow_read_write_fields; i++) {
4072                 unsigned long field = shadow_read_write_fields[i];
4073
4074                 clear_bit(field, vmx_vmwrite_bitmap);
4075                 clear_bit(field, vmx_vmread_bitmap);
4076                 if (vmcs_field_type(field) == VMCS_FIELD_TYPE_U64) {
4077                         clear_bit(field + 1, vmx_vmwrite_bitmap);
4078                         clear_bit(field + 1, vmx_vmread_bitmap);
4079                 }
4080         }
4081         for (i = 0; i < max_shadow_read_only_fields; i++) {
4082                 unsigned long field = shadow_read_only_fields[i];
4083
4084                 clear_bit(field, vmx_vmread_bitmap);
4085                 if (vmcs_field_type(field) == VMCS_FIELD_TYPE_U64)
4086                         clear_bit(field + 1, vmx_vmread_bitmap);
4087         }
4088 }
4089
4090 static __init int alloc_kvm_area(void)
4091 {
4092         int cpu;
4093
4094         for_each_possible_cpu(cpu) {
4095                 struct vmcs *vmcs;
4096
4097                 vmcs = alloc_vmcs_cpu(cpu);
4098                 if (!vmcs) {
4099                         free_kvm_area();
4100                         return -ENOMEM;
4101                 }
4102
4103                 per_cpu(vmxarea, cpu) = vmcs;
4104         }
4105         return 0;
4106 }
4107
4108 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
4109                 struct kvm_segment *save)
4110 {
4111         if (!emulate_invalid_guest_state) {
4112                 /*
4113                  * CS and SS RPL should be equal during guest entry according
4114                  * to VMX spec, but in reality it is not always so. Since vcpu
4115                  * is in the middle of the transition from real mode to
4116                  * protected mode it is safe to assume that RPL 0 is a good
4117                  * default value.
4118                  */
4119                 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
4120                         save->selector &= ~SEGMENT_RPL_MASK;
4121                 save->dpl = save->selector & SEGMENT_RPL_MASK;
4122                 save->s = 1;
4123         }
4124         vmx_set_segment(vcpu, save, seg);
4125 }
4126
4127 static void enter_pmode(struct kvm_vcpu *vcpu)
4128 {
4129         unsigned long flags;
4130         struct vcpu_vmx *vmx = to_vmx(vcpu);
4131
4132         /*
4133          * Update real mode segment cache. It may be not up-to-date if sement
4134          * register was written while vcpu was in a guest mode.
4135          */
4136         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
4137         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
4138         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
4139         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
4140         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
4141         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
4142
4143         vmx->rmode.vm86_active = 0;
4144
4145         vmx_segment_cache_clear(vmx);
4146
4147         vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
4148
4149         flags = vmcs_readl(GUEST_RFLAGS);
4150         flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
4151         flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
4152         vmcs_writel(GUEST_RFLAGS, flags);
4153
4154         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
4155                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
4156
4157         update_exception_bitmap(vcpu);
4158
4159         fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
4160         fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
4161         fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
4162         fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
4163         fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
4164         fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
4165 }
4166
4167 static void fix_rmode_seg(int seg, struct kvm_segment *save)
4168 {
4169         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
4170         struct kvm_segment var = *save;
4171
4172         var.dpl = 0x3;
4173         if (seg == VCPU_SREG_CS)
4174                 var.type = 0x3;
4175
4176         if (!emulate_invalid_guest_state) {
4177                 var.selector = var.base >> 4;
4178                 var.base = var.base & 0xffff0;
4179                 var.limit = 0xffff;
4180                 var.g = 0;
4181                 var.db = 0;
4182                 var.present = 1;
4183                 var.s = 1;
4184                 var.l = 0;
4185                 var.unusable = 0;
4186                 var.type = 0x3;
4187                 var.avl = 0;
4188                 if (save->base & 0xf)
4189                         printk_once(KERN_WARNING "kvm: segment base is not "
4190                                         "paragraph aligned when entering "
4191                                         "protected mode (seg=%d)", seg);
4192         }
4193
4194         vmcs_write16(sf->selector, var.selector);
4195         vmcs_writel(sf->base, var.base);
4196         vmcs_write32(sf->limit, var.limit);
4197         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
4198 }
4199
4200 static void enter_rmode(struct kvm_vcpu *vcpu)
4201 {
4202         unsigned long flags;
4203         struct vcpu_vmx *vmx = to_vmx(vcpu);
4204
4205         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
4206         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
4207         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
4208         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
4209         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
4210         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
4211         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
4212
4213         vmx->rmode.vm86_active = 1;
4214
4215         /*
4216          * Very old userspace does not call KVM_SET_TSS_ADDR before entering
4217          * vcpu. Warn the user that an update is overdue.
4218          */
4219         if (!vcpu->kvm->arch.tss_addr)
4220                 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
4221                              "called before entering vcpu\n");
4222
4223         vmx_segment_cache_clear(vmx);
4224
4225         vmcs_writel(GUEST_TR_BASE, vcpu->kvm->arch.tss_addr);
4226         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
4227         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
4228
4229         flags = vmcs_readl(GUEST_RFLAGS);
4230         vmx->rmode.save_rflags = flags;
4231
4232         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
4233
4234         vmcs_writel(GUEST_RFLAGS, flags);
4235         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
4236         update_exception_bitmap(vcpu);
4237
4238         fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
4239         fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
4240         fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
4241         fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
4242         fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
4243         fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
4244
4245         kvm_mmu_reset_context(vcpu);
4246 }
4247
4248 static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
4249 {
4250         struct vcpu_vmx *vmx = to_vmx(vcpu);
4251         struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
4252
4253         if (!msr)
4254                 return;
4255
4256         /*
4257          * Force kernel_gs_base reloading before EFER changes, as control
4258          * of this msr depends on is_long_mode().
4259          */
4260         vmx_load_host_state(to_vmx(vcpu));
4261         vcpu->arch.efer = efer;
4262         if (efer & EFER_LMA) {
4263                 vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
4264                 msr->data = efer;
4265         } else {
4266                 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
4267
4268                 msr->data = efer & ~EFER_LME;
4269         }
4270         setup_msrs(vmx);
4271 }
4272
4273 #ifdef CONFIG_X86_64
4274
4275 static void enter_lmode(struct kvm_vcpu *vcpu)
4276 {
4277         u32 guest_tr_ar;
4278
4279         vmx_segment_cache_clear(to_vmx(vcpu));
4280
4281         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
4282         if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
4283                 pr_debug_ratelimited("%s: tss fixup for long mode. \n",
4284                                      __func__);
4285                 vmcs_write32(GUEST_TR_AR_BYTES,
4286                              (guest_tr_ar & ~VMX_AR_TYPE_MASK)
4287                              | VMX_AR_TYPE_BUSY_64_TSS);
4288         }
4289         vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
4290 }
4291
4292 static void exit_lmode(struct kvm_vcpu *vcpu)
4293 {
4294         vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
4295         vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
4296 }
4297
4298 #endif
4299
4300 static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid)
4301 {
4302         if (enable_ept) {
4303                 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
4304                         return;
4305                 ept_sync_context(construct_eptp(vcpu, vcpu->arch.mmu.root_hpa));
4306         } else {
4307                 vpid_sync_context(vpid);
4308         }
4309 }
4310
4311 static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
4312 {
4313         __vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid);
4314 }
4315
4316 static void vmx_flush_tlb_ept_only(struct kvm_vcpu *vcpu)
4317 {
4318         if (enable_ept)
4319                 vmx_flush_tlb(vcpu);
4320 }
4321
4322 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
4323 {
4324         ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
4325
4326         vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
4327         vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
4328 }
4329
4330 static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
4331 {
4332         if (enable_ept && is_paging(vcpu))
4333                 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
4334         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
4335 }
4336
4337 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
4338 {
4339         ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
4340
4341         vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
4342         vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
4343 }
4344
4345 static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
4346 {
4347         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
4348
4349         if (!test_bit(VCPU_EXREG_PDPTR,
4350                       (unsigned long *)&vcpu->arch.regs_dirty))
4351                 return;
4352
4353         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
4354                 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
4355                 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
4356                 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
4357                 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
4358         }
4359 }
4360
4361 static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
4362 {
4363         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
4364
4365         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
4366                 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
4367                 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
4368                 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
4369                 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
4370         }
4371
4372         __set_bit(VCPU_EXREG_PDPTR,
4373                   (unsigned long *)&vcpu->arch.regs_avail);
4374         __set_bit(VCPU_EXREG_PDPTR,
4375                   (unsigned long *)&vcpu->arch.regs_dirty);
4376 }
4377
4378 static bool nested_guest_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
4379 {
4380         u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0;
4381         u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1;
4382         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4383
4384         if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high &
4385                 SECONDARY_EXEC_UNRESTRICTED_GUEST &&
4386             nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
4387                 fixed0 &= ~(X86_CR0_PE | X86_CR0_PG);
4388
4389         return fixed_bits_valid(val, fixed0, fixed1);
4390 }
4391
4392 static bool nested_host_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
4393 {
4394         u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0;
4395         u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1;
4396
4397         return fixed_bits_valid(val, fixed0, fixed1);
4398 }
4399
4400 static bool nested_cr4_valid(struct kvm_vcpu *vcpu, unsigned long val)
4401 {
4402         u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed0;
4403         u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed1;
4404
4405         return fixed_bits_valid(val, fixed0, fixed1);
4406 }
4407
4408 /* No difference in the restrictions on guest and host CR4 in VMX operation. */
4409 #define nested_guest_cr4_valid  nested_cr4_valid
4410 #define nested_host_cr4_valid   nested_cr4_valid
4411
4412 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
4413
4414 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
4415                                         unsigned long cr0,
4416                                         struct kvm_vcpu *vcpu)
4417 {
4418         if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
4419                 vmx_decache_cr3(vcpu);
4420         if (!(cr0 & X86_CR0_PG)) {
4421                 /* From paging/starting to nonpaging */
4422                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
4423                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
4424                              (CPU_BASED_CR3_LOAD_EXITING |
4425                               CPU_BASED_CR3_STORE_EXITING));
4426                 vcpu->arch.cr0 = cr0;
4427                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
4428         } else if (!is_paging(vcpu)) {
4429                 /* From nonpaging to paging */
4430                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
4431                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
4432                              ~(CPU_BASED_CR3_LOAD_EXITING |
4433                                CPU_BASED_CR3_STORE_EXITING));
4434                 vcpu->arch.cr0 = cr0;
4435                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
4436         }
4437
4438         if (!(cr0 & X86_CR0_WP))
4439                 *hw_cr0 &= ~X86_CR0_WP;
4440 }
4441
4442 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
4443 {
4444         struct vcpu_vmx *vmx = to_vmx(vcpu);
4445         unsigned long hw_cr0;
4446
4447         hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK);
4448         if (enable_unrestricted_guest)
4449                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
4450         else {
4451                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
4452
4453                 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
4454                         enter_pmode(vcpu);
4455
4456                 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
4457                         enter_rmode(vcpu);
4458         }
4459
4460 #ifdef CONFIG_X86_64
4461         if (vcpu->arch.efer & EFER_LME) {
4462                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
4463                         enter_lmode(vcpu);
4464                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
4465                         exit_lmode(vcpu);
4466         }
4467 #endif
4468
4469         if (enable_ept)
4470                 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
4471
4472         vmcs_writel(CR0_READ_SHADOW, cr0);
4473         vmcs_writel(GUEST_CR0, hw_cr0);
4474         vcpu->arch.cr0 = cr0;
4475
4476         /* depends on vcpu->arch.cr0 to be set to a new value */
4477         vmx->emulation_required = emulation_required(vcpu);
4478 }
4479
4480 static int get_ept_level(struct kvm_vcpu *vcpu)
4481 {
4482         if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48))
4483                 return 5;
4484         return 4;
4485 }
4486
4487 static u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa)
4488 {
4489         u64 eptp = VMX_EPTP_MT_WB;
4490
4491         eptp |= (get_ept_level(vcpu) == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
4492
4493         if (enable_ept_ad_bits &&
4494             (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
4495                 eptp |= VMX_EPTP_AD_ENABLE_BIT;
4496         eptp |= (root_hpa & PAGE_MASK);
4497
4498         return eptp;
4499 }
4500
4501 static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
4502 {
4503         unsigned long guest_cr3;
4504         u64 eptp;
4505
4506         guest_cr3 = cr3;
4507         if (enable_ept) {
4508                 eptp = construct_eptp(vcpu, cr3);
4509                 vmcs_write64(EPT_POINTER, eptp);
4510                 if (is_paging(vcpu) || is_guest_mode(vcpu))
4511                         guest_cr3 = kvm_read_cr3(vcpu);
4512                 else
4513                         guest_cr3 = vcpu->kvm->arch.ept_identity_map_addr;
4514                 ept_load_pdptrs(vcpu);
4515         }
4516
4517         vmx_flush_tlb(vcpu);
4518         vmcs_writel(GUEST_CR3, guest_cr3);
4519 }
4520
4521 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
4522 {
4523         /*
4524          * Pass through host's Machine Check Enable value to hw_cr4, which
4525          * is in force while we are in guest mode.  Do not let guests control
4526          * this bit, even if host CR4.MCE == 0.
4527          */
4528         unsigned long hw_cr4 =
4529                 (cr4_read_shadow() & X86_CR4_MCE) |
4530                 (cr4 & ~X86_CR4_MCE) |
4531                 (to_vmx(vcpu)->rmode.vm86_active ?
4532                  KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
4533
4534         if (cr4 & X86_CR4_VMXE) {
4535                 /*
4536                  * To use VMXON (and later other VMX instructions), a guest
4537                  * must first be able to turn on cr4.VMXE (see handle_vmon()).
4538                  * So basically the check on whether to allow nested VMX
4539                  * is here.
4540                  */
4541                 if (!nested_vmx_allowed(vcpu))
4542                         return 1;
4543         }
4544
4545         if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
4546                 return 1;
4547
4548         vcpu->arch.cr4 = cr4;
4549         if (enable_ept) {
4550                 if (!is_paging(vcpu)) {
4551                         hw_cr4 &= ~X86_CR4_PAE;
4552                         hw_cr4 |= X86_CR4_PSE;
4553                 } else if (!(cr4 & X86_CR4_PAE)) {
4554                         hw_cr4 &= ~X86_CR4_PAE;
4555                 }
4556         }
4557
4558         if (!enable_unrestricted_guest && !is_paging(vcpu))
4559                 /*
4560                  * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
4561                  * hardware.  To emulate this behavior, SMEP/SMAP/PKU needs
4562                  * to be manually disabled when guest switches to non-paging
4563                  * mode.
4564                  *
4565                  * If !enable_unrestricted_guest, the CPU is always running
4566                  * with CR0.PG=1 and CR4 needs to be modified.
4567                  * If enable_unrestricted_guest, the CPU automatically
4568                  * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
4569                  */
4570                 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
4571
4572         vmcs_writel(CR4_READ_SHADOW, cr4);
4573         vmcs_writel(GUEST_CR4, hw_cr4);
4574         return 0;
4575 }
4576
4577 static void vmx_get_segment(struct kvm_vcpu *vcpu,
4578                             struct kvm_segment *var, int seg)
4579 {
4580         struct vcpu_vmx *vmx = to_vmx(vcpu);
4581         u32 ar;
4582
4583         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
4584                 *var = vmx->rmode.segs[seg];
4585                 if (seg == VCPU_SREG_TR
4586                     || var->selector == vmx_read_guest_seg_selector(vmx, seg))
4587                         return;
4588                 var->base = vmx_read_guest_seg_base(vmx, seg);
4589                 var->selector = vmx_read_guest_seg_selector(vmx, seg);
4590                 return;
4591         }
4592         var->base = vmx_read_guest_seg_base(vmx, seg);
4593         var->limit = vmx_read_guest_seg_limit(vmx, seg);
4594         var->selector = vmx_read_guest_seg_selector(vmx, seg);
4595         ar = vmx_read_guest_seg_ar(vmx, seg);
4596         var->unusable = (ar >> 16) & 1;
4597         var->type = ar & 15;
4598         var->s = (ar >> 4) & 1;
4599         var->dpl = (ar >> 5) & 3;
4600         /*
4601          * Some userspaces do not preserve unusable property. Since usable
4602          * segment has to be present according to VMX spec we can use present
4603          * property to amend userspace bug by making unusable segment always
4604          * nonpresent. vmx_segment_access_rights() already marks nonpresent
4605          * segment as unusable.
4606          */
4607         var->present = !var->unusable;
4608         var->avl = (ar >> 12) & 1;
4609         var->l = (ar >> 13) & 1;
4610         var->db = (ar >> 14) & 1;
4611         var->g = (ar >> 15) & 1;
4612 }
4613
4614 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
4615 {
4616         struct kvm_segment s;
4617
4618         if (to_vmx(vcpu)->rmode.vm86_active) {
4619                 vmx_get_segment(vcpu, &s, seg);
4620                 return s.base;
4621         }
4622         return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
4623 }
4624
4625 static int vmx_get_cpl(struct kvm_vcpu *vcpu)
4626 {
4627         struct vcpu_vmx *vmx = to_vmx(vcpu);
4628
4629         if (unlikely(vmx->rmode.vm86_active))
4630                 return 0;
4631         else {
4632                 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
4633                 return VMX_AR_DPL(ar);
4634         }
4635 }
4636
4637 static u32 vmx_segment_access_rights(struct kvm_segment *var)
4638 {
4639         u32 ar;
4640
4641         if (var->unusable || !var->present)
4642                 ar = 1 << 16;
4643         else {
4644                 ar = var->type & 15;
4645                 ar |= (var->s & 1) << 4;
4646                 ar |= (var->dpl & 3) << 5;
4647                 ar |= (var->present & 1) << 7;
4648                 ar |= (var->avl & 1) << 12;
4649                 ar |= (var->l & 1) << 13;
4650                 ar |= (var->db & 1) << 14;
4651                 ar |= (var->g & 1) << 15;
4652         }
4653
4654         return ar;
4655 }
4656
4657 static void vmx_set_segment(struct kvm_vcpu *vcpu,
4658                             struct kvm_segment *var, int seg)
4659 {
4660         struct vcpu_vmx *vmx = to_vmx(vcpu);
4661         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
4662
4663         vmx_segment_cache_clear(vmx);
4664
4665         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
4666                 vmx->rmode.segs[seg] = *var;
4667                 if (seg == VCPU_SREG_TR)
4668                         vmcs_write16(sf->selector, var->selector);
4669                 else if (var->s)
4670                         fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
4671                 goto out;
4672         }
4673
4674         vmcs_writel(sf->base, var->base);
4675         vmcs_write32(sf->limit, var->limit);
4676         vmcs_write16(sf->selector, var->selector);
4677
4678         /*
4679          *   Fix the "Accessed" bit in AR field of segment registers for older
4680          * qemu binaries.
4681          *   IA32 arch specifies that at the time of processor reset the
4682          * "Accessed" bit in the AR field of segment registers is 1. And qemu
4683          * is setting it to 0 in the userland code. This causes invalid guest
4684          * state vmexit when "unrestricted guest" mode is turned on.
4685          *    Fix for this setup issue in cpu_reset is being pushed in the qemu
4686          * tree. Newer qemu binaries with that qemu fix would not need this
4687          * kvm hack.
4688          */
4689         if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
4690                 var->type |= 0x1; /* Accessed */
4691
4692         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
4693
4694 out:
4695         vmx->emulation_required = emulation_required(vcpu);
4696 }
4697
4698 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
4699 {
4700         u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
4701
4702         *db = (ar >> 14) & 1;
4703         *l = (ar >> 13) & 1;
4704 }
4705
4706 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
4707 {
4708         dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
4709         dt->address = vmcs_readl(GUEST_IDTR_BASE);
4710 }
4711
4712 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
4713 {
4714         vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
4715         vmcs_writel(GUEST_IDTR_BASE, dt->address);
4716 }
4717
4718 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
4719 {
4720         dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
4721         dt->address = vmcs_readl(GUEST_GDTR_BASE);
4722 }
4723
4724 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
4725 {
4726         vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
4727         vmcs_writel(GUEST_GDTR_BASE, dt->address);
4728 }
4729
4730 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
4731 {
4732         struct kvm_segment var;
4733         u32 ar;
4734
4735         vmx_get_segment(vcpu, &var, seg);
4736         var.dpl = 0x3;
4737         if (seg == VCPU_SREG_CS)
4738                 var.type = 0x3;
4739         ar = vmx_segment_access_rights(&var);
4740
4741         if (var.base != (var.selector << 4))
4742                 return false;
4743         if (var.limit != 0xffff)
4744                 return false;
4745         if (ar != 0xf3)
4746                 return false;
4747
4748         return true;
4749 }
4750
4751 static bool code_segment_valid(struct kvm_vcpu *vcpu)
4752 {
4753         struct kvm_segment cs;
4754         unsigned int cs_rpl;
4755
4756         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
4757         cs_rpl = cs.selector & SEGMENT_RPL_MASK;
4758
4759         if (cs.unusable)
4760                 return false;
4761         if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
4762                 return false;
4763         if (!cs.s)
4764                 return false;
4765         if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
4766                 if (cs.dpl > cs_rpl)
4767                         return false;
4768         } else {
4769                 if (cs.dpl != cs_rpl)
4770                         return false;
4771         }
4772         if (!cs.present)
4773                 return false;
4774
4775         /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
4776         return true;
4777 }
4778
4779 static bool stack_segment_valid(struct kvm_vcpu *vcpu)
4780 {
4781         struct kvm_segment ss;
4782         unsigned int ss_rpl;
4783
4784         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
4785         ss_rpl = ss.selector & SEGMENT_RPL_MASK;
4786
4787         if (ss.unusable)
4788                 return true;
4789         if (ss.type != 3 && ss.type != 7)
4790                 return false;
4791         if (!ss.s)
4792                 return false;
4793         if (ss.dpl != ss_rpl) /* DPL != RPL */
4794                 return false;
4795         if (!ss.present)
4796                 return false;
4797
4798         return true;
4799 }
4800
4801 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
4802 {
4803         struct kvm_segment var;
4804         unsigned int rpl;
4805
4806         vmx_get_segment(vcpu, &var, seg);
4807         rpl = var.selector & SEGMENT_RPL_MASK;
4808
4809         if (var.unusable)
4810                 return true;
4811         if (!var.s)
4812                 return false;
4813         if (!var.present)
4814                 return false;
4815         if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
4816                 if (var.dpl < rpl) /* DPL < RPL */
4817                         return false;
4818         }
4819
4820         /* TODO: Add other members to kvm_segment_field to allow checking for other access
4821          * rights flags
4822          */
4823         return true;
4824 }
4825
4826 static bool tr_valid(struct kvm_vcpu *vcpu)
4827 {
4828         struct kvm_segment tr;
4829
4830         vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
4831
4832         if (tr.unusable)
4833                 return false;
4834         if (tr.selector & SEGMENT_TI_MASK)      /* TI = 1 */
4835                 return false;
4836         if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
4837                 return false;
4838         if (!tr.present)
4839                 return false;
4840
4841         return true;
4842 }
4843
4844 static bool ldtr_valid(struct kvm_vcpu *vcpu)
4845 {
4846         struct kvm_segment ldtr;
4847
4848         vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
4849
4850         if (ldtr.unusable)
4851                 return true;
4852         if (ldtr.selector & SEGMENT_TI_MASK)    /* TI = 1 */
4853                 return false;
4854         if (ldtr.type != 2)
4855                 return false;
4856         if (!ldtr.present)
4857                 return false;
4858
4859         return true;
4860 }
4861
4862 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
4863 {
4864         struct kvm_segment cs, ss;
4865
4866         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
4867         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
4868
4869         return ((cs.selector & SEGMENT_RPL_MASK) ==
4870                  (ss.selector & SEGMENT_RPL_MASK));
4871 }
4872
4873 /*
4874  * Check if guest state is valid. Returns true if valid, false if
4875  * not.
4876  * We assume that registers are always usable
4877  */
4878 static bool guest_state_valid(struct kvm_vcpu *vcpu)
4879 {
4880         if (enable_unrestricted_guest)
4881                 return true;
4882
4883         /* real mode guest state checks */
4884         if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
4885                 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
4886                         return false;
4887                 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
4888                         return false;
4889                 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
4890                         return false;
4891                 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
4892                         return false;
4893                 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
4894                         return false;
4895                 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
4896                         return false;
4897         } else {
4898         /* protected mode guest state checks */
4899                 if (!cs_ss_rpl_check(vcpu))
4900                         return false;
4901                 if (!code_segment_valid(vcpu))
4902                         return false;
4903                 if (!stack_segment_valid(vcpu))
4904                         return false;
4905                 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
4906                         return false;
4907                 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
4908                         return false;
4909                 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
4910                         return false;
4911                 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
4912                         return false;
4913                 if (!tr_valid(vcpu))
4914                         return false;
4915                 if (!ldtr_valid(vcpu))
4916                         return false;
4917         }
4918         /* TODO:
4919          * - Add checks on RIP
4920          * - Add checks on RFLAGS
4921          */
4922
4923         return true;
4924 }
4925
4926 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
4927 {
4928         return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu));
4929 }
4930
4931 static int init_rmode_tss(struct kvm *kvm)
4932 {
4933         gfn_t fn;
4934         u16 data = 0;
4935         int idx, r;
4936
4937         idx = srcu_read_lock(&kvm->srcu);
4938         fn = kvm->arch.tss_addr >> PAGE_SHIFT;
4939         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
4940         if (r < 0)
4941                 goto out;
4942         data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
4943         r = kvm_write_guest_page(kvm, fn++, &data,
4944                         TSS_IOPB_BASE_OFFSET, sizeof(u16));
4945         if (r < 0)
4946                 goto out;
4947         r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
4948         if (r < 0)
4949                 goto out;
4950         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
4951         if (r < 0)
4952                 goto out;
4953         data = ~0;
4954         r = kvm_write_guest_page(kvm, fn, &data,
4955                                  RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
4956                                  sizeof(u8));
4957 out:
4958         srcu_read_unlock(&kvm->srcu, idx);
4959         return r;
4960 }
4961
4962 static int init_rmode_identity_map(struct kvm *kvm)
4963 {
4964         int i, idx, r = 0;
4965         kvm_pfn_t identity_map_pfn;
4966         u32 tmp;
4967
4968         if (!enable_ept)
4969                 return 0;
4970
4971         /* Protect kvm->arch.ept_identity_pagetable_done. */
4972         mutex_lock(&kvm->slots_lock);
4973
4974         if (likely(kvm->arch.ept_identity_pagetable_done))
4975                 goto out2;
4976
4977         identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
4978
4979         r = alloc_identity_pagetable(kvm);
4980         if (r < 0)
4981                 goto out2;
4982
4983         idx = srcu_read_lock(&kvm->srcu);
4984         r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
4985         if (r < 0)
4986                 goto out;
4987         /* Set up identity-mapping pagetable for EPT in real mode */
4988         for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
4989                 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
4990                         _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
4991                 r = kvm_write_guest_page(kvm, identity_map_pfn,
4992                                 &tmp, i * sizeof(tmp), sizeof(tmp));
4993                 if (r < 0)
4994                         goto out;
4995         }
4996         kvm->arch.ept_identity_pagetable_done = true;
4997
4998 out:
4999         srcu_read_unlock(&kvm->srcu, idx);
5000
5001 out2:
5002         mutex_unlock(&kvm->slots_lock);
5003         return r;
5004 }
5005
5006 static void seg_setup(int seg)
5007 {
5008         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
5009         unsigned int ar;
5010
5011         vmcs_write16(sf->selector, 0);
5012         vmcs_writel(sf->base, 0);
5013         vmcs_write32(sf->limit, 0xffff);
5014         ar = 0x93;
5015         if (seg == VCPU_SREG_CS)
5016                 ar |= 0x08; /* code segment */
5017
5018         vmcs_write32(sf->ar_bytes, ar);
5019 }
5020
5021 static int alloc_apic_access_page(struct kvm *kvm)
5022 {
5023         struct page *page;
5024         int r = 0;
5025
5026         mutex_lock(&kvm->slots_lock);
5027         if (kvm->arch.apic_access_page_done)
5028                 goto out;
5029         r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
5030                                     APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
5031         if (r)
5032                 goto out;
5033
5034         page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
5035         if (is_error_page(page)) {
5036                 r = -EFAULT;
5037                 goto out;
5038         }
5039
5040         /*
5041          * Do not pin the page in memory, so that memory hot-unplug
5042          * is able to migrate it.
5043          */
5044         put_page(page);
5045         kvm->arch.apic_access_page_done = true;
5046 out:
5047         mutex_unlock(&kvm->slots_lock);
5048         return r;
5049 }
5050
5051 static int alloc_identity_pagetable(struct kvm *kvm)
5052 {
5053         /* Called with kvm->slots_lock held. */
5054
5055         int r = 0;
5056
5057         BUG_ON(kvm->arch.ept_identity_pagetable_done);
5058
5059         r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
5060                                     kvm->arch.ept_identity_map_addr, PAGE_SIZE);
5061
5062         return r;
5063 }
5064
5065 static int allocate_vpid(void)
5066 {
5067         int vpid;
5068
5069         if (!enable_vpid)
5070                 return 0;
5071         spin_lock(&vmx_vpid_lock);
5072         vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
5073         if (vpid < VMX_NR_VPIDS)
5074                 __set_bit(vpid, vmx_vpid_bitmap);
5075         else
5076                 vpid = 0;
5077         spin_unlock(&vmx_vpid_lock);
5078         return vpid;
5079 }
5080
5081 static void free_vpid(int vpid)
5082 {
5083         if (!enable_vpid || vpid == 0)
5084                 return;
5085         spin_lock(&vmx_vpid_lock);
5086         __clear_bit(vpid, vmx_vpid_bitmap);
5087         spin_unlock(&vmx_vpid_lock);
5088 }
5089
5090 static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
5091                                                           u32 msr, int type)
5092 {
5093         int f = sizeof(unsigned long);
5094
5095         if (!cpu_has_vmx_msr_bitmap())
5096                 return;
5097
5098         /*
5099          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
5100          * have the write-low and read-high bitmap offsets the wrong way round.
5101          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
5102          */
5103         if (msr <= 0x1fff) {
5104                 if (type & MSR_TYPE_R)
5105                         /* read-low */
5106                         __clear_bit(msr, msr_bitmap + 0x000 / f);
5107
5108                 if (type & MSR_TYPE_W)
5109                         /* write-low */
5110                         __clear_bit(msr, msr_bitmap + 0x800 / f);
5111
5112         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
5113                 msr &= 0x1fff;
5114                 if (type & MSR_TYPE_R)
5115                         /* read-high */
5116                         __clear_bit(msr, msr_bitmap + 0x400 / f);
5117
5118                 if (type & MSR_TYPE_W)
5119                         /* write-high */
5120                         __clear_bit(msr, msr_bitmap + 0xc00 / f);
5121
5122         }
5123 }
5124
5125 static void __always_inline vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
5126                                                          u32 msr, int type)
5127 {
5128         int f = sizeof(unsigned long);
5129
5130         if (!cpu_has_vmx_msr_bitmap())
5131                 return;
5132
5133         /*
5134          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
5135          * have the write-low and read-high bitmap offsets the wrong way round.
5136          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
5137          */
5138         if (msr <= 0x1fff) {
5139                 if (type & MSR_TYPE_R)
5140                         /* read-low */
5141                         __set_bit(msr, msr_bitmap + 0x000 / f);
5142
5143                 if (type & MSR_TYPE_W)
5144                         /* write-low */
5145                         __set_bit(msr, msr_bitmap + 0x800 / f);
5146
5147         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
5148                 msr &= 0x1fff;
5149                 if (type & MSR_TYPE_R)
5150                         /* read-high */
5151                         __set_bit(msr, msr_bitmap + 0x400 / f);
5152
5153                 if (type & MSR_TYPE_W)
5154                         /* write-high */
5155                         __set_bit(msr, msr_bitmap + 0xc00 / f);
5156
5157         }
5158 }
5159
5160 static void __always_inline vmx_set_intercept_for_msr(unsigned long *msr_bitmap,
5161                                                       u32 msr, int type, bool value)
5162 {
5163         if (value)
5164                 vmx_enable_intercept_for_msr(msr_bitmap, msr, type);
5165         else
5166                 vmx_disable_intercept_for_msr(msr_bitmap, msr, type);
5167 }
5168
5169 /*
5170  * If a msr is allowed by L0, we should check whether it is allowed by L1.
5171  * The corresponding bit will be cleared unless both of L0 and L1 allow it.
5172  */
5173 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
5174                                                unsigned long *msr_bitmap_nested,
5175                                                u32 msr, int type)
5176 {
5177         int f = sizeof(unsigned long);
5178
5179         if (!cpu_has_vmx_msr_bitmap()) {
5180                 WARN_ON(1);
5181                 return;
5182         }
5183
5184         /*
5185          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
5186          * have the write-low and read-high bitmap offsets the wrong way round.
5187          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
5188          */
5189         if (msr <= 0x1fff) {
5190                 if (type & MSR_TYPE_R &&
5191                    !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
5192                         /* read-low */
5193                         __clear_bit(msr, msr_bitmap_nested + 0x000 / f);
5194
5195                 if (type & MSR_TYPE_W &&
5196                    !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
5197                         /* write-low */
5198                         __clear_bit(msr, msr_bitmap_nested + 0x800 / f);
5199
5200         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
5201                 msr &= 0x1fff;
5202                 if (type & MSR_TYPE_R &&
5203                    !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
5204                         /* read-high */
5205                         __clear_bit(msr, msr_bitmap_nested + 0x400 / f);
5206
5207                 if (type & MSR_TYPE_W &&
5208                    !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
5209                         /* write-high */
5210                         __clear_bit(msr, msr_bitmap_nested + 0xc00 / f);
5211
5212         }
5213 }
5214
5215 static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu)
5216 {
5217         u8 mode = 0;
5218
5219         if (cpu_has_secondary_exec_ctrls() &&
5220             (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) &
5221              SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
5222                 mode |= MSR_BITMAP_MODE_X2APIC;
5223                 if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
5224                         mode |= MSR_BITMAP_MODE_X2APIC_APICV;
5225         }
5226
5227         if (is_long_mode(vcpu))
5228                 mode |= MSR_BITMAP_MODE_LM;
5229
5230         return mode;
5231 }
5232
5233 #define X2APIC_MSR(r) (APIC_BASE_MSR + ((r) >> 4))
5234
5235 static void vmx_update_msr_bitmap_x2apic(unsigned long *msr_bitmap,
5236                                          u8 mode)
5237 {
5238         int msr;
5239
5240         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
5241                 unsigned word = msr / BITS_PER_LONG;
5242                 msr_bitmap[word] = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0;
5243                 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
5244         }
5245
5246         if (mode & MSR_BITMAP_MODE_X2APIC) {
5247                 /*
5248                  * TPR reads and writes can be virtualized even if virtual interrupt
5249                  * delivery is not in use.
5250                  */
5251                 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW);
5252                 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) {
5253                         vmx_enable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_R);
5254                         vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_EOI), MSR_TYPE_W);
5255                         vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W);
5256                 }
5257         }
5258 }
5259
5260 static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu)
5261 {
5262         struct vcpu_vmx *vmx = to_vmx(vcpu);
5263         unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
5264         u8 mode = vmx_msr_bitmap_mode(vcpu);
5265         u8 changed = mode ^ vmx->msr_bitmap_mode;
5266
5267         if (!changed)
5268                 return;
5269
5270         vmx_set_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW,
5271                                   !(mode & MSR_BITMAP_MODE_LM));
5272
5273         if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV))
5274                 vmx_update_msr_bitmap_x2apic(msr_bitmap, mode);
5275
5276         vmx->msr_bitmap_mode = mode;
5277 }
5278
5279 static bool vmx_get_enable_apicv(struct kvm_vcpu *vcpu)
5280 {
5281         return enable_apicv;
5282 }
5283
5284 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
5285 {
5286         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5287         gfn_t gfn;
5288
5289         /*
5290          * Don't need to mark the APIC access page dirty; it is never
5291          * written to by the CPU during APIC virtualization.
5292          */
5293
5294         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
5295                 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
5296                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
5297         }
5298
5299         if (nested_cpu_has_posted_intr(vmcs12)) {
5300                 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
5301                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
5302         }
5303 }
5304
5305
5306 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
5307 {
5308         struct vcpu_vmx *vmx = to_vmx(vcpu);
5309         int max_irr;
5310         void *vapic_page;
5311         u16 status;
5312
5313         if (!vmx->nested.pi_desc || !vmx->nested.pi_pending)
5314                 return;
5315
5316         vmx->nested.pi_pending = false;
5317         if (!pi_test_and_clear_on(vmx->nested.pi_desc))
5318                 return;
5319
5320         max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
5321         if (max_irr != 256) {
5322                 vapic_page = kmap(vmx->nested.virtual_apic_page);
5323                 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, vapic_page);
5324                 kunmap(vmx->nested.virtual_apic_page);
5325
5326                 status = vmcs_read16(GUEST_INTR_STATUS);
5327                 if ((u8)max_irr > ((u8)status & 0xff)) {
5328                         status &= ~0xff;
5329                         status |= (u8)max_irr;
5330                         vmcs_write16(GUEST_INTR_STATUS, status);
5331                 }
5332         }
5333
5334         nested_mark_vmcs12_pages_dirty(vcpu);
5335 }
5336
5337 static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
5338                                                      bool nested)
5339 {
5340 #ifdef CONFIG_SMP
5341         int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR;
5342
5343         if (vcpu->mode == IN_GUEST_MODE) {
5344                 /*
5345                  * The vector of interrupt to be delivered to vcpu had
5346                  * been set in PIR before this function.
5347                  *
5348                  * Following cases will be reached in this block, and
5349                  * we always send a notification event in all cases as
5350                  * explained below.
5351                  *
5352                  * Case 1: vcpu keeps in non-root mode. Sending a
5353                  * notification event posts the interrupt to vcpu.
5354                  *
5355                  * Case 2: vcpu exits to root mode and is still
5356                  * runnable. PIR will be synced to vIRR before the
5357                  * next vcpu entry. Sending a notification event in
5358                  * this case has no effect, as vcpu is not in root
5359                  * mode.
5360                  *
5361                  * Case 3: vcpu exits to root mode and is blocked.
5362                  * vcpu_block() has already synced PIR to vIRR and
5363                  * never blocks vcpu if vIRR is not cleared. Therefore,
5364                  * a blocked vcpu here does not wait for any requested
5365                  * interrupts in PIR, and sending a notification event
5366                  * which has no effect is safe here.
5367                  */
5368
5369                 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
5370                 return true;
5371         }
5372 #endif
5373         return false;
5374 }
5375
5376 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
5377                                                 int vector)
5378 {
5379         struct vcpu_vmx *vmx = to_vmx(vcpu);
5380
5381         if (is_guest_mode(vcpu) &&
5382             vector == vmx->nested.posted_intr_nv) {
5383                 /*
5384                  * If a posted intr is not recognized by hardware,
5385                  * we will accomplish it in the next vmentry.
5386                  */
5387                 vmx->nested.pi_pending = true;
5388                 kvm_make_request(KVM_REQ_EVENT, vcpu);
5389                 /* the PIR and ON have been set by L1. */
5390                 if (!kvm_vcpu_trigger_posted_interrupt(vcpu, true))
5391                         kvm_vcpu_kick(vcpu);
5392                 return 0;
5393         }
5394         return -1;
5395 }
5396 /*
5397  * Send interrupt to vcpu via posted interrupt way.
5398  * 1. If target vcpu is running(non-root mode), send posted interrupt
5399  * notification to vcpu and hardware will sync PIR to vIRR atomically.
5400  * 2. If target vcpu isn't running(root mode), kick it to pick up the
5401  * interrupt from PIR in next vmentry.
5402  */
5403 static void vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
5404 {
5405         struct vcpu_vmx *vmx = to_vmx(vcpu);
5406         int r;
5407
5408         r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
5409         if (!r)
5410                 return;
5411
5412         if (pi_test_and_set_pir(vector, &vmx->pi_desc))
5413                 return;
5414
5415         /* If a previous notification has sent the IPI, nothing to do.  */
5416         if (pi_test_and_set_on(&vmx->pi_desc))
5417                 return;
5418
5419         if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false))
5420                 kvm_vcpu_kick(vcpu);
5421 }
5422
5423 /*
5424  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
5425  * will not change in the lifetime of the guest.
5426  * Note that host-state that does change is set elsewhere. E.g., host-state
5427  * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
5428  */
5429 static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
5430 {
5431         u32 low32, high32;
5432         unsigned long tmpl;
5433         struct desc_ptr dt;
5434         unsigned long cr0, cr3, cr4;
5435
5436         cr0 = read_cr0();
5437         WARN_ON(cr0 & X86_CR0_TS);
5438         vmcs_writel(HOST_CR0, cr0);  /* 22.2.3 */
5439
5440         /*
5441          * Save the most likely value for this task's CR3 in the VMCS.
5442          * We can't use __get_current_cr3_fast() because we're not atomic.
5443          */
5444         cr3 = __read_cr3();
5445         vmcs_writel(HOST_CR3, cr3);             /* 22.2.3  FIXME: shadow tables */
5446         vmx->loaded_vmcs->vmcs_host_cr3 = cr3;
5447
5448         /* Save the most likely value for this task's CR4 in the VMCS. */
5449         cr4 = cr4_read_shadow();
5450         vmcs_writel(HOST_CR4, cr4);                     /* 22.2.3, 22.2.5 */
5451         vmx->loaded_vmcs->vmcs_host_cr4 = cr4;
5452
5453         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
5454 #ifdef CONFIG_X86_64
5455         /*
5456          * Load null selectors, so we can avoid reloading them in
5457          * __vmx_load_host_state(), in case userspace uses the null selectors
5458          * too (the expected case).
5459          */
5460         vmcs_write16(HOST_DS_SELECTOR, 0);
5461         vmcs_write16(HOST_ES_SELECTOR, 0);
5462 #else
5463         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
5464         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
5465 #endif
5466         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
5467         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
5468
5469         store_idt(&dt);
5470         vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
5471         vmx->host_idt_base = dt.address;
5472
5473         vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
5474
5475         rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
5476         vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
5477         rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
5478         vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */
5479
5480         if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
5481                 rdmsr(MSR_IA32_CR_PAT, low32, high32);
5482                 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
5483         }
5484 }
5485
5486 static void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
5487 {
5488         vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
5489         if (enable_ept)
5490                 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
5491         if (is_guest_mode(&vmx->vcpu))
5492                 vmx->vcpu.arch.cr4_guest_owned_bits &=
5493                         ~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
5494         vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
5495 }
5496
5497 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
5498 {
5499         u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
5500
5501         if (!kvm_vcpu_apicv_active(&vmx->vcpu))
5502                 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
5503         /* Enable the preemption timer dynamically */
5504         pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
5505         return pin_based_exec_ctrl;
5506 }
5507
5508 static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
5509 {
5510         struct vcpu_vmx *vmx = to_vmx(vcpu);
5511
5512         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
5513         if (cpu_has_secondary_exec_ctrls()) {
5514                 if (kvm_vcpu_apicv_active(vcpu))
5515                         vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
5516                                       SECONDARY_EXEC_APIC_REGISTER_VIRT |
5517                                       SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
5518                 else
5519                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
5520                                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
5521                                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
5522         }
5523
5524         if (cpu_has_vmx_msr_bitmap())
5525                 vmx_update_msr_bitmap(vcpu);
5526 }
5527
5528 static u32 vmx_exec_control(struct vcpu_vmx *vmx)
5529 {
5530         u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
5531
5532         if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
5533                 exec_control &= ~CPU_BASED_MOV_DR_EXITING;
5534
5535         if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
5536                 exec_control &= ~CPU_BASED_TPR_SHADOW;
5537 #ifdef CONFIG_X86_64
5538                 exec_control |= CPU_BASED_CR8_STORE_EXITING |
5539                                 CPU_BASED_CR8_LOAD_EXITING;
5540 #endif
5541         }
5542         if (!enable_ept)
5543                 exec_control |= CPU_BASED_CR3_STORE_EXITING |
5544                                 CPU_BASED_CR3_LOAD_EXITING  |
5545                                 CPU_BASED_INVLPG_EXITING;
5546         return exec_control;
5547 }
5548
5549 static bool vmx_rdrand_supported(void)
5550 {
5551         return vmcs_config.cpu_based_2nd_exec_ctrl &
5552                 SECONDARY_EXEC_RDRAND;
5553 }
5554
5555 static bool vmx_rdseed_supported(void)
5556 {
5557         return vmcs_config.cpu_based_2nd_exec_ctrl &
5558                 SECONDARY_EXEC_RDSEED;
5559 }
5560
5561 static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
5562 {
5563         struct kvm_vcpu *vcpu = &vmx->vcpu;
5564
5565         u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
5566         if (!cpu_need_virtualize_apic_accesses(vcpu))
5567                 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
5568         if (vmx->vpid == 0)
5569                 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
5570         if (!enable_ept) {
5571                 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
5572                 enable_unrestricted_guest = 0;
5573                 /* Enable INVPCID for non-ept guests may cause performance regression. */
5574                 exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
5575         }
5576         if (!enable_unrestricted_guest)
5577                 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
5578         if (!ple_gap)
5579                 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
5580         if (!kvm_vcpu_apicv_active(vcpu))
5581                 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
5582                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
5583         exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
5584         /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
5585            (handle_vmptrld).
5586            We can NOT enable shadow_vmcs here because we don't have yet
5587            a current VMCS12
5588         */
5589         exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
5590
5591         if (!enable_pml)
5592                 exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
5593
5594         if (vmx_xsaves_supported()) {
5595                 /* Exposing XSAVES only when XSAVE is exposed */
5596                 bool xsaves_enabled =
5597                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
5598                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVES);
5599
5600                 if (!xsaves_enabled)
5601                         exec_control &= ~SECONDARY_EXEC_XSAVES;
5602
5603                 if (nested) {
5604                         if (xsaves_enabled)
5605                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5606                                         SECONDARY_EXEC_XSAVES;
5607                         else
5608                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5609                                         ~SECONDARY_EXEC_XSAVES;
5610                 }
5611         }
5612
5613         if (vmx_rdtscp_supported()) {
5614                 bool rdtscp_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP);
5615                 if (!rdtscp_enabled)
5616                         exec_control &= ~SECONDARY_EXEC_RDTSCP;
5617
5618                 if (nested) {
5619                         if (rdtscp_enabled)
5620                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5621                                         SECONDARY_EXEC_RDTSCP;
5622                         else
5623                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5624                                         ~SECONDARY_EXEC_RDTSCP;
5625                 }
5626         }
5627
5628         if (vmx_invpcid_supported()) {
5629                 /* Exposing INVPCID only when PCID is exposed */
5630                 bool invpcid_enabled =
5631                         guest_cpuid_has(vcpu, X86_FEATURE_INVPCID) &&
5632                         guest_cpuid_has(vcpu, X86_FEATURE_PCID);
5633
5634                 if (!invpcid_enabled) {
5635                         exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
5636                         guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID);
5637                 }
5638
5639                 if (nested) {
5640                         if (invpcid_enabled)
5641                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5642                                         SECONDARY_EXEC_ENABLE_INVPCID;
5643                         else
5644                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5645                                         ~SECONDARY_EXEC_ENABLE_INVPCID;
5646                 }
5647         }
5648
5649         if (vmx_rdrand_supported()) {
5650                 bool rdrand_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDRAND);
5651                 if (rdrand_enabled)
5652                         exec_control &= ~SECONDARY_EXEC_RDRAND;
5653
5654                 if (nested) {
5655                         if (rdrand_enabled)
5656                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5657                                         SECONDARY_EXEC_RDRAND;
5658                         else
5659                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5660                                         ~SECONDARY_EXEC_RDRAND;
5661                 }
5662         }
5663
5664         if (vmx_rdseed_supported()) {
5665                 bool rdseed_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDSEED);
5666                 if (rdseed_enabled)
5667                         exec_control &= ~SECONDARY_EXEC_RDSEED;
5668
5669                 if (nested) {
5670                         if (rdseed_enabled)
5671                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5672                                         SECONDARY_EXEC_RDSEED;
5673                         else
5674                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5675                                         ~SECONDARY_EXEC_RDSEED;
5676                 }
5677         }
5678
5679         vmx->secondary_exec_control = exec_control;
5680 }
5681
5682 static void ept_set_mmio_spte_mask(void)
5683 {
5684         /*
5685          * EPT Misconfigurations can be generated if the value of bits 2:0
5686          * of an EPT paging-structure entry is 110b (write/execute).
5687          */
5688         kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK,
5689                                    VMX_EPT_MISCONFIG_WX_VALUE);
5690 }
5691
5692 #define VMX_XSS_EXIT_BITMAP 0
5693 /*
5694  * Sets up the vmcs for emulated real mode.
5695  */
5696 static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
5697 {
5698 #ifdef CONFIG_X86_64
5699         unsigned long a;
5700 #endif
5701         int i;
5702
5703         /* I/O */
5704         vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
5705         vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
5706
5707         if (enable_shadow_vmcs) {
5708                 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
5709                 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
5710         }
5711         if (cpu_has_vmx_msr_bitmap())
5712                 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
5713
5714         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
5715
5716         /* Control */
5717         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
5718         vmx->hv_deadline_tsc = -1;
5719
5720         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
5721
5722         if (cpu_has_secondary_exec_ctrls()) {
5723                 vmx_compute_secondary_exec_control(vmx);
5724                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
5725                              vmx->secondary_exec_control);
5726         }
5727
5728         if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
5729                 vmcs_write64(EOI_EXIT_BITMAP0, 0);
5730                 vmcs_write64(EOI_EXIT_BITMAP1, 0);
5731                 vmcs_write64(EOI_EXIT_BITMAP2, 0);
5732                 vmcs_write64(EOI_EXIT_BITMAP3, 0);
5733
5734                 vmcs_write16(GUEST_INTR_STATUS, 0);
5735
5736                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
5737                 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
5738         }
5739
5740         if (ple_gap) {
5741                 vmcs_write32(PLE_GAP, ple_gap);
5742                 vmx->ple_window = ple_window;
5743                 vmx->ple_window_dirty = true;
5744         }
5745
5746         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
5747         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
5748         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
5749
5750         vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
5751         vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
5752         vmx_set_constant_host_state(vmx);
5753 #ifdef CONFIG_X86_64
5754         rdmsrl(MSR_FS_BASE, a);
5755         vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
5756         rdmsrl(MSR_GS_BASE, a);
5757         vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
5758 #else
5759         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
5760         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
5761 #endif
5762
5763         if (cpu_has_vmx_vmfunc())
5764                 vmcs_write64(VM_FUNCTION_CONTROL, 0);
5765
5766         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
5767         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
5768         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
5769         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
5770         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
5771
5772         if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
5773                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
5774
5775         for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) {
5776                 u32 index = vmx_msr_index[i];
5777                 u32 data_low, data_high;
5778                 int j = vmx->nmsrs;
5779
5780                 if (rdmsr_safe(index, &data_low, &data_high) < 0)
5781                         continue;
5782                 if (wrmsr_safe(index, data_low, data_high) < 0)
5783                         continue;
5784                 vmx->guest_msrs[j].index = i;
5785                 vmx->guest_msrs[j].data = 0;
5786                 vmx->guest_msrs[j].mask = -1ull;
5787                 ++vmx->nmsrs;
5788         }
5789
5790         if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
5791                 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, vmx->arch_capabilities);
5792
5793         vm_exit_controls_init(vmx, vmcs_config.vmexit_ctrl);
5794
5795         /* 22.2.1, 20.8.1 */
5796         vm_entry_controls_init(vmx, vmcs_config.vmentry_ctrl);
5797
5798         vmx->vcpu.arch.cr0_guest_owned_bits = X86_CR0_TS;
5799         vmcs_writel(CR0_GUEST_HOST_MASK, ~X86_CR0_TS);
5800
5801         set_cr4_guest_host_mask(vmx);
5802
5803         if (vmx_xsaves_supported())
5804                 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
5805
5806         if (enable_pml) {
5807                 ASSERT(vmx->pml_pg);
5808                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
5809                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
5810         }
5811
5812         return 0;
5813 }
5814
5815 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
5816 {
5817         struct vcpu_vmx *vmx = to_vmx(vcpu);
5818         struct msr_data apic_base_msr;
5819         u64 cr0;
5820
5821         vmx->rmode.vm86_active = 0;
5822         vmx->spec_ctrl = 0;
5823
5824         vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
5825         kvm_set_cr8(vcpu, 0);
5826
5827         if (!init_event) {
5828                 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE |
5829                                      MSR_IA32_APICBASE_ENABLE;
5830                 if (kvm_vcpu_is_reset_bsp(vcpu))
5831                         apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
5832                 apic_base_msr.host_initiated = true;
5833                 kvm_set_apic_base(vcpu, &apic_base_msr);
5834         }
5835
5836         vmx_segment_cache_clear(vmx);
5837
5838         seg_setup(VCPU_SREG_CS);
5839         vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
5840         vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
5841
5842         seg_setup(VCPU_SREG_DS);
5843         seg_setup(VCPU_SREG_ES);
5844         seg_setup(VCPU_SREG_FS);
5845         seg_setup(VCPU_SREG_GS);
5846         seg_setup(VCPU_SREG_SS);
5847
5848         vmcs_write16(GUEST_TR_SELECTOR, 0);
5849         vmcs_writel(GUEST_TR_BASE, 0);
5850         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
5851         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
5852
5853         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
5854         vmcs_writel(GUEST_LDTR_BASE, 0);
5855         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
5856         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
5857
5858         if (!init_event) {
5859                 vmcs_write32(GUEST_SYSENTER_CS, 0);
5860                 vmcs_writel(GUEST_SYSENTER_ESP, 0);
5861                 vmcs_writel(GUEST_SYSENTER_EIP, 0);
5862                 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
5863         }
5864
5865         kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
5866         kvm_rip_write(vcpu, 0xfff0);
5867
5868         vmcs_writel(GUEST_GDTR_BASE, 0);
5869         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
5870
5871         vmcs_writel(GUEST_IDTR_BASE, 0);
5872         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
5873
5874         vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
5875         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
5876         vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
5877
5878         setup_msrs(vmx);
5879
5880         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
5881
5882         if (cpu_has_vmx_tpr_shadow() && !init_event) {
5883                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
5884                 if (cpu_need_tpr_shadow(vcpu))
5885                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
5886                                      __pa(vcpu->arch.apic->regs));
5887                 vmcs_write32(TPR_THRESHOLD, 0);
5888         }
5889
5890         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
5891
5892         if (vmx->vpid != 0)
5893                 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
5894
5895         cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
5896         vmx->vcpu.arch.cr0 = cr0;
5897         vmx_set_cr0(vcpu, cr0); /* enter rmode */
5898         vmx_set_cr4(vcpu, 0);
5899         vmx_set_efer(vcpu, 0);
5900
5901         update_exception_bitmap(vcpu);
5902
5903         vpid_sync_context(vmx->vpid);
5904 }
5905
5906 /*
5907  * In nested virtualization, check if L1 asked to exit on external interrupts.
5908  * For most existing hypervisors, this will always return true.
5909  */
5910 static bool nested_exit_on_intr(struct kvm_vcpu *vcpu)
5911 {
5912         return get_vmcs12(vcpu)->pin_based_vm_exec_control &
5913                 PIN_BASED_EXT_INTR_MASK;
5914 }
5915
5916 /*
5917  * In nested virtualization, check if L1 has set
5918  * VM_EXIT_ACK_INTR_ON_EXIT
5919  */
5920 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
5921 {
5922         return get_vmcs12(vcpu)->vm_exit_controls &
5923                 VM_EXIT_ACK_INTR_ON_EXIT;
5924 }
5925
5926 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
5927 {
5928         return get_vmcs12(vcpu)->pin_based_vm_exec_control &
5929                 PIN_BASED_NMI_EXITING;
5930 }
5931
5932 static void enable_irq_window(struct kvm_vcpu *vcpu)
5933 {
5934         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
5935                       CPU_BASED_VIRTUAL_INTR_PENDING);
5936 }
5937
5938 static void enable_nmi_window(struct kvm_vcpu *vcpu)
5939 {
5940         if (!cpu_has_virtual_nmis() ||
5941             vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
5942                 enable_irq_window(vcpu);
5943                 return;
5944         }
5945
5946         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
5947                       CPU_BASED_VIRTUAL_NMI_PENDING);
5948 }
5949
5950 static void vmx_inject_irq(struct kvm_vcpu *vcpu)
5951 {
5952         struct vcpu_vmx *vmx = to_vmx(vcpu);
5953         uint32_t intr;
5954         int irq = vcpu->arch.interrupt.nr;
5955
5956         trace_kvm_inj_virq(irq);
5957
5958         ++vcpu->stat.irq_injections;
5959         if (vmx->rmode.vm86_active) {
5960                 int inc_eip = 0;
5961                 if (vcpu->arch.interrupt.soft)
5962                         inc_eip = vcpu->arch.event_exit_inst_len;
5963                 if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
5964                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5965                 return;
5966         }
5967         intr = irq | INTR_INFO_VALID_MASK;
5968         if (vcpu->arch.interrupt.soft) {
5969                 intr |= INTR_TYPE_SOFT_INTR;
5970                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
5971                              vmx->vcpu.arch.event_exit_inst_len);
5972         } else
5973                 intr |= INTR_TYPE_EXT_INTR;
5974         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
5975 }
5976
5977 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
5978 {
5979         struct vcpu_vmx *vmx = to_vmx(vcpu);
5980
5981         if (!cpu_has_virtual_nmis()) {
5982                 /*
5983                  * Tracking the NMI-blocked state in software is built upon
5984                  * finding the next open IRQ window. This, in turn, depends on
5985                  * well-behaving guests: They have to keep IRQs disabled at
5986                  * least as long as the NMI handler runs. Otherwise we may
5987                  * cause NMI nesting, maybe breaking the guest. But as this is
5988                  * highly unlikely, we can live with the residual risk.
5989                  */
5990                 vmx->loaded_vmcs->soft_vnmi_blocked = 1;
5991                 vmx->loaded_vmcs->vnmi_blocked_time = 0;
5992         }
5993
5994         ++vcpu->stat.nmi_injections;
5995         vmx->loaded_vmcs->nmi_known_unmasked = false;
5996
5997         if (vmx->rmode.vm86_active) {
5998                 if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
5999                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
6000                 return;
6001         }
6002
6003         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
6004                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
6005 }
6006
6007 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
6008 {
6009         struct vcpu_vmx *vmx = to_vmx(vcpu);
6010         bool masked;
6011
6012         if (!cpu_has_virtual_nmis())
6013                 return vmx->loaded_vmcs->soft_vnmi_blocked;
6014         if (vmx->loaded_vmcs->nmi_known_unmasked)
6015                 return false;
6016         masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
6017         vmx->loaded_vmcs->nmi_known_unmasked = !masked;
6018         return masked;
6019 }
6020
6021 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
6022 {
6023         struct vcpu_vmx *vmx = to_vmx(vcpu);
6024
6025         if (!cpu_has_virtual_nmis()) {
6026                 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) {
6027                         vmx->loaded_vmcs->soft_vnmi_blocked = masked;
6028                         vmx->loaded_vmcs->vnmi_blocked_time = 0;
6029                 }
6030         } else {
6031                 vmx->loaded_vmcs->nmi_known_unmasked = !masked;
6032                 if (masked)
6033                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
6034                                       GUEST_INTR_STATE_NMI);
6035                 else
6036                         vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
6037                                         GUEST_INTR_STATE_NMI);
6038         }
6039 }
6040
6041 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
6042 {
6043         if (to_vmx(vcpu)->nested.nested_run_pending)
6044                 return 0;
6045
6046         if (!cpu_has_virtual_nmis() &&
6047             to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked)
6048                 return 0;
6049
6050         return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
6051                   (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
6052                    | GUEST_INTR_STATE_NMI));
6053 }
6054
6055 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
6056 {
6057         return (!to_vmx(vcpu)->nested.nested_run_pending &&
6058                 vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
6059                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
6060                         (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
6061 }
6062
6063 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
6064 {
6065         int ret;
6066
6067         ret = x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
6068                                     PAGE_SIZE * 3);
6069         if (ret)
6070                 return ret;
6071         kvm->arch.tss_addr = addr;
6072         return init_rmode_tss(kvm);
6073 }
6074
6075 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
6076 {
6077         switch (vec) {
6078         case BP_VECTOR:
6079                 /*
6080                  * Update instruction length as we may reinject the exception
6081                  * from user space while in guest debugging mode.
6082                  */
6083                 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
6084                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
6085                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
6086                         return false;
6087                 /* fall through */
6088         case DB_VECTOR:
6089                 if (vcpu->guest_debug &
6090                         (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
6091                         return false;
6092                 /* fall through */
6093         case DE_VECTOR:
6094         case OF_VECTOR:
6095         case BR_VECTOR:
6096         case UD_VECTOR:
6097         case DF_VECTOR:
6098         case SS_VECTOR:
6099         case GP_VECTOR:
6100         case MF_VECTOR:
6101                 return true;
6102         break;
6103         }
6104         return false;
6105 }
6106
6107 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
6108                                   int vec, u32 err_code)
6109 {
6110         /*
6111          * Instruction with address size override prefix opcode 0x67
6112          * Cause the #SS fault with 0 error code in VM86 mode.
6113          */
6114         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
6115                 if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
6116                         if (vcpu->arch.halt_request) {
6117                                 vcpu->arch.halt_request = 0;
6118                                 return kvm_vcpu_halt(vcpu);
6119                         }
6120                         return 1;
6121                 }
6122                 return 0;
6123         }
6124
6125         /*
6126          * Forward all other exceptions that are valid in real mode.
6127          * FIXME: Breaks guest debugging in real mode, needs to be fixed with
6128          *        the required debugging infrastructure rework.
6129          */
6130         kvm_queue_exception(vcpu, vec);
6131         return 1;
6132 }
6133
6134 /*
6135  * Trigger machine check on the host. We assume all the MSRs are already set up
6136  * by the CPU and that we still run on the same CPU as the MCE occurred on.
6137  * We pass a fake environment to the machine check handler because we want
6138  * the guest to be always treated like user space, no matter what context
6139  * it used internally.
6140  */
6141 static void kvm_machine_check(void)
6142 {
6143 #if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
6144         struct pt_regs regs = {
6145                 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
6146                 .flags = X86_EFLAGS_IF,
6147         };
6148
6149         do_machine_check(&regs, 0);
6150 #endif
6151 }
6152
6153 static int handle_machine_check(struct kvm_vcpu *vcpu)
6154 {
6155         /* already handled by vcpu_run */
6156         return 1;
6157 }
6158
6159 static int handle_exception(struct kvm_vcpu *vcpu)
6160 {
6161         struct vcpu_vmx *vmx = to_vmx(vcpu);
6162         struct kvm_run *kvm_run = vcpu->run;
6163         u32 intr_info, ex_no, error_code;
6164         unsigned long cr2, rip, dr6;
6165         u32 vect_info;
6166         enum emulation_result er;
6167
6168         vect_info = vmx->idt_vectoring_info;
6169         intr_info = vmx->exit_intr_info;
6170
6171         if (is_machine_check(intr_info))
6172                 return handle_machine_check(vcpu);
6173
6174         if (is_nmi(intr_info))
6175                 return 1;  /* already handled by vmx_vcpu_run() */
6176
6177         if (is_invalid_opcode(intr_info)) {
6178                 er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
6179                 if (er == EMULATE_USER_EXIT)
6180                         return 0;
6181                 if (er != EMULATE_DONE)
6182                         kvm_queue_exception(vcpu, UD_VECTOR);
6183                 return 1;
6184         }
6185
6186         error_code = 0;
6187         if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
6188                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
6189
6190         /*
6191          * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
6192          * MMIO, it is better to report an internal error.
6193          * See the comments in vmx_handle_exit.
6194          */
6195         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
6196             !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
6197                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6198                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
6199                 vcpu->run->internal.ndata = 3;
6200                 vcpu->run->internal.data[0] = vect_info;
6201                 vcpu->run->internal.data[1] = intr_info;
6202                 vcpu->run->internal.data[2] = error_code;
6203                 return 0;
6204         }
6205
6206         if (is_page_fault(intr_info)) {
6207                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
6208                 /* EPT won't cause page fault directly */
6209                 WARN_ON_ONCE(!vcpu->arch.apf.host_apf_reason && enable_ept);
6210                 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0,
6211                                 true);
6212         }
6213
6214         ex_no = intr_info & INTR_INFO_VECTOR_MASK;
6215
6216         if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
6217                 return handle_rmode_exception(vcpu, ex_no, error_code);
6218
6219         switch (ex_no) {
6220         case AC_VECTOR:
6221                 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
6222                 return 1;
6223         case DB_VECTOR:
6224                 dr6 = vmcs_readl(EXIT_QUALIFICATION);
6225                 if (!(vcpu->guest_debug &
6226                       (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
6227                         vcpu->arch.dr6 &= ~15;
6228                         vcpu->arch.dr6 |= dr6 | DR6_RTM;
6229                         if (is_icebp(intr_info))
6230                                 skip_emulated_instruction(vcpu);
6231
6232                         kvm_queue_exception(vcpu, DB_VECTOR);
6233                         return 1;
6234                 }
6235                 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
6236                 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
6237                 /* fall through */
6238         case BP_VECTOR:
6239                 /*
6240                  * Update instruction length as we may reinject #BP from
6241                  * user space while in guest debugging mode. Reading it for
6242                  * #DB as well causes no harm, it is not used in that case.
6243                  */
6244                 vmx->vcpu.arch.event_exit_inst_len =
6245                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
6246                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
6247                 rip = kvm_rip_read(vcpu);
6248                 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
6249                 kvm_run->debug.arch.exception = ex_no;
6250                 break;
6251         default:
6252                 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
6253                 kvm_run->ex.exception = ex_no;
6254                 kvm_run->ex.error_code = error_code;
6255                 break;
6256         }
6257         return 0;
6258 }
6259
6260 static int handle_external_interrupt(struct kvm_vcpu *vcpu)
6261 {
6262         ++vcpu->stat.irq_exits;
6263         return 1;
6264 }
6265
6266 static int handle_triple_fault(struct kvm_vcpu *vcpu)
6267 {
6268         vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
6269         vcpu->mmio_needed = 0;
6270         return 0;
6271 }
6272
6273 static int handle_io(struct kvm_vcpu *vcpu)
6274 {
6275         unsigned long exit_qualification;
6276         int size, in, string, ret;
6277         unsigned port;
6278
6279         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6280         string = (exit_qualification & 16) != 0;
6281         in = (exit_qualification & 8) != 0;
6282
6283         ++vcpu->stat.io_exits;
6284
6285         if (string || in)
6286                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
6287
6288         port = exit_qualification >> 16;
6289         size = (exit_qualification & 7) + 1;
6290
6291         ret = kvm_skip_emulated_instruction(vcpu);
6292
6293         /*
6294          * TODO: we might be squashing a KVM_GUESTDBG_SINGLESTEP-triggered
6295          * KVM_EXIT_DEBUG here.
6296          */
6297         return kvm_fast_pio_out(vcpu, size, port) && ret;
6298 }
6299
6300 static void
6301 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
6302 {
6303         /*
6304          * Patch in the VMCALL instruction:
6305          */
6306         hypercall[0] = 0x0f;
6307         hypercall[1] = 0x01;
6308         hypercall[2] = 0xc1;
6309 }
6310
6311 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
6312 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
6313 {
6314         if (is_guest_mode(vcpu)) {
6315                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6316                 unsigned long orig_val = val;
6317
6318                 /*
6319                  * We get here when L2 changed cr0 in a way that did not change
6320                  * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
6321                  * but did change L0 shadowed bits. So we first calculate the
6322                  * effective cr0 value that L1 would like to write into the
6323                  * hardware. It consists of the L2-owned bits from the new
6324                  * value combined with the L1-owned bits from L1's guest_cr0.
6325                  */
6326                 val = (val & ~vmcs12->cr0_guest_host_mask) |
6327                         (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
6328
6329                 if (!nested_guest_cr0_valid(vcpu, val))
6330                         return 1;
6331
6332                 if (kvm_set_cr0(vcpu, val))
6333                         return 1;
6334                 vmcs_writel(CR0_READ_SHADOW, orig_val);
6335                 return 0;
6336         } else {
6337                 if (to_vmx(vcpu)->nested.vmxon &&
6338                     !nested_host_cr0_valid(vcpu, val))
6339                         return 1;
6340
6341                 return kvm_set_cr0(vcpu, val);
6342         }
6343 }
6344
6345 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
6346 {
6347         if (is_guest_mode(vcpu)) {
6348                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6349                 unsigned long orig_val = val;
6350
6351                 /* analogously to handle_set_cr0 */
6352                 val = (val & ~vmcs12->cr4_guest_host_mask) |
6353                         (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
6354                 if (kvm_set_cr4(vcpu, val))
6355                         return 1;
6356                 vmcs_writel(CR4_READ_SHADOW, orig_val);
6357                 return 0;
6358         } else
6359                 return kvm_set_cr4(vcpu, val);
6360 }
6361
6362 static int handle_cr(struct kvm_vcpu *vcpu)
6363 {
6364         unsigned long exit_qualification, val;
6365         int cr;
6366         int reg;
6367         int err;
6368         int ret;
6369
6370         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6371         cr = exit_qualification & 15;
6372         reg = (exit_qualification >> 8) & 15;
6373         switch ((exit_qualification >> 4) & 3) {
6374         case 0: /* mov to cr */
6375                 val = kvm_register_readl(vcpu, reg);
6376                 trace_kvm_cr_write(cr, val);
6377                 switch (cr) {
6378                 case 0:
6379                         err = handle_set_cr0(vcpu, val);
6380                         return kvm_complete_insn_gp(vcpu, err);
6381                 case 3:
6382                         err = kvm_set_cr3(vcpu, val);
6383                         return kvm_complete_insn_gp(vcpu, err);
6384                 case 4:
6385                         err = handle_set_cr4(vcpu, val);
6386                         return kvm_complete_insn_gp(vcpu, err);
6387                 case 8: {
6388                                 u8 cr8_prev = kvm_get_cr8(vcpu);
6389                                 u8 cr8 = (u8)val;
6390                                 err = kvm_set_cr8(vcpu, cr8);
6391                                 ret = kvm_complete_insn_gp(vcpu, err);
6392                                 if (lapic_in_kernel(vcpu))
6393                                         return ret;
6394                                 if (cr8_prev <= cr8)
6395                                         return ret;
6396                                 /*
6397                                  * TODO: we might be squashing a
6398                                  * KVM_GUESTDBG_SINGLESTEP-triggered
6399                                  * KVM_EXIT_DEBUG here.
6400                                  */
6401                                 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
6402                                 return 0;
6403                         }
6404                 }
6405                 break;
6406         case 2: /* clts */
6407                 WARN_ONCE(1, "Guest should always own CR0.TS");
6408                 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
6409                 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
6410                 return kvm_skip_emulated_instruction(vcpu);
6411         case 1: /*mov from cr*/
6412                 switch (cr) {
6413                 case 3:
6414                         val = kvm_read_cr3(vcpu);
6415                         kvm_register_write(vcpu, reg, val);
6416                         trace_kvm_cr_read(cr, val);
6417                         return kvm_skip_emulated_instruction(vcpu);
6418                 case 8:
6419                         val = kvm_get_cr8(vcpu);
6420                         kvm_register_write(vcpu, reg, val);
6421                         trace_kvm_cr_read(cr, val);
6422                         return kvm_skip_emulated_instruction(vcpu);
6423                 }
6424                 break;
6425         case 3: /* lmsw */
6426                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
6427                 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
6428                 kvm_lmsw(vcpu, val);
6429
6430                 return kvm_skip_emulated_instruction(vcpu);
6431         default:
6432                 break;
6433         }
6434         vcpu->run->exit_reason = 0;
6435         vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
6436                (int)(exit_qualification >> 4) & 3, cr);
6437         return 0;
6438 }
6439
6440 static int handle_dr(struct kvm_vcpu *vcpu)
6441 {
6442         unsigned long exit_qualification;
6443         int dr, dr7, reg;
6444
6445         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6446         dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
6447
6448         /* First, if DR does not exist, trigger UD */
6449         if (!kvm_require_dr(vcpu, dr))
6450                 return 1;
6451
6452         /* Do not handle if the CPL > 0, will trigger GP on re-entry */
6453         if (!kvm_require_cpl(vcpu, 0))
6454                 return 1;
6455         dr7 = vmcs_readl(GUEST_DR7);
6456         if (dr7 & DR7_GD) {
6457                 /*
6458                  * As the vm-exit takes precedence over the debug trap, we
6459                  * need to emulate the latter, either for the host or the
6460                  * guest debugging itself.
6461                  */
6462                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
6463                         vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
6464                         vcpu->run->debug.arch.dr7 = dr7;
6465                         vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
6466                         vcpu->run->debug.arch.exception = DB_VECTOR;
6467                         vcpu->run->exit_reason = KVM_EXIT_DEBUG;
6468                         return 0;
6469                 } else {
6470                         vcpu->arch.dr6 &= ~15;
6471                         vcpu->arch.dr6 |= DR6_BD | DR6_RTM;
6472                         kvm_queue_exception(vcpu, DB_VECTOR);
6473                         return 1;
6474                 }
6475         }
6476
6477         if (vcpu->guest_debug == 0) {
6478                 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
6479                                 CPU_BASED_MOV_DR_EXITING);
6480
6481                 /*
6482                  * No more DR vmexits; force a reload of the debug registers
6483                  * and reenter on this instruction.  The next vmexit will
6484                  * retrieve the full state of the debug registers.
6485                  */
6486                 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
6487                 return 1;
6488         }
6489
6490         reg = DEBUG_REG_ACCESS_REG(exit_qualification);
6491         if (exit_qualification & TYPE_MOV_FROM_DR) {
6492                 unsigned long val;
6493
6494                 if (kvm_get_dr(vcpu, dr, &val))
6495                         return 1;
6496                 kvm_register_write(vcpu, reg, val);
6497         } else
6498                 if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
6499                         return 1;
6500
6501         return kvm_skip_emulated_instruction(vcpu);
6502 }
6503
6504 static u64 vmx_get_dr6(struct kvm_vcpu *vcpu)
6505 {
6506         return vcpu->arch.dr6;
6507 }
6508
6509 static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
6510 {
6511 }
6512
6513 static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
6514 {
6515         get_debugreg(vcpu->arch.db[0], 0);
6516         get_debugreg(vcpu->arch.db[1], 1);
6517         get_debugreg(vcpu->arch.db[2], 2);
6518         get_debugreg(vcpu->arch.db[3], 3);
6519         get_debugreg(vcpu->arch.dr6, 6);
6520         vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);
6521
6522         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
6523         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, CPU_BASED_MOV_DR_EXITING);
6524 }
6525
6526 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
6527 {
6528         vmcs_writel(GUEST_DR7, val);
6529 }
6530
6531 static int handle_cpuid(struct kvm_vcpu *vcpu)
6532 {
6533         return kvm_emulate_cpuid(vcpu);
6534 }
6535
6536 static int handle_rdmsr(struct kvm_vcpu *vcpu)
6537 {
6538         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
6539         struct msr_data msr_info;
6540
6541         msr_info.index = ecx;
6542         msr_info.host_initiated = false;
6543         if (vmx_get_msr(vcpu, &msr_info)) {
6544                 trace_kvm_msr_read_ex(ecx);
6545                 kvm_inject_gp(vcpu, 0);
6546                 return 1;
6547         }
6548
6549         trace_kvm_msr_read(ecx, msr_info.data);
6550
6551         /* FIXME: handling of bits 32:63 of rax, rdx */
6552         vcpu->arch.regs[VCPU_REGS_RAX] = msr_info.data & -1u;
6553         vcpu->arch.regs[VCPU_REGS_RDX] = (msr_info.data >> 32) & -1u;
6554         return kvm_skip_emulated_instruction(vcpu);
6555 }
6556
6557 static int handle_wrmsr(struct kvm_vcpu *vcpu)
6558 {
6559         struct msr_data msr;
6560         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
6561         u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
6562                 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
6563
6564         msr.data = data;
6565         msr.index = ecx;
6566         msr.host_initiated = false;
6567         if (kvm_set_msr(vcpu, &msr) != 0) {
6568                 trace_kvm_msr_write_ex(ecx, data);
6569                 kvm_inject_gp(vcpu, 0);
6570                 return 1;
6571         }
6572
6573         trace_kvm_msr_write(ecx, data);
6574         return kvm_skip_emulated_instruction(vcpu);
6575 }
6576
6577 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
6578 {
6579         kvm_apic_update_ppr(vcpu);
6580         return 1;
6581 }
6582
6583 static int handle_interrupt_window(struct kvm_vcpu *vcpu)
6584 {
6585         vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
6586                         CPU_BASED_VIRTUAL_INTR_PENDING);
6587
6588         kvm_make_request(KVM_REQ_EVENT, vcpu);
6589
6590         ++vcpu->stat.irq_window_exits;
6591         return 1;
6592 }
6593
6594 static int handle_halt(struct kvm_vcpu *vcpu)
6595 {
6596         return kvm_emulate_halt(vcpu);
6597 }
6598
6599 static int handle_vmcall(struct kvm_vcpu *vcpu)
6600 {
6601         return kvm_emulate_hypercall(vcpu);
6602 }
6603
6604 static int handle_invd(struct kvm_vcpu *vcpu)
6605 {
6606         return emulate_instruction(vcpu, 0) == EMULATE_DONE;
6607 }
6608
6609 static int handle_invlpg(struct kvm_vcpu *vcpu)
6610 {
6611         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6612
6613         kvm_mmu_invlpg(vcpu, exit_qualification);
6614         return kvm_skip_emulated_instruction(vcpu);
6615 }
6616
6617 static int handle_rdpmc(struct kvm_vcpu *vcpu)
6618 {
6619         int err;
6620
6621         err = kvm_rdpmc(vcpu);
6622         return kvm_complete_insn_gp(vcpu, err);
6623 }
6624
6625 static int handle_wbinvd(struct kvm_vcpu *vcpu)
6626 {
6627         return kvm_emulate_wbinvd(vcpu);
6628 }
6629
6630 static int handle_xsetbv(struct kvm_vcpu *vcpu)
6631 {
6632         u64 new_bv = kvm_read_edx_eax(vcpu);
6633         u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);
6634
6635         if (kvm_set_xcr(vcpu, index, new_bv) == 0)
6636                 return kvm_skip_emulated_instruction(vcpu);
6637         return 1;
6638 }
6639
6640 static int handle_xsaves(struct kvm_vcpu *vcpu)
6641 {
6642         kvm_skip_emulated_instruction(vcpu);
6643         WARN(1, "this should never happen\n");
6644         return 1;
6645 }
6646
6647 static int handle_xrstors(struct kvm_vcpu *vcpu)
6648 {
6649         kvm_skip_emulated_instruction(vcpu);
6650         WARN(1, "this should never happen\n");
6651         return 1;
6652 }
6653
6654 static int handle_apic_access(struct kvm_vcpu *vcpu)
6655 {
6656         if (likely(fasteoi)) {
6657                 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6658                 int access_type, offset;
6659
6660                 access_type = exit_qualification & APIC_ACCESS_TYPE;
6661                 offset = exit_qualification & APIC_ACCESS_OFFSET;
6662                 /*
6663                  * Sane guest uses MOV to write EOI, with written value
6664                  * not cared. So make a short-circuit here by avoiding
6665                  * heavy instruction emulation.
6666                  */
6667                 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
6668                     (offset == APIC_EOI)) {
6669                         kvm_lapic_set_eoi(vcpu);
6670                         return kvm_skip_emulated_instruction(vcpu);
6671                 }
6672         }
6673         return emulate_instruction(vcpu, 0) == EMULATE_DONE;
6674 }
6675
6676 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
6677 {
6678         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6679         int vector = exit_qualification & 0xff;
6680
6681         /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
6682         kvm_apic_set_eoi_accelerated(vcpu, vector);
6683         return 1;
6684 }
6685
6686 static int handle_apic_write(struct kvm_vcpu *vcpu)
6687 {
6688         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6689         u32 offset = exit_qualification & 0xfff;
6690
6691         /* APIC-write VM exit is trap-like and thus no need to adjust IP */
6692         kvm_apic_write_nodecode(vcpu, offset);
6693         return 1;
6694 }
6695
6696 static int handle_task_switch(struct kvm_vcpu *vcpu)
6697 {
6698         struct vcpu_vmx *vmx = to_vmx(vcpu);
6699         unsigned long exit_qualification;
6700         bool has_error_code = false;
6701         u32 error_code = 0;
6702         u16 tss_selector;
6703         int reason, type, idt_v, idt_index;
6704
6705         idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
6706         idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
6707         type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
6708
6709         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6710
6711         reason = (u32)exit_qualification >> 30;
6712         if (reason == TASK_SWITCH_GATE && idt_v) {
6713                 switch (type) {
6714                 case INTR_TYPE_NMI_INTR:
6715                         vcpu->arch.nmi_injected = false;
6716                         vmx_set_nmi_mask(vcpu, true);
6717                         break;
6718                 case INTR_TYPE_EXT_INTR:
6719                 case INTR_TYPE_SOFT_INTR:
6720                         kvm_clear_interrupt_queue(vcpu);
6721                         break;
6722                 case INTR_TYPE_HARD_EXCEPTION:
6723                         if (vmx->idt_vectoring_info &
6724                             VECTORING_INFO_DELIVER_CODE_MASK) {
6725                                 has_error_code = true;
6726                                 error_code =
6727                                         vmcs_read32(IDT_VECTORING_ERROR_CODE);
6728                         }
6729                         /* fall through */
6730                 case INTR_TYPE_SOFT_EXCEPTION:
6731                         kvm_clear_exception_queue(vcpu);
6732                         break;
6733                 default:
6734                         break;
6735                 }
6736         }
6737         tss_selector = exit_qualification;
6738
6739         if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
6740                        type != INTR_TYPE_EXT_INTR &&
6741                        type != INTR_TYPE_NMI_INTR))
6742                 skip_emulated_instruction(vcpu);
6743
6744         if (kvm_task_switch(vcpu, tss_selector,
6745                             type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
6746                             has_error_code, error_code) == EMULATE_FAIL) {
6747                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6748                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
6749                 vcpu->run->internal.ndata = 0;
6750                 return 0;
6751         }
6752
6753         /*
6754          * TODO: What about debug traps on tss switch?
6755          *       Are we supposed to inject them and update dr6?
6756          */
6757
6758         return 1;
6759 }
6760
6761 static int handle_ept_violation(struct kvm_vcpu *vcpu)
6762 {
6763         unsigned long exit_qualification;
6764         gpa_t gpa;
6765         u64 error_code;
6766
6767         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6768
6769         /*
6770          * EPT violation happened while executing iret from NMI,
6771          * "blocked by NMI" bit has to be set before next VM entry.
6772          * There are errata that may cause this bit to not be set:
6773          * AAK134, BY25.
6774          */
6775         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
6776                         cpu_has_virtual_nmis() &&
6777                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
6778                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
6779
6780         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
6781         trace_kvm_page_fault(gpa, exit_qualification);
6782
6783         /* Is it a read fault? */
6784         error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
6785                      ? PFERR_USER_MASK : 0;
6786         /* Is it a write fault? */
6787         error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
6788                       ? PFERR_WRITE_MASK : 0;
6789         /* Is it a fetch fault? */
6790         error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
6791                       ? PFERR_FETCH_MASK : 0;
6792         /* ept page table entry is present? */
6793         error_code |= (exit_qualification &
6794                        (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE |
6795                         EPT_VIOLATION_EXECUTABLE))
6796                       ? PFERR_PRESENT_MASK : 0;
6797
6798         error_code |= (exit_qualification & 0x100) != 0 ?
6799                PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
6800
6801         vcpu->arch.exit_qualification = exit_qualification;
6802         return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
6803 }
6804
6805 static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
6806 {
6807         int ret;
6808         gpa_t gpa;
6809
6810         /*
6811          * A nested guest cannot optimize MMIO vmexits, because we have an
6812          * nGPA here instead of the required GPA.
6813          */
6814         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
6815         if (!is_guest_mode(vcpu) &&
6816             !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
6817                 trace_kvm_fast_mmio(gpa);
6818                 /*
6819                  * Doing kvm_skip_emulated_instruction() depends on undefined
6820                  * behavior: Intel's manual doesn't mandate
6821                  * VM_EXIT_INSTRUCTION_LEN to be set in VMCS when EPT MISCONFIG
6822                  * occurs and while on real hardware it was observed to be set,
6823                  * other hypervisors (namely Hyper-V) don't set it, we end up
6824                  * advancing IP with some random value. Disable fast mmio when
6825                  * running nested and keep it for real hardware in hope that
6826                  * VM_EXIT_INSTRUCTION_LEN will always be set correctly.
6827                  */
6828                 if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
6829                         return kvm_skip_emulated_instruction(vcpu);
6830                 else
6831                         return x86_emulate_instruction(vcpu, gpa, EMULTYPE_SKIP,
6832                                                        NULL, 0) == EMULATE_DONE;
6833         }
6834
6835         ret = kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
6836         if (ret >= 0)
6837                 return ret;
6838
6839         /* It is the real ept misconfig */
6840         WARN_ON(1);
6841
6842         vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
6843         vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
6844
6845         return 0;
6846 }
6847
6848 static int handle_nmi_window(struct kvm_vcpu *vcpu)
6849 {
6850         vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
6851                         CPU_BASED_VIRTUAL_NMI_PENDING);
6852         ++vcpu->stat.nmi_window_exits;
6853         kvm_make_request(KVM_REQ_EVENT, vcpu);
6854
6855         return 1;
6856 }
6857
6858 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
6859 {
6860         struct vcpu_vmx *vmx = to_vmx(vcpu);
6861         enum emulation_result err = EMULATE_DONE;
6862         int ret = 1;
6863         u32 cpu_exec_ctrl;
6864         bool intr_window_requested;
6865         unsigned count = 130;
6866
6867         cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
6868         intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
6869
6870         while (vmx->emulation_required && count-- != 0) {
6871                 if (intr_window_requested && vmx_interrupt_allowed(vcpu))
6872                         return handle_interrupt_window(&vmx->vcpu);
6873
6874                 if (kvm_test_request(KVM_REQ_EVENT, vcpu))
6875                         return 1;
6876
6877                 err = emulate_instruction(vcpu, 0);
6878
6879                 if (err == EMULATE_USER_EXIT) {
6880                         ++vcpu->stat.mmio_exits;
6881                         ret = 0;
6882                         goto out;
6883                 }
6884
6885                 if (err != EMULATE_DONE)
6886                         goto emulation_error;
6887
6888                 if (vmx->emulation_required && !vmx->rmode.vm86_active &&
6889                     vcpu->arch.exception.pending)
6890                         goto emulation_error;
6891
6892                 if (vcpu->arch.halt_request) {
6893                         vcpu->arch.halt_request = 0;
6894                         ret = kvm_vcpu_halt(vcpu);
6895                         goto out;
6896                 }
6897
6898                 if (signal_pending(current))
6899                         goto out;
6900                 if (need_resched())
6901                         schedule();
6902         }
6903
6904 out:
6905         return ret;
6906
6907 emulation_error:
6908         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6909         vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
6910         vcpu->run->internal.ndata = 0;
6911         return 0;
6912 }
6913
6914 static int __grow_ple_window(int val)
6915 {
6916         if (ple_window_grow < 1)
6917                 return ple_window;
6918
6919         val = min(val, ple_window_actual_max);
6920
6921         if (ple_window_grow < ple_window)
6922                 val *= ple_window_grow;
6923         else
6924                 val += ple_window_grow;
6925
6926         return val;
6927 }
6928
6929 static int __shrink_ple_window(int val, int modifier, int minimum)
6930 {
6931         if (modifier < 1)
6932                 return ple_window;
6933
6934         if (modifier < ple_window)
6935                 val /= modifier;
6936         else
6937                 val -= modifier;
6938
6939         return max(val, minimum);
6940 }
6941
6942 static void grow_ple_window(struct kvm_vcpu *vcpu)
6943 {
6944         struct vcpu_vmx *vmx = to_vmx(vcpu);
6945         int old = vmx->ple_window;
6946
6947         vmx->ple_window = __grow_ple_window(old);
6948
6949         if (vmx->ple_window != old)
6950                 vmx->ple_window_dirty = true;
6951
6952         trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);
6953 }
6954
6955 static void shrink_ple_window(struct kvm_vcpu *vcpu)
6956 {
6957         struct vcpu_vmx *vmx = to_vmx(vcpu);
6958         int old = vmx->ple_window;
6959
6960         vmx->ple_window = __shrink_ple_window(old,
6961                                               ple_window_shrink, ple_window);
6962
6963         if (vmx->ple_window != old)
6964                 vmx->ple_window_dirty = true;
6965
6966         trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);
6967 }
6968
6969 /*
6970  * ple_window_actual_max is computed to be one grow_ple_window() below
6971  * ple_window_max. (See __grow_ple_window for the reason.)
6972  * This prevents overflows, because ple_window_max is int.
6973  * ple_window_max effectively rounded down to a multiple of ple_window_grow in
6974  * this process.
6975  * ple_window_max is also prevented from setting vmx->ple_window < ple_window.
6976  */
6977 static void update_ple_window_actual_max(void)
6978 {
6979         ple_window_actual_max =
6980                         __shrink_ple_window(max(ple_window_max, ple_window),
6981                                             ple_window_grow, INT_MIN);
6982 }
6983
6984 /*
6985  * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
6986  */
6987 static void wakeup_handler(void)
6988 {
6989         struct kvm_vcpu *vcpu;
6990         int cpu = smp_processor_id();
6991
6992         spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
6993         list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
6994                         blocked_vcpu_list) {
6995                 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
6996
6997                 if (pi_test_on(pi_desc) == 1)
6998                         kvm_vcpu_kick(vcpu);
6999         }
7000         spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
7001 }
7002
7003 void vmx_enable_tdp(void)
7004 {
7005         kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK,
7006                 enable_ept_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull,
7007                 enable_ept_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull,
7008                 0ull, VMX_EPT_EXECUTABLE_MASK,
7009                 cpu_has_vmx_ept_execute_only() ? 0ull : VMX_EPT_READABLE_MASK,
7010                 VMX_EPT_RWX_MASK, 0ull);
7011
7012         ept_set_mmio_spte_mask();
7013         kvm_enable_tdp();
7014 }
7015
7016 static __init int hardware_setup(void)
7017 {
7018         int r = -ENOMEM, i;
7019
7020         rdmsrl_safe(MSR_EFER, &host_efer);
7021
7022         for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
7023                 kvm_define_shared_msr(i, vmx_msr_index[i]);
7024
7025         for (i = 0; i < VMX_BITMAP_NR; i++) {
7026                 vmx_bitmap[i] = (unsigned long *)__get_free_page(GFP_KERNEL);
7027                 if (!vmx_bitmap[i])
7028                         goto out;
7029         }
7030
7031         memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
7032         memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
7033
7034         memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
7035
7036         memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
7037
7038         if (setup_vmcs_config(&vmcs_config) < 0) {
7039                 r = -EIO;
7040                 goto out;
7041         }
7042
7043         if (boot_cpu_has(X86_FEATURE_NX))
7044                 kvm_enable_efer_bits(EFER_NX);
7045
7046         if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() ||
7047                 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global()))
7048                 enable_vpid = 0;
7049
7050         if (!cpu_has_vmx_shadow_vmcs())
7051                 enable_shadow_vmcs = 0;
7052         if (enable_shadow_vmcs)
7053                 init_vmcs_shadow_fields();
7054
7055         if (!cpu_has_vmx_ept() ||
7056             !cpu_has_vmx_ept_4levels() ||
7057             !cpu_has_vmx_ept_mt_wb()) {
7058                 enable_ept = 0;
7059                 enable_unrestricted_guest = 0;
7060                 enable_ept_ad_bits = 0;
7061         }
7062
7063         if (!cpu_has_vmx_ept_ad_bits() || !enable_ept)
7064                 enable_ept_ad_bits = 0;
7065
7066         if (!cpu_has_vmx_unrestricted_guest())
7067                 enable_unrestricted_guest = 0;
7068
7069         if (!cpu_has_vmx_flexpriority())
7070                 flexpriority_enabled = 0;
7071
7072         /*
7073          * set_apic_access_page_addr() is used to reload apic access
7074          * page upon invalidation.  No need to do anything if not
7075          * using the APIC_ACCESS_ADDR VMCS field.
7076          */
7077         if (!flexpriority_enabled)
7078                 kvm_x86_ops->set_apic_access_page_addr = NULL;
7079
7080         if (!cpu_has_vmx_tpr_shadow())
7081                 kvm_x86_ops->update_cr8_intercept = NULL;
7082
7083         if (enable_ept && !cpu_has_vmx_ept_2m_page())
7084                 kvm_disable_largepages();
7085
7086         if (!cpu_has_vmx_ple())
7087                 ple_gap = 0;
7088
7089         if (!cpu_has_vmx_apicv()) {
7090                 enable_apicv = 0;
7091                 kvm_x86_ops->sync_pir_to_irr = NULL;
7092         }
7093
7094         if (cpu_has_vmx_tsc_scaling()) {
7095                 kvm_has_tsc_control = true;
7096                 kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
7097                 kvm_tsc_scaling_ratio_frac_bits = 48;
7098         }
7099
7100         set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
7101
7102         if (enable_ept)
7103                 vmx_enable_tdp();
7104         else
7105                 kvm_disable_tdp();
7106
7107         update_ple_window_actual_max();
7108
7109         /*
7110          * Only enable PML when hardware supports PML feature, and both EPT
7111          * and EPT A/D bit features are enabled -- PML depends on them to work.
7112          */
7113         if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
7114                 enable_pml = 0;
7115
7116         if (!enable_pml) {
7117                 kvm_x86_ops->slot_enable_log_dirty = NULL;
7118                 kvm_x86_ops->slot_disable_log_dirty = NULL;
7119                 kvm_x86_ops->flush_log_dirty = NULL;
7120                 kvm_x86_ops->enable_log_dirty_pt_masked = NULL;
7121         }
7122
7123         if (cpu_has_vmx_preemption_timer() && enable_preemption_timer) {
7124                 u64 vmx_msr;
7125
7126                 rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
7127                 cpu_preemption_timer_multi =
7128                          vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
7129         } else {
7130                 kvm_x86_ops->set_hv_timer = NULL;
7131                 kvm_x86_ops->cancel_hv_timer = NULL;
7132         }
7133
7134         kvm_set_posted_intr_wakeup_handler(wakeup_handler);
7135
7136         kvm_mce_cap_supported |= MCG_LMCE_P;
7137
7138         return alloc_kvm_area();
7139
7140 out:
7141         for (i = 0; i < VMX_BITMAP_NR; i++)
7142                 free_page((unsigned long)vmx_bitmap[i]);
7143
7144     return r;
7145 }
7146
7147 static __exit void hardware_unsetup(void)
7148 {
7149         int i;
7150
7151         for (i = 0; i < VMX_BITMAP_NR; i++)
7152                 free_page((unsigned long)vmx_bitmap[i]);
7153
7154         free_kvm_area();
7155 }
7156
7157 /*
7158  * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
7159  * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
7160  */
7161 static int handle_pause(struct kvm_vcpu *vcpu)
7162 {
7163         if (ple_gap)
7164                 grow_ple_window(vcpu);
7165
7166         /*
7167          * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting"
7168          * VM-execution control is ignored if CPL > 0. OTOH, KVM
7169          * never set PAUSE_EXITING and just set PLE if supported,
7170          * so the vcpu must be CPL=0 if it gets a PAUSE exit.
7171          */
7172         kvm_vcpu_on_spin(vcpu, true);
7173         return kvm_skip_emulated_instruction(vcpu);
7174 }
7175
7176 static int handle_nop(struct kvm_vcpu *vcpu)
7177 {
7178         return kvm_skip_emulated_instruction(vcpu);
7179 }
7180
7181 static int handle_mwait(struct kvm_vcpu *vcpu)
7182 {
7183         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
7184         return handle_nop(vcpu);
7185 }
7186
7187 static int handle_invalid_op(struct kvm_vcpu *vcpu)
7188 {
7189         kvm_queue_exception(vcpu, UD_VECTOR);
7190         return 1;
7191 }
7192
7193 static int handle_monitor_trap(struct kvm_vcpu *vcpu)
7194 {
7195         return 1;
7196 }
7197
7198 static int handle_monitor(struct kvm_vcpu *vcpu)
7199 {
7200         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
7201         return handle_nop(vcpu);
7202 }
7203
7204 /*
7205  * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
7206  * set the success or error code of an emulated VMX instruction, as specified
7207  * by Vol 2B, VMX Instruction Reference, "Conventions".
7208  */
7209 static void nested_vmx_succeed(struct kvm_vcpu *vcpu)
7210 {
7211         vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
7212                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
7213                             X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
7214 }
7215
7216 static void nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
7217 {
7218         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
7219                         & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
7220                             X86_EFLAGS_SF | X86_EFLAGS_OF))
7221                         | X86_EFLAGS_CF);
7222 }
7223
7224 static void nested_vmx_failValid(struct kvm_vcpu *vcpu,
7225                                         u32 vm_instruction_error)
7226 {
7227         if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {
7228                 /*
7229                  * failValid writes the error number to the current VMCS, which
7230                  * can't be done there isn't a current VMCS.
7231                  */
7232                 nested_vmx_failInvalid(vcpu);
7233                 return;
7234         }
7235         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
7236                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
7237                             X86_EFLAGS_SF | X86_EFLAGS_OF))
7238                         | X86_EFLAGS_ZF);
7239         get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
7240         /*
7241          * We don't need to force a shadow sync because
7242          * VM_INSTRUCTION_ERROR is not shadowed
7243          */
7244 }
7245
7246 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
7247 {
7248         /* TODO: not to reset guest simply here. */
7249         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
7250         pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
7251 }
7252
7253 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
7254 {
7255         struct vcpu_vmx *vmx =
7256                 container_of(timer, struct vcpu_vmx, nested.preemption_timer);
7257
7258         vmx->nested.preemption_timer_expired = true;
7259         kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
7260         kvm_vcpu_kick(&vmx->vcpu);
7261
7262         return HRTIMER_NORESTART;
7263 }
7264
7265 /*
7266  * Decode the memory-address operand of a vmx instruction, as recorded on an
7267  * exit caused by such an instruction (run by a guest hypervisor).
7268  * On success, returns 0. When the operand is invalid, returns 1 and throws
7269  * #UD or #GP.
7270  */
7271 static int get_vmx_mem_address(struct kvm_vcpu *vcpu,
7272                                  unsigned long exit_qualification,
7273                                  u32 vmx_instruction_info, bool wr, gva_t *ret)
7274 {
7275         gva_t off;
7276         bool exn;
7277         struct kvm_segment s;
7278
7279         /*
7280          * According to Vol. 3B, "Information for VM Exits Due to Instruction
7281          * Execution", on an exit, vmx_instruction_info holds most of the
7282          * addressing components of the operand. Only the displacement part
7283          * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
7284          * For how an actual address is calculated from all these components,
7285          * refer to Vol. 1, "Operand Addressing".
7286          */
7287         int  scaling = vmx_instruction_info & 3;
7288         int  addr_size = (vmx_instruction_info >> 7) & 7;
7289         bool is_reg = vmx_instruction_info & (1u << 10);
7290         int  seg_reg = (vmx_instruction_info >> 15) & 7;
7291         int  index_reg = (vmx_instruction_info >> 18) & 0xf;
7292         bool index_is_valid = !(vmx_instruction_info & (1u << 22));
7293         int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
7294         bool base_is_valid  = !(vmx_instruction_info & (1u << 27));
7295
7296         if (is_reg) {
7297                 kvm_queue_exception(vcpu, UD_VECTOR);
7298                 return 1;
7299         }
7300
7301         /* Addr = segment_base + offset */
7302         /* offset = base + [index * scale] + displacement */
7303         off = exit_qualification; /* holds the displacement */
7304         if (base_is_valid)
7305                 off += kvm_register_read(vcpu, base_reg);
7306         if (index_is_valid)
7307                 off += kvm_register_read(vcpu, index_reg)<<scaling;
7308         vmx_get_segment(vcpu, &s, seg_reg);
7309         *ret = s.base + off;
7310
7311         if (addr_size == 1) /* 32 bit */
7312                 *ret &= 0xffffffff;
7313
7314         /* Checks for #GP/#SS exceptions. */
7315         exn = false;
7316         if (is_long_mode(vcpu)) {
7317                 /* Long mode: #GP(0)/#SS(0) if the memory address is in a
7318                  * non-canonical form. This is the only check on the memory
7319                  * destination for long mode!
7320                  */
7321                 exn = is_noncanonical_address(*ret, vcpu);
7322         } else if (is_protmode(vcpu)) {
7323                 /* Protected mode: apply checks for segment validity in the
7324                  * following order:
7325                  * - segment type check (#GP(0) may be thrown)
7326                  * - usability check (#GP(0)/#SS(0))
7327                  * - limit check (#GP(0)/#SS(0))
7328                  */
7329                 if (wr)
7330                         /* #GP(0) if the destination operand is located in a
7331                          * read-only data segment or any code segment.
7332                          */
7333                         exn = ((s.type & 0xa) == 0 || (s.type & 8));
7334                 else
7335                         /* #GP(0) if the source operand is located in an
7336                          * execute-only code segment
7337                          */
7338                         exn = ((s.type & 0xa) == 8);
7339                 if (exn) {
7340                         kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
7341                         return 1;
7342                 }
7343                 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
7344                  */
7345                 exn = (s.unusable != 0);
7346                 /* Protected mode: #GP(0)/#SS(0) if the memory
7347                  * operand is outside the segment limit.
7348                  */
7349                 exn = exn || (off + sizeof(u64) > s.limit);
7350         }
7351         if (exn) {
7352                 kvm_queue_exception_e(vcpu,
7353                                       seg_reg == VCPU_SREG_SS ?
7354                                                 SS_VECTOR : GP_VECTOR,
7355                                       0);
7356                 return 1;
7357         }
7358
7359         return 0;
7360 }
7361
7362 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer)
7363 {
7364         gva_t gva;
7365         struct x86_exception e;
7366
7367         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
7368                         vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva))
7369                 return 1;
7370
7371         if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) {
7372                 kvm_inject_page_fault(vcpu, &e);
7373                 return 1;
7374         }
7375
7376         return 0;
7377 }
7378
7379 static int enter_vmx_operation(struct kvm_vcpu *vcpu)
7380 {
7381         struct vcpu_vmx *vmx = to_vmx(vcpu);
7382         struct vmcs *shadow_vmcs;
7383         int r;
7384
7385         r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
7386         if (r < 0)
7387                 goto out_vmcs02;
7388
7389         vmx->nested.cached_vmcs12 = kmalloc(VMCS12_SIZE, GFP_KERNEL);
7390         if (!vmx->nested.cached_vmcs12)
7391                 goto out_cached_vmcs12;
7392
7393         if (enable_shadow_vmcs) {
7394                 shadow_vmcs = alloc_vmcs();
7395                 if (!shadow_vmcs)
7396                         goto out_shadow_vmcs;
7397                 /* mark vmcs as shadow */
7398                 shadow_vmcs->revision_id |= (1u << 31);
7399                 /* init shadow vmcs */
7400                 vmcs_clear(shadow_vmcs);
7401                 vmx->vmcs01.shadow_vmcs = shadow_vmcs;
7402         }
7403
7404         hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
7405                      HRTIMER_MODE_REL_PINNED);
7406         vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
7407
7408         vmx->nested.vpid02 = allocate_vpid();
7409
7410         vmx->nested.vmxon = true;
7411         return 0;
7412
7413 out_shadow_vmcs:
7414         kfree(vmx->nested.cached_vmcs12);
7415
7416 out_cached_vmcs12:
7417         free_loaded_vmcs(&vmx->nested.vmcs02);
7418
7419 out_vmcs02:
7420         return -ENOMEM;
7421 }
7422
7423 /*
7424  * Emulate the VMXON instruction.
7425  * Currently, we just remember that VMX is active, and do not save or even
7426  * inspect the argument to VMXON (the so-called "VMXON pointer") because we
7427  * do not currently need to store anything in that guest-allocated memory
7428  * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
7429  * argument is different from the VMXON pointer (which the spec says they do).
7430  */
7431 static int handle_vmon(struct kvm_vcpu *vcpu)
7432 {
7433         int ret;
7434         gpa_t vmptr;
7435         struct page *page;
7436         struct vcpu_vmx *vmx = to_vmx(vcpu);
7437         const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
7438                 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
7439
7440         /*
7441          * The Intel VMX Instruction Reference lists a bunch of bits that are
7442          * prerequisite to running VMXON, most notably cr4.VMXE must be set to
7443          * 1 (see vmx_set_cr4() for when we allow the guest to set this).
7444          * Otherwise, we should fail with #UD.  But most faulting conditions
7445          * have already been checked by hardware, prior to the VM-exit for
7446          * VMXON.  We do test guest cr4.VMXE because processor CR4 always has
7447          * that bit set to 1 in non-root mode.
7448          */
7449         if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) {
7450                 kvm_queue_exception(vcpu, UD_VECTOR);
7451                 return 1;
7452         }
7453
7454         /* CPL=0 must be checked manually. */
7455         if (vmx_get_cpl(vcpu)) {
7456                 kvm_queue_exception(vcpu, UD_VECTOR);
7457                 return 1;
7458         }
7459
7460         if (vmx->nested.vmxon) {
7461                 nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
7462                 return kvm_skip_emulated_instruction(vcpu);
7463         }
7464
7465         if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
7466                         != VMXON_NEEDED_FEATURES) {
7467                 kvm_inject_gp(vcpu, 0);
7468                 return 1;
7469         }
7470
7471         if (nested_vmx_get_vmptr(vcpu, &vmptr))
7472                 return 1;
7473
7474         /*
7475          * SDM 3: 24.11.5
7476          * The first 4 bytes of VMXON region contain the supported
7477          * VMCS revision identifier
7478          *
7479          * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case;
7480          * which replaces physical address width with 32
7481          */
7482         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
7483                 nested_vmx_failInvalid(vcpu);
7484                 return kvm_skip_emulated_instruction(vcpu);
7485         }
7486
7487         page = kvm_vcpu_gpa_to_page(vcpu, vmptr);
7488         if (is_error_page(page)) {
7489                 nested_vmx_failInvalid(vcpu);
7490                 return kvm_skip_emulated_instruction(vcpu);
7491         }
7492         if (*(u32 *)kmap(page) != VMCS12_REVISION) {
7493                 kunmap(page);
7494                 kvm_release_page_clean(page);
7495                 nested_vmx_failInvalid(vcpu);
7496                 return kvm_skip_emulated_instruction(vcpu);
7497         }
7498         kunmap(page);
7499         kvm_release_page_clean(page);
7500
7501         vmx->nested.vmxon_ptr = vmptr;
7502         ret = enter_vmx_operation(vcpu);
7503         if (ret)
7504                 return ret;
7505
7506         nested_vmx_succeed(vcpu);
7507         return kvm_skip_emulated_instruction(vcpu);
7508 }
7509
7510 /*
7511  * Intel's VMX Instruction Reference specifies a common set of prerequisites
7512  * for running VMX instructions (except VMXON, whose prerequisites are
7513  * slightly different). It also specifies what exception to inject otherwise.
7514  * Note that many of these exceptions have priority over VM exits, so they
7515  * don't have to be checked again here.
7516  */
7517 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
7518 {
7519         if (vmx_get_cpl(vcpu)) {
7520                 kvm_queue_exception(vcpu, UD_VECTOR);
7521                 return 0;
7522         }
7523
7524         if (!to_vmx(vcpu)->nested.vmxon) {
7525                 kvm_queue_exception(vcpu, UD_VECTOR);
7526                 return 0;
7527         }
7528         return 1;
7529 }
7530
7531 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
7532 {
7533         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS);
7534         vmcs_write64(VMCS_LINK_POINTER, -1ull);
7535 }
7536
7537 static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
7538 {
7539         if (vmx->nested.current_vmptr == -1ull)
7540                 return;
7541
7542         if (enable_shadow_vmcs) {
7543                 /* copy to memory all shadowed fields in case
7544                    they were modified */
7545                 copy_shadow_to_vmcs12(vmx);
7546                 vmx->nested.sync_shadow_vmcs = false;
7547                 vmx_disable_shadow_vmcs(vmx);
7548         }
7549         vmx->nested.posted_intr_nv = -1;
7550
7551         /* Flush VMCS12 to guest memory */
7552         kvm_vcpu_write_guest_page(&vmx->vcpu,
7553                                   vmx->nested.current_vmptr >> PAGE_SHIFT,
7554                                   vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
7555
7556         vmx->nested.current_vmptr = -1ull;
7557 }
7558
7559 /*
7560  * Free whatever needs to be freed from vmx->nested when L1 goes down, or
7561  * just stops using VMX.
7562  */
7563 static void free_nested(struct vcpu_vmx *vmx)
7564 {
7565         if (!vmx->nested.vmxon)
7566                 return;
7567
7568         vmx->nested.vmxon = false;
7569         free_vpid(vmx->nested.vpid02);
7570         vmx->nested.posted_intr_nv = -1;
7571         vmx->nested.current_vmptr = -1ull;
7572         if (enable_shadow_vmcs) {
7573                 vmx_disable_shadow_vmcs(vmx);
7574                 vmcs_clear(vmx->vmcs01.shadow_vmcs);
7575                 free_vmcs(vmx->vmcs01.shadow_vmcs);
7576                 vmx->vmcs01.shadow_vmcs = NULL;
7577         }
7578         kfree(vmx->nested.cached_vmcs12);
7579         /* Unpin physical memory we referred to in the vmcs02 */
7580         if (vmx->nested.apic_access_page) {
7581                 kvm_release_page_dirty(vmx->nested.apic_access_page);
7582                 vmx->nested.apic_access_page = NULL;
7583         }
7584         if (vmx->nested.virtual_apic_page) {
7585                 kvm_release_page_dirty(vmx->nested.virtual_apic_page);
7586                 vmx->nested.virtual_apic_page = NULL;
7587         }
7588         if (vmx->nested.pi_desc_page) {
7589                 kunmap(vmx->nested.pi_desc_page);
7590                 kvm_release_page_dirty(vmx->nested.pi_desc_page);
7591                 vmx->nested.pi_desc_page = NULL;
7592                 vmx->nested.pi_desc = NULL;
7593         }
7594
7595         free_loaded_vmcs(&vmx->nested.vmcs02);
7596 }
7597
7598 /* Emulate the VMXOFF instruction */
7599 static int handle_vmoff(struct kvm_vcpu *vcpu)
7600 {
7601         if (!nested_vmx_check_permission(vcpu))
7602                 return 1;
7603         free_nested(to_vmx(vcpu));
7604         nested_vmx_succeed(vcpu);
7605         return kvm_skip_emulated_instruction(vcpu);
7606 }
7607
7608 /* Emulate the VMCLEAR instruction */
7609 static int handle_vmclear(struct kvm_vcpu *vcpu)
7610 {
7611         struct vcpu_vmx *vmx = to_vmx(vcpu);
7612         u32 zero = 0;
7613         gpa_t vmptr;
7614
7615         if (!nested_vmx_check_permission(vcpu))
7616                 return 1;
7617
7618         if (nested_vmx_get_vmptr(vcpu, &vmptr))
7619                 return 1;
7620
7621         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
7622                 nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS);
7623                 return kvm_skip_emulated_instruction(vcpu);
7624         }
7625
7626         if (vmptr == vmx->nested.vmxon_ptr) {
7627                 nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_VMXON_POINTER);
7628                 return kvm_skip_emulated_instruction(vcpu);
7629         }
7630
7631         if (vmptr == vmx->nested.current_vmptr)
7632                 nested_release_vmcs12(vmx);
7633
7634         kvm_vcpu_write_guest(vcpu,
7635                         vmptr + offsetof(struct vmcs12, launch_state),
7636                         &zero, sizeof(zero));
7637
7638         nested_vmx_succeed(vcpu);
7639         return kvm_skip_emulated_instruction(vcpu);
7640 }
7641
7642 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);
7643
7644 /* Emulate the VMLAUNCH instruction */
7645 static int handle_vmlaunch(struct kvm_vcpu *vcpu)
7646 {
7647         return nested_vmx_run(vcpu, true);
7648 }
7649
7650 /* Emulate the VMRESUME instruction */
7651 static int handle_vmresume(struct kvm_vcpu *vcpu)
7652 {
7653
7654         return nested_vmx_run(vcpu, false);
7655 }
7656
7657 /*
7658  * Read a vmcs12 field. Since these can have varying lengths and we return
7659  * one type, we chose the biggest type (u64) and zero-extend the return value
7660  * to that size. Note that the caller, handle_vmread, might need to use only
7661  * some of the bits we return here (e.g., on 32-bit guests, only 32 bits of
7662  * 64-bit fields are to be returned).
7663  */
7664 static inline int vmcs12_read_any(struct kvm_vcpu *vcpu,
7665                                   unsigned long field, u64 *ret)
7666 {
7667         short offset = vmcs_field_to_offset(field);
7668         char *p;
7669
7670         if (offset < 0)
7671                 return offset;
7672
7673         p = ((char *)(get_vmcs12(vcpu))) + offset;
7674
7675         switch (vmcs_field_type(field)) {
7676         case VMCS_FIELD_TYPE_NATURAL_WIDTH:
7677                 *ret = *((natural_width *)p);
7678                 return 0;
7679         case VMCS_FIELD_TYPE_U16:
7680                 *ret = *((u16 *)p);
7681                 return 0;
7682         case VMCS_FIELD_TYPE_U32:
7683                 *ret = *((u32 *)p);
7684                 return 0;
7685         case VMCS_FIELD_TYPE_U64:
7686                 *ret = *((u64 *)p);
7687                 return 0;
7688         default:
7689                 WARN_ON(1);
7690                 return -ENOENT;
7691         }
7692 }
7693
7694
7695 static inline int vmcs12_write_any(struct kvm_vcpu *vcpu,
7696                                    unsigned long field, u64 field_value){
7697         short offset = vmcs_field_to_offset(field);
7698         char *p = ((char *) get_vmcs12(vcpu)) + offset;
7699         if (offset < 0)
7700                 return offset;
7701
7702         switch (vmcs_field_type(field)) {
7703         case VMCS_FIELD_TYPE_U16:
7704                 *(u16 *)p = field_value;
7705                 return 0;
7706         case VMCS_FIELD_TYPE_U32:
7707                 *(u32 *)p = field_value;
7708                 return 0;
7709         case VMCS_FIELD_TYPE_U64:
7710                 *(u64 *)p = field_value;
7711                 return 0;
7712         case VMCS_FIELD_TYPE_NATURAL_WIDTH:
7713                 *(natural_width *)p = field_value;
7714                 return 0;
7715         default:
7716                 WARN_ON(1);
7717                 return -ENOENT;
7718         }
7719
7720 }
7721
7722 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
7723 {
7724         int i;
7725         unsigned long field;
7726         u64 field_value;
7727         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
7728         const unsigned long *fields = shadow_read_write_fields;
7729         const int num_fields = max_shadow_read_write_fields;
7730
7731         preempt_disable();
7732
7733         vmcs_load(shadow_vmcs);
7734
7735         for (i = 0; i < num_fields; i++) {
7736                 field = fields[i];
7737                 switch (vmcs_field_type(field)) {
7738                 case VMCS_FIELD_TYPE_U16:
7739                         field_value = vmcs_read16(field);
7740                         break;
7741                 case VMCS_FIELD_TYPE_U32:
7742                         field_value = vmcs_read32(field);
7743                         break;
7744                 case VMCS_FIELD_TYPE_U64:
7745                         field_value = vmcs_read64(field);
7746                         break;
7747                 case VMCS_FIELD_TYPE_NATURAL_WIDTH:
7748                         field_value = vmcs_readl(field);
7749                         break;
7750                 default:
7751                         WARN_ON(1);
7752                         continue;
7753                 }
7754                 vmcs12_write_any(&vmx->vcpu, field, field_value);
7755         }
7756
7757         vmcs_clear(shadow_vmcs);
7758         vmcs_load(vmx->loaded_vmcs->vmcs);
7759
7760         preempt_enable();
7761 }
7762
7763 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
7764 {
7765         const unsigned long *fields[] = {
7766                 shadow_read_write_fields,
7767                 shadow_read_only_fields
7768         };
7769         const int max_fields[] = {
7770                 max_shadow_read_write_fields,
7771                 max_shadow_read_only_fields
7772         };
7773         int i, q;
7774         unsigned long field;
7775         u64 field_value = 0;
7776         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
7777
7778         vmcs_load(shadow_vmcs);
7779
7780         for (q = 0; q < ARRAY_SIZE(fields); q++) {
7781                 for (i = 0; i < max_fields[q]; i++) {
7782                         field = fields[q][i];
7783                         vmcs12_read_any(&vmx->vcpu, field, &field_value);
7784
7785                         switch (vmcs_field_type(field)) {
7786                         case VMCS_FIELD_TYPE_U16:
7787                                 vmcs_write16(field, (u16)field_value);
7788                                 break;
7789                         case VMCS_FIELD_TYPE_U32:
7790                                 vmcs_write32(field, (u32)field_value);
7791                                 break;
7792                         case VMCS_FIELD_TYPE_U64:
7793                                 vmcs_write64(field, (u64)field_value);
7794                                 break;
7795                         case VMCS_FIELD_TYPE_NATURAL_WIDTH:
7796                                 vmcs_writel(field, (long)field_value);
7797                                 break;
7798                         default:
7799                                 WARN_ON(1);
7800                                 break;
7801                         }
7802                 }
7803         }
7804
7805         vmcs_clear(shadow_vmcs);
7806         vmcs_load(vmx->loaded_vmcs->vmcs);
7807 }
7808
7809 /*
7810  * VMX instructions which assume a current vmcs12 (i.e., that VMPTRLD was
7811  * used before) all generate the same failure when it is missing.
7812  */
7813 static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu)
7814 {
7815         struct vcpu_vmx *vmx = to_vmx(vcpu);
7816         if (vmx->nested.current_vmptr == -1ull) {
7817                 nested_vmx_failInvalid(vcpu);
7818                 return 0;
7819         }
7820         return 1;
7821 }
7822
7823 static int handle_vmread(struct kvm_vcpu *vcpu)
7824 {
7825         unsigned long field;
7826         u64 field_value;
7827         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7828         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7829         gva_t gva = 0;
7830
7831         if (!nested_vmx_check_permission(vcpu))
7832                 return 1;
7833
7834         if (!nested_vmx_check_vmcs12(vcpu))
7835                 return kvm_skip_emulated_instruction(vcpu);
7836
7837         /* Decode instruction info and find the field to read */
7838         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
7839         /* Read the field, zero-extended to a u64 field_value */
7840         if (vmcs12_read_any(vcpu, field, &field_value) < 0) {
7841                 nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
7842                 return kvm_skip_emulated_instruction(vcpu);
7843         }
7844         /*
7845          * Now copy part of this value to register or memory, as requested.
7846          * Note that the number of bits actually copied is 32 or 64 depending
7847          * on the guest's mode (32 or 64 bit), not on the given field's length.
7848          */
7849         if (vmx_instruction_info & (1u << 10)) {
7850                 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
7851                         field_value);
7852         } else {
7853                 if (get_vmx_mem_address(vcpu, exit_qualification,
7854                                 vmx_instruction_info, true, &gva))
7855                         return 1;
7856                 /* _system ok, nested_vmx_check_permission has verified cpl=0 */
7857                 kvm_write_guest_virt_system(vcpu, gva, &field_value,
7858                                             (is_long_mode(vcpu) ? 8 : 4), NULL);
7859         }
7860
7861         nested_vmx_succeed(vcpu);
7862         return kvm_skip_emulated_instruction(vcpu);
7863 }
7864
7865
7866 static int handle_vmwrite(struct kvm_vcpu *vcpu)
7867 {
7868         unsigned long field;
7869         gva_t gva;
7870         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7871         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7872         /* The value to write might be 32 or 64 bits, depending on L1's long
7873          * mode, and eventually we need to write that into a field of several
7874          * possible lengths. The code below first zero-extends the value to 64
7875          * bit (field_value), and then copies only the appropriate number of
7876          * bits into the vmcs12 field.
7877          */
7878         u64 field_value = 0;
7879         struct x86_exception e;
7880
7881         if (!nested_vmx_check_permission(vcpu))
7882                 return 1;
7883
7884         if (!nested_vmx_check_vmcs12(vcpu))
7885                 return kvm_skip_emulated_instruction(vcpu);
7886
7887         if (vmx_instruction_info & (1u << 10))
7888                 field_value = kvm_register_readl(vcpu,
7889                         (((vmx_instruction_info) >> 3) & 0xf));
7890         else {
7891                 if (get_vmx_mem_address(vcpu, exit_qualification,
7892                                 vmx_instruction_info, false, &gva))
7893                         return 1;
7894                 if (kvm_read_guest_virt(vcpu, gva, &field_value,
7895                                         (is_64_bit_mode(vcpu) ? 8 : 4), &e)) {
7896                         kvm_inject_page_fault(vcpu, &e);
7897                         return 1;
7898                 }
7899         }
7900
7901
7902         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
7903         if (vmcs_field_readonly(field)) {
7904                 nested_vmx_failValid(vcpu,
7905                         VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
7906                 return kvm_skip_emulated_instruction(vcpu);
7907         }
7908
7909         if (vmcs12_write_any(vcpu, field, field_value) < 0) {
7910                 nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
7911                 return kvm_skip_emulated_instruction(vcpu);
7912         }
7913
7914         nested_vmx_succeed(vcpu);
7915         return kvm_skip_emulated_instruction(vcpu);
7916 }
7917
7918 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
7919 {
7920         vmx->nested.current_vmptr = vmptr;
7921         if (enable_shadow_vmcs) {
7922                 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
7923                               SECONDARY_EXEC_SHADOW_VMCS);
7924                 vmcs_write64(VMCS_LINK_POINTER,
7925                              __pa(vmx->vmcs01.shadow_vmcs));
7926                 vmx->nested.sync_shadow_vmcs = true;
7927         }
7928 }
7929
7930 /* Emulate the VMPTRLD instruction */
7931 static int handle_vmptrld(struct kvm_vcpu *vcpu)
7932 {
7933         struct vcpu_vmx *vmx = to_vmx(vcpu);
7934         gpa_t vmptr;
7935
7936         if (!nested_vmx_check_permission(vcpu))
7937                 return 1;
7938
7939         if (nested_vmx_get_vmptr(vcpu, &vmptr))
7940                 return 1;
7941
7942         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
7943                 nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS);
7944                 return kvm_skip_emulated_instruction(vcpu);
7945         }
7946
7947         if (vmptr == vmx->nested.vmxon_ptr) {
7948                 nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_VMXON_POINTER);
7949                 return kvm_skip_emulated_instruction(vcpu);
7950         }
7951
7952         if (vmx->nested.current_vmptr != vmptr) {
7953                 struct vmcs12 *new_vmcs12;
7954                 struct page *page;
7955                 page = kvm_vcpu_gpa_to_page(vcpu, vmptr);
7956                 if (is_error_page(page)) {
7957                         nested_vmx_failInvalid(vcpu);
7958                         return kvm_skip_emulated_instruction(vcpu);
7959                 }
7960                 new_vmcs12 = kmap(page);
7961                 if (new_vmcs12->revision_id != VMCS12_REVISION) {
7962                         kunmap(page);
7963                         kvm_release_page_clean(page);
7964                         nested_vmx_failValid(vcpu,
7965                                 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
7966                         return kvm_skip_emulated_instruction(vcpu);
7967                 }
7968
7969                 nested_release_vmcs12(vmx);
7970                 /*
7971                  * Load VMCS12 from guest memory since it is not already
7972                  * cached.
7973                  */
7974                 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE);
7975                 kunmap(page);
7976                 kvm_release_page_clean(page);
7977
7978                 set_current_vmptr(vmx, vmptr);
7979         }
7980
7981         nested_vmx_succeed(vcpu);
7982         return kvm_skip_emulated_instruction(vcpu);
7983 }
7984
7985 /* Emulate the VMPTRST instruction */
7986 static int handle_vmptrst(struct kvm_vcpu *vcpu)
7987 {
7988         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7989         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7990         gva_t vmcs_gva;
7991         struct x86_exception e;
7992
7993         if (!nested_vmx_check_permission(vcpu))
7994                 return 1;
7995
7996         if (get_vmx_mem_address(vcpu, exit_qualification,
7997                         vmx_instruction_info, true, &vmcs_gva))
7998                 return 1;
7999         /* *_system ok, nested_vmx_check_permission has verified cpl=0 */
8000         if (kvm_write_guest_virt_system(vcpu, vmcs_gva,
8001                                         (void *)&to_vmx(vcpu)->nested.current_vmptr,
8002                                         sizeof(u64), &e)) {
8003                 kvm_inject_page_fault(vcpu, &e);
8004                 return 1;
8005         }
8006         nested_vmx_succeed(vcpu);
8007         return kvm_skip_emulated_instruction(vcpu);
8008 }
8009
8010 /* Emulate the INVEPT instruction */
8011 static int handle_invept(struct kvm_vcpu *vcpu)
8012 {
8013         struct vcpu_vmx *vmx = to_vmx(vcpu);
8014         u32 vmx_instruction_info, types;
8015         unsigned long type;
8016         gva_t gva;
8017         struct x86_exception e;
8018         struct {
8019                 u64 eptp, gpa;
8020         } operand;
8021
8022         if (!(vmx->nested.nested_vmx_secondary_ctls_high &
8023               SECONDARY_EXEC_ENABLE_EPT) ||
8024             !(vmx->nested.nested_vmx_ept_caps & VMX_EPT_INVEPT_BIT)) {
8025                 kvm_queue_exception(vcpu, UD_VECTOR);
8026                 return 1;
8027         }
8028
8029         if (!nested_vmx_check_permission(vcpu))
8030                 return 1;
8031
8032         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
8033         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
8034
8035         types = (vmx->nested.nested_vmx_ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
8036
8037         if (type >= 32 || !(types & (1 << type))) {
8038                 nested_vmx_failValid(vcpu,
8039                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
8040                 return kvm_skip_emulated_instruction(vcpu);
8041         }
8042
8043         /* According to the Intel VMX instruction reference, the memory
8044          * operand is read even if it isn't needed (e.g., for type==global)
8045          */
8046         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
8047                         vmx_instruction_info, false, &gva))
8048                 return 1;
8049         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
8050                 kvm_inject_page_fault(vcpu, &e);
8051                 return 1;
8052         }
8053
8054         switch (type) {
8055         case VMX_EPT_EXTENT_GLOBAL:
8056         /*
8057          * TODO: track mappings and invalidate
8058          * single context requests appropriately
8059          */
8060         case VMX_EPT_EXTENT_CONTEXT:
8061                 kvm_mmu_sync_roots(vcpu);
8062                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
8063                 nested_vmx_succeed(vcpu);
8064                 break;
8065         default:
8066                 BUG_ON(1);
8067                 break;
8068         }
8069
8070         return kvm_skip_emulated_instruction(vcpu);
8071 }
8072
8073 static int handle_invvpid(struct kvm_vcpu *vcpu)
8074 {
8075         struct vcpu_vmx *vmx = to_vmx(vcpu);
8076         u32 vmx_instruction_info;
8077         unsigned long type, types;
8078         gva_t gva;
8079         struct x86_exception e;
8080         struct {
8081                 u64 vpid;
8082                 u64 gla;
8083         } operand;
8084
8085         if (!(vmx->nested.nested_vmx_secondary_ctls_high &
8086               SECONDARY_EXEC_ENABLE_VPID) ||
8087                         !(vmx->nested.nested_vmx_vpid_caps & VMX_VPID_INVVPID_BIT)) {
8088                 kvm_queue_exception(vcpu, UD_VECTOR);
8089                 return 1;
8090         }
8091
8092         if (!nested_vmx_check_permission(vcpu))
8093                 return 1;
8094
8095         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
8096         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
8097
8098         types = (vmx->nested.nested_vmx_vpid_caps &
8099                         VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
8100
8101         if (type >= 32 || !(types & (1 << type))) {
8102                 nested_vmx_failValid(vcpu,
8103                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
8104                 return kvm_skip_emulated_instruction(vcpu);
8105         }
8106
8107         /* according to the intel vmx instruction reference, the memory
8108          * operand is read even if it isn't needed (e.g., for type==global)
8109          */
8110         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
8111                         vmx_instruction_info, false, &gva))
8112                 return 1;
8113         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
8114                 kvm_inject_page_fault(vcpu, &e);
8115                 return 1;
8116         }
8117         if (operand.vpid >> 16) {
8118                 nested_vmx_failValid(vcpu,
8119                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
8120                 return kvm_skip_emulated_instruction(vcpu);
8121         }
8122
8123         switch (type) {
8124         case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
8125                 if (is_noncanonical_address(operand.gla, vcpu)) {
8126                         nested_vmx_failValid(vcpu,
8127                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
8128                         return kvm_skip_emulated_instruction(vcpu);
8129                 }
8130                 /* fall through */
8131         case VMX_VPID_EXTENT_SINGLE_CONTEXT:
8132         case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
8133                 if (!operand.vpid) {
8134                         nested_vmx_failValid(vcpu,
8135                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
8136                         return kvm_skip_emulated_instruction(vcpu);
8137                 }
8138                 break;
8139         case VMX_VPID_EXTENT_ALL_CONTEXT:
8140                 break;
8141         default:
8142                 WARN_ON_ONCE(1);
8143                 return kvm_skip_emulated_instruction(vcpu);
8144         }
8145
8146         __vmx_flush_tlb(vcpu, vmx->nested.vpid02);
8147         nested_vmx_succeed(vcpu);
8148
8149         return kvm_skip_emulated_instruction(vcpu);
8150 }
8151
8152 static int handle_pml_full(struct kvm_vcpu *vcpu)
8153 {
8154         unsigned long exit_qualification;
8155
8156         trace_kvm_pml_full(vcpu->vcpu_id);
8157
8158         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
8159
8160         /*
8161          * PML buffer FULL happened while executing iret from NMI,
8162          * "blocked by NMI" bit has to be set before next VM entry.
8163          */
8164         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
8165                         cpu_has_virtual_nmis() &&
8166                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
8167                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
8168                                 GUEST_INTR_STATE_NMI);
8169
8170         /*
8171          * PML buffer already flushed at beginning of VMEXIT. Nothing to do
8172          * here.., and there's no userspace involvement needed for PML.
8173          */
8174         return 1;
8175 }
8176
8177 static int handle_preemption_timer(struct kvm_vcpu *vcpu)
8178 {
8179         kvm_lapic_expired_hv_timer(vcpu);
8180         return 1;
8181 }
8182
8183 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address)
8184 {
8185         struct vcpu_vmx *vmx = to_vmx(vcpu);
8186         int maxphyaddr = cpuid_maxphyaddr(vcpu);
8187
8188         /* Check for memory type validity */
8189         switch (address & VMX_EPTP_MT_MASK) {
8190         case VMX_EPTP_MT_UC:
8191                 if (!(vmx->nested.nested_vmx_ept_caps & VMX_EPTP_UC_BIT))
8192                         return false;
8193                 break;
8194         case VMX_EPTP_MT_WB:
8195                 if (!(vmx->nested.nested_vmx_ept_caps & VMX_EPTP_WB_BIT))
8196                         return false;
8197                 break;
8198         default:
8199                 return false;
8200         }
8201
8202         /* only 4 levels page-walk length are valid */
8203         if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4)
8204                 return false;
8205
8206         /* Reserved bits should not be set */
8207         if (address >> maxphyaddr || ((address >> 7) & 0x1f))
8208                 return false;
8209
8210         /* AD, if set, should be supported */
8211         if (address & VMX_EPTP_AD_ENABLE_BIT) {
8212                 if (!(vmx->nested.nested_vmx_ept_caps & VMX_EPT_AD_BIT))
8213                         return false;
8214         }
8215
8216         return true;
8217 }
8218
8219 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
8220                                      struct vmcs12 *vmcs12)
8221 {
8222         u32 index = vcpu->arch.regs[VCPU_REGS_RCX];
8223         u64 address;
8224         bool accessed_dirty;
8225         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
8226
8227         if (!nested_cpu_has_eptp_switching(vmcs12) ||
8228             !nested_cpu_has_ept(vmcs12))
8229                 return 1;
8230
8231         if (index >= VMFUNC_EPTP_ENTRIES)
8232                 return 1;
8233
8234
8235         if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT,
8236                                      &address, index * 8, 8))
8237                 return 1;
8238
8239         accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT);
8240
8241         /*
8242          * If the (L2) guest does a vmfunc to the currently
8243          * active ept pointer, we don't have to do anything else
8244          */
8245         if (vmcs12->ept_pointer != address) {
8246                 if (!valid_ept_address(vcpu, address))
8247                         return 1;
8248
8249                 kvm_mmu_unload(vcpu);
8250                 mmu->ept_ad = accessed_dirty;
8251                 mmu->base_role.ad_disabled = !accessed_dirty;
8252                 vmcs12->ept_pointer = address;
8253                 /*
8254                  * TODO: Check what's the correct approach in case
8255                  * mmu reload fails. Currently, we just let the next
8256                  * reload potentially fail
8257                  */
8258                 kvm_mmu_reload(vcpu);
8259         }
8260
8261         return 0;
8262 }
8263
8264 static int handle_vmfunc(struct kvm_vcpu *vcpu)
8265 {
8266         struct vcpu_vmx *vmx = to_vmx(vcpu);
8267         struct vmcs12 *vmcs12;
8268         u32 function = vcpu->arch.regs[VCPU_REGS_RAX];
8269
8270         /*
8271          * VMFUNC is only supported for nested guests, but we always enable the
8272          * secondary control for simplicity; for non-nested mode, fake that we
8273          * didn't by injecting #UD.
8274          */
8275         if (!is_guest_mode(vcpu)) {
8276                 kvm_queue_exception(vcpu, UD_VECTOR);
8277                 return 1;
8278         }
8279
8280         vmcs12 = get_vmcs12(vcpu);
8281         if ((vmcs12->vm_function_control & (1 << function)) == 0)
8282                 goto fail;
8283
8284         switch (function) {
8285         case 0:
8286                 if (nested_vmx_eptp_switching(vcpu, vmcs12))
8287                         goto fail;
8288                 break;
8289         default:
8290                 goto fail;
8291         }
8292         return kvm_skip_emulated_instruction(vcpu);
8293
8294 fail:
8295         nested_vmx_vmexit(vcpu, vmx->exit_reason,
8296                           vmcs_read32(VM_EXIT_INTR_INFO),
8297                           vmcs_readl(EXIT_QUALIFICATION));
8298         return 1;
8299 }
8300
8301 /*
8302  * The exit handlers return 1 if the exit was handled fully and guest execution
8303  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
8304  * to be done to userspace and return 0.
8305  */
8306 static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
8307         [EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
8308         [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
8309         [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
8310         [EXIT_REASON_NMI_WINDOW]              = handle_nmi_window,
8311         [EXIT_REASON_IO_INSTRUCTION]          = handle_io,
8312         [EXIT_REASON_CR_ACCESS]               = handle_cr,
8313         [EXIT_REASON_DR_ACCESS]               = handle_dr,
8314         [EXIT_REASON_CPUID]                   = handle_cpuid,
8315         [EXIT_REASON_MSR_READ]                = handle_rdmsr,
8316         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
8317         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
8318         [EXIT_REASON_HLT]                     = handle_halt,
8319         [EXIT_REASON_INVD]                    = handle_invd,
8320         [EXIT_REASON_INVLPG]                  = handle_invlpg,
8321         [EXIT_REASON_RDPMC]                   = handle_rdpmc,
8322         [EXIT_REASON_VMCALL]                  = handle_vmcall,
8323         [EXIT_REASON_VMCLEAR]                 = handle_vmclear,
8324         [EXIT_REASON_VMLAUNCH]                = handle_vmlaunch,
8325         [EXIT_REASON_VMPTRLD]                 = handle_vmptrld,
8326         [EXIT_REASON_VMPTRST]                 = handle_vmptrst,
8327         [EXIT_REASON_VMREAD]                  = handle_vmread,
8328         [EXIT_REASON_VMRESUME]                = handle_vmresume,
8329         [EXIT_REASON_VMWRITE]                 = handle_vmwrite,
8330         [EXIT_REASON_VMOFF]                   = handle_vmoff,
8331         [EXIT_REASON_VMON]                    = handle_vmon,
8332         [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
8333         [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
8334         [EXIT_REASON_APIC_WRITE]              = handle_apic_write,
8335         [EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,
8336         [EXIT_REASON_WBINVD]                  = handle_wbinvd,
8337         [EXIT_REASON_XSETBV]                  = handle_xsetbv,
8338         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
8339         [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
8340         [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
8341         [EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
8342         [EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
8343         [EXIT_REASON_MWAIT_INSTRUCTION]       = handle_mwait,
8344         [EXIT_REASON_MONITOR_TRAP_FLAG]       = handle_monitor_trap,
8345         [EXIT_REASON_MONITOR_INSTRUCTION]     = handle_monitor,
8346         [EXIT_REASON_INVEPT]                  = handle_invept,
8347         [EXIT_REASON_INVVPID]                 = handle_invvpid,
8348         [EXIT_REASON_RDRAND]                  = handle_invalid_op,
8349         [EXIT_REASON_RDSEED]                  = handle_invalid_op,
8350         [EXIT_REASON_XSAVES]                  = handle_xsaves,
8351         [EXIT_REASON_XRSTORS]                 = handle_xrstors,
8352         [EXIT_REASON_PML_FULL]                = handle_pml_full,
8353         [EXIT_REASON_VMFUNC]                  = handle_vmfunc,
8354         [EXIT_REASON_PREEMPTION_TIMER]        = handle_preemption_timer,
8355 };
8356
8357 static const int kvm_vmx_max_exit_handlers =
8358         ARRAY_SIZE(kvm_vmx_exit_handlers);
8359
8360 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
8361                                        struct vmcs12 *vmcs12)
8362 {
8363         unsigned long exit_qualification;
8364         gpa_t bitmap, last_bitmap;
8365         unsigned int port;
8366         int size;
8367         u8 b;
8368
8369         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
8370                 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
8371
8372         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
8373
8374         port = exit_qualification >> 16;
8375         size = (exit_qualification & 7) + 1;
8376
8377         last_bitmap = (gpa_t)-1;
8378         b = -1;
8379
8380         while (size > 0) {
8381                 if (port < 0x8000)
8382                         bitmap = vmcs12->io_bitmap_a;
8383                 else if (port < 0x10000)
8384                         bitmap = vmcs12->io_bitmap_b;
8385                 else
8386                         return true;
8387                 bitmap += (port & 0x7fff) / 8;
8388
8389                 if (last_bitmap != bitmap)
8390                         if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
8391                                 return true;
8392                 if (b & (1 << (port & 7)))
8393                         return true;
8394
8395                 port++;
8396                 size--;
8397                 last_bitmap = bitmap;
8398         }
8399
8400         return false;
8401 }
8402
8403 /*
8404  * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
8405  * rather than handle it ourselves in L0. I.e., check whether L1 expressed
8406  * disinterest in the current event (read or write a specific MSR) by using an
8407  * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
8408  */
8409 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
8410         struct vmcs12 *vmcs12, u32 exit_reason)
8411 {
8412         u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX];
8413         gpa_t bitmap;
8414
8415         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
8416                 return true;
8417
8418         /*
8419          * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
8420          * for the four combinations of read/write and low/high MSR numbers.
8421          * First we need to figure out which of the four to use:
8422          */
8423         bitmap = vmcs12->msr_bitmap;
8424         if (exit_reason == EXIT_REASON_MSR_WRITE)
8425                 bitmap += 2048;
8426         if (msr_index >= 0xc0000000) {
8427                 msr_index -= 0xc0000000;
8428                 bitmap += 1024;
8429         }
8430
8431         /* Then read the msr_index'th bit from this bitmap: */
8432         if (msr_index < 1024*8) {
8433                 unsigned char b;
8434                 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
8435                         return true;
8436                 return 1 & (b >> (msr_index & 7));
8437         } else
8438                 return true; /* let L1 handle the wrong parameter */
8439 }
8440
8441 /*
8442  * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
8443  * rather than handle it ourselves in L0. I.e., check if L1 wanted to
8444  * intercept (via guest_host_mask etc.) the current event.
8445  */
8446 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
8447         struct vmcs12 *vmcs12)
8448 {
8449         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
8450         int cr = exit_qualification & 15;
8451         int reg;
8452         unsigned long val;
8453
8454         switch ((exit_qualification >> 4) & 3) {
8455         case 0: /* mov to cr */
8456                 reg = (exit_qualification >> 8) & 15;
8457                 val = kvm_register_readl(vcpu, reg);
8458                 switch (cr) {
8459                 case 0:
8460                         if (vmcs12->cr0_guest_host_mask &
8461                             (val ^ vmcs12->cr0_read_shadow))
8462                                 return true;
8463                         break;
8464                 case 3:
8465                         if ((vmcs12->cr3_target_count >= 1 &&
8466                                         vmcs12->cr3_target_value0 == val) ||
8467                                 (vmcs12->cr3_target_count >= 2 &&
8468                                         vmcs12->cr3_target_value1 == val) ||
8469                                 (vmcs12->cr3_target_count >= 3 &&
8470                                         vmcs12->cr3_target_value2 == val) ||
8471                                 (vmcs12->cr3_target_count >= 4 &&
8472                                         vmcs12->cr3_target_value3 == val))
8473                                 return false;
8474                         if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
8475                                 return true;
8476                         break;
8477                 case 4:
8478                         if (vmcs12->cr4_guest_host_mask &
8479                             (vmcs12->cr4_read_shadow ^ val))
8480                                 return true;
8481                         break;
8482                 case 8:
8483                         if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
8484                                 return true;
8485                         break;
8486                 }
8487                 break;
8488         case 2: /* clts */
8489                 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
8490                     (vmcs12->cr0_read_shadow & X86_CR0_TS))
8491                         return true;
8492                 break;
8493         case 1: /* mov from cr */
8494                 switch (cr) {
8495                 case 3:
8496                         if (vmcs12->cpu_based_vm_exec_control &
8497                             CPU_BASED_CR3_STORE_EXITING)
8498                                 return true;
8499                         break;
8500                 case 8:
8501                         if (vmcs12->cpu_based_vm_exec_control &
8502                             CPU_BASED_CR8_STORE_EXITING)
8503                                 return true;
8504                         break;
8505                 }
8506                 break;
8507         case 3: /* lmsw */
8508                 /*
8509                  * lmsw can change bits 1..3 of cr0, and only set bit 0 of
8510                  * cr0. Other attempted changes are ignored, with no exit.
8511                  */
8512                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
8513                 if (vmcs12->cr0_guest_host_mask & 0xe &
8514                     (val ^ vmcs12->cr0_read_shadow))
8515                         return true;
8516                 if ((vmcs12->cr0_guest_host_mask & 0x1) &&
8517                     !(vmcs12->cr0_read_shadow & 0x1) &&
8518                     (val & 0x1))
8519                         return true;
8520                 break;
8521         }
8522         return false;
8523 }
8524
8525 /*
8526  * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
8527  * should handle it ourselves in L0 (and then continue L2). Only call this
8528  * when in is_guest_mode (L2).
8529  */
8530 static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
8531 {
8532         u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
8533         struct vcpu_vmx *vmx = to_vmx(vcpu);
8534         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
8535
8536         if (vmx->nested.nested_run_pending)
8537                 return false;
8538
8539         if (unlikely(vmx->fail)) {
8540                 pr_info_ratelimited("%s failed vm entry %x\n", __func__,
8541                                     vmcs_read32(VM_INSTRUCTION_ERROR));
8542                 return true;
8543         }
8544
8545         /*
8546          * The host physical addresses of some pages of guest memory
8547          * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
8548          * Page). The CPU may write to these pages via their host
8549          * physical address while L2 is running, bypassing any
8550          * address-translation-based dirty tracking (e.g. EPT write
8551          * protection).
8552          *
8553          * Mark them dirty on every exit from L2 to prevent them from
8554          * getting out of sync with dirty tracking.
8555          */
8556         nested_mark_vmcs12_pages_dirty(vcpu);
8557
8558         trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
8559                                 vmcs_readl(EXIT_QUALIFICATION),
8560                                 vmx->idt_vectoring_info,
8561                                 intr_info,
8562                                 vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
8563                                 KVM_ISA_VMX);
8564
8565         switch (exit_reason) {
8566         case EXIT_REASON_EXCEPTION_NMI:
8567                 if (is_nmi(intr_info))
8568                         return false;
8569                 else if (is_page_fault(intr_info))
8570                         return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept;
8571                 else if (is_no_device(intr_info) &&
8572                          !(vmcs12->guest_cr0 & X86_CR0_TS))
8573                         return false;
8574                 else if (is_debug(intr_info) &&
8575                          vcpu->guest_debug &
8576                          (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
8577                         return false;
8578                 else if (is_breakpoint(intr_info) &&
8579                          vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
8580                         return false;
8581                 return vmcs12->exception_bitmap &
8582                                 (1u << (intr_info & INTR_INFO_VECTOR_MASK));
8583         case EXIT_REASON_EXTERNAL_INTERRUPT:
8584                 return false;
8585         case EXIT_REASON_TRIPLE_FAULT:
8586                 return true;
8587         case EXIT_REASON_PENDING_INTERRUPT:
8588                 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
8589         case EXIT_REASON_NMI_WINDOW:
8590                 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
8591         case EXIT_REASON_TASK_SWITCH:
8592                 return true;
8593         case EXIT_REASON_CPUID:
8594                 return true;
8595         case EXIT_REASON_HLT:
8596                 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
8597         case EXIT_REASON_INVD:
8598                 return true;
8599         case EXIT_REASON_INVLPG:
8600                 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
8601         case EXIT_REASON_RDPMC:
8602                 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
8603         case EXIT_REASON_RDRAND:
8604                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND);
8605         case EXIT_REASON_RDSEED:
8606                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED);
8607         case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
8608                 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
8609         case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
8610         case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
8611         case EXIT_REASON_VMPTRST: case EXIT_REASON_VMREAD:
8612         case EXIT_REASON_VMRESUME: case EXIT_REASON_VMWRITE:
8613         case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
8614         case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
8615                 /*
8616                  * VMX instructions trap unconditionally. This allows L1 to
8617                  * emulate them for its L2 guest, i.e., allows 3-level nesting!
8618                  */
8619                 return true;
8620         case EXIT_REASON_CR_ACCESS:
8621                 return nested_vmx_exit_handled_cr(vcpu, vmcs12);
8622         case EXIT_REASON_DR_ACCESS:
8623                 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
8624         case EXIT_REASON_IO_INSTRUCTION:
8625                 return nested_vmx_exit_handled_io(vcpu, vmcs12);
8626         case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
8627                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
8628         case EXIT_REASON_MSR_READ:
8629         case EXIT_REASON_MSR_WRITE:
8630                 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
8631         case EXIT_REASON_INVALID_STATE:
8632                 return true;
8633         case EXIT_REASON_MWAIT_INSTRUCTION:
8634                 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
8635         case EXIT_REASON_MONITOR_TRAP_FLAG:
8636                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG);
8637         case EXIT_REASON_MONITOR_INSTRUCTION:
8638                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
8639         case EXIT_REASON_PAUSE_INSTRUCTION:
8640                 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
8641                         nested_cpu_has2(vmcs12,
8642                                 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
8643         case EXIT_REASON_MCE_DURING_VMENTRY:
8644                 return false;
8645         case EXIT_REASON_TPR_BELOW_THRESHOLD:
8646                 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
8647         case EXIT_REASON_APIC_ACCESS:
8648                 return nested_cpu_has2(vmcs12,
8649                         SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
8650         case EXIT_REASON_APIC_WRITE:
8651         case EXIT_REASON_EOI_INDUCED:
8652                 /* apic_write and eoi_induced should exit unconditionally. */
8653                 return true;
8654         case EXIT_REASON_EPT_VIOLATION:
8655                 /*
8656                  * L0 always deals with the EPT violation. If nested EPT is
8657                  * used, and the nested mmu code discovers that the address is
8658                  * missing in the guest EPT table (EPT12), the EPT violation
8659                  * will be injected with nested_ept_inject_page_fault()
8660                  */
8661                 return false;
8662         case EXIT_REASON_EPT_MISCONFIG:
8663                 /*
8664                  * L2 never uses directly L1's EPT, but rather L0's own EPT
8665                  * table (shadow on EPT) or a merged EPT table that L0 built
8666                  * (EPT on EPT). So any problems with the structure of the
8667                  * table is L0's fault.
8668                  */
8669                 return false;
8670         case EXIT_REASON_INVPCID:
8671                 return
8672                         nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) &&
8673                         nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
8674         case EXIT_REASON_WBINVD:
8675                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
8676         case EXIT_REASON_XSETBV:
8677                 return true;
8678         case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
8679                 /*
8680                  * This should never happen, since it is not possible to
8681                  * set XSS to a non-zero value---neither in L1 nor in L2.
8682                  * If if it were, XSS would have to be checked against
8683                  * the XSS exit bitmap in vmcs12.
8684                  */
8685                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
8686         case EXIT_REASON_PREEMPTION_TIMER:
8687                 return false;
8688         case EXIT_REASON_PML_FULL:
8689                 /* We emulate PML support to L1. */
8690                 return false;
8691         case EXIT_REASON_VMFUNC:
8692                 /* VM functions are emulated through L2->L0 vmexits. */
8693                 return false;
8694         default:
8695                 return true;
8696         }
8697 }
8698
8699 static int nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason)
8700 {
8701         u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
8702
8703         /*
8704          * At this point, the exit interruption info in exit_intr_info
8705          * is only valid for EXCEPTION_NMI exits.  For EXTERNAL_INTERRUPT
8706          * we need to query the in-kernel LAPIC.
8707          */
8708         WARN_ON(exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT);
8709         if ((exit_intr_info &
8710              (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) ==
8711             (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) {
8712                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
8713                 vmcs12->vm_exit_intr_error_code =
8714                         vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
8715         }
8716
8717         nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info,
8718                           vmcs_readl(EXIT_QUALIFICATION));
8719         return 1;
8720 }
8721
8722 static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
8723 {
8724         *info1 = vmcs_readl(EXIT_QUALIFICATION);
8725         *info2 = vmcs_read32(VM_EXIT_INTR_INFO);
8726 }
8727
8728 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
8729 {
8730         if (vmx->pml_pg) {
8731                 __free_page(vmx->pml_pg);
8732                 vmx->pml_pg = NULL;
8733         }
8734 }
8735
8736 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
8737 {
8738         struct vcpu_vmx *vmx = to_vmx(vcpu);
8739         u64 *pml_buf;
8740         u16 pml_idx;
8741
8742         pml_idx = vmcs_read16(GUEST_PML_INDEX);
8743
8744         /* Do nothing if PML buffer is empty */
8745         if (pml_idx == (PML_ENTITY_NUM - 1))
8746                 return;
8747
8748         /* PML index always points to next available PML buffer entity */
8749         if (pml_idx >= PML_ENTITY_NUM)
8750                 pml_idx = 0;
8751         else
8752                 pml_idx++;
8753
8754         pml_buf = page_address(vmx->pml_pg);
8755         for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
8756                 u64 gpa;
8757
8758                 gpa = pml_buf[pml_idx];
8759                 WARN_ON(gpa & (PAGE_SIZE - 1));
8760                 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
8761         }
8762
8763         /* reset PML index */
8764         vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
8765 }
8766
8767 /*
8768  * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
8769  * Called before reporting dirty_bitmap to userspace.
8770  */
8771 static void kvm_flush_pml_buffers(struct kvm *kvm)
8772 {
8773         int i;
8774         struct kvm_vcpu *vcpu;
8775         /*
8776          * We only need to kick vcpu out of guest mode here, as PML buffer
8777          * is flushed at beginning of all VMEXITs, and it's obvious that only
8778          * vcpus running in guest are possible to have unflushed GPAs in PML
8779          * buffer.
8780          */
8781         kvm_for_each_vcpu(i, vcpu, kvm)
8782                 kvm_vcpu_kick(vcpu);
8783 }
8784
8785 static void vmx_dump_sel(char *name, uint32_t sel)
8786 {
8787         pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
8788                name, vmcs_read16(sel),
8789                vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
8790                vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
8791                vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
8792 }
8793
8794 static void vmx_dump_dtsel(char *name, uint32_t limit)
8795 {
8796         pr_err("%s                           limit=0x%08x, base=0x%016lx\n",
8797                name, vmcs_read32(limit),
8798                vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
8799 }
8800
8801 static void dump_vmcs(void)
8802 {
8803         u32 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
8804         u32 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
8805         u32 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
8806         u32 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
8807         u32 secondary_exec_control = 0;
8808         unsigned long cr4 = vmcs_readl(GUEST_CR4);
8809         u64 efer = vmcs_read64(GUEST_IA32_EFER);
8810         int i, n;
8811
8812         if (cpu_has_secondary_exec_ctrls())
8813                 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
8814
8815         pr_err("*** Guest State ***\n");
8816         pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
8817                vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
8818                vmcs_readl(CR0_GUEST_HOST_MASK));
8819         pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
8820                cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
8821         pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
8822         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
8823             (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA))
8824         {
8825                 pr_err("PDPTR0 = 0x%016llx  PDPTR1 = 0x%016llx\n",
8826                        vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
8827                 pr_err("PDPTR2 = 0x%016llx  PDPTR3 = 0x%016llx\n",
8828                        vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
8829         }
8830         pr_err("RSP = 0x%016lx  RIP = 0x%016lx\n",
8831                vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
8832         pr_err("RFLAGS=0x%08lx         DR7 = 0x%016lx\n",
8833                vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
8834         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
8835                vmcs_readl(GUEST_SYSENTER_ESP),
8836                vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
8837         vmx_dump_sel("CS:  ", GUEST_CS_SELECTOR);
8838         vmx_dump_sel("DS:  ", GUEST_DS_SELECTOR);
8839         vmx_dump_sel("SS:  ", GUEST_SS_SELECTOR);
8840         vmx_dump_sel("ES:  ", GUEST_ES_SELECTOR);
8841         vmx_dump_sel("FS:  ", GUEST_FS_SELECTOR);
8842         vmx_dump_sel("GS:  ", GUEST_GS_SELECTOR);
8843         vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
8844         vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
8845         vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
8846         vmx_dump_sel("TR:  ", GUEST_TR_SELECTOR);
8847         if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
8848             (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
8849                 pr_err("EFER =     0x%016llx  PAT = 0x%016llx\n",
8850                        efer, vmcs_read64(GUEST_IA32_PAT));
8851         pr_err("DebugCtl = 0x%016llx  DebugExceptions = 0x%016lx\n",
8852                vmcs_read64(GUEST_IA32_DEBUGCTL),
8853                vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
8854         if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
8855                 pr_err("PerfGlobCtl = 0x%016llx\n",
8856                        vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
8857         if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
8858                 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
8859         pr_err("Interruptibility = %08x  ActivityState = %08x\n",
8860                vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
8861                vmcs_read32(GUEST_ACTIVITY_STATE));
8862         if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
8863                 pr_err("InterruptStatus = %04x\n",
8864                        vmcs_read16(GUEST_INTR_STATUS));
8865
8866         pr_err("*** Host State ***\n");
8867         pr_err("RIP = 0x%016lx  RSP = 0x%016lx\n",
8868                vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
8869         pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
8870                vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
8871                vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
8872                vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
8873                vmcs_read16(HOST_TR_SELECTOR));
8874         pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
8875                vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
8876                vmcs_readl(HOST_TR_BASE));
8877         pr_err("GDTBase=%016lx IDTBase=%016lx\n",
8878                vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
8879         pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
8880                vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
8881                vmcs_readl(HOST_CR4));
8882         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
8883                vmcs_readl(HOST_IA32_SYSENTER_ESP),
8884                vmcs_read32(HOST_IA32_SYSENTER_CS),
8885                vmcs_readl(HOST_IA32_SYSENTER_EIP));
8886         if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
8887                 pr_err("EFER = 0x%016llx  PAT = 0x%016llx\n",
8888                        vmcs_read64(HOST_IA32_EFER),
8889                        vmcs_read64(HOST_IA32_PAT));
8890         if (vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
8891                 pr_err("PerfGlobCtl = 0x%016llx\n",
8892                        vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
8893
8894         pr_err("*** Control State ***\n");
8895         pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
8896                pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
8897         pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
8898         pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
8899                vmcs_read32(EXCEPTION_BITMAP),
8900                vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
8901                vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
8902         pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
8903                vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
8904                vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
8905                vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
8906         pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
8907                vmcs_read32(VM_EXIT_INTR_INFO),
8908                vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
8909                vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
8910         pr_err("        reason=%08x qualification=%016lx\n",
8911                vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
8912         pr_err("IDTVectoring: info=%08x errcode=%08x\n",
8913                vmcs_read32(IDT_VECTORING_INFO_FIELD),
8914                vmcs_read32(IDT_VECTORING_ERROR_CODE));
8915         pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
8916         if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
8917                 pr_err("TSC Multiplier = 0x%016llx\n",
8918                        vmcs_read64(TSC_MULTIPLIER));
8919         if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW)
8920                 pr_err("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
8921         if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
8922                 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
8923         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
8924                 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
8925         n = vmcs_read32(CR3_TARGET_COUNT);
8926         for (i = 0; i + 1 < n; i += 4)
8927                 pr_err("CR3 target%u=%016lx target%u=%016lx\n",
8928                        i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2),
8929                        i + 1, vmcs_readl(CR3_TARGET_VALUE0 + i * 2 + 2));
8930         if (i < n)
8931                 pr_err("CR3 target%u=%016lx\n",
8932                        i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2));
8933         if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
8934                 pr_err("PLE Gap=%08x Window=%08x\n",
8935                        vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
8936         if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
8937                 pr_err("Virtual processor ID = 0x%04x\n",
8938                        vmcs_read16(VIRTUAL_PROCESSOR_ID));
8939 }
8940
8941 /*
8942  * The guest has exited.  See if we can fix it or if we need userspace
8943  * assistance.
8944  */
8945 static int vmx_handle_exit(struct kvm_vcpu *vcpu)
8946 {
8947         struct vcpu_vmx *vmx = to_vmx(vcpu);
8948         u32 exit_reason = vmx->exit_reason;
8949         u32 vectoring_info = vmx->idt_vectoring_info;
8950
8951         trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
8952
8953         /*
8954          * Flush logged GPAs PML buffer, this will make dirty_bitmap more
8955          * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
8956          * querying dirty_bitmap, we only need to kick all vcpus out of guest
8957          * mode as if vcpus is in root mode, the PML buffer must has been
8958          * flushed already.
8959          */
8960         if (enable_pml)
8961                 vmx_flush_pml_buffer(vcpu);
8962
8963         /* If guest state is invalid, start emulating */
8964         if (vmx->emulation_required)
8965                 return handle_invalid_guest_state(vcpu);
8966
8967         if (is_guest_mode(vcpu) && nested_vmx_exit_reflected(vcpu, exit_reason))
8968                 return nested_vmx_reflect_vmexit(vcpu, exit_reason);
8969
8970         if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
8971                 dump_vmcs();
8972                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
8973                 vcpu->run->fail_entry.hardware_entry_failure_reason
8974                         = exit_reason;
8975                 return 0;
8976         }
8977
8978         if (unlikely(vmx->fail)) {
8979                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
8980                 vcpu->run->fail_entry.hardware_entry_failure_reason
8981                         = vmcs_read32(VM_INSTRUCTION_ERROR);
8982                 return 0;
8983         }
8984
8985         /*
8986          * Note:
8987          * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
8988          * delivery event since it indicates guest is accessing MMIO.
8989          * The vm-exit can be triggered again after return to guest that
8990          * will cause infinite loop.
8991          */
8992         if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
8993                         (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
8994                         exit_reason != EXIT_REASON_EPT_VIOLATION &&
8995                         exit_reason != EXIT_REASON_PML_FULL &&
8996                         exit_reason != EXIT_REASON_TASK_SWITCH)) {
8997                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
8998                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
8999                 vcpu->run->internal.ndata = 3;
9000                 vcpu->run->internal.data[0] = vectoring_info;
9001                 vcpu->run->internal.data[1] = exit_reason;
9002                 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
9003                 if (exit_reason == EXIT_REASON_EPT_MISCONFIG) {
9004                         vcpu->run->internal.ndata++;
9005                         vcpu->run->internal.data[3] =
9006                                 vmcs_read64(GUEST_PHYSICAL_ADDRESS);
9007                 }
9008                 return 0;
9009         }
9010
9011         if (unlikely(!cpu_has_virtual_nmis() &&
9012                      vmx->loaded_vmcs->soft_vnmi_blocked)) {
9013                 if (vmx_interrupt_allowed(vcpu)) {
9014                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
9015                 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL &&
9016                            vcpu->arch.nmi_pending) {
9017                         /*
9018                          * This CPU don't support us in finding the end of an
9019                          * NMI-blocked window if the guest runs with IRQs
9020                          * disabled. So we pull the trigger after 1 s of
9021                          * futile waiting, but inform the user about this.
9022                          */
9023                         printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
9024                                "state on VCPU %d after 1 s timeout\n",
9025                                __func__, vcpu->vcpu_id);
9026                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
9027                 }
9028         }
9029
9030         if (exit_reason < kvm_vmx_max_exit_handlers
9031             && kvm_vmx_exit_handlers[exit_reason])
9032                 return kvm_vmx_exit_handlers[exit_reason](vcpu);
9033         else {
9034                 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
9035                                 exit_reason);
9036                 kvm_queue_exception(vcpu, UD_VECTOR);
9037                 return 1;
9038         }
9039 }
9040
9041 /*
9042  * Software based L1D cache flush which is used when microcode providing
9043  * the cache control MSR is not loaded.
9044  *
9045  * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to
9046  * flush it is required to read in 64 KiB because the replacement algorithm
9047  * is not exactly LRU. This could be sized at runtime via topology
9048  * information but as all relevant affected CPUs have 32KiB L1D cache size
9049  * there is no point in doing so.
9050  */
9051 #define L1D_CACHE_ORDER 4
9052 static void *vmx_l1d_flush_pages;
9053
9054 static void __maybe_unused vmx_l1d_flush(void)
9055 {
9056         int size = PAGE_SIZE << L1D_CACHE_ORDER;
9057
9058         if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
9059                 wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
9060                 return;
9061         }
9062
9063         asm volatile(
9064                 /* First ensure the pages are in the TLB */
9065                 "xorl   %%eax, %%eax\n"
9066                 ".Lpopulate_tlb:\n\t"
9067                 "movzbl (%[empty_zp], %%" _ASM_AX "), %%ecx\n\t"
9068                 "addl   $4096, %%eax\n\t"
9069                 "cmpl   %%eax, %[size]\n\t"
9070                 "jne    .Lpopulate_tlb\n\t"
9071                 "xorl   %%eax, %%eax\n\t"
9072                 "cpuid\n\t"
9073                 /* Now fill the cache */
9074                 "xorl   %%eax, %%eax\n"
9075                 ".Lfill_cache:\n"
9076                 "movzbl (%[empty_zp], %%" _ASM_AX "), %%ecx\n\t"
9077                 "addl   $64, %%eax\n\t"
9078                 "cmpl   %%eax, %[size]\n\t"
9079                 "jne    .Lfill_cache\n\t"
9080                 "lfence\n"
9081                 :: [empty_zp] "r" (vmx_l1d_flush_pages),
9082                     [size] "r" (size)
9083                 : "eax", "ebx", "ecx", "edx");
9084 }
9085
9086 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
9087 {
9088         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
9089
9090         if (is_guest_mode(vcpu) &&
9091                 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
9092                 return;
9093
9094         if (irr == -1 || tpr < irr) {
9095                 vmcs_write32(TPR_THRESHOLD, 0);
9096                 return;
9097         }
9098
9099         vmcs_write32(TPR_THRESHOLD, irr);
9100 }
9101
9102 static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
9103 {
9104         u32 sec_exec_control;
9105
9106         /* Postpone execution until vmcs01 is the current VMCS. */
9107         if (is_guest_mode(vcpu)) {
9108                 to_vmx(vcpu)->nested.change_vmcs01_virtual_x2apic_mode = true;
9109                 return;
9110         }
9111
9112         if (!cpu_has_vmx_virtualize_x2apic_mode())
9113                 return;
9114
9115         if (!cpu_need_tpr_shadow(vcpu))
9116                 return;
9117
9118         sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
9119
9120         if (set) {
9121                 sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
9122                 sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
9123         } else {
9124                 sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
9125                 sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
9126                 vmx_flush_tlb_ept_only(vcpu);
9127         }
9128         vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
9129
9130         vmx_update_msr_bitmap(vcpu);
9131 }
9132
9133 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
9134 {
9135         struct vcpu_vmx *vmx = to_vmx(vcpu);
9136
9137         /*
9138          * Currently we do not handle the nested case where L2 has an
9139          * APIC access page of its own; that page is still pinned.
9140          * Hence, we skip the case where the VCPU is in guest mode _and_
9141          * L1 prepared an APIC access page for L2.
9142          *
9143          * For the case where L1 and L2 share the same APIC access page
9144          * (flexpriority=Y but SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES clear
9145          * in the vmcs12), this function will only update either the vmcs01
9146          * or the vmcs02.  If the former, the vmcs02 will be updated by
9147          * prepare_vmcs02.  If the latter, the vmcs01 will be updated in
9148          * the next L2->L1 exit.
9149          */
9150         if (!is_guest_mode(vcpu) ||
9151             !nested_cpu_has2(get_vmcs12(&vmx->vcpu),
9152                              SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
9153                 vmcs_write64(APIC_ACCESS_ADDR, hpa);
9154                 vmx_flush_tlb_ept_only(vcpu);
9155         }
9156 }
9157
9158 static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
9159 {
9160         u16 status;
9161         u8 old;
9162
9163         if (max_isr == -1)
9164                 max_isr = 0;
9165
9166         status = vmcs_read16(GUEST_INTR_STATUS);
9167         old = status >> 8;
9168         if (max_isr != old) {
9169                 status &= 0xff;
9170                 status |= max_isr << 8;
9171                 vmcs_write16(GUEST_INTR_STATUS, status);
9172         }
9173 }
9174
9175 static void vmx_set_rvi(int vector)
9176 {
9177         u16 status;
9178         u8 old;
9179
9180         if (vector == -1)
9181                 vector = 0;
9182
9183         status = vmcs_read16(GUEST_INTR_STATUS);
9184         old = (u8)status & 0xff;
9185         if ((u8)vector != old) {
9186                 status &= ~0xff;
9187                 status |= (u8)vector;
9188                 vmcs_write16(GUEST_INTR_STATUS, status);
9189         }
9190 }
9191
9192 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
9193 {
9194         if (!is_guest_mode(vcpu)) {
9195                 vmx_set_rvi(max_irr);
9196                 return;
9197         }
9198
9199         if (max_irr == -1)
9200                 return;
9201
9202         /*
9203          * In guest mode.  If a vmexit is needed, vmx_check_nested_events
9204          * handles it.
9205          */
9206         if (nested_exit_on_intr(vcpu))
9207                 return;
9208
9209         /*
9210          * Else, fall back to pre-APICv interrupt injection since L2
9211          * is run without virtual interrupt delivery.
9212          */
9213         if (!kvm_event_needs_reinjection(vcpu) &&
9214             vmx_interrupt_allowed(vcpu)) {
9215                 kvm_queue_interrupt(vcpu, max_irr, false);
9216                 vmx_inject_irq(vcpu);
9217         }
9218 }
9219
9220 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
9221 {
9222         struct vcpu_vmx *vmx = to_vmx(vcpu);
9223         int max_irr;
9224
9225         WARN_ON(!vcpu->arch.apicv_active);
9226         if (pi_test_on(&vmx->pi_desc)) {
9227                 pi_clear_on(&vmx->pi_desc);
9228                 /*
9229                  * IOMMU can write to PIR.ON, so the barrier matters even on UP.
9230                  * But on x86 this is just a compiler barrier anyway.
9231                  */
9232                 smp_mb__after_atomic();
9233                 max_irr = kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
9234         } else {
9235                 max_irr = kvm_lapic_find_highest_irr(vcpu);
9236         }
9237         vmx_hwapic_irr_update(vcpu, max_irr);
9238         return max_irr;
9239 }
9240
9241 static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
9242 {
9243         if (!kvm_vcpu_apicv_active(vcpu))
9244                 return;
9245
9246         vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
9247         vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
9248         vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
9249         vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
9250 }
9251
9252 static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu)
9253 {
9254         struct vcpu_vmx *vmx = to_vmx(vcpu);
9255
9256         pi_clear_on(&vmx->pi_desc);
9257         memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir));
9258 }
9259
9260 static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
9261 {
9262         u32 exit_intr_info = 0;
9263         u16 basic_exit_reason = (u16)vmx->exit_reason;
9264
9265         if (!(basic_exit_reason == EXIT_REASON_MCE_DURING_VMENTRY
9266               || basic_exit_reason == EXIT_REASON_EXCEPTION_NMI))
9267                 return;
9268
9269         if (!(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
9270                 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
9271         vmx->exit_intr_info = exit_intr_info;
9272
9273         /* if exit due to PF check for async PF */
9274         if (is_page_fault(exit_intr_info))
9275                 vmx->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
9276
9277         /* Handle machine checks before interrupts are enabled */
9278         if (basic_exit_reason == EXIT_REASON_MCE_DURING_VMENTRY ||
9279             is_machine_check(exit_intr_info))
9280                 kvm_machine_check();
9281
9282         /* We need to handle NMIs before interrupts are enabled */
9283         if (is_nmi(exit_intr_info)) {
9284                 kvm_before_handle_nmi(&vmx->vcpu);
9285                 asm("int $2");
9286                 kvm_after_handle_nmi(&vmx->vcpu);
9287         }
9288 }
9289
9290 static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
9291 {
9292         u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
9293
9294         if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK))
9295                         == (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) {
9296                 unsigned int vector;
9297                 unsigned long entry;
9298                 gate_desc *desc;
9299                 struct vcpu_vmx *vmx = to_vmx(vcpu);
9300 #ifdef CONFIG_X86_64
9301                 unsigned long tmp;
9302 #endif
9303
9304                 vector =  exit_intr_info & INTR_INFO_VECTOR_MASK;
9305                 desc = (gate_desc *)vmx->host_idt_base + vector;
9306                 entry = gate_offset(desc);
9307                 asm volatile(
9308 #ifdef CONFIG_X86_64
9309                         "mov %%" _ASM_SP ", %[sp]\n\t"
9310                         "and $0xfffffffffffffff0, %%" _ASM_SP "\n\t"
9311                         "push $%c[ss]\n\t"
9312                         "push %[sp]\n\t"
9313 #endif
9314                         "pushf\n\t"
9315                         __ASM_SIZE(push) " $%c[cs]\n\t"
9316                         CALL_NOSPEC
9317                         :
9318 #ifdef CONFIG_X86_64
9319                         [sp]"=&r"(tmp),
9320 #endif
9321                         ASM_CALL_CONSTRAINT
9322                         :
9323                         THUNK_TARGET(entry),
9324                         [ss]"i"(__KERNEL_DS),
9325                         [cs]"i"(__KERNEL_CS)
9326                         );
9327         }
9328 }
9329 STACK_FRAME_NON_STANDARD(vmx_handle_external_intr);
9330
9331 static bool vmx_has_emulated_msr(int index)
9332 {
9333         switch (index) {
9334         case MSR_IA32_SMBASE:
9335                 /*
9336                  * We cannot do SMM unless we can run the guest in big
9337                  * real mode.
9338                  */
9339                 return enable_unrestricted_guest || emulate_invalid_guest_state;
9340         case MSR_AMD64_VIRT_SPEC_CTRL:
9341                 /* This is AMD only.  */
9342                 return false;
9343         default:
9344                 return true;
9345         }
9346 }
9347
9348 static bool vmx_mpx_supported(void)
9349 {
9350         return (vmcs_config.vmexit_ctrl & VM_EXIT_CLEAR_BNDCFGS) &&
9351                 (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_BNDCFGS);
9352 }
9353
9354 static bool vmx_xsaves_supported(void)
9355 {
9356         return vmcs_config.cpu_based_2nd_exec_ctrl &
9357                 SECONDARY_EXEC_XSAVES;
9358 }
9359
9360 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
9361 {
9362         u32 exit_intr_info;
9363         bool unblock_nmi;
9364         u8 vector;
9365         bool idtv_info_valid;
9366
9367         idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
9368
9369         if (cpu_has_virtual_nmis()) {
9370                 if (vmx->loaded_vmcs->nmi_known_unmasked)
9371                         return;
9372                 /*
9373                  * Can't use vmx->exit_intr_info since we're not sure what
9374                  * the exit reason is.
9375                  */
9376                 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
9377                 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
9378                 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
9379                 /*
9380                  * SDM 3: 27.7.1.2 (September 2008)
9381                  * Re-set bit "block by NMI" before VM entry if vmexit caused by
9382                  * a guest IRET fault.
9383                  * SDM 3: 23.2.2 (September 2008)
9384                  * Bit 12 is undefined in any of the following cases:
9385                  *  If the VM exit sets the valid bit in the IDT-vectoring
9386                  *   information field.
9387                  *  If the VM exit is due to a double fault.
9388                  */
9389                 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
9390                     vector != DF_VECTOR && !idtv_info_valid)
9391                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
9392                                       GUEST_INTR_STATE_NMI);
9393                 else
9394                         vmx->loaded_vmcs->nmi_known_unmasked =
9395                                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
9396                                   & GUEST_INTR_STATE_NMI);
9397         } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked))
9398                 vmx->loaded_vmcs->vnmi_blocked_time +=
9399                         ktime_to_ns(ktime_sub(ktime_get(),
9400                                               vmx->loaded_vmcs->entry_time));
9401 }
9402
9403 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
9404                                       u32 idt_vectoring_info,
9405                                       int instr_len_field,
9406                                       int error_code_field)
9407 {
9408         u8 vector;
9409         int type;
9410         bool idtv_info_valid;
9411
9412         idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
9413
9414         vcpu->arch.nmi_injected = false;
9415         kvm_clear_exception_queue(vcpu);
9416         kvm_clear_interrupt_queue(vcpu);
9417
9418         if (!idtv_info_valid)
9419                 return;
9420
9421         kvm_make_request(KVM_REQ_EVENT, vcpu);
9422
9423         vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
9424         type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
9425
9426         switch (type) {
9427         case INTR_TYPE_NMI_INTR:
9428                 vcpu->arch.nmi_injected = true;
9429                 /*
9430                  * SDM 3: 27.7.1.2 (September 2008)
9431                  * Clear bit "block by NMI" before VM entry if a NMI
9432                  * delivery faulted.
9433                  */
9434                 vmx_set_nmi_mask(vcpu, false);
9435                 break;
9436         case INTR_TYPE_SOFT_EXCEPTION:
9437                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
9438                 /* fall through */
9439         case INTR_TYPE_HARD_EXCEPTION:
9440                 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
9441                         u32 err = vmcs_read32(error_code_field);
9442                         kvm_requeue_exception_e(vcpu, vector, err);
9443                 } else
9444                         kvm_requeue_exception(vcpu, vector);
9445                 break;
9446         case INTR_TYPE_SOFT_INTR:
9447                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
9448                 /* fall through */
9449         case INTR_TYPE_EXT_INTR:
9450                 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
9451                 break;
9452         default:
9453                 break;
9454         }
9455 }
9456
9457 static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
9458 {
9459         __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
9460                                   VM_EXIT_INSTRUCTION_LEN,
9461                                   IDT_VECTORING_ERROR_CODE);
9462 }
9463
9464 static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
9465 {
9466         __vmx_complete_interrupts(vcpu,
9467                                   vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
9468                                   VM_ENTRY_INSTRUCTION_LEN,
9469                                   VM_ENTRY_EXCEPTION_ERROR_CODE);
9470
9471         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
9472 }
9473
9474 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
9475 {
9476         int i, nr_msrs;
9477         struct perf_guest_switch_msr *msrs;
9478
9479         msrs = perf_guest_get_msrs(&nr_msrs);
9480
9481         if (!msrs)
9482                 return;
9483
9484         for (i = 0; i < nr_msrs; i++)
9485                 if (msrs[i].host == msrs[i].guest)
9486                         clear_atomic_switch_msr(vmx, msrs[i].msr);
9487                 else
9488                         add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
9489                                         msrs[i].host);
9490 }
9491
9492 static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
9493 {
9494         struct vcpu_vmx *vmx = to_vmx(vcpu);
9495         u64 tscl;
9496         u32 delta_tsc;
9497
9498         if (vmx->hv_deadline_tsc == -1)
9499                 return;
9500
9501         tscl = rdtsc();
9502         if (vmx->hv_deadline_tsc > tscl)
9503                 /* sure to be 32 bit only because checked on set_hv_timer */
9504                 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
9505                         cpu_preemption_timer_multi);
9506         else
9507                 delta_tsc = 0;
9508
9509         vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc);
9510 }
9511
9512 static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
9513 {
9514         struct vcpu_vmx *vmx = to_vmx(vcpu);
9515         unsigned long debugctlmsr, cr3, cr4;
9516
9517         /* Record the guest's net vcpu time for enforced NMI injections. */
9518         if (unlikely(!cpu_has_virtual_nmis() &&
9519                      vmx->loaded_vmcs->soft_vnmi_blocked))
9520                 vmx->loaded_vmcs->entry_time = ktime_get();
9521
9522         /* Don't enter VMX if guest state is invalid, let the exit handler
9523            start emulation until we arrive back to a valid state */
9524         if (vmx->emulation_required)
9525                 return;
9526
9527         if (vmx->ple_window_dirty) {
9528                 vmx->ple_window_dirty = false;
9529                 vmcs_write32(PLE_WINDOW, vmx->ple_window);
9530         }
9531
9532         if (vmx->nested.sync_shadow_vmcs) {
9533                 copy_vmcs12_to_shadow(vmx);
9534                 vmx->nested.sync_shadow_vmcs = false;
9535         }
9536
9537         if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
9538                 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
9539         if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
9540                 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
9541
9542         cr3 = __get_current_cr3_fast();
9543         if (unlikely(cr3 != vmx->loaded_vmcs->vmcs_host_cr3)) {
9544                 vmcs_writel(HOST_CR3, cr3);
9545                 vmx->loaded_vmcs->vmcs_host_cr3 = cr3;
9546         }
9547
9548         cr4 = cr4_read_shadow();
9549         if (unlikely(cr4 != vmx->loaded_vmcs->vmcs_host_cr4)) {
9550                 vmcs_writel(HOST_CR4, cr4);
9551                 vmx->loaded_vmcs->vmcs_host_cr4 = cr4;
9552         }
9553
9554         /* When single-stepping over STI and MOV SS, we must clear the
9555          * corresponding interruptibility bits in the guest state. Otherwise
9556          * vmentry fails as it then expects bit 14 (BS) in pending debug
9557          * exceptions being set, but that's not correct for the guest debugging
9558          * case. */
9559         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
9560                 vmx_set_interrupt_shadow(vcpu, 0);
9561
9562         if (static_cpu_has(X86_FEATURE_PKU) &&
9563             kvm_read_cr4_bits(vcpu, X86_CR4_PKE) &&
9564             vcpu->arch.pkru != vmx->host_pkru)
9565                 __write_pkru(vcpu->arch.pkru);
9566
9567         atomic_switch_perf_msrs(vmx);
9568         debugctlmsr = get_debugctlmsr();
9569
9570         vmx_arm_hv_timer(vcpu);
9571
9572         /*
9573          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
9574          * it's non-zero. Since vmentry is serialising on affected CPUs, there
9575          * is no need to worry about the conditional branch over the wrmsr
9576          * being speculatively taken.
9577          */
9578         x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0);
9579
9580         vmx->__launched = vmx->loaded_vmcs->launched;
9581
9582         asm(
9583                 /* Store host registers */
9584                 "push %%" _ASM_DX "; push %%" _ASM_BP ";"
9585                 "push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
9586                 "push %%" _ASM_CX " \n\t"
9587                 "cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
9588                 "je 1f \n\t"
9589                 "mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
9590                 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
9591                 "1: \n\t"
9592                 /* Reload cr2 if changed */
9593                 "mov %c[cr2](%0), %%" _ASM_AX " \n\t"
9594                 "mov %%cr2, %%" _ASM_DX " \n\t"
9595                 "cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
9596                 "je 2f \n\t"
9597                 "mov %%" _ASM_AX", %%cr2 \n\t"
9598                 "2: \n\t"
9599                 /* Check if vmlaunch of vmresume is needed */
9600                 "cmpl $0, %c[launched](%0) \n\t"
9601                 /* Load guest registers.  Don't clobber flags. */
9602                 "mov %c[rax](%0), %%" _ASM_AX " \n\t"
9603                 "mov %c[rbx](%0), %%" _ASM_BX " \n\t"
9604                 "mov %c[rdx](%0), %%" _ASM_DX " \n\t"
9605                 "mov %c[rsi](%0), %%" _ASM_SI " \n\t"
9606                 "mov %c[rdi](%0), %%" _ASM_DI " \n\t"
9607                 "mov %c[rbp](%0), %%" _ASM_BP " \n\t"
9608 #ifdef CONFIG_X86_64
9609                 "mov %c[r8](%0),  %%r8  \n\t"
9610                 "mov %c[r9](%0),  %%r9  \n\t"
9611                 "mov %c[r10](%0), %%r10 \n\t"
9612                 "mov %c[r11](%0), %%r11 \n\t"
9613                 "mov %c[r12](%0), %%r12 \n\t"
9614                 "mov %c[r13](%0), %%r13 \n\t"
9615                 "mov %c[r14](%0), %%r14 \n\t"
9616                 "mov %c[r15](%0), %%r15 \n\t"
9617 #endif
9618                 "mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */
9619
9620                 /* Enter guest mode */
9621                 "jne 1f \n\t"
9622                 __ex(ASM_VMX_VMLAUNCH) "\n\t"
9623                 "jmp 2f \n\t"
9624                 "1: " __ex(ASM_VMX_VMRESUME) "\n\t"
9625                 "2: "
9626                 /* Save guest registers, load host registers, keep flags */
9627                 "mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
9628                 "pop %0 \n\t"
9629                 "setbe %c[fail](%0)\n\t"
9630                 "mov %%" _ASM_AX ", %c[rax](%0) \n\t"
9631                 "mov %%" _ASM_BX ", %c[rbx](%0) \n\t"
9632                 __ASM_SIZE(pop) " %c[rcx](%0) \n\t"
9633                 "mov %%" _ASM_DX ", %c[rdx](%0) \n\t"
9634                 "mov %%" _ASM_SI ", %c[rsi](%0) \n\t"
9635                 "mov %%" _ASM_DI ", %c[rdi](%0) \n\t"
9636                 "mov %%" _ASM_BP ", %c[rbp](%0) \n\t"
9637 #ifdef CONFIG_X86_64
9638                 "mov %%r8,  %c[r8](%0) \n\t"
9639                 "mov %%r9,  %c[r9](%0) \n\t"
9640                 "mov %%r10, %c[r10](%0) \n\t"
9641                 "mov %%r11, %c[r11](%0) \n\t"
9642                 "mov %%r12, %c[r12](%0) \n\t"
9643                 "mov %%r13, %c[r13](%0) \n\t"
9644                 "mov %%r14, %c[r14](%0) \n\t"
9645                 "mov %%r15, %c[r15](%0) \n\t"
9646                 "xor %%r8d,  %%r8d \n\t"
9647                 "xor %%r9d,  %%r9d \n\t"
9648                 "xor %%r10d, %%r10d \n\t"
9649                 "xor %%r11d, %%r11d \n\t"
9650                 "xor %%r12d, %%r12d \n\t"
9651                 "xor %%r13d, %%r13d \n\t"
9652                 "xor %%r14d, %%r14d \n\t"
9653                 "xor %%r15d, %%r15d \n\t"
9654 #endif
9655                 "mov %%cr2, %%" _ASM_AX "   \n\t"
9656                 "mov %%" _ASM_AX ", %c[cr2](%0) \n\t"
9657
9658                 "xor %%eax, %%eax \n\t"
9659                 "xor %%ebx, %%ebx \n\t"
9660                 "xor %%esi, %%esi \n\t"
9661                 "xor %%edi, %%edi \n\t"
9662                 "pop  %%" _ASM_BP "; pop  %%" _ASM_DX " \n\t"
9663                 ".pushsection .rodata \n\t"
9664                 ".global vmx_return \n\t"
9665                 "vmx_return: " _ASM_PTR " 2b \n\t"
9666                 ".popsection"
9667               : : "c"(vmx), "d"((unsigned long)HOST_RSP),
9668                 [launched]"i"(offsetof(struct vcpu_vmx, __launched)),
9669                 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
9670                 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
9671                 [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
9672                 [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
9673                 [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
9674                 [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
9675                 [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
9676                 [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
9677                 [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
9678 #ifdef CONFIG_X86_64
9679                 [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
9680                 [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
9681                 [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
9682                 [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
9683                 [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
9684                 [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
9685                 [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
9686                 [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
9687 #endif
9688                 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
9689                 [wordsize]"i"(sizeof(ulong))
9690               : "cc", "memory"
9691 #ifdef CONFIG_X86_64
9692                 , "rax", "rbx", "rdi", "rsi"
9693                 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
9694 #else
9695                 , "eax", "ebx", "edi", "esi"
9696 #endif
9697               );
9698
9699         /*
9700          * We do not use IBRS in the kernel. If this vCPU has used the
9701          * SPEC_CTRL MSR it may have left it on; save the value and
9702          * turn it off. This is much more efficient than blindly adding
9703          * it to the atomic save/restore list. Especially as the former
9704          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
9705          *
9706          * For non-nested case:
9707          * If the L01 MSR bitmap does not intercept the MSR, then we need to
9708          * save it.
9709          *
9710          * For nested case:
9711          * If the L02 MSR bitmap does not intercept the MSR, then we need to
9712          * save it.
9713          */
9714         if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
9715                 vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
9716
9717         x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
9718
9719         /* Eliminate branch target predictions from guest mode */
9720         vmexit_fill_RSB();
9721
9722         /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
9723         if (debugctlmsr)
9724                 update_debugctlmsr(debugctlmsr);
9725
9726 #ifndef CONFIG_X86_64
9727         /*
9728          * The sysexit path does not restore ds/es, so we must set them to
9729          * a reasonable value ourselves.
9730          *
9731          * We can't defer this to vmx_load_host_state() since that function
9732          * may be executed in interrupt context, which saves and restore segments
9733          * around it, nullifying its effect.
9734          */
9735         loadsegment(ds, __USER_DS);
9736         loadsegment(es, __USER_DS);
9737 #endif
9738
9739         vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
9740                                   | (1 << VCPU_EXREG_RFLAGS)
9741                                   | (1 << VCPU_EXREG_PDPTR)
9742                                   | (1 << VCPU_EXREG_SEGMENTS)
9743                                   | (1 << VCPU_EXREG_CR3));
9744         vcpu->arch.regs_dirty = 0;
9745
9746         /*
9747          * eager fpu is enabled if PKEY is supported and CR4 is switched
9748          * back on host, so it is safe to read guest PKRU from current
9749          * XSAVE.
9750          */
9751         if (static_cpu_has(X86_FEATURE_PKU) &&
9752             kvm_read_cr4_bits(vcpu, X86_CR4_PKE)) {
9753                 vcpu->arch.pkru = __read_pkru();
9754                 if (vcpu->arch.pkru != vmx->host_pkru)
9755                         __write_pkru(vmx->host_pkru);
9756         }
9757
9758         /*
9759          * the KVM_REQ_EVENT optimization bit is only on for one entry, and if
9760          * we did not inject a still-pending event to L1 now because of
9761          * nested_run_pending, we need to re-enable this bit.
9762          */
9763         if (vmx->nested.nested_run_pending)
9764                 kvm_make_request(KVM_REQ_EVENT, vcpu);
9765
9766         vmx->nested.nested_run_pending = 0;
9767         vmx->idt_vectoring_info = 0;
9768
9769         vmx->exit_reason = vmx->fail ? 0xdead : vmcs_read32(VM_EXIT_REASON);
9770         if (vmx->fail || (vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
9771                 return;
9772
9773         vmx->loaded_vmcs->launched = 1;
9774         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
9775
9776         vmx_complete_atomic_exit(vmx);
9777         vmx_recover_nmi_blocking(vmx);
9778         vmx_complete_interrupts(vmx);
9779 }
9780 STACK_FRAME_NON_STANDARD(vmx_vcpu_run);
9781
9782 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
9783 {
9784         struct vcpu_vmx *vmx = to_vmx(vcpu);
9785         int cpu;
9786
9787         if (vmx->loaded_vmcs == vmcs)
9788                 return;
9789
9790         cpu = get_cpu();
9791         vmx->loaded_vmcs = vmcs;
9792         vmx_vcpu_put(vcpu);
9793         vmx_vcpu_load(vcpu, cpu);
9794         vcpu->cpu = cpu;
9795         put_cpu();
9796 }
9797
9798 /*
9799  * Ensure that the current vmcs of the logical processor is the
9800  * vmcs01 of the vcpu before calling free_nested().
9801  */
9802 static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
9803 {
9804        struct vcpu_vmx *vmx = to_vmx(vcpu);
9805        int r;
9806
9807        r = vcpu_load(vcpu);
9808        BUG_ON(r);
9809        vmx_switch_vmcs(vcpu, &vmx->vmcs01);
9810        free_nested(vmx);
9811        vcpu_put(vcpu);
9812 }
9813
9814 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
9815 {
9816         struct vcpu_vmx *vmx = to_vmx(vcpu);
9817
9818         if (enable_pml)
9819                 vmx_destroy_pml_buffer(vmx);
9820         free_vpid(vmx->vpid);
9821         leave_guest_mode(vcpu);
9822         vmx_free_vcpu_nested(vcpu);
9823         free_loaded_vmcs(vmx->loaded_vmcs);
9824         kfree(vmx->guest_msrs);
9825         kvm_vcpu_uninit(vcpu);
9826         kmem_cache_free(kvm_vcpu_cache, vmx);
9827 }
9828
9829 static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
9830 {
9831         int err;
9832         struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
9833         unsigned long *msr_bitmap;
9834         int cpu;
9835
9836         if (!vmx)
9837                 return ERR_PTR(-ENOMEM);
9838
9839         vmx->vpid = allocate_vpid();
9840
9841         err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
9842         if (err)
9843                 goto free_vcpu;
9844
9845         err = -ENOMEM;
9846
9847         /*
9848          * If PML is turned on, failure on enabling PML just results in failure
9849          * of creating the vcpu, therefore we can simplify PML logic (by
9850          * avoiding dealing with cases, such as enabling PML partially on vcpus
9851          * for the guest, etc.
9852          */
9853         if (enable_pml) {
9854                 vmx->pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO);
9855                 if (!vmx->pml_pg)
9856                         goto uninit_vcpu;
9857         }
9858
9859         vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
9860         BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0])
9861                      > PAGE_SIZE);
9862
9863         if (!vmx->guest_msrs)
9864                 goto free_pml;
9865
9866         err = alloc_loaded_vmcs(&vmx->vmcs01);
9867         if (err < 0)
9868                 goto free_msrs;
9869
9870         msr_bitmap = vmx->vmcs01.msr_bitmap;
9871         vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE, MSR_TYPE_RW);
9872         vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE, MSR_TYPE_RW);
9873         vmx_disable_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
9874         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
9875         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW);
9876         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW);
9877         vmx->msr_bitmap_mode = 0;
9878
9879         vmx->loaded_vmcs = &vmx->vmcs01;
9880         cpu = get_cpu();
9881         vmx_vcpu_load(&vmx->vcpu, cpu);
9882         vmx->vcpu.cpu = cpu;
9883         err = vmx_vcpu_setup(vmx);
9884         vmx_vcpu_put(&vmx->vcpu);
9885         put_cpu();
9886         if (err)
9887                 goto free_vmcs;
9888         if (cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
9889                 err = alloc_apic_access_page(kvm);
9890                 if (err)
9891                         goto free_vmcs;
9892         }
9893
9894         if (enable_ept) {
9895                 if (!kvm->arch.ept_identity_map_addr)
9896                         kvm->arch.ept_identity_map_addr =
9897                                 VMX_EPT_IDENTITY_PAGETABLE_ADDR;
9898                 err = init_rmode_identity_map(kvm);
9899                 if (err)
9900                         goto free_vmcs;
9901         }
9902
9903         if (nested)
9904                 nested_vmx_setup_ctls_msrs(vmx);
9905
9906         vmx->nested.posted_intr_nv = -1;
9907         vmx->nested.current_vmptr = -1ull;
9908
9909         vmx->msr_ia32_feature_control_valid_bits = FEATURE_CONTROL_LOCKED;
9910
9911         /*
9912          * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
9913          * or POSTED_INTR_WAKEUP_VECTOR.
9914          */
9915         vmx->pi_desc.nv = POSTED_INTR_VECTOR;
9916         vmx->pi_desc.sn = 1;
9917
9918         return &vmx->vcpu;
9919
9920 free_vmcs:
9921         free_loaded_vmcs(vmx->loaded_vmcs);
9922 free_msrs:
9923         kfree(vmx->guest_msrs);
9924 free_pml:
9925         vmx_destroy_pml_buffer(vmx);
9926 uninit_vcpu:
9927         kvm_vcpu_uninit(&vmx->vcpu);
9928 free_vcpu:
9929         free_vpid(vmx->vpid);
9930         kmem_cache_free(kvm_vcpu_cache, vmx);
9931         return ERR_PTR(err);
9932 }
9933
9934 #define L1TF_MSG "SMT enabled with L1TF CPU bug present. Refer to CVE-2018-3620 for details.\n"
9935
9936 static int vmx_vm_init(struct kvm *kvm)
9937 {
9938         if (boot_cpu_has(X86_BUG_L1TF) && cpu_smt_control == CPU_SMT_ENABLED) {
9939                 if (nosmt) {
9940                         pr_err(L1TF_MSG);
9941                         return -EOPNOTSUPP;
9942                 }
9943                 pr_warn(L1TF_MSG);
9944         }
9945         return 0;
9946 }
9947
9948 static void __init vmx_check_processor_compat(void *rtn)
9949 {
9950         struct vmcs_config vmcs_conf;
9951
9952         *(int *)rtn = 0;
9953         if (setup_vmcs_config(&vmcs_conf) < 0)
9954                 *(int *)rtn = -EIO;
9955         if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
9956                 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
9957                                 smp_processor_id());
9958                 *(int *)rtn = -EIO;
9959         }
9960 }
9961
9962 static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
9963 {
9964         u8 cache;
9965         u64 ipat = 0;
9966
9967         /* For VT-d and EPT combination
9968          * 1. MMIO: always map as UC
9969          * 2. EPT with VT-d:
9970          *   a. VT-d without snooping control feature: can't guarantee the
9971          *      result, try to trust guest.
9972          *   b. VT-d with snooping control feature: snooping control feature of
9973          *      VT-d engine can guarantee the cache correctness. Just set it
9974          *      to WB to keep consistent with host. So the same as item 3.
9975          * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
9976          *    consistent with host MTRR
9977          */
9978         if (is_mmio) {
9979                 cache = MTRR_TYPE_UNCACHABLE;
9980                 goto exit;
9981         }
9982
9983         if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
9984                 ipat = VMX_EPT_IPAT_BIT;
9985                 cache = MTRR_TYPE_WRBACK;
9986                 goto exit;
9987         }
9988
9989         if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
9990                 ipat = VMX_EPT_IPAT_BIT;
9991                 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
9992                         cache = MTRR_TYPE_WRBACK;
9993                 else
9994                         cache = MTRR_TYPE_UNCACHABLE;
9995                 goto exit;
9996         }
9997
9998         cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
9999
10000 exit:
10001         return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat;
10002 }
10003
10004 static int vmx_get_lpage_level(void)
10005 {
10006         if (enable_ept && !cpu_has_vmx_ept_1g_page())
10007                 return PT_DIRECTORY_LEVEL;
10008         else
10009                 /* For shadow and EPT supported 1GB page */
10010                 return PT_PDPE_LEVEL;
10011 }
10012
10013 static void vmcs_set_secondary_exec_control(u32 new_ctl)
10014 {
10015         /*
10016          * These bits in the secondary execution controls field
10017          * are dynamic, the others are mostly based on the hypervisor
10018          * architecture and the guest's CPUID.  Do not touch the
10019          * dynamic bits.
10020          */
10021         u32 mask =
10022                 SECONDARY_EXEC_SHADOW_VMCS |
10023                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
10024                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
10025
10026         u32 cur_ctl = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
10027
10028         vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
10029                      (new_ctl & ~mask) | (cur_ctl & mask));
10030 }
10031
10032 /*
10033  * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
10034  * (indicating "allowed-1") if they are supported in the guest's CPUID.
10035  */
10036 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
10037 {
10038         struct vcpu_vmx *vmx = to_vmx(vcpu);
10039         struct kvm_cpuid_entry2 *entry;
10040
10041         vmx->nested.nested_vmx_cr0_fixed1 = 0xffffffff;
10042         vmx->nested.nested_vmx_cr4_fixed1 = X86_CR4_PCE;
10043
10044 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do {            \
10045         if (entry && (entry->_reg & (_cpuid_mask)))                     \
10046                 vmx->nested.nested_vmx_cr4_fixed1 |= (_cr4_mask);       \
10047 } while (0)
10048
10049         entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
10050         cr4_fixed1_update(X86_CR4_VME,        edx, bit(X86_FEATURE_VME));
10051         cr4_fixed1_update(X86_CR4_PVI,        edx, bit(X86_FEATURE_VME));
10052         cr4_fixed1_update(X86_CR4_TSD,        edx, bit(X86_FEATURE_TSC));
10053         cr4_fixed1_update(X86_CR4_DE,         edx, bit(X86_FEATURE_DE));
10054         cr4_fixed1_update(X86_CR4_PSE,        edx, bit(X86_FEATURE_PSE));
10055         cr4_fixed1_update(X86_CR4_PAE,        edx, bit(X86_FEATURE_PAE));
10056         cr4_fixed1_update(X86_CR4_MCE,        edx, bit(X86_FEATURE_MCE));
10057         cr4_fixed1_update(X86_CR4_PGE,        edx, bit(X86_FEATURE_PGE));
10058         cr4_fixed1_update(X86_CR4_OSFXSR,     edx, bit(X86_FEATURE_FXSR));
10059         cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, bit(X86_FEATURE_XMM));
10060         cr4_fixed1_update(X86_CR4_VMXE,       ecx, bit(X86_FEATURE_VMX));
10061         cr4_fixed1_update(X86_CR4_SMXE,       ecx, bit(X86_FEATURE_SMX));
10062         cr4_fixed1_update(X86_CR4_PCIDE,      ecx, bit(X86_FEATURE_PCID));
10063         cr4_fixed1_update(X86_CR4_OSXSAVE,    ecx, bit(X86_FEATURE_XSAVE));
10064
10065         entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
10066         cr4_fixed1_update(X86_CR4_FSGSBASE,   ebx, bit(X86_FEATURE_FSGSBASE));
10067         cr4_fixed1_update(X86_CR4_SMEP,       ebx, bit(X86_FEATURE_SMEP));
10068         cr4_fixed1_update(X86_CR4_SMAP,       ebx, bit(X86_FEATURE_SMAP));
10069         cr4_fixed1_update(X86_CR4_PKE,        ecx, bit(X86_FEATURE_PKU));
10070         /* TODO: Use X86_CR4_UMIP and X86_FEATURE_UMIP macros */
10071         cr4_fixed1_update(bit(11),            ecx, bit(2));
10072
10073 #undef cr4_fixed1_update
10074 }
10075
10076 static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
10077 {
10078         struct vcpu_vmx *vmx = to_vmx(vcpu);
10079
10080         if (cpu_has_secondary_exec_ctrls()) {
10081                 vmx_compute_secondary_exec_control(vmx);
10082                 vmcs_set_secondary_exec_control(vmx->secondary_exec_control);
10083         }
10084
10085         if (nested_vmx_allowed(vcpu))
10086                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
10087                         FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
10088         else
10089                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
10090                         ~FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
10091
10092         if (nested_vmx_allowed(vcpu))
10093                 nested_vmx_cr_fixed1_bits_update(vcpu);
10094 }
10095
10096 static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
10097 {
10098         if (func == 1 && nested)
10099                 entry->ecx |= bit(X86_FEATURE_VMX);
10100 }
10101
10102 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
10103                 struct x86_exception *fault)
10104 {
10105         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
10106         struct vcpu_vmx *vmx = to_vmx(vcpu);
10107         u32 exit_reason;
10108         unsigned long exit_qualification = vcpu->arch.exit_qualification;
10109
10110         if (vmx->nested.pml_full) {
10111                 exit_reason = EXIT_REASON_PML_FULL;
10112                 vmx->nested.pml_full = false;
10113                 exit_qualification &= INTR_INFO_UNBLOCK_NMI;
10114         } else if (fault->error_code & PFERR_RSVD_MASK)
10115                 exit_reason = EXIT_REASON_EPT_MISCONFIG;
10116         else
10117                 exit_reason = EXIT_REASON_EPT_VIOLATION;
10118
10119         nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification);
10120         vmcs12->guest_physical_address = fault->address;
10121 }
10122
10123 static bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu)
10124 {
10125         return nested_ept_get_cr3(vcpu) & VMX_EPTP_AD_ENABLE_BIT;
10126 }
10127
10128 /* Callbacks for nested_ept_init_mmu_context: */
10129
10130 static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu)
10131 {
10132         /* return the page table to be shadowed - in our case, EPT12 */
10133         return get_vmcs12(vcpu)->ept_pointer;
10134 }
10135
10136 static int nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
10137 {
10138         WARN_ON(mmu_is_nested(vcpu));
10139         if (!valid_ept_address(vcpu, nested_ept_get_cr3(vcpu)))
10140                 return 1;
10141
10142         kvm_mmu_unload(vcpu);
10143         kvm_init_shadow_ept_mmu(vcpu,
10144                         to_vmx(vcpu)->nested.nested_vmx_ept_caps &
10145                         VMX_EPT_EXECUTE_ONLY_BIT,
10146                         nested_ept_ad_enabled(vcpu));
10147         vcpu->arch.mmu.set_cr3           = vmx_set_cr3;
10148         vcpu->arch.mmu.get_cr3           = nested_ept_get_cr3;
10149         vcpu->arch.mmu.inject_page_fault = nested_ept_inject_page_fault;
10150
10151         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
10152         return 0;
10153 }
10154
10155 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
10156 {
10157         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
10158 }
10159
10160 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
10161                                             u16 error_code)
10162 {
10163         bool inequality, bit;
10164
10165         bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
10166         inequality =
10167                 (error_code & vmcs12->page_fault_error_code_mask) !=
10168                  vmcs12->page_fault_error_code_match;
10169         return inequality ^ bit;
10170 }
10171
10172 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
10173                 struct x86_exception *fault)
10174 {
10175         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
10176
10177         WARN_ON(!is_guest_mode(vcpu));
10178
10179         if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) &&
10180                 !to_vmx(vcpu)->nested.nested_run_pending) {
10181                 vmcs12->vm_exit_intr_error_code = fault->error_code;
10182                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
10183                                   PF_VECTOR | INTR_TYPE_HARD_EXCEPTION |
10184                                   INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK,
10185                                   fault->address);
10186         } else {
10187                 kvm_inject_page_fault(vcpu, fault);
10188         }
10189 }
10190
10191 static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
10192                                                struct vmcs12 *vmcs12);
10193
10194 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
10195                                         struct vmcs12 *vmcs12)
10196 {
10197         struct vcpu_vmx *vmx = to_vmx(vcpu);
10198         struct page *page;
10199         u64 hpa;
10200
10201         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
10202                 /*
10203                  * Translate L1 physical address to host physical
10204                  * address for vmcs02. Keep the page pinned, so this
10205                  * physical address remains valid. We keep a reference
10206                  * to it so we can release it later.
10207                  */
10208                 if (vmx->nested.apic_access_page) { /* shouldn't happen */
10209                         kvm_release_page_dirty(vmx->nested.apic_access_page);
10210                         vmx->nested.apic_access_page = NULL;
10211                 }
10212                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr);
10213                 /*
10214                  * If translation failed, no matter: This feature asks
10215                  * to exit when accessing the given address, and if it
10216                  * can never be accessed, this feature won't do
10217                  * anything anyway.
10218                  */
10219                 if (!is_error_page(page)) {
10220                         vmx->nested.apic_access_page = page;
10221                         hpa = page_to_phys(vmx->nested.apic_access_page);
10222                         vmcs_write64(APIC_ACCESS_ADDR, hpa);
10223                 } else {
10224                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
10225                                         SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
10226                 }
10227         } else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) &&
10228                    cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
10229                 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
10230                               SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
10231                 kvm_vcpu_reload_apic_access_page(vcpu);
10232         }
10233
10234         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
10235                 if (vmx->nested.virtual_apic_page) { /* shouldn't happen */
10236                         kvm_release_page_dirty(vmx->nested.virtual_apic_page);
10237                         vmx->nested.virtual_apic_page = NULL;
10238                 }
10239                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->virtual_apic_page_addr);
10240
10241                 /*
10242                  * If translation failed, VM entry will fail because
10243                  * prepare_vmcs02 set VIRTUAL_APIC_PAGE_ADDR to -1ull.
10244                  * Failing the vm entry is _not_ what the processor
10245                  * does but it's basically the only possibility we
10246                  * have.  We could still enter the guest if CR8 load
10247                  * exits are enabled, CR8 store exits are enabled, and
10248                  * virtualize APIC access is disabled; in this case
10249                  * the processor would never use the TPR shadow and we
10250                  * could simply clear the bit from the execution
10251                  * control.  But such a configuration is useless, so
10252                  * let's keep the code simple.
10253                  */
10254                 if (!is_error_page(page)) {
10255                         vmx->nested.virtual_apic_page = page;
10256                         hpa = page_to_phys(vmx->nested.virtual_apic_page);
10257                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, hpa);
10258                 }
10259         }
10260
10261         if (nested_cpu_has_posted_intr(vmcs12)) {
10262                 if (vmx->nested.pi_desc_page) { /* shouldn't happen */
10263                         kunmap(vmx->nested.pi_desc_page);
10264                         kvm_release_page_dirty(vmx->nested.pi_desc_page);
10265                         vmx->nested.pi_desc_page = NULL;
10266                 }
10267                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->posted_intr_desc_addr);
10268                 if (is_error_page(page))
10269                         return;
10270                 vmx->nested.pi_desc_page = page;
10271                 vmx->nested.pi_desc = kmap(vmx->nested.pi_desc_page);
10272                 vmx->nested.pi_desc =
10273                         (struct pi_desc *)((void *)vmx->nested.pi_desc +
10274                         (unsigned long)(vmcs12->posted_intr_desc_addr &
10275                         (PAGE_SIZE - 1)));
10276                 vmcs_write64(POSTED_INTR_DESC_ADDR,
10277                         page_to_phys(vmx->nested.pi_desc_page) +
10278                         (unsigned long)(vmcs12->posted_intr_desc_addr &
10279                         (PAGE_SIZE - 1)));
10280         }
10281         if (cpu_has_vmx_msr_bitmap() &&
10282             nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS) &&
10283             nested_vmx_merge_msr_bitmap(vcpu, vmcs12))
10284                 vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
10285                               CPU_BASED_USE_MSR_BITMAPS);
10286         else
10287                 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
10288                                 CPU_BASED_USE_MSR_BITMAPS);
10289 }
10290
10291 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
10292 {
10293         u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
10294         struct vcpu_vmx *vmx = to_vmx(vcpu);
10295
10296         if (vcpu->arch.virtual_tsc_khz == 0)
10297                 return;
10298
10299         /* Make sure short timeouts reliably trigger an immediate vmexit.
10300          * hrtimer_start does not guarantee this. */
10301         if (preemption_timeout <= 1) {
10302                 vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
10303                 return;
10304         }
10305
10306         preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
10307         preemption_timeout *= 1000000;
10308         do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
10309         hrtimer_start(&vmx->nested.preemption_timer,
10310                       ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
10311 }
10312
10313 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu,
10314                                                struct vmcs12 *vmcs12)
10315 {
10316         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
10317                 return 0;
10318
10319         if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) ||
10320             !page_address_valid(vcpu, vmcs12->io_bitmap_b))
10321                 return -EINVAL;
10322
10323         return 0;
10324 }
10325
10326 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
10327                                                 struct vmcs12 *vmcs12)
10328 {
10329         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
10330                 return 0;
10331
10332         if (!page_address_valid(vcpu, vmcs12->msr_bitmap))
10333                 return -EINVAL;
10334
10335         return 0;
10336 }
10337
10338 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
10339                                                 struct vmcs12 *vmcs12)
10340 {
10341         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
10342                 return 0;
10343
10344         if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))
10345                 return -EINVAL;
10346
10347         return 0;
10348 }
10349
10350 /*
10351  * Merge L0's and L1's MSR bitmap, return false to indicate that
10352  * we do not use the hardware.
10353  */
10354 static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
10355                                                struct vmcs12 *vmcs12)
10356 {
10357         int msr;
10358         struct page *page;
10359         unsigned long *msr_bitmap_l1;
10360         unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
10361         /*
10362          * pred_cmd & spec_ctrl are trying to verify two things:
10363          *
10364          * 1. L0 gave a permission to L1 to actually passthrough the MSR. This
10365          *    ensures that we do not accidentally generate an L02 MSR bitmap
10366          *    from the L12 MSR bitmap that is too permissive.
10367          * 2. That L1 or L2s have actually used the MSR. This avoids
10368          *    unnecessarily merging of the bitmap if the MSR is unused. This
10369          *    works properly because we only update the L01 MSR bitmap lazily.
10370          *    So even if L0 should pass L1 these MSRs, the L01 bitmap is only
10371          *    updated to reflect this when L1 (or its L2s) actually write to
10372          *    the MSR.
10373          */
10374         bool pred_cmd = !msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD);
10375         bool spec_ctrl = !msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL);
10376
10377         if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
10378             !pred_cmd && !spec_ctrl)
10379                 return false;
10380
10381         page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->msr_bitmap);
10382         if (is_error_page(page))
10383                 return false;
10384         msr_bitmap_l1 = (unsigned long *)kmap(page);
10385
10386         memset(msr_bitmap_l0, 0xff, PAGE_SIZE);
10387
10388         if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
10389                 if (nested_cpu_has_apic_reg_virt(vmcs12))
10390                         for (msr = 0x800; msr <= 0x8ff; msr++)
10391                                 nested_vmx_disable_intercept_for_msr(
10392                                         msr_bitmap_l1, msr_bitmap_l0,
10393                                         msr, MSR_TYPE_R);
10394
10395                 nested_vmx_disable_intercept_for_msr(
10396                                 msr_bitmap_l1, msr_bitmap_l0,
10397                                 APIC_BASE_MSR + (APIC_TASKPRI >> 4),
10398                                 MSR_TYPE_R | MSR_TYPE_W);
10399
10400                 if (nested_cpu_has_vid(vmcs12)) {
10401                         nested_vmx_disable_intercept_for_msr(
10402                                 msr_bitmap_l1, msr_bitmap_l0,
10403                                 APIC_BASE_MSR + (APIC_EOI >> 4),
10404                                 MSR_TYPE_W);
10405                         nested_vmx_disable_intercept_for_msr(
10406                                 msr_bitmap_l1, msr_bitmap_l0,
10407                                 APIC_BASE_MSR + (APIC_SELF_IPI >> 4),
10408                                 MSR_TYPE_W);
10409                 }
10410         }
10411
10412         if (spec_ctrl)
10413                 nested_vmx_disable_intercept_for_msr(
10414                                         msr_bitmap_l1, msr_bitmap_l0,
10415                                         MSR_IA32_SPEC_CTRL,
10416                                         MSR_TYPE_R | MSR_TYPE_W);
10417
10418         if (pred_cmd)
10419                 nested_vmx_disable_intercept_for_msr(
10420                                         msr_bitmap_l1, msr_bitmap_l0,
10421                                         MSR_IA32_PRED_CMD,
10422                                         MSR_TYPE_W);
10423
10424         kunmap(page);
10425         kvm_release_page_clean(page);
10426
10427         return true;
10428 }
10429
10430 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu,
10431                                           struct vmcs12 *vmcs12)
10432 {
10433         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
10434             !page_address_valid(vcpu, vmcs12->apic_access_addr))
10435                 return -EINVAL;
10436         else
10437                 return 0;
10438 }
10439
10440 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
10441                                            struct vmcs12 *vmcs12)
10442 {
10443         if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
10444             !nested_cpu_has_apic_reg_virt(vmcs12) &&
10445             !nested_cpu_has_vid(vmcs12) &&
10446             !nested_cpu_has_posted_intr(vmcs12))
10447                 return 0;
10448
10449         /*
10450          * If virtualize x2apic mode is enabled,
10451          * virtualize apic access must be disabled.
10452          */
10453         if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&
10454             nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
10455                 return -EINVAL;
10456
10457         /*
10458          * If virtual interrupt delivery is enabled,
10459          * we must exit on external interrupts.
10460          */
10461         if (nested_cpu_has_vid(vmcs12) &&
10462            !nested_exit_on_intr(vcpu))
10463                 return -EINVAL;
10464
10465         /*
10466          * bits 15:8 should be zero in posted_intr_nv,
10467          * the descriptor address has been already checked
10468          * in nested_get_vmcs12_pages.
10469          */
10470         if (nested_cpu_has_posted_intr(vmcs12) &&
10471            (!nested_cpu_has_vid(vmcs12) ||
10472             !nested_exit_intr_ack_set(vcpu) ||
10473             vmcs12->posted_intr_nv & 0xff00))
10474                 return -EINVAL;
10475
10476         /* tpr shadow is needed by all apicv features. */
10477         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
10478                 return -EINVAL;
10479
10480         return 0;
10481 }
10482
10483 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
10484                                        unsigned long count_field,
10485                                        unsigned long addr_field)
10486 {
10487         int maxphyaddr;
10488         u64 count, addr;
10489
10490         if (vmcs12_read_any(vcpu, count_field, &count) ||
10491             vmcs12_read_any(vcpu, addr_field, &addr)) {
10492                 WARN_ON(1);
10493                 return -EINVAL;
10494         }
10495         if (count == 0)
10496                 return 0;
10497         maxphyaddr = cpuid_maxphyaddr(vcpu);
10498         if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
10499             (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) {
10500                 pr_debug_ratelimited(
10501                         "nVMX: invalid MSR switch (0x%lx, %d, %llu, 0x%08llx)",
10502                         addr_field, maxphyaddr, count, addr);
10503                 return -EINVAL;
10504         }
10505         return 0;
10506 }
10507
10508 static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu,
10509                                                 struct vmcs12 *vmcs12)
10510 {
10511         if (vmcs12->vm_exit_msr_load_count == 0 &&
10512             vmcs12->vm_exit_msr_store_count == 0 &&
10513             vmcs12->vm_entry_msr_load_count == 0)
10514                 return 0; /* Fast path */
10515         if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT,
10516                                         VM_EXIT_MSR_LOAD_ADDR) ||
10517             nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT,
10518                                         VM_EXIT_MSR_STORE_ADDR) ||
10519             nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT,
10520                                         VM_ENTRY_MSR_LOAD_ADDR))
10521                 return -EINVAL;
10522         return 0;
10523 }
10524
10525 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu,
10526                                          struct vmcs12 *vmcs12)
10527 {
10528         u64 address = vmcs12->pml_address;
10529         int maxphyaddr = cpuid_maxphyaddr(vcpu);
10530
10531         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_PML)) {
10532                 if (!nested_cpu_has_ept(vmcs12) ||
10533                     !IS_ALIGNED(address, 4096)  ||
10534                     address >> maxphyaddr)
10535                         return -EINVAL;
10536         }
10537
10538         return 0;
10539 }
10540
10541 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
10542                                        struct vmx_msr_entry *e)
10543 {
10544         /* x2APIC MSR accesses are not allowed */
10545         if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)
10546                 return -EINVAL;
10547         if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */
10548             e->index == MSR_IA32_UCODE_REV)
10549                 return -EINVAL;
10550         if (e->reserved != 0)
10551                 return -EINVAL;
10552         return 0;
10553 }
10554
10555 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
10556                                      struct vmx_msr_entry *e)
10557 {
10558         if (e->index == MSR_FS_BASE ||
10559             e->index == MSR_GS_BASE ||
10560             e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */
10561             nested_vmx_msr_check_common(vcpu, e))
10562                 return -EINVAL;
10563         return 0;
10564 }
10565
10566 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
10567                                       struct vmx_msr_entry *e)
10568 {
10569         if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */
10570             nested_vmx_msr_check_common(vcpu, e))
10571                 return -EINVAL;
10572         return 0;
10573 }
10574
10575 /*
10576  * Load guest's/host's msr at nested entry/exit.
10577  * return 0 for success, entry index for failure.
10578  */
10579 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
10580 {
10581         u32 i;
10582         struct vmx_msr_entry e;
10583         struct msr_data msr;
10584
10585         msr.host_initiated = false;
10586         for (i = 0; i < count; i++) {
10587                 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
10588                                         &e, sizeof(e))) {
10589                         pr_debug_ratelimited(
10590                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
10591                                 __func__, i, gpa + i * sizeof(e));
10592                         goto fail;
10593                 }
10594                 if (nested_vmx_load_msr_check(vcpu, &e)) {
10595                         pr_debug_ratelimited(
10596                                 "%s check failed (%u, 0x%x, 0x%x)\n",
10597                                 __func__, i, e.index, e.reserved);
10598                         goto fail;
10599                 }
10600                 msr.index = e.index;
10601                 msr.data = e.value;
10602                 if (kvm_set_msr(vcpu, &msr)) {
10603                         pr_debug_ratelimited(
10604                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
10605                                 __func__, i, e.index, e.value);
10606                         goto fail;
10607                 }
10608         }
10609         return 0;
10610 fail:
10611         return i + 1;
10612 }
10613
10614 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
10615 {
10616         u32 i;
10617         struct vmx_msr_entry e;
10618
10619         for (i = 0; i < count; i++) {
10620                 struct msr_data msr_info;
10621                 if (kvm_vcpu_read_guest(vcpu,
10622                                         gpa + i * sizeof(e),
10623                                         &e, 2 * sizeof(u32))) {
10624                         pr_debug_ratelimited(
10625                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
10626                                 __func__, i, gpa + i * sizeof(e));
10627                         return -EINVAL;
10628                 }
10629                 if (nested_vmx_store_msr_check(vcpu, &e)) {
10630                         pr_debug_ratelimited(
10631                                 "%s check failed (%u, 0x%x, 0x%x)\n",
10632                                 __func__, i, e.index, e.reserved);
10633                         return -EINVAL;
10634                 }
10635                 msr_info.host_initiated = false;
10636                 msr_info.index = e.index;
10637                 if (kvm_get_msr(vcpu, &msr_info)) {
10638                         pr_debug_ratelimited(
10639                                 "%s cannot read MSR (%u, 0x%x)\n",
10640                                 __func__, i, e.index);
10641                         return -EINVAL;
10642                 }
10643                 if (kvm_vcpu_write_guest(vcpu,
10644                                          gpa + i * sizeof(e) +
10645                                              offsetof(struct vmx_msr_entry, value),
10646                                          &msr_info.data, sizeof(msr_info.data))) {
10647                         pr_debug_ratelimited(
10648                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
10649                                 __func__, i, e.index, msr_info.data);
10650                         return -EINVAL;
10651                 }
10652         }
10653         return 0;
10654 }
10655
10656 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
10657 {
10658         unsigned long invalid_mask;
10659
10660         invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
10661         return (val & invalid_mask) == 0;
10662 }
10663
10664 /*
10665  * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are
10666  * emulating VM entry into a guest with EPT enabled.
10667  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
10668  * is assigned to entry_failure_code on failure.
10669  */
10670 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
10671                                u32 *entry_failure_code)
10672 {
10673         if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) {
10674                 if (!nested_cr3_valid(vcpu, cr3)) {
10675                         *entry_failure_code = ENTRY_FAIL_DEFAULT;
10676                         return 1;
10677                 }
10678
10679                 /*
10680                  * If PAE paging and EPT are both on, CR3 is not used by the CPU and
10681                  * must not be dereferenced.
10682                  */
10683                 if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) &&
10684                     !nested_ept) {
10685                         if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) {
10686                                 *entry_failure_code = ENTRY_FAIL_PDPTE;
10687                                 return 1;
10688                         }
10689                 }
10690
10691                 vcpu->arch.cr3 = cr3;
10692                 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
10693         }
10694
10695         kvm_mmu_reset_context(vcpu);
10696         return 0;
10697 }
10698
10699 /*
10700  * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
10701  * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
10702  * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
10703  * guest in a way that will both be appropriate to L1's requests, and our
10704  * needs. In addition to modifying the active vmcs (which is vmcs02), this
10705  * function also has additional necessary side-effects, like setting various
10706  * vcpu->arch fields.
10707  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
10708  * is assigned to entry_failure_code on failure.
10709  */
10710 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
10711                           bool from_vmentry, u32 *entry_failure_code)
10712 {
10713         struct vcpu_vmx *vmx = to_vmx(vcpu);
10714         u32 exec_control, vmcs12_exec_ctrl;
10715
10716         vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
10717         vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
10718         vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
10719         vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
10720         vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
10721         vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
10722         vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
10723         vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
10724         vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
10725         vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
10726         vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
10727         vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
10728         vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
10729         vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
10730         vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
10731         vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
10732         vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
10733         vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
10734         vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
10735         vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
10736         vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
10737         vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
10738         vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
10739         vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
10740         vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
10741         vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
10742         vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
10743         vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
10744         vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
10745         vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
10746         vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
10747         vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
10748         vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
10749         vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
10750         vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
10751         vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
10752
10753         if (from_vmentry &&
10754             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
10755                 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
10756                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
10757         } else {
10758                 kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
10759                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
10760         }
10761         if (from_vmentry) {
10762                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
10763                              vmcs12->vm_entry_intr_info_field);
10764                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
10765                              vmcs12->vm_entry_exception_error_code);
10766                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
10767                              vmcs12->vm_entry_instruction_len);
10768                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
10769                              vmcs12->guest_interruptibility_info);
10770                 vmx->loaded_vmcs->nmi_known_unmasked =
10771                         !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI);
10772         } else {
10773                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
10774         }
10775         vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
10776         vmx_set_rflags(vcpu, vmcs12->guest_rflags);
10777         vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
10778                 vmcs12->guest_pending_dbg_exceptions);
10779         vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
10780         vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
10781
10782         if (nested_cpu_has_xsaves(vmcs12))
10783                 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
10784         vmcs_write64(VMCS_LINK_POINTER, -1ull);
10785
10786         exec_control = vmcs12->pin_based_vm_exec_control;
10787
10788         /* Preemption timer setting is only taken from vmcs01.  */
10789         exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
10790         exec_control |= vmcs_config.pin_based_exec_ctrl;
10791         if (vmx->hv_deadline_tsc == -1)
10792                 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
10793
10794         /* Posted interrupts setting is only taken from vmcs12.  */
10795         if (nested_cpu_has_posted_intr(vmcs12)) {
10796                 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
10797                 vmx->nested.pi_pending = false;
10798                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
10799         } else {
10800                 exec_control &= ~PIN_BASED_POSTED_INTR;
10801         }
10802
10803         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control);
10804
10805         vmx->nested.preemption_timer_expired = false;
10806         if (nested_cpu_has_preemption_timer(vmcs12))
10807                 vmx_start_preemption_timer(vcpu);
10808
10809         /*
10810          * Whether page-faults are trapped is determined by a combination of
10811          * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
10812          * If enable_ept, L0 doesn't care about page faults and we should
10813          * set all of these to L1's desires. However, if !enable_ept, L0 does
10814          * care about (at least some) page faults, and because it is not easy
10815          * (if at all possible?) to merge L0 and L1's desires, we simply ask
10816          * to exit on each and every L2 page fault. This is done by setting
10817          * MASK=MATCH=0 and (see below) EB.PF=1.
10818          * Note that below we don't need special code to set EB.PF beyond the
10819          * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
10820          * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
10821          * !enable_ept, EB.PF is 1, so the "or" will always be 1.
10822          */
10823         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
10824                 enable_ept ? vmcs12->page_fault_error_code_mask : 0);
10825         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
10826                 enable_ept ? vmcs12->page_fault_error_code_match : 0);
10827
10828         if (cpu_has_secondary_exec_ctrls()) {
10829                 exec_control = vmx->secondary_exec_control;
10830
10831                 /* Take the following fields only from vmcs12 */
10832                 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
10833                                   SECONDARY_EXEC_ENABLE_INVPCID |
10834                                   SECONDARY_EXEC_RDTSCP |
10835                                   SECONDARY_EXEC_XSAVES |
10836                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
10837                                   SECONDARY_EXEC_APIC_REGISTER_VIRT |
10838                                   SECONDARY_EXEC_ENABLE_VMFUNC);
10839                 if (nested_cpu_has(vmcs12,
10840                                    CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) {
10841                         vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control &
10842                                 ~SECONDARY_EXEC_ENABLE_PML;
10843                         exec_control |= vmcs12_exec_ctrl;
10844                 }
10845
10846                 /* All VMFUNCs are currently emulated through L0 vmexits.  */
10847                 if (exec_control & SECONDARY_EXEC_ENABLE_VMFUNC)
10848                         vmcs_write64(VM_FUNCTION_CONTROL, 0);
10849
10850                 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
10851                         vmcs_write64(EOI_EXIT_BITMAP0,
10852                                 vmcs12->eoi_exit_bitmap0);
10853                         vmcs_write64(EOI_EXIT_BITMAP1,
10854                                 vmcs12->eoi_exit_bitmap1);
10855                         vmcs_write64(EOI_EXIT_BITMAP2,
10856                                 vmcs12->eoi_exit_bitmap2);
10857                         vmcs_write64(EOI_EXIT_BITMAP3,
10858                                 vmcs12->eoi_exit_bitmap3);
10859                         vmcs_write16(GUEST_INTR_STATUS,
10860                                 vmcs12->guest_intr_status);
10861                 }
10862
10863                 /*
10864                  * Write an illegal value to APIC_ACCESS_ADDR. Later,
10865                  * nested_get_vmcs12_pages will either fix it up or
10866                  * remove the VM execution control.
10867                  */
10868                 if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)
10869                         vmcs_write64(APIC_ACCESS_ADDR, -1ull);
10870
10871                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
10872         }
10873
10874
10875         /*
10876          * Set host-state according to L0's settings (vmcs12 is irrelevant here)
10877          * Some constant fields are set here by vmx_set_constant_host_state().
10878          * Other fields are different per CPU, and will be set later when
10879          * vmx_vcpu_load() is called, and when vmx_save_host_state() is called.
10880          */
10881         vmx_set_constant_host_state(vmx);
10882
10883         /*
10884          * Set the MSR load/store lists to match L0's settings.
10885          */
10886         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
10887         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
10888         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
10889         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
10890         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
10891
10892         /*
10893          * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
10894          * entry, but only if the current (host) sp changed from the value
10895          * we wrote last (vmx->host_rsp). This cache is no longer relevant
10896          * if we switch vmcs, and rather than hold a separate cache per vmcs,
10897          * here we just force the write to happen on entry.
10898          */
10899         vmx->host_rsp = 0;
10900
10901         exec_control = vmx_exec_control(vmx); /* L0's desires */
10902         exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
10903         exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
10904         exec_control &= ~CPU_BASED_TPR_SHADOW;
10905         exec_control |= vmcs12->cpu_based_vm_exec_control;
10906
10907         /*
10908          * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR. Later, if
10909          * nested_get_vmcs12_pages can't fix it up, the illegal value
10910          * will result in a VM entry failure.
10911          */
10912         if (exec_control & CPU_BASED_TPR_SHADOW) {
10913                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull);
10914                 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
10915         } else {
10916 #ifdef CONFIG_X86_64
10917                 exec_control |= CPU_BASED_CR8_LOAD_EXITING |
10918                                 CPU_BASED_CR8_STORE_EXITING;
10919 #endif
10920         }
10921
10922         /*
10923          * Merging of IO bitmap not currently supported.
10924          * Rather, exit every time.
10925          */
10926         exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
10927         exec_control |= CPU_BASED_UNCOND_IO_EXITING;
10928
10929         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
10930
10931         /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
10932          * bitwise-or of what L1 wants to trap for L2, and what we want to
10933          * trap. Note that CR0.TS also needs updating - we do this later.
10934          */
10935         update_exception_bitmap(vcpu);
10936         vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
10937         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
10938
10939         /* L2->L1 exit controls are emulated - the hardware exit is to L0 so
10940          * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
10941          * bits are further modified by vmx_set_efer() below.
10942          */
10943         vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
10944
10945         /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are
10946          * emulated by vmx_set_efer(), below.
10947          */
10948         vm_entry_controls_init(vmx,
10949                 (vmcs12->vm_entry_controls & ~VM_ENTRY_LOAD_IA32_EFER &
10950                         ~VM_ENTRY_IA32E_MODE) |
10951                 (vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));
10952
10953         if (from_vmentry &&
10954             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
10955                 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
10956                 vcpu->arch.pat = vmcs12->guest_ia32_pat;
10957         } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
10958                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
10959         }
10960
10961         set_cr4_guest_host_mask(vmx);
10962
10963         if (from_vmentry &&
10964             vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)
10965                 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
10966
10967         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
10968                 vmcs_write64(TSC_OFFSET,
10969                         vcpu->arch.tsc_offset + vmcs12->tsc_offset);
10970         else
10971                 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
10972         if (kvm_has_tsc_control)
10973                 decache_tsc_multiplier(vmx);
10974
10975         if (cpu_has_vmx_msr_bitmap())
10976                 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
10977
10978         if (enable_vpid) {
10979                 /*
10980                  * There is no direct mapping between vpid02 and vpid12, the
10981                  * vpid02 is per-vCPU for L0 and reused while the value of
10982                  * vpid12 is changed w/ one invvpid during nested vmentry.
10983                  * The vpid12 is allocated by L1 for L2, so it will not
10984                  * influence global bitmap(for vpid01 and vpid02 allocation)
10985                  * even if spawn a lot of nested vCPUs.
10986                  */
10987                 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) {
10988                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
10989                         if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
10990                                 vmx->nested.last_vpid = vmcs12->virtual_processor_id;
10991                                 __vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02);
10992                         }
10993                 } else {
10994                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
10995                         vmx_flush_tlb(vcpu);
10996                 }
10997
10998         }
10999
11000         if (enable_pml) {
11001                 /*
11002                  * Conceptually we want to copy the PML address and index from
11003                  * vmcs01 here, and then back to vmcs01 on nested vmexit. But,
11004                  * since we always flush the log on each vmexit, this happens
11005                  * to be equivalent to simply resetting the fields in vmcs02.
11006                  */
11007                 ASSERT(vmx->pml_pg);
11008                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
11009                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
11010         }
11011
11012         if (nested_cpu_has_ept(vmcs12)) {
11013                 if (nested_ept_init_mmu_context(vcpu)) {
11014                         *entry_failure_code = ENTRY_FAIL_DEFAULT;
11015                         return 1;
11016                 }
11017         } else if (nested_cpu_has2(vmcs12,
11018                                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
11019                 vmx_flush_tlb_ept_only(vcpu);
11020         }
11021
11022         /*
11023          * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those
11024          * bits which we consider mandatory enabled.
11025          * The CR0_READ_SHADOW is what L2 should have expected to read given
11026          * the specifications by L1; It's not enough to take
11027          * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
11028          * have more bits than L1 expected.
11029          */
11030         vmx_set_cr0(vcpu, vmcs12->guest_cr0);
11031         vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
11032
11033         vmx_set_cr4(vcpu, vmcs12->guest_cr4);
11034         vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
11035
11036         if (from_vmentry &&
11037             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
11038                 vcpu->arch.efer = vmcs12->guest_ia32_efer;
11039         else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
11040                 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
11041         else
11042                 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
11043         /* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
11044         vmx_set_efer(vcpu, vcpu->arch.efer);
11045
11046         /* Shadow page tables on either EPT or shadow page tables. */
11047         if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12),
11048                                 entry_failure_code))
11049                 return 1;
11050
11051         if (!enable_ept)
11052                 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
11053
11054         /*
11055          * L1 may access the L2's PDPTR, so save them to construct vmcs12
11056          */
11057         if (enable_ept) {
11058                 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
11059                 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
11060                 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
11061                 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
11062         }
11063
11064         kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
11065         kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
11066         return 0;
11067 }
11068
11069 static int check_vmentry_prereqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
11070 {
11071         struct vcpu_vmx *vmx = to_vmx(vcpu);
11072
11073         if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
11074             vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT)
11075                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11076
11077         if (nested_vmx_check_io_bitmap_controls(vcpu, vmcs12))
11078                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11079
11080         if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12))
11081                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11082
11083         if (nested_vmx_check_apic_access_controls(vcpu, vmcs12))
11084                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11085
11086         if (nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12))
11087                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11088
11089         if (nested_vmx_check_apicv_controls(vcpu, vmcs12))
11090                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11091
11092         if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12))
11093                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11094
11095         if (nested_vmx_check_pml_controls(vcpu, vmcs12))
11096                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11097
11098         if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
11099                                 vmx->nested.nested_vmx_procbased_ctls_low,
11100                                 vmx->nested.nested_vmx_procbased_ctls_high) ||
11101             (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
11102              !vmx_control_verify(vmcs12->secondary_vm_exec_control,
11103                                  vmx->nested.nested_vmx_secondary_ctls_low,
11104                                  vmx->nested.nested_vmx_secondary_ctls_high)) ||
11105             !vmx_control_verify(vmcs12->pin_based_vm_exec_control,
11106                                 vmx->nested.nested_vmx_pinbased_ctls_low,
11107                                 vmx->nested.nested_vmx_pinbased_ctls_high) ||
11108             !vmx_control_verify(vmcs12->vm_exit_controls,
11109                                 vmx->nested.nested_vmx_exit_ctls_low,
11110                                 vmx->nested.nested_vmx_exit_ctls_high) ||
11111             !vmx_control_verify(vmcs12->vm_entry_controls,
11112                                 vmx->nested.nested_vmx_entry_ctls_low,
11113                                 vmx->nested.nested_vmx_entry_ctls_high))
11114                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11115
11116         if (nested_cpu_has_vmfunc(vmcs12)) {
11117                 if (vmcs12->vm_function_control &
11118                     ~vmx->nested.nested_vmx_vmfunc_controls)
11119                         return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11120
11121                 if (nested_cpu_has_eptp_switching(vmcs12)) {
11122                         if (!nested_cpu_has_ept(vmcs12) ||
11123                             !page_address_valid(vcpu, vmcs12->eptp_list_address))
11124                                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11125                 }
11126         }
11127
11128         if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu))
11129                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
11130
11131         if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) ||
11132             !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) ||
11133             !nested_cr3_valid(vcpu, vmcs12->host_cr3))
11134                 return VMXERR_ENTRY_INVALID_HOST_STATE_FIELD;
11135
11136         return 0;
11137 }
11138
11139 static int check_vmentry_postreqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
11140                                   u32 *exit_qual)
11141 {
11142         bool ia32e;
11143
11144         *exit_qual = ENTRY_FAIL_DEFAULT;
11145
11146         if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) ||
11147             !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))
11148                 return 1;
11149
11150         if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_SHADOW_VMCS) &&
11151             vmcs12->vmcs_link_pointer != -1ull) {
11152                 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR;
11153                 return 1;
11154         }
11155
11156         /*
11157          * If the load IA32_EFER VM-entry control is 1, the following checks
11158          * are performed on the field for the IA32_EFER MSR:
11159          * - Bits reserved in the IA32_EFER MSR must be 0.
11160          * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
11161          *   the IA-32e mode guest VM-exit control. It must also be identical
11162          *   to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
11163          *   CR0.PG) is 1.
11164          */
11165         if (to_vmx(vcpu)->nested.nested_run_pending &&
11166             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
11167                 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
11168                 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) ||
11169                     ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) ||
11170                     ((vmcs12->guest_cr0 & X86_CR0_PG) &&
11171                      ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))
11172                         return 1;
11173         }
11174
11175         /*
11176          * If the load IA32_EFER VM-exit control is 1, bits reserved in the
11177          * IA32_EFER MSR must be 0 in the field for that register. In addition,
11178          * the values of the LMA and LME bits in the field must each be that of
11179          * the host address-space size VM-exit control.
11180          */
11181         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
11182                 ia32e = (vmcs12->vm_exit_controls &
11183                          VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
11184                 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) ||
11185                     ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) ||
11186                     ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))
11187                         return 1;
11188         }
11189
11190         return 0;
11191 }
11192
11193 static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry)
11194 {
11195         struct vcpu_vmx *vmx = to_vmx(vcpu);
11196         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
11197         u32 msr_entry_idx;
11198         u32 exit_qual;
11199
11200         enter_guest_mode(vcpu);
11201
11202         if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
11203                 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
11204
11205         vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
11206         vmx_segment_cache_clear(vmx);
11207
11208         if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &exit_qual)) {
11209                 leave_guest_mode(vcpu);
11210                 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
11211                 nested_vmx_entry_failure(vcpu, vmcs12,
11212                                          EXIT_REASON_INVALID_STATE, exit_qual);
11213                 return 1;
11214         }
11215
11216         nested_get_vmcs12_pages(vcpu, vmcs12);
11217
11218         msr_entry_idx = nested_vmx_load_msr(vcpu,
11219                                             vmcs12->vm_entry_msr_load_addr,
11220                                             vmcs12->vm_entry_msr_load_count);
11221         if (msr_entry_idx) {
11222                 leave_guest_mode(vcpu);
11223                 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
11224                 nested_vmx_entry_failure(vcpu, vmcs12,
11225                                 EXIT_REASON_MSR_LOAD_FAIL, msr_entry_idx);
11226                 return 1;
11227         }
11228
11229         /*
11230          * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
11231          * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
11232          * returned as far as L1 is concerned. It will only return (and set
11233          * the success flag) when L2 exits (see nested_vmx_vmexit()).
11234          */
11235         return 0;
11236 }
11237
11238 /*
11239  * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
11240  * for running an L2 nested guest.
11241  */
11242 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
11243 {
11244         struct vmcs12 *vmcs12;
11245         struct vcpu_vmx *vmx = to_vmx(vcpu);
11246         u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu);
11247         u32 exit_qual;
11248         int ret;
11249
11250         if (!nested_vmx_check_permission(vcpu))
11251                 return 1;
11252
11253         if (!nested_vmx_check_vmcs12(vcpu))
11254                 goto out;
11255
11256         vmcs12 = get_vmcs12(vcpu);
11257
11258         if (enable_shadow_vmcs)
11259                 copy_shadow_to_vmcs12(vmx);
11260
11261         /*
11262          * The nested entry process starts with enforcing various prerequisites
11263          * on vmcs12 as required by the Intel SDM, and act appropriately when
11264          * they fail: As the SDM explains, some conditions should cause the
11265          * instruction to fail, while others will cause the instruction to seem
11266          * to succeed, but return an EXIT_REASON_INVALID_STATE.
11267          * To speed up the normal (success) code path, we should avoid checking
11268          * for misconfigurations which will anyway be caught by the processor
11269          * when using the merged vmcs02.
11270          */
11271         if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS) {
11272                 nested_vmx_failValid(vcpu,
11273                                      VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS);
11274                 goto out;
11275         }
11276
11277         if (vmcs12->launch_state == launch) {
11278                 nested_vmx_failValid(vcpu,
11279                         launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
11280                                : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
11281                 goto out;
11282         }
11283
11284         ret = check_vmentry_prereqs(vcpu, vmcs12);
11285         if (ret) {
11286                 nested_vmx_failValid(vcpu, ret);
11287                 goto out;
11288         }
11289
11290         /*
11291          * After this point, the trap flag no longer triggers a singlestep trap
11292          * on the vm entry instructions; don't call kvm_skip_emulated_instruction.
11293          * This is not 100% correct; for performance reasons, we delegate most
11294          * of the checks on host state to the processor.  If those fail,
11295          * the singlestep trap is missed.
11296          */
11297         skip_emulated_instruction(vcpu);
11298
11299         ret = check_vmentry_postreqs(vcpu, vmcs12, &exit_qual);
11300         if (ret) {
11301                 nested_vmx_entry_failure(vcpu, vmcs12,
11302                                          EXIT_REASON_INVALID_STATE, exit_qual);
11303                 return 1;
11304         }
11305
11306         /*
11307          * We're finally done with prerequisite checking, and can start with
11308          * the nested entry.
11309          */
11310
11311         ret = enter_vmx_non_root_mode(vcpu, true);
11312         if (ret)
11313                 return ret;
11314
11315         /*
11316          * If we're entering a halted L2 vcpu and the L2 vcpu won't be woken
11317          * by event injection, halt vcpu.
11318          */
11319         if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
11320             !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK))
11321                 return kvm_vcpu_halt(vcpu);
11322
11323         vmx->nested.nested_run_pending = 1;
11324
11325         return 1;
11326
11327 out:
11328         return kvm_skip_emulated_instruction(vcpu);
11329 }
11330
11331 /*
11332  * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
11333  * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
11334  * This function returns the new value we should put in vmcs12.guest_cr0.
11335  * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
11336  *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
11337  *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
11338  *     didn't trap the bit, because if L1 did, so would L0).
11339  *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
11340  *     been modified by L2, and L1 knows it. So just leave the old value of
11341  *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
11342  *     isn't relevant, because if L0 traps this bit it can set it to anything.
11343  *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
11344  *     changed these bits, and therefore they need to be updated, but L0
11345  *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
11346  *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
11347  */
11348 static inline unsigned long
11349 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
11350 {
11351         return
11352         /*1*/   (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
11353         /*2*/   (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
11354         /*3*/   (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
11355                         vcpu->arch.cr0_guest_owned_bits));
11356 }
11357
11358 static inline unsigned long
11359 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
11360 {
11361         return
11362         /*1*/   (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
11363         /*2*/   (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
11364         /*3*/   (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
11365                         vcpu->arch.cr4_guest_owned_bits));
11366 }
11367
11368 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
11369                                        struct vmcs12 *vmcs12)
11370 {
11371         u32 idt_vectoring;
11372         unsigned int nr;
11373
11374         if (vcpu->arch.exception.injected) {
11375                 nr = vcpu->arch.exception.nr;
11376                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
11377
11378                 if (kvm_exception_is_soft(nr)) {
11379                         vmcs12->vm_exit_instruction_len =
11380                                 vcpu->arch.event_exit_inst_len;
11381                         idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
11382                 } else
11383                         idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
11384
11385                 if (vcpu->arch.exception.has_error_code) {
11386                         idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
11387                         vmcs12->idt_vectoring_error_code =
11388                                 vcpu->arch.exception.error_code;
11389                 }
11390
11391                 vmcs12->idt_vectoring_info_field = idt_vectoring;
11392         } else if (vcpu->arch.nmi_injected) {
11393                 vmcs12->idt_vectoring_info_field =
11394                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
11395         } else if (vcpu->arch.interrupt.pending) {
11396                 nr = vcpu->arch.interrupt.nr;
11397                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
11398
11399                 if (vcpu->arch.interrupt.soft) {
11400                         idt_vectoring |= INTR_TYPE_SOFT_INTR;
11401                         vmcs12->vm_entry_instruction_len =
11402                                 vcpu->arch.event_exit_inst_len;
11403                 } else
11404                         idt_vectoring |= INTR_TYPE_EXT_INTR;
11405
11406                 vmcs12->idt_vectoring_info_field = idt_vectoring;
11407         }
11408 }
11409
11410 static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
11411 {
11412         struct vcpu_vmx *vmx = to_vmx(vcpu);
11413         unsigned long exit_qual;
11414         bool block_nested_events =
11415             vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
11416
11417         if (vcpu->arch.exception.pending &&
11418                 nested_vmx_check_exception(vcpu, &exit_qual)) {
11419                 if (block_nested_events)
11420                         return -EBUSY;
11421                 nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
11422                 return 0;
11423         }
11424
11425         if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
11426             vmx->nested.preemption_timer_expired) {
11427                 if (block_nested_events)
11428                         return -EBUSY;
11429                 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
11430                 return 0;
11431         }
11432
11433         if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
11434                 if (block_nested_events)
11435                         return -EBUSY;
11436                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
11437                                   NMI_VECTOR | INTR_TYPE_NMI_INTR |
11438                                   INTR_INFO_VALID_MASK, 0);
11439                 /*
11440                  * The NMI-triggered VM exit counts as injection:
11441                  * clear this one and block further NMIs.
11442                  */
11443                 vcpu->arch.nmi_pending = 0;
11444                 vmx_set_nmi_mask(vcpu, true);
11445                 return 0;
11446         }
11447
11448         if ((kvm_cpu_has_interrupt(vcpu) || external_intr) &&
11449             nested_exit_on_intr(vcpu)) {
11450                 if (block_nested_events)
11451                         return -EBUSY;
11452                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
11453                 return 0;
11454         }
11455
11456         vmx_complete_nested_posted_interrupt(vcpu);
11457         return 0;
11458 }
11459
11460 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
11461 {
11462         ktime_t remaining =
11463                 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
11464         u64 value;
11465
11466         if (ktime_to_ns(remaining) <= 0)
11467                 return 0;
11468
11469         value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
11470         do_div(value, 1000000);
11471         return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
11472 }
11473
11474 /*
11475  * Update the guest state fields of vmcs12 to reflect changes that
11476  * occurred while L2 was running. (The "IA-32e mode guest" bit of the
11477  * VM-entry controls is also updated, since this is really a guest
11478  * state bit.)
11479  */
11480 static void sync_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
11481 {
11482         vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
11483         vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
11484
11485         vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
11486         vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP);
11487         vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
11488
11489         vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
11490         vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
11491         vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
11492         vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
11493         vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
11494         vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
11495         vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
11496         vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
11497         vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
11498         vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
11499         vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
11500         vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
11501         vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
11502         vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
11503         vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
11504         vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
11505         vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
11506         vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
11507         vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
11508         vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
11509         vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
11510         vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
11511         vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
11512         vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
11513         vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
11514         vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
11515         vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
11516         vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
11517         vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
11518         vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
11519         vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
11520         vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
11521         vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
11522         vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
11523         vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
11524         vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
11525
11526         vmcs12->guest_interruptibility_info =
11527                 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
11528         vmcs12->guest_pending_dbg_exceptions =
11529                 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
11530         if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
11531                 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
11532         else
11533                 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
11534
11535         if (nested_cpu_has_preemption_timer(vmcs12)) {
11536                 if (vmcs12->vm_exit_controls &
11537                     VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
11538                         vmcs12->vmx_preemption_timer_value =
11539                                 vmx_get_preemption_timer_value(vcpu);
11540                 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
11541         }
11542
11543         /*
11544          * In some cases (usually, nested EPT), L2 is allowed to change its
11545          * own CR3 without exiting. If it has changed it, we must keep it.
11546          * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
11547          * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
11548          *
11549          * Additionally, restore L2's PDPTR to vmcs12.
11550          */
11551         if (enable_ept) {
11552                 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
11553                 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
11554                 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
11555                 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
11556                 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
11557         }
11558
11559         vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
11560
11561         if (nested_cpu_has_vid(vmcs12))
11562                 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
11563
11564         vmcs12->vm_entry_controls =
11565                 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
11566                 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
11567
11568         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) {
11569                 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
11570                 vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
11571         }
11572
11573         /* TODO: These cannot have changed unless we have MSR bitmaps and
11574          * the relevant bit asks not to trap the change */
11575         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
11576                 vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT);
11577         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
11578                 vmcs12->guest_ia32_efer = vcpu->arch.efer;
11579         vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
11580         vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
11581         vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
11582         if (kvm_mpx_supported())
11583                 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
11584 }
11585
11586 /*
11587  * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
11588  * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
11589  * and this function updates it to reflect the changes to the guest state while
11590  * L2 was running (and perhaps made some exits which were handled directly by L0
11591  * without going back to L1), and to reflect the exit reason.
11592  * Note that we do not have to copy here all VMCS fields, just those that
11593  * could have changed by the L2 guest or the exit - i.e., the guest-state and
11594  * exit-information fields only. Other fields are modified by L1 with VMWRITE,
11595  * which already writes to vmcs12 directly.
11596  */
11597 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
11598                            u32 exit_reason, u32 exit_intr_info,
11599                            unsigned long exit_qualification)
11600 {
11601         /* update guest state fields: */
11602         sync_vmcs12(vcpu, vmcs12);
11603
11604         /* update exit information fields: */
11605
11606         vmcs12->vm_exit_reason = exit_reason;
11607         vmcs12->exit_qualification = exit_qualification;
11608         vmcs12->vm_exit_intr_info = exit_intr_info;
11609
11610         vmcs12->idt_vectoring_info_field = 0;
11611         vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
11612         vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
11613
11614         if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
11615                 vmcs12->launch_state = 1;
11616
11617                 /* vm_entry_intr_info_field is cleared on exit. Emulate this
11618                  * instead of reading the real value. */
11619                 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
11620
11621                 /*
11622                  * Transfer the event that L0 or L1 may wanted to inject into
11623                  * L2 to IDT_VECTORING_INFO_FIELD.
11624                  */
11625                 vmcs12_save_pending_event(vcpu, vmcs12);
11626         }
11627
11628         /*
11629          * Drop what we picked up for L2 via vmx_complete_interrupts. It is
11630          * preserved above and would only end up incorrectly in L1.
11631          */
11632         vcpu->arch.nmi_injected = false;
11633         kvm_clear_exception_queue(vcpu);
11634         kvm_clear_interrupt_queue(vcpu);
11635 }
11636
11637 static void load_vmcs12_mmu_host_state(struct kvm_vcpu *vcpu,
11638                         struct vmcs12 *vmcs12)
11639 {
11640         u32 entry_failure_code;
11641
11642         nested_ept_uninit_mmu_context(vcpu);
11643
11644         /*
11645          * Only PDPTE load can fail as the value of cr3 was checked on entry and
11646          * couldn't have changed.
11647          */
11648         if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code))
11649                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
11650
11651         if (!enable_ept)
11652                 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
11653 }
11654
11655 /*
11656  * A part of what we need to when the nested L2 guest exits and we want to
11657  * run its L1 parent, is to reset L1's guest state to the host state specified
11658  * in vmcs12.
11659  * This function is to be called not only on normal nested exit, but also on
11660  * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
11661  * Failures During or After Loading Guest State").
11662  * This function should be called when the active VMCS is L1's (vmcs01).
11663  */
11664 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
11665                                    struct vmcs12 *vmcs12)
11666 {
11667         struct kvm_segment seg;
11668
11669         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
11670                 vcpu->arch.efer = vmcs12->host_ia32_efer;
11671         else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
11672                 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
11673         else
11674                 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
11675         vmx_set_efer(vcpu, vcpu->arch.efer);
11676
11677         kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp);
11678         kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip);
11679         vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
11680         /*
11681          * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
11682          * actually changed, because vmx_set_cr0 refers to efer set above.
11683          *
11684          * CR0_GUEST_HOST_MASK is already set in the original vmcs01
11685          * (KVM doesn't change it);
11686          */
11687         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
11688         vmx_set_cr0(vcpu, vmcs12->host_cr0);
11689
11690         /* Same as above - no reason to call set_cr4_guest_host_mask().  */
11691         vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
11692         vmx_set_cr4(vcpu, vmcs12->host_cr4);
11693
11694         load_vmcs12_mmu_host_state(vcpu, vmcs12);
11695
11696         if (enable_vpid) {
11697                 /*
11698                  * Trivially support vpid by letting L2s share their parent
11699                  * L1's vpid. TODO: move to a more elaborate solution, giving
11700                  * each L2 its own vpid and exposing the vpid feature to L1.
11701                  */
11702                 vmx_flush_tlb(vcpu);
11703         }
11704         /* Restore posted intr vector. */
11705         if (nested_cpu_has_posted_intr(vmcs12))
11706                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
11707
11708         vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
11709         vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
11710         vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
11711         vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
11712         vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
11713         vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF);
11714         vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);
11715
11716         /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1.  */
11717         if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
11718                 vmcs_write64(GUEST_BNDCFGS, 0);
11719
11720         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
11721                 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
11722                 vcpu->arch.pat = vmcs12->host_ia32_pat;
11723         }
11724         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
11725                 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
11726                         vmcs12->host_ia32_perf_global_ctrl);
11727
11728         /* Set L1 segment info according to Intel SDM
11729             27.5.2 Loading Host Segment and Descriptor-Table Registers */
11730         seg = (struct kvm_segment) {
11731                 .base = 0,
11732                 .limit = 0xFFFFFFFF,
11733                 .selector = vmcs12->host_cs_selector,
11734                 .type = 11,
11735                 .present = 1,
11736                 .s = 1,
11737                 .g = 1
11738         };
11739         if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
11740                 seg.l = 1;
11741         else
11742                 seg.db = 1;
11743         vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
11744         seg = (struct kvm_segment) {
11745                 .base = 0,
11746                 .limit = 0xFFFFFFFF,
11747                 .type = 3,
11748                 .present = 1,
11749                 .s = 1,
11750                 .db = 1,
11751                 .g = 1
11752         };
11753         seg.selector = vmcs12->host_ds_selector;
11754         vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
11755         seg.selector = vmcs12->host_es_selector;
11756         vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
11757         seg.selector = vmcs12->host_ss_selector;
11758         vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
11759         seg.selector = vmcs12->host_fs_selector;
11760         seg.base = vmcs12->host_fs_base;
11761         vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
11762         seg.selector = vmcs12->host_gs_selector;
11763         seg.base = vmcs12->host_gs_base;
11764         vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
11765         seg = (struct kvm_segment) {
11766                 .base = vmcs12->host_tr_base,
11767                 .limit = 0x67,
11768                 .selector = vmcs12->host_tr_selector,
11769                 .type = 11,
11770                 .present = 1
11771         };
11772         vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
11773
11774         kvm_set_dr(vcpu, 7, 0x400);
11775         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
11776
11777         if (cpu_has_vmx_msr_bitmap())
11778                 vmx_update_msr_bitmap(vcpu);
11779
11780         if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
11781                                 vmcs12->vm_exit_msr_load_count))
11782                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
11783 }
11784
11785 /*
11786  * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
11787  * and modify vmcs12 to make it see what it would expect to see there if
11788  * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
11789  */
11790 static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
11791                               u32 exit_intr_info,
11792                               unsigned long exit_qualification)
11793 {
11794         struct vcpu_vmx *vmx = to_vmx(vcpu);
11795         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
11796
11797         /* trying to cancel vmlaunch/vmresume is a bug */
11798         WARN_ON_ONCE(vmx->nested.nested_run_pending);
11799
11800         /*
11801          * The only expected VM-instruction error is "VM entry with
11802          * invalid control field(s)." Anything else indicates a
11803          * problem with L0.
11804          */
11805         WARN_ON_ONCE(vmx->fail && (vmcs_read32(VM_INSTRUCTION_ERROR) !=
11806                                    VMXERR_ENTRY_INVALID_CONTROL_FIELD));
11807
11808         leave_guest_mode(vcpu);
11809
11810         if (likely(!vmx->fail)) {
11811                 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
11812                                exit_qualification);
11813
11814                 if (nested_vmx_store_msr(vcpu, vmcs12->vm_exit_msr_store_addr,
11815                                          vmcs12->vm_exit_msr_store_count))
11816                         nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL);
11817         }
11818
11819         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
11820         vm_entry_controls_reset_shadow(vmx);
11821         vm_exit_controls_reset_shadow(vmx);
11822         vmx_segment_cache_clear(vmx);
11823
11824         /* Update any VMCS fields that might have changed while L2 ran */
11825         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
11826         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
11827         vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
11828         if (vmx->hv_deadline_tsc == -1)
11829                 vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
11830                                 PIN_BASED_VMX_PREEMPTION_TIMER);
11831         else
11832                 vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
11833                               PIN_BASED_VMX_PREEMPTION_TIMER);
11834         if (kvm_has_tsc_control)
11835                 decache_tsc_multiplier(vmx);
11836
11837         if (vmx->nested.change_vmcs01_virtual_x2apic_mode) {
11838                 vmx->nested.change_vmcs01_virtual_x2apic_mode = false;
11839                 vmx_set_virtual_x2apic_mode(vcpu,
11840                                 vcpu->arch.apic_base & X2APIC_ENABLE);
11841         } else if (!nested_cpu_has_ept(vmcs12) &&
11842                    nested_cpu_has2(vmcs12,
11843                                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
11844                 vmx_flush_tlb_ept_only(vcpu);
11845         }
11846
11847         /* This is needed for same reason as it was needed in prepare_vmcs02 */
11848         vmx->host_rsp = 0;
11849
11850         /* Unpin physical memory we referred to in vmcs02 */
11851         if (vmx->nested.apic_access_page) {
11852                 kvm_release_page_dirty(vmx->nested.apic_access_page);
11853                 vmx->nested.apic_access_page = NULL;
11854         }
11855         if (vmx->nested.virtual_apic_page) {
11856                 kvm_release_page_dirty(vmx->nested.virtual_apic_page);
11857                 vmx->nested.virtual_apic_page = NULL;
11858         }
11859         if (vmx->nested.pi_desc_page) {
11860                 kunmap(vmx->nested.pi_desc_page);
11861                 kvm_release_page_dirty(vmx->nested.pi_desc_page);
11862                 vmx->nested.pi_desc_page = NULL;
11863                 vmx->nested.pi_desc = NULL;
11864         }
11865
11866         /*
11867          * We are now running in L2, mmu_notifier will force to reload the
11868          * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1.
11869          */
11870         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
11871
11872         if (enable_shadow_vmcs)
11873                 vmx->nested.sync_shadow_vmcs = true;
11874
11875         /* in case we halted in L2 */
11876         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11877
11878         if (likely(!vmx->fail)) {
11879                 /*
11880                  * TODO: SDM says that with acknowledge interrupt on
11881                  * exit, bit 31 of the VM-exit interrupt information
11882                  * (valid interrupt) is always set to 1 on
11883                  * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't
11884                  * need kvm_cpu_has_interrupt().  See the commit
11885                  * message for details.
11886                  */
11887                 if (nested_exit_intr_ack_set(vcpu) &&
11888                     exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT &&
11889                     kvm_cpu_has_interrupt(vcpu)) {
11890                         int irq = kvm_cpu_get_interrupt(vcpu);
11891                         WARN_ON(irq < 0);
11892                         vmcs12->vm_exit_intr_info = irq |
11893                                 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
11894                 }
11895
11896                 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
11897                                                vmcs12->exit_qualification,
11898                                                vmcs12->idt_vectoring_info_field,
11899                                                vmcs12->vm_exit_intr_info,
11900                                                vmcs12->vm_exit_intr_error_code,
11901                                                KVM_ISA_VMX);
11902
11903                 load_vmcs12_host_state(vcpu, vmcs12);
11904
11905                 return;
11906         }
11907
11908         /*
11909          * After an early L2 VM-entry failure, we're now back
11910          * in L1 which thinks it just finished a VMLAUNCH or
11911          * VMRESUME instruction, so we need to set the failure
11912          * flag and the VM-instruction error field of the VMCS
11913          * accordingly.
11914          */
11915         nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
11916
11917         load_vmcs12_mmu_host_state(vcpu, vmcs12);
11918
11919         /*
11920          * The emulated instruction was already skipped in
11921          * nested_vmx_run, but the updated RIP was never
11922          * written back to the vmcs01.
11923          */
11924         skip_emulated_instruction(vcpu);
11925         vmx->fail = 0;
11926 }
11927
11928 /*
11929  * Forcibly leave nested mode in order to be able to reset the VCPU later on.
11930  */
11931 static void vmx_leave_nested(struct kvm_vcpu *vcpu)
11932 {
11933         if (is_guest_mode(vcpu)) {
11934                 to_vmx(vcpu)->nested.nested_run_pending = 0;
11935                 nested_vmx_vmexit(vcpu, -1, 0, 0);
11936         }
11937         free_nested(to_vmx(vcpu));
11938 }
11939
11940 /*
11941  * L1's failure to enter L2 is a subset of a normal exit, as explained in
11942  * 23.7 "VM-entry failures during or after loading guest state" (this also
11943  * lists the acceptable exit-reason and exit-qualification parameters).
11944  * It should only be called before L2 actually succeeded to run, and when
11945  * vmcs01 is current (it doesn't leave_guest_mode() or switch vmcss).
11946  */
11947 static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
11948                         struct vmcs12 *vmcs12,
11949                         u32 reason, unsigned long qualification)
11950 {
11951         load_vmcs12_host_state(vcpu, vmcs12);
11952         vmcs12->vm_exit_reason = reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
11953         vmcs12->exit_qualification = qualification;
11954         nested_vmx_succeed(vcpu);
11955         if (enable_shadow_vmcs)
11956                 to_vmx(vcpu)->nested.sync_shadow_vmcs = true;
11957 }
11958
11959 static int vmx_check_intercept(struct kvm_vcpu *vcpu,
11960                                struct x86_instruction_info *info,
11961                                enum x86_intercept_stage stage)
11962 {
11963         return X86EMUL_CONTINUE;
11964 }
11965
11966 #ifdef CONFIG_X86_64
11967 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */
11968 static inline int u64_shl_div_u64(u64 a, unsigned int shift,
11969                                   u64 divisor, u64 *result)
11970 {
11971         u64 low = a << shift, high = a >> (64 - shift);
11972
11973         /* To avoid the overflow on divq */
11974         if (high >= divisor)
11975                 return 1;
11976
11977         /* Low hold the result, high hold rem which is discarded */
11978         asm("divq %2\n\t" : "=a" (low), "=d" (high) :
11979             "rm" (divisor), "0" (low), "1" (high));
11980         *result = low;
11981
11982         return 0;
11983 }
11984
11985 static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc)
11986 {
11987         struct vcpu_vmx *vmx = to_vmx(vcpu);
11988         u64 tscl = rdtsc();
11989         u64 guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
11990         u64 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
11991
11992         /* Convert to host delta tsc if tsc scaling is enabled */
11993         if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
11994                         u64_shl_div_u64(delta_tsc,
11995                                 kvm_tsc_scaling_ratio_frac_bits,
11996                                 vcpu->arch.tsc_scaling_ratio,
11997                                 &delta_tsc))
11998                 return -ERANGE;
11999
12000         /*
12001          * If the delta tsc can't fit in the 32 bit after the multi shift,
12002          * we can't use the preemption timer.
12003          * It's possible that it fits on later vmentries, but checking
12004          * on every vmentry is costly so we just use an hrtimer.
12005          */
12006         if (delta_tsc >> (cpu_preemption_timer_multi + 32))
12007                 return -ERANGE;
12008
12009         vmx->hv_deadline_tsc = tscl + delta_tsc;
12010         vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
12011                         PIN_BASED_VMX_PREEMPTION_TIMER);
12012
12013         return delta_tsc == 0;
12014 }
12015
12016 static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
12017 {
12018         struct vcpu_vmx *vmx = to_vmx(vcpu);
12019         vmx->hv_deadline_tsc = -1;
12020         vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
12021                         PIN_BASED_VMX_PREEMPTION_TIMER);
12022 }
12023 #endif
12024
12025 static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
12026 {
12027         if (ple_gap)
12028                 shrink_ple_window(vcpu);
12029 }
12030
12031 static void vmx_slot_enable_log_dirty(struct kvm *kvm,
12032                                      struct kvm_memory_slot *slot)
12033 {
12034         kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
12035         kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
12036 }
12037
12038 static void vmx_slot_disable_log_dirty(struct kvm *kvm,
12039                                        struct kvm_memory_slot *slot)
12040 {
12041         kvm_mmu_slot_set_dirty(kvm, slot);
12042 }
12043
12044 static void vmx_flush_log_dirty(struct kvm *kvm)
12045 {
12046         kvm_flush_pml_buffers(kvm);
12047 }
12048
12049 static int vmx_write_pml_buffer(struct kvm_vcpu *vcpu)
12050 {
12051         struct vmcs12 *vmcs12;
12052         struct vcpu_vmx *vmx = to_vmx(vcpu);
12053         gpa_t gpa;
12054         struct page *page = NULL;
12055         u64 *pml_address;
12056
12057         if (is_guest_mode(vcpu)) {
12058                 WARN_ON_ONCE(vmx->nested.pml_full);
12059
12060                 /*
12061                  * Check if PML is enabled for the nested guest.
12062                  * Whether eptp bit 6 is set is already checked
12063                  * as part of A/D emulation.
12064                  */
12065                 vmcs12 = get_vmcs12(vcpu);
12066                 if (!nested_cpu_has_pml(vmcs12))
12067                         return 0;
12068
12069                 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) {
12070                         vmx->nested.pml_full = true;
12071                         return 1;
12072                 }
12073
12074                 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS) & ~0xFFFull;
12075
12076                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->pml_address);
12077                 if (is_error_page(page))
12078                         return 0;
12079
12080                 pml_address = kmap(page);
12081                 pml_address[vmcs12->guest_pml_index--] = gpa;
12082                 kunmap(page);
12083                 kvm_release_page_clean(page);
12084         }
12085
12086         return 0;
12087 }
12088
12089 static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
12090                                            struct kvm_memory_slot *memslot,
12091                                            gfn_t offset, unsigned long mask)
12092 {
12093         kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
12094 }
12095
12096 static void __pi_post_block(struct kvm_vcpu *vcpu)
12097 {
12098         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
12099         struct pi_desc old, new;
12100         unsigned int dest;
12101
12102         do {
12103                 old.control = new.control = pi_desc->control;
12104                 WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR,
12105                      "Wakeup handler not enabled while the VCPU is blocked\n");
12106
12107                 dest = cpu_physical_id(vcpu->cpu);
12108
12109                 if (x2apic_enabled())
12110                         new.ndst = dest;
12111                 else
12112                         new.ndst = (dest << 8) & 0xFF00;
12113
12114                 /* set 'NV' to 'notification vector' */
12115                 new.nv = POSTED_INTR_VECTOR;
12116         } while (cmpxchg64(&pi_desc->control, old.control,
12117                            new.control) != old.control);
12118
12119         if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) {
12120                 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
12121                 list_del(&vcpu->blocked_vcpu_list);
12122                 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
12123                 vcpu->pre_pcpu = -1;
12124         }
12125 }
12126
12127 /*
12128  * This routine does the following things for vCPU which is going
12129  * to be blocked if VT-d PI is enabled.
12130  * - Store the vCPU to the wakeup list, so when interrupts happen
12131  *   we can find the right vCPU to wake up.
12132  * - Change the Posted-interrupt descriptor as below:
12133  *      'NDST' <-- vcpu->pre_pcpu
12134  *      'NV' <-- POSTED_INTR_WAKEUP_VECTOR
12135  * - If 'ON' is set during this process, which means at least one
12136  *   interrupt is posted for this vCPU, we cannot block it, in
12137  *   this case, return 1, otherwise, return 0.
12138  *
12139  */
12140 static int pi_pre_block(struct kvm_vcpu *vcpu)
12141 {
12142         unsigned int dest;
12143         struct pi_desc old, new;
12144         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
12145
12146         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
12147                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
12148                 !kvm_vcpu_apicv_active(vcpu))
12149                 return 0;
12150
12151         WARN_ON(irqs_disabled());
12152         local_irq_disable();
12153         if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) {
12154                 vcpu->pre_pcpu = vcpu->cpu;
12155                 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
12156                 list_add_tail(&vcpu->blocked_vcpu_list,
12157                               &per_cpu(blocked_vcpu_on_cpu,
12158                                        vcpu->pre_pcpu));
12159                 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
12160         }
12161
12162         do {
12163                 old.control = new.control = pi_desc->control;
12164
12165                 WARN((pi_desc->sn == 1),
12166                      "Warning: SN field of posted-interrupts "
12167                      "is set before blocking\n");
12168
12169                 /*
12170                  * Since vCPU can be preempted during this process,
12171                  * vcpu->cpu could be different with pre_pcpu, we
12172                  * need to set pre_pcpu as the destination of wakeup
12173                  * notification event, then we can find the right vCPU
12174                  * to wakeup in wakeup handler if interrupts happen
12175                  * when the vCPU is in blocked state.
12176                  */
12177                 dest = cpu_physical_id(vcpu->pre_pcpu);
12178
12179                 if (x2apic_enabled())
12180                         new.ndst = dest;
12181                 else
12182                         new.ndst = (dest << 8) & 0xFF00;
12183
12184                 /* set 'NV' to 'wakeup vector' */
12185                 new.nv = POSTED_INTR_WAKEUP_VECTOR;
12186         } while (cmpxchg64(&pi_desc->control, old.control,
12187                            new.control) != old.control);
12188
12189         /* We should not block the vCPU if an interrupt is posted for it.  */
12190         if (pi_test_on(pi_desc) == 1)
12191                 __pi_post_block(vcpu);
12192
12193         local_irq_enable();
12194         return (vcpu->pre_pcpu == -1);
12195 }
12196
12197 static int vmx_pre_block(struct kvm_vcpu *vcpu)
12198 {
12199         if (pi_pre_block(vcpu))
12200                 return 1;
12201
12202         if (kvm_lapic_hv_timer_in_use(vcpu))
12203                 kvm_lapic_switch_to_sw_timer(vcpu);
12204
12205         return 0;
12206 }
12207
12208 static void pi_post_block(struct kvm_vcpu *vcpu)
12209 {
12210         if (vcpu->pre_pcpu == -1)
12211                 return;
12212
12213         WARN_ON(irqs_disabled());
12214         local_irq_disable();
12215         __pi_post_block(vcpu);
12216         local_irq_enable();
12217 }
12218
12219 static void vmx_post_block(struct kvm_vcpu *vcpu)
12220 {
12221         if (kvm_x86_ops->set_hv_timer)
12222                 kvm_lapic_switch_to_hv_timer(vcpu);
12223
12224         pi_post_block(vcpu);
12225 }
12226
12227 /*
12228  * vmx_update_pi_irte - set IRTE for Posted-Interrupts
12229  *
12230  * @kvm: kvm
12231  * @host_irq: host irq of the interrupt
12232  * @guest_irq: gsi of the interrupt
12233  * @set: set or unset PI
12234  * returns 0 on success, < 0 on failure
12235  */
12236 static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
12237                               uint32_t guest_irq, bool set)
12238 {
12239         struct kvm_kernel_irq_routing_entry *e;
12240         struct kvm_irq_routing_table *irq_rt;
12241         struct kvm_lapic_irq irq;
12242         struct kvm_vcpu *vcpu;
12243         struct vcpu_data vcpu_info;
12244         int idx, ret = 0;
12245
12246         if (!kvm_arch_has_assigned_device(kvm) ||
12247                 !irq_remapping_cap(IRQ_POSTING_CAP) ||
12248                 !kvm_vcpu_apicv_active(kvm->vcpus[0]))
12249                 return 0;
12250
12251         idx = srcu_read_lock(&kvm->irq_srcu);
12252         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
12253         if (guest_irq >= irq_rt->nr_rt_entries ||
12254             hlist_empty(&irq_rt->map[guest_irq])) {
12255                 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
12256                              guest_irq, irq_rt->nr_rt_entries);
12257                 goto out;
12258         }
12259
12260         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
12261                 if (e->type != KVM_IRQ_ROUTING_MSI)
12262                         continue;
12263                 /*
12264                  * VT-d PI cannot support posting multicast/broadcast
12265                  * interrupts to a vCPU, we still use interrupt remapping
12266                  * for these kind of interrupts.
12267                  *
12268                  * For lowest-priority interrupts, we only support
12269                  * those with single CPU as the destination, e.g. user
12270                  * configures the interrupts via /proc/irq or uses
12271                  * irqbalance to make the interrupts single-CPU.
12272                  *
12273                  * We will support full lowest-priority interrupt later.
12274                  */
12275
12276                 kvm_set_msi_irq(kvm, e, &irq);
12277                 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
12278                         /*
12279                          * Make sure the IRTE is in remapped mode if
12280                          * we don't handle it in posted mode.
12281                          */
12282                         ret = irq_set_vcpu_affinity(host_irq, NULL);
12283                         if (ret < 0) {
12284                                 printk(KERN_INFO
12285                                    "failed to back to remapped mode, irq: %u\n",
12286                                    host_irq);
12287                                 goto out;
12288                         }
12289
12290                         continue;
12291                 }
12292
12293                 vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
12294                 vcpu_info.vector = irq.vector;
12295
12296                 trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,
12297                                 vcpu_info.vector, vcpu_info.pi_desc_addr, set);
12298
12299                 if (set)
12300                         ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
12301                 else
12302                         ret = irq_set_vcpu_affinity(host_irq, NULL);
12303
12304                 if (ret < 0) {
12305                         printk(KERN_INFO "%s: failed to update PI IRTE\n",
12306                                         __func__);
12307                         goto out;
12308                 }
12309         }
12310
12311         ret = 0;
12312 out:
12313         srcu_read_unlock(&kvm->irq_srcu, idx);
12314         return ret;
12315 }
12316
12317 static void vmx_setup_mce(struct kvm_vcpu *vcpu)
12318 {
12319         if (vcpu->arch.mcg_cap & MCG_LMCE_P)
12320                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
12321                         FEATURE_CONTROL_LMCE;
12322         else
12323                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
12324                         ~FEATURE_CONTROL_LMCE;
12325 }
12326
12327 static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
12328         .cpu_has_kvm_support = cpu_has_kvm_support,
12329         .disabled_by_bios = vmx_disabled_by_bios,
12330         .hardware_setup = hardware_setup,
12331         .hardware_unsetup = hardware_unsetup,
12332         .check_processor_compatibility = vmx_check_processor_compat,
12333         .hardware_enable = hardware_enable,
12334         .hardware_disable = hardware_disable,
12335         .cpu_has_accelerated_tpr = report_flexpriority,
12336         .has_emulated_msr = vmx_has_emulated_msr,
12337
12338         .vm_init = vmx_vm_init,
12339
12340         .vcpu_create = vmx_create_vcpu,
12341         .vcpu_free = vmx_free_vcpu,
12342         .vcpu_reset = vmx_vcpu_reset,
12343
12344         .prepare_guest_switch = vmx_save_host_state,
12345         .vcpu_load = vmx_vcpu_load,
12346         .vcpu_put = vmx_vcpu_put,
12347
12348         .update_bp_intercept = update_exception_bitmap,
12349         .get_msr = vmx_get_msr,
12350         .set_msr = vmx_set_msr,
12351         .get_segment_base = vmx_get_segment_base,
12352         .get_segment = vmx_get_segment,
12353         .set_segment = vmx_set_segment,
12354         .get_cpl = vmx_get_cpl,
12355         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
12356         .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
12357         .decache_cr3 = vmx_decache_cr3,
12358         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
12359         .set_cr0 = vmx_set_cr0,
12360         .set_cr3 = vmx_set_cr3,
12361         .set_cr4 = vmx_set_cr4,
12362         .set_efer = vmx_set_efer,
12363         .get_idt = vmx_get_idt,
12364         .set_idt = vmx_set_idt,
12365         .get_gdt = vmx_get_gdt,
12366         .set_gdt = vmx_set_gdt,
12367         .get_dr6 = vmx_get_dr6,
12368         .set_dr6 = vmx_set_dr6,
12369         .set_dr7 = vmx_set_dr7,
12370         .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
12371         .cache_reg = vmx_cache_reg,
12372         .get_rflags = vmx_get_rflags,
12373         .set_rflags = vmx_set_rflags,
12374
12375         .tlb_flush = vmx_flush_tlb,
12376
12377         .run = vmx_vcpu_run,
12378         .handle_exit = vmx_handle_exit,
12379         .skip_emulated_instruction = skip_emulated_instruction,
12380         .set_interrupt_shadow = vmx_set_interrupt_shadow,
12381         .get_interrupt_shadow = vmx_get_interrupt_shadow,
12382         .patch_hypercall = vmx_patch_hypercall,
12383         .set_irq = vmx_inject_irq,
12384         .set_nmi = vmx_inject_nmi,
12385         .queue_exception = vmx_queue_exception,
12386         .cancel_injection = vmx_cancel_injection,
12387         .interrupt_allowed = vmx_interrupt_allowed,
12388         .nmi_allowed = vmx_nmi_allowed,
12389         .get_nmi_mask = vmx_get_nmi_mask,
12390         .set_nmi_mask = vmx_set_nmi_mask,
12391         .enable_nmi_window = enable_nmi_window,
12392         .enable_irq_window = enable_irq_window,
12393         .update_cr8_intercept = update_cr8_intercept,
12394         .set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
12395         .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
12396         .get_enable_apicv = vmx_get_enable_apicv,
12397         .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
12398         .load_eoi_exitmap = vmx_load_eoi_exitmap,
12399         .apicv_post_state_restore = vmx_apicv_post_state_restore,
12400         .hwapic_irr_update = vmx_hwapic_irr_update,
12401         .hwapic_isr_update = vmx_hwapic_isr_update,
12402         .sync_pir_to_irr = vmx_sync_pir_to_irr,
12403         .deliver_posted_interrupt = vmx_deliver_posted_interrupt,
12404
12405         .set_tss_addr = vmx_set_tss_addr,
12406         .get_tdp_level = get_ept_level,
12407         .get_mt_mask = vmx_get_mt_mask,
12408
12409         .get_exit_info = vmx_get_exit_info,
12410
12411         .get_lpage_level = vmx_get_lpage_level,
12412
12413         .cpuid_update = vmx_cpuid_update,
12414
12415         .rdtscp_supported = vmx_rdtscp_supported,
12416         .invpcid_supported = vmx_invpcid_supported,
12417
12418         .set_supported_cpuid = vmx_set_supported_cpuid,
12419
12420         .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
12421
12422         .write_tsc_offset = vmx_write_tsc_offset,
12423
12424         .set_tdp_cr3 = vmx_set_cr3,
12425
12426         .check_intercept = vmx_check_intercept,
12427         .handle_external_intr = vmx_handle_external_intr,
12428         .mpx_supported = vmx_mpx_supported,
12429         .xsaves_supported = vmx_xsaves_supported,
12430
12431         .check_nested_events = vmx_check_nested_events,
12432
12433         .sched_in = vmx_sched_in,
12434
12435         .slot_enable_log_dirty = vmx_slot_enable_log_dirty,
12436         .slot_disable_log_dirty = vmx_slot_disable_log_dirty,
12437         .flush_log_dirty = vmx_flush_log_dirty,
12438         .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
12439         .write_log_dirty = vmx_write_pml_buffer,
12440
12441         .pre_block = vmx_pre_block,
12442         .post_block = vmx_post_block,
12443
12444         .pmu_ops = &intel_pmu_ops,
12445
12446         .update_pi_irte = vmx_update_pi_irte,
12447
12448 #ifdef CONFIG_X86_64
12449         .set_hv_timer = vmx_set_hv_timer,
12450         .cancel_hv_timer = vmx_cancel_hv_timer,
12451 #endif
12452
12453         .setup_mce = vmx_setup_mce,
12454 };
12455
12456 static int __init vmx_setup_l1d_flush(void)
12457 {
12458         struct page *page;
12459
12460         if (vmentry_l1d_flush == VMENTER_L1D_FLUSH_NEVER ||
12461             !boot_cpu_has_bug(X86_BUG_L1TF))
12462                 return 0;
12463
12464         if (!boot_cpu_has(X86_FEATURE_FLUSH_L1D)) {
12465                 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
12466                 if (!page)
12467                         return -ENOMEM;
12468                 vmx_l1d_flush_pages = page_address(page);
12469         }
12470
12471         static_branch_enable(&vmx_l1d_should_flush);
12472         return 0;
12473 }
12474
12475 static void vmx_free_l1d_flush_pages(void)
12476 {
12477         if (vmx_l1d_flush_pages) {
12478                 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER);
12479                 vmx_l1d_flush_pages = NULL;
12480         }
12481 }
12482
12483 static int __init vmx_init(void)
12484 {
12485         int r;
12486
12487         r = vmx_setup_l1d_flush();
12488         if (r)
12489                 return r;
12490
12491         r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
12492                      __alignof__(struct vcpu_vmx), THIS_MODULE);
12493         if (r) {
12494                 vmx_free_l1d_flush_pages();
12495                 return r;
12496         }
12497
12498 #ifdef CONFIG_KEXEC_CORE
12499         rcu_assign_pointer(crash_vmclear_loaded_vmcss,
12500                            crash_vmclear_local_loaded_vmcss);
12501 #endif
12502
12503         return 0;
12504 }
12505
12506 static void __exit vmx_exit(void)
12507 {
12508 #ifdef CONFIG_KEXEC_CORE
12509         RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
12510         synchronize_rcu();
12511 #endif
12512
12513         kvm_exit();
12514
12515         vmx_free_l1d_flush_pages();
12516 }
12517
12518 module_init(vmx_init)
12519 module_exit(vmx_exit)