arch/x86/kvm/vmx.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.  See
  15  * the COPYING file in the top-level directory.
  16  *
  17  */
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "cpuid.h"
  22 #include "lapic.h"
  23
  24 #include <linux/kvm_host.h>
  25 #include <linux/module.h>
  26 #include <linux/kernel.h>
  27 #include <linux/mm.h>
  28 #include <linux/highmem.h>
  29 #include <linux/sched.h>
  30 #include <linux/moduleparam.h>
  31 #include <linux/mod_devicetable.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34 #include <linux/tboot.h>
  35 #include <linux/hrtimer.h>
  36 #include <linux/frame.h>
  37 #include <linux/nospec.h>
  38 #include "kvm_cache_regs.h"
  39 #include "x86.h"
  40
  41 #include <asm/cpu.h>
  42 #include <asm/io.h>
  43 #include <asm/desc.h>
  44 #include <asm/vmx.h>
  45 #include <asm/virtext.h>
  46 #include <asm/mce.h>
  47 #include <asm/fpu/internal.h>
  48 #include <asm/perf_event.h>
  49 #include <asm/debugreg.h>
  50 #include <asm/kexec.h>
  51 #include <asm/apic.h>
  52 #include <asm/irq_remapping.h>
  53 #include <asm/mmu_context.h>
  54 #include <asm/nospec-branch.h>
  55
  56 #include "trace.h"
  57 #include "pmu.h"
  58
  59 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  60 #define __ex_clear(x, reg) \
  61         ____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
  62
  63 MODULE_AUTHOR("Qumranet");
  64 MODULE_LICENSE("GPL");
  65
  66 static const struct x86_cpu_id vmx_cpu_id[] = {
  67         X86_FEATURE_MATCH(X86_FEATURE_VMX),
  68         {}
  69 };
  70 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
  71
  72 static bool __read_mostly enable_vpid = 1;
  73 module_param_named(vpid, enable_vpid, bool, 0444);
  74
  75 static bool __read_mostly flexpriority_enabled = 1;
  76 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  77
  78 static bool __read_mostly enable_ept = 1;
  79 module_param_named(ept, enable_ept, bool, S_IRUGO);
  80
  81 static bool __read_mostly enable_unrestricted_guest = 1;
  82 module_param_named(unrestricted_guest,
  83                         enable_unrestricted_guest, bool, S_IRUGO);
  84
  85 static bool __read_mostly enable_ept_ad_bits = 1;
  86 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
  87
  88 static bool __read_mostly emulate_invalid_guest_state = true;
  89 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
  90
  91 static bool __read_mostly fasteoi = 1;
  92 module_param(fasteoi, bool, S_IRUGO);
  93
  94 static bool __read_mostly enable_apicv = 1;
  95 module_param(enable_apicv, bool, S_IRUGO);
  96
  97 static bool __read_mostly enable_shadow_vmcs = 1;
  98 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
  99 /*
 100  * If nested=1, nested virtualization is supported, i.e., guests may use
 101  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 102  * use VMX instructions.
 103  */
 104 static bool __read_mostly nested = 0;
 105 module_param(nested, bool, S_IRUGO);
 106
 107 static u64 __read_mostly host_xss;
 108
 109 static bool __read_mostly enable_pml = 1;
 110 module_param_named(pml, enable_pml, bool, S_IRUGO);
 111
 112 #define MSR_TYPE_R      1
 113 #define MSR_TYPE_W      2
 114 #define MSR_TYPE_RW     3
 115
 116 #define MSR_BITMAP_MODE_X2APIC          1
 117 #define MSR_BITMAP_MODE_X2APIC_APICV    2
 118 #define MSR_BITMAP_MODE_LM              4
 119
 120 #define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL
 121
 122 /* Guest_tsc -> host_tsc conversion requires 64-bit division.  */
 123 static int __read_mostly cpu_preemption_timer_multi;
 124 static bool __read_mostly enable_preemption_timer = 1;
 125 #ifdef CONFIG_X86_64
 126 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
 127 #endif
 128
 129 #define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
 130 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST (X86_CR0_WP | X86_CR0_NE)
 131 #define KVM_VM_CR0_ALWAYS_ON                                            \
 132         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
 133 #define KVM_CR4_GUEST_OWNED_BITS                                      \
 134         (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
 135          | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_TSD)
 136
 137 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
 138 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
 139
 140 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
 141
 142 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
 143
 144 /*
 145  * Hyper-V requires all of these, so mark them as supported even though
 146  * they are just treated the same as all-context.
 147  */
 148 #define VMX_VPID_EXTENT_SUPPORTED_MASK          \
 149         (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |  \
 150         VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |    \
 151         VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |    \
 152         VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
 153
 154 /*
 155  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 156  * ple_gap:    upper bound on the amount of time between two successive
 157  *             executions of PAUSE in a loop. Also indicate if ple enabled.
 158  *             According to test, this time is usually smaller than 128 cycles.
 159  * ple_window: upper bound on the amount of time a guest is allowed to execute
 160  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 161  *             less than 2^12 cycles
 162  * Time is measured based on a counter that runs at the same rate as the TSC,
 163  * refer SDM volume 3b section 21.6.13 & 22.1.3.
 164  */
 165 #define KVM_VMX_DEFAULT_PLE_GAP           128
 166 #define KVM_VMX_DEFAULT_PLE_WINDOW        4096
 167 #define KVM_VMX_DEFAULT_PLE_WINDOW_GROW   2
 168 #define KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK 0
 169 #define KVM_VMX_DEFAULT_PLE_WINDOW_MAX    \
 170                 INT_MAX / KVM_VMX_DEFAULT_PLE_WINDOW_GROW
 171
 172 static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
 173 module_param(ple_gap, int, S_IRUGO);
 174
 175 static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
 176 module_param(ple_window, int, S_IRUGO);
 177
 178 /* Default doubles per-vcpu window every exit. */
 179 static int ple_window_grow = KVM_VMX_DEFAULT_PLE_WINDOW_GROW;
 180 module_param(ple_window_grow, int, S_IRUGO);
 181
 182 /* Default resets per-vcpu window every exit to ple_window. */
 183 static int ple_window_shrink = KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK;
 184 module_param(ple_window_shrink, int, S_IRUGO);
 185
 186 /* Default is to compute the maximum so we can never overflow. */
 187 static int ple_window_actual_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 188 static int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 189 module_param(ple_window_max, int, S_IRUGO);
 190
 191 extern const ulong vmx_return;
 192
 193 #define NR_AUTOLOAD_MSRS 8
 194
 195 struct vmcs {
 196         u32 revision_id;
 197         u32 abort;
 198         char data[0];
 199 };
 200
 201 /*
 202  * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
 203  * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
 204  * loaded on this CPU (so we can clear them if the CPU goes down).
 205  */
 206 struct loaded_vmcs {
 207         struct vmcs *vmcs;
 208         struct vmcs *shadow_vmcs;
 209         int cpu;
 210         bool launched;
 211         bool nmi_known_unmasked;
 212         unsigned long vmcs_host_cr3;    /* May not match real cr3 */
 213         unsigned long vmcs_host_cr4;    /* May not match real cr4 */
 214         /* Support for vnmi-less CPUs */
 215         int soft_vnmi_blocked;
 216         ktime_t entry_time;
 217         s64 vnmi_blocked_time;
 218         unsigned long *msr_bitmap;
 219         struct list_head loaded_vmcss_on_cpu_link;
 220 };
 221
 222 struct shared_msr_entry {
 223         unsigned index;
 224         u64 data;
 225         u64 mask;
 226 };
 227
 228 /*
 229  * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
 230  * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
 231  * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
 232  * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
 233  * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
 234  * More than one of these structures may exist, if L1 runs multiple L2 guests.
 235  * nested_vmx_run() will use the data here to build the vmcs02: a VMCS for the
 236  * underlying hardware which will be used to run L2.
 237  * This structure is packed to ensure that its layout is identical across
 238  * machines (necessary for live migration).
 239  * If there are changes in this struct, VMCS12_REVISION must be changed.
 240  */
 241 typedef u64 natural_width;
 242 struct __packed vmcs12 {
 243         /* According to the Intel spec, a VMCS region must start with the
 244          * following two fields. Then follow implementation-specific data.
 245          */
 246         u32 revision_id;
 247         u32 abort;
 248
 249         u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
 250         u32 padding[7]; /* room for future expansion */
 251
 252         u64 io_bitmap_a;
 253         u64 io_bitmap_b;
 254         u64 msr_bitmap;
 255         u64 vm_exit_msr_store_addr;
 256         u64 vm_exit_msr_load_addr;
 257         u64 vm_entry_msr_load_addr;
 258         u64 tsc_offset;
 259         u64 virtual_apic_page_addr;
 260         u64 apic_access_addr;
 261         u64 posted_intr_desc_addr;
 262         u64 vm_function_control;
 263         u64 ept_pointer;
 264         u64 eoi_exit_bitmap0;
 265         u64 eoi_exit_bitmap1;
 266         u64 eoi_exit_bitmap2;
 267         u64 eoi_exit_bitmap3;
 268         u64 eptp_list_address;
 269         u64 xss_exit_bitmap;
 270         u64 guest_physical_address;
 271         u64 vmcs_link_pointer;
 272         u64 pml_address;
 273         u64 guest_ia32_debugctl;
 274         u64 guest_ia32_pat;
 275         u64 guest_ia32_efer;
 276         u64 guest_ia32_perf_global_ctrl;
 277         u64 guest_pdptr0;
 278         u64 guest_pdptr1;
 279         u64 guest_pdptr2;
 280         u64 guest_pdptr3;
 281         u64 guest_bndcfgs;
 282         u64 host_ia32_pat;
 283         u64 host_ia32_efer;
 284         u64 host_ia32_perf_global_ctrl;
 285         u64 padding64[8]; /* room for future expansion */
 286         /*
 287          * To allow migration of L1 (complete with its L2 guests) between
 288          * machines of different natural widths (32 or 64 bit), we cannot have
 289          * unsigned long fields with no explict size. We use u64 (aliased
 290          * natural_width) instead. Luckily, x86 is little-endian.
 291          */
 292         natural_width cr0_guest_host_mask;
 293         natural_width cr4_guest_host_mask;
 294         natural_width cr0_read_shadow;
 295         natural_width cr4_read_shadow;
 296         natural_width cr3_target_value0;
 297         natural_width cr3_target_value1;
 298         natural_width cr3_target_value2;
 299         natural_width cr3_target_value3;
 300         natural_width exit_qualification;
 301         natural_width guest_linear_address;
 302         natural_width guest_cr0;
 303         natural_width guest_cr3;
 304         natural_width guest_cr4;
 305         natural_width guest_es_base;
 306         natural_width guest_cs_base;
 307         natural_width guest_ss_base;
 308         natural_width guest_ds_base;
 309         natural_width guest_fs_base;
 310         natural_width guest_gs_base;
 311         natural_width guest_ldtr_base;
 312         natural_width guest_tr_base;
 313         natural_width guest_gdtr_base;
 314         natural_width guest_idtr_base;
 315         natural_width guest_dr7;
 316         natural_width guest_rsp;
 317         natural_width guest_rip;
 318         natural_width guest_rflags;
 319         natural_width guest_pending_dbg_exceptions;
 320         natural_width guest_sysenter_esp;
 321         natural_width guest_sysenter_eip;
 322         natural_width host_cr0;
 323         natural_width host_cr3;
 324         natural_width host_cr4;
 325         natural_width host_fs_base;
 326         natural_width host_gs_base;
 327         natural_width host_tr_base;
 328         natural_width host_gdtr_base;
 329         natural_width host_idtr_base;
 330         natural_width host_ia32_sysenter_esp;
 331         natural_width host_ia32_sysenter_eip;
 332         natural_width host_rsp;
 333         natural_width host_rip;
 334         natural_width paddingl[8]; /* room for future expansion */
 335         u32 pin_based_vm_exec_control;
 336         u32 cpu_based_vm_exec_control;
 337         u32 exception_bitmap;
 338         u32 page_fault_error_code_mask;
 339         u32 page_fault_error_code_match;
 340         u32 cr3_target_count;
 341         u32 vm_exit_controls;
 342         u32 vm_exit_msr_store_count;
 343         u32 vm_exit_msr_load_count;
 344         u32 vm_entry_controls;
 345         u32 vm_entry_msr_load_count;
 346         u32 vm_entry_intr_info_field;
 347         u32 vm_entry_exception_error_code;
 348         u32 vm_entry_instruction_len;
 349         u32 tpr_threshold;
 350         u32 secondary_vm_exec_control;
 351         u32 vm_instruction_error;
 352         u32 vm_exit_reason;
 353         u32 vm_exit_intr_info;
 354         u32 vm_exit_intr_error_code;
 355         u32 idt_vectoring_info_field;
 356         u32 idt_vectoring_error_code;
 357         u32 vm_exit_instruction_len;
 358         u32 vmx_instruction_info;
 359         u32 guest_es_limit;
 360         u32 guest_cs_limit;
 361         u32 guest_ss_limit;
 362         u32 guest_ds_limit;
 363         u32 guest_fs_limit;
 364         u32 guest_gs_limit;
 365         u32 guest_ldtr_limit;
 366         u32 guest_tr_limit;
 367         u32 guest_gdtr_limit;
 368         u32 guest_idtr_limit;
 369         u32 guest_es_ar_bytes;
 370         u32 guest_cs_ar_bytes;
 371         u32 guest_ss_ar_bytes;
 372         u32 guest_ds_ar_bytes;
 373         u32 guest_fs_ar_bytes;
 374         u32 guest_gs_ar_bytes;
 375         u32 guest_ldtr_ar_bytes;
 376         u32 guest_tr_ar_bytes;
 377         u32 guest_interruptibility_info;
 378         u32 guest_activity_state;
 379         u32 guest_sysenter_cs;
 380         u32 host_ia32_sysenter_cs;
 381         u32 vmx_preemption_timer_value;
 382         u32 padding32[7]; /* room for future expansion */
 383         u16 virtual_processor_id;
 384         u16 posted_intr_nv;
 385         u16 guest_es_selector;
 386         u16 guest_cs_selector;
 387         u16 guest_ss_selector;
 388         u16 guest_ds_selector;
 389         u16 guest_fs_selector;
 390         u16 guest_gs_selector;
 391         u16 guest_ldtr_selector;
 392         u16 guest_tr_selector;
 393         u16 guest_intr_status;
 394         u16 guest_pml_index;
 395         u16 host_es_selector;
 396         u16 host_cs_selector;
 397         u16 host_ss_selector;
 398         u16 host_ds_selector;
 399         u16 host_fs_selector;
 400         u16 host_gs_selector;
 401         u16 host_tr_selector;
 402 };
 403
 404 /*
 405  * VMCS12_REVISION is an arbitrary id that should be changed if the content or
 406  * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
 407  * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
 408  */
 409 #define VMCS12_REVISION 0x11e57ed0
 410
 411 /*
 412  * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
 413  * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
 414  * current implementation, 4K are reserved to avoid future complications.
 415  */
 416 #define VMCS12_SIZE 0x1000
 417
 418 /*
 419  * The nested_vmx structure is part of vcpu_vmx, and holds information we need
 420  * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
 421  */
 422 struct nested_vmx {
 423         /* Has the level1 guest done vmxon? */
 424         bool vmxon;
 425         gpa_t vmxon_ptr;
 426         bool pml_full;
 427
 428         /* The guest-physical address of the current VMCS L1 keeps for L2 */
 429         gpa_t current_vmptr;
 430         /*
 431          * Cache of the guest's VMCS, existing outside of guest memory.
 432          * Loaded from guest memory during VMPTRLD. Flushed to guest
 433          * memory during VMCLEAR and VMPTRLD.
 434          */
 435         struct vmcs12 *cached_vmcs12;
 436         /*
 437          * Indicates if the shadow vmcs must be updated with the
 438          * data hold by vmcs12
 439          */
 440         bool sync_shadow_vmcs;
 441
 442         bool change_vmcs01_virtual_x2apic_mode;
 443         /* L2 must run next, and mustn't decide to exit to L1. */
 444         bool nested_run_pending;
 445
 446         struct loaded_vmcs vmcs02;
 447
 448         /*
 449          * Guest pages referred to in the vmcs02 with host-physical
 450          * pointers, so we must keep them pinned while L2 runs.
 451          */
 452         struct page *apic_access_page;
 453         struct page *virtual_apic_page;
 454         struct page *pi_desc_page;
 455         struct pi_desc *pi_desc;
 456         bool pi_pending;
 457         u16 posted_intr_nv;
 458
 459         struct hrtimer preemption_timer;
 460         bool preemption_timer_expired;
 461
 462         /* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */
 463         u64 vmcs01_debugctl;
 464
 465         u16 vpid02;
 466         u16 last_vpid;
 467
 468         /*
 469          * We only store the "true" versions of the VMX capability MSRs. We
 470          * generate the "non-true" versions by setting the must-be-1 bits
 471          * according to the SDM.
 472          */
 473         u32 nested_vmx_procbased_ctls_low;
 474         u32 nested_vmx_procbased_ctls_high;
 475         u32 nested_vmx_secondary_ctls_low;
 476         u32 nested_vmx_secondary_ctls_high;
 477         u32 nested_vmx_pinbased_ctls_low;
 478         u32 nested_vmx_pinbased_ctls_high;
 479         u32 nested_vmx_exit_ctls_low;
 480         u32 nested_vmx_exit_ctls_high;
 481         u32 nested_vmx_entry_ctls_low;
 482         u32 nested_vmx_entry_ctls_high;
 483         u32 nested_vmx_misc_low;
 484         u32 nested_vmx_misc_high;
 485         u32 nested_vmx_ept_caps;
 486         u32 nested_vmx_vpid_caps;
 487         u64 nested_vmx_basic;
 488         u64 nested_vmx_cr0_fixed0;
 489         u64 nested_vmx_cr0_fixed1;
 490         u64 nested_vmx_cr4_fixed0;
 491         u64 nested_vmx_cr4_fixed1;
 492         u64 nested_vmx_vmcs_enum;
 493         u64 nested_vmx_vmfunc_controls;
 494 };
 495
 496 #define POSTED_INTR_ON  0
 497 #define POSTED_INTR_SN  1
 498
 499 /* Posted-Interrupt Descriptor */
 500 struct pi_desc {
 501         u32 pir[8];     /* Posted interrupt requested */
 502         union {
 503                 struct {
 504                                 /* bit 256 - Outstanding Notification */
 505                         u16     on      : 1,
 506                                 /* bit 257 - Suppress Notification */
 507                                 sn      : 1,
 508                                 /* bit 271:258 - Reserved */
 509                                 rsvd_1  : 14;
 510                                 /* bit 279:272 - Notification Vector */
 511                         u8      nv;
 512                                 /* bit 287:280 - Reserved */
 513                         u8      rsvd_2;
 514                                 /* bit 319:288 - Notification Destination */
 515                         u32     ndst;
 516                 };
 517                 u64 control;
 518         };
 519         u32 rsvd[6];
 520 } __aligned(64);
 521
 522 static bool pi_test_and_set_on(struct pi_desc *pi_desc)
 523 {
 524         return test_and_set_bit(POSTED_INTR_ON,
 525                         (unsigned long *)&pi_desc->control);
 526 }
 527
 528 static bool pi_test_and_clear_on(struct pi_desc *pi_desc)
 529 {
 530         return test_and_clear_bit(POSTED_INTR_ON,
 531                         (unsigned long *)&pi_desc->control);
 532 }
 533
 534 static int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
 535 {
 536         return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
 537 }
 538
 539 static inline void pi_clear_sn(struct pi_desc *pi_desc)
 540 {
 541         return clear_bit(POSTED_INTR_SN,
 542                         (unsigned long *)&pi_desc->control);
 543 }
 544
 545 static inline void pi_set_sn(struct pi_desc *pi_desc)
 546 {
 547         return set_bit(POSTED_INTR_SN,
 548                         (unsigned long *)&pi_desc->control);
 549 }
 550
 551 static inline void pi_clear_on(struct pi_desc *pi_desc)
 552 {
 553         clear_bit(POSTED_INTR_ON,
 554                   (unsigned long *)&pi_desc->control);
 555 }
 556
 557 static inline int pi_test_on(struct pi_desc *pi_desc)
 558 {
 559         return test_bit(POSTED_INTR_ON,
 560                         (unsigned long *)&pi_desc->control);
 561 }
 562
 563 static inline int pi_test_sn(struct pi_desc *pi_desc)
 564 {
 565         return test_bit(POSTED_INTR_SN,
 566                         (unsigned long *)&pi_desc->control);
 567 }
 568
 569 struct vcpu_vmx {
 570         struct kvm_vcpu       vcpu;
 571         unsigned long         host_rsp;
 572         u8                    fail;
 573         u8                    msr_bitmap_mode;
 574         u32                   exit_intr_info;
 575         u32                   idt_vectoring_info;
 576         ulong                 rflags;
 577         struct shared_msr_entry *guest_msrs;
 578         int                   nmsrs;
 579         int                   save_nmsrs;
 580         unsigned long         host_idt_base;
 581 #ifdef CONFIG_X86_64
 582         u64                   msr_host_kernel_gs_base;
 583         u64                   msr_guest_kernel_gs_base;
 584 #endif
 585
 586         u32 vm_entry_controls_shadow;
 587         u32 vm_exit_controls_shadow;
 588         u32 secondary_exec_control;
 589
 590         /*
 591          * loaded_vmcs points to the VMCS currently used in this vcpu. For a
 592          * non-nested (L1) guest, it always points to vmcs01. For a nested
 593          * guest (L2), it points to a different VMCS.
 594          */
 595         struct loaded_vmcs    vmcs01;
 596         struct loaded_vmcs   *loaded_vmcs;
 597         bool                  __launched; /* temporary, used in vmx_vcpu_run */
 598         struct msr_autoload {
 599                 unsigned nr;
 600                 struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
 601                 struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
 602         } msr_autoload;
 603         struct {
 604                 int           loaded;
 605                 u16           fs_sel, gs_sel, ldt_sel;
 606 #ifdef CONFIG_X86_64
 607                 u16           ds_sel, es_sel;
 608 #endif
 609                 int           gs_ldt_reload_needed;
 610                 int           fs_reload_needed;
 611                 u64           msr_host_bndcfgs;
 612         } host_state;
 613         struct {
 614                 int vm86_active;
 615                 ulong save_rflags;
 616                 struct kvm_segment segs[8];
 617         } rmode;
 618         struct {
 619                 u32 bitmask; /* 4 bits per segment (1 bit per field) */
 620                 struct kvm_save_segment {
 621                         u16 selector;
 622                         unsigned long base;
 623                         u32 limit;
 624                         u32 ar;
 625                 } seg[8];
 626         } segment_cache;
 627         int vpid;
 628         bool emulation_required;
 629
 630         u32 exit_reason;
 631
 632         /* Posted interrupt descriptor */
 633         struct pi_desc pi_desc;
 634
 635         /* Support for a guest hypervisor (nested VMX) */
 636         struct nested_vmx nested;
 637
 638         /* Dynamic PLE window. */
 639         int ple_window;
 640         bool ple_window_dirty;
 641
 642         /* Support for PML */
 643 #define PML_ENTITY_NUM          512
 644         struct page *pml_pg;
 645
 646         /* apic deadline value in host tsc */
 647         u64 hv_deadline_tsc;
 648
 649         u64 current_tsc_ratio;
 650
 651         u32 host_pkru;
 652
 653         /*
 654          * Only bits masked by msr_ia32_feature_control_valid_bits can be set in
 655          * msr_ia32_feature_control. FEATURE_CONTROL_LOCKED is always included
 656          * in msr_ia32_feature_control_valid_bits.
 657          */
 658         u64 msr_ia32_feature_control;
 659         u64 msr_ia32_feature_control_valid_bits;
 660 };
 661
 662 enum segment_cache_field {
 663         SEG_FIELD_SEL = 0,
 664         SEG_FIELD_BASE = 1,
 665         SEG_FIELD_LIMIT = 2,
 666         SEG_FIELD_AR = 3,
 667
 668         SEG_FIELD_NR = 4
 669 };
 670
 671 static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
 672 {
 673         return container_of(vcpu, struct vcpu_vmx, vcpu);
 674 }
 675
 676 static struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
 677 {
 678         return &(to_vmx(vcpu)->pi_desc);
 679 }
 680
 681 #define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
 682 #define FIELD(number, name)     [number] = VMCS12_OFFSET(name)
 683 #define FIELD64(number, name)   [number] = VMCS12_OFFSET(name), \
 684                                 [number##_HIGH] = VMCS12_OFFSET(name)+4
 685
 686
 687 static unsigned long shadow_read_only_fields[] = {
 688         /*
 689          * We do NOT shadow fields that are modified when L0
 690          * traps and emulates any vmx instruction (e.g. VMPTRLD,
 691          * VMXON...) executed by L1.
 692          * For example, VM_INSTRUCTION_ERROR is read
 693          * by L1 if a vmx instruction fails (part of the error path).
 694          * Note the code assumes this logic. If for some reason
 695          * we start shadowing these fields then we need to
 696          * force a shadow sync when L0 emulates vmx instructions
 697          * (e.g. force a sync if VM_INSTRUCTION_ERROR is modified
 698          * by nested_vmx_failValid)
 699          */
 700         VM_EXIT_REASON,
 701         VM_EXIT_INTR_INFO,
 702         VM_EXIT_INSTRUCTION_LEN,
 703         IDT_VECTORING_INFO_FIELD,
 704         IDT_VECTORING_ERROR_CODE,
 705         VM_EXIT_INTR_ERROR_CODE,
 706         EXIT_QUALIFICATION,
 707         GUEST_LINEAR_ADDRESS,
 708         GUEST_PHYSICAL_ADDRESS
 709 };
 710 static int max_shadow_read_only_fields =
 711         ARRAY_SIZE(shadow_read_only_fields);
 712
 713 static unsigned long shadow_read_write_fields[] = {
 714         TPR_THRESHOLD,
 715         GUEST_RIP,
 716         GUEST_RSP,
 717         GUEST_CR0,
 718         GUEST_CR3,
 719         GUEST_CR4,
 720         GUEST_INTERRUPTIBILITY_INFO,
 721         GUEST_RFLAGS,
 722         GUEST_CS_SELECTOR,
 723         GUEST_CS_AR_BYTES,
 724         GUEST_CS_LIMIT,
 725         GUEST_CS_BASE,
 726         GUEST_ES_BASE,
 727         GUEST_BNDCFGS,
 728         CR0_GUEST_HOST_MASK,
 729         CR0_READ_SHADOW,
 730         CR4_READ_SHADOW,
 731         TSC_OFFSET,
 732         EXCEPTION_BITMAP,
 733         CPU_BASED_VM_EXEC_CONTROL,
 734         VM_ENTRY_EXCEPTION_ERROR_CODE,
 735         VM_ENTRY_INTR_INFO_FIELD,
 736         VM_ENTRY_INSTRUCTION_LEN,
 737         VM_ENTRY_EXCEPTION_ERROR_CODE,
 738         HOST_FS_BASE,
 739         HOST_GS_BASE,
 740         HOST_FS_SELECTOR,
 741         HOST_GS_SELECTOR
 742 };
 743 static int max_shadow_read_write_fields =
 744         ARRAY_SIZE(shadow_read_write_fields);
 745
 746 static const unsigned short vmcs_field_to_offset_table[] = {
 747         FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
 748         FIELD(POSTED_INTR_NV, posted_intr_nv),
 749         FIELD(GUEST_ES_SELECTOR, guest_es_selector),
 750         FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
 751         FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
 752         FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
 753         FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
 754         FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
 755         FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
 756         FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
 757         FIELD(GUEST_INTR_STATUS, guest_intr_status),
 758         FIELD(GUEST_PML_INDEX, guest_pml_index),
 759         FIELD(HOST_ES_SELECTOR, host_es_selector),
 760         FIELD(HOST_CS_SELECTOR, host_cs_selector),
 761         FIELD(HOST_SS_SELECTOR, host_ss_selector),
 762         FIELD(HOST_DS_SELECTOR, host_ds_selector),
 763         FIELD(HOST_FS_SELECTOR, host_fs_selector),
 764         FIELD(HOST_GS_SELECTOR, host_gs_selector),
 765         FIELD(HOST_TR_SELECTOR, host_tr_selector),
 766         FIELD64(IO_BITMAP_A, io_bitmap_a),
 767         FIELD64(IO_BITMAP_B, io_bitmap_b),
 768         FIELD64(MSR_BITMAP, msr_bitmap),
 769         FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
 770         FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
 771         FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
 772         FIELD64(TSC_OFFSET, tsc_offset),
 773         FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
 774         FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
 775         FIELD64(POSTED_INTR_DESC_ADDR, posted_intr_desc_addr),
 776         FIELD64(VM_FUNCTION_CONTROL, vm_function_control),
 777         FIELD64(EPT_POINTER, ept_pointer),
 778         FIELD64(EOI_EXIT_BITMAP0, eoi_exit_bitmap0),
 779         FIELD64(EOI_EXIT_BITMAP1, eoi_exit_bitmap1),
 780         FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
 781         FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
 782         FIELD64(EPTP_LIST_ADDRESS, eptp_list_address),
 783         FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
 784         FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
 785         FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
 786         FIELD64(PML_ADDRESS, pml_address),
 787         FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
 788         FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
 789         FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
 790         FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
 791         FIELD64(GUEST_PDPTR0, guest_pdptr0),
 792         FIELD64(GUEST_PDPTR1, guest_pdptr1),
 793         FIELD64(GUEST_PDPTR2, guest_pdptr2),
 794         FIELD64(GUEST_PDPTR3, guest_pdptr3),
 795         FIELD64(GUEST_BNDCFGS, guest_bndcfgs),
 796         FIELD64(HOST_IA32_PAT, host_ia32_pat),
 797         FIELD64(HOST_IA32_EFER, host_ia32_efer),
 798         FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
 799         FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
 800         FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
 801         FIELD(EXCEPTION_BITMAP, exception_bitmap),
 802         FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
 803         FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
 804         FIELD(CR3_TARGET_COUNT, cr3_target_count),
 805         FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
 806         FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
 807         FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
 808         FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
 809         FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
 810         FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
 811         FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
 812         FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
 813         FIELD(TPR_THRESHOLD, tpr_threshold),
 814         FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
 815         FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
 816         FIELD(VM_EXIT_REASON, vm_exit_reason),
 817         FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
 818         FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
 819         FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
 820         FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
 821         FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
 822         FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
 823         FIELD(GUEST_ES_LIMIT, guest_es_limit),
 824         FIELD(GUEST_CS_LIMIT, guest_cs_limit),
 825         FIELD(GUEST_SS_LIMIT, guest_ss_limit),
 826         FIELD(GUEST_DS_LIMIT, guest_ds_limit),
 827         FIELD(GUEST_FS_LIMIT, guest_fs_limit),
 828         FIELD(GUEST_GS_LIMIT, guest_gs_limit),
 829         FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
 830         FIELD(GUEST_TR_LIMIT, guest_tr_limit),
 831         FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
 832         FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
 833         FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
 834         FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
 835         FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
 836         FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
 837         FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
 838         FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
 839         FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
 840         FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
 841         FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
 842         FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
 843         FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
 844         FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
 845         FIELD(VMX_PREEMPTION_TIMER_VALUE, vmx_preemption_timer_value),
 846         FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
 847         FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
 848         FIELD(CR0_READ_SHADOW, cr0_read_shadow),
 849         FIELD(CR4_READ_SHADOW, cr4_read_shadow),
 850         FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
 851         FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
 852         FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
 853         FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
 854         FIELD(EXIT_QUALIFICATION, exit_qualification),
 855         FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
 856         FIELD(GUEST_CR0, guest_cr0),
 857         FIELD(GUEST_CR3, guest_cr3),
 858         FIELD(GUEST_CR4, guest_cr4),
 859         FIELD(GUEST_ES_BASE, guest_es_base),
 860         FIELD(GUEST_CS_BASE, guest_cs_base),
 861         FIELD(GUEST_SS_BASE, guest_ss_base),
 862         FIELD(GUEST_DS_BASE, guest_ds_base),
 863         FIELD(GUEST_FS_BASE, guest_fs_base),
 864         FIELD(GUEST_GS_BASE, guest_gs_base),
 865         FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
 866         FIELD(GUEST_TR_BASE, guest_tr_base),
 867         FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
 868         FIELD(GUEST_IDTR_BASE, guest_idtr_base),
 869         FIELD(GUEST_DR7, guest_dr7),
 870         FIELD(GUEST_RSP, guest_rsp),
 871         FIELD(GUEST_RIP, guest_rip),
 872         FIELD(GUEST_RFLAGS, guest_rflags),
 873         FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
 874         FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
 875         FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
 876         FIELD(HOST_CR0, host_cr0),
 877         FIELD(HOST_CR3, host_cr3),
 878         FIELD(HOST_CR4, host_cr4),
 879         FIELD(HOST_FS_BASE, host_fs_base),
 880         FIELD(HOST_GS_BASE, host_gs_base),
 881         FIELD(HOST_TR_BASE, host_tr_base),
 882         FIELD(HOST_GDTR_BASE, host_gdtr_base),
 883         FIELD(HOST_IDTR_BASE, host_idtr_base),
 884         FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
 885         FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
 886         FIELD(HOST_RSP, host_rsp),
 887         FIELD(HOST_RIP, host_rip),
 888 };
 889
 890 static inline short vmcs_field_to_offset(unsigned long field)
 891 {
 892         const size_t size = ARRAY_SIZE(vmcs_field_to_offset_table);
 893         unsigned short offset;
 894
 895         BUILD_BUG_ON(size > SHRT_MAX);
 896         if (field >= size)
 897                 return -ENOENT;
 898
 899         field = array_index_nospec(field, size);
 900         offset = vmcs_field_to_offset_table[field];
 901         if (offset == 0)
 902                 return -ENOENT;
 903         return offset;
 904 }
 905
 906 static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
 907 {
 908         return to_vmx(vcpu)->nested.cached_vmcs12;
 909 }
 910
 911 static bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu);
 912 static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu);
 913 static u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa);
 914 static bool vmx_xsaves_supported(void);
 915 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
 916 static void vmx_set_segment(struct kvm_vcpu *vcpu,
 917                             struct kvm_segment *var, int seg);
 918 static void vmx_get_segment(struct kvm_vcpu *vcpu,
 919                             struct kvm_segment *var, int seg);
 920 static bool guest_state_valid(struct kvm_vcpu *vcpu);
 921 static u32 vmx_segment_access_rights(struct kvm_segment *var);
 922 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx);
 923 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
 924 static int alloc_identity_pagetable(struct kvm *kvm);
 925 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
 926 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
 927 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
 928                                             u16 error_code);
 929 static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);
 930 static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
 931                                                           u32 msr, int type);
 932
 933 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 934 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
 935 /*
 936  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
 937  * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
 938  */
 939 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
 940
 941 /*
 942  * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
 943  * can find which vCPU should be waken up.
 944  */
 945 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
 946 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
 947
 948 enum {
 949         VMX_IO_BITMAP_A,
 950         VMX_IO_BITMAP_B,
 951         VMX_VMREAD_BITMAP,
 952         VMX_VMWRITE_BITMAP,
 953         VMX_BITMAP_NR
 954 };
 955
 956 static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
 957
 958 #define vmx_io_bitmap_a                      (vmx_bitmap[VMX_IO_BITMAP_A])
 959 #define vmx_io_bitmap_b                      (vmx_bitmap[VMX_IO_BITMAP_B])
 960 #define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
 961 #define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
 962
 963 static bool cpu_has_load_ia32_efer;
 964 static bool cpu_has_load_perf_global_ctrl;
 965
 966 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
 967 static DEFINE_SPINLOCK(vmx_vpid_lock);
 968
 969 static struct vmcs_config {
 970         int size;
 971         int order;
 972         u32 basic_cap;
 973         u32 revision_id;
 974         u32 pin_based_exec_ctrl;
 975         u32 cpu_based_exec_ctrl;
 976         u32 cpu_based_2nd_exec_ctrl;
 977         u32 vmexit_ctrl;
 978         u32 vmentry_ctrl;
 979 } vmcs_config;
 980
 981 static struct vmx_capability {
 982         u32 ept;
 983         u32 vpid;
 984 } vmx_capability;
 985
 986 #define VMX_SEGMENT_FIELD(seg)                                  \
 987         [VCPU_SREG_##seg] = {                                   \
 988                 .selector = GUEST_##seg##_SELECTOR,             \
 989                 .base = GUEST_##seg##_BASE,                     \
 990                 .limit = GUEST_##seg##_LIMIT,                   \
 991                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
 992         }
 993
 994 static const struct kvm_vmx_segment_field {
 995         unsigned selector;
 996         unsigned base;
 997         unsigned limit;
 998         unsigned ar_bytes;
 999 } kvm_vmx_segment_fields[] = {
1000         VMX_SEGMENT_FIELD(CS),
1001         VMX_SEGMENT_FIELD(DS),
1002         VMX_SEGMENT_FIELD(ES),
1003         VMX_SEGMENT_FIELD(FS),
1004         VMX_SEGMENT_FIELD(GS),
1005         VMX_SEGMENT_FIELD(SS),
1006         VMX_SEGMENT_FIELD(TR),
1007         VMX_SEGMENT_FIELD(LDTR),
1008 };
1009
1010 static u64 host_efer;
1011
1012 static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
1013
1014 /*
1015  * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
1016  * away by decrementing the array size.
1017  */
1018 static const u32 vmx_msr_index[] = {
1019 #ifdef CONFIG_X86_64
1020         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
1021 #endif
1022         MSR_EFER, MSR_TSC_AUX, MSR_STAR,
1023 };
1024
1025 static inline bool is_exception_n(u32 intr_info, u8 vector)
1026 {
1027         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
1028                              INTR_INFO_VALID_MASK)) ==
1029                 (INTR_TYPE_HARD_EXCEPTION | vector | INTR_INFO_VALID_MASK);
1030 }
1031
1032 static inline bool is_debug(u32 intr_info)
1033 {
1034         return is_exception_n(intr_info, DB_VECTOR);
1035 }
1036
1037 static inline bool is_breakpoint(u32 intr_info)
1038 {
1039         return is_exception_n(intr_info, BP_VECTOR);
1040 }
1041
1042 static inline bool is_page_fault(u32 intr_info)
1043 {
1044         return is_exception_n(intr_info, PF_VECTOR);
1045 }
1046
1047 static inline bool is_no_device(u32 intr_info)
1048 {
1049         return is_exception_n(intr_info, NM_VECTOR);
1050 }
1051
1052 static inline bool is_invalid_opcode(u32 intr_info)
1053 {
1054         return is_exception_n(intr_info, UD_VECTOR);
1055 }
1056
1057 static inline bool is_external_interrupt(u32 intr_info)
1058 {
1059         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1060                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
1061 }
1062
1063 static inline bool is_machine_check(u32 intr_info)
1064 {
1065         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
1066                              INTR_INFO_VALID_MASK)) ==
1067                 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
1068 }
1069
1070 static inline bool cpu_has_vmx_msr_bitmap(void)
1071 {
1072         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
1073 }
1074
1075 static inline bool cpu_has_vmx_tpr_shadow(void)
1076 {
1077         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
1078 }
1079
1080 static inline bool cpu_need_tpr_shadow(struct kvm_vcpu *vcpu)
1081 {
1082         return cpu_has_vmx_tpr_shadow() && lapic_in_kernel(vcpu);
1083 }
1084
1085 static inline bool cpu_has_secondary_exec_ctrls(void)
1086 {
1087         return vmcs_config.cpu_based_exec_ctrl &
1088                 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1089 }
1090
1091 static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
1092 {
1093         return vmcs_config.cpu_based_2nd_exec_ctrl &
1094                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
1095 }
1096
1097 static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
1098 {
1099         return vmcs_config.cpu_based_2nd_exec_ctrl &
1100                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
1101 }
1102
1103 static inline bool cpu_has_vmx_apic_register_virt(void)
1104 {
1105         return vmcs_config.cpu_based_2nd_exec_ctrl &
1106                 SECONDARY_EXEC_APIC_REGISTER_VIRT;
1107 }
1108
1109 static inline bool cpu_has_vmx_virtual_intr_delivery(void)
1110 {
1111         return vmcs_config.cpu_based_2nd_exec_ctrl &
1112                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
1113 }
1114
1115 /*
1116  * Comment's format: document - errata name - stepping - processor name.
1117  * Refer from
1118  * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
1119  */
1120 static u32 vmx_preemption_cpu_tfms[] = {
1121 /* 323344.pdf - BA86   - D0 - Xeon 7500 Series */
1122 0x000206E6,
1123 /* 323056.pdf - AAX65  - C2 - Xeon L3406 */
1124 /* 322814.pdf - AAT59  - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
1125 /* 322911.pdf - AAU65  - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
1126 0x00020652,
1127 /* 322911.pdf - AAU65  - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
1128 0x00020655,
1129 /* 322373.pdf - AAO95  - B1 - Xeon 3400 Series */
1130 /* 322166.pdf - AAN92  - B1 - i7-800 and i5-700 Desktop */
1131 /*
1132  * 320767.pdf - AAP86  - B1 -
1133  * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
1134  */
1135 0x000106E5,
1136 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
1137 0x000106A0,
1138 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
1139 0x000106A1,
1140 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
1141 0x000106A4,
1142  /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
1143  /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
1144  /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
1145 0x000106A5,
1146 };
1147
1148 static inline bool cpu_has_broken_vmx_preemption_timer(void)
1149 {
1150         u32 eax = cpuid_eax(0x00000001), i;
1151
1152         /* Clear the reserved bits */
1153         eax &= ~(0x3U << 14 | 0xfU << 28);
1154         for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
1155                 if (eax == vmx_preemption_cpu_tfms[i])
1156                         return true;
1157
1158         return false;
1159 }
1160
1161 static inline bool cpu_has_vmx_preemption_timer(void)
1162 {
1163         return vmcs_config.pin_based_exec_ctrl &
1164                 PIN_BASED_VMX_PREEMPTION_TIMER;
1165 }
1166
1167 static inline bool cpu_has_vmx_posted_intr(void)
1168 {
1169         return IS_ENABLED(CONFIG_X86_LOCAL_APIC) &&
1170                 vmcs_config.pin_based_exec_ctrl & PIN_BASED_POSTED_INTR;
1171 }
1172
1173 static inline bool cpu_has_vmx_apicv(void)
1174 {
1175         return cpu_has_vmx_apic_register_virt() &&
1176                 cpu_has_vmx_virtual_intr_delivery() &&
1177                 cpu_has_vmx_posted_intr();
1178 }
1179
1180 static inline bool cpu_has_vmx_flexpriority(void)
1181 {
1182         return cpu_has_vmx_tpr_shadow() &&
1183                 cpu_has_vmx_virtualize_apic_accesses();
1184 }
1185
1186 static inline bool cpu_has_vmx_ept_execute_only(void)
1187 {
1188         return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
1189 }
1190
1191 static inline bool cpu_has_vmx_ept_2m_page(void)
1192 {
1193         return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
1194 }
1195
1196 static inline bool cpu_has_vmx_ept_1g_page(void)
1197 {
1198         return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
1199 }
1200
1201 static inline bool cpu_has_vmx_ept_4levels(void)
1202 {
1203         return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
1204 }
1205
1206 static inline bool cpu_has_vmx_ept_mt_wb(void)
1207 {
1208         return vmx_capability.ept & VMX_EPTP_WB_BIT;
1209 }
1210
1211 static inline bool cpu_has_vmx_ept_5levels(void)
1212 {
1213         return vmx_capability.ept & VMX_EPT_PAGE_WALK_5_BIT;
1214 }
1215
1216 static inline bool cpu_has_vmx_ept_ad_bits(void)
1217 {
1218         return vmx_capability.ept & VMX_EPT_AD_BIT;
1219 }
1220
1221 static inline bool cpu_has_vmx_invept_context(void)
1222 {
1223         return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
1224 }
1225
1226 static inline bool cpu_has_vmx_invept_global(void)
1227 {
1228         return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
1229 }
1230
1231 static inline bool cpu_has_vmx_invvpid_single(void)
1232 {
1233         return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
1234 }
1235
1236 static inline bool cpu_has_vmx_invvpid_global(void)
1237 {
1238         return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
1239 }
1240
1241 static inline bool cpu_has_vmx_invvpid(void)
1242 {
1243         return vmx_capability.vpid & VMX_VPID_INVVPID_BIT;
1244 }
1245
1246 static inline bool cpu_has_vmx_ept(void)
1247 {
1248         return vmcs_config.cpu_based_2nd_exec_ctrl &
1249                 SECONDARY_EXEC_ENABLE_EPT;
1250 }
1251
1252 static inline bool cpu_has_vmx_unrestricted_guest(void)
1253 {
1254         return vmcs_config.cpu_based_2nd_exec_ctrl &
1255                 SECONDARY_EXEC_UNRESTRICTED_GUEST;
1256 }
1257
1258 static inline bool cpu_has_vmx_ple(void)
1259 {
1260         return vmcs_config.cpu_based_2nd_exec_ctrl &
1261                 SECONDARY_EXEC_PAUSE_LOOP_EXITING;
1262 }
1263
1264 static inline bool cpu_has_vmx_basic_inout(void)
1265 {
1266         return  (((u64)vmcs_config.basic_cap << 32) & VMX_BASIC_INOUT);
1267 }
1268
1269 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
1270 {
1271         return flexpriority_enabled && lapic_in_kernel(vcpu);
1272 }
1273
1274 static inline bool cpu_has_vmx_vpid(void)
1275 {
1276         return vmcs_config.cpu_based_2nd_exec_ctrl &
1277                 SECONDARY_EXEC_ENABLE_VPID;
1278 }
1279
1280 static inline bool cpu_has_vmx_rdtscp(void)
1281 {
1282         return vmcs_config.cpu_based_2nd_exec_ctrl &
1283                 SECONDARY_EXEC_RDTSCP;
1284 }
1285
1286 static inline bool cpu_has_vmx_invpcid(void)
1287 {
1288         return vmcs_config.cpu_based_2nd_exec_ctrl &
1289                 SECONDARY_EXEC_ENABLE_INVPCID;
1290 }
1291
1292 static inline bool cpu_has_virtual_nmis(void)
1293 {
1294         return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
1295 }
1296
1297 static inline bool cpu_has_vmx_wbinvd_exit(void)
1298 {
1299         return vmcs_config.cpu_based_2nd_exec_ctrl &
1300                 SECONDARY_EXEC_WBINVD_EXITING;
1301 }
1302
1303 static inline bool cpu_has_vmx_shadow_vmcs(void)
1304 {
1305         u64 vmx_msr;
1306         rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
1307         /* check if the cpu supports writing r/o exit information fields */
1308         if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS))
1309                 return false;
1310
1311         return vmcs_config.cpu_based_2nd_exec_ctrl &
1312                 SECONDARY_EXEC_SHADOW_VMCS;
1313 }
1314
1315 static inline bool cpu_has_vmx_pml(void)
1316 {
1317         return vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_ENABLE_PML;
1318 }
1319
1320 static inline bool cpu_has_vmx_tsc_scaling(void)
1321 {
1322         return vmcs_config.cpu_based_2nd_exec_ctrl &
1323                 SECONDARY_EXEC_TSC_SCALING;
1324 }
1325
1326 static inline bool cpu_has_vmx_vmfunc(void)
1327 {
1328         return vmcs_config.cpu_based_2nd_exec_ctrl &
1329                 SECONDARY_EXEC_ENABLE_VMFUNC;
1330 }
1331
1332 static inline bool report_flexpriority(void)
1333 {
1334         return flexpriority_enabled;
1335 }
1336
1337 static inline unsigned nested_cpu_vmx_misc_cr3_count(struct kvm_vcpu *vcpu)
1338 {
1339         return vmx_misc_cr3_count(to_vmx(vcpu)->nested.nested_vmx_misc_low);
1340 }
1341
1342 static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
1343 {
1344         return vmcs12->cpu_based_vm_exec_control & bit;
1345 }
1346
1347 static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
1348 {
1349         return (vmcs12->cpu_based_vm_exec_control &
1350                         CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
1351                 (vmcs12->secondary_vm_exec_control & bit);
1352 }
1353
1354 static inline bool nested_cpu_has_preemption_timer(struct vmcs12 *vmcs12)
1355 {
1356         return vmcs12->pin_based_vm_exec_control &
1357                 PIN_BASED_VMX_PREEMPTION_TIMER;
1358 }
1359
1360 static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12)
1361 {
1362         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT);
1363 }
1364
1365 static inline bool nested_cpu_has_xsaves(struct vmcs12 *vmcs12)
1366 {
1367         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
1368 }
1369
1370 static inline bool nested_cpu_has_pml(struct vmcs12 *vmcs12)
1371 {
1372         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_PML);
1373 }
1374
1375 static inline bool nested_cpu_has_virt_x2apic_mode(struct vmcs12 *vmcs12)
1376 {
1377         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
1378 }
1379
1380 static inline bool nested_cpu_has_vpid(struct vmcs12 *vmcs12)
1381 {
1382         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VPID);
1383 }
1384
1385 static inline bool nested_cpu_has_apic_reg_virt(struct vmcs12 *vmcs12)
1386 {
1387         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_APIC_REGISTER_VIRT);
1388 }
1389
1390 static inline bool nested_cpu_has_vid(struct vmcs12 *vmcs12)
1391 {
1392         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
1393 }
1394
1395 static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12)
1396 {
1397         return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR;
1398 }
1399
1400 static inline bool nested_cpu_has_vmfunc(struct vmcs12 *vmcs12)
1401 {
1402         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VMFUNC);
1403 }
1404
1405 static inline bool nested_cpu_has_eptp_switching(struct vmcs12 *vmcs12)
1406 {
1407         return nested_cpu_has_vmfunc(vmcs12) &&
1408                 (vmcs12->vm_function_control &
1409                  VMX_VMFUNC_EPTP_SWITCHING);
1410 }
1411
1412 static inline bool is_nmi(u32 intr_info)
1413 {
1414         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1415                 == (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK);
1416 }
1417
1418 static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
1419                               u32 exit_intr_info,
1420                               unsigned long exit_qualification);
1421 static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
1422                         struct vmcs12 *vmcs12,
1423                         u32 reason, unsigned long qualification);
1424
1425 static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
1426 {
1427         int i;
1428
1429         for (i = 0; i < vmx->nmsrs; ++i)
1430                 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
1431                         return i;
1432         return -1;
1433 }
1434
1435 static inline void __invvpid(int ext, u16 vpid, gva_t gva)
1436 {
1437     struct {
1438         u64 vpid : 16;
1439         u64 rsvd : 48;
1440         u64 gva;
1441     } operand = { vpid, 0, gva };
1442
1443     asm volatile (__ex(ASM_VMX_INVVPID)
1444                   /* CF==1 or ZF==1 --> rc = -1 */
1445                   "; ja 1f ; ud2 ; 1:"
1446                   : : "a"(&operand), "c"(ext) : "cc", "memory");
1447 }
1448
1449 static inline void __invept(int ext, u64 eptp, gpa_t gpa)
1450 {
1451         struct {
1452                 u64 eptp, gpa;
1453         } operand = {eptp, gpa};
1454
1455         asm volatile (__ex(ASM_VMX_INVEPT)
1456                         /* CF==1 or ZF==1 --> rc = -1 */
1457                         "; ja 1f ; ud2 ; 1:\n"
1458                         : : "a" (&operand), "c" (ext) : "cc", "memory");
1459 }
1460
1461 static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
1462 {
1463         int i;
1464
1465         i = __find_msr_index(vmx, msr);
1466         if (i >= 0)
1467                 return &vmx->guest_msrs[i];
1468         return NULL;
1469 }
1470
1471 static void vmcs_clear(struct vmcs *vmcs)
1472 {
1473         u64 phys_addr = __pa(vmcs);
1474         u8 error;
1475
1476         asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
1477                       : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1478                       : "cc", "memory");
1479         if (error)
1480                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
1481                        vmcs, phys_addr);
1482 }
1483
1484 static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
1485 {
1486         vmcs_clear(loaded_vmcs->vmcs);
1487         if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
1488                 vmcs_clear(loaded_vmcs->shadow_vmcs);
1489         loaded_vmcs->cpu = -1;
1490         loaded_vmcs->launched = 0;
1491 }
1492
1493 static void vmcs_load(struct vmcs *vmcs)
1494 {
1495         u64 phys_addr = __pa(vmcs);
1496         u8 error;
1497
1498         asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
1499                         : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1500                         : "cc", "memory");
1501         if (error)
1502                 printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
1503                        vmcs, phys_addr);
1504 }
1505
1506 #ifdef CONFIG_KEXEC_CORE
1507 /*
1508  * This bitmap is used to indicate whether the vmclear
1509  * operation is enabled on all cpus. All disabled by
1510  * default.
1511  */
1512 static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;
1513
1514 static inline void crash_enable_local_vmclear(int cpu)
1515 {
1516         cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
1517 }
1518
1519 static inline void crash_disable_local_vmclear(int cpu)
1520 {
1521         cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
1522 }
1523
1524 static inline int crash_local_vmclear_enabled(int cpu)
1525 {
1526         return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
1527 }
1528
1529 static void crash_vmclear_local_loaded_vmcss(void)
1530 {
1531         int cpu = raw_smp_processor_id();
1532         struct loaded_vmcs *v;
1533
1534         if (!crash_local_vmclear_enabled(cpu))
1535                 return;
1536
1537         list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
1538                             loaded_vmcss_on_cpu_link)
1539                 vmcs_clear(v->vmcs);
1540 }
1541 #else
1542 static inline void crash_enable_local_vmclear(int cpu) { }
1543 static inline void crash_disable_local_vmclear(int cpu) { }
1544 #endif /* CONFIG_KEXEC_CORE */
1545
1546 static void __loaded_vmcs_clear(void *arg)
1547 {
1548         struct loaded_vmcs *loaded_vmcs = arg;
1549         int cpu = raw_smp_processor_id();
1550
1551         if (loaded_vmcs->cpu != cpu)
1552                 return; /* vcpu migration can race with cpu offline */
1553         if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
1554                 per_cpu(current_vmcs, cpu) = NULL;
1555         crash_disable_local_vmclear(cpu);
1556         list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
1557
1558         /*
1559          * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
1560          * is before setting loaded_vmcs->vcpu to -1 which is done in
1561          * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
1562          * then adds the vmcs into percpu list before it is deleted.
1563          */
1564         smp_wmb();
1565
1566         loaded_vmcs_init(loaded_vmcs);
1567         crash_enable_local_vmclear(cpu);
1568 }
1569
1570 static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
1571 {
1572         int cpu = loaded_vmcs->cpu;
1573
1574         if (cpu != -1)
1575                 smp_call_function_single(cpu,
1576                          __loaded_vmcs_clear, loaded_vmcs, 1);
1577 }
1578
1579 static inline void vpid_sync_vcpu_single(int vpid)
1580 {
1581         if (vpid == 0)
1582                 return;
1583
1584         if (cpu_has_vmx_invvpid_single())
1585                 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
1586 }
1587
1588 static inline void vpid_sync_vcpu_global(void)
1589 {
1590         if (cpu_has_vmx_invvpid_global())
1591                 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
1592 }
1593
1594 static inline void vpid_sync_context(int vpid)
1595 {
1596         if (cpu_has_vmx_invvpid_single())
1597                 vpid_sync_vcpu_single(vpid);
1598         else
1599                 vpid_sync_vcpu_global();
1600 }
1601
1602 static inline void ept_sync_global(void)
1603 {
1604         if (cpu_has_vmx_invept_global())
1605                 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
1606 }
1607
1608 static inline void ept_sync_context(u64 eptp)
1609 {
1610         if (enable_ept) {
1611                 if (cpu_has_vmx_invept_context())
1612                         __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
1613                 else
1614                         ept_sync_global();
1615         }
1616 }
1617
1618 static __always_inline void vmcs_check16(unsigned long field)
1619 {
1620         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
1621                          "16-bit accessor invalid for 64-bit field");
1622         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
1623                          "16-bit accessor invalid for 64-bit high field");
1624         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
1625                          "16-bit accessor invalid for 32-bit high field");
1626         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
1627                          "16-bit accessor invalid for natural width field");
1628 }
1629
1630 static __always_inline void vmcs_check32(unsigned long field)
1631 {
1632         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
1633                          "32-bit accessor invalid for 16-bit field");
1634         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
1635                          "32-bit accessor invalid for natural width field");
1636 }
1637
1638 static __always_inline void vmcs_check64(unsigned long field)
1639 {
1640         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
1641                          "64-bit accessor invalid for 16-bit field");
1642         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
1643                          "64-bit accessor invalid for 64-bit high field");
1644         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
1645                          "64-bit accessor invalid for 32-bit field");
1646         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
1647                          "64-bit accessor invalid for natural width field");
1648 }
1649
1650 static __always_inline void vmcs_checkl(unsigned long field)
1651 {
1652         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
1653                          "Natural width accessor invalid for 16-bit field");
1654         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
1655                          "Natural width accessor invalid for 64-bit field");
1656         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
1657                          "Natural width accessor invalid for 64-bit high field");
1658         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
1659                          "Natural width accessor invalid for 32-bit field");
1660 }
1661
1662 static __always_inline unsigned long __vmcs_readl(unsigned long field)
1663 {
1664         unsigned long value;
1665
1666         asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
1667                       : "=a"(value) : "d"(field) : "cc");
1668         return value;
1669 }
1670
1671 static __always_inline u16 vmcs_read16(unsigned long field)
1672 {
1673         vmcs_check16(field);
1674         return __vmcs_readl(field);
1675 }
1676
1677 static __always_inline u32 vmcs_read32(unsigned long field)
1678 {
1679         vmcs_check32(field);
1680         return __vmcs_readl(field);
1681 }
1682
1683 static __always_inline u64 vmcs_read64(unsigned long field)
1684 {
1685         vmcs_check64(field);
1686 #ifdef CONFIG_X86_64
1687         return __vmcs_readl(field);
1688 #else
1689         return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
1690 #endif
1691 }
1692
1693 static __always_inline unsigned long vmcs_readl(unsigned long field)
1694 {
1695         vmcs_checkl(field);
1696         return __vmcs_readl(field);
1697 }
1698
1699 static noinline void vmwrite_error(unsigned long field, unsigned long value)
1700 {
1701         printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
1702                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
1703         dump_stack();
1704 }
1705
1706 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
1707 {
1708         u8 error;
1709
1710         asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
1711                        : "=q"(error) : "a"(value), "d"(field) : "cc");
1712         if (unlikely(error))
1713                 vmwrite_error(field, value);
1714 }
1715
1716 static __always_inline void vmcs_write16(unsigned long field, u16 value)
1717 {
1718         vmcs_check16(field);
1719         __vmcs_writel(field, value);
1720 }
1721
1722 static __always_inline void vmcs_write32(unsigned long field, u32 value)
1723 {
1724         vmcs_check32(field);
1725         __vmcs_writel(field, value);
1726 }
1727
1728 static __always_inline void vmcs_write64(unsigned long field, u64 value)
1729 {
1730         vmcs_check64(field);
1731         __vmcs_writel(field, value);
1732 #ifndef CONFIG_X86_64
1733         asm volatile ("");
1734         __vmcs_writel(field+1, value >> 32);
1735 #endif
1736 }
1737
1738 static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
1739 {
1740         vmcs_checkl(field);
1741         __vmcs_writel(field, value);
1742 }
1743
1744 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
1745 {
1746         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
1747                          "vmcs_clear_bits does not support 64-bit fields");
1748         __vmcs_writel(field, __vmcs_readl(field) & ~mask);
1749 }
1750
1751 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
1752 {
1753         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
1754                          "vmcs_set_bits does not support 64-bit fields");
1755         __vmcs_writel(field, __vmcs_readl(field) | mask);
1756 }
1757
1758 static inline void vm_entry_controls_reset_shadow(struct vcpu_vmx *vmx)
1759 {
1760         vmx->vm_entry_controls_shadow = vmcs_read32(VM_ENTRY_CONTROLS);
1761 }
1762
1763 static inline void vm_entry_controls_init(struct vcpu_vmx *vmx, u32 val)
1764 {
1765         vmcs_write32(VM_ENTRY_CONTROLS, val);
1766         vmx->vm_entry_controls_shadow = val;
1767 }
1768
1769 static inline void vm_entry_controls_set(struct vcpu_vmx *vmx, u32 val)
1770 {
1771         if (vmx->vm_entry_controls_shadow != val)
1772                 vm_entry_controls_init(vmx, val);
1773 }
1774
1775 static inline u32 vm_entry_controls_get(struct vcpu_vmx *vmx)
1776 {
1777         return vmx->vm_entry_controls_shadow;
1778 }
1779
1780
1781 static inline void vm_entry_controls_setbit(struct vcpu_vmx *vmx, u32 val)
1782 {
1783         vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) | val);
1784 }
1785
1786 static inline void vm_entry_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
1787 {
1788         vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) & ~val);
1789 }
1790
1791 static inline void vm_exit_controls_reset_shadow(struct vcpu_vmx *vmx)
1792 {
1793         vmx->vm_exit_controls_shadow = vmcs_read32(VM_EXIT_CONTROLS);
1794 }
1795
1796 static inline void vm_exit_controls_init(struct vcpu_vmx *vmx, u32 val)
1797 {
1798         vmcs_write32(VM_EXIT_CONTROLS, val);
1799         vmx->vm_exit_controls_shadow = val;
1800 }
1801
1802 static inline void vm_exit_controls_set(struct vcpu_vmx *vmx, u32 val)
1803 {
1804         if (vmx->vm_exit_controls_shadow != val)
1805                 vm_exit_controls_init(vmx, val);
1806 }
1807
1808 static inline u32 vm_exit_controls_get(struct vcpu_vmx *vmx)
1809 {
1810         return vmx->vm_exit_controls_shadow;
1811 }
1812
1813
1814 static inline void vm_exit_controls_setbit(struct vcpu_vmx *vmx, u32 val)
1815 {
1816         vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) | val);
1817 }
1818
1819 static inline void vm_exit_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
1820 {
1821         vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) & ~val);
1822 }
1823
1824 static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
1825 {
1826         vmx->segment_cache.bitmask = 0;
1827 }
1828
1829 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
1830                                        unsigned field)
1831 {
1832         bool ret;
1833         u32 mask = 1 << (seg * SEG_FIELD_NR + field);
1834
1835         if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
1836                 vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
1837                 vmx->segment_cache.bitmask = 0;
1838         }
1839         ret = vmx->segment_cache.bitmask & mask;
1840         vmx->segment_cache.bitmask |= mask;
1841         return ret;
1842 }
1843
1844 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
1845 {
1846         u16 *p = &vmx->segment_cache.seg[seg].selector;
1847
1848         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
1849                 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
1850         return *p;
1851 }
1852
1853 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
1854 {
1855         ulong *p = &vmx->segment_cache.seg[seg].base;
1856
1857         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
1858                 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
1859         return *p;
1860 }
1861
1862 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
1863 {
1864         u32 *p = &vmx->segment_cache.seg[seg].limit;
1865
1866         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
1867                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
1868         return *p;
1869 }
1870
1871 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
1872 {
1873         u32 *p = &vmx->segment_cache.seg[seg].ar;
1874
1875         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
1876                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
1877         return *p;
1878 }
1879
1880 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
1881 {
1882         u32 eb;
1883
1884         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
1885              (1u << DB_VECTOR) | (1u << AC_VECTOR);
1886         if ((vcpu->guest_debug &
1887              (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
1888             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
1889                 eb |= 1u << BP_VECTOR;
1890         if (to_vmx(vcpu)->rmode.vm86_active)
1891                 eb = ~0;
1892         if (enable_ept)
1893                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
1894
1895         /* When we are running a nested L2 guest and L1 specified for it a
1896          * certain exception bitmap, we must trap the same exceptions and pass
1897          * them to L1. When running L2, we will only handle the exceptions
1898          * specified above if L1 did not want them.
1899          */
1900         if (is_guest_mode(vcpu))
1901                 eb |= get_vmcs12(vcpu)->exception_bitmap;
1902
1903         vmcs_write32(EXCEPTION_BITMAP, eb);
1904 }
1905
1906 /*
1907  * Check if MSR is intercepted for L01 MSR bitmap.
1908  */
1909 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
1910 {
1911         unsigned long *msr_bitmap;
1912         int f = sizeof(unsigned long);
1913
1914         if (!cpu_has_vmx_msr_bitmap())
1915                 return true;
1916
1917         msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
1918
1919         if (msr <= 0x1fff) {
1920                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
1921         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
1922                 msr &= 0x1fff;
1923                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
1924         }
1925
1926         return true;
1927 }
1928
1929 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
1930                 unsigned long entry, unsigned long exit)
1931 {
1932         vm_entry_controls_clearbit(vmx, entry);
1933         vm_exit_controls_clearbit(vmx, exit);
1934 }
1935
1936 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
1937 {
1938         unsigned i;
1939         struct msr_autoload *m = &vmx->msr_autoload;
1940
1941         switch (msr) {
1942         case MSR_EFER:
1943                 if (cpu_has_load_ia32_efer) {
1944                         clear_atomic_switch_msr_special(vmx,
1945                                         VM_ENTRY_LOAD_IA32_EFER,
1946                                         VM_EXIT_LOAD_IA32_EFER);
1947                         return;
1948                 }
1949                 break;
1950         case MSR_CORE_PERF_GLOBAL_CTRL:
1951                 if (cpu_has_load_perf_global_ctrl) {
1952                         clear_atomic_switch_msr_special(vmx,
1953                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
1954                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
1955                         return;
1956                 }
1957                 break;
1958         }
1959
1960         for (i = 0; i < m->nr; ++i)
1961                 if (m->guest[i].index == msr)
1962                         break;
1963
1964         if (i == m->nr)
1965                 return;
1966         --m->nr;
1967         m->guest[i] = m->guest[m->nr];
1968         m->host[i] = m->host[m->nr];
1969         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
1970         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
1971 }
1972
1973 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
1974                 unsigned long entry, unsigned long exit,
1975                 unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
1976                 u64 guest_val, u64 host_val)
1977 {
1978         vmcs_write64(guest_val_vmcs, guest_val);
1979         vmcs_write64(host_val_vmcs, host_val);
1980         vm_entry_controls_setbit(vmx, entry);
1981         vm_exit_controls_setbit(vmx, exit);
1982 }
1983
1984 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
1985                                   u64 guest_val, u64 host_val)
1986 {
1987         unsigned i;
1988         struct msr_autoload *m = &vmx->msr_autoload;
1989
1990         switch (msr) {
1991         case MSR_EFER:
1992                 if (cpu_has_load_ia32_efer) {
1993                         add_atomic_switch_msr_special(vmx,
1994                                         VM_ENTRY_LOAD_IA32_EFER,
1995                                         VM_EXIT_LOAD_IA32_EFER,
1996                                         GUEST_IA32_EFER,
1997                                         HOST_IA32_EFER,
1998                                         guest_val, host_val);
1999                         return;
2000                 }
2001                 break;
2002         case MSR_CORE_PERF_GLOBAL_CTRL:
2003                 if (cpu_has_load_perf_global_ctrl) {
2004                         add_atomic_switch_msr_special(vmx,
2005                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
2006                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
2007                                         GUEST_IA32_PERF_GLOBAL_CTRL,
2008                                         HOST_IA32_PERF_GLOBAL_CTRL,
2009                                         guest_val, host_val);
2010                         return;
2011                 }
2012                 break;
2013         case MSR_IA32_PEBS_ENABLE:
2014                 /* PEBS needs a quiescent period after being disabled (to write
2015                  * a record).  Disabling PEBS through VMX MSR swapping doesn't
2016                  * provide that period, so a CPU could write host's record into
2017                  * guest's memory.
2018                  */
2019                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
2020         }
2021
2022         for (i = 0; i < m->nr; ++i)
2023                 if (m->guest[i].index == msr)
2024                         break;
2025
2026         if (i == NR_AUTOLOAD_MSRS) {
2027                 printk_once(KERN_WARNING "Not enough msr switch entries. "
2028                                 "Can't add msr %x\n", msr);
2029                 return;
2030         } else if (i == m->nr) {
2031                 ++m->nr;
2032                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
2033                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
2034         }
2035
2036         m->guest[i].index = msr;
2037         m->guest[i].value = guest_val;
2038         m->host[i].index = msr;
2039         m->host[i].value = host_val;
2040 }
2041
2042 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
2043 {
2044         u64 guest_efer = vmx->vcpu.arch.efer;
2045         u64 ignore_bits = 0;
2046
2047         if (!enable_ept) {
2048                 /*
2049                  * NX is needed to handle CR0.WP=1, CR4.SMEP=1.  Testing
2050                  * host CPUID is more efficient than testing guest CPUID
2051                  * or CR4.  Host SMEP is anyway a requirement for guest SMEP.
2052                  */
2053                 if (boot_cpu_has(X86_FEATURE_SMEP))
2054                         guest_efer |= EFER_NX;
2055                 else if (!(guest_efer & EFER_NX))
2056                         ignore_bits |= EFER_NX;
2057         }
2058
2059         /*
2060          * LMA and LME handled by hardware; SCE meaningless outside long mode.
2061          */
2062         ignore_bits |= EFER_SCE;
2063 #ifdef CONFIG_X86_64
2064         ignore_bits |= EFER_LMA | EFER_LME;
2065         /* SCE is meaningful only in long mode on Intel */
2066         if (guest_efer & EFER_LMA)
2067                 ignore_bits &= ~(u64)EFER_SCE;
2068 #endif
2069
2070         clear_atomic_switch_msr(vmx, MSR_EFER);
2071
2072         /*
2073          * On EPT, we can't emulate NX, so we must switch EFER atomically.
2074          * On CPUs that support "load IA32_EFER", always switch EFER
2075          * atomically, since it's faster than switching it manually.
2076          */
2077         if (cpu_has_load_ia32_efer ||
2078             (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
2079                 if (!(guest_efer & EFER_LMA))
2080                         guest_efer &= ~EFER_LME;
2081                 if (guest_efer != host_efer)
2082                         add_atomic_switch_msr(vmx, MSR_EFER,
2083                                               guest_efer, host_efer);
2084                 return false;
2085         } else {
2086                 guest_efer &= ~ignore_bits;
2087                 guest_efer |= host_efer & ignore_bits;
2088
2089                 vmx->guest_msrs[efer_offset].data = guest_efer;
2090                 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
2091
2092                 return true;
2093         }
2094 }
2095
2096 #ifdef CONFIG_X86_32
2097 /*
2098  * On 32-bit kernels, VM exits still load the FS and GS bases from the
2099  * VMCS rather than the segment table.  KVM uses this helper to figure
2100  * out the current bases to poke them into the VMCS before entry.
2101  */
2102 static unsigned long segment_base(u16 selector)
2103 {
2104         struct desc_struct *table;
2105         unsigned long v;
2106
2107         if (!(selector & ~SEGMENT_RPL_MASK))
2108                 return 0;
2109
2110         table = get_current_gdt_ro();
2111
2112         if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2113                 u16 ldt_selector = kvm_read_ldt();
2114
2115                 if (!(ldt_selector & ~SEGMENT_RPL_MASK))
2116                         return 0;
2117
2118                 table = (struct desc_struct *)segment_base(ldt_selector);
2119         }
2120         v = get_desc_base(&table[selector >> 3]);
2121         return v;
2122 }
2123 #endif
2124
2125 static void vmx_save_host_state(struct kvm_vcpu *vcpu)
2126 {
2127         struct vcpu_vmx *vmx = to_vmx(vcpu);
2128         int i;
2129
2130         if (vmx->host_state.loaded)
2131                 return;
2132
2133         vmx->host_state.loaded = 1;
2134         /*
2135          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
2136          * allow segment selectors with cpl > 0 or ti == 1.
2137          */
2138         vmx->host_state.ldt_sel = kvm_read_ldt();
2139         vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
2140         savesegment(fs, vmx->host_state.fs_sel);
2141         if (!(vmx->host_state.fs_sel & 7)) {
2142                 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
2143                 vmx->host_state.fs_reload_needed = 0;
2144         } else {
2145                 vmcs_write16(HOST_FS_SELECTOR, 0);
2146                 vmx->host_state.fs_reload_needed = 1;
2147         }
2148         savesegment(gs, vmx->host_state.gs_sel);
2149         if (!(vmx->host_state.gs_sel & 7))
2150                 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
2151         else {
2152                 vmcs_write16(HOST_GS_SELECTOR, 0);
2153                 vmx->host_state.gs_ldt_reload_needed = 1;
2154         }
2155
2156 #ifdef CONFIG_X86_64
2157         savesegment(ds, vmx->host_state.ds_sel);
2158         savesegment(es, vmx->host_state.es_sel);
2159 #endif
2160
2161 #ifdef CONFIG_X86_64
2162         vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
2163         vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
2164 #else
2165         vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
2166         vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
2167 #endif
2168
2169 #ifdef CONFIG_X86_64
2170         rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
2171         if (is_long_mode(&vmx->vcpu))
2172                 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
2173 #endif
2174         if (boot_cpu_has(X86_FEATURE_MPX))
2175                 rdmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
2176         for (i = 0; i < vmx->save_nmsrs; ++i)
2177                 kvm_set_shared_msr(vmx->guest_msrs[i].index,
2178                                    vmx->guest_msrs[i].data,
2179                                    vmx->guest_msrs[i].mask);
2180 }
2181
2182 static void __vmx_load_host_state(struct vcpu_vmx *vmx)
2183 {
2184         if (!vmx->host_state.loaded)
2185                 return;
2186
2187         ++vmx->vcpu.stat.host_state_reload;
2188         vmx->host_state.loaded = 0;
2189 #ifdef CONFIG_X86_64
2190         if (is_long_mode(&vmx->vcpu))
2191                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
2192 #endif
2193         if (vmx->host_state.gs_ldt_reload_needed) {
2194                 kvm_load_ldt(vmx->host_state.ldt_sel);
2195 #ifdef CONFIG_X86_64
2196                 load_gs_index(vmx->host_state.gs_sel);
2197 #else
2198                 loadsegment(gs, vmx->host_state.gs_sel);
2199 #endif
2200         }
2201         if (vmx->host_state.fs_reload_needed)
2202                 loadsegment(fs, vmx->host_state.fs_sel);
2203 #ifdef CONFIG_X86_64
2204         if (unlikely(vmx->host_state.ds_sel | vmx->host_state.es_sel)) {
2205                 loadsegment(ds, vmx->host_state.ds_sel);
2206                 loadsegment(es, vmx->host_state.es_sel);
2207         }
2208 #endif
2209         invalidate_tss_limit();
2210 #ifdef CONFIG_X86_64
2211         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
2212 #endif
2213         if (vmx->host_state.msr_host_bndcfgs)
2214                 wrmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
2215         load_fixmap_gdt(raw_smp_processor_id());
2216 }
2217
2218 static void vmx_load_host_state(struct vcpu_vmx *vmx)
2219 {
2220         preempt_disable();
2221         __vmx_load_host_state(vmx);
2222         preempt_enable();
2223 }
2224
2225 static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
2226 {
2227         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
2228         struct pi_desc old, new;
2229         unsigned int dest;
2230
2231         /*
2232          * In case of hot-plug or hot-unplug, we may have to undo
2233          * vmx_vcpu_pi_put even if there is no assigned device.  And we
2234          * always keep PI.NDST up to date for simplicity: it makes the
2235          * code easier, and CPU migration is not a fast path.
2236          */
2237         if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu)
2238                 return;
2239
2240         /*
2241          * First handle the simple case where no cmpxchg is necessary; just
2242          * allow posting non-urgent interrupts.
2243          *
2244          * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change
2245          * PI.NDST: pi_post_block will do it for us and the wakeup_handler
2246          * expects the VCPU to be on the blocked_vcpu_list that matches
2247          * PI.NDST.
2248          */
2249         if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR ||
2250             vcpu->cpu == cpu) {
2251                 pi_clear_sn(pi_desc);
2252                 return;
2253         }
2254
2255         /* The full case.  */
2256         do {
2257                 old.control = new.control = pi_desc->control;
2258
2259                 dest = cpu_physical_id(cpu);
2260
2261                 if (x2apic_enabled())
2262                         new.ndst = dest;
2263                 else
2264                         new.ndst = (dest << 8) & 0xFF00;
2265
2266                 new.sn = 0;
2267         } while (cmpxchg64(&pi_desc->control, old.control,
2268                            new.control) != old.control);
2269 }
2270
2271 static void decache_tsc_multiplier(struct vcpu_vmx *vmx)
2272 {
2273         vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio;
2274         vmcs_write64(TSC_MULTIPLIER, vmx->current_tsc_ratio);
2275 }
2276
2277 /*
2278  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
2279  * vcpu mutex is already taken.
2280  */
2281 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2282 {
2283         struct vcpu_vmx *vmx = to_vmx(vcpu);
2284         bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
2285
2286         if (!already_loaded) {
2287                 loaded_vmcs_clear(vmx->loaded_vmcs);
2288                 local_irq_disable();
2289                 crash_disable_local_vmclear(cpu);
2290
2291                 /*
2292                  * Read loaded_vmcs->cpu should be before fetching
2293                  * loaded_vmcs->loaded_vmcss_on_cpu_link.
2294                  * See the comments in __loaded_vmcs_clear().
2295                  */
2296                 smp_rmb();
2297
2298                 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
2299                          &per_cpu(loaded_vmcss_on_cpu, cpu));
2300                 crash_enable_local_vmclear(cpu);
2301                 local_irq_enable();
2302         }
2303
2304         if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
2305                 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
2306                 vmcs_load(vmx->loaded_vmcs->vmcs);
2307                 indirect_branch_prediction_barrier();
2308         }
2309
2310         if (!already_loaded) {
2311                 void *gdt = get_current_gdt_ro();
2312                 unsigned long sysenter_esp;
2313
2314                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2315
2316                 /*
2317                  * Linux uses per-cpu TSS and GDT, so set these when switching
2318                  * processors.  See 22.2.4.
2319                  */
2320                 vmcs_writel(HOST_TR_BASE,
2321                             (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
2322                 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt);   /* 22.2.4 */
2323
2324                 /*
2325                  * VM exits change the host TR limit to 0x67 after a VM
2326                  * exit.  This is okay, since 0x67 covers everything except
2327                  * the IO bitmap and have have code to handle the IO bitmap
2328                  * being lost after a VM exit.
2329                  */
2330                 BUILD_BUG_ON(IO_BITMAP_OFFSET - 1 != 0x67);
2331
2332                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
2333                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
2334
2335                 vmx->loaded_vmcs->cpu = cpu;
2336         }
2337
2338         /* Setup TSC multiplier */
2339         if (kvm_has_tsc_control &&
2340             vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
2341                 decache_tsc_multiplier(vmx);
2342
2343         vmx_vcpu_pi_load(vcpu, cpu);
2344         vmx->host_pkru = read_pkru();
2345 }
2346
2347 static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
2348 {
2349         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
2350
2351         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
2352                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
2353                 !kvm_vcpu_apicv_active(vcpu))
2354                 return;
2355
2356         /* Set SN when the vCPU is preempted */
2357         if (vcpu->preempted)
2358                 pi_set_sn(pi_desc);
2359 }
2360
2361 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
2362 {
2363         vmx_vcpu_pi_put(vcpu);
2364
2365         __vmx_load_host_state(to_vmx(vcpu));
2366 }
2367
2368 static bool emulation_required(struct kvm_vcpu *vcpu)
2369 {
2370         return emulate_invalid_guest_state && !guest_state_valid(vcpu);
2371 }
2372
2373 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
2374
2375 /*
2376  * Return the cr0 value that a nested guest would read. This is a combination
2377  * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
2378  * its hypervisor (cr0_read_shadow).
2379  */
2380 static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
2381 {
2382         return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
2383                 (fields->cr0_read_shadow & fields->cr0_guest_host_mask);
2384 }
2385 static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
2386 {
2387         return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
2388                 (fields->cr4_read_shadow & fields->cr4_guest_host_mask);
2389 }
2390
2391 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
2392 {
2393         unsigned long rflags, save_rflags;
2394
2395         if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
2396                 __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
2397                 rflags = vmcs_readl(GUEST_RFLAGS);
2398                 if (to_vmx(vcpu)->rmode.vm86_active) {
2399                         rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
2400                         save_rflags = to_vmx(vcpu)->rmode.save_rflags;
2401                         rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
2402                 }
2403                 to_vmx(vcpu)->rflags = rflags;
2404         }
2405         return to_vmx(vcpu)->rflags;
2406 }
2407
2408 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2409 {
2410         unsigned long old_rflags = vmx_get_rflags(vcpu);
2411
2412         __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
2413         to_vmx(vcpu)->rflags = rflags;
2414         if (to_vmx(vcpu)->rmode.vm86_active) {
2415                 to_vmx(vcpu)->rmode.save_rflags = rflags;
2416                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
2417         }
2418         vmcs_writel(GUEST_RFLAGS, rflags);
2419
2420         if ((old_rflags ^ to_vmx(vcpu)->rflags) & X86_EFLAGS_VM)
2421                 to_vmx(vcpu)->emulation_required = emulation_required(vcpu);
2422 }
2423
2424 static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
2425 {
2426         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
2427         int ret = 0;
2428
2429         if (interruptibility & GUEST_INTR_STATE_STI)
2430                 ret |= KVM_X86_SHADOW_INT_STI;
2431         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
2432                 ret |= KVM_X86_SHADOW_INT_MOV_SS;
2433
2434         return ret;
2435 }
2436
2437 static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
2438 {
2439         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
2440         u32 interruptibility = interruptibility_old;
2441
2442         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
2443
2444         if (mask & KVM_X86_SHADOW_INT_MOV_SS)
2445                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
2446         else if (mask & KVM_X86_SHADOW_INT_STI)
2447                 interruptibility |= GUEST_INTR_STATE_STI;
2448
2449         if ((interruptibility != interruptibility_old))
2450                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
2451 }
2452
2453 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
2454 {
2455         unsigned long rip;
2456
2457         rip = kvm_rip_read(vcpu);
2458         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
2459         kvm_rip_write(vcpu, rip);
2460
2461         /* skipping an emulated instruction also counts */
2462         vmx_set_interrupt_shadow(vcpu, 0);
2463 }
2464
2465 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu,
2466                                                unsigned long exit_qual)
2467 {
2468         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
2469         unsigned int nr = vcpu->arch.exception.nr;
2470         u32 intr_info = nr | INTR_INFO_VALID_MASK;
2471
2472         if (vcpu->arch.exception.has_error_code) {
2473                 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code;
2474                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
2475         }
2476
2477         if (kvm_exception_is_soft(nr))
2478                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
2479         else
2480                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
2481
2482         if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
2483             vmx_get_nmi_mask(vcpu))
2484                 intr_info |= INTR_INFO_UNBLOCK_NMI;
2485
2486         nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
2487 }
2488
2489 /*
2490  * KVM wants to inject page-faults which it got to the guest. This function
2491  * checks whether in a nested guest, we need to inject them to L1 or L2.
2492  */
2493 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual)
2494 {
2495         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
2496         unsigned int nr = vcpu->arch.exception.nr;
2497
2498         if (nr == PF_VECTOR) {
2499                 if (vcpu->arch.exception.nested_apf) {
2500                         *exit_qual = vcpu->arch.apf.nested_apf_token;
2501                         return 1;
2502                 }
2503                 /*
2504                  * FIXME: we must not write CR2 when L1 intercepts an L2 #PF exception.
2505                  * The fix is to add the ancillary datum (CR2 or DR6) to structs
2506                  * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6
2507                  * can be written only when inject_pending_event runs.  This should be
2508                  * conditional on a new capability---if the capability is disabled,
2509                  * kvm_multiple_exception would write the ancillary information to
2510                  * CR2 or DR6, for backwards ABI-compatibility.
2511                  */
2512                 if (nested_vmx_is_page_fault_vmexit(vmcs12,
2513                                                     vcpu->arch.exception.error_code)) {
2514                         *exit_qual = vcpu->arch.cr2;
2515                         return 1;
2516                 }
2517         } else {
2518                 if (vmcs12->exception_bitmap & (1u << nr)) {
2519                         if (nr == DB_VECTOR)
2520                                 *exit_qual = vcpu->arch.dr6;
2521                         else
2522                                 *exit_qual = 0;
2523                         return 1;
2524                 }
2525         }
2526
2527         return 0;
2528 }
2529
2530 static void vmx_queue_exception(struct kvm_vcpu *vcpu)
2531 {
2532         struct vcpu_vmx *vmx = to_vmx(vcpu);
2533         unsigned nr = vcpu->arch.exception.nr;
2534         bool has_error_code = vcpu->arch.exception.has_error_code;
2535         u32 error_code = vcpu->arch.exception.error_code;
2536         u32 intr_info = nr | INTR_INFO_VALID_MASK;
2537
2538         if (has_error_code) {
2539                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
2540                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
2541         }
2542
2543         if (vmx->rmode.vm86_active) {
2544                 int inc_eip = 0;
2545                 if (kvm_exception_is_soft(nr))
2546                         inc_eip = vcpu->arch.event_exit_inst_len;
2547                 if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
2548                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2549                 return;
2550         }
2551
2552         if (kvm_exception_is_soft(nr)) {
2553                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2554                              vmx->vcpu.arch.event_exit_inst_len);
2555                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
2556         } else
2557                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
2558
2559         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
2560 }
2561
2562 static bool vmx_rdtscp_supported(void)
2563 {
2564         return cpu_has_vmx_rdtscp();
2565 }
2566
2567 static bool vmx_invpcid_supported(void)
2568 {
2569         return cpu_has_vmx_invpcid() && enable_ept;
2570 }
2571
2572 /*
2573  * Swap MSR entry in host/guest MSR entry array.
2574  */
2575 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
2576 {
2577         struct shared_msr_entry tmp;
2578
2579         tmp = vmx->guest_msrs[to];
2580         vmx->guest_msrs[to] = vmx->guest_msrs[from];
2581         vmx->guest_msrs[from] = tmp;
2582 }
2583
2584 /*
2585  * Set up the vmcs to automatically save and restore system
2586  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
2587  * mode, as fiddling with msrs is very expensive.
2588  */
2589 static void setup_msrs(struct vcpu_vmx *vmx)
2590 {
2591         int save_nmsrs, index;
2592
2593         save_nmsrs = 0;
2594 #ifdef CONFIG_X86_64
2595         if (is_long_mode(&vmx->vcpu)) {
2596                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
2597                 if (index >= 0)
2598                         move_msr_up(vmx, index, save_nmsrs++);
2599                 index = __find_msr_index(vmx, MSR_LSTAR);
2600                 if (index >= 0)
2601                         move_msr_up(vmx, index, save_nmsrs++);
2602                 index = __find_msr_index(vmx, MSR_CSTAR);
2603                 if (index >= 0)
2604                         move_msr_up(vmx, index, save_nmsrs++);
2605                 index = __find_msr_index(vmx, MSR_TSC_AUX);
2606                 if (index >= 0 && guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP))
2607                         move_msr_up(vmx, index, save_nmsrs++);
2608                 /*
2609                  * MSR_STAR is only needed on long mode guests, and only
2610                  * if efer.sce is enabled.
2611                  */
2612                 index = __find_msr_index(vmx, MSR_STAR);
2613                 if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
2614                         move_msr_up(vmx, index, save_nmsrs++);
2615         }
2616 #endif
2617         index = __find_msr_index(vmx, MSR_EFER);
2618         if (index >= 0 && update_transition_efer(vmx, index))
2619                 move_msr_up(vmx, index, save_nmsrs++);
2620
2621         vmx->save_nmsrs = save_nmsrs;
2622
2623         if (cpu_has_vmx_msr_bitmap())
2624                 vmx_update_msr_bitmap(&vmx->vcpu);
2625 }
2626
2627 /*
2628  * reads and returns guest's timestamp counter "register"
2629  * guest_tsc = (host_tsc * tsc multiplier) >> 48 + tsc_offset
2630  * -- Intel TSC Scaling for Virtualization White Paper, sec 1.3
2631  */
2632 static u64 guest_read_tsc(struct kvm_vcpu *vcpu)
2633 {
2634         u64 host_tsc, tsc_offset;
2635
2636         host_tsc = rdtsc();
2637         tsc_offset = vmcs_read64(TSC_OFFSET);
2638         return kvm_scale_tsc(vcpu, host_tsc) + tsc_offset;
2639 }
2640
2641 /*
2642  * writes 'offset' into guest's timestamp counter offset register
2643  */
2644 static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
2645 {
2646         if (is_guest_mode(vcpu)) {
2647                 /*
2648                  * We're here if L1 chose not to trap WRMSR to TSC. According
2649                  * to the spec, this should set L1's TSC; The offset that L1
2650                  * set for L2 remains unchanged, and still needs to be added
2651                  * to the newly set TSC to get L2's TSC.
2652                  */
2653                 struct vmcs12 *vmcs12;
2654                 /* recalculate vmcs02.TSC_OFFSET: */
2655                 vmcs12 = get_vmcs12(vcpu);
2656                 vmcs_write64(TSC_OFFSET, offset +
2657                         (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ?
2658                          vmcs12->tsc_offset : 0));
2659         } else {
2660                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
2661                                            vmcs_read64(TSC_OFFSET), offset);
2662                 vmcs_write64(TSC_OFFSET, offset);
2663         }
2664 }
2665
2666 /*
2667  * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
2668  * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
2669  * all guests if the "nested" module option is off, and can also be disabled
2670  * for a single guest by disabling its VMX cpuid bit.
2671  */
2672 static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
2673 {
2674         return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
2675 }
2676
2677 /*
2678  * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
2679  * returned for the various VMX controls MSRs when nested VMX is enabled.
2680  * The same values should also be used to verify that vmcs12 control fields are
2681  * valid during nested entry from L1 to L2.
2682  * Each of these control msrs has a low and high 32-bit half: A low bit is on
2683  * if the corresponding bit in the (32-bit) control field *must* be on, and a
2684  * bit in the high half is on if the corresponding bit in the control field
2685  * may be on. See also vmx_control_verify().
2686  */
2687 static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
2688 {
2689         /*
2690          * Note that as a general rule, the high half of the MSRs (bits in
2691          * the control fields which may be 1) should be initialized by the
2692          * intersection of the underlying hardware's MSR (i.e., features which
2693          * can be supported) and the list of features we want to expose -
2694          * because they are known to be properly supported in our code.
2695          * Also, usually, the low half of the MSRs (bits which must be 1) can
2696          * be set to 0, meaning that L1 may turn off any of these bits. The
2697          * reason is that if one of these bits is necessary, it will appear
2698          * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
2699          * fields of vmcs01 and vmcs02, will turn these bits off - and
2700          * nested_vmx_exit_reflected() will not pass related exits to L1.
2701          * These rules have exceptions below.
2702          */
2703
2704         /* pin-based controls */
2705         rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
2706                 vmx->nested.nested_vmx_pinbased_ctls_low,
2707                 vmx->nested.nested_vmx_pinbased_ctls_high);
2708         vmx->nested.nested_vmx_pinbased_ctls_low |=
2709                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
2710         vmx->nested.nested_vmx_pinbased_ctls_high &=
2711                 PIN_BASED_EXT_INTR_MASK |
2712                 PIN_BASED_NMI_EXITING |
2713                 PIN_BASED_VIRTUAL_NMIS;
2714         vmx->nested.nested_vmx_pinbased_ctls_high |=
2715                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
2716                 PIN_BASED_VMX_PREEMPTION_TIMER;
2717         if (kvm_vcpu_apicv_active(&vmx->vcpu))
2718                 vmx->nested.nested_vmx_pinbased_ctls_high |=
2719                         PIN_BASED_POSTED_INTR;
2720
2721         /* exit controls */
2722         rdmsr(MSR_IA32_VMX_EXIT_CTLS,
2723                 vmx->nested.nested_vmx_exit_ctls_low,
2724                 vmx->nested.nested_vmx_exit_ctls_high);
2725         vmx->nested.nested_vmx_exit_ctls_low =
2726                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
2727
2728         vmx->nested.nested_vmx_exit_ctls_high &=
2729 #ifdef CONFIG_X86_64
2730                 VM_EXIT_HOST_ADDR_SPACE_SIZE |
2731 #endif
2732                 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
2733         vmx->nested.nested_vmx_exit_ctls_high |=
2734                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
2735                 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
2736                 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
2737
2738         if (kvm_mpx_supported())
2739                 vmx->nested.nested_vmx_exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
2740
2741         /* We support free control of debug control saving. */
2742         vmx->nested.nested_vmx_exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
2743
2744         /* entry controls */
2745         rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
2746                 vmx->nested.nested_vmx_entry_ctls_low,
2747                 vmx->nested.nested_vmx_entry_ctls_high);
2748         vmx->nested.nested_vmx_entry_ctls_low =
2749                 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
2750         vmx->nested.nested_vmx_entry_ctls_high &=
2751 #ifdef CONFIG_X86_64
2752                 VM_ENTRY_IA32E_MODE |
2753 #endif
2754                 VM_ENTRY_LOAD_IA32_PAT;
2755         vmx->nested.nested_vmx_entry_ctls_high |=
2756                 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
2757         if (kvm_mpx_supported())
2758                 vmx->nested.nested_vmx_entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
2759
2760         /* We support free control of debug control loading. */
2761         vmx->nested.nested_vmx_entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
2762
2763         /* cpu-based controls */
2764         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
2765                 vmx->nested.nested_vmx_procbased_ctls_low,
2766                 vmx->nested.nested_vmx_procbased_ctls_high);
2767         vmx->nested.nested_vmx_procbased_ctls_low =
2768                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
2769         vmx->nested.nested_vmx_procbased_ctls_high &=
2770                 CPU_BASED_VIRTUAL_INTR_PENDING |
2771                 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING |
2772                 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
2773                 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
2774                 CPU_BASED_CR3_STORE_EXITING |
2775 #ifdef CONFIG_X86_64
2776                 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
2777 #endif
2778                 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
2779                 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
2780                 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
2781                 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
2782                 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2783         /*
2784          * We can allow some features even when not supported by the
2785          * hardware. For example, L1 can specify an MSR bitmap - and we
2786          * can use it to avoid exits to L1 - even when L0 runs L2
2787          * without MSR bitmaps.
2788          */
2789         vmx->nested.nested_vmx_procbased_ctls_high |=
2790                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
2791                 CPU_BASED_USE_MSR_BITMAPS;
2792
2793         /* We support free control of CR3 access interception. */
2794         vmx->nested.nested_vmx_procbased_ctls_low &=
2795                 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
2796
2797         /*
2798          * secondary cpu-based controls.  Do not include those that
2799          * depend on CPUID bits, they are added later by vmx_cpuid_update.
2800          */
2801         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
2802                 vmx->nested.nested_vmx_secondary_ctls_low,
2803                 vmx->nested.nested_vmx_secondary_ctls_high);
2804         vmx->nested.nested_vmx_secondary_ctls_low = 0;
2805         vmx->nested.nested_vmx_secondary_ctls_high &=
2806                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2807                 SECONDARY_EXEC_DESC |
2808                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2809                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2810                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2811                 SECONDARY_EXEC_WBINVD_EXITING;
2812
2813         if (enable_ept) {
2814                 /* nested EPT: emulate EPT also to L1 */
2815                 vmx->nested.nested_vmx_secondary_ctls_high |=
2816                         SECONDARY_EXEC_ENABLE_EPT;
2817                 vmx->nested.nested_vmx_ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
2818                          VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT;
2819                 if (cpu_has_vmx_ept_execute_only())
2820                         vmx->nested.nested_vmx_ept_caps |=
2821                                 VMX_EPT_EXECUTE_ONLY_BIT;
2822                 vmx->nested.nested_vmx_ept_caps &= vmx_capability.ept;
2823                 vmx->nested.nested_vmx_ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
2824                         VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
2825                         VMX_EPT_1GB_PAGE_BIT;
2826                 if (enable_ept_ad_bits) {
2827                         vmx->nested.nested_vmx_secondary_ctls_high |=
2828                                 SECONDARY_EXEC_ENABLE_PML;
2829                         vmx->nested.nested_vmx_ept_caps |= VMX_EPT_AD_BIT;
2830                 }
2831         } else
2832                 vmx->nested.nested_vmx_ept_caps = 0;
2833
2834         if (cpu_has_vmx_vmfunc()) {
2835                 vmx->nested.nested_vmx_secondary_ctls_high |=
2836                         SECONDARY_EXEC_ENABLE_VMFUNC;
2837                 /*
2838                  * Advertise EPTP switching unconditionally
2839                  * since we emulate it
2840                  */
2841                 if (enable_ept)
2842                         vmx->nested.nested_vmx_vmfunc_controls =
2843                                 VMX_VMFUNC_EPTP_SWITCHING;
2844         }
2845
2846         /*
2847          * Old versions of KVM use the single-context version without
2848          * checking for support, so declare that it is supported even
2849          * though it is treated as global context.  The alternative is
2850          * not failing the single-context invvpid, and it is worse.
2851          */
2852         if (enable_vpid) {
2853                 vmx->nested.nested_vmx_secondary_ctls_high |=
2854                         SECONDARY_EXEC_ENABLE_VPID;
2855                 vmx->nested.nested_vmx_vpid_caps = VMX_VPID_INVVPID_BIT |
2856                         VMX_VPID_EXTENT_SUPPORTED_MASK;
2857         } else
2858                 vmx->nested.nested_vmx_vpid_caps = 0;
2859
2860         if (enable_unrestricted_guest)
2861                 vmx->nested.nested_vmx_secondary_ctls_high |=
2862                         SECONDARY_EXEC_UNRESTRICTED_GUEST;
2863
2864         /* miscellaneous data */
2865         rdmsr(MSR_IA32_VMX_MISC,
2866                 vmx->nested.nested_vmx_misc_low,
2867                 vmx->nested.nested_vmx_misc_high);
2868         vmx->nested.nested_vmx_misc_low &= VMX_MISC_SAVE_EFER_LMA;
2869         vmx->nested.nested_vmx_misc_low |=
2870                 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
2871                 VMX_MISC_ACTIVITY_HLT;
2872         vmx->nested.nested_vmx_misc_high = 0;
2873
2874         /*
2875          * This MSR reports some information about VMX support. We
2876          * should return information about the VMX we emulate for the
2877          * guest, and the VMCS structure we give it - not about the
2878          * VMX support of the underlying hardware.
2879          */
2880         vmx->nested.nested_vmx_basic =
2881                 VMCS12_REVISION |
2882                 VMX_BASIC_TRUE_CTLS |
2883                 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
2884                 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
2885
2886         if (cpu_has_vmx_basic_inout())
2887                 vmx->nested.nested_vmx_basic |= VMX_BASIC_INOUT;
2888
2889         /*
2890          * These MSRs specify bits which the guest must keep fixed on
2891          * while L1 is in VMXON mode (in L1's root mode, or running an L2).
2892          * We picked the standard core2 setting.
2893          */
2894 #define VMXON_CR0_ALWAYSON     (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
2895 #define VMXON_CR4_ALWAYSON     X86_CR4_VMXE
2896         vmx->nested.nested_vmx_cr0_fixed0 = VMXON_CR0_ALWAYSON;
2897         vmx->nested.nested_vmx_cr4_fixed0 = VMXON_CR4_ALWAYSON;
2898
2899         /* These MSRs specify bits which the guest must keep fixed off. */
2900         rdmsrl(MSR_IA32_VMX_CR0_FIXED1, vmx->nested.nested_vmx_cr0_fixed1);
2901         rdmsrl(MSR_IA32_VMX_CR4_FIXED1, vmx->nested.nested_vmx_cr4_fixed1);
2902
2903         /* highest index: VMX_PREEMPTION_TIMER_VALUE */
2904         vmx->nested.nested_vmx_vmcs_enum = 0x2e;
2905 }
2906
2907 /*
2908  * if fixed0[i] == 1: val[i] must be 1
2909  * if fixed1[i] == 0: val[i] must be 0
2910  */
2911 static inline bool fixed_bits_valid(u64 val, u64 fixed0, u64 fixed1)
2912 {
2913         return ((val & fixed1) | fixed0) == val;
2914 }
2915
2916 static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
2917 {
2918         return fixed_bits_valid(control, low, high);
2919 }
2920
2921 static inline u64 vmx_control_msr(u32 low, u32 high)
2922 {
2923         return low | ((u64)high << 32);
2924 }
2925
2926 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
2927 {
2928         superset &= mask;
2929         subset &= mask;
2930
2931         return (superset | subset) == superset;
2932 }
2933
2934 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
2935 {
2936         const u64 feature_and_reserved =
2937                 /* feature (except bit 48; see below) */
2938                 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
2939                 /* reserved */
2940                 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
2941         u64 vmx_basic = vmx->nested.nested_vmx_basic;
2942
2943         if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
2944                 return -EINVAL;
2945
2946         /*
2947          * KVM does not emulate a version of VMX that constrains physical
2948          * addresses of VMX structures (e.g. VMCS) to 32-bits.
2949          */
2950         if (data & BIT_ULL(48))
2951                 return -EINVAL;
2952
2953         if (vmx_basic_vmcs_revision_id(vmx_basic) !=
2954             vmx_basic_vmcs_revision_id(data))
2955                 return -EINVAL;
2956
2957         if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
2958                 return -EINVAL;
2959
2960         vmx->nested.nested_vmx_basic = data;
2961         return 0;
2962 }
2963
2964 static int
2965 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
2966 {
2967         u64 supported;
2968         u32 *lowp, *highp;
2969
2970         switch (msr_index) {
2971         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
2972                 lowp = &vmx->nested.nested_vmx_pinbased_ctls_low;
2973                 highp = &vmx->nested.nested_vmx_pinbased_ctls_high;
2974                 break;
2975         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
2976                 lowp = &vmx->nested.nested_vmx_procbased_ctls_low;
2977                 highp = &vmx->nested.nested_vmx_procbased_ctls_high;
2978                 break;
2979         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
2980                 lowp = &vmx->nested.nested_vmx_exit_ctls_low;
2981                 highp = &vmx->nested.nested_vmx_exit_ctls_high;
2982                 break;
2983         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
2984                 lowp = &vmx->nested.nested_vmx_entry_ctls_low;
2985                 highp = &vmx->nested.nested_vmx_entry_ctls_high;
2986                 break;
2987         case MSR_IA32_VMX_PROCBASED_CTLS2:
2988                 lowp = &vmx->nested.nested_vmx_secondary_ctls_low;
2989                 highp = &vmx->nested.nested_vmx_secondary_ctls_high;
2990                 break;
2991         default:
2992                 BUG();
2993         }
2994
2995         supported = vmx_control_msr(*lowp, *highp);
2996
2997         /* Check must-be-1 bits are still 1. */
2998         if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
2999                 return -EINVAL;
3000
3001         /* Check must-be-0 bits are still 0. */
3002         if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
3003                 return -EINVAL;
3004
3005         *lowp = data;
3006         *highp = data >> 32;
3007         return 0;
3008 }
3009
3010 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
3011 {
3012         const u64 feature_and_reserved_bits =
3013                 /* feature */
3014                 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
3015                 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
3016                 /* reserved */
3017                 GENMASK_ULL(13, 9) | BIT_ULL(31);
3018         u64 vmx_misc;
3019
3020         vmx_misc = vmx_control_msr(vmx->nested.nested_vmx_misc_low,
3021                                    vmx->nested.nested_vmx_misc_high);
3022
3023         if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
3024                 return -EINVAL;
3025
3026         if ((vmx->nested.nested_vmx_pinbased_ctls_high &
3027              PIN_BASED_VMX_PREEMPTION_TIMER) &&
3028             vmx_misc_preemption_timer_rate(data) !=
3029             vmx_misc_preemption_timer_rate(vmx_misc))
3030                 return -EINVAL;
3031
3032         if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
3033                 return -EINVAL;
3034
3035         if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
3036                 return -EINVAL;
3037
3038         if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
3039                 return -EINVAL;
3040
3041         vmx->nested.nested_vmx_misc_low = data;
3042         vmx->nested.nested_vmx_misc_high = data >> 32;
3043         return 0;
3044 }
3045
3046 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
3047 {
3048         u64 vmx_ept_vpid_cap;
3049
3050         vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.nested_vmx_ept_caps,
3051                                            vmx->nested.nested_vmx_vpid_caps);
3052
3053         /* Every bit is either reserved or a feature bit. */
3054         if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
3055                 return -EINVAL;
3056
3057         vmx->nested.nested_vmx_ept_caps = data;
3058         vmx->nested.nested_vmx_vpid_caps = data >> 32;
3059         return 0;
3060 }
3061
3062 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
3063 {
3064         u64 *msr;
3065
3066         switch (msr_index) {
3067         case MSR_IA32_VMX_CR0_FIXED0:
3068                 msr = &vmx->nested.nested_vmx_cr0_fixed0;
3069                 break;
3070         case MSR_IA32_VMX_CR4_FIXED0:
3071                 msr = &vmx->nested.nested_vmx_cr4_fixed0;
3072                 break;
3073         default:
3074                 BUG();
3075         }
3076
3077         /*
3078          * 1 bits (which indicates bits which "must-be-1" during VMX operation)
3079          * must be 1 in the restored value.
3080          */
3081         if (!is_bitwise_subset(data, *msr, -1ULL))
3082                 return -EINVAL;
3083
3084         *msr = data;
3085         return 0;
3086 }
3087
3088 /*
3089  * Called when userspace is restoring VMX MSRs.
3090  *
3091  * Returns 0 on success, non-0 otherwise.
3092  */
3093 static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
3094 {
3095         struct vcpu_vmx *vmx = to_vmx(vcpu);
3096
3097         switch (msr_index) {
3098         case MSR_IA32_VMX_BASIC:
3099                 return vmx_restore_vmx_basic(vmx, data);
3100         case MSR_IA32_VMX_PINBASED_CTLS:
3101         case MSR_IA32_VMX_PROCBASED_CTLS:
3102         case MSR_IA32_VMX_EXIT_CTLS:
3103         case MSR_IA32_VMX_ENTRY_CTLS:
3104                 /*
3105                  * The "non-true" VMX capability MSRs are generated from the
3106                  * "true" MSRs, so we do not support restoring them directly.
3107                  *
3108                  * If userspace wants to emulate VMX_BASIC[55]=0, userspace
3109                  * should restore the "true" MSRs with the must-be-1 bits
3110                  * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
3111                  * DEFAULT SETTINGS".
3112                  */
3113                 return -EINVAL;
3114         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
3115         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
3116         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
3117         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
3118         case MSR_IA32_VMX_PROCBASED_CTLS2:
3119                 return vmx_restore_control_msr(vmx, msr_index, data);
3120         case MSR_IA32_VMX_MISC:
3121                 return vmx_restore_vmx_misc(vmx, data);
3122         case MSR_IA32_VMX_CR0_FIXED0:
3123         case MSR_IA32_VMX_CR4_FIXED0:
3124                 return vmx_restore_fixed0_msr(vmx, msr_index, data);
3125         case MSR_IA32_VMX_CR0_FIXED1:
3126         case MSR_IA32_VMX_CR4_FIXED1:
3127                 /*
3128                  * These MSRs are generated based on the vCPU's CPUID, so we
3129                  * do not support restoring them directly.
3130                  */
3131                 return -EINVAL;
3132         case MSR_IA32_VMX_EPT_VPID_CAP:
3133                 return vmx_restore_vmx_ept_vpid_cap(vmx, data);
3134         case MSR_IA32_VMX_VMCS_ENUM:
3135                 vmx->nested.nested_vmx_vmcs_enum = data;
3136                 return 0;
3137         default:
3138                 /*
3139                  * The rest of the VMX capability MSRs do not support restore.
3140                  */
3141                 return -EINVAL;
3142         }
3143 }
3144
3145 /* Returns 0 on success, non-0 otherwise. */
3146 static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
3147 {
3148         struct vcpu_vmx *vmx = to_vmx(vcpu);
3149
3150         switch (msr_index) {
3151         case MSR_IA32_VMX_BASIC:
3152                 *pdata = vmx->nested.nested_vmx_basic;
3153                 break;
3154         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
3155         case MSR_IA32_VMX_PINBASED_CTLS:
3156                 *pdata = vmx_control_msr(
3157                         vmx->nested.nested_vmx_pinbased_ctls_low,
3158                         vmx->nested.nested_vmx_pinbased_ctls_high);
3159                 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
3160                         *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
3161                 break;
3162         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
3163         case MSR_IA32_VMX_PROCBASED_CTLS:
3164                 *pdata = vmx_control_msr(
3165                         vmx->nested.nested_vmx_procbased_ctls_low,
3166                         vmx->nested.nested_vmx_procbased_ctls_high);
3167                 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
3168                         *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
3169                 break;
3170         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
3171         case MSR_IA32_VMX_EXIT_CTLS:
3172                 *pdata = vmx_control_msr(
3173                         vmx->nested.nested_vmx_exit_ctls_low,
3174                         vmx->nested.nested_vmx_exit_ctls_high);
3175                 if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
3176                         *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
3177                 break;
3178         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
3179         case MSR_IA32_VMX_ENTRY_CTLS:
3180                 *pdata = vmx_control_msr(
3181                         vmx->nested.nested_vmx_entry_ctls_low,
3182                         vmx->nested.nested_vmx_entry_ctls_high);
3183                 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
3184                         *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
3185                 break;
3186         case MSR_IA32_VMX_MISC:
3187                 *pdata = vmx_control_msr(
3188                         vmx->nested.nested_vmx_misc_low,
3189                         vmx->nested.nested_vmx_misc_high);
3190                 break;
3191         case MSR_IA32_VMX_CR0_FIXED0:
3192                 *pdata = vmx->nested.nested_vmx_cr0_fixed0;
3193                 break;
3194         case MSR_IA32_VMX_CR0_FIXED1:
3195                 *pdata = vmx->nested.nested_vmx_cr0_fixed1;
3196                 break;
3197         case MSR_IA32_VMX_CR4_FIXED0:
3198                 *pdata = vmx->nested.nested_vmx_cr4_fixed0;
3199                 break;
3200         case MSR_IA32_VMX_CR4_FIXED1:
3201                 *pdata = vmx->nested.nested_vmx_cr4_fixed1;
3202                 break;
3203         case MSR_IA32_VMX_VMCS_ENUM:
3204                 *pdata = vmx->nested.nested_vmx_vmcs_enum;
3205                 break;
3206         case MSR_IA32_VMX_PROCBASED_CTLS2:
3207                 *pdata = vmx_control_msr(
3208                         vmx->nested.nested_vmx_secondary_ctls_low,
3209                         vmx->nested.nested_vmx_secondary_ctls_high);
3210                 break;
3211         case MSR_IA32_VMX_EPT_VPID_CAP:
3212                 *pdata = vmx->nested.nested_vmx_ept_caps |
3213                         ((u64)vmx->nested.nested_vmx_vpid_caps << 32);
3214                 break;
3215         case MSR_IA32_VMX_VMFUNC:
3216                 *pdata = vmx->nested.nested_vmx_vmfunc_controls;
3217                 break;
3218         default:
3219                 return 1;
3220         }
3221
3222         return 0;
3223 }
3224
3225 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
3226                                                  uint64_t val)
3227 {
3228         uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
3229
3230         return !(val & ~valid_bits);
3231 }
3232
3233 /*
3234  * Reads an msr value (of 'msr_index') into 'pdata'.
3235  * Returns 0 on success, non-0 otherwise.
3236  * Assumes vcpu_load() was already called.
3237  */
3238 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3239 {
3240         struct shared_msr_entry *msr;
3241
3242         switch (msr_info->index) {
3243 #ifdef CONFIG_X86_64
3244         case MSR_FS_BASE:
3245                 msr_info->data = vmcs_readl(GUEST_FS_BASE);
3246                 break;
3247         case MSR_GS_BASE:
3248                 msr_info->data = vmcs_readl(GUEST_GS_BASE);
3249                 break;
3250         case MSR_KERNEL_GS_BASE:
3251                 vmx_load_host_state(to_vmx(vcpu));
3252                 msr_info->data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
3253                 break;
3254 #endif
3255         case MSR_EFER:
3256                 return kvm_get_msr_common(vcpu, msr_info);
3257         case MSR_IA32_TSC:
3258                 msr_info->data = guest_read_tsc(vcpu);
3259                 break;
3260         case MSR_IA32_SYSENTER_CS:
3261                 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
3262                 break;
3263         case MSR_IA32_SYSENTER_EIP:
3264                 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
3265                 break;
3266         case MSR_IA32_SYSENTER_ESP:
3267                 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
3268                 break;
3269         case MSR_IA32_BNDCFGS:
3270                 if (!kvm_mpx_supported() ||
3271                     (!msr_info->host_initiated &&
3272                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
3273                         return 1;
3274                 msr_info->data = vmcs_read64(GUEST_BNDCFGS);
3275                 break;
3276         case MSR_IA32_MCG_EXT_CTL:
3277                 if (!msr_info->host_initiated &&
3278                     !(to_vmx(vcpu)->msr_ia32_feature_control &
3279                       FEATURE_CONTROL_LMCE))
3280                         return 1;
3281                 msr_info->data = vcpu->arch.mcg_ext_ctl;
3282                 break;
3283         case MSR_IA32_FEATURE_CONTROL:
3284                 msr_info->data = to_vmx(vcpu)->msr_ia32_feature_control;
3285                 break;
3286         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
3287                 if (!nested_vmx_allowed(vcpu))
3288                         return 1;
3289                 return vmx_get_vmx_msr(vcpu, msr_info->index, &msr_info->data);
3290         case MSR_IA32_XSS:
3291                 if (!vmx_xsaves_supported())
3292                         return 1;
3293                 msr_info->data = vcpu->arch.ia32_xss;
3294                 break;
3295         case MSR_TSC_AUX:
3296                 if (!msr_info->host_initiated &&
3297                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
3298                         return 1;
3299                 /* Otherwise falls through */
3300         default:
3301                 msr = find_msr_entry(to_vmx(vcpu), msr_info->index);
3302                 if (msr) {
3303                         msr_info->data = msr->data;
3304                         break;
3305                 }
3306                 return kvm_get_msr_common(vcpu, msr_info);
3307         }
3308
3309         return 0;
3310 }
3311
3312 static void vmx_leave_nested(struct kvm_vcpu *vcpu);
3313
3314 /*
3315  * Writes msr value into into the appropriate "register".
3316  * Returns 0 on success, non-0 otherwise.
3317  * Assumes vcpu_load() was already called.
3318  */
3319 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3320 {
3321         struct vcpu_vmx *vmx = to_vmx(vcpu);
3322         struct shared_msr_entry *msr;
3323         int ret = 0;
3324         u32 msr_index = msr_info->index;
3325         u64 data = msr_info->data;
3326
3327         switch (msr_index) {
3328         case MSR_EFER:
3329                 ret = kvm_set_msr_common(vcpu, msr_info);
3330                 break;
3331 #ifdef CONFIG_X86_64
3332         case MSR_FS_BASE:
3333                 vmx_segment_cache_clear(vmx);
3334                 vmcs_writel(GUEST_FS_BASE, data);
3335                 break;
3336         case MSR_GS_BASE:
3337                 vmx_segment_cache_clear(vmx);
3338                 vmcs_writel(GUEST_GS_BASE, data);
3339                 break;
3340         case MSR_KERNEL_GS_BASE:
3341                 vmx_load_host_state(vmx);
3342                 vmx->msr_guest_kernel_gs_base = data;
3343                 break;
3344 #endif
3345         case MSR_IA32_SYSENTER_CS:
3346                 vmcs_write32(GUEST_SYSENTER_CS, data);
3347                 break;
3348         case MSR_IA32_SYSENTER_EIP:
3349                 vmcs_writel(GUEST_SYSENTER_EIP, data);
3350                 break;
3351         case MSR_IA32_SYSENTER_ESP:
3352                 vmcs_writel(GUEST_SYSENTER_ESP, data);
3353                 break;
3354         case MSR_IA32_BNDCFGS:
3355                 if (!kvm_mpx_supported() ||
3356                     (!msr_info->host_initiated &&
3357                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
3358                         return 1;
3359                 if (is_noncanonical_address(data & PAGE_MASK, vcpu) ||
3360                     (data & MSR_IA32_BNDCFGS_RSVD))
3361                         return 1;
3362                 vmcs_write64(GUEST_BNDCFGS, data);
3363                 break;
3364         case MSR_IA32_TSC:
3365                 kvm_write_tsc(vcpu, msr_info);
3366                 break;
3367         case MSR_IA32_PRED_CMD:
3368                 if (!msr_info->host_initiated &&
3369                     !guest_cpuid_has(vcpu, X86_FEATURE_IBPB) &&
3370                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
3371                         return 1;
3372
3373                 if (data & ~PRED_CMD_IBPB)
3374                         return 1;
3375
3376                 if (!data)
3377                         break;
3378
3379                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
3380
3381                 /*
3382                  * For non-nested:
3383                  * When it's written (to non-zero) for the first time, pass
3384                  * it through.
3385                  *
3386                  * For nested:
3387                  * The handling of the MSR bitmap for L2 guests is done in
3388                  * nested_vmx_merge_msr_bitmap. We should not touch the
3389                  * vmcs02.msr_bitmap here since it gets completely overwritten
3390                  * in the merging.
3391                  */
3392                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, MSR_IA32_PRED_CMD,
3393                                               MSR_TYPE_W);
3394                 break;
3395         case MSR_IA32_CR_PAT:
3396                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
3397                         if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
3398                                 return 1;
3399                         vmcs_write64(GUEST_IA32_PAT, data);
3400                         vcpu->arch.pat = data;
3401                         break;
3402                 }
3403                 ret = kvm_set_msr_common(vcpu, msr_info);
3404                 break;
3405         case MSR_IA32_TSC_ADJUST:
3406                 ret = kvm_set_msr_common(vcpu, msr_info);
3407                 break;
3408         case MSR_IA32_MCG_EXT_CTL:
3409                 if ((!msr_info->host_initiated &&
3410                      !(to_vmx(vcpu)->msr_ia32_feature_control &
3411                        FEATURE_CONTROL_LMCE)) ||
3412                     (data & ~MCG_EXT_CTL_LMCE_EN))
3413                         return 1;
3414                 vcpu->arch.mcg_ext_ctl = data;
3415                 break;
3416         case MSR_IA32_FEATURE_CONTROL:
3417                 if (!vmx_feature_control_msr_valid(vcpu, data) ||
3418                     (to_vmx(vcpu)->msr_ia32_feature_control &
3419                      FEATURE_CONTROL_LOCKED && !msr_info->host_initiated))
3420                         return 1;
3421                 vmx->msr_ia32_feature_control = data;
3422                 if (msr_info->host_initiated && data == 0)
3423                         vmx_leave_nested(vcpu);
3424                 break;
3425         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
3426                 if (!msr_info->host_initiated)
3427                         return 1; /* they are read-only */
3428                 if (!nested_vmx_allowed(vcpu))
3429                         return 1;
3430                 return vmx_set_vmx_msr(vcpu, msr_index, data);
3431         case MSR_IA32_XSS:
3432                 if (!vmx_xsaves_supported())
3433                         return 1;
3434                 /*
3435                  * The only supported bit as of Skylake is bit 8, but
3436                  * it is not supported on KVM.
3437                  */
3438                 if (data != 0)
3439                         return 1;
3440                 vcpu->arch.ia32_xss = data;
3441                 if (vcpu->arch.ia32_xss != host_xss)
3442                         add_atomic_switch_msr(vmx, MSR_IA32_XSS,
3443                                 vcpu->arch.ia32_xss, host_xss);
3444                 else
3445                         clear_atomic_switch_msr(vmx, MSR_IA32_XSS);
3446                 break;
3447         case MSR_TSC_AUX:
3448                 if (!msr_info->host_initiated &&
3449                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
3450                         return 1;
3451                 /* Check reserved bit, higher 32 bits should be zero */
3452                 if ((data >> 32) != 0)
3453                         return 1;
3454                 /* Otherwise falls through */
3455         default:
3456                 msr = find_msr_entry(vmx, msr_index);
3457                 if (msr) {
3458                         u64 old_msr_data = msr->data;
3459                         msr->data = data;
3460                         if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
3461                                 preempt_disable();
3462                                 ret = kvm_set_shared_msr(msr->index, msr->data,
3463                                                          msr->mask);
3464                                 preempt_enable();
3465                                 if (ret)
3466                                         msr->data = old_msr_data;
3467                         }
3468                         break;
3469                 }
3470                 ret = kvm_set_msr_common(vcpu, msr_info);
3471         }
3472
3473         return ret;
3474 }
3475
3476 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
3477 {
3478         __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
3479         switch (reg) {
3480         case VCPU_REGS_RSP:
3481                 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
3482                 break;
3483         case VCPU_REGS_RIP:
3484                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
3485                 break;
3486         case VCPU_EXREG_PDPTR:
3487                 if (enable_ept)
3488                         ept_save_pdptrs(vcpu);
3489                 break;
3490         default:
3491                 break;
3492         }
3493 }
3494
3495 static __init int cpu_has_kvm_support(void)
3496 {
3497         return cpu_has_vmx();
3498 }
3499
3500 static __init int vmx_disabled_by_bios(void)
3501 {
3502         u64 msr;
3503
3504         rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
3505         if (msr & FEATURE_CONTROL_LOCKED) {
3506                 /* launched w/ TXT and VMX disabled */
3507                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
3508                         && tboot_enabled())
3509                         return 1;
3510                 /* launched w/o TXT and VMX only enabled w/ TXT */
3511                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
3512                         && (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
3513                         && !tboot_enabled()) {
3514                         printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
3515                                 "activate TXT before enabling KVM\n");
3516                         return 1;
3517                 }
3518                 /* launched w/o TXT and VMX disabled */
3519                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
3520                         && !tboot_enabled())
3521                         return 1;
3522         }
3523
3524         return 0;
3525 }
3526
3527 static void kvm_cpu_vmxon(u64 addr)
3528 {
3529         cr4_set_bits(X86_CR4_VMXE);
3530         intel_pt_handle_vmx(1);
3531
3532         asm volatile (ASM_VMX_VMXON_RAX
3533                         : : "a"(&addr), "m"(addr)
3534                         : "memory", "cc");
3535 }
3536
3537 static int hardware_enable(void)
3538 {
3539         int cpu = raw_smp_processor_id();
3540         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
3541         u64 old, test_bits;
3542
3543         if (cr4_read_shadow() & X86_CR4_VMXE)
3544                 return -EBUSY;
3545
3546         INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
3547         INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
3548         spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
3549
3550         /*
3551          * Now we can enable the vmclear operation in kdump
3552          * since the loaded_vmcss_on_cpu list on this cpu
3553          * has been initialized.
3554          *
3555          * Though the cpu is not in VMX operation now, there
3556          * is no problem to enable the vmclear operation
3557          * for the loaded_vmcss_on_cpu list is empty!
3558          */
3559         crash_enable_local_vmclear(cpu);
3560
3561         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
3562
3563         test_bits = FEATURE_CONTROL_LOCKED;
3564         test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
3565         if (tboot_enabled())
3566                 test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;
3567
3568         if ((old & test_bits) != test_bits) {
3569                 /* enable and lock */
3570                 wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
3571         }
3572         kvm_cpu_vmxon(phys_addr);
3573         ept_sync_global();
3574
3575         return 0;
3576 }
3577
3578 static void vmclear_local_loaded_vmcss(void)
3579 {
3580         int cpu = raw_smp_processor_id();
3581         struct loaded_vmcs *v, *n;
3582
3583         list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
3584                                  loaded_vmcss_on_cpu_link)
3585                 __loaded_vmcs_clear(v);
3586 }
3587
3588
3589 /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
3590  * tricks.
3591  */
3592 static void kvm_cpu_vmxoff(void)
3593 {
3594         asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
3595
3596         intel_pt_handle_vmx(0);
3597         cr4_clear_bits(X86_CR4_VMXE);
3598 }
3599
3600 static void hardware_disable(void)
3601 {
3602         vmclear_local_loaded_vmcss();
3603         kvm_cpu_vmxoff();
3604 }
3605
3606 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
3607                                       u32 msr, u32 *result)
3608 {
3609         u32 vmx_msr_low, vmx_msr_high;
3610         u32 ctl = ctl_min | ctl_opt;
3611
3612         rdmsr(msr, vmx_msr_low, vmx_msr_high);
3613
3614         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
3615         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
3616
3617         /* Ensure minimum (required) set of control bits are supported. */
3618         if (ctl_min & ~ctl)
3619                 return -EIO;
3620
3621         *result = ctl;
3622         return 0;
3623 }
3624
3625 static __init bool allow_1_setting(u32 msr, u32 ctl)
3626 {
3627         u32 vmx_msr_low, vmx_msr_high;
3628
3629         rdmsr(msr, vmx_msr_low, vmx_msr_high);
3630         return vmx_msr_high & ctl;
3631 }
3632
3633 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
3634 {
3635         u32 vmx_msr_low, vmx_msr_high;
3636         u32 min, opt, min2, opt2;
3637         u32 _pin_based_exec_control = 0;
3638         u32 _cpu_based_exec_control = 0;
3639         u32 _cpu_based_2nd_exec_control = 0;
3640         u32 _vmexit_control = 0;
3641         u32 _vmentry_control = 0;
3642
3643         min = CPU_BASED_HLT_EXITING |
3644 #ifdef CONFIG_X86_64
3645               CPU_BASED_CR8_LOAD_EXITING |
3646               CPU_BASED_CR8_STORE_EXITING |
3647 #endif
3648               CPU_BASED_CR3_LOAD_EXITING |
3649               CPU_BASED_CR3_STORE_EXITING |
3650               CPU_BASED_USE_IO_BITMAPS |
3651               CPU_BASED_MOV_DR_EXITING |
3652               CPU_BASED_USE_TSC_OFFSETING |
3653               CPU_BASED_INVLPG_EXITING |
3654               CPU_BASED_RDPMC_EXITING;
3655
3656         if (!kvm_mwait_in_guest())
3657                 min |= CPU_BASED_MWAIT_EXITING |
3658                         CPU_BASED_MONITOR_EXITING;
3659
3660         opt = CPU_BASED_TPR_SHADOW |
3661               CPU_BASED_USE_MSR_BITMAPS |
3662               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
3663         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
3664                                 &_cpu_based_exec_control) < 0)
3665                 return -EIO;
3666 #ifdef CONFIG_X86_64
3667         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
3668                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
3669                                            ~CPU_BASED_CR8_STORE_EXITING;
3670 #endif
3671         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
3672                 min2 = 0;
3673                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
3674                         SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
3675                         SECONDARY_EXEC_WBINVD_EXITING |
3676                         SECONDARY_EXEC_ENABLE_VPID |
3677                         SECONDARY_EXEC_ENABLE_EPT |
3678                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
3679                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
3680                         SECONDARY_EXEC_RDTSCP |
3681                         SECONDARY_EXEC_ENABLE_INVPCID |
3682                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
3683                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
3684                         SECONDARY_EXEC_SHADOW_VMCS |
3685                         SECONDARY_EXEC_XSAVES |
3686                         SECONDARY_EXEC_RDSEED |
3687                         SECONDARY_EXEC_RDRAND |
3688                         SECONDARY_EXEC_ENABLE_PML |
3689                         SECONDARY_EXEC_TSC_SCALING |
3690                         SECONDARY_EXEC_ENABLE_VMFUNC;
3691                 if (adjust_vmx_controls(min2, opt2,
3692                                         MSR_IA32_VMX_PROCBASED_CTLS2,
3693                                         &_cpu_based_2nd_exec_control) < 0)
3694                         return -EIO;
3695         }
3696 #ifndef CONFIG_X86_64
3697         if (!(_cpu_based_2nd_exec_control &
3698                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
3699                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
3700 #endif
3701
3702         if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
3703                 _cpu_based_2nd_exec_control &= ~(
3704                                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
3705                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
3706                                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
3707
3708         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
3709                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
3710                    enabled */
3711                 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
3712                                              CPU_BASED_CR3_STORE_EXITING |
3713                                              CPU_BASED_INVLPG_EXITING);
3714                 rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
3715                       vmx_capability.ept, vmx_capability.vpid);
3716         }
3717
3718         min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
3719 #ifdef CONFIG_X86_64
3720         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
3721 #endif
3722         opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT |
3723                 VM_EXIT_CLEAR_BNDCFGS;
3724         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
3725                                 &_vmexit_control) < 0)
3726                 return -EIO;
3727
3728         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
3729         opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
3730                  PIN_BASED_VMX_PREEMPTION_TIMER;
3731         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
3732                                 &_pin_based_exec_control) < 0)
3733                 return -EIO;
3734
3735         if (cpu_has_broken_vmx_preemption_timer())
3736                 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
3737         if (!(_cpu_based_2nd_exec_control &
3738                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
3739                 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
3740
3741         min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
3742         opt = VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS;
3743         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
3744                                 &_vmentry_control) < 0)
3745                 return -EIO;
3746
3747         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
3748
3749         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
3750         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
3751                 return -EIO;
3752
3753 #ifdef CONFIG_X86_64
3754         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
3755         if (vmx_msr_high & (1u<<16))
3756                 return -EIO;
3757 #endif
3758
3759         /* Require Write-Back (WB) memory type for VMCS accesses. */
3760         if (((vmx_msr_high >> 18) & 15) != 6)
3761                 return -EIO;
3762
3763         vmcs_conf->size = vmx_msr_high & 0x1fff;
3764         vmcs_conf->order = get_order(vmcs_conf->size);
3765         vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
3766         vmcs_conf->revision_id = vmx_msr_low;
3767
3768         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
3769         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
3770         vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
3771         vmcs_conf->vmexit_ctrl         = _vmexit_control;
3772         vmcs_conf->vmentry_ctrl        = _vmentry_control;
3773
3774         cpu_has_load_ia32_efer =
3775                 allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
3776                                 VM_ENTRY_LOAD_IA32_EFER)
3777                 && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
3778                                    VM_EXIT_LOAD_IA32_EFER);
3779
3780         cpu_has_load_perf_global_ctrl =
3781                 allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
3782                                 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
3783                 && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
3784                                    VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
3785
3786         /*
3787          * Some cpus support VM_ENTRY_(LOAD|SAVE)_IA32_PERF_GLOBAL_CTRL
3788          * but due to errata below it can't be used. Workaround is to use
3789          * msr load mechanism to switch IA32_PERF_GLOBAL_CTRL.
3790          *
3791          * VM Exit May Incorrectly Clear IA32_PERF_GLOBAL_CTRL [34:32]
3792          *
3793          * AAK155             (model 26)
3794          * AAP115             (model 30)
3795          * AAT100             (model 37)
3796          * BC86,AAY89,BD102   (model 44)
3797          * BA97               (model 46)
3798          *
3799          */
3800         if (cpu_has_load_perf_global_ctrl && boot_cpu_data.x86 == 0x6) {
3801                 switch (boot_cpu_data.x86_model) {
3802                 case 26:
3803                 case 30:
3804                 case 37:
3805                 case 44:
3806                 case 46:
3807                         cpu_has_load_perf_global_ctrl = false;
3808                         printk_once(KERN_WARNING"kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
3809                                         "does not work properly. Using workaround\n");
3810                         break;
3811                 default:
3812                         break;
3813                 }
3814         }
3815
3816         if (boot_cpu_has(X86_FEATURE_XSAVES))
3817                 rdmsrl(MSR_IA32_XSS, host_xss);
3818
3819         return 0;
3820 }
3821
3822 static struct vmcs *alloc_vmcs_cpu(int cpu)
3823 {
3824         int node = cpu_to_node(cpu);
3825         struct page *pages;
3826         struct vmcs *vmcs;
3827
3828         pages = __alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);
3829         if (!pages)
3830                 return NULL;
3831         vmcs = page_address(pages);
3832         memset(vmcs, 0, vmcs_config.size);
3833         vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
3834         return vmcs;
3835 }
3836
3837 static void free_vmcs(struct vmcs *vmcs)
3838 {
3839         free_pages((unsigned long)vmcs, vmcs_config.order);
3840 }
3841
3842 /*
3843  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
3844  */
3845 static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
3846 {
3847         if (!loaded_vmcs->vmcs)
3848                 return;
3849         loaded_vmcs_clear(loaded_vmcs);
3850         free_vmcs(loaded_vmcs->vmcs);
3851         loaded_vmcs->vmcs = NULL;
3852         if (loaded_vmcs->msr_bitmap)
3853                 free_page((unsigned long)loaded_vmcs->msr_bitmap);
3854         WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
3855 }
3856
3857 static struct vmcs *alloc_vmcs(void)
3858 {
3859         return alloc_vmcs_cpu(raw_smp_processor_id());
3860 }
3861
3862 static int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
3863 {
3864         loaded_vmcs->vmcs = alloc_vmcs();
3865         if (!loaded_vmcs->vmcs)
3866                 return -ENOMEM;
3867
3868         loaded_vmcs->shadow_vmcs = NULL;
3869         loaded_vmcs_init(loaded_vmcs);
3870
3871         if (cpu_has_vmx_msr_bitmap()) {
3872                 loaded_vmcs->msr_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
3873                 if (!loaded_vmcs->msr_bitmap)
3874                         goto out_vmcs;
3875                 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
3876         }
3877         return 0;
3878
3879 out_vmcs:
3880         free_loaded_vmcs(loaded_vmcs);
3881         return -ENOMEM;
3882 }
3883
3884 static void free_kvm_area(void)
3885 {
3886         int cpu;
3887
3888         for_each_possible_cpu(cpu) {
3889                 free_vmcs(per_cpu(vmxarea, cpu));
3890                 per_cpu(vmxarea, cpu) = NULL;
3891         }
3892 }
3893
3894 enum vmcs_field_type {
3895         VMCS_FIELD_TYPE_U16 = 0,
3896         VMCS_FIELD_TYPE_U64 = 1,
3897         VMCS_FIELD_TYPE_U32 = 2,
3898         VMCS_FIELD_TYPE_NATURAL_WIDTH = 3
3899 };
3900
3901 static inline int vmcs_field_type(unsigned long field)
3902 {
3903         if (0x1 & field)        /* the *_HIGH fields are all 32 bit */
3904                 return VMCS_FIELD_TYPE_U32;
3905         return (field >> 13) & 0x3 ;
3906 }
3907
3908 static inline int vmcs_field_readonly(unsigned long field)
3909 {
3910         return (((field >> 10) & 0x3) == 1);
3911 }
3912
3913 static void init_vmcs_shadow_fields(void)
3914 {
3915         int i, j;
3916
3917         /* No checks for read only fields yet */
3918
3919         for (i = j = 0; i < max_shadow_read_write_fields; i++) {
3920                 switch (shadow_read_write_fields[i]) {
3921                 case GUEST_BNDCFGS:
3922                         if (!kvm_mpx_supported())
3923                                 continue;
3924                         break;
3925                 default:
3926                         break;
3927                 }
3928
3929                 if (j < i)
3930                         shadow_read_write_fields[j] =
3931                                 shadow_read_write_fields[i];
3932                 j++;
3933         }
3934         max_shadow_read_write_fields = j;
3935
3936         /* shadowed fields guest access without vmexit */
3937         for (i = 0; i < max_shadow_read_write_fields; i++) {
3938                 unsigned long field = shadow_read_write_fields[i];
3939
3940                 clear_bit(field, vmx_vmwrite_bitmap);
3941                 clear_bit(field, vmx_vmread_bitmap);
3942                 if (vmcs_field_type(field) == VMCS_FIELD_TYPE_U64) {
3943                         clear_bit(field + 1, vmx_vmwrite_bitmap);
3944                         clear_bit(field + 1, vmx_vmread_bitmap);
3945                 }
3946         }
3947         for (i = 0; i < max_shadow_read_only_fields; i++) {
3948                 unsigned long field = shadow_read_only_fields[i];
3949
3950                 clear_bit(field, vmx_vmread_bitmap);
3951                 if (vmcs_field_type(field) == VMCS_FIELD_TYPE_U64)
3952                         clear_bit(field + 1, vmx_vmread_bitmap);
3953         }
3954 }
3955
3956 static __init int alloc_kvm_area(void)
3957 {
3958         int cpu;
3959
3960         for_each_possible_cpu(cpu) {
3961                 struct vmcs *vmcs;
3962
3963                 vmcs = alloc_vmcs_cpu(cpu);
3964                 if (!vmcs) {
3965                         free_kvm_area();
3966                         return -ENOMEM;
3967                 }
3968
3969                 per_cpu(vmxarea, cpu) = vmcs;
3970         }
3971         return 0;
3972 }
3973
3974 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
3975                 struct kvm_segment *save)
3976 {
3977         if (!emulate_invalid_guest_state) {
3978                 /*
3979                  * CS and SS RPL should be equal during guest entry according
3980                  * to VMX spec, but in reality it is not always so. Since vcpu
3981                  * is in the middle of the transition from real mode to
3982                  * protected mode it is safe to assume that RPL 0 is a good
3983                  * default value.
3984                  */
3985                 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
3986                         save->selector &= ~SEGMENT_RPL_MASK;
3987                 save->dpl = save->selector & SEGMENT_RPL_MASK;
3988                 save->s = 1;
3989         }
3990         vmx_set_segment(vcpu, save, seg);
3991 }
3992
3993 static void enter_pmode(struct kvm_vcpu *vcpu)
3994 {
3995         unsigned long flags;
3996         struct vcpu_vmx *vmx = to_vmx(vcpu);
3997
3998         /*
3999          * Update real mode segment cache. It may be not up-to-date if sement
4000          * register was written while vcpu was in a guest mode.
4001          */
4002         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
4003         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
4004         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
4005         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
4006         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
4007         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
4008
4009         vmx->rmode.vm86_active = 0;
4010
4011         vmx_segment_cache_clear(vmx);
4012
4013         vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
4014
4015         flags = vmcs_readl(GUEST_RFLAGS);
4016         flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
4017         flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
4018         vmcs_writel(GUEST_RFLAGS, flags);
4019
4020         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
4021                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
4022
4023         update_exception_bitmap(vcpu);
4024
4025         fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
4026         fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
4027         fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
4028         fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
4029         fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
4030         fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
4031 }
4032
4033 static void fix_rmode_seg(int seg, struct kvm_segment *save)
4034 {
4035         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
4036         struct kvm_segment var = *save;
4037
4038         var.dpl = 0x3;
4039         if (seg == VCPU_SREG_CS)
4040                 var.type = 0x3;
4041
4042         if (!emulate_invalid_guest_state) {
4043                 var.selector = var.base >> 4;
4044                 var.base = var.base & 0xffff0;
4045                 var.limit = 0xffff;
4046                 var.g = 0;
4047                 var.db = 0;
4048                 var.present = 1;
4049                 var.s = 1;
4050                 var.l = 0;
4051                 var.unusable = 0;
4052                 var.type = 0x3;
4053                 var.avl = 0;
4054                 if (save->base & 0xf)
4055                         printk_once(KERN_WARNING "kvm: segment base is not "
4056                                         "paragraph aligned when entering "
4057                                         "protected mode (seg=%d)", seg);
4058         }
4059
4060         vmcs_write16(sf->selector, var.selector);
4061         vmcs_writel(sf->base, var.base);
4062         vmcs_write32(sf->limit, var.limit);
4063         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
4064 }
4065
4066 static void enter_rmode(struct kvm_vcpu *vcpu)
4067 {
4068         unsigned long flags;
4069         struct vcpu_vmx *vmx = to_vmx(vcpu);
4070
4071         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
4072         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
4073         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
4074         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
4075         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
4076         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
4077         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
4078
4079         vmx->rmode.vm86_active = 1;
4080
4081         /*
4082          * Very old userspace does not call KVM_SET_TSS_ADDR before entering
4083          * vcpu. Warn the user that an update is overdue.
4084          */
4085         if (!vcpu->kvm->arch.tss_addr)
4086                 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
4087                              "called before entering vcpu\n");
4088
4089         vmx_segment_cache_clear(vmx);
4090
4091         vmcs_writel(GUEST_TR_BASE, vcpu->kvm->arch.tss_addr);
4092         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
4093         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
4094
4095         flags = vmcs_readl(GUEST_RFLAGS);
4096         vmx->rmode.save_rflags = flags;
4097
4098         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
4099
4100         vmcs_writel(GUEST_RFLAGS, flags);
4101         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
4102         update_exception_bitmap(vcpu);
4103
4104         fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
4105         fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
4106         fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
4107         fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
4108         fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
4109         fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
4110
4111         kvm_mmu_reset_context(vcpu);
4112 }
4113
4114 static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
4115 {
4116         struct vcpu_vmx *vmx = to_vmx(vcpu);
4117         struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
4118
4119         if (!msr)
4120                 return;
4121
4122         /*
4123          * Force kernel_gs_base reloading before EFER changes, as control
4124          * of this msr depends on is_long_mode().
4125          */
4126         vmx_load_host_state(to_vmx(vcpu));
4127         vcpu->arch.efer = efer;
4128         if (efer & EFER_LMA) {
4129                 vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
4130                 msr->data = efer;
4131         } else {
4132                 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
4133
4134                 msr->data = efer & ~EFER_LME;
4135         }
4136         setup_msrs(vmx);
4137 }
4138
4139 #ifdef CONFIG_X86_64
4140
4141 static void enter_lmode(struct kvm_vcpu *vcpu)
4142 {
4143         u32 guest_tr_ar;
4144
4145         vmx_segment_cache_clear(to_vmx(vcpu));
4146
4147         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
4148         if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
4149                 pr_debug_ratelimited("%s: tss fixup for long mode. \n",
4150                                      __func__);
4151                 vmcs_write32(GUEST_TR_AR_BYTES,
4152                              (guest_tr_ar & ~VMX_AR_TYPE_MASK)
4153                              | VMX_AR_TYPE_BUSY_64_TSS);
4154         }
4155         vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
4156 }
4157
4158 static void exit_lmode(struct kvm_vcpu *vcpu)
4159 {
4160         vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
4161         vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
4162 }
4163
4164 #endif
4165
4166 static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid)
4167 {
4168         if (enable_ept) {
4169                 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
4170                         return;
4171                 ept_sync_context(construct_eptp(vcpu, vcpu->arch.mmu.root_hpa));
4172         } else {
4173                 vpid_sync_context(vpid);
4174         }
4175 }
4176
4177 static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
4178 {
4179         __vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid);
4180 }
4181
4182 static void vmx_flush_tlb_ept_only(struct kvm_vcpu *vcpu)
4183 {
4184         if (enable_ept)
4185                 vmx_flush_tlb(vcpu);
4186 }
4187
4188 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
4189 {
4190         ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
4191
4192         vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
4193         vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
4194 }
4195
4196 static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
4197 {
4198         if (enable_ept && is_paging(vcpu))
4199                 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
4200         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
4201 }
4202
4203 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
4204 {
4205         ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
4206
4207         vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
4208         vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
4209 }
4210
4211 static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
4212 {
4213         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
4214
4215         if (!test_bit(VCPU_EXREG_PDPTR,
4216                       (unsigned long *)&vcpu->arch.regs_dirty))
4217                 return;
4218
4219         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
4220                 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
4221                 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
4222                 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
4223                 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
4224         }
4225 }
4226
4227 static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
4228 {
4229         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
4230
4231         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
4232                 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
4233                 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
4234                 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
4235                 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
4236         }
4237
4238         __set_bit(VCPU_EXREG_PDPTR,
4239                   (unsigned long *)&vcpu->arch.regs_avail);
4240         __set_bit(VCPU_EXREG_PDPTR,
4241                   (unsigned long *)&vcpu->arch.regs_dirty);
4242 }
4243
4244 static bool nested_guest_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
4245 {
4246         u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0;
4247         u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1;
4248         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4249
4250         if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high &
4251                 SECONDARY_EXEC_UNRESTRICTED_GUEST &&
4252             nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
4253                 fixed0 &= ~(X86_CR0_PE | X86_CR0_PG);
4254
4255         return fixed_bits_valid(val, fixed0, fixed1);
4256 }
4257
4258 static bool nested_host_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
4259 {
4260         u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0;
4261         u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1;
4262
4263         return fixed_bits_valid(val, fixed0, fixed1);
4264 }
4265
4266 static bool nested_cr4_valid(struct kvm_vcpu *vcpu, unsigned long val)
4267 {
4268         u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed0;
4269         u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed1;
4270
4271         return fixed_bits_valid(val, fixed0, fixed1);
4272 }
4273
4274 /* No difference in the restrictions on guest and host CR4 in VMX operation. */
4275 #define nested_guest_cr4_valid  nested_cr4_valid
4276 #define nested_host_cr4_valid   nested_cr4_valid
4277
4278 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
4279
4280 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
4281                                         unsigned long cr0,
4282                                         struct kvm_vcpu *vcpu)
4283 {
4284         if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
4285                 vmx_decache_cr3(vcpu);
4286         if (!(cr0 & X86_CR0_PG)) {
4287                 /* From paging/starting to nonpaging */
4288                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
4289                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
4290                              (CPU_BASED_CR3_LOAD_EXITING |
4291                               CPU_BASED_CR3_STORE_EXITING));
4292                 vcpu->arch.cr0 = cr0;
4293                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
4294         } else if (!is_paging(vcpu)) {
4295                 /* From nonpaging to paging */
4296                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
4297                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
4298                              ~(CPU_BASED_CR3_LOAD_EXITING |
4299                                CPU_BASED_CR3_STORE_EXITING));
4300                 vcpu->arch.cr0 = cr0;
4301                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
4302         }
4303
4304         if (!(cr0 & X86_CR0_WP))
4305                 *hw_cr0 &= ~X86_CR0_WP;
4306 }
4307
4308 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
4309 {
4310         struct vcpu_vmx *vmx = to_vmx(vcpu);
4311         unsigned long hw_cr0;
4312
4313         hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK);
4314         if (enable_unrestricted_guest)
4315                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
4316         else {
4317                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
4318
4319                 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
4320                         enter_pmode(vcpu);
4321
4322                 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
4323                         enter_rmode(vcpu);
4324         }
4325
4326 #ifdef CONFIG_X86_64
4327         if (vcpu->arch.efer & EFER_LME) {
4328                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
4329                         enter_lmode(vcpu);
4330                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
4331                         exit_lmode(vcpu);
4332         }
4333 #endif
4334
4335         if (enable_ept)
4336                 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
4337
4338         vmcs_writel(CR0_READ_SHADOW, cr0);
4339         vmcs_writel(GUEST_CR0, hw_cr0);
4340         vcpu->arch.cr0 = cr0;
4341
4342         /* depends on vcpu->arch.cr0 to be set to a new value */
4343         vmx->emulation_required = emulation_required(vcpu);
4344 }
4345
4346 static int get_ept_level(struct kvm_vcpu *vcpu)
4347 {
4348         if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48))
4349                 return 5;
4350         return 4;
4351 }
4352
4353 static u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa)
4354 {
4355         u64 eptp = VMX_EPTP_MT_WB;
4356
4357         eptp |= (get_ept_level(vcpu) == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
4358
4359         if (enable_ept_ad_bits &&
4360             (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
4361                 eptp |= VMX_EPTP_AD_ENABLE_BIT;
4362         eptp |= (root_hpa & PAGE_MASK);
4363
4364         return eptp;
4365 }
4366
4367 static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
4368 {
4369         unsigned long guest_cr3;
4370         u64 eptp;
4371
4372         guest_cr3 = cr3;
4373         if (enable_ept) {
4374                 eptp = construct_eptp(vcpu, cr3);
4375                 vmcs_write64(EPT_POINTER, eptp);
4376                 if (is_paging(vcpu) || is_guest_mode(vcpu))
4377                         guest_cr3 = kvm_read_cr3(vcpu);
4378                 else
4379                         guest_cr3 = vcpu->kvm->arch.ept_identity_map_addr;
4380                 ept_load_pdptrs(vcpu);
4381         }
4382
4383         vmx_flush_tlb(vcpu);
4384         vmcs_writel(GUEST_CR3, guest_cr3);
4385 }
4386
4387 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
4388 {
4389         /*
4390          * Pass through host's Machine Check Enable value to hw_cr4, which
4391          * is in force while we are in guest mode.  Do not let guests control
4392          * this bit, even if host CR4.MCE == 0.
4393          */
4394         unsigned long hw_cr4 =
4395                 (cr4_read_shadow() & X86_CR4_MCE) |
4396                 (cr4 & ~X86_CR4_MCE) |
4397                 (to_vmx(vcpu)->rmode.vm86_active ?
4398                  KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
4399
4400         if (cr4 & X86_CR4_VMXE) {
4401                 /*
4402                  * To use VMXON (and later other VMX instructions), a guest
4403                  * must first be able to turn on cr4.VMXE (see handle_vmon()).
4404                  * So basically the check on whether to allow nested VMX
4405                  * is here.
4406                  */
4407                 if (!nested_vmx_allowed(vcpu))
4408                         return 1;
4409         }
4410
4411         if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
4412                 return 1;
4413
4414         vcpu->arch.cr4 = cr4;
4415         if (enable_ept) {
4416                 if (!is_paging(vcpu)) {
4417                         hw_cr4 &= ~X86_CR4_PAE;
4418                         hw_cr4 |= X86_CR4_PSE;
4419                 } else if (!(cr4 & X86_CR4_PAE)) {
4420                         hw_cr4 &= ~X86_CR4_PAE;
4421                 }
4422         }
4423
4424         if (!enable_unrestricted_guest && !is_paging(vcpu))
4425                 /*
4426                  * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
4427                  * hardware.  To emulate this behavior, SMEP/SMAP/PKU needs
4428                  * to be manually disabled when guest switches to non-paging
4429                  * mode.
4430                  *
4431                  * If !enable_unrestricted_guest, the CPU is always running
4432                  * with CR0.PG=1 and CR4 needs to be modified.
4433                  * If enable_unrestricted_guest, the CPU automatically
4434                  * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
4435                  */
4436                 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
4437
4438         vmcs_writel(CR4_READ_SHADOW, cr4);
4439         vmcs_writel(GUEST_CR4, hw_cr4);
4440         return 0;
4441 }
4442
4443 static void vmx_get_segment(struct kvm_vcpu *vcpu,
4444                             struct kvm_segment *var, int seg)
4445 {
4446         struct vcpu_vmx *vmx = to_vmx(vcpu);
4447         u32 ar;
4448
4449         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
4450                 *var = vmx->rmode.segs[seg];
4451                 if (seg == VCPU_SREG_TR
4452                     || var->selector == vmx_read_guest_seg_selector(vmx, seg))
4453                         return;
4454                 var->base = vmx_read_guest_seg_base(vmx, seg);
4455                 var->selector = vmx_read_guest_seg_selector(vmx, seg);
4456                 return;
4457         }
4458         var->base = vmx_read_guest_seg_base(vmx, seg);
4459         var->limit = vmx_read_guest_seg_limit(vmx, seg);
4460         var->selector = vmx_read_guest_seg_selector(vmx, seg);
4461         ar = vmx_read_guest_seg_ar(vmx, seg);
4462         var->unusable = (ar >> 16) & 1;
4463         var->type = ar & 15;
4464         var->s = (ar >> 4) & 1;
4465         var->dpl = (ar >> 5) & 3;
4466         /*
4467          * Some userspaces do not preserve unusable property. Since usable
4468          * segment has to be present according to VMX spec we can use present
4469          * property to amend userspace bug by making unusable segment always
4470          * nonpresent. vmx_segment_access_rights() already marks nonpresent
4471          * segment as unusable.
4472          */
4473         var->present = !var->unusable;
4474         var->avl = (ar >> 12) & 1;
4475         var->l = (ar >> 13) & 1;
4476         var->db = (ar >> 14) & 1;
4477         var->g = (ar >> 15) & 1;
4478 }
4479
4480 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
4481 {
4482         struct kvm_segment s;
4483
4484         if (to_vmx(vcpu)->rmode.vm86_active) {
4485                 vmx_get_segment(vcpu, &s, seg);
4486                 return s.base;
4487         }
4488         return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
4489 }
4490
4491 static int vmx_get_cpl(struct kvm_vcpu *vcpu)
4492 {
4493         struct vcpu_vmx *vmx = to_vmx(vcpu);
4494
4495         if (unlikely(vmx->rmode.vm86_active))
4496                 return 0;
4497         else {
4498                 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
4499                 return VMX_AR_DPL(ar);
4500         }
4501 }
4502
4503 static u32 vmx_segment_access_rights(struct kvm_segment *var)
4504 {
4505         u32 ar;
4506
4507         if (var->unusable || !var->present)
4508                 ar = 1 << 16;
4509         else {
4510                 ar = var->type & 15;
4511                 ar |= (var->s & 1) << 4;
4512                 ar |= (var->dpl & 3) << 5;
4513                 ar |= (var->present & 1) << 7;
4514                 ar |= (var->avl & 1) << 12;
4515                 ar |= (var->l & 1) << 13;
4516                 ar |= (var->db & 1) << 14;
4517                 ar |= (var->g & 1) << 15;
4518         }
4519
4520         return ar;
4521 }
4522
4523 static void vmx_set_segment(struct kvm_vcpu *vcpu,
4524                             struct kvm_segment *var, int seg)
4525 {
4526         struct vcpu_vmx *vmx = to_vmx(vcpu);
4527         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
4528
4529         vmx_segment_cache_clear(vmx);
4530
4531         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
4532                 vmx->rmode.segs[seg] = *var;
4533                 if (seg == VCPU_SREG_TR)
4534                         vmcs_write16(sf->selector, var->selector);
4535                 else if (var->s)
4536                         fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
4537                 goto out;
4538         }
4539
4540         vmcs_writel(sf->base, var->base);
4541         vmcs_write32(sf->limit, var->limit);
4542         vmcs_write16(sf->selector, var->selector);
4543
4544         /*
4545          *   Fix the "Accessed" bit in AR field of segment registers for older
4546          * qemu binaries.
4547          *   IA32 arch specifies that at the time of processor reset the
4548          * "Accessed" bit in the AR field of segment registers is 1. And qemu
4549          * is setting it to 0 in the userland code. This causes invalid guest
4550          * state vmexit when "unrestricted guest" mode is turned on.
4551          *    Fix for this setup issue in cpu_reset is being pushed in the qemu
4552          * tree. Newer qemu binaries with that qemu fix would not need this
4553          * kvm hack.
4554          */
4555         if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
4556                 var->type |= 0x1; /* Accessed */
4557
4558         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
4559
4560 out:
4561         vmx->emulation_required = emulation_required(vcpu);
4562 }
4563
4564 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
4565 {
4566         u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
4567
4568         *db = (ar >> 14) & 1;
4569         *l = (ar >> 13) & 1;
4570 }
4571
4572 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
4573 {
4574         dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
4575         dt->address = vmcs_readl(GUEST_IDTR_BASE);
4576 }
4577
4578 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
4579 {
4580         vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
4581         vmcs_writel(GUEST_IDTR_BASE, dt->address);
4582 }
4583
4584 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
4585 {
4586         dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
4587         dt->address = vmcs_readl(GUEST_GDTR_BASE);
4588 }
4589
4590 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
4591 {
4592         vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
4593         vmcs_writel(GUEST_GDTR_BASE, dt->address);
4594 }
4595
4596 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
4597 {
4598         struct kvm_segment var;
4599         u32 ar;
4600
4601         vmx_get_segment(vcpu, &var, seg);
4602         var.dpl = 0x3;
4603         if (seg == VCPU_SREG_CS)
4604                 var.type = 0x3;
4605         ar = vmx_segment_access_rights(&var);
4606
4607         if (var.base != (var.selector << 4))
4608                 return false;
4609         if (var.limit != 0xffff)
4610                 return false;
4611         if (ar != 0xf3)
4612                 return false;
4613
4614         return true;
4615 }
4616
4617 static bool code_segment_valid(struct kvm_vcpu *vcpu)
4618 {
4619         struct kvm_segment cs;
4620         unsigned int cs_rpl;
4621
4622         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
4623         cs_rpl = cs.selector & SEGMENT_RPL_MASK;
4624
4625         if (cs.unusable)
4626                 return false;
4627         if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
4628                 return false;
4629         if (!cs.s)
4630                 return false;
4631         if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
4632                 if (cs.dpl > cs_rpl)
4633                         return false;
4634         } else {
4635                 if (cs.dpl != cs_rpl)
4636                         return false;
4637         }
4638         if (!cs.present)
4639                 return false;
4640
4641         /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
4642         return true;
4643 }
4644
4645 static bool stack_segment_valid(struct kvm_vcpu *vcpu)
4646 {
4647         struct kvm_segment ss;
4648         unsigned int ss_rpl;
4649
4650         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
4651         ss_rpl = ss.selector & SEGMENT_RPL_MASK;
4652
4653         if (ss.unusable)
4654                 return true;
4655         if (ss.type != 3 && ss.type != 7)
4656                 return false;
4657         if (!ss.s)
4658                 return false;
4659         if (ss.dpl != ss_rpl) /* DPL != RPL */
4660                 return false;
4661         if (!ss.present)
4662                 return false;
4663
4664         return true;
4665 }
4666
4667 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
4668 {
4669         struct kvm_segment var;
4670         unsigned int rpl;
4671
4672         vmx_get_segment(vcpu, &var, seg);
4673         rpl = var.selector & SEGMENT_RPL_MASK;
4674
4675         if (var.unusable)
4676                 return true;
4677         if (!var.s)
4678                 return false;
4679         if (!var.present)
4680                 return false;
4681         if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
4682                 if (var.dpl < rpl) /* DPL < RPL */
4683                         return false;
4684         }
4685
4686         /* TODO: Add other members to kvm_segment_field to allow checking for other access
4687          * rights flags
4688          */
4689         return true;
4690 }
4691
4692 static bool tr_valid(struct kvm_vcpu *vcpu)
4693 {
4694         struct kvm_segment tr;
4695
4696         vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
4697
4698         if (tr.unusable)
4699                 return false;
4700         if (tr.selector & SEGMENT_TI_MASK)      /* TI = 1 */
4701                 return false;
4702         if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
4703                 return false;
4704         if (!tr.present)
4705                 return false;
4706
4707         return true;
4708 }
4709
4710 static bool ldtr_valid(struct kvm_vcpu *vcpu)
4711 {
4712         struct kvm_segment ldtr;
4713
4714         vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
4715
4716         if (ldtr.unusable)
4717                 return true;
4718         if (ldtr.selector & SEGMENT_TI_MASK)    /* TI = 1 */
4719                 return false;
4720         if (ldtr.type != 2)
4721                 return false;
4722         if (!ldtr.present)
4723                 return false;
4724
4725         return true;
4726 }
4727
4728 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
4729 {
4730         struct kvm_segment cs, ss;
4731
4732         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
4733         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
4734
4735         return ((cs.selector & SEGMENT_RPL_MASK) ==
4736                  (ss.selector & SEGMENT_RPL_MASK));
4737 }
4738
4739 /*
4740  * Check if guest state is valid. Returns true if valid, false if
4741  * not.
4742  * We assume that registers are always usable
4743  */
4744 static bool guest_state_valid(struct kvm_vcpu *vcpu)
4745 {
4746         if (enable_unrestricted_guest)
4747                 return true;
4748
4749         /* real mode guest state checks */
4750         if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
4751                 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
4752                         return false;
4753                 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
4754                         return false;
4755                 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
4756                         return false;
4757                 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
4758                         return false;
4759                 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
4760                         return false;
4761                 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
4762                         return false;
4763         } else {
4764         /* protected mode guest state checks */
4765                 if (!cs_ss_rpl_check(vcpu))
4766                         return false;
4767                 if (!code_segment_valid(vcpu))
4768                         return false;
4769                 if (!stack_segment_valid(vcpu))
4770                         return false;
4771                 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
4772                         return false;
4773                 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
4774                         return false;
4775                 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
4776                         return false;
4777                 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
4778                         return false;
4779                 if (!tr_valid(vcpu))
4780                         return false;
4781                 if (!ldtr_valid(vcpu))
4782                         return false;
4783         }
4784         /* TODO:
4785          * - Add checks on RIP
4786          * - Add checks on RFLAGS
4787          */
4788
4789         return true;
4790 }
4791
4792 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
4793 {
4794         return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu));
4795 }
4796
4797 static int init_rmode_tss(struct kvm *kvm)
4798 {
4799         gfn_t fn;
4800         u16 data = 0;
4801         int idx, r;
4802
4803         idx = srcu_read_lock(&kvm->srcu);
4804         fn = kvm->arch.tss_addr >> PAGE_SHIFT;
4805         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
4806         if (r < 0)
4807                 goto out;
4808         data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
4809         r = kvm_write_guest_page(kvm, fn++, &data,
4810                         TSS_IOPB_BASE_OFFSET, sizeof(u16));
4811         if (r < 0)
4812                 goto out;
4813         r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
4814         if (r < 0)
4815                 goto out;
4816         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
4817         if (r < 0)
4818                 goto out;
4819         data = ~0;
4820         r = kvm_write_guest_page(kvm, fn, &data,
4821                                  RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
4822                                  sizeof(u8));
4823 out:
4824         srcu_read_unlock(&kvm->srcu, idx);
4825         return r;
4826 }
4827
4828 static int init_rmode_identity_map(struct kvm *kvm)
4829 {
4830         int i, idx, r = 0;
4831         kvm_pfn_t identity_map_pfn;
4832         u32 tmp;
4833
4834         if (!enable_ept)
4835                 return 0;
4836
4837         /* Protect kvm->arch.ept_identity_pagetable_done. */
4838         mutex_lock(&kvm->slots_lock);
4839
4840         if (likely(kvm->arch.ept_identity_pagetable_done))
4841                 goto out2;
4842
4843         identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
4844
4845         r = alloc_identity_pagetable(kvm);
4846         if (r < 0)
4847                 goto out2;
4848
4849         idx = srcu_read_lock(&kvm->srcu);
4850         r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
4851         if (r < 0)
4852                 goto out;
4853         /* Set up identity-mapping pagetable for EPT in real mode */
4854         for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
4855                 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
4856                         _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
4857                 r = kvm_write_guest_page(kvm, identity_map_pfn,
4858                                 &tmp, i * sizeof(tmp), sizeof(tmp));
4859                 if (r < 0)
4860                         goto out;
4861         }
4862         kvm->arch.ept_identity_pagetable_done = true;
4863
4864 out:
4865         srcu_read_unlock(&kvm->srcu, idx);
4866
4867 out2:
4868         mutex_unlock(&kvm->slots_lock);
4869         return r;
4870 }
4871
4872 static void seg_setup(int seg)
4873 {
4874         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
4875         unsigned int ar;
4876
4877         vmcs_write16(sf->selector, 0);
4878         vmcs_writel(sf->base, 0);
4879         vmcs_write32(sf->limit, 0xffff);
4880         ar = 0x93;
4881         if (seg == VCPU_SREG_CS)
4882                 ar |= 0x08; /* code segment */
4883
4884         vmcs_write32(sf->ar_bytes, ar);
4885 }
4886
4887 static int alloc_apic_access_page(struct kvm *kvm)
4888 {
4889         struct page *page;
4890         int r = 0;
4891
4892         mutex_lock(&kvm->slots_lock);
4893         if (kvm->arch.apic_access_page_done)
4894                 goto out;
4895         r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
4896                                     APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
4897         if (r)
4898                 goto out;
4899
4900         page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
4901         if (is_error_page(page)) {
4902                 r = -EFAULT;
4903                 goto out;
4904         }
4905
4906         /*
4907          * Do not pin the page in memory, so that memory hot-unplug
4908          * is able to migrate it.
4909          */
4910         put_page(page);
4911         kvm->arch.apic_access_page_done = true;
4912 out:
4913         mutex_unlock(&kvm->slots_lock);
4914         return r;
4915 }
4916
4917 static int alloc_identity_pagetable(struct kvm *kvm)
4918 {
4919         /* Called with kvm->slots_lock held. */
4920
4921         int r = 0;
4922
4923         BUG_ON(kvm->arch.ept_identity_pagetable_done);
4924
4925         r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
4926                                     kvm->arch.ept_identity_map_addr, PAGE_SIZE);
4927
4928         return r;
4929 }
4930
4931 static int allocate_vpid(void)
4932 {
4933         int vpid;
4934
4935         if (!enable_vpid)
4936                 return 0;
4937         spin_lock(&vmx_vpid_lock);
4938         vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
4939         if (vpid < VMX_NR_VPIDS)
4940                 __set_bit(vpid, vmx_vpid_bitmap);
4941         else
4942                 vpid = 0;
4943         spin_unlock(&vmx_vpid_lock);
4944         return vpid;
4945 }
4946
4947 static void free_vpid(int vpid)
4948 {
4949         if (!enable_vpid || vpid == 0)
4950                 return;
4951         spin_lock(&vmx_vpid_lock);
4952         __clear_bit(vpid, vmx_vpid_bitmap);
4953         spin_unlock(&vmx_vpid_lock);
4954 }
4955
4956 static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
4957                                                           u32 msr, int type)
4958 {
4959         int f = sizeof(unsigned long);
4960
4961         if (!cpu_has_vmx_msr_bitmap())
4962                 return;
4963
4964         /*
4965          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
4966          * have the write-low and read-high bitmap offsets the wrong way round.
4967          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
4968          */
4969         if (msr <= 0x1fff) {
4970                 if (type & MSR_TYPE_R)
4971                         /* read-low */
4972                         __clear_bit(msr, msr_bitmap + 0x000 / f);
4973
4974                 if (type & MSR_TYPE_W)
4975                         /* write-low */
4976                         __clear_bit(msr, msr_bitmap + 0x800 / f);
4977
4978         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
4979                 msr &= 0x1fff;
4980                 if (type & MSR_TYPE_R)
4981                         /* read-high */
4982                         __clear_bit(msr, msr_bitmap + 0x400 / f);
4983
4984                 if (type & MSR_TYPE_W)
4985                         /* write-high */
4986                         __clear_bit(msr, msr_bitmap + 0xc00 / f);
4987
4988         }
4989 }
4990
4991 static void __always_inline vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
4992                                                          u32 msr, int type)
4993 {
4994         int f = sizeof(unsigned long);
4995
4996         if (!cpu_has_vmx_msr_bitmap())
4997                 return;
4998
4999         /*
5000          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
5001          * have the write-low and read-high bitmap offsets the wrong way round.
5002          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
5003          */
5004         if (msr <= 0x1fff) {
5005                 if (type & MSR_TYPE_R)
5006                         /* read-low */
5007                         __set_bit(msr, msr_bitmap + 0x000 / f);
5008
5009                 if (type & MSR_TYPE_W)
5010                         /* write-low */
5011                         __set_bit(msr, msr_bitmap + 0x800 / f);
5012
5013         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
5014                 msr &= 0x1fff;
5015                 if (type & MSR_TYPE_R)
5016                         /* read-high */
5017                         __set_bit(msr, msr_bitmap + 0x400 / f);
5018
5019                 if (type & MSR_TYPE_W)
5020                         /* write-high */
5021                         __set_bit(msr, msr_bitmap + 0xc00 / f);
5022
5023         }
5024 }
5025
5026 static void __always_inline vmx_set_intercept_for_msr(unsigned long *msr_bitmap,
5027                                                       u32 msr, int type, bool value)
5028 {
5029         if (value)
5030                 vmx_enable_intercept_for_msr(msr_bitmap, msr, type);
5031         else
5032                 vmx_disable_intercept_for_msr(msr_bitmap, msr, type);
5033 }
5034
5035 /*
5036  * If a msr is allowed by L0, we should check whether it is allowed by L1.
5037  * The corresponding bit will be cleared unless both of L0 and L1 allow it.
5038  */
5039 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
5040                                                unsigned long *msr_bitmap_nested,
5041                                                u32 msr, int type)
5042 {
5043         int f = sizeof(unsigned long);
5044
5045         if (!cpu_has_vmx_msr_bitmap()) {
5046                 WARN_ON(1);
5047                 return;
5048         }
5049
5050         /*
5051          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
5052          * have the write-low and read-high bitmap offsets the wrong way round.
5053          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
5054          */
5055         if (msr <= 0x1fff) {
5056                 if (type & MSR_TYPE_R &&
5057                    !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
5058                         /* read-low */
5059                         __clear_bit(msr, msr_bitmap_nested + 0x000 / f);
5060
5061                 if (type & MSR_TYPE_W &&
5062                    !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
5063                         /* write-low */
5064                         __clear_bit(msr, msr_bitmap_nested + 0x800 / f);
5065
5066         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
5067                 msr &= 0x1fff;
5068                 if (type & MSR_TYPE_R &&
5069                    !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
5070                         /* read-high */
5071                         __clear_bit(msr, msr_bitmap_nested + 0x400 / f);
5072
5073                 if (type & MSR_TYPE_W &&
5074                    !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
5075                         /* write-high */
5076                         __clear_bit(msr, msr_bitmap_nested + 0xc00 / f);
5077
5078         }
5079 }
5080
5081 static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu)
5082 {
5083         u8 mode = 0;
5084
5085         if (cpu_has_secondary_exec_ctrls() &&
5086             (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) &
5087              SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
5088                 mode |= MSR_BITMAP_MODE_X2APIC;
5089                 if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
5090                         mode |= MSR_BITMAP_MODE_X2APIC_APICV;
5091         }
5092
5093         if (is_long_mode(vcpu))
5094                 mode |= MSR_BITMAP_MODE_LM;
5095
5096         return mode;
5097 }
5098
5099 #define X2APIC_MSR(r) (APIC_BASE_MSR + ((r) >> 4))
5100
5101 static void vmx_update_msr_bitmap_x2apic(unsigned long *msr_bitmap,
5102                                          u8 mode)
5103 {
5104         int msr;
5105
5106         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
5107                 unsigned word = msr / BITS_PER_LONG;
5108                 msr_bitmap[word] = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0;
5109                 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
5110         }
5111
5112         if (mode & MSR_BITMAP_MODE_X2APIC) {
5113                 /*
5114                  * TPR reads and writes can be virtualized even if virtual interrupt
5115                  * delivery is not in use.
5116                  */
5117                 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW);
5118                 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) {
5119                         vmx_enable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_R);
5120                         vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_EOI), MSR_TYPE_W);
5121                         vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W);
5122                 }
5123         }
5124 }
5125
5126 static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu)
5127 {
5128         struct vcpu_vmx *vmx = to_vmx(vcpu);
5129         unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
5130         u8 mode = vmx_msr_bitmap_mode(vcpu);
5131         u8 changed = mode ^ vmx->msr_bitmap_mode;
5132
5133         if (!changed)
5134                 return;
5135
5136         vmx_set_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW,
5137                                   !(mode & MSR_BITMAP_MODE_LM));
5138
5139         if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV))
5140                 vmx_update_msr_bitmap_x2apic(msr_bitmap, mode);
5141
5142         vmx->msr_bitmap_mode = mode;
5143 }
5144
5145 static bool vmx_get_enable_apicv(struct kvm_vcpu *vcpu)
5146 {
5147         return enable_apicv;
5148 }
5149
5150 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
5151 {
5152         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5153         gfn_t gfn;
5154
5155         /*
5156          * Don't need to mark the APIC access page dirty; it is never
5157          * written to by the CPU during APIC virtualization.
5158          */
5159
5160         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
5161                 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
5162                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
5163         }
5164
5165         if (nested_cpu_has_posted_intr(vmcs12)) {
5166                 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
5167                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
5168         }
5169 }
5170
5171
5172 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
5173 {
5174         struct vcpu_vmx *vmx = to_vmx(vcpu);
5175         int max_irr;
5176         void *vapic_page;
5177         u16 status;
5178
5179         if (!vmx->nested.pi_desc || !vmx->nested.pi_pending)
5180                 return;
5181
5182         vmx->nested.pi_pending = false;
5183         if (!pi_test_and_clear_on(vmx->nested.pi_desc))
5184                 return;
5185
5186         max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
5187         if (max_irr != 256) {
5188                 vapic_page = kmap(vmx->nested.virtual_apic_page);
5189                 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, vapic_page);
5190                 kunmap(vmx->nested.virtual_apic_page);
5191
5192                 status = vmcs_read16(GUEST_INTR_STATUS);
5193                 if ((u8)max_irr > ((u8)status & 0xff)) {
5194                         status &= ~0xff;
5195                         status |= (u8)max_irr;
5196                         vmcs_write16(GUEST_INTR_STATUS, status);
5197                 }
5198         }
5199
5200         nested_mark_vmcs12_pages_dirty(vcpu);
5201 }
5202
5203 static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
5204                                                      bool nested)
5205 {
5206 #ifdef CONFIG_SMP
5207         int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR;
5208
5209         if (vcpu->mode == IN_GUEST_MODE) {
5210                 /*
5211                  * The vector of interrupt to be delivered to vcpu had
5212                  * been set in PIR before this function.
5213                  *
5214                  * Following cases will be reached in this block, and
5215                  * we always send a notification event in all cases as
5216                  * explained below.
5217                  *
5218                  * Case 1: vcpu keeps in non-root mode. Sending a
5219                  * notification event posts the interrupt to vcpu.
5220                  *
5221                  * Case 2: vcpu exits to root mode and is still
5222                  * runnable. PIR will be synced to vIRR before the
5223                  * next vcpu entry. Sending a notification event in
5224                  * this case has no effect, as vcpu is not in root
5225                  * mode.
5226                  *
5227                  * Case 3: vcpu exits to root mode and is blocked.
5228                  * vcpu_block() has already synced PIR to vIRR and
5229                  * never blocks vcpu if vIRR is not cleared. Therefore,
5230                  * a blocked vcpu here does not wait for any requested
5231                  * interrupts in PIR, and sending a notification event
5232                  * which has no effect is safe here.
5233                  */
5234
5235                 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
5236                 return true;
5237         }
5238 #endif
5239         return false;
5240 }
5241
5242 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
5243                                                 int vector)
5244 {
5245         struct vcpu_vmx *vmx = to_vmx(vcpu);
5246
5247         if (is_guest_mode(vcpu) &&
5248             vector == vmx->nested.posted_intr_nv) {
5249                 /* the PIR and ON have been set by L1. */
5250                 kvm_vcpu_trigger_posted_interrupt(vcpu, true);
5251                 /*
5252                  * If a posted intr is not recognized by hardware,
5253                  * we will accomplish it in the next vmentry.
5254                  */
5255                 vmx->nested.pi_pending = true;
5256                 kvm_make_request(KVM_REQ_EVENT, vcpu);
5257                 return 0;
5258         }
5259         return -1;
5260 }
5261 /*
5262  * Send interrupt to vcpu via posted interrupt way.
5263  * 1. If target vcpu is running(non-root mode), send posted interrupt
5264  * notification to vcpu and hardware will sync PIR to vIRR atomically.
5265  * 2. If target vcpu isn't running(root mode), kick it to pick up the
5266  * interrupt from PIR in next vmentry.
5267  */
5268 static void vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
5269 {
5270         struct vcpu_vmx *vmx = to_vmx(vcpu);
5271         int r;
5272
5273         r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
5274         if (!r)
5275                 return;
5276
5277         if (pi_test_and_set_pir(vector, &vmx->pi_desc))
5278                 return;
5279
5280         /* If a previous notification has sent the IPI, nothing to do.  */
5281         if (pi_test_and_set_on(&vmx->pi_desc))
5282                 return;
5283
5284         if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false))
5285                 kvm_vcpu_kick(vcpu);
5286 }
5287
5288 /*
5289  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
5290  * will not change in the lifetime of the guest.
5291  * Note that host-state that does change is set elsewhere. E.g., host-state
5292  * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
5293  */
5294 static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
5295 {
5296         u32 low32, high32;
5297         unsigned long tmpl;
5298         struct desc_ptr dt;
5299         unsigned long cr0, cr3, cr4;
5300
5301         cr0 = read_cr0();
5302         WARN_ON(cr0 & X86_CR0_TS);
5303         vmcs_writel(HOST_CR0, cr0);  /* 22.2.3 */
5304
5305         /*
5306          * Save the most likely value for this task's CR3 in the VMCS.
5307          * We can't use __get_current_cr3_fast() because we're not atomic.
5308          */
5309         cr3 = __read_cr3();
5310         vmcs_writel(HOST_CR3, cr3);             /* 22.2.3  FIXME: shadow tables */
5311         vmx->loaded_vmcs->vmcs_host_cr3 = cr3;
5312
5313         /* Save the most likely value for this task's CR4 in the VMCS. */
5314         cr4 = cr4_read_shadow();
5315         vmcs_writel(HOST_CR4, cr4);                     /* 22.2.3, 22.2.5 */
5316         vmx->loaded_vmcs->vmcs_host_cr4 = cr4;
5317
5318         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
5319 #ifdef CONFIG_X86_64
5320         /*
5321          * Load null selectors, so we can avoid reloading them in
5322          * __vmx_load_host_state(), in case userspace uses the null selectors
5323          * too (the expected case).
5324          */
5325         vmcs_write16(HOST_DS_SELECTOR, 0);
5326         vmcs_write16(HOST_ES_SELECTOR, 0);
5327 #else
5328         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
5329         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
5330 #endif
5331         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
5332         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
5333
5334         store_idt(&dt);
5335         vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
5336         vmx->host_idt_base = dt.address;
5337
5338         vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
5339
5340         rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
5341         vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
5342         rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
5343         vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */
5344
5345         if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
5346                 rdmsr(MSR_IA32_CR_PAT, low32, high32);
5347                 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
5348         }
5349 }
5350
5351 static void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
5352 {
5353         vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
5354         if (enable_ept)
5355                 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
5356         if (is_guest_mode(&vmx->vcpu))
5357                 vmx->vcpu.arch.cr4_guest_owned_bits &=
5358                         ~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
5359         vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
5360 }
5361
5362 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
5363 {
5364         u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
5365
5366         if (!kvm_vcpu_apicv_active(&vmx->vcpu))
5367                 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
5368         /* Enable the preemption timer dynamically */
5369         pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
5370         return pin_based_exec_ctrl;
5371 }
5372
5373 static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
5374 {
5375         struct vcpu_vmx *vmx = to_vmx(vcpu);
5376
5377         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
5378         if (cpu_has_secondary_exec_ctrls()) {
5379                 if (kvm_vcpu_apicv_active(vcpu))
5380                         vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
5381                                       SECONDARY_EXEC_APIC_REGISTER_VIRT |
5382                                       SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
5383                 else
5384                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
5385                                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
5386                                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
5387         }
5388
5389         if (cpu_has_vmx_msr_bitmap())
5390                 vmx_update_msr_bitmap(vcpu);
5391 }
5392
5393 static u32 vmx_exec_control(struct vcpu_vmx *vmx)
5394 {
5395         u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
5396
5397         if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
5398                 exec_control &= ~CPU_BASED_MOV_DR_EXITING;
5399
5400         if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
5401                 exec_control &= ~CPU_BASED_TPR_SHADOW;
5402 #ifdef CONFIG_X86_64
5403                 exec_control |= CPU_BASED_CR8_STORE_EXITING |
5404                                 CPU_BASED_CR8_LOAD_EXITING;
5405 #endif
5406         }
5407         if (!enable_ept)
5408                 exec_control |= CPU_BASED_CR3_STORE_EXITING |
5409                                 CPU_BASED_CR3_LOAD_EXITING  |
5410                                 CPU_BASED_INVLPG_EXITING;
5411         return exec_control;
5412 }
5413
5414 static bool vmx_rdrand_supported(void)
5415 {
5416         return vmcs_config.cpu_based_2nd_exec_ctrl &
5417                 SECONDARY_EXEC_RDRAND;
5418 }
5419
5420 static bool vmx_rdseed_supported(void)
5421 {
5422         return vmcs_config.cpu_based_2nd_exec_ctrl &
5423                 SECONDARY_EXEC_RDSEED;
5424 }
5425
5426 static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
5427 {
5428         struct kvm_vcpu *vcpu = &vmx->vcpu;
5429
5430         u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
5431         if (!cpu_need_virtualize_apic_accesses(vcpu))
5432                 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
5433         if (vmx->vpid == 0)
5434                 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
5435         if (!enable_ept) {
5436                 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
5437                 enable_unrestricted_guest = 0;
5438                 /* Enable INVPCID for non-ept guests may cause performance regression. */
5439                 exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
5440         }
5441         if (!enable_unrestricted_guest)
5442                 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
5443         if (!ple_gap)
5444                 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
5445         if (!kvm_vcpu_apicv_active(vcpu))
5446                 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
5447                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
5448         exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
5449         /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
5450            (handle_vmptrld).
5451            We can NOT enable shadow_vmcs here because we don't have yet
5452            a current VMCS12
5453         */
5454         exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
5455
5456         if (!enable_pml)
5457                 exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
5458
5459         if (vmx_xsaves_supported()) {
5460                 /* Exposing XSAVES only when XSAVE is exposed */
5461                 bool xsaves_enabled =
5462                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
5463                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVES);
5464
5465                 if (!xsaves_enabled)
5466                         exec_control &= ~SECONDARY_EXEC_XSAVES;
5467
5468                 if (nested) {
5469                         if (xsaves_enabled)
5470                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5471                                         SECONDARY_EXEC_XSAVES;
5472                         else
5473                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5474                                         ~SECONDARY_EXEC_XSAVES;
5475                 }
5476         }
5477
5478         if (vmx_rdtscp_supported()) {
5479                 bool rdtscp_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP);
5480                 if (!rdtscp_enabled)
5481                         exec_control &= ~SECONDARY_EXEC_RDTSCP;
5482
5483                 if (nested) {
5484                         if (rdtscp_enabled)
5485                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5486                                         SECONDARY_EXEC_RDTSCP;
5487                         else
5488                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5489                                         ~SECONDARY_EXEC_RDTSCP;
5490                 }
5491         }
5492
5493         if (vmx_invpcid_supported()) {
5494                 /* Exposing INVPCID only when PCID is exposed */
5495                 bool invpcid_enabled =
5496                         guest_cpuid_has(vcpu, X86_FEATURE_INVPCID) &&
5497                         guest_cpuid_has(vcpu, X86_FEATURE_PCID);
5498
5499                 if (!invpcid_enabled) {
5500                         exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
5501                         guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID);
5502                 }
5503
5504                 if (nested) {
5505                         if (invpcid_enabled)
5506                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5507                                         SECONDARY_EXEC_ENABLE_INVPCID;
5508                         else
5509                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5510                                         ~SECONDARY_EXEC_ENABLE_INVPCID;
5511                 }
5512         }
5513
5514         if (vmx_rdrand_supported()) {
5515                 bool rdrand_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDRAND);
5516                 if (rdrand_enabled)
5517                         exec_control &= ~SECONDARY_EXEC_RDRAND;
5518
5519                 if (nested) {
5520                         if (rdrand_enabled)
5521                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5522                                         SECONDARY_EXEC_RDRAND;
5523                         else
5524                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5525                                         ~SECONDARY_EXEC_RDRAND;
5526                 }
5527         }
5528
5529         if (vmx_rdseed_supported()) {
5530                 bool rdseed_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDSEED);
5531                 if (rdseed_enabled)
5532                         exec_control &= ~SECONDARY_EXEC_RDSEED;
5533
5534                 if (nested) {
5535                         if (rdseed_enabled)
5536                                 vmx->nested.nested_vmx_secondary_ctls_high |=
5537                                         SECONDARY_EXEC_RDSEED;
5538                         else
5539                                 vmx->nested.nested_vmx_secondary_ctls_high &=
5540                                         ~SECONDARY_EXEC_RDSEED;
5541                 }
5542         }
5543
5544         vmx->secondary_exec_control = exec_control;
5545 }
5546
5547 static void ept_set_mmio_spte_mask(void)
5548 {
5549         /*
5550          * EPT Misconfigurations can be generated if the value of bits 2:0
5551          * of an EPT paging-structure entry is 110b (write/execute).
5552          */
5553         kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK,
5554                                    VMX_EPT_MISCONFIG_WX_VALUE);
5555 }
5556
5557 #define VMX_XSS_EXIT_BITMAP 0
5558 /*
5559  * Sets up the vmcs for emulated real mode.
5560  */
5561 static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
5562 {
5563 #ifdef CONFIG_X86_64
5564         unsigned long a;
5565 #endif
5566         int i;
5567
5568         /* I/O */
5569         vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
5570         vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
5571
5572         if (enable_shadow_vmcs) {
5573                 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
5574                 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
5575         }
5576         if (cpu_has_vmx_msr_bitmap())
5577                 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
5578
5579         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
5580
5581         /* Control */
5582         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
5583         vmx->hv_deadline_tsc = -1;
5584
5585         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
5586
5587         if (cpu_has_secondary_exec_ctrls()) {
5588                 vmx_compute_secondary_exec_control(vmx);
5589                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
5590                              vmx->secondary_exec_control);
5591         }
5592
5593         if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
5594                 vmcs_write64(EOI_EXIT_BITMAP0, 0);
5595                 vmcs_write64(EOI_EXIT_BITMAP1, 0);
5596                 vmcs_write64(EOI_EXIT_BITMAP2, 0);
5597                 vmcs_write64(EOI_EXIT_BITMAP3, 0);
5598
5599                 vmcs_write16(GUEST_INTR_STATUS, 0);
5600
5601                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
5602                 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
5603         }
5604
5605         if (ple_gap) {
5606                 vmcs_write32(PLE_GAP, ple_gap);
5607                 vmx->ple_window = ple_window;
5608                 vmx->ple_window_dirty = true;
5609         }
5610
5611         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
5612         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
5613         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
5614
5615         vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
5616         vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
5617         vmx_set_constant_host_state(vmx);
5618 #ifdef CONFIG_X86_64
5619         rdmsrl(MSR_FS_BASE, a);
5620         vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
5621         rdmsrl(MSR_GS_BASE, a);
5622         vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
5623 #else
5624         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
5625         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
5626 #endif
5627
5628         if (cpu_has_vmx_vmfunc())
5629                 vmcs_write64(VM_FUNCTION_CONTROL, 0);
5630
5631         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
5632         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
5633         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
5634         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
5635         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
5636
5637         if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
5638                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
5639
5640         for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) {
5641                 u32 index = vmx_msr_index[i];
5642                 u32 data_low, data_high;
5643                 int j = vmx->nmsrs;
5644
5645                 if (rdmsr_safe(index, &data_low, &data_high) < 0)
5646                         continue;
5647                 if (wrmsr_safe(index, data_low, data_high) < 0)
5648                         continue;
5649                 vmx->guest_msrs[j].index = i;
5650                 vmx->guest_msrs[j].data = 0;
5651                 vmx->guest_msrs[j].mask = -1ull;
5652                 ++vmx->nmsrs;
5653         }
5654
5655
5656         vm_exit_controls_init(vmx, vmcs_config.vmexit_ctrl);
5657
5658         /* 22.2.1, 20.8.1 */
5659         vm_entry_controls_init(vmx, vmcs_config.vmentry_ctrl);
5660
5661         vmx->vcpu.arch.cr0_guest_owned_bits = X86_CR0_TS;
5662         vmcs_writel(CR0_GUEST_HOST_MASK, ~X86_CR0_TS);
5663
5664         set_cr4_guest_host_mask(vmx);
5665
5666         if (vmx_xsaves_supported())
5667                 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
5668
5669         if (enable_pml) {
5670                 ASSERT(vmx->pml_pg);
5671                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
5672                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
5673         }
5674
5675         return 0;
5676 }
5677
5678 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
5679 {
5680         struct vcpu_vmx *vmx = to_vmx(vcpu);
5681         struct msr_data apic_base_msr;
5682         u64 cr0;
5683
5684         vmx->rmode.vm86_active = 0;
5685
5686         vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
5687         kvm_set_cr8(vcpu, 0);
5688
5689         if (!init_event) {
5690                 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE |
5691                                      MSR_IA32_APICBASE_ENABLE;
5692                 if (kvm_vcpu_is_reset_bsp(vcpu))
5693                         apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
5694                 apic_base_msr.host_initiated = true;
5695                 kvm_set_apic_base(vcpu, &apic_base_msr);
5696         }
5697
5698         vmx_segment_cache_clear(vmx);
5699
5700         seg_setup(VCPU_SREG_CS);
5701         vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
5702         vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
5703
5704         seg_setup(VCPU_SREG_DS);
5705         seg_setup(VCPU_SREG_ES);
5706         seg_setup(VCPU_SREG_FS);
5707         seg_setup(VCPU_SREG_GS);
5708         seg_setup(VCPU_SREG_SS);
5709
5710         vmcs_write16(GUEST_TR_SELECTOR, 0);
5711         vmcs_writel(GUEST_TR_BASE, 0);
5712         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
5713         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
5714
5715         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
5716         vmcs_writel(GUEST_LDTR_BASE, 0);
5717         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
5718         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
5719
5720         if (!init_event) {
5721                 vmcs_write32(GUEST_SYSENTER_CS, 0);
5722                 vmcs_writel(GUEST_SYSENTER_ESP, 0);
5723                 vmcs_writel(GUEST_SYSENTER_EIP, 0);
5724                 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
5725         }
5726
5727         kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
5728         kvm_rip_write(vcpu, 0xfff0);
5729
5730         vmcs_writel(GUEST_GDTR_BASE, 0);
5731         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
5732
5733         vmcs_writel(GUEST_IDTR_BASE, 0);
5734         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
5735
5736         vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
5737         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
5738         vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
5739
5740         setup_msrs(vmx);
5741
5742         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
5743
5744         if (cpu_has_vmx_tpr_shadow() && !init_event) {
5745                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
5746                 if (cpu_need_tpr_shadow(vcpu))
5747                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
5748                                      __pa(vcpu->arch.apic->regs));
5749                 vmcs_write32(TPR_THRESHOLD, 0);
5750         }
5751
5752         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
5753
5754         if (vmx->vpid != 0)
5755                 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
5756
5757         cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
5758         vmx->vcpu.arch.cr0 = cr0;
5759         vmx_set_cr0(vcpu, cr0); /* enter rmode */
5760         vmx_set_cr4(vcpu, 0);
5761         vmx_set_efer(vcpu, 0);
5762
5763         update_exception_bitmap(vcpu);
5764
5765         vpid_sync_context(vmx->vpid);
5766 }
5767
5768 /*
5769  * In nested virtualization, check if L1 asked to exit on external interrupts.
5770  * For most existing hypervisors, this will always return true.
5771  */
5772 static bool nested_exit_on_intr(struct kvm_vcpu *vcpu)
5773 {
5774         return get_vmcs12(vcpu)->pin_based_vm_exec_control &
5775                 PIN_BASED_EXT_INTR_MASK;
5776 }
5777
5778 /*
5779  * In nested virtualization, check if L1 has set
5780  * VM_EXIT_ACK_INTR_ON_EXIT
5781  */
5782 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
5783 {
5784         return get_vmcs12(vcpu)->vm_exit_controls &
5785                 VM_EXIT_ACK_INTR_ON_EXIT;
5786 }
5787
5788 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
5789 {
5790         return get_vmcs12(vcpu)->pin_based_vm_exec_control &
5791                 PIN_BASED_NMI_EXITING;
5792 }
5793
5794 static void enable_irq_window(struct kvm_vcpu *vcpu)
5795 {
5796         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
5797                       CPU_BASED_VIRTUAL_INTR_PENDING);
5798 }
5799
5800 static void enable_nmi_window(struct kvm_vcpu *vcpu)
5801 {
5802         if (!cpu_has_virtual_nmis() ||
5803             vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
5804                 enable_irq_window(vcpu);
5805                 return;
5806         }
5807
5808         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
5809                       CPU_BASED_VIRTUAL_NMI_PENDING);
5810 }
5811
5812 static void vmx_inject_irq(struct kvm_vcpu *vcpu)
5813 {
5814         struct vcpu_vmx *vmx = to_vmx(vcpu);
5815         uint32_t intr;
5816         int irq = vcpu->arch.interrupt.nr;
5817
5818         trace_kvm_inj_virq(irq);
5819
5820         ++vcpu->stat.irq_injections;
5821         if (vmx->rmode.vm86_active) {
5822                 int inc_eip = 0;
5823                 if (vcpu->arch.interrupt.soft)
5824                         inc_eip = vcpu->arch.event_exit_inst_len;
5825                 if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
5826                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5827                 return;
5828         }
5829         intr = irq | INTR_INFO_VALID_MASK;
5830         if (vcpu->arch.interrupt.soft) {
5831                 intr |= INTR_TYPE_SOFT_INTR;
5832                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
5833                              vmx->vcpu.arch.event_exit_inst_len);
5834         } else
5835                 intr |= INTR_TYPE_EXT_INTR;
5836         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
5837 }
5838
5839 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
5840 {
5841         struct vcpu_vmx *vmx = to_vmx(vcpu);
5842
5843         if (!cpu_has_virtual_nmis()) {
5844                 /*
5845                  * Tracking the NMI-blocked state in software is built upon
5846                  * finding the next open IRQ window. This, in turn, depends on
5847                  * well-behaving guests: They have to keep IRQs disabled at
5848                  * least as long as the NMI handler runs. Otherwise we may
5849                  * cause NMI nesting, maybe breaking the guest. But as this is
5850                  * highly unlikely, we can live with the residual risk.
5851                  */
5852                 vmx->loaded_vmcs->soft_vnmi_blocked = 1;
5853                 vmx->loaded_vmcs->vnmi_blocked_time = 0;
5854         }
5855
5856         ++vcpu->stat.nmi_injections;
5857         vmx->loaded_vmcs->nmi_known_unmasked = false;
5858
5859         if (vmx->rmode.vm86_active) {
5860                 if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
5861                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5862                 return;
5863         }
5864
5865         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
5866                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
5867 }
5868
5869 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
5870 {
5871         struct vcpu_vmx *vmx = to_vmx(vcpu);
5872         bool masked;
5873
5874         if (!cpu_has_virtual_nmis())
5875                 return vmx->loaded_vmcs->soft_vnmi_blocked;
5876         if (vmx->loaded_vmcs->nmi_known_unmasked)
5877                 return false;
5878         masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
5879         vmx->loaded_vmcs->nmi_known_unmasked = !masked;
5880         return masked;
5881 }
5882
5883 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
5884 {
5885         struct vcpu_vmx *vmx = to_vmx(vcpu);
5886
5887         if (!cpu_has_virtual_nmis()) {
5888                 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) {
5889                         vmx->loaded_vmcs->soft_vnmi_blocked = masked;
5890                         vmx->loaded_vmcs->vnmi_blocked_time = 0;
5891                 }
5892         } else {
5893                 vmx->loaded_vmcs->nmi_known_unmasked = !masked;
5894                 if (masked)
5895                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
5896                                       GUEST_INTR_STATE_NMI);
5897                 else
5898                         vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
5899                                         GUEST_INTR_STATE_NMI);
5900         }
5901 }
5902
5903 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
5904 {
5905         if (to_vmx(vcpu)->nested.nested_run_pending)
5906                 return 0;
5907
5908         if (!cpu_has_virtual_nmis() &&
5909             to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked)
5910                 return 0;
5911
5912         return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
5913                   (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
5914                    | GUEST_INTR_STATE_NMI));
5915 }
5916
5917 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
5918 {
5919         return (!to_vmx(vcpu)->nested.nested_run_pending &&
5920                 vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
5921                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
5922                         (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
5923 }
5924
5925 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
5926 {
5927         int ret;
5928
5929         ret = x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
5930                                     PAGE_SIZE * 3);
5931         if (ret)
5932                 return ret;
5933         kvm->arch.tss_addr = addr;
5934         return init_rmode_tss(kvm);
5935 }
5936
5937 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
5938 {
5939         switch (vec) {
5940         case BP_VECTOR:
5941                 /*
5942                  * Update instruction length as we may reinject the exception
5943                  * from user space while in guest debugging mode.
5944                  */
5945                 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
5946                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
5947                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
5948                         return false;
5949                 /* fall through */
5950         case DB_VECTOR:
5951                 if (vcpu->guest_debug &
5952                         (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
5953                         return false;
5954                 /* fall through */
5955         case DE_VECTOR:
5956         case OF_VECTOR:
5957         case BR_VECTOR:
5958         case UD_VECTOR:
5959         case DF_VECTOR:
5960         case SS_VECTOR:
5961         case GP_VECTOR:
5962         case MF_VECTOR:
5963                 return true;
5964         break;
5965         }
5966         return false;
5967 }
5968
5969 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
5970                                   int vec, u32 err_code)
5971 {
5972         /*
5973          * Instruction with address size override prefix opcode 0x67
5974          * Cause the #SS fault with 0 error code in VM86 mode.
5975          */
5976         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
5977                 if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
5978                         if (vcpu->arch.halt_request) {
5979                                 vcpu->arch.halt_request = 0;
5980                                 return kvm_vcpu_halt(vcpu);
5981                         }
5982                         return 1;
5983                 }
5984                 return 0;
5985         }
5986
5987         /*
5988          * Forward all other exceptions that are valid in real mode.
5989          * FIXME: Breaks guest debugging in real mode, needs to be fixed with
5990          *        the required debugging infrastructure rework.
5991          */
5992         kvm_queue_exception(vcpu, vec);
5993         return 1;
5994 }
5995
5996 /*
5997  * Trigger machine check on the host. We assume all the MSRs are already set up
5998  * by the CPU and that we still run on the same CPU as the MCE occurred on.
5999  * We pass a fake environment to the machine check handler because we want
6000  * the guest to be always treated like user space, no matter what context
6001  * it used internally.
6002  */
6003 static void kvm_machine_check(void)
6004 {
6005 #if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
6006         struct pt_regs regs = {
6007                 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
6008                 .flags = X86_EFLAGS_IF,
6009         };
6010
6011         do_machine_check(&regs, 0);
6012 #endif
6013 }
6014
6015 static int handle_machine_check(struct kvm_vcpu *vcpu)
6016 {
6017         /* already handled by vcpu_run */
6018         return 1;
6019 }
6020
6021 static int handle_exception(struct kvm_vcpu *vcpu)
6022 {
6023         struct vcpu_vmx *vmx = to_vmx(vcpu);
6024         struct kvm_run *kvm_run = vcpu->run;
6025         u32 intr_info, ex_no, error_code;
6026         unsigned long cr2, rip, dr6;
6027         u32 vect_info;
6028         enum emulation_result er;
6029
6030         vect_info = vmx->idt_vectoring_info;
6031         intr_info = vmx->exit_intr_info;
6032
6033         if (is_machine_check(intr_info))
6034                 return handle_machine_check(vcpu);
6035
6036         if (is_nmi(intr_info))
6037                 return 1;  /* already handled by vmx_vcpu_run() */
6038
6039         if (is_invalid_opcode(intr_info)) {
6040                 er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
6041                 if (er == EMULATE_USER_EXIT)
6042                         return 0;
6043                 if (er != EMULATE_DONE)
6044                         kvm_queue_exception(vcpu, UD_VECTOR);
6045                 return 1;
6046         }
6047
6048         error_code = 0;
6049         if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
6050                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
6051
6052         /*
6053          * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
6054          * MMIO, it is better to report an internal error.
6055          * See the comments in vmx_handle_exit.
6056          */
6057         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
6058             !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
6059                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6060                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
6061                 vcpu->run->internal.ndata = 3;
6062                 vcpu->run->internal.data[0] = vect_info;
6063                 vcpu->run->internal.data[1] = intr_info;
6064                 vcpu->run->internal.data[2] = error_code;
6065                 return 0;
6066         }
6067
6068         if (is_page_fault(intr_info)) {
6069                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
6070                 /* EPT won't cause page fault directly */
6071                 WARN_ON_ONCE(!vcpu->arch.apf.host_apf_reason && enable_ept);
6072                 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0,
6073                                 true);
6074         }
6075
6076         ex_no = intr_info & INTR_INFO_VECTOR_MASK;
6077
6078         if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
6079                 return handle_rmode_exception(vcpu, ex_no, error_code);
6080
6081         switch (ex_no) {
6082         case AC_VECTOR:
6083                 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
6084                 return 1;
6085         case DB_VECTOR:
6086                 dr6 = vmcs_readl(EXIT_QUALIFICATION);
6087                 if (!(vcpu->guest_debug &
6088                       (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
6089                         vcpu->arch.dr6 &= ~15;
6090                         vcpu->arch.dr6 |= dr6 | DR6_RTM;
6091                         if (!(dr6 & ~DR6_RESERVED)) /* icebp */
6092                                 skip_emulated_instruction(vcpu);
6093
6094                         kvm_queue_exception(vcpu, DB_VECTOR);
6095                         return 1;
6096                 }
6097                 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
6098                 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
6099                 /* fall through */
6100         case BP_VECTOR:
6101                 /*
6102                  * Update instruction length as we may reinject #BP from
6103                  * user space while in guest debugging mode. Reading it for
6104                  * #DB as well causes no harm, it is not used in that case.
6105                  */
6106                 vmx->vcpu.arch.event_exit_inst_len =
6107                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
6108                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
6109                 rip = kvm_rip_read(vcpu);
6110                 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
6111                 kvm_run->debug.arch.exception = ex_no;
6112                 break;
6113         default:
6114                 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
6115                 kvm_run->ex.exception = ex_no;
6116                 kvm_run->ex.error_code = error_code;
6117                 break;
6118         }
6119         return 0;
6120 }
6121
6122 static int handle_external_interrupt(struct kvm_vcpu *vcpu)
6123 {
6124         ++vcpu->stat.irq_exits;
6125         return 1;
6126 }
6127
6128 static int handle_triple_fault(struct kvm_vcpu *vcpu)
6129 {
6130         vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
6131         vcpu->mmio_needed = 0;
6132         return 0;
6133 }
6134
6135 static int handle_io(struct kvm_vcpu *vcpu)
6136 {
6137         unsigned long exit_qualification;
6138         int size, in, string, ret;
6139         unsigned port;
6140
6141         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6142         string = (exit_qualification & 16) != 0;
6143         in = (exit_qualification & 8) != 0;
6144
6145         ++vcpu->stat.io_exits;
6146
6147         if (string || in)
6148                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
6149
6150         port = exit_qualification >> 16;
6151         size = (exit_qualification & 7) + 1;
6152
6153         ret = kvm_skip_emulated_instruction(vcpu);
6154
6155         /*
6156          * TODO: we might be squashing a KVM_GUESTDBG_SINGLESTEP-triggered
6157          * KVM_EXIT_DEBUG here.
6158          */
6159         return kvm_fast_pio_out(vcpu, size, port) && ret;
6160 }
6161
6162 static void
6163 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
6164 {
6165         /*
6166          * Patch in the VMCALL instruction:
6167          */
6168         hypercall[0] = 0x0f;
6169         hypercall[1] = 0x01;
6170         hypercall[2] = 0xc1;
6171 }
6172
6173 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
6174 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
6175 {
6176         if (is_guest_mode(vcpu)) {
6177                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6178                 unsigned long orig_val = val;
6179
6180                 /*
6181                  * We get here when L2 changed cr0 in a way that did not change
6182                  * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
6183                  * but did change L0 shadowed bits. So we first calculate the
6184                  * effective cr0 value that L1 would like to write into the
6185                  * hardware. It consists of the L2-owned bits from the new
6186                  * value combined with the L1-owned bits from L1's guest_cr0.
6187                  */
6188                 val = (val & ~vmcs12->cr0_guest_host_mask) |
6189                         (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
6190
6191                 if (!nested_guest_cr0_valid(vcpu, val))
6192                         return 1;
6193
6194                 if (kvm_set_cr0(vcpu, val))
6195                         return 1;
6196                 vmcs_writel(CR0_READ_SHADOW, orig_val);
6197                 return 0;
6198         } else {
6199                 if (to_vmx(vcpu)->nested.vmxon &&
6200                     !nested_host_cr0_valid(vcpu, val))
6201                         return 1;
6202
6203                 return kvm_set_cr0(vcpu, val);
6204         }
6205 }
6206
6207 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
6208 {
6209         if (is_guest_mode(vcpu)) {
6210                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6211                 unsigned long orig_val = val;
6212
6213                 /* analogously to handle_set_cr0 */
6214                 val = (val & ~vmcs12->cr4_guest_host_mask) |
6215                         (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
6216                 if (kvm_set_cr4(vcpu, val))
6217                         return 1;
6218                 vmcs_writel(CR4_READ_SHADOW, orig_val);
6219                 return 0;
6220         } else
6221                 return kvm_set_cr4(vcpu, val);
6222 }
6223
6224 static int handle_cr(struct kvm_vcpu *vcpu)
6225 {
6226         unsigned long exit_qualification, val;
6227         int cr;
6228         int reg;
6229         int err;
6230         int ret;
6231
6232         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6233         cr = exit_qualification & 15;
6234         reg = (exit_qualification >> 8) & 15;
6235         switch ((exit_qualification >> 4) & 3) {
6236         case 0: /* mov to cr */
6237                 val = kvm_register_readl(vcpu, reg);
6238                 trace_kvm_cr_write(cr, val);
6239                 switch (cr) {
6240                 case 0:
6241                         err = handle_set_cr0(vcpu, val);
6242                         return kvm_complete_insn_gp(vcpu, err);
6243                 case 3:
6244                         err = kvm_set_cr3(vcpu, val);
6245                         return kvm_complete_insn_gp(vcpu, err);
6246                 case 4:
6247                         err = handle_set_cr4(vcpu, val);
6248                         return kvm_complete_insn_gp(vcpu, err);
6249                 case 8: {
6250                                 u8 cr8_prev = kvm_get_cr8(vcpu);
6251                                 u8 cr8 = (u8)val;
6252                                 err = kvm_set_cr8(vcpu, cr8);
6253                                 ret = kvm_complete_insn_gp(vcpu, err);
6254                                 if (lapic_in_kernel(vcpu))
6255                                         return ret;
6256                                 if (cr8_prev <= cr8)
6257                                         return ret;
6258                                 /*
6259                                  * TODO: we might be squashing a
6260                                  * KVM_GUESTDBG_SINGLESTEP-triggered
6261                                  * KVM_EXIT_DEBUG here.
6262                                  */
6263                                 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
6264                                 return 0;
6265                         }
6266                 }
6267                 break;
6268         case 2: /* clts */
6269                 WARN_ONCE(1, "Guest should always own CR0.TS");
6270                 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
6271                 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
6272                 return kvm_skip_emulated_instruction(vcpu);
6273         case 1: /*mov from cr*/
6274                 switch (cr) {
6275                 case 3:
6276                         val = kvm_read_cr3(vcpu);
6277                         kvm_register_write(vcpu, reg, val);
6278                         trace_kvm_cr_read(cr, val);
6279                         return kvm_skip_emulated_instruction(vcpu);
6280                 case 8:
6281                         val = kvm_get_cr8(vcpu);
6282                         kvm_register_write(vcpu, reg, val);
6283                         trace_kvm_cr_read(cr, val);
6284                         return kvm_skip_emulated_instruction(vcpu);
6285                 }
6286                 break;
6287         case 3: /* lmsw */
6288                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
6289                 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
6290                 kvm_lmsw(vcpu, val);
6291
6292                 return kvm_skip_emulated_instruction(vcpu);
6293         default:
6294                 break;
6295         }
6296         vcpu->run->exit_reason = 0;
6297         vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
6298                (int)(exit_qualification >> 4) & 3, cr);
6299         return 0;
6300 }
6301
6302 static int handle_dr(struct kvm_vcpu *vcpu)
6303 {
6304         unsigned long exit_qualification;
6305         int dr, dr7, reg;
6306
6307         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6308         dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
6309
6310         /* First, if DR does not exist, trigger UD */
6311         if (!kvm_require_dr(vcpu, dr))
6312                 return 1;
6313
6314         /* Do not handle if the CPL > 0, will trigger GP on re-entry */
6315         if (!kvm_require_cpl(vcpu, 0))
6316                 return 1;
6317         dr7 = vmcs_readl(GUEST_DR7);
6318         if (dr7 & DR7_GD) {
6319                 /*
6320                  * As the vm-exit takes precedence over the debug trap, we
6321                  * need to emulate the latter, either for the host or the
6322                  * guest debugging itself.
6323                  */
6324                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
6325                         vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
6326                         vcpu->run->debug.arch.dr7 = dr7;
6327                         vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
6328                         vcpu->run->debug.arch.exception = DB_VECTOR;
6329                         vcpu->run->exit_reason = KVM_EXIT_DEBUG;
6330                         return 0;
6331                 } else {
6332                         vcpu->arch.dr6 &= ~15;
6333                         vcpu->arch.dr6 |= DR6_BD | DR6_RTM;
6334                         kvm_queue_exception(vcpu, DB_VECTOR);
6335                         return 1;
6336                 }
6337         }
6338
6339         if (vcpu->guest_debug == 0) {
6340                 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
6341                                 CPU_BASED_MOV_DR_EXITING);
6342
6343                 /*
6344                  * No more DR vmexits; force a reload of the debug registers
6345                  * and reenter on this instruction.  The next vmexit will
6346                  * retrieve the full state of the debug registers.
6347                  */
6348                 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
6349                 return 1;
6350         }
6351
6352         reg = DEBUG_REG_ACCESS_REG(exit_qualification);
6353         if (exit_qualification & TYPE_MOV_FROM_DR) {
6354                 unsigned long val;
6355
6356                 if (kvm_get_dr(vcpu, dr, &val))
6357                         return 1;
6358                 kvm_register_write(vcpu, reg, val);
6359         } else
6360                 if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
6361                         return 1;
6362
6363         return kvm_skip_emulated_instruction(vcpu);
6364 }
6365
6366 static u64 vmx_get_dr6(struct kvm_vcpu *vcpu)
6367 {
6368         return vcpu->arch.dr6;
6369 }
6370
6371 static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
6372 {
6373 }
6374
6375 static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
6376 {
6377         get_debugreg(vcpu->arch.db[0], 0);
6378         get_debugreg(vcpu->arch.db[1], 1);
6379         get_debugreg(vcpu->arch.db[2], 2);
6380         get_debugreg(vcpu->arch.db[3], 3);
6381         get_debugreg(vcpu->arch.dr6, 6);
6382         vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);
6383
6384         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
6385         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, CPU_BASED_MOV_DR_EXITING);
6386 }
6387
6388 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
6389 {
6390         vmcs_writel(GUEST_DR7, val);
6391 }
6392
6393 static int handle_cpuid(struct kvm_vcpu *vcpu)
6394 {
6395         return kvm_emulate_cpuid(vcpu);
6396 }
6397
6398 static int handle_rdmsr(struct kvm_vcpu *vcpu)
6399 {
6400         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
6401         struct msr_data msr_info;
6402
6403         msr_info.index = ecx;
6404         msr_info.host_initiated = false;
6405         if (vmx_get_msr(vcpu, &msr_info)) {
6406                 trace_kvm_msr_read_ex(ecx);
6407                 kvm_inject_gp(vcpu, 0);
6408                 return 1;
6409         }
6410
6411         trace_kvm_msr_read(ecx, msr_info.data);
6412
6413         /* FIXME: handling of bits 32:63 of rax, rdx */
6414         vcpu->arch.regs[VCPU_REGS_RAX] = msr_info.data & -1u;
6415         vcpu->arch.regs[VCPU_REGS_RDX] = (msr_info.data >> 32) & -1u;
6416         return kvm_skip_emulated_instruction(vcpu);
6417 }
6418
6419 static int handle_wrmsr(struct kvm_vcpu *vcpu)
6420 {
6421         struct msr_data msr;
6422         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
6423         u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
6424                 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
6425
6426         msr.data = data;
6427         msr.index = ecx;
6428         msr.host_initiated = false;
6429         if (kvm_set_msr(vcpu, &msr) != 0) {
6430                 trace_kvm_msr_write_ex(ecx, data);
6431                 kvm_inject_gp(vcpu, 0);
6432                 return 1;
6433         }
6434
6435         trace_kvm_msr_write(ecx, data);
6436         return kvm_skip_emulated_instruction(vcpu);
6437 }
6438
6439 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
6440 {
6441         kvm_apic_update_ppr(vcpu);
6442         return 1;
6443 }
6444
6445 static int handle_interrupt_window(struct kvm_vcpu *vcpu)
6446 {
6447         vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
6448                         CPU_BASED_VIRTUAL_INTR_PENDING);
6449
6450         kvm_make_request(KVM_REQ_EVENT, vcpu);
6451
6452         ++vcpu->stat.irq_window_exits;
6453         return 1;
6454 }
6455
6456 static int handle_halt(struct kvm_vcpu *vcpu)
6457 {
6458         return kvm_emulate_halt(vcpu);
6459 }
6460
6461 static int handle_vmcall(struct kvm_vcpu *vcpu)
6462 {
6463         return kvm_emulate_hypercall(vcpu);
6464 }
6465
6466 static int handle_invd(struct kvm_vcpu *vcpu)
6467 {
6468         return emulate_instruction(vcpu, 0) == EMULATE_DONE;
6469 }
6470
6471 static int handle_invlpg(struct kvm_vcpu *vcpu)
6472 {
6473         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6474
6475         kvm_mmu_invlpg(vcpu, exit_qualification);
6476         return kvm_skip_emulated_instruction(vcpu);
6477 }
6478
6479 static int handle_rdpmc(struct kvm_vcpu *vcpu)
6480 {
6481         int err;
6482
6483         err = kvm_rdpmc(vcpu);
6484         return kvm_complete_insn_gp(vcpu, err);
6485 }
6486
6487 static int handle_wbinvd(struct kvm_vcpu *vcpu)
6488 {
6489         return kvm_emulate_wbinvd(vcpu);
6490 }
6491
6492 static int handle_xsetbv(struct kvm_vcpu *vcpu)
6493 {
6494         u64 new_bv = kvm_read_edx_eax(vcpu);
6495         u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);
6496
6497         if (kvm_set_xcr(vcpu, index, new_bv) == 0)
6498                 return kvm_skip_emulated_instruction(vcpu);
6499         return 1;
6500 }
6501
6502 static int handle_xsaves(struct kvm_vcpu *vcpu)
6503 {
6504         kvm_skip_emulated_instruction(vcpu);
6505         WARN(1, "this should never happen\n");
6506         return 1;
6507 }
6508
6509 static int handle_xrstors(struct kvm_vcpu *vcpu)
6510 {
6511         kvm_skip_emulated_instruction(vcpu);
6512         WARN(1, "this should never happen\n");
6513         return 1;
6514 }
6515
6516 static int handle_apic_access(struct kvm_vcpu *vcpu)
6517 {
6518         if (likely(fasteoi)) {
6519                 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6520                 int access_type, offset;
6521
6522                 access_type = exit_qualification & APIC_ACCESS_TYPE;
6523                 offset = exit_qualification & APIC_ACCESS_OFFSET;
6524                 /*
6525                  * Sane guest uses MOV to write EOI, with written value
6526                  * not cared. So make a short-circuit here by avoiding
6527                  * heavy instruction emulation.
6528                  */
6529                 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
6530                     (offset == APIC_EOI)) {
6531                         kvm_lapic_set_eoi(vcpu);
6532                         return kvm_skip_emulated_instruction(vcpu);
6533                 }
6534         }
6535         return emulate_instruction(vcpu, 0) == EMULATE_DONE;
6536 }
6537
6538 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
6539 {
6540         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6541         int vector = exit_qualification & 0xff;
6542
6543         /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
6544         kvm_apic_set_eoi_accelerated(vcpu, vector);
6545         return 1;
6546 }
6547
6548 static int handle_apic_write(struct kvm_vcpu *vcpu)
6549 {
6550         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6551         u32 offset = exit_qualification & 0xfff;
6552
6553         /* APIC-write VM exit is trap-like and thus no need to adjust IP */
6554         kvm_apic_write_nodecode(vcpu, offset);
6555         return 1;
6556 }
6557
6558 static int handle_task_switch(struct kvm_vcpu *vcpu)
6559 {
6560         struct vcpu_vmx *vmx = to_vmx(vcpu);
6561         unsigned long exit_qualification;
6562         bool has_error_code = false;
6563         u32 error_code = 0;
6564         u16 tss_selector;
6565         int reason, type, idt_v, idt_index;
6566
6567         idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
6568         idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
6569         type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
6570
6571         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6572
6573         reason = (u32)exit_qualification >> 30;
6574         if (reason == TASK_SWITCH_GATE && idt_v) {
6575                 switch (type) {
6576                 case INTR_TYPE_NMI_INTR:
6577                         vcpu->arch.nmi_injected = false;
6578                         vmx_set_nmi_mask(vcpu, true);
6579                         break;
6580                 case INTR_TYPE_EXT_INTR:
6581                 case INTR_TYPE_SOFT_INTR:
6582                         kvm_clear_interrupt_queue(vcpu);
6583                         break;
6584                 case INTR_TYPE_HARD_EXCEPTION:
6585                         if (vmx->idt_vectoring_info &
6586                             VECTORING_INFO_DELIVER_CODE_MASK) {
6587                                 has_error_code = true;
6588                                 error_code =
6589                                         vmcs_read32(IDT_VECTORING_ERROR_CODE);
6590                         }
6591                         /* fall through */
6592                 case INTR_TYPE_SOFT_EXCEPTION:
6593                         kvm_clear_exception_queue(vcpu);
6594                         break;
6595                 default:
6596                         break;
6597                 }
6598         }
6599         tss_selector = exit_qualification;
6600
6601         if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
6602                        type != INTR_TYPE_EXT_INTR &&
6603                        type != INTR_TYPE_NMI_INTR))
6604                 skip_emulated_instruction(vcpu);
6605
6606         if (kvm_task_switch(vcpu, tss_selector,
6607                             type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
6608                             has_error_code, error_code) == EMULATE_FAIL) {
6609                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6610                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
6611                 vcpu->run->internal.ndata = 0;
6612                 return 0;
6613         }
6614
6615         /*
6616          * TODO: What about debug traps on tss switch?
6617          *       Are we supposed to inject them and update dr6?
6618          */
6619
6620         return 1;
6621 }
6622
6623 static int handle_ept_violation(struct kvm_vcpu *vcpu)
6624 {
6625         unsigned long exit_qualification;
6626         gpa_t gpa;
6627         u64 error_code;
6628
6629         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6630
6631         /*
6632          * EPT violation happened while executing iret from NMI,
6633          * "blocked by NMI" bit has to be set before next VM entry.
6634          * There are errata that may cause this bit to not be set:
6635          * AAK134, BY25.
6636          */
6637         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
6638                         cpu_has_virtual_nmis() &&
6639                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
6640                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
6641
6642         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
6643         trace_kvm_page_fault(gpa, exit_qualification);
6644
6645         /* Is it a read fault? */
6646         error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
6647                      ? PFERR_USER_MASK : 0;
6648         /* Is it a write fault? */
6649         error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
6650                       ? PFERR_WRITE_MASK : 0;
6651         /* Is it a fetch fault? */
6652         error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
6653                       ? PFERR_FETCH_MASK : 0;
6654         /* ept page table entry is present? */
6655         error_code |= (exit_qualification &
6656                        (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE |
6657                         EPT_VIOLATION_EXECUTABLE))
6658                       ? PFERR_PRESENT_MASK : 0;
6659
6660         error_code |= (exit_qualification & 0x100) != 0 ?
6661                PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
6662
6663         vcpu->arch.exit_qualification = exit_qualification;
6664         return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
6665 }
6666
6667 static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
6668 {
6669         int ret;
6670         gpa_t gpa;
6671
6672         /*
6673          * A nested guest cannot optimize MMIO vmexits, because we have an
6674          * nGPA here instead of the required GPA.
6675          */
6676         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
6677         if (!is_guest_mode(vcpu) &&
6678             !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
6679                 trace_kvm_fast_mmio(gpa);
6680                 return kvm_skip_emulated_instruction(vcpu);
6681         }
6682
6683         ret = kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
6684         if (ret >= 0)
6685                 return ret;
6686
6687         /* It is the real ept misconfig */
6688         WARN_ON(1);
6689
6690         vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
6691         vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
6692
6693         return 0;
6694 }
6695
6696 static int handle_nmi_window(struct kvm_vcpu *vcpu)
6697 {
6698         vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
6699                         CPU_BASED_VIRTUAL_NMI_PENDING);
6700         ++vcpu->stat.nmi_window_exits;
6701         kvm_make_request(KVM_REQ_EVENT, vcpu);
6702
6703         return 1;
6704 }
6705
6706 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
6707 {
6708         struct vcpu_vmx *vmx = to_vmx(vcpu);
6709         enum emulation_result err = EMULATE_DONE;
6710         int ret = 1;
6711         u32 cpu_exec_ctrl;
6712         bool intr_window_requested;
6713         unsigned count = 130;
6714
6715         cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
6716         intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
6717
6718         while (vmx->emulation_required && count-- != 0) {
6719                 if (intr_window_requested && vmx_interrupt_allowed(vcpu))
6720                         return handle_interrupt_window(&vmx->vcpu);
6721
6722                 if (kvm_test_request(KVM_REQ_EVENT, vcpu))
6723                         return 1;
6724
6725                 err = emulate_instruction(vcpu, 0);
6726
6727                 if (err == EMULATE_USER_EXIT) {
6728                         ++vcpu->stat.mmio_exits;
6729                         ret = 0;
6730                         goto out;
6731                 }
6732
6733                 if (err != EMULATE_DONE) {
6734                         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6735                         vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
6736                         vcpu->run->internal.ndata = 0;
6737                         return 0;
6738                 }
6739
6740                 if (vcpu->arch.halt_request) {
6741                         vcpu->arch.halt_request = 0;
6742                         ret = kvm_vcpu_halt(vcpu);
6743                         goto out;
6744                 }
6745
6746                 if (signal_pending(current))
6747                         goto out;
6748                 if (need_resched())
6749                         schedule();
6750         }
6751
6752 out:
6753         return ret;
6754 }
6755
6756 static int __grow_ple_window(int val)
6757 {
6758         if (ple_window_grow < 1)
6759                 return ple_window;
6760
6761         val = min(val, ple_window_actual_max);
6762
6763         if (ple_window_grow < ple_window)
6764                 val *= ple_window_grow;
6765         else
6766                 val += ple_window_grow;
6767
6768         return val;
6769 }
6770
6771 static int __shrink_ple_window(int val, int modifier, int minimum)
6772 {
6773         if (modifier < 1)
6774                 return ple_window;
6775
6776         if (modifier < ple_window)
6777                 val /= modifier;
6778         else
6779                 val -= modifier;
6780
6781         return max(val, minimum);
6782 }
6783
6784 static void grow_ple_window(struct kvm_vcpu *vcpu)
6785 {
6786         struct vcpu_vmx *vmx = to_vmx(vcpu);
6787         int old = vmx->ple_window;
6788
6789         vmx->ple_window = __grow_ple_window(old);
6790
6791         if (vmx->ple_window != old)
6792                 vmx->ple_window_dirty = true;
6793
6794         trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);
6795 }
6796
6797 static void shrink_ple_window(struct kvm_vcpu *vcpu)
6798 {
6799         struct vcpu_vmx *vmx = to_vmx(vcpu);
6800         int old = vmx->ple_window;
6801
6802         vmx->ple_window = __shrink_ple_window(old,
6803                                               ple_window_shrink, ple_window);
6804
6805         if (vmx->ple_window != old)
6806                 vmx->ple_window_dirty = true;
6807
6808         trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);
6809 }
6810
6811 /*
6812  * ple_window_actual_max is computed to be one grow_ple_window() below
6813  * ple_window_max. (See __grow_ple_window for the reason.)
6814  * This prevents overflows, because ple_window_max is int.
6815  * ple_window_max effectively rounded down to a multiple of ple_window_grow in
6816  * this process.
6817  * ple_window_max is also prevented from setting vmx->ple_window < ple_window.
6818  */
6819 static void update_ple_window_actual_max(void)
6820 {
6821         ple_window_actual_max =
6822                         __shrink_ple_window(max(ple_window_max, ple_window),
6823                                             ple_window_grow, INT_MIN);
6824 }
6825
6826 /*
6827  * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
6828  */
6829 static void wakeup_handler(void)
6830 {
6831         struct kvm_vcpu *vcpu;
6832         int cpu = smp_processor_id();
6833
6834         spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
6835         list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
6836                         blocked_vcpu_list) {
6837                 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
6838
6839                 if (pi_test_on(pi_desc) == 1)
6840                         kvm_vcpu_kick(vcpu);
6841         }
6842         spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
6843 }
6844
6845 void vmx_enable_tdp(void)
6846 {
6847         kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK,
6848                 enable_ept_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull,
6849                 enable_ept_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull,
6850                 0ull, VMX_EPT_EXECUTABLE_MASK,
6851                 cpu_has_vmx_ept_execute_only() ? 0ull : VMX_EPT_READABLE_MASK,
6852                 VMX_EPT_RWX_MASK, 0ull);
6853
6854         ept_set_mmio_spte_mask();
6855         kvm_enable_tdp();
6856 }
6857
6858 static __init int hardware_setup(void)
6859 {
6860         int r = -ENOMEM, i;
6861
6862         rdmsrl_safe(MSR_EFER, &host_efer);
6863
6864         for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
6865                 kvm_define_shared_msr(i, vmx_msr_index[i]);
6866
6867         for (i = 0; i < VMX_BITMAP_NR; i++) {
6868                 vmx_bitmap[i] = (unsigned long *)__get_free_page(GFP_KERNEL);
6869                 if (!vmx_bitmap[i])
6870                         goto out;
6871         }
6872
6873         vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
6874         memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
6875         memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
6876
6877         memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
6878
6879         memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
6880
6881         if (setup_vmcs_config(&vmcs_config) < 0) {
6882                 r = -EIO;
6883                 goto out;
6884         }
6885
6886         if (boot_cpu_has(X86_FEATURE_NX))
6887                 kvm_enable_efer_bits(EFER_NX);
6888
6889         if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() ||
6890                 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global()))
6891                 enable_vpid = 0;
6892
6893         if (!cpu_has_vmx_shadow_vmcs())
6894                 enable_shadow_vmcs = 0;
6895         if (enable_shadow_vmcs)
6896                 init_vmcs_shadow_fields();
6897
6898         if (!cpu_has_vmx_ept() ||
6899             !cpu_has_vmx_ept_4levels() ||
6900             !cpu_has_vmx_ept_mt_wb()) {
6901                 enable_ept = 0;
6902                 enable_unrestricted_guest = 0;
6903                 enable_ept_ad_bits = 0;
6904         }
6905
6906         if (!cpu_has_vmx_ept_ad_bits() || !enable_ept)
6907                 enable_ept_ad_bits = 0;
6908
6909         if (!cpu_has_vmx_unrestricted_guest())
6910                 enable_unrestricted_guest = 0;
6911
6912         if (!cpu_has_vmx_flexpriority())
6913                 flexpriority_enabled = 0;
6914
6915         /*
6916          * set_apic_access_page_addr() is used to reload apic access
6917          * page upon invalidation.  No need to do anything if not
6918          * using the APIC_ACCESS_ADDR VMCS field.
6919          */
6920         if (!flexpriority_enabled)
6921                 kvm_x86_ops->set_apic_access_page_addr = NULL;
6922
6923         if (!cpu_has_vmx_tpr_shadow())
6924                 kvm_x86_ops->update_cr8_intercept = NULL;
6925
6926         if (enable_ept && !cpu_has_vmx_ept_2m_page())
6927                 kvm_disable_largepages();
6928
6929         if (!cpu_has_vmx_ple())
6930                 ple_gap = 0;
6931
6932         if (!cpu_has_vmx_apicv()) {
6933                 enable_apicv = 0;
6934                 kvm_x86_ops->sync_pir_to_irr = NULL;
6935         }
6936
6937         if (cpu_has_vmx_tsc_scaling()) {
6938                 kvm_has_tsc_control = true;
6939                 kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
6940                 kvm_tsc_scaling_ratio_frac_bits = 48;
6941         }
6942
6943         set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
6944
6945         if (enable_ept)
6946                 vmx_enable_tdp();
6947         else
6948                 kvm_disable_tdp();
6949
6950         update_ple_window_actual_max();
6951
6952         /*
6953          * Only enable PML when hardware supports PML feature, and both EPT
6954          * and EPT A/D bit features are enabled -- PML depends on them to work.
6955          */
6956         if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
6957                 enable_pml = 0;
6958
6959         if (!enable_pml) {
6960                 kvm_x86_ops->slot_enable_log_dirty = NULL;
6961                 kvm_x86_ops->slot_disable_log_dirty = NULL;
6962                 kvm_x86_ops->flush_log_dirty = NULL;
6963                 kvm_x86_ops->enable_log_dirty_pt_masked = NULL;
6964         }
6965
6966         if (cpu_has_vmx_preemption_timer() && enable_preemption_timer) {
6967                 u64 vmx_msr;
6968
6969                 rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
6970                 cpu_preemption_timer_multi =
6971                          vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
6972         } else {
6973                 kvm_x86_ops->set_hv_timer = NULL;
6974                 kvm_x86_ops->cancel_hv_timer = NULL;
6975         }
6976
6977         kvm_set_posted_intr_wakeup_handler(wakeup_handler);
6978
6979         kvm_mce_cap_supported |= MCG_LMCE_P;
6980
6981         return alloc_kvm_area();
6982
6983 out:
6984         for (i = 0; i < VMX_BITMAP_NR; i++)
6985                 free_page((unsigned long)vmx_bitmap[i]);
6986
6987     return r;
6988 }
6989
6990 static __exit void hardware_unsetup(void)
6991 {
6992         int i;
6993
6994         for (i = 0; i < VMX_BITMAP_NR; i++)
6995                 free_page((unsigned long)vmx_bitmap[i]);
6996
6997         free_kvm_area();
6998 }
6999
7000 /*
7001  * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
7002  * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
7003  */
7004 static int handle_pause(struct kvm_vcpu *vcpu)
7005 {
7006         if (ple_gap)
7007                 grow_ple_window(vcpu);
7008
7009         /*
7010          * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting"
7011          * VM-execution control is ignored if CPL > 0. OTOH, KVM
7012          * never set PAUSE_EXITING and just set PLE if supported,
7013          * so the vcpu must be CPL=0 if it gets a PAUSE exit.
7014          */
7015         kvm_vcpu_on_spin(vcpu, true);
7016         return kvm_skip_emulated_instruction(vcpu);
7017 }
7018
7019 static int handle_nop(struct kvm_vcpu *vcpu)
7020 {
7021         return kvm_skip_emulated_instruction(vcpu);
7022 }
7023
7024 static int handle_mwait(struct kvm_vcpu *vcpu)
7025 {
7026         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
7027         return handle_nop(vcpu);
7028 }
7029
7030 static int handle_invalid_op(struct kvm_vcpu *vcpu)
7031 {
7032         kvm_queue_exception(vcpu, UD_VECTOR);
7033         return 1;
7034 }
7035
7036 static int handle_monitor_trap(struct kvm_vcpu *vcpu)
7037 {
7038         return 1;
7039 }
7040
7041 static int handle_monitor(struct kvm_vcpu *vcpu)
7042 {
7043         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
7044         return handle_nop(vcpu);
7045 }
7046
7047 /*
7048  * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
7049  * set the success or error code of an emulated VMX instruction, as specified
7050  * by Vol 2B, VMX Instruction Reference, "Conventions".
7051  */
7052 static void nested_vmx_succeed(struct kvm_vcpu *vcpu)
7053 {
7054         vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
7055                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
7056                             X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
7057 }
7058
7059 static void nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
7060 {
7061         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
7062                         & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
7063                             X86_EFLAGS_SF | X86_EFLAGS_OF))
7064                         | X86_EFLAGS_CF);
7065 }
7066
7067 static void nested_vmx_failValid(struct kvm_vcpu *vcpu,
7068                                         u32 vm_instruction_error)
7069 {
7070         if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {
7071                 /*
7072                  * failValid writes the error number to the current VMCS, which
7073                  * can't be done there isn't a current VMCS.
7074                  */
7075                 nested_vmx_failInvalid(vcpu);
7076                 return;
7077         }
7078         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
7079                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
7080                             X86_EFLAGS_SF | X86_EFLAGS_OF))
7081                         | X86_EFLAGS_ZF);
7082         get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
7083         /*
7084          * We don't need to force a shadow sync because
7085          * VM_INSTRUCTION_ERROR is not shadowed
7086          */
7087 }
7088
7089 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
7090 {
7091         /* TODO: not to reset guest simply here. */
7092         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
7093         pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
7094 }
7095
7096 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
7097 {
7098         struct vcpu_vmx *vmx =
7099                 container_of(timer, struct vcpu_vmx, nested.preemption_timer);
7100
7101         vmx->nested.preemption_timer_expired = true;
7102         kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
7103         kvm_vcpu_kick(&vmx->vcpu);
7104
7105         return HRTIMER_NORESTART;
7106 }
7107
7108 /*
7109  * Decode the memory-address operand of a vmx instruction, as recorded on an
7110  * exit caused by such an instruction (run by a guest hypervisor).
7111  * On success, returns 0. When the operand is invalid, returns 1 and throws
7112  * #UD or #GP.
7113  */
7114 static int get_vmx_mem_address(struct kvm_vcpu *vcpu,
7115                                  unsigned long exit_qualification,
7116                                  u32 vmx_instruction_info, bool wr, gva_t *ret)
7117 {
7118         gva_t off;
7119         bool exn;
7120         struct kvm_segment s;
7121
7122         /*
7123          * According to Vol. 3B, "Information for VM Exits Due to Instruction
7124          * Execution", on an exit, vmx_instruction_info holds most of the
7125          * addressing components of the operand. Only the displacement part
7126          * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
7127          * For how an actual address is calculated from all these components,
7128          * refer to Vol. 1, "Operand Addressing".
7129          */
7130         int  scaling = vmx_instruction_info & 3;
7131         int  addr_size = (vmx_instruction_info >> 7) & 7;
7132         bool is_reg = vmx_instruction_info & (1u << 10);
7133         int  seg_reg = (vmx_instruction_info >> 15) & 7;
7134         int  index_reg = (vmx_instruction_info >> 18) & 0xf;
7135         bool index_is_valid = !(vmx_instruction_info & (1u << 22));
7136         int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
7137         bool base_is_valid  = !(vmx_instruction_info & (1u << 27));
7138
7139         if (is_reg) {
7140                 kvm_queue_exception(vcpu, UD_VECTOR);
7141                 return 1;
7142         }
7143
7144         /* Addr = segment_base + offset */
7145         /* offset = base + [index * scale] + displacement */
7146         off = exit_qualification; /* holds the displacement */
7147         if (base_is_valid)
7148                 off += kvm_register_read(vcpu, base_reg);
7149         if (index_is_valid)
7150                 off += kvm_register_read(vcpu, index_reg)<<scaling;
7151         vmx_get_segment(vcpu, &s, seg_reg);
7152         *ret = s.base + off;
7153
7154         if (addr_size == 1) /* 32 bit */
7155                 *ret &= 0xffffffff;
7156
7157         /* Checks for #GP/#SS exceptions. */
7158         exn = false;
7159         if (is_long_mode(vcpu)) {
7160                 /* Long mode: #GP(0)/#SS(0) if the memory address is in a
7161                  * non-canonical form. This is the only check on the memory
7162                  * destination for long mode!
7163                  */
7164                 exn = is_noncanonical_address(*ret, vcpu);
7165         } else if (is_protmode(vcpu)) {
7166                 /* Protected mode: apply checks for segment validity in the
7167                  * following order:
7168                  * - segment type check (#GP(0) may be thrown)
7169                  * - usability check (#GP(0)/#SS(0))
7170                  * - limit check (#GP(0)/#SS(0))
7171                  */
7172                 if (wr)
7173                         /* #GP(0) if the destination operand is located in a
7174                          * read-only data segment or any code segment.
7175                          */
7176                         exn = ((s.type & 0xa) == 0 || (s.type & 8));
7177                 else
7178                         /* #GP(0) if the source operand is located in an
7179                          * execute-only code segment
7180                          */
7181                         exn = ((s.type & 0xa) == 8);
7182                 if (exn) {
7183                         kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
7184                         return 1;
7185                 }
7186                 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
7187                  */
7188                 exn = (s.unusable != 0);
7189                 /* Protected mode: #GP(0)/#SS(0) if the memory
7190                  * operand is outside the segment limit.
7191                  */
7192                 exn = exn || (off + sizeof(u64) > s.limit);
7193         }
7194         if (exn) {
7195                 kvm_queue_exception_e(vcpu,
7196                                       seg_reg == VCPU_SREG_SS ?
7197                                                 SS_VECTOR : GP_VECTOR,
7198                                       0);
7199                 return 1;
7200         }
7201
7202         return 0;
7203 }
7204
7205 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer)
7206 {
7207         gva_t gva;
7208         struct x86_exception e;
7209
7210         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
7211                         vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva))
7212                 return 1;
7213
7214         if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, vmpointer,
7215                                 sizeof(*vmpointer), &e)) {
7216                 kvm_inject_page_fault(vcpu, &e);
7217                 return 1;
7218         }
7219
7220         return 0;
7221 }
7222
7223 static int enter_vmx_operation(struct kvm_vcpu *vcpu)
7224 {
7225         struct vcpu_vmx *vmx = to_vmx(vcpu);
7226         struct vmcs *shadow_vmcs;
7227         int r;
7228
7229         r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
7230         if (r < 0)
7231                 goto out_vmcs02;
7232
7233         vmx->nested.cached_vmcs12 = kmalloc(VMCS12_SIZE, GFP_KERNEL);
7234         if (!vmx->nested.cached_vmcs12)
7235                 goto out_cached_vmcs12;
7236
7237         if (enable_shadow_vmcs) {
7238                 shadow_vmcs = alloc_vmcs();
7239                 if (!shadow_vmcs)
7240                         goto out_shadow_vmcs;
7241                 /* mark vmcs as shadow */
7242                 shadow_vmcs->revision_id |= (1u << 31);
7243                 /* init shadow vmcs */
7244                 vmcs_clear(shadow_vmcs);
7245                 vmx->vmcs01.shadow_vmcs = shadow_vmcs;
7246         }
7247
7248         hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
7249                      HRTIMER_MODE_REL_PINNED);
7250         vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
7251
7252         vmx->nested.vmxon = true;
7253         return 0;
7254
7255 out_shadow_vmcs:
7256         kfree(vmx->nested.cached_vmcs12);
7257
7258 out_cached_vmcs12:
7259         free_loaded_vmcs(&vmx->nested.vmcs02);
7260
7261 out_vmcs02:
7262         return -ENOMEM;
7263 }
7264
7265 /*
7266  * Emulate the VMXON instruction.
7267  * Currently, we just remember that VMX is active, and do not save or even
7268  * inspect the argument to VMXON (the so-called "VMXON pointer") because we
7269  * do not currently need to store anything in that guest-allocated memory
7270  * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
7271  * argument is different from the VMXON pointer (which the spec says they do).
7272  */
7273 static int handle_vmon(struct kvm_vcpu *vcpu)
7274 {
7275         int ret;
7276         gpa_t vmptr;
7277         struct page *page;
7278         struct vcpu_vmx *vmx = to_vmx(vcpu);
7279         const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
7280                 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
7281
7282         /*
7283          * The Intel VMX Instruction Reference lists a bunch of bits that are
7284          * prerequisite to running VMXON, most notably cr4.VMXE must be set to
7285          * 1 (see vmx_set_cr4() for when we allow the guest to set this).
7286          * Otherwise, we should fail with #UD.  But most faulting conditions
7287          * have already been checked by hardware, prior to the VM-exit for
7288          * VMXON.  We do test guest cr4.VMXE because processor CR4 always has
7289          * that bit set to 1 in non-root mode.
7290          */
7291         if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) {
7292                 kvm_queue_exception(vcpu, UD_VECTOR);
7293                 return 1;
7294         }
7295
7296         if (vmx->nested.vmxon) {
7297                 nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
7298                 return kvm_skip_emulated_instruction(vcpu);
7299         }
7300
7301         if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
7302                         != VMXON_NEEDED_FEATURES) {
7303                 kvm_inject_gp(vcpu, 0);
7304                 return 1;
7305         }
7306
7307         if (nested_vmx_get_vmptr(vcpu, &vmptr))
7308                 return 1;
7309
7310         /*
7311          * SDM 3: 24.11.5
7312          * The first 4 bytes of VMXON region contain the supported
7313          * VMCS revision identifier
7314          *
7315          * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case;
7316          * which replaces physical address width with 32
7317          */
7318         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
7319                 nested_vmx_failInvalid(vcpu);
7320                 return kvm_skip_emulated_instruction(vcpu);
7321         }
7322
7323         page = kvm_vcpu_gpa_to_page(vcpu, vmptr);
7324         if (is_error_page(page)) {
7325                 nested_vmx_failInvalid(vcpu);
7326                 return kvm_skip_emulated_instruction(vcpu);
7327         }
7328         if (*(u32 *)kmap(page) != VMCS12_REVISION) {
7329                 kunmap(page);
7330                 kvm_release_page_clean(page);
7331                 nested_vmx_failInvalid(vcpu);
7332                 return kvm_skip_emulated_instruction(vcpu);
7333         }
7334         kunmap(page);
7335         kvm_release_page_clean(page);
7336
7337         vmx->nested.vmxon_ptr = vmptr;
7338         ret = enter_vmx_operation(vcpu);
7339         if (ret)
7340                 return ret;
7341
7342         nested_vmx_succeed(vcpu);
7343         return kvm_skip_emulated_instruction(vcpu);
7344 }
7345
7346 /*
7347  * Intel's VMX Instruction Reference specifies a common set of prerequisites
7348  * for running VMX instructions (except VMXON, whose prerequisites are
7349  * slightly different). It also specifies what exception to inject otherwise.
7350  * Note that many of these exceptions have priority over VM exits, so they
7351  * don't have to be checked again here.
7352  */
7353 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
7354 {
7355         if (!to_vmx(vcpu)->nested.vmxon) {
7356                 kvm_queue_exception(vcpu, UD_VECTOR);
7357                 return 0;
7358         }
7359         return 1;
7360 }
7361
7362 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
7363 {
7364         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS);
7365         vmcs_write64(VMCS_LINK_POINTER, -1ull);
7366 }
7367
7368 static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
7369 {
7370         if (vmx->nested.current_vmptr == -1ull)
7371                 return;
7372
7373         if (enable_shadow_vmcs) {
7374                 /* copy to memory all shadowed fields in case
7375                    they were modified */
7376                 copy_shadow_to_vmcs12(vmx);
7377                 vmx->nested.sync_shadow_vmcs = false;
7378                 vmx_disable_shadow_vmcs(vmx);
7379         }
7380         vmx->nested.posted_intr_nv = -1;
7381
7382         /* Flush VMCS12 to guest memory */
7383         kvm_vcpu_write_guest_page(&vmx->vcpu,
7384                                   vmx->nested.current_vmptr >> PAGE_SHIFT,
7385                                   vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
7386
7387         vmx->nested.current_vmptr = -1ull;
7388 }
7389
7390 /*
7391  * Free whatever needs to be freed from vmx->nested when L1 goes down, or
7392  * just stops using VMX.
7393  */
7394 static void free_nested(struct vcpu_vmx *vmx)
7395 {
7396         if (!vmx->nested.vmxon)
7397                 return;
7398
7399         vmx->nested.vmxon = false;
7400         free_vpid(vmx->nested.vpid02);
7401         vmx->nested.posted_intr_nv = -1;
7402         vmx->nested.current_vmptr = -1ull;
7403         if (enable_shadow_vmcs) {
7404                 vmx_disable_shadow_vmcs(vmx);
7405                 vmcs_clear(vmx->vmcs01.shadow_vmcs);
7406                 free_vmcs(vmx->vmcs01.shadow_vmcs);
7407                 vmx->vmcs01.shadow_vmcs = NULL;
7408         }
7409         kfree(vmx->nested.cached_vmcs12);
7410         /* Unpin physical memory we referred to in the vmcs02 */
7411         if (vmx->nested.apic_access_page) {
7412                 kvm_release_page_dirty(vmx->nested.apic_access_page);
7413                 vmx->nested.apic_access_page = NULL;
7414         }
7415         if (vmx->nested.virtual_apic_page) {
7416                 kvm_release_page_dirty(vmx->nested.virtual_apic_page);
7417                 vmx->nested.virtual_apic_page = NULL;
7418         }
7419         if (vmx->nested.pi_desc_page) {
7420                 kunmap(vmx->nested.pi_desc_page);
7421                 kvm_release_page_dirty(vmx->nested.pi_desc_page);
7422                 vmx->nested.pi_desc_page = NULL;
7423                 vmx->nested.pi_desc = NULL;
7424         }
7425
7426         free_loaded_vmcs(&vmx->nested.vmcs02);
7427 }
7428
7429 /* Emulate the VMXOFF instruction */
7430 static int handle_vmoff(struct kvm_vcpu *vcpu)
7431 {
7432         if (!nested_vmx_check_permission(vcpu))
7433                 return 1;
7434         free_nested(to_vmx(vcpu));
7435         nested_vmx_succeed(vcpu);
7436         return kvm_skip_emulated_instruction(vcpu);
7437 }
7438
7439 /* Emulate the VMCLEAR instruction */
7440 static int handle_vmclear(struct kvm_vcpu *vcpu)
7441 {
7442         struct vcpu_vmx *vmx = to_vmx(vcpu);
7443         u32 zero = 0;
7444         gpa_t vmptr;
7445
7446         if (!nested_vmx_check_permission(vcpu))
7447                 return 1;
7448
7449         if (nested_vmx_get_vmptr(vcpu, &vmptr))
7450                 return 1;
7451
7452         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
7453                 nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS);
7454                 return kvm_skip_emulated_instruction(vcpu);
7455         }
7456
7457         if (vmptr == vmx->nested.vmxon_ptr) {
7458                 nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_VMXON_POINTER);
7459                 return kvm_skip_emulated_instruction(vcpu);
7460         }
7461
7462         if (vmptr == vmx->nested.current_vmptr)
7463                 nested_release_vmcs12(vmx);
7464
7465         kvm_vcpu_write_guest(vcpu,
7466                         vmptr + offsetof(struct vmcs12, launch_state),
7467                         &zero, sizeof(zero));
7468
7469         nested_vmx_succeed(vcpu);
7470         return kvm_skip_emulated_instruction(vcpu);
7471 }
7472
7473 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);
7474
7475 /* Emulate the VMLAUNCH instruction */
7476 static int handle_vmlaunch(struct kvm_vcpu *vcpu)
7477 {
7478         return nested_vmx_run(vcpu, true);
7479 }
7480
7481 /* Emulate the VMRESUME instruction */
7482 static int handle_vmresume(struct kvm_vcpu *vcpu)
7483 {
7484
7485         return nested_vmx_run(vcpu, false);
7486 }
7487
7488 /*
7489  * Read a vmcs12 field. Since these can have varying lengths and we return
7490  * one type, we chose the biggest type (u64) and zero-extend the return value
7491  * to that size. Note that the caller, handle_vmread, might need to use only
7492  * some of the bits we return here (e.g., on 32-bit guests, only 32 bits of
7493  * 64-bit fields are to be returned).
7494  */
7495 static inline int vmcs12_read_any(struct kvm_vcpu *vcpu,
7496                                   unsigned long field, u64 *ret)
7497 {
7498         short offset = vmcs_field_to_offset(field);
7499         char *p;
7500
7501         if (offset < 0)
7502                 return offset;
7503
7504         p = ((char *)(get_vmcs12(vcpu))) + offset;
7505
7506         switch (vmcs_field_type(field)) {
7507         case VMCS_FIELD_TYPE_NATURAL_WIDTH:
7508                 *ret = *((natural_width *)p);
7509                 return 0;
7510         case VMCS_FIELD_TYPE_U16:
7511                 *ret = *((u16 *)p);
7512                 return 0;
7513         case VMCS_FIELD_TYPE_U32:
7514                 *ret = *((u32 *)p);
7515                 return 0;
7516         case VMCS_FIELD_TYPE_U64:
7517                 *ret = *((u64 *)p);
7518                 return 0;
7519         default:
7520                 WARN_ON(1);
7521                 return -ENOENT;
7522         }
7523 }
7524
7525
7526 static inline int vmcs12_write_any(struct kvm_vcpu *vcpu,
7527                                    unsigned long field, u64 field_value){
7528         short offset = vmcs_field_to_offset(field);
7529         char *p = ((char *) get_vmcs12(vcpu)) + offset;
7530         if (offset < 0)
7531                 return offset;
7532
7533         switch (vmcs_field_type(field)) {
7534         case VMCS_FIELD_TYPE_U16:
7535                 *(u16 *)p = field_value;
7536                 return 0;
7537         case VMCS_FIELD_TYPE_U32:
7538                 *(u32 *)p = field_value;
7539                 return 0;
7540         case VMCS_FIELD_TYPE_U64:
7541                 *(u64 *)p = field_value;
7542                 return 0;
7543         case VMCS_FIELD_TYPE_NATURAL_WIDTH:
7544                 *(natural_width *)p = field_value;
7545                 return 0;
7546         default:
7547                 WARN_ON(1);
7548                 return -ENOENT;
7549         }
7550
7551 }
7552
7553 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
7554 {
7555         int i;
7556         unsigned long field;
7557         u64 field_value;
7558         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
7559         const unsigned long *fields = shadow_read_write_fields;
7560         const int num_fields = max_shadow_read_write_fields;
7561
7562         preempt_disable();
7563
7564         vmcs_load(shadow_vmcs);
7565
7566         for (i = 0; i < num_fields; i++) {
7567                 field = fields[i];
7568                 switch (vmcs_field_type(field)) {
7569                 case VMCS_FIELD_TYPE_U16:
7570                         field_value = vmcs_read16(field);
7571                         break;
7572                 case VMCS_FIELD_TYPE_U32:
7573                         field_value = vmcs_read32(field);
7574                         break;
7575                 case VMCS_FIELD_TYPE_U64:
7576                         field_value = vmcs_read64(field);
7577                         break;
7578                 case VMCS_FIELD_TYPE_NATURAL_WIDTH:
7579                         field_value = vmcs_readl(field);
7580                         break;
7581                 default:
7582                         WARN_ON(1);
7583                         continue;
7584                 }
7585                 vmcs12_write_any(&vmx->vcpu, field, field_value);
7586         }
7587
7588         vmcs_clear(shadow_vmcs);
7589         vmcs_load(vmx->loaded_vmcs->vmcs);
7590
7591         preempt_enable();
7592 }
7593
7594 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
7595 {
7596         const unsigned long *fields[] = {
7597                 shadow_read_write_fields,
7598                 shadow_read_only_fields
7599         };
7600         const int max_fields[] = {
7601                 max_shadow_read_write_fields,
7602                 max_shadow_read_only_fields
7603         };
7604         int i, q;
7605         unsigned long field;
7606         u64 field_value = 0;
7607         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
7608
7609         vmcs_load(shadow_vmcs);
7610
7611         for (q = 0; q < ARRAY_SIZE(fields); q++) {
7612                 for (i = 0; i < max_fields[q]; i++) {
7613                         field = fields[q][i];
7614                         vmcs12_read_any(&vmx->vcpu, field, &field_value);
7615
7616                         switch (vmcs_field_type(field)) {
7617                         case VMCS_FIELD_TYPE_U16:
7618                                 vmcs_write16(field, (u16)field_value);
7619                                 break;
7620                         case VMCS_FIELD_TYPE_U32:
7621                                 vmcs_write32(field, (u32)field_value);
7622                                 break;
7623                         case VMCS_FIELD_TYPE_U64:
7624                                 vmcs_write64(field, (u64)field_value);
7625                                 break;
7626                         case VMCS_FIELD_TYPE_NATURAL_WIDTH:
7627                                 vmcs_writel(field, (long)field_value);
7628                                 break;
7629                         default:
7630                                 WARN_ON(1);
7631                                 break;
7632                         }
7633                 }
7634         }
7635
7636         vmcs_clear(shadow_vmcs);
7637         vmcs_load(vmx->loaded_vmcs->vmcs);
7638 }
7639
7640 /*
7641  * VMX instructions which assume a current vmcs12 (i.e., that VMPTRLD was
7642  * used before) all generate the same failure when it is missing.
7643  */
7644 static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu)
7645 {
7646         struct vcpu_vmx *vmx = to_vmx(vcpu);
7647         if (vmx->nested.current_vmptr == -1ull) {
7648                 nested_vmx_failInvalid(vcpu);
7649                 return 0;
7650         }
7651         return 1;
7652 }
7653
7654 static int handle_vmread(struct kvm_vcpu *vcpu)
7655 {
7656         unsigned long field;
7657         u64 field_value;
7658         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7659         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7660         gva_t gva = 0;
7661
7662         if (!nested_vmx_check_permission(vcpu))
7663                 return 1;
7664
7665         if (!nested_vmx_check_vmcs12(vcpu))
7666                 return kvm_skip_emulated_instruction(vcpu);
7667
7668         /* Decode instruction info and find the field to read */
7669         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
7670         /* Read the field, zero-extended to a u64 field_value */
7671         if (vmcs12_read_any(vcpu, field, &field_value) < 0) {
7672                 nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
7673                 return kvm_skip_emulated_instruction(vcpu);
7674         }
7675         /*
7676          * Now copy part of this value to register or memory, as requested.
7677          * Note that the number of bits actually copied is 32 or 64 depending
7678          * on the guest's mode (32 or 64 bit), not on the given field's length.
7679          */
7680         if (vmx_instruction_info & (1u << 10)) {
7681                 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
7682                         field_value);
7683         } else {
7684                 if (get_vmx_mem_address(vcpu, exit_qualification,
7685                                 vmx_instruction_info, true, &gva))
7686                         return 1;
7687                 /* _system ok, as hardware has verified cpl=0 */
7688                 kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, gva,
7689                              &field_value, (is_long_mode(vcpu) ? 8 : 4), NULL);
7690         }
7691
7692         nested_vmx_succeed(vcpu);
7693         return kvm_skip_emulated_instruction(vcpu);
7694 }
7695
7696
7697 static int handle_vmwrite(struct kvm_vcpu *vcpu)
7698 {
7699         unsigned long field;
7700         gva_t gva;
7701         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7702         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7703         /* The value to write might be 32 or 64 bits, depending on L1's long
7704          * mode, and eventually we need to write that into a field of several
7705          * possible lengths. The code below first zero-extends the value to 64
7706          * bit (field_value), and then copies only the appropriate number of
7707          * bits into the vmcs12 field.
7708          */
7709         u64 field_value = 0;
7710         struct x86_exception e;
7711
7712         if (!nested_vmx_check_permission(vcpu))
7713                 return 1;
7714
7715         if (!nested_vmx_check_vmcs12(vcpu))
7716                 return kvm_skip_emulated_instruction(vcpu);
7717
7718         if (vmx_instruction_info & (1u << 10))
7719                 field_value = kvm_register_readl(vcpu,
7720                         (((vmx_instruction_info) >> 3) & 0xf));
7721         else {
7722                 if (get_vmx_mem_address(vcpu, exit_qualification,
7723                                 vmx_instruction_info, false, &gva))
7724                         return 1;
7725                 if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva,
7726                            &field_value, (is_64_bit_mode(vcpu) ? 8 : 4), &e)) {
7727                         kvm_inject_page_fault(vcpu, &e);
7728                         return 1;
7729                 }
7730         }
7731
7732
7733         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
7734         if (vmcs_field_readonly(field)) {
7735                 nested_vmx_failValid(vcpu,
7736                         VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
7737                 return kvm_skip_emulated_instruction(vcpu);
7738         }
7739
7740         if (vmcs12_write_any(vcpu, field, field_value) < 0) {
7741                 nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
7742                 return kvm_skip_emulated_instruction(vcpu);
7743         }
7744
7745         nested_vmx_succeed(vcpu);
7746         return kvm_skip_emulated_instruction(vcpu);
7747 }
7748
7749 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
7750 {
7751         vmx->nested.current_vmptr = vmptr;
7752         if (enable_shadow_vmcs) {
7753                 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
7754                               SECONDARY_EXEC_SHADOW_VMCS);
7755                 vmcs_write64(VMCS_LINK_POINTER,
7756                              __pa(vmx->vmcs01.shadow_vmcs));
7757                 vmx->nested.sync_shadow_vmcs = true;
7758         }
7759 }
7760
7761 /* Emulate the VMPTRLD instruction */
7762 static int handle_vmptrld(struct kvm_vcpu *vcpu)
7763 {
7764         struct vcpu_vmx *vmx = to_vmx(vcpu);
7765         gpa_t vmptr;
7766
7767         if (!nested_vmx_check_permission(vcpu))
7768                 return 1;
7769
7770         if (nested_vmx_get_vmptr(vcpu, &vmptr))
7771                 return 1;
7772
7773         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
7774                 nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS);
7775                 return kvm_skip_emulated_instruction(vcpu);
7776         }
7777
7778         if (vmptr == vmx->nested.vmxon_ptr) {
7779                 nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_VMXON_POINTER);
7780                 return kvm_skip_emulated_instruction(vcpu);
7781         }
7782
7783         if (vmx->nested.current_vmptr != vmptr) {
7784                 struct vmcs12 *new_vmcs12;
7785                 struct page *page;
7786                 page = kvm_vcpu_gpa_to_page(vcpu, vmptr);
7787                 if (is_error_page(page)) {
7788                         nested_vmx_failInvalid(vcpu);
7789                         return kvm_skip_emulated_instruction(vcpu);
7790                 }
7791                 new_vmcs12 = kmap(page);
7792                 if (new_vmcs12->revision_id != VMCS12_REVISION) {
7793                         kunmap(page);
7794                         kvm_release_page_clean(page);
7795                         nested_vmx_failValid(vcpu,
7796                                 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
7797                         return kvm_skip_emulated_instruction(vcpu);
7798                 }
7799
7800                 nested_release_vmcs12(vmx);
7801                 /*
7802                  * Load VMCS12 from guest memory since it is not already
7803                  * cached.
7804                  */
7805                 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE);
7806                 kunmap(page);
7807                 kvm_release_page_clean(page);
7808
7809                 set_current_vmptr(vmx, vmptr);
7810         }
7811
7812         nested_vmx_succeed(vcpu);
7813         return kvm_skip_emulated_instruction(vcpu);
7814 }
7815
7816 /* Emulate the VMPTRST instruction */
7817 static int handle_vmptrst(struct kvm_vcpu *vcpu)
7818 {
7819         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7820         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7821         gva_t vmcs_gva;
7822         struct x86_exception e;
7823
7824         if (!nested_vmx_check_permission(vcpu))
7825                 return 1;
7826
7827         if (get_vmx_mem_address(vcpu, exit_qualification,
7828                         vmx_instruction_info, true, &vmcs_gva))
7829                 return 1;
7830         /* ok to use *_system, as hardware has verified cpl=0 */
7831         if (kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, vmcs_gva,
7832                                  (void *)&to_vmx(vcpu)->nested.current_vmptr,
7833                                  sizeof(u64), &e)) {
7834                 kvm_inject_page_fault(vcpu, &e);
7835                 return 1;
7836         }
7837         nested_vmx_succeed(vcpu);
7838         return kvm_skip_emulated_instruction(vcpu);
7839 }
7840
7841 /* Emulate the INVEPT instruction */
7842 static int handle_invept(struct kvm_vcpu *vcpu)
7843 {
7844         struct vcpu_vmx *vmx = to_vmx(vcpu);
7845         u32 vmx_instruction_info, types;
7846         unsigned long type;
7847         gva_t gva;
7848         struct x86_exception e;
7849         struct {
7850                 u64 eptp, gpa;
7851         } operand;
7852
7853         if (!(vmx->nested.nested_vmx_secondary_ctls_high &
7854               SECONDARY_EXEC_ENABLE_EPT) ||
7855             !(vmx->nested.nested_vmx_ept_caps & VMX_EPT_INVEPT_BIT)) {
7856                 kvm_queue_exception(vcpu, UD_VECTOR);
7857                 return 1;
7858         }
7859
7860         if (!nested_vmx_check_permission(vcpu))
7861                 return 1;
7862
7863         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7864         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
7865
7866         types = (vmx->nested.nested_vmx_ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
7867
7868         if (type >= 32 || !(types & (1 << type))) {
7869                 nested_vmx_failValid(vcpu,
7870                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
7871                 return kvm_skip_emulated_instruction(vcpu);
7872         }
7873
7874         /* According to the Intel VMX instruction reference, the memory
7875          * operand is read even if it isn't needed (e.g., for type==global)
7876          */
7877         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
7878                         vmx_instruction_info, false, &gva))
7879                 return 1;
7880         if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &operand,
7881                                 sizeof(operand), &e)) {
7882                 kvm_inject_page_fault(vcpu, &e);
7883                 return 1;
7884         }
7885
7886         switch (type) {
7887         case VMX_EPT_EXTENT_GLOBAL:
7888         /*
7889          * TODO: track mappings and invalidate
7890          * single context requests appropriately
7891          */
7892         case VMX_EPT_EXTENT_CONTEXT:
7893                 kvm_mmu_sync_roots(vcpu);
7894                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
7895                 nested_vmx_succeed(vcpu);
7896                 break;
7897         default:
7898                 BUG_ON(1);
7899                 break;
7900         }
7901
7902         return kvm_skip_emulated_instruction(vcpu);
7903 }
7904
7905 static int handle_invvpid(struct kvm_vcpu *vcpu)
7906 {
7907         struct vcpu_vmx *vmx = to_vmx(vcpu);
7908         u32 vmx_instruction_info;
7909         unsigned long type, types;
7910         gva_t gva;
7911         struct x86_exception e;
7912         struct {
7913                 u64 vpid;
7914                 u64 gla;
7915         } operand;
7916
7917         if (!(vmx->nested.nested_vmx_secondary_ctls_high &
7918               SECONDARY_EXEC_ENABLE_VPID) ||
7919                         !(vmx->nested.nested_vmx_vpid_caps & VMX_VPID_INVVPID_BIT)) {
7920                 kvm_queue_exception(vcpu, UD_VECTOR);
7921                 return 1;
7922         }
7923
7924         if (!nested_vmx_check_permission(vcpu))
7925                 return 1;
7926
7927         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7928         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
7929
7930         types = (vmx->nested.nested_vmx_vpid_caps &
7931                         VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
7932
7933         if (type >= 32 || !(types & (1 << type))) {
7934                 nested_vmx_failValid(vcpu,
7935                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
7936                 return kvm_skip_emulated_instruction(vcpu);
7937         }
7938
7939         /* according to the intel vmx instruction reference, the memory
7940          * operand is read even if it isn't needed (e.g., for type==global)
7941          */
7942         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
7943                         vmx_instruction_info, false, &gva))
7944                 return 1;
7945         if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &operand,
7946                                 sizeof(operand), &e)) {
7947                 kvm_inject_page_fault(vcpu, &e);
7948                 return 1;
7949         }
7950         if (operand.vpid >> 16) {
7951                 nested_vmx_failValid(vcpu,
7952                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
7953                 return kvm_skip_emulated_instruction(vcpu);
7954         }
7955
7956         switch (type) {
7957         case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
7958                 if (is_noncanonical_address(operand.gla, vcpu)) {
7959                         nested_vmx_failValid(vcpu,
7960                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
7961                         return kvm_skip_emulated_instruction(vcpu);
7962                 }
7963                 /* fall through */
7964         case VMX_VPID_EXTENT_SINGLE_CONTEXT:
7965         case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
7966                 if (!operand.vpid) {
7967                         nested_vmx_failValid(vcpu,
7968                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
7969                         return kvm_skip_emulated_instruction(vcpu);
7970                 }
7971                 break;
7972         case VMX_VPID_EXTENT_ALL_CONTEXT:
7973                 break;
7974         default:
7975                 WARN_ON_ONCE(1);
7976                 return kvm_skip_emulated_instruction(vcpu);
7977         }
7978
7979         __vmx_flush_tlb(vcpu, vmx->nested.vpid02);
7980         nested_vmx_succeed(vcpu);
7981
7982         return kvm_skip_emulated_instruction(vcpu);
7983 }
7984
7985 static int handle_pml_full(struct kvm_vcpu *vcpu)
7986 {
7987         unsigned long exit_qualification;
7988
7989         trace_kvm_pml_full(vcpu->vcpu_id);
7990
7991         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7992
7993         /*
7994          * PML buffer FULL happened while executing iret from NMI,
7995          * "blocked by NMI" bit has to be set before next VM entry.
7996          */
7997         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
7998                         cpu_has_virtual_nmis() &&
7999                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
8000                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
8001                                 GUEST_INTR_STATE_NMI);
8002
8003         /*
8004          * PML buffer already flushed at beginning of VMEXIT. Nothing to do
8005          * here.., and there's no userspace involvement needed for PML.
8006          */
8007         return 1;
8008 }
8009
8010 static int handle_preemption_timer(struct kvm_vcpu *vcpu)
8011 {
8012         kvm_lapic_expired_hv_timer(vcpu);
8013         return 1;
8014 }
8015
8016 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address)
8017 {
8018         struct vcpu_vmx *vmx = to_vmx(vcpu);
8019         int maxphyaddr = cpuid_maxphyaddr(vcpu);
8020
8021         /* Check for memory type validity */
8022         switch (address & VMX_EPTP_MT_MASK) {
8023         case VMX_EPTP_MT_UC:
8024                 if (!(vmx->nested.nested_vmx_ept_caps & VMX_EPTP_UC_BIT))
8025                         return false;
8026                 break;
8027         case VMX_EPTP_MT_WB:
8028                 if (!(vmx->nested.nested_vmx_ept_caps & VMX_EPTP_WB_BIT))
8029                         return false;
8030                 break;
8031         default:
8032                 return false;
8033         }
8034
8035         /* only 4 levels page-walk length are valid */
8036         if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4)
8037                 return false;
8038
8039         /* Reserved bits should not be set */
8040         if (address >> maxphyaddr || ((address >> 7) & 0x1f))
8041                 return false;
8042
8043         /* AD, if set, should be supported */
8044         if (address & VMX_EPTP_AD_ENABLE_BIT) {
8045                 if (!(vmx->nested.nested_vmx_ept_caps & VMX_EPT_AD_BIT))
8046                         return false;
8047         }
8048
8049         return true;
8050 }
8051
8052 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
8053                                      struct vmcs12 *vmcs12)
8054 {
8055         u32 index = vcpu->arch.regs[VCPU_REGS_RCX];
8056         u64 address;
8057         bool accessed_dirty;
8058         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
8059
8060         if (!nested_cpu_has_eptp_switching(vmcs12) ||
8061             !nested_cpu_has_ept(vmcs12))
8062                 return 1;
8063
8064         if (index >= VMFUNC_EPTP_ENTRIES)
8065                 return 1;
8066
8067
8068         if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT,
8069                                      &address, index * 8, 8))
8070                 return 1;
8071
8072         accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT);
8073
8074         /*
8075          * If the (L2) guest does a vmfunc to the currently
8076          * active ept pointer, we don't have to do anything else
8077          */
8078         if (vmcs12->ept_pointer != address) {
8079                 if (!valid_ept_address(vcpu, address))
8080                         return 1;
8081
8082                 kvm_mmu_unload(vcpu);
8083                 mmu->ept_ad = accessed_dirty;
8084                 mmu->base_role.ad_disabled = !accessed_dirty;
8085                 vmcs12->ept_pointer = address;
8086                 /*
8087                  * TODO: Check what's the correct approach in case
8088                  * mmu reload fails. Currently, we just let the next
8089                  * reload potentially fail
8090                  */
8091                 kvm_mmu_reload(vcpu);
8092         }
8093
8094         return 0;
8095 }
8096
8097 static int handle_vmfunc(struct kvm_vcpu *vcpu)
8098 {
8099         struct vcpu_vmx *vmx = to_vmx(vcpu);
8100         struct vmcs12 *vmcs12;
8101         u32 function = vcpu->arch.regs[VCPU_REGS_RAX];
8102
8103         /*
8104          * VMFUNC is only supported for nested guests, but we always enable the
8105          * secondary control for simplicity; for non-nested mode, fake that we
8106          * didn't by injecting #UD.
8107          */
8108         if (!is_guest_mode(vcpu)) {
8109                 kvm_queue_exception(vcpu, UD_VECTOR);
8110                 return 1;
8111         }
8112
8113         vmcs12 = get_vmcs12(vcpu);
8114         if ((vmcs12->vm_function_control & (1 << function)) == 0)
8115                 goto fail;
8116
8117         switch (function) {
8118         case 0:
8119                 if (nested_vmx_eptp_switching(vcpu, vmcs12))
8120                         goto fail;
8121                 break;
8122         default:
8123                 goto fail;
8124         }
8125         return kvm_skip_emulated_instruction(vcpu);
8126
8127 fail:
8128         nested_vmx_vmexit(vcpu, vmx->exit_reason,
8129                           vmcs_read32(VM_EXIT_INTR_INFO),
8130                           vmcs_readl(EXIT_QUALIFICATION));
8131         return 1;
8132 }
8133
8134 /*
8135  * The exit handlers return 1 if the exit was handled fully and guest execution
8136  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
8137  * to be done to userspace and return 0.
8138  */
8139 static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
8140         [EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
8141         [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
8142         [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
8143         [EXIT_REASON_NMI_WINDOW]              = handle_nmi_window,
8144         [EXIT_REASON_IO_INSTRUCTION]          = handle_io,
8145         [EXIT_REASON_CR_ACCESS]               = handle_cr,
8146         [EXIT_REASON_DR_ACCESS]               = handle_dr,
8147         [EXIT_REASON_CPUID]                   = handle_cpuid,
8148         [EXIT_REASON_MSR_READ]                = handle_rdmsr,
8149         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
8150         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
8151         [EXIT_REASON_HLT]                     = handle_halt,
8152         [EXIT_REASON_INVD]                    = handle_invd,
8153         [EXIT_REASON_INVLPG]                  = handle_invlpg,
8154         [EXIT_REASON_RDPMC]                   = handle_rdpmc,
8155         [EXIT_REASON_VMCALL]                  = handle_vmcall,
8156         [EXIT_REASON_VMCLEAR]                 = handle_vmclear,
8157         [EXIT_REASON_VMLAUNCH]                = handle_vmlaunch,
8158         [EXIT_REASON_VMPTRLD]                 = handle_vmptrld,
8159         [EXIT_REASON_VMPTRST]                 = handle_vmptrst,
8160         [EXIT_REASON_VMREAD]                  = handle_vmread,
8161         [EXIT_REASON_VMRESUME]                = handle_vmresume,
8162         [EXIT_REASON_VMWRITE]                 = handle_vmwrite,
8163         [EXIT_REASON_VMOFF]                   = handle_vmoff,
8164         [EXIT_REASON_VMON]                    = handle_vmon,
8165         [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
8166         [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
8167         [EXIT_REASON_APIC_WRITE]              = handle_apic_write,
8168         [EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,
8169         [EXIT_REASON_WBINVD]                  = handle_wbinvd,
8170         [EXIT_REASON_XSETBV]                  = handle_xsetbv,
8171         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
8172         [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
8173         [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
8174         [EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
8175         [EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
8176         [EXIT_REASON_MWAIT_INSTRUCTION]       = handle_mwait,
8177         [EXIT_REASON_MONITOR_TRAP_FLAG]       = handle_monitor_trap,
8178         [EXIT_REASON_MONITOR_INSTRUCTION]     = handle_monitor,
8179         [EXIT_REASON_INVEPT]                  = handle_invept,
8180         [EXIT_REASON_INVVPID]                 = handle_invvpid,
8181         [EXIT_REASON_RDRAND]                  = handle_invalid_op,
8182         [EXIT_REASON_RDSEED]                  = handle_invalid_op,
8183         [EXIT_REASON_XSAVES]                  = handle_xsaves,
8184         [EXIT_REASON_XRSTORS]                 = handle_xrstors,
8185         [EXIT_REASON_PML_FULL]                = handle_pml_full,
8186         [EXIT_REASON_VMFUNC]                  = handle_vmfunc,
8187         [EXIT_REASON_PREEMPTION_TIMER]        = handle_preemption_timer,
8188 };
8189
8190 static const int kvm_vmx_max_exit_handlers =
8191         ARRAY_SIZE(kvm_vmx_exit_handlers);
8192
8193 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
8194                                        struct vmcs12 *vmcs12)
8195 {
8196         unsigned long exit_qualification;
8197         gpa_t bitmap, last_bitmap;
8198         unsigned int port;
8199         int size;
8200         u8 b;
8201
8202         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
8203                 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
8204
8205         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
8206
8207         port = exit_qualification >> 16;
8208         size = (exit_qualification & 7) + 1;
8209
8210         last_bitmap = (gpa_t)-1;
8211         b = -1;
8212
8213         while (size > 0) {
8214                 if (port < 0x8000)
8215                         bitmap = vmcs12->io_bitmap_a;
8216                 else if (port < 0x10000)
8217                         bitmap = vmcs12->io_bitmap_b;
8218                 else
8219                         return true;
8220                 bitmap += (port & 0x7fff) / 8;
8221
8222                 if (last_bitmap != bitmap)
8223                         if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
8224                                 return true;
8225                 if (b & (1 << (port & 7)))
8226                         return true;
8227
8228                 port++;
8229                 size--;
8230                 last_bitmap = bitmap;
8231         }
8232
8233         return false;
8234 }
8235
8236 /*
8237  * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
8238  * rather than handle it ourselves in L0. I.e., check whether L1 expressed
8239  * disinterest in the current event (read or write a specific MSR) by using an
8240  * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
8241  */
8242 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
8243         struct vmcs12 *vmcs12, u32 exit_reason)
8244 {
8245         u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX];
8246         gpa_t bitmap;
8247
8248         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
8249                 return true;
8250
8251         /*
8252          * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
8253          * for the four combinations of read/write and low/high MSR numbers.
8254          * First we need to figure out which of the four to use:
8255          */
8256         bitmap = vmcs12->msr_bitmap;
8257         if (exit_reason == EXIT_REASON_MSR_WRITE)
8258                 bitmap += 2048;
8259         if (msr_index >= 0xc0000000) {
8260                 msr_index -= 0xc0000000;
8261                 bitmap += 1024;
8262         }
8263
8264         /* Then read the msr_index'th bit from this bitmap: */
8265         if (msr_index < 1024*8) {
8266                 unsigned char b;
8267                 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
8268                         return true;
8269                 return 1 & (b >> (msr_index & 7));
8270         } else
8271                 return true; /* let L1 handle the wrong parameter */
8272 }
8273
8274 /*
8275  * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
8276  * rather than handle it ourselves in L0. I.e., check if L1 wanted to
8277  * intercept (via guest_host_mask etc.) the current event.
8278  */
8279 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
8280         struct vmcs12 *vmcs12)
8281 {
8282         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
8283         int cr = exit_qualification & 15;
8284         int reg;
8285         unsigned long val;
8286
8287         switch ((exit_qualification >> 4) & 3) {
8288         case 0: /* mov to cr */
8289                 reg = (exit_qualification >> 8) & 15;
8290                 val = kvm_register_readl(vcpu, reg);
8291                 switch (cr) {
8292                 case 0:
8293                         if (vmcs12->cr0_guest_host_mask &
8294                             (val ^ vmcs12->cr0_read_shadow))
8295                                 return true;
8296                         break;
8297                 case 3:
8298                         if ((vmcs12->cr3_target_count >= 1 &&
8299                                         vmcs12->cr3_target_value0 == val) ||
8300                                 (vmcs12->cr3_target_count >= 2 &&
8301                                         vmcs12->cr3_target_value1 == val) ||
8302                                 (vmcs12->cr3_target_count >= 3 &&
8303                                         vmcs12->cr3_target_value2 == val) ||
8304                                 (vmcs12->cr3_target_count >= 4 &&
8305                                         vmcs12->cr3_target_value3 == val))
8306                                 return false;
8307                         if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
8308                                 return true;
8309                         break;
8310                 case 4:
8311                         if (vmcs12->cr4_guest_host_mask &
8312                             (vmcs12->cr4_read_shadow ^ val))
8313                                 return true;
8314                         break;
8315                 case 8:
8316                         if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
8317                                 return true;
8318                         break;
8319                 }
8320                 break;
8321         case 2: /* clts */
8322                 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
8323                     (vmcs12->cr0_read_shadow & X86_CR0_TS))
8324                         return true;
8325                 break;
8326         case 1: /* mov from cr */
8327                 switch (cr) {
8328                 case 3:
8329                         if (vmcs12->cpu_based_vm_exec_control &
8330                             CPU_BASED_CR3_STORE_EXITING)
8331                                 return true;
8332                         break;
8333                 case 8:
8334                         if (vmcs12->cpu_based_vm_exec_control &
8335                             CPU_BASED_CR8_STORE_EXITING)
8336                                 return true;
8337                         break;
8338                 }
8339                 break;
8340         case 3: /* lmsw */
8341                 /*
8342                  * lmsw can change bits 1..3 of cr0, and only set bit 0 of
8343                  * cr0. Other attempted changes are ignored, with no exit.
8344                  */
8345                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
8346                 if (vmcs12->cr0_guest_host_mask & 0xe &
8347                     (val ^ vmcs12->cr0_read_shadow))
8348                         return true;
8349                 if ((vmcs12->cr0_guest_host_mask & 0x1) &&
8350                     !(vmcs12->cr0_read_shadow & 0x1) &&
8351                     (val & 0x1))
8352                         return true;
8353                 break;
8354         }
8355         return false;
8356 }
8357
8358 /*
8359  * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
8360  * should handle it ourselves in L0 (and then continue L2). Only call this
8361  * when in is_guest_mode (L2).
8362  */
8363 static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
8364 {
8365         u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
8366         struct vcpu_vmx *vmx = to_vmx(vcpu);
8367         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
8368
8369         if (vmx->nested.nested_run_pending)
8370                 return false;
8371
8372         if (unlikely(vmx->fail)) {
8373                 pr_info_ratelimited("%s failed vm entry %x\n", __func__,
8374                                     vmcs_read32(VM_INSTRUCTION_ERROR));
8375                 return true;
8376         }
8377
8378         /*
8379          * The host physical addresses of some pages of guest memory
8380          * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
8381          * Page). The CPU may write to these pages via their host
8382          * physical address while L2 is running, bypassing any
8383          * address-translation-based dirty tracking (e.g. EPT write
8384          * protection).
8385          *
8386          * Mark them dirty on every exit from L2 to prevent them from
8387          * getting out of sync with dirty tracking.
8388          */
8389         nested_mark_vmcs12_pages_dirty(vcpu);
8390
8391         trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
8392                                 vmcs_readl(EXIT_QUALIFICATION),
8393                                 vmx->idt_vectoring_info,
8394                                 intr_info,
8395                                 vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
8396                                 KVM_ISA_VMX);
8397
8398         switch (exit_reason) {
8399         case EXIT_REASON_EXCEPTION_NMI:
8400                 if (is_nmi(intr_info))
8401                         return false;
8402                 else if (is_page_fault(intr_info))
8403                         return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept;
8404                 else if (is_no_device(intr_info) &&
8405                          !(vmcs12->guest_cr0 & X86_CR0_TS))
8406                         return false;
8407                 else if (is_debug(intr_info) &&
8408                          vcpu->guest_debug &
8409                          (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
8410                         return false;
8411                 else if (is_breakpoint(intr_info) &&
8412                          vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
8413                         return false;
8414                 return vmcs12->exception_bitmap &
8415                                 (1u << (intr_info & INTR_INFO_VECTOR_MASK));
8416         case EXIT_REASON_EXTERNAL_INTERRUPT:
8417                 return false;
8418         case EXIT_REASON_TRIPLE_FAULT:
8419                 return true;
8420         case EXIT_REASON_PENDING_INTERRUPT:
8421                 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
8422         case EXIT_REASON_NMI_WINDOW:
8423                 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
8424         case EXIT_REASON_TASK_SWITCH:
8425                 return true;
8426         case EXIT_REASON_CPUID:
8427                 return true;
8428         case EXIT_REASON_HLT:
8429                 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
8430         case EXIT_REASON_INVD:
8431                 return true;
8432         case EXIT_REASON_INVLPG:
8433                 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
8434         case EXIT_REASON_RDPMC:
8435                 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
8436         case EXIT_REASON_RDRAND:
8437                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND);
8438         case EXIT_REASON_RDSEED:
8439                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED);
8440         case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
8441                 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
8442         case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
8443         case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
8444         case EXIT_REASON_VMPTRST: case EXIT_REASON_VMREAD:
8445         case EXIT_REASON_VMRESUME: case EXIT_REASON_VMWRITE:
8446         case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
8447         case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
8448                 /*
8449                  * VMX instructions trap unconditionally. This allows L1 to
8450                  * emulate them for its L2 guest, i.e., allows 3-level nesting!
8451                  */
8452                 return true;
8453         case EXIT_REASON_CR_ACCESS:
8454                 return nested_vmx_exit_handled_cr(vcpu, vmcs12);
8455         case EXIT_REASON_DR_ACCESS:
8456                 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
8457         case EXIT_REASON_IO_INSTRUCTION:
8458                 return nested_vmx_exit_handled_io(vcpu, vmcs12);
8459         case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
8460                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
8461         case EXIT_REASON_MSR_READ:
8462         case EXIT_REASON_MSR_WRITE:
8463                 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
8464         case EXIT_REASON_INVALID_STATE:
8465                 return true;
8466         case EXIT_REASON_MWAIT_INSTRUCTION:
8467                 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
8468         case EXIT_REASON_MONITOR_TRAP_FLAG:
8469                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG);
8470         case EXIT_REASON_MONITOR_INSTRUCTION:
8471                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
8472         case EXIT_REASON_PAUSE_INSTRUCTION:
8473                 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
8474                         nested_cpu_has2(vmcs12,
8475                                 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
8476         case EXIT_REASON_MCE_DURING_VMENTRY:
8477                 return false;
8478         case EXIT_REASON_TPR_BELOW_THRESHOLD:
8479                 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
8480         case EXIT_REASON_APIC_ACCESS:
8481                 return nested_cpu_has2(vmcs12,
8482                         SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
8483         case EXIT_REASON_APIC_WRITE:
8484         case EXIT_REASON_EOI_INDUCED:
8485                 /* apic_write and eoi_induced should exit unconditionally. */
8486                 return true;
8487         case EXIT_REASON_EPT_VIOLATION:
8488                 /*
8489                  * L0 always deals with the EPT violation. If nested EPT is
8490                  * used, and the nested mmu code discovers that the address is
8491                  * missing in the guest EPT table (EPT12), the EPT violation
8492                  * will be injected with nested_ept_inject_page_fault()
8493                  */
8494                 return false;
8495         case EXIT_REASON_EPT_MISCONFIG:
8496                 /*
8497                  * L2 never uses directly L1's EPT, but rather L0's own EPT
8498                  * table (shadow on EPT) or a merged EPT table that L0 built
8499                  * (EPT on EPT). So any problems with the structure of the
8500                  * table is L0's fault.
8501                  */
8502                 return false;
8503         case EXIT_REASON_INVPCID:
8504                 return
8505                         nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) &&
8506                         nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
8507         case EXIT_REASON_WBINVD:
8508                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
8509         case EXIT_REASON_XSETBV:
8510                 return true;
8511         case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
8512                 /*
8513                  * This should never happen, since it is not possible to
8514                  * set XSS to a non-zero value---neither in L1 nor in L2.
8515                  * If if it were, XSS would have to be checked against
8516                  * the XSS exit bitmap in vmcs12.
8517                  */
8518                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
8519         case EXIT_REASON_PREEMPTION_TIMER:
8520                 return false;
8521         case EXIT_REASON_PML_FULL:
8522                 /* We emulate PML support to L1. */
8523                 return false;
8524         case EXIT_REASON_VMFUNC:
8525                 /* VM functions are emulated through L2->L0 vmexits. */
8526                 return false;
8527         default:
8528                 return true;
8529         }
8530 }
8531
8532 static int nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason)
8533 {
8534         u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
8535
8536         /*
8537          * At this point, the exit interruption info in exit_intr_info
8538          * is only valid for EXCEPTION_NMI exits.  For EXTERNAL_INTERRUPT
8539          * we need to query the in-kernel LAPIC.
8540          */
8541         WARN_ON(exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT);
8542         if ((exit_intr_info &
8543              (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) ==
8544             (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) {
8545                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
8546                 vmcs12->vm_exit_intr_error_code =
8547                         vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
8548         }
8549
8550         nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info,
8551                           vmcs_readl(EXIT_QUALIFICATION));
8552         return 1;
8553 }
8554
8555 static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
8556 {
8557         *info1 = vmcs_readl(EXIT_QUALIFICATION);
8558         *info2 = vmcs_read32(VM_EXIT_INTR_INFO);
8559 }
8560
8561 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
8562 {
8563         if (vmx->pml_pg) {
8564                 __free_page(vmx->pml_pg);
8565                 vmx->pml_pg = NULL;
8566         }
8567 }
8568
8569 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
8570 {
8571         struct vcpu_vmx *vmx = to_vmx(vcpu);
8572         u64 *pml_buf;
8573         u16 pml_idx;
8574
8575         pml_idx = vmcs_read16(GUEST_PML_INDEX);
8576
8577         /* Do nothing if PML buffer is empty */
8578         if (pml_idx == (PML_ENTITY_NUM - 1))
8579                 return;
8580
8581         /* PML index always points to next available PML buffer entity */
8582         if (pml_idx >= PML_ENTITY_NUM)
8583                 pml_idx = 0;
8584         else
8585                 pml_idx++;
8586
8587         pml_buf = page_address(vmx->pml_pg);
8588         for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
8589                 u64 gpa;
8590
8591                 gpa = pml_buf[pml_idx];
8592                 WARN_ON(gpa & (PAGE_SIZE - 1));
8593                 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
8594         }
8595
8596         /* reset PML index */
8597         vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
8598 }
8599
8600 /*
8601  * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
8602  * Called before reporting dirty_bitmap to userspace.
8603  */
8604 static void kvm_flush_pml_buffers(struct kvm *kvm)
8605 {
8606         int i;
8607         struct kvm_vcpu *vcpu;
8608         /*
8609          * We only need to kick vcpu out of guest mode here, as PML buffer
8610          * is flushed at beginning of all VMEXITs, and it's obvious that only
8611          * vcpus running in guest are possible to have unflushed GPAs in PML
8612          * buffer.
8613          */
8614         kvm_for_each_vcpu(i, vcpu, kvm)
8615                 kvm_vcpu_kick(vcpu);
8616 }
8617
8618 static void vmx_dump_sel(char *name, uint32_t sel)
8619 {
8620         pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
8621                name, vmcs_read16(sel),
8622                vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
8623                vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
8624                vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
8625 }
8626
8627 static void vmx_dump_dtsel(char *name, uint32_t limit)
8628 {
8629         pr_err("%s                           limit=0x%08x, base=0x%016lx\n",
8630                name, vmcs_read32(limit),
8631                vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
8632 }
8633
8634 static void dump_vmcs(void)
8635 {
8636         u32 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
8637         u32 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
8638         u32 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
8639         u32 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
8640         u32 secondary_exec_control = 0;
8641         unsigned long cr4 = vmcs_readl(GUEST_CR4);
8642         u64 efer = vmcs_read64(GUEST_IA32_EFER);
8643         int i, n;
8644
8645         if (cpu_has_secondary_exec_ctrls())
8646                 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
8647
8648         pr_err("*** Guest State ***\n");
8649         pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
8650                vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
8651                vmcs_readl(CR0_GUEST_HOST_MASK));
8652         pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
8653                cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
8654         pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
8655         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
8656             (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA))
8657         {
8658                 pr_err("PDPTR0 = 0x%016llx  PDPTR1 = 0x%016llx\n",
8659                        vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
8660                 pr_err("PDPTR2 = 0x%016llx  PDPTR3 = 0x%016llx\n",
8661                        vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
8662         }
8663         pr_err("RSP = 0x%016lx  RIP = 0x%016lx\n",
8664                vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
8665         pr_err("RFLAGS=0x%08lx         DR7 = 0x%016lx\n",
8666                vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
8667         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
8668                vmcs_readl(GUEST_SYSENTER_ESP),
8669                vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
8670         vmx_dump_sel("CS:  ", GUEST_CS_SELECTOR);
8671         vmx_dump_sel("DS:  ", GUEST_DS_SELECTOR);
8672         vmx_dump_sel("SS:  ", GUEST_SS_SELECTOR);
8673         vmx_dump_sel("ES:  ", GUEST_ES_SELECTOR);
8674         vmx_dump_sel("FS:  ", GUEST_FS_SELECTOR);
8675         vmx_dump_sel("GS:  ", GUEST_GS_SELECTOR);
8676         vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
8677         vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
8678         vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
8679         vmx_dump_sel("TR:  ", GUEST_TR_SELECTOR);
8680         if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
8681             (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
8682                 pr_err("EFER =     0x%016llx  PAT = 0x%016llx\n",
8683                        efer, vmcs_read64(GUEST_IA32_PAT));
8684         pr_err("DebugCtl = 0x%016llx  DebugExceptions = 0x%016lx\n",
8685                vmcs_read64(GUEST_IA32_DEBUGCTL),
8686                vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
8687         if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
8688                 pr_err("PerfGlobCtl = 0x%016llx\n",
8689                        vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
8690         if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
8691                 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
8692         pr_err("Interruptibility = %08x  ActivityState = %08x\n",
8693                vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
8694                vmcs_read32(GUEST_ACTIVITY_STATE));
8695         if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
8696                 pr_err("InterruptStatus = %04x\n",
8697                        vmcs_read16(GUEST_INTR_STATUS));
8698
8699         pr_err("*** Host State ***\n");
8700         pr_err("RIP = 0x%016lx  RSP = 0x%016lx\n",
8701                vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
8702         pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
8703                vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
8704                vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
8705                vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
8706                vmcs_read16(HOST_TR_SELECTOR));
8707         pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
8708                vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
8709                vmcs_readl(HOST_TR_BASE));
8710         pr_err("GDTBase=%016lx IDTBase=%016lx\n",
8711                vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
8712         pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
8713                vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
8714                vmcs_readl(HOST_CR4));
8715         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
8716                vmcs_readl(HOST_IA32_SYSENTER_ESP),
8717                vmcs_read32(HOST_IA32_SYSENTER_CS),
8718                vmcs_readl(HOST_IA32_SYSENTER_EIP));
8719         if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
8720                 pr_err("EFER = 0x%016llx  PAT = 0x%016llx\n",
8721                        vmcs_read64(HOST_IA32_EFER),
8722                        vmcs_read64(HOST_IA32_PAT));
8723         if (vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
8724                 pr_err("PerfGlobCtl = 0x%016llx\n",
8725                        vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
8726
8727         pr_err("*** Control State ***\n");
8728         pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
8729                pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
8730         pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
8731         pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
8732                vmcs_read32(EXCEPTION_BITMAP),
8733                vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
8734                vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
8735         pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
8736                vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
8737                vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
8738                vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
8739         pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
8740                vmcs_read32(VM_EXIT_INTR_INFO),
8741                vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
8742                vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
8743         pr_err("        reason=%08x qualification=%016lx\n",
8744                vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
8745         pr_err("IDTVectoring: info=%08x errcode=%08x\n",
8746                vmcs_read32(IDT_VECTORING_INFO_FIELD),
8747                vmcs_read32(IDT_VECTORING_ERROR_CODE));
8748         pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
8749         if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
8750                 pr_err("TSC Multiplier = 0x%016llx\n",
8751                        vmcs_read64(TSC_MULTIPLIER));
8752         if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW)
8753                 pr_err("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
8754         if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
8755                 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
8756         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
8757                 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
8758         n = vmcs_read32(CR3_TARGET_COUNT);
8759         for (i = 0; i + 1 < n; i += 4)
8760                 pr_err("CR3 target%u=%016lx target%u=%016lx\n",
8761                        i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2),
8762                        i + 1, vmcs_readl(CR3_TARGET_VALUE0 + i * 2 + 2));
8763         if (i < n)
8764                 pr_err("CR3 target%u=%016lx\n",
8765                        i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2));
8766         if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
8767                 pr_err("PLE Gap=%08x Window=%08x\n",
8768                        vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
8769         if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
8770                 pr_err("Virtual processor ID = 0x%04x\n",
8771                        vmcs_read16(VIRTUAL_PROCESSOR_ID));
8772 }
8773
8774 /*
8775  * The guest has exited.  See if we can fix it or if we need userspace
8776  * assistance.
8777  */
8778 static int vmx_handle_exit(struct kvm_vcpu *vcpu)
8779 {
8780         struct vcpu_vmx *vmx = to_vmx(vcpu);
8781         u32 exit_reason = vmx->exit_reason;
8782         u32 vectoring_info = vmx->idt_vectoring_info;
8783
8784         trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
8785
8786         /*
8787          * Flush logged GPAs PML buffer, this will make dirty_bitmap more
8788          * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
8789          * querying dirty_bitmap, we only need to kick all vcpus out of guest
8790          * mode as if vcpus is in root mode, the PML buffer must has been
8791          * flushed already.
8792          */
8793         if (enable_pml)
8794                 vmx_flush_pml_buffer(vcpu);
8795
8796         /* If guest state is invalid, start emulating */
8797         if (vmx->emulation_required)
8798                 return handle_invalid_guest_state(vcpu);
8799
8800         if (is_guest_mode(vcpu) && nested_vmx_exit_reflected(vcpu, exit_reason))
8801                 return nested_vmx_reflect_vmexit(vcpu, exit_reason);
8802
8803         if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
8804                 dump_vmcs();
8805                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
8806                 vcpu->run->fail_entry.hardware_entry_failure_reason
8807                         = exit_reason;
8808                 return 0;
8809         }
8810
8811         if (unlikely(vmx->fail)) {
8812                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
8813                 vcpu->run->fail_entry.hardware_entry_failure_reason
8814                         = vmcs_read32(VM_INSTRUCTION_ERROR);
8815                 return 0;
8816         }
8817
8818         /*
8819          * Note:
8820          * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
8821          * delivery event since it indicates guest is accessing MMIO.
8822          * The vm-exit can be triggered again after return to guest that
8823          * will cause infinite loop.
8824          */
8825         if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
8826                         (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
8827                         exit_reason != EXIT_REASON_EPT_VIOLATION &&
8828                         exit_reason != EXIT_REASON_PML_FULL &&
8829                         exit_reason != EXIT_REASON_TASK_SWITCH)) {
8830                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
8831                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
8832                 vcpu->run->internal.ndata = 3;
8833                 vcpu->run->internal.data[0] = vectoring_info;
8834                 vcpu->run->internal.data[1] = exit_reason;
8835                 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
8836                 if (exit_reason == EXIT_REASON_EPT_MISCONFIG) {
8837                         vcpu->run->internal.ndata++;
8838                         vcpu->run->internal.data[3] =
8839                                 vmcs_read64(GUEST_PHYSICAL_ADDRESS);
8840                 }
8841                 return 0;
8842         }
8843
8844         if (unlikely(!cpu_has_virtual_nmis() &&
8845                      vmx->loaded_vmcs->soft_vnmi_blocked)) {
8846                 if (vmx_interrupt_allowed(vcpu)) {
8847                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
8848                 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL &&
8849                            vcpu->arch.nmi_pending) {
8850                         /*
8851                          * This CPU don't support us in finding the end of an
8852                          * NMI-blocked window if the guest runs with IRQs
8853                          * disabled. So we pull the trigger after 1 s of
8854                          * futile waiting, but inform the user about this.
8855                          */
8856                         printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
8857                                "state on VCPU %d after 1 s timeout\n",
8858                                __func__, vcpu->vcpu_id);
8859                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
8860                 }
8861         }
8862
8863         if (exit_reason < kvm_vmx_max_exit_handlers
8864             && kvm_vmx_exit_handlers[exit_reason])
8865                 return kvm_vmx_exit_handlers[exit_reason](vcpu);
8866         else {
8867                 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
8868                                 exit_reason);
8869                 kvm_queue_exception(vcpu, UD_VECTOR);
8870                 return 1;
8871         }
8872 }
8873
8874 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
8875 {
8876         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
8877
8878         if (is_guest_mode(vcpu) &&
8879                 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
8880                 return;
8881
8882         if (irr == -1 || tpr < irr) {
8883                 vmcs_write32(TPR_THRESHOLD, 0);
8884                 return;
8885         }
8886
8887         vmcs_write32(TPR_THRESHOLD, irr);
8888 }
8889
8890 static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
8891 {
8892         u32 sec_exec_control;
8893
8894         /* Postpone execution until vmcs01 is the current VMCS. */
8895         if (is_guest_mode(vcpu)) {
8896                 to_vmx(vcpu)->nested.change_vmcs01_virtual_x2apic_mode = true;
8897                 return;
8898         }
8899
8900         if (!cpu_has_vmx_virtualize_x2apic_mode())
8901                 return;
8902
8903         if (!cpu_need_tpr_shadow(vcpu))
8904                 return;
8905
8906         sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
8907
8908         if (set) {
8909                 sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
8910                 sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
8911         } else {
8912                 sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
8913                 sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
8914                 vmx_flush_tlb_ept_only(vcpu);
8915         }
8916         vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
8917
8918         vmx_update_msr_bitmap(vcpu);
8919 }
8920
8921 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
8922 {
8923         struct vcpu_vmx *vmx = to_vmx(vcpu);
8924
8925         /*
8926          * Currently we do not handle the nested case where L2 has an
8927          * APIC access page of its own; that page is still pinned.
8928          * Hence, we skip the case where the VCPU is in guest mode _and_
8929          * L1 prepared an APIC access page for L2.
8930          *
8931          * For the case where L1 and L2 share the same APIC access page
8932          * (flexpriority=Y but SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES clear
8933          * in the vmcs12), this function will only update either the vmcs01
8934          * or the vmcs02.  If the former, the vmcs02 will be updated by
8935          * prepare_vmcs02.  If the latter, the vmcs01 will be updated in
8936          * the next L2->L1 exit.
8937          */
8938         if (!is_guest_mode(vcpu) ||
8939             !nested_cpu_has2(get_vmcs12(&vmx->vcpu),
8940                              SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
8941                 vmcs_write64(APIC_ACCESS_ADDR, hpa);
8942                 vmx_flush_tlb_ept_only(vcpu);
8943         }
8944 }
8945
8946 static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
8947 {
8948         u16 status;
8949         u8 old;
8950
8951         if (max_isr == -1)
8952                 max_isr = 0;
8953
8954         status = vmcs_read16(GUEST_INTR_STATUS);
8955         old = status >> 8;
8956         if (max_isr != old) {
8957                 status &= 0xff;
8958                 status |= max_isr << 8;
8959                 vmcs_write16(GUEST_INTR_STATUS, status);
8960         }
8961 }
8962
8963 static void vmx_set_rvi(int vector)
8964 {
8965         u16 status;
8966         u8 old;
8967
8968         if (vector == -1)
8969                 vector = 0;
8970
8971         status = vmcs_read16(GUEST_INTR_STATUS);
8972         old = (u8)status & 0xff;
8973         if ((u8)vector != old) {
8974                 status &= ~0xff;
8975                 status |= (u8)vector;
8976                 vmcs_write16(GUEST_INTR_STATUS, status);
8977         }
8978 }
8979
8980 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
8981 {
8982         if (!is_guest_mode(vcpu)) {
8983                 vmx_set_rvi(max_irr);
8984                 return;
8985         }
8986
8987         if (max_irr == -1)
8988                 return;
8989
8990         /*
8991          * In guest mode.  If a vmexit is needed, vmx_check_nested_events
8992          * handles it.
8993          */
8994         if (nested_exit_on_intr(vcpu))
8995                 return;
8996
8997         /*
8998          * Else, fall back to pre-APICv interrupt injection since L2
8999          * is run without virtual interrupt delivery.
9000          */
9001         if (!kvm_event_needs_reinjection(vcpu) &&
9002             vmx_interrupt_allowed(vcpu)) {
9003                 kvm_queue_interrupt(vcpu, max_irr, false);
9004                 vmx_inject_irq(vcpu);
9005         }
9006 }
9007
9008 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
9009 {
9010         struct vcpu_vmx *vmx = to_vmx(vcpu);
9011         int max_irr;
9012
9013         WARN_ON(!vcpu->arch.apicv_active);
9014         if (pi_test_on(&vmx->pi_desc)) {
9015                 pi_clear_on(&vmx->pi_desc);
9016                 /*
9017                  * IOMMU can write to PIR.ON, so the barrier matters even on UP.
9018                  * But on x86 this is just a compiler barrier anyway.
9019                  */
9020                 smp_mb__after_atomic();
9021                 max_irr = kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
9022         } else {
9023                 max_irr = kvm_lapic_find_highest_irr(vcpu);
9024         }
9025         vmx_hwapic_irr_update(vcpu, max_irr);
9026         return max_irr;
9027 }
9028
9029 static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
9030 {
9031         if (!kvm_vcpu_apicv_active(vcpu))
9032                 return;
9033
9034         vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
9035         vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
9036         vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
9037         vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
9038 }
9039
9040 static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu)
9041 {
9042         struct vcpu_vmx *vmx = to_vmx(vcpu);
9043
9044         pi_clear_on(&vmx->pi_desc);
9045         memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir));
9046 }
9047
9048 static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
9049 {
9050         u32 exit_intr_info = 0;
9051         u16 basic_exit_reason = (u16)vmx->exit_reason;
9052
9053         if (!(basic_exit_reason == EXIT_REASON_MCE_DURING_VMENTRY
9054               || basic_exit_reason == EXIT_REASON_EXCEPTION_NMI))
9055                 return;
9056
9057         if (!(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
9058                 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
9059         vmx->exit_intr_info = exit_intr_info;
9060
9061         /* if exit due to PF check for async PF */
9062         if (is_page_fault(exit_intr_info))
9063                 vmx->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
9064
9065         /* Handle machine checks before interrupts are enabled */
9066         if (basic_exit_reason == EXIT_REASON_MCE_DURING_VMENTRY ||
9067             is_machine_check(exit_intr_info))
9068                 kvm_machine_check();
9069
9070         /* We need to handle NMIs before interrupts are enabled */
9071         if (is_nmi(exit_intr_info)) {
9072                 kvm_before_handle_nmi(&vmx->vcpu);
9073                 asm("int $2");
9074                 kvm_after_handle_nmi(&vmx->vcpu);
9075         }
9076 }
9077
9078 static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
9079 {
9080         u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
9081
9082         if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK))
9083                         == (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) {
9084                 unsigned int vector;
9085                 unsigned long entry;
9086                 gate_desc *desc;
9087                 struct vcpu_vmx *vmx = to_vmx(vcpu);
9088 #ifdef CONFIG_X86_64
9089                 unsigned long tmp;
9090 #endif
9091
9092                 vector =  exit_intr_info & INTR_INFO_VECTOR_MASK;
9093                 desc = (gate_desc *)vmx->host_idt_base + vector;
9094                 entry = gate_offset(desc);
9095                 asm volatile(
9096 #ifdef CONFIG_X86_64
9097                         "mov %%" _ASM_SP ", %[sp]\n\t"
9098                         "and $0xfffffffffffffff0, %%" _ASM_SP "\n\t"
9099                         "push $%c[ss]\n\t"
9100                         "push %[sp]\n\t"
9101 #endif
9102                         "pushf\n\t"
9103                         __ASM_SIZE(push) " $%c[cs]\n\t"
9104                         CALL_NOSPEC
9105                         :
9106 #ifdef CONFIG_X86_64
9107                         [sp]"=&r"(tmp),
9108 #endif
9109                         ASM_CALL_CONSTRAINT
9110                         :
9111                         THUNK_TARGET(entry),
9112                         [ss]"i"(__KERNEL_DS),
9113                         [cs]"i"(__KERNEL_CS)
9114                         );
9115         }
9116 }
9117 STACK_FRAME_NON_STANDARD(vmx_handle_external_intr);
9118
9119 static bool vmx_has_high_real_mode_segbase(void)
9120 {
9121         return enable_unrestricted_guest || emulate_invalid_guest_state;
9122 }
9123
9124 static bool vmx_mpx_supported(void)
9125 {
9126         return (vmcs_config.vmexit_ctrl & VM_EXIT_CLEAR_BNDCFGS) &&
9127                 (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_BNDCFGS);
9128 }
9129
9130 static bool vmx_xsaves_supported(void)
9131 {
9132         return vmcs_config.cpu_based_2nd_exec_ctrl &
9133                 SECONDARY_EXEC_XSAVES;
9134 }
9135
9136 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
9137 {
9138         u32 exit_intr_info;
9139         bool unblock_nmi;
9140         u8 vector;
9141         bool idtv_info_valid;
9142
9143         idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
9144
9145         if (cpu_has_virtual_nmis()) {
9146                 if (vmx->loaded_vmcs->nmi_known_unmasked)
9147                         return;
9148                 /*
9149                  * Can't use vmx->exit_intr_info since we're not sure what
9150                  * the exit reason is.
9151                  */
9152                 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
9153                 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
9154                 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
9155                 /*
9156                  * SDM 3: 27.7.1.2 (September 2008)
9157                  * Re-set bit "block by NMI" before VM entry if vmexit caused by
9158                  * a guest IRET fault.
9159                  * SDM 3: 23.2.2 (September 2008)
9160                  * Bit 12 is undefined in any of the following cases:
9161                  *  If the VM exit sets the valid bit in the IDT-vectoring
9162                  *   information field.
9163                  *  If the VM exit is due to a double fault.
9164                  */
9165                 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
9166                     vector != DF_VECTOR && !idtv_info_valid)
9167                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
9168                                       GUEST_INTR_STATE_NMI);
9169                 else
9170                         vmx->loaded_vmcs->nmi_known_unmasked =
9171                                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
9172                                   & GUEST_INTR_STATE_NMI);
9173         } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked))
9174                 vmx->loaded_vmcs->vnmi_blocked_time +=
9175                         ktime_to_ns(ktime_sub(ktime_get(),
9176                                               vmx->loaded_vmcs->entry_time));
9177 }
9178
9179 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
9180                                       u32 idt_vectoring_info,
9181                                       int instr_len_field,
9182                                       int error_code_field)
9183 {
9184         u8 vector;
9185         int type;
9186         bool idtv_info_valid;
9187
9188         idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
9189
9190         vcpu->arch.nmi_injected = false;
9191         kvm_clear_exception_queue(vcpu);
9192         kvm_clear_interrupt_queue(vcpu);
9193
9194         if (!idtv_info_valid)
9195                 return;
9196
9197         kvm_make_request(KVM_REQ_EVENT, vcpu);
9198
9199         vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
9200         type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
9201
9202         switch (type) {
9203         case INTR_TYPE_NMI_INTR:
9204                 vcpu->arch.nmi_injected = true;
9205                 /*
9206                  * SDM 3: 27.7.1.2 (September 2008)
9207                  * Clear bit "block by NMI" before VM entry if a NMI
9208                  * delivery faulted.
9209                  */
9210                 vmx_set_nmi_mask(vcpu, false);
9211                 break;
9212         case INTR_TYPE_SOFT_EXCEPTION:
9213                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
9214                 /* fall through */
9215         case INTR_TYPE_HARD_EXCEPTION:
9216                 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
9217                         u32 err = vmcs_read32(error_code_field);
9218                         kvm_requeue_exception_e(vcpu, vector, err);
9219                 } else
9220                         kvm_requeue_exception(vcpu, vector);
9221                 break;
9222         case INTR_TYPE_SOFT_INTR:
9223                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
9224                 /* fall through */
9225         case INTR_TYPE_EXT_INTR:
9226                 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
9227                 break;
9228         default:
9229                 break;
9230         }
9231 }
9232
9233 static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
9234 {
9235         __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
9236                                   VM_EXIT_INSTRUCTION_LEN,
9237                                   IDT_VECTORING_ERROR_CODE);
9238 }
9239
9240 static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
9241 {
9242         __vmx_complete_interrupts(vcpu,
9243                                   vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
9244                                   VM_ENTRY_INSTRUCTION_LEN,
9245                                   VM_ENTRY_EXCEPTION_ERROR_CODE);
9246
9247         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
9248 }
9249
9250 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
9251 {
9252         int i, nr_msrs;
9253         struct perf_guest_switch_msr *msrs;
9254
9255         msrs = perf_guest_get_msrs(&nr_msrs);
9256
9257         if (!msrs)
9258                 return;
9259
9260         for (i = 0; i < nr_msrs; i++)
9261                 if (msrs[i].host == msrs[i].guest)
9262                         clear_atomic_switch_msr(vmx, msrs[i].msr);
9263                 else
9264                         add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
9265                                         msrs[i].host);
9266 }
9267
9268 static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
9269 {
9270         struct vcpu_vmx *vmx = to_vmx(vcpu);
9271         u64 tscl;
9272         u32 delta_tsc;
9273
9274         if (vmx->hv_deadline_tsc == -1)
9275                 return;
9276
9277         tscl = rdtsc();
9278         if (vmx->hv_deadline_tsc > tscl)
9279                 /* sure to be 32 bit only because checked on set_hv_timer */
9280                 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
9281                         cpu_preemption_timer_multi);
9282         else
9283                 delta_tsc = 0;
9284
9285         vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc);
9286 }
9287
9288 static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
9289 {
9290         struct vcpu_vmx *vmx = to_vmx(vcpu);
9291         unsigned long debugctlmsr, cr3, cr4;
9292
9293         /* Record the guest's net vcpu time for enforced NMI injections. */
9294         if (unlikely(!cpu_has_virtual_nmis() &&
9295                      vmx->loaded_vmcs->soft_vnmi_blocked))
9296                 vmx->loaded_vmcs->entry_time = ktime_get();
9297
9298         /* Don't enter VMX if guest state is invalid, let the exit handler
9299            start emulation until we arrive back to a valid state */
9300         if (vmx->emulation_required)
9301                 return;
9302
9303         if (vmx->ple_window_dirty) {
9304                 vmx->ple_window_dirty = false;
9305                 vmcs_write32(PLE_WINDOW, vmx->ple_window);
9306         }
9307
9308         if (vmx->nested.sync_shadow_vmcs) {
9309                 copy_vmcs12_to_shadow(vmx);
9310                 vmx->nested.sync_shadow_vmcs = false;
9311         }
9312
9313         if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
9314                 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
9315         if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
9316                 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
9317
9318         cr3 = __get_current_cr3_fast();
9319         if (unlikely(cr3 != vmx->loaded_vmcs->vmcs_host_cr3)) {
9320                 vmcs_writel(HOST_CR3, cr3);
9321                 vmx->loaded_vmcs->vmcs_host_cr3 = cr3;
9322         }
9323
9324         cr4 = cr4_read_shadow();
9325         if (unlikely(cr4 != vmx->loaded_vmcs->vmcs_host_cr4)) {
9326                 vmcs_writel(HOST_CR4, cr4);
9327                 vmx->loaded_vmcs->vmcs_host_cr4 = cr4;
9328         }
9329
9330         /* When single-stepping over STI and MOV SS, we must clear the
9331          * corresponding interruptibility bits in the guest state. Otherwise
9332          * vmentry fails as it then expects bit 14 (BS) in pending debug
9333          * exceptions being set, but that's not correct for the guest debugging
9334          * case. */
9335         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
9336                 vmx_set_interrupt_shadow(vcpu, 0);
9337
9338         if (static_cpu_has(X86_FEATURE_PKU) &&
9339             kvm_read_cr4_bits(vcpu, X86_CR4_PKE) &&
9340             vcpu->arch.pkru != vmx->host_pkru)
9341                 __write_pkru(vcpu->arch.pkru);
9342
9343         atomic_switch_perf_msrs(vmx);
9344         debugctlmsr = get_debugctlmsr();
9345
9346         vmx_arm_hv_timer(vcpu);
9347
9348         vmx->__launched = vmx->loaded_vmcs->launched;
9349         asm(
9350                 /* Store host registers */
9351                 "push %%" _ASM_DX "; push %%" _ASM_BP ";"
9352                 "push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
9353                 "push %%" _ASM_CX " \n\t"
9354                 "cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
9355                 "je 1f \n\t"
9356                 "mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
9357                 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
9358                 "1: \n\t"
9359                 /* Reload cr2 if changed */
9360                 "mov %c[cr2](%0), %%" _ASM_AX " \n\t"
9361                 "mov %%cr2, %%" _ASM_DX " \n\t"
9362                 "cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
9363                 "je 2f \n\t"
9364                 "mov %%" _ASM_AX", %%cr2 \n\t"
9365                 "2: \n\t"
9366                 /* Check if vmlaunch of vmresume is needed */
9367                 "cmpl $0, %c[launched](%0) \n\t"
9368                 /* Load guest registers.  Don't clobber flags. */
9369                 "mov %c[rax](%0), %%" _ASM_AX " \n\t"
9370                 "mov %c[rbx](%0), %%" _ASM_BX " \n\t"
9371                 "mov %c[rdx](%0), %%" _ASM_DX " \n\t"
9372                 "mov %c[rsi](%0), %%" _ASM_SI " \n\t"
9373                 "mov %c[rdi](%0), %%" _ASM_DI " \n\t"
9374                 "mov %c[rbp](%0), %%" _ASM_BP " \n\t"
9375 #ifdef CONFIG_X86_64
9376                 "mov %c[r8](%0),  %%r8  \n\t"
9377                 "mov %c[r9](%0),  %%r9  \n\t"
9378                 "mov %c[r10](%0), %%r10 \n\t"
9379                 "mov %c[r11](%0), %%r11 \n\t"
9380                 "mov %c[r12](%0), %%r12 \n\t"
9381                 "mov %c[r13](%0), %%r13 \n\t"
9382                 "mov %c[r14](%0), %%r14 \n\t"
9383                 "mov %c[r15](%0), %%r15 \n\t"
9384 #endif
9385                 "mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */
9386
9387                 /* Enter guest mode */
9388                 "jne 1f \n\t"
9389                 __ex(ASM_VMX_VMLAUNCH) "\n\t"
9390                 "jmp 2f \n\t"
9391                 "1: " __ex(ASM_VMX_VMRESUME) "\n\t"
9392                 "2: "
9393                 /* Save guest registers, load host registers, keep flags */
9394                 "mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
9395                 "pop %0 \n\t"
9396                 "setbe %c[fail](%0)\n\t"
9397                 "mov %%" _ASM_AX ", %c[rax](%0) \n\t"
9398                 "mov %%" _ASM_BX ", %c[rbx](%0) \n\t"
9399                 __ASM_SIZE(pop) " %c[rcx](%0) \n\t"
9400                 "mov %%" _ASM_DX ", %c[rdx](%0) \n\t"
9401                 "mov %%" _ASM_SI ", %c[rsi](%0) \n\t"
9402                 "mov %%" _ASM_DI ", %c[rdi](%0) \n\t"
9403                 "mov %%" _ASM_BP ", %c[rbp](%0) \n\t"
9404 #ifdef CONFIG_X86_64
9405                 "mov %%r8,  %c[r8](%0) \n\t"
9406                 "mov %%r9,  %c[r9](%0) \n\t"
9407                 "mov %%r10, %c[r10](%0) \n\t"
9408                 "mov %%r11, %c[r11](%0) \n\t"
9409                 "mov %%r12, %c[r12](%0) \n\t"
9410                 "mov %%r13, %c[r13](%0) \n\t"
9411                 "mov %%r14, %c[r14](%0) \n\t"
9412                 "mov %%r15, %c[r15](%0) \n\t"
9413                 "xor %%r8d,  %%r8d \n\t"
9414                 "xor %%r9d,  %%r9d \n\t"
9415                 "xor %%r10d, %%r10d \n\t"
9416                 "xor %%r11d, %%r11d \n\t"
9417                 "xor %%r12d, %%r12d \n\t"
9418                 "xor %%r13d, %%r13d \n\t"
9419                 "xor %%r14d, %%r14d \n\t"
9420                 "xor %%r15d, %%r15d \n\t"
9421 #endif
9422                 "mov %%cr2, %%" _ASM_AX "   \n\t"
9423                 "mov %%" _ASM_AX ", %c[cr2](%0) \n\t"
9424
9425                 "xor %%eax, %%eax \n\t"
9426                 "xor %%ebx, %%ebx \n\t"
9427                 "xor %%esi, %%esi \n\t"
9428                 "xor %%edi, %%edi \n\t"
9429                 "pop  %%" _ASM_BP "; pop  %%" _ASM_DX " \n\t"
9430                 ".pushsection .rodata \n\t"
9431                 ".global vmx_return \n\t"
9432                 "vmx_return: " _ASM_PTR " 2b \n\t"
9433                 ".popsection"
9434               : : "c"(vmx), "d"((unsigned long)HOST_RSP),
9435                 [launched]"i"(offsetof(struct vcpu_vmx, __launched)),
9436                 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
9437                 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
9438                 [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
9439                 [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
9440                 [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
9441                 [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
9442                 [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
9443                 [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
9444                 [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
9445 #ifdef CONFIG_X86_64
9446                 [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
9447                 [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
9448                 [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
9449                 [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
9450                 [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
9451                 [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
9452                 [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
9453                 [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
9454 #endif
9455                 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
9456                 [wordsize]"i"(sizeof(ulong))
9457               : "cc", "memory"
9458 #ifdef CONFIG_X86_64
9459                 , "rax", "rbx", "rdi", "rsi"
9460                 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
9461 #else
9462                 , "eax", "ebx", "edi", "esi"
9463 #endif
9464               );
9465
9466         /* Eliminate branch target predictions from guest mode */
9467         vmexit_fill_RSB();
9468
9469         /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
9470         if (debugctlmsr)
9471                 update_debugctlmsr(debugctlmsr);
9472
9473 #ifndef CONFIG_X86_64
9474         /*
9475          * The sysexit path does not restore ds/es, so we must set them to
9476          * a reasonable value ourselves.
9477          *
9478          * We can't defer this to vmx_load_host_state() since that function
9479          * may be executed in interrupt context, which saves and restore segments
9480          * around it, nullifying its effect.
9481          */
9482         loadsegment(ds, __USER_DS);
9483         loadsegment(es, __USER_DS);
9484 #endif
9485
9486         vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
9487                                   | (1 << VCPU_EXREG_RFLAGS)
9488                                   | (1 << VCPU_EXREG_PDPTR)
9489                                   | (1 << VCPU_EXREG_SEGMENTS)
9490                                   | (1 << VCPU_EXREG_CR3));
9491         vcpu->arch.regs_dirty = 0;
9492
9493         /*
9494          * eager fpu is enabled if PKEY is supported and CR4 is switched
9495          * back on host, so it is safe to read guest PKRU from current
9496          * XSAVE.
9497          */
9498         if (static_cpu_has(X86_FEATURE_PKU) &&
9499             kvm_read_cr4_bits(vcpu, X86_CR4_PKE)) {
9500                 vcpu->arch.pkru = __read_pkru();
9501                 if (vcpu->arch.pkru != vmx->host_pkru)
9502                         __write_pkru(vmx->host_pkru);
9503         }
9504
9505         /*
9506          * the KVM_REQ_EVENT optimization bit is only on for one entry, and if
9507          * we did not inject a still-pending event to L1 now because of
9508          * nested_run_pending, we need to re-enable this bit.
9509          */
9510         if (vmx->nested.nested_run_pending)
9511                 kvm_make_request(KVM_REQ_EVENT, vcpu);
9512
9513         vmx->nested.nested_run_pending = 0;
9514         vmx->idt_vectoring_info = 0;
9515
9516         vmx->exit_reason = vmx->fail ? 0xdead : vmcs_read32(VM_EXIT_REASON);
9517         if (vmx->fail || (vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
9518                 return;
9519
9520         vmx->loaded_vmcs->launched = 1;
9521         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
9522
9523         vmx_complete_atomic_exit(vmx);
9524         vmx_recover_nmi_blocking(vmx);
9525         vmx_complete_interrupts(vmx);
9526 }
9527 STACK_FRAME_NON_STANDARD(vmx_vcpu_run);
9528
9529 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
9530 {
9531         struct vcpu_vmx *vmx = to_vmx(vcpu);
9532         int cpu;
9533
9534         if (vmx->loaded_vmcs == vmcs)
9535                 return;
9536
9537         cpu = get_cpu();
9538         vmx->loaded_vmcs = vmcs;
9539         vmx_vcpu_put(vcpu);
9540         vmx_vcpu_load(vcpu, cpu);
9541         vcpu->cpu = cpu;
9542         put_cpu();
9543 }
9544
9545 /*
9546  * Ensure that the current vmcs of the logical processor is the
9547  * vmcs01 of the vcpu before calling free_nested().
9548  */
9549 static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
9550 {
9551        struct vcpu_vmx *vmx = to_vmx(vcpu);
9552        int r;
9553
9554        r = vcpu_load(vcpu);
9555        BUG_ON(r);
9556        vmx_switch_vmcs(vcpu, &vmx->vmcs01);
9557        free_nested(vmx);
9558        vcpu_put(vcpu);
9559 }
9560
9561 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
9562 {
9563         struct vcpu_vmx *vmx = to_vmx(vcpu);
9564
9565         if (enable_pml)
9566                 vmx_destroy_pml_buffer(vmx);
9567         free_vpid(vmx->vpid);
9568         leave_guest_mode(vcpu);
9569         vmx_free_vcpu_nested(vcpu);
9570         free_loaded_vmcs(vmx->loaded_vmcs);
9571         kfree(vmx->guest_msrs);
9572         kvm_vcpu_uninit(vcpu);
9573         kmem_cache_free(kvm_vcpu_cache, vmx);
9574 }
9575
9576 static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
9577 {
9578         int err;
9579         struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
9580         unsigned long *msr_bitmap;
9581         int cpu;
9582
9583         if (!vmx)
9584                 return ERR_PTR(-ENOMEM);
9585
9586         vmx->vpid = allocate_vpid();
9587
9588         err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
9589         if (err)
9590                 goto free_vcpu;
9591
9592         err = -ENOMEM;
9593
9594         /*
9595          * If PML is turned on, failure on enabling PML just results in failure
9596          * of creating the vcpu, therefore we can simplify PML logic (by
9597          * avoiding dealing with cases, such as enabling PML partially on vcpus
9598          * for the guest, etc.
9599          */
9600         if (enable_pml) {
9601                 vmx->pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO);
9602                 if (!vmx->pml_pg)
9603                         goto uninit_vcpu;
9604         }
9605
9606         vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
9607         BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0])
9608                      > PAGE_SIZE);
9609
9610         if (!vmx->guest_msrs)
9611                 goto free_pml;
9612
9613         err = alloc_loaded_vmcs(&vmx->vmcs01);
9614         if (err < 0)
9615                 goto free_msrs;
9616
9617         msr_bitmap = vmx->vmcs01.msr_bitmap;
9618         vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE, MSR_TYPE_RW);
9619         vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE, MSR_TYPE_RW);
9620         vmx_disable_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
9621         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
9622         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW);
9623         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW);
9624         vmx->msr_bitmap_mode = 0;
9625
9626         vmx->loaded_vmcs = &vmx->vmcs01;
9627         cpu = get_cpu();
9628         vmx_vcpu_load(&vmx->vcpu, cpu);
9629         vmx->vcpu.cpu = cpu;
9630         err = vmx_vcpu_setup(vmx);
9631         vmx_vcpu_put(&vmx->vcpu);
9632         put_cpu();
9633         if (err)
9634                 goto free_vmcs;
9635         if (cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
9636                 err = alloc_apic_access_page(kvm);
9637                 if (err)
9638                         goto free_vmcs;
9639         }
9640
9641         if (enable_ept) {
9642                 if (!kvm->arch.ept_identity_map_addr)
9643                         kvm->arch.ept_identity_map_addr =
9644                                 VMX_EPT_IDENTITY_PAGETABLE_ADDR;
9645                 err = init_rmode_identity_map(kvm);
9646                 if (err)
9647                         goto free_vmcs;
9648         }
9649
9650         if (nested) {
9651                 nested_vmx_setup_ctls_msrs(vmx);
9652                 vmx->nested.vpid02 = allocate_vpid();
9653         }
9654
9655         vmx->nested.posted_intr_nv = -1;
9656         vmx->nested.current_vmptr = -1ull;
9657
9658         vmx->msr_ia32_feature_control_valid_bits = FEATURE_CONTROL_LOCKED;
9659
9660         /*
9661          * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
9662          * or POSTED_INTR_WAKEUP_VECTOR.
9663          */
9664         vmx->pi_desc.nv = POSTED_INTR_VECTOR;
9665         vmx->pi_desc.sn = 1;
9666
9667         return &vmx->vcpu;
9668
9669 free_vmcs:
9670         free_vpid(vmx->nested.vpid02);
9671         free_loaded_vmcs(vmx->loaded_vmcs);
9672 free_msrs:
9673         kfree(vmx->guest_msrs);
9674 free_pml:
9675         vmx_destroy_pml_buffer(vmx);
9676 uninit_vcpu:
9677         kvm_vcpu_uninit(&vmx->vcpu);
9678 free_vcpu:
9679         free_vpid(vmx->vpid);
9680         kmem_cache_free(kvm_vcpu_cache, vmx);
9681         return ERR_PTR(err);
9682 }
9683
9684 static void __init vmx_check_processor_compat(void *rtn)
9685 {
9686         struct vmcs_config vmcs_conf;
9687
9688         *(int *)rtn = 0;
9689         if (setup_vmcs_config(&vmcs_conf) < 0)
9690                 *(int *)rtn = -EIO;
9691         if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
9692                 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
9693                                 smp_processor_id());
9694                 *(int *)rtn = -EIO;
9695         }
9696 }
9697
9698 static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
9699 {
9700         u8 cache;
9701         u64 ipat = 0;
9702
9703         /* For VT-d and EPT combination
9704          * 1. MMIO: always map as UC
9705          * 2. EPT with VT-d:
9706          *   a. VT-d without snooping control feature: can't guarantee the
9707          *      result, try to trust guest.
9708          *   b. VT-d with snooping control feature: snooping control feature of
9709          *      VT-d engine can guarantee the cache correctness. Just set it
9710          *      to WB to keep consistent with host. So the same as item 3.
9711          * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
9712          *    consistent with host MTRR
9713          */
9714         if (is_mmio) {
9715                 cache = MTRR_TYPE_UNCACHABLE;
9716                 goto exit;
9717         }
9718
9719         if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
9720                 ipat = VMX_EPT_IPAT_BIT;
9721                 cache = MTRR_TYPE_WRBACK;
9722                 goto exit;
9723         }
9724
9725         if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
9726                 ipat = VMX_EPT_IPAT_BIT;
9727                 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
9728                         cache = MTRR_TYPE_WRBACK;
9729                 else
9730                         cache = MTRR_TYPE_UNCACHABLE;
9731                 goto exit;
9732         }
9733
9734         cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
9735
9736 exit:
9737         return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat;
9738 }
9739
9740 static int vmx_get_lpage_level(void)
9741 {
9742         if (enable_ept && !cpu_has_vmx_ept_1g_page())
9743                 return PT_DIRECTORY_LEVEL;
9744         else
9745                 /* For shadow and EPT supported 1GB page */
9746                 return PT_PDPE_LEVEL;
9747 }
9748
9749 static void vmcs_set_secondary_exec_control(u32 new_ctl)
9750 {
9751         /*
9752          * These bits in the secondary execution controls field
9753          * are dynamic, the others are mostly based on the hypervisor
9754          * architecture and the guest's CPUID.  Do not touch the
9755          * dynamic bits.
9756          */
9757         u32 mask =
9758                 SECONDARY_EXEC_SHADOW_VMCS |
9759                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
9760                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
9761
9762         u32 cur_ctl = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
9763
9764         vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
9765                      (new_ctl & ~mask) | (cur_ctl & mask));
9766 }
9767
9768 /*
9769  * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
9770  * (indicating "allowed-1") if they are supported in the guest's CPUID.
9771  */
9772 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
9773 {
9774         struct vcpu_vmx *vmx = to_vmx(vcpu);
9775         struct kvm_cpuid_entry2 *entry;
9776
9777         vmx->nested.nested_vmx_cr0_fixed1 = 0xffffffff;
9778         vmx->nested.nested_vmx_cr4_fixed1 = X86_CR4_PCE;
9779
9780 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do {            \
9781         if (entry && (entry->_reg & (_cpuid_mask)))                     \
9782                 vmx->nested.nested_vmx_cr4_fixed1 |= (_cr4_mask);       \
9783 } while (0)
9784
9785         entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
9786         cr4_fixed1_update(X86_CR4_VME,        edx, bit(X86_FEATURE_VME));
9787         cr4_fixed1_update(X86_CR4_PVI,        edx, bit(X86_FEATURE_VME));
9788         cr4_fixed1_update(X86_CR4_TSD,        edx, bit(X86_FEATURE_TSC));
9789         cr4_fixed1_update(X86_CR4_DE,         edx, bit(X86_FEATURE_DE));
9790         cr4_fixed1_update(X86_CR4_PSE,        edx, bit(X86_FEATURE_PSE));
9791         cr4_fixed1_update(X86_CR4_PAE,        edx, bit(X86_FEATURE_PAE));
9792         cr4_fixed1_update(X86_CR4_MCE,        edx, bit(X86_FEATURE_MCE));
9793         cr4_fixed1_update(X86_CR4_PGE,        edx, bit(X86_FEATURE_PGE));
9794         cr4_fixed1_update(X86_CR4_OSFXSR,     edx, bit(X86_FEATURE_FXSR));
9795         cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, bit(X86_FEATURE_XMM));
9796         cr4_fixed1_update(X86_CR4_VMXE,       ecx, bit(X86_FEATURE_VMX));
9797         cr4_fixed1_update(X86_CR4_SMXE,       ecx, bit(X86_FEATURE_SMX));
9798         cr4_fixed1_update(X86_CR4_PCIDE,      ecx, bit(X86_FEATURE_PCID));
9799         cr4_fixed1_update(X86_CR4_OSXSAVE,    ecx, bit(X86_FEATURE_XSAVE));
9800
9801         entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
9802         cr4_fixed1_update(X86_CR4_FSGSBASE,   ebx, bit(X86_FEATURE_FSGSBASE));
9803         cr4_fixed1_update(X86_CR4_SMEP,       ebx, bit(X86_FEATURE_SMEP));
9804         cr4_fixed1_update(X86_CR4_SMAP,       ebx, bit(X86_FEATURE_SMAP));
9805         cr4_fixed1_update(X86_CR4_PKE,        ecx, bit(X86_FEATURE_PKU));
9806         /* TODO: Use X86_CR4_UMIP and X86_FEATURE_UMIP macros */
9807         cr4_fixed1_update(bit(11),            ecx, bit(2));
9808
9809 #undef cr4_fixed1_update
9810 }
9811
9812 static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
9813 {
9814         struct vcpu_vmx *vmx = to_vmx(vcpu);
9815
9816         if (cpu_has_secondary_exec_ctrls()) {
9817                 vmx_compute_secondary_exec_control(vmx);
9818                 vmcs_set_secondary_exec_control(vmx->secondary_exec_control);
9819         }
9820
9821         if (nested_vmx_allowed(vcpu))
9822                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
9823                         FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
9824         else
9825                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
9826                         ~FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
9827
9828         if (nested_vmx_allowed(vcpu))
9829                 nested_vmx_cr_fixed1_bits_update(vcpu);
9830 }
9831
9832 static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
9833 {
9834         if (func == 1 && nested)
9835                 entry->ecx |= bit(X86_FEATURE_VMX);
9836 }
9837
9838 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
9839                 struct x86_exception *fault)
9840 {
9841         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
9842         struct vcpu_vmx *vmx = to_vmx(vcpu);
9843         u32 exit_reason;
9844         unsigned long exit_qualification = vcpu->arch.exit_qualification;
9845
9846         if (vmx->nested.pml_full) {
9847                 exit_reason = EXIT_REASON_PML_FULL;
9848                 vmx->nested.pml_full = false;
9849                 exit_qualification &= INTR_INFO_UNBLOCK_NMI;
9850         } else if (fault->error_code & PFERR_RSVD_MASK)
9851                 exit_reason = EXIT_REASON_EPT_MISCONFIG;
9852         else
9853                 exit_reason = EXIT_REASON_EPT_VIOLATION;
9854
9855         nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification);
9856         vmcs12->guest_physical_address = fault->address;
9857 }
9858
9859 static bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu)
9860 {
9861         return nested_ept_get_cr3(vcpu) & VMX_EPTP_AD_ENABLE_BIT;
9862 }
9863
9864 /* Callbacks for nested_ept_init_mmu_context: */
9865
9866 static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu)
9867 {
9868         /* return the page table to be shadowed - in our case, EPT12 */
9869         return get_vmcs12(vcpu)->ept_pointer;
9870 }
9871
9872 static int nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
9873 {
9874         WARN_ON(mmu_is_nested(vcpu));
9875         if (!valid_ept_address(vcpu, nested_ept_get_cr3(vcpu)))
9876                 return 1;
9877
9878         kvm_mmu_unload(vcpu);
9879         kvm_init_shadow_ept_mmu(vcpu,
9880                         to_vmx(vcpu)->nested.nested_vmx_ept_caps &
9881                         VMX_EPT_EXECUTE_ONLY_BIT,
9882                         nested_ept_ad_enabled(vcpu));
9883         vcpu->arch.mmu.set_cr3           = vmx_set_cr3;
9884         vcpu->arch.mmu.get_cr3           = nested_ept_get_cr3;
9885         vcpu->arch.mmu.inject_page_fault = nested_ept_inject_page_fault;
9886
9887         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
9888         return 0;
9889 }
9890
9891 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
9892 {
9893         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
9894 }
9895
9896 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
9897                                             u16 error_code)
9898 {
9899         bool inequality, bit;
9900
9901         bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
9902         inequality =
9903                 (error_code & vmcs12->page_fault_error_code_mask) !=
9904                  vmcs12->page_fault_error_code_match;
9905         return inequality ^ bit;
9906 }
9907
9908 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
9909                 struct x86_exception *fault)
9910 {
9911         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
9912
9913         WARN_ON(!is_guest_mode(vcpu));
9914
9915         if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) &&
9916                 !to_vmx(vcpu)->nested.nested_run_pending) {
9917                 vmcs12->vm_exit_intr_error_code = fault->error_code;
9918                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
9919                                   PF_VECTOR | INTR_TYPE_HARD_EXCEPTION |
9920                                   INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK,
9921                                   fault->address);
9922         } else {
9923                 kvm_inject_page_fault(vcpu, fault);
9924         }
9925 }
9926
9927 static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
9928                                                struct vmcs12 *vmcs12);
9929
9930 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
9931                                         struct vmcs12 *vmcs12)
9932 {
9933         struct vcpu_vmx *vmx = to_vmx(vcpu);
9934         struct page *page;
9935         u64 hpa;
9936
9937         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
9938                 /*
9939                  * Translate L1 physical address to host physical
9940                  * address for vmcs02. Keep the page pinned, so this
9941                  * physical address remains valid. We keep a reference
9942                  * to it so we can release it later.
9943                  */
9944                 if (vmx->nested.apic_access_page) { /* shouldn't happen */
9945                         kvm_release_page_dirty(vmx->nested.apic_access_page);
9946                         vmx->nested.apic_access_page = NULL;
9947                 }
9948                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr);
9949                 /*
9950                  * If translation failed, no matter: This feature asks
9951                  * to exit when accessing the given address, and if it
9952                  * can never be accessed, this feature won't do
9953                  * anything anyway.
9954                  */
9955                 if (!is_error_page(page)) {
9956                         vmx->nested.apic_access_page = page;
9957                         hpa = page_to_phys(vmx->nested.apic_access_page);
9958                         vmcs_write64(APIC_ACCESS_ADDR, hpa);
9959                 } else {
9960                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
9961                                         SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
9962                 }
9963         } else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) &&
9964                    cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
9965                 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
9966                               SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
9967                 kvm_vcpu_reload_apic_access_page(vcpu);
9968         }
9969
9970         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
9971                 if (vmx->nested.virtual_apic_page) { /* shouldn't happen */
9972                         kvm_release_page_dirty(vmx->nested.virtual_apic_page);
9973                         vmx->nested.virtual_apic_page = NULL;
9974                 }
9975                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->virtual_apic_page_addr);
9976
9977                 /*
9978                  * If translation failed, VM entry will fail because
9979                  * prepare_vmcs02 set VIRTUAL_APIC_PAGE_ADDR to -1ull.
9980                  * Failing the vm entry is _not_ what the processor
9981                  * does but it's basically the only possibility we
9982                  * have.  We could still enter the guest if CR8 load
9983                  * exits are enabled, CR8 store exits are enabled, and
9984                  * virtualize APIC access is disabled; in this case
9985                  * the processor would never use the TPR shadow and we
9986                  * could simply clear the bit from the execution
9987                  * control.  But such a configuration is useless, so
9988                  * let's keep the code simple.
9989                  */
9990                 if (!is_error_page(page)) {
9991                         vmx->nested.virtual_apic_page = page;
9992                         hpa = page_to_phys(vmx->nested.virtual_apic_page);
9993                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, hpa);
9994                 }
9995         }
9996
9997         if (nested_cpu_has_posted_intr(vmcs12)) {
9998                 if (vmx->nested.pi_desc_page) { /* shouldn't happen */
9999                         kunmap(vmx->nested.pi_desc_page);
10000                         kvm_release_page_dirty(vmx->nested.pi_desc_page);
10001                         vmx->nested.pi_desc_page = NULL;
10002                 }
10003                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->posted_intr_desc_addr);
10004                 if (is_error_page(page))
10005                         return;
10006                 vmx->nested.pi_desc_page = page;
10007                 vmx->nested.pi_desc = kmap(vmx->nested.pi_desc_page);
10008                 vmx->nested.pi_desc =
10009                         (struct pi_desc *)((void *)vmx->nested.pi_desc +
10010                         (unsigned long)(vmcs12->posted_intr_desc_addr &
10011                         (PAGE_SIZE - 1)));
10012                 vmcs_write64(POSTED_INTR_DESC_ADDR,
10013                         page_to_phys(vmx->nested.pi_desc_page) +
10014                         (unsigned long)(vmcs12->posted_intr_desc_addr &
10015                         (PAGE_SIZE - 1)));
10016         }
10017         if (cpu_has_vmx_msr_bitmap() &&
10018             nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS) &&
10019             nested_vmx_merge_msr_bitmap(vcpu, vmcs12))
10020                 ;
10021         else
10022                 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
10023                                 CPU_BASED_USE_MSR_BITMAPS);
10024 }
10025
10026 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
10027 {
10028         u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
10029         struct vcpu_vmx *vmx = to_vmx(vcpu);
10030
10031         if (vcpu->arch.virtual_tsc_khz == 0)
10032                 return;
10033
10034         /* Make sure short timeouts reliably trigger an immediate vmexit.
10035          * hrtimer_start does not guarantee this. */
10036         if (preemption_timeout <= 1) {
10037                 vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
10038                 return;
10039         }
10040
10041         preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
10042         preemption_timeout *= 1000000;
10043         do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
10044         hrtimer_start(&vmx->nested.preemption_timer,
10045                       ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
10046 }
10047
10048 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu,
10049                                                struct vmcs12 *vmcs12)
10050 {
10051         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
10052                 return 0;
10053
10054         if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) ||
10055             !page_address_valid(vcpu, vmcs12->io_bitmap_b))
10056                 return -EINVAL;
10057
10058         return 0;
10059 }
10060
10061 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
10062                                                 struct vmcs12 *vmcs12)
10063 {
10064         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
10065                 return 0;
10066
10067         if (!page_address_valid(vcpu, vmcs12->msr_bitmap))
10068                 return -EINVAL;
10069
10070         return 0;
10071 }
10072
10073 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
10074                                                 struct vmcs12 *vmcs12)
10075 {
10076         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
10077                 return 0;
10078
10079         if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))
10080                 return -EINVAL;
10081
10082         return 0;
10083 }
10084
10085 /*
10086  * Merge L0's and L1's MSR bitmap, return false to indicate that
10087  * we do not use the hardware.
10088  */
10089 static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
10090                                                struct vmcs12 *vmcs12)
10091 {
10092         int msr;
10093         struct page *page;
10094         unsigned long *msr_bitmap_l1;
10095         unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
10096         /*
10097          * pred_cmd is trying to verify two things:
10098          *
10099          * 1. L0 gave a permission to L1 to actually passthrough the MSR. This
10100          *    ensures that we do not accidentally generate an L02 MSR bitmap
10101          *    from the L12 MSR bitmap that is too permissive.
10102          * 2. That L1 or L2s have actually used the MSR. This avoids
10103          *    unnecessarily merging of the bitmap if the MSR is unused. This
10104          *    works properly because we only update the L01 MSR bitmap lazily.
10105          *    So even if L0 should pass L1 these MSRs, the L01 bitmap is only
10106          *    updated to reflect this when L1 (or its L2s) actually write to
10107          *    the MSR.
10108          */
10109         bool pred_cmd = msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD);
10110
10111         if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
10112             !pred_cmd)
10113                 return false;
10114
10115         page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->msr_bitmap);
10116         if (is_error_page(page))
10117                 return false;
10118         msr_bitmap_l1 = (unsigned long *)kmap(page);
10119
10120         memset(msr_bitmap_l0, 0xff, PAGE_SIZE);
10121
10122         if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
10123                 if (nested_cpu_has_apic_reg_virt(vmcs12))
10124                         for (msr = 0x800; msr <= 0x8ff; msr++)
10125                                 nested_vmx_disable_intercept_for_msr(
10126                                         msr_bitmap_l1, msr_bitmap_l0,
10127                                         msr, MSR_TYPE_R);
10128
10129                 nested_vmx_disable_intercept_for_msr(
10130                                 msr_bitmap_l1, msr_bitmap_l0,
10131                                 APIC_BASE_MSR + (APIC_TASKPRI >> 4),
10132                                 MSR_TYPE_R | MSR_TYPE_W);
10133
10134                 if (nested_cpu_has_vid(vmcs12)) {
10135                         nested_vmx_disable_intercept_for_msr(
10136                                 msr_bitmap_l1, msr_bitmap_l0,
10137                                 APIC_BASE_MSR + (APIC_EOI >> 4),
10138                                 MSR_TYPE_W);
10139                         nested_vmx_disable_intercept_for_msr(
10140                                 msr_bitmap_l1, msr_bitmap_l0,
10141                                 APIC_BASE_MSR + (APIC_SELF_IPI >> 4),
10142                                 MSR_TYPE_W);
10143                 }
10144         }
10145
10146         if (pred_cmd)
10147                 nested_vmx_disable_intercept_for_msr(
10148                                         msr_bitmap_l1, msr_bitmap_l0,
10149                                         MSR_IA32_PRED_CMD,
10150                                         MSR_TYPE_W);
10151
10152         kunmap(page);
10153         kvm_release_page_clean(page);
10154
10155         return true;
10156 }
10157
10158 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
10159                                            struct vmcs12 *vmcs12)
10160 {
10161         if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
10162             !nested_cpu_has_apic_reg_virt(vmcs12) &&
10163             !nested_cpu_has_vid(vmcs12) &&
10164             !nested_cpu_has_posted_intr(vmcs12))
10165                 return 0;
10166
10167         /*
10168          * If virtualize x2apic mode is enabled,
10169          * virtualize apic access must be disabled.
10170          */
10171         if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&
10172             nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
10173                 return -EINVAL;
10174
10175         /*
10176          * If virtual interrupt delivery is enabled,
10177          * we must exit on external interrupts.
10178          */
10179         if (nested_cpu_has_vid(vmcs12) &&
10180            !nested_exit_on_intr(vcpu))
10181                 return -EINVAL;
10182
10183         /*
10184          * bits 15:8 should be zero in posted_intr_nv,
10185          * the descriptor address has been already checked
10186          * in nested_get_vmcs12_pages.
10187          */
10188         if (nested_cpu_has_posted_intr(vmcs12) &&
10189            (!nested_cpu_has_vid(vmcs12) ||
10190             !nested_exit_intr_ack_set(vcpu) ||
10191             vmcs12->posted_intr_nv & 0xff00))
10192                 return -EINVAL;
10193
10194         /* tpr shadow is needed by all apicv features. */
10195         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
10196                 return -EINVAL;
10197
10198         return 0;
10199 }
10200
10201 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
10202                                        unsigned long count_field,
10203                                        unsigned long addr_field)
10204 {
10205         int maxphyaddr;
10206         u64 count, addr;
10207
10208         if (vmcs12_read_any(vcpu, count_field, &count) ||
10209             vmcs12_read_any(vcpu, addr_field, &addr)) {
10210                 WARN_ON(1);
10211                 return -EINVAL;
10212         }
10213         if (count == 0)
10214                 return 0;
10215         maxphyaddr = cpuid_maxphyaddr(vcpu);
10216         if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
10217             (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) {
10218                 pr_debug_ratelimited(
10219                         "nVMX: invalid MSR switch (0x%lx, %d, %llu, 0x%08llx)",
10220                         addr_field, maxphyaddr, count, addr);
10221                 return -EINVAL;
10222         }
10223         return 0;
10224 }
10225
10226 static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu,
10227                                                 struct vmcs12 *vmcs12)
10228 {
10229         if (vmcs12->vm_exit_msr_load_count == 0 &&
10230             vmcs12->vm_exit_msr_store_count == 0 &&
10231             vmcs12->vm_entry_msr_load_count == 0)
10232                 return 0; /* Fast path */
10233         if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT,
10234                                         VM_EXIT_MSR_LOAD_ADDR) ||
10235             nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT,
10236                                         VM_EXIT_MSR_STORE_ADDR) ||
10237             nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT,
10238                                         VM_ENTRY_MSR_LOAD_ADDR))
10239                 return -EINVAL;
10240         return 0;
10241 }
10242
10243 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu,
10244                                          struct vmcs12 *vmcs12)
10245 {
10246         u64 address = vmcs12->pml_address;
10247         int maxphyaddr = cpuid_maxphyaddr(vcpu);
10248
10249         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_PML)) {
10250                 if (!nested_cpu_has_ept(vmcs12) ||
10251                     !IS_ALIGNED(address, 4096)  ||
10252                     address >> maxphyaddr)
10253                         return -EINVAL;
10254         }
10255
10256         return 0;
10257 }
10258
10259 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
10260                                        struct vmx_msr_entry *e)
10261 {
10262         /* x2APIC MSR accesses are not allowed */
10263         if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)
10264                 return -EINVAL;
10265         if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */
10266             e->index == MSR_IA32_UCODE_REV)
10267                 return -EINVAL;
10268         if (e->reserved != 0)
10269                 return -EINVAL;
10270         return 0;
10271 }
10272
10273 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
10274                                      struct vmx_msr_entry *e)
10275 {
10276         if (e->index == MSR_FS_BASE ||
10277             e->index == MSR_GS_BASE ||
10278             e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */
10279             nested_vmx_msr_check_common(vcpu, e))
10280                 return -EINVAL;
10281         return 0;
10282 }
10283
10284 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
10285                                       struct vmx_msr_entry *e)
10286 {
10287         if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */
10288             nested_vmx_msr_check_common(vcpu, e))
10289                 return -EINVAL;
10290         return 0;
10291 }
10292
10293 /*
10294  * Load guest's/host's msr at nested entry/exit.
10295  * return 0 for success, entry index for failure.
10296  */
10297 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
10298 {
10299         u32 i;
10300         struct vmx_msr_entry e;
10301         struct msr_data msr;
10302
10303         msr.host_initiated = false;
10304         for (i = 0; i < count; i++) {
10305                 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
10306                                         &e, sizeof(e))) {
10307                         pr_debug_ratelimited(
10308                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
10309                                 __func__, i, gpa + i * sizeof(e));
10310                         goto fail;
10311                 }
10312                 if (nested_vmx_load_msr_check(vcpu, &e)) {
10313                         pr_debug_ratelimited(
10314                                 "%s check failed (%u, 0x%x, 0x%x)\n",
10315                                 __func__, i, e.index, e.reserved);
10316                         goto fail;
10317                 }
10318                 msr.index = e.index;
10319                 msr.data = e.value;
10320                 if (kvm_set_msr(vcpu, &msr)) {
10321                         pr_debug_ratelimited(
10322                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
10323                                 __func__, i, e.index, e.value);
10324                         goto fail;
10325                 }
10326         }
10327         return 0;
10328 fail:
10329         return i + 1;
10330 }
10331
10332 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
10333 {
10334         u32 i;
10335         struct vmx_msr_entry e;
10336
10337         for (i = 0; i < count; i++) {
10338                 struct msr_data msr_info;
10339                 if (kvm_vcpu_read_guest(vcpu,
10340                                         gpa + i * sizeof(e),
10341                                         &e, 2 * sizeof(u32))) {
10342                         pr_debug_ratelimited(
10343                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
10344                                 __func__, i, gpa + i * sizeof(e));
10345                         return -EINVAL;
10346                 }
10347                 if (nested_vmx_store_msr_check(vcpu, &e)) {
10348                         pr_debug_ratelimited(
10349                                 "%s check failed (%u, 0x%x, 0x%x)\n",
10350                                 __func__, i, e.index, e.reserved);
10351                         return -EINVAL;
10352                 }
10353                 msr_info.host_initiated = false;
10354                 msr_info.index = e.index;
10355                 if (kvm_get_msr(vcpu, &msr_info)) {
10356                         pr_debug_ratelimited(
10357                                 "%s cannot read MSR (%u, 0x%x)\n",
10358                                 __func__, i, e.index);
10359                         return -EINVAL;
10360                 }
10361                 if (kvm_vcpu_write_guest(vcpu,
10362                                          gpa + i * sizeof(e) +
10363                                              offsetof(struct vmx_msr_entry, value),
10364                                          &msr_info.data, sizeof(msr_info.data))) {
10365                         pr_debug_ratelimited(
10366                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
10367                                 __func__, i, e.index, msr_info.data);
10368                         return -EINVAL;
10369                 }
10370         }
10371         return 0;
10372 }
10373
10374 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
10375 {
10376         unsigned long invalid_mask;
10377
10378         invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
10379         return (val & invalid_mask) == 0;
10380 }
10381
10382 /*
10383  * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are
10384  * emulating VM entry into a guest with EPT enabled.
10385  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
10386  * is assigned to entry_failure_code on failure.
10387  */
10388 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
10389                                u32 *entry_failure_code)
10390 {
10391         if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) {
10392                 if (!nested_cr3_valid(vcpu, cr3)) {
10393                         *entry_failure_code = ENTRY_FAIL_DEFAULT;
10394                         return 1;
10395                 }
10396
10397                 /*
10398                  * If PAE paging and EPT are both on, CR3 is not used by the CPU and
10399                  * must not be dereferenced.
10400                  */
10401                 if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) &&
10402                     !nested_ept) {
10403                         if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) {
10404                                 *entry_failure_code = ENTRY_FAIL_PDPTE;
10405                                 return 1;
10406                         }
10407                 }
10408
10409                 vcpu->arch.cr3 = cr3;
10410                 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
10411         }
10412
10413         kvm_mmu_reset_context(vcpu);
10414         return 0;
10415 }
10416
10417 /*
10418  * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
10419  * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
10420  * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
10421  * guest in a way that will both be appropriate to L1's requests, and our
10422  * needs. In addition to modifying the active vmcs (which is vmcs02), this
10423  * function also has additional necessary side-effects, like setting various
10424  * vcpu->arch fields.
10425  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
10426  * is assigned to entry_failure_code on failure.
10427  */
10428 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
10429                           bool from_vmentry, u32 *entry_failure_code)
10430 {
10431         struct vcpu_vmx *vmx = to_vmx(vcpu);
10432         u32 exec_control, vmcs12_exec_ctrl;
10433
10434         vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
10435         vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
10436         vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
10437         vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
10438         vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
10439         vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
10440         vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
10441         vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
10442         vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
10443         vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
10444         vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
10445         vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
10446         vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
10447         vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
10448         vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
10449         vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
10450         vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
10451         vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
10452         vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
10453         vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
10454         vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
10455         vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
10456         vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
10457         vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
10458         vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
10459         vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
10460         vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
10461         vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
10462         vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
10463         vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
10464         vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
10465         vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
10466         vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
10467         vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
10468         vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
10469         vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
10470
10471         if (from_vmentry &&
10472             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
10473                 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
10474                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
10475         } else {
10476                 kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
10477                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
10478         }
10479         if (from_vmentry) {
10480                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
10481                              vmcs12->vm_entry_intr_info_field);
10482                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
10483                              vmcs12->vm_entry_exception_error_code);
10484                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
10485                              vmcs12->vm_entry_instruction_len);
10486                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
10487                              vmcs12->guest_interruptibility_info);
10488                 vmx->loaded_vmcs->nmi_known_unmasked =
10489                         !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI);
10490         } else {
10491                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
10492         }
10493         vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
10494         vmx_set_rflags(vcpu, vmcs12->guest_rflags);
10495         vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
10496                 vmcs12->guest_pending_dbg_exceptions);
10497         vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
10498         vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
10499
10500         if (nested_cpu_has_xsaves(vmcs12))
10501                 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
10502         vmcs_write64(VMCS_LINK_POINTER, -1ull);
10503
10504         exec_control = vmcs12->pin_based_vm_exec_control;
10505
10506         /* Preemption timer setting is only taken from vmcs01.  */
10507         exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
10508         exec_control |= vmcs_config.pin_based_exec_ctrl;
10509         if (vmx->hv_deadline_tsc == -1)
10510                 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
10511
10512         /* Posted interrupts setting is only taken from vmcs12.  */
10513         if (nested_cpu_has_posted_intr(vmcs12)) {
10514                 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
10515                 vmx->nested.pi_pending = false;
10516                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
10517         } else {
10518                 exec_control &= ~PIN_BASED_POSTED_INTR;
10519         }
10520
10521         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control);
10522
10523         vmx->nested.preemption_timer_expired = false;
10524         if (nested_cpu_has_preemption_timer(vmcs12))
10525                 vmx_start_preemption_timer(vcpu);
10526
10527         /*
10528          * Whether page-faults are trapped is determined by a combination of
10529          * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
10530          * If enable_ept, L0 doesn't care about page faults and we should
10531          * set all of these to L1's desires. However, if !enable_ept, L0 does
10532          * care about (at least some) page faults, and because it is not easy
10533          * (if at all possible?) to merge L0 and L1's desires, we simply ask
10534          * to exit on each and every L2 page fault. This is done by setting
10535          * MASK=MATCH=0 and (see below) EB.PF=1.
10536          * Note that below we don't need special code to set EB.PF beyond the
10537          * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
10538          * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
10539          * !enable_ept, EB.PF is 1, so the "or" will always be 1.
10540          */
10541         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
10542                 enable_ept ? vmcs12->page_fault_error_code_mask : 0);
10543         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
10544                 enable_ept ? vmcs12->page_fault_error_code_match : 0);
10545
10546         if (cpu_has_secondary_exec_ctrls()) {
10547                 exec_control = vmx->secondary_exec_control;
10548
10549                 /* Take the following fields only from vmcs12 */
10550                 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
10551                                   SECONDARY_EXEC_ENABLE_INVPCID |
10552                                   SECONDARY_EXEC_RDTSCP |
10553                                   SECONDARY_EXEC_XSAVES |
10554                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
10555                                   SECONDARY_EXEC_APIC_REGISTER_VIRT |
10556                                   SECONDARY_EXEC_ENABLE_VMFUNC);
10557                 if (nested_cpu_has(vmcs12,
10558                                    CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) {
10559                         vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control &
10560                                 ~SECONDARY_EXEC_ENABLE_PML;
10561                         exec_control |= vmcs12_exec_ctrl;
10562                 }
10563
10564                 /* All VMFUNCs are currently emulated through L0 vmexits.  */
10565                 if (exec_control & SECONDARY_EXEC_ENABLE_VMFUNC)
10566                         vmcs_write64(VM_FUNCTION_CONTROL, 0);
10567
10568                 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
10569                         vmcs_write64(EOI_EXIT_BITMAP0,
10570                                 vmcs12->eoi_exit_bitmap0);
10571                         vmcs_write64(EOI_EXIT_BITMAP1,
10572                                 vmcs12->eoi_exit_bitmap1);
10573                         vmcs_write64(EOI_EXIT_BITMAP2,
10574                                 vmcs12->eoi_exit_bitmap2);
10575                         vmcs_write64(EOI_EXIT_BITMAP3,
10576                                 vmcs12->eoi_exit_bitmap3);
10577                         vmcs_write16(GUEST_INTR_STATUS,
10578                                 vmcs12->guest_intr_status);
10579                 }
10580
10581                 /*
10582                  * Write an illegal value to APIC_ACCESS_ADDR. Later,
10583                  * nested_get_vmcs12_pages will either fix it up or
10584                  * remove the VM execution control.
10585                  */
10586                 if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)
10587                         vmcs_write64(APIC_ACCESS_ADDR, -1ull);
10588
10589                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
10590         }
10591
10592
10593         /*
10594          * Set host-state according to L0's settings (vmcs12 is irrelevant here)
10595          * Some constant fields are set here by vmx_set_constant_host_state().
10596          * Other fields are different per CPU, and will be set later when
10597          * vmx_vcpu_load() is called, and when vmx_save_host_state() is called.
10598          */
10599         vmx_set_constant_host_state(vmx);
10600
10601         /*
10602          * Set the MSR load/store lists to match L0's settings.
10603          */
10604         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
10605         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
10606         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
10607         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
10608         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
10609
10610         /*
10611          * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
10612          * entry, but only if the current (host) sp changed from the value
10613          * we wrote last (vmx->host_rsp). This cache is no longer relevant
10614          * if we switch vmcs, and rather than hold a separate cache per vmcs,
10615          * here we just force the write to happen on entry.
10616          */
10617         vmx->host_rsp = 0;
10618
10619         exec_control = vmx_exec_control(vmx); /* L0's desires */
10620         exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
10621         exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
10622         exec_control &= ~CPU_BASED_TPR_SHADOW;
10623         exec_control |= vmcs12->cpu_based_vm_exec_control;
10624
10625         /*
10626          * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR. Later, if
10627          * nested_get_vmcs12_pages can't fix it up, the illegal value
10628          * will result in a VM entry failure.
10629          */
10630         if (exec_control & CPU_BASED_TPR_SHADOW) {
10631                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull);
10632                 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
10633         } else {
10634 #ifdef CONFIG_X86_64
10635                 exec_control |= CPU_BASED_CR8_LOAD_EXITING |
10636                                 CPU_BASED_CR8_STORE_EXITING;
10637 #endif
10638         }
10639
10640         /*
10641          * Merging of IO bitmap not currently supported.
10642          * Rather, exit every time.
10643          */
10644         exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
10645         exec_control |= CPU_BASED_UNCOND_IO_EXITING;
10646
10647         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
10648
10649         /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
10650          * bitwise-or of what L1 wants to trap for L2, and what we want to
10651          * trap. Note that CR0.TS also needs updating - we do this later.
10652          */
10653         update_exception_bitmap(vcpu);
10654         vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
10655         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
10656
10657         /* L2->L1 exit controls are emulated - the hardware exit is to L0 so
10658          * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
10659          * bits are further modified by vmx_set_efer() below.
10660          */
10661         vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
10662
10663         /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are
10664          * emulated by vmx_set_efer(), below.
10665          */
10666         vm_entry_controls_init(vmx,
10667                 (vmcs12->vm_entry_controls & ~VM_ENTRY_LOAD_IA32_EFER &
10668                         ~VM_ENTRY_IA32E_MODE) |
10669                 (vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));
10670
10671         if (from_vmentry &&
10672             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
10673                 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
10674                 vcpu->arch.pat = vmcs12->guest_ia32_pat;
10675         } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
10676                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
10677         }
10678
10679         set_cr4_guest_host_mask(vmx);
10680
10681         if (from_vmentry &&
10682             vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)
10683                 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
10684
10685         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
10686                 vmcs_write64(TSC_OFFSET,
10687                         vcpu->arch.tsc_offset + vmcs12->tsc_offset);
10688         else
10689                 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
10690         if (kvm_has_tsc_control)
10691                 decache_tsc_multiplier(vmx);
10692
10693         if (cpu_has_vmx_msr_bitmap())
10694                 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
10695
10696         if (enable_vpid) {
10697                 /*
10698                  * There is no direct mapping between vpid02 and vpid12, the
10699                  * vpid02 is per-vCPU for L0 and reused while the value of
10700                  * vpid12 is changed w/ one invvpid during nested vmentry.
10701                  * The vpid12 is allocated by L1 for L2, so it will not
10702                  * influence global bitmap(for vpid01 and vpid02 allocation)
10703                  * even if spawn a lot of nested vCPUs.
10704                  */
10705                 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) {
10706                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
10707                         if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
10708                                 vmx->nested.last_vpid = vmcs12->virtual_processor_id;
10709                                 __vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02);
10710                         }
10711                 } else {
10712                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
10713                         vmx_flush_tlb(vcpu);
10714                 }
10715
10716         }
10717
10718         if (enable_pml) {
10719                 /*
10720                  * Conceptually we want to copy the PML address and index from
10721                  * vmcs01 here, and then back to vmcs01 on nested vmexit. But,
10722                  * since we always flush the log on each vmexit, this happens
10723                  * to be equivalent to simply resetting the fields in vmcs02.
10724                  */
10725                 ASSERT(vmx->pml_pg);
10726                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
10727                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
10728         }
10729
10730         if (nested_cpu_has_ept(vmcs12)) {
10731                 if (nested_ept_init_mmu_context(vcpu)) {
10732                         *entry_failure_code = ENTRY_FAIL_DEFAULT;
10733                         return 1;
10734                 }
10735         } else if (nested_cpu_has2(vmcs12,
10736                                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
10737                 vmx_flush_tlb_ept_only(vcpu);
10738         }
10739
10740         /*
10741          * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those
10742          * bits which we consider mandatory enabled.
10743          * The CR0_READ_SHADOW is what L2 should have expected to read given
10744          * the specifications by L1; It's not enough to take
10745          * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
10746          * have more bits than L1 expected.
10747          */
10748         vmx_set_cr0(vcpu, vmcs12->guest_cr0);
10749         vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
10750
10751         vmx_set_cr4(vcpu, vmcs12->guest_cr4);
10752         vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
10753
10754         if (from_vmentry &&
10755             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
10756                 vcpu->arch.efer = vmcs12->guest_ia32_efer;
10757         else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
10758                 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
10759         else
10760                 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
10761         /* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
10762         vmx_set_efer(vcpu, vcpu->arch.efer);
10763
10764         /* Shadow page tables on either EPT or shadow page tables. */
10765         if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12),
10766                                 entry_failure_code))
10767                 return 1;
10768
10769         if (!enable_ept)
10770                 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
10771
10772         /*
10773          * L1 may access the L2's PDPTR, so save them to construct vmcs12
10774          */
10775         if (enable_ept) {
10776                 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
10777                 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
10778                 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
10779                 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
10780         }
10781
10782         kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
10783         kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
10784         return 0;
10785 }
10786
10787 static int check_vmentry_prereqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
10788 {
10789         struct vcpu_vmx *vmx = to_vmx(vcpu);
10790
10791         if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
10792             vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT)
10793                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10794
10795         if (nested_vmx_check_io_bitmap_controls(vcpu, vmcs12))
10796                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10797
10798         if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12))
10799                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10800
10801         if (nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12))
10802                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10803
10804         if (nested_vmx_check_apicv_controls(vcpu, vmcs12))
10805                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10806
10807         if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12))
10808                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10809
10810         if (nested_vmx_check_pml_controls(vcpu, vmcs12))
10811                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10812
10813         if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
10814                                 vmx->nested.nested_vmx_procbased_ctls_low,
10815                                 vmx->nested.nested_vmx_procbased_ctls_high) ||
10816             (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
10817              !vmx_control_verify(vmcs12->secondary_vm_exec_control,
10818                                  vmx->nested.nested_vmx_secondary_ctls_low,
10819                                  vmx->nested.nested_vmx_secondary_ctls_high)) ||
10820             !vmx_control_verify(vmcs12->pin_based_vm_exec_control,
10821                                 vmx->nested.nested_vmx_pinbased_ctls_low,
10822                                 vmx->nested.nested_vmx_pinbased_ctls_high) ||
10823             !vmx_control_verify(vmcs12->vm_exit_controls,
10824                                 vmx->nested.nested_vmx_exit_ctls_low,
10825                                 vmx->nested.nested_vmx_exit_ctls_high) ||
10826             !vmx_control_verify(vmcs12->vm_entry_controls,
10827                                 vmx->nested.nested_vmx_entry_ctls_low,
10828                                 vmx->nested.nested_vmx_entry_ctls_high))
10829                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10830
10831         if (nested_cpu_has_vmfunc(vmcs12)) {
10832                 if (vmcs12->vm_function_control &
10833                     ~vmx->nested.nested_vmx_vmfunc_controls)
10834                         return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10835
10836                 if (nested_cpu_has_eptp_switching(vmcs12)) {
10837                         if (!nested_cpu_has_ept(vmcs12) ||
10838                             !page_address_valid(vcpu, vmcs12->eptp_list_address))
10839                                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10840                 }
10841         }
10842
10843         if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu))
10844                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
10845
10846         if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) ||
10847             !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) ||
10848             !nested_cr3_valid(vcpu, vmcs12->host_cr3))
10849                 return VMXERR_ENTRY_INVALID_HOST_STATE_FIELD;
10850
10851         return 0;
10852 }
10853
10854 static int check_vmentry_postreqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
10855                                   u32 *exit_qual)
10856 {
10857         bool ia32e;
10858
10859         *exit_qual = ENTRY_FAIL_DEFAULT;
10860
10861         if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) ||
10862             !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))
10863                 return 1;
10864
10865         if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_SHADOW_VMCS) &&
10866             vmcs12->vmcs_link_pointer != -1ull) {
10867                 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR;
10868                 return 1;
10869         }
10870
10871         /*
10872          * If the load IA32_EFER VM-entry control is 1, the following checks
10873          * are performed on the field for the IA32_EFER MSR:
10874          * - Bits reserved in the IA32_EFER MSR must be 0.
10875          * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
10876          *   the IA-32e mode guest VM-exit control. It must also be identical
10877          *   to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
10878          *   CR0.PG) is 1.
10879          */
10880         if (to_vmx(vcpu)->nested.nested_run_pending &&
10881             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
10882                 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
10883                 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) ||
10884                     ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) ||
10885                     ((vmcs12->guest_cr0 & X86_CR0_PG) &&
10886                      ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))
10887                         return 1;
10888         }
10889
10890         /*
10891          * If the load IA32_EFER VM-exit control is 1, bits reserved in the
10892          * IA32_EFER MSR must be 0 in the field for that register. In addition,
10893          * the values of the LMA and LME bits in the field must each be that of
10894          * the host address-space size VM-exit control.
10895          */
10896         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
10897                 ia32e = (vmcs12->vm_exit_controls &
10898                          VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
10899                 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) ||
10900                     ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) ||
10901                     ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))
10902                         return 1;
10903         }
10904
10905         return 0;
10906 }
10907
10908 static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry)
10909 {
10910         struct vcpu_vmx *vmx = to_vmx(vcpu);
10911         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
10912         u32 msr_entry_idx;
10913         u32 exit_qual;
10914
10915         enter_guest_mode(vcpu);
10916
10917         if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
10918                 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
10919
10920         vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
10921         vmx_segment_cache_clear(vmx);
10922
10923         if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &exit_qual)) {
10924                 leave_guest_mode(vcpu);
10925                 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
10926                 nested_vmx_entry_failure(vcpu, vmcs12,
10927                                          EXIT_REASON_INVALID_STATE, exit_qual);
10928                 return 1;
10929         }
10930
10931         nested_get_vmcs12_pages(vcpu, vmcs12);
10932
10933         msr_entry_idx = nested_vmx_load_msr(vcpu,
10934                                             vmcs12->vm_entry_msr_load_addr,
10935                                             vmcs12->vm_entry_msr_load_count);
10936         if (msr_entry_idx) {
10937                 leave_guest_mode(vcpu);
10938                 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
10939                 nested_vmx_entry_failure(vcpu, vmcs12,
10940                                 EXIT_REASON_MSR_LOAD_FAIL, msr_entry_idx);
10941                 return 1;
10942         }
10943
10944         /*
10945          * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
10946          * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
10947          * returned as far as L1 is concerned. It will only return (and set
10948          * the success flag) when L2 exits (see nested_vmx_vmexit()).
10949          */
10950         return 0;
10951 }
10952
10953 /*
10954  * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
10955  * for running an L2 nested guest.
10956  */
10957 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
10958 {
10959         struct vmcs12 *vmcs12;
10960         struct vcpu_vmx *vmx = to_vmx(vcpu);
10961         u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu);
10962         u32 exit_qual;
10963         int ret;
10964
10965         if (!nested_vmx_check_permission(vcpu))
10966                 return 1;
10967
10968         if (!nested_vmx_check_vmcs12(vcpu))
10969                 goto out;
10970
10971         vmcs12 = get_vmcs12(vcpu);
10972
10973         if (enable_shadow_vmcs)
10974                 copy_shadow_to_vmcs12(vmx);
10975
10976         /*
10977          * The nested entry process starts with enforcing various prerequisites
10978          * on vmcs12 as required by the Intel SDM, and act appropriately when
10979          * they fail: As the SDM explains, some conditions should cause the
10980          * instruction to fail, while others will cause the instruction to seem
10981          * to succeed, but return an EXIT_REASON_INVALID_STATE.
10982          * To speed up the normal (success) code path, we should avoid checking
10983          * for misconfigurations which will anyway be caught by the processor
10984          * when using the merged vmcs02.
10985          */
10986         if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS) {
10987                 nested_vmx_failValid(vcpu,
10988                                      VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS);
10989                 goto out;
10990         }
10991
10992         if (vmcs12->launch_state == launch) {
10993                 nested_vmx_failValid(vcpu,
10994                         launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
10995                                : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
10996                 goto out;
10997         }
10998
10999         ret = check_vmentry_prereqs(vcpu, vmcs12);
11000         if (ret) {
11001                 nested_vmx_failValid(vcpu, ret);
11002                 goto out;
11003         }
11004
11005         /*
11006          * After this point, the trap flag no longer triggers a singlestep trap
11007          * on the vm entry instructions; don't call kvm_skip_emulated_instruction.
11008          * This is not 100% correct; for performance reasons, we delegate most
11009          * of the checks on host state to the processor.  If those fail,
11010          * the singlestep trap is missed.
11011          */
11012         skip_emulated_instruction(vcpu);
11013
11014         ret = check_vmentry_postreqs(vcpu, vmcs12, &exit_qual);
11015         if (ret) {
11016                 nested_vmx_entry_failure(vcpu, vmcs12,
11017                                          EXIT_REASON_INVALID_STATE, exit_qual);
11018                 return 1;
11019         }
11020
11021         /*
11022          * We're finally done with prerequisite checking, and can start with
11023          * the nested entry.
11024          */
11025
11026         ret = enter_vmx_non_root_mode(vcpu, true);
11027         if (ret)
11028                 return ret;
11029
11030         if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT)
11031                 return kvm_vcpu_halt(vcpu);
11032
11033         vmx->nested.nested_run_pending = 1;
11034
11035         return 1;
11036
11037 out:
11038         return kvm_skip_emulated_instruction(vcpu);
11039 }
11040
11041 /*
11042  * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
11043  * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
11044  * This function returns the new value we should put in vmcs12.guest_cr0.
11045  * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
11046  *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
11047  *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
11048  *     didn't trap the bit, because if L1 did, so would L0).
11049  *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
11050  *     been modified by L2, and L1 knows it. So just leave the old value of
11051  *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
11052  *     isn't relevant, because if L0 traps this bit it can set it to anything.
11053  *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
11054  *     changed these bits, and therefore they need to be updated, but L0
11055  *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
11056  *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
11057  */
11058 static inline unsigned long
11059 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
11060 {
11061         return
11062         /*1*/   (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
11063         /*2*/   (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
11064         /*3*/   (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
11065                         vcpu->arch.cr0_guest_owned_bits));
11066 }
11067
11068 static inline unsigned long
11069 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
11070 {
11071         return
11072         /*1*/   (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
11073         /*2*/   (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
11074         /*3*/   (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
11075                         vcpu->arch.cr4_guest_owned_bits));
11076 }
11077
11078 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
11079                                        struct vmcs12 *vmcs12)
11080 {
11081         u32 idt_vectoring;
11082         unsigned int nr;
11083
11084         if (vcpu->arch.exception.injected) {
11085                 nr = vcpu->arch.exception.nr;
11086                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
11087
11088                 if (kvm_exception_is_soft(nr)) {
11089                         vmcs12->vm_exit_instruction_len =
11090                                 vcpu->arch.event_exit_inst_len;
11091                         idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
11092                 } else
11093                         idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
11094
11095                 if (vcpu->arch.exception.has_error_code) {
11096                         idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
11097                         vmcs12->idt_vectoring_error_code =
11098                                 vcpu->arch.exception.error_code;
11099                 }
11100
11101                 vmcs12->idt_vectoring_info_field = idt_vectoring;
11102         } else if (vcpu->arch.nmi_injected) {
11103                 vmcs12->idt_vectoring_info_field =
11104                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
11105         } else if (vcpu->arch.interrupt.pending) {
11106                 nr = vcpu->arch.interrupt.nr;
11107                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
11108
11109                 if (vcpu->arch.interrupt.soft) {
11110                         idt_vectoring |= INTR_TYPE_SOFT_INTR;
11111                         vmcs12->vm_entry_instruction_len =
11112                                 vcpu->arch.event_exit_inst_len;
11113                 } else
11114                         idt_vectoring |= INTR_TYPE_EXT_INTR;
11115
11116                 vmcs12->idt_vectoring_info_field = idt_vectoring;
11117         }
11118 }
11119
11120 static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
11121 {
11122         struct vcpu_vmx *vmx = to_vmx(vcpu);
11123         unsigned long exit_qual;
11124         bool block_nested_events =
11125             vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
11126
11127         if (vcpu->arch.exception.pending &&
11128                 nested_vmx_check_exception(vcpu, &exit_qual)) {
11129                 if (block_nested_events)
11130                         return -EBUSY;
11131                 nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
11132                 vcpu->arch.exception.pending = false;
11133                 return 0;
11134         }
11135
11136         if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
11137             vmx->nested.preemption_timer_expired) {
11138                 if (block_nested_events)
11139                         return -EBUSY;
11140                 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
11141                 return 0;
11142         }
11143
11144         if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
11145                 if (block_nested_events)
11146                         return -EBUSY;
11147                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
11148                                   NMI_VECTOR | INTR_TYPE_NMI_INTR |
11149                                   INTR_INFO_VALID_MASK, 0);
11150                 /*
11151                  * The NMI-triggered VM exit counts as injection:
11152                  * clear this one and block further NMIs.
11153                  */
11154                 vcpu->arch.nmi_pending = 0;
11155                 vmx_set_nmi_mask(vcpu, true);
11156                 return 0;
11157         }
11158
11159         if ((kvm_cpu_has_interrupt(vcpu) || external_intr) &&
11160             nested_exit_on_intr(vcpu)) {
11161                 if (block_nested_events)
11162                         return -EBUSY;
11163                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
11164                 return 0;
11165         }
11166
11167         vmx_complete_nested_posted_interrupt(vcpu);
11168         return 0;
11169 }
11170
11171 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
11172 {
11173         ktime_t remaining =
11174                 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
11175         u64 value;
11176
11177         if (ktime_to_ns(remaining) <= 0)
11178                 return 0;
11179
11180         value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
11181         do_div(value, 1000000);
11182         return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
11183 }
11184
11185 /*
11186  * Update the guest state fields of vmcs12 to reflect changes that
11187  * occurred while L2 was running. (The "IA-32e mode guest" bit of the
11188  * VM-entry controls is also updated, since this is really a guest
11189  * state bit.)
11190  */
11191 static void sync_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
11192 {
11193         vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
11194         vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
11195
11196         vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
11197         vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP);
11198         vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
11199
11200         vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
11201         vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
11202         vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
11203         vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
11204         vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
11205         vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
11206         vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
11207         vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
11208         vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
11209         vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
11210         vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
11211         vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
11212         vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
11213         vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
11214         vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
11215         vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
11216         vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
11217         vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
11218         vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
11219         vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
11220         vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
11221         vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
11222         vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
11223         vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
11224         vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
11225         vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
11226         vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
11227         vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
11228         vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
11229         vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
11230         vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
11231         vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
11232         vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
11233         vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
11234         vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
11235         vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
11236
11237         vmcs12->guest_interruptibility_info =
11238                 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
11239         vmcs12->guest_pending_dbg_exceptions =
11240                 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
11241         if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
11242                 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
11243         else
11244                 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
11245
11246         if (nested_cpu_has_preemption_timer(vmcs12)) {
11247                 if (vmcs12->vm_exit_controls &
11248                     VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
11249                         vmcs12->vmx_preemption_timer_value =
11250                                 vmx_get_preemption_timer_value(vcpu);
11251                 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
11252         }
11253
11254         /*
11255          * In some cases (usually, nested EPT), L2 is allowed to change its
11256          * own CR3 without exiting. If it has changed it, we must keep it.
11257          * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
11258          * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
11259          *
11260          * Additionally, restore L2's PDPTR to vmcs12.
11261          */
11262         if (enable_ept) {
11263                 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
11264                 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
11265                 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
11266                 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
11267                 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
11268         }
11269
11270         vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
11271
11272         if (nested_cpu_has_vid(vmcs12))
11273                 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
11274
11275         vmcs12->vm_entry_controls =
11276                 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
11277                 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
11278
11279         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) {
11280                 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
11281                 vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
11282         }
11283
11284         /* TODO: These cannot have changed unless we have MSR bitmaps and
11285          * the relevant bit asks not to trap the change */
11286         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
11287                 vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT);
11288         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
11289                 vmcs12->guest_ia32_efer = vcpu->arch.efer;
11290         vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
11291         vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
11292         vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
11293         if (kvm_mpx_supported())
11294                 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
11295 }
11296
11297 /*
11298  * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
11299  * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
11300  * and this function updates it to reflect the changes to the guest state while
11301  * L2 was running (and perhaps made some exits which were handled directly by L0
11302  * without going back to L1), and to reflect the exit reason.
11303  * Note that we do not have to copy here all VMCS fields, just those that
11304  * could have changed by the L2 guest or the exit - i.e., the guest-state and
11305  * exit-information fields only. Other fields are modified by L1 with VMWRITE,
11306  * which already writes to vmcs12 directly.
11307  */
11308 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
11309                            u32 exit_reason, u32 exit_intr_info,
11310                            unsigned long exit_qualification)
11311 {
11312         /* update guest state fields: */
11313         sync_vmcs12(vcpu, vmcs12);
11314
11315         /* update exit information fields: */
11316
11317         vmcs12->vm_exit_reason = exit_reason;
11318         vmcs12->exit_qualification = exit_qualification;
11319         vmcs12->vm_exit_intr_info = exit_intr_info;
11320
11321         vmcs12->idt_vectoring_info_field = 0;
11322         vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
11323         vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
11324
11325         if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
11326                 vmcs12->launch_state = 1;
11327
11328                 /* vm_entry_intr_info_field is cleared on exit. Emulate this
11329                  * instead of reading the real value. */
11330                 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
11331
11332                 /*
11333                  * Transfer the event that L0 or L1 may wanted to inject into
11334                  * L2 to IDT_VECTORING_INFO_FIELD.
11335                  */
11336                 vmcs12_save_pending_event(vcpu, vmcs12);
11337         }
11338
11339         /*
11340          * Drop what we picked up for L2 via vmx_complete_interrupts. It is
11341          * preserved above and would only end up incorrectly in L1.
11342          */
11343         vcpu->arch.nmi_injected = false;
11344         kvm_clear_exception_queue(vcpu);
11345         kvm_clear_interrupt_queue(vcpu);
11346 }
11347
11348 static void load_vmcs12_mmu_host_state(struct kvm_vcpu *vcpu,
11349                         struct vmcs12 *vmcs12)
11350 {
11351         u32 entry_failure_code;
11352
11353         nested_ept_uninit_mmu_context(vcpu);
11354
11355         /*
11356          * Only PDPTE load can fail as the value of cr3 was checked on entry and
11357          * couldn't have changed.
11358          */
11359         if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code))
11360                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
11361
11362         if (!enable_ept)
11363                 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
11364 }
11365
11366 /*
11367  * A part of what we need to when the nested L2 guest exits and we want to
11368  * run its L1 parent, is to reset L1's guest state to the host state specified
11369  * in vmcs12.
11370  * This function is to be called not only on normal nested exit, but also on
11371  * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
11372  * Failures During or After Loading Guest State").
11373  * This function should be called when the active VMCS is L1's (vmcs01).
11374  */
11375 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
11376                                    struct vmcs12 *vmcs12)
11377 {
11378         struct kvm_segment seg;
11379
11380         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
11381                 vcpu->arch.efer = vmcs12->host_ia32_efer;
11382         else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
11383                 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
11384         else
11385                 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
11386         vmx_set_efer(vcpu, vcpu->arch.efer);
11387
11388         kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp);
11389         kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip);
11390         vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
11391         /*
11392          * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
11393          * actually changed, because vmx_set_cr0 refers to efer set above.
11394          *
11395          * CR0_GUEST_HOST_MASK is already set in the original vmcs01
11396          * (KVM doesn't change it);
11397          */
11398         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
11399         vmx_set_cr0(vcpu, vmcs12->host_cr0);
11400
11401         /* Same as above - no reason to call set_cr4_guest_host_mask().  */
11402         vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
11403         vmx_set_cr4(vcpu, vmcs12->host_cr4);
11404
11405         load_vmcs12_mmu_host_state(vcpu, vmcs12);
11406
11407         if (enable_vpid) {
11408                 /*
11409                  * Trivially support vpid by letting L2s share their parent
11410                  * L1's vpid. TODO: move to a more elaborate solution, giving
11411                  * each L2 its own vpid and exposing the vpid feature to L1.
11412                  */
11413                 vmx_flush_tlb(vcpu);
11414         }
11415         /* Restore posted intr vector. */
11416         if (nested_cpu_has_posted_intr(vmcs12))
11417                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
11418
11419         vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
11420         vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
11421         vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
11422         vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
11423         vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
11424         vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF);
11425         vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);
11426
11427         /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1.  */
11428         if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
11429                 vmcs_write64(GUEST_BNDCFGS, 0);
11430
11431         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
11432                 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
11433                 vcpu->arch.pat = vmcs12->host_ia32_pat;
11434         }
11435         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
11436                 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
11437                         vmcs12->host_ia32_perf_global_ctrl);
11438
11439         /* Set L1 segment info according to Intel SDM
11440             27.5.2 Loading Host Segment and Descriptor-Table Registers */
11441         seg = (struct kvm_segment) {
11442                 .base = 0,
11443                 .limit = 0xFFFFFFFF,
11444                 .selector = vmcs12->host_cs_selector,
11445                 .type = 11,
11446                 .present = 1,
11447                 .s = 1,
11448                 .g = 1
11449         };
11450         if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
11451                 seg.l = 1;
11452         else
11453                 seg.db = 1;
11454         vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
11455         seg = (struct kvm_segment) {
11456                 .base = 0,
11457                 .limit = 0xFFFFFFFF,
11458                 .type = 3,
11459                 .present = 1,
11460                 .s = 1,
11461                 .db = 1,
11462                 .g = 1
11463         };
11464         seg.selector = vmcs12->host_ds_selector;
11465         vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
11466         seg.selector = vmcs12->host_es_selector;
11467         vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
11468         seg.selector = vmcs12->host_ss_selector;
11469         vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
11470         seg.selector = vmcs12->host_fs_selector;
11471         seg.base = vmcs12->host_fs_base;
11472         vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
11473         seg.selector = vmcs12->host_gs_selector;
11474         seg.base = vmcs12->host_gs_base;
11475         vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
11476         seg = (struct kvm_segment) {
11477                 .base = vmcs12->host_tr_base,
11478                 .limit = 0x67,
11479                 .selector = vmcs12->host_tr_selector,
11480                 .type = 11,
11481                 .present = 1
11482         };
11483         vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
11484
11485         kvm_set_dr(vcpu, 7, 0x400);
11486         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
11487
11488         if (cpu_has_vmx_msr_bitmap())
11489                 vmx_update_msr_bitmap(vcpu);
11490
11491         if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
11492                                 vmcs12->vm_exit_msr_load_count))
11493                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
11494 }
11495
11496 /*
11497  * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
11498  * and modify vmcs12 to make it see what it would expect to see there if
11499  * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
11500  */
11501 static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
11502                               u32 exit_intr_info,
11503                               unsigned long exit_qualification)
11504 {
11505         struct vcpu_vmx *vmx = to_vmx(vcpu);
11506         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
11507
11508         /* trying to cancel vmlaunch/vmresume is a bug */
11509         WARN_ON_ONCE(vmx->nested.nested_run_pending);
11510
11511         /*
11512          * The only expected VM-instruction error is "VM entry with
11513          * invalid control field(s)." Anything else indicates a
11514          * problem with L0.
11515          */
11516         WARN_ON_ONCE(vmx->fail && (vmcs_read32(VM_INSTRUCTION_ERROR) !=
11517                                    VMXERR_ENTRY_INVALID_CONTROL_FIELD));
11518
11519         leave_guest_mode(vcpu);
11520
11521         if (likely(!vmx->fail)) {
11522                 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
11523                                exit_qualification);
11524
11525                 if (nested_vmx_store_msr(vcpu, vmcs12->vm_exit_msr_store_addr,
11526                                          vmcs12->vm_exit_msr_store_count))
11527                         nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL);
11528         }
11529
11530         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
11531         vm_entry_controls_reset_shadow(vmx);
11532         vm_exit_controls_reset_shadow(vmx);
11533         vmx_segment_cache_clear(vmx);
11534
11535         /* Update any VMCS fields that might have changed while L2 ran */
11536         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
11537         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
11538         vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
11539         if (vmx->hv_deadline_tsc == -1)
11540                 vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
11541                                 PIN_BASED_VMX_PREEMPTION_TIMER);
11542         else
11543                 vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
11544                               PIN_BASED_VMX_PREEMPTION_TIMER);
11545         if (kvm_has_tsc_control)
11546                 decache_tsc_multiplier(vmx);
11547
11548         if (vmx->nested.change_vmcs01_virtual_x2apic_mode) {
11549                 vmx->nested.change_vmcs01_virtual_x2apic_mode = false;
11550                 vmx_set_virtual_x2apic_mode(vcpu,
11551                                 vcpu->arch.apic_base & X2APIC_ENABLE);
11552         } else if (!nested_cpu_has_ept(vmcs12) &&
11553                    nested_cpu_has2(vmcs12,
11554                                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
11555                 vmx_flush_tlb_ept_only(vcpu);
11556         }
11557
11558         /* This is needed for same reason as it was needed in prepare_vmcs02 */
11559         vmx->host_rsp = 0;
11560
11561         /* Unpin physical memory we referred to in vmcs02 */
11562         if (vmx->nested.apic_access_page) {
11563                 kvm_release_page_dirty(vmx->nested.apic_access_page);
11564                 vmx->nested.apic_access_page = NULL;
11565         }
11566         if (vmx->nested.virtual_apic_page) {
11567                 kvm_release_page_dirty(vmx->nested.virtual_apic_page);
11568                 vmx->nested.virtual_apic_page = NULL;
11569         }
11570         if (vmx->nested.pi_desc_page) {
11571                 kunmap(vmx->nested.pi_desc_page);
11572                 kvm_release_page_dirty(vmx->nested.pi_desc_page);
11573                 vmx->nested.pi_desc_page = NULL;
11574                 vmx->nested.pi_desc = NULL;
11575         }
11576
11577         /*
11578          * We are now running in L2, mmu_notifier will force to reload the
11579          * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1.
11580          */
11581         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
11582
11583         if (enable_shadow_vmcs)
11584                 vmx->nested.sync_shadow_vmcs = true;
11585
11586         /* in case we halted in L2 */
11587         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11588
11589         if (likely(!vmx->fail)) {
11590                 /*
11591                  * TODO: SDM says that with acknowledge interrupt on
11592                  * exit, bit 31 of the VM-exit interrupt information
11593                  * (valid interrupt) is always set to 1 on
11594                  * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't
11595                  * need kvm_cpu_has_interrupt().  See the commit
11596                  * message for details.
11597                  */
11598                 if (nested_exit_intr_ack_set(vcpu) &&
11599                     exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT &&
11600                     kvm_cpu_has_interrupt(vcpu)) {
11601                         int irq = kvm_cpu_get_interrupt(vcpu);
11602                         WARN_ON(irq < 0);
11603                         vmcs12->vm_exit_intr_info = irq |
11604                                 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
11605                 }
11606
11607                 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
11608                                                vmcs12->exit_qualification,
11609                                                vmcs12->idt_vectoring_info_field,
11610                                                vmcs12->vm_exit_intr_info,
11611                                                vmcs12->vm_exit_intr_error_code,
11612                                                KVM_ISA_VMX);
11613
11614                 load_vmcs12_host_state(vcpu, vmcs12);
11615
11616                 return;
11617         }
11618
11619         /*
11620          * After an early L2 VM-entry failure, we're now back
11621          * in L1 which thinks it just finished a VMLAUNCH or
11622          * VMRESUME instruction, so we need to set the failure
11623          * flag and the VM-instruction error field of the VMCS
11624          * accordingly.
11625          */
11626         nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
11627
11628         load_vmcs12_mmu_host_state(vcpu, vmcs12);
11629
11630         /*
11631          * The emulated instruction was already skipped in
11632          * nested_vmx_run, but the updated RIP was never
11633          * written back to the vmcs01.
11634          */
11635         skip_emulated_instruction(vcpu);
11636         vmx->fail = 0;
11637 }
11638
11639 /*
11640  * Forcibly leave nested mode in order to be able to reset the VCPU later on.
11641  */
11642 static void vmx_leave_nested(struct kvm_vcpu *vcpu)
11643 {
11644         if (is_guest_mode(vcpu)) {
11645                 to_vmx(vcpu)->nested.nested_run_pending = 0;
11646                 nested_vmx_vmexit(vcpu, -1, 0, 0);
11647         }
11648         free_nested(to_vmx(vcpu));
11649 }
11650
11651 /*
11652  * L1's failure to enter L2 is a subset of a normal exit, as explained in
11653  * 23.7 "VM-entry failures during or after loading guest state" (this also
11654  * lists the acceptable exit-reason and exit-qualification parameters).
11655  * It should only be called before L2 actually succeeded to run, and when
11656  * vmcs01 is current (it doesn't leave_guest_mode() or switch vmcss).
11657  */
11658 static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
11659                         struct vmcs12 *vmcs12,
11660                         u32 reason, unsigned long qualification)
11661 {
11662         load_vmcs12_host_state(vcpu, vmcs12);
11663         vmcs12->vm_exit_reason = reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
11664         vmcs12->exit_qualification = qualification;
11665         nested_vmx_succeed(vcpu);
11666         if (enable_shadow_vmcs)
11667                 to_vmx(vcpu)->nested.sync_shadow_vmcs = true;
11668 }
11669
11670 static int vmx_check_intercept(struct kvm_vcpu *vcpu,
11671                                struct x86_instruction_info *info,
11672                                enum x86_intercept_stage stage)
11673 {
11674         return X86EMUL_CONTINUE;
11675 }
11676
11677 #ifdef CONFIG_X86_64
11678 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */
11679 static inline int u64_shl_div_u64(u64 a, unsigned int shift,
11680                                   u64 divisor, u64 *result)
11681 {
11682         u64 low = a << shift, high = a >> (64 - shift);
11683
11684         /* To avoid the overflow on divq */
11685         if (high >= divisor)
11686                 return 1;
11687
11688         /* Low hold the result, high hold rem which is discarded */
11689         asm("divq %2\n\t" : "=a" (low), "=d" (high) :
11690             "rm" (divisor), "0" (low), "1" (high));
11691         *result = low;
11692
11693         return 0;
11694 }
11695
11696 static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc)
11697 {
11698         struct vcpu_vmx *vmx = to_vmx(vcpu);
11699         u64 tscl = rdtsc();
11700         u64 guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
11701         u64 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
11702
11703         /* Convert to host delta tsc if tsc scaling is enabled */
11704         if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
11705                         u64_shl_div_u64(delta_tsc,
11706                                 kvm_tsc_scaling_ratio_frac_bits,
11707                                 vcpu->arch.tsc_scaling_ratio,
11708                                 &delta_tsc))
11709                 return -ERANGE;
11710
11711         /*
11712          * If the delta tsc can't fit in the 32 bit after the multi shift,
11713          * we can't use the preemption timer.
11714          * It's possible that it fits on later vmentries, but checking
11715          * on every vmentry is costly so we just use an hrtimer.
11716          */
11717         if (delta_tsc >> (cpu_preemption_timer_multi + 32))
11718                 return -ERANGE;
11719
11720         vmx->hv_deadline_tsc = tscl + delta_tsc;
11721         vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
11722                         PIN_BASED_VMX_PREEMPTION_TIMER);
11723
11724         return delta_tsc == 0;
11725 }
11726
11727 static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
11728 {
11729         struct vcpu_vmx *vmx = to_vmx(vcpu);
11730         vmx->hv_deadline_tsc = -1;
11731         vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
11732                         PIN_BASED_VMX_PREEMPTION_TIMER);
11733 }
11734 #endif
11735
11736 static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
11737 {
11738         if (ple_gap)
11739                 shrink_ple_window(vcpu);
11740 }
11741
11742 static void vmx_slot_enable_log_dirty(struct kvm *kvm,
11743                                      struct kvm_memory_slot *slot)
11744 {
11745         kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
11746         kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
11747 }
11748
11749 static void vmx_slot_disable_log_dirty(struct kvm *kvm,
11750                                        struct kvm_memory_slot *slot)
11751 {
11752         kvm_mmu_slot_set_dirty(kvm, slot);
11753 }
11754
11755 static void vmx_flush_log_dirty(struct kvm *kvm)
11756 {
11757         kvm_flush_pml_buffers(kvm);
11758 }
11759
11760 static int vmx_write_pml_buffer(struct kvm_vcpu *vcpu)
11761 {
11762         struct vmcs12 *vmcs12;
11763         struct vcpu_vmx *vmx = to_vmx(vcpu);
11764         gpa_t gpa;
11765         struct page *page = NULL;
11766         u64 *pml_address;
11767
11768         if (is_guest_mode(vcpu)) {
11769                 WARN_ON_ONCE(vmx->nested.pml_full);
11770
11771                 /*
11772                  * Check if PML is enabled for the nested guest.
11773                  * Whether eptp bit 6 is set is already checked
11774                  * as part of A/D emulation.
11775                  */
11776                 vmcs12 = get_vmcs12(vcpu);
11777                 if (!nested_cpu_has_pml(vmcs12))
11778                         return 0;
11779
11780                 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) {
11781                         vmx->nested.pml_full = true;
11782                         return 1;
11783                 }
11784
11785                 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS) & ~0xFFFull;
11786
11787                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->pml_address);
11788                 if (is_error_page(page))
11789                         return 0;
11790
11791                 pml_address = kmap(page);
11792                 pml_address[vmcs12->guest_pml_index--] = gpa;
11793                 kunmap(page);
11794                 kvm_release_page_clean(page);
11795         }
11796
11797         return 0;
11798 }
11799
11800 static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
11801                                            struct kvm_memory_slot *memslot,
11802                                            gfn_t offset, unsigned long mask)
11803 {
11804         kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
11805 }
11806
11807 static void __pi_post_block(struct kvm_vcpu *vcpu)
11808 {
11809         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
11810         struct pi_desc old, new;
11811         unsigned int dest;
11812
11813         do {
11814                 old.control = new.control = pi_desc->control;
11815                 WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR,
11816                      "Wakeup handler not enabled while the VCPU is blocked\n");
11817
11818                 dest = cpu_physical_id(vcpu->cpu);
11819
11820                 if (x2apic_enabled())
11821                         new.ndst = dest;
11822                 else
11823                         new.ndst = (dest << 8) & 0xFF00;
11824
11825                 /* set 'NV' to 'notification vector' */
11826                 new.nv = POSTED_INTR_VECTOR;
11827         } while (cmpxchg64(&pi_desc->control, old.control,
11828                            new.control) != old.control);
11829
11830         if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) {
11831                 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
11832                 list_del(&vcpu->blocked_vcpu_list);
11833                 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
11834                 vcpu->pre_pcpu = -1;
11835         }
11836 }
11837
11838 /*
11839  * This routine does the following things for vCPU which is going
11840  * to be blocked if VT-d PI is enabled.
11841  * - Store the vCPU to the wakeup list, so when interrupts happen
11842  *   we can find the right vCPU to wake up.
11843  * - Change the Posted-interrupt descriptor as below:
11844  *      'NDST' <-- vcpu->pre_pcpu
11845  *      'NV' <-- POSTED_INTR_WAKEUP_VECTOR
11846  * - If 'ON' is set during this process, which means at least one
11847  *   interrupt is posted for this vCPU, we cannot block it, in
11848  *   this case, return 1, otherwise, return 0.
11849  *
11850  */
11851 static int pi_pre_block(struct kvm_vcpu *vcpu)
11852 {
11853         unsigned int dest;
11854         struct pi_desc old, new;
11855         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
11856
11857         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
11858                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
11859                 !kvm_vcpu_apicv_active(vcpu))
11860                 return 0;
11861
11862         WARN_ON(irqs_disabled());
11863         local_irq_disable();
11864         if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) {
11865                 vcpu->pre_pcpu = vcpu->cpu;
11866                 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
11867                 list_add_tail(&vcpu->blocked_vcpu_list,
11868                               &per_cpu(blocked_vcpu_on_cpu,
11869                                        vcpu->pre_pcpu));
11870                 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
11871         }
11872
11873         do {
11874                 old.control = new.control = pi_desc->control;
11875
11876                 WARN((pi_desc->sn == 1),
11877                      "Warning: SN field of posted-interrupts "
11878                      "is set before blocking\n");
11879
11880                 /*
11881                  * Since vCPU can be preempted during this process,
11882                  * vcpu->cpu could be different with pre_pcpu, we
11883                  * need to set pre_pcpu as the destination of wakeup
11884                  * notification event, then we can find the right vCPU
11885                  * to wakeup in wakeup handler if interrupts happen
11886                  * when the vCPU is in blocked state.
11887                  */
11888                 dest = cpu_physical_id(vcpu->pre_pcpu);
11889
11890                 if (x2apic_enabled())
11891                         new.ndst = dest;
11892                 else
11893                         new.ndst = (dest << 8) & 0xFF00;
11894
11895                 /* set 'NV' to 'wakeup vector' */
11896                 new.nv = POSTED_INTR_WAKEUP_VECTOR;
11897         } while (cmpxchg64(&pi_desc->control, old.control,
11898                            new.control) != old.control);
11899
11900         /* We should not block the vCPU if an interrupt is posted for it.  */
11901         if (pi_test_on(pi_desc) == 1)
11902                 __pi_post_block(vcpu);
11903
11904         local_irq_enable();
11905         return (vcpu->pre_pcpu == -1);
11906 }
11907
11908 static int vmx_pre_block(struct kvm_vcpu *vcpu)
11909 {
11910         if (pi_pre_block(vcpu))
11911                 return 1;
11912
11913         if (kvm_lapic_hv_timer_in_use(vcpu))
11914                 kvm_lapic_switch_to_sw_timer(vcpu);
11915
11916         return 0;
11917 }
11918
11919 static void pi_post_block(struct kvm_vcpu *vcpu)
11920 {
11921         if (vcpu->pre_pcpu == -1)
11922                 return;
11923
11924         WARN_ON(irqs_disabled());
11925         local_irq_disable();
11926         __pi_post_block(vcpu);
11927         local_irq_enable();
11928 }
11929
11930 static void vmx_post_block(struct kvm_vcpu *vcpu)
11931 {
11932         if (kvm_x86_ops->set_hv_timer)
11933                 kvm_lapic_switch_to_hv_timer(vcpu);
11934
11935         pi_post_block(vcpu);
11936 }
11937
11938 /*
11939  * vmx_update_pi_irte - set IRTE for Posted-Interrupts
11940  *
11941  * @kvm: kvm
11942  * @host_irq: host irq of the interrupt
11943  * @guest_irq: gsi of the interrupt
11944  * @set: set or unset PI
11945  * returns 0 on success, < 0 on failure
11946  */
11947 static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
11948                               uint32_t guest_irq, bool set)
11949 {
11950         struct kvm_kernel_irq_routing_entry *e;
11951         struct kvm_irq_routing_table *irq_rt;
11952         struct kvm_lapic_irq irq;
11953         struct kvm_vcpu *vcpu;
11954         struct vcpu_data vcpu_info;
11955         int idx, ret = 0;
11956
11957         if (!kvm_arch_has_assigned_device(kvm) ||
11958                 !irq_remapping_cap(IRQ_POSTING_CAP) ||
11959                 !kvm_vcpu_apicv_active(kvm->vcpus[0]))
11960                 return 0;
11961
11962         idx = srcu_read_lock(&kvm->irq_srcu);
11963         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
11964         if (guest_irq >= irq_rt->nr_rt_entries ||
11965             hlist_empty(&irq_rt->map[guest_irq])) {
11966                 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
11967                              guest_irq, irq_rt->nr_rt_entries);
11968                 goto out;
11969         }
11970
11971         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
11972                 if (e->type != KVM_IRQ_ROUTING_MSI)
11973                         continue;
11974                 /*
11975                  * VT-d PI cannot support posting multicast/broadcast
11976                  * interrupts to a vCPU, we still use interrupt remapping
11977                  * for these kind of interrupts.
11978                  *
11979                  * For lowest-priority interrupts, we only support
11980                  * those with single CPU as the destination, e.g. user
11981                  * configures the interrupts via /proc/irq or uses
11982                  * irqbalance to make the interrupts single-CPU.
11983                  *
11984                  * We will support full lowest-priority interrupt later.
11985                  */
11986
11987                 kvm_set_msi_irq(kvm, e, &irq);
11988                 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
11989                         /*
11990                          * Make sure the IRTE is in remapped mode if
11991                          * we don't handle it in posted mode.
11992                          */
11993                         ret = irq_set_vcpu_affinity(host_irq, NULL);
11994                         if (ret < 0) {
11995                                 printk(KERN_INFO
11996                                    "failed to back to remapped mode, irq: %u\n",
11997                                    host_irq);
11998                                 goto out;
11999                         }
12000
12001                         continue;
12002                 }
12003
12004                 vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
12005                 vcpu_info.vector = irq.vector;
12006
12007                 trace_kvm_pi_irte_update(vcpu->vcpu_id, host_irq, e->gsi,
12008                                 vcpu_info.vector, vcpu_info.pi_desc_addr, set);
12009
12010                 if (set)
12011                         ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
12012                 else
12013                         ret = irq_set_vcpu_affinity(host_irq, NULL);
12014
12015                 if (ret < 0) {
12016                         printk(KERN_INFO "%s: failed to update PI IRTE\n",
12017                                         __func__);
12018                         goto out;
12019                 }
12020         }
12021
12022         ret = 0;
12023 out:
12024         srcu_read_unlock(&kvm->irq_srcu, idx);
12025         return ret;
12026 }
12027
12028 static void vmx_setup_mce(struct kvm_vcpu *vcpu)
12029 {
12030         if (vcpu->arch.mcg_cap & MCG_LMCE_P)
12031                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
12032                         FEATURE_CONTROL_LMCE;
12033         else
12034                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
12035                         ~FEATURE_CONTROL_LMCE;
12036 }
12037
12038 static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
12039         .cpu_has_kvm_support = cpu_has_kvm_support,
12040         .disabled_by_bios = vmx_disabled_by_bios,
12041         .hardware_setup = hardware_setup,
12042         .hardware_unsetup = hardware_unsetup,
12043         .check_processor_compatibility = vmx_check_processor_compat,
12044         .hardware_enable = hardware_enable,
12045         .hardware_disable = hardware_disable,
12046         .cpu_has_accelerated_tpr = report_flexpriority,
12047         .cpu_has_high_real_mode_segbase = vmx_has_high_real_mode_segbase,
12048
12049         .vcpu_create = vmx_create_vcpu,
12050         .vcpu_free = vmx_free_vcpu,
12051         .vcpu_reset = vmx_vcpu_reset,
12052
12053         .prepare_guest_switch = vmx_save_host_state,
12054         .vcpu_load = vmx_vcpu_load,
12055         .vcpu_put = vmx_vcpu_put,
12056
12057         .update_bp_intercept = update_exception_bitmap,
12058         .get_msr = vmx_get_msr,
12059         .set_msr = vmx_set_msr,
12060         .get_segment_base = vmx_get_segment_base,
12061         .get_segment = vmx_get_segment,
12062         .set_segment = vmx_set_segment,
12063         .get_cpl = vmx_get_cpl,
12064         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
12065         .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
12066         .decache_cr3 = vmx_decache_cr3,
12067         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
12068         .set_cr0 = vmx_set_cr0,
12069         .set_cr3 = vmx_set_cr3,
12070         .set_cr4 = vmx_set_cr4,
12071         .set_efer = vmx_set_efer,
12072         .get_idt = vmx_get_idt,
12073         .set_idt = vmx_set_idt,
12074         .get_gdt = vmx_get_gdt,
12075         .set_gdt = vmx_set_gdt,
12076         .get_dr6 = vmx_get_dr6,
12077         .set_dr6 = vmx_set_dr6,
12078         .set_dr7 = vmx_set_dr7,
12079         .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
12080         .cache_reg = vmx_cache_reg,
12081         .get_rflags = vmx_get_rflags,
12082         .set_rflags = vmx_set_rflags,
12083
12084         .tlb_flush = vmx_flush_tlb,
12085
12086         .run = vmx_vcpu_run,
12087         .handle_exit = vmx_handle_exit,
12088         .skip_emulated_instruction = skip_emulated_instruction,
12089         .set_interrupt_shadow = vmx_set_interrupt_shadow,
12090         .get_interrupt_shadow = vmx_get_interrupt_shadow,
12091         .patch_hypercall = vmx_patch_hypercall,
12092         .set_irq = vmx_inject_irq,
12093         .set_nmi = vmx_inject_nmi,
12094         .queue_exception = vmx_queue_exception,
12095         .cancel_injection = vmx_cancel_injection,
12096         .interrupt_allowed = vmx_interrupt_allowed,
12097         .nmi_allowed = vmx_nmi_allowed,
12098         .get_nmi_mask = vmx_get_nmi_mask,
12099         .set_nmi_mask = vmx_set_nmi_mask,
12100         .enable_nmi_window = enable_nmi_window,
12101         .enable_irq_window = enable_irq_window,
12102         .update_cr8_intercept = update_cr8_intercept,
12103         .set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
12104         .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
12105         .get_enable_apicv = vmx_get_enable_apicv,
12106         .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
12107         .load_eoi_exitmap = vmx_load_eoi_exitmap,
12108         .apicv_post_state_restore = vmx_apicv_post_state_restore,
12109         .hwapic_irr_update = vmx_hwapic_irr_update,
12110         .hwapic_isr_update = vmx_hwapic_isr_update,
12111         .sync_pir_to_irr = vmx_sync_pir_to_irr,
12112         .deliver_posted_interrupt = vmx_deliver_posted_interrupt,
12113
12114         .set_tss_addr = vmx_set_tss_addr,
12115         .get_tdp_level = get_ept_level,
12116         .get_mt_mask = vmx_get_mt_mask,
12117
12118         .get_exit_info = vmx_get_exit_info,
12119
12120         .get_lpage_level = vmx_get_lpage_level,
12121
12122         .cpuid_update = vmx_cpuid_update,
12123
12124         .rdtscp_supported = vmx_rdtscp_supported,
12125         .invpcid_supported = vmx_invpcid_supported,
12126
12127         .set_supported_cpuid = vmx_set_supported_cpuid,
12128
12129         .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
12130
12131         .write_tsc_offset = vmx_write_tsc_offset,
12132
12133         .set_tdp_cr3 = vmx_set_cr3,
12134
12135         .check_intercept = vmx_check_intercept,
12136         .handle_external_intr = vmx_handle_external_intr,
12137         .mpx_supported = vmx_mpx_supported,
12138         .xsaves_supported = vmx_xsaves_supported,
12139
12140         .check_nested_events = vmx_check_nested_events,
12141
12142         .sched_in = vmx_sched_in,
12143
12144         .slot_enable_log_dirty = vmx_slot_enable_log_dirty,
12145         .slot_disable_log_dirty = vmx_slot_disable_log_dirty,
12146         .flush_log_dirty = vmx_flush_log_dirty,
12147         .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
12148         .write_log_dirty = vmx_write_pml_buffer,
12149
12150         .pre_block = vmx_pre_block,
12151         .post_block = vmx_post_block,
12152
12153         .pmu_ops = &intel_pmu_ops,
12154
12155         .update_pi_irte = vmx_update_pi_irte,
12156
12157 #ifdef CONFIG_X86_64
12158         .set_hv_timer = vmx_set_hv_timer,
12159         .cancel_hv_timer = vmx_cancel_hv_timer,
12160 #endif
12161
12162         .setup_mce = vmx_setup_mce,
12163 };
12164
12165 static int __init vmx_init(void)
12166 {
12167         int r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
12168                      __alignof__(struct vcpu_vmx), THIS_MODULE);
12169         if (r)
12170                 return r;
12171
12172 #ifdef CONFIG_KEXEC_CORE
12173         rcu_assign_pointer(crash_vmclear_loaded_vmcss,
12174                            crash_vmclear_local_loaded_vmcss);
12175 #endif
12176
12177         return 0;
12178 }
12179
12180 static void __exit vmx_exit(void)
12181 {
12182 #ifdef CONFIG_KEXEC_CORE
12183         RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
12184         synchronize_rcu();
12185 #endif
12186
12187         kvm_exit();
12188 }
12189
12190 module_init(vmx_init)
12191 module_exit(vmx_exit)