arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/cpu.h>
  13 #include <linux/module.h>
  14 #include <linux/nospec.h>
  15 #include <linux/prctl.h>
  16 #include <linux/sched/smt.h>
  17 #include <linux/pgtable.h>
  18 #include <linux/bpf.h>
  19
  20 #include <asm/spec-ctrl.h>
  21 #include <asm/cmdline.h>
  22 #include <asm/bugs.h>
  23 #include <asm/processor.h>
  24 #include <asm/processor-flags.h>
  25 #include <asm/fpu/api.h>
  26 #include <asm/msr.h>
  27 #include <asm/vmx.h>
  28 #include <asm/paravirt.h>
  29 #include <asm/intel-family.h>
  30 #include <asm/e820/api.h>
  31 #include <asm/hypervisor.h>
  32 #include <asm/tlbflush.h>
  33
  34 #include "cpu.h"
  35
  36 static void __init spectre_v1_select_mitigation(void);
  37 static void __init spectre_v2_select_mitigation(void);
  38 static void __init retbleed_select_mitigation(void);
  39 static void __init spectre_v2_user_select_mitigation(void);
  40 static void __init ssb_select_mitigation(void);
  41 static void __init l1tf_select_mitigation(void);
  42 static void __init mds_select_mitigation(void);
  43 static void __init md_clear_update_mitigation(void);
  44 static void __init md_clear_select_mitigation(void);
  45 static void __init taa_select_mitigation(void);
  46 static void __init mmio_select_mitigation(void);
  47 static void __init srbds_select_mitigation(void);
  48 static void __init l1d_flush_select_mitigation(void);
  49 static void __init gds_select_mitigation(void);
  50 static void __init srso_select_mitigation(void);
  51
  52 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
  53 u64 x86_spec_ctrl_base;
  54 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  55
  56 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
  57 DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
  58 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current);
  59
  60 static DEFINE_MUTEX(spec_ctrl_mutex);
  61
  62 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
  63 static void update_spec_ctrl(u64 val)
  64 {
  65         this_cpu_write(x86_spec_ctrl_current, val);
  66         wrmsrl(MSR_IA32_SPEC_CTRL, val);
  67 }
  68
  69 /*
  70  * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
  71  * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
  72  */
  73 void update_spec_ctrl_cond(u64 val)
  74 {
  75         if (this_cpu_read(x86_spec_ctrl_current) == val)
  76                 return;
  77
  78         this_cpu_write(x86_spec_ctrl_current, val);
  79
  80         /*
  81          * When KERNEL_IBRS this MSR is written on return-to-user, unless
  82          * forced the update can be delayed until that time.
  83          */
  84         if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS))
  85                 wrmsrl(MSR_IA32_SPEC_CTRL, val);
  86 }
  87
  88 u64 spec_ctrl_current(void)
  89 {
  90         return this_cpu_read(x86_spec_ctrl_current);
  91 }
  92 EXPORT_SYMBOL_GPL(spec_ctrl_current);
  93
  94 /*
  95  * AMD specific MSR info for Speculative Store Bypass control.
  96  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  97  */
  98 u64 __ro_after_init x86_amd_ls_cfg_base;
  99 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
 100
 101 /* Control conditional STIBP in switch_to() */
 102 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
 103 /* Control conditional IBPB in switch_mm() */
 104 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 105 /* Control unconditional IBPB in switch_mm() */
 106 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 107
 108 /* Control MDS CPU buffer clear before returning to user space */
 109 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
 110 EXPORT_SYMBOL_GPL(mds_user_clear);
 111 /* Control MDS CPU buffer clear before idling (halt, mwait) */
 112 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
 113 EXPORT_SYMBOL_GPL(mds_idle_clear);
 114
 115 /*
 116  * Controls whether l1d flush based mitigations are enabled,
 117  * based on hw features and admin setting via boot parameter
 118  * defaults to false
 119  */
 120 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
 121
 122 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */
 123 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear);
 124 EXPORT_SYMBOL_GPL(mmio_stale_data_clear);
 125
 126 void __init cpu_select_mitigations(void)
 127 {
 128         /*
 129          * Read the SPEC_CTRL MSR to account for reserved bits which may
 130          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
 131          * init code as it is not enumerated and depends on the family.
 132          */
 133         if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) {
 134                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 135
 136                 /*
 137                  * Previously running kernel (kexec), may have some controls
 138                  * turned ON. Clear them and let the mitigations setup below
 139                  * rediscover them based on configuration.
 140                  */
 141                 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK;
 142         }
 143
 144         /* Select the proper CPU mitigations before patching alternatives: */
 145         spectre_v1_select_mitigation();
 146         spectre_v2_select_mitigation();
 147         /*
 148          * retbleed_select_mitigation() relies on the state set by
 149          * spectre_v2_select_mitigation(); specifically it wants to know about
 150          * spectre_v2=ibrs.
 151          */
 152         retbleed_select_mitigation();
 153         /*
 154          * spectre_v2_user_select_mitigation() relies on the state set by
 155          * retbleed_select_mitigation(); specifically the STIBP selection is
 156          * forced for UNRET or IBPB.
 157          */
 158         spectre_v2_user_select_mitigation();
 159         ssb_select_mitigation();
 160         l1tf_select_mitigation();
 161         md_clear_select_mitigation();
 162         srbds_select_mitigation();
 163         l1d_flush_select_mitigation();
 164         gds_select_mitigation();
 165         srso_select_mitigation();
 166 }
 167
 168 /*
 169  * NOTE: This function is *only* called for SVM, since Intel uses
 170  * MSR_IA32_SPEC_CTRL for SSBD.
 171  */
 172 void
 173 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest)
 174 {
 175         u64 guestval, hostval;
 176         struct thread_info *ti = current_thread_info();
 177
 178         /*
 179          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 180          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 181          */
 182         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 183             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 184                 return;
 185
 186         /*
 187          * If the host has SSBD mitigation enabled, force it in the host's
 188          * virtual MSR value. If its not permanently enabled, evaluate
 189          * current's TIF_SSBD thread flag.
 190          */
 191         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 192                 hostval = SPEC_CTRL_SSBD;
 193         else
 194                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 195
 196         /* Sanitize the guest value */
 197         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 198
 199         if (hostval != guestval) {
 200                 unsigned long tif;
 201
 202                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 203                                  ssbd_spec_ctrl_to_tif(hostval);
 204
 205                 speculation_ctrl_update(tif);
 206         }
 207 }
 208 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 209
 210 static void x86_amd_ssb_disable(void)
 211 {
 212         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 213
 214         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 215                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 216         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 217                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 218 }
 219
 220 #undef pr_fmt
 221 #define pr_fmt(fmt)     "MDS: " fmt
 222
 223 /* Default mitigation for MDS-affected CPUs */
 224 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 225 static bool mds_nosmt __ro_after_init = false;
 226
 227 static const char * const mds_strings[] = {
 228         [MDS_MITIGATION_OFF]    = "Vulnerable",
 229         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 230         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 231 };
 232
 233 static void __init mds_select_mitigation(void)
 234 {
 235         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
 236                 mds_mitigation = MDS_MITIGATION_OFF;
 237                 return;
 238         }
 239
 240         if (mds_mitigation == MDS_MITIGATION_FULL) {
 241                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 242                         mds_mitigation = MDS_MITIGATION_VMWERV;
 243
 244                 static_branch_enable(&mds_user_clear);
 245
 246                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 247                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
 248                         cpu_smt_disable(false);
 249         }
 250 }
 251
 252 static int __init mds_cmdline(char *str)
 253 {
 254         if (!boot_cpu_has_bug(X86_BUG_MDS))
 255                 return 0;
 256
 257         if (!str)
 258                 return -EINVAL;
 259
 260         if (!strcmp(str, "off"))
 261                 mds_mitigation = MDS_MITIGATION_OFF;
 262         else if (!strcmp(str, "full"))
 263                 mds_mitigation = MDS_MITIGATION_FULL;
 264         else if (!strcmp(str, "full,nosmt")) {
 265                 mds_mitigation = MDS_MITIGATION_FULL;
 266                 mds_nosmt = true;
 267         }
 268
 269         return 0;
 270 }
 271 early_param("mds", mds_cmdline);
 272
 273 #undef pr_fmt
 274 #define pr_fmt(fmt)     "TAA: " fmt
 275
 276 enum taa_mitigations {
 277         TAA_MITIGATION_OFF,
 278         TAA_MITIGATION_UCODE_NEEDED,
 279         TAA_MITIGATION_VERW,
 280         TAA_MITIGATION_TSX_DISABLED,
 281 };
 282
 283 /* Default mitigation for TAA-affected CPUs */
 284 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
 285 static bool taa_nosmt __ro_after_init;
 286
 287 static const char * const taa_strings[] = {
 288         [TAA_MITIGATION_OFF]            = "Vulnerable",
 289         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
 290         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
 291         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
 292 };
 293
 294 static void __init taa_select_mitigation(void)
 295 {
 296         u64 ia32_cap;
 297
 298         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
 299                 taa_mitigation = TAA_MITIGATION_OFF;
 300                 return;
 301         }
 302
 303         /* TSX previously disabled by tsx=off */
 304         if (!boot_cpu_has(X86_FEATURE_RTM)) {
 305                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
 306                 return;
 307         }
 308
 309         if (cpu_mitigations_off()) {
 310                 taa_mitigation = TAA_MITIGATION_OFF;
 311                 return;
 312         }
 313
 314         /*
 315          * TAA mitigation via VERW is turned off if both
 316          * tsx_async_abort=off and mds=off are specified.
 317          */
 318         if (taa_mitigation == TAA_MITIGATION_OFF &&
 319             mds_mitigation == MDS_MITIGATION_OFF)
 320                 return;
 321
 322         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 323                 taa_mitigation = TAA_MITIGATION_VERW;
 324         else
 325                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 326
 327         /*
 328          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
 329          * A microcode update fixes this behavior to clear CPU buffers. It also
 330          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
 331          * ARCH_CAP_TSX_CTRL_MSR bit.
 332          *
 333          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
 334          * update is required.
 335          */
 336         ia32_cap = x86_read_arch_cap_msr();
 337         if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
 338             !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
 339                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 340
 341         /*
 342          * TSX is enabled, select alternate mitigation for TAA which is
 343          * the same as MDS. Enable MDS static branch to clear CPU buffers.
 344          *
 345          * For guests that can't determine whether the correct microcode is
 346          * present on host, enable the mitigation for UCODE_NEEDED as well.
 347          */
 348         static_branch_enable(&mds_user_clear);
 349
 350         if (taa_nosmt || cpu_mitigations_auto_nosmt())
 351                 cpu_smt_disable(false);
 352 }
 353
 354 static int __init tsx_async_abort_parse_cmdline(char *str)
 355 {
 356         if (!boot_cpu_has_bug(X86_BUG_TAA))
 357                 return 0;
 358
 359         if (!str)
 360                 return -EINVAL;
 361
 362         if (!strcmp(str, "off")) {
 363                 taa_mitigation = TAA_MITIGATION_OFF;
 364         } else if (!strcmp(str, "full")) {
 365                 taa_mitigation = TAA_MITIGATION_VERW;
 366         } else if (!strcmp(str, "full,nosmt")) {
 367                 taa_mitigation = TAA_MITIGATION_VERW;
 368                 taa_nosmt = true;
 369         }
 370
 371         return 0;
 372 }
 373 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
 374
 375 #undef pr_fmt
 376 #define pr_fmt(fmt)     "MMIO Stale Data: " fmt
 377
 378 enum mmio_mitigations {
 379         MMIO_MITIGATION_OFF,
 380         MMIO_MITIGATION_UCODE_NEEDED,
 381         MMIO_MITIGATION_VERW,
 382 };
 383
 384 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */
 385 static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW;
 386 static bool mmio_nosmt __ro_after_init = false;
 387
 388 static const char * const mmio_strings[] = {
 389         [MMIO_MITIGATION_OFF]           = "Vulnerable",
 390         [MMIO_MITIGATION_UCODE_NEEDED]  = "Vulnerable: Clear CPU buffers attempted, no microcode",
 391         [MMIO_MITIGATION_VERW]          = "Mitigation: Clear CPU buffers",
 392 };
 393
 394 static void __init mmio_select_mitigation(void)
 395 {
 396         u64 ia32_cap;
 397
 398         if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) ||
 399              boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) ||
 400              cpu_mitigations_off()) {
 401                 mmio_mitigation = MMIO_MITIGATION_OFF;
 402                 return;
 403         }
 404
 405         if (mmio_mitigation == MMIO_MITIGATION_OFF)
 406                 return;
 407
 408         ia32_cap = x86_read_arch_cap_msr();
 409
 410         /*
 411          * Enable CPU buffer clear mitigation for host and VMM, if also affected
 412          * by MDS or TAA. Otherwise, enable mitigation for VMM only.
 413          */
 414         if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) &&
 415                                               boot_cpu_has(X86_FEATURE_RTM)))
 416                 static_branch_enable(&mds_user_clear);
 417         else
 418                 static_branch_enable(&mmio_stale_data_clear);
 419
 420         /*
 421          * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
 422          * be propagated to uncore buffers, clearing the Fill buffers on idle
 423          * is required irrespective of SMT state.
 424          */
 425         if (!(ia32_cap & ARCH_CAP_FBSDP_NO))
 426                 static_branch_enable(&mds_idle_clear);
 427
 428         /*
 429          * Check if the system has the right microcode.
 430          *
 431          * CPU Fill buffer clear mitigation is enumerated by either an explicit
 432          * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
 433          * affected systems.
 434          */
 435         if ((ia32_cap & ARCH_CAP_FB_CLEAR) ||
 436             (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
 437              boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
 438              !(ia32_cap & ARCH_CAP_MDS_NO)))
 439                 mmio_mitigation = MMIO_MITIGATION_VERW;
 440         else
 441                 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
 442
 443         if (mmio_nosmt || cpu_mitigations_auto_nosmt())
 444                 cpu_smt_disable(false);
 445 }
 446
 447 static int __init mmio_stale_data_parse_cmdline(char *str)
 448 {
 449         if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 450                 return 0;
 451
 452         if (!str)
 453                 return -EINVAL;
 454
 455         if (!strcmp(str, "off")) {
 456                 mmio_mitigation = MMIO_MITIGATION_OFF;
 457         } else if (!strcmp(str, "full")) {
 458                 mmio_mitigation = MMIO_MITIGATION_VERW;
 459         } else if (!strcmp(str, "full,nosmt")) {
 460                 mmio_mitigation = MMIO_MITIGATION_VERW;
 461                 mmio_nosmt = true;
 462         }
 463
 464         return 0;
 465 }
 466 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline);
 467
 468 #undef pr_fmt
 469 #define pr_fmt(fmt)     "" fmt
 470
 471 static void __init md_clear_update_mitigation(void)
 472 {
 473         if (cpu_mitigations_off())
 474                 return;
 475
 476         if (!static_key_enabled(&mds_user_clear))
 477                 goto out;
 478
 479         /*
 480          * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data
 481          * mitigation, if necessary.
 482          */
 483         if (mds_mitigation == MDS_MITIGATION_OFF &&
 484             boot_cpu_has_bug(X86_BUG_MDS)) {
 485                 mds_mitigation = MDS_MITIGATION_FULL;
 486                 mds_select_mitigation();
 487         }
 488         if (taa_mitigation == TAA_MITIGATION_OFF &&
 489             boot_cpu_has_bug(X86_BUG_TAA)) {
 490                 taa_mitigation = TAA_MITIGATION_VERW;
 491                 taa_select_mitigation();
 492         }
 493         if (mmio_mitigation == MMIO_MITIGATION_OFF &&
 494             boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) {
 495                 mmio_mitigation = MMIO_MITIGATION_VERW;
 496                 mmio_select_mitigation();
 497         }
 498 out:
 499         if (boot_cpu_has_bug(X86_BUG_MDS))
 500                 pr_info("MDS: %s\n", mds_strings[mds_mitigation]);
 501         if (boot_cpu_has_bug(X86_BUG_TAA))
 502                 pr_info("TAA: %s\n", taa_strings[taa_mitigation]);
 503         if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 504                 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]);
 505         else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
 506                 pr_info("MMIO Stale Data: Unknown: No mitigations\n");
 507 }
 508
 509 static void __init md_clear_select_mitigation(void)
 510 {
 511         mds_select_mitigation();
 512         taa_select_mitigation();
 513         mmio_select_mitigation();
 514
 515         /*
 516          * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update
 517          * and print their mitigation after MDS, TAA and MMIO Stale Data
 518          * mitigation selection is done.
 519          */
 520         md_clear_update_mitigation();
 521 }
 522
 523 #undef pr_fmt
 524 #define pr_fmt(fmt)     "SRBDS: " fmt
 525
 526 enum srbds_mitigations {
 527         SRBDS_MITIGATION_OFF,
 528         SRBDS_MITIGATION_UCODE_NEEDED,
 529         SRBDS_MITIGATION_FULL,
 530         SRBDS_MITIGATION_TSX_OFF,
 531         SRBDS_MITIGATION_HYPERVISOR,
 532 };
 533
 534 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL;
 535
 536 static const char * const srbds_strings[] = {
 537         [SRBDS_MITIGATION_OFF]          = "Vulnerable",
 538         [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
 539         [SRBDS_MITIGATION_FULL]         = "Mitigation: Microcode",
 540         [SRBDS_MITIGATION_TSX_OFF]      = "Mitigation: TSX disabled",
 541         [SRBDS_MITIGATION_HYPERVISOR]   = "Unknown: Dependent on hypervisor status",
 542 };
 543
 544 static bool srbds_off;
 545
 546 void update_srbds_msr(void)
 547 {
 548         u64 mcu_ctrl;
 549
 550         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 551                 return;
 552
 553         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 554                 return;
 555
 556         if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
 557                 return;
 558
 559         /*
 560          * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX
 561          * being disabled and it hasn't received the SRBDS MSR microcode.
 562          */
 563         if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 564                 return;
 565
 566         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 567
 568         switch (srbds_mitigation) {
 569         case SRBDS_MITIGATION_OFF:
 570         case SRBDS_MITIGATION_TSX_OFF:
 571                 mcu_ctrl |= RNGDS_MITG_DIS;
 572                 break;
 573         case SRBDS_MITIGATION_FULL:
 574                 mcu_ctrl &= ~RNGDS_MITG_DIS;
 575                 break;
 576         default:
 577                 break;
 578         }
 579
 580         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 581 }
 582
 583 static void __init srbds_select_mitigation(void)
 584 {
 585         u64 ia32_cap;
 586
 587         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 588                 return;
 589
 590         /*
 591          * Check to see if this is one of the MDS_NO systems supporting TSX that
 592          * are only exposed to SRBDS when TSX is enabled or when CPU is affected
 593          * by Processor MMIO Stale Data vulnerability.
 594          */
 595         ia32_cap = x86_read_arch_cap_msr();
 596         if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) &&
 597             !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 598                 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
 599         else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 600                 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
 601         else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 602                 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
 603         else if (cpu_mitigations_off() || srbds_off)
 604                 srbds_mitigation = SRBDS_MITIGATION_OFF;
 605
 606         update_srbds_msr();
 607         pr_info("%s\n", srbds_strings[srbds_mitigation]);
 608 }
 609
 610 static int __init srbds_parse_cmdline(char *str)
 611 {
 612         if (!str)
 613                 return -EINVAL;
 614
 615         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 616                 return 0;
 617
 618         srbds_off = !strcmp(str, "off");
 619         return 0;
 620 }
 621 early_param("srbds", srbds_parse_cmdline);
 622
 623 #undef pr_fmt
 624 #define pr_fmt(fmt)     "L1D Flush : " fmt
 625
 626 enum l1d_flush_mitigations {
 627         L1D_FLUSH_OFF = 0,
 628         L1D_FLUSH_ON,
 629 };
 630
 631 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF;
 632
 633 static void __init l1d_flush_select_mitigation(void)
 634 {
 635         if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D))
 636                 return;
 637
 638         static_branch_enable(&switch_mm_cond_l1d_flush);
 639         pr_info("Conditional flush on switch_mm() enabled\n");
 640 }
 641
 642 static int __init l1d_flush_parse_cmdline(char *str)
 643 {
 644         if (!strcmp(str, "on"))
 645                 l1d_flush_mitigation = L1D_FLUSH_ON;
 646
 647         return 0;
 648 }
 649 early_param("l1d_flush", l1d_flush_parse_cmdline);
 650
 651 #undef pr_fmt
 652 #define pr_fmt(fmt)     "GDS: " fmt
 653
 654 enum gds_mitigations {
 655         GDS_MITIGATION_OFF,
 656         GDS_MITIGATION_UCODE_NEEDED,
 657         GDS_MITIGATION_FORCE,
 658         GDS_MITIGATION_FULL,
 659         GDS_MITIGATION_FULL_LOCKED,
 660         GDS_MITIGATION_HYPERVISOR,
 661 };
 662
 663 #if IS_ENABLED(CONFIG_GDS_FORCE_MITIGATION)
 664 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FORCE;
 665 #else
 666 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FULL;
 667 #endif
 668
 669 static const char * const gds_strings[] = {
 670         [GDS_MITIGATION_OFF]            = "Vulnerable",
 671         [GDS_MITIGATION_UCODE_NEEDED]   = "Vulnerable: No microcode",
 672         [GDS_MITIGATION_FORCE]          = "Mitigation: AVX disabled, no microcode",
 673         [GDS_MITIGATION_FULL]           = "Mitigation: Microcode",
 674         [GDS_MITIGATION_FULL_LOCKED]    = "Mitigation: Microcode (locked)",
 675         [GDS_MITIGATION_HYPERVISOR]     = "Unknown: Dependent on hypervisor status",
 676 };
 677
 678 bool gds_ucode_mitigated(void)
 679 {
 680         return (gds_mitigation == GDS_MITIGATION_FULL ||
 681                 gds_mitigation == GDS_MITIGATION_FULL_LOCKED);
 682 }
 683 EXPORT_SYMBOL_GPL(gds_ucode_mitigated);
 684
 685 void update_gds_msr(void)
 686 {
 687         u64 mcu_ctrl_after;
 688         u64 mcu_ctrl;
 689
 690         switch (gds_mitigation) {
 691         case GDS_MITIGATION_OFF:
 692                 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 693                 mcu_ctrl |= GDS_MITG_DIS;
 694                 break;
 695         case GDS_MITIGATION_FULL_LOCKED:
 696                 /*
 697                  * The LOCKED state comes from the boot CPU. APs might not have
 698                  * the same state. Make sure the mitigation is enabled on all
 699                  * CPUs.
 700                  */
 701         case GDS_MITIGATION_FULL:
 702                 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 703                 mcu_ctrl &= ~GDS_MITG_DIS;
 704                 break;
 705         case GDS_MITIGATION_FORCE:
 706         case GDS_MITIGATION_UCODE_NEEDED:
 707         case GDS_MITIGATION_HYPERVISOR:
 708                 return;
 709         };
 710
 711         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 712
 713         /*
 714          * Check to make sure that the WRMSR value was not ignored. Writes to
 715          * GDS_MITG_DIS will be ignored if this processor is locked but the boot
 716          * processor was not.
 717          */
 718         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after);
 719         WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after);
 720 }
 721
 722 static void __init gds_select_mitigation(void)
 723 {
 724         u64 mcu_ctrl;
 725
 726         if (!boot_cpu_has_bug(X86_BUG_GDS))
 727                 return;
 728
 729         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
 730                 gds_mitigation = GDS_MITIGATION_HYPERVISOR;
 731                 goto out;
 732         }
 733
 734         if (cpu_mitigations_off())
 735                 gds_mitigation = GDS_MITIGATION_OFF;
 736         /* Will verify below that mitigation _can_ be disabled */
 737
 738         /* No microcode */
 739         if (!(x86_read_arch_cap_msr() & ARCH_CAP_GDS_CTRL)) {
 740                 if (gds_mitigation == GDS_MITIGATION_FORCE) {
 741                         /*
 742                          * This only needs to be done on the boot CPU so do it
 743                          * here rather than in update_gds_msr()
 744                          */
 745                         setup_clear_cpu_cap(X86_FEATURE_AVX);
 746                         pr_warn("Microcode update needed! Disabling AVX as mitigation.\n");
 747                 } else {
 748                         gds_mitigation = GDS_MITIGATION_UCODE_NEEDED;
 749                 }
 750                 goto out;
 751         }
 752
 753         /* Microcode has mitigation, use it */
 754         if (gds_mitigation == GDS_MITIGATION_FORCE)
 755                 gds_mitigation = GDS_MITIGATION_FULL;
 756
 757         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 758         if (mcu_ctrl & GDS_MITG_LOCKED) {
 759                 if (gds_mitigation == GDS_MITIGATION_OFF)
 760                         pr_warn("Mitigation locked. Disable failed.\n");
 761
 762                 /*
 763                  * The mitigation is selected from the boot CPU. All other CPUs
 764                  * _should_ have the same state. If the boot CPU isn't locked
 765                  * but others are then update_gds_msr() will WARN() of the state
 766                  * mismatch. If the boot CPU is locked update_gds_msr() will
 767                  * ensure the other CPUs have the mitigation enabled.
 768                  */
 769                 gds_mitigation = GDS_MITIGATION_FULL_LOCKED;
 770         }
 771
 772         update_gds_msr();
 773 out:
 774         pr_info("%s\n", gds_strings[gds_mitigation]);
 775 }
 776
 777 static int __init gds_parse_cmdline(char *str)
 778 {
 779         if (!str)
 780                 return -EINVAL;
 781
 782         if (!boot_cpu_has_bug(X86_BUG_GDS))
 783                 return 0;
 784
 785         if (!strcmp(str, "off"))
 786                 gds_mitigation = GDS_MITIGATION_OFF;
 787         else if (!strcmp(str, "force"))
 788                 gds_mitigation = GDS_MITIGATION_FORCE;
 789
 790         return 0;
 791 }
 792 early_param("gather_data_sampling", gds_parse_cmdline);
 793
 794 #undef pr_fmt
 795 #define pr_fmt(fmt)     "Spectre V1 : " fmt
 796
 797 enum spectre_v1_mitigation {
 798         SPECTRE_V1_MITIGATION_NONE,
 799         SPECTRE_V1_MITIGATION_AUTO,
 800 };
 801
 802 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
 803         SPECTRE_V1_MITIGATION_AUTO;
 804
 805 static const char * const spectre_v1_strings[] = {
 806         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
 807         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
 808 };
 809
 810 /*
 811  * Does SMAP provide full mitigation against speculative kernel access to
 812  * userspace?
 813  */
 814 static bool smap_works_speculatively(void)
 815 {
 816         if (!boot_cpu_has(X86_FEATURE_SMAP))
 817                 return false;
 818
 819         /*
 820          * On CPUs which are vulnerable to Meltdown, SMAP does not
 821          * prevent speculative access to user data in the L1 cache.
 822          * Consider SMAP to be non-functional as a mitigation on these
 823          * CPUs.
 824          */
 825         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
 826                 return false;
 827
 828         return true;
 829 }
 830
 831 static void __init spectre_v1_select_mitigation(void)
 832 {
 833         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
 834                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 835                 return;
 836         }
 837
 838         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
 839                 /*
 840                  * With Spectre v1, a user can speculatively control either
 841                  * path of a conditional swapgs with a user-controlled GS
 842                  * value.  The mitigation is to add lfences to both code paths.
 843                  *
 844                  * If FSGSBASE is enabled, the user can put a kernel address in
 845                  * GS, in which case SMAP provides no protection.
 846                  *
 847                  * If FSGSBASE is disabled, the user can only put a user space
 848                  * address in GS.  That makes an attack harder, but still
 849                  * possible if there's no SMAP protection.
 850                  */
 851                 if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
 852                     !smap_works_speculatively()) {
 853                         /*
 854                          * Mitigation can be provided from SWAPGS itself or
 855                          * PTI as the CR3 write in the Meltdown mitigation
 856                          * is serializing.
 857                          *
 858                          * If neither is there, mitigate with an LFENCE to
 859                          * stop speculation through swapgs.
 860                          */
 861                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
 862                             !boot_cpu_has(X86_FEATURE_PTI))
 863                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
 864
 865                         /*
 866                          * Enable lfences in the kernel entry (non-swapgs)
 867                          * paths, to prevent user entry from speculatively
 868                          * skipping swapgs.
 869                          */
 870                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
 871                 }
 872         }
 873
 874         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
 875 }
 876
 877 static int __init nospectre_v1_cmdline(char *str)
 878 {
 879         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 880         return 0;
 881 }
 882 early_param("nospectre_v1", nospectre_v1_cmdline);
 883
 884 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 885         SPECTRE_V2_NONE;
 886
 887 #undef pr_fmt
 888 #define pr_fmt(fmt)     "RETBleed: " fmt
 889
 890 enum retbleed_mitigation {
 891         RETBLEED_MITIGATION_NONE,
 892         RETBLEED_MITIGATION_UNRET,
 893         RETBLEED_MITIGATION_IBPB,
 894         RETBLEED_MITIGATION_IBRS,
 895         RETBLEED_MITIGATION_EIBRS,
 896 };
 897
 898 enum retbleed_mitigation_cmd {
 899         RETBLEED_CMD_OFF,
 900         RETBLEED_CMD_AUTO,
 901         RETBLEED_CMD_UNRET,
 902         RETBLEED_CMD_IBPB,
 903 };
 904
 905 static const char * const retbleed_strings[] = {
 906         [RETBLEED_MITIGATION_NONE]      = "Vulnerable",
 907         [RETBLEED_MITIGATION_UNRET]     = "Mitigation: untrained return thunk",
 908         [RETBLEED_MITIGATION_IBPB]      = "Mitigation: IBPB",
 909         [RETBLEED_MITIGATION_IBRS]      = "Mitigation: IBRS",
 910         [RETBLEED_MITIGATION_EIBRS]     = "Mitigation: Enhanced IBRS",
 911 };
 912
 913 static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
 914         RETBLEED_MITIGATION_NONE;
 915 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init =
 916         RETBLEED_CMD_AUTO;
 917
 918 static int __ro_after_init retbleed_nosmt = false;
 919
 920 static int __init retbleed_parse_cmdline(char *str)
 921 {
 922         if (!str)
 923                 return -EINVAL;
 924
 925         while (str) {
 926                 char *next = strchr(str, ',');
 927                 if (next) {
 928                         *next = 0;
 929                         next++;
 930                 }
 931
 932                 if (!strcmp(str, "off")) {
 933                         retbleed_cmd = RETBLEED_CMD_OFF;
 934                 } else if (!strcmp(str, "auto")) {
 935                         retbleed_cmd = RETBLEED_CMD_AUTO;
 936                 } else if (!strcmp(str, "unret")) {
 937                         retbleed_cmd = RETBLEED_CMD_UNRET;
 938                 } else if (!strcmp(str, "ibpb")) {
 939                         retbleed_cmd = RETBLEED_CMD_IBPB;
 940                 } else if (!strcmp(str, "nosmt")) {
 941                         retbleed_nosmt = true;
 942                 } else {
 943                         pr_err("Ignoring unknown retbleed option (%s).", str);
 944                 }
 945
 946                 str = next;
 947         }
 948
 949         return 0;
 950 }
 951 early_param("retbleed", retbleed_parse_cmdline);
 952
 953 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
 954 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
 955
 956 static void __init retbleed_select_mitigation(void)
 957 {
 958         bool mitigate_smt = false;
 959
 960         if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())
 961                 return;
 962
 963         switch (retbleed_cmd) {
 964         case RETBLEED_CMD_OFF:
 965                 return;
 966
 967         case RETBLEED_CMD_UNRET:
 968                 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) {
 969                         retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
 970                 } else {
 971                         pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n");
 972                         goto do_cmd_auto;
 973                 }
 974                 break;
 975
 976         case RETBLEED_CMD_IBPB:
 977                 if (!boot_cpu_has(X86_FEATURE_IBPB)) {
 978                         pr_err("WARNING: CPU does not support IBPB.\n");
 979                         goto do_cmd_auto;
 980                 } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
 981                         retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
 982                 } else {
 983                         pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
 984                         goto do_cmd_auto;
 985                 }
 986                 break;
 987
 988 do_cmd_auto:
 989         case RETBLEED_CMD_AUTO:
 990         default:
 991                 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
 992                     boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
 993                         if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY))
 994                                 retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
 995                         else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY) && boot_cpu_has(X86_FEATURE_IBPB))
 996                                 retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
 997                 }
 998
 999                 /*
1000                  * The Intel mitigation (IBRS or eIBRS) was already selected in
1001                  * spectre_v2_select_mitigation().  'retbleed_mitigation' will
1002                  * be set accordingly below.
1003                  */
1004
1005                 break;
1006         }
1007
1008         switch (retbleed_mitigation) {
1009         case RETBLEED_MITIGATION_UNRET:
1010                 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1011                 setup_force_cpu_cap(X86_FEATURE_UNRET);
1012
1013                 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
1014                     boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
1015                         pr_err(RETBLEED_UNTRAIN_MSG);
1016
1017                 mitigate_smt = true;
1018                 break;
1019
1020         case RETBLEED_MITIGATION_IBPB:
1021                 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
1022                 mitigate_smt = true;
1023                 break;
1024
1025         default:
1026                 break;
1027         }
1028
1029         if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&
1030             (retbleed_nosmt || cpu_mitigations_auto_nosmt()))
1031                 cpu_smt_disable(false);
1032
1033         /*
1034          * Let IBRS trump all on Intel without affecting the effects of the
1035          * retbleed= cmdline option.
1036          */
1037         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1038                 switch (spectre_v2_enabled) {
1039                 case SPECTRE_V2_IBRS:
1040                         retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
1041                         break;
1042                 case SPECTRE_V2_EIBRS:
1043                 case SPECTRE_V2_EIBRS_RETPOLINE:
1044                 case SPECTRE_V2_EIBRS_LFENCE:
1045                         retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
1046                         break;
1047                 default:
1048                         pr_err(RETBLEED_INTEL_MSG);
1049                 }
1050         }
1051
1052         pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
1053 }
1054
1055 #undef pr_fmt
1056 #define pr_fmt(fmt)     "Spectre V2 : " fmt
1057
1058 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
1059         SPECTRE_V2_USER_NONE;
1060 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
1061         SPECTRE_V2_USER_NONE;
1062
1063 #ifdef CONFIG_RETPOLINE
1064 static bool spectre_v2_bad_module;
1065
1066 bool retpoline_module_ok(bool has_retpoline)
1067 {
1068         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
1069                 return true;
1070
1071         pr_err("System may be vulnerable to spectre v2\n");
1072         spectre_v2_bad_module = true;
1073         return false;
1074 }
1075
1076 static inline const char *spectre_v2_module_string(void)
1077 {
1078         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
1079 }
1080 #else
1081 static inline const char *spectre_v2_module_string(void) { return ""; }
1082 #endif
1083
1084 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
1085 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
1086 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
1087 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n"
1088
1089 #ifdef CONFIG_BPF_SYSCALL
1090 void unpriv_ebpf_notify(int new_state)
1091 {
1092         if (new_state)
1093                 return;
1094
1095         /* Unprivileged eBPF is enabled */
1096
1097         switch (spectre_v2_enabled) {
1098         case SPECTRE_V2_EIBRS:
1099                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1100                 break;
1101         case SPECTRE_V2_EIBRS_LFENCE:
1102                 if (sched_smt_active())
1103                         pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1104                 break;
1105         default:
1106                 break;
1107         }
1108 }
1109 #endif
1110
1111 static inline bool match_option(const char *arg, int arglen, const char *opt)
1112 {
1113         int len = strlen(opt);
1114
1115         return len == arglen && !strncmp(arg, opt, len);
1116 }
1117
1118 /* The kernel command line selection for spectre v2 */
1119 enum spectre_v2_mitigation_cmd {
1120         SPECTRE_V2_CMD_NONE,
1121         SPECTRE_V2_CMD_AUTO,
1122         SPECTRE_V2_CMD_FORCE,
1123         SPECTRE_V2_CMD_RETPOLINE,
1124         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
1125         SPECTRE_V2_CMD_RETPOLINE_LFENCE,
1126         SPECTRE_V2_CMD_EIBRS,
1127         SPECTRE_V2_CMD_EIBRS_RETPOLINE,
1128         SPECTRE_V2_CMD_EIBRS_LFENCE,
1129         SPECTRE_V2_CMD_IBRS,
1130 };
1131
1132 enum spectre_v2_user_cmd {
1133         SPECTRE_V2_USER_CMD_NONE,
1134         SPECTRE_V2_USER_CMD_AUTO,
1135         SPECTRE_V2_USER_CMD_FORCE,
1136         SPECTRE_V2_USER_CMD_PRCTL,
1137         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
1138         SPECTRE_V2_USER_CMD_SECCOMP,
1139         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
1140 };
1141
1142 static const char * const spectre_v2_user_strings[] = {
1143         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
1144         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
1145         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
1146         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
1147         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
1148 };
1149
1150 static const struct {
1151         const char                      *option;
1152         enum spectre_v2_user_cmd        cmd;
1153         bool                            secure;
1154 } v2_user_options[] __initconst = {
1155         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
1156         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
1157         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
1158         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
1159         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
1160         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
1161         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
1162 };
1163
1164 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
1165 {
1166         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1167                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
1168 }
1169
1170 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd;
1171
1172 static enum spectre_v2_user_cmd __init
1173 spectre_v2_parse_user_cmdline(void)
1174 {
1175         char arg[20];
1176         int ret, i;
1177
1178         switch (spectre_v2_cmd) {
1179         case SPECTRE_V2_CMD_NONE:
1180                 return SPECTRE_V2_USER_CMD_NONE;
1181         case SPECTRE_V2_CMD_FORCE:
1182                 return SPECTRE_V2_USER_CMD_FORCE;
1183         default:
1184                 break;
1185         }
1186
1187         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
1188                                   arg, sizeof(arg));
1189         if (ret < 0)
1190                 return SPECTRE_V2_USER_CMD_AUTO;
1191
1192         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
1193                 if (match_option(arg, ret, v2_user_options[i].option)) {
1194                         spec_v2_user_print_cond(v2_user_options[i].option,
1195                                                 v2_user_options[i].secure);
1196                         return v2_user_options[i].cmd;
1197                 }
1198         }
1199
1200         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
1201         return SPECTRE_V2_USER_CMD_AUTO;
1202 }
1203
1204 static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
1205 {
1206         return mode == SPECTRE_V2_EIBRS ||
1207                mode == SPECTRE_V2_EIBRS_RETPOLINE ||
1208                mode == SPECTRE_V2_EIBRS_LFENCE;
1209 }
1210
1211 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
1212 {
1213         return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS;
1214 }
1215
1216 static void __init
1217 spectre_v2_user_select_mitigation(void)
1218 {
1219         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
1220         bool smt_possible = IS_ENABLED(CONFIG_SMP);
1221         enum spectre_v2_user_cmd cmd;
1222
1223         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
1224                 return;
1225
1226         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
1227             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
1228                 smt_possible = false;
1229
1230         cmd = spectre_v2_parse_user_cmdline();
1231         switch (cmd) {
1232         case SPECTRE_V2_USER_CMD_NONE:
1233                 goto set_mode;
1234         case SPECTRE_V2_USER_CMD_FORCE:
1235                 mode = SPECTRE_V2_USER_STRICT;
1236                 break;
1237         case SPECTRE_V2_USER_CMD_AUTO:
1238         case SPECTRE_V2_USER_CMD_PRCTL:
1239         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1240                 mode = SPECTRE_V2_USER_PRCTL;
1241                 break;
1242         case SPECTRE_V2_USER_CMD_SECCOMP:
1243         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1244                 if (IS_ENABLED(CONFIG_SECCOMP))
1245                         mode = SPECTRE_V2_USER_SECCOMP;
1246                 else
1247                         mode = SPECTRE_V2_USER_PRCTL;
1248                 break;
1249         }
1250
1251         /* Initialize Indirect Branch Prediction Barrier */
1252         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1253                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
1254
1255                 spectre_v2_user_ibpb = mode;
1256                 switch (cmd) {
1257                 case SPECTRE_V2_USER_CMD_FORCE:
1258                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1259                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1260                         static_branch_enable(&switch_mm_always_ibpb);
1261                         spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
1262                         break;
1263                 case SPECTRE_V2_USER_CMD_PRCTL:
1264                 case SPECTRE_V2_USER_CMD_AUTO:
1265                 case SPECTRE_V2_USER_CMD_SECCOMP:
1266                         static_branch_enable(&switch_mm_cond_ibpb);
1267                         break;
1268                 default:
1269                         break;
1270                 }
1271
1272                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
1273                         static_key_enabled(&switch_mm_always_ibpb) ?
1274                         "always-on" : "conditional");
1275         }
1276
1277         /*
1278          * If no STIBP, enhanced IBRS is enabled, or SMT impossible, STIBP
1279          * is not required.
1280          *
1281          * Enhanced IBRS also protects against cross-thread branch target
1282          * injection in user-mode as the IBRS bit remains always set which
1283          * implicitly enables cross-thread protections.  However, in legacy IBRS
1284          * mode, the IBRS bit is set only on kernel entry and cleared on return
1285          * to userspace. This disables the implicit cross-thread protection,
1286          * so allow for STIBP to be selected in that case.
1287          */
1288         if (!boot_cpu_has(X86_FEATURE_STIBP) ||
1289             !smt_possible ||
1290             spectre_v2_in_eibrs_mode(spectre_v2_enabled))
1291                 return;
1292
1293         /*
1294          * At this point, an STIBP mode other than "off" has been set.
1295          * If STIBP support is not being forced, check if STIBP always-on
1296          * is preferred.
1297          */
1298         if (mode != SPECTRE_V2_USER_STRICT &&
1299             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
1300                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1301
1302         if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
1303             retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
1304                 if (mode != SPECTRE_V2_USER_STRICT &&
1305                     mode != SPECTRE_V2_USER_STRICT_PREFERRED)
1306                         pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");
1307                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1308         }
1309
1310         spectre_v2_user_stibp = mode;
1311
1312 set_mode:
1313         pr_info("%s\n", spectre_v2_user_strings[mode]);
1314 }
1315
1316 static const char * const spectre_v2_strings[] = {
1317         [SPECTRE_V2_NONE]                       = "Vulnerable",
1318         [SPECTRE_V2_RETPOLINE]                  = "Mitigation: Retpolines",
1319         [SPECTRE_V2_LFENCE]                     = "Mitigation: LFENCE",
1320         [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced IBRS",
1321         [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced IBRS + LFENCE",
1322         [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced IBRS + Retpolines",
1323         [SPECTRE_V2_IBRS]                       = "Mitigation: IBRS",
1324 };
1325
1326 static const struct {
1327         const char *option;
1328         enum spectre_v2_mitigation_cmd cmd;
1329         bool secure;
1330 } mitigation_options[] __initconst = {
1331         { "off",                SPECTRE_V2_CMD_NONE,              false },
1332         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
1333         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
1334         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
1335         { "retpoline,lfence",   SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
1336         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
1337         { "eibrs",              SPECTRE_V2_CMD_EIBRS,             false },
1338         { "eibrs,lfence",       SPECTRE_V2_CMD_EIBRS_LFENCE,      false },
1339         { "eibrs,retpoline",    SPECTRE_V2_CMD_EIBRS_RETPOLINE,   false },
1340         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
1341         { "ibrs",               SPECTRE_V2_CMD_IBRS,              false },
1342 };
1343
1344 static void __init spec_v2_print_cond(const char *reason, bool secure)
1345 {
1346         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1347                 pr_info("%s selected on command line.\n", reason);
1348 }
1349
1350 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
1351 {
1352         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
1353         char arg[20];
1354         int ret, i;
1355
1356         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
1357             cpu_mitigations_off())
1358                 return SPECTRE_V2_CMD_NONE;
1359
1360         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
1361         if (ret < 0)
1362                 return SPECTRE_V2_CMD_AUTO;
1363
1364         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
1365                 if (!match_option(arg, ret, mitigation_options[i].option))
1366                         continue;
1367                 cmd = mitigation_options[i].cmd;
1368                 break;
1369         }
1370
1371         if (i >= ARRAY_SIZE(mitigation_options)) {
1372                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1373                 return SPECTRE_V2_CMD_AUTO;
1374         }
1375
1376         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
1377              cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1378              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
1379              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1380              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1381             !IS_ENABLED(CONFIG_RETPOLINE)) {
1382                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1383                        mitigation_options[i].option);
1384                 return SPECTRE_V2_CMD_AUTO;
1385         }
1386
1387         if ((cmd == SPECTRE_V2_CMD_EIBRS ||
1388              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1389              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1390             !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1391                 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
1392                        mitigation_options[i].option);
1393                 return SPECTRE_V2_CMD_AUTO;
1394         }
1395
1396         if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1397              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
1398             !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
1399                 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
1400                        mitigation_options[i].option);
1401                 return SPECTRE_V2_CMD_AUTO;
1402         }
1403
1404         if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY)) {
1405                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1406                        mitigation_options[i].option);
1407                 return SPECTRE_V2_CMD_AUTO;
1408         }
1409
1410         if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
1411                 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",
1412                        mitigation_options[i].option);
1413                 return SPECTRE_V2_CMD_AUTO;
1414         }
1415
1416         if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) {
1417                 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",
1418                        mitigation_options[i].option);
1419                 return SPECTRE_V2_CMD_AUTO;
1420         }
1421
1422         if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_has(X86_FEATURE_XENPV)) {
1423                 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",
1424                        mitigation_options[i].option);
1425                 return SPECTRE_V2_CMD_AUTO;
1426         }
1427
1428         spec_v2_print_cond(mitigation_options[i].option,
1429                            mitigation_options[i].secure);
1430         return cmd;
1431 }
1432
1433 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
1434 {
1435         if (!IS_ENABLED(CONFIG_RETPOLINE)) {
1436                 pr_err("Kernel not compiled with retpoline; no mitigation available!");
1437                 return SPECTRE_V2_NONE;
1438         }
1439
1440         return SPECTRE_V2_RETPOLINE;
1441 }
1442
1443 /* Disable in-kernel use of non-RSB RET predictors */
1444 static void __init spec_ctrl_disable_kernel_rrsba(void)
1445 {
1446         u64 ia32_cap;
1447
1448         if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL))
1449                 return;
1450
1451         ia32_cap = x86_read_arch_cap_msr();
1452
1453         if (ia32_cap & ARCH_CAP_RRSBA) {
1454                 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S;
1455                 update_spec_ctrl(x86_spec_ctrl_base);
1456         }
1457 }
1458
1459 static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode)
1460 {
1461         /*
1462          * Similar to context switches, there are two types of RSB attacks
1463          * after VM exit:
1464          *
1465          * 1) RSB underflow
1466          *
1467          * 2) Poisoned RSB entry
1468          *
1469          * When retpoline is enabled, both are mitigated by filling/clearing
1470          * the RSB.
1471          *
1472          * When IBRS is enabled, while #1 would be mitigated by the IBRS branch
1473          * prediction isolation protections, RSB still needs to be cleared
1474          * because of #2.  Note that SMEP provides no protection here, unlike
1475          * user-space-poisoned RSB entries.
1476          *
1477          * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB
1478          * bug is present then a LITE version of RSB protection is required,
1479          * just a single call needs to retire before a RET is executed.
1480          */
1481         switch (mode) {
1482         case SPECTRE_V2_NONE:
1483                 return;
1484
1485         case SPECTRE_V2_EIBRS_LFENCE:
1486         case SPECTRE_V2_EIBRS:
1487                 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
1488                         setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE);
1489                         pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");
1490                 }
1491                 return;
1492
1493         case SPECTRE_V2_EIBRS_RETPOLINE:
1494         case SPECTRE_V2_RETPOLINE:
1495         case SPECTRE_V2_LFENCE:
1496         case SPECTRE_V2_IBRS:
1497                 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT);
1498                 pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n");
1499                 return;
1500         }
1501
1502         pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit");
1503         dump_stack();
1504 }
1505
1506 static void __init spectre_v2_select_mitigation(void)
1507 {
1508         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
1509         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
1510
1511         /*
1512          * If the CPU is not affected and the command line mode is NONE or AUTO
1513          * then nothing to do.
1514          */
1515         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
1516             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
1517                 return;
1518
1519         switch (cmd) {
1520         case SPECTRE_V2_CMD_NONE:
1521                 return;
1522
1523         case SPECTRE_V2_CMD_FORCE:
1524         case SPECTRE_V2_CMD_AUTO:
1525                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1526                         mode = SPECTRE_V2_EIBRS;
1527                         break;
1528                 }
1529
1530                 if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) &&
1531                     boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1532                     retbleed_cmd != RETBLEED_CMD_OFF &&
1533                     boot_cpu_has(X86_FEATURE_IBRS) &&
1534                     boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1535                         mode = SPECTRE_V2_IBRS;
1536                         break;
1537                 }
1538
1539                 mode = spectre_v2_select_retpoline();
1540                 break;
1541
1542         case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
1543                 pr_err(SPECTRE_V2_LFENCE_MSG);
1544                 mode = SPECTRE_V2_LFENCE;
1545                 break;
1546
1547         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
1548                 mode = SPECTRE_V2_RETPOLINE;
1549                 break;
1550
1551         case SPECTRE_V2_CMD_RETPOLINE:
1552                 mode = spectre_v2_select_retpoline();
1553                 break;
1554
1555         case SPECTRE_V2_CMD_IBRS:
1556                 mode = SPECTRE_V2_IBRS;
1557                 break;
1558
1559         case SPECTRE_V2_CMD_EIBRS:
1560                 mode = SPECTRE_V2_EIBRS;
1561                 break;
1562
1563         case SPECTRE_V2_CMD_EIBRS_LFENCE:
1564                 mode = SPECTRE_V2_EIBRS_LFENCE;
1565                 break;
1566
1567         case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
1568                 mode = SPECTRE_V2_EIBRS_RETPOLINE;
1569                 break;
1570         }
1571
1572         if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
1573                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1574
1575         if (spectre_v2_in_ibrs_mode(mode)) {
1576                 x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
1577                 update_spec_ctrl(x86_spec_ctrl_base);
1578         }
1579
1580         switch (mode) {
1581         case SPECTRE_V2_NONE:
1582         case SPECTRE_V2_EIBRS:
1583                 break;
1584
1585         case SPECTRE_V2_IBRS:
1586                 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS);
1587                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED))
1588                         pr_warn(SPECTRE_V2_IBRS_PERF_MSG);
1589                 break;
1590
1591         case SPECTRE_V2_LFENCE:
1592         case SPECTRE_V2_EIBRS_LFENCE:
1593                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
1594                 fallthrough;
1595
1596         case SPECTRE_V2_RETPOLINE:
1597         case SPECTRE_V2_EIBRS_RETPOLINE:
1598                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
1599                 break;
1600         }
1601
1602         /*
1603          * Disable alternate RSB predictions in kernel when indirect CALLs and
1604          * JMPs gets protection against BHI and Intramode-BTI, but RET
1605          * prediction from a non-RSB predictor is still a risk.
1606          */
1607         if (mode == SPECTRE_V2_EIBRS_LFENCE ||
1608             mode == SPECTRE_V2_EIBRS_RETPOLINE ||
1609             mode == SPECTRE_V2_RETPOLINE)
1610                 spec_ctrl_disable_kernel_rrsba();
1611
1612         spectre_v2_enabled = mode;
1613         pr_info("%s\n", spectre_v2_strings[mode]);
1614
1615         /*
1616          * If Spectre v2 protection has been enabled, fill the RSB during a
1617          * context switch.  In general there are two types of RSB attacks
1618          * across context switches, for which the CALLs/RETs may be unbalanced.
1619          *
1620          * 1) RSB underflow
1621          *
1622          *    Some Intel parts have "bottomless RSB".  When the RSB is empty,
1623          *    speculated return targets may come from the branch predictor,
1624          *    which could have a user-poisoned BTB or BHB entry.
1625          *
1626          *    AMD has it even worse: *all* returns are speculated from the BTB,
1627          *    regardless of the state of the RSB.
1628          *
1629          *    When IBRS or eIBRS is enabled, the "user -> kernel" attack
1630          *    scenario is mitigated by the IBRS branch prediction isolation
1631          *    properties, so the RSB buffer filling wouldn't be necessary to
1632          *    protect against this type of attack.
1633          *
1634          *    The "user -> user" attack scenario is mitigated by RSB filling.
1635          *
1636          * 2) Poisoned RSB entry
1637          *
1638          *    If the 'next' in-kernel return stack is shorter than 'prev',
1639          *    'next' could be tricked into speculating with a user-poisoned RSB
1640          *    entry.
1641          *
1642          *    The "user -> kernel" attack scenario is mitigated by SMEP and
1643          *    eIBRS.
1644          *
1645          *    The "user -> user" scenario, also known as SpectreBHB, requires
1646          *    RSB clearing.
1647          *
1648          * So to mitigate all cases, unconditionally fill RSB on context
1649          * switches.
1650          *
1651          * FIXME: Is this pointless for retbleed-affected AMD?
1652          */
1653         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
1654         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1655
1656         spectre_v2_determine_rsb_fill_type_at_vmexit(mode);
1657
1658         /*
1659          * Retpoline protects the kernel, but doesn't protect firmware.  IBRS
1660          * and Enhanced IBRS protect firmware too, so enable IBRS around
1661          * firmware calls only when IBRS / Enhanced IBRS aren't otherwise
1662          * enabled.
1663          *
1664          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1665          * the user might select retpoline on the kernel command line and if
1666          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1667          * enable IBRS around firmware calls.
1668          */
1669         if (boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1670             boot_cpu_has(X86_FEATURE_IBPB) &&
1671             (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1672              boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) {
1673
1674                 if (retbleed_cmd != RETBLEED_CMD_IBPB) {
1675                         setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW);
1676                         pr_info("Enabling Speculation Barrier for firmware calls\n");
1677                 }
1678
1679         } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) {
1680                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
1681                 pr_info("Enabling Restricted Speculation for firmware calls\n");
1682         }
1683
1684         /* Set up IBPB and STIBP depending on the general spectre V2 command */
1685         spectre_v2_cmd = cmd;
1686 }
1687
1688 static void update_stibp_msr(void * __unused)
1689 {
1690         u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP);
1691         update_spec_ctrl(val);
1692 }
1693
1694 /* Update x86_spec_ctrl_base in case SMT state changed. */
1695 static void update_stibp_strict(void)
1696 {
1697         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
1698
1699         if (sched_smt_active())
1700                 mask |= SPEC_CTRL_STIBP;
1701
1702         if (mask == x86_spec_ctrl_base)
1703                 return;
1704
1705         pr_info("Update user space SMT mitigation: STIBP %s\n",
1706                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
1707         x86_spec_ctrl_base = mask;
1708         on_each_cpu(update_stibp_msr, NULL, 1);
1709 }
1710
1711 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1712 static void update_indir_branch_cond(void)
1713 {
1714         if (sched_smt_active())
1715                 static_branch_enable(&switch_to_cond_stibp);
1716         else
1717                 static_branch_disable(&switch_to_cond_stibp);
1718 }
1719
1720 #undef pr_fmt
1721 #define pr_fmt(fmt) fmt
1722
1723 /* Update the static key controlling the MDS CPU buffer clear in idle */
1724 static void update_mds_branch_idle(void)
1725 {
1726         u64 ia32_cap = x86_read_arch_cap_msr();
1727
1728         /*
1729          * Enable the idle clearing if SMT is active on CPUs which are
1730          * affected only by MSBDS and not any other MDS variant.
1731          *
1732          * The other variants cannot be mitigated when SMT is enabled, so
1733          * clearing the buffers on idle just to prevent the Store Buffer
1734          * repartitioning leak would be a window dressing exercise.
1735          */
1736         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
1737                 return;
1738
1739         if (sched_smt_active()) {
1740                 static_branch_enable(&mds_idle_clear);
1741         } else if (mmio_mitigation == MMIO_MITIGATION_OFF ||
1742                    (ia32_cap & ARCH_CAP_FBSDP_NO)) {
1743                 static_branch_disable(&mds_idle_clear);
1744         }
1745 }
1746
1747 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1748 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1749 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
1750
1751 void cpu_bugs_smt_update(void)
1752 {
1753         mutex_lock(&spec_ctrl_mutex);
1754
1755         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1756             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1757                 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1758
1759         switch (spectre_v2_user_stibp) {
1760         case SPECTRE_V2_USER_NONE:
1761                 break;
1762         case SPECTRE_V2_USER_STRICT:
1763         case SPECTRE_V2_USER_STRICT_PREFERRED:
1764                 update_stibp_strict();
1765                 break;
1766         case SPECTRE_V2_USER_PRCTL:
1767         case SPECTRE_V2_USER_SECCOMP:
1768                 update_indir_branch_cond();
1769                 break;
1770         }
1771
1772         switch (mds_mitigation) {
1773         case MDS_MITIGATION_FULL:
1774         case MDS_MITIGATION_VMWERV:
1775                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
1776                         pr_warn_once(MDS_MSG_SMT);
1777                 update_mds_branch_idle();
1778                 break;
1779         case MDS_MITIGATION_OFF:
1780                 break;
1781         }
1782
1783         switch (taa_mitigation) {
1784         case TAA_MITIGATION_VERW:
1785         case TAA_MITIGATION_UCODE_NEEDED:
1786                 if (sched_smt_active())
1787                         pr_warn_once(TAA_MSG_SMT);
1788                 break;
1789         case TAA_MITIGATION_TSX_DISABLED:
1790         case TAA_MITIGATION_OFF:
1791                 break;
1792         }
1793
1794         switch (mmio_mitigation) {
1795         case MMIO_MITIGATION_VERW:
1796         case MMIO_MITIGATION_UCODE_NEEDED:
1797                 if (sched_smt_active())
1798                         pr_warn_once(MMIO_MSG_SMT);
1799                 break;
1800         case MMIO_MITIGATION_OFF:
1801                 break;
1802         }
1803
1804         mutex_unlock(&spec_ctrl_mutex);
1805 }
1806
1807 #undef pr_fmt
1808 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
1809
1810 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
1811
1812 /* The kernel command line selection */
1813 enum ssb_mitigation_cmd {
1814         SPEC_STORE_BYPASS_CMD_NONE,
1815         SPEC_STORE_BYPASS_CMD_AUTO,
1816         SPEC_STORE_BYPASS_CMD_ON,
1817         SPEC_STORE_BYPASS_CMD_PRCTL,
1818         SPEC_STORE_BYPASS_CMD_SECCOMP,
1819 };
1820
1821 static const char * const ssb_strings[] = {
1822         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
1823         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
1824         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
1825         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
1826 };
1827
1828 static const struct {
1829         const char *option;
1830         enum ssb_mitigation_cmd cmd;
1831 } ssb_mitigation_options[]  __initconst = {
1832         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
1833         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
1834         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
1835         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
1836         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
1837 };
1838
1839 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
1840 {
1841         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
1842         char arg[20];
1843         int ret, i;
1844
1845         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
1846             cpu_mitigations_off()) {
1847                 return SPEC_STORE_BYPASS_CMD_NONE;
1848         } else {
1849                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
1850                                           arg, sizeof(arg));
1851                 if (ret < 0)
1852                         return SPEC_STORE_BYPASS_CMD_AUTO;
1853
1854                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
1855                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
1856                                 continue;
1857
1858                         cmd = ssb_mitigation_options[i].cmd;
1859                         break;
1860                 }
1861
1862                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
1863                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1864                         return SPEC_STORE_BYPASS_CMD_AUTO;
1865                 }
1866         }
1867
1868         return cmd;
1869 }
1870
1871 static enum ssb_mitigation __init __ssb_select_mitigation(void)
1872 {
1873         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
1874         enum ssb_mitigation_cmd cmd;
1875
1876         if (!boot_cpu_has(X86_FEATURE_SSBD))
1877                 return mode;
1878
1879         cmd = ssb_parse_cmdline();
1880         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
1881             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
1882              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
1883                 return mode;
1884
1885         switch (cmd) {
1886         case SPEC_STORE_BYPASS_CMD_SECCOMP:
1887                 /*
1888                  * Choose prctl+seccomp as the default mode if seccomp is
1889                  * enabled.
1890                  */
1891                 if (IS_ENABLED(CONFIG_SECCOMP))
1892                         mode = SPEC_STORE_BYPASS_SECCOMP;
1893                 else
1894                         mode = SPEC_STORE_BYPASS_PRCTL;
1895                 break;
1896         case SPEC_STORE_BYPASS_CMD_ON:
1897                 mode = SPEC_STORE_BYPASS_DISABLE;
1898                 break;
1899         case SPEC_STORE_BYPASS_CMD_AUTO:
1900         case SPEC_STORE_BYPASS_CMD_PRCTL:
1901                 mode = SPEC_STORE_BYPASS_PRCTL;
1902                 break;
1903         case SPEC_STORE_BYPASS_CMD_NONE:
1904                 break;
1905         }
1906
1907         /*
1908          * We have three CPU feature flags that are in play here:
1909          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1910          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1911          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1912          */
1913         if (mode == SPEC_STORE_BYPASS_DISABLE) {
1914                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
1915                 /*
1916                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1917                  * use a completely different MSR and bit dependent on family.
1918                  */
1919                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
1920                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1921                         x86_amd_ssb_disable();
1922                 } else {
1923                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
1924                         update_spec_ctrl(x86_spec_ctrl_base);
1925                 }
1926         }
1927
1928         return mode;
1929 }
1930
1931 static void ssb_select_mitigation(void)
1932 {
1933         ssb_mode = __ssb_select_mitigation();
1934
1935         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1936                 pr_info("%s\n", ssb_strings[ssb_mode]);
1937 }
1938
1939 #undef pr_fmt
1940 #define pr_fmt(fmt)     "Speculation prctl: " fmt
1941
1942 static void task_update_spec_tif(struct task_struct *tsk)
1943 {
1944         /* Force the update of the real TIF bits */
1945         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
1946
1947         /*
1948          * Immediately update the speculation control MSRs for the current
1949          * task, but for a non-current task delay setting the CPU
1950          * mitigation until it is scheduled next.
1951          *
1952          * This can only happen for SECCOMP mitigation. For PRCTL it's
1953          * always the current task.
1954          */
1955         if (tsk == current)
1956                 speculation_ctrl_update_current();
1957 }
1958
1959 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl)
1960 {
1961
1962         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
1963                 return -EPERM;
1964
1965         switch (ctrl) {
1966         case PR_SPEC_ENABLE:
1967                 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
1968                 return 0;
1969         case PR_SPEC_DISABLE:
1970                 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
1971                 return 0;
1972         default:
1973                 return -ERANGE;
1974         }
1975 }
1976
1977 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
1978 {
1979         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
1980             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
1981                 return -ENXIO;
1982
1983         switch (ctrl) {
1984         case PR_SPEC_ENABLE:
1985                 /* If speculation is force disabled, enable is not allowed */
1986                 if (task_spec_ssb_force_disable(task))
1987                         return -EPERM;
1988                 task_clear_spec_ssb_disable(task);
1989                 task_clear_spec_ssb_noexec(task);
1990                 task_update_spec_tif(task);
1991                 break;
1992         case PR_SPEC_DISABLE:
1993                 task_set_spec_ssb_disable(task);
1994                 task_clear_spec_ssb_noexec(task);
1995                 task_update_spec_tif(task);
1996                 break;
1997         case PR_SPEC_FORCE_DISABLE:
1998                 task_set_spec_ssb_disable(task);
1999                 task_set_spec_ssb_force_disable(task);
2000                 task_clear_spec_ssb_noexec(task);
2001                 task_update_spec_tif(task);
2002                 break;
2003         case PR_SPEC_DISABLE_NOEXEC:
2004                 if (task_spec_ssb_force_disable(task))
2005                         return -EPERM;
2006                 task_set_spec_ssb_disable(task);
2007                 task_set_spec_ssb_noexec(task);
2008                 task_update_spec_tif(task);
2009                 break;
2010         default:
2011                 return -ERANGE;
2012         }
2013         return 0;
2014 }
2015
2016 static bool is_spec_ib_user_controlled(void)
2017 {
2018         return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
2019                 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2020                 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
2021                 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
2022 }
2023
2024 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
2025 {
2026         switch (ctrl) {
2027         case PR_SPEC_ENABLE:
2028                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2029                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2030                         return 0;
2031
2032                 /*
2033                  * With strict mode for both IBPB and STIBP, the instruction
2034                  * code paths avoid checking this task flag and instead,
2035                  * unconditionally run the instruction. However, STIBP and IBPB
2036                  * are independent and either can be set to conditionally
2037                  * enabled regardless of the mode of the other.
2038                  *
2039                  * If either is set to conditional, allow the task flag to be
2040                  * updated, unless it was force-disabled by a previous prctl
2041                  * call. Currently, this is possible on an AMD CPU which has the
2042                  * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
2043                  * kernel is booted with 'spectre_v2_user=seccomp', then
2044                  * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
2045                  * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
2046                  */
2047                 if (!is_spec_ib_user_controlled() ||
2048                     task_spec_ib_force_disable(task))
2049                         return -EPERM;
2050
2051                 task_clear_spec_ib_disable(task);
2052                 task_update_spec_tif(task);
2053                 break;
2054         case PR_SPEC_DISABLE:
2055         case PR_SPEC_FORCE_DISABLE:
2056                 /*
2057                  * Indirect branch speculation is always allowed when
2058                  * mitigation is force disabled.
2059                  */
2060                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2061                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2062                         return -EPERM;
2063
2064                 if (!is_spec_ib_user_controlled())
2065                         return 0;
2066
2067                 task_set_spec_ib_disable(task);
2068                 if (ctrl == PR_SPEC_FORCE_DISABLE)
2069                         task_set_spec_ib_force_disable(task);
2070                 task_update_spec_tif(task);
2071                 if (task == current)
2072                         indirect_branch_prediction_barrier();
2073                 break;
2074         default:
2075                 return -ERANGE;
2076         }
2077         return 0;
2078 }
2079
2080 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
2081                              unsigned long ctrl)
2082 {
2083         switch (which) {
2084         case PR_SPEC_STORE_BYPASS:
2085                 return ssb_prctl_set(task, ctrl);
2086         case PR_SPEC_INDIRECT_BRANCH:
2087                 return ib_prctl_set(task, ctrl);
2088         case PR_SPEC_L1D_FLUSH:
2089                 return l1d_flush_prctl_set(task, ctrl);
2090         default:
2091                 return -ENODEV;
2092         }
2093 }
2094
2095 #ifdef CONFIG_SECCOMP
2096 void arch_seccomp_spec_mitigate(struct task_struct *task)
2097 {
2098         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
2099                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2100         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2101             spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
2102                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2103 }
2104 #endif
2105
2106 static int l1d_flush_prctl_get(struct task_struct *task)
2107 {
2108         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
2109                 return PR_SPEC_FORCE_DISABLE;
2110
2111         if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))
2112                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2113         else
2114                 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2115 }
2116
2117 static int ssb_prctl_get(struct task_struct *task)
2118 {
2119         switch (ssb_mode) {
2120         case SPEC_STORE_BYPASS_DISABLE:
2121                 return PR_SPEC_DISABLE;
2122         case SPEC_STORE_BYPASS_SECCOMP:
2123         case SPEC_STORE_BYPASS_PRCTL:
2124                 if (task_spec_ssb_force_disable(task))
2125                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2126                 if (task_spec_ssb_noexec(task))
2127                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
2128                 if (task_spec_ssb_disable(task))
2129                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2130                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2131         default:
2132                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
2133                         return PR_SPEC_ENABLE;
2134                 return PR_SPEC_NOT_AFFECTED;
2135         }
2136 }
2137
2138 static int ib_prctl_get(struct task_struct *task)
2139 {
2140         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
2141                 return PR_SPEC_NOT_AFFECTED;
2142
2143         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2144             spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2145                 return PR_SPEC_ENABLE;
2146         else if (is_spec_ib_user_controlled()) {
2147                 if (task_spec_ib_force_disable(task))
2148                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2149                 if (task_spec_ib_disable(task))
2150                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2151                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2152         } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
2153             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2154             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
2155                 return PR_SPEC_DISABLE;
2156         else
2157                 return PR_SPEC_NOT_AFFECTED;
2158 }
2159
2160 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
2161 {
2162         switch (which) {
2163         case PR_SPEC_STORE_BYPASS:
2164                 return ssb_prctl_get(task);
2165         case PR_SPEC_INDIRECT_BRANCH:
2166                 return ib_prctl_get(task);
2167         case PR_SPEC_L1D_FLUSH:
2168                 return l1d_flush_prctl_get(task);
2169         default:
2170                 return -ENODEV;
2171         }
2172 }
2173
2174 void x86_spec_ctrl_setup_ap(void)
2175 {
2176         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
2177                 update_spec_ctrl(x86_spec_ctrl_base);
2178
2179         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
2180                 x86_amd_ssb_disable();
2181 }
2182
2183 bool itlb_multihit_kvm_mitigation;
2184 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
2185
2186 #undef pr_fmt
2187 #define pr_fmt(fmt)     "L1TF: " fmt
2188
2189 /* Default mitigation for L1TF-affected CPUs */
2190 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
2191 #if IS_ENABLED(CONFIG_KVM_INTEL)
2192 EXPORT_SYMBOL_GPL(l1tf_mitigation);
2193 #endif
2194 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
2195 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
2196
2197 /*
2198  * These CPUs all support 44bits physical address space internally in the
2199  * cache but CPUID can report a smaller number of physical address bits.
2200  *
2201  * The L1TF mitigation uses the top most address bit for the inversion of
2202  * non present PTEs. When the installed memory reaches into the top most
2203  * address bit due to memory holes, which has been observed on machines
2204  * which report 36bits physical address bits and have 32G RAM installed,
2205  * then the mitigation range check in l1tf_select_mitigation() triggers.
2206  * This is a false positive because the mitigation is still possible due to
2207  * the fact that the cache uses 44bit internally. Use the cache bits
2208  * instead of the reported physical bits and adjust them on the affected
2209  * machines to 44bit if the reported bits are less than 44.
2210  */
2211 static void override_cache_bits(struct cpuinfo_x86 *c)
2212 {
2213         if (c->x86 != 6)
2214                 return;
2215
2216         switch (c->x86_model) {
2217         case INTEL_FAM6_NEHALEM:
2218         case INTEL_FAM6_WESTMERE:
2219         case INTEL_FAM6_SANDYBRIDGE:
2220         case INTEL_FAM6_IVYBRIDGE:
2221         case INTEL_FAM6_HASWELL:
2222         case INTEL_FAM6_HASWELL_L:
2223         case INTEL_FAM6_HASWELL_G:
2224         case INTEL_FAM6_BROADWELL:
2225         case INTEL_FAM6_BROADWELL_G:
2226         case INTEL_FAM6_SKYLAKE_L:
2227         case INTEL_FAM6_SKYLAKE:
2228         case INTEL_FAM6_KABYLAKE_L:
2229         case INTEL_FAM6_KABYLAKE:
2230                 if (c->x86_cache_bits < 44)
2231                         c->x86_cache_bits = 44;
2232                 break;
2233         }
2234 }
2235
2236 static void __init l1tf_select_mitigation(void)
2237 {
2238         u64 half_pa;
2239
2240         if (!boot_cpu_has_bug(X86_BUG_L1TF))
2241                 return;
2242
2243         if (cpu_mitigations_off())
2244                 l1tf_mitigation = L1TF_MITIGATION_OFF;
2245         else if (cpu_mitigations_auto_nosmt())
2246                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2247
2248         override_cache_bits(&boot_cpu_data);
2249
2250         switch (l1tf_mitigation) {
2251         case L1TF_MITIGATION_OFF:
2252         case L1TF_MITIGATION_FLUSH_NOWARN:
2253         case L1TF_MITIGATION_FLUSH:
2254                 break;
2255         case L1TF_MITIGATION_FLUSH_NOSMT:
2256         case L1TF_MITIGATION_FULL:
2257                 cpu_smt_disable(false);
2258                 break;
2259         case L1TF_MITIGATION_FULL_FORCE:
2260                 cpu_smt_disable(true);
2261                 break;
2262         }
2263
2264 #if CONFIG_PGTABLE_LEVELS == 2
2265         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
2266         return;
2267 #endif
2268
2269         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
2270         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
2271                         e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
2272                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
2273                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
2274                                 half_pa);
2275                 pr_info("However, doing so will make a part of your RAM unusable.\n");
2276                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
2277                 return;
2278         }
2279
2280         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
2281 }
2282
2283 static int __init l1tf_cmdline(char *str)
2284 {
2285         if (!boot_cpu_has_bug(X86_BUG_L1TF))
2286                 return 0;
2287
2288         if (!str)
2289                 return -EINVAL;
2290
2291         if (!strcmp(str, "off"))
2292                 l1tf_mitigation = L1TF_MITIGATION_OFF;
2293         else if (!strcmp(str, "flush,nowarn"))
2294                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
2295         else if (!strcmp(str, "flush"))
2296                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
2297         else if (!strcmp(str, "flush,nosmt"))
2298                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2299         else if (!strcmp(str, "full"))
2300                 l1tf_mitigation = L1TF_MITIGATION_FULL;
2301         else if (!strcmp(str, "full,force"))
2302                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
2303
2304         return 0;
2305 }
2306 early_param("l1tf", l1tf_cmdline);
2307
2308 #undef pr_fmt
2309 #define pr_fmt(fmt)     "Speculative Return Stack Overflow: " fmt
2310
2311 enum srso_mitigation {
2312         SRSO_MITIGATION_NONE,
2313         SRSO_MITIGATION_MICROCODE,
2314         SRSO_MITIGATION_SAFE_RET,
2315 };
2316
2317 enum srso_mitigation_cmd {
2318         SRSO_CMD_OFF,
2319         SRSO_CMD_MICROCODE,
2320         SRSO_CMD_SAFE_RET,
2321 };
2322
2323 static const char * const srso_strings[] = {
2324         [SRSO_MITIGATION_NONE]           = "Vulnerable",
2325         [SRSO_MITIGATION_MICROCODE]      = "Mitigation: microcode",
2326         [SRSO_MITIGATION_SAFE_RET]       = "Mitigation: safe RET",
2327 };
2328
2329 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE;
2330 static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET;
2331
2332 static int __init srso_parse_cmdline(char *str)
2333 {
2334         if (!str)
2335                 return -EINVAL;
2336
2337         if (!strcmp(str, "off"))
2338                 srso_cmd = SRSO_CMD_OFF;
2339         else if (!strcmp(str, "microcode"))
2340                 srso_cmd = SRSO_CMD_MICROCODE;
2341         else if (!strcmp(str, "safe-ret"))
2342                 srso_cmd = SRSO_CMD_SAFE_RET;
2343         else
2344                 pr_err("Ignoring unknown SRSO option (%s).", str);
2345
2346         return 0;
2347 }
2348 early_param("spec_rstack_overflow", srso_parse_cmdline);
2349
2350 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options."
2351
2352 static void __init srso_select_mitigation(void)
2353 {
2354         bool has_microcode;
2355
2356         if (!boot_cpu_has_bug(X86_BUG_SRSO) || cpu_mitigations_off())
2357                 return;
2358
2359         /*
2360          * The first check is for the kernel running as a guest in order
2361          * for guests to verify whether IBPB is a viable mitigation.
2362          */
2363         has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE) || cpu_has_ibpb_brtype_microcode();
2364         if (!has_microcode) {
2365                 pr_warn("IBPB-extending microcode not applied!\n");
2366                 pr_warn(SRSO_NOTICE);
2367         } else {
2368                 /*
2369                  * Enable the synthetic (even if in a real CPUID leaf)
2370                  * flag for guests.
2371                  */
2372                 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
2373         }
2374
2375         switch (srso_cmd) {
2376         case SRSO_CMD_OFF:
2377                 return;
2378
2379         case SRSO_CMD_MICROCODE:
2380                 if (has_microcode) {
2381                         srso_mitigation = SRSO_MITIGATION_MICROCODE;
2382                         pr_warn(SRSO_NOTICE);
2383                 }
2384                 break;
2385
2386         case SRSO_CMD_SAFE_RET:
2387                 if (IS_ENABLED(CONFIG_CPU_SRSO)) {
2388                         if (boot_cpu_data.x86 == 0x19)
2389                                 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
2390                         else
2391                                 setup_force_cpu_cap(X86_FEATURE_SRSO);
2392                         srso_mitigation = SRSO_MITIGATION_SAFE_RET;
2393                 } else {
2394                         pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
2395                         return;
2396                 }
2397                 break;
2398
2399         default:
2400                 break;
2401
2402         }
2403
2404         pr_info("%s%s\n", srso_strings[srso_mitigation], (has_microcode ? "" : ", no microcode"));
2405 }
2406
2407 #undef pr_fmt
2408 #define pr_fmt(fmt) fmt
2409
2410 #ifdef CONFIG_SYSFS
2411
2412 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
2413
2414 #if IS_ENABLED(CONFIG_KVM_INTEL)
2415 static const char * const l1tf_vmx_states[] = {
2416         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
2417         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
2418         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
2419         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
2420         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
2421         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
2422 };
2423
2424 static ssize_t l1tf_show_state(char *buf)
2425 {
2426         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
2427                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
2428
2429         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
2430             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
2431              sched_smt_active())) {
2432                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
2433                                l1tf_vmx_states[l1tf_vmx_mitigation]);
2434         }
2435
2436         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
2437                        l1tf_vmx_states[l1tf_vmx_mitigation],
2438                        sched_smt_active() ? "vulnerable" : "disabled");
2439 }
2440
2441 static ssize_t itlb_multihit_show_state(char *buf)
2442 {
2443         if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
2444             !boot_cpu_has(X86_FEATURE_VMX))
2445                 return sprintf(buf, "KVM: Mitigation: VMX unsupported\n");
2446         else if (!(cr4_read_shadow() & X86_CR4_VMXE))
2447                 return sprintf(buf, "KVM: Mitigation: VMX disabled\n");
2448         else if (itlb_multihit_kvm_mitigation)
2449                 return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
2450         else
2451                 return sprintf(buf, "KVM: Vulnerable\n");
2452 }
2453 #else
2454 static ssize_t l1tf_show_state(char *buf)
2455 {
2456         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
2457 }
2458
2459 static ssize_t itlb_multihit_show_state(char *buf)
2460 {
2461         return sprintf(buf, "Processor vulnerable\n");
2462 }
2463 #endif
2464
2465 static ssize_t mds_show_state(char *buf)
2466 {
2467         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2468                 return sprintf(buf, "%s; SMT Host state unknown\n",
2469                                mds_strings[mds_mitigation]);
2470         }
2471
2472         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
2473                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2474                                (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
2475                                 sched_smt_active() ? "mitigated" : "disabled"));
2476         }
2477
2478         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2479                        sched_smt_active() ? "vulnerable" : "disabled");
2480 }
2481
2482 static ssize_t tsx_async_abort_show_state(char *buf)
2483 {
2484         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
2485             (taa_mitigation == TAA_MITIGATION_OFF))
2486                 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
2487
2488         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2489                 return sprintf(buf, "%s; SMT Host state unknown\n",
2490                                taa_strings[taa_mitigation]);
2491         }
2492
2493         return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
2494                        sched_smt_active() ? "vulnerable" : "disabled");
2495 }
2496
2497 static ssize_t mmio_stale_data_show_state(char *buf)
2498 {
2499         if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
2500                 return sysfs_emit(buf, "Unknown: No mitigations\n");
2501
2502         if (mmio_mitigation == MMIO_MITIGATION_OFF)
2503                 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]);
2504
2505         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2506                 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2507                                   mmio_strings[mmio_mitigation]);
2508         }
2509
2510         return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation],
2511                           sched_smt_active() ? "vulnerable" : "disabled");
2512 }
2513
2514 static char *stibp_state(void)
2515 {
2516         if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
2517                 return "";
2518
2519         switch (spectre_v2_user_stibp) {
2520         case SPECTRE_V2_USER_NONE:
2521                 return ", STIBP: disabled";
2522         case SPECTRE_V2_USER_STRICT:
2523                 return ", STIBP: forced";
2524         case SPECTRE_V2_USER_STRICT_PREFERRED:
2525                 return ", STIBP: always-on";
2526         case SPECTRE_V2_USER_PRCTL:
2527         case SPECTRE_V2_USER_SECCOMP:
2528                 if (static_key_enabled(&switch_to_cond_stibp))
2529                         return ", STIBP: conditional";
2530         }
2531         return "";
2532 }
2533
2534 static char *ibpb_state(void)
2535 {
2536         if (boot_cpu_has(X86_FEATURE_IBPB)) {
2537                 if (static_key_enabled(&switch_mm_always_ibpb))
2538                         return ", IBPB: always-on";
2539                 if (static_key_enabled(&switch_mm_cond_ibpb))
2540                         return ", IBPB: conditional";
2541                 return ", IBPB: disabled";
2542         }
2543         return "";
2544 }
2545
2546 static char *pbrsb_eibrs_state(void)
2547 {
2548         if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
2549                 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) ||
2550                     boot_cpu_has(X86_FEATURE_RSB_VMEXIT))
2551                         return ", PBRSB-eIBRS: SW sequence";
2552                 else
2553                         return ", PBRSB-eIBRS: Vulnerable";
2554         } else {
2555                 return ", PBRSB-eIBRS: Not affected";
2556         }
2557 }
2558
2559 static ssize_t spectre_v2_show_state(char *buf)
2560 {
2561         if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
2562                 return sprintf(buf, "Vulnerable: LFENCE\n");
2563
2564         if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
2565                 return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
2566
2567         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
2568             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
2569                 return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
2570
2571         return sprintf(buf, "%s%s%s%s%s%s%s\n",
2572                        spectre_v2_strings[spectre_v2_enabled],
2573                        ibpb_state(),
2574                        boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
2575                        stibp_state(),
2576                        boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
2577                        pbrsb_eibrs_state(),
2578                        spectre_v2_module_string());
2579 }
2580
2581 static ssize_t srbds_show_state(char *buf)
2582 {
2583         return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]);
2584 }
2585
2586 static ssize_t retbleed_show_state(char *buf)
2587 {
2588         if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
2589             retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
2590             if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
2591                 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
2592                     return sprintf(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n");
2593
2594             return sprintf(buf, "%s; SMT %s\n",
2595                            retbleed_strings[retbleed_mitigation],
2596                            !sched_smt_active() ? "disabled" :
2597                            spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2598                            spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ?
2599                            "enabled with STIBP protection" : "vulnerable");
2600         }
2601
2602         return sprintf(buf, "%s\n", retbleed_strings[retbleed_mitigation]);
2603 }
2604
2605 static ssize_t gds_show_state(char *buf)
2606 {
2607         return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]);
2608 }
2609
2610 static ssize_t srso_show_state(char *buf)
2611 {
2612         return sysfs_emit(buf, "%s%s\n",
2613                           srso_strings[srso_mitigation],
2614                           (cpu_has_ibpb_brtype_microcode() ? "" : ", no microcode"));
2615 }
2616
2617 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
2618                                char *buf, unsigned int bug)
2619 {
2620         if (!boot_cpu_has_bug(bug))
2621                 return sprintf(buf, "Not affected\n");
2622
2623         switch (bug) {
2624         case X86_BUG_CPU_MELTDOWN:
2625                 if (boot_cpu_has(X86_FEATURE_PTI))
2626                         return sprintf(buf, "Mitigation: PTI\n");
2627
2628                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
2629                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
2630
2631                 break;
2632
2633         case X86_BUG_SPECTRE_V1:
2634                 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
2635
2636         case X86_BUG_SPECTRE_V2:
2637                 return spectre_v2_show_state(buf);
2638
2639         case X86_BUG_SPEC_STORE_BYPASS:
2640                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
2641
2642         case X86_BUG_L1TF:
2643                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
2644                         return l1tf_show_state(buf);
2645                 break;
2646
2647         case X86_BUG_MDS:
2648                 return mds_show_state(buf);
2649
2650         case X86_BUG_TAA:
2651                 return tsx_async_abort_show_state(buf);
2652
2653         case X86_BUG_ITLB_MULTIHIT:
2654                 return itlb_multihit_show_state(buf);
2655
2656         case X86_BUG_SRBDS:
2657                 return srbds_show_state(buf);
2658
2659         case X86_BUG_MMIO_STALE_DATA:
2660         case X86_BUG_MMIO_UNKNOWN:
2661                 return mmio_stale_data_show_state(buf);
2662
2663         case X86_BUG_RETBLEED:
2664                 return retbleed_show_state(buf);
2665
2666         case X86_BUG_GDS:
2667                 return gds_show_state(buf);
2668
2669         case X86_BUG_SRSO:
2670                 return srso_show_state(buf);
2671
2672         default:
2673                 break;
2674         }
2675
2676         return sprintf(buf, "Vulnerable\n");
2677 }
2678
2679 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
2680 {
2681         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
2682 }
2683
2684 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
2685 {
2686         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
2687 }
2688
2689 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
2690 {
2691         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
2692 }
2693
2694 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
2695 {
2696         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
2697 }
2698
2699 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
2700 {
2701         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
2702 }
2703
2704 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
2705 {
2706         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
2707 }
2708
2709 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
2710 {
2711         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
2712 }
2713
2714 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
2715 {
2716         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
2717 }
2718
2719 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
2720 {
2721         return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
2722 }
2723
2724 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf)
2725 {
2726         if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
2727                 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN);
2728         else
2729                 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA);
2730 }
2731
2732 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf)
2733 {
2734         return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED);
2735 }
2736
2737 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf)
2738 {
2739         return cpu_show_common(dev, attr, buf, X86_BUG_GDS);
2740 }
2741
2742 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf)
2743 {
2744         return cpu_show_common(dev, attr, buf, X86_BUG_SRSO);
2745 }
2746 #endif