1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 1994 Linus Torvalds
5 * Cyrix stuff, June 1998 by:
6 * - Rafael R. Reilova (moved everything from head.S),
7 * <rreilova@ececs.uc.edu>
8 * - Channing Corn (tests & fixes),
9 * - Andrew D. Balsa (code cleanup).
11 #include <linux/init.h>
12 #include <linux/utsname.h>
13 #include <linux/cpu.h>
14 #include <linux/module.h>
15 #include <linux/nospec.h>
16 #include <linux/prctl.h>
17 #include <linux/sched/smt.h>
18 #include <linux/pgtable.h>
19 #include <linux/bpf.h>
21 #include <asm/spec-ctrl.h>
22 #include <asm/cmdline.h>
24 #include <asm/processor.h>
25 #include <asm/processor-flags.h>
26 #include <asm/fpu/api.h>
29 #include <asm/paravirt.h>
30 #include <asm/alternative.h>
31 #include <asm/set_memory.h>
32 #include <asm/intel-family.h>
33 #include <asm/e820/api.h>
34 #include <asm/hypervisor.h>
35 #include <asm/tlbflush.h>
39 static void __init spectre_v1_select_mitigation(void);
40 static void __init spectre_v2_select_mitigation(void);
41 static void __init retbleed_select_mitigation(void);
42 static void __init spectre_v2_user_select_mitigation(void);
43 static void __init ssb_select_mitigation(void);
44 static void __init l1tf_select_mitigation(void);
45 static void __init mds_select_mitigation(void);
46 static void __init md_clear_update_mitigation(void);
47 static void __init md_clear_select_mitigation(void);
48 static void __init taa_select_mitigation(void);
49 static void __init mmio_select_mitigation(void);
50 static void __init srbds_select_mitigation(void);
51 static void __init l1d_flush_select_mitigation(void);
53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
54 u64 x86_spec_ctrl_base;
55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
58 DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
59 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current);
61 static DEFINE_MUTEX(spec_ctrl_mutex);
64 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
65 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
67 void write_spec_ctrl_current(u64 val, bool force)
69 if (this_cpu_read(x86_spec_ctrl_current) == val)
72 this_cpu_write(x86_spec_ctrl_current, val);
75 * When KERNEL_IBRS this MSR is written on return-to-user, unless
76 * forced the update can be delayed until that time.
78 if (force || !cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS))
79 wrmsrl(MSR_IA32_SPEC_CTRL, val);
83 * The vendor and possibly platform specific bits which can be modified in
86 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
89 * AMD specific MSR info for Speculative Store Bypass control.
90 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
92 u64 __ro_after_init x86_amd_ls_cfg_base;
93 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
95 /* Control conditional STIBP in switch_to() */
96 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
97 /* Control conditional IBPB in switch_mm() */
98 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
99 /* Control unconditional IBPB in switch_mm() */
100 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
102 /* Control MDS CPU buffer clear before returning to user space */
103 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
104 EXPORT_SYMBOL_GPL(mds_user_clear);
105 /* Control MDS CPU buffer clear before idling (halt, mwait) */
106 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
107 EXPORT_SYMBOL_GPL(mds_idle_clear);
110 * Controls whether l1d flush based mitigations are enabled,
111 * based on hw features and admin setting via boot parameter
114 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
116 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */
117 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear);
118 EXPORT_SYMBOL_GPL(mmio_stale_data_clear);
120 void __init check_bugs(void)
125 * identify_boot_cpu() initialized SMT support information, let the
128 cpu_smt_check_topology();
130 if (!IS_ENABLED(CONFIG_SMP)) {
132 print_cpu_info(&boot_cpu_data);
136 * Read the SPEC_CTRL MSR to account for reserved bits which may
137 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
138 * init code as it is not enumerated and depends on the family.
140 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
141 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
143 /* Allow STIBP in MSR_SPEC_CTRL if supported */
144 if (boot_cpu_has(X86_FEATURE_STIBP))
145 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
147 /* Select the proper CPU mitigations before patching alternatives: */
148 spectre_v1_select_mitigation();
149 spectre_v2_select_mitigation();
151 * retbleed_select_mitigation() relies on the state set by
152 * spectre_v2_select_mitigation(); specifically it wants to know about
155 retbleed_select_mitigation();
157 * spectre_v2_user_select_mitigation() relies on the state set by
158 * retbleed_select_mitigation(); specifically the STIBP selection is
161 spectre_v2_user_select_mitigation();
162 ssb_select_mitigation();
163 l1tf_select_mitigation();
164 md_clear_select_mitigation();
165 srbds_select_mitigation();
166 l1d_flush_select_mitigation();
172 * Check whether we are able to run this kernel safely on SMP.
174 * - i386 is no longer supported.
175 * - In order to run on anything without a TSC, we need to be
176 * compiled for a i486.
178 if (boot_cpu_data.x86 < 4)
179 panic("Kernel requires i486+ for 'invlpg' and other features");
181 init_utsname()->machine[1] =
182 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
183 alternative_instructions();
185 fpu__init_check_bugs();
186 #else /* CONFIG_X86_64 */
187 alternative_instructions();
190 * Make sure the first 2MB area is not mapped by huge pages
191 * There are typically fixed size MTRRs in there and overlapping
192 * MTRRs into large pages causes slow downs.
194 * Right now we don't do that with gbpages because there seems
195 * very little benefit for that case.
198 set_memory_4k((unsigned long)__va(0), 1);
203 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
205 u64 msrval, guestval, hostval = x86_spec_ctrl_base;
206 struct thread_info *ti = current_thread_info();
208 /* Is MSR_SPEC_CTRL implemented ? */
209 if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
211 * Restrict guest_spec_ctrl to supported values. Clear the
212 * modifiable bits in the host base value and or the
213 * modifiable bits from the guest value.
215 guestval = hostval & ~x86_spec_ctrl_mask;
216 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
218 /* SSBD controlled in MSR_SPEC_CTRL */
219 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
220 static_cpu_has(X86_FEATURE_AMD_SSBD))
221 hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
223 /* Conditional STIBP enabled? */
224 if (static_branch_unlikely(&switch_to_cond_stibp))
225 hostval |= stibp_tif_to_spec_ctrl(ti->flags);
227 if (hostval != guestval) {
228 msrval = setguest ? guestval : hostval;
229 wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
234 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
235 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
237 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
238 !static_cpu_has(X86_FEATURE_VIRT_SSBD))
242 * If the host has SSBD mitigation enabled, force it in the host's
243 * virtual MSR value. If its not permanently enabled, evaluate
244 * current's TIF_SSBD thread flag.
246 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
247 hostval = SPEC_CTRL_SSBD;
249 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
251 /* Sanitize the guest value */
252 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
254 if (hostval != guestval) {
257 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
258 ssbd_spec_ctrl_to_tif(hostval);
260 speculation_ctrl_update(tif);
263 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
265 static void x86_amd_ssb_disable(void)
267 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
269 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
270 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
271 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
272 wrmsrl(MSR_AMD64_LS_CFG, msrval);
276 #define pr_fmt(fmt) "MDS: " fmt
278 /* Default mitigation for MDS-affected CPUs */
279 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
280 static bool mds_nosmt __ro_after_init = false;
282 static const char * const mds_strings[] = {
283 [MDS_MITIGATION_OFF] = "Vulnerable",
284 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers",
285 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
288 static void __init mds_select_mitigation(void)
290 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
291 mds_mitigation = MDS_MITIGATION_OFF;
295 if (mds_mitigation == MDS_MITIGATION_FULL) {
296 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
297 mds_mitigation = MDS_MITIGATION_VMWERV;
299 static_branch_enable(&mds_user_clear);
301 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
302 (mds_nosmt || cpu_mitigations_auto_nosmt()))
303 cpu_smt_disable(false);
307 static int __init mds_cmdline(char *str)
309 if (!boot_cpu_has_bug(X86_BUG_MDS))
315 if (!strcmp(str, "off"))
316 mds_mitigation = MDS_MITIGATION_OFF;
317 else if (!strcmp(str, "full"))
318 mds_mitigation = MDS_MITIGATION_FULL;
319 else if (!strcmp(str, "full,nosmt")) {
320 mds_mitigation = MDS_MITIGATION_FULL;
326 early_param("mds", mds_cmdline);
329 #define pr_fmt(fmt) "TAA: " fmt
331 enum taa_mitigations {
333 TAA_MITIGATION_UCODE_NEEDED,
335 TAA_MITIGATION_TSX_DISABLED,
338 /* Default mitigation for TAA-affected CPUs */
339 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
340 static bool taa_nosmt __ro_after_init;
342 static const char * const taa_strings[] = {
343 [TAA_MITIGATION_OFF] = "Vulnerable",
344 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode",
345 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers",
346 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled",
349 static void __init taa_select_mitigation(void)
353 if (!boot_cpu_has_bug(X86_BUG_TAA)) {
354 taa_mitigation = TAA_MITIGATION_OFF;
358 /* TSX previously disabled by tsx=off */
359 if (!boot_cpu_has(X86_FEATURE_RTM)) {
360 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
364 if (cpu_mitigations_off()) {
365 taa_mitigation = TAA_MITIGATION_OFF;
370 * TAA mitigation via VERW is turned off if both
371 * tsx_async_abort=off and mds=off are specified.
373 if (taa_mitigation == TAA_MITIGATION_OFF &&
374 mds_mitigation == MDS_MITIGATION_OFF)
377 if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
378 taa_mitigation = TAA_MITIGATION_VERW;
380 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
383 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
384 * A microcode update fixes this behavior to clear CPU buffers. It also
385 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
386 * ARCH_CAP_TSX_CTRL_MSR bit.
388 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
389 * update is required.
391 ia32_cap = x86_read_arch_cap_msr();
392 if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
393 !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
394 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
397 * TSX is enabled, select alternate mitigation for TAA which is
398 * the same as MDS. Enable MDS static branch to clear CPU buffers.
400 * For guests that can't determine whether the correct microcode is
401 * present on host, enable the mitigation for UCODE_NEEDED as well.
403 static_branch_enable(&mds_user_clear);
405 if (taa_nosmt || cpu_mitigations_auto_nosmt())
406 cpu_smt_disable(false);
409 static int __init tsx_async_abort_parse_cmdline(char *str)
411 if (!boot_cpu_has_bug(X86_BUG_TAA))
417 if (!strcmp(str, "off")) {
418 taa_mitigation = TAA_MITIGATION_OFF;
419 } else if (!strcmp(str, "full")) {
420 taa_mitigation = TAA_MITIGATION_VERW;
421 } else if (!strcmp(str, "full,nosmt")) {
422 taa_mitigation = TAA_MITIGATION_VERW;
428 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
431 #define pr_fmt(fmt) "MMIO Stale Data: " fmt
433 enum mmio_mitigations {
435 MMIO_MITIGATION_UCODE_NEEDED,
436 MMIO_MITIGATION_VERW,
439 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */
440 static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW;
441 static bool mmio_nosmt __ro_after_init = false;
443 static const char * const mmio_strings[] = {
444 [MMIO_MITIGATION_OFF] = "Vulnerable",
445 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode",
446 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers",
449 static void __init mmio_select_mitigation(void)
453 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) ||
454 cpu_mitigations_off()) {
455 mmio_mitigation = MMIO_MITIGATION_OFF;
459 if (mmio_mitigation == MMIO_MITIGATION_OFF)
462 ia32_cap = x86_read_arch_cap_msr();
465 * Enable CPU buffer clear mitigation for host and VMM, if also affected
466 * by MDS or TAA. Otherwise, enable mitigation for VMM only.
468 if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) &&
469 boot_cpu_has(X86_FEATURE_RTM)))
470 static_branch_enable(&mds_user_clear);
472 static_branch_enable(&mmio_stale_data_clear);
475 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
476 * be propagated to uncore buffers, clearing the Fill buffers on idle
477 * is required irrespective of SMT state.
479 if (!(ia32_cap & ARCH_CAP_FBSDP_NO))
480 static_branch_enable(&mds_idle_clear);
483 * Check if the system has the right microcode.
485 * CPU Fill buffer clear mitigation is enumerated by either an explicit
486 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
489 if ((ia32_cap & ARCH_CAP_FB_CLEAR) ||
490 (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
491 boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
492 !(ia32_cap & ARCH_CAP_MDS_NO)))
493 mmio_mitigation = MMIO_MITIGATION_VERW;
495 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
497 if (mmio_nosmt || cpu_mitigations_auto_nosmt())
498 cpu_smt_disable(false);
501 static int __init mmio_stale_data_parse_cmdline(char *str)
503 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
509 if (!strcmp(str, "off")) {
510 mmio_mitigation = MMIO_MITIGATION_OFF;
511 } else if (!strcmp(str, "full")) {
512 mmio_mitigation = MMIO_MITIGATION_VERW;
513 } else if (!strcmp(str, "full,nosmt")) {
514 mmio_mitigation = MMIO_MITIGATION_VERW;
520 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline);
523 #define pr_fmt(fmt) "" fmt
525 static void __init md_clear_update_mitigation(void)
527 if (cpu_mitigations_off())
530 if (!static_key_enabled(&mds_user_clear))
534 * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data
535 * mitigation, if necessary.
537 if (mds_mitigation == MDS_MITIGATION_OFF &&
538 boot_cpu_has_bug(X86_BUG_MDS)) {
539 mds_mitigation = MDS_MITIGATION_FULL;
540 mds_select_mitigation();
542 if (taa_mitigation == TAA_MITIGATION_OFF &&
543 boot_cpu_has_bug(X86_BUG_TAA)) {
544 taa_mitigation = TAA_MITIGATION_VERW;
545 taa_select_mitigation();
547 if (mmio_mitigation == MMIO_MITIGATION_OFF &&
548 boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) {
549 mmio_mitigation = MMIO_MITIGATION_VERW;
550 mmio_select_mitigation();
553 if (boot_cpu_has_bug(X86_BUG_MDS))
554 pr_info("MDS: %s\n", mds_strings[mds_mitigation]);
555 if (boot_cpu_has_bug(X86_BUG_TAA))
556 pr_info("TAA: %s\n", taa_strings[taa_mitigation]);
557 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
558 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]);
561 static void __init md_clear_select_mitigation(void)
563 mds_select_mitigation();
564 taa_select_mitigation();
565 mmio_select_mitigation();
568 * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update
569 * and print their mitigation after MDS, TAA and MMIO Stale Data
570 * mitigation selection is done.
572 md_clear_update_mitigation();
576 #define pr_fmt(fmt) "SRBDS: " fmt
578 enum srbds_mitigations {
579 SRBDS_MITIGATION_OFF,
580 SRBDS_MITIGATION_UCODE_NEEDED,
581 SRBDS_MITIGATION_FULL,
582 SRBDS_MITIGATION_TSX_OFF,
583 SRBDS_MITIGATION_HYPERVISOR,
586 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL;
588 static const char * const srbds_strings[] = {
589 [SRBDS_MITIGATION_OFF] = "Vulnerable",
590 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
591 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode",
592 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled",
593 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status",
596 static bool srbds_off;
598 void update_srbds_msr(void)
602 if (!boot_cpu_has_bug(X86_BUG_SRBDS))
605 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
608 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
612 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX
613 * being disabled and it hasn't received the SRBDS MSR microcode.
615 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
618 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
620 switch (srbds_mitigation) {
621 case SRBDS_MITIGATION_OFF:
622 case SRBDS_MITIGATION_TSX_OFF:
623 mcu_ctrl |= RNGDS_MITG_DIS;
625 case SRBDS_MITIGATION_FULL:
626 mcu_ctrl &= ~RNGDS_MITG_DIS;
632 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
635 static void __init srbds_select_mitigation(void)
639 if (!boot_cpu_has_bug(X86_BUG_SRBDS))
643 * Check to see if this is one of the MDS_NO systems supporting TSX that
644 * are only exposed to SRBDS when TSX is enabled or when CPU is affected
645 * by Processor MMIO Stale Data vulnerability.
647 ia32_cap = x86_read_arch_cap_msr();
648 if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) &&
649 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
650 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
651 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
652 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
653 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
654 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
655 else if (cpu_mitigations_off() || srbds_off)
656 srbds_mitigation = SRBDS_MITIGATION_OFF;
659 pr_info("%s\n", srbds_strings[srbds_mitigation]);
662 static int __init srbds_parse_cmdline(char *str)
667 if (!boot_cpu_has_bug(X86_BUG_SRBDS))
670 srbds_off = !strcmp(str, "off");
673 early_param("srbds", srbds_parse_cmdline);
676 #define pr_fmt(fmt) "L1D Flush : " fmt
678 enum l1d_flush_mitigations {
683 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF;
685 static void __init l1d_flush_select_mitigation(void)
687 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D))
690 static_branch_enable(&switch_mm_cond_l1d_flush);
691 pr_info("Conditional flush on switch_mm() enabled\n");
694 static int __init l1d_flush_parse_cmdline(char *str)
696 if (!strcmp(str, "on"))
697 l1d_flush_mitigation = L1D_FLUSH_ON;
701 early_param("l1d_flush", l1d_flush_parse_cmdline);
704 #define pr_fmt(fmt) "Spectre V1 : " fmt
706 enum spectre_v1_mitigation {
707 SPECTRE_V1_MITIGATION_NONE,
708 SPECTRE_V1_MITIGATION_AUTO,
711 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
712 SPECTRE_V1_MITIGATION_AUTO;
714 static const char * const spectre_v1_strings[] = {
715 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
716 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
720 * Does SMAP provide full mitigation against speculative kernel access to
723 static bool smap_works_speculatively(void)
725 if (!boot_cpu_has(X86_FEATURE_SMAP))
729 * On CPUs which are vulnerable to Meltdown, SMAP does not
730 * prevent speculative access to user data in the L1 cache.
731 * Consider SMAP to be non-functional as a mitigation on these
734 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
740 static void __init spectre_v1_select_mitigation(void)
742 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
743 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
747 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
749 * With Spectre v1, a user can speculatively control either
750 * path of a conditional swapgs with a user-controlled GS
751 * value. The mitigation is to add lfences to both code paths.
753 * If FSGSBASE is enabled, the user can put a kernel address in
754 * GS, in which case SMAP provides no protection.
756 * If FSGSBASE is disabled, the user can only put a user space
757 * address in GS. That makes an attack harder, but still
758 * possible if there's no SMAP protection.
760 if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
761 !smap_works_speculatively()) {
763 * Mitigation can be provided from SWAPGS itself or
764 * PTI as the CR3 write in the Meltdown mitigation
767 * If neither is there, mitigate with an LFENCE to
768 * stop speculation through swapgs.
770 if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
771 !boot_cpu_has(X86_FEATURE_PTI))
772 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
775 * Enable lfences in the kernel entry (non-swapgs)
776 * paths, to prevent user entry from speculatively
779 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
783 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
786 static int __init nospectre_v1_cmdline(char *str)
788 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
791 early_param("nospectre_v1", nospectre_v1_cmdline);
793 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
797 #define pr_fmt(fmt) "RETBleed: " fmt
799 enum retbleed_mitigation {
800 RETBLEED_MITIGATION_NONE,
801 RETBLEED_MITIGATION_UNRET,
802 RETBLEED_MITIGATION_IBRS,
803 RETBLEED_MITIGATION_EIBRS,
806 enum retbleed_mitigation_cmd {
812 const char * const retbleed_strings[] = {
813 [RETBLEED_MITIGATION_NONE] = "Vulnerable",
814 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk",
815 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS",
816 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS",
819 static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
820 RETBLEED_MITIGATION_NONE;
821 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init =
824 static int __ro_after_init retbleed_nosmt = false;
826 static int __init retbleed_parse_cmdline(char *str)
832 char *next = strchr(str, ',');
838 if (!strcmp(str, "off")) {
839 retbleed_cmd = RETBLEED_CMD_OFF;
840 } else if (!strcmp(str, "auto")) {
841 retbleed_cmd = RETBLEED_CMD_AUTO;
842 } else if (!strcmp(str, "unret")) {
843 retbleed_cmd = RETBLEED_CMD_UNRET;
844 } else if (!strcmp(str, "nosmt")) {
845 retbleed_nosmt = true;
847 pr_err("Ignoring unknown retbleed option (%s).", str);
855 early_param("retbleed", retbleed_parse_cmdline);
857 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
858 #define RETBLEED_COMPILER_MSG "WARNING: kernel not compiled with RETPOLINE or -mfunction-return capable compiler!\n"
859 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
861 static void __init retbleed_select_mitigation(void)
863 if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())
866 switch (retbleed_cmd) {
867 case RETBLEED_CMD_OFF:
870 case RETBLEED_CMD_UNRET:
871 retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
874 case RETBLEED_CMD_AUTO:
876 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
877 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
878 retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
881 * The Intel mitigation (IBRS) was already selected in
882 * spectre_v2_select_mitigation().
888 switch (retbleed_mitigation) {
889 case RETBLEED_MITIGATION_UNRET:
891 if (!IS_ENABLED(CONFIG_RETPOLINE) ||
892 !IS_ENABLED(CONFIG_CC_HAS_RETURN_THUNK)) {
893 pr_err(RETBLEED_COMPILER_MSG);
894 retbleed_mitigation = RETBLEED_MITIGATION_NONE;
898 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
899 setup_force_cpu_cap(X86_FEATURE_UNRET);
901 if (!boot_cpu_has(X86_FEATURE_STIBP) &&
902 (retbleed_nosmt || cpu_mitigations_auto_nosmt()))
903 cpu_smt_disable(false);
905 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
906 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
907 pr_err(RETBLEED_UNTRAIN_MSG);
915 * Let IBRS trump all on Intel without affecting the effects of the
916 * retbleed= cmdline option.
918 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
919 switch (spectre_v2_enabled) {
920 case SPECTRE_V2_IBRS:
921 retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
923 case SPECTRE_V2_EIBRS:
924 case SPECTRE_V2_EIBRS_RETPOLINE:
925 case SPECTRE_V2_EIBRS_LFENCE:
926 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
929 pr_err(RETBLEED_INTEL_MSG);
933 pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
937 #define pr_fmt(fmt) "Spectre V2 : " fmt
939 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
940 SPECTRE_V2_USER_NONE;
941 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
942 SPECTRE_V2_USER_NONE;
944 #ifdef CONFIG_RETPOLINE
945 static bool spectre_v2_bad_module;
947 bool retpoline_module_ok(bool has_retpoline)
949 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
952 pr_err("System may be vulnerable to spectre v2\n");
953 spectre_v2_bad_module = true;
957 static inline const char *spectre_v2_module_string(void)
959 return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
962 static inline const char *spectre_v2_module_string(void) { return ""; }
965 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
966 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
967 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
969 #ifdef CONFIG_BPF_SYSCALL
970 void unpriv_ebpf_notify(int new_state)
975 /* Unprivileged eBPF is enabled */
977 switch (spectre_v2_enabled) {
978 case SPECTRE_V2_EIBRS:
979 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
981 case SPECTRE_V2_EIBRS_LFENCE:
982 if (sched_smt_active())
983 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
991 static inline bool match_option(const char *arg, int arglen, const char *opt)
993 int len = strlen(opt);
995 return len == arglen && !strncmp(arg, opt, len);
998 /* The kernel command line selection for spectre v2 */
999 enum spectre_v2_mitigation_cmd {
1000 SPECTRE_V2_CMD_NONE,
1001 SPECTRE_V2_CMD_AUTO,
1002 SPECTRE_V2_CMD_FORCE,
1003 SPECTRE_V2_CMD_RETPOLINE,
1004 SPECTRE_V2_CMD_RETPOLINE_GENERIC,
1005 SPECTRE_V2_CMD_RETPOLINE_LFENCE,
1006 SPECTRE_V2_CMD_EIBRS,
1007 SPECTRE_V2_CMD_EIBRS_RETPOLINE,
1008 SPECTRE_V2_CMD_EIBRS_LFENCE,
1009 SPECTRE_V2_CMD_IBRS,
1012 enum spectre_v2_user_cmd {
1013 SPECTRE_V2_USER_CMD_NONE,
1014 SPECTRE_V2_USER_CMD_AUTO,
1015 SPECTRE_V2_USER_CMD_FORCE,
1016 SPECTRE_V2_USER_CMD_PRCTL,
1017 SPECTRE_V2_USER_CMD_PRCTL_IBPB,
1018 SPECTRE_V2_USER_CMD_SECCOMP,
1019 SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
1022 static const char * const spectre_v2_user_strings[] = {
1023 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable",
1024 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection",
1025 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection",
1026 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl",
1027 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl",
1030 static const struct {
1032 enum spectre_v2_user_cmd cmd;
1034 } v2_user_options[] __initconst = {
1035 { "auto", SPECTRE_V2_USER_CMD_AUTO, false },
1036 { "off", SPECTRE_V2_USER_CMD_NONE, false },
1037 { "on", SPECTRE_V2_USER_CMD_FORCE, true },
1038 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false },
1039 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false },
1040 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false },
1041 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false },
1044 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
1046 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1047 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
1050 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd;
1052 static enum spectre_v2_user_cmd __init
1053 spectre_v2_parse_user_cmdline(void)
1058 switch (spectre_v2_cmd) {
1059 case SPECTRE_V2_CMD_NONE:
1060 return SPECTRE_V2_USER_CMD_NONE;
1061 case SPECTRE_V2_CMD_FORCE:
1062 return SPECTRE_V2_USER_CMD_FORCE;
1067 ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
1070 return SPECTRE_V2_USER_CMD_AUTO;
1072 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
1073 if (match_option(arg, ret, v2_user_options[i].option)) {
1074 spec_v2_user_print_cond(v2_user_options[i].option,
1075 v2_user_options[i].secure);
1076 return v2_user_options[i].cmd;
1080 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
1081 return SPECTRE_V2_USER_CMD_AUTO;
1084 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
1086 return mode == SPECTRE_V2_IBRS ||
1087 mode == SPECTRE_V2_EIBRS ||
1088 mode == SPECTRE_V2_EIBRS_RETPOLINE ||
1089 mode == SPECTRE_V2_EIBRS_LFENCE;
1093 spectre_v2_user_select_mitigation(void)
1095 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
1096 bool smt_possible = IS_ENABLED(CONFIG_SMP);
1097 enum spectre_v2_user_cmd cmd;
1099 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
1102 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
1103 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
1104 smt_possible = false;
1106 cmd = spectre_v2_parse_user_cmdline();
1108 case SPECTRE_V2_USER_CMD_NONE:
1110 case SPECTRE_V2_USER_CMD_FORCE:
1111 mode = SPECTRE_V2_USER_STRICT;
1113 case SPECTRE_V2_USER_CMD_AUTO:
1114 case SPECTRE_V2_USER_CMD_PRCTL:
1115 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1116 mode = SPECTRE_V2_USER_PRCTL;
1118 case SPECTRE_V2_USER_CMD_SECCOMP:
1119 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1120 if (IS_ENABLED(CONFIG_SECCOMP))
1121 mode = SPECTRE_V2_USER_SECCOMP;
1123 mode = SPECTRE_V2_USER_PRCTL;
1127 /* Initialize Indirect Branch Prediction Barrier */
1128 if (boot_cpu_has(X86_FEATURE_IBPB)) {
1129 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
1131 spectre_v2_user_ibpb = mode;
1133 case SPECTRE_V2_USER_CMD_FORCE:
1134 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1135 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1136 static_branch_enable(&switch_mm_always_ibpb);
1137 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
1139 case SPECTRE_V2_USER_CMD_PRCTL:
1140 case SPECTRE_V2_USER_CMD_AUTO:
1141 case SPECTRE_V2_USER_CMD_SECCOMP:
1142 static_branch_enable(&switch_mm_cond_ibpb);
1148 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
1149 static_key_enabled(&switch_mm_always_ibpb) ?
1150 "always-on" : "conditional");
1154 * If no STIBP, IBRS or enhanced IBRS is enabled, or SMT impossible,
1155 * STIBP is not required.
1157 if (!boot_cpu_has(X86_FEATURE_STIBP) ||
1159 spectre_v2_in_ibrs_mode(spectre_v2_enabled))
1163 * At this point, an STIBP mode other than "off" has been set.
1164 * If STIBP support is not being forced, check if STIBP always-on
1167 if (mode != SPECTRE_V2_USER_STRICT &&
1168 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
1169 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1171 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET) {
1172 if (mode != SPECTRE_V2_USER_STRICT &&
1173 mode != SPECTRE_V2_USER_STRICT_PREFERRED)
1174 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation'\n");
1175 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1178 spectre_v2_user_stibp = mode;
1181 pr_info("%s\n", spectre_v2_user_strings[mode]);
1184 static const char * const spectre_v2_strings[] = {
1185 [SPECTRE_V2_NONE] = "Vulnerable",
1186 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines",
1187 [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE",
1188 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS",
1189 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE",
1190 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines",
1191 [SPECTRE_V2_IBRS] = "Mitigation: IBRS",
1194 static const struct {
1196 enum spectre_v2_mitigation_cmd cmd;
1198 } mitigation_options[] __initconst = {
1199 { "off", SPECTRE_V2_CMD_NONE, false },
1200 { "on", SPECTRE_V2_CMD_FORCE, true },
1201 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false },
1202 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
1203 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
1204 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
1205 { "eibrs", SPECTRE_V2_CMD_EIBRS, false },
1206 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false },
1207 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false },
1208 { "auto", SPECTRE_V2_CMD_AUTO, false },
1209 { "ibrs", SPECTRE_V2_CMD_IBRS, false },
1212 static void __init spec_v2_print_cond(const char *reason, bool secure)
1214 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1215 pr_info("%s selected on command line.\n", reason);
1218 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
1220 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
1224 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
1225 cpu_mitigations_off())
1226 return SPECTRE_V2_CMD_NONE;
1228 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
1230 return SPECTRE_V2_CMD_AUTO;
1232 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
1233 if (!match_option(arg, ret, mitigation_options[i].option))
1235 cmd = mitigation_options[i].cmd;
1239 if (i >= ARRAY_SIZE(mitigation_options)) {
1240 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1241 return SPECTRE_V2_CMD_AUTO;
1244 if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
1245 cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1246 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
1247 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1248 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1249 !IS_ENABLED(CONFIG_RETPOLINE)) {
1250 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1251 mitigation_options[i].option);
1252 return SPECTRE_V2_CMD_AUTO;
1255 if ((cmd == SPECTRE_V2_CMD_EIBRS ||
1256 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1257 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1258 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1259 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
1260 mitigation_options[i].option);
1261 return SPECTRE_V2_CMD_AUTO;
1264 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1265 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
1266 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
1267 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
1268 mitigation_options[i].option);
1269 return SPECTRE_V2_CMD_AUTO;
1272 if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
1273 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",
1274 mitigation_options[i].option);
1275 return SPECTRE_V2_CMD_AUTO;
1278 if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) {
1279 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",
1280 mitigation_options[i].option);
1281 return SPECTRE_V2_CMD_AUTO;
1284 if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_has(X86_FEATURE_XENPV)) {
1285 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",
1286 mitigation_options[i].option);
1287 return SPECTRE_V2_CMD_AUTO;
1290 spec_v2_print_cond(mitigation_options[i].option,
1291 mitigation_options[i].secure);
1295 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
1297 if (!IS_ENABLED(CONFIG_RETPOLINE)) {
1298 pr_err("Kernel not compiled with retpoline; no mitigation available!");
1299 return SPECTRE_V2_NONE;
1302 return SPECTRE_V2_RETPOLINE;
1305 static void __init spectre_v2_select_mitigation(void)
1307 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
1308 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
1311 * If the CPU is not affected and the command line mode is NONE or AUTO
1312 * then nothing to do.
1314 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
1315 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
1319 case SPECTRE_V2_CMD_NONE:
1322 case SPECTRE_V2_CMD_FORCE:
1323 case SPECTRE_V2_CMD_AUTO:
1324 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1325 mode = SPECTRE_V2_EIBRS;
1329 if (boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1330 retbleed_cmd != RETBLEED_CMD_OFF &&
1331 boot_cpu_has(X86_FEATURE_IBRS) &&
1332 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1333 mode = SPECTRE_V2_IBRS;
1337 mode = spectre_v2_select_retpoline();
1340 case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
1341 pr_err(SPECTRE_V2_LFENCE_MSG);
1342 mode = SPECTRE_V2_LFENCE;
1345 case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
1346 mode = SPECTRE_V2_RETPOLINE;
1349 case SPECTRE_V2_CMD_RETPOLINE:
1350 mode = spectre_v2_select_retpoline();
1353 case SPECTRE_V2_CMD_IBRS:
1354 mode = SPECTRE_V2_IBRS;
1357 case SPECTRE_V2_CMD_EIBRS:
1358 mode = SPECTRE_V2_EIBRS;
1361 case SPECTRE_V2_CMD_EIBRS_LFENCE:
1362 mode = SPECTRE_V2_EIBRS_LFENCE;
1365 case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
1366 mode = SPECTRE_V2_EIBRS_RETPOLINE;
1370 if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
1371 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1373 if (spectre_v2_in_ibrs_mode(mode)) {
1374 /* Force it so VMEXIT will restore correctly */
1375 x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
1376 write_spec_ctrl_current(x86_spec_ctrl_base, true);
1380 case SPECTRE_V2_NONE:
1381 case SPECTRE_V2_EIBRS:
1384 case SPECTRE_V2_IBRS:
1385 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS);
1388 case SPECTRE_V2_LFENCE:
1389 case SPECTRE_V2_EIBRS_LFENCE:
1390 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
1393 case SPECTRE_V2_RETPOLINE:
1394 case SPECTRE_V2_EIBRS_RETPOLINE:
1395 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
1399 spectre_v2_enabled = mode;
1400 pr_info("%s\n", spectre_v2_strings[mode]);
1403 * If spectre v2 protection has been enabled, unconditionally fill
1404 * RSB during a context switch; this protects against two independent
1407 * - RSB underflow (and switch to BTB) on Skylake+
1408 * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
1410 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
1411 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1414 * Retpoline protects the kernel, but doesn't protect firmware. IBRS
1415 * and Enhanced IBRS protect firmware too, so enable IBRS around
1416 * firmware calls only when IBRS / Enhanced IBRS aren't otherwise
1419 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1420 * the user might select retpoline on the kernel command line and if
1421 * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1422 * enable IBRS around firmware calls.
1424 if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) {
1425 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
1426 pr_info("Enabling Restricted Speculation for firmware calls\n");
1429 /* Set up IBPB and STIBP depending on the general spectre V2 command */
1430 spectre_v2_cmd = cmd;
1433 static void update_stibp_msr(void * __unused)
1435 write_spec_ctrl_current(x86_spec_ctrl_base, true);
1438 /* Update x86_spec_ctrl_base in case SMT state changed. */
1439 static void update_stibp_strict(void)
1441 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
1443 if (sched_smt_active())
1444 mask |= SPEC_CTRL_STIBP;
1446 if (mask == x86_spec_ctrl_base)
1449 pr_info("Update user space SMT mitigation: STIBP %s\n",
1450 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
1451 x86_spec_ctrl_base = mask;
1452 on_each_cpu(update_stibp_msr, NULL, 1);
1455 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1456 static void update_indir_branch_cond(void)
1458 if (sched_smt_active())
1459 static_branch_enable(&switch_to_cond_stibp);
1461 static_branch_disable(&switch_to_cond_stibp);
1465 #define pr_fmt(fmt) fmt
1467 /* Update the static key controlling the MDS CPU buffer clear in idle */
1468 static void update_mds_branch_idle(void)
1470 u64 ia32_cap = x86_read_arch_cap_msr();
1473 * Enable the idle clearing if SMT is active on CPUs which are
1474 * affected only by MSBDS and not any other MDS variant.
1476 * The other variants cannot be mitigated when SMT is enabled, so
1477 * clearing the buffers on idle just to prevent the Store Buffer
1478 * repartitioning leak would be a window dressing exercise.
1480 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
1483 if (sched_smt_active()) {
1484 static_branch_enable(&mds_idle_clear);
1485 } else if (mmio_mitigation == MMIO_MITIGATION_OFF ||
1486 (ia32_cap & ARCH_CAP_FBSDP_NO)) {
1487 static_branch_disable(&mds_idle_clear);
1491 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1492 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1493 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
1495 void cpu_bugs_smt_update(void)
1497 mutex_lock(&spec_ctrl_mutex);
1499 if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1500 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1501 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1503 switch (spectre_v2_user_stibp) {
1504 case SPECTRE_V2_USER_NONE:
1506 case SPECTRE_V2_USER_STRICT:
1507 case SPECTRE_V2_USER_STRICT_PREFERRED:
1508 update_stibp_strict();
1510 case SPECTRE_V2_USER_PRCTL:
1511 case SPECTRE_V2_USER_SECCOMP:
1512 update_indir_branch_cond();
1516 switch (mds_mitigation) {
1517 case MDS_MITIGATION_FULL:
1518 case MDS_MITIGATION_VMWERV:
1519 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
1520 pr_warn_once(MDS_MSG_SMT);
1521 update_mds_branch_idle();
1523 case MDS_MITIGATION_OFF:
1527 switch (taa_mitigation) {
1528 case TAA_MITIGATION_VERW:
1529 case TAA_MITIGATION_UCODE_NEEDED:
1530 if (sched_smt_active())
1531 pr_warn_once(TAA_MSG_SMT);
1533 case TAA_MITIGATION_TSX_DISABLED:
1534 case TAA_MITIGATION_OFF:
1538 switch (mmio_mitigation) {
1539 case MMIO_MITIGATION_VERW:
1540 case MMIO_MITIGATION_UCODE_NEEDED:
1541 if (sched_smt_active())
1542 pr_warn_once(MMIO_MSG_SMT);
1544 case MMIO_MITIGATION_OFF:
1548 mutex_unlock(&spec_ctrl_mutex);
1552 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt
1554 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
1556 /* The kernel command line selection */
1557 enum ssb_mitigation_cmd {
1558 SPEC_STORE_BYPASS_CMD_NONE,
1559 SPEC_STORE_BYPASS_CMD_AUTO,
1560 SPEC_STORE_BYPASS_CMD_ON,
1561 SPEC_STORE_BYPASS_CMD_PRCTL,
1562 SPEC_STORE_BYPASS_CMD_SECCOMP,
1565 static const char * const ssb_strings[] = {
1566 [SPEC_STORE_BYPASS_NONE] = "Vulnerable",
1567 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled",
1568 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl",
1569 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
1572 static const struct {
1574 enum ssb_mitigation_cmd cmd;
1575 } ssb_mitigation_options[] __initconst = {
1576 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */
1577 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */
1578 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */
1579 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */
1580 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
1583 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
1585 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
1589 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
1590 cpu_mitigations_off()) {
1591 return SPEC_STORE_BYPASS_CMD_NONE;
1593 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
1596 return SPEC_STORE_BYPASS_CMD_AUTO;
1598 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
1599 if (!match_option(arg, ret, ssb_mitigation_options[i].option))
1602 cmd = ssb_mitigation_options[i].cmd;
1606 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
1607 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1608 return SPEC_STORE_BYPASS_CMD_AUTO;
1615 static enum ssb_mitigation __init __ssb_select_mitigation(void)
1617 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
1618 enum ssb_mitigation_cmd cmd;
1620 if (!boot_cpu_has(X86_FEATURE_SSBD))
1623 cmd = ssb_parse_cmdline();
1624 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
1625 (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
1626 cmd == SPEC_STORE_BYPASS_CMD_AUTO))
1630 case SPEC_STORE_BYPASS_CMD_SECCOMP:
1632 * Choose prctl+seccomp as the default mode if seccomp is
1635 if (IS_ENABLED(CONFIG_SECCOMP))
1636 mode = SPEC_STORE_BYPASS_SECCOMP;
1638 mode = SPEC_STORE_BYPASS_PRCTL;
1640 case SPEC_STORE_BYPASS_CMD_ON:
1641 mode = SPEC_STORE_BYPASS_DISABLE;
1643 case SPEC_STORE_BYPASS_CMD_AUTO:
1644 case SPEC_STORE_BYPASS_CMD_PRCTL:
1645 mode = SPEC_STORE_BYPASS_PRCTL;
1647 case SPEC_STORE_BYPASS_CMD_NONE:
1652 * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper
1653 * bit in the mask to allow guests to use the mitigation even in the
1654 * case where the host does not enable it.
1656 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
1657 static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1658 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
1662 * We have three CPU feature flags that are in play here:
1663 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1664 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1665 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1667 if (mode == SPEC_STORE_BYPASS_DISABLE) {
1668 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
1670 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1671 * use a completely different MSR and bit dependent on family.
1673 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
1674 !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1675 x86_amd_ssb_disable();
1677 x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
1678 write_spec_ctrl_current(x86_spec_ctrl_base, true);
1685 static void ssb_select_mitigation(void)
1687 ssb_mode = __ssb_select_mitigation();
1689 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1690 pr_info("%s\n", ssb_strings[ssb_mode]);
1694 #define pr_fmt(fmt) "Speculation prctl: " fmt
1696 static void task_update_spec_tif(struct task_struct *tsk)
1698 /* Force the update of the real TIF bits */
1699 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
1702 * Immediately update the speculation control MSRs for the current
1703 * task, but for a non-current task delay setting the CPU
1704 * mitigation until it is scheduled next.
1706 * This can only happen for SECCOMP mitigation. For PRCTL it's
1707 * always the current task.
1710 speculation_ctrl_update_current();
1713 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl)
1716 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
1720 case PR_SPEC_ENABLE:
1721 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
1723 case PR_SPEC_DISABLE:
1724 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
1731 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
1733 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
1734 ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
1738 case PR_SPEC_ENABLE:
1739 /* If speculation is force disabled, enable is not allowed */
1740 if (task_spec_ssb_force_disable(task))
1742 task_clear_spec_ssb_disable(task);
1743 task_clear_spec_ssb_noexec(task);
1744 task_update_spec_tif(task);
1746 case PR_SPEC_DISABLE:
1747 task_set_spec_ssb_disable(task);
1748 task_clear_spec_ssb_noexec(task);
1749 task_update_spec_tif(task);
1751 case PR_SPEC_FORCE_DISABLE:
1752 task_set_spec_ssb_disable(task);
1753 task_set_spec_ssb_force_disable(task);
1754 task_clear_spec_ssb_noexec(task);
1755 task_update_spec_tif(task);
1757 case PR_SPEC_DISABLE_NOEXEC:
1758 if (task_spec_ssb_force_disable(task))
1760 task_set_spec_ssb_disable(task);
1761 task_set_spec_ssb_noexec(task);
1762 task_update_spec_tif(task);
1770 static bool is_spec_ib_user_controlled(void)
1772 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
1773 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
1774 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
1775 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
1778 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
1781 case PR_SPEC_ENABLE:
1782 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1783 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1787 * With strict mode for both IBPB and STIBP, the instruction
1788 * code paths avoid checking this task flag and instead,
1789 * unconditionally run the instruction. However, STIBP and IBPB
1790 * are independent and either can be set to conditionally
1791 * enabled regardless of the mode of the other.
1793 * If either is set to conditional, allow the task flag to be
1794 * updated, unless it was force-disabled by a previous prctl
1795 * call. Currently, this is possible on an AMD CPU which has the
1796 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
1797 * kernel is booted with 'spectre_v2_user=seccomp', then
1798 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
1799 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
1801 if (!is_spec_ib_user_controlled() ||
1802 task_spec_ib_force_disable(task))
1805 task_clear_spec_ib_disable(task);
1806 task_update_spec_tif(task);
1808 case PR_SPEC_DISABLE:
1809 case PR_SPEC_FORCE_DISABLE:
1811 * Indirect branch speculation is always allowed when
1812 * mitigation is force disabled.
1814 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1815 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1818 if (!is_spec_ib_user_controlled())
1821 task_set_spec_ib_disable(task);
1822 if (ctrl == PR_SPEC_FORCE_DISABLE)
1823 task_set_spec_ib_force_disable(task);
1824 task_update_spec_tif(task);
1832 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
1836 case PR_SPEC_STORE_BYPASS:
1837 return ssb_prctl_set(task, ctrl);
1838 case PR_SPEC_INDIRECT_BRANCH:
1839 return ib_prctl_set(task, ctrl);
1840 case PR_SPEC_L1D_FLUSH:
1841 return l1d_flush_prctl_set(task, ctrl);
1847 #ifdef CONFIG_SECCOMP
1848 void arch_seccomp_spec_mitigate(struct task_struct *task)
1850 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
1851 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1852 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
1853 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
1854 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1858 static int l1d_flush_prctl_get(struct task_struct *task)
1860 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
1861 return PR_SPEC_FORCE_DISABLE;
1863 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))
1864 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1866 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1869 static int ssb_prctl_get(struct task_struct *task)
1872 case SPEC_STORE_BYPASS_DISABLE:
1873 return PR_SPEC_DISABLE;
1874 case SPEC_STORE_BYPASS_SECCOMP:
1875 case SPEC_STORE_BYPASS_PRCTL:
1876 if (task_spec_ssb_force_disable(task))
1877 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1878 if (task_spec_ssb_noexec(task))
1879 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
1880 if (task_spec_ssb_disable(task))
1881 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1882 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1884 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1885 return PR_SPEC_ENABLE;
1886 return PR_SPEC_NOT_AFFECTED;
1890 static int ib_prctl_get(struct task_struct *task)
1892 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
1893 return PR_SPEC_NOT_AFFECTED;
1895 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1896 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1897 return PR_SPEC_ENABLE;
1898 else if (is_spec_ib_user_controlled()) {
1899 if (task_spec_ib_force_disable(task))
1900 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1901 if (task_spec_ib_disable(task))
1902 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1903 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1904 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
1905 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
1906 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
1907 return PR_SPEC_DISABLE;
1909 return PR_SPEC_NOT_AFFECTED;
1912 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
1915 case PR_SPEC_STORE_BYPASS:
1916 return ssb_prctl_get(task);
1917 case PR_SPEC_INDIRECT_BRANCH:
1918 return ib_prctl_get(task);
1919 case PR_SPEC_L1D_FLUSH:
1920 return l1d_flush_prctl_get(task);
1926 void x86_spec_ctrl_setup_ap(void)
1928 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1929 write_spec_ctrl_current(x86_spec_ctrl_base, true);
1931 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1932 x86_amd_ssb_disable();
1935 bool itlb_multihit_kvm_mitigation;
1936 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
1939 #define pr_fmt(fmt) "L1TF: " fmt
1941 /* Default mitigation for L1TF-affected CPUs */
1942 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
1943 #if IS_ENABLED(CONFIG_KVM_INTEL)
1944 EXPORT_SYMBOL_GPL(l1tf_mitigation);
1946 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
1947 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1950 * These CPUs all support 44bits physical address space internally in the
1951 * cache but CPUID can report a smaller number of physical address bits.
1953 * The L1TF mitigation uses the top most address bit for the inversion of
1954 * non present PTEs. When the installed memory reaches into the top most
1955 * address bit due to memory holes, which has been observed on machines
1956 * which report 36bits physical address bits and have 32G RAM installed,
1957 * then the mitigation range check in l1tf_select_mitigation() triggers.
1958 * This is a false positive because the mitigation is still possible due to
1959 * the fact that the cache uses 44bit internally. Use the cache bits
1960 * instead of the reported physical bits and adjust them on the affected
1961 * machines to 44bit if the reported bits are less than 44.
1963 static void override_cache_bits(struct cpuinfo_x86 *c)
1968 switch (c->x86_model) {
1969 case INTEL_FAM6_NEHALEM:
1970 case INTEL_FAM6_WESTMERE:
1971 case INTEL_FAM6_SANDYBRIDGE:
1972 case INTEL_FAM6_IVYBRIDGE:
1973 case INTEL_FAM6_HASWELL:
1974 case INTEL_FAM6_HASWELL_L:
1975 case INTEL_FAM6_HASWELL_G:
1976 case INTEL_FAM6_BROADWELL:
1977 case INTEL_FAM6_BROADWELL_G:
1978 case INTEL_FAM6_SKYLAKE_L:
1979 case INTEL_FAM6_SKYLAKE:
1980 case INTEL_FAM6_KABYLAKE_L:
1981 case INTEL_FAM6_KABYLAKE:
1982 if (c->x86_cache_bits < 44)
1983 c->x86_cache_bits = 44;
1988 static void __init l1tf_select_mitigation(void)
1992 if (!boot_cpu_has_bug(X86_BUG_L1TF))
1995 if (cpu_mitigations_off())
1996 l1tf_mitigation = L1TF_MITIGATION_OFF;
1997 else if (cpu_mitigations_auto_nosmt())
1998 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2000 override_cache_bits(&boot_cpu_data);
2002 switch (l1tf_mitigation) {
2003 case L1TF_MITIGATION_OFF:
2004 case L1TF_MITIGATION_FLUSH_NOWARN:
2005 case L1TF_MITIGATION_FLUSH:
2007 case L1TF_MITIGATION_FLUSH_NOSMT:
2008 case L1TF_MITIGATION_FULL:
2009 cpu_smt_disable(false);
2011 case L1TF_MITIGATION_FULL_FORCE:
2012 cpu_smt_disable(true);
2016 #if CONFIG_PGTABLE_LEVELS == 2
2017 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
2021 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
2022 if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
2023 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
2024 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
2025 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
2027 pr_info("However, doing so will make a part of your RAM unusable.\n");
2028 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
2032 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
2035 static int __init l1tf_cmdline(char *str)
2037 if (!boot_cpu_has_bug(X86_BUG_L1TF))
2043 if (!strcmp(str, "off"))
2044 l1tf_mitigation = L1TF_MITIGATION_OFF;
2045 else if (!strcmp(str, "flush,nowarn"))
2046 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
2047 else if (!strcmp(str, "flush"))
2048 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
2049 else if (!strcmp(str, "flush,nosmt"))
2050 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2051 else if (!strcmp(str, "full"))
2052 l1tf_mitigation = L1TF_MITIGATION_FULL;
2053 else if (!strcmp(str, "full,force"))
2054 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
2058 early_param("l1tf", l1tf_cmdline);
2061 #define pr_fmt(fmt) fmt
2065 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
2067 #if IS_ENABLED(CONFIG_KVM_INTEL)
2068 static const char * const l1tf_vmx_states[] = {
2069 [VMENTER_L1D_FLUSH_AUTO] = "auto",
2070 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable",
2071 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes",
2072 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes",
2073 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled",
2074 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary"
2077 static ssize_t l1tf_show_state(char *buf)
2079 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
2080 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
2082 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
2083 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
2084 sched_smt_active())) {
2085 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
2086 l1tf_vmx_states[l1tf_vmx_mitigation]);
2089 return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
2090 l1tf_vmx_states[l1tf_vmx_mitigation],
2091 sched_smt_active() ? "vulnerable" : "disabled");
2094 static ssize_t itlb_multihit_show_state(char *buf)
2096 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
2097 !boot_cpu_has(X86_FEATURE_VMX))
2098 return sprintf(buf, "KVM: Mitigation: VMX unsupported\n");
2099 else if (!(cr4_read_shadow() & X86_CR4_VMXE))
2100 return sprintf(buf, "KVM: Mitigation: VMX disabled\n");
2101 else if (itlb_multihit_kvm_mitigation)
2102 return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
2104 return sprintf(buf, "KVM: Vulnerable\n");
2107 static ssize_t l1tf_show_state(char *buf)
2109 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
2112 static ssize_t itlb_multihit_show_state(char *buf)
2114 return sprintf(buf, "Processor vulnerable\n");
2118 static ssize_t mds_show_state(char *buf)
2120 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2121 return sprintf(buf, "%s; SMT Host state unknown\n",
2122 mds_strings[mds_mitigation]);
2125 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
2126 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2127 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
2128 sched_smt_active() ? "mitigated" : "disabled"));
2131 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2132 sched_smt_active() ? "vulnerable" : "disabled");
2135 static ssize_t tsx_async_abort_show_state(char *buf)
2137 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
2138 (taa_mitigation == TAA_MITIGATION_OFF))
2139 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
2141 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2142 return sprintf(buf, "%s; SMT Host state unknown\n",
2143 taa_strings[taa_mitigation]);
2146 return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
2147 sched_smt_active() ? "vulnerable" : "disabled");
2150 static ssize_t mmio_stale_data_show_state(char *buf)
2152 if (mmio_mitigation == MMIO_MITIGATION_OFF)
2153 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]);
2155 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2156 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2157 mmio_strings[mmio_mitigation]);
2160 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation],
2161 sched_smt_active() ? "vulnerable" : "disabled");
2164 static char *stibp_state(void)
2166 if (spectre_v2_in_ibrs_mode(spectre_v2_enabled))
2169 switch (spectre_v2_user_stibp) {
2170 case SPECTRE_V2_USER_NONE:
2171 return ", STIBP: disabled";
2172 case SPECTRE_V2_USER_STRICT:
2173 return ", STIBP: forced";
2174 case SPECTRE_V2_USER_STRICT_PREFERRED:
2175 return ", STIBP: always-on";
2176 case SPECTRE_V2_USER_PRCTL:
2177 case SPECTRE_V2_USER_SECCOMP:
2178 if (static_key_enabled(&switch_to_cond_stibp))
2179 return ", STIBP: conditional";
2184 static char *ibpb_state(void)
2186 if (boot_cpu_has(X86_FEATURE_IBPB)) {
2187 if (static_key_enabled(&switch_mm_always_ibpb))
2188 return ", IBPB: always-on";
2189 if (static_key_enabled(&switch_mm_cond_ibpb))
2190 return ", IBPB: conditional";
2191 return ", IBPB: disabled";
2196 static ssize_t spectre_v2_show_state(char *buf)
2198 if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
2199 return sprintf(buf, "Vulnerable: LFENCE\n");
2201 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
2202 return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
2204 if (sched_smt_active() && unprivileged_ebpf_enabled() &&
2205 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
2206 return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
2208 return sprintf(buf, "%s%s%s%s%s%s\n",
2209 spectre_v2_strings[spectre_v2_enabled],
2211 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
2213 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
2214 spectre_v2_module_string());
2217 static ssize_t srbds_show_state(char *buf)
2219 return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]);
2222 static ssize_t retbleed_show_state(char *buf)
2224 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET) {
2225 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
2226 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
2227 return sprintf(buf, "Vulnerable: untrained return thunk on non-Zen uarch\n");
2229 return sprintf(buf, "%s; SMT %s\n",
2230 retbleed_strings[retbleed_mitigation],
2231 !sched_smt_active() ? "disabled" :
2232 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2233 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ?
2234 "enabled with STIBP protection" : "vulnerable");
2237 return sprintf(buf, "%s\n", retbleed_strings[retbleed_mitigation]);
2240 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
2241 char *buf, unsigned int bug)
2243 if (!boot_cpu_has_bug(bug))
2244 return sprintf(buf, "Not affected\n");
2247 case X86_BUG_CPU_MELTDOWN:
2248 if (boot_cpu_has(X86_FEATURE_PTI))
2249 return sprintf(buf, "Mitigation: PTI\n");
2251 if (hypervisor_is_type(X86_HYPER_XEN_PV))
2252 return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
2256 case X86_BUG_SPECTRE_V1:
2257 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
2259 case X86_BUG_SPECTRE_V2:
2260 return spectre_v2_show_state(buf);
2262 case X86_BUG_SPEC_STORE_BYPASS:
2263 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
2266 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
2267 return l1tf_show_state(buf);
2271 return mds_show_state(buf);
2274 return tsx_async_abort_show_state(buf);
2276 case X86_BUG_ITLB_MULTIHIT:
2277 return itlb_multihit_show_state(buf);
2280 return srbds_show_state(buf);
2282 case X86_BUG_MMIO_STALE_DATA:
2283 return mmio_stale_data_show_state(buf);
2285 case X86_BUG_RETBLEED:
2286 return retbleed_show_state(buf);
2292 return sprintf(buf, "Vulnerable\n");
2295 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
2297 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
2300 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
2302 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
2305 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
2307 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
2310 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
2312 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
2315 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
2317 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
2320 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
2322 return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
2325 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
2327 return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
2330 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
2332 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
2335 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
2337 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
2340 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf)
2342 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA);
2345 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf)
2347 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED);