arch/x86/kernel/cpu/bugs.c

   1 /*
   2  *  Copyright (C) 1994  Linus Torvalds
   3  *
   4  *  Cyrix stuff, June 1998 by:
   5  *      - Rafael R. Reilova (moved everything from head.S),
   6  *        <rreilova@ececs.uc.edu>
   7  *      - Channing Corn (tests & fixes),
   8  *      - Andrew D. Balsa (code cleanup).
   9  */
  10 #include <linux/init.h>
  11 #include <linux/utsname.h>
  12 #include <linux/cpu.h>
  13 #include <linux/module.h>
  14 #include <linux/nospec.h>
  15 #include <linux/prctl.h>
  16 #include <linux/sched/smt.h>
  17
  18 #include <asm/spec-ctrl.h>
  19 #include <asm/cmdline.h>
  20 #include <asm/bugs.h>
  21 #include <asm/processor.h>
  22 #include <asm/processor-flags.h>
  23 #include <asm/fpu/internal.h>
  24 #include <asm/msr.h>
  25 #include <asm/vmx.h>
  26 #include <asm/paravirt.h>
  27 #include <asm/alternative.h>
  28 #include <asm/hypervisor.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/cacheflush.h>
  31 #include <asm/intel-family.h>
  32 #include <asm/e820.h>
  33
  34 static void __init spectre_v2_select_mitigation(void);
  35 static void __init ssb_select_mitigation(void);
  36 static void __init l1tf_select_mitigation(void);
  37 static void __init mds_select_mitigation(void);
  38
  39 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  40 u64 x86_spec_ctrl_base;
  41 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  42 static DEFINE_MUTEX(spec_ctrl_mutex);
  43
  44 /*
  45  * The vendor and possibly platform specific bits which can be modified in
  46  * x86_spec_ctrl_base.
  47  */
  48 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  49
  50 /*
  51  * AMD specific MSR info for Speculative Store Bypass control.
  52  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  53  */
  54 u64 __ro_after_init x86_amd_ls_cfg_base;
  55 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  56
  57 /* Control conditional STIPB in switch_to() */
  58 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
  59 /* Control conditional IBPB in switch_mm() */
  60 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
  61 /* Control unconditional IBPB in switch_mm() */
  62 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
  63
  64 /* Control MDS CPU buffer clear before returning to user space */
  65 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
  66 EXPORT_SYMBOL_GPL(mds_user_clear);
  67 /* Control MDS CPU buffer clear before idling (halt, mwait) */
  68 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
  69 EXPORT_SYMBOL_GPL(mds_idle_clear);
  70
  71 void __init check_bugs(void)
  72 {
  73         identify_boot_cpu();
  74
  75         /*
  76          * identify_boot_cpu() initialized SMT support information, let the
  77          * core code know.
  78          */
  79         cpu_smt_check_topology_early();
  80
  81         if (!IS_ENABLED(CONFIG_SMP)) {
  82                 pr_info("CPU: ");
  83                 print_cpu_info(&boot_cpu_data);
  84         }
  85
  86         /*
  87          * Read the SPEC_CTRL MSR to account for reserved bits which may
  88          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  89          * init code as it is not enumerated and depends on the family.
  90          */
  91         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  92                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  93
  94         /* Allow STIBP in MSR_SPEC_CTRL if supported */
  95         if (boot_cpu_has(X86_FEATURE_STIBP))
  96                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
  97
  98         /* Select the proper spectre mitigation before patching alternatives */
  99         spectre_v2_select_mitigation();
 100
 101         /*
 102          * Select proper mitigation for any exposure to the Speculative Store
 103          * Bypass vulnerability.
 104          */
 105         ssb_select_mitigation();
 106
 107         l1tf_select_mitigation();
 108
 109         mds_select_mitigation();
 110
 111 #ifdef CONFIG_X86_32
 112         /*
 113          * Check whether we are able to run this kernel safely on SMP.
 114          *
 115          * - i386 is no longer supported.
 116          * - In order to run on anything without a TSC, we need to be
 117          *   compiled for a i486.
 118          */
 119         if (boot_cpu_data.x86 < 4)
 120                 panic("Kernel requires i486+ for 'invlpg' and other features");
 121
 122         init_utsname()->machine[1] =
 123                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 124         alternative_instructions();
 125
 126         fpu__init_check_bugs();
 127 #else /* CONFIG_X86_64 */
 128         alternative_instructions();
 129
 130         /*
 131          * Make sure the first 2MB area is not mapped by huge pages
 132          * There are typically fixed size MTRRs in there and overlapping
 133          * MTRRs into large pages causes slow downs.
 134          *
 135          * Right now we don't do that with gbpages because there seems
 136          * very little benefit for that case.
 137          */
 138         if (!direct_gbpages)
 139                 set_memory_4k((unsigned long)__va(0), 1);
 140 #endif
 141 }
 142
 143 void
 144 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 145 {
 146         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 147         struct thread_info *ti = current_thread_info();
 148
 149         /* Is MSR_SPEC_CTRL implemented ? */
 150         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 151                 /*
 152                  * Restrict guest_spec_ctrl to supported values. Clear the
 153                  * modifiable bits in the host base value and or the
 154                  * modifiable bits from the guest value.
 155                  */
 156                 guestval = hostval & ~x86_spec_ctrl_mask;
 157                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 158
 159                 /* SSBD controlled in MSR_SPEC_CTRL */
 160                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 161                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 162                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 163
 164                 /* Conditional STIBP enabled? */
 165                 if (static_branch_unlikely(&switch_to_cond_stibp))
 166                         hostval |= stibp_tif_to_spec_ctrl(ti->flags);
 167
 168                 if (hostval != guestval) {
 169                         msrval = setguest ? guestval : hostval;
 170                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 171                 }
 172         }
 173
 174         /*
 175          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 176          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 177          */
 178         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 179             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 180                 return;
 181
 182         /*
 183          * If the host has SSBD mitigation enabled, force it in the host's
 184          * virtual MSR value. If its not permanently enabled, evaluate
 185          * current's TIF_SSBD thread flag.
 186          */
 187         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 188                 hostval = SPEC_CTRL_SSBD;
 189         else
 190                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 191
 192         /* Sanitize the guest value */
 193         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 194
 195         if (hostval != guestval) {
 196                 unsigned long tif;
 197
 198                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 199                                  ssbd_spec_ctrl_to_tif(hostval);
 200
 201                 speculation_ctrl_update(tif);
 202         }
 203 }
 204 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 205
 206 static void x86_amd_ssb_disable(void)
 207 {
 208         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 209
 210         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 211                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 212         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 213                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 214 }
 215
 216 #undef pr_fmt
 217 #define pr_fmt(fmt)     "MDS: " fmt
 218
 219 /* Default mitigation for L1TF-affected CPUs */
 220 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 221
 222 static const char * const mds_strings[] = {
 223         [MDS_MITIGATION_OFF]    = "Vulnerable",
 224         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 225         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 226 };
 227
 228 static void __init mds_select_mitigation(void)
 229 {
 230         if (!boot_cpu_has_bug(X86_BUG_MDS)) {
 231                 mds_mitigation = MDS_MITIGATION_OFF;
 232                 return;
 233         }
 234
 235         if (mds_mitigation == MDS_MITIGATION_FULL) {
 236                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 237                         mds_mitigation = MDS_MITIGATION_VMWERV;
 238                 static_branch_enable(&mds_user_clear);
 239         }
 240         pr_info("%s\n", mds_strings[mds_mitigation]);
 241 }
 242
 243 static int __init mds_cmdline(char *str)
 244 {
 245         if (!boot_cpu_has_bug(X86_BUG_MDS))
 246                 return 0;
 247
 248         if (!str)
 249                 return -EINVAL;
 250
 251         if (!strcmp(str, "off"))
 252                 mds_mitigation = MDS_MITIGATION_OFF;
 253         else if (!strcmp(str, "full"))
 254                 mds_mitigation = MDS_MITIGATION_FULL;
 255
 256         return 0;
 257 }
 258 early_param("mds", mds_cmdline);
 259
 260 #undef pr_fmt
 261 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 262
 263 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 264         SPECTRE_V2_NONE;
 265
 266 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
 267         SPECTRE_V2_USER_NONE;
 268
 269 #ifdef RETPOLINE
 270 static bool spectre_v2_bad_module;
 271
 272 bool retpoline_module_ok(bool has_retpoline)
 273 {
 274         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 275                 return true;
 276
 277         pr_err("System may be vulnerable to spectre v2\n");
 278         spectre_v2_bad_module = true;
 279         return false;
 280 }
 281
 282 static inline const char *spectre_v2_module_string(void)
 283 {
 284         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 285 }
 286 #else
 287 static inline const char *spectre_v2_module_string(void) { return ""; }
 288 #endif
 289
 290 static inline bool match_option(const char *arg, int arglen, const char *opt)
 291 {
 292         int len = strlen(opt);
 293
 294         return len == arglen && !strncmp(arg, opt, len);
 295 }
 296
 297 /* The kernel command line selection for spectre v2 */
 298 enum spectre_v2_mitigation_cmd {
 299         SPECTRE_V2_CMD_NONE,
 300         SPECTRE_V2_CMD_AUTO,
 301         SPECTRE_V2_CMD_FORCE,
 302         SPECTRE_V2_CMD_RETPOLINE,
 303         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 304         SPECTRE_V2_CMD_RETPOLINE_AMD,
 305 };
 306
 307 enum spectre_v2_user_cmd {
 308         SPECTRE_V2_USER_CMD_NONE,
 309         SPECTRE_V2_USER_CMD_AUTO,
 310         SPECTRE_V2_USER_CMD_FORCE,
 311         SPECTRE_V2_USER_CMD_PRCTL,
 312         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
 313         SPECTRE_V2_USER_CMD_SECCOMP,
 314         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
 315 };
 316
 317 static const char * const spectre_v2_user_strings[] = {
 318         [SPECTRE_V2_USER_NONE]          = "User space: Vulnerable",
 319         [SPECTRE_V2_USER_STRICT]        = "User space: Mitigation: STIBP protection",
 320         [SPECTRE_V2_USER_PRCTL]         = "User space: Mitigation: STIBP via prctl",
 321         [SPECTRE_V2_USER_SECCOMP]       = "User space: Mitigation: STIBP via seccomp and prctl",
 322 };
 323
 324 static const struct {
 325         const char                      *option;
 326         enum spectre_v2_user_cmd        cmd;
 327         bool                            secure;
 328 } v2_user_options[] __initdata = {
 329         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
 330         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
 331         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
 332         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
 333         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
 334         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
 335         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
 336 };
 337
 338 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
 339 {
 340         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 341                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
 342 }
 343
 344 static enum spectre_v2_user_cmd __init
 345 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
 346 {
 347         char arg[20];
 348         int ret, i;
 349
 350         switch (v2_cmd) {
 351         case SPECTRE_V2_CMD_NONE:
 352                 return SPECTRE_V2_USER_CMD_NONE;
 353         case SPECTRE_V2_CMD_FORCE:
 354                 return SPECTRE_V2_USER_CMD_FORCE;
 355         default:
 356                 break;
 357         }
 358
 359         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
 360                                   arg, sizeof(arg));
 361         if (ret < 0)
 362                 return SPECTRE_V2_USER_CMD_AUTO;
 363
 364         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
 365                 if (match_option(arg, ret, v2_user_options[i].option)) {
 366                         spec_v2_user_print_cond(v2_user_options[i].option,
 367                                                 v2_user_options[i].secure);
 368                         return v2_user_options[i].cmd;
 369                 }
 370         }
 371
 372         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
 373         return SPECTRE_V2_USER_CMD_AUTO;
 374 }
 375
 376 static void __init
 377 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
 378 {
 379         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
 380         bool smt_possible = IS_ENABLED(CONFIG_SMP);
 381         enum spectre_v2_user_cmd cmd;
 382
 383         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
 384                 return;
 385
 386         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 387             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 388                 smt_possible = false;
 389
 390         cmd = spectre_v2_parse_user_cmdline(v2_cmd);
 391         switch (cmd) {
 392         case SPECTRE_V2_USER_CMD_NONE:
 393                 goto set_mode;
 394         case SPECTRE_V2_USER_CMD_FORCE:
 395                 mode = SPECTRE_V2_USER_STRICT;
 396                 break;
 397         case SPECTRE_V2_USER_CMD_PRCTL:
 398         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 399                 mode = SPECTRE_V2_USER_PRCTL;
 400                 break;
 401         case SPECTRE_V2_USER_CMD_AUTO:
 402         case SPECTRE_V2_USER_CMD_SECCOMP:
 403         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 404                 if (IS_ENABLED(CONFIG_SECCOMP))
 405                         mode = SPECTRE_V2_USER_SECCOMP;
 406                 else
 407                         mode = SPECTRE_V2_USER_PRCTL;
 408                 break;
 409         }
 410
 411         /* Initialize Indirect Branch Prediction Barrier */
 412         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 413                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 414
 415                 switch (cmd) {
 416                 case SPECTRE_V2_USER_CMD_FORCE:
 417                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 418                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 419                         static_branch_enable(&switch_mm_always_ibpb);
 420                         break;
 421                 case SPECTRE_V2_USER_CMD_PRCTL:
 422                 case SPECTRE_V2_USER_CMD_AUTO:
 423                 case SPECTRE_V2_USER_CMD_SECCOMP:
 424                         static_branch_enable(&switch_mm_cond_ibpb);
 425                         break;
 426                 default:
 427                         break;
 428                 }
 429
 430                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
 431                         static_key_enabled(&switch_mm_always_ibpb) ?
 432                         "always-on" : "conditional");
 433         }
 434
 435         /* If enhanced IBRS is enabled no STIPB required */
 436         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 437                 return;
 438
 439         /*
 440          * If SMT is not possible or STIBP is not available clear the STIPB
 441          * mode.
 442          */
 443         if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP))
 444                 mode = SPECTRE_V2_USER_NONE;
 445 set_mode:
 446         spectre_v2_user = mode;
 447         /* Only print the STIBP mode when SMT possible */
 448         if (smt_possible)
 449                 pr_info("%s\n", spectre_v2_user_strings[mode]);
 450 }
 451
 452 static const char * const spectre_v2_strings[] = {
 453         [SPECTRE_V2_NONE]                       = "Vulnerable",
 454         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
 455         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
 456         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 457         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 458         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
 459 };
 460
 461 static const struct {
 462         const char *option;
 463         enum spectre_v2_mitigation_cmd cmd;
 464         bool secure;
 465 } mitigation_options[] __initdata = {
 466         { "off",                SPECTRE_V2_CMD_NONE,              false },
 467         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
 468         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
 469         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 470         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 471         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 472 };
 473
 474 static void __init spec_v2_print_cond(const char *reason, bool secure)
 475 {
 476         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 477                 pr_info("%s selected on command line.\n", reason);
 478 }
 479
 480 static inline bool retp_compiler(void)
 481 {
 482         return __is_defined(RETPOLINE);
 483 }
 484
 485 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 486 {
 487         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 488         char arg[20];
 489         int ret, i;
 490
 491         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 492                 return SPECTRE_V2_CMD_NONE;
 493
 494         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 495         if (ret < 0)
 496                 return SPECTRE_V2_CMD_AUTO;
 497
 498         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 499                 if (!match_option(arg, ret, mitigation_options[i].option))
 500                         continue;
 501                 cmd = mitigation_options[i].cmd;
 502                 break;
 503         }
 504
 505         if (i >= ARRAY_SIZE(mitigation_options)) {
 506                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 507                 return SPECTRE_V2_CMD_AUTO;
 508         }
 509
 510         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 511              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 512              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 513             !IS_ENABLED(CONFIG_RETPOLINE)) {
 514                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 515                 return SPECTRE_V2_CMD_AUTO;
 516         }
 517
 518         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 519             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 520                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 521                 return SPECTRE_V2_CMD_AUTO;
 522         }
 523
 524         spec_v2_print_cond(mitigation_options[i].option,
 525                            mitigation_options[i].secure);
 526         return cmd;
 527 }
 528
 529 static void __init spectre_v2_select_mitigation(void)
 530 {
 531         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 532         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 533
 534         /*
 535          * If the CPU is not affected and the command line mode is NONE or AUTO
 536          * then nothing to do.
 537          */
 538         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 539             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 540                 return;
 541
 542         switch (cmd) {
 543         case SPECTRE_V2_CMD_NONE:
 544                 return;
 545
 546         case SPECTRE_V2_CMD_FORCE:
 547         case SPECTRE_V2_CMD_AUTO:
 548                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 549                         mode = SPECTRE_V2_IBRS_ENHANCED;
 550                         /* Force it so VMEXIT will restore correctly */
 551                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
 552                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 553                         goto specv2_set_mode;
 554                 }
 555                 if (IS_ENABLED(CONFIG_RETPOLINE))
 556                         goto retpoline_auto;
 557                 break;
 558         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 559                 if (IS_ENABLED(CONFIG_RETPOLINE))
 560                         goto retpoline_amd;
 561                 break;
 562         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 563                 if (IS_ENABLED(CONFIG_RETPOLINE))
 564                         goto retpoline_generic;
 565                 break;
 566         case SPECTRE_V2_CMD_RETPOLINE:
 567                 if (IS_ENABLED(CONFIG_RETPOLINE))
 568                         goto retpoline_auto;
 569                 break;
 570         }
 571         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 572         return;
 573
 574 retpoline_auto:
 575         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 576         retpoline_amd:
 577                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 578                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 579                         goto retpoline_generic;
 580                 }
 581                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 582                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 583                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 584                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 585         } else {
 586         retpoline_generic:
 587                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 588                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 589                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 590         }
 591
 592 specv2_set_mode:
 593         spectre_v2_enabled = mode;
 594         pr_info("%s\n", spectre_v2_strings[mode]);
 595
 596         /*
 597          * If spectre v2 protection has been enabled, unconditionally fill
 598          * RSB during a context switch; this protects against two independent
 599          * issues:
 600          *
 601          *      - RSB underflow (and switch to BTB) on Skylake+
 602          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
 603          */
 604         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 605         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
 606
 607         /*
 608          * Retpoline means the kernel is safe because it has no indirect
 609          * branches. Enhanced IBRS protects firmware too, so, enable restricted
 610          * speculation around firmware calls only when Enhanced IBRS isn't
 611          * supported.
 612          *
 613          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
 614          * the user might select retpoline on the kernel command line and if
 615          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
 616          * enable IBRS around firmware calls.
 617          */
 618         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
 619                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 620                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 621         }
 622
 623         /* Set up IBPB and STIBP depending on the general spectre V2 command */
 624         spectre_v2_user_select_mitigation(cmd);
 625
 626         /* Enable STIBP if appropriate */
 627         arch_smt_update();
 628 }
 629
 630 static void update_stibp_msr(void * __unused)
 631 {
 632         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 633 }
 634
 635 /* Update x86_spec_ctrl_base in case SMT state changed. */
 636 static void update_stibp_strict(void)
 637 {
 638         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
 639
 640         if (sched_smt_active())
 641                 mask |= SPEC_CTRL_STIBP;
 642
 643         if (mask == x86_spec_ctrl_base)
 644                 return;
 645
 646         pr_info("Update user space SMT mitigation: STIBP %s\n",
 647                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
 648         x86_spec_ctrl_base = mask;
 649         on_each_cpu(update_stibp_msr, NULL, 1);
 650 }
 651
 652 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
 653 static void update_indir_branch_cond(void)
 654 {
 655         if (sched_smt_active())
 656                 static_branch_enable(&switch_to_cond_stibp);
 657         else
 658                 static_branch_disable(&switch_to_cond_stibp);
 659 }
 660
 661 /* Update the static key controlling the MDS CPU buffer clear in idle */
 662 static void update_mds_branch_idle(void)
 663 {
 664         /*
 665          * Enable the idle clearing if SMT is active on CPUs which are
 666          * affected only by MSBDS and not any other MDS variant.
 667          *
 668          * The other variants cannot be mitigated when SMT is enabled, so
 669          * clearing the buffers on idle just to prevent the Store Buffer
 670          * repartitioning leak would be a window dressing exercise.
 671          */
 672         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
 673                 return;
 674
 675         if (sched_smt_active())
 676                 static_branch_enable(&mds_idle_clear);
 677         else
 678                 static_branch_disable(&mds_idle_clear);
 679 }
 680
 681 void arch_smt_update(void)
 682 {
 683         /* Enhanced IBRS implies STIBP. No update required. */
 684         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 685                 return;
 686
 687         mutex_lock(&spec_ctrl_mutex);
 688
 689         switch (spectre_v2_user) {
 690         case SPECTRE_V2_USER_NONE:
 691                 break;
 692         case SPECTRE_V2_USER_STRICT:
 693                 update_stibp_strict();
 694                 break;
 695         case SPECTRE_V2_USER_PRCTL:
 696         case SPECTRE_V2_USER_SECCOMP:
 697                 update_indir_branch_cond();
 698                 break;
 699         }
 700
 701         switch (mds_mitigation) {
 702         case MDS_MITIGATION_FULL:
 703         case MDS_MITIGATION_VMWERV:
 704                 update_mds_branch_idle();
 705                 break;
 706         case MDS_MITIGATION_OFF:
 707                 break;
 708         }
 709
 710         mutex_unlock(&spec_ctrl_mutex);
 711 }
 712
 713 #undef pr_fmt
 714 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 715
 716 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 717
 718 /* The kernel command line selection */
 719 enum ssb_mitigation_cmd {
 720         SPEC_STORE_BYPASS_CMD_NONE,
 721         SPEC_STORE_BYPASS_CMD_AUTO,
 722         SPEC_STORE_BYPASS_CMD_ON,
 723         SPEC_STORE_BYPASS_CMD_PRCTL,
 724         SPEC_STORE_BYPASS_CMD_SECCOMP,
 725 };
 726
 727 static const char * const ssb_strings[] = {
 728         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 729         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 730         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 731         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 732 };
 733
 734 static const struct {
 735         const char *option;
 736         enum ssb_mitigation_cmd cmd;
 737 } ssb_mitigation_options[]  __initdata = {
 738         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 739         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 740         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 741         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 742         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 743 };
 744
 745 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 746 {
 747         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 748         char arg[20];
 749         int ret, i;
 750
 751         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 752                 return SPEC_STORE_BYPASS_CMD_NONE;
 753         } else {
 754                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 755                                           arg, sizeof(arg));
 756                 if (ret < 0)
 757                         return SPEC_STORE_BYPASS_CMD_AUTO;
 758
 759                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 760                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 761                                 continue;
 762
 763                         cmd = ssb_mitigation_options[i].cmd;
 764                         break;
 765                 }
 766
 767                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 768                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 769                         return SPEC_STORE_BYPASS_CMD_AUTO;
 770                 }
 771         }
 772
 773         return cmd;
 774 }
 775
 776 static enum ssb_mitigation __init __ssb_select_mitigation(void)
 777 {
 778         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 779         enum ssb_mitigation_cmd cmd;
 780
 781         if (!boot_cpu_has(X86_FEATURE_SSBD))
 782                 return mode;
 783
 784         cmd = ssb_parse_cmdline();
 785         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 786             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 787              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 788                 return mode;
 789
 790         switch (cmd) {
 791         case SPEC_STORE_BYPASS_CMD_AUTO:
 792         case SPEC_STORE_BYPASS_CMD_SECCOMP:
 793                 /*
 794                  * Choose prctl+seccomp as the default mode if seccomp is
 795                  * enabled.
 796                  */
 797                 if (IS_ENABLED(CONFIG_SECCOMP))
 798                         mode = SPEC_STORE_BYPASS_SECCOMP;
 799                 else
 800                         mode = SPEC_STORE_BYPASS_PRCTL;
 801                 break;
 802         case SPEC_STORE_BYPASS_CMD_ON:
 803                 mode = SPEC_STORE_BYPASS_DISABLE;
 804                 break;
 805         case SPEC_STORE_BYPASS_CMD_PRCTL:
 806                 mode = SPEC_STORE_BYPASS_PRCTL;
 807                 break;
 808         case SPEC_STORE_BYPASS_CMD_NONE:
 809                 break;
 810         }
 811
 812         /*
 813          * We have three CPU feature flags that are in play here:
 814          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
 815          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
 816          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
 817          */
 818         if (mode == SPEC_STORE_BYPASS_DISABLE) {
 819                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 820                 /*
 821                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
 822                  * use a completely different MSR and bit dependent on family.
 823                  */
 824                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
 825                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
 826                         x86_amd_ssb_disable();
 827                 } else {
 828                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
 829                         x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
 830                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 831                 }
 832         }
 833
 834         return mode;
 835 }
 836
 837 static void ssb_select_mitigation(void)
 838 {
 839         ssb_mode = __ssb_select_mitigation();
 840
 841         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 842                 pr_info("%s\n", ssb_strings[ssb_mode]);
 843 }
 844
 845 #undef pr_fmt
 846 #define pr_fmt(fmt)     "Speculation prctl: " fmt
 847
 848 static void task_update_spec_tif(struct task_struct *tsk)
 849 {
 850         /* Force the update of the real TIF bits */
 851         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
 852
 853         /*
 854          * Immediately update the speculation control MSRs for the current
 855          * task, but for a non-current task delay setting the CPU
 856          * mitigation until it is scheduled next.
 857          *
 858          * This can only happen for SECCOMP mitigation. For PRCTL it's
 859          * always the current task.
 860          */
 861         if (tsk == current)
 862                 speculation_ctrl_update_current();
 863 }
 864
 865 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
 866 {
 867         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
 868             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
 869                 return -ENXIO;
 870
 871         switch (ctrl) {
 872         case PR_SPEC_ENABLE:
 873                 /* If speculation is force disabled, enable is not allowed */
 874                 if (task_spec_ssb_force_disable(task))
 875                         return -EPERM;
 876                 task_clear_spec_ssb_disable(task);
 877                 task_update_spec_tif(task);
 878                 break;
 879         case PR_SPEC_DISABLE:
 880                 task_set_spec_ssb_disable(task);
 881                 task_update_spec_tif(task);
 882                 break;
 883         case PR_SPEC_FORCE_DISABLE:
 884                 task_set_spec_ssb_disable(task);
 885                 task_set_spec_ssb_force_disable(task);
 886                 task_update_spec_tif(task);
 887                 break;
 888         default:
 889                 return -ERANGE;
 890         }
 891         return 0;
 892 }
 893
 894 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
 895 {
 896         switch (ctrl) {
 897         case PR_SPEC_ENABLE:
 898                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
 899                         return 0;
 900                 /*
 901                  * Indirect branch speculation is always disabled in strict
 902                  * mode.
 903                  */
 904                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT)
 905                         return -EPERM;
 906                 task_clear_spec_ib_disable(task);
 907                 task_update_spec_tif(task);
 908                 break;
 909         case PR_SPEC_DISABLE:
 910         case PR_SPEC_FORCE_DISABLE:
 911                 /*
 912                  * Indirect branch speculation is always allowed when
 913                  * mitigation is force disabled.
 914                  */
 915                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
 916                         return -EPERM;
 917                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT)
 918                         return 0;
 919                 task_set_spec_ib_disable(task);
 920                 if (ctrl == PR_SPEC_FORCE_DISABLE)
 921                         task_set_spec_ib_force_disable(task);
 922                 task_update_spec_tif(task);
 923                 break;
 924         default:
 925                 return -ERANGE;
 926         }
 927         return 0;
 928 }
 929
 930 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
 931                              unsigned long ctrl)
 932 {
 933         switch (which) {
 934         case PR_SPEC_STORE_BYPASS:
 935                 return ssb_prctl_set(task, ctrl);
 936         case PR_SPEC_INDIRECT_BRANCH:
 937                 return ib_prctl_set(task, ctrl);
 938         default:
 939                 return -ENODEV;
 940         }
 941 }
 942
 943 #ifdef CONFIG_SECCOMP
 944 void arch_seccomp_spec_mitigate(struct task_struct *task)
 945 {
 946         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
 947                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 948         if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP)
 949                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 950 }
 951 #endif
 952
 953 static int ssb_prctl_get(struct task_struct *task)
 954 {
 955         switch (ssb_mode) {
 956         case SPEC_STORE_BYPASS_DISABLE:
 957                 return PR_SPEC_DISABLE;
 958         case SPEC_STORE_BYPASS_SECCOMP:
 959         case SPEC_STORE_BYPASS_PRCTL:
 960                 if (task_spec_ssb_force_disable(task))
 961                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 962                 if (task_spec_ssb_disable(task))
 963                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 964                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 965         default:
 966                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 967                         return PR_SPEC_ENABLE;
 968                 return PR_SPEC_NOT_AFFECTED;
 969         }
 970 }
 971
 972 static int ib_prctl_get(struct task_struct *task)
 973 {
 974         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 975                 return PR_SPEC_NOT_AFFECTED;
 976
 977         switch (spectre_v2_user) {
 978         case SPECTRE_V2_USER_NONE:
 979                 return PR_SPEC_ENABLE;
 980         case SPECTRE_V2_USER_PRCTL:
 981         case SPECTRE_V2_USER_SECCOMP:
 982                 if (task_spec_ib_force_disable(task))
 983                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 984                 if (task_spec_ib_disable(task))
 985                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 986                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 987         case SPECTRE_V2_USER_STRICT:
 988                 return PR_SPEC_DISABLE;
 989         default:
 990                 return PR_SPEC_NOT_AFFECTED;
 991         }
 992 }
 993
 994 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
 995 {
 996         switch (which) {
 997         case PR_SPEC_STORE_BYPASS:
 998                 return ssb_prctl_get(task);
 999         case PR_SPEC_INDIRECT_BRANCH:
1000                 return ib_prctl_get(task);
1001         default:
1002                 return -ENODEV;
1003         }
1004 }
1005
1006 void x86_spec_ctrl_setup_ap(void)
1007 {
1008         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1009                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1010
1011         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1012                 x86_amd_ssb_disable();
1013 }
1014
1015 #undef pr_fmt
1016 #define pr_fmt(fmt)     "L1TF: " fmt
1017
1018 /* Default mitigation for L1TF-affected CPUs */
1019 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
1020 #if IS_ENABLED(CONFIG_KVM_INTEL)
1021 EXPORT_SYMBOL_GPL(l1tf_mitigation);
1022 #endif
1023 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
1024 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1025
1026 /*
1027  * These CPUs all support 44bits physical address space internally in the
1028  * cache but CPUID can report a smaller number of physical address bits.
1029  *
1030  * The L1TF mitigation uses the top most address bit for the inversion of
1031  * non present PTEs. When the installed memory reaches into the top most
1032  * address bit due to memory holes, which has been observed on machines
1033  * which report 36bits physical address bits and have 32G RAM installed,
1034  * then the mitigation range check in l1tf_select_mitigation() triggers.
1035  * This is a false positive because the mitigation is still possible due to
1036  * the fact that the cache uses 44bit internally. Use the cache bits
1037  * instead of the reported physical bits and adjust them on the affected
1038  * machines to 44bit if the reported bits are less than 44.
1039  */
1040 static void override_cache_bits(struct cpuinfo_x86 *c)
1041 {
1042         if (c->x86 != 6)
1043                 return;
1044
1045         switch (c->x86_model) {
1046         case INTEL_FAM6_NEHALEM:
1047         case INTEL_FAM6_WESTMERE:
1048         case INTEL_FAM6_SANDYBRIDGE:
1049         case INTEL_FAM6_IVYBRIDGE:
1050         case INTEL_FAM6_HASWELL_CORE:
1051         case INTEL_FAM6_HASWELL_ULT:
1052         case INTEL_FAM6_HASWELL_GT3E:
1053         case INTEL_FAM6_BROADWELL_CORE:
1054         case INTEL_FAM6_BROADWELL_GT3E:
1055         case INTEL_FAM6_SKYLAKE_MOBILE:
1056         case INTEL_FAM6_SKYLAKE_DESKTOP:
1057         case INTEL_FAM6_KABYLAKE_MOBILE:
1058         case INTEL_FAM6_KABYLAKE_DESKTOP:
1059                 if (c->x86_cache_bits < 44)
1060                         c->x86_cache_bits = 44;
1061                 break;
1062         }
1063 }
1064
1065 static void __init l1tf_select_mitigation(void)
1066 {
1067         u64 half_pa;
1068
1069         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1070                 return;
1071
1072         override_cache_bits(&boot_cpu_data);
1073
1074         switch (l1tf_mitigation) {
1075         case L1TF_MITIGATION_OFF:
1076         case L1TF_MITIGATION_FLUSH_NOWARN:
1077         case L1TF_MITIGATION_FLUSH:
1078                 break;
1079         case L1TF_MITIGATION_FLUSH_NOSMT:
1080         case L1TF_MITIGATION_FULL:
1081                 cpu_smt_disable(false);
1082                 break;
1083         case L1TF_MITIGATION_FULL_FORCE:
1084                 cpu_smt_disable(true);
1085                 break;
1086         }
1087
1088 #if CONFIG_PGTABLE_LEVELS == 2
1089         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1090         return;
1091 #endif
1092
1093         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
1094         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
1095                         e820_any_mapped(half_pa, ULLONG_MAX - half_pa, E820_RAM)) {
1096                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
1097                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1098                                 half_pa);
1099                 pr_info("However, doing so will make a part of your RAM unusable.\n");
1100                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
1101                 return;
1102         }
1103
1104         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
1105 }
1106
1107 static int __init l1tf_cmdline(char *str)
1108 {
1109         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1110                 return 0;
1111
1112         if (!str)
1113                 return -EINVAL;
1114
1115         if (!strcmp(str, "off"))
1116                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1117         else if (!strcmp(str, "flush,nowarn"))
1118                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
1119         else if (!strcmp(str, "flush"))
1120                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
1121         else if (!strcmp(str, "flush,nosmt"))
1122                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1123         else if (!strcmp(str, "full"))
1124                 l1tf_mitigation = L1TF_MITIGATION_FULL;
1125         else if (!strcmp(str, "full,force"))
1126                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
1127
1128         return 0;
1129 }
1130 early_param("l1tf", l1tf_cmdline);
1131
1132 #undef pr_fmt
1133
1134 #ifdef CONFIG_SYSFS
1135
1136 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
1137
1138 #if IS_ENABLED(CONFIG_KVM_INTEL)
1139 static const char * const l1tf_vmx_states[] = {
1140         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
1141         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
1142         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
1143         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
1144         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
1145         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
1146 };
1147
1148 static ssize_t l1tf_show_state(char *buf)
1149 {
1150         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
1151                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1152
1153         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
1154             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
1155              sched_smt_active())) {
1156                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
1157                                l1tf_vmx_states[l1tf_vmx_mitigation]);
1158         }
1159
1160         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
1161                        l1tf_vmx_states[l1tf_vmx_mitigation],
1162                        sched_smt_active() ? "vulnerable" : "disabled");
1163 }
1164 #else
1165 static ssize_t l1tf_show_state(char *buf)
1166 {
1167         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1168 }
1169 #endif
1170
1171 static ssize_t mds_show_state(char *buf)
1172 {
1173 #ifdef CONFIG_HYPERVISOR_GUEST
1174         if (x86_hyper) {
1175                 return sprintf(buf, "%s; SMT Host state unknown\n",
1176                                mds_strings[mds_mitigation]);
1177         }
1178 #endif
1179
1180         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
1181                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1182                                sched_smt_active() ? "mitigated" : "disabled");
1183         }
1184
1185         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1186                        sched_smt_active() ? "vulnerable" : "disabled");
1187 }
1188
1189 static char *stibp_state(void)
1190 {
1191         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
1192                 return "";
1193
1194         switch (spectre_v2_user) {
1195         case SPECTRE_V2_USER_NONE:
1196                 return ", STIBP: disabled";
1197         case SPECTRE_V2_USER_STRICT:
1198                 return ", STIBP: forced";
1199         case SPECTRE_V2_USER_PRCTL:
1200         case SPECTRE_V2_USER_SECCOMP:
1201                 if (static_key_enabled(&switch_to_cond_stibp))
1202                         return ", STIBP: conditional";
1203         }
1204         return "";
1205 }
1206
1207 static char *ibpb_state(void)
1208 {
1209         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1210                 if (static_key_enabled(&switch_mm_always_ibpb))
1211                         return ", IBPB: always-on";
1212                 if (static_key_enabled(&switch_mm_cond_ibpb))
1213                         return ", IBPB: conditional";
1214                 return ", IBPB: disabled";
1215         }
1216         return "";
1217 }
1218
1219 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
1220                                char *buf, unsigned int bug)
1221 {
1222         if (!boot_cpu_has_bug(bug))
1223                 return sprintf(buf, "Not affected\n");
1224
1225         switch (bug) {
1226         case X86_BUG_CPU_MELTDOWN:
1227                 if (boot_cpu_has(X86_FEATURE_KAISER))
1228                         return sprintf(buf, "Mitigation: PTI\n");
1229
1230                 break;
1231
1232         case X86_BUG_SPECTRE_V1:
1233                 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
1234
1235         case X86_BUG_SPECTRE_V2:
1236                 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
1237                                ibpb_state(),
1238                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1239                                stibp_state(),
1240                                boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1241                                spectre_v2_module_string());
1242
1243         case X86_BUG_SPEC_STORE_BYPASS:
1244                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1245
1246         case X86_BUG_L1TF:
1247                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
1248                         return l1tf_show_state(buf);
1249                 break;
1250
1251         case X86_BUG_MDS:
1252                 return mds_show_state(buf);
1253
1254         default:
1255                 break;
1256         }
1257
1258         return sprintf(buf, "Vulnerable\n");
1259 }
1260
1261 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1262 {
1263         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1264 }
1265
1266 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
1267 {
1268         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
1269 }
1270
1271 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
1272 {
1273         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
1274 }
1275
1276 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1277 {
1278         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1279 }
1280
1281 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1282 {
1283         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1284 }
1285
1286 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
1287 {
1288         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
1289 }
1290 #endif