arch/x86/kernel/cpu/bugs.c

   1 /*
   2  *  Copyright (C) 1994  Linus Torvalds
   3  *
   4  *  Cyrix stuff, June 1998 by:
   5  *      - Rafael R. Reilova (moved everything from head.S),
   6  *        <rreilova@ececs.uc.edu>
   7  *      - Channing Corn (tests & fixes),
   8  *      - Andrew D. Balsa (code cleanup).
   9  */
  10 #include <linux/init.h>
  11 #include <linux/utsname.h>
  12 #include <linux/cpu.h>
  13 #include <linux/module.h>
  14 #include <linux/nospec.h>
  15 #include <linux/prctl.h>
  16 #include <linux/sched/smt.h>
  17
  18 #include <asm/spec-ctrl.h>
  19 #include <asm/cmdline.h>
  20 #include <asm/bugs.h>
  21 #include <asm/processor.h>
  22 #include <asm/processor-flags.h>
  23 #include <asm/fpu/internal.h>
  24 #include <asm/msr.h>
  25 #include <asm/vmx.h>
  26 #include <asm/paravirt.h>
  27 #include <asm/alternative.h>
  28 #include <asm/hypervisor.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/cacheflush.h>
  31 #include <asm/intel-family.h>
  32 #include <asm/e820.h>
  33
  34 static void __init spectre_v2_select_mitigation(void);
  35 static void __init ssb_select_mitigation(void);
  36 static void __init l1tf_select_mitigation(void);
  37 static void __init mds_select_mitigation(void);
  38
  39 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  40 u64 x86_spec_ctrl_base;
  41 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  42 static DEFINE_MUTEX(spec_ctrl_mutex);
  43
  44 /*
  45  * The vendor and possibly platform specific bits which can be modified in
  46  * x86_spec_ctrl_base.
  47  */
  48 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  49
  50 /*
  51  * AMD specific MSR info for Speculative Store Bypass control.
  52  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  53  */
  54 u64 __ro_after_init x86_amd_ls_cfg_base;
  55 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  56
  57 /* Control conditional STIPB in switch_to() */
  58 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
  59 /* Control conditional IBPB in switch_mm() */
  60 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
  61 /* Control unconditional IBPB in switch_mm() */
  62 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
  63
  64 /* Control MDS CPU buffer clear before returning to user space */
  65 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
  66 EXPORT_SYMBOL_GPL(mds_user_clear);
  67 /* Control MDS CPU buffer clear before idling (halt, mwait) */
  68 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
  69 EXPORT_SYMBOL_GPL(mds_idle_clear);
  70
  71 void __init check_bugs(void)
  72 {
  73         identify_boot_cpu();
  74
  75         /*
  76          * identify_boot_cpu() initialized SMT support information, let the
  77          * core code know.
  78          */
  79         cpu_smt_check_topology_early();
  80
  81         if (!IS_ENABLED(CONFIG_SMP)) {
  82                 pr_info("CPU: ");
  83                 print_cpu_info(&boot_cpu_data);
  84         }
  85
  86         /*
  87          * Read the SPEC_CTRL MSR to account for reserved bits which may
  88          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  89          * init code as it is not enumerated and depends on the family.
  90          */
  91         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  92                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  93
  94         /* Allow STIBP in MSR_SPEC_CTRL if supported */
  95         if (boot_cpu_has(X86_FEATURE_STIBP))
  96                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
  97
  98         /* Select the proper spectre mitigation before patching alternatives */
  99         spectre_v2_select_mitigation();
 100
 101         /*
 102          * Select proper mitigation for any exposure to the Speculative Store
 103          * Bypass vulnerability.
 104          */
 105         ssb_select_mitigation();
 106
 107         l1tf_select_mitigation();
 108
 109         mds_select_mitigation();
 110
 111 #ifdef CONFIG_X86_32
 112         /*
 113          * Check whether we are able to run this kernel safely on SMP.
 114          *
 115          * - i386 is no longer supported.
 116          * - In order to run on anything without a TSC, we need to be
 117          *   compiled for a i486.
 118          */
 119         if (boot_cpu_data.x86 < 4)
 120                 panic("Kernel requires i486+ for 'invlpg' and other features");
 121
 122         init_utsname()->machine[1] =
 123                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 124         alternative_instructions();
 125
 126         fpu__init_check_bugs();
 127 #else /* CONFIG_X86_64 */
 128         alternative_instructions();
 129
 130         /*
 131          * Make sure the first 2MB area is not mapped by huge pages
 132          * There are typically fixed size MTRRs in there and overlapping
 133          * MTRRs into large pages causes slow downs.
 134          *
 135          * Right now we don't do that with gbpages because there seems
 136          * very little benefit for that case.
 137          */
 138         if (!direct_gbpages)
 139                 set_memory_4k((unsigned long)__va(0), 1);
 140 #endif
 141 }
 142
 143 void
 144 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 145 {
 146         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 147         struct thread_info *ti = current_thread_info();
 148
 149         /* Is MSR_SPEC_CTRL implemented ? */
 150         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 151                 /*
 152                  * Restrict guest_spec_ctrl to supported values. Clear the
 153                  * modifiable bits in the host base value and or the
 154                  * modifiable bits from the guest value.
 155                  */
 156                 guestval = hostval & ~x86_spec_ctrl_mask;
 157                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 158
 159                 /* SSBD controlled in MSR_SPEC_CTRL */
 160                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 161                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 162                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 163
 164                 /* Conditional STIBP enabled? */
 165                 if (static_branch_unlikely(&switch_to_cond_stibp))
 166                         hostval |= stibp_tif_to_spec_ctrl(ti->flags);
 167
 168                 if (hostval != guestval) {
 169                         msrval = setguest ? guestval : hostval;
 170                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 171                 }
 172         }
 173
 174         /*
 175          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 176          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 177          */
 178         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 179             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 180                 return;
 181
 182         /*
 183          * If the host has SSBD mitigation enabled, force it in the host's
 184          * virtual MSR value. If its not permanently enabled, evaluate
 185          * current's TIF_SSBD thread flag.
 186          */
 187         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 188                 hostval = SPEC_CTRL_SSBD;
 189         else
 190                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 191
 192         /* Sanitize the guest value */
 193         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 194
 195         if (hostval != guestval) {
 196                 unsigned long tif;
 197
 198                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 199                                  ssbd_spec_ctrl_to_tif(hostval);
 200
 201                 speculation_ctrl_update(tif);
 202         }
 203 }
 204 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 205
 206 static void x86_amd_ssb_disable(void)
 207 {
 208         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 209
 210         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 211                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 212         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 213                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 214 }
 215
 216 #undef pr_fmt
 217 #define pr_fmt(fmt)     "MDS: " fmt
 218
 219 /* Default mitigation for L1TF-affected CPUs */
 220 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 221
 222 static const char * const mds_strings[] = {
 223         [MDS_MITIGATION_OFF]    = "Vulnerable",
 224         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers"
 225 };
 226
 227 static void __init mds_select_mitigation(void)
 228 {
 229         if (!boot_cpu_has_bug(X86_BUG_MDS)) {
 230                 mds_mitigation = MDS_MITIGATION_OFF;
 231                 return;
 232         }
 233
 234         if (mds_mitigation == MDS_MITIGATION_FULL) {
 235                 if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 236                         static_branch_enable(&mds_user_clear);
 237                 else
 238                         mds_mitigation = MDS_MITIGATION_OFF;
 239         }
 240         pr_info("%s\n", mds_strings[mds_mitigation]);
 241 }
 242
 243 static int __init mds_cmdline(char *str)
 244 {
 245         if (!boot_cpu_has_bug(X86_BUG_MDS))
 246                 return 0;
 247
 248         if (!str)
 249                 return -EINVAL;
 250
 251         if (!strcmp(str, "off"))
 252                 mds_mitigation = MDS_MITIGATION_OFF;
 253         else if (!strcmp(str, "full"))
 254                 mds_mitigation = MDS_MITIGATION_FULL;
 255
 256         return 0;
 257 }
 258 early_param("mds", mds_cmdline);
 259
 260 #undef pr_fmt
 261 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 262
 263 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 264         SPECTRE_V2_NONE;
 265
 266 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
 267         SPECTRE_V2_USER_NONE;
 268
 269 #ifdef RETPOLINE
 270 static bool spectre_v2_bad_module;
 271
 272 bool retpoline_module_ok(bool has_retpoline)
 273 {
 274         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 275                 return true;
 276
 277         pr_err("System may be vulnerable to spectre v2\n");
 278         spectre_v2_bad_module = true;
 279         return false;
 280 }
 281
 282 static inline const char *spectre_v2_module_string(void)
 283 {
 284         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 285 }
 286 #else
 287 static inline const char *spectre_v2_module_string(void) { return ""; }
 288 #endif
 289
 290 static inline bool match_option(const char *arg, int arglen, const char *opt)
 291 {
 292         int len = strlen(opt);
 293
 294         return len == arglen && !strncmp(arg, opt, len);
 295 }
 296
 297 /* The kernel command line selection for spectre v2 */
 298 enum spectre_v2_mitigation_cmd {
 299         SPECTRE_V2_CMD_NONE,
 300         SPECTRE_V2_CMD_AUTO,
 301         SPECTRE_V2_CMD_FORCE,
 302         SPECTRE_V2_CMD_RETPOLINE,
 303         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 304         SPECTRE_V2_CMD_RETPOLINE_AMD,
 305 };
 306
 307 enum spectre_v2_user_cmd {
 308         SPECTRE_V2_USER_CMD_NONE,
 309         SPECTRE_V2_USER_CMD_AUTO,
 310         SPECTRE_V2_USER_CMD_FORCE,
 311         SPECTRE_V2_USER_CMD_PRCTL,
 312         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
 313         SPECTRE_V2_USER_CMD_SECCOMP,
 314         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
 315 };
 316
 317 static const char * const spectre_v2_user_strings[] = {
 318         [SPECTRE_V2_USER_NONE]          = "User space: Vulnerable",
 319         [SPECTRE_V2_USER_STRICT]        = "User space: Mitigation: STIBP protection",
 320         [SPECTRE_V2_USER_PRCTL]         = "User space: Mitigation: STIBP via prctl",
 321         [SPECTRE_V2_USER_SECCOMP]       = "User space: Mitigation: STIBP via seccomp and prctl",
 322 };
 323
 324 static const struct {
 325         const char                      *option;
 326         enum spectre_v2_user_cmd        cmd;
 327         bool                            secure;
 328 } v2_user_options[] __initdata = {
 329         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
 330         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
 331         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
 332         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
 333         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
 334         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
 335         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
 336 };
 337
 338 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
 339 {
 340         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 341                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
 342 }
 343
 344 static enum spectre_v2_user_cmd __init
 345 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
 346 {
 347         char arg[20];
 348         int ret, i;
 349
 350         switch (v2_cmd) {
 351         case SPECTRE_V2_CMD_NONE:
 352                 return SPECTRE_V2_USER_CMD_NONE;
 353         case SPECTRE_V2_CMD_FORCE:
 354                 return SPECTRE_V2_USER_CMD_FORCE;
 355         default:
 356                 break;
 357         }
 358
 359         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
 360                                   arg, sizeof(arg));
 361         if (ret < 0)
 362                 return SPECTRE_V2_USER_CMD_AUTO;
 363
 364         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
 365                 if (match_option(arg, ret, v2_user_options[i].option)) {
 366                         spec_v2_user_print_cond(v2_user_options[i].option,
 367                                                 v2_user_options[i].secure);
 368                         return v2_user_options[i].cmd;
 369                 }
 370         }
 371
 372         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
 373         return SPECTRE_V2_USER_CMD_AUTO;
 374 }
 375
 376 static void __init
 377 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
 378 {
 379         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
 380         bool smt_possible = IS_ENABLED(CONFIG_SMP);
 381         enum spectre_v2_user_cmd cmd;
 382
 383         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
 384                 return;
 385
 386         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 387             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 388                 smt_possible = false;
 389
 390         cmd = spectre_v2_parse_user_cmdline(v2_cmd);
 391         switch (cmd) {
 392         case SPECTRE_V2_USER_CMD_NONE:
 393                 goto set_mode;
 394         case SPECTRE_V2_USER_CMD_FORCE:
 395                 mode = SPECTRE_V2_USER_STRICT;
 396                 break;
 397         case SPECTRE_V2_USER_CMD_PRCTL:
 398         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 399                 mode = SPECTRE_V2_USER_PRCTL;
 400                 break;
 401         case SPECTRE_V2_USER_CMD_AUTO:
 402         case SPECTRE_V2_USER_CMD_SECCOMP:
 403         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 404                 if (IS_ENABLED(CONFIG_SECCOMP))
 405                         mode = SPECTRE_V2_USER_SECCOMP;
 406                 else
 407                         mode = SPECTRE_V2_USER_PRCTL;
 408                 break;
 409         }
 410
 411         /* Initialize Indirect Branch Prediction Barrier */
 412         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 413                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 414
 415                 switch (cmd) {
 416                 case SPECTRE_V2_USER_CMD_FORCE:
 417                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 418                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 419                         static_branch_enable(&switch_mm_always_ibpb);
 420                         break;
 421                 case SPECTRE_V2_USER_CMD_PRCTL:
 422                 case SPECTRE_V2_USER_CMD_AUTO:
 423                 case SPECTRE_V2_USER_CMD_SECCOMP:
 424                         static_branch_enable(&switch_mm_cond_ibpb);
 425                         break;
 426                 default:
 427                         break;
 428                 }
 429
 430                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
 431                         static_key_enabled(&switch_mm_always_ibpb) ?
 432                         "always-on" : "conditional");
 433         }
 434
 435         /* If enhanced IBRS is enabled no STIPB required */
 436         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 437                 return;
 438
 439         /*
 440          * If SMT is not possible or STIBP is not available clear the STIPB
 441          * mode.
 442          */
 443         if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP))
 444                 mode = SPECTRE_V2_USER_NONE;
 445 set_mode:
 446         spectre_v2_user = mode;
 447         /* Only print the STIBP mode when SMT possible */
 448         if (smt_possible)
 449                 pr_info("%s\n", spectre_v2_user_strings[mode]);
 450 }
 451
 452 static const char * const spectre_v2_strings[] = {
 453         [SPECTRE_V2_NONE]                       = "Vulnerable",
 454         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
 455         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
 456         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 457         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 458         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
 459 };
 460
 461 static const struct {
 462         const char *option;
 463         enum spectre_v2_mitigation_cmd cmd;
 464         bool secure;
 465 } mitigation_options[] __initdata = {
 466         { "off",                SPECTRE_V2_CMD_NONE,              false },
 467         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
 468         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
 469         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 470         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 471         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 472 };
 473
 474 static void __init spec_v2_print_cond(const char *reason, bool secure)
 475 {
 476         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 477                 pr_info("%s selected on command line.\n", reason);
 478 }
 479
 480 static inline bool retp_compiler(void)
 481 {
 482         return __is_defined(RETPOLINE);
 483 }
 484
 485 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 486 {
 487         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 488         char arg[20];
 489         int ret, i;
 490
 491         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 492                 return SPECTRE_V2_CMD_NONE;
 493
 494         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 495         if (ret < 0)
 496                 return SPECTRE_V2_CMD_AUTO;
 497
 498         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 499                 if (!match_option(arg, ret, mitigation_options[i].option))
 500                         continue;
 501                 cmd = mitigation_options[i].cmd;
 502                 break;
 503         }
 504
 505         if (i >= ARRAY_SIZE(mitigation_options)) {
 506                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 507                 return SPECTRE_V2_CMD_AUTO;
 508         }
 509
 510         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 511              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 512              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 513             !IS_ENABLED(CONFIG_RETPOLINE)) {
 514                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 515                 return SPECTRE_V2_CMD_AUTO;
 516         }
 517
 518         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 519             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 520                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 521                 return SPECTRE_V2_CMD_AUTO;
 522         }
 523
 524         spec_v2_print_cond(mitigation_options[i].option,
 525                            mitigation_options[i].secure);
 526         return cmd;
 527 }
 528
 529 static void __init spectre_v2_select_mitigation(void)
 530 {
 531         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 532         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 533
 534         /*
 535          * If the CPU is not affected and the command line mode is NONE or AUTO
 536          * then nothing to do.
 537          */
 538         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 539             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 540                 return;
 541
 542         switch (cmd) {
 543         case SPECTRE_V2_CMD_NONE:
 544                 return;
 545
 546         case SPECTRE_V2_CMD_FORCE:
 547         case SPECTRE_V2_CMD_AUTO:
 548                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 549                         mode = SPECTRE_V2_IBRS_ENHANCED;
 550                         /* Force it so VMEXIT will restore correctly */
 551                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
 552                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 553                         goto specv2_set_mode;
 554                 }
 555                 if (IS_ENABLED(CONFIG_RETPOLINE))
 556                         goto retpoline_auto;
 557                 break;
 558         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 559                 if (IS_ENABLED(CONFIG_RETPOLINE))
 560                         goto retpoline_amd;
 561                 break;
 562         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 563                 if (IS_ENABLED(CONFIG_RETPOLINE))
 564                         goto retpoline_generic;
 565                 break;
 566         case SPECTRE_V2_CMD_RETPOLINE:
 567                 if (IS_ENABLED(CONFIG_RETPOLINE))
 568                         goto retpoline_auto;
 569                 break;
 570         }
 571         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 572         return;
 573
 574 retpoline_auto:
 575         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 576         retpoline_amd:
 577                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 578                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 579                         goto retpoline_generic;
 580                 }
 581                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 582                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 583                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 584                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 585         } else {
 586         retpoline_generic:
 587                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 588                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 589                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 590         }
 591
 592 specv2_set_mode:
 593         spectre_v2_enabled = mode;
 594         pr_info("%s\n", spectre_v2_strings[mode]);
 595
 596         /*
 597          * If spectre v2 protection has been enabled, unconditionally fill
 598          * RSB during a context switch; this protects against two independent
 599          * issues:
 600          *
 601          *      - RSB underflow (and switch to BTB) on Skylake+
 602          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
 603          */
 604         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 605         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
 606
 607         /*
 608          * Retpoline means the kernel is safe because it has no indirect
 609          * branches. Enhanced IBRS protects firmware too, so, enable restricted
 610          * speculation around firmware calls only when Enhanced IBRS isn't
 611          * supported.
 612          *
 613          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
 614          * the user might select retpoline on the kernel command line and if
 615          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
 616          * enable IBRS around firmware calls.
 617          */
 618         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
 619                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 620                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 621         }
 622
 623         /* Set up IBPB and STIBP depending on the general spectre V2 command */
 624         spectre_v2_user_select_mitigation(cmd);
 625
 626         /* Enable STIBP if appropriate */
 627         arch_smt_update();
 628 }
 629
 630 static void update_stibp_msr(void * __unused)
 631 {
 632         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 633 }
 634
 635 /* Update x86_spec_ctrl_base in case SMT state changed. */
 636 static void update_stibp_strict(void)
 637 {
 638         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
 639
 640         if (sched_smt_active())
 641                 mask |= SPEC_CTRL_STIBP;
 642
 643         if (mask == x86_spec_ctrl_base)
 644                 return;
 645
 646         pr_info("Update user space SMT mitigation: STIBP %s\n",
 647                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
 648         x86_spec_ctrl_base = mask;
 649         on_each_cpu(update_stibp_msr, NULL, 1);
 650 }
 651
 652 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
 653 static void update_indir_branch_cond(void)
 654 {
 655         if (sched_smt_active())
 656                 static_branch_enable(&switch_to_cond_stibp);
 657         else
 658                 static_branch_disable(&switch_to_cond_stibp);
 659 }
 660
 661 /* Update the static key controlling the MDS CPU buffer clear in idle */
 662 static void update_mds_branch_idle(void)
 663 {
 664         /*
 665          * Enable the idle clearing if SMT is active on CPUs which are
 666          * affected only by MSBDS and not any other MDS variant.
 667          *
 668          * The other variants cannot be mitigated when SMT is enabled, so
 669          * clearing the buffers on idle just to prevent the Store Buffer
 670          * repartitioning leak would be a window dressing exercise.
 671          */
 672         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
 673                 return;
 674
 675         if (sched_smt_active())
 676                 static_branch_enable(&mds_idle_clear);
 677         else
 678                 static_branch_disable(&mds_idle_clear);
 679 }
 680
 681 void arch_smt_update(void)
 682 {
 683         /* Enhanced IBRS implies STIBP. No update required. */
 684         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 685                 return;
 686
 687         mutex_lock(&spec_ctrl_mutex);
 688
 689         switch (spectre_v2_user) {
 690         case SPECTRE_V2_USER_NONE:
 691                 break;
 692         case SPECTRE_V2_USER_STRICT:
 693                 update_stibp_strict();
 694                 break;
 695         case SPECTRE_V2_USER_PRCTL:
 696         case SPECTRE_V2_USER_SECCOMP:
 697                 update_indir_branch_cond();
 698                 break;
 699         }
 700
 701         if (mds_mitigation == MDS_MITIGATION_FULL)
 702                 update_mds_branch_idle();
 703
 704         mutex_unlock(&spec_ctrl_mutex);
 705 }
 706
 707 #undef pr_fmt
 708 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 709
 710 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 711
 712 /* The kernel command line selection */
 713 enum ssb_mitigation_cmd {
 714         SPEC_STORE_BYPASS_CMD_NONE,
 715         SPEC_STORE_BYPASS_CMD_AUTO,
 716         SPEC_STORE_BYPASS_CMD_ON,
 717         SPEC_STORE_BYPASS_CMD_PRCTL,
 718         SPEC_STORE_BYPASS_CMD_SECCOMP,
 719 };
 720
 721 static const char * const ssb_strings[] = {
 722         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 723         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 724         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 725         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 726 };
 727
 728 static const struct {
 729         const char *option;
 730         enum ssb_mitigation_cmd cmd;
 731 } ssb_mitigation_options[]  __initdata = {
 732         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 733         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 734         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 735         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 736         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 737 };
 738
 739 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 740 {
 741         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 742         char arg[20];
 743         int ret, i;
 744
 745         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 746                 return SPEC_STORE_BYPASS_CMD_NONE;
 747         } else {
 748                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 749                                           arg, sizeof(arg));
 750                 if (ret < 0)
 751                         return SPEC_STORE_BYPASS_CMD_AUTO;
 752
 753                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 754                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 755                                 continue;
 756
 757                         cmd = ssb_mitigation_options[i].cmd;
 758                         break;
 759                 }
 760
 761                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 762                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 763                         return SPEC_STORE_BYPASS_CMD_AUTO;
 764                 }
 765         }
 766
 767         return cmd;
 768 }
 769
 770 static enum ssb_mitigation __init __ssb_select_mitigation(void)
 771 {
 772         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 773         enum ssb_mitigation_cmd cmd;
 774
 775         if (!boot_cpu_has(X86_FEATURE_SSBD))
 776                 return mode;
 777
 778         cmd = ssb_parse_cmdline();
 779         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 780             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 781              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 782                 return mode;
 783
 784         switch (cmd) {
 785         case SPEC_STORE_BYPASS_CMD_AUTO:
 786         case SPEC_STORE_BYPASS_CMD_SECCOMP:
 787                 /*
 788                  * Choose prctl+seccomp as the default mode if seccomp is
 789                  * enabled.
 790                  */
 791                 if (IS_ENABLED(CONFIG_SECCOMP))
 792                         mode = SPEC_STORE_BYPASS_SECCOMP;
 793                 else
 794                         mode = SPEC_STORE_BYPASS_PRCTL;
 795                 break;
 796         case SPEC_STORE_BYPASS_CMD_ON:
 797                 mode = SPEC_STORE_BYPASS_DISABLE;
 798                 break;
 799         case SPEC_STORE_BYPASS_CMD_PRCTL:
 800                 mode = SPEC_STORE_BYPASS_PRCTL;
 801                 break;
 802         case SPEC_STORE_BYPASS_CMD_NONE:
 803                 break;
 804         }
 805
 806         /*
 807          * We have three CPU feature flags that are in play here:
 808          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
 809          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
 810          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
 811          */
 812         if (mode == SPEC_STORE_BYPASS_DISABLE) {
 813                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 814                 /*
 815                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
 816                  * use a completely different MSR and bit dependent on family.
 817                  */
 818                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
 819                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
 820                         x86_amd_ssb_disable();
 821                 } else {
 822                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
 823                         x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
 824                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 825                 }
 826         }
 827
 828         return mode;
 829 }
 830
 831 static void ssb_select_mitigation(void)
 832 {
 833         ssb_mode = __ssb_select_mitigation();
 834
 835         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 836                 pr_info("%s\n", ssb_strings[ssb_mode]);
 837 }
 838
 839 #undef pr_fmt
 840 #define pr_fmt(fmt)     "Speculation prctl: " fmt
 841
 842 static void task_update_spec_tif(struct task_struct *tsk)
 843 {
 844         /* Force the update of the real TIF bits */
 845         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
 846
 847         /*
 848          * Immediately update the speculation control MSRs for the current
 849          * task, but for a non-current task delay setting the CPU
 850          * mitigation until it is scheduled next.
 851          *
 852          * This can only happen for SECCOMP mitigation. For PRCTL it's
 853          * always the current task.
 854          */
 855         if (tsk == current)
 856                 speculation_ctrl_update_current();
 857 }
 858
 859 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
 860 {
 861         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
 862             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
 863                 return -ENXIO;
 864
 865         switch (ctrl) {
 866         case PR_SPEC_ENABLE:
 867                 /* If speculation is force disabled, enable is not allowed */
 868                 if (task_spec_ssb_force_disable(task))
 869                         return -EPERM;
 870                 task_clear_spec_ssb_disable(task);
 871                 task_update_spec_tif(task);
 872                 break;
 873         case PR_SPEC_DISABLE:
 874                 task_set_spec_ssb_disable(task);
 875                 task_update_spec_tif(task);
 876                 break;
 877         case PR_SPEC_FORCE_DISABLE:
 878                 task_set_spec_ssb_disable(task);
 879                 task_set_spec_ssb_force_disable(task);
 880                 task_update_spec_tif(task);
 881                 break;
 882         default:
 883                 return -ERANGE;
 884         }
 885         return 0;
 886 }
 887
 888 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
 889 {
 890         switch (ctrl) {
 891         case PR_SPEC_ENABLE:
 892                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
 893                         return 0;
 894                 /*
 895                  * Indirect branch speculation is always disabled in strict
 896                  * mode.
 897                  */
 898                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT)
 899                         return -EPERM;
 900                 task_clear_spec_ib_disable(task);
 901                 task_update_spec_tif(task);
 902                 break;
 903         case PR_SPEC_DISABLE:
 904         case PR_SPEC_FORCE_DISABLE:
 905                 /*
 906                  * Indirect branch speculation is always allowed when
 907                  * mitigation is force disabled.
 908                  */
 909                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
 910                         return -EPERM;
 911                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT)
 912                         return 0;
 913                 task_set_spec_ib_disable(task);
 914                 if (ctrl == PR_SPEC_FORCE_DISABLE)
 915                         task_set_spec_ib_force_disable(task);
 916                 task_update_spec_tif(task);
 917                 break;
 918         default:
 919                 return -ERANGE;
 920         }
 921         return 0;
 922 }
 923
 924 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
 925                              unsigned long ctrl)
 926 {
 927         switch (which) {
 928         case PR_SPEC_STORE_BYPASS:
 929                 return ssb_prctl_set(task, ctrl);
 930         case PR_SPEC_INDIRECT_BRANCH:
 931                 return ib_prctl_set(task, ctrl);
 932         default:
 933                 return -ENODEV;
 934         }
 935 }
 936
 937 #ifdef CONFIG_SECCOMP
 938 void arch_seccomp_spec_mitigate(struct task_struct *task)
 939 {
 940         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
 941                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 942         if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP)
 943                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 944 }
 945 #endif
 946
 947 static int ssb_prctl_get(struct task_struct *task)
 948 {
 949         switch (ssb_mode) {
 950         case SPEC_STORE_BYPASS_DISABLE:
 951                 return PR_SPEC_DISABLE;
 952         case SPEC_STORE_BYPASS_SECCOMP:
 953         case SPEC_STORE_BYPASS_PRCTL:
 954                 if (task_spec_ssb_force_disable(task))
 955                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 956                 if (task_spec_ssb_disable(task))
 957                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 958                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 959         default:
 960                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 961                         return PR_SPEC_ENABLE;
 962                 return PR_SPEC_NOT_AFFECTED;
 963         }
 964 }
 965
 966 static int ib_prctl_get(struct task_struct *task)
 967 {
 968         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 969                 return PR_SPEC_NOT_AFFECTED;
 970
 971         switch (spectre_v2_user) {
 972         case SPECTRE_V2_USER_NONE:
 973                 return PR_SPEC_ENABLE;
 974         case SPECTRE_V2_USER_PRCTL:
 975         case SPECTRE_V2_USER_SECCOMP:
 976                 if (task_spec_ib_force_disable(task))
 977                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 978                 if (task_spec_ib_disable(task))
 979                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 980                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 981         case SPECTRE_V2_USER_STRICT:
 982                 return PR_SPEC_DISABLE;
 983         default:
 984                 return PR_SPEC_NOT_AFFECTED;
 985         }
 986 }
 987
 988 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
 989 {
 990         switch (which) {
 991         case PR_SPEC_STORE_BYPASS:
 992                 return ssb_prctl_get(task);
 993         case PR_SPEC_INDIRECT_BRANCH:
 994                 return ib_prctl_get(task);
 995         default:
 996                 return -ENODEV;
 997         }
 998 }
 999
1000 void x86_spec_ctrl_setup_ap(void)
1001 {
1002         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1003                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1004
1005         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1006                 x86_amd_ssb_disable();
1007 }
1008
1009 #undef pr_fmt
1010 #define pr_fmt(fmt)     "L1TF: " fmt
1011
1012 /* Default mitigation for L1TF-affected CPUs */
1013 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
1014 #if IS_ENABLED(CONFIG_KVM_INTEL)
1015 EXPORT_SYMBOL_GPL(l1tf_mitigation);
1016 #endif
1017 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
1018 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1019
1020 /*
1021  * These CPUs all support 44bits physical address space internally in the
1022  * cache but CPUID can report a smaller number of physical address bits.
1023  *
1024  * The L1TF mitigation uses the top most address bit for the inversion of
1025  * non present PTEs. When the installed memory reaches into the top most
1026  * address bit due to memory holes, which has been observed on machines
1027  * which report 36bits physical address bits and have 32G RAM installed,
1028  * then the mitigation range check in l1tf_select_mitigation() triggers.
1029  * This is a false positive because the mitigation is still possible due to
1030  * the fact that the cache uses 44bit internally. Use the cache bits
1031  * instead of the reported physical bits and adjust them on the affected
1032  * machines to 44bit if the reported bits are less than 44.
1033  */
1034 static void override_cache_bits(struct cpuinfo_x86 *c)
1035 {
1036         if (c->x86 != 6)
1037                 return;
1038
1039         switch (c->x86_model) {
1040         case INTEL_FAM6_NEHALEM:
1041         case INTEL_FAM6_WESTMERE:
1042         case INTEL_FAM6_SANDYBRIDGE:
1043         case INTEL_FAM6_IVYBRIDGE:
1044         case INTEL_FAM6_HASWELL_CORE:
1045         case INTEL_FAM6_HASWELL_ULT:
1046         case INTEL_FAM6_HASWELL_GT3E:
1047         case INTEL_FAM6_BROADWELL_CORE:
1048         case INTEL_FAM6_BROADWELL_GT3E:
1049         case INTEL_FAM6_SKYLAKE_MOBILE:
1050         case INTEL_FAM6_SKYLAKE_DESKTOP:
1051         case INTEL_FAM6_KABYLAKE_MOBILE:
1052         case INTEL_FAM6_KABYLAKE_DESKTOP:
1053                 if (c->x86_cache_bits < 44)
1054                         c->x86_cache_bits = 44;
1055                 break;
1056         }
1057 }
1058
1059 static void __init l1tf_select_mitigation(void)
1060 {
1061         u64 half_pa;
1062
1063         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1064                 return;
1065
1066         override_cache_bits(&boot_cpu_data);
1067
1068         switch (l1tf_mitigation) {
1069         case L1TF_MITIGATION_OFF:
1070         case L1TF_MITIGATION_FLUSH_NOWARN:
1071         case L1TF_MITIGATION_FLUSH:
1072                 break;
1073         case L1TF_MITIGATION_FLUSH_NOSMT:
1074         case L1TF_MITIGATION_FULL:
1075                 cpu_smt_disable(false);
1076                 break;
1077         case L1TF_MITIGATION_FULL_FORCE:
1078                 cpu_smt_disable(true);
1079                 break;
1080         }
1081
1082 #if CONFIG_PGTABLE_LEVELS == 2
1083         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1084         return;
1085 #endif
1086
1087         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
1088         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
1089                         e820_any_mapped(half_pa, ULLONG_MAX - half_pa, E820_RAM)) {
1090                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
1091                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1092                                 half_pa);
1093                 pr_info("However, doing so will make a part of your RAM unusable.\n");
1094                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html might help you decide.\n");
1095                 return;
1096         }
1097
1098         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
1099 }
1100
1101 static int __init l1tf_cmdline(char *str)
1102 {
1103         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1104                 return 0;
1105
1106         if (!str)
1107                 return -EINVAL;
1108
1109         if (!strcmp(str, "off"))
1110                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1111         else if (!strcmp(str, "flush,nowarn"))
1112                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
1113         else if (!strcmp(str, "flush"))
1114                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
1115         else if (!strcmp(str, "flush,nosmt"))
1116                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1117         else if (!strcmp(str, "full"))
1118                 l1tf_mitigation = L1TF_MITIGATION_FULL;
1119         else if (!strcmp(str, "full,force"))
1120                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
1121
1122         return 0;
1123 }
1124 early_param("l1tf", l1tf_cmdline);
1125
1126 #undef pr_fmt
1127
1128 #ifdef CONFIG_SYSFS
1129
1130 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
1131
1132 #if IS_ENABLED(CONFIG_KVM_INTEL)
1133 static const char * const l1tf_vmx_states[] = {
1134         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
1135         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
1136         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
1137         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
1138         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
1139         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
1140 };
1141
1142 static ssize_t l1tf_show_state(char *buf)
1143 {
1144         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
1145                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1146
1147         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
1148             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
1149              sched_smt_active())) {
1150                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
1151                                l1tf_vmx_states[l1tf_vmx_mitigation]);
1152         }
1153
1154         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
1155                        l1tf_vmx_states[l1tf_vmx_mitigation],
1156                        sched_smt_active() ? "vulnerable" : "disabled");
1157 }
1158 #else
1159 static ssize_t l1tf_show_state(char *buf)
1160 {
1161         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1162 }
1163 #endif
1164
1165 static ssize_t mds_show_state(char *buf)
1166 {
1167 #ifdef CONFIG_HYPERVISOR_GUEST
1168         if (x86_hyper) {
1169                 return sprintf(buf, "%s; SMT Host state unknown\n",
1170                                mds_strings[mds_mitigation]);
1171         }
1172 #endif
1173
1174         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
1175                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1176                                sched_smt_active() ? "mitigated" : "disabled");
1177         }
1178
1179         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1180                        sched_smt_active() ? "vulnerable" : "disabled");
1181 }
1182
1183 static char *stibp_state(void)
1184 {
1185         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
1186                 return "";
1187
1188         switch (spectre_v2_user) {
1189         case SPECTRE_V2_USER_NONE:
1190                 return ", STIBP: disabled";
1191         case SPECTRE_V2_USER_STRICT:
1192                 return ", STIBP: forced";
1193         case SPECTRE_V2_USER_PRCTL:
1194         case SPECTRE_V2_USER_SECCOMP:
1195                 if (static_key_enabled(&switch_to_cond_stibp))
1196                         return ", STIBP: conditional";
1197         }
1198         return "";
1199 }
1200
1201 static char *ibpb_state(void)
1202 {
1203         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1204                 if (static_key_enabled(&switch_mm_always_ibpb))
1205                         return ", IBPB: always-on";
1206                 if (static_key_enabled(&switch_mm_cond_ibpb))
1207                         return ", IBPB: conditional";
1208                 return ", IBPB: disabled";
1209         }
1210         return "";
1211 }
1212
1213 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
1214                                char *buf, unsigned int bug)
1215 {
1216         if (!boot_cpu_has_bug(bug))
1217                 return sprintf(buf, "Not affected\n");
1218
1219         switch (bug) {
1220         case X86_BUG_CPU_MELTDOWN:
1221                 if (boot_cpu_has(X86_FEATURE_KAISER))
1222                         return sprintf(buf, "Mitigation: PTI\n");
1223
1224                 break;
1225
1226         case X86_BUG_SPECTRE_V1:
1227                 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
1228
1229         case X86_BUG_SPECTRE_V2:
1230                 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
1231                                ibpb_state(),
1232                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1233                                stibp_state(),
1234                                boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1235                                spectre_v2_module_string());
1236
1237         case X86_BUG_SPEC_STORE_BYPASS:
1238                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1239
1240         case X86_BUG_L1TF:
1241                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
1242                         return l1tf_show_state(buf);
1243                 break;
1244
1245         case X86_BUG_MDS:
1246                 return mds_show_state(buf);
1247
1248         default:
1249                 break;
1250         }
1251
1252         return sprintf(buf, "Vulnerable\n");
1253 }
1254
1255 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1256 {
1257         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1258 }
1259
1260 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
1261 {
1262         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
1263 }
1264
1265 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
1266 {
1267         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
1268 }
1269
1270 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1271 {
1272         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1273 }
1274
1275 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1276 {
1277         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1278 }
1279
1280 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
1281 {
1282         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
1283 }
1284 #endif