arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15 #include <linux/nospec.h>
  16 #include <linux/prctl.h>
  17 #include <linux/sched/smt.h>
  18
  19 #include <asm/spec-ctrl.h>
  20 #include <asm/cmdline.h>
  21 #include <asm/bugs.h>
  22 #include <asm/processor.h>
  23 #include <asm/processor-flags.h>
  24 #include <asm/fpu/internal.h>
  25 #include <asm/msr.h>
  26 #include <asm/vmx.h>
  27 #include <asm/paravirt.h>
  28 #include <asm/alternative.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/set_memory.h>
  31 #include <asm/intel-family.h>
  32 #include <asm/e820/api.h>
  33 #include <asm/hypervisor.h>
  34
  35 static void __init spectre_v2_select_mitigation(void);
  36 static void __init ssb_select_mitigation(void);
  37 static void __init l1tf_select_mitigation(void);
  38
  39 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  40 u64 x86_spec_ctrl_base;
  41 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  42 static DEFINE_MUTEX(spec_ctrl_mutex);
  43
  44 /*
  45  * The vendor and possibly platform specific bits which can be modified in
  46  * x86_spec_ctrl_base.
  47  */
  48 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  49
  50 /*
  51  * AMD specific MSR info for Speculative Store Bypass control.
  52  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  53  */
  54 u64 __ro_after_init x86_amd_ls_cfg_base;
  55 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  56
  57 void __init check_bugs(void)
  58 {
  59         identify_boot_cpu();
  60
  61         /*
  62          * identify_boot_cpu() initialized SMT support information, let the
  63          * core code know.
  64          */
  65         cpu_smt_check_topology_early();
  66
  67         if (!IS_ENABLED(CONFIG_SMP)) {
  68                 pr_info("CPU: ");
  69                 print_cpu_info(&boot_cpu_data);
  70         }
  71
  72         /*
  73          * Read the SPEC_CTRL MSR to account for reserved bits which may
  74          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  75          * init code as it is not enumerated and depends on the family.
  76          */
  77         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  78                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  79
  80         /* Allow STIBP in MSR_SPEC_CTRL if supported */
  81         if (boot_cpu_has(X86_FEATURE_STIBP))
  82                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
  83
  84         /* Select the proper spectre mitigation before patching alternatives */
  85         spectre_v2_select_mitigation();
  86
  87         /*
  88          * Select proper mitigation for any exposure to the Speculative Store
  89          * Bypass vulnerability.
  90          */
  91         ssb_select_mitigation();
  92
  93         l1tf_select_mitigation();
  94
  95 #ifdef CONFIG_X86_32
  96         /*
  97          * Check whether we are able to run this kernel safely on SMP.
  98          *
  99          * - i386 is no longer supported.
 100          * - In order to run on anything without a TSC, we need to be
 101          *   compiled for a i486.
 102          */
 103         if (boot_cpu_data.x86 < 4)
 104                 panic("Kernel requires i486+ for 'invlpg' and other features");
 105
 106         init_utsname()->machine[1] =
 107                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 108         alternative_instructions();
 109
 110         fpu__init_check_bugs();
 111 #else /* CONFIG_X86_64 */
 112         alternative_instructions();
 113
 114         /*
 115          * Make sure the first 2MB area is not mapped by huge pages
 116          * There are typically fixed size MTRRs in there and overlapping
 117          * MTRRs into large pages causes slow downs.
 118          *
 119          * Right now we don't do that with gbpages because there seems
 120          * very little benefit for that case.
 121          */
 122         if (!direct_gbpages)
 123                 set_memory_4k((unsigned long)__va(0), 1);
 124 #endif
 125 }
 126
 127 void
 128 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 129 {
 130         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 131         struct thread_info *ti = current_thread_info();
 132
 133         /* Is MSR_SPEC_CTRL implemented ? */
 134         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 135                 /*
 136                  * Restrict guest_spec_ctrl to supported values. Clear the
 137                  * modifiable bits in the host base value and or the
 138                  * modifiable bits from the guest value.
 139                  */
 140                 guestval = hostval & ~x86_spec_ctrl_mask;
 141                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 142
 143                 /* SSBD controlled in MSR_SPEC_CTRL */
 144                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 145                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 146                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 147
 148                 if (hostval != guestval) {
 149                         msrval = setguest ? guestval : hostval;
 150                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 151                 }
 152         }
 153
 154         /*
 155          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 156          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 157          */
 158         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 159             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 160                 return;
 161
 162         /*
 163          * If the host has SSBD mitigation enabled, force it in the host's
 164          * virtual MSR value. If its not permanently enabled, evaluate
 165          * current's TIF_SSBD thread flag.
 166          */
 167         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 168                 hostval = SPEC_CTRL_SSBD;
 169         else
 170                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 171
 172         /* Sanitize the guest value */
 173         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 174
 175         if (hostval != guestval) {
 176                 unsigned long tif;
 177
 178                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 179                                  ssbd_spec_ctrl_to_tif(hostval);
 180
 181                 speculation_ctrl_update(tif);
 182         }
 183 }
 184 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 185
 186 static void x86_amd_ssb_disable(void)
 187 {
 188         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 189
 190         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 191                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 192         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 193                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 194 }
 195
 196 #undef pr_fmt
 197 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 198
 199 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 200         SPECTRE_V2_NONE;
 201
 202 #ifdef RETPOLINE
 203 static bool spectre_v2_bad_module;
 204
 205 bool retpoline_module_ok(bool has_retpoline)
 206 {
 207         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 208                 return true;
 209
 210         pr_err("System may be vulnerable to spectre v2\n");
 211         spectre_v2_bad_module = true;
 212         return false;
 213 }
 214
 215 static inline const char *spectre_v2_module_string(void)
 216 {
 217         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 218 }
 219 #else
 220 static inline const char *spectre_v2_module_string(void) { return ""; }
 221 #endif
 222
 223 static inline bool match_option(const char *arg, int arglen, const char *opt)
 224 {
 225         int len = strlen(opt);
 226
 227         return len == arglen && !strncmp(arg, opt, len);
 228 }
 229
 230 /* The kernel command line selection for spectre v2 */
 231 enum spectre_v2_mitigation_cmd {
 232         SPECTRE_V2_CMD_NONE,
 233         SPECTRE_V2_CMD_AUTO,
 234         SPECTRE_V2_CMD_FORCE,
 235         SPECTRE_V2_CMD_RETPOLINE,
 236         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 237         SPECTRE_V2_CMD_RETPOLINE_AMD,
 238 };
 239
 240 static const char *spectre_v2_strings[] = {
 241         [SPECTRE_V2_NONE]                       = "Vulnerable",
 242         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 243         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 244         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
 245 };
 246
 247 static const struct {
 248         const char *option;
 249         enum spectre_v2_mitigation_cmd cmd;
 250         bool secure;
 251 } mitigation_options[] = {
 252         { "off",                SPECTRE_V2_CMD_NONE,              false },
 253         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
 254         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
 255         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 256         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 257         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 258 };
 259
 260 static void __init spec2_print_if_insecure(const char *reason)
 261 {
 262         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 263                 pr_info("%s selected on command line.\n", reason);
 264 }
 265
 266 static void __init spec2_print_if_secure(const char *reason)
 267 {
 268         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 269                 pr_info("%s selected on command line.\n", reason);
 270 }
 271
 272 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 273 {
 274         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 275         char arg[20];
 276         int ret, i;
 277
 278         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 279                 return SPECTRE_V2_CMD_NONE;
 280
 281         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 282         if (ret < 0)
 283                 return SPECTRE_V2_CMD_AUTO;
 284
 285         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 286                 if (!match_option(arg, ret, mitigation_options[i].option))
 287                         continue;
 288                 cmd = mitigation_options[i].cmd;
 289                 break;
 290         }
 291
 292         if (i >= ARRAY_SIZE(mitigation_options)) {
 293                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 294                 return SPECTRE_V2_CMD_AUTO;
 295         }
 296
 297         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 298              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 299              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 300             !IS_ENABLED(CONFIG_RETPOLINE)) {
 301                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 302                 return SPECTRE_V2_CMD_AUTO;
 303         }
 304
 305         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 306             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON &&
 307             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 308                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 309                 return SPECTRE_V2_CMD_AUTO;
 310         }
 311
 312         if (mitigation_options[i].secure)
 313                 spec2_print_if_secure(mitigation_options[i].option);
 314         else
 315                 spec2_print_if_insecure(mitigation_options[i].option);
 316
 317         return cmd;
 318 }
 319
 320 static void __init spectre_v2_select_mitigation(void)
 321 {
 322         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 323         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 324
 325         /*
 326          * If the CPU is not affected and the command line mode is NONE or AUTO
 327          * then nothing to do.
 328          */
 329         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 330             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 331                 return;
 332
 333         switch (cmd) {
 334         case SPECTRE_V2_CMD_NONE:
 335                 return;
 336
 337         case SPECTRE_V2_CMD_FORCE:
 338         case SPECTRE_V2_CMD_AUTO:
 339                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 340                         mode = SPECTRE_V2_IBRS_ENHANCED;
 341                         /* Force it so VMEXIT will restore correctly */
 342                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
 343                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 344                         goto specv2_set_mode;
 345                 }
 346                 if (IS_ENABLED(CONFIG_RETPOLINE))
 347                         goto retpoline_auto;
 348                 break;
 349         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 350                 if (IS_ENABLED(CONFIG_RETPOLINE))
 351                         goto retpoline_amd;
 352                 break;
 353         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 354                 if (IS_ENABLED(CONFIG_RETPOLINE))
 355                         goto retpoline_generic;
 356                 break;
 357         case SPECTRE_V2_CMD_RETPOLINE:
 358                 if (IS_ENABLED(CONFIG_RETPOLINE))
 359                         goto retpoline_auto;
 360                 break;
 361         }
 362         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 363         return;
 364
 365 retpoline_auto:
 366         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
 367             boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
 368         retpoline_amd:
 369                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 370                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 371                         goto retpoline_generic;
 372                 }
 373                 mode = SPECTRE_V2_RETPOLINE_AMD;
 374                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 375                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 376         } else {
 377         retpoline_generic:
 378                 mode = SPECTRE_V2_RETPOLINE_GENERIC;
 379                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 380         }
 381
 382 specv2_set_mode:
 383         spectre_v2_enabled = mode;
 384         pr_info("%s\n", spectre_v2_strings[mode]);
 385
 386         /*
 387          * If spectre v2 protection has been enabled, unconditionally fill
 388          * RSB during a context switch; this protects against two independent
 389          * issues:
 390          *
 391          *      - RSB underflow (and switch to BTB) on Skylake+
 392          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
 393          */
 394         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 395         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
 396
 397         /* Initialize Indirect Branch Prediction Barrier if supported */
 398         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 399                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 400                 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
 401         }
 402
 403         /*
 404          * Retpoline means the kernel is safe because it has no indirect
 405          * branches. Enhanced IBRS protects firmware too, so, enable restricted
 406          * speculation around firmware calls only when Enhanced IBRS isn't
 407          * supported.
 408          *
 409          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
 410          * the user might select retpoline on the kernel command line and if
 411          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
 412          * enable IBRS around firmware calls.
 413          */
 414         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
 415                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 416                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 417         }
 418
 419         /* Enable STIBP if appropriate */
 420         arch_smt_update();
 421 }
 422
 423 static bool stibp_needed(void)
 424 {
 425         if (spectre_v2_enabled == SPECTRE_V2_NONE)
 426                 return false;
 427
 428         /* Enhanced IBRS makes using STIBP unnecessary. */
 429         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 430                 return false;
 431
 432         if (!boot_cpu_has(X86_FEATURE_STIBP))
 433                 return false;
 434
 435         return true;
 436 }
 437
 438 static void update_stibp_msr(void *info)
 439 {
 440         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 441 }
 442
 443 void arch_smt_update(void)
 444 {
 445         u64 mask;
 446
 447         if (!stibp_needed())
 448                 return;
 449
 450         mutex_lock(&spec_ctrl_mutex);
 451
 452         mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
 453         if (sched_smt_active())
 454                 mask |= SPEC_CTRL_STIBP;
 455
 456         if (mask != x86_spec_ctrl_base) {
 457                 pr_info("Spectre v2 cross-process SMT mitigation: %s STIBP\n",
 458                         mask & SPEC_CTRL_STIBP ? "Enabling" : "Disabling");
 459                 x86_spec_ctrl_base = mask;
 460                 on_each_cpu(update_stibp_msr, NULL, 1);
 461         }
 462         mutex_unlock(&spec_ctrl_mutex);
 463 }
 464
 465 #undef pr_fmt
 466 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 467
 468 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 469
 470 /* The kernel command line selection */
 471 enum ssb_mitigation_cmd {
 472         SPEC_STORE_BYPASS_CMD_NONE,
 473         SPEC_STORE_BYPASS_CMD_AUTO,
 474         SPEC_STORE_BYPASS_CMD_ON,
 475         SPEC_STORE_BYPASS_CMD_PRCTL,
 476         SPEC_STORE_BYPASS_CMD_SECCOMP,
 477 };
 478
 479 static const char *ssb_strings[] = {
 480         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 481         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 482         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 483         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 484 };
 485
 486 static const struct {
 487         const char *option;
 488         enum ssb_mitigation_cmd cmd;
 489 } ssb_mitigation_options[] = {
 490         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 491         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 492         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 493         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 494         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 495 };
 496
 497 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 498 {
 499         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 500         char arg[20];
 501         int ret, i;
 502
 503         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 504                 return SPEC_STORE_BYPASS_CMD_NONE;
 505         } else {
 506                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 507                                           arg, sizeof(arg));
 508                 if (ret < 0)
 509                         return SPEC_STORE_BYPASS_CMD_AUTO;
 510
 511                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 512                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 513                                 continue;
 514
 515                         cmd = ssb_mitigation_options[i].cmd;
 516                         break;
 517                 }
 518
 519                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 520                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 521                         return SPEC_STORE_BYPASS_CMD_AUTO;
 522                 }
 523         }
 524
 525         return cmd;
 526 }
 527
 528 static enum ssb_mitigation __init __ssb_select_mitigation(void)
 529 {
 530         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 531         enum ssb_mitigation_cmd cmd;
 532
 533         if (!boot_cpu_has(X86_FEATURE_SSBD))
 534                 return mode;
 535
 536         cmd = ssb_parse_cmdline();
 537         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 538             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 539              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 540                 return mode;
 541
 542         switch (cmd) {
 543         case SPEC_STORE_BYPASS_CMD_AUTO:
 544         case SPEC_STORE_BYPASS_CMD_SECCOMP:
 545                 /*
 546                  * Choose prctl+seccomp as the default mode if seccomp is
 547                  * enabled.
 548                  */
 549                 if (IS_ENABLED(CONFIG_SECCOMP))
 550                         mode = SPEC_STORE_BYPASS_SECCOMP;
 551                 else
 552                         mode = SPEC_STORE_BYPASS_PRCTL;
 553                 break;
 554         case SPEC_STORE_BYPASS_CMD_ON:
 555                 mode = SPEC_STORE_BYPASS_DISABLE;
 556                 break;
 557         case SPEC_STORE_BYPASS_CMD_PRCTL:
 558                 mode = SPEC_STORE_BYPASS_PRCTL;
 559                 break;
 560         case SPEC_STORE_BYPASS_CMD_NONE:
 561                 break;
 562         }
 563
 564         /*
 565          * We have three CPU feature flags that are in play here:
 566          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
 567          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
 568          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
 569          */
 570         if (mode == SPEC_STORE_BYPASS_DISABLE) {
 571                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 572                 /*
 573                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
 574                  * use a completely different MSR and bit dependent on family.
 575                  */
 576                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
 577                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
 578                         x86_amd_ssb_disable();
 579                 } else {
 580                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
 581                         x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
 582                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 583                 }
 584         }
 585
 586         return mode;
 587 }
 588
 589 static void ssb_select_mitigation(void)
 590 {
 591         ssb_mode = __ssb_select_mitigation();
 592
 593         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 594                 pr_info("%s\n", ssb_strings[ssb_mode]);
 595 }
 596
 597 #undef pr_fmt
 598 #define pr_fmt(fmt)     "Speculation prctl: " fmt
 599
 600 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
 601 {
 602         bool update;
 603
 604         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
 605             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
 606                 return -ENXIO;
 607
 608         switch (ctrl) {
 609         case PR_SPEC_ENABLE:
 610                 /* If speculation is force disabled, enable is not allowed */
 611                 if (task_spec_ssb_force_disable(task))
 612                         return -EPERM;
 613                 task_clear_spec_ssb_disable(task);
 614                 update = test_and_clear_tsk_thread_flag(task, TIF_SSBD);
 615                 break;
 616         case PR_SPEC_DISABLE:
 617                 task_set_spec_ssb_disable(task);
 618                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 619                 break;
 620         case PR_SPEC_FORCE_DISABLE:
 621                 task_set_spec_ssb_disable(task);
 622                 task_set_spec_ssb_force_disable(task);
 623                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 624                 break;
 625         default:
 626                 return -ERANGE;
 627         }
 628
 629         /*
 630          * If being set on non-current task, delay setting the CPU
 631          * mitigation until it is next scheduled.
 632          */
 633         if (task == current && update)
 634                 speculation_ctrl_update_current();
 635
 636         return 0;
 637 }
 638
 639 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
 640                              unsigned long ctrl)
 641 {
 642         switch (which) {
 643         case PR_SPEC_STORE_BYPASS:
 644                 return ssb_prctl_set(task, ctrl);
 645         default:
 646                 return -ENODEV;
 647         }
 648 }
 649
 650 #ifdef CONFIG_SECCOMP
 651 void arch_seccomp_spec_mitigate(struct task_struct *task)
 652 {
 653         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
 654                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 655 }
 656 #endif
 657
 658 static int ssb_prctl_get(struct task_struct *task)
 659 {
 660         switch (ssb_mode) {
 661         case SPEC_STORE_BYPASS_DISABLE:
 662                 return PR_SPEC_DISABLE;
 663         case SPEC_STORE_BYPASS_SECCOMP:
 664         case SPEC_STORE_BYPASS_PRCTL:
 665                 if (task_spec_ssb_force_disable(task))
 666                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 667                 if (task_spec_ssb_disable(task))
 668                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 669                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 670         default:
 671                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 672                         return PR_SPEC_ENABLE;
 673                 return PR_SPEC_NOT_AFFECTED;
 674         }
 675 }
 676
 677 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
 678 {
 679         switch (which) {
 680         case PR_SPEC_STORE_BYPASS:
 681                 return ssb_prctl_get(task);
 682         default:
 683                 return -ENODEV;
 684         }
 685 }
 686
 687 void x86_spec_ctrl_setup_ap(void)
 688 {
 689         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
 690                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 691
 692         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
 693                 x86_amd_ssb_disable();
 694 }
 695
 696 #undef pr_fmt
 697 #define pr_fmt(fmt)     "L1TF: " fmt
 698
 699 /* Default mitigation for L1TF-affected CPUs */
 700 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
 701 #if IS_ENABLED(CONFIG_KVM_INTEL)
 702 EXPORT_SYMBOL_GPL(l1tf_mitigation);
 703 #endif
 704 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
 705 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
 706
 707 /*
 708  * These CPUs all support 44bits physical address space internally in the
 709  * cache but CPUID can report a smaller number of physical address bits.
 710  *
 711  * The L1TF mitigation uses the top most address bit for the inversion of
 712  * non present PTEs. When the installed memory reaches into the top most
 713  * address bit due to memory holes, which has been observed on machines
 714  * which report 36bits physical address bits and have 32G RAM installed,
 715  * then the mitigation range check in l1tf_select_mitigation() triggers.
 716  * This is a false positive because the mitigation is still possible due to
 717  * the fact that the cache uses 44bit internally. Use the cache bits
 718  * instead of the reported physical bits and adjust them on the affected
 719  * machines to 44bit if the reported bits are less than 44.
 720  */
 721 static void override_cache_bits(struct cpuinfo_x86 *c)
 722 {
 723         if (c->x86 != 6)
 724                 return;
 725
 726         switch (c->x86_model) {
 727         case INTEL_FAM6_NEHALEM:
 728         case INTEL_FAM6_WESTMERE:
 729         case INTEL_FAM6_SANDYBRIDGE:
 730         case INTEL_FAM6_IVYBRIDGE:
 731         case INTEL_FAM6_HASWELL_CORE:
 732         case INTEL_FAM6_HASWELL_ULT:
 733         case INTEL_FAM6_HASWELL_GT3E:
 734         case INTEL_FAM6_BROADWELL_CORE:
 735         case INTEL_FAM6_BROADWELL_GT3E:
 736         case INTEL_FAM6_SKYLAKE_MOBILE:
 737         case INTEL_FAM6_SKYLAKE_DESKTOP:
 738         case INTEL_FAM6_KABYLAKE_MOBILE:
 739         case INTEL_FAM6_KABYLAKE_DESKTOP:
 740                 if (c->x86_cache_bits < 44)
 741                         c->x86_cache_bits = 44;
 742                 break;
 743         }
 744 }
 745
 746 static void __init l1tf_select_mitigation(void)
 747 {
 748         u64 half_pa;
 749
 750         if (!boot_cpu_has_bug(X86_BUG_L1TF))
 751                 return;
 752
 753         override_cache_bits(&boot_cpu_data);
 754
 755         switch (l1tf_mitigation) {
 756         case L1TF_MITIGATION_OFF:
 757         case L1TF_MITIGATION_FLUSH_NOWARN:
 758         case L1TF_MITIGATION_FLUSH:
 759                 break;
 760         case L1TF_MITIGATION_FLUSH_NOSMT:
 761         case L1TF_MITIGATION_FULL:
 762                 cpu_smt_disable(false);
 763                 break;
 764         case L1TF_MITIGATION_FULL_FORCE:
 765                 cpu_smt_disable(true);
 766                 break;
 767         }
 768
 769 #if CONFIG_PGTABLE_LEVELS == 2
 770         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
 771         return;
 772 #endif
 773
 774         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
 775         if (e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
 776                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
 777                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
 778                                 half_pa);
 779                 pr_info("However, doing so will make a part of your RAM unusable.\n");
 780                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html might help you decide.\n");
 781                 return;
 782         }
 783
 784         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
 785 }
 786
 787 static int __init l1tf_cmdline(char *str)
 788 {
 789         if (!boot_cpu_has_bug(X86_BUG_L1TF))
 790                 return 0;
 791
 792         if (!str)
 793                 return -EINVAL;
 794
 795         if (!strcmp(str, "off"))
 796                 l1tf_mitigation = L1TF_MITIGATION_OFF;
 797         else if (!strcmp(str, "flush,nowarn"))
 798                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
 799         else if (!strcmp(str, "flush"))
 800                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
 801         else if (!strcmp(str, "flush,nosmt"))
 802                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
 803         else if (!strcmp(str, "full"))
 804                 l1tf_mitigation = L1TF_MITIGATION_FULL;
 805         else if (!strcmp(str, "full,force"))
 806                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
 807
 808         return 0;
 809 }
 810 early_param("l1tf", l1tf_cmdline);
 811
 812 #undef pr_fmt
 813
 814 #ifdef CONFIG_SYSFS
 815
 816 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
 817
 818 #if IS_ENABLED(CONFIG_KVM_INTEL)
 819 static const char *l1tf_vmx_states[] = {
 820         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
 821         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
 822         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
 823         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
 824         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
 825         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
 826 };
 827
 828 static ssize_t l1tf_show_state(char *buf)
 829 {
 830         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
 831                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
 832
 833         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
 834             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
 835              sched_smt_active())) {
 836                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
 837                                l1tf_vmx_states[l1tf_vmx_mitigation]);
 838         }
 839
 840         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
 841                        l1tf_vmx_states[l1tf_vmx_mitigation],
 842                        sched_smt_active() ? "vulnerable" : "disabled");
 843 }
 844 #else
 845 static ssize_t l1tf_show_state(char *buf)
 846 {
 847         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
 848 }
 849 #endif
 850
 851 static char *stibp_state(void)
 852 {
 853         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 854                 return "";
 855
 856         if (x86_spec_ctrl_base & SPEC_CTRL_STIBP)
 857                 return ", STIBP";
 858         else
 859                 return "";
 860 }
 861
 862 static char *ibpb_state(void)
 863 {
 864         if (boot_cpu_has(X86_FEATURE_USE_IBPB))
 865                 return ", IBPB";
 866         else
 867                 return "";
 868 }
 869
 870 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 871                                char *buf, unsigned int bug)
 872 {
 873         if (!boot_cpu_has_bug(bug))
 874                 return sprintf(buf, "Not affected\n");
 875
 876         switch (bug) {
 877         case X86_BUG_CPU_MELTDOWN:
 878                 if (boot_cpu_has(X86_FEATURE_PTI))
 879                         return sprintf(buf, "Mitigation: PTI\n");
 880
 881                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
 882                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
 883
 884                 break;
 885
 886         case X86_BUG_SPECTRE_V1:
 887                 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
 888
 889         case X86_BUG_SPECTRE_V2:
 890                 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
 891                                ibpb_state(),
 892                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
 893                                stibp_state(),
 894                                boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
 895                                spectre_v2_module_string());
 896
 897         case X86_BUG_SPEC_STORE_BYPASS:
 898                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
 899
 900         case X86_BUG_L1TF:
 901                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
 902                         return l1tf_show_state(buf);
 903                 break;
 904         default:
 905                 break;
 906         }
 907
 908         return sprintf(buf, "Vulnerable\n");
 909 }
 910
 911 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
 912 {
 913         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
 914 }
 915
 916 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
 917 {
 918         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
 919 }
 920
 921 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
 922 {
 923         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
 924 }
 925
 926 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
 927 {
 928         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
 929 }
 930
 931 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
 932 {
 933         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
 934 }
 935 #endif