arch/x86/include/asm/nospec-branch.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2
   3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
   4 #define _ASM_X86_NOSPEC_BRANCH_H_
   5
   6 #include <linux/static_key.h>
   7
   8 #include <asm/alternative.h>
   9 #include <asm/alternative-asm.h>
  10 #include <asm/cpufeatures.h>
  11 #include <asm/msr-index.h>
  12
  13 /*
  14  * Fill the CPU return stack buffer.
  15  *
  16  * Each entry in the RSB, if used for a speculative 'ret', contains an
  17  * infinite 'pause; lfence; jmp' loop to capture speculative execution.
  18  *
  19  * This is required in various cases for retpoline and IBRS-based
  20  * mitigations for the Spectre variant 2 vulnerability. Sometimes to
  21  * eliminate potentially bogus entries from the RSB, and sometimes
  22  * purely to ensure that it doesn't get empty, which on some CPUs would
  23  * allow predictions from other (unwanted!) sources to be used.
  24  *
  25  * We define a CPP macro such that it can be used from both .S files and
  26  * inline assembly. It's possible to do a .macro and then include that
  27  * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
  28  */
  29
  30 #define RSB_CLEAR_LOOPS         32      /* To forcibly overwrite all entries */
  31 #define RSB_FILL_LOOPS          16      /* To avoid underflow */
  32
  33 /*
  34  * Google experimented with loop-unrolling and this turned out to be
  35  * the optimal version — two calls, each with their own speculation
  36  * trap should their return address end up getting used, in a loop.
  37  */
  38 #define __FILL_RETURN_BUFFER(reg, nr, sp)       \
  39         mov     $(nr/2), reg;                   \
  40 771:                                            \
  41         call    772f;                           \
  42 773:    /* speculation trap */                  \
  43         pause;                                  \
  44         lfence;                                 \
  45         jmp     773b;                           \
  46 772:                                            \
  47         call    774f;                           \
  48 775:    /* speculation trap */                  \
  49         pause;                                  \
  50         lfence;                                 \
  51         jmp     775b;                           \
  52 774:                                            \
  53         dec     reg;                            \
  54         jnz     771b;                           \
  55         add     $(BITS_PER_LONG/8) * nr, sp;
  56
  57 #ifdef __ASSEMBLY__
  58
  59 /*
  60  * These are the bare retpoline primitives for indirect jmp and call.
  61  * Do not use these directly; they only exist to make the ALTERNATIVE
  62  * invocation below less ugly.
  63  */
  64 .macro RETPOLINE_JMP reg:req
  65         call    .Ldo_rop_\@
  66 .Lspec_trap_\@:
  67         pause
  68         lfence
  69         jmp     .Lspec_trap_\@
  70 .Ldo_rop_\@:
  71         mov     \reg, (%_ASM_SP)
  72         ret
  73 .endm
  74
  75 /*
  76  * This is a wrapper around RETPOLINE_JMP so the called function in reg
  77  * returns to the instruction after the macro.
  78  */
  79 .macro RETPOLINE_CALL reg:req
  80         jmp     .Ldo_call_\@
  81 .Ldo_retpoline_jmp_\@:
  82         RETPOLINE_JMP \reg
  83 .Ldo_call_\@:
  84         call    .Ldo_retpoline_jmp_\@
  85 .endm
  86
  87 /*
  88  * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
  89  * indirect jmp/call which may be susceptible to the Spectre variant 2
  90  * attack.
  91  */
  92 .macro JMP_NOSPEC reg:req
  93 #ifdef CONFIG_RETPOLINE
  94         ALTERNATIVE_2 __stringify(jmp *\reg),                           \
  95                 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
  96                 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
  97 #else
  98         jmp     *\reg
  99 #endif
 100 .endm
 101
 102 .macro CALL_NOSPEC reg:req
 103 #ifdef CONFIG_RETPOLINE
 104         ALTERNATIVE_2 __stringify(call *\reg),                          \
 105                 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
 106                 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
 107 #else
 108         call    *\reg
 109 #endif
 110 .endm
 111
 112  /*
 113   * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
 114   * monstrosity above, manually.
 115   */
 116 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
 117 #ifdef CONFIG_RETPOLINE
 118         ALTERNATIVE "jmp .Lskip_rsb_\@",                                \
 119                 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))    \
 120                 \ftr
 121 .Lskip_rsb_\@:
 122 #endif
 123 .endm
 124
 125 #else /* __ASSEMBLY__ */
 126
 127 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
 128
 129 /*
 130  * Since the inline asm uses the %V modifier which is only in newer GCC,
 131  * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
 132  */
 133 # define CALL_NOSPEC                                            \
 134         ALTERNATIVE(                                            \
 135         "call *%[thunk_target]\n",                              \
 136         "call __x86_indirect_thunk_%V[thunk_target]\n",         \
 137         X86_FEATURE_RETPOLINE)
 138 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
 139
 140 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
 141 /*
 142  * For i386 we use the original ret-equivalent retpoline, because
 143  * otherwise we'll run out of registers. We don't care about CET
 144  * here, anyway.
 145  */
 146 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",     \
 147         "       jmp    904f;\n"                                 \
 148         "       .align 16\n"                                    \
 149         "901:   call   903f;\n"                                 \
 150         "902:   pause;\n"                                       \
 151         "       lfence;\n"                                      \
 152         "       jmp    902b;\n"                                 \
 153         "       .align 16\n"                                    \
 154         "903:   addl   $4, %%esp;\n"                            \
 155         "       pushl  %[thunk_target];\n"                      \
 156         "       ret;\n"                                         \
 157         "       .align 16\n"                                    \
 158         "904:   call   901b;\n",                                \
 159         X86_FEATURE_RETPOLINE)
 160
 161 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 162 #else /* No retpoline for C / inline asm */
 163 # define CALL_NOSPEC "call *%[thunk_target]\n"
 164 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 165 #endif
 166
 167 /* The Spectre V2 mitigation variants */
 168 enum spectre_v2_mitigation {
 169         SPECTRE_V2_NONE,
 170         SPECTRE_V2_RETPOLINE_MINIMAL,
 171         SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
 172         SPECTRE_V2_RETPOLINE_GENERIC,
 173         SPECTRE_V2_RETPOLINE_AMD,
 174         SPECTRE_V2_IBRS_ENHANCED,
 175 };
 176
 177 /* The indirect branch speculation control variants */
 178 enum spectre_v2_user_mitigation {
 179         SPECTRE_V2_USER_NONE,
 180         SPECTRE_V2_USER_STRICT,
 181         SPECTRE_V2_USER_PRCTL,
 182         SPECTRE_V2_USER_SECCOMP,
 183 };
 184
 185 /* The Speculative Store Bypass disable variants */
 186 enum ssb_mitigation {
 187         SPEC_STORE_BYPASS_NONE,
 188         SPEC_STORE_BYPASS_DISABLE,
 189         SPEC_STORE_BYPASS_PRCTL,
 190         SPEC_STORE_BYPASS_SECCOMP,
 191 };
 192
 193 extern char __indirect_thunk_start[];
 194 extern char __indirect_thunk_end[];
 195
 196 /*
 197  * On VMEXIT we must ensure that no RSB predictions learned in the guest
 198  * can be followed in the host, by overwriting the RSB completely. Both
 199  * retpoline and IBRS mitigations for Spectre v2 need this; only on future
 200  * CPUs with IBRS_ALL *might* it be avoided.
 201  */
 202 static inline void vmexit_fill_RSB(void)
 203 {
 204 #ifdef CONFIG_RETPOLINE
 205         unsigned long loops;
 206
 207         asm volatile (ALTERNATIVE("jmp 910f",
 208                                   __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
 209                                   X86_FEATURE_RETPOLINE)
 210                       "910:"
 211                       : "=r" (loops), ASM_CALL_CONSTRAINT
 212                       : : "memory" );
 213 #endif
 214 }
 215
 216 static __always_inline
 217 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
 218 {
 219         asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
 220                 : : "c" (msr),
 221                     "a" ((u32)val),
 222                     "d" ((u32)(val >> 32)),
 223                     [feature] "i" (feature)
 224                 : "memory");
 225 }
 226
 227 static inline void indirect_branch_prediction_barrier(void)
 228 {
 229         u64 val = PRED_CMD_IBPB;
 230
 231         alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB);
 232 }
 233
 234 /* The Intel SPEC CTRL MSR base value cache */
 235 extern u64 x86_spec_ctrl_base;
 236
 237 /*
 238  * With retpoline, we must use IBRS to restrict branch prediction
 239  * before calling into firmware.
 240  *
 241  * (Implemented as CPP macros due to header hell.)
 242  */
 243 #define firmware_restrict_branch_speculation_start()                    \
 244 do {                                                                    \
 245         u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS;                  \
 246                                                                         \
 247         preempt_disable();                                              \
 248         alternative_msr_write(MSR_IA32_SPEC_CTRL, val,                  \
 249                               X86_FEATURE_USE_IBRS_FW);                 \
 250 } while (0)
 251
 252 #define firmware_restrict_branch_speculation_end()                      \
 253 do {                                                                    \
 254         u64 val = x86_spec_ctrl_base;                                   \
 255                                                                         \
 256         alternative_msr_write(MSR_IA32_SPEC_CTRL, val,                  \
 257                               X86_FEATURE_USE_IBRS_FW);                 \
 258         preempt_enable();                                               \
 259 } while (0)
 260
 261 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
 262 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 263 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 264
 265 #include <asm/segment.h>
 266
 267 /**
 268  * mds_clear_cpu_buffers - Mitigation for MDS vulnerability
 269  *
 270  * This uses the otherwise unused and obsolete VERW instruction in
 271  * combination with microcode which triggers a CPU buffer flush when the
 272  * instruction is executed.
 273  */
 274 static inline void mds_clear_cpu_buffers(void)
 275 {
 276         static const u16 ds = __KERNEL_DS;
 277
 278         /*
 279          * Has to be the memory-operand variant because only that
 280          * guarantees the CPU buffer flush functionality according to
 281          * documentation. The register-operand variant does not.
 282          * Works with any segment selector, but a valid writable
 283          * data segment is the fastest variant.
 284          *
 285          * "cc" clobber is required because VERW modifies ZF.
 286          */
 287         asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
 288 }
 289
 290 #endif /* __ASSEMBLY__ */
 291
 292 /*
 293  * Below is used in the eBPF JIT compiler and emits the byte sequence
 294  * for the following assembly:
 295  *
 296  * With retpolines configured:
 297  *
 298  *    callq do_rop
 299  *  spec_trap:
 300  *    pause
 301  *    lfence
 302  *    jmp spec_trap
 303  *  do_rop:
 304  *    mov %rax,(%rsp)
 305  *    retq
 306  *
 307  * Without retpolines configured:
 308  *
 309  *    jmp *%rax
 310  */
 311 #ifdef CONFIG_RETPOLINE
 312 # define RETPOLINE_RAX_BPF_JIT_SIZE     17
 313 # define RETPOLINE_RAX_BPF_JIT()                                \
 314         EMIT1_off32(0xE8, 7);    /* callq do_rop */             \
 315         /* spec_trap: */                                        \
 316         EMIT2(0xF3, 0x90);       /* pause */                    \
 317         EMIT3(0x0F, 0xAE, 0xE8); /* lfence */                   \
 318         EMIT2(0xEB, 0xF9);       /* jmp spec_trap */            \
 319         /* do_rop: */                                           \
 320         EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */    \
 321         EMIT1(0xC3);             /* retq */
 322 #else
 323 # define RETPOLINE_RAX_BPF_JIT_SIZE     2
 324 # define RETPOLINE_RAX_BPF_JIT()                                \
 325         EMIT2(0xFF, 0xE0);       /* jmp *%rax */
 326 #endif
 327
 328 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */