select ARCH_32BIT_OFF_T
select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE if HAVE_KRETPROBES && FRAME_POINTER && !ARM_UNWIND
select ARCH_HAS_BINFMT_FLAT
+ select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL if MMU
select ARCH_HAS_DMA_WRITE_COMBINE if !ARM_DMA_MEM_BUFFERABLE
select ARCH_HAS_ELF_RANDOMIZE
select ARCH_USE_CMPXCHG_LOCKREF
select ARCH_USE_MEMTEST
select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU
+ select ARCH_WANT_GENERAL_HUGETLB
select ARCH_WANT_IPC_PARSE_VERSION
select ARCH_WANT_LD_ORPHAN_WARN
select BINFMT_FLAT_ARGVP_ENVP_ON_STACK
select GENERIC_CPU_AUTOPROBE
select GENERIC_EARLY_IOREMAP
select GENERIC_IDLE_POLL_SETUP
+ select GENERIC_IRQ_MULTI_HANDLER
select GENERIC_IRQ_PROBE
select GENERIC_IRQ_SHOW
select GENERIC_IRQ_SHOW_LEVEL
select HAVE_ARCH_AUDITSYSCALL if AEABI && !OABI_COMPAT
select HAVE_ARCH_BITREVERSE if (CPU_32v7M || CPU_32v7) && !CPU_32v6
select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU
+ select HAVE_ARCH_KFENCE if MMU && !XIP_KERNEL
select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL
select HAVE_ARCH_MMAP_RND_BITS if MMU
select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32
select HAVE_CONTEXT_TRACKING
select HAVE_C_RECORDMCOUNT
+ select HAVE_BUILDTIME_MCOUNT_SORT
select HAVE_DEBUG_KMEMLEAK if !XIP_KERNEL
select HAVE_DMA_CONTIGUOUS if MMU
select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU
select HAVE_EXIT_THREAD
select HAVE_FAST_GUP if ARM_LPAE
select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
- select HAVE_FUNCTION_GRAPH_TRACER if !THUMB2_KERNEL && !CC_IS_CLANG
- select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG)
+ select HAVE_FUNCTION_GRAPH_TRACER
+ select HAVE_FUNCTION_TRACER if !XIP_KERNEL
- select HAVE_FUTEX_CMPXCHG if FUTEX
select HAVE_GCC_PLUGINS
select HAVE_HW_BREAKPOINT if PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7)
select HAVE_IRQ_TIME_ACCOUNTING
select PERF_USE_VMALLOC
select RTC_LIB
select SYS_SUPPORTS_APM_EMULATION
- select THREAD_INFO_IN_TASK if CURRENT_POINTER_IN_TPIDRURO
+ select THREAD_INFO_IN_TASK
+ select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS
select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M
# Above selects are sorted alphabetically; please add new ones
# according to that. Thanks.
Europe. There is an ARM Linux project with a web page at
<http://www.arm.linux.org.uk/>.
+ config ARM_HAS_GROUP_RELOCS
+ def_bool y
+ depends on !LD_IS_LLD || LLD_VERSION >= 140000
+ depends on !COMPILE_TEST
+ help
+ Whether or not to use R_ARM_ALU_PC_Gn or R_ARM_LDR_PC_Gn group
+ relocations, which have been around for a long time, but were not
+ supported in LLD until version 14. The combined range is -/+ 256 MiB,
+ which is usually sufficient, but not for allyesconfig, so we disable
+ this feature when doing compile testing.
+
config ARM_HAS_SG_CHAIN
bool
config FIQ
bool
- config NEED_RET_TO_USER
- bool
-
config ARCH_MTD_XIP
bool
select AUTO_ZRELADDR
select TIMER_OF
select COMMON_CLK
- select GENERIC_IRQ_MULTI_HANDLER
select HAVE_PCI
select PCI_DOMAINS_GENERIC if PCI
select SPARSE_IRQ
select ARM_AMBA
imply ARM_PATCH_PHYS_VIRT
select ARM_VIC
- select GENERIC_IRQ_MULTI_HANDLER
select AUTO_ZRELADDR
select CLKSRC_MMIO
select CPU_ARM920T
select CPU_XSCALE
select GPIO_IOP
select GPIOLIB
- select NEED_RET_TO_USER
select FORCE_PCI
select PLAT_IOP
help
select ARCH_SUPPORTS_BIG_ENDIAN
select CPU_XSCALE
select DMABOUNCE if PCI
- select GENERIC_IRQ_MULTI_HANDLER
select GPIO_IXP4XX
select GPIOLIB
select HAVE_PCI
config ARCH_DOVE
bool "Marvell Dove"
select CPU_PJ4
- select GENERIC_IRQ_MULTI_HANDLER
select GPIOLIB
select HAVE_PCI
select MVEBU_MBUS
select CLKSRC_MMIO
select TIMER_OF
select CPU_XSCALE if !CPU_XSC3
- select GENERIC_IRQ_MULTI_HANDLER
select GPIO_PXA
select GPIOLIB
select IRQ_DOMAIN
select COMMON_CLK
select CPU_FREQ
select CPU_SA1100
- select GENERIC_IRQ_MULTI_HANDLER
select GPIOLIB
select IRQ_DOMAIN
select ISA
select CLKSRC_SAMSUNG_PWM
select GPIO_SAMSUNG
select GPIOLIB
- select GENERIC_IRQ_MULTI_HANDLER
- select HAVE_S3C2410_I2C if I2C
select NEED_MACH_IO_H
select S3C2410_WATCHDOG
select SAMSUNG_ATAGS
select ARCH_OMAP
select CLKSRC_MMIO
select GENERIC_IRQ_CHIP
- select GENERIC_IRQ_MULTI_HANDLER
select GPIOLIB
select HAVE_LEGACY_CLK
select IRQ_DOMAIN
config CURRENT_POINTER_IN_TPIDRURO
def_bool y
- depends on SMP && CPU_32v6K && !CPU_V6
+ depends on CPU_32v6K && !CPU_V6
+
+ config IRQSTACKS
+ def_bool y
+ select HAVE_IRQ_EXIT_ON_IRQ_STACK
+ select HAVE_SOFTIRQ_ON_OWN_STACK
config ARM_CPU_TOPOLOGY
bool "Support cpu topology definition"
bool "High Memory Support"
depends on MMU
select KMAP_LOCAL
+ select KMAP_LOCAL_NON_LINEAR_PTE_ARRAY
help
The address space of ARM processors is only 4 Gigabytes large
and it has to accommodate user address space, kernel address
def_bool y
depends on ARM_PMU
-config ARCH_WANT_GENERAL_HUGETLB
- def_bool y
-
config ARM_MODULE_PLTS
bool "Use PLTs to allow module memory to spill over into vmalloc area"
depends on MODULES
help
Say Y if you want to run Linux in a Virtual Machine on Xen on ARM.
+ config CC_HAVE_STACKPROTECTOR_TLS
+ def_bool $(cc-option,-mtp=cp15 -mstack-protector-guard=tls -mstack-protector-guard-offset=0)
+
config STACKPROTECTOR_PER_TASK
bool "Use a unique stack canary value for each task"
- depends on GCC_PLUGINS && STACKPROTECTOR && THREAD_INFO_IN_TASK && !XIP_DEFLATED_DATA
- select GCC_PLUGIN_ARM_SSP_PER_TASK
+ depends on STACKPROTECTOR && CURRENT_POINTER_IN_TPIDRURO && !XIP_DEFLATED_DATA
+ depends on GCC_PLUGINS || CC_HAVE_STACKPROTECTOR_TLS
+ select GCC_PLUGIN_ARM_SSP_PER_TASK if !CC_HAVE_STACKPROTECTOR_TLS
default y
help
Due to the fact that GCC uses an ordinary symbol reference from
config UNWINDER_ARM
bool "ARM EABI stack unwinder"
- depends on AEABI && !FUNCTION_GRAPH_TRACER
+ depends on AEABI
- # https://github.com/ClangBuiltLinux/linux/issues/732
- depends on !LD_IS_LLD || LLD_VERSION >= 110000
select ARM_UNWIND
help
This option enables stack unwinding support in the kernel
config ARM_UNWIND
bool
+config BACKTRACE_VERBOSE
+ bool "Verbose backtrace"
+ depends on EXPERT
+ help
+ When the kernel produces a warning or oops, the kernel prints a
+ trace of the call chain. This option controls whether we include
+ the numeric addresses or only include the symbolic information.
+
+ In most cases, say N here, unless you are intending to debug the
+ kernel and have access to the kernel binary image.
+
config FRAME_POINTER
bool
Say Y here if you want kernel low-level debugging support
on i.MX25.
- config DEBUG_IMX21_IMX27_UART
- bool "i.MX21 and i.MX27 Debug UART"
- depends on SOC_IMX21 || SOC_IMX27
+ config DEBUG_IMX27_UART
+ bool "i.MX27 Debug UART"
+ depends on SOC_IMX27
help
Say Y here if you want kernel low-level debugging support
- on i.MX21 or i.MX27.
+ on i.MX27.
config DEBUG_IMX28_UART
bool "i.MX28 Debug UART"
int "i.MX Debug UART Port Selection"
depends on DEBUG_IMX1_UART || \
DEBUG_IMX25_UART || \
- DEBUG_IMX21_IMX27_UART || \
+ DEBUG_IMX27_UART || \
DEBUG_IMX31_UART || \
DEBUG_IMX35_UART || \
DEBUG_IMX50_UART || \
default "debug/icedcc.S" if DEBUG_ICEDCC
default "debug/imx.S" if DEBUG_IMX1_UART || \
DEBUG_IMX25_UART || \
- DEBUG_IMX21_IMX27_UART || \
+ DEBUG_IMX27_UART || \
DEBUG_IMX31_UART || \
DEBUG_IMX35_UART || \
DEBUG_IMX50_UART || \
DEBUG_IMX51_UART || \
- DEBUG_IMX53_UART ||\
+ DEBUG_IMX53_UART || \
DEBUG_IMX6Q_UART || \
DEBUG_IMX6SL_UART || \
DEBUG_IMX6SX_UART || \
OBJS += debug.o
AFLAGS_head.o += -DDEBUG
endif
-FONTC = $(srctree)/lib/fonts/font_acorn_8x8.c
# string library code (-Os is enforced to keep it much smaller)
OBJS += string.o
CPPFLAGS_vmlinux.lds += -DMALLOC_SIZE="$(MALLOC_SIZE)"
compress-$(CONFIG_KERNEL_GZIP) = gzip
-compress-$(CONFIG_KERNEL_LZO) = lzo
-compress-$(CONFIG_KERNEL_LZMA) = lzma
-compress-$(CONFIG_KERNEL_XZ) = xzkern
-compress-$(CONFIG_KERNEL_LZ4) = lz4
+compress-$(CONFIG_KERNEL_LZO) = lzo_with_size
+compress-$(CONFIG_KERNEL_LZMA) = lzma_with_size
+compress-$(CONFIG_KERNEL_XZ) = xzkern_with_size
+compress-$(CONFIG_KERNEL_LZ4) = lz4_with_size
libfdt_objs := fdt_rw.o fdt_ro.o fdt_wip.o fdt.o
OBJS += $(libfdt_objs) fdt_check_mem_start.o
endif
- # -fstack-protector-strong triggers protection checks in this code,
- # but it is being used too early to link to meaningful stack_chk logic.
- $(foreach o, $(libfdt_objs) atags_to_fdt.o fdt_check_mem_start.o, \
- $(eval CFLAGS_$(o) := -I $(srctree)/scripts/dtc/libfdt -fno-stack-protector))
-
targets := vmlinux vmlinux.lds piggy_data piggy.o \
- lib1funcs.o ashldi3.o bswapsdi2.o \
head.o $(OBJS)
-clean-files += lib1funcs.S ashldi3.S bswapsdi2.S hyp-stub.S
-
KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
ccflags-y := -fpic $(call cc-option,-mno-single-pic-base,) -fno-builtin \
+ -I$(srctree)/scripts/dtc/libfdt -fno-stack-protector \
-I$(obj) $(DISABLE_ARM_SSP_PER_TASK_PLUGIN)
ccflags-remove-$(CONFIG_FUNCTION_TRACER) += -pg
asflags-y := -DZIMAGE
# Next argument is a linker script
LDFLAGS_vmlinux += -T
-# For __aeabi_uidivmod
-lib1funcs = $(obj)/lib1funcs.o
-
-$(obj)/lib1funcs.S: $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.S
- $(call cmd,shipped)
-
-# For __aeabi_llsl
-ashldi3 = $(obj)/ashldi3.o
-
-$(obj)/ashldi3.S: $(srctree)/arch/$(SRCARCH)/lib/ashldi3.S
- $(call cmd,shipped)
-
-# For __bswapsi2, __bswapdi2
-bswapsdi2 = $(obj)/bswapsdi2.o
-
-$(obj)/bswapsdi2.S: $(srctree)/arch/$(SRCARCH)/lib/bswapsdi2.S
- $(call cmd,shipped)
+OBJS += lib1funcs.o ashldi3.o bswapsdi2.o
# We need to prevent any GOTOFF relocs being used with references
# to symbols in the .bss section since we cannot relocate them
efi-obj-$(CONFIG_EFI_STUB) := $(objtree)/drivers/firmware/efi/libstub/lib.a
$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.o \
- $(addprefix $(obj)/, $(OBJS)) $(lib1funcs) $(ashldi3) \
- $(bswapsdi2) $(efi-obj-y) FORCE
+ $(addprefix $(obj)/, $(OBJS)) \
+ $(efi-obj-y) FORCE
@$(check_for_multiple_zreladdr)
$(call if_changed,ld)
@$(check_for_bad_syms)
$(obj)/piggy.o: $(obj)/piggy_data
CFLAGS_font.o := -Dstatic=
-
-$(obj)/font.c: $(FONTC)
- $(call cmd,shipped)
-
AFLAGS_hyp-stub.o := -Wa,-march=armv7-a
-
-$(obj)/hyp-stub.S: $(srctree)/arch/$(SRCARCH)/kernel/hyp-stub.S
- $(call cmd,shipped)
#define IMM12_MASK 0xfff
+ /* the frame pointer used for stack unwinding */
+ ARM( fpreg .req r11 )
+ THUMB( fpreg .req r7 )
+
/*
* Enable and disable interrupts
*/
.endm
#endif
+#if __LINUX_ARM_ARCH__ < 7
+ .macro dsb, args
+ mcr p15, 0, r0, c7, c10, 4
+ .endm
+
+ .macro isb, args
+ mcr p15, 0, r0, c7, c5, 4
+ .endm
+#endif
+
.macro asm_trace_hardirqs_off, save=1
#if defined(CONFIG_TRACE_IRQFLAGS)
.if \save
.endm
.endr
- .macro get_current, rd
- #ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
- mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register
- #else
- get_thread_info \rd
- ldr \rd, [\rd, #TI_TASK]
- #endif
- .endm
-
- .macro set_current, rn
- #ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
- mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register
- #endif
- .endm
-
- .macro reload_current, t1:req, t2:req
- #ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
- adr_l \t1, __entry_task @ get __entry_task base address
- mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset
- ldr \t1, [\t1, \t2] @ load variable
- mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO
- #endif
- .endm
-
/*
* Get current thread_info.
*/
.macro get_thread_info, rd
- #ifdef CONFIG_THREAD_INFO_IN_TASK
/* thread_info is the first member of struct task_struct */
get_current \rd
- #else
- ARM( mov \rd, sp, lsr #THREAD_SIZE_ORDER + PAGE_SHIFT )
- THUMB( mov \rd, sp )
- THUMB( lsr \rd, \rd, #THREAD_SIZE_ORDER + PAGE_SHIFT )
- mov \rd, \rd, lsl #THREAD_SIZE_ORDER + PAGE_SHIFT
- #endif
.endm
/*
*/
#define ALT_UP(instr...) \
.pushsection ".alt.smp.init", "a" ;\
+ .align 2 ;\
.long 9998b - . ;\
9997: instr ;\
.if . - 9997b == 2 ;\
.popsection
#define ALT_UP_B(label) \
.pushsection ".alt.smp.init", "a" ;\
+ .align 2 ;\
.long 9998b - . ;\
W(b) . + (label - 9998b) ;\
.popsection
#define ALT_UP_B(label) b label
#endif
+ /*
+ * this_cpu_offset - load the per-CPU offset of this CPU into
+ * register 'rd'
+ */
+ .macro this_cpu_offset, rd:req
+ #ifdef CONFIG_SMP
+ ALT_SMP(mrc p15, 0, \rd, c13, c0, 4)
+ #ifdef CONFIG_CPU_V6
+ ALT_UP_B(.L1_\@)
+ .L0_\@:
+ .subsection 1
+ .L1_\@: ldr_va \rd, __per_cpu_offset
+ b .L0_\@
+ .previous
+ #endif
+ #else
+ mov \rd, #0
+ #endif
+ .endm
+
+ /*
+ * set_current - store the task pointer of this CPU's current task
+ */
+ .macro set_current, rn:req, tmp:req
+ #if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP)
+ 9998: mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register
+ #ifdef CONFIG_CPU_V6
+ ALT_UP_B(.L0_\@)
+ .subsection 1
+ .L0_\@: str_va \rn, __current, \tmp
+ b .L1_\@
+ .previous
+ .L1_\@:
+ #endif
+ #else
+ str_va \rn, __current, \tmp
+ #endif
+ .endm
+
+ /*
+ * get_current - load the task pointer of this CPU's current task
+ */
+ .macro get_current, rd:req
+ #if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP)
+ 9998: mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register
+ #ifdef CONFIG_CPU_V6
+ ALT_UP_B(.L0_\@)
+ .subsection 1
+ .L0_\@: ldr_va \rd, __current
+ b .L1_\@
+ .previous
+ .L1_\@:
+ #endif
+ #else
+ ldr_va \rd, __current
+ #endif
+ .endm
+
+ /*
+ * reload_current - reload the task pointer of this CPU's current task
+ * into the TLS register
+ */
+ .macro reload_current, t1:req, t2:req
+ #if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP)
+ #ifdef CONFIG_CPU_V6
+ ALT_SMP(nop)
+ ALT_UP_B(.L0_\@)
+ #endif
+ ldr_this_cpu \t1, __entry_task, \t1, \t2
+ mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO
+ .L0_\@:
+ #endif
+ .endm
+
/*
* Instruction barrier
*/
/*
* mov_l - move a constant value or [relocated] address into a register
*/
- .macro mov_l, dst:req, imm:req
+ .macro mov_l, dst:req, imm:req, cond
.if __LINUX_ARM_ARCH__ < 7
- ldr \dst, =\imm
+ ldr\cond \dst, =\imm
.else
- movw \dst, #:lower16:\imm
- movt \dst, #:upper16:\imm
+ movw\cond \dst, #:lower16:\imm
+ movt\cond \dst, #:upper16:\imm
.endif
.endm
__adldst_l str, \src, \sym, \tmp, \cond
.endm
+ .macro __ldst_va, op, reg, tmp, sym, cond
+ #if __LINUX_ARM_ARCH__ >= 7 || \
+ !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \
+ (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
+ mov_l \tmp, \sym, \cond
+ \op\cond \reg, [\tmp]
+ #else
+ /*
+ * Avoid a literal load, by emitting a sequence of ADD/LDR instructions
+ * with the appropriate relocations. The combined sequence has a range
+ * of -/+ 256 MiB, which should be sufficient for the core kernel and
+ * for modules loaded into the module region.
+ */
+ .globl \sym
+ .reloc .L0_\@, R_ARM_ALU_PC_G0_NC, \sym
+ .reloc .L1_\@, R_ARM_ALU_PC_G1_NC, \sym
+ .reloc .L2_\@, R_ARM_LDR_PC_G2, \sym
+ .L0_\@: sub\cond \tmp, pc, #8
+ .L1_\@: sub\cond \tmp, \tmp, #4
+ .L2_\@: \op\cond \reg, [\tmp, #0]
+ #endif
+ .endm
+
+ /*
+ * ldr_va - load a 32-bit word from the virtual address of \sym
+ */
+ .macro ldr_va, rd:req, sym:req, cond
+ __ldst_va ldr, \rd, \rd, \sym, \cond
+ .endm
+
+ /*
+ * str_va - store a 32-bit word to the virtual address of \sym
+ */
+ .macro str_va, rn:req, sym:req, tmp:req, cond
+ __ldst_va str, \rn, \tmp, \sym, \cond
+ .endm
+
+ /*
+ * ldr_this_cpu_armv6 - Load a 32-bit word from the per-CPU variable 'sym',
+ * without using a temp register. Supported in ARM mode
+ * only.
+ */
+ .macro ldr_this_cpu_armv6, rd:req, sym:req
+ this_cpu_offset \rd
+ .globl \sym
+ .reloc .L0_\@, R_ARM_ALU_PC_G0_NC, \sym
+ .reloc .L1_\@, R_ARM_ALU_PC_G1_NC, \sym
+ .reloc .L2_\@, R_ARM_LDR_PC_G2, \sym
+ add \rd, \rd, pc
+ .L0_\@: sub \rd, \rd, #4
+ .L1_\@: sub \rd, \rd, #0
+ .L2_\@: ldr \rd, [\rd, #4]
+ .endm
+
+ /*
+ * ldr_this_cpu - Load a 32-bit word from the per-CPU variable 'sym'
+ * into register 'rd', which may be the stack pointer,
+ * using 't1' and 't2' as general temp registers. These
+ * are permitted to overlap with 'rd' if != sp
+ */
+ .macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req
+ #if __LINUX_ARM_ARCH__ >= 7 || \
+ !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \
+ (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
+ this_cpu_offset \t1
+ mov_l \t2, \sym
+ ldr \rd, [\t1, \t2]
+ #else
+ ldr_this_cpu_armv6 \rd, \sym
+ #endif
+ .endm
+
/*
* rev_l - byte-swap a 32-bit value
*
.endif
.endm
+ /*
+ * bl_r - branch and link to register
+ *
+ * @dst: target to branch to
+ * @c: conditional opcode suffix
+ */
+ .macro bl_r, dst:req, c
+ .if __LINUX_ARM_ARCH__ < 6
+ mov\c lr, pc
+ mov\c pc, \dst
+ .else
+ blx\c \dst
+ .endif
+ .endm
+
#endif /* __ASM_ASSEMBLER_H__ */
*/
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
extern void flush_dcache_page(struct page *);
-void flush_dcache_folio(struct folio *folio);
#define ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 1
static inline void flush_kernel_vmap_range(void *addr, int size)
* however some exceptions may exist. Caveat emptor.
*
* - The clobber list is dictated by the call to v7_flush_dcache_*.
- * fp is preserved to the stack explicitly prior disabling the cache
- * since adding it to the clobber list is incompatible with having
- * CONFIG_FRAME_POINTER=y. ip is saved as well if ever r12-clobbering
- * trampoline are inserted by the linker and to keep sp 64-bit aligned.
*/
#define v7_exit_coherency_flush(level) \
asm volatile( \
".arch armv7-a \n\t" \
- "stmfd sp!, {fp, ip} \n\t" \
"mrc p15, 0, r0, c1, c0, 0 @ get SCTLR \n\t" \
"bic r0, r0, #"__stringify(CR_C)" \n\t" \
"mcr p15, 0, r0, c1, c0, 0 @ set SCTLR \n\t" \
"bic r0, r0, #(1 << 6) @ disable local coherency \n\t" \
"mcr p15, 0, r0, c1, c0, 1 @ set ACTLR \n\t" \
"isb \n\t" \
- "dsb \n\t" \
- "ldmfd sp!, {fp, ip}" \
- : : : "r0","r1","r2","r3","r4","r5","r6","r7", \
- "r9","r10","lr","memory" )
+ "dsb" \
+ : : : "r0","r1","r2","r3","r4","r5","r6", \
+ "r9","r10","ip","lr","memory" )
void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
void *kaddr, unsigned long len);
#define __ASM_ARM_SWITCH_TO_H
#include <linux/thread_info.h>
+ #include <asm/smp_plat.h>
/*
* For v7 SMP cores running a preemptible kernel we may be pre-empted
*/
extern struct task_struct *__switch_to(struct task_struct *, struct thread_info *, struct thread_info *);
-static inline void set_ti_cpu(struct task_struct *p)
-{
-#ifdef CONFIG_THREAD_INFO_IN_TASK
- /*
- * The core code no longer maintains the thread_info::cpu field once
- * CONFIG_THREAD_INFO_IN_TASK is in effect, but we rely on it for
- * raw_smp_processor_id(), which cannot access struct task_struct*
- * directly for reasons of circular #inclusion hell.
- */
- task_thread_info(p)->cpu = task_cpu(p);
-#endif
-}
-
#define switch_to(prev,next,last) \
do { \
__complete_pending_tlbi(); \
- if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) \
- set_ti_cpu(next); \
+ if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || is_smp()) \
__this_cpu_write(__entry_task, next); \
last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \
} while (0)
CFLAGS_REMOVE_ftrace.o = -pg
CFLAGS_REMOVE_insn.o = -pg
CFLAGS_REMOVE_patch.o = -pg
+ CFLAGS_REMOVE_unwind.o = -pg
endif
CFLAGS_REMOVE_return_address.o = -pg
obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o
+obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o
+
extra-y := $(head-y) vmlinux.lds
#include <asm/glue-df.h>
#include <asm/glue-pf.h>
#include <asm/vfpmacros.h>
- #ifndef CONFIG_GENERIC_IRQ_MULTI_HANDLER
- #include <mach/entry-macro.S>
- #endif
#include <asm/thread_notify.h>
#include <asm/unwind.h>
#include <asm/unistd.h>
#include <asm/uaccess-asm.h>
#include "entry-header.S"
- #include <asm/entry-macro-multi.S>
#include <asm/probes.h>
/*
* Interrupt handling.
*/
- .macro irq_handler
- #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
- mov r0, sp
- bl generic_handle_arch_irq
- #else
- arch_irq_handler_default
+ .macro irq_handler, from_user:req
+ mov r1, sp
+ ldr_this_cpu r2, irq_stack_ptr, r2, r3
+ .if \from_user == 0
+ @
+ @ If we took the interrupt while running in the kernel, we may already
+ @ be using the IRQ stack, so revert to the original value in that case.
+ @
+ subs r3, r2, r1 @ SP above bottom of IRQ stack?
+ rsbscs r3, r3, #THREAD_SIZE @ ... and below the top?
+ #ifdef CONFIG_VMAP_STACK
+ ldr_va r3, high_memory, cc @ End of the linear region
+ cmpcc r3, r1 @ Stack pointer was below it?
#endif
+ bcc 0f @ If not, switch to the IRQ stack
+ mov r0, r1
+ bl generic_handle_arch_irq
+ b 1f
+ 0:
+ .endif
+
+ mov_l r0, generic_handle_arch_irq
+ bl call_with_stack
+ 1:
.endm
.macro pabt_helper
#define SPFIX(code...)
#endif
- .macro svc_entry, stack_hole=0, trace=1, uaccess=1
+ .macro svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1
UNWIND(.fnstart )
- UNWIND(.save {r0 - pc} )
- sub sp, sp, #(SVC_REGS_SIZE + \stack_hole - 4)
+ sub sp, sp, #(SVC_REGS_SIZE + \stack_hole)
+ THUMB( add sp, r1 ) @ get SP in a GPR without
+ THUMB( sub r1, sp, r1 ) @ using a temp register
+
+ .if \overflow_check
+ UNWIND(.save {r0 - pc} )
+ do_overflow_check (SVC_REGS_SIZE + \stack_hole)
+ .endif
+
#ifdef CONFIG_THUMB2_KERNEL
- SPFIX( str r0, [sp] ) @ temporarily saved
- SPFIX( mov r0, sp )
- SPFIX( tst r0, #4 ) @ test original stack alignment
- SPFIX( ldr r0, [sp] ) @ restored
+ tst r1, #4 @ test stack pointer alignment
+ sub r1, sp, r1 @ restore original R1
+ sub sp, r1 @ restore original SP
#else
SPFIX( tst sp, #4 )
#endif
- SPFIX( subeq sp, sp, #4 )
- stmia sp, {r1 - r12}
+ SPFIX( subne sp, sp, #4 )
+
+ ARM( stmib sp, {r1 - r12} )
+ THUMB( stmia sp, {r0 - r12} ) @ No STMIB in Thumb-2
ldmia r0, {r3 - r5}
- add r7, sp, #S_SP - 4 @ here for interlock avoidance
+ add r7, sp, #S_SP @ here for interlock avoidance
mov r6, #-1 @ "" "" "" ""
- add r2, sp, #(SVC_REGS_SIZE + \stack_hole - 4)
- SPFIX( addeq r2, r2, #4 )
- str r3, [sp, #-4]! @ save the "real" r0 copied
+ add r2, sp, #(SVC_REGS_SIZE + \stack_hole)
+ SPFIX( addne r2, r2, #4 )
+ str r3, [sp] @ save the "real" r0 copied
@ from the exception stack
mov r3, lr
.align 5
__irq_svc:
svc_entry
- irq_handler
+ irq_handler from_user=0
#ifdef CONFIG_PREEMPTION
ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
__irq_usr:
usr_entry
kuser_cmpxchg_check
- irq_handler
+ irq_handler from_user=1
get_thread_info tsk
mov why, #0
b ret_to_user_from_irq
tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2
reteq lr
and r8, r0, #0x00000f00 @ mask out CP number
- THUMB( lsr r8, r8, #8 )
mov r7, #1
- add r6, r10, #TI_USED_CP
- ARM( strb r7, [r6, r8, lsr #8] ) @ set appropriate used_cp[]
- THUMB( strb r7, [r6, r8] ) @ set appropriate used_cp[]
+ add r6, r10, r8, lsr #8 @ add used_cp[] array offset first
+ strb r7, [r6, #TI_USED_CP] @ set appropriate used_cp[]
#ifdef CONFIG_IWMMXT
@ Test if we need to give access to iWMMXt coprocessors
ldr r5, [r10, #TI_FLAGS]
bcs iwmmxt_task_enable
#endif
ARM( add pc, pc, r8, lsr #6 )
- THUMB( lsl r8, r8, #2 )
+ THUMB( lsr r8, r8, #6 )
THUMB( add pc, r8 )
nop
ldr r6, [r2, #TI_CPU_DOMAIN]
#endif
switch_tls r1, r4, r5, r3, r7
- #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP)
- ldr r7, [r2, #TI_TASK]
+ #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \
+ !defined(CONFIG_STACKPROTECTOR_PER_TASK)
ldr r8, =__stack_chk_guard
.if (TSK_STACK_CANARY > IMM12_MASK)
- add r7, r7, #TSK_STACK_CANARY & ~IMM12_MASK
+ add r9, r2, #TSK_STACK_CANARY & ~IMM12_MASK
+ ldr r9, [r9, #TSK_STACK_CANARY & IMM12_MASK]
+ .else
+ ldr r9, [r2, #TSK_STACK_CANARY & IMM12_MASK]
.endif
- ldr r7, [r7, #TSK_STACK_CANARY & IMM12_MASK]
- #elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO)
- mov r7, r2 @ Preserve 'next'
#endif
+ mov r7, r2 @ Preserve 'next'
#ifdef CONFIG_CPU_USE_DOMAINS
mcr p15, 0, r6, c3, c0, 0 @ Set domain register
#endif
ldr r0, =thread_notify_head
mov r1, #THREAD_NOTIFY_SWITCH
bl atomic_notifier_call_chain
- #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP)
- str r7, [r8]
+ #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \
+ !defined(CONFIG_STACKPROTECTOR_PER_TASK)
+ str r9, [r8]
#endif
- THUMB( mov ip, r4 )
mov r0, r5
- set_current r7
- ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously
- THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously
- THUMB( ldr sp, [ip], #4 )
- THUMB( ldr pc, [ip] )
+ #if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK)
+ set_current r7, r8
+ ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously
+ #else
+ mov r1, r7
+ ldmia r4, {r4 - sl, fp, ip, lr} @ Load all regs saved previously
+ #ifdef CONFIG_VMAP_STACK
+ @
+ @ Do a dummy read from the new stack while running from the old one so
+ @ that we can rely on do_translation_fault() to fix up any stale PMD
+ @ entries covering the vmalloc region.
+ @
+ ldr r2, [ip]
+ #endif
+
+ @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what
+ @ effectuates the task switch, as that is what causes the observable
+ @ values of current and current_thread_info to change. When
+ @ CONFIG_THREAD_INFO_IN_TASK=y, setting current (and therefore
+ @ current_thread_info) is done explicitly, and the update of SP just
+ @ switches us to another stack, with few other side effects. In order
+ @ to prevent this distinction from causing any inconsistencies, let's
+ @ keep the 'set_current' call as close as we can to the update of SP.
+ set_current r1, r2
+ mov sp, ip
+ ret lr
+ #endif
UNWIND(.fnend )
ENDPROC(__switch_to)
+ #ifdef CONFIG_VMAP_STACK
+ .text
+ .align 2
+ __bad_stack:
+ @
+ @ We've just detected an overflow. We need to load the address of this
+ @ CPU's overflow stack into the stack pointer register. We have only one
+ @ scratch register so let's use a sequence of ADDs including one
+ @ involving the PC, and decorate them with PC-relative group
+ @ relocations. As these are ARM only, switch to ARM mode first.
+ @
+ @ We enter here with IP clobbered and its value stashed on the mode
+ @ stack.
+ @
+ THUMB( bx pc )
+ THUMB( nop )
+ THUMB( .arm )
+ ldr_this_cpu_armv6 ip, overflow_stack_ptr
+
+ str sp, [ip, #-4]! @ Preserve original SP value
+ mov sp, ip @ Switch to overflow stack
+ pop {ip} @ Original SP in IP
+
+ #if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
+ mov ip, ip @ mov expected by unwinder
+ push {fp, ip, lr, pc} @ GCC flavor frame record
+ #else
+ str ip, [sp, #-8]! @ store original SP
+ push {fpreg, lr} @ Clang flavor frame record
+ #endif
+ UNWIND( ldr ip, [r0, #4] ) @ load exception LR
+ UNWIND( str ip, [sp, #12] ) @ store in the frame record
+ ldr ip, [r0, #12] @ reload IP
+
+ @ Store the original GPRs to the new stack.
+ svc_entry uaccess=0, overflow_check=0
+
+ UNWIND( .save {sp, pc} )
+ UNWIND( .save {fpreg, lr} )
+ UNWIND( .setfp fpreg, sp )
+
+ ldr fpreg, [sp, #S_SP] @ Add our frame record
+ @ to the linked list
+ #if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
+ ldr r1, [fp, #4] @ reload SP at entry
+ add fp, fp, #12
+ #else
+ ldr r1, [fpreg, #8]
+ #endif
+ str r1, [sp, #S_SP] @ store in pt_regs
+
+ @ Stash the regs for handle_bad_stack
+ mov r0, sp
+
+ @ Time to die
+ bl handle_bad_stack
+ nop
+ UNWIND( .fnend )
+ ENDPROC(__bad_stack)
+ #endif
+
__INIT
/*
sub lr, lr, #\correction
.endif
- @
- @ Save r0, lr_<exception> (parent PC) and spsr_<exception>
- @ (parent CPSR)
- @
+ @ Save r0, lr_<exception> (parent PC)
stmia sp, {r0, lr} @ save r0, lr
- mrs lr, spsr
+
+ @ Save spsr_<exception> (parent CPSR)
+2: mrs lr, spsr
str lr, [sp, #8] @ save spsr
@
movs pc, lr @ branch to handler in SVC mode
ENDPROC(vector_\name)
+#ifdef CONFIG_HARDEN_BRANCH_HISTORY
+ .subsection 1
+ .align 5
+vector_bhb_loop8_\name:
+ .if \correction
+ sub lr, lr, #\correction
+ .endif
+
+ @ Save r0, lr_<exception> (parent PC)
+ stmia sp, {r0, lr}
+
+ @ bhb workaround
+ mov r0, #8
+3: b . + 4
+ subs r0, r0, #1
+ bne 3b
+ dsb
+ isb
+ b 2b
+ENDPROC(vector_bhb_loop8_\name)
+
+vector_bhb_bpiall_\name:
+ .if \correction
+ sub lr, lr, #\correction
+ .endif
+
+ @ Save r0, lr_<exception> (parent PC)
+ stmia sp, {r0, lr}
+
+ @ bhb workaround
+ mcr p15, 0, r0, c7, c5, 6 @ BPIALL
+ @ isb not needed due to "movs pc, lr" in the vector stub
+ @ which gives a "context synchronisation".
+ b 2b
+ENDPROC(vector_bhb_bpiall_\name)
+ .previous
+#endif
+
.align 2
@ handler addresses follow this label
1:
.section .stubs, "ax", %progbits
@ This must be the first word
.word vector_swi
+#ifdef CONFIG_HARDEN_BRANCH_HISTORY
+ .word vector_bhb_loop8_swi
+ .word vector_bhb_bpiall_swi
+#endif
vector_rst:
ARM( swi SYS_ERROR0 )
* FIQ "NMI" handler
*-----------------------------------------------------------------------------
* Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86
- * systems.
+ * systems. This must be the last vector stub, so lets place it in its own
+ * subsection.
*/
+ .subsection 2
vector_stub fiq, FIQ_MODE, 4
.long __fiq_usr @ 0 (USR_26 / USR_32)
W(b) vector_irq
W(b) vector_fiq
+#ifdef CONFIG_HARDEN_BRANCH_HISTORY
+ .section .vectors.bhb.loop8, "ax", %progbits
+.L__vectors_bhb_loop8_start:
+ W(b) vector_rst
+ W(b) vector_bhb_loop8_und
+ W(ldr) pc, .L__vectors_bhb_loop8_start + 0x1004
+ W(b) vector_bhb_loop8_pabt
+ W(b) vector_bhb_loop8_dabt
+ W(b) vector_addrexcptn
+ W(b) vector_bhb_loop8_irq
+ W(b) vector_bhb_loop8_fiq
+
+ .section .vectors.bhb.bpiall, "ax", %progbits
+.L__vectors_bhb_bpiall_start:
+ W(b) vector_rst
+ W(b) vector_bhb_bpiall_und
+ W(ldr) pc, .L__vectors_bhb_bpiall_start + 0x1008
+ W(b) vector_bhb_bpiall_pabt
+ W(b) vector_bhb_bpiall_dabt
+ W(b) vector_addrexcptn
+ W(b) vector_bhb_bpiall_irq
+ W(b) vector_bhb_bpiall_fiq
+#endif
+
.data
.align 2
.equ NR_syscalls, __NR_syscalls
- #ifdef CONFIG_NEED_RET_TO_USER
- #include <mach/entry-macro.S>
- #else
- .macro arch_ret_to_user, tmp1, tmp2
- .endm
+ .macro arch_ret_to_user, tmp
+ #ifdef CONFIG_ARCH_IOP32X
+ mrc p15, 0, \tmp, c15, c1, 0
+ tst \tmp, #(1 << 6)
+ bicne \tmp, \tmp, #(1 << 6)
+ mcrne p15, 0, \tmp, c15, c1, 0 @ Disable cp6 access
#endif
+ .endm
#include "entry-header.S"
/* perform architecture specific actions before user return */
- arch_ret_to_user r1, lr
+ arch_ret_to_user r1
restore_user_regs fast = 1, offset = S_OFF
UNWIND(.fnend )
asm_trace_hardirqs_on save = 0
/* perform architecture specific actions before user return */
- arch_ret_to_user r1, lr
+ arch_ret_to_user r1
ct_user_enter save = 0
restore_user_regs fast = 0, offset = 0
*/
.align 5
+#ifdef CONFIG_HARDEN_BRANCH_HISTORY
+ENTRY(vector_bhb_loop8_swi)
+ sub sp, sp, #PT_REGS_SIZE
+ stmia sp, {r0 - r12}
+ mov r8, #8
+1: b 2f
+2: subs r8, r8, #1
+ bne 1b
+ dsb
+ isb
+ b 3f
+ENDPROC(vector_bhb_loop8_swi)
+
+ .align 5
+ENTRY(vector_bhb_bpiall_swi)
+ sub sp, sp, #PT_REGS_SIZE
+ stmia sp, {r0 - r12}
+ mcr p15, 0, r8, c7, c5, 6 @ BPIALL
+ isb
+ b 3f
+ENDPROC(vector_bhb_bpiall_swi)
+#endif
+ .align 5
ENTRY(vector_swi)
#ifdef CONFIG_CPU_V7M
v7m_exception_entry
#else
sub sp, sp, #PT_REGS_SIZE
stmia sp, {r0 - r12} @ Calling r0 - r12
+3:
ARM( add r8, sp, #S_PC )
ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr
THUMB( mov r8, sp )
secondary_data.swapper_pg_dir = get_arch_pgd(swapper_pg_dir);
#endif
secondary_data.task = idle;
- if (IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK))
- task_thread_info(idle)->cpu = cpu;
-
sync_cache_w(&secondary_data);
/*
check_cpu_icache_size(cpuid);
}
+ static void set_current(struct task_struct *cur)
+ {
+ /* Set TPIDRURO */
+ asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory");
+ }
+
/*
* This is the secondary CPU boot entry. We're using this CPUs
* idle thread stack, but a set of temporary page tables.
/*
* Main handler for inter-processor interrupts
*/
- asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs)
- {
- handle_IPI(ipinr, regs);
- }
-
static void do_handle_IPI(int ipinr)
{
unsigned int cpu = smp_processor_id();
#include <linux/atomic.h>
#include <asm/cacheflush.h>
#include <asm/exception.h>
+#include <asm/spectre.h>
#include <asm/unistd.h>
#include <asm/traps.h>
#include <asm/ptrace.h>
#include <asm/unwind.h>
#include <asm/tls.h>
+ #include <asm/stacktrace.h>
#include <asm/system_misc.h>
#include <asm/opcodes.h>
__setup("user_debug=", user_debug_setup);
#endif
- static void dump_mem(const char *, const char *, unsigned long, unsigned long);
-
void dump_backtrace_entry(unsigned long where, unsigned long from,
unsigned long frame, const char *loglvl)
{
unsigned long end = frame + 4 + sizeof(struct pt_regs);
-#ifdef CONFIG_KALLSYMS
+ if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER) &&
+ IS_ENABLED(CONFIG_CC_IS_GCC) &&
+ end > ALIGN(frame, THREAD_SIZE)) {
+ /*
+ * If we are walking past the end of the stack, it may be due
+ * to the fact that we are on an IRQ or overflow stack. In this
+ * case, we can load the address of the other stack from the
+ * frame record.
+ */
+ frame = ((unsigned long *)frame)[-2] - 4;
+ end = frame + 4 + sizeof(struct pt_regs);
+ }
+
+#ifndef CONFIG_KALLSYMS
+ printk("%sFunction entered at [<%08lx>] from [<%08lx>]\n",
+ loglvl, where, from);
+#elif defined CONFIG_BACKTRACE_VERBOSE
printk("%s[<%08lx>] (%ps) from [<%08lx>] (%pS)\n",
loglvl, where, (void *)where, from, (void *)from);
#else
- printk("%sFunction entered at [<%08lx>] from [<%08lx>]\n",
- loglvl, where, from);
+ printk("%s %ps from %pS\n", loglvl, (void *)where, (void *)from);
#endif
if (in_entry_text(from) && end <= ALIGN(frame, THREAD_SIZE))
static int verify_stack(unsigned long sp)
{
if (sp < PAGE_OFFSET ||
- (sp > (unsigned long)high_memory && high_memory != NULL))
+ (!IS_ENABLED(CONFIG_VMAP_STACK) &&
+ sp > (unsigned long)high_memory && high_memory != NULL))
return -EFAULT;
return 0;
/*
* Dump out the contents of some memory nicely...
*/
- static void dump_mem(const char *lvl, const char *str, unsigned long bottom,
- unsigned long top)
+ void dump_mem(const char *lvl, const char *str, unsigned long bottom,
+ unsigned long top)
{
unsigned long first;
int i;
if (!user_mode(regs) || in_interrupt()) {
dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp,
- THREAD_SIZE + (unsigned long)task_stack_page(tsk));
+ ALIGN(regs->ARM_sp - THREAD_SIZE, THREAD_ALIGN)
+ + THREAD_SIZE);
dump_backtrace(regs, tsk, KERN_EMERG);
dump_instr(KERN_EMERG, regs);
}
if (panic_on_oops)
panic("Fatal exception");
if (signr)
- do_exit(signr);
+ make_task_dead(signr);
}
/*
}
#endif
+#ifndef CONFIG_CPU_V7M
+static void copy_from_lma(void *vma, void *lma_start, void *lma_end)
+{
+ memcpy(vma, lma_start, lma_end - lma_start);
+}
+
+static void flush_vectors(void *vma, size_t offset, size_t size)
+{
+ unsigned long start = (unsigned long)vma + offset;
+ unsigned long end = start + size;
+
+ flush_icache_range(start, end);
+}
+
+#ifdef CONFIG_HARDEN_BRANCH_HISTORY
+int spectre_bhb_update_vectors(unsigned int method)
+{
+ extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[];
+ extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[];
+ void *vec_start, *vec_end;
+
+ if (system_state >= SYSTEM_FREEING_INITMEM) {
+ pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n",
+ smp_processor_id());
+ return SPECTRE_VULNERABLE;
+ }
+
+ switch (method) {
+ case SPECTRE_V2_METHOD_LOOP8:
+ vec_start = __vectors_bhb_loop8_start;
+ vec_end = __vectors_bhb_loop8_end;
+ break;
+
+ case SPECTRE_V2_METHOD_BPIALL:
+ vec_start = __vectors_bhb_bpiall_start;
+ vec_end = __vectors_bhb_bpiall_end;
+ break;
+
+ default:
+ pr_err("CPU%u: unknown Spectre BHB state %d\n",
+ smp_processor_id(), method);
+ return SPECTRE_VULNERABLE;
+ }
+
+ copy_from_lma(vectors_page, vec_start, vec_end);
+ flush_vectors(vectors_page, 0, vec_end - vec_start);
+
+ return SPECTRE_MITIGATED;
+}
+#endif
+
void __init early_trap_init(void *vectors_base)
{
-#ifndef CONFIG_CPU_V7M
- unsigned long vectors = (unsigned long)vectors_base;
extern char __stubs_start[], __stubs_end[];
extern char __vectors_start[], __vectors_end[];
unsigned i;
* into the vector page, mapped at 0xffff0000, and ensure these
* are visible to the instruction stream.
*/
- memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start);
- memcpy((void *)vectors + 0x1000, __stubs_start, __stubs_end - __stubs_start);
+ copy_from_lma(vectors_base, __vectors_start, __vectors_end);
+ copy_from_lma(vectors_base + 0x1000, __stubs_start, __stubs_end);
kuser_init(vectors_base);
- flush_icache_range(vectors, vectors + PAGE_SIZE * 2);
+ flush_vectors(vectors_base, 0, PAGE_SIZE * 2);
+}
#else /* ifndef CONFIG_CPU_V7M */
+void __init early_trap_init(void *vectors_base)
+{
/*
* on V7-M there is no need to copy the vector table to a dedicated
* memory area. The address is configurable and so a table in the kernel
* image can be used.
*/
-#endif
}
+#endif
+
+ #ifdef CONFIG_VMAP_STACK
+
+ DECLARE_PER_CPU(u8 *, irq_stack_ptr);
+
+ asmlinkage DEFINE_PER_CPU(u8 *, overflow_stack_ptr);
+
+ static int __init allocate_overflow_stacks(void)
+ {
+ u8 *stack;
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ stack = (u8 *)__get_free_page(GFP_KERNEL);
+ if (WARN_ON(!stack))
+ return -ENOMEM;
+ per_cpu(overflow_stack_ptr, cpu) = &stack[OVERFLOW_STACK_SIZE];
+ }
+ return 0;
+ }
+ early_initcall(allocate_overflow_stacks);
+
+ asmlinkage void handle_bad_stack(struct pt_regs *regs)
+ {
+ unsigned long tsk_stk = (unsigned long)current->stack;
+ #ifdef CONFIG_IRQSTACKS
+ unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
+ #endif
+ unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr);
+
+ console_verbose();
+ pr_emerg("Insufficient stack space to handle exception!");
+
+ pr_emerg("Task stack: [0x%08lx..0x%08lx]\n",
+ tsk_stk, tsk_stk + THREAD_SIZE);
+ #ifdef CONFIG_IRQSTACKS
+ pr_emerg("IRQ stack: [0x%08lx..0x%08lx]\n",
+ irq_stk - THREAD_SIZE, irq_stk);
+ #endif
+ pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n",
+ ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk);
+
+ die("kernel stack overflow", regs, 0);
+ }
+
+ #ifndef CONFIG_ARM_LPAE
+ /*
+ * Normally, we rely on the logic in do_translation_fault() to update stale PMD
+ * entries covering the vmalloc space in a task's page tables when it first
+ * accesses the region in question. Unfortunately, this is not sufficient when
+ * the task stack resides in the vmalloc region, as do_translation_fault() is a
+ * C function that needs a stack to run.
+ *
+ * So we need to ensure that these PMD entries are up to date *before* the MM
+ * switch. As we already have some logic in the MM switch path that takes care
+ * of this, let's trigger it by bumping the counter every time the core vmalloc
+ * code modifies a PMD entry in the vmalloc region. Use release semantics on
+ * the store so that other CPUs observing the counter's new value are
+ * guaranteed to see the updated page table entries as well.
+ */
+ void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
+ {
+ if (start < VMALLOC_END && end > VMALLOC_START)
+ atomic_inc_return_release(&init_mm.context.vmalloc_seq);
+ }
+ #endif
+ #endif
select CPU_PABRT_V6
select CPU_THUMB_CAPABLE
select CPU_TLB_V6 if MMU
+ select SMP_ON_UP if SMP
# ARMv6k
config CPU_V6K
config CPU_SPECTRE
bool
+ select GENERIC_CPU_VULNERABILITIES
config HARDEN_BRANCH_PREDICTOR
bool "Harden the branch predictor against aliasing attacks" if EXPERT
If unsure, say Y.
+config HARDEN_BRANCH_HISTORY
+ bool "Harden Spectre style attacks against branch history" if EXPERT
+ depends on CPU_SPECTRE
+ default y
+ help
+ Speculation attacks against some high-performance processors can
+ make use of branch history to influence future speculation. When
+ taking an exception, a sequence of branches overwrites the branch
+ history, or branch history is invalidated.
+
config TLS_REG_EMUL
bool
select NEED_KUSER_HELPERS
void __check_vmalloc_seq(struct mm_struct *mm)
{
- unsigned int seq;
+ int seq;
do {
- seq = init_mm.context.vmalloc_seq;
+ seq = atomic_read(&init_mm.context.vmalloc_seq);
memcpy(pgd_offset(mm, VMALLOC_START),
pgd_offset_k(VMALLOC_START),
sizeof(pgd_t) * (pgd_index(VMALLOC_END) -
pgd_index(VMALLOC_START)));
- mm->context.vmalloc_seq = seq;
- } while (seq != init_mm.context.vmalloc_seq);
+ /*
+ * Use a store-release so that other CPUs that observe the
+ * counter's new value are guaranteed to see the results of the
+ * memcpy as well.
+ */
+ atomic_set_release(&mm->context.vmalloc_seq, seq);
+ } while (seq != atomic_read(&init_mm.context.vmalloc_seq));
}
#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
* Note: this is still racy on SMP machines.
*/
pmd_clear(pmdp);
- init_mm.context.vmalloc_seq++;
+ atomic_inc_return_release(&init_mm.context.vmalloc_seq);
/*
* Free the page table, if there was one.
* Ensure that the active_mm is up to date - we want to
* catch any use-after-iounmap cases.
*/
- if (current->active_mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)
- __check_vmalloc_seq(current->active_mm);
+ check_vmalloc_seq(current->active_mm);
flush_tlb_kernel_range(virt, end);
}
pci_ioremap_mem_type = mem_type;
}
-int pci_ioremap_io(unsigned int offset, phys_addr_t phys_addr)
+int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
{
- BUG_ON(offset + SZ_64K - 1 > IO_SPACE_LIMIT);
+ unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
- return ioremap_page_range(PCI_IO_VIRT_BASE + offset,
- PCI_IO_VIRT_BASE + offset + SZ_64K,
- phys_addr,
+ if (!(res->flags & IORESOURCE_IO))
+ return -EINVAL;
+
+ if (res->end > IO_SPACE_LIMIT)
+ return -EINVAL;
+
+ return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
__pgprot(get_mem_type(pci_ioremap_mem_type)->prot_pte));
}
-EXPORT_SYMBOL_GPL(pci_ioremap_io);
+EXPORT_SYMBOL(pci_remap_iospace);
void __iomem *pci_remap_cfgspace(resource_size_t res_cookie, size_t size)
{
#define NVIC_ISER 0x000
#define NVIC_ICER 0x080
-#define NVIC_IPR 0x300
+#define NVIC_IPR 0x400
#define NVIC_MAX_BANKS 16
/*
static struct irq_domain *nvic_irq_domain;
- static void __nvic_handle_irq(irq_hw_number_t hwirq)
+ static void __irq_entry nvic_handle_irq(struct pt_regs *regs)
{
- generic_handle_domain_irq(nvic_irq_domain, hwirq);
- }
+ unsigned long icsr = readl_relaxed(BASEADDR_V7M_SCB + V7M_SCB_ICSR);
+ irq_hw_number_t hwirq = (icsr & V7M_SCB_ICSR_VECTACTIVE) - 16;
- /*
- * TODO: restructure the ARMv7M entry logic so that this entry logic can live
- * in arch code.
- */
- asmlinkage void __exception_irq_entry
- nvic_handle_irq(irq_hw_number_t hwirq, struct pt_regs *regs)
- {
- struct pt_regs *old_regs;
-
- irq_enter();
- old_regs = set_irq_regs(regs);
- __nvic_handle_irq(hwirq);
- set_irq_regs(old_regs);
- irq_exit();
+ generic_handle_domain_irq(nvic_irq_domain, hwirq);
}
static int nvic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
if (!nvic_irq_domain) {
pr_warn("Failed to allocate irq domain\n");
+ iounmap(nvic_base);
return -ENOMEM;
}
if (ret) {
pr_warn("Failed to allocate irq chips\n");
irq_domain_remove(nvic_irq_domain);
+ iounmap(nvic_base);
return ret;
}
for (i = 0; i < irqs; i += 4)
writel_relaxed(0, nvic_base + NVIC_IPR + i);
+ set_handle_irq(nvic_handle_irq);
return 0;
}
IRQCHIP_DECLARE(armv7m_nvic, "arm,armv7m-nvic", nvic_of_init);
OSDN Git Service
