From 650b55b707fdfa764e9f2b81314d3eb4216fb962 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Fri, 15 May 2020 12:12:55 +1000 Subject: [PATCH] powerpc: Add prefixed instructions to instruction data type For powerpc64, redefine the ppc_inst type so both word and prefixed instructions can be represented. On powerpc32 the type will remain the same. Update places which had assumed instructions to be 4 bytes long. Signed-off-by: Jordan Niethe Reviewed-by: Alistair Popple [mpe: Rework the get_user_inst() macros to be parameterised, and don't assign to the dest if an error occurred. Use CONFIG_PPC64 not __powerpc64__ in a few places. Address other comments from Christophe. Fix some sparse complaints.] Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20200506034050.24806-24-jniethe5@gmail.com --- arch/powerpc/include/asm/inst.h | 70 ++++++++++++++++++++++++++++++++--- arch/powerpc/include/asm/kprobes.h | 2 +- arch/powerpc/include/asm/ppc-opcode.h | 3 ++ arch/powerpc/include/asm/uaccess.h | 36 ++++++++++++++++++ arch/powerpc/include/asm/uprobes.h | 2 +- arch/powerpc/kernel/crash_dump.c | 2 +- arch/powerpc/kernel/optprobes.c | 6 ++- arch/powerpc/kernel/optprobes_head.S | 3 ++ arch/powerpc/lib/code-patching.c | 13 ++++++- arch/powerpc/lib/feature-fixups.c | 5 ++- arch/powerpc/lib/inst.c | 40 ++++++++++++++++++++ arch/powerpc/lib/sstep.c | 4 +- arch/powerpc/xmon/xmon.c | 4 +- arch/powerpc/xmon/xmon_bpts.S | 2 + 14 files changed, 175 insertions(+), 17 deletions(-) diff --git a/arch/powerpc/include/asm/inst.h b/arch/powerpc/include/asm/inst.h index c7ea70e73073..d82e0c99cfa1 100644 --- a/arch/powerpc/include/asm/inst.h +++ b/arch/powerpc/include/asm/inst.h @@ -2,29 +2,80 @@ #ifndef _ASM_POWERPC_INST_H #define _ASM_POWERPC_INST_H +#include + /* * Instruction data type for POWER */ struct ppc_inst { u32 val; +#ifdef CONFIG_PPC64 + u32 suffix; +#endif } __packed; -#define ppc_inst(x) ((struct ppc_inst){ .val = x }) - static inline u32 ppc_inst_val(struct ppc_inst x) { return x.val; } -static inline int ppc_inst_len(struct ppc_inst x) +static inline int ppc_inst_primary_opcode(struct ppc_inst x) { - return sizeof(struct ppc_inst); + return ppc_inst_val(x) >> 26; } -static inline int ppc_inst_primary_opcode(struct ppc_inst x) +#ifdef CONFIG_PPC64 +#define ppc_inst(x) ((struct ppc_inst){ .val = (x), .suffix = 0xff }) + +#define ppc_inst_prefix(x, y) ((struct ppc_inst){ .val = (x), .suffix = (y) }) + +static inline u32 ppc_inst_suffix(struct ppc_inst x) { - return ppc_inst_val(x) >> 26; + return x.suffix; +} + +static inline bool ppc_inst_prefixed(struct ppc_inst x) +{ + return (ppc_inst_primary_opcode(x) == 1) && ppc_inst_suffix(x) != 0xff; +} + +static inline struct ppc_inst ppc_inst_swab(struct ppc_inst x) +{ + return ppc_inst_prefix(swab32(ppc_inst_val(x)), + swab32(ppc_inst_suffix(x))); +} + +static inline struct ppc_inst ppc_inst_read(const struct ppc_inst *ptr) +{ + u32 val, suffix; + + val = *(u32 *)ptr; + if ((val >> 26) == OP_PREFIX) { + suffix = *((u32 *)ptr + 1); + return ppc_inst_prefix(val, suffix); + } else { + return ppc_inst(val); + } +} + +static inline bool ppc_inst_equal(struct ppc_inst x, struct ppc_inst y) +{ + return *(u64 *)&x == *(u64 *)&y; +} + +#else + +#define ppc_inst(x) ((struct ppc_inst){ .val = x }) + +static inline bool ppc_inst_prefixed(struct ppc_inst x) +{ + return false; +} + +static inline u32 ppc_inst_suffix(struct ppc_inst x) +{ + return 0; } static inline struct ppc_inst ppc_inst_swab(struct ppc_inst x) @@ -42,6 +93,13 @@ static inline bool ppc_inst_equal(struct ppc_inst x, struct ppc_inst y) return ppc_inst_val(x) == ppc_inst_val(y); } +#endif /* CONFIG_PPC64 */ + +static inline int ppc_inst_len(struct ppc_inst x) +{ + return ppc_inst_prefixed(x) ? 8 : 4; +} + int probe_user_read_inst(struct ppc_inst *inst, struct ppc_inst __user *nip); diff --git a/arch/powerpc/include/asm/kprobes.h b/arch/powerpc/include/asm/kprobes.h index 66b3f2983b22..4fc0e15e23a5 100644 --- a/arch/powerpc/include/asm/kprobes.h +++ b/arch/powerpc/include/asm/kprobes.h @@ -43,7 +43,7 @@ extern kprobe_opcode_t optprobe_template_ret[]; extern kprobe_opcode_t optprobe_template_end[]; /* Fixed instruction size for powerpc */ -#define MAX_INSN_SIZE 1 +#define MAX_INSN_SIZE 2 #define MAX_OPTIMIZED_LENGTH sizeof(kprobe_opcode_t) /* 4 bytes */ #define MAX_OPTINSN_SIZE (optprobe_template_end - optprobe_template_entry) #define RELATIVEJUMP_SIZE sizeof(kprobe_opcode_t) /* 4 bytes */ diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index c1df75edde44..2a39c716c343 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -158,6 +158,9 @@ /* VMX Vector Store Instructions */ #define OP_31_XOP_STVX 231 +/* Prefixed Instructions */ +#define OP_PREFIX 1 + #define OP_31 31 #define OP_LWZ 32 #define OP_STFS 52 diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 0d2d0c3dc527..c0523efa1458 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -105,6 +105,40 @@ static inline int __access_ok(unsigned long addr, unsigned long size, #define __put_user_inatomic(x, ptr) \ __put_user_nosleep((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) +#ifdef CONFIG_PPC64 + +#define ___get_user_instr(gu_op, dest, ptr) \ +({ \ + long __gui_ret = 0; \ + unsigned long __gui_ptr = (unsigned long)ptr; \ + struct ppc_inst __gui_inst; \ + unsigned int __prefix, __suffix; \ + __gui_ret = gu_op(__prefix, (unsigned int __user *)__gui_ptr); \ + if (__gui_ret == 0) { \ + if ((__prefix >> 26) == OP_PREFIX) { \ + __gui_ret = gu_op(__suffix, \ + (unsigned int __user *)__gui_ptr + 1); \ + __gui_inst = ppc_inst_prefix(__prefix, \ + __suffix); \ + } else { \ + __gui_inst = ppc_inst(__prefix); \ + } \ + if (__gui_ret == 0) \ + (dest) = __gui_inst; \ + } \ + __gui_ret; \ +}) + +#define get_user_instr(x, ptr) \ + ___get_user_instr(get_user, x, ptr) + +#define __get_user_instr(x, ptr) \ + ___get_user_instr(__get_user, x, ptr) + +#define __get_user_instr_inatomic(x, ptr) \ + ___get_user_instr(__get_user_inatomic, x, ptr) + +#else /* !CONFIG_PPC64 */ #define get_user_instr(x, ptr) \ get_user((x).val, (u32 __user *)(ptr)) @@ -114,6 +148,8 @@ static inline int __access_ok(unsigned long addr, unsigned long size, #define __get_user_instr_inatomic(x, ptr) \ __get_user_nosleep((x).val, (u32 __user *)(ptr), sizeof(u32)) +#endif /* CONFIG_PPC64 */ + extern long __put_user_bad(void); /* diff --git a/arch/powerpc/include/asm/uprobes.h b/arch/powerpc/include/asm/uprobes.h index 7e3b329ba2d3..5bf65f5d44a9 100644 --- a/arch/powerpc/include/asm/uprobes.h +++ b/arch/powerpc/include/asm/uprobes.h @@ -15,7 +15,7 @@ typedef ppc_opcode_t uprobe_opcode_t; -#define MAX_UINSN_BYTES 4 +#define MAX_UINSN_BYTES 8 #define UPROBE_XOL_SLOT_BYTES (MAX_UINSN_BYTES) /* The following alias is needed for reference from arch-agnostic code */ diff --git a/arch/powerpc/kernel/crash_dump.c b/arch/powerpc/kernel/crash_dump.c index 72bafb47e757..735e89337398 100644 --- a/arch/powerpc/kernel/crash_dump.c +++ b/arch/powerpc/kernel/crash_dump.c @@ -46,7 +46,7 @@ static void __init create_trampoline(unsigned long addr) * two instructions it doesn't require any registers. */ patch_instruction(p, ppc_inst(PPC_INST_NOP)); - patch_branch(++p, addr + PHYSICAL_START, 0); + patch_branch((void *)p + 4, addr + PHYSICAL_START, 0); } void __init setup_kdump_trampoline(void) diff --git a/arch/powerpc/kernel/optprobes.c b/arch/powerpc/kernel/optprobes.c index 8eea8dbb93fa..3ac105e7faae 100644 --- a/arch/powerpc/kernel/optprobes.c +++ b/arch/powerpc/kernel/optprobes.c @@ -198,7 +198,7 @@ void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr) int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) { - struct ppc_inst branch_op_callback, branch_emulate_step; + struct ppc_inst branch_op_callback, branch_emulate_step, temp; kprobe_opcode_t *op_callback_addr, *emulate_step_addr, *buff; long b_offset; unsigned long nip, size; @@ -282,7 +282,9 @@ int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) /* * 3. load instruction to be emulated into relevant register, and */ - patch_imm32_load_insns(*p->ainsn.insn, buff + TMPL_INSN_IDX); + temp = ppc_inst_read((struct ppc_inst *)p->ainsn.insn); + patch_imm64_load_insns(ppc_inst_val(temp) | ((u64)ppc_inst_suffix(temp) << 32), + 4, buff + TMPL_INSN_IDX); /* * 4. branch back from trampoline diff --git a/arch/powerpc/kernel/optprobes_head.S b/arch/powerpc/kernel/optprobes_head.S index cf383520843f..ff8ba4d3824d 100644 --- a/arch/powerpc/kernel/optprobes_head.S +++ b/arch/powerpc/kernel/optprobes_head.S @@ -94,6 +94,9 @@ optprobe_template_insn: /* 2, Pass instruction to be emulated in r4 */ nop nop + nop + nop + nop .global optprobe_template_call_emulate optprobe_template_call_emulate: diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index d946f7d6bb32..e9a0ea1c7ba4 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -24,7 +24,18 @@ static int __patch_instruction(struct ppc_inst *exec_addr, struct ppc_inst instr { int err = 0; - __put_user_asm(ppc_inst_val(instr), patch_addr, err, "stw"); + if (!ppc_inst_prefixed(instr)) { + __put_user_asm(ppc_inst_val(instr), patch_addr, err, "stw"); + } else { +#ifdef CONFIG_CPU_LITTLE_ENDIAN + __put_user_asm((u64)ppc_inst_suffix(instr) << 32 | + ppc_inst_val(instr), patch_addr, err, "std"); +#else + __put_user_asm((u64)ppc_inst_val(instr) << 32 | + ppc_inst_suffix(instr), patch_addr, err, "std"); +#endif + } + if (err) return err; diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c index 0c9ffdef8096..1fb845f60f43 100644 --- a/arch/powerpc/lib/feature-fixups.c +++ b/arch/powerpc/lib/feature-fixups.c @@ -84,12 +84,13 @@ static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) src = alt_start; dest = start; - for (; src < alt_end; src++, dest++) { + for (; src < alt_end; src = (void *)src + ppc_inst_len(ppc_inst_read(src)), + (dest = (void *)dest + ppc_inst_len(ppc_inst_read(dest)))) { if (patch_alt_instruction(src, dest, alt_start, alt_end)) return 1; } - for (; dest < end; dest++) + for (; dest < end; dest = (void *)dest + ppc_inst_len(ppc_inst(PPC_INST_NOP))) raw_patch_instruction(dest, ppc_inst(PPC_INST_NOP)); return 0; diff --git a/arch/powerpc/lib/inst.c b/arch/powerpc/lib/inst.c index bf3126ee399d..aedfd6e31e53 100644 --- a/arch/powerpc/lib/inst.c +++ b/arch/powerpc/lib/inst.c @@ -4,8 +4,47 @@ */ #include +#include #include +#include +#ifdef CONFIG_PPC64 +int probe_user_read_inst(struct ppc_inst *inst, + struct ppc_inst __user *nip) +{ + unsigned int val, suffix; + int err; + + err = probe_user_read(&val, nip, sizeof(val)); + if (err) + return err; + if (get_op(val) == OP_PREFIX) { + err = probe_user_read(&suffix, (void __user *)nip + 4, 4); + *inst = ppc_inst_prefix(val, suffix); + } else { + *inst = ppc_inst(val); + } + return err; +} + +int probe_kernel_read_inst(struct ppc_inst *inst, + struct ppc_inst *src) +{ + unsigned int val, suffix; + int err; + + err = probe_kernel_read(&val, src, sizeof(val)); + if (err) + return err; + if (get_op(val) == OP_PREFIX) { + err = probe_kernel_read(&suffix, (void *)src + 4, 4); + *inst = ppc_inst_prefix(val, suffix); + } else { + *inst = ppc_inst(val); + } + return err; +} +#else /* !CONFIG_PPC64 */ int probe_user_read_inst(struct ppc_inst *inst, struct ppc_inst __user *nip) { @@ -31,3 +70,4 @@ int probe_kernel_read_inst(struct ppc_inst *inst, return err; } +#endif /* CONFIG_PPC64 */ diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c index 95a56bb1ba3f..ecd756c346fd 100644 --- a/arch/powerpc/lib/sstep.c +++ b/arch/powerpc/lib/sstep.c @@ -1169,10 +1169,12 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, unsigned long int imm; unsigned long int val, val2; unsigned int mb, me, sh; - unsigned int word; + unsigned int word, suffix; long ival; word = ppc_inst_val(instr); + suffix = ppc_inst_suffix(instr); + op->type = COMPUTE; opcode = ppc_inst_primary_opcode(instr); diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index 00b24f357c2b..ac8ccf333d51 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -758,8 +758,8 @@ static int xmon_bpt(struct pt_regs *regs) /* Are we at the trap at bp->instr[1] for some bp? */ bp = in_breakpoint_table(regs->nip, &offset); - if (bp != NULL && offset == 4) { - regs->nip = bp->address + 4; + if (bp != NULL && (offset == 4 || offset == 8)) { + regs->nip = bp->address + offset; atomic_dec(&bp->ref_count); return 1; } diff --git a/arch/powerpc/xmon/xmon_bpts.S b/arch/powerpc/xmon/xmon_bpts.S index f3ad0ab50854..69726814cd27 100644 --- a/arch/powerpc/xmon/xmon_bpts.S +++ b/arch/powerpc/xmon/xmon_bpts.S @@ -4,6 +4,8 @@ #include #include "xmon_bpts.h" +/* Prefixed instructions can not cross 64 byte boundaries */ +.align 6 .global bpt_table bpt_table: .space NBPTS * BPT_SIZE -- 2.11.0