powerpc: bpf: eliminate zero extension code-gen

Cc: Naveen N. Rao <naveen.n.rao@linux.ibm.com>
Cc: Sandipan Das <sandipan@linux.ibm.com>
Signed-off-by: Jiong Wang <jiong.wang@netronome.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 21a1dcd..0ebd946 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -504,6 +504,9 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
                case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
                        /* slw clears top 32 bits */
                        PPC_SLW(dst_reg, dst_reg, src_reg);
+                       /* skip zero extension move, but set address map. */
+                       if (insn_is_zext(&insn[i + 1]))
+                               addrs[++i] = ctx->idx * 4;
                        break;
                case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
                        PPC_SLD(dst_reg, dst_reg, src_reg);
@@ -511,6 +514,8 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
                case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
                        /* with imm 0, we still need to clear top 32 bits */
                        PPC_SLWI(dst_reg, dst_reg, imm);
+                       if (insn_is_zext(&insn[i + 1]))
+                               addrs[++i] = ctx->idx * 4;
                        break;
                case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
                        if (imm != 0)
@@ -518,12 +523,16 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
                        break;
                case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
                        PPC_SRW(dst_reg, dst_reg, src_reg);
+                       if (insn_is_zext(&insn[i + 1]))
+                               addrs[++i] = ctx->idx * 4;
                        break;
                case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
                        PPC_SRD(dst_reg, dst_reg, src_reg);
                        break;
                case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
                        PPC_SRWI(dst_reg, dst_reg, imm);
+                       if (insn_is_zext(&insn[i + 1]))
+                               addrs[++i] = ctx->idx * 4;
                        break;
                case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
                        if (imm != 0)
@@ -548,6 +557,11 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
                 */
                case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
                case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
+                       if (imm == 1) {
+                               /* special mov32 for zext */
+                               PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
+                               break;
+                       }
                        PPC_MR(dst_reg, src_reg);
                        goto bpf_alu32_trunc;
                case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
@@ -555,11 +569,13 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
                        PPC_LI32(dst_reg, imm);
                        if (imm < 0)
                                goto bpf_alu32_trunc;
+                       else if (insn_is_zext(&insn[i + 1]))
+                               addrs[++i] = ctx->idx * 4;
                        break;
 
 bpf_alu32_trunc:
                /* Truncate to 32-bits */
-               if (BPF_CLASS(code) == BPF_ALU)
+               if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)
                        PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
                break;
 
@@ -618,10 +634,13 @@ emit_clear:
                        case 16:
                                /* zero-extend 16 bits into 64 bits */
                                PPC_RLDICL(dst_reg, dst_reg, 0, 48);
+                               if (insn_is_zext(&insn[i + 1]))
+                                       addrs[++i] = ctx->idx * 4;
                                break;
                        case 32:
-                               /* zero-extend 32 bits into 64 bits */
-                               PPC_RLDICL(dst_reg, dst_reg, 0, 32);
+                               if (!fp->aux->verifier_zext)
+                                       /* zero-extend 32 bits into 64 bits */
+                                       PPC_RLDICL(dst_reg, dst_reg, 0, 32);
                                break;
                        case 64:
                                /* nop */
@@ -698,14 +717,20 @@ emit_clear:
                /* dst = *(u8 *)(ul) (src + off) */
                case BPF_LDX | BPF_MEM | BPF_B:
                        PPC_LBZ(dst_reg, src_reg, off);
+                       if (insn_is_zext(&insn[i + 1]))
+                               addrs[++i] = ctx->idx * 4;
                        break;
                /* dst = *(u16 *)(ul) (src + off) */
                case BPF_LDX | BPF_MEM | BPF_H:
                        PPC_LHZ(dst_reg, src_reg, off);
+                       if (insn_is_zext(&insn[i + 1]))
+                               addrs[++i] = ctx->idx * 4;
                        break;
                /* dst = *(u32 *)(ul) (src + off) */
                case BPF_LDX | BPF_MEM | BPF_W:
                        PPC_LWZ(dst_reg, src_reg, off);
+                       if (insn_is_zext(&insn[i + 1]))
+                               addrs[++i] = ctx->idx * 4;
                        break;
                /* dst = *(u64 *)(ul) (src + off) */
                case BPF_LDX | BPF_MEM | BPF_DW:
@@ -1046,6 +1071,11 @@ struct powerpc64_jit_data {
        struct codegen_context ctx;
 };
 
+bool bpf_jit_needs_zext(void)
+{
+       return true;
+}
+
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 {
        u32 proglen;