bpf: Resolve symbols with ftrace_lookup_symbols for kprobe multi link

Using kallsyms_lookup_names function to speed up symbols lookup in
kprobe multi link attachment and replacing with it the current
kprobe_multi_resolve_syms function.

This speeds up bpftrace kprobe attachment:

  # perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* {  } i:ms:1 { exit(); }'
  ...
  6.5681 +- 0.0225 seconds time elapsed  ( +-  0.34% )

After:

  # perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* {  } i:ms:1 { exit(); }'
  ...
  0.5661 +- 0.0275 seconds time elapsed  ( +-  4.85% )

Acked-by: Andrii Nakryiko <andrii@kernel.org>
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20220510122616.2652285-5-jolsa@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index f15b826..7fd11c1 100644 (file)
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -2229,6 +2229,59 @@ struct bpf_kprobe_multi_run_ctx {
        unsigned long entry_ip;
 };
 
+struct user_syms {
+       const char **syms;
+       char *buf;
+};
+
+static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt)
+{
+       unsigned long __user usymbol;
+       const char **syms = NULL;
+       char *buf = NULL, *p;
+       int err = -ENOMEM;
+       unsigned int i;
+
+       syms = kvmalloc(cnt * sizeof(*syms), GFP_KERNEL);
+       if (!syms)
+               goto error;
+
+       buf = kvmalloc(cnt * KSYM_NAME_LEN, GFP_KERNEL);
+       if (!buf)
+               goto error;
+
+       for (p = buf, i = 0; i < cnt; i++) {
+               if (__get_user(usymbol, usyms + i)) {
+                       err = -EFAULT;
+                       goto error;
+               }
+               err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN);
+               if (err == KSYM_NAME_LEN)
+                       err = -E2BIG;
+               if (err < 0)
+                       goto error;
+               syms[i] = p;
+               p += err + 1;
+       }
+
+       us->syms = syms;
+       us->buf = buf;
+       return 0;
+
+error:
+       if (err) {
+               kvfree(syms);
+               kvfree(buf);
+       }
+       return err;
+}
+
+static void free_user_syms(struct user_syms *us)
+{
+       kvfree(us->syms);
+       kvfree(us->buf);
+}
+
 static void bpf_kprobe_multi_link_release(struct bpf_link *link)
 {
        struct bpf_kprobe_multi_link *kmulti_link;
@@ -2349,53 +2402,12 @@ kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip,
        kprobe_multi_link_prog_run(link, entry_ip, regs);
 }
 
-static int
-kprobe_multi_resolve_syms(const void __user *usyms, u32 cnt,
-                         unsigned long *addrs)
+static int symbols_cmp(const void *a, const void *b)
 {
-       unsigned long addr, size;
-       const char __user **syms;
-       int err = -ENOMEM;
-       unsigned int i;
-       char *func;
-
-       size = cnt * sizeof(*syms);
-       syms = kvzalloc(size, GFP_KERNEL);
-       if (!syms)
-               return -ENOMEM;
+       const char **str_a = (const char **) a;
+       const char **str_b = (const char **) b;
 
-       func = kmalloc(KSYM_NAME_LEN, GFP_KERNEL);
-       if (!func)
-               goto error;
-
-       if (copy_from_user(syms, usyms, size)) {
-               err = -EFAULT;
-               goto error;
-       }
-
-       for (i = 0; i < cnt; i++) {
-               err = strncpy_from_user(func, syms[i], KSYM_NAME_LEN);
-               if (err == KSYM_NAME_LEN)
-                       err = -E2BIG;
-               if (err < 0)
-                       goto error;
-               err = -EINVAL;
-               addr = kallsyms_lookup_name(func);
-               if (!addr)
-                       goto error;
-               if (!kallsyms_lookup_size_offset(addr, &size, NULL))
-                       goto error;
-               addr = ftrace_location_range(addr, addr + size - 1);
-               if (!addr)
-                       goto error;
-               addrs[i] = addr;
-       }
-
-       err = 0;
-error:
-       kvfree(syms);
-       kfree(func);
-       return err;
+       return strcmp(*str_a, *str_b);
 }
 
 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
@@ -2441,7 +2453,15 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
                        goto error;
                }
        } else {
-               err = kprobe_multi_resolve_syms(usyms, cnt, addrs);
+               struct user_syms us;
+
+               err = copy_user_syms(&us, usyms, cnt);
+               if (err)
+                       goto error;
+
+               sort(us.syms, cnt, sizeof(*us.syms), symbols_cmp, NULL);
+               err = ftrace_lookup_symbols(us.syms, cnt, addrs);
+               free_user_syms(&us);
                if (err)
                        goto error;
        }