kernel: bpf: move syscall allocations to stack

These are really small, freed in the same function, very frequent.
Allocating them on stack will improve performance.

Signed-off-by: Yaroslav Furman <yaro330@gmail.com>

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 04fc102..bd3d8f2 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -225,7 +225,7 @@ static int map_lookup_elem(union bpf_attr *attr)
        void __user *uvalue = u64_to_ptr(attr->value);
        int ufd = attr->map_fd;
        struct bpf_map *map;
-       void *key, *value, *ptr;
+       void *key, *value, *value_onstack[64], *ptr;
        struct fd f;
        int err;
 
@@ -247,9 +247,14 @@ static int map_lookup_elem(union bpf_attr *attr)
                goto free_key;
 
        err = -ENOMEM;
-       value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
-       if (!value)
-               goto free_key;
+        if (value_size < ARRAY_SIZE(value_onstack)) {
+            value = value_onstack;
+            memset(value, 0, sizeof(value));
+       } else {
+           value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
+           if (!value)
+                goto free_key;
+       }
 
        rcu_read_lock();
        ptr = map->ops->map_lookup_elem(map, key);
@@ -268,7 +273,8 @@ static int map_lookup_elem(union bpf_attr *attr)
        err = 0;
 
 free_value:
-       kfree(value);
+        if (value != value_onstack)
+            kfree(value);
 free_key:
        kfree(key);
 err_put:
@@ -378,7 +384,7 @@ static int map_get_next_key(union bpf_attr *attr)
        void __user *unext_key = u64_to_ptr(attr->next_key);
        int ufd = attr->map_fd;
        struct bpf_map *map;
-       void *key, *next_key;
+       void *key, *next_key, *next_key_onstack[64];
        struct fd f;
        int err;
 
@@ -404,9 +410,15 @@ static int map_get_next_key(union bpf_attr *attr)
        }
 
        err = -ENOMEM;
-       next_key = kmalloc(map->key_size, GFP_USER);
-       if (!next_key)
-               goto free_key;
+
+       if (map->key_size < ARRAY_SIZE(next_key_onstack)) {
+               next_key = next_key_onstack;
+               memset(next_key, 0, sizeof(next_key));
+       } else {
+                next_key = kmalloc(map->key_size, GFP_USER);
+                if (!next_key)
+                   goto free_key;
+       }
 
        rcu_read_lock();
        err = map->ops->map_get_next_key(map, key, next_key);
@@ -421,7 +433,8 @@ static int map_get_next_key(union bpf_attr *attr)
        err = 0;
 
 free_next_key:
-       kfree(next_key);
+        if (next_key != next_key_onstack)
+               kfree(next_key);
 free_key:
        kfree(key);
 err_put: