kernel/bpf/bpf_inode_storage.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019 Facebook
   4  * Copyright 2020 Google LLC.
   5  */
   6
   7 #include <linux/rculist.h>
   8 #include <linux/list.h>
   9 #include <linux/hash.h>
  10 #include <linux/types.h>
  11 #include <linux/spinlock.h>
  12 #include <linux/bpf.h>
  13 #include <linux/bpf_local_storage.h>
  14 #include <net/sock.h>
  15 #include <uapi/linux/sock_diag.h>
  16 #include <uapi/linux/btf.h>
  17 #include <linux/bpf_lsm.h>
  18 #include <linux/btf_ids.h>
  19 #include <linux/fdtable.h>
  20 #include <linux/rcupdate_trace.h>
  21
  22 DEFINE_BPF_STORAGE_CACHE(inode_cache);
  23
  24 static struct bpf_local_storage __rcu **
  25 inode_storage_ptr(void *owner)
  26 {
  27         struct inode *inode = owner;
  28         struct bpf_storage_blob *bsb;
  29
  30         bsb = bpf_inode(inode);
  31         if (!bsb)
  32                 return NULL;
  33         return &bsb->storage;
  34 }
  35
  36 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
  37                                                            struct bpf_map *map,
  38                                                            bool cacheit_lockit)
  39 {
  40         struct bpf_local_storage *inode_storage;
  41         struct bpf_local_storage_map *smap;
  42         struct bpf_storage_blob *bsb;
  43
  44         bsb = bpf_inode(inode);
  45         if (!bsb)
  46                 return NULL;
  47
  48         inode_storage =
  49                 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
  50         if (!inode_storage)
  51                 return NULL;
  52
  53         smap = (struct bpf_local_storage_map *)map;
  54         return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
  55 }
  56
  57 void bpf_inode_storage_free(struct inode *inode)
  58 {
  59         struct bpf_local_storage *local_storage;
  60         bool free_inode_storage = false;
  61         struct bpf_storage_blob *bsb;
  62
  63         bsb = bpf_inode(inode);
  64         if (!bsb)
  65                 return;
  66
  67         rcu_read_lock();
  68
  69         local_storage = rcu_dereference(bsb->storage);
  70         if (!local_storage) {
  71                 rcu_read_unlock();
  72                 return;
  73         }
  74
  75         raw_spin_lock_bh(&local_storage->lock);
  76         free_inode_storage = bpf_local_storage_unlink_nolock(local_storage);
  77         raw_spin_unlock_bh(&local_storage->lock);
  78         rcu_read_unlock();
  79
  80         if (free_inode_storage)
  81                 kfree_rcu(local_storage, rcu);
  82 }
  83
  84 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
  85 {
  86         struct bpf_local_storage_data *sdata;
  87         struct file *f;
  88         int fd;
  89
  90         fd = *(int *)key;
  91         f = fget_raw(fd);
  92         if (!f)
  93                 return ERR_PTR(-EBADF);
  94
  95         sdata = inode_storage_lookup(f->f_inode, map, true);
  96         fput(f);
  97         return sdata ? sdata->data : NULL;
  98 }
  99
 100 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
 101                                          void *value, u64 map_flags)
 102 {
 103         struct bpf_local_storage_data *sdata;
 104         struct file *f;
 105         int fd;
 106
 107         fd = *(int *)key;
 108         f = fget_raw(fd);
 109         if (!f)
 110                 return -EBADF;
 111         if (!inode_storage_ptr(f->f_inode)) {
 112                 fput(f);
 113                 return -EBADF;
 114         }
 115
 116         sdata = bpf_local_storage_update(f->f_inode,
 117                                          (struct bpf_local_storage_map *)map,
 118                                          value, map_flags, GFP_ATOMIC);
 119         fput(f);
 120         return PTR_ERR_OR_ZERO(sdata);
 121 }
 122
 123 static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
 124 {
 125         struct bpf_local_storage_data *sdata;
 126
 127         sdata = inode_storage_lookup(inode, map, false);
 128         if (!sdata)
 129                 return -ENOENT;
 130
 131         bpf_selem_unlink(SELEM(sdata), true);
 132
 133         return 0;
 134 }
 135
 136 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
 137 {
 138         struct file *f;
 139         int fd, err;
 140
 141         fd = *(int *)key;
 142         f = fget_raw(fd);
 143         if (!f)
 144                 return -EBADF;
 145
 146         err = inode_storage_delete(f->f_inode, map);
 147         fput(f);
 148         return err;
 149 }
 150
 151 /* *gfp_flags* is a hidden argument provided by the verifier */
 152 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
 153            void *, value, u64, flags, gfp_t, gfp_flags)
 154 {
 155         struct bpf_local_storage_data *sdata;
 156
 157         WARN_ON_ONCE(!bpf_rcu_lock_held());
 158         if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
 159                 return (unsigned long)NULL;
 160
 161         /* explicitly check that the inode_storage_ptr is not
 162          * NULL as inode_storage_lookup returns NULL in this case and
 163          * bpf_local_storage_update expects the owner to have a
 164          * valid storage pointer.
 165          */
 166         if (!inode || !inode_storage_ptr(inode))
 167                 return (unsigned long)NULL;
 168
 169         sdata = inode_storage_lookup(inode, map, true);
 170         if (sdata)
 171                 return (unsigned long)sdata->data;
 172
 173         /* This helper must only called from where the inode is guaranteed
 174          * to have a refcount and cannot be freed.
 175          */
 176         if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
 177                 sdata = bpf_local_storage_update(
 178                         inode, (struct bpf_local_storage_map *)map, value,
 179                         BPF_NOEXIST, gfp_flags);
 180                 return IS_ERR(sdata) ? (unsigned long)NULL :
 181                                              (unsigned long)sdata->data;
 182         }
 183
 184         return (unsigned long)NULL;
 185 }
 186
 187 BPF_CALL_2(bpf_inode_storage_delete,
 188            struct bpf_map *, map, struct inode *, inode)
 189 {
 190         WARN_ON_ONCE(!bpf_rcu_lock_held());
 191         if (!inode)
 192                 return -EINVAL;
 193
 194         /* This helper must only called from where the inode is guaranteed
 195          * to have a refcount and cannot be freed.
 196          */
 197         return inode_storage_delete(inode, map);
 198 }
 199
 200 static int notsupp_get_next_key(struct bpf_map *map, void *key,
 201                                 void *next_key)
 202 {
 203         return -ENOTSUPP;
 204 }
 205
 206 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
 207 {
 208         return bpf_local_storage_map_alloc(attr, &inode_cache);
 209 }
 210
 211 static void inode_storage_map_free(struct bpf_map *map)
 212 {
 213         bpf_local_storage_map_free(map, &inode_cache, NULL);
 214 }
 215
 216 const struct bpf_map_ops inode_storage_map_ops = {
 217         .map_meta_equal = bpf_map_meta_equal,
 218         .map_alloc_check = bpf_local_storage_map_alloc_check,
 219         .map_alloc = inode_storage_map_alloc,
 220         .map_free = inode_storage_map_free,
 221         .map_get_next_key = notsupp_get_next_key,
 222         .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
 223         .map_update_elem = bpf_fd_inode_storage_update_elem,
 224         .map_delete_elem = bpf_fd_inode_storage_delete_elem,
 225         .map_check_btf = bpf_local_storage_map_check_btf,
 226         .map_btf_id = &bpf_local_storage_map_btf_id[0],
 227         .map_owner_storage_ptr = inode_storage_ptr,
 228 };
 229
 230 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
 231
 232 const struct bpf_func_proto bpf_inode_storage_get_proto = {
 233         .func           = bpf_inode_storage_get,
 234         .gpl_only       = false,
 235         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 236         .arg1_type      = ARG_CONST_MAP_PTR,
 237         .arg2_type      = ARG_PTR_TO_BTF_ID,
 238         .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
 239         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 240         .arg4_type      = ARG_ANYTHING,
 241 };
 242
 243 const struct bpf_func_proto bpf_inode_storage_delete_proto = {
 244         .func           = bpf_inode_storage_delete,
 245         .gpl_only       = false,
 246         .ret_type       = RET_INTEGER,
 247         .arg1_type      = ARG_CONST_MAP_PTR,
 248         .arg2_type      = ARG_PTR_TO_BTF_ID,
 249         .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
 250 };