kernel/bpf/bpf_inode_storage.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019 Facebook
   4  * Copyright 2020 Google LLC.
   5  */
   6
   7 #include <linux/rculist.h>
   8 #include <linux/list.h>
   9 #include <linux/hash.h>
  10 #include <linux/types.h>
  11 #include <linux/spinlock.h>
  12 #include <linux/bpf.h>
  13 #include <linux/bpf_local_storage.h>
  14 #include <net/sock.h>
  15 #include <uapi/linux/sock_diag.h>
  16 #include <uapi/linux/btf.h>
  17 #include <linux/bpf_lsm.h>
  18 #include <linux/btf_ids.h>
  19 #include <linux/fdtable.h>
  20 #include <linux/rcupdate_trace.h>
  21
  22 DEFINE_BPF_STORAGE_CACHE(inode_cache);
  23
  24 static struct bpf_local_storage __rcu **
  25 inode_storage_ptr(void *owner)
  26 {
  27         struct inode *inode = owner;
  28         struct bpf_storage_blob *bsb;
  29
  30         bsb = bpf_inode(inode);
  31         if (!bsb)
  32                 return NULL;
  33         return &bsb->storage;
  34 }
  35
  36 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
  37                                                            struct bpf_map *map,
  38                                                            bool cacheit_lockit)
  39 {
  40         struct bpf_local_storage *inode_storage;
  41         struct bpf_local_storage_map *smap;
  42         struct bpf_storage_blob *bsb;
  43
  44         bsb = bpf_inode(inode);
  45         if (!bsb)
  46                 return NULL;
  47
  48         inode_storage =
  49                 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
  50         if (!inode_storage)
  51                 return NULL;
  52
  53         smap = (struct bpf_local_storage_map *)map;
  54         return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
  55 }
  56
  57 void bpf_inode_storage_free(struct inode *inode)
  58 {
  59         struct bpf_local_storage *local_storage;
  60         struct bpf_storage_blob *bsb;
  61
  62         bsb = bpf_inode(inode);
  63         if (!bsb)
  64                 return;
  65
  66         rcu_read_lock();
  67
  68         local_storage = rcu_dereference(bsb->storage);
  69         if (!local_storage) {
  70                 rcu_read_unlock();
  71                 return;
  72         }
  73
  74         bpf_local_storage_destroy(local_storage);
  75         rcu_read_unlock();
  76 }
  77
  78 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
  79 {
  80         struct bpf_local_storage_data *sdata;
  81         struct fd f = fdget_raw(*(int *)key);
  82
  83         if (!f.file)
  84                 return ERR_PTR(-EBADF);
  85
  86         sdata = inode_storage_lookup(file_inode(f.file), map, true);
  87         fdput(f);
  88         return sdata ? sdata->data : NULL;
  89 }
  90
  91 static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
  92                                              void *value, u64 map_flags)
  93 {
  94         struct bpf_local_storage_data *sdata;
  95         struct fd f = fdget_raw(*(int *)key);
  96
  97         if (!f.file)
  98                 return -EBADF;
  99         if (!inode_storage_ptr(file_inode(f.file))) {
 100                 fdput(f);
 101                 return -EBADF;
 102         }
 103
 104         sdata = bpf_local_storage_update(file_inode(f.file),
 105                                          (struct bpf_local_storage_map *)map,
 106                                          value, map_flags, GFP_ATOMIC);
 107         fdput(f);
 108         return PTR_ERR_OR_ZERO(sdata);
 109 }
 110
 111 static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
 112 {
 113         struct bpf_local_storage_data *sdata;
 114
 115         sdata = inode_storage_lookup(inode, map, false);
 116         if (!sdata)
 117                 return -ENOENT;
 118
 119         bpf_selem_unlink(SELEM(sdata), false);
 120
 121         return 0;
 122 }
 123
 124 static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
 125 {
 126         struct fd f = fdget_raw(*(int *)key);
 127         int err;
 128
 129         if (!f.file)
 130                 return -EBADF;
 131
 132         err = inode_storage_delete(file_inode(f.file), map);
 133         fdput(f);
 134         return err;
 135 }
 136
 137 /* *gfp_flags* is a hidden argument provided by the verifier */
 138 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
 139            void *, value, u64, flags, gfp_t, gfp_flags)
 140 {
 141         struct bpf_local_storage_data *sdata;
 142
 143         WARN_ON_ONCE(!bpf_rcu_lock_held());
 144         if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
 145                 return (unsigned long)NULL;
 146
 147         /* explicitly check that the inode_storage_ptr is not
 148          * NULL as inode_storage_lookup returns NULL in this case and
 149          * bpf_local_storage_update expects the owner to have a
 150          * valid storage pointer.
 151          */
 152         if (!inode || !inode_storage_ptr(inode))
 153                 return (unsigned long)NULL;
 154
 155         sdata = inode_storage_lookup(inode, map, true);
 156         if (sdata)
 157                 return (unsigned long)sdata->data;
 158
 159         /* This helper must only called from where the inode is guaranteed
 160          * to have a refcount and cannot be freed.
 161          */
 162         if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
 163                 sdata = bpf_local_storage_update(
 164                         inode, (struct bpf_local_storage_map *)map, value,
 165                         BPF_NOEXIST, gfp_flags);
 166                 return IS_ERR(sdata) ? (unsigned long)NULL :
 167                                              (unsigned long)sdata->data;
 168         }
 169
 170         return (unsigned long)NULL;
 171 }
 172
 173 BPF_CALL_2(bpf_inode_storage_delete,
 174            struct bpf_map *, map, struct inode *, inode)
 175 {
 176         WARN_ON_ONCE(!bpf_rcu_lock_held());
 177         if (!inode)
 178                 return -EINVAL;
 179
 180         /* This helper must only called from where the inode is guaranteed
 181          * to have a refcount and cannot be freed.
 182          */
 183         return inode_storage_delete(inode, map);
 184 }
 185
 186 static int notsupp_get_next_key(struct bpf_map *map, void *key,
 187                                 void *next_key)
 188 {
 189         return -ENOTSUPP;
 190 }
 191
 192 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
 193 {
 194         return bpf_local_storage_map_alloc(attr, &inode_cache, false);
 195 }
 196
 197 static void inode_storage_map_free(struct bpf_map *map)
 198 {
 199         bpf_local_storage_map_free(map, &inode_cache, NULL);
 200 }
 201
 202 const struct bpf_map_ops inode_storage_map_ops = {
 203         .map_meta_equal = bpf_map_meta_equal,
 204         .map_alloc_check = bpf_local_storage_map_alloc_check,
 205         .map_alloc = inode_storage_map_alloc,
 206         .map_free = inode_storage_map_free,
 207         .map_get_next_key = notsupp_get_next_key,
 208         .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
 209         .map_update_elem = bpf_fd_inode_storage_update_elem,
 210         .map_delete_elem = bpf_fd_inode_storage_delete_elem,
 211         .map_check_btf = bpf_local_storage_map_check_btf,
 212         .map_mem_usage = bpf_local_storage_map_mem_usage,
 213         .map_btf_id = &bpf_local_storage_map_btf_id[0],
 214         .map_owner_storage_ptr = inode_storage_ptr,
 215 };
 216
 217 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
 218
 219 const struct bpf_func_proto bpf_inode_storage_get_proto = {
 220         .func           = bpf_inode_storage_get,
 221         .gpl_only       = false,
 222         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 223         .arg1_type      = ARG_CONST_MAP_PTR,
 224         .arg2_type      = ARG_PTR_TO_BTF_ID_OR_NULL,
 225         .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
 226         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 227         .arg4_type      = ARG_ANYTHING,
 228 };
 229
 230 const struct bpf_func_proto bpf_inode_storage_delete_proto = {
 231         .func           = bpf_inode_storage_delete,
 232         .gpl_only       = false,
 233         .ret_type       = RET_INTEGER,
 234         .arg1_type      = ARG_CONST_MAP_PTR,
 235         .arg2_type      = ARG_PTR_TO_BTF_ID_OR_NULL,
 236         .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
 237 };