include/linux/bpf_local_storage.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Copyright (c) 2019 Facebook
   4  * Copyright 2020 Google LLC.
   5  */
   6
   7 #ifndef _BPF_LOCAL_STORAGE_H
   8 #define _BPF_LOCAL_STORAGE_H
   9
  10 #include <linux/bpf.h>
  11 #include <linux/filter.h>
  12 #include <linux/rculist.h>
  13 #include <linux/list.h>
  14 #include <linux/hash.h>
  15 #include <linux/types.h>
  16 #include <uapi/linux/btf.h>
  17
  18 #define BPF_LOCAL_STORAGE_CACHE_SIZE    16
  19
  20 #define bpf_rcu_lock_held()                                                    \
  21         (rcu_read_lock_held() || rcu_read_lock_trace_held() ||                 \
  22          rcu_read_lock_bh_held())
  23 struct bpf_local_storage_map_bucket {
  24         struct hlist_head list;
  25         raw_spinlock_t lock;
  26 };
  27
  28 /* Thp map is not the primary owner of a bpf_local_storage_elem.
  29  * Instead, the container object (eg. sk->sk_bpf_storage) is.
  30  *
  31  * The map (bpf_local_storage_map) is for two purposes
  32  * 1. Define the size of the "local storage".  It is
  33  *    the map's value_size.
  34  *
  35  * 2. Maintain a list to keep track of all elems such
  36  *    that they can be cleaned up during the map destruction.
  37  *
  38  * When a bpf local storage is being looked up for a
  39  * particular object,  the "bpf_map" pointer is actually used
  40  * as the "key" to search in the list of elem in
  41  * the respective bpf_local_storage owned by the object.
  42  *
  43  * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
  44  * as the searching key.
  45  */
  46 struct bpf_local_storage_map {
  47         struct bpf_map map;
  48         /* Lookup elem does not require accessing the map.
  49          *
  50          * Updating/Deleting requires a bucket lock to
  51          * link/unlink the elem from the map.  Having
  52          * multiple buckets to improve contention.
  53          */
  54         struct bpf_local_storage_map_bucket *buckets;
  55         u32 bucket_log;
  56         u16 elem_size;
  57         u16 cache_idx;
  58 };
  59
  60 struct bpf_local_storage_data {
  61         /* smap is used as the searching key when looking up
  62          * from the object's bpf_local_storage.
  63          *
  64          * Put it in the same cacheline as the data to minimize
  65          * the number of cachelines accessed during the cache hit case.
  66          */
  67         struct bpf_local_storage_map __rcu *smap;
  68         u8 data[] __aligned(8);
  69 };
  70
  71 /* Linked to bpf_local_storage and bpf_local_storage_map */
  72 struct bpf_local_storage_elem {
  73         struct hlist_node map_node;     /* Linked to bpf_local_storage_map */
  74         struct hlist_node snode;        /* Linked to bpf_local_storage */
  75         struct bpf_local_storage __rcu *local_storage;
  76         struct rcu_head rcu;
  77         bool can_use_smap; /* Is it safe to access smap in bpf_selem_free_* RCU
  78                             * callbacks? bpf_local_storage_map_free only
  79                             * executes rcu_barrier when there are special
  80                             * fields, this field remembers that to ensure we
  81                             * don't access already freed smap in sdata.
  82                             */
  83         /* 8 bytes hole */
  84         /* The data is stored in another cacheline to minimize
  85          * the number of cachelines access during a cache hit.
  86          */
  87         struct bpf_local_storage_data sdata ____cacheline_aligned;
  88 };
  89
  90 struct bpf_local_storage {
  91         struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
  92         struct hlist_head list; /* List of bpf_local_storage_elem */
  93         void *owner;            /* The object that owns the above "list" of
  94                                  * bpf_local_storage_elem.
  95                                  */
  96         struct rcu_head rcu;
  97         raw_spinlock_t lock;    /* Protect adding/removing from the "list" */
  98 };
  99
 100 /* U16_MAX is much more than enough for sk local storage
 101  * considering a tcp_sock is ~2k.
 102  */
 103 #define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE                                       \
 104         min_t(u32,                                                             \
 105               (KMALLOC_MAX_SIZE - MAX_BPF_STACK -                              \
 106                sizeof(struct bpf_local_storage_elem)),                         \
 107               (U16_MAX - sizeof(struct bpf_local_storage_elem)))
 108
 109 #define SELEM(_SDATA)                                                          \
 110         container_of((_SDATA), struct bpf_local_storage_elem, sdata)
 111 #define SDATA(_SELEM) (&(_SELEM)->sdata)
 112
 113 #define BPF_LOCAL_STORAGE_CACHE_SIZE    16
 114
 115 struct bpf_local_storage_cache {
 116         spinlock_t idx_lock;
 117         u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
 118 };
 119
 120 #define DEFINE_BPF_STORAGE_CACHE(name)                          \
 121 static struct bpf_local_storage_cache name = {                  \
 122         .idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock),        \
 123 }
 124
 125 /* Helper functions for bpf_local_storage */
 126 int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
 127
 128 struct bpf_map *
 129 bpf_local_storage_map_alloc(union bpf_attr *attr,
 130                             struct bpf_local_storage_cache *cache);
 131
 132 struct bpf_local_storage_data *
 133 bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
 134                          struct bpf_local_storage_map *smap,
 135                          bool cacheit_lockit);
 136
 137 bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage);
 138
 139 void bpf_local_storage_map_free(struct bpf_map *map,
 140                                 struct bpf_local_storage_cache *cache,
 141                                 int __percpu *busy_counter);
 142
 143 int bpf_local_storage_map_check_btf(const struct bpf_map *map,
 144                                     const struct btf *btf,
 145                                     const struct btf_type *key_type,
 146                                     const struct btf_type *value_type);
 147
 148 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
 149                                    struct bpf_local_storage_elem *selem);
 150
 151 void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu);
 152
 153 void bpf_selem_link_map(struct bpf_local_storage_map *smap,
 154                         struct bpf_local_storage_elem *selem);
 155
 156 void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
 157
 158 struct bpf_local_storage_elem *
 159 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
 160                 bool charge_mem, gfp_t gfp_flags);
 161
 162 int
 163 bpf_local_storage_alloc(void *owner,
 164                         struct bpf_local_storage_map *smap,
 165                         struct bpf_local_storage_elem *first_selem,
 166                         gfp_t gfp_flags);
 167
 168 struct bpf_local_storage_data *
 169 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
 170                          void *value, u64 map_flags, gfp_t gfp_flags);
 171
 172 void bpf_local_storage_free_rcu(struct rcu_head *rcu);
 173
 174 #endif /* _BPF_LOCAL_STORAGE_H */