kernel/bpf/tcx.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2023 Isovalent */
   3
   4 #include <linux/bpf.h>
   5 #include <linux/bpf_mprog.h>
   6 #include <linux/netdevice.h>
   7
   8 #include <net/tcx.h>
   9
  10 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
  11 {
  12         bool created, ingress = attr->attach_type == BPF_TCX_INGRESS;
  13         struct net *net = current->nsproxy->net_ns;
  14         struct bpf_mprog_entry *entry, *entry_new;
  15         struct bpf_prog *replace_prog = NULL;
  16         struct net_device *dev;
  17         int ret;
  18
  19         rtnl_lock();
  20         dev = __dev_get_by_index(net, attr->target_ifindex);
  21         if (!dev) {
  22                 ret = -ENODEV;
  23                 goto out;
  24         }
  25         if (attr->attach_flags & BPF_F_REPLACE) {
  26                 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
  27                                                  prog->type);
  28                 if (IS_ERR(replace_prog)) {
  29                         ret = PTR_ERR(replace_prog);
  30                         replace_prog = NULL;
  31                         goto out;
  32                 }
  33         }
  34         entry = tcx_entry_fetch_or_create(dev, ingress, &created);
  35         if (!entry) {
  36                 ret = -ENOMEM;
  37                 goto out;
  38         }
  39         ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
  40                                attr->attach_flags, attr->relative_fd,
  41                                attr->expected_revision);
  42         if (!ret) {
  43                 if (entry != entry_new) {
  44                         tcx_entry_update(dev, entry_new, ingress);
  45                         tcx_entry_sync();
  46                         tcx_skeys_inc(ingress);
  47                 }
  48                 bpf_mprog_commit(entry);
  49         } else if (created) {
  50                 tcx_entry_free(entry);
  51         }
  52 out:
  53         if (replace_prog)
  54                 bpf_prog_put(replace_prog);
  55         rtnl_unlock();
  56         return ret;
  57 }
  58
  59 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
  60 {
  61         bool ingress = attr->attach_type == BPF_TCX_INGRESS;
  62         struct net *net = current->nsproxy->net_ns;
  63         struct bpf_mprog_entry *entry, *entry_new;
  64         struct net_device *dev;
  65         int ret;
  66
  67         rtnl_lock();
  68         dev = __dev_get_by_index(net, attr->target_ifindex);
  69         if (!dev) {
  70                 ret = -ENODEV;
  71                 goto out;
  72         }
  73         entry = tcx_entry_fetch(dev, ingress);
  74         if (!entry) {
  75                 ret = -ENOENT;
  76                 goto out;
  77         }
  78         ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
  79                                attr->relative_fd, attr->expected_revision);
  80         if (!ret) {
  81                 if (!tcx_entry_is_active(entry_new))
  82                         entry_new = NULL;
  83                 tcx_entry_update(dev, entry_new, ingress);
  84                 tcx_entry_sync();
  85                 tcx_skeys_dec(ingress);
  86                 bpf_mprog_commit(entry);
  87                 if (!entry_new)
  88                         tcx_entry_free(entry);
  89         }
  90 out:
  91         rtnl_unlock();
  92         return ret;
  93 }
  94
  95 void tcx_uninstall(struct net_device *dev, bool ingress)
  96 {
  97         struct bpf_mprog_entry *entry, *entry_new = NULL;
  98         struct bpf_tuple tuple = {};
  99         struct bpf_mprog_fp *fp;
 100         struct bpf_mprog_cp *cp;
 101         bool active;
 102
 103         entry = tcx_entry_fetch(dev, ingress);
 104         if (!entry)
 105                 return;
 106         active = tcx_entry(entry)->miniq_active;
 107         if (active)
 108                 bpf_mprog_clear_all(entry, &entry_new);
 109         tcx_entry_update(dev, entry_new, ingress);
 110         tcx_entry_sync();
 111         bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
 112                 if (tuple.link)
 113                         tcx_link(tuple.link)->dev = NULL;
 114                 else
 115                         bpf_prog_put(tuple.prog);
 116                 tcx_skeys_dec(ingress);
 117         }
 118         if (!active)
 119                 tcx_entry_free(entry);
 120 }
 121
 122 int tcx_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
 123 {
 124         bool ingress = attr->query.attach_type == BPF_TCX_INGRESS;
 125         struct net *net = current->nsproxy->net_ns;
 126         struct bpf_mprog_entry *entry;
 127         struct net_device *dev;
 128         int ret;
 129
 130         rtnl_lock();
 131         dev = __dev_get_by_index(net, attr->query.target_ifindex);
 132         if (!dev) {
 133                 ret = -ENODEV;
 134                 goto out;
 135         }
 136         entry = tcx_entry_fetch(dev, ingress);
 137         if (!entry) {
 138                 ret = -ENOENT;
 139                 goto out;
 140         }
 141         ret = bpf_mprog_query(attr, uattr, entry);
 142 out:
 143         rtnl_unlock();
 144         return ret;
 145 }
 146
 147 static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd,
 148                                 u64 revision)
 149 {
 150         struct tcx_link *tcx = tcx_link(link);
 151         bool created, ingress = tcx->location == BPF_TCX_INGRESS;
 152         struct bpf_mprog_entry *entry, *entry_new;
 153         struct net_device *dev = tcx->dev;
 154         int ret;
 155
 156         ASSERT_RTNL();
 157         entry = tcx_entry_fetch_or_create(dev, ingress, &created);
 158         if (!entry)
 159                 return -ENOMEM;
 160         ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
 161                                id_or_fd, revision);
 162         if (!ret) {
 163                 if (entry != entry_new) {
 164                         tcx_entry_update(dev, entry_new, ingress);
 165                         tcx_entry_sync();
 166                         tcx_skeys_inc(ingress);
 167                 }
 168                 bpf_mprog_commit(entry);
 169         } else if (created) {
 170                 tcx_entry_free(entry);
 171         }
 172         return ret;
 173 }
 174
 175 static void tcx_link_release(struct bpf_link *link)
 176 {
 177         struct tcx_link *tcx = tcx_link(link);
 178         bool ingress = tcx->location == BPF_TCX_INGRESS;
 179         struct bpf_mprog_entry *entry, *entry_new;
 180         struct net_device *dev;
 181         int ret = 0;
 182
 183         rtnl_lock();
 184         dev = tcx->dev;
 185         if (!dev)
 186                 goto out;
 187         entry = tcx_entry_fetch(dev, ingress);
 188         if (!entry) {
 189                 ret = -ENOENT;
 190                 goto out;
 191         }
 192         ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
 193         if (!ret) {
 194                 if (!tcx_entry_is_active(entry_new))
 195                         entry_new = NULL;
 196                 tcx_entry_update(dev, entry_new, ingress);
 197                 tcx_entry_sync();
 198                 tcx_skeys_dec(ingress);
 199                 bpf_mprog_commit(entry);
 200                 if (!entry_new)
 201                         tcx_entry_free(entry);
 202                 tcx->dev = NULL;
 203         }
 204 out:
 205         WARN_ON_ONCE(ret);
 206         rtnl_unlock();
 207 }
 208
 209 static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog,
 210                            struct bpf_prog *oprog)
 211 {
 212         struct tcx_link *tcx = tcx_link(link);
 213         bool ingress = tcx->location == BPF_TCX_INGRESS;
 214         struct bpf_mprog_entry *entry, *entry_new;
 215         struct net_device *dev;
 216         int ret = 0;
 217
 218         rtnl_lock();
 219         dev = tcx->dev;
 220         if (!dev) {
 221                 ret = -ENOLINK;
 222                 goto out;
 223         }
 224         if (oprog && link->prog != oprog) {
 225                 ret = -EPERM;
 226                 goto out;
 227         }
 228         oprog = link->prog;
 229         if (oprog == nprog) {
 230                 bpf_prog_put(nprog);
 231                 goto out;
 232         }
 233         entry = tcx_entry_fetch(dev, ingress);
 234         if (!entry) {
 235                 ret = -ENOENT;
 236                 goto out;
 237         }
 238         ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
 239                                BPF_F_REPLACE | BPF_F_ID,
 240                                link->prog->aux->id, 0);
 241         if (!ret) {
 242                 WARN_ON_ONCE(entry != entry_new);
 243                 oprog = xchg(&link->prog, nprog);
 244                 bpf_prog_put(oprog);
 245                 bpf_mprog_commit(entry);
 246         }
 247 out:
 248         rtnl_unlock();
 249         return ret;
 250 }
 251
 252 static void tcx_link_dealloc(struct bpf_link *link)
 253 {
 254         kfree(tcx_link(link));
 255 }
 256
 257 static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
 258 {
 259         const struct tcx_link *tcx = tcx_link_const(link);
 260         u32 ifindex = 0;
 261
 262         rtnl_lock();
 263         if (tcx->dev)
 264                 ifindex = tcx->dev->ifindex;
 265         rtnl_unlock();
 266
 267         seq_printf(seq, "ifindex:\t%u\n", ifindex);
 268         seq_printf(seq, "attach_type:\t%u (%s)\n",
 269                    tcx->location,
 270                    tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress");
 271 }
 272
 273 static int tcx_link_fill_info(const struct bpf_link *link,
 274                               struct bpf_link_info *info)
 275 {
 276         const struct tcx_link *tcx = tcx_link_const(link);
 277         u32 ifindex = 0;
 278
 279         rtnl_lock();
 280         if (tcx->dev)
 281                 ifindex = tcx->dev->ifindex;
 282         rtnl_unlock();
 283
 284         info->tcx.ifindex = ifindex;
 285         info->tcx.attach_type = tcx->location;
 286         return 0;
 287 }
 288
 289 static int tcx_link_detach(struct bpf_link *link)
 290 {
 291         tcx_link_release(link);
 292         return 0;
 293 }
 294
 295 static const struct bpf_link_ops tcx_link_lops = {
 296         .release        = tcx_link_release,
 297         .detach         = tcx_link_detach,
 298         .dealloc        = tcx_link_dealloc,
 299         .update_prog    = tcx_link_update,
 300         .show_fdinfo    = tcx_link_fdinfo,
 301         .fill_link_info = tcx_link_fill_info,
 302 };
 303
 304 static int tcx_link_init(struct tcx_link *tcx,
 305                          struct bpf_link_primer *link_primer,
 306                          const union bpf_attr *attr,
 307                          struct net_device *dev,
 308                          struct bpf_prog *prog)
 309 {
 310         bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog);
 311         tcx->location = attr->link_create.attach_type;
 312         tcx->dev = dev;
 313         return bpf_link_prime(&tcx->link, link_primer);
 314 }
 315
 316 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 317 {
 318         struct net *net = current->nsproxy->net_ns;
 319         struct bpf_link_primer link_primer;
 320         struct net_device *dev;
 321         struct tcx_link *tcx;
 322         int ret;
 323
 324         rtnl_lock();
 325         dev = __dev_get_by_index(net, attr->link_create.target_ifindex);
 326         if (!dev) {
 327                 ret = -ENODEV;
 328                 goto out;
 329         }
 330         tcx = kzalloc(sizeof(*tcx), GFP_USER);
 331         if (!tcx) {
 332                 ret = -ENOMEM;
 333                 goto out;
 334         }
 335         ret = tcx_link_init(tcx, &link_primer, attr, dev, prog);
 336         if (ret) {
 337                 kfree(tcx);
 338                 goto out;
 339         }
 340         ret = tcx_link_prog_attach(&tcx->link, attr->link_create.flags,
 341                                    attr->link_create.tcx.relative_fd,
 342                                    attr->link_create.tcx.expected_revision);
 343         if (ret) {
 344                 tcx->dev = NULL;
 345                 bpf_link_cleanup(&link_primer);
 346                 goto out;
 347         }
 348         ret = bpf_link_settle(&link_primer);
 349 out:
 350         rtnl_unlock();
 351         return ret;
 352 }