kernel/bpf/syscall.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   3  */
   4 #include <linux/bpf.h>
   5 #include <linux/bpf-cgroup.h>
   6 #include <linux/bpf_trace.h>
   7 #include <linux/bpf_lirc.h>
   8 #include <linux/bpf_verifier.h>
   9 #include <linux/bsearch.h>
  10 #include <linux/btf.h>
  11 #include <linux/syscalls.h>
  12 #include <linux/slab.h>
  13 #include <linux/sched/signal.h>
  14 #include <linux/vmalloc.h>
  15 #include <linux/mmzone.h>
  16 #include <linux/anon_inodes.h>
  17 #include <linux/fdtable.h>
  18 #include <linux/file.h>
  19 #include <linux/fs.h>
  20 #include <linux/license.h>
  21 #include <linux/filter.h>
  22 #include <linux/kernel.h>
  23 #include <linux/idr.h>
  24 #include <linux/cred.h>
  25 #include <linux/timekeeping.h>
  26 #include <linux/ctype.h>
  27 #include <linux/nospec.h>
  28 #include <linux/audit.h>
  29 #include <uapi/linux/btf.h>
  30 #include <linux/pgtable.h>
  31 #include <linux/bpf_lsm.h>
  32 #include <linux/poll.h>
  33 #include <linux/sort.h>
  34 #include <linux/bpf-netns.h>
  35 #include <linux/rcupdate_trace.h>
  36 #include <linux/memcontrol.h>
  37 #include <linux/trace_events.h>
  38 #include <net/netfilter/nf_bpf_link.h>
  39
  40 #include <net/tcx.h>
  41
  42 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  43                           (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  44                           (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  45 #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
  46 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  47 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
  48                         IS_FD_HASH(map))
  49
  50 #define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)
  51
  52 DEFINE_PER_CPU(int, bpf_prog_active);
  53 static DEFINE_IDR(prog_idr);
  54 static DEFINE_SPINLOCK(prog_idr_lock);
  55 static DEFINE_IDR(map_idr);
  56 static DEFINE_SPINLOCK(map_idr_lock);
  57 static DEFINE_IDR(link_idr);
  58 static DEFINE_SPINLOCK(link_idr_lock);
  59
  60 int sysctl_unprivileged_bpf_disabled __read_mostly =
  61         IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;
  62
  63 static const struct bpf_map_ops * const bpf_map_types[] = {
  64 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
  65 #define BPF_MAP_TYPE(_id, _ops) \
  66         [_id] = &_ops,
  67 #define BPF_LINK_TYPE(_id, _name)
  68 #include <linux/bpf_types.h>
  69 #undef BPF_PROG_TYPE
  70 #undef BPF_MAP_TYPE
  71 #undef BPF_LINK_TYPE
  72 };
  73
  74 /*
  75  * If we're handed a bigger struct than we know of, ensure all the unknown bits
  76  * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  77  * we don't know about yet.
  78  *
  79  * There is a ToCToU between this function call and the following
  80  * copy_from_user() call. However, this is not a concern since this function is
  81  * meant to be a future-proofing of bits.
  82  */
  83 int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
  84                              size_t expected_size,
  85                              size_t actual_size)
  86 {
  87         int res;
  88
  89         if (unlikely(actual_size > PAGE_SIZE))  /* silly large */
  90                 return -E2BIG;
  91
  92         if (actual_size <= expected_size)
  93                 return 0;
  94
  95         if (uaddr.is_kernel)
  96                 res = memchr_inv(uaddr.kernel + expected_size, 0,
  97                                  actual_size - expected_size) == NULL;
  98         else
  99                 res = check_zeroed_user(uaddr.user + expected_size,
 100                                         actual_size - expected_size);
 101         if (res < 0)
 102                 return res;
 103         return res ? 0 : -E2BIG;
 104 }
 105
 106 const struct bpf_map_ops bpf_map_offload_ops = {
 107         .map_meta_equal = bpf_map_meta_equal,
 108         .map_alloc = bpf_map_offload_map_alloc,
 109         .map_free = bpf_map_offload_map_free,
 110         .map_check_btf = map_check_no_btf,
 111         .map_mem_usage = bpf_map_offload_map_mem_usage,
 112 };
 113
 114 static void bpf_map_write_active_inc(struct bpf_map *map)
 115 {
 116         atomic64_inc(&map->writecnt);
 117 }
 118
 119 static void bpf_map_write_active_dec(struct bpf_map *map)
 120 {
 121         atomic64_dec(&map->writecnt);
 122 }
 123
 124 bool bpf_map_write_active(const struct bpf_map *map)
 125 {
 126         return atomic64_read(&map->writecnt) != 0;
 127 }
 128
 129 static u32 bpf_map_value_size(const struct bpf_map *map)
 130 {
 131         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 132             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 133             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 134             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 135                 return round_up(map->value_size, 8) * num_possible_cpus();
 136         else if (IS_FD_MAP(map))
 137                 return sizeof(u32);
 138         else
 139                 return  map->value_size;
 140 }
 141
 142 static void maybe_wait_bpf_programs(struct bpf_map *map)
 143 {
 144         /* Wait for any running BPF programs to complete so that
 145          * userspace, when we return to it, knows that all programs
 146          * that could be running use the new map value.
 147          */
 148         if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
 149             map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 150                 synchronize_rcu();
 151 }
 152
 153 static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,
 154                                 void *key, void *value, __u64 flags)
 155 {
 156         int err;
 157
 158         /* Need to create a kthread, thus must support schedule */
 159         if (bpf_map_is_offloaded(map)) {
 160                 return bpf_map_offload_update_elem(map, key, value, flags);
 161         } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
 162                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 163                 return map->ops->map_update_elem(map, key, value, flags);
 164         } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
 165                    map->map_type == BPF_MAP_TYPE_SOCKMAP) {
 166                 return sock_map_update_elem_sys(map, key, value, flags);
 167         } else if (IS_FD_PROG_ARRAY(map)) {
 168                 return bpf_fd_array_map_update_elem(map, map_file, key, value,
 169                                                     flags);
 170         }
 171
 172         bpf_disable_instrumentation();
 173         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 174             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 175                 err = bpf_percpu_hash_update(map, key, value, flags);
 176         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 177                 err = bpf_percpu_array_update(map, key, value, flags);
 178         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 179                 err = bpf_percpu_cgroup_storage_update(map, key, value,
 180                                                        flags);
 181         } else if (IS_FD_ARRAY(map)) {
 182                 rcu_read_lock();
 183                 err = bpf_fd_array_map_update_elem(map, map_file, key, value,
 184                                                    flags);
 185                 rcu_read_unlock();
 186         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 187                 rcu_read_lock();
 188                 err = bpf_fd_htab_map_update_elem(map, map_file, key, value,
 189                                                   flags);
 190                 rcu_read_unlock();
 191         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 192                 /* rcu_read_lock() is not needed */
 193                 err = bpf_fd_reuseport_array_update_elem(map, key, value,
 194                                                          flags);
 195         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 196                    map->map_type == BPF_MAP_TYPE_STACK ||
 197                    map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
 198                 err = map->ops->map_push_elem(map, value, flags);
 199         } else {
 200                 rcu_read_lock();
 201                 err = map->ops->map_update_elem(map, key, value, flags);
 202                 rcu_read_unlock();
 203         }
 204         bpf_enable_instrumentation();
 205         maybe_wait_bpf_programs(map);
 206
 207         return err;
 208 }
 209
 210 static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
 211                               __u64 flags)
 212 {
 213         void *ptr;
 214         int err;
 215
 216         if (bpf_map_is_offloaded(map))
 217                 return bpf_map_offload_lookup_elem(map, key, value);
 218
 219         bpf_disable_instrumentation();
 220         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 221             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 222                 err = bpf_percpu_hash_copy(map, key, value);
 223         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 224                 err = bpf_percpu_array_copy(map, key, value);
 225         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 226                 err = bpf_percpu_cgroup_storage_copy(map, key, value);
 227         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 228                 err = bpf_stackmap_copy(map, key, value);
 229         } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
 230                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 231         } else if (IS_FD_HASH(map)) {
 232                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 233         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 234                 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
 235         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 236                    map->map_type == BPF_MAP_TYPE_STACK ||
 237                    map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
 238                 err = map->ops->map_peek_elem(map, value);
 239         } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 240                 /* struct_ops map requires directly updating "value" */
 241                 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
 242         } else {
 243                 rcu_read_lock();
 244                 if (map->ops->map_lookup_elem_sys_only)
 245                         ptr = map->ops->map_lookup_elem_sys_only(map, key);
 246                 else
 247                         ptr = map->ops->map_lookup_elem(map, key);
 248                 if (IS_ERR(ptr)) {
 249                         err = PTR_ERR(ptr);
 250                 } else if (!ptr) {
 251                         err = -ENOENT;
 252                 } else {
 253                         err = 0;
 254                         if (flags & BPF_F_LOCK)
 255                                 /* lock 'ptr' and copy everything but lock */
 256                                 copy_map_value_locked(map, value, ptr, true);
 257                         else
 258                                 copy_map_value(map, value, ptr);
 259                         /* mask lock and timer, since value wasn't zero inited */
 260                         check_and_init_map_value(map, value);
 261                 }
 262                 rcu_read_unlock();
 263         }
 264
 265         bpf_enable_instrumentation();
 266         maybe_wait_bpf_programs(map);
 267
 268         return err;
 269 }
 270
 271 /* Please, do not use this function outside from the map creation path
 272  * (e.g. in map update path) without taking care of setting the active
 273  * memory cgroup (see at bpf_map_kmalloc_node() for example).
 274  */
 275 static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
 276 {
 277         /* We really just want to fail instead of triggering OOM killer
 278          * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
 279          * which is used for lower order allocation requests.
 280          *
 281          * It has been observed that higher order allocation requests done by
 282          * vmalloc with __GFP_NORETRY being set might fail due to not trying
 283          * to reclaim memory from the page cache, thus we set
 284          * __GFP_RETRY_MAYFAIL to avoid such situations.
 285          */
 286
 287         gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO);
 288         unsigned int flags = 0;
 289         unsigned long align = 1;
 290         void *area;
 291
 292         if (size >= SIZE_MAX)
 293                 return NULL;
 294
 295         /* kmalloc()'ed memory can't be mmap()'ed */
 296         if (mmapable) {
 297                 BUG_ON(!PAGE_ALIGNED(size));
 298                 align = SHMLBA;
 299                 flags = VM_USERMAP;
 300         } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 301                 area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
 302                                     numa_node);
 303                 if (area != NULL)
 304                         return area;
 305         }
 306
 307         return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
 308                         gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
 309                         flags, numa_node, __builtin_return_address(0));
 310 }
 311
 312 void *bpf_map_area_alloc(u64 size, int numa_node)
 313 {
 314         return __bpf_map_area_alloc(size, numa_node, false);
 315 }
 316
 317 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
 318 {
 319         return __bpf_map_area_alloc(size, numa_node, true);
 320 }
 321
 322 void bpf_map_area_free(void *area)
 323 {
 324         kvfree(area);
 325 }
 326
 327 static u32 bpf_map_flags_retain_permanent(u32 flags)
 328 {
 329         /* Some map creation flags are not tied to the map object but
 330          * rather to the map fd instead, so they have no meaning upon
 331          * map object inspection since multiple file descriptors with
 332          * different (access) properties can exist here. Thus, given
 333          * this has zero meaning for the map itself, lets clear these
 334          * from here.
 335          */
 336         return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
 337 }
 338
 339 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
 340 {
 341         map->map_type = attr->map_type;
 342         map->key_size = attr->key_size;
 343         map->value_size = attr->value_size;
 344         map->max_entries = attr->max_entries;
 345         map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
 346         map->numa_node = bpf_map_attr_numa_node(attr);
 347         map->map_extra = attr->map_extra;
 348 }
 349
 350 static int bpf_map_alloc_id(struct bpf_map *map)
 351 {
 352         int id;
 353
 354         idr_preload(GFP_KERNEL);
 355         spin_lock_bh(&map_idr_lock);
 356         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 357         if (id > 0)
 358                 map->id = id;
 359         spin_unlock_bh(&map_idr_lock);
 360         idr_preload_end();
 361
 362         if (WARN_ON_ONCE(!id))
 363                 return -ENOSPC;
 364
 365         return id > 0 ? 0 : id;
 366 }
 367
 368 void bpf_map_free_id(struct bpf_map *map)
 369 {
 370         unsigned long flags;
 371
 372         /* Offloaded maps are removed from the IDR store when their device
 373          * disappears - even if someone holds an fd to them they are unusable,
 374          * the memory is gone, all ops will fail; they are simply waiting for
 375          * refcnt to drop to be freed.
 376          */
 377         if (!map->id)
 378                 return;
 379
 380         spin_lock_irqsave(&map_idr_lock, flags);
 381
 382         idr_remove(&map_idr, map->id);
 383         map->id = 0;
 384
 385         spin_unlock_irqrestore(&map_idr_lock, flags);
 386 }
 387
 388 #ifdef CONFIG_MEMCG_KMEM
 389 static void bpf_map_save_memcg(struct bpf_map *map)
 390 {
 391         /* Currently if a map is created by a process belonging to the root
 392          * memory cgroup, get_obj_cgroup_from_current() will return NULL.
 393          * So we have to check map->objcg for being NULL each time it's
 394          * being used.
 395          */
 396         if (memcg_bpf_enabled())
 397                 map->objcg = get_obj_cgroup_from_current();
 398 }
 399
 400 static void bpf_map_release_memcg(struct bpf_map *map)
 401 {
 402         if (map->objcg)
 403                 obj_cgroup_put(map->objcg);
 404 }
 405
 406 static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map)
 407 {
 408         if (map->objcg)
 409                 return get_mem_cgroup_from_objcg(map->objcg);
 410
 411         return root_mem_cgroup;
 412 }
 413
 414 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
 415                            int node)
 416 {
 417         struct mem_cgroup *memcg, *old_memcg;
 418         void *ptr;
 419
 420         memcg = bpf_map_get_memcg(map);
 421         old_memcg = set_active_memcg(memcg);
 422         ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
 423         set_active_memcg(old_memcg);
 424         mem_cgroup_put(memcg);
 425
 426         return ptr;
 427 }
 428
 429 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
 430 {
 431         struct mem_cgroup *memcg, *old_memcg;
 432         void *ptr;
 433
 434         memcg = bpf_map_get_memcg(map);
 435         old_memcg = set_active_memcg(memcg);
 436         ptr = kzalloc(size, flags | __GFP_ACCOUNT);
 437         set_active_memcg(old_memcg);
 438         mem_cgroup_put(memcg);
 439
 440         return ptr;
 441 }
 442
 443 void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
 444                        gfp_t flags)
 445 {
 446         struct mem_cgroup *memcg, *old_memcg;
 447         void *ptr;
 448
 449         memcg = bpf_map_get_memcg(map);
 450         old_memcg = set_active_memcg(memcg);
 451         ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);
 452         set_active_memcg(old_memcg);
 453         mem_cgroup_put(memcg);
 454
 455         return ptr;
 456 }
 457
 458 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
 459                                     size_t align, gfp_t flags)
 460 {
 461         struct mem_cgroup *memcg, *old_memcg;
 462         void __percpu *ptr;
 463
 464         memcg = bpf_map_get_memcg(map);
 465         old_memcg = set_active_memcg(memcg);
 466         ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
 467         set_active_memcg(old_memcg);
 468         mem_cgroup_put(memcg);
 469
 470         return ptr;
 471 }
 472
 473 #else
 474 static void bpf_map_save_memcg(struct bpf_map *map)
 475 {
 476 }
 477
 478 static void bpf_map_release_memcg(struct bpf_map *map)
 479 {
 480 }
 481 #endif
 482
 483 static int btf_field_cmp(const void *a, const void *b)
 484 {
 485         const struct btf_field *f1 = a, *f2 = b;
 486
 487         if (f1->offset < f2->offset)
 488                 return -1;
 489         else if (f1->offset > f2->offset)
 490                 return 1;
 491         return 0;
 492 }
 493
 494 struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset,
 495                                   u32 field_mask)
 496 {
 497         struct btf_field *field;
 498
 499         if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask))
 500                 return NULL;
 501         field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp);
 502         if (!field || !(field->type & field_mask))
 503                 return NULL;
 504         return field;
 505 }
 506
 507 void btf_record_free(struct btf_record *rec)
 508 {
 509         int i;
 510
 511         if (IS_ERR_OR_NULL(rec))
 512                 return;
 513         for (i = 0; i < rec->cnt; i++) {
 514                 switch (rec->fields[i].type) {
 515                 case BPF_KPTR_UNREF:
 516                 case BPF_KPTR_REF:
 517                         if (rec->fields[i].kptr.module)
 518                                 module_put(rec->fields[i].kptr.module);
 519                         btf_put(rec->fields[i].kptr.btf);
 520                         break;
 521                 case BPF_LIST_HEAD:
 522                 case BPF_LIST_NODE:
 523                 case BPF_RB_ROOT:
 524                 case BPF_RB_NODE:
 525                 case BPF_SPIN_LOCK:
 526                 case BPF_TIMER:
 527                 case BPF_REFCOUNT:
 528                         /* Nothing to release */
 529                         break;
 530                 default:
 531                         WARN_ON_ONCE(1);
 532                         continue;
 533                 }
 534         }
 535         kfree(rec);
 536 }
 537
 538 void bpf_map_free_record(struct bpf_map *map)
 539 {
 540         btf_record_free(map->record);
 541         map->record = NULL;
 542 }
 543
 544 struct btf_record *btf_record_dup(const struct btf_record *rec)
 545 {
 546         const struct btf_field *fields;
 547         struct btf_record *new_rec;
 548         int ret, size, i;
 549
 550         if (IS_ERR_OR_NULL(rec))
 551                 return NULL;
 552         size = offsetof(struct btf_record, fields[rec->cnt]);
 553         new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN);
 554         if (!new_rec)
 555                 return ERR_PTR(-ENOMEM);
 556         /* Do a deep copy of the btf_record */
 557         fields = rec->fields;
 558         new_rec->cnt = 0;
 559         for (i = 0; i < rec->cnt; i++) {
 560                 switch (fields[i].type) {
 561                 case BPF_KPTR_UNREF:
 562                 case BPF_KPTR_REF:
 563                         btf_get(fields[i].kptr.btf);
 564                         if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
 565                                 ret = -ENXIO;
 566                                 goto free;
 567                         }
 568                         break;
 569                 case BPF_LIST_HEAD:
 570                 case BPF_LIST_NODE:
 571                 case BPF_RB_ROOT:
 572                 case BPF_RB_NODE:
 573                 case BPF_SPIN_LOCK:
 574                 case BPF_TIMER:
 575                 case BPF_REFCOUNT:
 576                         /* Nothing to acquire */
 577                         break;
 578                 default:
 579                         ret = -EFAULT;
 580                         WARN_ON_ONCE(1);
 581                         goto free;
 582                 }
 583                 new_rec->cnt++;
 584         }
 585         return new_rec;
 586 free:
 587         btf_record_free(new_rec);
 588         return ERR_PTR(ret);
 589 }
 590
 591 bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b)
 592 {
 593         bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b);
 594         int size;
 595
 596         if (!a_has_fields && !b_has_fields)
 597                 return true;
 598         if (a_has_fields != b_has_fields)
 599                 return false;
 600         if (rec_a->cnt != rec_b->cnt)
 601                 return false;
 602         size = offsetof(struct btf_record, fields[rec_a->cnt]);
 603         /* btf_parse_fields uses kzalloc to allocate a btf_record, so unused
 604          * members are zeroed out. So memcmp is safe to do without worrying
 605          * about padding/unused fields.
 606          *
 607          * While spin_lock, timer, and kptr have no relation to map BTF,
 608          * list_head metadata is specific to map BTF, the btf and value_rec
 609          * members in particular. btf is the map BTF, while value_rec points to
 610          * btf_record in that map BTF.
 611          *
 612          * So while by default, we don't rely on the map BTF (which the records
 613          * were parsed from) matching for both records, which is not backwards
 614          * compatible, in case list_head is part of it, we implicitly rely on
 615          * that by way of depending on memcmp succeeding for it.
 616          */
 617         return !memcmp(rec_a, rec_b, size);
 618 }
 619
 620 void bpf_obj_free_timer(const struct btf_record *rec, void *obj)
 621 {
 622         if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER)))
 623                 return;
 624         bpf_timer_cancel_and_free(obj + rec->timer_off);
 625 }
 626
 627 extern void __bpf_obj_drop_impl(void *p, const struct btf_record *rec);
 628
 629 void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
 630 {
 631         const struct btf_field *fields;
 632         int i;
 633
 634         if (IS_ERR_OR_NULL(rec))
 635                 return;
 636         fields = rec->fields;
 637         for (i = 0; i < rec->cnt; i++) {
 638                 struct btf_struct_meta *pointee_struct_meta;
 639                 const struct btf_field *field = &fields[i];
 640                 void *field_ptr = obj + field->offset;
 641                 void *xchgd_field;
 642
 643                 switch (fields[i].type) {
 644                 case BPF_SPIN_LOCK:
 645                         break;
 646                 case BPF_TIMER:
 647                         bpf_timer_cancel_and_free(field_ptr);
 648                         break;
 649                 case BPF_KPTR_UNREF:
 650                         WRITE_ONCE(*(u64 *)field_ptr, 0);
 651                         break;
 652                 case BPF_KPTR_REF:
 653                         xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);
 654                         if (!xchgd_field)
 655                                 break;
 656
 657                         if (!btf_is_kernel(field->kptr.btf)) {
 658                                 pointee_struct_meta = btf_find_struct_meta(field->kptr.btf,
 659                                                                            field->kptr.btf_id);
 660                                 migrate_disable();
 661                                 __bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ?
 662                                                                  pointee_struct_meta->record :
 663                                                                  NULL);
 664                                 migrate_enable();
 665                         } else {
 666                                 field->kptr.dtor(xchgd_field);
 667                         }
 668                         break;
 669                 case BPF_LIST_HEAD:
 670                         if (WARN_ON_ONCE(rec->spin_lock_off < 0))
 671                                 continue;
 672                         bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off);
 673                         break;
 674                 case BPF_RB_ROOT:
 675                         if (WARN_ON_ONCE(rec->spin_lock_off < 0))
 676                                 continue;
 677                         bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off);
 678                         break;
 679                 case BPF_LIST_NODE:
 680                 case BPF_RB_NODE:
 681                 case BPF_REFCOUNT:
 682                         break;
 683                 default:
 684                         WARN_ON_ONCE(1);
 685                         continue;
 686                 }
 687         }
 688 }
 689
 690 /* called from workqueue */
 691 static void bpf_map_free_deferred(struct work_struct *work)
 692 {
 693         struct bpf_map *map = container_of(work, struct bpf_map, work);
 694         struct btf_record *rec = map->record;
 695         struct btf *btf = map->btf;
 696
 697         security_bpf_map_free(map);
 698         bpf_map_release_memcg(map);
 699         /* implementation dependent freeing */
 700         map->ops->map_free(map);
 701         /* Delay freeing of btf_record for maps, as map_free
 702          * callback usually needs access to them. It is better to do it here
 703          * than require each callback to do the free itself manually.
 704          *
 705          * Note that the btf_record stashed in map->inner_map_meta->record was
 706          * already freed using the map_free callback for map in map case which
 707          * eventually calls bpf_map_free_meta, since inner_map_meta is only a
 708          * template bpf_map struct used during verification.
 709          */
 710         btf_record_free(rec);
 711         /* Delay freeing of btf for maps, as map_free callback may need
 712          * struct_meta info which will be freed with btf_put().
 713          */
 714         btf_put(btf);
 715 }
 716
 717 static void bpf_map_put_uref(struct bpf_map *map)
 718 {
 719         if (atomic64_dec_and_test(&map->usercnt)) {
 720                 if (map->ops->map_release_uref)
 721                         map->ops->map_release_uref(map);
 722         }
 723 }
 724
 725 static void bpf_map_free_in_work(struct bpf_map *map)
 726 {
 727         INIT_WORK(&map->work, bpf_map_free_deferred);
 728         /* Avoid spawning kworkers, since they all might contend
 729          * for the same mutex like slab_mutex.
 730          */
 731         queue_work(system_unbound_wq, &map->work);
 732 }
 733
 734 static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
 735 {
 736         bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
 737 }
 738
 739 static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)
 740 {
 741         if (rcu_trace_implies_rcu_gp())
 742                 bpf_map_free_rcu_gp(rcu);
 743         else
 744                 call_rcu(rcu, bpf_map_free_rcu_gp);
 745 }
 746
 747 /* decrement map refcnt and schedule it for freeing via workqueue
 748  * (underlying map implementation ops->map_free() might sleep)
 749  */
 750 void bpf_map_put(struct bpf_map *map)
 751 {
 752         if (atomic64_dec_and_test(&map->refcnt)) {
 753                 /* bpf_map_free_id() must be called first */
 754                 bpf_map_free_id(map);
 755
 756                 if (READ_ONCE(map->free_after_mult_rcu_gp))
 757                         call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);
 758                 else
 759                         bpf_map_free_in_work(map);
 760         }
 761 }
 762 EXPORT_SYMBOL_GPL(bpf_map_put);
 763
 764 void bpf_map_put_with_uref(struct bpf_map *map)
 765 {
 766         bpf_map_put_uref(map);
 767         bpf_map_put(map);
 768 }
 769
 770 static int bpf_map_release(struct inode *inode, struct file *filp)
 771 {
 772         struct bpf_map *map = filp->private_data;
 773
 774         if (map->ops->map_release)
 775                 map->ops->map_release(map, filp);
 776
 777         bpf_map_put_with_uref(map);
 778         return 0;
 779 }
 780
 781 static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
 782 {
 783         fmode_t mode = f.file->f_mode;
 784
 785         /* Our file permissions may have been overridden by global
 786          * map permissions facing syscall side.
 787          */
 788         if (READ_ONCE(map->frozen))
 789                 mode &= ~FMODE_CAN_WRITE;
 790         return mode;
 791 }
 792
 793 #ifdef CONFIG_PROC_FS
 794 /* Show the memory usage of a bpf map */
 795 static u64 bpf_map_memory_usage(const struct bpf_map *map)
 796 {
 797         return map->ops->map_mem_usage(map);
 798 }
 799
 800 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 801 {
 802         struct bpf_map *map = filp->private_data;
 803         u32 type = 0, jited = 0;
 804
 805         if (map_type_contains_progs(map)) {
 806                 spin_lock(&map->owner.lock);
 807                 type  = map->owner.type;
 808                 jited = map->owner.jited;
 809                 spin_unlock(&map->owner.lock);
 810         }
 811
 812         seq_printf(m,
 813                    "map_type:\t%u\n"
 814                    "key_size:\t%u\n"
 815                    "value_size:\t%u\n"
 816                    "max_entries:\t%u\n"
 817                    "map_flags:\t%#x\n"
 818                    "map_extra:\t%#llx\n"
 819                    "memlock:\t%llu\n"
 820                    "map_id:\t%u\n"
 821                    "frozen:\t%u\n",
 822                    map->map_type,
 823                    map->key_size,
 824                    map->value_size,
 825                    map->max_entries,
 826                    map->map_flags,
 827                    (unsigned long long)map->map_extra,
 828                    bpf_map_memory_usage(map),
 829                    map->id,
 830                    READ_ONCE(map->frozen));
 831         if (type) {
 832                 seq_printf(m, "owner_prog_type:\t%u\n", type);
 833                 seq_printf(m, "owner_jited:\t%u\n", jited);
 834         }
 835 }
 836 #endif
 837
 838 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
 839                               loff_t *ppos)
 840 {
 841         /* We need this handler such that alloc_file() enables
 842          * f_mode with FMODE_CAN_READ.
 843          */
 844         return -EINVAL;
 845 }
 846
 847 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
 848                                size_t siz, loff_t *ppos)
 849 {
 850         /* We need this handler such that alloc_file() enables
 851          * f_mode with FMODE_CAN_WRITE.
 852          */
 853         return -EINVAL;
 854 }
 855
 856 /* called for any extra memory-mapped regions (except initial) */
 857 static void bpf_map_mmap_open(struct vm_area_struct *vma)
 858 {
 859         struct bpf_map *map = vma->vm_file->private_data;
 860
 861         if (vma->vm_flags & VM_MAYWRITE)
 862                 bpf_map_write_active_inc(map);
 863 }
 864
 865 /* called for all unmapped memory region (including initial) */
 866 static void bpf_map_mmap_close(struct vm_area_struct *vma)
 867 {
 868         struct bpf_map *map = vma->vm_file->private_data;
 869
 870         if (vma->vm_flags & VM_MAYWRITE)
 871                 bpf_map_write_active_dec(map);
 872 }
 873
 874 static const struct vm_operations_struct bpf_map_default_vmops = {
 875         .open           = bpf_map_mmap_open,
 876         .close          = bpf_map_mmap_close,
 877 };
 878
 879 static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
 880 {
 881         struct bpf_map *map = filp->private_data;
 882         int err;
 883
 884         if (!map->ops->map_mmap || !IS_ERR_OR_NULL(map->record))
 885                 return -ENOTSUPP;
 886
 887         if (!(vma->vm_flags & VM_SHARED))
 888                 return -EINVAL;
 889
 890         mutex_lock(&map->freeze_mutex);
 891
 892         if (vma->vm_flags & VM_WRITE) {
 893                 if (map->frozen) {
 894                         err = -EPERM;
 895                         goto out;
 896                 }
 897                 /* map is meant to be read-only, so do not allow mapping as
 898                  * writable, because it's possible to leak a writable page
 899                  * reference and allows user-space to still modify it after
 900                  * freezing, while verifier will assume contents do not change
 901                  */
 902                 if (map->map_flags & BPF_F_RDONLY_PROG) {
 903                         err = -EACCES;
 904                         goto out;
 905                 }
 906         }
 907
 908         /* set default open/close callbacks */
 909         vma->vm_ops = &bpf_map_default_vmops;
 910         vma->vm_private_data = map;
 911         vm_flags_clear(vma, VM_MAYEXEC);
 912         if (!(vma->vm_flags & VM_WRITE))
 913                 /* disallow re-mapping with PROT_WRITE */
 914                 vm_flags_clear(vma, VM_MAYWRITE);
 915
 916         err = map->ops->map_mmap(map, vma);
 917         if (err)
 918                 goto out;
 919
 920         if (vma->vm_flags & VM_MAYWRITE)
 921                 bpf_map_write_active_inc(map);
 922 out:
 923         mutex_unlock(&map->freeze_mutex);
 924         return err;
 925 }
 926
 927 static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
 928 {
 929         struct bpf_map *map = filp->private_data;
 930
 931         if (map->ops->map_poll)
 932                 return map->ops->map_poll(map, filp, pts);
 933
 934         return EPOLLERR;
 935 }
 936
 937 const struct file_operations bpf_map_fops = {
 938 #ifdef CONFIG_PROC_FS
 939         .show_fdinfo    = bpf_map_show_fdinfo,
 940 #endif
 941         .release        = bpf_map_release,
 942         .read           = bpf_dummy_read,
 943         .write          = bpf_dummy_write,
 944         .mmap           = bpf_map_mmap,
 945         .poll           = bpf_map_poll,
 946 };
 947
 948 int bpf_map_new_fd(struct bpf_map *map, int flags)
 949 {
 950         int ret;
 951
 952         ret = security_bpf_map(map, OPEN_FMODE(flags));
 953         if (ret < 0)
 954                 return ret;
 955
 956         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 957                                 flags | O_CLOEXEC);
 958 }
 959
 960 int bpf_get_file_flag(int flags)
 961 {
 962         if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
 963                 return -EINVAL;
 964         if (flags & BPF_F_RDONLY)
 965                 return O_RDONLY;
 966         if (flags & BPF_F_WRONLY)
 967                 return O_WRONLY;
 968         return O_RDWR;
 969 }
 970
 971 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 972 #define CHECK_ATTR(CMD) \
 973         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 974                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 975                    sizeof(*attr) - \
 976                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 977                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 978
 979 /* dst and src must have at least "size" number of bytes.
 980  * Return strlen on success and < 0 on error.
 981  */
 982 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
 983 {
 984         const char *end = src + size;
 985         const char *orig_src = src;
 986
 987         memset(dst, 0, size);
 988         /* Copy all isalnum(), '_' and '.' chars. */
 989         while (src < end && *src) {
 990                 if (!isalnum(*src) &&
 991                     *src != '_' && *src != '.')
 992                         return -EINVAL;
 993                 *dst++ = *src++;
 994         }
 995
 996         /* No '\0' found in "size" number of bytes */
 997         if (src == end)
 998                 return -EINVAL;
 999
1000         return src - orig_src;
1001 }
1002
1003 int map_check_no_btf(const struct bpf_map *map,
1004                      const struct btf *btf,
1005                      const struct btf_type *key_type,
1006                      const struct btf_type *value_type)
1007 {
1008         return -ENOTSUPP;
1009 }
1010
1011 static int map_check_btf(struct bpf_map *map, const struct btf *btf,
1012                          u32 btf_key_id, u32 btf_value_id)
1013 {
1014         const struct btf_type *key_type, *value_type;
1015         u32 key_size, value_size;
1016         int ret = 0;
1017
1018         /* Some maps allow key to be unspecified. */
1019         if (btf_key_id) {
1020                 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
1021                 if (!key_type || key_size != map->key_size)
1022                         return -EINVAL;
1023         } else {
1024                 key_type = btf_type_by_id(btf, 0);
1025                 if (!map->ops->map_check_btf)
1026                         return -EINVAL;
1027         }
1028
1029         value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
1030         if (!value_type || value_size != map->value_size)
1031                 return -EINVAL;
1032
1033         map->record = btf_parse_fields(btf, value_type,
1034                                        BPF_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
1035                                        BPF_RB_ROOT | BPF_REFCOUNT,
1036                                        map->value_size);
1037         if (!IS_ERR_OR_NULL(map->record)) {
1038                 int i;
1039
1040                 if (!bpf_capable()) {
1041                         ret = -EPERM;
1042                         goto free_map_tab;
1043                 }
1044                 if (map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) {
1045                         ret = -EACCES;
1046                         goto free_map_tab;
1047                 }
1048                 for (i = 0; i < sizeof(map->record->field_mask) * 8; i++) {
1049                         switch (map->record->field_mask & (1 << i)) {
1050                         case 0:
1051                                 continue;
1052                         case BPF_SPIN_LOCK:
1053                                 if (map->map_type != BPF_MAP_TYPE_HASH &&
1054                                     map->map_type != BPF_MAP_TYPE_ARRAY &&
1055                                     map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
1056                                     map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1057                                     map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1058                                     map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1059                                     map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1060                                         ret = -EOPNOTSUPP;
1061                                         goto free_map_tab;
1062                                 }
1063                                 break;
1064                         case BPF_TIMER:
1065                                 if (map->map_type != BPF_MAP_TYPE_HASH &&
1066                                     map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1067                                     map->map_type != BPF_MAP_TYPE_ARRAY) {
1068                                         ret = -EOPNOTSUPP;
1069                                         goto free_map_tab;
1070                                 }
1071                                 break;
1072                         case BPF_KPTR_UNREF:
1073                         case BPF_KPTR_REF:
1074                         case BPF_REFCOUNT:
1075                                 if (map->map_type != BPF_MAP_TYPE_HASH &&
1076                                     map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
1077                                     map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1078                                     map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH &&
1079                                     map->map_type != BPF_MAP_TYPE_ARRAY &&
1080                                     map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
1081                                     map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1082                                     map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1083                                     map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1084                                     map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1085                                         ret = -EOPNOTSUPP;
1086                                         goto free_map_tab;
1087                                 }
1088                                 break;
1089                         case BPF_LIST_HEAD:
1090                         case BPF_RB_ROOT:
1091                                 if (map->map_type != BPF_MAP_TYPE_HASH &&
1092                                     map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1093                                     map->map_type != BPF_MAP_TYPE_ARRAY) {
1094                                         ret = -EOPNOTSUPP;
1095                                         goto free_map_tab;
1096                                 }
1097                                 break;
1098                         default:
1099                                 /* Fail if map_type checks are missing for a field type */
1100                                 ret = -EOPNOTSUPP;
1101                                 goto free_map_tab;
1102                         }
1103                 }
1104         }
1105
1106         ret = btf_check_and_fixup_fields(btf, map->record);
1107         if (ret < 0)
1108                 goto free_map_tab;
1109
1110         if (map->ops->map_check_btf) {
1111                 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
1112                 if (ret < 0)
1113                         goto free_map_tab;
1114         }
1115
1116         return ret;
1117 free_map_tab:
1118         bpf_map_free_record(map);
1119         return ret;
1120 }
1121
1122 #define BPF_MAP_CREATE_LAST_FIELD map_extra
1123 /* called via syscall */
1124 static int map_create(union bpf_attr *attr)
1125 {
1126         const struct bpf_map_ops *ops;
1127         int numa_node = bpf_map_attr_numa_node(attr);
1128         u32 map_type = attr->map_type;
1129         struct bpf_map *map;
1130         int f_flags;
1131         int err;
1132
1133         err = CHECK_ATTR(BPF_MAP_CREATE);
1134         if (err)
1135                 return -EINVAL;
1136
1137         if (attr->btf_vmlinux_value_type_id) {
1138                 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
1139                     attr->btf_key_type_id || attr->btf_value_type_id)
1140                         return -EINVAL;
1141         } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
1142                 return -EINVAL;
1143         }
1144
1145         if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER &&
1146             attr->map_extra != 0)
1147                 return -EINVAL;
1148
1149         f_flags = bpf_get_file_flag(attr->map_flags);
1150         if (f_flags < 0)
1151                 return f_flags;
1152
1153         if (numa_node != NUMA_NO_NODE &&
1154             ((unsigned int)numa_node >= nr_node_ids ||
1155              !node_online(numa_node)))
1156                 return -EINVAL;
1157
1158         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
1159         map_type = attr->map_type;
1160         if (map_type >= ARRAY_SIZE(bpf_map_types))
1161                 return -EINVAL;
1162         map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types));
1163         ops = bpf_map_types[map_type];
1164         if (!ops)
1165                 return -EINVAL;
1166
1167         if (ops->map_alloc_check) {
1168                 err = ops->map_alloc_check(attr);
1169                 if (err)
1170                         return err;
1171         }
1172         if (attr->map_ifindex)
1173                 ops = &bpf_map_offload_ops;
1174         if (!ops->map_mem_usage)
1175                 return -EINVAL;
1176
1177         /* Intent here is for unprivileged_bpf_disabled to block BPF map
1178          * creation for unprivileged users; other actions depend
1179          * on fd availability and access to bpffs, so are dependent on
1180          * object creation success. Even with unprivileged BPF disabled,
1181          * capability checks are still carried out.
1182          */
1183         if (sysctl_unprivileged_bpf_disabled && !bpf_capable())
1184                 return -EPERM;
1185
1186         /* check privileged map type permissions */
1187         switch (map_type) {
1188         case BPF_MAP_TYPE_ARRAY:
1189         case BPF_MAP_TYPE_PERCPU_ARRAY:
1190         case BPF_MAP_TYPE_PROG_ARRAY:
1191         case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1192         case BPF_MAP_TYPE_CGROUP_ARRAY:
1193         case BPF_MAP_TYPE_ARRAY_OF_MAPS:
1194         case BPF_MAP_TYPE_HASH:
1195         case BPF_MAP_TYPE_PERCPU_HASH:
1196         case BPF_MAP_TYPE_HASH_OF_MAPS:
1197         case BPF_MAP_TYPE_RINGBUF:
1198         case BPF_MAP_TYPE_USER_RINGBUF:
1199         case BPF_MAP_TYPE_CGROUP_STORAGE:
1200         case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
1201                 /* unprivileged */
1202                 break;
1203         case BPF_MAP_TYPE_SK_STORAGE:
1204         case BPF_MAP_TYPE_INODE_STORAGE:
1205         case BPF_MAP_TYPE_TASK_STORAGE:
1206         case BPF_MAP_TYPE_CGRP_STORAGE:
1207         case BPF_MAP_TYPE_BLOOM_FILTER:
1208         case BPF_MAP_TYPE_LPM_TRIE:
1209         case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
1210         case BPF_MAP_TYPE_STACK_TRACE:
1211         case BPF_MAP_TYPE_QUEUE:
1212         case BPF_MAP_TYPE_STACK:
1213         case BPF_MAP_TYPE_LRU_HASH:
1214         case BPF_MAP_TYPE_LRU_PERCPU_HASH:
1215         case BPF_MAP_TYPE_STRUCT_OPS:
1216         case BPF_MAP_TYPE_CPUMAP:
1217                 if (!bpf_capable())
1218                         return -EPERM;
1219                 break;
1220         case BPF_MAP_TYPE_SOCKMAP:
1221         case BPF_MAP_TYPE_SOCKHASH:
1222         case BPF_MAP_TYPE_DEVMAP:
1223         case BPF_MAP_TYPE_DEVMAP_HASH:
1224         case BPF_MAP_TYPE_XSKMAP:
1225                 if (!capable(CAP_NET_ADMIN))
1226                         return -EPERM;
1227                 break;
1228         default:
1229                 WARN(1, "unsupported map type %d", map_type);
1230                 return -EPERM;
1231         }
1232
1233         map = ops->map_alloc(attr);
1234         if (IS_ERR(map))
1235                 return PTR_ERR(map);
1236         map->ops = ops;
1237         map->map_type = map_type;
1238
1239         err = bpf_obj_name_cpy(map->name, attr->map_name,
1240                                sizeof(attr->map_name));
1241         if (err < 0)
1242                 goto free_map;
1243
1244         atomic64_set(&map->refcnt, 1);
1245         atomic64_set(&map->usercnt, 1);
1246         mutex_init(&map->freeze_mutex);
1247         spin_lock_init(&map->owner.lock);
1248
1249         if (attr->btf_key_type_id || attr->btf_value_type_id ||
1250             /* Even the map's value is a kernel's struct,
1251              * the bpf_prog.o must have BTF to begin with
1252              * to figure out the corresponding kernel's
1253              * counter part.  Thus, attr->btf_fd has
1254              * to be valid also.
1255              */
1256             attr->btf_vmlinux_value_type_id) {
1257                 struct btf *btf;
1258
1259                 btf = btf_get_by_fd(attr->btf_fd);
1260                 if (IS_ERR(btf)) {
1261                         err = PTR_ERR(btf);
1262                         goto free_map;
1263                 }
1264                 if (btf_is_kernel(btf)) {
1265                         btf_put(btf);
1266                         err = -EACCES;
1267                         goto free_map;
1268                 }
1269                 map->btf = btf;
1270
1271                 if (attr->btf_value_type_id) {
1272                         err = map_check_btf(map, btf, attr->btf_key_type_id,
1273                                             attr->btf_value_type_id);
1274                         if (err)
1275                                 goto free_map;
1276                 }
1277
1278                 map->btf_key_type_id = attr->btf_key_type_id;
1279                 map->btf_value_type_id = attr->btf_value_type_id;
1280                 map->btf_vmlinux_value_type_id =
1281                         attr->btf_vmlinux_value_type_id;
1282         }
1283
1284         err = security_bpf_map_alloc(map);
1285         if (err)
1286                 goto free_map;
1287
1288         err = bpf_map_alloc_id(map);
1289         if (err)
1290                 goto free_map_sec;
1291
1292         bpf_map_save_memcg(map);
1293
1294         err = bpf_map_new_fd(map, f_flags);
1295         if (err < 0) {
1296                 /* failed to allocate fd.
1297                  * bpf_map_put_with_uref() is needed because the above
1298                  * bpf_map_alloc_id() has published the map
1299                  * to the userspace and the userspace may
1300                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
1301                  */
1302                 bpf_map_put_with_uref(map);
1303                 return err;
1304         }
1305
1306         return err;
1307
1308 free_map_sec:
1309         security_bpf_map_free(map);
1310 free_map:
1311         btf_put(map->btf);
1312         map->ops->map_free(map);
1313         return err;
1314 }
1315
1316 /* if error is returned, fd is released.
1317  * On success caller should complete fd access with matching fdput()
1318  */
1319 struct bpf_map *__bpf_map_get(struct fd f)
1320 {
1321         if (!f.file)
1322                 return ERR_PTR(-EBADF);
1323         if (f.file->f_op != &bpf_map_fops) {
1324                 fdput(f);
1325                 return ERR_PTR(-EINVAL);
1326         }
1327
1328         return f.file->private_data;
1329 }
1330
1331 void bpf_map_inc(struct bpf_map *map)
1332 {
1333         atomic64_inc(&map->refcnt);
1334 }
1335 EXPORT_SYMBOL_GPL(bpf_map_inc);
1336
1337 void bpf_map_inc_with_uref(struct bpf_map *map)
1338 {
1339         atomic64_inc(&map->refcnt);
1340         atomic64_inc(&map->usercnt);
1341 }
1342 EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
1343
1344 struct bpf_map *bpf_map_get(u32 ufd)
1345 {
1346         struct fd f = fdget(ufd);
1347         struct bpf_map *map;
1348
1349         map = __bpf_map_get(f);
1350         if (IS_ERR(map))
1351                 return map;
1352
1353         bpf_map_inc(map);
1354         fdput(f);
1355
1356         return map;
1357 }
1358 EXPORT_SYMBOL(bpf_map_get);
1359
1360 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
1361 {
1362         struct fd f = fdget(ufd);
1363         struct bpf_map *map;
1364
1365         map = __bpf_map_get(f);
1366         if (IS_ERR(map))
1367                 return map;
1368
1369         bpf_map_inc_with_uref(map);
1370         fdput(f);
1371
1372         return map;
1373 }
1374
1375 /* map_idr_lock should have been held or the map should have been
1376  * protected by rcu read lock.
1377  */
1378 struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
1379 {
1380         int refold;
1381
1382         refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
1383         if (!refold)
1384                 return ERR_PTR(-ENOENT);
1385         if (uref)
1386                 atomic64_inc(&map->usercnt);
1387
1388         return map;
1389 }
1390
1391 struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
1392 {
1393         spin_lock_bh(&map_idr_lock);
1394         map = __bpf_map_inc_not_zero(map, false);
1395         spin_unlock_bh(&map_idr_lock);
1396
1397         return map;
1398 }
1399 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1400
1401 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
1402 {
1403         return -ENOTSUPP;
1404 }
1405
1406 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
1407 {
1408         if (key_size)
1409                 return vmemdup_user(ukey, key_size);
1410
1411         if (ukey)
1412                 return ERR_PTR(-EINVAL);
1413
1414         return NULL;
1415 }
1416
1417 static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
1418 {
1419         if (key_size)
1420                 return kvmemdup_bpfptr(ukey, key_size);
1421
1422         if (!bpfptr_is_null(ukey))
1423                 return ERR_PTR(-EINVAL);
1424
1425         return NULL;
1426 }
1427
1428 /* last field in 'union bpf_attr' used by this command */
1429 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1430
1431 static int map_lookup_elem(union bpf_attr *attr)
1432 {
1433         void __user *ukey = u64_to_user_ptr(attr->key);
1434         void __user *uvalue = u64_to_user_ptr(attr->value);
1435         int ufd = attr->map_fd;
1436         struct bpf_map *map;
1437         void *key, *value;
1438         u32 value_size;
1439         struct fd f;
1440         int err;
1441
1442         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1443                 return -EINVAL;
1444
1445         if (attr->flags & ~BPF_F_LOCK)
1446                 return -EINVAL;
1447
1448         f = fdget(ufd);
1449         map = __bpf_map_get(f);
1450         if (IS_ERR(map))
1451                 return PTR_ERR(map);
1452         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1453                 err = -EPERM;
1454                 goto err_put;
1455         }
1456
1457         if ((attr->flags & BPF_F_LOCK) &&
1458             !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1459                 err = -EINVAL;
1460                 goto err_put;
1461         }
1462
1463         key = __bpf_copy_key(ukey, map->key_size);
1464         if (IS_ERR(key)) {
1465                 err = PTR_ERR(key);
1466                 goto err_put;
1467         }
1468
1469         value_size = bpf_map_value_size(map);
1470
1471         err = -ENOMEM;
1472         value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1473         if (!value)
1474                 goto free_key;
1475
1476         if (map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
1477                 if (copy_from_user(value, uvalue, value_size))
1478                         err = -EFAULT;
1479                 else
1480                         err = bpf_map_copy_value(map, key, value, attr->flags);
1481                 goto free_value;
1482         }
1483
1484         err = bpf_map_copy_value(map, key, value, attr->flags);
1485         if (err)
1486                 goto free_value;
1487
1488         err = -EFAULT;
1489         if (copy_to_user(uvalue, value, value_size) != 0)
1490                 goto free_value;
1491
1492         err = 0;
1493
1494 free_value:
1495         kvfree(value);
1496 free_key:
1497         kvfree(key);
1498 err_put:
1499         fdput(f);
1500         return err;
1501 }
1502
1503
1504 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1505
1506 static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
1507 {
1508         bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1509         bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel);
1510         int ufd = attr->map_fd;
1511         struct bpf_map *map;
1512         void *key, *value;
1513         u32 value_size;
1514         struct fd f;
1515         int err;
1516
1517         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1518                 return -EINVAL;
1519
1520         f = fdget(ufd);
1521         map = __bpf_map_get(f);
1522         if (IS_ERR(map))
1523                 return PTR_ERR(map);
1524         bpf_map_write_active_inc(map);
1525         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1526                 err = -EPERM;
1527                 goto err_put;
1528         }
1529
1530         if ((attr->flags & BPF_F_LOCK) &&
1531             !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1532                 err = -EINVAL;
1533                 goto err_put;
1534         }
1535
1536         key = ___bpf_copy_key(ukey, map->key_size);
1537         if (IS_ERR(key)) {
1538                 err = PTR_ERR(key);
1539                 goto err_put;
1540         }
1541
1542         value_size = bpf_map_value_size(map);
1543         value = kvmemdup_bpfptr(uvalue, value_size);
1544         if (IS_ERR(value)) {
1545                 err = PTR_ERR(value);
1546                 goto free_key;
1547         }
1548
1549         err = bpf_map_update_value(map, f.file, key, value, attr->flags);
1550
1551         kvfree(value);
1552 free_key:
1553         kvfree(key);
1554 err_put:
1555         bpf_map_write_active_dec(map);
1556         fdput(f);
1557         return err;
1558 }
1559
1560 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1561
1562 static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr)
1563 {
1564         bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1565         int ufd = attr->map_fd;
1566         struct bpf_map *map;
1567         struct fd f;
1568         void *key;
1569         int err;
1570
1571         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1572                 return -EINVAL;
1573
1574         f = fdget(ufd);
1575         map = __bpf_map_get(f);
1576         if (IS_ERR(map))
1577                 return PTR_ERR(map);
1578         bpf_map_write_active_inc(map);
1579         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1580                 err = -EPERM;
1581                 goto err_put;
1582         }
1583
1584         key = ___bpf_copy_key(ukey, map->key_size);
1585         if (IS_ERR(key)) {
1586                 err = PTR_ERR(key);
1587                 goto err_put;
1588         }
1589
1590         if (bpf_map_is_offloaded(map)) {
1591                 err = bpf_map_offload_delete_elem(map, key);
1592                 goto out;
1593         } else if (IS_FD_PROG_ARRAY(map) ||
1594                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1595                 /* These maps require sleepable context */
1596                 err = map->ops->map_delete_elem(map, key);
1597                 goto out;
1598         }
1599
1600         bpf_disable_instrumentation();
1601         rcu_read_lock();
1602         err = map->ops->map_delete_elem(map, key);
1603         rcu_read_unlock();
1604         bpf_enable_instrumentation();
1605         maybe_wait_bpf_programs(map);
1606 out:
1607         kvfree(key);
1608 err_put:
1609         bpf_map_write_active_dec(map);
1610         fdput(f);
1611         return err;
1612 }
1613
1614 /* last field in 'union bpf_attr' used by this command */
1615 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1616
1617 static int map_get_next_key(union bpf_attr *attr)
1618 {
1619         void __user *ukey = u64_to_user_ptr(attr->key);
1620         void __user *unext_key = u64_to_user_ptr(attr->next_key);
1621         int ufd = attr->map_fd;
1622         struct bpf_map *map;
1623         void *key, *next_key;
1624         struct fd f;
1625         int err;
1626
1627         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1628                 return -EINVAL;
1629
1630         f = fdget(ufd);
1631         map = __bpf_map_get(f);
1632         if (IS_ERR(map))
1633                 return PTR_ERR(map);
1634         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1635                 err = -EPERM;
1636                 goto err_put;
1637         }
1638
1639         if (ukey) {
1640                 key = __bpf_copy_key(ukey, map->key_size);
1641                 if (IS_ERR(key)) {
1642                         err = PTR_ERR(key);
1643                         goto err_put;
1644                 }
1645         } else {
1646                 key = NULL;
1647         }
1648
1649         err = -ENOMEM;
1650         next_key = kvmalloc(map->key_size, GFP_USER);
1651         if (!next_key)
1652                 goto free_key;
1653
1654         if (bpf_map_is_offloaded(map)) {
1655                 err = bpf_map_offload_get_next_key(map, key, next_key);
1656                 goto out;
1657         }
1658
1659         rcu_read_lock();
1660         err = map->ops->map_get_next_key(map, key, next_key);
1661         rcu_read_unlock();
1662 out:
1663         if (err)
1664                 goto free_next_key;
1665
1666         err = -EFAULT;
1667         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1668                 goto free_next_key;
1669
1670         err = 0;
1671
1672 free_next_key:
1673         kvfree(next_key);
1674 free_key:
1675         kvfree(key);
1676 err_put:
1677         fdput(f);
1678         return err;
1679 }
1680
1681 int generic_map_delete_batch(struct bpf_map *map,
1682                              const union bpf_attr *attr,
1683                              union bpf_attr __user *uattr)
1684 {
1685         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1686         u32 cp, max_count;
1687         int err = 0;
1688         void *key;
1689
1690         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1691                 return -EINVAL;
1692
1693         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1694             !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1695                 return -EINVAL;
1696         }
1697
1698         max_count = attr->batch.count;
1699         if (!max_count)
1700                 return 0;
1701
1702         key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1703         if (!key)
1704                 return -ENOMEM;
1705
1706         for (cp = 0; cp < max_count; cp++) {
1707                 err = -EFAULT;
1708                 if (copy_from_user(key, keys + cp * map->key_size,
1709                                    map->key_size))
1710                         break;
1711
1712                 if (bpf_map_is_offloaded(map)) {
1713                         err = bpf_map_offload_delete_elem(map, key);
1714                         break;
1715                 }
1716
1717                 bpf_disable_instrumentation();
1718                 rcu_read_lock();
1719                 err = map->ops->map_delete_elem(map, key);
1720                 rcu_read_unlock();
1721                 bpf_enable_instrumentation();
1722                 if (err)
1723                         break;
1724                 cond_resched();
1725         }
1726         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1727                 err = -EFAULT;
1728
1729         kvfree(key);
1730
1731         maybe_wait_bpf_programs(map);
1732         return err;
1733 }
1734
1735 int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
1736                              const union bpf_attr *attr,
1737                              union bpf_attr __user *uattr)
1738 {
1739         void __user *values = u64_to_user_ptr(attr->batch.values);
1740         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1741         u32 value_size, cp, max_count;
1742         void *key, *value;
1743         int err = 0;
1744
1745         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1746                 return -EINVAL;
1747
1748         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1749             !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1750                 return -EINVAL;
1751         }
1752
1753         value_size = bpf_map_value_size(map);
1754
1755         max_count = attr->batch.count;
1756         if (!max_count)
1757                 return 0;
1758
1759         key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1760         if (!key)
1761                 return -ENOMEM;
1762
1763         value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1764         if (!value) {
1765                 kvfree(key);
1766                 return -ENOMEM;
1767         }
1768
1769         for (cp = 0; cp < max_count; cp++) {
1770                 err = -EFAULT;
1771                 if (copy_from_user(key, keys + cp * map->key_size,
1772                     map->key_size) ||
1773                     copy_from_user(value, values + cp * value_size, value_size))
1774                         break;
1775
1776                 err = bpf_map_update_value(map, map_file, key, value,
1777                                            attr->batch.elem_flags);
1778
1779                 if (err)
1780                         break;
1781                 cond_resched();
1782         }
1783
1784         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1785                 err = -EFAULT;
1786
1787         kvfree(value);
1788         kvfree(key);
1789         return err;
1790 }
1791
1792 #define MAP_LOOKUP_RETRIES 3
1793
1794 int generic_map_lookup_batch(struct bpf_map *map,
1795                                     const union bpf_attr *attr,
1796                                     union bpf_attr __user *uattr)
1797 {
1798         void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
1799         void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
1800         void __user *values = u64_to_user_ptr(attr->batch.values);
1801         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1802         void *buf, *buf_prevkey, *prev_key, *key, *value;
1803         int err, retry = MAP_LOOKUP_RETRIES;
1804         u32 value_size, cp, max_count;
1805
1806         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1807                 return -EINVAL;
1808
1809         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1810             !btf_record_has_field(map->record, BPF_SPIN_LOCK))
1811                 return -EINVAL;
1812
1813         value_size = bpf_map_value_size(map);
1814
1815         max_count = attr->batch.count;
1816         if (!max_count)
1817                 return 0;
1818
1819         if (put_user(0, &uattr->batch.count))
1820                 return -EFAULT;
1821
1822         buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1823         if (!buf_prevkey)
1824                 return -ENOMEM;
1825
1826         buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
1827         if (!buf) {
1828                 kvfree(buf_prevkey);
1829                 return -ENOMEM;
1830         }
1831
1832         err = -EFAULT;
1833         prev_key = NULL;
1834         if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
1835                 goto free_buf;
1836         key = buf;
1837         value = key + map->key_size;
1838         if (ubatch)
1839                 prev_key = buf_prevkey;
1840
1841         for (cp = 0; cp < max_count;) {
1842                 rcu_read_lock();
1843                 err = map->ops->map_get_next_key(map, prev_key, key);
1844                 rcu_read_unlock();
1845                 if (err)
1846                         break;
1847                 err = bpf_map_copy_value(map, key, value,
1848                                          attr->batch.elem_flags);
1849
1850                 if (err == -ENOENT) {
1851                         if (retry) {
1852                                 retry--;
1853                                 continue;
1854                         }
1855                         err = -EINTR;
1856                         break;
1857                 }
1858
1859                 if (err)
1860                         goto free_buf;
1861
1862                 if (copy_to_user(keys + cp * map->key_size, key,
1863                                  map->key_size)) {
1864                         err = -EFAULT;
1865                         goto free_buf;
1866                 }
1867                 if (copy_to_user(values + cp * value_size, value, value_size)) {
1868                         err = -EFAULT;
1869                         goto free_buf;
1870                 }
1871
1872                 if (!prev_key)
1873                         prev_key = buf_prevkey;
1874
1875                 swap(prev_key, key);
1876                 retry = MAP_LOOKUP_RETRIES;
1877                 cp++;
1878                 cond_resched();
1879         }
1880
1881         if (err == -EFAULT)
1882                 goto free_buf;
1883
1884         if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
1885                     (cp && copy_to_user(uobatch, prev_key, map->key_size))))
1886                 err = -EFAULT;
1887
1888 free_buf:
1889         kvfree(buf_prevkey);
1890         kvfree(buf);
1891         return err;
1892 }
1893
1894 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
1895
1896 static int map_lookup_and_delete_elem(union bpf_attr *attr)
1897 {
1898         void __user *ukey = u64_to_user_ptr(attr->key);
1899         void __user *uvalue = u64_to_user_ptr(attr->value);
1900         int ufd = attr->map_fd;
1901         struct bpf_map *map;
1902         void *key, *value;
1903         u32 value_size;
1904         struct fd f;
1905         int err;
1906
1907         if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1908                 return -EINVAL;
1909
1910         if (attr->flags & ~BPF_F_LOCK)
1911                 return -EINVAL;
1912
1913         f = fdget(ufd);
1914         map = __bpf_map_get(f);
1915         if (IS_ERR(map))
1916                 return PTR_ERR(map);
1917         bpf_map_write_active_inc(map);
1918         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
1919             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1920                 err = -EPERM;
1921                 goto err_put;
1922         }
1923
1924         if (attr->flags &&
1925             (map->map_type == BPF_MAP_TYPE_QUEUE ||
1926              map->map_type == BPF_MAP_TYPE_STACK)) {
1927                 err = -EINVAL;
1928                 goto err_put;
1929         }
1930
1931         if ((attr->flags & BPF_F_LOCK) &&
1932             !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1933                 err = -EINVAL;
1934                 goto err_put;
1935         }
1936
1937         key = __bpf_copy_key(ukey, map->key_size);
1938         if (IS_ERR(key)) {
1939                 err = PTR_ERR(key);
1940                 goto err_put;
1941         }
1942
1943         value_size = bpf_map_value_size(map);
1944
1945         err = -ENOMEM;
1946         value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1947         if (!value)
1948                 goto free_key;
1949
1950         err = -ENOTSUPP;
1951         if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1952             map->map_type == BPF_MAP_TYPE_STACK) {
1953                 err = map->ops->map_pop_elem(map, value);
1954         } else if (map->map_type == BPF_MAP_TYPE_HASH ||
1955                    map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1956                    map->map_type == BPF_MAP_TYPE_LRU_HASH ||
1957                    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
1958                 if (!bpf_map_is_offloaded(map)) {
1959                         bpf_disable_instrumentation();
1960                         rcu_read_lock();
1961                         err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
1962                         rcu_read_unlock();
1963                         bpf_enable_instrumentation();
1964                 }
1965         }
1966
1967         if (err)
1968                 goto free_value;
1969
1970         if (copy_to_user(uvalue, value, value_size) != 0) {
1971                 err = -EFAULT;
1972                 goto free_value;
1973         }
1974
1975         err = 0;
1976
1977 free_value:
1978         kvfree(value);
1979 free_key:
1980         kvfree(key);
1981 err_put:
1982         bpf_map_write_active_dec(map);
1983         fdput(f);
1984         return err;
1985 }
1986
1987 #define BPF_MAP_FREEZE_LAST_FIELD map_fd
1988
1989 static int map_freeze(const union bpf_attr *attr)
1990 {
1991         int err = 0, ufd = attr->map_fd;
1992         struct bpf_map *map;
1993         struct fd f;
1994
1995         if (CHECK_ATTR(BPF_MAP_FREEZE))
1996                 return -EINVAL;
1997
1998         f = fdget(ufd);
1999         map = __bpf_map_get(f);
2000         if (IS_ERR(map))
2001                 return PTR_ERR(map);
2002
2003         if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || !IS_ERR_OR_NULL(map->record)) {
2004                 fdput(f);
2005                 return -ENOTSUPP;
2006         }
2007
2008         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
2009                 fdput(f);
2010                 return -EPERM;
2011         }
2012
2013         mutex_lock(&map->freeze_mutex);
2014         if (bpf_map_write_active(map)) {
2015                 err = -EBUSY;
2016                 goto err_put;
2017         }
2018         if (READ_ONCE(map->frozen)) {
2019                 err = -EBUSY;
2020                 goto err_put;
2021         }
2022
2023         WRITE_ONCE(map->frozen, true);
2024 err_put:
2025         mutex_unlock(&map->freeze_mutex);
2026         fdput(f);
2027         return err;
2028 }
2029
2030 static const struct bpf_prog_ops * const bpf_prog_types[] = {
2031 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2032         [_id] = & _name ## _prog_ops,
2033 #define BPF_MAP_TYPE(_id, _ops)
2034 #define BPF_LINK_TYPE(_id, _name)
2035 #include <linux/bpf_types.h>
2036 #undef BPF_PROG_TYPE
2037 #undef BPF_MAP_TYPE
2038 #undef BPF_LINK_TYPE
2039 };
2040
2041 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
2042 {
2043         const struct bpf_prog_ops *ops;
2044
2045         if (type >= ARRAY_SIZE(bpf_prog_types))
2046                 return -EINVAL;
2047         type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
2048         ops = bpf_prog_types[type];
2049         if (!ops)
2050                 return -EINVAL;
2051
2052         if (!bpf_prog_is_offloaded(prog->aux))
2053                 prog->aux->ops = ops;
2054         else
2055                 prog->aux->ops = &bpf_offload_prog_ops;
2056         prog->type = type;
2057         return 0;
2058 }
2059
2060 enum bpf_audit {
2061         BPF_AUDIT_LOAD,
2062         BPF_AUDIT_UNLOAD,
2063         BPF_AUDIT_MAX,
2064 };
2065
2066 static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
2067         [BPF_AUDIT_LOAD]   = "LOAD",
2068         [BPF_AUDIT_UNLOAD] = "UNLOAD",
2069 };
2070
2071 static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
2072 {
2073         struct audit_context *ctx = NULL;
2074         struct audit_buffer *ab;
2075
2076         if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
2077                 return;
2078         if (audit_enabled == AUDIT_OFF)
2079                 return;
2080         if (!in_irq() && !irqs_disabled())
2081                 ctx = audit_context();
2082         ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
2083         if (unlikely(!ab))
2084                 return;
2085         audit_log_format(ab, "prog-id=%u op=%s",
2086                          prog->aux->id, bpf_audit_str[op]);
2087         audit_log_end(ab);
2088 }
2089
2090 static int bpf_prog_alloc_id(struct bpf_prog *prog)
2091 {
2092         int id;
2093
2094         idr_preload(GFP_KERNEL);
2095         spin_lock_bh(&prog_idr_lock);
2096         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
2097         if (id > 0)
2098                 prog->aux->id = id;
2099         spin_unlock_bh(&prog_idr_lock);
2100         idr_preload_end();
2101
2102         /* id is in [1, INT_MAX) */
2103         if (WARN_ON_ONCE(!id))
2104                 return -ENOSPC;
2105
2106         return id > 0 ? 0 : id;
2107 }
2108
2109 void bpf_prog_free_id(struct bpf_prog *prog)
2110 {
2111         unsigned long flags;
2112
2113         /* cBPF to eBPF migrations are currently not in the idr store.
2114          * Offloaded programs are removed from the store when their device
2115          * disappears - even if someone grabs an fd to them they are unusable,
2116          * simply waiting for refcnt to drop to be freed.
2117          */
2118         if (!prog->aux->id)
2119                 return;
2120
2121         spin_lock_irqsave(&prog_idr_lock, flags);
2122         idr_remove(&prog_idr, prog->aux->id);
2123         prog->aux->id = 0;
2124         spin_unlock_irqrestore(&prog_idr_lock, flags);
2125 }
2126
2127 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
2128 {
2129         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
2130
2131         kvfree(aux->func_info);
2132         kfree(aux->func_info_aux);
2133         free_uid(aux->user);
2134         security_bpf_prog_free(aux);
2135         bpf_prog_free(aux->prog);
2136 }
2137
2138 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
2139 {
2140         bpf_prog_kallsyms_del_all(prog);
2141         btf_put(prog->aux->btf);
2142         module_put(prog->aux->mod);
2143         kvfree(prog->aux->jited_linfo);
2144         kvfree(prog->aux->linfo);
2145         kfree(prog->aux->kfunc_tab);
2146         if (prog->aux->attach_btf)
2147                 btf_put(prog->aux->attach_btf);
2148
2149         if (deferred) {
2150                 if (prog->aux->sleepable)
2151                         call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
2152                 else
2153                         call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
2154         } else {
2155                 __bpf_prog_put_rcu(&prog->aux->rcu);
2156         }
2157 }
2158
2159 static void bpf_prog_put_deferred(struct work_struct *work)
2160 {
2161         struct bpf_prog_aux *aux;
2162         struct bpf_prog *prog;
2163
2164         aux = container_of(work, struct bpf_prog_aux, work);
2165         prog = aux->prog;
2166         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
2167         bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
2168         bpf_prog_free_id(prog);
2169         __bpf_prog_put_noref(prog, true);
2170 }
2171
2172 static void __bpf_prog_put(struct bpf_prog *prog)
2173 {
2174         struct bpf_prog_aux *aux = prog->aux;
2175
2176         if (atomic64_dec_and_test(&aux->refcnt)) {
2177                 if (in_irq() || irqs_disabled()) {
2178                         INIT_WORK(&aux->work, bpf_prog_put_deferred);
2179                         schedule_work(&aux->work);
2180                 } else {
2181                         bpf_prog_put_deferred(&aux->work);
2182                 }
2183         }
2184 }
2185
2186 void bpf_prog_put(struct bpf_prog *prog)
2187 {
2188         __bpf_prog_put(prog);
2189 }
2190 EXPORT_SYMBOL_GPL(bpf_prog_put);
2191
2192 static int bpf_prog_release(struct inode *inode, struct file *filp)
2193 {
2194         struct bpf_prog *prog = filp->private_data;
2195
2196         bpf_prog_put(prog);
2197         return 0;
2198 }
2199
2200 struct bpf_prog_kstats {
2201         u64 nsecs;
2202         u64 cnt;
2203         u64 misses;
2204 };
2205
2206 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog)
2207 {
2208         struct bpf_prog_stats *stats;
2209         unsigned int flags;
2210
2211         stats = this_cpu_ptr(prog->stats);
2212         flags = u64_stats_update_begin_irqsave(&stats->syncp);
2213         u64_stats_inc(&stats->misses);
2214         u64_stats_update_end_irqrestore(&stats->syncp, flags);
2215 }
2216
2217 static void bpf_prog_get_stats(const struct bpf_prog *prog,
2218                                struct bpf_prog_kstats *stats)
2219 {
2220         u64 nsecs = 0, cnt = 0, misses = 0;
2221         int cpu;
2222
2223         for_each_possible_cpu(cpu) {
2224                 const struct bpf_prog_stats *st;
2225                 unsigned int start;
2226                 u64 tnsecs, tcnt, tmisses;
2227
2228                 st = per_cpu_ptr(prog->stats, cpu);
2229                 do {
2230                         start = u64_stats_fetch_begin(&st->syncp);
2231                         tnsecs = u64_stats_read(&st->nsecs);
2232                         tcnt = u64_stats_read(&st->cnt);
2233                         tmisses = u64_stats_read(&st->misses);
2234                 } while (u64_stats_fetch_retry(&st->syncp, start));
2235                 nsecs += tnsecs;
2236                 cnt += tcnt;
2237                 misses += tmisses;
2238         }
2239         stats->nsecs = nsecs;
2240         stats->cnt = cnt;
2241         stats->misses = misses;
2242 }
2243
2244 #ifdef CONFIG_PROC_FS
2245 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
2246 {
2247         const struct bpf_prog *prog = filp->private_data;
2248         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2249         struct bpf_prog_kstats stats;
2250
2251         bpf_prog_get_stats(prog, &stats);
2252         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2253         seq_printf(m,
2254                    "prog_type:\t%u\n"
2255                    "prog_jited:\t%u\n"
2256                    "prog_tag:\t%s\n"
2257                    "memlock:\t%llu\n"
2258                    "prog_id:\t%u\n"
2259                    "run_time_ns:\t%llu\n"
2260                    "run_cnt:\t%llu\n"
2261                    "recursion_misses:\t%llu\n"
2262                    "verified_insns:\t%u\n",
2263                    prog->type,
2264                    prog->jited,
2265                    prog_tag,
2266                    prog->pages * 1ULL << PAGE_SHIFT,
2267                    prog->aux->id,
2268                    stats.nsecs,
2269                    stats.cnt,
2270                    stats.misses,
2271                    prog->aux->verified_insns);
2272 }
2273 #endif
2274
2275 const struct file_operations bpf_prog_fops = {
2276 #ifdef CONFIG_PROC_FS
2277         .show_fdinfo    = bpf_prog_show_fdinfo,
2278 #endif
2279         .release        = bpf_prog_release,
2280         .read           = bpf_dummy_read,
2281         .write          = bpf_dummy_write,
2282 };
2283
2284 int bpf_prog_new_fd(struct bpf_prog *prog)
2285 {
2286         int ret;
2287
2288         ret = security_bpf_prog(prog);
2289         if (ret < 0)
2290                 return ret;
2291
2292         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
2293                                 O_RDWR | O_CLOEXEC);
2294 }
2295
2296 static struct bpf_prog *____bpf_prog_get(struct fd f)
2297 {
2298         if (!f.file)
2299                 return ERR_PTR(-EBADF);
2300         if (f.file->f_op != &bpf_prog_fops) {
2301                 fdput(f);
2302                 return ERR_PTR(-EINVAL);
2303         }
2304
2305         return f.file->private_data;
2306 }
2307
2308 void bpf_prog_add(struct bpf_prog *prog, int i)
2309 {
2310         atomic64_add(i, &prog->aux->refcnt);
2311 }
2312 EXPORT_SYMBOL_GPL(bpf_prog_add);
2313
2314 void bpf_prog_sub(struct bpf_prog *prog, int i)
2315 {
2316         /* Only to be used for undoing previous bpf_prog_add() in some
2317          * error path. We still know that another entity in our call
2318          * path holds a reference to the program, thus atomic_sub() can
2319          * be safely used in such cases!
2320          */
2321         WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
2322 }
2323 EXPORT_SYMBOL_GPL(bpf_prog_sub);
2324
2325 void bpf_prog_inc(struct bpf_prog *prog)
2326 {
2327         atomic64_inc(&prog->aux->refcnt);
2328 }
2329 EXPORT_SYMBOL_GPL(bpf_prog_inc);
2330
2331 /* prog_idr_lock should have been held */
2332 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
2333 {
2334         int refold;
2335
2336         refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
2337
2338         if (!refold)
2339                 return ERR_PTR(-ENOENT);
2340
2341         return prog;
2342 }
2343 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
2344
2345 bool bpf_prog_get_ok(struct bpf_prog *prog,
2346                             enum bpf_prog_type *attach_type, bool attach_drv)
2347 {
2348         /* not an attachment, just a refcount inc, always allow */
2349         if (!attach_type)
2350                 return true;
2351
2352         if (prog->type != *attach_type)
2353                 return false;
2354         if (bpf_prog_is_offloaded(prog->aux) && !attach_drv)
2355                 return false;
2356
2357         return true;
2358 }
2359
2360 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
2361                                        bool attach_drv)
2362 {
2363         struct fd f = fdget(ufd);
2364         struct bpf_prog *prog;
2365
2366         prog = ____bpf_prog_get(f);
2367         if (IS_ERR(prog))
2368                 return prog;
2369         if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
2370                 prog = ERR_PTR(-EINVAL);
2371                 goto out;
2372         }
2373
2374         bpf_prog_inc(prog);
2375 out:
2376         fdput(f);
2377         return prog;
2378 }
2379
2380 struct bpf_prog *bpf_prog_get(u32 ufd)
2381 {
2382         return __bpf_prog_get(ufd, NULL, false);
2383 }
2384
2385 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2386                                        bool attach_drv)
2387 {
2388         return __bpf_prog_get(ufd, &type, attach_drv);
2389 }
2390 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
2391
2392 /* Initially all BPF programs could be loaded w/o specifying
2393  * expected_attach_type. Later for some of them specifying expected_attach_type
2394  * at load time became required so that program could be validated properly.
2395  * Programs of types that are allowed to be loaded both w/ and w/o (for
2396  * backward compatibility) expected_attach_type, should have the default attach
2397  * type assigned to expected_attach_type for the latter case, so that it can be
2398  * validated later at attach time.
2399  *
2400  * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
2401  * prog type requires it but has some attach types that have to be backward
2402  * compatible.
2403  */
2404 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
2405 {
2406         switch (attr->prog_type) {
2407         case BPF_PROG_TYPE_CGROUP_SOCK:
2408                 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
2409                  * exist so checking for non-zero is the way to go here.
2410                  */
2411                 if (!attr->expected_attach_type)
2412                         attr->expected_attach_type =
2413                                 BPF_CGROUP_INET_SOCK_CREATE;
2414                 break;
2415         case BPF_PROG_TYPE_SK_REUSEPORT:
2416                 if (!attr->expected_attach_type)
2417                         attr->expected_attach_type =
2418                                 BPF_SK_REUSEPORT_SELECT;
2419                 break;
2420         }
2421 }
2422
2423 static int
2424 bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
2425                            enum bpf_attach_type expected_attach_type,
2426                            struct btf *attach_btf, u32 btf_id,
2427                            struct bpf_prog *dst_prog)
2428 {
2429         if (btf_id) {
2430                 if (btf_id > BTF_MAX_TYPE)
2431                         return -EINVAL;
2432
2433                 if (!attach_btf && !dst_prog)
2434                         return -EINVAL;
2435
2436                 switch (prog_type) {
2437                 case BPF_PROG_TYPE_TRACING:
2438                 case BPF_PROG_TYPE_LSM:
2439                 case BPF_PROG_TYPE_STRUCT_OPS:
2440                 case BPF_PROG_TYPE_EXT:
2441                         break;
2442                 default:
2443                         return -EINVAL;
2444                 }
2445         }
2446
2447         if (attach_btf && (!btf_id || dst_prog))
2448                 return -EINVAL;
2449
2450         if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
2451             prog_type != BPF_PROG_TYPE_EXT)
2452                 return -EINVAL;
2453
2454         switch (prog_type) {
2455         case BPF_PROG_TYPE_CGROUP_SOCK:
2456                 switch (expected_attach_type) {
2457                 case BPF_CGROUP_INET_SOCK_CREATE:
2458                 case BPF_CGROUP_INET_SOCK_RELEASE:
2459                 case BPF_CGROUP_INET4_POST_BIND:
2460                 case BPF_CGROUP_INET6_POST_BIND:
2461                         return 0;
2462                 default:
2463                         return -EINVAL;
2464                 }
2465         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2466                 switch (expected_attach_type) {
2467                 case BPF_CGROUP_INET4_BIND:
2468                 case BPF_CGROUP_INET6_BIND:
2469                 case BPF_CGROUP_INET4_CONNECT:
2470                 case BPF_CGROUP_INET6_CONNECT:
2471                 case BPF_CGROUP_INET4_GETPEERNAME:
2472                 case BPF_CGROUP_INET6_GETPEERNAME:
2473                 case BPF_CGROUP_INET4_GETSOCKNAME:
2474                 case BPF_CGROUP_INET6_GETSOCKNAME:
2475                 case BPF_CGROUP_UDP4_SENDMSG:
2476                 case BPF_CGROUP_UDP6_SENDMSG:
2477                 case BPF_CGROUP_UDP4_RECVMSG:
2478                 case BPF_CGROUP_UDP6_RECVMSG:
2479                         return 0;
2480                 default:
2481                         return -EINVAL;
2482                 }
2483         case BPF_PROG_TYPE_CGROUP_SKB:
2484                 switch (expected_attach_type) {
2485                 case BPF_CGROUP_INET_INGRESS:
2486                 case BPF_CGROUP_INET_EGRESS:
2487                         return 0;
2488                 default:
2489                         return -EINVAL;
2490                 }
2491         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2492                 switch (expected_attach_type) {
2493                 case BPF_CGROUP_SETSOCKOPT:
2494                 case BPF_CGROUP_GETSOCKOPT:
2495                         return 0;
2496                 default:
2497                         return -EINVAL;
2498                 }
2499         case BPF_PROG_TYPE_SK_LOOKUP:
2500                 if (expected_attach_type == BPF_SK_LOOKUP)
2501                         return 0;
2502                 return -EINVAL;
2503         case BPF_PROG_TYPE_SK_REUSEPORT:
2504                 switch (expected_attach_type) {
2505                 case BPF_SK_REUSEPORT_SELECT:
2506                 case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:
2507                         return 0;
2508                 default:
2509                         return -EINVAL;
2510                 }
2511         case BPF_PROG_TYPE_NETFILTER:
2512                 if (expected_attach_type == BPF_NETFILTER)
2513                         return 0;
2514                 return -EINVAL;
2515         case BPF_PROG_TYPE_SYSCALL:
2516         case BPF_PROG_TYPE_EXT:
2517                 if (expected_attach_type)
2518                         return -EINVAL;
2519                 fallthrough;
2520         default:
2521                 return 0;
2522         }
2523 }
2524
2525 static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
2526 {
2527         switch (prog_type) {
2528         case BPF_PROG_TYPE_SCHED_CLS:
2529         case BPF_PROG_TYPE_SCHED_ACT:
2530         case BPF_PROG_TYPE_XDP:
2531         case BPF_PROG_TYPE_LWT_IN:
2532         case BPF_PROG_TYPE_LWT_OUT:
2533         case BPF_PROG_TYPE_LWT_XMIT:
2534         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2535         case BPF_PROG_TYPE_SK_SKB:
2536         case BPF_PROG_TYPE_SK_MSG:
2537         case BPF_PROG_TYPE_FLOW_DISSECTOR:
2538         case BPF_PROG_TYPE_CGROUP_DEVICE:
2539         case BPF_PROG_TYPE_CGROUP_SOCK:
2540         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2541         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2542         case BPF_PROG_TYPE_CGROUP_SYSCTL:
2543         case BPF_PROG_TYPE_SOCK_OPS:
2544         case BPF_PROG_TYPE_EXT: /* extends any prog */
2545         case BPF_PROG_TYPE_NETFILTER:
2546                 return true;
2547         case BPF_PROG_TYPE_CGROUP_SKB:
2548                 /* always unpriv */
2549         case BPF_PROG_TYPE_SK_REUSEPORT:
2550                 /* equivalent to SOCKET_FILTER. need CAP_BPF only */
2551         default:
2552                 return false;
2553         }
2554 }
2555
2556 static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
2557 {
2558         switch (prog_type) {
2559         case BPF_PROG_TYPE_KPROBE:
2560         case BPF_PROG_TYPE_TRACEPOINT:
2561         case BPF_PROG_TYPE_PERF_EVENT:
2562         case BPF_PROG_TYPE_RAW_TRACEPOINT:
2563         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2564         case BPF_PROG_TYPE_TRACING:
2565         case BPF_PROG_TYPE_LSM:
2566         case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
2567         case BPF_PROG_TYPE_EXT: /* extends any prog */
2568                 return true;
2569         default:
2570                 return false;
2571         }
2572 }
2573
2574 /* last field in 'union bpf_attr' used by this command */
2575 #define BPF_PROG_LOAD_LAST_FIELD log_true_size
2576
2577 static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
2578 {
2579         enum bpf_prog_type type = attr->prog_type;
2580         struct bpf_prog *prog, *dst_prog = NULL;
2581         struct btf *attach_btf = NULL;
2582         int err;
2583         char license[128];
2584
2585         if (CHECK_ATTR(BPF_PROG_LOAD))
2586                 return -EINVAL;
2587
2588         if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2589                                  BPF_F_ANY_ALIGNMENT |
2590                                  BPF_F_TEST_STATE_FREQ |
2591                                  BPF_F_SLEEPABLE |
2592                                  BPF_F_TEST_RND_HI32 |
2593                                  BPF_F_XDP_HAS_FRAGS |
2594                                  BPF_F_XDP_DEV_BOUND_ONLY))
2595                 return -EINVAL;
2596
2597         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2598             (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2599             !bpf_capable())
2600                 return -EPERM;
2601
2602         /* Intent here is for unprivileged_bpf_disabled to block BPF program
2603          * creation for unprivileged users; other actions depend
2604          * on fd availability and access to bpffs, so are dependent on
2605          * object creation success. Even with unprivileged BPF disabled,
2606          * capability checks are still carried out for these
2607          * and other operations.
2608          */
2609         if (sysctl_unprivileged_bpf_disabled && !bpf_capable())
2610                 return -EPERM;
2611
2612         if (attr->insn_cnt == 0 ||
2613             attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
2614                 return -E2BIG;
2615         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
2616             type != BPF_PROG_TYPE_CGROUP_SKB &&
2617             !bpf_capable())
2618                 return -EPERM;
2619
2620         if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN))
2621                 return -EPERM;
2622         if (is_perfmon_prog_type(type) && !perfmon_capable())
2623                 return -EPERM;
2624
2625         /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
2626          * or btf, we need to check which one it is
2627          */
2628         if (attr->attach_prog_fd) {
2629                 dst_prog = bpf_prog_get(attr->attach_prog_fd);
2630                 if (IS_ERR(dst_prog)) {
2631                         dst_prog = NULL;
2632                         attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
2633                         if (IS_ERR(attach_btf))
2634                                 return -EINVAL;
2635                         if (!btf_is_kernel(attach_btf)) {
2636                                 /* attaching through specifying bpf_prog's BTF
2637                                  * objects directly might be supported eventually
2638                                  */
2639                                 btf_put(attach_btf);
2640                                 return -ENOTSUPP;
2641                         }
2642                 }
2643         } else if (attr->attach_btf_id) {
2644                 /* fall back to vmlinux BTF, if BTF type ID is specified */
2645                 attach_btf = bpf_get_btf_vmlinux();
2646                 if (IS_ERR(attach_btf))
2647                         return PTR_ERR(attach_btf);
2648                 if (!attach_btf)
2649                         return -EINVAL;
2650                 btf_get(attach_btf);
2651         }
2652
2653         bpf_prog_load_fixup_attach_type(attr);
2654         if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2655                                        attach_btf, attr->attach_btf_id,
2656                                        dst_prog)) {
2657                 if (dst_prog)
2658                         bpf_prog_put(dst_prog);
2659                 if (attach_btf)
2660                         btf_put(attach_btf);
2661                 return -EINVAL;
2662         }
2663
2664         /* plain bpf_prog allocation */
2665         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2666         if (!prog) {
2667                 if (dst_prog)
2668                         bpf_prog_put(dst_prog);
2669                 if (attach_btf)
2670                         btf_put(attach_btf);
2671                 return -ENOMEM;
2672         }
2673
2674         prog->expected_attach_type = attr->expected_attach_type;
2675         prog->aux->attach_btf = attach_btf;
2676         prog->aux->attach_btf_id = attr->attach_btf_id;
2677         prog->aux->dst_prog = dst_prog;
2678         prog->aux->dev_bound = !!attr->prog_ifindex;
2679         prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
2680         prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS;
2681
2682         err = security_bpf_prog_alloc(prog->aux);
2683         if (err)
2684                 goto free_prog;
2685
2686         prog->aux->user = get_current_user();
2687         prog->len = attr->insn_cnt;
2688
2689         err = -EFAULT;
2690         if (copy_from_bpfptr(prog->insns,
2691                              make_bpfptr(attr->insns, uattr.is_kernel),
2692                              bpf_prog_insn_size(prog)) != 0)
2693                 goto free_prog_sec;
2694         /* copy eBPF program license from user space */
2695         if (strncpy_from_bpfptr(license,
2696                                 make_bpfptr(attr->license, uattr.is_kernel),
2697                                 sizeof(license) - 1) < 0)
2698                 goto free_prog_sec;
2699         license[sizeof(license) - 1] = 0;
2700
2701         /* eBPF programs must be GPL compatible to use GPL-ed functions */
2702         prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0;
2703
2704         prog->orig_prog = NULL;
2705         prog->jited = 0;
2706
2707         atomic64_set(&prog->aux->refcnt, 1);
2708
2709         if (bpf_prog_is_dev_bound(prog->aux)) {
2710                 err = bpf_prog_dev_bound_init(prog, attr);
2711                 if (err)
2712                         goto free_prog_sec;
2713         }
2714
2715         if (type == BPF_PROG_TYPE_EXT && dst_prog &&
2716             bpf_prog_is_dev_bound(dst_prog->aux)) {
2717                 err = bpf_prog_dev_bound_inherit(prog, dst_prog);
2718                 if (err)
2719                         goto free_prog_sec;
2720         }
2721
2722         /* find program type: socket_filter vs tracing_filter */
2723         err = find_prog_type(type, prog);
2724         if (err < 0)
2725                 goto free_prog_sec;
2726
2727         prog->aux->load_time = ktime_get_boottime_ns();
2728         err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
2729                                sizeof(attr->prog_name));
2730         if (err < 0)
2731                 goto free_prog_sec;
2732
2733         /* run eBPF verifier */
2734         err = bpf_check(&prog, attr, uattr, uattr_size);
2735         if (err < 0)
2736                 goto free_used_maps;
2737
2738         prog = bpf_prog_select_runtime(prog, &err);
2739         if (err < 0)
2740                 goto free_used_maps;
2741
2742         err = bpf_prog_alloc_id(prog);
2743         if (err)
2744                 goto free_used_maps;
2745
2746         /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2747          * effectively publicly exposed. However, retrieving via
2748          * bpf_prog_get_fd_by_id() will take another reference,
2749          * therefore it cannot be gone underneath us.
2750          *
2751          * Only for the time /after/ successful bpf_prog_new_fd()
2752          * and before returning to userspace, we might just hold
2753          * one reference and any parallel close on that fd could
2754          * rip everything out. Hence, below notifications must
2755          * happen before bpf_prog_new_fd().
2756          *
2757          * Also, any failure handling from this point onwards must
2758          * be using bpf_prog_put() given the program is exposed.
2759          */
2760         bpf_prog_kallsyms_add(prog);
2761         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
2762         bpf_audit_prog(prog, BPF_AUDIT_LOAD);
2763
2764         err = bpf_prog_new_fd(prog);
2765         if (err < 0)
2766                 bpf_prog_put(prog);
2767         return err;
2768
2769 free_used_maps:
2770         /* In case we have subprogs, we need to wait for a grace
2771          * period before we can tear down JIT memory since symbols
2772          * are already exposed under kallsyms.
2773          */
2774         __bpf_prog_put_noref(prog, prog->aux->func_cnt);
2775         return err;
2776 free_prog_sec:
2777         free_uid(prog->aux->user);
2778         security_bpf_prog_free(prog->aux);
2779 free_prog:
2780         if (prog->aux->attach_btf)
2781                 btf_put(prog->aux->attach_btf);
2782         bpf_prog_free(prog);
2783         return err;
2784 }
2785
2786 #define BPF_OBJ_LAST_FIELD path_fd
2787
2788 static int bpf_obj_pin(const union bpf_attr *attr)
2789 {
2790         int path_fd;
2791
2792         if (CHECK_ATTR(BPF_OBJ) || attr->file_flags & ~BPF_F_PATH_FD)
2793                 return -EINVAL;
2794
2795         /* path_fd has to be accompanied by BPF_F_PATH_FD flag */
2796         if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
2797                 return -EINVAL;
2798
2799         path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
2800         return bpf_obj_pin_user(attr->bpf_fd, path_fd,
2801                                 u64_to_user_ptr(attr->pathname));
2802 }
2803
2804 static int bpf_obj_get(const union bpf_attr *attr)
2805 {
2806         int path_fd;
2807
2808         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
2809             attr->file_flags & ~(BPF_OBJ_FLAG_MASK | BPF_F_PATH_FD))
2810                 return -EINVAL;
2811
2812         /* path_fd has to be accompanied by BPF_F_PATH_FD flag */
2813         if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
2814                 return -EINVAL;
2815
2816         path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
2817         return bpf_obj_get_user(path_fd, u64_to_user_ptr(attr->pathname),
2818                                 attr->file_flags);
2819 }
2820
2821 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2822                    const struct bpf_link_ops *ops, struct bpf_prog *prog)
2823 {
2824         atomic64_set(&link->refcnt, 1);
2825         link->type = type;
2826         link->id = 0;
2827         link->ops = ops;
2828         link->prog = prog;
2829 }
2830
2831 static void bpf_link_free_id(int id)
2832 {
2833         if (!id)
2834                 return;
2835
2836         spin_lock_bh(&link_idr_lock);
2837         idr_remove(&link_idr, id);
2838         spin_unlock_bh(&link_idr_lock);
2839 }
2840
2841 /* Clean up bpf_link and corresponding anon_inode file and FD. After
2842  * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
2843  * anon_inode's release() call. This helper marks bpf_link as
2844  * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
2845  * is not decremented, it's the responsibility of a calling code that failed
2846  * to complete bpf_link initialization.
2847  * This helper eventually calls link's dealloc callback, but does not call
2848  * link's release callback.
2849  */
2850 void bpf_link_cleanup(struct bpf_link_primer *primer)
2851 {
2852         primer->link->prog = NULL;
2853         bpf_link_free_id(primer->id);
2854         fput(primer->file);
2855         put_unused_fd(primer->fd);
2856 }
2857
2858 void bpf_link_inc(struct bpf_link *link)
2859 {
2860         atomic64_inc(&link->refcnt);
2861 }
2862
2863 /* bpf_link_free is guaranteed to be called from process context */
2864 static void bpf_link_free(struct bpf_link *link)
2865 {
2866         bpf_link_free_id(link->id);
2867         if (link->prog) {
2868                 /* detach BPF program, clean up used resources */
2869                 link->ops->release(link);
2870                 bpf_prog_put(link->prog);
2871         }
2872         /* free bpf_link and its containing memory */
2873         link->ops->dealloc(link);
2874 }
2875
2876 static void bpf_link_put_deferred(struct work_struct *work)
2877 {
2878         struct bpf_link *link = container_of(work, struct bpf_link, work);
2879
2880         bpf_link_free(link);
2881 }
2882
2883 /* bpf_link_put might be called from atomic context. It needs to be called
2884  * from sleepable context in order to acquire sleeping locks during the process.
2885  */
2886 void bpf_link_put(struct bpf_link *link)
2887 {
2888         if (!atomic64_dec_and_test(&link->refcnt))
2889                 return;
2890
2891         INIT_WORK(&link->work, bpf_link_put_deferred);
2892         schedule_work(&link->work);
2893 }
2894 EXPORT_SYMBOL(bpf_link_put);
2895
2896 static void bpf_link_put_direct(struct bpf_link *link)
2897 {
2898         if (!atomic64_dec_and_test(&link->refcnt))
2899                 return;
2900         bpf_link_free(link);
2901 }
2902
2903 static int bpf_link_release(struct inode *inode, struct file *filp)
2904 {
2905         struct bpf_link *link = filp->private_data;
2906
2907         bpf_link_put_direct(link);
2908         return 0;
2909 }
2910
2911 #ifdef CONFIG_PROC_FS
2912 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
2913 #define BPF_MAP_TYPE(_id, _ops)
2914 #define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
2915 static const char *bpf_link_type_strs[] = {
2916         [BPF_LINK_TYPE_UNSPEC] = "<invalid>",
2917 #include <linux/bpf_types.h>
2918 };
2919 #undef BPF_PROG_TYPE
2920 #undef BPF_MAP_TYPE
2921 #undef BPF_LINK_TYPE
2922
2923 static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
2924 {
2925         const struct bpf_link *link = filp->private_data;
2926         const struct bpf_prog *prog = link->prog;
2927         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2928
2929         seq_printf(m,
2930                    "link_type:\t%s\n"
2931                    "link_id:\t%u\n",
2932                    bpf_link_type_strs[link->type],
2933                    link->id);
2934         if (prog) {
2935                 bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2936                 seq_printf(m,
2937                            "prog_tag:\t%s\n"
2938                            "prog_id:\t%u\n",
2939                            prog_tag,
2940                            prog->aux->id);
2941         }
2942         if (link->ops->show_fdinfo)
2943                 link->ops->show_fdinfo(link, m);
2944 }
2945 #endif
2946
2947 static const struct file_operations bpf_link_fops = {
2948 #ifdef CONFIG_PROC_FS
2949         .show_fdinfo    = bpf_link_show_fdinfo,
2950 #endif
2951         .release        = bpf_link_release,
2952         .read           = bpf_dummy_read,
2953         .write          = bpf_dummy_write,
2954 };
2955
2956 static int bpf_link_alloc_id(struct bpf_link *link)
2957 {
2958         int id;
2959
2960         idr_preload(GFP_KERNEL);
2961         spin_lock_bh(&link_idr_lock);
2962         id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
2963         spin_unlock_bh(&link_idr_lock);
2964         idr_preload_end();
2965
2966         return id;
2967 }
2968
2969 /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
2970  * reserving unused FD and allocating ID from link_idr. This is to be paired
2971  * with bpf_link_settle() to install FD and ID and expose bpf_link to
2972  * user-space, if bpf_link is successfully attached. If not, bpf_link and
2973  * pre-allocated resources are to be freed with bpf_cleanup() call. All the
2974  * transient state is passed around in struct bpf_link_primer.
2975  * This is preferred way to create and initialize bpf_link, especially when
2976  * there are complicated and expensive operations in between creating bpf_link
2977  * itself and attaching it to BPF hook. By using bpf_link_prime() and
2978  * bpf_link_settle() kernel code using bpf_link doesn't have to perform
2979  * expensive (and potentially failing) roll back operations in a rare case
2980  * that file, FD, or ID can't be allocated.
2981  */
2982 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
2983 {
2984         struct file *file;
2985         int fd, id;
2986
2987         fd = get_unused_fd_flags(O_CLOEXEC);
2988         if (fd < 0)
2989                 return fd;
2990
2991
2992         id = bpf_link_alloc_id(link);
2993         if (id < 0) {
2994                 put_unused_fd(fd);
2995                 return id;
2996         }
2997
2998         file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC);
2999         if (IS_ERR(file)) {
3000                 bpf_link_free_id(id);
3001                 put_unused_fd(fd);
3002                 return PTR_ERR(file);
3003         }
3004
3005         primer->link = link;
3006         primer->file = file;
3007         primer->fd = fd;
3008         primer->id = id;
3009         return 0;
3010 }
3011
3012 int bpf_link_settle(struct bpf_link_primer *primer)
3013 {
3014         /* make bpf_link fetchable by ID */
3015         spin_lock_bh(&link_idr_lock);
3016         primer->link->id = primer->id;
3017         spin_unlock_bh(&link_idr_lock);
3018         /* make bpf_link fetchable by FD */
3019         fd_install(primer->fd, primer->file);
3020         /* pass through installed FD */
3021         return primer->fd;
3022 }
3023
3024 int bpf_link_new_fd(struct bpf_link *link)
3025 {
3026         return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC);
3027 }
3028
3029 struct bpf_link *bpf_link_get_from_fd(u32 ufd)
3030 {
3031         struct fd f = fdget(ufd);
3032         struct bpf_link *link;
3033
3034         if (!f.file)
3035                 return ERR_PTR(-EBADF);
3036         if (f.file->f_op != &bpf_link_fops) {
3037                 fdput(f);
3038                 return ERR_PTR(-EINVAL);
3039         }
3040
3041         link = f.file->private_data;
3042         bpf_link_inc(link);
3043         fdput(f);
3044
3045         return link;
3046 }
3047 EXPORT_SYMBOL(bpf_link_get_from_fd);
3048
3049 static void bpf_tracing_link_release(struct bpf_link *link)
3050 {
3051         struct bpf_tracing_link *tr_link =
3052                 container_of(link, struct bpf_tracing_link, link.link);
3053
3054         WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
3055                                                 tr_link->trampoline));
3056
3057         bpf_trampoline_put(tr_link->trampoline);
3058
3059         /* tgt_prog is NULL if target is a kernel function */
3060         if (tr_link->tgt_prog)
3061                 bpf_prog_put(tr_link->tgt_prog);
3062 }
3063
3064 static void bpf_tracing_link_dealloc(struct bpf_link *link)
3065 {
3066         struct bpf_tracing_link *tr_link =
3067                 container_of(link, struct bpf_tracing_link, link.link);
3068
3069         kfree(tr_link);
3070 }
3071
3072 static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
3073                                          struct seq_file *seq)
3074 {
3075         struct bpf_tracing_link *tr_link =
3076                 container_of(link, struct bpf_tracing_link, link.link);
3077         u32 target_btf_id, target_obj_id;
3078
3079         bpf_trampoline_unpack_key(tr_link->trampoline->key,
3080                                   &target_obj_id, &target_btf_id);
3081         seq_printf(seq,
3082                    "attach_type:\t%d\n"
3083                    "target_obj_id:\t%u\n"
3084                    "target_btf_id:\t%u\n",
3085                    tr_link->attach_type,
3086                    target_obj_id,
3087                    target_btf_id);
3088 }
3089
3090 static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
3091                                            struct bpf_link_info *info)
3092 {
3093         struct bpf_tracing_link *tr_link =
3094                 container_of(link, struct bpf_tracing_link, link.link);
3095
3096         info->tracing.attach_type = tr_link->attach_type;
3097         bpf_trampoline_unpack_key(tr_link->trampoline->key,
3098                                   &info->tracing.target_obj_id,
3099                                   &info->tracing.target_btf_id);
3100
3101         return 0;
3102 }
3103
3104 static const struct bpf_link_ops bpf_tracing_link_lops = {
3105         .release = bpf_tracing_link_release,
3106         .dealloc = bpf_tracing_link_dealloc,
3107         .show_fdinfo = bpf_tracing_link_show_fdinfo,
3108         .fill_link_info = bpf_tracing_link_fill_link_info,
3109 };
3110
3111 static int bpf_tracing_prog_attach(struct bpf_prog *prog,
3112                                    int tgt_prog_fd,
3113                                    u32 btf_id,
3114                                    u64 bpf_cookie)
3115 {
3116         struct bpf_link_primer link_primer;
3117         struct bpf_prog *tgt_prog = NULL;
3118         struct bpf_trampoline *tr = NULL;
3119         struct bpf_tracing_link *link;
3120         u64 key = 0;
3121         int err;
3122
3123         switch (prog->type) {
3124         case BPF_PROG_TYPE_TRACING:
3125                 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
3126                     prog->expected_attach_type != BPF_TRACE_FEXIT &&
3127                     prog->expected_attach_type != BPF_MODIFY_RETURN) {
3128                         err = -EINVAL;
3129                         goto out_put_prog;
3130                 }
3131                 break;
3132         case BPF_PROG_TYPE_EXT:
3133                 if (prog->expected_attach_type != 0) {
3134                         err = -EINVAL;
3135                         goto out_put_prog;
3136                 }
3137                 break;
3138         case BPF_PROG_TYPE_LSM:
3139                 if (prog->expected_attach_type != BPF_LSM_MAC) {
3140                         err = -EINVAL;
3141                         goto out_put_prog;
3142                 }
3143                 break;
3144         default:
3145                 err = -EINVAL;
3146                 goto out_put_prog;
3147         }
3148
3149         if (!!tgt_prog_fd != !!btf_id) {
3150                 err = -EINVAL;
3151                 goto out_put_prog;
3152         }
3153
3154         if (tgt_prog_fd) {
3155                 /* For now we only allow new targets for BPF_PROG_TYPE_EXT */
3156                 if (prog->type != BPF_PROG_TYPE_EXT) {
3157                         err = -EINVAL;
3158                         goto out_put_prog;
3159                 }
3160
3161                 tgt_prog = bpf_prog_get(tgt_prog_fd);
3162                 if (IS_ERR(tgt_prog)) {
3163                         err = PTR_ERR(tgt_prog);
3164                         tgt_prog = NULL;
3165                         goto out_put_prog;
3166                 }
3167
3168                 key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
3169         }
3170
3171         link = kzalloc(sizeof(*link), GFP_USER);
3172         if (!link) {
3173                 err = -ENOMEM;
3174                 goto out_put_prog;
3175         }
3176         bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
3177                       &bpf_tracing_link_lops, prog);
3178         link->attach_type = prog->expected_attach_type;
3179         link->link.cookie = bpf_cookie;
3180
3181         mutex_lock(&prog->aux->dst_mutex);
3182
3183         /* There are a few possible cases here:
3184          *
3185          * - if prog->aux->dst_trampoline is set, the program was just loaded
3186          *   and not yet attached to anything, so we can use the values stored
3187          *   in prog->aux
3188          *
3189          * - if prog->aux->dst_trampoline is NULL, the program has already been
3190          *   attached to a target and its initial target was cleared (below)
3191          *
3192          * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
3193          *   target_btf_id using the link_create API.
3194          *
3195          * - if tgt_prog == NULL when this function was called using the old
3196          *   raw_tracepoint_open API, and we need a target from prog->aux
3197          *
3198          * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program
3199          *   was detached and is going for re-attachment.
3200          */
3201         if (!prog->aux->dst_trampoline && !tgt_prog) {
3202                 /*
3203                  * Allow re-attach for TRACING and LSM programs. If it's
3204                  * currently linked, bpf_trampoline_link_prog will fail.
3205                  * EXT programs need to specify tgt_prog_fd, so they
3206                  * re-attach in separate code path.
3207                  */
3208                 if (prog->type != BPF_PROG_TYPE_TRACING &&
3209                     prog->type != BPF_PROG_TYPE_LSM) {
3210                         err = -EINVAL;
3211                         goto out_unlock;
3212                 }
3213                 btf_id = prog->aux->attach_btf_id;
3214                 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
3215         }
3216
3217         if (!prog->aux->dst_trampoline ||
3218             (key && key != prog->aux->dst_trampoline->key)) {
3219                 /* If there is no saved target, or the specified target is
3220                  * different from the destination specified at load time, we
3221                  * need a new trampoline and a check for compatibility
3222                  */
3223                 struct bpf_attach_target_info tgt_info = {};
3224
3225                 err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
3226                                               &tgt_info);
3227                 if (err)
3228                         goto out_unlock;
3229
3230                 if (tgt_info.tgt_mod) {
3231                         module_put(prog->aux->mod);
3232                         prog->aux->mod = tgt_info.tgt_mod;
3233                 }
3234
3235                 tr = bpf_trampoline_get(key, &tgt_info);
3236                 if (!tr) {
3237                         err = -ENOMEM;
3238                         goto out_unlock;
3239                 }
3240         } else {
3241                 /* The caller didn't specify a target, or the target was the
3242                  * same as the destination supplied during program load. This
3243                  * means we can reuse the trampoline and reference from program
3244                  * load time, and there is no need to allocate a new one. This
3245                  * can only happen once for any program, as the saved values in
3246                  * prog->aux are cleared below.
3247                  */
3248                 tr = prog->aux->dst_trampoline;
3249                 tgt_prog = prog->aux->dst_prog;
3250         }
3251
3252         err = bpf_link_prime(&link->link.link, &link_primer);
3253         if (err)
3254                 goto out_unlock;
3255
3256         err = bpf_trampoline_link_prog(&link->link, tr);
3257         if (err) {
3258                 bpf_link_cleanup(&link_primer);
3259                 link = NULL;
3260                 goto out_unlock;
3261         }
3262
3263         link->tgt_prog = tgt_prog;
3264         link->trampoline = tr;
3265
3266         /* Always clear the trampoline and target prog from prog->aux to make
3267          * sure the original attach destination is not kept alive after a
3268          * program is (re-)attached to another target.
3269          */
3270         if (prog->aux->dst_prog &&
3271             (tgt_prog_fd || tr != prog->aux->dst_trampoline))
3272                 /* got extra prog ref from syscall, or attaching to different prog */
3273                 bpf_prog_put(prog->aux->dst_prog);
3274         if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
3275                 /* we allocated a new trampoline, so free the old one */
3276                 bpf_trampoline_put(prog->aux->dst_trampoline);
3277
3278         prog->aux->dst_prog = NULL;
3279         prog->aux->dst_trampoline = NULL;
3280         mutex_unlock(&prog->aux->dst_mutex);
3281
3282         return bpf_link_settle(&link_primer);
3283 out_unlock:
3284         if (tr && tr != prog->aux->dst_trampoline)
3285                 bpf_trampoline_put(tr);
3286         mutex_unlock(&prog->aux->dst_mutex);
3287         kfree(link);
3288 out_put_prog:
3289         if (tgt_prog_fd && tgt_prog)
3290                 bpf_prog_put(tgt_prog);
3291         return err;
3292 }
3293
3294 struct bpf_raw_tp_link {
3295         struct bpf_link link;
3296         struct bpf_raw_event_map *btp;
3297 };
3298
3299 static void bpf_raw_tp_link_release(struct bpf_link *link)
3300 {
3301         struct bpf_raw_tp_link *raw_tp =
3302                 container_of(link, struct bpf_raw_tp_link, link);
3303
3304         bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
3305         bpf_put_raw_tracepoint(raw_tp->btp);
3306 }
3307
3308 static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
3309 {
3310         struct bpf_raw_tp_link *raw_tp =
3311                 container_of(link, struct bpf_raw_tp_link, link);
3312
3313         kfree(raw_tp);
3314 }
3315
3316 static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
3317                                         struct seq_file *seq)
3318 {
3319         struct bpf_raw_tp_link *raw_tp_link =
3320                 container_of(link, struct bpf_raw_tp_link, link);
3321
3322         seq_printf(seq,
3323                    "tp_name:\t%s\n",
3324                    raw_tp_link->btp->tp->name);
3325 }
3326
3327 static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen,
3328                             u32 len)
3329 {
3330         if (ulen >= len + 1) {
3331                 if (copy_to_user(ubuf, buf, len + 1))
3332                         return -EFAULT;
3333         } else {
3334                 char zero = '\0';
3335
3336                 if (copy_to_user(ubuf, buf, ulen - 1))
3337                         return -EFAULT;
3338                 if (put_user(zero, ubuf + ulen - 1))
3339                         return -EFAULT;
3340                 return -ENOSPC;
3341         }
3342
3343         return 0;
3344 }
3345
3346 static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
3347                                           struct bpf_link_info *info)
3348 {
3349         struct bpf_raw_tp_link *raw_tp_link =
3350                 container_of(link, struct bpf_raw_tp_link, link);
3351         char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
3352         const char *tp_name = raw_tp_link->btp->tp->name;
3353         u32 ulen = info->raw_tracepoint.tp_name_len;
3354         size_t tp_len = strlen(tp_name);
3355
3356         if (!ulen ^ !ubuf)
3357                 return -EINVAL;
3358
3359         info->raw_tracepoint.tp_name_len = tp_len + 1;
3360
3361         if (!ubuf)
3362                 return 0;
3363
3364         return bpf_copy_to_user(ubuf, tp_name, ulen, tp_len);
3365 }
3366
3367 static const struct bpf_link_ops bpf_raw_tp_link_lops = {
3368         .release = bpf_raw_tp_link_release,
3369         .dealloc = bpf_raw_tp_link_dealloc,
3370         .show_fdinfo = bpf_raw_tp_link_show_fdinfo,
3371         .fill_link_info = bpf_raw_tp_link_fill_link_info,
3372 };
3373
3374 #ifdef CONFIG_PERF_EVENTS
3375 struct bpf_perf_link {
3376         struct bpf_link link;
3377         struct file *perf_file;
3378 };
3379
3380 static void bpf_perf_link_release(struct bpf_link *link)
3381 {
3382         struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3383         struct perf_event *event = perf_link->perf_file->private_data;
3384
3385         perf_event_free_bpf_prog(event);
3386         fput(perf_link->perf_file);
3387 }
3388
3389 static void bpf_perf_link_dealloc(struct bpf_link *link)
3390 {
3391         struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3392
3393         kfree(perf_link);
3394 }
3395
3396 static int bpf_perf_link_fill_common(const struct perf_event *event,
3397                                      char __user *uname, u32 ulen,
3398                                      u64 *probe_offset, u64 *probe_addr,
3399                                      u32 *fd_type)
3400 {
3401         const char *buf;
3402         u32 prog_id;
3403         size_t len;
3404         int err;
3405
3406         if (!ulen ^ !uname)
3407                 return -EINVAL;
3408
3409         err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf,
3410                                       probe_offset, probe_addr);
3411         if (err)
3412                 return err;
3413         if (!uname)
3414                 return 0;
3415         if (buf) {
3416                 len = strlen(buf);
3417                 err = bpf_copy_to_user(uname, buf, ulen, len);
3418                 if (err)
3419                         return err;
3420         } else {
3421                 char zero = '\0';
3422
3423                 if (put_user(zero, uname))
3424                         return -EFAULT;
3425         }
3426         return 0;
3427 }
3428
3429 #ifdef CONFIG_KPROBE_EVENTS
3430 static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
3431                                      struct bpf_link_info *info)
3432 {
3433         char __user *uname;
3434         u64 addr, offset;
3435         u32 ulen, type;
3436         int err;
3437
3438         uname = u64_to_user_ptr(info->perf_event.kprobe.func_name);
3439         ulen = info->perf_event.kprobe.name_len;
3440         err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr,
3441                                         &type);
3442         if (err)
3443                 return err;
3444         if (type == BPF_FD_TYPE_KRETPROBE)
3445                 info->perf_event.type = BPF_PERF_EVENT_KRETPROBE;
3446         else
3447                 info->perf_event.type = BPF_PERF_EVENT_KPROBE;
3448
3449         info->perf_event.kprobe.offset = offset;
3450         if (!kallsyms_show_value(current_cred()))
3451                 addr = 0;
3452         info->perf_event.kprobe.addr = addr;
3453         return 0;
3454 }
3455 #endif
3456
3457 #ifdef CONFIG_UPROBE_EVENTS
3458 static int bpf_perf_link_fill_uprobe(const struct perf_event *event,
3459                                      struct bpf_link_info *info)
3460 {
3461         char __user *uname;
3462         u64 addr, offset;
3463         u32 ulen, type;
3464         int err;
3465
3466         uname = u64_to_user_ptr(info->perf_event.uprobe.file_name);
3467         ulen = info->perf_event.uprobe.name_len;
3468         err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr,
3469                                         &type);
3470         if (err)
3471                 return err;
3472
3473         if (type == BPF_FD_TYPE_URETPROBE)
3474                 info->perf_event.type = BPF_PERF_EVENT_URETPROBE;
3475         else
3476                 info->perf_event.type = BPF_PERF_EVENT_UPROBE;
3477         info->perf_event.uprobe.offset = offset;
3478         return 0;
3479 }
3480 #endif
3481
3482 static int bpf_perf_link_fill_probe(const struct perf_event *event,
3483                                     struct bpf_link_info *info)
3484 {
3485 #ifdef CONFIG_KPROBE_EVENTS
3486         if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE)
3487                 return bpf_perf_link_fill_kprobe(event, info);
3488 #endif
3489 #ifdef CONFIG_UPROBE_EVENTS
3490         if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE)
3491                 return bpf_perf_link_fill_uprobe(event, info);
3492 #endif
3493         return -EOPNOTSUPP;
3494 }
3495
3496 static int bpf_perf_link_fill_tracepoint(const struct perf_event *event,
3497                                          struct bpf_link_info *info)
3498 {
3499         char __user *uname;
3500         u32 ulen;
3501
3502         uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name);
3503         ulen = info->perf_event.tracepoint.name_len;
3504         info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT;
3505         return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL);
3506 }
3507
3508 static int bpf_perf_link_fill_perf_event(const struct perf_event *event,
3509                                          struct bpf_link_info *info)
3510 {
3511         info->perf_event.event.type = event->attr.type;
3512         info->perf_event.event.config = event->attr.config;
3513         info->perf_event.type = BPF_PERF_EVENT_EVENT;
3514         return 0;
3515 }
3516
3517 static int bpf_perf_link_fill_link_info(const struct bpf_link *link,
3518                                         struct bpf_link_info *info)
3519 {
3520         struct bpf_perf_link *perf_link;
3521         const struct perf_event *event;
3522
3523         perf_link = container_of(link, struct bpf_perf_link, link);
3524         event = perf_get_event(perf_link->perf_file);
3525         if (IS_ERR(event))
3526                 return PTR_ERR(event);
3527
3528         switch (event->prog->type) {
3529         case BPF_PROG_TYPE_PERF_EVENT:
3530                 return bpf_perf_link_fill_perf_event(event, info);
3531         case BPF_PROG_TYPE_TRACEPOINT:
3532                 return bpf_perf_link_fill_tracepoint(event, info);
3533         case BPF_PROG_TYPE_KPROBE:
3534                 return bpf_perf_link_fill_probe(event, info);
3535         default:
3536                 return -EOPNOTSUPP;
3537         }
3538 }
3539
3540 static const struct bpf_link_ops bpf_perf_link_lops = {
3541         .release = bpf_perf_link_release,
3542         .dealloc = bpf_perf_link_dealloc,
3543         .fill_link_info = bpf_perf_link_fill_link_info,
3544 };
3545
3546 static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3547 {
3548         struct bpf_link_primer link_primer;
3549         struct bpf_perf_link *link;
3550         struct perf_event *event;
3551         struct file *perf_file;
3552         int err;
3553
3554         if (attr->link_create.flags)
3555                 return -EINVAL;
3556
3557         perf_file = perf_event_get(attr->link_create.target_fd);
3558         if (IS_ERR(perf_file))
3559                 return PTR_ERR(perf_file);
3560
3561         link = kzalloc(sizeof(*link), GFP_USER);
3562         if (!link) {
3563                 err = -ENOMEM;
3564                 goto out_put_file;
3565         }
3566         bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog);
3567         link->perf_file = perf_file;
3568
3569         err = bpf_link_prime(&link->link, &link_primer);
3570         if (err) {
3571                 kfree(link);
3572                 goto out_put_file;
3573         }
3574
3575         event = perf_file->private_data;
3576         err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie);
3577         if (err) {
3578                 bpf_link_cleanup(&link_primer);
3579                 goto out_put_file;
3580         }
3581         /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */
3582         bpf_prog_inc(prog);
3583
3584         return bpf_link_settle(&link_primer);
3585
3586 out_put_file:
3587         fput(perf_file);
3588         return err;
3589 }
3590 #else
3591 static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3592 {
3593         return -EOPNOTSUPP;
3594 }
3595 #endif /* CONFIG_PERF_EVENTS */
3596
3597 static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
3598                                   const char __user *user_tp_name)
3599 {
3600         struct bpf_link_primer link_primer;
3601         struct bpf_raw_tp_link *link;
3602         struct bpf_raw_event_map *btp;
3603         const char *tp_name;
3604         char buf[128];
3605         int err;
3606
3607         switch (prog->type) {
3608         case BPF_PROG_TYPE_TRACING:
3609         case BPF_PROG_TYPE_EXT:
3610         case BPF_PROG_TYPE_LSM:
3611                 if (user_tp_name)
3612                         /* The attach point for this category of programs
3613                          * should be specified via btf_id during program load.
3614                          */
3615                         return -EINVAL;
3616                 if (prog->type == BPF_PROG_TYPE_TRACING &&
3617                     prog->expected_attach_type == BPF_TRACE_RAW_TP) {
3618                         tp_name = prog->aux->attach_func_name;
3619                         break;
3620                 }
3621                 return bpf_tracing_prog_attach(prog, 0, 0, 0);
3622         case BPF_PROG_TYPE_RAW_TRACEPOINT:
3623         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
3624                 if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0)
3625                         return -EFAULT;
3626                 buf[sizeof(buf) - 1] = 0;
3627                 tp_name = buf;
3628                 break;
3629         default:
3630                 return -EINVAL;
3631         }
3632
3633         btp = bpf_get_raw_tracepoint(tp_name);
3634         if (!btp)
3635                 return -ENOENT;
3636
3637         link = kzalloc(sizeof(*link), GFP_USER);
3638         if (!link) {
3639                 err = -ENOMEM;
3640                 goto out_put_btp;
3641         }
3642         bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
3643                       &bpf_raw_tp_link_lops, prog);
3644         link->btp = btp;
3645
3646         err = bpf_link_prime(&link->link, &link_primer);
3647         if (err) {
3648                 kfree(link);
3649                 goto out_put_btp;
3650         }
3651
3652         err = bpf_probe_register(link->btp, prog);
3653         if (err) {
3654                 bpf_link_cleanup(&link_primer);
3655                 goto out_put_btp;
3656         }
3657
3658         return bpf_link_settle(&link_primer);
3659
3660 out_put_btp:
3661         bpf_put_raw_tracepoint(btp);
3662         return err;
3663 }
3664
3665 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
3666
3667 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
3668 {
3669         struct bpf_prog *prog;
3670         int fd;
3671
3672         if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
3673                 return -EINVAL;
3674
3675         prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
3676         if (IS_ERR(prog))
3677                 return PTR_ERR(prog);
3678
3679         fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name));
3680         if (fd < 0)
3681                 bpf_prog_put(prog);
3682         return fd;
3683 }
3684
3685 static enum bpf_prog_type
3686 attach_type_to_prog_type(enum bpf_attach_type attach_type)
3687 {
3688         switch (attach_type) {
3689         case BPF_CGROUP_INET_INGRESS:
3690         case BPF_CGROUP_INET_EGRESS:
3691                 return BPF_PROG_TYPE_CGROUP_SKB;
3692         case BPF_CGROUP_INET_SOCK_CREATE:
3693         case BPF_CGROUP_INET_SOCK_RELEASE:
3694         case BPF_CGROUP_INET4_POST_BIND:
3695         case BPF_CGROUP_INET6_POST_BIND:
3696                 return BPF_PROG_TYPE_CGROUP_SOCK;
3697         case BPF_CGROUP_INET4_BIND:
3698         case BPF_CGROUP_INET6_BIND:
3699         case BPF_CGROUP_INET4_CONNECT:
3700         case BPF_CGROUP_INET6_CONNECT:
3701         case BPF_CGROUP_INET4_GETPEERNAME:
3702         case BPF_CGROUP_INET6_GETPEERNAME:
3703         case BPF_CGROUP_INET4_GETSOCKNAME:
3704         case BPF_CGROUP_INET6_GETSOCKNAME:
3705         case BPF_CGROUP_UDP4_SENDMSG:
3706         case BPF_CGROUP_UDP6_SENDMSG:
3707         case BPF_CGROUP_UDP4_RECVMSG:
3708         case BPF_CGROUP_UDP6_RECVMSG:
3709                 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
3710         case BPF_CGROUP_SOCK_OPS:
3711                 return BPF_PROG_TYPE_SOCK_OPS;
3712         case BPF_CGROUP_DEVICE:
3713                 return BPF_PROG_TYPE_CGROUP_DEVICE;
3714         case BPF_SK_MSG_VERDICT:
3715                 return BPF_PROG_TYPE_SK_MSG;
3716         case BPF_SK_SKB_STREAM_PARSER:
3717         case BPF_SK_SKB_STREAM_VERDICT:
3718         case BPF_SK_SKB_VERDICT:
3719                 return BPF_PROG_TYPE_SK_SKB;
3720         case BPF_LIRC_MODE2:
3721                 return BPF_PROG_TYPE_LIRC_MODE2;
3722         case BPF_FLOW_DISSECTOR:
3723                 return BPF_PROG_TYPE_FLOW_DISSECTOR;
3724         case BPF_CGROUP_SYSCTL:
3725                 return BPF_PROG_TYPE_CGROUP_SYSCTL;
3726         case BPF_CGROUP_GETSOCKOPT:
3727         case BPF_CGROUP_SETSOCKOPT:
3728                 return BPF_PROG_TYPE_CGROUP_SOCKOPT;
3729         case BPF_TRACE_ITER:
3730         case BPF_TRACE_RAW_TP:
3731         case BPF_TRACE_FENTRY:
3732         case BPF_TRACE_FEXIT:
3733         case BPF_MODIFY_RETURN:
3734                 return BPF_PROG_TYPE_TRACING;
3735         case BPF_LSM_MAC:
3736                 return BPF_PROG_TYPE_LSM;
3737         case BPF_SK_LOOKUP:
3738                 return BPF_PROG_TYPE_SK_LOOKUP;
3739         case BPF_XDP:
3740                 return BPF_PROG_TYPE_XDP;
3741         case BPF_LSM_CGROUP:
3742                 return BPF_PROG_TYPE_LSM;
3743         case BPF_TCX_INGRESS:
3744         case BPF_TCX_EGRESS:
3745                 return BPF_PROG_TYPE_SCHED_CLS;
3746         default:
3747                 return BPF_PROG_TYPE_UNSPEC;
3748         }
3749 }
3750
3751 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
3752                                              enum bpf_attach_type attach_type)
3753 {
3754         enum bpf_prog_type ptype;
3755
3756         switch (prog->type) {
3757         case BPF_PROG_TYPE_CGROUP_SOCK:
3758         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3759         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3760         case BPF_PROG_TYPE_SK_LOOKUP:
3761                 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
3762         case BPF_PROG_TYPE_CGROUP_SKB:
3763                 if (!capable(CAP_NET_ADMIN))
3764                         /* cg-skb progs can be loaded by unpriv user.
3765                          * check permissions at attach time.
3766                          */
3767                         return -EPERM;
3768                 return prog->enforce_expected_attach_type &&
3769                         prog->expected_attach_type != attach_type ?
3770                         -EINVAL : 0;
3771         case BPF_PROG_TYPE_EXT:
3772                 return 0;
3773         case BPF_PROG_TYPE_NETFILTER:
3774                 if (attach_type != BPF_NETFILTER)
3775                         return -EINVAL;
3776                 return 0;
3777         case BPF_PROG_TYPE_PERF_EVENT:
3778         case BPF_PROG_TYPE_TRACEPOINT:
3779                 if (attach_type != BPF_PERF_EVENT)
3780                         return -EINVAL;
3781                 return 0;
3782         case BPF_PROG_TYPE_KPROBE:
3783                 if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI &&
3784                     attach_type != BPF_TRACE_KPROBE_MULTI)
3785                         return -EINVAL;
3786                 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
3787                     attach_type != BPF_TRACE_UPROBE_MULTI)
3788                         return -EINVAL;
3789                 if (attach_type != BPF_PERF_EVENT &&
3790                     attach_type != BPF_TRACE_KPROBE_MULTI &&
3791                     attach_type != BPF_TRACE_UPROBE_MULTI)
3792                         return -EINVAL;
3793                 return 0;
3794         case BPF_PROG_TYPE_SCHED_CLS:
3795                 if (attach_type != BPF_TCX_INGRESS &&
3796                     attach_type != BPF_TCX_EGRESS)
3797                         return -EINVAL;
3798                 return 0;
3799         default:
3800                 ptype = attach_type_to_prog_type(attach_type);
3801                 if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type)
3802                         return -EINVAL;
3803                 return 0;
3804         }
3805 }
3806
3807 #define BPF_PROG_ATTACH_LAST_FIELD expected_revision
3808
3809 #define BPF_F_ATTACH_MASK_BASE  \
3810         (BPF_F_ALLOW_OVERRIDE | \
3811          BPF_F_ALLOW_MULTI |    \
3812          BPF_F_REPLACE)
3813
3814 #define BPF_F_ATTACH_MASK_MPROG \
3815         (BPF_F_REPLACE |        \
3816          BPF_F_BEFORE |         \
3817          BPF_F_AFTER |          \
3818          BPF_F_ID |             \
3819          BPF_F_LINK)
3820
3821 static int bpf_prog_attach(const union bpf_attr *attr)
3822 {
3823         enum bpf_prog_type ptype;
3824         struct bpf_prog *prog;
3825         int ret;
3826
3827         if (CHECK_ATTR(BPF_PROG_ATTACH))
3828                 return -EINVAL;
3829
3830         ptype = attach_type_to_prog_type(attr->attach_type);
3831         if (ptype == BPF_PROG_TYPE_UNSPEC)
3832                 return -EINVAL;
3833         if (bpf_mprog_supported(ptype)) {
3834                 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
3835                         return -EINVAL;
3836         } else {
3837                 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_BASE)
3838                         return -EINVAL;
3839                 if (attr->relative_fd ||
3840                     attr->expected_revision)
3841                         return -EINVAL;
3842         }
3843
3844         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
3845         if (IS_ERR(prog))
3846                 return PTR_ERR(prog);
3847
3848         if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
3849                 bpf_prog_put(prog);
3850                 return -EINVAL;
3851         }
3852
3853         switch (ptype) {
3854         case BPF_PROG_TYPE_SK_SKB:
3855         case BPF_PROG_TYPE_SK_MSG:
3856                 ret = sock_map_get_from_fd(attr, prog);
3857                 break;
3858         case BPF_PROG_TYPE_LIRC_MODE2:
3859                 ret = lirc_prog_attach(attr, prog);
3860                 break;
3861         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3862                 ret = netns_bpf_prog_attach(attr, prog);
3863                 break;
3864         case BPF_PROG_TYPE_CGROUP_DEVICE:
3865         case BPF_PROG_TYPE_CGROUP_SKB:
3866         case BPF_PROG_TYPE_CGROUP_SOCK:
3867         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3868         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3869         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3870         case BPF_PROG_TYPE_SOCK_OPS:
3871         case BPF_PROG_TYPE_LSM:
3872                 if (ptype == BPF_PROG_TYPE_LSM &&
3873                     prog->expected_attach_type != BPF_LSM_CGROUP)
3874                         ret = -EINVAL;
3875                 else
3876                         ret = cgroup_bpf_prog_attach(attr, ptype, prog);
3877                 break;
3878         case BPF_PROG_TYPE_SCHED_CLS:
3879                 ret = tcx_prog_attach(attr, prog);
3880                 break;
3881         default:
3882                 ret = -EINVAL;
3883         }
3884
3885         if (ret)
3886                 bpf_prog_put(prog);
3887         return ret;
3888 }
3889
3890 #define BPF_PROG_DETACH_LAST_FIELD expected_revision
3891
3892 static int bpf_prog_detach(const union bpf_attr *attr)
3893 {
3894         struct bpf_prog *prog = NULL;
3895         enum bpf_prog_type ptype;
3896         int ret;
3897
3898         if (CHECK_ATTR(BPF_PROG_DETACH))
3899                 return -EINVAL;
3900
3901         ptype = attach_type_to_prog_type(attr->attach_type);
3902         if (bpf_mprog_supported(ptype)) {
3903                 if (ptype == BPF_PROG_TYPE_UNSPEC)
3904                         return -EINVAL;
3905                 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
3906                         return -EINVAL;
3907                 if (attr->attach_bpf_fd) {
3908                         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
3909                         if (IS_ERR(prog))
3910                                 return PTR_ERR(prog);
3911                 }
3912         } else if (attr->attach_flags ||
3913                    attr->relative_fd ||
3914                    attr->expected_revision) {
3915                 return -EINVAL;
3916         }
3917
3918         switch (ptype) {
3919         case BPF_PROG_TYPE_SK_MSG:
3920         case BPF_PROG_TYPE_SK_SKB:
3921                 ret = sock_map_prog_detach(attr, ptype);
3922                 break;
3923         case BPF_PROG_TYPE_LIRC_MODE2:
3924                 ret = lirc_prog_detach(attr);
3925                 break;
3926         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3927                 ret = netns_bpf_prog_detach(attr, ptype);
3928                 break;
3929         case BPF_PROG_TYPE_CGROUP_DEVICE:
3930         case BPF_PROG_TYPE_CGROUP_SKB:
3931         case BPF_PROG_TYPE_CGROUP_SOCK:
3932         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3933         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3934         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3935         case BPF_PROG_TYPE_SOCK_OPS:
3936         case BPF_PROG_TYPE_LSM:
3937                 ret = cgroup_bpf_prog_detach(attr, ptype);
3938                 break;
3939         case BPF_PROG_TYPE_SCHED_CLS:
3940                 ret = tcx_prog_detach(attr, prog);
3941                 break;
3942         default:
3943                 ret = -EINVAL;
3944         }
3945
3946         if (prog)
3947                 bpf_prog_put(prog);
3948         return ret;
3949 }
3950
3951 #define BPF_PROG_QUERY_LAST_FIELD query.revision
3952
3953 static int bpf_prog_query(const union bpf_attr *attr,
3954                           union bpf_attr __user *uattr)
3955 {
3956         if (!capable(CAP_NET_ADMIN))
3957                 return -EPERM;
3958         if (CHECK_ATTR(BPF_PROG_QUERY))
3959                 return -EINVAL;
3960         if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
3961                 return -EINVAL;
3962
3963         switch (attr->query.attach_type) {
3964         case BPF_CGROUP_INET_INGRESS:
3965         case BPF_CGROUP_INET_EGRESS:
3966         case BPF_CGROUP_INET_SOCK_CREATE:
3967         case BPF_CGROUP_INET_SOCK_RELEASE:
3968         case BPF_CGROUP_INET4_BIND:
3969         case BPF_CGROUP_INET6_BIND:
3970         case BPF_CGROUP_INET4_POST_BIND:
3971         case BPF_CGROUP_INET6_POST_BIND:
3972         case BPF_CGROUP_INET4_CONNECT:
3973         case BPF_CGROUP_INET6_CONNECT:
3974         case BPF_CGROUP_INET4_GETPEERNAME:
3975         case BPF_CGROUP_INET6_GETPEERNAME:
3976         case BPF_CGROUP_INET4_GETSOCKNAME:
3977         case BPF_CGROUP_INET6_GETSOCKNAME:
3978         case BPF_CGROUP_UDP4_SENDMSG:
3979         case BPF_CGROUP_UDP6_SENDMSG:
3980         case BPF_CGROUP_UDP4_RECVMSG:
3981         case BPF_CGROUP_UDP6_RECVMSG:
3982         case BPF_CGROUP_SOCK_OPS:
3983         case BPF_CGROUP_DEVICE:
3984         case BPF_CGROUP_SYSCTL:
3985         case BPF_CGROUP_GETSOCKOPT:
3986         case BPF_CGROUP_SETSOCKOPT:
3987         case BPF_LSM_CGROUP:
3988                 return cgroup_bpf_prog_query(attr, uattr);
3989         case BPF_LIRC_MODE2:
3990                 return lirc_prog_query(attr, uattr);
3991         case BPF_FLOW_DISSECTOR:
3992         case BPF_SK_LOOKUP:
3993                 return netns_bpf_prog_query(attr, uattr);
3994         case BPF_SK_SKB_STREAM_PARSER:
3995         case BPF_SK_SKB_STREAM_VERDICT:
3996         case BPF_SK_MSG_VERDICT:
3997         case BPF_SK_SKB_VERDICT:
3998                 return sock_map_bpf_prog_query(attr, uattr);
3999         case BPF_TCX_INGRESS:
4000         case BPF_TCX_EGRESS:
4001                 return tcx_prog_query(attr, uattr);
4002         default:
4003                 return -EINVAL;
4004         }
4005 }
4006
4007 #define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size
4008
4009 static int bpf_prog_test_run(const union bpf_attr *attr,
4010                              union bpf_attr __user *uattr)
4011 {
4012         struct bpf_prog *prog;
4013         int ret = -ENOTSUPP;
4014
4015         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
4016                 return -EINVAL;
4017
4018         if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
4019             (!attr->test.ctx_size_in && attr->test.ctx_in))
4020                 return -EINVAL;
4021
4022         if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
4023             (!attr->test.ctx_size_out && attr->test.ctx_out))
4024                 return -EINVAL;
4025
4026         prog = bpf_prog_get(attr->test.prog_fd);
4027         if (IS_ERR(prog))
4028                 return PTR_ERR(prog);
4029
4030         if (prog->aux->ops->test_run)
4031                 ret = prog->aux->ops->test_run(prog, attr, uattr);
4032
4033         bpf_prog_put(prog);
4034         return ret;
4035 }
4036
4037 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
4038
4039 static int bpf_obj_get_next_id(const union bpf_attr *attr,
4040                                union bpf_attr __user *uattr,
4041                                struct idr *idr,
4042                                spinlock_t *lock)
4043 {
4044         u32 next_id = attr->start_id;
4045         int err = 0;
4046
4047         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
4048                 return -EINVAL;
4049
4050         if (!capable(CAP_SYS_ADMIN))
4051                 return -EPERM;
4052
4053         next_id++;
4054         spin_lock_bh(lock);
4055         if (!idr_get_next(idr, &next_id))
4056                 err = -ENOENT;
4057         spin_unlock_bh(lock);
4058
4059         if (!err)
4060                 err = put_user(next_id, &uattr->next_id);
4061
4062         return err;
4063 }
4064
4065 struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
4066 {
4067         struct bpf_map *map;
4068
4069         spin_lock_bh(&map_idr_lock);
4070 again:
4071         map = idr_get_next(&map_idr, id);
4072         if (map) {
4073                 map = __bpf_map_inc_not_zero(map, false);
4074                 if (IS_ERR(map)) {
4075                         (*id)++;
4076                         goto again;
4077                 }
4078         }
4079         spin_unlock_bh(&map_idr_lock);
4080
4081         return map;
4082 }
4083
4084 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
4085 {
4086         struct bpf_prog *prog;
4087
4088         spin_lock_bh(&prog_idr_lock);
4089 again:
4090         prog = idr_get_next(&prog_idr, id);
4091         if (prog) {
4092                 prog = bpf_prog_inc_not_zero(prog);
4093                 if (IS_ERR(prog)) {
4094                         (*id)++;
4095                         goto again;
4096                 }
4097         }
4098         spin_unlock_bh(&prog_idr_lock);
4099
4100         return prog;
4101 }
4102
4103 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
4104
4105 struct bpf_prog *bpf_prog_by_id(u32 id)
4106 {
4107         struct bpf_prog *prog;
4108
4109         if (!id)
4110                 return ERR_PTR(-ENOENT);
4111
4112         spin_lock_bh(&prog_idr_lock);
4113         prog = idr_find(&prog_idr, id);
4114         if (prog)
4115                 prog = bpf_prog_inc_not_zero(prog);
4116         else
4117                 prog = ERR_PTR(-ENOENT);
4118         spin_unlock_bh(&prog_idr_lock);
4119         return prog;
4120 }
4121
4122 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
4123 {
4124         struct bpf_prog *prog;
4125         u32 id = attr->prog_id;
4126         int fd;
4127
4128         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
4129                 return -EINVAL;
4130
4131         if (!capable(CAP_SYS_ADMIN))
4132                 return -EPERM;
4133
4134         prog = bpf_prog_by_id(id);
4135         if (IS_ERR(prog))
4136                 return PTR_ERR(prog);
4137
4138         fd = bpf_prog_new_fd(prog);
4139         if (fd < 0)
4140                 bpf_prog_put(prog);
4141
4142         return fd;
4143 }
4144
4145 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
4146
4147 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
4148 {
4149         struct bpf_map *map;
4150         u32 id = attr->map_id;
4151         int f_flags;
4152         int fd;
4153
4154         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
4155             attr->open_flags & ~BPF_OBJ_FLAG_MASK)
4156                 return -EINVAL;
4157
4158         if (!capable(CAP_SYS_ADMIN))
4159                 return -EPERM;
4160
4161         f_flags = bpf_get_file_flag(attr->open_flags);
4162         if (f_flags < 0)
4163                 return f_flags;
4164
4165         spin_lock_bh(&map_idr_lock);
4166         map = idr_find(&map_idr, id);
4167         if (map)
4168                 map = __bpf_map_inc_not_zero(map, true);
4169         else
4170                 map = ERR_PTR(-ENOENT);
4171         spin_unlock_bh(&map_idr_lock);
4172
4173         if (IS_ERR(map))
4174                 return PTR_ERR(map);
4175
4176         fd = bpf_map_new_fd(map, f_flags);
4177         if (fd < 0)
4178                 bpf_map_put_with_uref(map);
4179
4180         return fd;
4181 }
4182
4183 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
4184                                               unsigned long addr, u32 *off,
4185                                               u32 *type)
4186 {
4187         const struct bpf_map *map;
4188         int i;
4189
4190         mutex_lock(&prog->aux->used_maps_mutex);
4191         for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
4192                 map = prog->aux->used_maps[i];
4193                 if (map == (void *)addr) {
4194                         *type = BPF_PSEUDO_MAP_FD;
4195                         goto out;
4196                 }
4197                 if (!map->ops->map_direct_value_meta)
4198                         continue;
4199                 if (!map->ops->map_direct_value_meta(map, addr, off)) {
4200                         *type = BPF_PSEUDO_MAP_VALUE;
4201                         goto out;
4202                 }
4203         }
4204         map = NULL;
4205
4206 out:
4207         mutex_unlock(&prog->aux->used_maps_mutex);
4208         return map;
4209 }
4210
4211 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
4212                                               const struct cred *f_cred)
4213 {
4214         const struct bpf_map *map;
4215         struct bpf_insn *insns;
4216         u32 off, type;
4217         u64 imm;
4218         u8 code;
4219         int i;
4220
4221         insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
4222                         GFP_USER);
4223         if (!insns)
4224                 return insns;
4225
4226         for (i = 0; i < prog->len; i++) {
4227                 code = insns[i].code;
4228
4229                 if (code == (BPF_JMP | BPF_TAIL_CALL)) {
4230                         insns[i].code = BPF_JMP | BPF_CALL;
4231                         insns[i].imm = BPF_FUNC_tail_call;
4232                         /* fall-through */
4233                 }
4234                 if (code == (BPF_JMP | BPF_CALL) ||
4235                     code == (BPF_JMP | BPF_CALL_ARGS)) {
4236                         if (code == (BPF_JMP | BPF_CALL_ARGS))
4237                                 insns[i].code = BPF_JMP | BPF_CALL;
4238                         if (!bpf_dump_raw_ok(f_cred))
4239                                 insns[i].imm = 0;
4240                         continue;
4241                 }
4242                 if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
4243                         insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
4244                         continue;
4245                 }
4246
4247                 if (code != (BPF_LD | BPF_IMM | BPF_DW))
4248                         continue;
4249
4250                 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
4251                 map = bpf_map_from_imm(prog, imm, &off, &type);
4252                 if (map) {
4253                         insns[i].src_reg = type;
4254                         insns[i].imm = map->id;
4255                         insns[i + 1].imm = off;
4256                         continue;
4257                 }
4258         }
4259
4260         return insns;
4261 }
4262
4263 static int set_info_rec_size(struct bpf_prog_info *info)
4264 {
4265         /*
4266          * Ensure info.*_rec_size is the same as kernel expected size
4267          *
4268          * or
4269          *
4270          * Only allow zero *_rec_size if both _rec_size and _cnt are
4271          * zero.  In this case, the kernel will set the expected
4272          * _rec_size back to the info.
4273          */
4274
4275         if ((info->nr_func_info || info->func_info_rec_size) &&
4276             info->func_info_rec_size != sizeof(struct bpf_func_info))
4277                 return -EINVAL;
4278
4279         if ((info->nr_line_info || info->line_info_rec_size) &&
4280             info->line_info_rec_size != sizeof(struct bpf_line_info))
4281                 return -EINVAL;
4282
4283         if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
4284             info->jited_line_info_rec_size != sizeof(__u64))
4285                 return -EINVAL;
4286
4287         info->func_info_rec_size = sizeof(struct bpf_func_info);
4288         info->line_info_rec_size = sizeof(struct bpf_line_info);
4289         info->jited_line_info_rec_size = sizeof(__u64);
4290
4291         return 0;
4292 }
4293
4294 static int bpf_prog_get_info_by_fd(struct file *file,
4295                                    struct bpf_prog *prog,
4296                                    const union bpf_attr *attr,
4297                                    union bpf_attr __user *uattr)
4298 {
4299         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
4300         struct btf *attach_btf = bpf_prog_get_target_btf(prog);
4301         struct bpf_prog_info info;
4302         u32 info_len = attr->info.info_len;
4303         struct bpf_prog_kstats stats;
4304         char __user *uinsns;
4305         u32 ulen;
4306         int err;
4307
4308         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
4309         if (err)
4310                 return err;
4311         info_len = min_t(u32, sizeof(info), info_len);
4312
4313         memset(&info, 0, sizeof(info));
4314         if (copy_from_user(&info, uinfo, info_len))
4315                 return -EFAULT;
4316
4317         info.type = prog->type;
4318         info.id = prog->aux->id;
4319         info.load_time = prog->aux->load_time;
4320         info.created_by_uid = from_kuid_munged(current_user_ns(),
4321                                                prog->aux->user->uid);
4322         info.gpl_compatible = prog->gpl_compatible;
4323
4324         memcpy(info.tag, prog->tag, sizeof(prog->tag));
4325         memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
4326
4327         mutex_lock(&prog->aux->used_maps_mutex);
4328         ulen = info.nr_map_ids;
4329         info.nr_map_ids = prog->aux->used_map_cnt;
4330         ulen = min_t(u32, info.nr_map_ids, ulen);
4331         if (ulen) {
4332                 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
4333                 u32 i;
4334
4335                 for (i = 0; i < ulen; i++)
4336                         if (put_user(prog->aux->used_maps[i]->id,
4337                                      &user_map_ids[i])) {
4338                                 mutex_unlock(&prog->aux->used_maps_mutex);
4339                                 return -EFAULT;
4340                         }
4341         }
4342         mutex_unlock(&prog->aux->used_maps_mutex);
4343
4344         err = set_info_rec_size(&info);
4345         if (err)
4346                 return err;
4347
4348         bpf_prog_get_stats(prog, &stats);
4349         info.run_time_ns = stats.nsecs;
4350         info.run_cnt = stats.cnt;
4351         info.recursion_misses = stats.misses;
4352
4353         info.verified_insns = prog->aux->verified_insns;
4354
4355         if (!bpf_capable()) {
4356                 info.jited_prog_len = 0;
4357                 info.xlated_prog_len = 0;
4358                 info.nr_jited_ksyms = 0;
4359                 info.nr_jited_func_lens = 0;
4360                 info.nr_func_info = 0;
4361                 info.nr_line_info = 0;
4362                 info.nr_jited_line_info = 0;
4363                 goto done;
4364         }
4365
4366         ulen = info.xlated_prog_len;
4367         info.xlated_prog_len = bpf_prog_insn_size(prog);
4368         if (info.xlated_prog_len && ulen) {
4369                 struct bpf_insn *insns_sanitized;
4370                 bool fault;
4371
4372                 if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) {
4373                         info.xlated_prog_insns = 0;
4374                         goto done;
4375                 }
4376                 insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
4377                 if (!insns_sanitized)
4378                         return -ENOMEM;
4379                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
4380                 ulen = min_t(u32, info.xlated_prog_len, ulen);
4381                 fault = copy_to_user(uinsns, insns_sanitized, ulen);
4382                 kfree(insns_sanitized);
4383                 if (fault)
4384                         return -EFAULT;
4385         }
4386
4387         if (bpf_prog_is_offloaded(prog->aux)) {
4388                 err = bpf_prog_offload_info_fill(&info, prog);
4389                 if (err)
4390                         return err;
4391                 goto done;
4392         }
4393
4394         /* NOTE: the following code is supposed to be skipped for offload.
4395          * bpf_prog_offload_info_fill() is the place to fill similar fields
4396          * for offload.
4397          */
4398         ulen = info.jited_prog_len;
4399         if (prog->aux->func_cnt) {
4400                 u32 i;
4401
4402                 info.jited_prog_len = 0;
4403                 for (i = 0; i < prog->aux->func_cnt; i++)
4404                         info.jited_prog_len += prog->aux->func[i]->jited_len;
4405         } else {
4406                 info.jited_prog_len = prog->jited_len;
4407         }
4408
4409         if (info.jited_prog_len && ulen) {
4410                 if (bpf_dump_raw_ok(file->f_cred)) {
4411                         uinsns = u64_to_user_ptr(info.jited_prog_insns);
4412                         ulen = min_t(u32, info.jited_prog_len, ulen);
4413
4414                         /* for multi-function programs, copy the JITed
4415                          * instructions for all the functions
4416                          */
4417                         if (prog->aux->func_cnt) {
4418                                 u32 len, free, i;
4419                                 u8 *img;
4420
4421                                 free = ulen;
4422                                 for (i = 0; i < prog->aux->func_cnt; i++) {
4423                                         len = prog->aux->func[i]->jited_len;
4424                                         len = min_t(u32, len, free);
4425                                         img = (u8 *) prog->aux->func[i]->bpf_func;
4426                                         if (copy_to_user(uinsns, img, len))
4427                                                 return -EFAULT;
4428                                         uinsns += len;
4429                                         free -= len;
4430                                         if (!free)
4431                                                 break;
4432                                 }
4433                         } else {
4434                                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
4435                                         return -EFAULT;
4436                         }
4437                 } else {
4438                         info.jited_prog_insns = 0;
4439                 }
4440         }
4441
4442         ulen = info.nr_jited_ksyms;
4443         info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
4444         if (ulen) {
4445                 if (bpf_dump_raw_ok(file->f_cred)) {
4446                         unsigned long ksym_addr;
4447                         u64 __user *user_ksyms;
4448                         u32 i;
4449
4450                         /* copy the address of the kernel symbol
4451                          * corresponding to each function
4452                          */
4453                         ulen = min_t(u32, info.nr_jited_ksyms, ulen);
4454                         user_ksyms = u64_to_user_ptr(info.jited_ksyms);
4455                         if (prog->aux->func_cnt) {
4456                                 for (i = 0; i < ulen; i++) {
4457                                         ksym_addr = (unsigned long)
4458                                                 prog->aux->func[i]->bpf_func;
4459                                         if (put_user((u64) ksym_addr,
4460                                                      &user_ksyms[i]))
4461                                                 return -EFAULT;
4462                                 }
4463                         } else {
4464                                 ksym_addr = (unsigned long) prog->bpf_func;
4465                                 if (put_user((u64) ksym_addr, &user_ksyms[0]))
4466                                         return -EFAULT;
4467                         }
4468                 } else {
4469                         info.jited_ksyms = 0;
4470                 }
4471         }
4472
4473         ulen = info.nr_jited_func_lens;
4474         info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
4475         if (ulen) {
4476                 if (bpf_dump_raw_ok(file->f_cred)) {
4477                         u32 __user *user_lens;
4478                         u32 func_len, i;
4479
4480                         /* copy the JITed image lengths for each function */
4481                         ulen = min_t(u32, info.nr_jited_func_lens, ulen);
4482                         user_lens = u64_to_user_ptr(info.jited_func_lens);
4483                         if (prog->aux->func_cnt) {
4484                                 for (i = 0; i < ulen; i++) {
4485                                         func_len =
4486                                                 prog->aux->func[i]->jited_len;
4487                                         if (put_user(func_len, &user_lens[i]))
4488                                                 return -EFAULT;
4489                                 }
4490                         } else {
4491                                 func_len = prog->jited_len;
4492                                 if (put_user(func_len, &user_lens[0]))
4493                                         return -EFAULT;
4494                         }
4495                 } else {
4496                         info.jited_func_lens = 0;
4497                 }
4498         }
4499
4500         if (prog->aux->btf)
4501                 info.btf_id = btf_obj_id(prog->aux->btf);
4502         info.attach_btf_id = prog->aux->attach_btf_id;
4503         if (attach_btf)
4504                 info.attach_btf_obj_id = btf_obj_id(attach_btf);
4505
4506         ulen = info.nr_func_info;
4507         info.nr_func_info = prog->aux->func_info_cnt;
4508         if (info.nr_func_info && ulen) {
4509                 char __user *user_finfo;
4510
4511                 user_finfo = u64_to_user_ptr(info.func_info);
4512                 ulen = min_t(u32, info.nr_func_info, ulen);
4513                 if (copy_to_user(user_finfo, prog->aux->func_info,
4514                                  info.func_info_rec_size * ulen))
4515                         return -EFAULT;
4516         }
4517
4518         ulen = info.nr_line_info;
4519         info.nr_line_info = prog->aux->nr_linfo;
4520         if (info.nr_line_info && ulen) {
4521                 __u8 __user *user_linfo;
4522
4523                 user_linfo = u64_to_user_ptr(info.line_info);
4524                 ulen = min_t(u32, info.nr_line_info, ulen);
4525                 if (copy_to_user(user_linfo, prog->aux->linfo,
4526                                  info.line_info_rec_size * ulen))
4527                         return -EFAULT;
4528         }
4529
4530         ulen = info.nr_jited_line_info;
4531         if (prog->aux->jited_linfo)
4532                 info.nr_jited_line_info = prog->aux->nr_linfo;
4533         else
4534                 info.nr_jited_line_info = 0;
4535         if (info.nr_jited_line_info && ulen) {
4536                 if (bpf_dump_raw_ok(file->f_cred)) {
4537                         unsigned long line_addr;
4538                         __u64 __user *user_linfo;
4539                         u32 i;
4540
4541                         user_linfo = u64_to_user_ptr(info.jited_line_info);
4542                         ulen = min_t(u32, info.nr_jited_line_info, ulen);
4543                         for (i = 0; i < ulen; i++) {
4544                                 line_addr = (unsigned long)prog->aux->jited_linfo[i];
4545                                 if (put_user((__u64)line_addr, &user_linfo[i]))
4546                                         return -EFAULT;
4547                         }
4548                 } else {
4549                         info.jited_line_info = 0;
4550                 }
4551         }
4552
4553         ulen = info.nr_prog_tags;
4554         info.nr_prog_tags = prog->aux->func_cnt ? : 1;
4555         if (ulen) {
4556                 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
4557                 u32 i;
4558
4559                 user_prog_tags = u64_to_user_ptr(info.prog_tags);
4560                 ulen = min_t(u32, info.nr_prog_tags, ulen);
4561                 if (prog->aux->func_cnt) {
4562                         for (i = 0; i < ulen; i++) {
4563                                 if (copy_to_user(user_prog_tags[i],
4564                                                  prog->aux->func[i]->tag,
4565                                                  BPF_TAG_SIZE))
4566                                         return -EFAULT;
4567                         }
4568                 } else {
4569                         if (copy_to_user(user_prog_tags[0],
4570                                          prog->tag, BPF_TAG_SIZE))
4571                                 return -EFAULT;
4572                 }
4573         }
4574
4575 done:
4576         if (copy_to_user(uinfo, &info, info_len) ||
4577             put_user(info_len, &uattr->info.info_len))
4578                 return -EFAULT;
4579
4580         return 0;
4581 }
4582
4583 static int bpf_map_get_info_by_fd(struct file *file,
4584                                   struct bpf_map *map,
4585                                   const union bpf_attr *attr,
4586                                   union bpf_attr __user *uattr)
4587 {
4588         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
4589         struct bpf_map_info info;
4590         u32 info_len = attr->info.info_len;
4591         int err;
4592
4593         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
4594         if (err)
4595                 return err;
4596         info_len = min_t(u32, sizeof(info), info_len);
4597
4598         memset(&info, 0, sizeof(info));
4599         info.type = map->map_type;
4600         info.id = map->id;
4601         info.key_size = map->key_size;
4602         info.value_size = map->value_size;
4603         info.max_entries = map->max_entries;
4604         info.map_flags = map->map_flags;
4605         info.map_extra = map->map_extra;
4606         memcpy(info.name, map->name, sizeof(map->name));
4607
4608         if (map->btf) {
4609                 info.btf_id = btf_obj_id(map->btf);
4610                 info.btf_key_type_id = map->btf_key_type_id;
4611                 info.btf_value_type_id = map->btf_value_type_id;
4612         }
4613         info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
4614
4615         if (bpf_map_is_offloaded(map)) {
4616                 err = bpf_map_offload_info_fill(&info, map);
4617                 if (err)
4618                         return err;
4619         }
4620
4621         if (copy_to_user(uinfo, &info, info_len) ||
4622             put_user(info_len, &uattr->info.info_len))
4623                 return -EFAULT;
4624
4625         return 0;
4626 }
4627
4628 static int bpf_btf_get_info_by_fd(struct file *file,
4629                                   struct btf *btf,
4630                                   const union bpf_attr *attr,
4631                                   union bpf_attr __user *uattr)
4632 {
4633         struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
4634         u32 info_len = attr->info.info_len;
4635         int err;
4636
4637         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
4638         if (err)
4639                 return err;
4640
4641         return btf_get_info_by_fd(btf, attr, uattr);
4642 }
4643
4644 static int bpf_link_get_info_by_fd(struct file *file,
4645                                   struct bpf_link *link,
4646                                   const union bpf_attr *attr,
4647                                   union bpf_attr __user *uattr)
4648 {
4649         struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
4650         struct bpf_link_info info;
4651         u32 info_len = attr->info.info_len;
4652         int err;
4653
4654         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
4655         if (err)
4656                 return err;
4657         info_len = min_t(u32, sizeof(info), info_len);
4658
4659         memset(&info, 0, sizeof(info));
4660         if (copy_from_user(&info, uinfo, info_len))
4661                 return -EFAULT;
4662
4663         info.type = link->type;
4664         info.id = link->id;
4665         if (link->prog)
4666                 info.prog_id = link->prog->aux->id;
4667
4668         if (link->ops->fill_link_info) {
4669                 err = link->ops->fill_link_info(link, &info);
4670                 if (err)
4671                         return err;
4672         }
4673
4674         if (copy_to_user(uinfo, &info, info_len) ||
4675             put_user(info_len, &uattr->info.info_len))
4676                 return -EFAULT;
4677
4678         return 0;
4679 }
4680
4681
4682 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
4683
4684 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
4685                                   union bpf_attr __user *uattr)
4686 {
4687         int ufd = attr->info.bpf_fd;
4688         struct fd f;
4689         int err;
4690
4691         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
4692                 return -EINVAL;
4693
4694         f = fdget(ufd);
4695         if (!f.file)
4696                 return -EBADFD;
4697
4698         if (f.file->f_op == &bpf_prog_fops)
4699                 err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr,
4700                                               uattr);
4701         else if (f.file->f_op == &bpf_map_fops)
4702                 err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr,
4703                                              uattr);
4704         else if (f.file->f_op == &btf_fops)
4705                 err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr);
4706         else if (f.file->f_op == &bpf_link_fops)
4707                 err = bpf_link_get_info_by_fd(f.file, f.file->private_data,
4708                                               attr, uattr);
4709         else
4710                 err = -EINVAL;
4711
4712         fdput(f);
4713         return err;
4714 }
4715
4716 #define BPF_BTF_LOAD_LAST_FIELD btf_log_true_size
4717
4718 static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
4719 {
4720         if (CHECK_ATTR(BPF_BTF_LOAD))
4721                 return -EINVAL;
4722
4723         if (!bpf_capable())
4724                 return -EPERM;
4725
4726         return btf_new_fd(attr, uattr, uattr_size);
4727 }
4728
4729 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
4730
4731 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
4732 {
4733         if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
4734                 return -EINVAL;
4735
4736         if (!capable(CAP_SYS_ADMIN))
4737                 return -EPERM;
4738
4739         return btf_get_fd_by_id(attr->btf_id);
4740 }
4741
4742 static int bpf_task_fd_query_copy(const union bpf_attr *attr,
4743                                     union bpf_attr __user *uattr,
4744                                     u32 prog_id, u32 fd_type,
4745                                     const char *buf, u64 probe_offset,
4746                                     u64 probe_addr)
4747 {
4748         char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
4749         u32 len = buf ? strlen(buf) : 0, input_len;
4750         int err = 0;
4751
4752         if (put_user(len, &uattr->task_fd_query.buf_len))
4753                 return -EFAULT;
4754         input_len = attr->task_fd_query.buf_len;
4755         if (input_len && ubuf) {
4756                 if (!len) {
4757                         /* nothing to copy, just make ubuf NULL terminated */
4758                         char zero = '\0';
4759
4760                         if (put_user(zero, ubuf))
4761                                 return -EFAULT;
4762                 } else if (input_len >= len + 1) {
4763                         /* ubuf can hold the string with NULL terminator */
4764                         if (copy_to_user(ubuf, buf, len + 1))
4765                                 return -EFAULT;
4766                 } else {
4767                         /* ubuf cannot hold the string with NULL terminator,
4768                          * do a partial copy with NULL terminator.
4769                          */
4770                         char zero = '\0';
4771
4772                         err = -ENOSPC;
4773                         if (copy_to_user(ubuf, buf, input_len - 1))
4774                                 return -EFAULT;
4775                         if (put_user(zero, ubuf + input_len - 1))
4776                                 return -EFAULT;
4777                 }
4778         }
4779
4780         if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
4781             put_user(fd_type, &uattr->task_fd_query.fd_type) ||
4782             put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
4783             put_user(probe_addr, &uattr->task_fd_query.probe_addr))
4784                 return -EFAULT;
4785
4786         return err;
4787 }
4788
4789 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
4790
4791 static int bpf_task_fd_query(const union bpf_attr *attr,
4792                              union bpf_attr __user *uattr)
4793 {
4794         pid_t pid = attr->task_fd_query.pid;
4795         u32 fd = attr->task_fd_query.fd;
4796         const struct perf_event *event;
4797         struct task_struct *task;
4798         struct file *file;
4799         int err;
4800
4801         if (CHECK_ATTR(BPF_TASK_FD_QUERY))
4802                 return -EINVAL;
4803
4804         if (!capable(CAP_SYS_ADMIN))
4805                 return -EPERM;
4806
4807         if (attr->task_fd_query.flags != 0)
4808                 return -EINVAL;
4809
4810         rcu_read_lock();
4811         task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
4812         rcu_read_unlock();
4813         if (!task)
4814                 return -ENOENT;
4815
4816         err = 0;
4817         file = fget_task(task, fd);
4818         put_task_struct(task);
4819         if (!file)
4820                 return -EBADF;
4821
4822         if (file->f_op == &bpf_link_fops) {
4823                 struct bpf_link *link = file->private_data;
4824
4825                 if (link->ops == &bpf_raw_tp_link_lops) {
4826                         struct bpf_raw_tp_link *raw_tp =
4827                                 container_of(link, struct bpf_raw_tp_link, link);
4828                         struct bpf_raw_event_map *btp = raw_tp->btp;
4829
4830                         err = bpf_task_fd_query_copy(attr, uattr,
4831                                                      raw_tp->link.prog->aux->id,
4832                                                      BPF_FD_TYPE_RAW_TRACEPOINT,
4833                                                      btp->tp->name, 0, 0);
4834                         goto put_file;
4835                 }
4836                 goto out_not_supp;
4837         }
4838
4839         event = perf_get_event(file);
4840         if (!IS_ERR(event)) {
4841                 u64 probe_offset, probe_addr;
4842                 u32 prog_id, fd_type;
4843                 const char *buf;
4844
4845                 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
4846                                               &buf, &probe_offset,
4847                                               &probe_addr);
4848                 if (!err)
4849                         err = bpf_task_fd_query_copy(attr, uattr, prog_id,
4850                                                      fd_type, buf,
4851                                                      probe_offset,
4852                                                      probe_addr);
4853                 goto put_file;
4854         }
4855
4856 out_not_supp:
4857         err = -ENOTSUPP;
4858 put_file:
4859         fput(file);
4860         return err;
4861 }
4862
4863 #define BPF_MAP_BATCH_LAST_FIELD batch.flags
4864
4865 #define BPF_DO_BATCH(fn, ...)                   \
4866         do {                                    \
4867                 if (!fn) {                      \
4868                         err = -ENOTSUPP;        \
4869                         goto err_put;           \
4870                 }                               \
4871                 err = fn(__VA_ARGS__);          \
4872         } while (0)
4873
4874 static int bpf_map_do_batch(const union bpf_attr *attr,
4875                             union bpf_attr __user *uattr,
4876                             int cmd)
4877 {
4878         bool has_read  = cmd == BPF_MAP_LOOKUP_BATCH ||
4879                          cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH;
4880         bool has_write = cmd != BPF_MAP_LOOKUP_BATCH;
4881         struct bpf_map *map;
4882         int err, ufd;
4883         struct fd f;
4884
4885         if (CHECK_ATTR(BPF_MAP_BATCH))
4886                 return -EINVAL;
4887
4888         ufd = attr->batch.map_fd;
4889         f = fdget(ufd);
4890         map = __bpf_map_get(f);
4891         if (IS_ERR(map))
4892                 return PTR_ERR(map);
4893         if (has_write)
4894                 bpf_map_write_active_inc(map);
4895         if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
4896                 err = -EPERM;
4897                 goto err_put;
4898         }
4899         if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
4900                 err = -EPERM;
4901                 goto err_put;
4902         }
4903
4904         if (cmd == BPF_MAP_LOOKUP_BATCH)
4905                 BPF_DO_BATCH(map->ops->map_lookup_batch, map, attr, uattr);
4906         else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
4907                 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch, map, attr, uattr);
4908         else if (cmd == BPF_MAP_UPDATE_BATCH)
4909                 BPF_DO_BATCH(map->ops->map_update_batch, map, f.file, attr, uattr);
4910         else
4911                 BPF_DO_BATCH(map->ops->map_delete_batch, map, attr, uattr);
4912 err_put:
4913         if (has_write)
4914                 bpf_map_write_active_dec(map);
4915         fdput(f);
4916         return err;
4917 }
4918
4919 #define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.pid
4920 static int link_create(union bpf_attr *attr, bpfptr_t uattr)
4921 {
4922         struct bpf_prog *prog;
4923         int ret;
4924
4925         if (CHECK_ATTR(BPF_LINK_CREATE))
4926                 return -EINVAL;
4927
4928         if (attr->link_create.attach_type == BPF_STRUCT_OPS)
4929                 return bpf_struct_ops_link_create(attr);
4930
4931         prog = bpf_prog_get(attr->link_create.prog_fd);
4932         if (IS_ERR(prog))
4933                 return PTR_ERR(prog);
4934
4935         ret = bpf_prog_attach_check_attach_type(prog,
4936                                                 attr->link_create.attach_type);
4937         if (ret)
4938                 goto out;
4939
4940         switch (prog->type) {
4941         case BPF_PROG_TYPE_CGROUP_SKB:
4942         case BPF_PROG_TYPE_CGROUP_SOCK:
4943         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4944         case BPF_PROG_TYPE_SOCK_OPS:
4945         case BPF_PROG_TYPE_CGROUP_DEVICE:
4946         case BPF_PROG_TYPE_CGROUP_SYSCTL:
4947         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4948                 ret = cgroup_bpf_link_attach(attr, prog);
4949                 break;
4950         case BPF_PROG_TYPE_EXT:
4951                 ret = bpf_tracing_prog_attach(prog,
4952                                               attr->link_create.target_fd,
4953                                               attr->link_create.target_btf_id,
4954                                               attr->link_create.tracing.cookie);
4955                 break;
4956         case BPF_PROG_TYPE_LSM:
4957         case BPF_PROG_TYPE_TRACING:
4958                 if (attr->link_create.attach_type != prog->expected_attach_type) {
4959                         ret = -EINVAL;
4960                         goto out;
4961                 }
4962                 if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
4963                         ret = bpf_raw_tp_link_attach(prog, NULL);
4964                 else if (prog->expected_attach_type == BPF_TRACE_ITER)
4965                         ret = bpf_iter_link_attach(attr, uattr, prog);
4966                 else if (prog->expected_attach_type == BPF_LSM_CGROUP)
4967                         ret = cgroup_bpf_link_attach(attr, prog);
4968                 else
4969                         ret = bpf_tracing_prog_attach(prog,
4970                                                       attr->link_create.target_fd,
4971                                                       attr->link_create.target_btf_id,
4972                                                       attr->link_create.tracing.cookie);
4973                 break;
4974         case BPF_PROG_TYPE_FLOW_DISSECTOR:
4975         case BPF_PROG_TYPE_SK_LOOKUP:
4976                 ret = netns_bpf_link_create(attr, prog);
4977                 break;
4978 #ifdef CONFIG_NET
4979         case BPF_PROG_TYPE_XDP:
4980                 ret = bpf_xdp_link_attach(attr, prog);
4981                 break;
4982         case BPF_PROG_TYPE_SCHED_CLS:
4983                 ret = tcx_link_attach(attr, prog);
4984                 break;
4985         case BPF_PROG_TYPE_NETFILTER:
4986                 ret = bpf_nf_link_attach(attr, prog);
4987                 break;
4988 #endif
4989         case BPF_PROG_TYPE_PERF_EVENT:
4990         case BPF_PROG_TYPE_TRACEPOINT:
4991                 ret = bpf_perf_link_attach(attr, prog);
4992                 break;
4993         case BPF_PROG_TYPE_KPROBE:
4994                 if (attr->link_create.attach_type == BPF_PERF_EVENT)
4995                         ret = bpf_perf_link_attach(attr, prog);
4996                 else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI)
4997                         ret = bpf_kprobe_multi_link_attach(attr, prog);
4998                 else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI)
4999                         ret = bpf_uprobe_multi_link_attach(attr, prog);
5000                 break;
5001         default:
5002                 ret = -EINVAL;
5003         }
5004
5005 out:
5006         if (ret < 0)
5007                 bpf_prog_put(prog);
5008         return ret;
5009 }
5010
5011 static int link_update_map(struct bpf_link *link, union bpf_attr *attr)
5012 {
5013         struct bpf_map *new_map, *old_map = NULL;
5014         int ret;
5015
5016         new_map = bpf_map_get(attr->link_update.new_map_fd);
5017         if (IS_ERR(new_map))
5018                 return PTR_ERR(new_map);
5019
5020         if (attr->link_update.flags & BPF_F_REPLACE) {
5021                 old_map = bpf_map_get(attr->link_update.old_map_fd);
5022                 if (IS_ERR(old_map)) {
5023                         ret = PTR_ERR(old_map);
5024                         goto out_put;
5025                 }
5026         } else if (attr->link_update.old_map_fd) {
5027                 ret = -EINVAL;
5028                 goto out_put;
5029         }
5030
5031         ret = link->ops->update_map(link, new_map, old_map);
5032
5033         if (old_map)
5034                 bpf_map_put(old_map);
5035 out_put:
5036         bpf_map_put(new_map);
5037         return ret;
5038 }
5039
5040 #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
5041
5042 static int link_update(union bpf_attr *attr)
5043 {
5044         struct bpf_prog *old_prog = NULL, *new_prog;
5045         struct bpf_link *link;
5046         u32 flags;
5047         int ret;
5048
5049         if (CHECK_ATTR(BPF_LINK_UPDATE))
5050                 return -EINVAL;
5051
5052         flags = attr->link_update.flags;
5053         if (flags & ~BPF_F_REPLACE)
5054                 return -EINVAL;
5055
5056         link = bpf_link_get_from_fd(attr->link_update.link_fd);
5057         if (IS_ERR(link))
5058                 return PTR_ERR(link);
5059
5060         if (link->ops->update_map) {
5061                 ret = link_update_map(link, attr);
5062                 goto out_put_link;
5063         }
5064
5065         new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
5066         if (IS_ERR(new_prog)) {
5067                 ret = PTR_ERR(new_prog);
5068                 goto out_put_link;
5069         }
5070
5071         if (flags & BPF_F_REPLACE) {
5072                 old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
5073                 if (IS_ERR(old_prog)) {
5074                         ret = PTR_ERR(old_prog);
5075                         old_prog = NULL;
5076                         goto out_put_progs;
5077                 }
5078         } else if (attr->link_update.old_prog_fd) {
5079                 ret = -EINVAL;
5080                 goto out_put_progs;
5081         }
5082
5083         if (link->ops->update_prog)
5084                 ret = link->ops->update_prog(link, new_prog, old_prog);
5085         else
5086                 ret = -EINVAL;
5087
5088 out_put_progs:
5089         if (old_prog)
5090                 bpf_prog_put(old_prog);
5091         if (ret)
5092                 bpf_prog_put(new_prog);
5093 out_put_link:
5094         bpf_link_put_direct(link);
5095         return ret;
5096 }
5097
5098 #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
5099
5100 static int link_detach(union bpf_attr *attr)
5101 {
5102         struct bpf_link *link;
5103         int ret;
5104
5105         if (CHECK_ATTR(BPF_LINK_DETACH))
5106                 return -EINVAL;
5107
5108         link = bpf_link_get_from_fd(attr->link_detach.link_fd);
5109         if (IS_ERR(link))
5110                 return PTR_ERR(link);
5111
5112         if (link->ops->detach)
5113                 ret = link->ops->detach(link);
5114         else
5115                 ret = -EOPNOTSUPP;
5116
5117         bpf_link_put_direct(link);
5118         return ret;
5119 }
5120
5121 static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
5122 {
5123         return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
5124 }
5125
5126 struct bpf_link *bpf_link_by_id(u32 id)
5127 {
5128         struct bpf_link *link;
5129
5130         if (!id)
5131                 return ERR_PTR(-ENOENT);
5132
5133         spin_lock_bh(&link_idr_lock);
5134         /* before link is "settled", ID is 0, pretend it doesn't exist yet */
5135         link = idr_find(&link_idr, id);
5136         if (link) {
5137                 if (link->id)
5138                         link = bpf_link_inc_not_zero(link);
5139                 else
5140                         link = ERR_PTR(-EAGAIN);
5141         } else {
5142                 link = ERR_PTR(-ENOENT);
5143         }
5144         spin_unlock_bh(&link_idr_lock);
5145         return link;
5146 }
5147
5148 struct bpf_link *bpf_link_get_curr_or_next(u32 *id)
5149 {
5150         struct bpf_link *link;
5151
5152         spin_lock_bh(&link_idr_lock);
5153 again:
5154         link = idr_get_next(&link_idr, id);
5155         if (link) {
5156                 link = bpf_link_inc_not_zero(link);
5157                 if (IS_ERR(link)) {
5158                         (*id)++;
5159                         goto again;
5160                 }
5161         }
5162         spin_unlock_bh(&link_idr_lock);
5163
5164         return link;
5165 }
5166
5167 #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
5168
5169 static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
5170 {
5171         struct bpf_link *link;
5172         u32 id = attr->link_id;
5173         int fd;
5174
5175         if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
5176                 return -EINVAL;
5177
5178         if (!capable(CAP_SYS_ADMIN))
5179                 return -EPERM;
5180
5181         link = bpf_link_by_id(id);
5182         if (IS_ERR(link))
5183                 return PTR_ERR(link);
5184
5185         fd = bpf_link_new_fd(link);
5186         if (fd < 0)
5187                 bpf_link_put_direct(link);
5188
5189         return fd;
5190 }
5191
5192 DEFINE_MUTEX(bpf_stats_enabled_mutex);
5193
5194 static int bpf_stats_release(struct inode *inode, struct file *file)
5195 {
5196         mutex_lock(&bpf_stats_enabled_mutex);
5197         static_key_slow_dec(&bpf_stats_enabled_key.key);
5198         mutex_unlock(&bpf_stats_enabled_mutex);
5199         return 0;
5200 }
5201
5202 static const struct file_operations bpf_stats_fops = {
5203         .release = bpf_stats_release,
5204 };
5205
5206 static int bpf_enable_runtime_stats(void)
5207 {
5208         int fd;
5209
5210         mutex_lock(&bpf_stats_enabled_mutex);
5211
5212         /* Set a very high limit to avoid overflow */
5213         if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
5214                 mutex_unlock(&bpf_stats_enabled_mutex);
5215                 return -EBUSY;
5216         }
5217
5218         fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
5219         if (fd >= 0)
5220                 static_key_slow_inc(&bpf_stats_enabled_key.key);
5221
5222         mutex_unlock(&bpf_stats_enabled_mutex);
5223         return fd;
5224 }
5225
5226 #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
5227
5228 static int bpf_enable_stats(union bpf_attr *attr)
5229 {
5230
5231         if (CHECK_ATTR(BPF_ENABLE_STATS))
5232                 return -EINVAL;
5233
5234         if (!capable(CAP_SYS_ADMIN))
5235                 return -EPERM;
5236
5237         switch (attr->enable_stats.type) {
5238         case BPF_STATS_RUN_TIME:
5239                 return bpf_enable_runtime_stats();
5240         default:
5241                 break;
5242         }
5243         return -EINVAL;
5244 }
5245
5246 #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
5247
5248 static int bpf_iter_create(union bpf_attr *attr)
5249 {
5250         struct bpf_link *link;
5251         int err;
5252
5253         if (CHECK_ATTR(BPF_ITER_CREATE))
5254                 return -EINVAL;
5255
5256         if (attr->iter_create.flags)
5257                 return -EINVAL;
5258
5259         link = bpf_link_get_from_fd(attr->iter_create.link_fd);
5260         if (IS_ERR(link))
5261                 return PTR_ERR(link);
5262
5263         err = bpf_iter_new_fd(link);
5264         bpf_link_put_direct(link);
5265
5266         return err;
5267 }
5268
5269 #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
5270
5271 static int bpf_prog_bind_map(union bpf_attr *attr)
5272 {
5273         struct bpf_prog *prog;
5274         struct bpf_map *map;
5275         struct bpf_map **used_maps_old, **used_maps_new;
5276         int i, ret = 0;
5277
5278         if (CHECK_ATTR(BPF_PROG_BIND_MAP))
5279                 return -EINVAL;
5280
5281         if (attr->prog_bind_map.flags)
5282                 return -EINVAL;
5283
5284         prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
5285         if (IS_ERR(prog))
5286                 return PTR_ERR(prog);
5287
5288         map = bpf_map_get(attr->prog_bind_map.map_fd);
5289         if (IS_ERR(map)) {
5290                 ret = PTR_ERR(map);
5291                 goto out_prog_put;
5292         }
5293
5294         mutex_lock(&prog->aux->used_maps_mutex);
5295
5296         used_maps_old = prog->aux->used_maps;
5297
5298         for (i = 0; i < prog->aux->used_map_cnt; i++)
5299                 if (used_maps_old[i] == map) {
5300                         bpf_map_put(map);
5301                         goto out_unlock;
5302                 }
5303
5304         used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
5305                                       sizeof(used_maps_new[0]),
5306                                       GFP_KERNEL);
5307         if (!used_maps_new) {
5308                 ret = -ENOMEM;
5309                 goto out_unlock;
5310         }
5311
5312         memcpy(used_maps_new, used_maps_old,
5313                sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
5314         used_maps_new[prog->aux->used_map_cnt] = map;
5315
5316         prog->aux->used_map_cnt++;
5317         prog->aux->used_maps = used_maps_new;
5318
5319         kfree(used_maps_old);
5320
5321 out_unlock:
5322         mutex_unlock(&prog->aux->used_maps_mutex);
5323
5324         if (ret)
5325                 bpf_map_put(map);
5326 out_prog_put:
5327         bpf_prog_put(prog);
5328         return ret;
5329 }
5330
5331 static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size)
5332 {
5333         union bpf_attr attr;
5334         int err;
5335
5336         err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
5337         if (err)
5338                 return err;
5339         size = min_t(u32, size, sizeof(attr));
5340
5341         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
5342         memset(&attr, 0, sizeof(attr));
5343         if (copy_from_bpfptr(&attr, uattr, size) != 0)
5344                 return -EFAULT;
5345
5346         err = security_bpf(cmd, &attr, size);
5347         if (err < 0)
5348                 return err;
5349
5350         switch (cmd) {
5351         case BPF_MAP_CREATE:
5352                 err = map_create(&attr);
5353                 break;
5354         case BPF_MAP_LOOKUP_ELEM:
5355                 err = map_lookup_elem(&attr);
5356                 break;
5357         case BPF_MAP_UPDATE_ELEM:
5358                 err = map_update_elem(&attr, uattr);
5359                 break;
5360         case BPF_MAP_DELETE_ELEM:
5361                 err = map_delete_elem(&attr, uattr);
5362                 break;
5363         case BPF_MAP_GET_NEXT_KEY:
5364                 err = map_get_next_key(&attr);
5365                 break;
5366         case BPF_MAP_FREEZE:
5367                 err = map_freeze(&attr);
5368                 break;
5369         case BPF_PROG_LOAD:
5370                 err = bpf_prog_load(&attr, uattr, size);
5371                 break;
5372         case BPF_OBJ_PIN:
5373                 err = bpf_obj_pin(&attr);
5374                 break;
5375         case BPF_OBJ_GET:
5376                 err = bpf_obj_get(&attr);
5377                 break;
5378         case BPF_PROG_ATTACH:
5379                 err = bpf_prog_attach(&attr);
5380                 break;
5381         case BPF_PROG_DETACH:
5382                 err = bpf_prog_detach(&attr);
5383                 break;
5384         case BPF_PROG_QUERY:
5385                 err = bpf_prog_query(&attr, uattr.user);
5386                 break;
5387         case BPF_PROG_TEST_RUN:
5388                 err = bpf_prog_test_run(&attr, uattr.user);
5389                 break;
5390         case BPF_PROG_GET_NEXT_ID:
5391                 err = bpf_obj_get_next_id(&attr, uattr.user,
5392                                           &prog_idr, &prog_idr_lock);
5393                 break;
5394         case BPF_MAP_GET_NEXT_ID:
5395                 err = bpf_obj_get_next_id(&attr, uattr.user,
5396                                           &map_idr, &map_idr_lock);
5397                 break;
5398         case BPF_BTF_GET_NEXT_ID:
5399                 err = bpf_obj_get_next_id(&attr, uattr.user,
5400                                           &btf_idr, &btf_idr_lock);
5401                 break;
5402         case BPF_PROG_GET_FD_BY_ID:
5403                 err = bpf_prog_get_fd_by_id(&attr);
5404                 break;
5405         case BPF_MAP_GET_FD_BY_ID:
5406                 err = bpf_map_get_fd_by_id(&attr);
5407                 break;
5408         case BPF_OBJ_GET_INFO_BY_FD:
5409                 err = bpf_obj_get_info_by_fd(&attr, uattr.user);
5410                 break;
5411         case BPF_RAW_TRACEPOINT_OPEN:
5412                 err = bpf_raw_tracepoint_open(&attr);
5413                 break;
5414         case BPF_BTF_LOAD:
5415                 err = bpf_btf_load(&attr, uattr, size);
5416                 break;
5417         case BPF_BTF_GET_FD_BY_ID:
5418                 err = bpf_btf_get_fd_by_id(&attr);
5419                 break;
5420         case BPF_TASK_FD_QUERY:
5421                 err = bpf_task_fd_query(&attr, uattr.user);
5422                 break;
5423         case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
5424                 err = map_lookup_and_delete_elem(&attr);
5425                 break;
5426         case BPF_MAP_LOOKUP_BATCH:
5427                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH);
5428                 break;
5429         case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
5430                 err = bpf_map_do_batch(&attr, uattr.user,
5431                                        BPF_MAP_LOOKUP_AND_DELETE_BATCH);
5432                 break;
5433         case BPF_MAP_UPDATE_BATCH:
5434                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH);
5435                 break;
5436         case BPF_MAP_DELETE_BATCH:
5437                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH);
5438                 break;
5439         case BPF_LINK_CREATE:
5440                 err = link_create(&attr, uattr);
5441                 break;
5442         case BPF_LINK_UPDATE:
5443                 err = link_update(&attr);
5444                 break;
5445         case BPF_LINK_GET_FD_BY_ID:
5446                 err = bpf_link_get_fd_by_id(&attr);
5447                 break;
5448         case BPF_LINK_GET_NEXT_ID:
5449                 err = bpf_obj_get_next_id(&attr, uattr.user,
5450                                           &link_idr, &link_idr_lock);
5451                 break;
5452         case BPF_ENABLE_STATS:
5453                 err = bpf_enable_stats(&attr);
5454                 break;
5455         case BPF_ITER_CREATE:
5456                 err = bpf_iter_create(&attr);
5457                 break;
5458         case BPF_LINK_DETACH:
5459                 err = link_detach(&attr);
5460                 break;
5461         case BPF_PROG_BIND_MAP:
5462                 err = bpf_prog_bind_map(&attr);
5463                 break;
5464         default:
5465                 err = -EINVAL;
5466                 break;
5467         }
5468
5469         return err;
5470 }
5471
5472 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
5473 {
5474         return __sys_bpf(cmd, USER_BPFPTR(uattr), size);
5475 }
5476
5477 static bool syscall_prog_is_valid_access(int off, int size,
5478                                          enum bpf_access_type type,
5479                                          const struct bpf_prog *prog,
5480                                          struct bpf_insn_access_aux *info)
5481 {
5482         if (off < 0 || off >= U16_MAX)
5483                 return false;
5484         if (off % size != 0)
5485                 return false;
5486         return true;
5487 }
5488
5489 BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
5490 {
5491         switch (cmd) {
5492         case BPF_MAP_CREATE:
5493         case BPF_MAP_DELETE_ELEM:
5494         case BPF_MAP_UPDATE_ELEM:
5495         case BPF_MAP_FREEZE:
5496         case BPF_MAP_GET_FD_BY_ID:
5497         case BPF_PROG_LOAD:
5498         case BPF_BTF_LOAD:
5499         case BPF_LINK_CREATE:
5500         case BPF_RAW_TRACEPOINT_OPEN:
5501                 break;
5502         default:
5503                 return -EINVAL;
5504         }
5505         return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size);
5506 }
5507
5508
5509 /* To shut up -Wmissing-prototypes.
5510  * This function is used by the kernel light skeleton
5511  * to load bpf programs when modules are loaded or during kernel boot.
5512  * See tools/lib/bpf/skel_internal.h
5513  */
5514 int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size);
5515
5516 int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size)
5517 {
5518         struct bpf_prog * __maybe_unused prog;
5519         struct bpf_tramp_run_ctx __maybe_unused run_ctx;
5520
5521         switch (cmd) {
5522 #ifdef CONFIG_BPF_JIT /* __bpf_prog_enter_sleepable used by trampoline and JIT */
5523         case BPF_PROG_TEST_RUN:
5524                 if (attr->test.data_in || attr->test.data_out ||
5525                     attr->test.ctx_out || attr->test.duration ||
5526                     attr->test.repeat || attr->test.flags)
5527                         return -EINVAL;
5528
5529                 prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL);
5530                 if (IS_ERR(prog))
5531                         return PTR_ERR(prog);
5532
5533                 if (attr->test.ctx_size_in < prog->aux->max_ctx_offset ||
5534                     attr->test.ctx_size_in > U16_MAX) {
5535                         bpf_prog_put(prog);
5536                         return -EINVAL;
5537                 }
5538
5539                 run_ctx.bpf_cookie = 0;
5540                 if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) {
5541                         /* recursion detected */
5542                         __bpf_prog_exit_sleepable_recur(prog, 0, &run_ctx);
5543                         bpf_prog_put(prog);
5544                         return -EBUSY;
5545                 }
5546                 attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);
5547                 __bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */,
5548                                                 &run_ctx);
5549                 bpf_prog_put(prog);
5550                 return 0;
5551 #endif
5552         default:
5553                 return ____bpf_sys_bpf(cmd, attr, size);
5554         }
5555 }
5556 EXPORT_SYMBOL(kern_sys_bpf);
5557
5558 static const struct bpf_func_proto bpf_sys_bpf_proto = {
5559         .func           = bpf_sys_bpf,
5560         .gpl_only       = false,
5561         .ret_type       = RET_INTEGER,
5562         .arg1_type      = ARG_ANYTHING,
5563         .arg2_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
5564         .arg3_type      = ARG_CONST_SIZE,
5565 };
5566
5567 const struct bpf_func_proto * __weak
5568 tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
5569 {
5570         return bpf_base_func_proto(func_id);
5571 }
5572
5573 BPF_CALL_1(bpf_sys_close, u32, fd)
5574 {
5575         /* When bpf program calls this helper there should not be
5576          * an fdget() without matching completed fdput().
5577          * This helper is allowed in the following callchain only:
5578          * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close
5579          */
5580         return close_fd(fd);
5581 }
5582
5583 static const struct bpf_func_proto bpf_sys_close_proto = {
5584         .func           = bpf_sys_close,
5585         .gpl_only       = false,
5586         .ret_type       = RET_INTEGER,
5587         .arg1_type      = ARG_ANYTHING,
5588 };
5589
5590 BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flags, u64 *, res)
5591 {
5592         if (flags)
5593                 return -EINVAL;
5594
5595         if (name_sz <= 1 || name[name_sz - 1])
5596                 return -EINVAL;
5597
5598         if (!bpf_dump_raw_ok(current_cred()))
5599                 return -EPERM;
5600
5601         *res = kallsyms_lookup_name(name);
5602         return *res ? 0 : -ENOENT;
5603 }
5604
5605 static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = {
5606         .func           = bpf_kallsyms_lookup_name,
5607         .gpl_only       = false,
5608         .ret_type       = RET_INTEGER,
5609         .arg1_type      = ARG_PTR_TO_MEM,
5610         .arg2_type      = ARG_CONST_SIZE_OR_ZERO,
5611         .arg3_type      = ARG_ANYTHING,
5612         .arg4_type      = ARG_PTR_TO_LONG,
5613 };
5614
5615 static const struct bpf_func_proto *
5616 syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
5617 {
5618         switch (func_id) {
5619         case BPF_FUNC_sys_bpf:
5620                 return !perfmon_capable() ? NULL : &bpf_sys_bpf_proto;
5621         case BPF_FUNC_btf_find_by_name_kind:
5622                 return &bpf_btf_find_by_name_kind_proto;
5623         case BPF_FUNC_sys_close:
5624                 return &bpf_sys_close_proto;
5625         case BPF_FUNC_kallsyms_lookup_name:
5626                 return &bpf_kallsyms_lookup_name_proto;
5627         default:
5628                 return tracing_prog_func_proto(func_id, prog);
5629         }
5630 }
5631
5632 const struct bpf_verifier_ops bpf_syscall_verifier_ops = {
5633         .get_func_proto  = syscall_prog_func_proto,
5634         .is_valid_access = syscall_prog_is_valid_access,
5635 };
5636
5637 const struct bpf_prog_ops bpf_syscall_prog_ops = {
5638         .test_run = bpf_prog_test_run_syscall,
5639 };
5640
5641 #ifdef CONFIG_SYSCTL
5642 static int bpf_stats_handler(struct ctl_table *table, int write,
5643                              void *buffer, size_t *lenp, loff_t *ppos)
5644 {
5645         struct static_key *key = (struct static_key *)table->data;
5646         static int saved_val;
5647         int val, ret;
5648         struct ctl_table tmp = {
5649                 .data   = &val,
5650                 .maxlen = sizeof(val),
5651                 .mode   = table->mode,
5652                 .extra1 = SYSCTL_ZERO,
5653                 .extra2 = SYSCTL_ONE,
5654         };
5655
5656         if (write && !capable(CAP_SYS_ADMIN))
5657                 return -EPERM;
5658
5659         mutex_lock(&bpf_stats_enabled_mutex);
5660         val = saved_val;
5661         ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
5662         if (write && !ret && val != saved_val) {
5663                 if (val)
5664                         static_key_slow_inc(key);
5665                 else
5666                         static_key_slow_dec(key);
5667                 saved_val = val;
5668         }
5669         mutex_unlock(&bpf_stats_enabled_mutex);
5670         return ret;
5671 }
5672
5673 void __weak unpriv_ebpf_notify(int new_state)
5674 {
5675 }
5676
5677 static int bpf_unpriv_handler(struct ctl_table *table, int write,
5678                               void *buffer, size_t *lenp, loff_t *ppos)
5679 {
5680         int ret, unpriv_enable = *(int *)table->data;
5681         bool locked_state = unpriv_enable == 1;
5682         struct ctl_table tmp = *table;
5683
5684         if (write && !capable(CAP_SYS_ADMIN))
5685                 return -EPERM;
5686
5687         tmp.data = &unpriv_enable;
5688         ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
5689         if (write && !ret) {
5690                 if (locked_state && unpriv_enable != 1)
5691                         return -EPERM;
5692                 *(int *)table->data = unpriv_enable;
5693         }
5694
5695         if (write)
5696                 unpriv_ebpf_notify(unpriv_enable);
5697
5698         return ret;
5699 }
5700
5701 static struct ctl_table bpf_syscall_table[] = {
5702         {
5703                 .procname       = "unprivileged_bpf_disabled",
5704                 .data           = &sysctl_unprivileged_bpf_disabled,
5705                 .maxlen         = sizeof(sysctl_unprivileged_bpf_disabled),
5706                 .mode           = 0644,
5707                 .proc_handler   = bpf_unpriv_handler,
5708                 .extra1         = SYSCTL_ZERO,
5709                 .extra2         = SYSCTL_TWO,
5710         },
5711         {
5712                 .procname       = "bpf_stats_enabled",
5713                 .data           = &bpf_stats_enabled_key.key,
5714                 .mode           = 0644,
5715                 .proc_handler   = bpf_stats_handler,
5716         },
5717         { }
5718 };
5719
5720 static int __init bpf_syscall_sysctl_init(void)
5721 {
5722         register_sysctl_init("kernel", bpf_syscall_table);
5723         return 0;
5724 }
5725 late_initcall(bpf_syscall_sysctl_init);
5726 #endif /* CONFIG_SYSCTL */