kernel/bpf/syscall.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   3  */
   4 #include <linux/bpf.h>
   5 #include <linux/bpf_trace.h>
   6 #include <linux/bpf_lirc.h>
   7 #include <linux/bpf_verifier.h>
   8 #include <linux/btf.h>
   9 #include <linux/syscalls.h>
  10 #include <linux/slab.h>
  11 #include <linux/sched/signal.h>
  12 #include <linux/vmalloc.h>
  13 #include <linux/mmzone.h>
  14 #include <linux/anon_inodes.h>
  15 #include <linux/fdtable.h>
  16 #include <linux/file.h>
  17 #include <linux/fs.h>
  18 #include <linux/license.h>
  19 #include <linux/filter.h>
  20 #include <linux/kernel.h>
  21 #include <linux/idr.h>
  22 #include <linux/cred.h>
  23 #include <linux/timekeeping.h>
  24 #include <linux/ctype.h>
  25 #include <linux/nospec.h>
  26 #include <linux/audit.h>
  27 #include <uapi/linux/btf.h>
  28 #include <linux/pgtable.h>
  29 #include <linux/bpf_lsm.h>
  30 #include <linux/poll.h>
  31 #include <linux/bpf-netns.h>
  32 #include <linux/rcupdate_trace.h>
  33 #include <linux/memcontrol.h>
  34
  35 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  36                           (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  37                           (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  38 #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
  39 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  40 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
  41                         IS_FD_HASH(map))
  42
  43 #define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)
  44
  45 DEFINE_PER_CPU(int, bpf_prog_active);
  46 static DEFINE_IDR(prog_idr);
  47 static DEFINE_SPINLOCK(prog_idr_lock);
  48 static DEFINE_IDR(map_idr);
  49 static DEFINE_SPINLOCK(map_idr_lock);
  50 static DEFINE_IDR(link_idr);
  51 static DEFINE_SPINLOCK(link_idr_lock);
  52
  53 int sysctl_unprivileged_bpf_disabled __read_mostly =
  54         IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;
  55
  56 static const struct bpf_map_ops * const bpf_map_types[] = {
  57 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
  58 #define BPF_MAP_TYPE(_id, _ops) \
  59         [_id] = &_ops,
  60 #define BPF_LINK_TYPE(_id, _name)
  61 #include <linux/bpf_types.h>
  62 #undef BPF_PROG_TYPE
  63 #undef BPF_MAP_TYPE
  64 #undef BPF_LINK_TYPE
  65 };
  66
  67 /*
  68  * If we're handed a bigger struct than we know of, ensure all the unknown bits
  69  * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  70  * we don't know about yet.
  71  *
  72  * There is a ToCToU between this function call and the following
  73  * copy_from_user() call. However, this is not a concern since this function is
  74  * meant to be a future-proofing of bits.
  75  */
  76 int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
  77                              size_t expected_size,
  78                              size_t actual_size)
  79 {
  80         int res;
  81
  82         if (unlikely(actual_size > PAGE_SIZE))  /* silly large */
  83                 return -E2BIG;
  84
  85         if (actual_size <= expected_size)
  86                 return 0;
  87
  88         if (uaddr.is_kernel)
  89                 res = memchr_inv(uaddr.kernel + expected_size, 0,
  90                                  actual_size - expected_size) == NULL;
  91         else
  92                 res = check_zeroed_user(uaddr.user + expected_size,
  93                                         actual_size - expected_size);
  94         if (res < 0)
  95                 return res;
  96         return res ? 0 : -E2BIG;
  97 }
  98
  99 const struct bpf_map_ops bpf_map_offload_ops = {
 100         .map_meta_equal = bpf_map_meta_equal,
 101         .map_alloc = bpf_map_offload_map_alloc,
 102         .map_free = bpf_map_offload_map_free,
 103         .map_check_btf = map_check_no_btf,
 104 };
 105
 106 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
 107 {
 108         const struct bpf_map_ops *ops;
 109         u32 type = attr->map_type;
 110         struct bpf_map *map;
 111         int err;
 112
 113         if (type >= ARRAY_SIZE(bpf_map_types))
 114                 return ERR_PTR(-EINVAL);
 115         type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));
 116         ops = bpf_map_types[type];
 117         if (!ops)
 118                 return ERR_PTR(-EINVAL);
 119
 120         if (ops->map_alloc_check) {
 121                 err = ops->map_alloc_check(attr);
 122                 if (err)
 123                         return ERR_PTR(err);
 124         }
 125         if (attr->map_ifindex)
 126                 ops = &bpf_map_offload_ops;
 127         map = ops->map_alloc(attr);
 128         if (IS_ERR(map))
 129                 return map;
 130         map->ops = ops;
 131         map->map_type = type;
 132         return map;
 133 }
 134
 135 static u32 bpf_map_value_size(const struct bpf_map *map)
 136 {
 137         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 138             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 139             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 140             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 141                 return round_up(map->value_size, 8) * num_possible_cpus();
 142         else if (IS_FD_MAP(map))
 143                 return sizeof(u32);
 144         else
 145                 return  map->value_size;
 146 }
 147
 148 static void maybe_wait_bpf_programs(struct bpf_map *map)
 149 {
 150         /* Wait for any running BPF programs to complete so that
 151          * userspace, when we return to it, knows that all programs
 152          * that could be running use the new map value.
 153          */
 154         if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
 155             map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 156                 synchronize_rcu();
 157 }
 158
 159 static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key,
 160                                 void *value, __u64 flags)
 161 {
 162         int err;
 163
 164         /* Need to create a kthread, thus must support schedule */
 165         if (bpf_map_is_dev_bound(map)) {
 166                 return bpf_map_offload_update_elem(map, key, value, flags);
 167         } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
 168                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 169                 return map->ops->map_update_elem(map, key, value, flags);
 170         } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
 171                    map->map_type == BPF_MAP_TYPE_SOCKMAP) {
 172                 return sock_map_update_elem_sys(map, key, value, flags);
 173         } else if (IS_FD_PROG_ARRAY(map)) {
 174                 return bpf_fd_array_map_update_elem(map, f.file, key, value,
 175                                                     flags);
 176         }
 177
 178         bpf_disable_instrumentation();
 179         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 180             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 181                 err = bpf_percpu_hash_update(map, key, value, flags);
 182         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 183                 err = bpf_percpu_array_update(map, key, value, flags);
 184         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 185                 err = bpf_percpu_cgroup_storage_update(map, key, value,
 186                                                        flags);
 187         } else if (IS_FD_ARRAY(map)) {
 188                 rcu_read_lock();
 189                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 190                                                    flags);
 191                 rcu_read_unlock();
 192         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 193                 rcu_read_lock();
 194                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
 195                                                   flags);
 196                 rcu_read_unlock();
 197         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 198                 /* rcu_read_lock() is not needed */
 199                 err = bpf_fd_reuseport_array_update_elem(map, key, value,
 200                                                          flags);
 201         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 202                    map->map_type == BPF_MAP_TYPE_STACK) {
 203                 err = map->ops->map_push_elem(map, value, flags);
 204         } else {
 205                 rcu_read_lock();
 206                 err = map->ops->map_update_elem(map, key, value, flags);
 207                 rcu_read_unlock();
 208         }
 209         bpf_enable_instrumentation();
 210         maybe_wait_bpf_programs(map);
 211
 212         return err;
 213 }
 214
 215 static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
 216                               __u64 flags)
 217 {
 218         void *ptr;
 219         int err;
 220
 221         if (bpf_map_is_dev_bound(map))
 222                 return bpf_map_offload_lookup_elem(map, key, value);
 223
 224         bpf_disable_instrumentation();
 225         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 226             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 227                 err = bpf_percpu_hash_copy(map, key, value);
 228         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 229                 err = bpf_percpu_array_copy(map, key, value);
 230         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 231                 err = bpf_percpu_cgroup_storage_copy(map, key, value);
 232         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 233                 err = bpf_stackmap_copy(map, key, value);
 234         } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
 235                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 236         } else if (IS_FD_HASH(map)) {
 237                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 238         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 239                 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
 240         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 241                    map->map_type == BPF_MAP_TYPE_STACK) {
 242                 err = map->ops->map_peek_elem(map, value);
 243         } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 244                 /* struct_ops map requires directly updating "value" */
 245                 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
 246         } else {
 247                 rcu_read_lock();
 248                 if (map->ops->map_lookup_elem_sys_only)
 249                         ptr = map->ops->map_lookup_elem_sys_only(map, key);
 250                 else
 251                         ptr = map->ops->map_lookup_elem(map, key);
 252                 if (IS_ERR(ptr)) {
 253                         err = PTR_ERR(ptr);
 254                 } else if (!ptr) {
 255                         err = -ENOENT;
 256                 } else {
 257                         err = 0;
 258                         if (flags & BPF_F_LOCK)
 259                                 /* lock 'ptr' and copy everything but lock */
 260                                 copy_map_value_locked(map, value, ptr, true);
 261                         else
 262                                 copy_map_value(map, value, ptr);
 263                         /* mask lock and timer, since value wasn't zero inited */
 264                         check_and_init_map_value(map, value);
 265                 }
 266                 rcu_read_unlock();
 267         }
 268
 269         bpf_enable_instrumentation();
 270         maybe_wait_bpf_programs(map);
 271
 272         return err;
 273 }
 274
 275 /* Please, do not use this function outside from the map creation path
 276  * (e.g. in map update path) without taking care of setting the active
 277  * memory cgroup (see at bpf_map_kmalloc_node() for example).
 278  */
 279 static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
 280 {
 281         /* We really just want to fail instead of triggering OOM killer
 282          * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
 283          * which is used for lower order allocation requests.
 284          *
 285          * It has been observed that higher order allocation requests done by
 286          * vmalloc with __GFP_NORETRY being set might fail due to not trying
 287          * to reclaim memory from the page cache, thus we set
 288          * __GFP_RETRY_MAYFAIL to avoid such situations.
 289          */
 290
 291         const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_ACCOUNT;
 292         unsigned int flags = 0;
 293         unsigned long align = 1;
 294         void *area;
 295
 296         if (size >= SIZE_MAX)
 297                 return NULL;
 298
 299         /* kmalloc()'ed memory can't be mmap()'ed */
 300         if (mmapable) {
 301                 BUG_ON(!PAGE_ALIGNED(size));
 302                 align = SHMLBA;
 303                 flags = VM_USERMAP;
 304         } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 305                 area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
 306                                     numa_node);
 307                 if (area != NULL)
 308                         return area;
 309         }
 310
 311         return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
 312                         gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
 313                         flags, numa_node, __builtin_return_address(0));
 314 }
 315
 316 void *bpf_map_area_alloc(u64 size, int numa_node)
 317 {
 318         return __bpf_map_area_alloc(size, numa_node, false);
 319 }
 320
 321 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
 322 {
 323         return __bpf_map_area_alloc(size, numa_node, true);
 324 }
 325
 326 void bpf_map_area_free(void *area)
 327 {
 328         kvfree(area);
 329 }
 330
 331 static u32 bpf_map_flags_retain_permanent(u32 flags)
 332 {
 333         /* Some map creation flags are not tied to the map object but
 334          * rather to the map fd instead, so they have no meaning upon
 335          * map object inspection since multiple file descriptors with
 336          * different (access) properties can exist here. Thus, given
 337          * this has zero meaning for the map itself, lets clear these
 338          * from here.
 339          */
 340         return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
 341 }
 342
 343 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
 344 {
 345         map->map_type = attr->map_type;
 346         map->key_size = attr->key_size;
 347         map->value_size = attr->value_size;
 348         map->max_entries = attr->max_entries;
 349         map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
 350         map->numa_node = bpf_map_attr_numa_node(attr);
 351 }
 352
 353 static int bpf_map_alloc_id(struct bpf_map *map)
 354 {
 355         int id;
 356
 357         idr_preload(GFP_KERNEL);
 358         spin_lock_bh(&map_idr_lock);
 359         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 360         if (id > 0)
 361                 map->id = id;
 362         spin_unlock_bh(&map_idr_lock);
 363         idr_preload_end();
 364
 365         if (WARN_ON_ONCE(!id))
 366                 return -ENOSPC;
 367
 368         return id > 0 ? 0 : id;
 369 }
 370
 371 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
 372 {
 373         unsigned long flags;
 374
 375         /* Offloaded maps are removed from the IDR store when their device
 376          * disappears - even if someone holds an fd to them they are unusable,
 377          * the memory is gone, all ops will fail; they are simply waiting for
 378          * refcnt to drop to be freed.
 379          */
 380         if (!map->id)
 381                 return;
 382
 383         if (do_idr_lock)
 384                 spin_lock_irqsave(&map_idr_lock, flags);
 385         else
 386                 __acquire(&map_idr_lock);
 387
 388         idr_remove(&map_idr, map->id);
 389         map->id = 0;
 390
 391         if (do_idr_lock)
 392                 spin_unlock_irqrestore(&map_idr_lock, flags);
 393         else
 394                 __release(&map_idr_lock);
 395 }
 396
 397 #ifdef CONFIG_MEMCG_KMEM
 398 static void bpf_map_save_memcg(struct bpf_map *map)
 399 {
 400         map->memcg = get_mem_cgroup_from_mm(current->mm);
 401 }
 402
 403 static void bpf_map_release_memcg(struct bpf_map *map)
 404 {
 405         mem_cgroup_put(map->memcg);
 406 }
 407
 408 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
 409                            int node)
 410 {
 411         struct mem_cgroup *old_memcg;
 412         void *ptr;
 413
 414         old_memcg = set_active_memcg(map->memcg);
 415         ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
 416         set_active_memcg(old_memcg);
 417
 418         return ptr;
 419 }
 420
 421 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
 422 {
 423         struct mem_cgroup *old_memcg;
 424         void *ptr;
 425
 426         old_memcg = set_active_memcg(map->memcg);
 427         ptr = kzalloc(size, flags | __GFP_ACCOUNT);
 428         set_active_memcg(old_memcg);
 429
 430         return ptr;
 431 }
 432
 433 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
 434                                     size_t align, gfp_t flags)
 435 {
 436         struct mem_cgroup *old_memcg;
 437         void __percpu *ptr;
 438
 439         old_memcg = set_active_memcg(map->memcg);
 440         ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
 441         set_active_memcg(old_memcg);
 442
 443         return ptr;
 444 }
 445
 446 #else
 447 static void bpf_map_save_memcg(struct bpf_map *map)
 448 {
 449 }
 450
 451 static void bpf_map_release_memcg(struct bpf_map *map)
 452 {
 453 }
 454 #endif
 455
 456 /* called from workqueue */
 457 static void bpf_map_free_deferred(struct work_struct *work)
 458 {
 459         struct bpf_map *map = container_of(work, struct bpf_map, work);
 460
 461         security_bpf_map_free(map);
 462         bpf_map_release_memcg(map);
 463         /* implementation dependent freeing */
 464         map->ops->map_free(map);
 465 }
 466
 467 static void bpf_map_put_uref(struct bpf_map *map)
 468 {
 469         if (atomic64_dec_and_test(&map->usercnt)) {
 470                 if (map->ops->map_release_uref)
 471                         map->ops->map_release_uref(map);
 472         }
 473 }
 474
 475 /* decrement map refcnt and schedule it for freeing via workqueue
 476  * (unrelying map implementation ops->map_free() might sleep)
 477  */
 478 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 479 {
 480         if (atomic64_dec_and_test(&map->refcnt)) {
 481                 /* bpf_map_free_id() must be called first */
 482                 bpf_map_free_id(map, do_idr_lock);
 483                 btf_put(map->btf);
 484                 INIT_WORK(&map->work, bpf_map_free_deferred);
 485                 schedule_work(&map->work);
 486         }
 487 }
 488
 489 void bpf_map_put(struct bpf_map *map)
 490 {
 491         __bpf_map_put(map, true);
 492 }
 493 EXPORT_SYMBOL_GPL(bpf_map_put);
 494
 495 void bpf_map_put_with_uref(struct bpf_map *map)
 496 {
 497         bpf_map_put_uref(map);
 498         bpf_map_put(map);
 499 }
 500
 501 static int bpf_map_release(struct inode *inode, struct file *filp)
 502 {
 503         struct bpf_map *map = filp->private_data;
 504
 505         if (map->ops->map_release)
 506                 map->ops->map_release(map, filp);
 507
 508         bpf_map_put_with_uref(map);
 509         return 0;
 510 }
 511
 512 static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
 513 {
 514         fmode_t mode = f.file->f_mode;
 515
 516         /* Our file permissions may have been overridden by global
 517          * map permissions facing syscall side.
 518          */
 519         if (READ_ONCE(map->frozen))
 520                 mode &= ~FMODE_CAN_WRITE;
 521         return mode;
 522 }
 523
 524 #ifdef CONFIG_PROC_FS
 525 /* Provides an approximation of the map's memory footprint.
 526  * Used only to provide a backward compatibility and display
 527  * a reasonable "memlock" info.
 528  */
 529 static unsigned long bpf_map_memory_footprint(const struct bpf_map *map)
 530 {
 531         unsigned long size;
 532
 533         size = round_up(map->key_size + bpf_map_value_size(map), 8);
 534
 535         return round_up(map->max_entries * size, PAGE_SIZE);
 536 }
 537
 538 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 539 {
 540         const struct bpf_map *map = filp->private_data;
 541         const struct bpf_array *array;
 542         u32 type = 0, jited = 0;
 543
 544         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 545                 array = container_of(map, struct bpf_array, map);
 546                 spin_lock(&array->aux->owner.lock);
 547                 type  = array->aux->owner.type;
 548                 jited = array->aux->owner.jited;
 549                 spin_unlock(&array->aux->owner.lock);
 550         }
 551
 552         seq_printf(m,
 553                    "map_type:\t%u\n"
 554                    "key_size:\t%u\n"
 555                    "value_size:\t%u\n"
 556                    "max_entries:\t%u\n"
 557                    "map_flags:\t%#x\n"
 558                    "memlock:\t%lu\n"
 559                    "map_id:\t%u\n"
 560                    "frozen:\t%u\n",
 561                    map->map_type,
 562                    map->key_size,
 563                    map->value_size,
 564                    map->max_entries,
 565                    map->map_flags,
 566                    bpf_map_memory_footprint(map),
 567                    map->id,
 568                    READ_ONCE(map->frozen));
 569         if (type) {
 570                 seq_printf(m, "owner_prog_type:\t%u\n", type);
 571                 seq_printf(m, "owner_jited:\t%u\n", jited);
 572         }
 573 }
 574 #endif
 575
 576 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
 577                               loff_t *ppos)
 578 {
 579         /* We need this handler such that alloc_file() enables
 580          * f_mode with FMODE_CAN_READ.
 581          */
 582         return -EINVAL;
 583 }
 584
 585 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
 586                                size_t siz, loff_t *ppos)
 587 {
 588         /* We need this handler such that alloc_file() enables
 589          * f_mode with FMODE_CAN_WRITE.
 590          */
 591         return -EINVAL;
 592 }
 593
 594 /* called for any extra memory-mapped regions (except initial) */
 595 static void bpf_map_mmap_open(struct vm_area_struct *vma)
 596 {
 597         struct bpf_map *map = vma->vm_file->private_data;
 598
 599         if (vma->vm_flags & VM_MAYWRITE) {
 600                 mutex_lock(&map->freeze_mutex);
 601                 map->writecnt++;
 602                 mutex_unlock(&map->freeze_mutex);
 603         }
 604 }
 605
 606 /* called for all unmapped memory region (including initial) */
 607 static void bpf_map_mmap_close(struct vm_area_struct *vma)
 608 {
 609         struct bpf_map *map = vma->vm_file->private_data;
 610
 611         if (vma->vm_flags & VM_MAYWRITE) {
 612                 mutex_lock(&map->freeze_mutex);
 613                 map->writecnt--;
 614                 mutex_unlock(&map->freeze_mutex);
 615         }
 616 }
 617
 618 static const struct vm_operations_struct bpf_map_default_vmops = {
 619         .open           = bpf_map_mmap_open,
 620         .close          = bpf_map_mmap_close,
 621 };
 622
 623 static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
 624 {
 625         struct bpf_map *map = filp->private_data;
 626         int err;
 627
 628         if (!map->ops->map_mmap || map_value_has_spin_lock(map) ||
 629             map_value_has_timer(map))
 630                 return -ENOTSUPP;
 631
 632         if (!(vma->vm_flags & VM_SHARED))
 633                 return -EINVAL;
 634
 635         mutex_lock(&map->freeze_mutex);
 636
 637         if (vma->vm_flags & VM_WRITE) {
 638                 if (map->frozen) {
 639                         err = -EPERM;
 640                         goto out;
 641                 }
 642                 /* map is meant to be read-only, so do not allow mapping as
 643                  * writable, because it's possible to leak a writable page
 644                  * reference and allows user-space to still modify it after
 645                  * freezing, while verifier will assume contents do not change
 646                  */
 647                 if (map->map_flags & BPF_F_RDONLY_PROG) {
 648                         err = -EACCES;
 649                         goto out;
 650                 }
 651         }
 652
 653         /* set default open/close callbacks */
 654         vma->vm_ops = &bpf_map_default_vmops;
 655         vma->vm_private_data = map;
 656         vma->vm_flags &= ~VM_MAYEXEC;
 657         if (!(vma->vm_flags & VM_WRITE))
 658                 /* disallow re-mapping with PROT_WRITE */
 659                 vma->vm_flags &= ~VM_MAYWRITE;
 660
 661         err = map->ops->map_mmap(map, vma);
 662         if (err)
 663                 goto out;
 664
 665         if (vma->vm_flags & VM_MAYWRITE)
 666                 map->writecnt++;
 667 out:
 668         mutex_unlock(&map->freeze_mutex);
 669         return err;
 670 }
 671
 672 static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
 673 {
 674         struct bpf_map *map = filp->private_data;
 675
 676         if (map->ops->map_poll)
 677                 return map->ops->map_poll(map, filp, pts);
 678
 679         return EPOLLERR;
 680 }
 681
 682 const struct file_operations bpf_map_fops = {
 683 #ifdef CONFIG_PROC_FS
 684         .show_fdinfo    = bpf_map_show_fdinfo,
 685 #endif
 686         .release        = bpf_map_release,
 687         .read           = bpf_dummy_read,
 688         .write          = bpf_dummy_write,
 689         .mmap           = bpf_map_mmap,
 690         .poll           = bpf_map_poll,
 691 };
 692
 693 int bpf_map_new_fd(struct bpf_map *map, int flags)
 694 {
 695         int ret;
 696
 697         ret = security_bpf_map(map, OPEN_FMODE(flags));
 698         if (ret < 0)
 699                 return ret;
 700
 701         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 702                                 flags | O_CLOEXEC);
 703 }
 704
 705 int bpf_get_file_flag(int flags)
 706 {
 707         if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
 708                 return -EINVAL;
 709         if (flags & BPF_F_RDONLY)
 710                 return O_RDONLY;
 711         if (flags & BPF_F_WRONLY)
 712                 return O_WRONLY;
 713         return O_RDWR;
 714 }
 715
 716 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 717 #define CHECK_ATTR(CMD) \
 718         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 719                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 720                    sizeof(*attr) - \
 721                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 722                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 723
 724 /* dst and src must have at least "size" number of bytes.
 725  * Return strlen on success and < 0 on error.
 726  */
 727 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
 728 {
 729         const char *end = src + size;
 730         const char *orig_src = src;
 731
 732         memset(dst, 0, size);
 733         /* Copy all isalnum(), '_' and '.' chars. */
 734         while (src < end && *src) {
 735                 if (!isalnum(*src) &&
 736                     *src != '_' && *src != '.')
 737                         return -EINVAL;
 738                 *dst++ = *src++;
 739         }
 740
 741         /* No '\0' found in "size" number of bytes */
 742         if (src == end)
 743                 return -EINVAL;
 744
 745         return src - orig_src;
 746 }
 747
 748 int map_check_no_btf(const struct bpf_map *map,
 749                      const struct btf *btf,
 750                      const struct btf_type *key_type,
 751                      const struct btf_type *value_type)
 752 {
 753         return -ENOTSUPP;
 754 }
 755
 756 static int map_check_btf(struct bpf_map *map, const struct btf *btf,
 757                          u32 btf_key_id, u32 btf_value_id)
 758 {
 759         const struct btf_type *key_type, *value_type;
 760         u32 key_size, value_size;
 761         int ret = 0;
 762
 763         /* Some maps allow key to be unspecified. */
 764         if (btf_key_id) {
 765                 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
 766                 if (!key_type || key_size != map->key_size)
 767                         return -EINVAL;
 768         } else {
 769                 key_type = btf_type_by_id(btf, 0);
 770                 if (!map->ops->map_check_btf)
 771                         return -EINVAL;
 772         }
 773
 774         value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
 775         if (!value_type || value_size != map->value_size)
 776                 return -EINVAL;
 777
 778         map->spin_lock_off = btf_find_spin_lock(btf, value_type);
 779
 780         if (map_value_has_spin_lock(map)) {
 781                 if (map->map_flags & BPF_F_RDONLY_PROG)
 782                         return -EACCES;
 783                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 784                     map->map_type != BPF_MAP_TYPE_ARRAY &&
 785                     map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
 786                     map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
 787                     map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
 788                     map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
 789                         return -ENOTSUPP;
 790                 if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
 791                     map->value_size) {
 792                         WARN_ONCE(1,
 793                                   "verifier bug spin_lock_off %d value_size %d\n",
 794                                   map->spin_lock_off, map->value_size);
 795                         return -EFAULT;
 796                 }
 797         }
 798
 799         map->timer_off = btf_find_timer(btf, value_type);
 800         if (map_value_has_timer(map)) {
 801                 if (map->map_flags & BPF_F_RDONLY_PROG)
 802                         return -EACCES;
 803                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 804                     map->map_type != BPF_MAP_TYPE_LRU_HASH &&
 805                     map->map_type != BPF_MAP_TYPE_ARRAY)
 806                         return -EOPNOTSUPP;
 807         }
 808
 809         if (map->ops->map_check_btf)
 810                 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
 811
 812         return ret;
 813 }
 814
 815 #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id
 816 /* called via syscall */
 817 static int map_create(union bpf_attr *attr)
 818 {
 819         int numa_node = bpf_map_attr_numa_node(attr);
 820         struct bpf_map *map;
 821         int f_flags;
 822         int err;
 823
 824         err = CHECK_ATTR(BPF_MAP_CREATE);
 825         if (err)
 826                 return -EINVAL;
 827
 828         if (attr->btf_vmlinux_value_type_id) {
 829                 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
 830                     attr->btf_key_type_id || attr->btf_value_type_id)
 831                         return -EINVAL;
 832         } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
 833                 return -EINVAL;
 834         }
 835
 836         f_flags = bpf_get_file_flag(attr->map_flags);
 837         if (f_flags < 0)
 838                 return f_flags;
 839
 840         if (numa_node != NUMA_NO_NODE &&
 841             ((unsigned int)numa_node >= nr_node_ids ||
 842              !node_online(numa_node)))
 843                 return -EINVAL;
 844
 845         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 846         map = find_and_alloc_map(attr);
 847         if (IS_ERR(map))
 848                 return PTR_ERR(map);
 849
 850         err = bpf_obj_name_cpy(map->name, attr->map_name,
 851                                sizeof(attr->map_name));
 852         if (err < 0)
 853                 goto free_map;
 854
 855         atomic64_set(&map->refcnt, 1);
 856         atomic64_set(&map->usercnt, 1);
 857         mutex_init(&map->freeze_mutex);
 858
 859         map->spin_lock_off = -EINVAL;
 860         map->timer_off = -EINVAL;
 861         if (attr->btf_key_type_id || attr->btf_value_type_id ||
 862             /* Even the map's value is a kernel's struct,
 863              * the bpf_prog.o must have BTF to begin with
 864              * to figure out the corresponding kernel's
 865              * counter part.  Thus, attr->btf_fd has
 866              * to be valid also.
 867              */
 868             attr->btf_vmlinux_value_type_id) {
 869                 struct btf *btf;
 870
 871                 btf = btf_get_by_fd(attr->btf_fd);
 872                 if (IS_ERR(btf)) {
 873                         err = PTR_ERR(btf);
 874                         goto free_map;
 875                 }
 876                 if (btf_is_kernel(btf)) {
 877                         btf_put(btf);
 878                         err = -EACCES;
 879                         goto free_map;
 880                 }
 881                 map->btf = btf;
 882
 883                 if (attr->btf_value_type_id) {
 884                         err = map_check_btf(map, btf, attr->btf_key_type_id,
 885                                             attr->btf_value_type_id);
 886                         if (err)
 887                                 goto free_map;
 888                 }
 889
 890                 map->btf_key_type_id = attr->btf_key_type_id;
 891                 map->btf_value_type_id = attr->btf_value_type_id;
 892                 map->btf_vmlinux_value_type_id =
 893                         attr->btf_vmlinux_value_type_id;
 894         }
 895
 896         err = security_bpf_map_alloc(map);
 897         if (err)
 898                 goto free_map;
 899
 900         err = bpf_map_alloc_id(map);
 901         if (err)
 902                 goto free_map_sec;
 903
 904         bpf_map_save_memcg(map);
 905
 906         err = bpf_map_new_fd(map, f_flags);
 907         if (err < 0) {
 908                 /* failed to allocate fd.
 909                  * bpf_map_put_with_uref() is needed because the above
 910                  * bpf_map_alloc_id() has published the map
 911                  * to the userspace and the userspace may
 912                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
 913                  */
 914                 bpf_map_put_with_uref(map);
 915                 return err;
 916         }
 917
 918         return err;
 919
 920 free_map_sec:
 921         security_bpf_map_free(map);
 922 free_map:
 923         btf_put(map->btf);
 924         map->ops->map_free(map);
 925         return err;
 926 }
 927
 928 /* if error is returned, fd is released.
 929  * On success caller should complete fd access with matching fdput()
 930  */
 931 struct bpf_map *__bpf_map_get(struct fd f)
 932 {
 933         if (!f.file)
 934                 return ERR_PTR(-EBADF);
 935         if (f.file->f_op != &bpf_map_fops) {
 936                 fdput(f);
 937                 return ERR_PTR(-EINVAL);
 938         }
 939
 940         return f.file->private_data;
 941 }
 942
 943 void bpf_map_inc(struct bpf_map *map)
 944 {
 945         atomic64_inc(&map->refcnt);
 946 }
 947 EXPORT_SYMBOL_GPL(bpf_map_inc);
 948
 949 void bpf_map_inc_with_uref(struct bpf_map *map)
 950 {
 951         atomic64_inc(&map->refcnt);
 952         atomic64_inc(&map->usercnt);
 953 }
 954 EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
 955
 956 struct bpf_map *bpf_map_get(u32 ufd)
 957 {
 958         struct fd f = fdget(ufd);
 959         struct bpf_map *map;
 960
 961         map = __bpf_map_get(f);
 962         if (IS_ERR(map))
 963                 return map;
 964
 965         bpf_map_inc(map);
 966         fdput(f);
 967
 968         return map;
 969 }
 970
 971 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 972 {
 973         struct fd f = fdget(ufd);
 974         struct bpf_map *map;
 975
 976         map = __bpf_map_get(f);
 977         if (IS_ERR(map))
 978                 return map;
 979
 980         bpf_map_inc_with_uref(map);
 981         fdput(f);
 982
 983         return map;
 984 }
 985
 986 /* map_idr_lock should have been held */
 987 static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
 988 {
 989         int refold;
 990
 991         refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
 992         if (!refold)
 993                 return ERR_PTR(-ENOENT);
 994         if (uref)
 995                 atomic64_inc(&map->usercnt);
 996
 997         return map;
 998 }
 999
1000 struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
1001 {
1002         spin_lock_bh(&map_idr_lock);
1003         map = __bpf_map_inc_not_zero(map, false);
1004         spin_unlock_bh(&map_idr_lock);
1005
1006         return map;
1007 }
1008 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1009
1010 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
1011 {
1012         return -ENOTSUPP;
1013 }
1014
1015 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
1016 {
1017         if (key_size)
1018                 return vmemdup_user(ukey, key_size);
1019
1020         if (ukey)
1021                 return ERR_PTR(-EINVAL);
1022
1023         return NULL;
1024 }
1025
1026 static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
1027 {
1028         if (key_size)
1029                 return kvmemdup_bpfptr(ukey, key_size);
1030
1031         if (!bpfptr_is_null(ukey))
1032                 return ERR_PTR(-EINVAL);
1033
1034         return NULL;
1035 }
1036
1037 /* last field in 'union bpf_attr' used by this command */
1038 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1039
1040 static int map_lookup_elem(union bpf_attr *attr)
1041 {
1042         void __user *ukey = u64_to_user_ptr(attr->key);
1043         void __user *uvalue = u64_to_user_ptr(attr->value);
1044         int ufd = attr->map_fd;
1045         struct bpf_map *map;
1046         void *key, *value;
1047         u32 value_size;
1048         struct fd f;
1049         int err;
1050
1051         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1052                 return -EINVAL;
1053
1054         if (attr->flags & ~BPF_F_LOCK)
1055                 return -EINVAL;
1056
1057         f = fdget(ufd);
1058         map = __bpf_map_get(f);
1059         if (IS_ERR(map))
1060                 return PTR_ERR(map);
1061         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1062                 err = -EPERM;
1063                 goto err_put;
1064         }
1065
1066         if ((attr->flags & BPF_F_LOCK) &&
1067             !map_value_has_spin_lock(map)) {
1068                 err = -EINVAL;
1069                 goto err_put;
1070         }
1071
1072         key = __bpf_copy_key(ukey, map->key_size);
1073         if (IS_ERR(key)) {
1074                 err = PTR_ERR(key);
1075                 goto err_put;
1076         }
1077
1078         value_size = bpf_map_value_size(map);
1079
1080         err = -ENOMEM;
1081         value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1082         if (!value)
1083                 goto free_key;
1084
1085         err = bpf_map_copy_value(map, key, value, attr->flags);
1086         if (err)
1087                 goto free_value;
1088
1089         err = -EFAULT;
1090         if (copy_to_user(uvalue, value, value_size) != 0)
1091                 goto free_value;
1092
1093         err = 0;
1094
1095 free_value:
1096         kvfree(value);
1097 free_key:
1098         kvfree(key);
1099 err_put:
1100         fdput(f);
1101         return err;
1102 }
1103
1104
1105 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1106
1107 static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
1108 {
1109         bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1110         bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel);
1111         int ufd = attr->map_fd;
1112         struct bpf_map *map;
1113         void *key, *value;
1114         u32 value_size;
1115         struct fd f;
1116         int err;
1117
1118         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1119                 return -EINVAL;
1120
1121         f = fdget(ufd);
1122         map = __bpf_map_get(f);
1123         if (IS_ERR(map))
1124                 return PTR_ERR(map);
1125         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1126                 err = -EPERM;
1127                 goto err_put;
1128         }
1129
1130         if ((attr->flags & BPF_F_LOCK) &&
1131             !map_value_has_spin_lock(map)) {
1132                 err = -EINVAL;
1133                 goto err_put;
1134         }
1135
1136         key = ___bpf_copy_key(ukey, map->key_size);
1137         if (IS_ERR(key)) {
1138                 err = PTR_ERR(key);
1139                 goto err_put;
1140         }
1141
1142         value_size = bpf_map_value_size(map);
1143
1144         err = -ENOMEM;
1145         value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1146         if (!value)
1147                 goto free_key;
1148
1149         err = -EFAULT;
1150         if (copy_from_bpfptr(value, uvalue, value_size) != 0)
1151                 goto free_value;
1152
1153         err = bpf_map_update_value(map, f, key, value, attr->flags);
1154
1155 free_value:
1156         kvfree(value);
1157 free_key:
1158         kvfree(key);
1159 err_put:
1160         fdput(f);
1161         return err;
1162 }
1163
1164 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1165
1166 static int map_delete_elem(union bpf_attr *attr)
1167 {
1168         void __user *ukey = u64_to_user_ptr(attr->key);
1169         int ufd = attr->map_fd;
1170         struct bpf_map *map;
1171         struct fd f;
1172         void *key;
1173         int err;
1174
1175         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1176                 return -EINVAL;
1177
1178         f = fdget(ufd);
1179         map = __bpf_map_get(f);
1180         if (IS_ERR(map))
1181                 return PTR_ERR(map);
1182         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1183                 err = -EPERM;
1184                 goto err_put;
1185         }
1186
1187         key = __bpf_copy_key(ukey, map->key_size);
1188         if (IS_ERR(key)) {
1189                 err = PTR_ERR(key);
1190                 goto err_put;
1191         }
1192
1193         if (bpf_map_is_dev_bound(map)) {
1194                 err = bpf_map_offload_delete_elem(map, key);
1195                 goto out;
1196         } else if (IS_FD_PROG_ARRAY(map) ||
1197                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1198                 /* These maps require sleepable context */
1199                 err = map->ops->map_delete_elem(map, key);
1200                 goto out;
1201         }
1202
1203         bpf_disable_instrumentation();
1204         rcu_read_lock();
1205         err = map->ops->map_delete_elem(map, key);
1206         rcu_read_unlock();
1207         bpf_enable_instrumentation();
1208         maybe_wait_bpf_programs(map);
1209 out:
1210         kvfree(key);
1211 err_put:
1212         fdput(f);
1213         return err;
1214 }
1215
1216 /* last field in 'union bpf_attr' used by this command */
1217 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1218
1219 static int map_get_next_key(union bpf_attr *attr)
1220 {
1221         void __user *ukey = u64_to_user_ptr(attr->key);
1222         void __user *unext_key = u64_to_user_ptr(attr->next_key);
1223         int ufd = attr->map_fd;
1224         struct bpf_map *map;
1225         void *key, *next_key;
1226         struct fd f;
1227         int err;
1228
1229         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1230                 return -EINVAL;
1231
1232         f = fdget(ufd);
1233         map = __bpf_map_get(f);
1234         if (IS_ERR(map))
1235                 return PTR_ERR(map);
1236         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1237                 err = -EPERM;
1238                 goto err_put;
1239         }
1240
1241         if (ukey) {
1242                 key = __bpf_copy_key(ukey, map->key_size);
1243                 if (IS_ERR(key)) {
1244                         err = PTR_ERR(key);
1245                         goto err_put;
1246                 }
1247         } else {
1248                 key = NULL;
1249         }
1250
1251         err = -ENOMEM;
1252         next_key = kvmalloc(map->key_size, GFP_USER);
1253         if (!next_key)
1254                 goto free_key;
1255
1256         if (bpf_map_is_dev_bound(map)) {
1257                 err = bpf_map_offload_get_next_key(map, key, next_key);
1258                 goto out;
1259         }
1260
1261         rcu_read_lock();
1262         err = map->ops->map_get_next_key(map, key, next_key);
1263         rcu_read_unlock();
1264 out:
1265         if (err)
1266                 goto free_next_key;
1267
1268         err = -EFAULT;
1269         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1270                 goto free_next_key;
1271
1272         err = 0;
1273
1274 free_next_key:
1275         kvfree(next_key);
1276 free_key:
1277         kvfree(key);
1278 err_put:
1279         fdput(f);
1280         return err;
1281 }
1282
1283 int generic_map_delete_batch(struct bpf_map *map,
1284                              const union bpf_attr *attr,
1285                              union bpf_attr __user *uattr)
1286 {
1287         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1288         u32 cp, max_count;
1289         int err = 0;
1290         void *key;
1291
1292         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1293                 return -EINVAL;
1294
1295         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1296             !map_value_has_spin_lock(map)) {
1297                 return -EINVAL;
1298         }
1299
1300         max_count = attr->batch.count;
1301         if (!max_count)
1302                 return 0;
1303
1304         key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1305         if (!key)
1306                 return -ENOMEM;
1307
1308         for (cp = 0; cp < max_count; cp++) {
1309                 err = -EFAULT;
1310                 if (copy_from_user(key, keys + cp * map->key_size,
1311                                    map->key_size))
1312                         break;
1313
1314                 if (bpf_map_is_dev_bound(map)) {
1315                         err = bpf_map_offload_delete_elem(map, key);
1316                         break;
1317                 }
1318
1319                 bpf_disable_instrumentation();
1320                 rcu_read_lock();
1321                 err = map->ops->map_delete_elem(map, key);
1322                 rcu_read_unlock();
1323                 bpf_enable_instrumentation();
1324                 maybe_wait_bpf_programs(map);
1325                 if (err)
1326                         break;
1327         }
1328         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1329                 err = -EFAULT;
1330
1331         kvfree(key);
1332         return err;
1333 }
1334
1335 int generic_map_update_batch(struct bpf_map *map,
1336                              const union bpf_attr *attr,
1337                              union bpf_attr __user *uattr)
1338 {
1339         void __user *values = u64_to_user_ptr(attr->batch.values);
1340         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1341         u32 value_size, cp, max_count;
1342         int ufd = attr->batch.map_fd;
1343         void *key, *value;
1344         struct fd f;
1345         int err = 0;
1346
1347         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1348                 return -EINVAL;
1349
1350         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1351             !map_value_has_spin_lock(map)) {
1352                 return -EINVAL;
1353         }
1354
1355         value_size = bpf_map_value_size(map);
1356
1357         max_count = attr->batch.count;
1358         if (!max_count)
1359                 return 0;
1360
1361         key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1362         if (!key)
1363                 return -ENOMEM;
1364
1365         value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1366         if (!value) {
1367                 kvfree(key);
1368                 return -ENOMEM;
1369         }
1370
1371         f = fdget(ufd); /* bpf_map_do_batch() guarantees ufd is valid */
1372         for (cp = 0; cp < max_count; cp++) {
1373                 err = -EFAULT;
1374                 if (copy_from_user(key, keys + cp * map->key_size,
1375                     map->key_size) ||
1376                     copy_from_user(value, values + cp * value_size, value_size))
1377                         break;
1378
1379                 err = bpf_map_update_value(map, f, key, value,
1380                                            attr->batch.elem_flags);
1381
1382                 if (err)
1383                         break;
1384         }
1385
1386         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1387                 err = -EFAULT;
1388
1389         kvfree(value);
1390         kvfree(key);
1391         fdput(f);
1392         return err;
1393 }
1394
1395 #define MAP_LOOKUP_RETRIES 3
1396
1397 int generic_map_lookup_batch(struct bpf_map *map,
1398                                     const union bpf_attr *attr,
1399                                     union bpf_attr __user *uattr)
1400 {
1401         void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
1402         void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
1403         void __user *values = u64_to_user_ptr(attr->batch.values);
1404         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1405         void *buf, *buf_prevkey, *prev_key, *key, *value;
1406         int err, retry = MAP_LOOKUP_RETRIES;
1407         u32 value_size, cp, max_count;
1408
1409         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1410                 return -EINVAL;
1411
1412         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1413             !map_value_has_spin_lock(map))
1414                 return -EINVAL;
1415
1416         value_size = bpf_map_value_size(map);
1417
1418         max_count = attr->batch.count;
1419         if (!max_count)
1420                 return 0;
1421
1422         if (put_user(0, &uattr->batch.count))
1423                 return -EFAULT;
1424
1425         buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1426         if (!buf_prevkey)
1427                 return -ENOMEM;
1428
1429         buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
1430         if (!buf) {
1431                 kvfree(buf_prevkey);
1432                 return -ENOMEM;
1433         }
1434
1435         err = -EFAULT;
1436         prev_key = NULL;
1437         if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
1438                 goto free_buf;
1439         key = buf;
1440         value = key + map->key_size;
1441         if (ubatch)
1442                 prev_key = buf_prevkey;
1443
1444         for (cp = 0; cp < max_count;) {
1445                 rcu_read_lock();
1446                 err = map->ops->map_get_next_key(map, prev_key, key);
1447                 rcu_read_unlock();
1448                 if (err)
1449                         break;
1450                 err = bpf_map_copy_value(map, key, value,
1451                                          attr->batch.elem_flags);
1452
1453                 if (err == -ENOENT) {
1454                         if (retry) {
1455                                 retry--;
1456                                 continue;
1457                         }
1458                         err = -EINTR;
1459                         break;
1460                 }
1461
1462                 if (err)
1463                         goto free_buf;
1464
1465                 if (copy_to_user(keys + cp * map->key_size, key,
1466                                  map->key_size)) {
1467                         err = -EFAULT;
1468                         goto free_buf;
1469                 }
1470                 if (copy_to_user(values + cp * value_size, value, value_size)) {
1471                         err = -EFAULT;
1472                         goto free_buf;
1473                 }
1474
1475                 if (!prev_key)
1476                         prev_key = buf_prevkey;
1477
1478                 swap(prev_key, key);
1479                 retry = MAP_LOOKUP_RETRIES;
1480                 cp++;
1481         }
1482
1483         if (err == -EFAULT)
1484                 goto free_buf;
1485
1486         if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
1487                     (cp && copy_to_user(uobatch, prev_key, map->key_size))))
1488                 err = -EFAULT;
1489
1490 free_buf:
1491         kvfree(buf_prevkey);
1492         kvfree(buf);
1493         return err;
1494 }
1495
1496 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
1497
1498 static int map_lookup_and_delete_elem(union bpf_attr *attr)
1499 {
1500         void __user *ukey = u64_to_user_ptr(attr->key);
1501         void __user *uvalue = u64_to_user_ptr(attr->value);
1502         int ufd = attr->map_fd;
1503         struct bpf_map *map;
1504         void *key, *value;
1505         u32 value_size;
1506         struct fd f;
1507         int err;
1508
1509         if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1510                 return -EINVAL;
1511
1512         if (attr->flags & ~BPF_F_LOCK)
1513                 return -EINVAL;
1514
1515         f = fdget(ufd);
1516         map = __bpf_map_get(f);
1517         if (IS_ERR(map))
1518                 return PTR_ERR(map);
1519         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
1520             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1521                 err = -EPERM;
1522                 goto err_put;
1523         }
1524
1525         if (attr->flags &&
1526             (map->map_type == BPF_MAP_TYPE_QUEUE ||
1527              map->map_type == BPF_MAP_TYPE_STACK)) {
1528                 err = -EINVAL;
1529                 goto err_put;
1530         }
1531
1532         if ((attr->flags & BPF_F_LOCK) &&
1533             !map_value_has_spin_lock(map)) {
1534                 err = -EINVAL;
1535                 goto err_put;
1536         }
1537
1538         key = __bpf_copy_key(ukey, map->key_size);
1539         if (IS_ERR(key)) {
1540                 err = PTR_ERR(key);
1541                 goto err_put;
1542         }
1543
1544         value_size = bpf_map_value_size(map);
1545
1546         err = -ENOMEM;
1547         value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1548         if (!value)
1549                 goto free_key;
1550
1551         err = -ENOTSUPP;
1552         if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1553             map->map_type == BPF_MAP_TYPE_STACK) {
1554                 err = map->ops->map_pop_elem(map, value);
1555         } else if (map->map_type == BPF_MAP_TYPE_HASH ||
1556                    map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1557                    map->map_type == BPF_MAP_TYPE_LRU_HASH ||
1558                    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
1559                 if (!bpf_map_is_dev_bound(map)) {
1560                         bpf_disable_instrumentation();
1561                         rcu_read_lock();
1562                         err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
1563                         rcu_read_unlock();
1564                         bpf_enable_instrumentation();
1565                 }
1566         }
1567
1568         if (err)
1569                 goto free_value;
1570
1571         if (copy_to_user(uvalue, value, value_size) != 0) {
1572                 err = -EFAULT;
1573                 goto free_value;
1574         }
1575
1576         err = 0;
1577
1578 free_value:
1579         kvfree(value);
1580 free_key:
1581         kvfree(key);
1582 err_put:
1583         fdput(f);
1584         return err;
1585 }
1586
1587 #define BPF_MAP_FREEZE_LAST_FIELD map_fd
1588
1589 static int map_freeze(const union bpf_attr *attr)
1590 {
1591         int err = 0, ufd = attr->map_fd;
1592         struct bpf_map *map;
1593         struct fd f;
1594
1595         if (CHECK_ATTR(BPF_MAP_FREEZE))
1596                 return -EINVAL;
1597
1598         f = fdget(ufd);
1599         map = __bpf_map_get(f);
1600         if (IS_ERR(map))
1601                 return PTR_ERR(map);
1602
1603         if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS ||
1604             map_value_has_timer(map)) {
1605                 fdput(f);
1606                 return -ENOTSUPP;
1607         }
1608
1609         mutex_lock(&map->freeze_mutex);
1610
1611         if (map->writecnt) {
1612                 err = -EBUSY;
1613                 goto err_put;
1614         }
1615         if (READ_ONCE(map->frozen)) {
1616                 err = -EBUSY;
1617                 goto err_put;
1618         }
1619         if (!bpf_capable()) {
1620                 err = -EPERM;
1621                 goto err_put;
1622         }
1623
1624         WRITE_ONCE(map->frozen, true);
1625 err_put:
1626         mutex_unlock(&map->freeze_mutex);
1627         fdput(f);
1628         return err;
1629 }
1630
1631 static const struct bpf_prog_ops * const bpf_prog_types[] = {
1632 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
1633         [_id] = & _name ## _prog_ops,
1634 #define BPF_MAP_TYPE(_id, _ops)
1635 #define BPF_LINK_TYPE(_id, _name)
1636 #include <linux/bpf_types.h>
1637 #undef BPF_PROG_TYPE
1638 #undef BPF_MAP_TYPE
1639 #undef BPF_LINK_TYPE
1640 };
1641
1642 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
1643 {
1644         const struct bpf_prog_ops *ops;
1645
1646         if (type >= ARRAY_SIZE(bpf_prog_types))
1647                 return -EINVAL;
1648         type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
1649         ops = bpf_prog_types[type];
1650         if (!ops)
1651                 return -EINVAL;
1652
1653         if (!bpf_prog_is_dev_bound(prog->aux))
1654                 prog->aux->ops = ops;
1655         else
1656                 prog->aux->ops = &bpf_offload_prog_ops;
1657         prog->type = type;
1658         return 0;
1659 }
1660
1661 enum bpf_audit {
1662         BPF_AUDIT_LOAD,
1663         BPF_AUDIT_UNLOAD,
1664         BPF_AUDIT_MAX,
1665 };
1666
1667 static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
1668         [BPF_AUDIT_LOAD]   = "LOAD",
1669         [BPF_AUDIT_UNLOAD] = "UNLOAD",
1670 };
1671
1672 static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
1673 {
1674         struct audit_context *ctx = NULL;
1675         struct audit_buffer *ab;
1676
1677         if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
1678                 return;
1679         if (audit_enabled == AUDIT_OFF)
1680                 return;
1681         if (op == BPF_AUDIT_LOAD)
1682                 ctx = audit_context();
1683         ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
1684         if (unlikely(!ab))
1685                 return;
1686         audit_log_format(ab, "prog-id=%u op=%s",
1687                          prog->aux->id, bpf_audit_str[op]);
1688         audit_log_end(ab);
1689 }
1690
1691 static int bpf_prog_alloc_id(struct bpf_prog *prog)
1692 {
1693         int id;
1694
1695         idr_preload(GFP_KERNEL);
1696         spin_lock_bh(&prog_idr_lock);
1697         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
1698         if (id > 0)
1699                 prog->aux->id = id;
1700         spin_unlock_bh(&prog_idr_lock);
1701         idr_preload_end();
1702
1703         /* id is in [1, INT_MAX) */
1704         if (WARN_ON_ONCE(!id))
1705                 return -ENOSPC;
1706
1707         return id > 0 ? 0 : id;
1708 }
1709
1710 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
1711 {
1712         unsigned long flags;
1713
1714         /* cBPF to eBPF migrations are currently not in the idr store.
1715          * Offloaded programs are removed from the store when their device
1716          * disappears - even if someone grabs an fd to them they are unusable,
1717          * simply waiting for refcnt to drop to be freed.
1718          */
1719         if (!prog->aux->id)
1720                 return;
1721
1722         if (do_idr_lock)
1723                 spin_lock_irqsave(&prog_idr_lock, flags);
1724         else
1725                 __acquire(&prog_idr_lock);
1726
1727         idr_remove(&prog_idr, prog->aux->id);
1728         prog->aux->id = 0;
1729
1730         if (do_idr_lock)
1731                 spin_unlock_irqrestore(&prog_idr_lock, flags);
1732         else
1733                 __release(&prog_idr_lock);
1734 }
1735
1736 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
1737 {
1738         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
1739
1740         kvfree(aux->func_info);
1741         kfree(aux->func_info_aux);
1742         free_uid(aux->user);
1743         security_bpf_prog_free(aux);
1744         bpf_prog_free(aux->prog);
1745 }
1746
1747 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
1748 {
1749         bpf_prog_kallsyms_del_all(prog);
1750         btf_put(prog->aux->btf);
1751         kvfree(prog->aux->jited_linfo);
1752         kvfree(prog->aux->linfo);
1753         kfree(prog->aux->kfunc_tab);
1754         if (prog->aux->attach_btf)
1755                 btf_put(prog->aux->attach_btf);
1756
1757         if (deferred) {
1758                 if (prog->aux->sleepable)
1759                         call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
1760                 else
1761                         call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1762         } else {
1763                 __bpf_prog_put_rcu(&prog->aux->rcu);
1764         }
1765 }
1766
1767 static void bpf_prog_put_deferred(struct work_struct *work)
1768 {
1769         struct bpf_prog_aux *aux;
1770         struct bpf_prog *prog;
1771
1772         aux = container_of(work, struct bpf_prog_aux, work);
1773         prog = aux->prog;
1774         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
1775         bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
1776         __bpf_prog_put_noref(prog, true);
1777 }
1778
1779 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
1780 {
1781         struct bpf_prog_aux *aux = prog->aux;
1782
1783         if (atomic64_dec_and_test(&aux->refcnt)) {
1784                 /* bpf_prog_free_id() must be called first */
1785                 bpf_prog_free_id(prog, do_idr_lock);
1786
1787                 if (in_irq() || irqs_disabled()) {
1788                         INIT_WORK(&aux->work, bpf_prog_put_deferred);
1789                         schedule_work(&aux->work);
1790                 } else {
1791                         bpf_prog_put_deferred(&aux->work);
1792                 }
1793         }
1794 }
1795
1796 void bpf_prog_put(struct bpf_prog *prog)
1797 {
1798         __bpf_prog_put(prog, true);
1799 }
1800 EXPORT_SYMBOL_GPL(bpf_prog_put);
1801
1802 static int bpf_prog_release(struct inode *inode, struct file *filp)
1803 {
1804         struct bpf_prog *prog = filp->private_data;
1805
1806         bpf_prog_put(prog);
1807         return 0;
1808 }
1809
1810 static void bpf_prog_get_stats(const struct bpf_prog *prog,
1811                                struct bpf_prog_stats *stats)
1812 {
1813         u64 nsecs = 0, cnt = 0, misses = 0;
1814         int cpu;
1815
1816         for_each_possible_cpu(cpu) {
1817                 const struct bpf_prog_stats *st;
1818                 unsigned int start;
1819                 u64 tnsecs, tcnt, tmisses;
1820
1821                 st = per_cpu_ptr(prog->stats, cpu);
1822                 do {
1823                         start = u64_stats_fetch_begin_irq(&st->syncp);
1824                         tnsecs = st->nsecs;
1825                         tcnt = st->cnt;
1826                         tmisses = st->misses;
1827                 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
1828                 nsecs += tnsecs;
1829                 cnt += tcnt;
1830                 misses += tmisses;
1831         }
1832         stats->nsecs = nsecs;
1833         stats->cnt = cnt;
1834         stats->misses = misses;
1835 }
1836
1837 #ifdef CONFIG_PROC_FS
1838 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
1839 {
1840         const struct bpf_prog *prog = filp->private_data;
1841         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
1842         struct bpf_prog_stats stats;
1843
1844         bpf_prog_get_stats(prog, &stats);
1845         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
1846         seq_printf(m,
1847                    "prog_type:\t%u\n"
1848                    "prog_jited:\t%u\n"
1849                    "prog_tag:\t%s\n"
1850                    "memlock:\t%llu\n"
1851                    "prog_id:\t%u\n"
1852                    "run_time_ns:\t%llu\n"
1853                    "run_cnt:\t%llu\n"
1854                    "recursion_misses:\t%llu\n",
1855                    prog->type,
1856                    prog->jited,
1857                    prog_tag,
1858                    prog->pages * 1ULL << PAGE_SHIFT,
1859                    prog->aux->id,
1860                    stats.nsecs,
1861                    stats.cnt,
1862                    stats.misses);
1863 }
1864 #endif
1865
1866 const struct file_operations bpf_prog_fops = {
1867 #ifdef CONFIG_PROC_FS
1868         .show_fdinfo    = bpf_prog_show_fdinfo,
1869 #endif
1870         .release        = bpf_prog_release,
1871         .read           = bpf_dummy_read,
1872         .write          = bpf_dummy_write,
1873 };
1874
1875 int bpf_prog_new_fd(struct bpf_prog *prog)
1876 {
1877         int ret;
1878
1879         ret = security_bpf_prog(prog);
1880         if (ret < 0)
1881                 return ret;
1882
1883         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
1884                                 O_RDWR | O_CLOEXEC);
1885 }
1886
1887 static struct bpf_prog *____bpf_prog_get(struct fd f)
1888 {
1889         if (!f.file)
1890                 return ERR_PTR(-EBADF);
1891         if (f.file->f_op != &bpf_prog_fops) {
1892                 fdput(f);
1893                 return ERR_PTR(-EINVAL);
1894         }
1895
1896         return f.file->private_data;
1897 }
1898
1899 void bpf_prog_add(struct bpf_prog *prog, int i)
1900 {
1901         atomic64_add(i, &prog->aux->refcnt);
1902 }
1903 EXPORT_SYMBOL_GPL(bpf_prog_add);
1904
1905 void bpf_prog_sub(struct bpf_prog *prog, int i)
1906 {
1907         /* Only to be used for undoing previous bpf_prog_add() in some
1908          * error path. We still know that another entity in our call
1909          * path holds a reference to the program, thus atomic_sub() can
1910          * be safely used in such cases!
1911          */
1912         WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
1913 }
1914 EXPORT_SYMBOL_GPL(bpf_prog_sub);
1915
1916 void bpf_prog_inc(struct bpf_prog *prog)
1917 {
1918         atomic64_inc(&prog->aux->refcnt);
1919 }
1920 EXPORT_SYMBOL_GPL(bpf_prog_inc);
1921
1922 /* prog_idr_lock should have been held */
1923 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
1924 {
1925         int refold;
1926
1927         refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1928
1929         if (!refold)
1930                 return ERR_PTR(-ENOENT);
1931
1932         return prog;
1933 }
1934 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
1935
1936 bool bpf_prog_get_ok(struct bpf_prog *prog,
1937                             enum bpf_prog_type *attach_type, bool attach_drv)
1938 {
1939         /* not an attachment, just a refcount inc, always allow */
1940         if (!attach_type)
1941                 return true;
1942
1943         if (prog->type != *attach_type)
1944                 return false;
1945         if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)
1946                 return false;
1947
1948         return true;
1949 }
1950
1951 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
1952                                        bool attach_drv)
1953 {
1954         struct fd f = fdget(ufd);
1955         struct bpf_prog *prog;
1956
1957         prog = ____bpf_prog_get(f);
1958         if (IS_ERR(prog))
1959                 return prog;
1960         if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
1961                 prog = ERR_PTR(-EINVAL);
1962                 goto out;
1963         }
1964
1965         bpf_prog_inc(prog);
1966 out:
1967         fdput(f);
1968         return prog;
1969 }
1970
1971 struct bpf_prog *bpf_prog_get(u32 ufd)
1972 {
1973         return __bpf_prog_get(ufd, NULL, false);
1974 }
1975
1976 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
1977                                        bool attach_drv)
1978 {
1979         return __bpf_prog_get(ufd, &type, attach_drv);
1980 }
1981 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
1982
1983 /* Initially all BPF programs could be loaded w/o specifying
1984  * expected_attach_type. Later for some of them specifying expected_attach_type
1985  * at load time became required so that program could be validated properly.
1986  * Programs of types that are allowed to be loaded both w/ and w/o (for
1987  * backward compatibility) expected_attach_type, should have the default attach
1988  * type assigned to expected_attach_type for the latter case, so that it can be
1989  * validated later at attach time.
1990  *
1991  * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1992  * prog type requires it but has some attach types that have to be backward
1993  * compatible.
1994  */
1995 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
1996 {
1997         switch (attr->prog_type) {
1998         case BPF_PROG_TYPE_CGROUP_SOCK:
1999                 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
2000                  * exist so checking for non-zero is the way to go here.
2001                  */
2002                 if (!attr->expected_attach_type)
2003                         attr->expected_attach_type =
2004                                 BPF_CGROUP_INET_SOCK_CREATE;
2005                 break;
2006         case BPF_PROG_TYPE_SK_REUSEPORT:
2007                 if (!attr->expected_attach_type)
2008                         attr->expected_attach_type =
2009                                 BPF_SK_REUSEPORT_SELECT;
2010                 break;
2011         }
2012 }
2013
2014 static int
2015 bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
2016                            enum bpf_attach_type expected_attach_type,
2017                            struct btf *attach_btf, u32 btf_id,
2018                            struct bpf_prog *dst_prog)
2019 {
2020         if (btf_id) {
2021                 if (btf_id > BTF_MAX_TYPE)
2022                         return -EINVAL;
2023
2024                 if (!attach_btf && !dst_prog)
2025                         return -EINVAL;
2026
2027                 switch (prog_type) {
2028                 case BPF_PROG_TYPE_TRACING:
2029                 case BPF_PROG_TYPE_LSM:
2030                 case BPF_PROG_TYPE_STRUCT_OPS:
2031                 case BPF_PROG_TYPE_EXT:
2032                         break;
2033                 default:
2034                         return -EINVAL;
2035                 }
2036         }
2037
2038         if (attach_btf && (!btf_id || dst_prog))
2039                 return -EINVAL;
2040
2041         if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
2042             prog_type != BPF_PROG_TYPE_EXT)
2043                 return -EINVAL;
2044
2045         switch (prog_type) {
2046         case BPF_PROG_TYPE_CGROUP_SOCK:
2047                 switch (expected_attach_type) {
2048                 case BPF_CGROUP_INET_SOCK_CREATE:
2049                 case BPF_CGROUP_INET_SOCK_RELEASE:
2050                 case BPF_CGROUP_INET4_POST_BIND:
2051                 case BPF_CGROUP_INET6_POST_BIND:
2052                         return 0;
2053                 default:
2054                         return -EINVAL;
2055                 }
2056         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2057                 switch (expected_attach_type) {
2058                 case BPF_CGROUP_INET4_BIND:
2059                 case BPF_CGROUP_INET6_BIND:
2060                 case BPF_CGROUP_INET4_CONNECT:
2061                 case BPF_CGROUP_INET6_CONNECT:
2062                 case BPF_CGROUP_INET4_GETPEERNAME:
2063                 case BPF_CGROUP_INET6_GETPEERNAME:
2064                 case BPF_CGROUP_INET4_GETSOCKNAME:
2065                 case BPF_CGROUP_INET6_GETSOCKNAME:
2066                 case BPF_CGROUP_UDP4_SENDMSG:
2067                 case BPF_CGROUP_UDP6_SENDMSG:
2068                 case BPF_CGROUP_UDP4_RECVMSG:
2069                 case BPF_CGROUP_UDP6_RECVMSG:
2070                         return 0;
2071                 default:
2072                         return -EINVAL;
2073                 }
2074         case BPF_PROG_TYPE_CGROUP_SKB:
2075                 switch (expected_attach_type) {
2076                 case BPF_CGROUP_INET_INGRESS:
2077                 case BPF_CGROUP_INET_EGRESS:
2078                         return 0;
2079                 default:
2080                         return -EINVAL;
2081                 }
2082         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2083                 switch (expected_attach_type) {
2084                 case BPF_CGROUP_SETSOCKOPT:
2085                 case BPF_CGROUP_GETSOCKOPT:
2086                         return 0;
2087                 default:
2088                         return -EINVAL;
2089                 }
2090         case BPF_PROG_TYPE_SK_LOOKUP:
2091                 if (expected_attach_type == BPF_SK_LOOKUP)
2092                         return 0;
2093                 return -EINVAL;
2094         case BPF_PROG_TYPE_SK_REUSEPORT:
2095                 switch (expected_attach_type) {
2096                 case BPF_SK_REUSEPORT_SELECT:
2097                 case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:
2098                         return 0;
2099                 default:
2100                         return -EINVAL;
2101                 }
2102         case BPF_PROG_TYPE_SYSCALL:
2103         case BPF_PROG_TYPE_EXT:
2104                 if (expected_attach_type)
2105                         return -EINVAL;
2106                 fallthrough;
2107         default:
2108                 return 0;
2109         }
2110 }
2111
2112 static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
2113 {
2114         switch (prog_type) {
2115         case BPF_PROG_TYPE_SCHED_CLS:
2116         case BPF_PROG_TYPE_SCHED_ACT:
2117         case BPF_PROG_TYPE_XDP:
2118         case BPF_PROG_TYPE_LWT_IN:
2119         case BPF_PROG_TYPE_LWT_OUT:
2120         case BPF_PROG_TYPE_LWT_XMIT:
2121         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2122         case BPF_PROG_TYPE_SK_SKB:
2123         case BPF_PROG_TYPE_SK_MSG:
2124         case BPF_PROG_TYPE_LIRC_MODE2:
2125         case BPF_PROG_TYPE_FLOW_DISSECTOR:
2126         case BPF_PROG_TYPE_CGROUP_DEVICE:
2127         case BPF_PROG_TYPE_CGROUP_SOCK:
2128         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2129         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2130         case BPF_PROG_TYPE_CGROUP_SYSCTL:
2131         case BPF_PROG_TYPE_SOCK_OPS:
2132         case BPF_PROG_TYPE_EXT: /* extends any prog */
2133                 return true;
2134         case BPF_PROG_TYPE_CGROUP_SKB:
2135                 /* always unpriv */
2136         case BPF_PROG_TYPE_SK_REUSEPORT:
2137                 /* equivalent to SOCKET_FILTER. need CAP_BPF only */
2138         default:
2139                 return false;
2140         }
2141 }
2142
2143 static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
2144 {
2145         switch (prog_type) {
2146         case BPF_PROG_TYPE_KPROBE:
2147         case BPF_PROG_TYPE_TRACEPOINT:
2148         case BPF_PROG_TYPE_PERF_EVENT:
2149         case BPF_PROG_TYPE_RAW_TRACEPOINT:
2150         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2151         case BPF_PROG_TYPE_TRACING:
2152         case BPF_PROG_TYPE_LSM:
2153         case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
2154         case BPF_PROG_TYPE_EXT: /* extends any prog */
2155                 return true;
2156         default:
2157                 return false;
2158         }
2159 }
2160
2161 /* last field in 'union bpf_attr' used by this command */
2162 #define BPF_PROG_LOAD_LAST_FIELD fd_array
2163
2164 static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr)
2165 {
2166         enum bpf_prog_type type = attr->prog_type;
2167         struct bpf_prog *prog, *dst_prog = NULL;
2168         struct btf *attach_btf = NULL;
2169         int err;
2170         char license[128];
2171         bool is_gpl;
2172
2173         if (CHECK_ATTR(BPF_PROG_LOAD))
2174                 return -EINVAL;
2175
2176         if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2177                                  BPF_F_ANY_ALIGNMENT |
2178                                  BPF_F_TEST_STATE_FREQ |
2179                                  BPF_F_SLEEPABLE |
2180                                  BPF_F_TEST_RND_HI32))
2181                 return -EINVAL;
2182
2183         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2184             (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2185             !bpf_capable())
2186                 return -EPERM;
2187
2188         /* copy eBPF program license from user space */
2189         if (strncpy_from_bpfptr(license,
2190                                 make_bpfptr(attr->license, uattr.is_kernel),
2191                                 sizeof(license) - 1) < 0)
2192                 return -EFAULT;
2193         license[sizeof(license) - 1] = 0;
2194
2195         /* eBPF programs must be GPL compatible to use GPL-ed functions */
2196         is_gpl = license_is_gpl_compatible(license);
2197
2198         if (attr->insn_cnt == 0 ||
2199             attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
2200                 return -E2BIG;
2201         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
2202             type != BPF_PROG_TYPE_CGROUP_SKB &&
2203             !bpf_capable())
2204                 return -EPERM;
2205
2206         if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN))
2207                 return -EPERM;
2208         if (is_perfmon_prog_type(type) && !perfmon_capable())
2209                 return -EPERM;
2210
2211         /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
2212          * or btf, we need to check which one it is
2213          */
2214         if (attr->attach_prog_fd) {
2215                 dst_prog = bpf_prog_get(attr->attach_prog_fd);
2216                 if (IS_ERR(dst_prog)) {
2217                         dst_prog = NULL;
2218                         attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
2219                         if (IS_ERR(attach_btf))
2220                                 return -EINVAL;
2221                         if (!btf_is_kernel(attach_btf)) {
2222                                 /* attaching through specifying bpf_prog's BTF
2223                                  * objects directly might be supported eventually
2224                                  */
2225                                 btf_put(attach_btf);
2226                                 return -ENOTSUPP;
2227                         }
2228                 }
2229         } else if (attr->attach_btf_id) {
2230                 /* fall back to vmlinux BTF, if BTF type ID is specified */
2231                 attach_btf = bpf_get_btf_vmlinux();
2232                 if (IS_ERR(attach_btf))
2233                         return PTR_ERR(attach_btf);
2234                 if (!attach_btf)
2235                         return -EINVAL;
2236                 btf_get(attach_btf);
2237         }
2238
2239         bpf_prog_load_fixup_attach_type(attr);
2240         if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2241                                        attach_btf, attr->attach_btf_id,
2242                                        dst_prog)) {
2243                 if (dst_prog)
2244                         bpf_prog_put(dst_prog);
2245                 if (attach_btf)
2246                         btf_put(attach_btf);
2247                 return -EINVAL;
2248         }
2249
2250         /* plain bpf_prog allocation */
2251         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2252         if (!prog) {
2253                 if (dst_prog)
2254                         bpf_prog_put(dst_prog);
2255                 if (attach_btf)
2256                         btf_put(attach_btf);
2257                 return -ENOMEM;
2258         }
2259
2260         prog->expected_attach_type = attr->expected_attach_type;
2261         prog->aux->attach_btf = attach_btf;
2262         prog->aux->attach_btf_id = attr->attach_btf_id;
2263         prog->aux->dst_prog = dst_prog;
2264         prog->aux->offload_requested = !!attr->prog_ifindex;
2265         prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
2266
2267         err = security_bpf_prog_alloc(prog->aux);
2268         if (err)
2269                 goto free_prog;
2270
2271         prog->aux->user = get_current_user();
2272         prog->len = attr->insn_cnt;
2273
2274         err = -EFAULT;
2275         if (copy_from_bpfptr(prog->insns,
2276                              make_bpfptr(attr->insns, uattr.is_kernel),
2277                              bpf_prog_insn_size(prog)) != 0)
2278                 goto free_prog_sec;
2279
2280         prog->orig_prog = NULL;
2281         prog->jited = 0;
2282
2283         atomic64_set(&prog->aux->refcnt, 1);
2284         prog->gpl_compatible = is_gpl ? 1 : 0;
2285
2286         if (bpf_prog_is_dev_bound(prog->aux)) {
2287                 err = bpf_prog_offload_init(prog, attr);
2288                 if (err)
2289                         goto free_prog_sec;
2290         }
2291
2292         /* find program type: socket_filter vs tracing_filter */
2293         err = find_prog_type(type, prog);
2294         if (err < 0)
2295                 goto free_prog_sec;
2296
2297         prog->aux->load_time = ktime_get_boottime_ns();
2298         err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
2299                                sizeof(attr->prog_name));
2300         if (err < 0)
2301                 goto free_prog_sec;
2302
2303         /* run eBPF verifier */
2304         err = bpf_check(&prog, attr, uattr);
2305         if (err < 0)
2306                 goto free_used_maps;
2307
2308         prog = bpf_prog_select_runtime(prog, &err);
2309         if (err < 0)
2310                 goto free_used_maps;
2311
2312         err = bpf_prog_alloc_id(prog);
2313         if (err)
2314                 goto free_used_maps;
2315
2316         /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2317          * effectively publicly exposed. However, retrieving via
2318          * bpf_prog_get_fd_by_id() will take another reference,
2319          * therefore it cannot be gone underneath us.
2320          *
2321          * Only for the time /after/ successful bpf_prog_new_fd()
2322          * and before returning to userspace, we might just hold
2323          * one reference and any parallel close on that fd could
2324          * rip everything out. Hence, below notifications must
2325          * happen before bpf_prog_new_fd().
2326          *
2327          * Also, any failure handling from this point onwards must
2328          * be using bpf_prog_put() given the program is exposed.
2329          */
2330         bpf_prog_kallsyms_add(prog);
2331         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
2332         bpf_audit_prog(prog, BPF_AUDIT_LOAD);
2333
2334         err = bpf_prog_new_fd(prog);
2335         if (err < 0)
2336                 bpf_prog_put(prog);
2337         return err;
2338
2339 free_used_maps:
2340         /* In case we have subprogs, we need to wait for a grace
2341          * period before we can tear down JIT memory since symbols
2342          * are already exposed under kallsyms.
2343          */
2344         __bpf_prog_put_noref(prog, prog->aux->func_cnt);
2345         return err;
2346 free_prog_sec:
2347         free_uid(prog->aux->user);
2348         security_bpf_prog_free(prog->aux);
2349 free_prog:
2350         if (prog->aux->attach_btf)
2351                 btf_put(prog->aux->attach_btf);
2352         bpf_prog_free(prog);
2353         return err;
2354 }
2355
2356 #define BPF_OBJ_LAST_FIELD file_flags
2357
2358 static int bpf_obj_pin(const union bpf_attr *attr)
2359 {
2360         if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
2361                 return -EINVAL;
2362
2363         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
2364 }
2365
2366 static int bpf_obj_get(const union bpf_attr *attr)
2367 {
2368         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
2369             attr->file_flags & ~BPF_OBJ_FLAG_MASK)
2370                 return -EINVAL;
2371
2372         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
2373                                 attr->file_flags);
2374 }
2375
2376 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2377                    const struct bpf_link_ops *ops, struct bpf_prog *prog)
2378 {
2379         atomic64_set(&link->refcnt, 1);
2380         link->type = type;
2381         link->id = 0;
2382         link->ops = ops;
2383         link->prog = prog;
2384 }
2385
2386 static void bpf_link_free_id(int id)
2387 {
2388         if (!id)
2389                 return;
2390
2391         spin_lock_bh(&link_idr_lock);
2392         idr_remove(&link_idr, id);
2393         spin_unlock_bh(&link_idr_lock);
2394 }
2395
2396 /* Clean up bpf_link and corresponding anon_inode file and FD. After
2397  * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
2398  * anon_inode's release() call. This helper marksbpf_link as
2399  * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
2400  * is not decremented, it's the responsibility of a calling code that failed
2401  * to complete bpf_link initialization.
2402  */
2403 void bpf_link_cleanup(struct bpf_link_primer *primer)
2404 {
2405         primer->link->prog = NULL;
2406         bpf_link_free_id(primer->id);
2407         fput(primer->file);
2408         put_unused_fd(primer->fd);
2409 }
2410
2411 void bpf_link_inc(struct bpf_link *link)
2412 {
2413         atomic64_inc(&link->refcnt);
2414 }
2415
2416 /* bpf_link_free is guaranteed to be called from process context */
2417 static void bpf_link_free(struct bpf_link *link)
2418 {
2419         bpf_link_free_id(link->id);
2420         if (link->prog) {
2421                 /* detach BPF program, clean up used resources */
2422                 link->ops->release(link);
2423                 bpf_prog_put(link->prog);
2424         }
2425         /* free bpf_link and its containing memory */
2426         link->ops->dealloc(link);
2427 }
2428
2429 static void bpf_link_put_deferred(struct work_struct *work)
2430 {
2431         struct bpf_link *link = container_of(work, struct bpf_link, work);
2432
2433         bpf_link_free(link);
2434 }
2435
2436 /* bpf_link_put can be called from atomic context, but ensures that resources
2437  * are freed from process context
2438  */
2439 void bpf_link_put(struct bpf_link *link)
2440 {
2441         if (!atomic64_dec_and_test(&link->refcnt))
2442                 return;
2443
2444         if (in_atomic()) {
2445                 INIT_WORK(&link->work, bpf_link_put_deferred);
2446                 schedule_work(&link->work);
2447         } else {
2448                 bpf_link_free(link);
2449         }
2450 }
2451
2452 static int bpf_link_release(struct inode *inode, struct file *filp)
2453 {
2454         struct bpf_link *link = filp->private_data;
2455
2456         bpf_link_put(link);
2457         return 0;
2458 }
2459
2460 #ifdef CONFIG_PROC_FS
2461 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
2462 #define BPF_MAP_TYPE(_id, _ops)
2463 #define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
2464 static const char *bpf_link_type_strs[] = {
2465         [BPF_LINK_TYPE_UNSPEC] = "<invalid>",
2466 #include <linux/bpf_types.h>
2467 };
2468 #undef BPF_PROG_TYPE
2469 #undef BPF_MAP_TYPE
2470 #undef BPF_LINK_TYPE
2471
2472 static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
2473 {
2474         const struct bpf_link *link = filp->private_data;
2475         const struct bpf_prog *prog = link->prog;
2476         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2477
2478         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2479         seq_printf(m,
2480                    "link_type:\t%s\n"
2481                    "link_id:\t%u\n"
2482                    "prog_tag:\t%s\n"
2483                    "prog_id:\t%u\n",
2484                    bpf_link_type_strs[link->type],
2485                    link->id,
2486                    prog_tag,
2487                    prog->aux->id);
2488         if (link->ops->show_fdinfo)
2489                 link->ops->show_fdinfo(link, m);
2490 }
2491 #endif
2492
2493 static const struct file_operations bpf_link_fops = {
2494 #ifdef CONFIG_PROC_FS
2495         .show_fdinfo    = bpf_link_show_fdinfo,
2496 #endif
2497         .release        = bpf_link_release,
2498         .read           = bpf_dummy_read,
2499         .write          = bpf_dummy_write,
2500 };
2501
2502 static int bpf_link_alloc_id(struct bpf_link *link)
2503 {
2504         int id;
2505
2506         idr_preload(GFP_KERNEL);
2507         spin_lock_bh(&link_idr_lock);
2508         id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
2509         spin_unlock_bh(&link_idr_lock);
2510         idr_preload_end();
2511
2512         return id;
2513 }
2514
2515 /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
2516  * reserving unused FD and allocating ID from link_idr. This is to be paired
2517  * with bpf_link_settle() to install FD and ID and expose bpf_link to
2518  * user-space, if bpf_link is successfully attached. If not, bpf_link and
2519  * pre-allocated resources are to be freed with bpf_cleanup() call. All the
2520  * transient state is passed around in struct bpf_link_primer.
2521  * This is preferred way to create and initialize bpf_link, especially when
2522  * there are complicated and expensive operations inbetween creating bpf_link
2523  * itself and attaching it to BPF hook. By using bpf_link_prime() and
2524  * bpf_link_settle() kernel code using bpf_link doesn't have to perform
2525  * expensive (and potentially failing) roll back operations in a rare case
2526  * that file, FD, or ID can't be allocated.
2527  */
2528 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
2529 {
2530         struct file *file;
2531         int fd, id;
2532
2533         fd = get_unused_fd_flags(O_CLOEXEC);
2534         if (fd < 0)
2535                 return fd;
2536
2537
2538         id = bpf_link_alloc_id(link);
2539         if (id < 0) {
2540                 put_unused_fd(fd);
2541                 return id;
2542         }
2543
2544         file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC);
2545         if (IS_ERR(file)) {
2546                 bpf_link_free_id(id);
2547                 put_unused_fd(fd);
2548                 return PTR_ERR(file);
2549         }
2550
2551         primer->link = link;
2552         primer->file = file;
2553         primer->fd = fd;
2554         primer->id = id;
2555         return 0;
2556 }
2557
2558 int bpf_link_settle(struct bpf_link_primer *primer)
2559 {
2560         /* make bpf_link fetchable by ID */
2561         spin_lock_bh(&link_idr_lock);
2562         primer->link->id = primer->id;
2563         spin_unlock_bh(&link_idr_lock);
2564         /* make bpf_link fetchable by FD */
2565         fd_install(primer->fd, primer->file);
2566         /* pass through installed FD */
2567         return primer->fd;
2568 }
2569
2570 int bpf_link_new_fd(struct bpf_link *link)
2571 {
2572         return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC);
2573 }
2574
2575 struct bpf_link *bpf_link_get_from_fd(u32 ufd)
2576 {
2577         struct fd f = fdget(ufd);
2578         struct bpf_link *link;
2579
2580         if (!f.file)
2581                 return ERR_PTR(-EBADF);
2582         if (f.file->f_op != &bpf_link_fops) {
2583                 fdput(f);
2584                 return ERR_PTR(-EINVAL);
2585         }
2586
2587         link = f.file->private_data;
2588         bpf_link_inc(link);
2589         fdput(f);
2590
2591         return link;
2592 }
2593
2594 struct bpf_tracing_link {
2595         struct bpf_link link;
2596         enum bpf_attach_type attach_type;
2597         struct bpf_trampoline *trampoline;
2598         struct bpf_prog *tgt_prog;
2599 };
2600
2601 static void bpf_tracing_link_release(struct bpf_link *link)
2602 {
2603         struct bpf_tracing_link *tr_link =
2604                 container_of(link, struct bpf_tracing_link, link);
2605
2606         WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
2607                                                 tr_link->trampoline));
2608
2609         bpf_trampoline_put(tr_link->trampoline);
2610
2611         /* tgt_prog is NULL if target is a kernel function */
2612         if (tr_link->tgt_prog)
2613                 bpf_prog_put(tr_link->tgt_prog);
2614 }
2615
2616 static void bpf_tracing_link_dealloc(struct bpf_link *link)
2617 {
2618         struct bpf_tracing_link *tr_link =
2619                 container_of(link, struct bpf_tracing_link, link);
2620
2621         kfree(tr_link);
2622 }
2623
2624 static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
2625                                          struct seq_file *seq)
2626 {
2627         struct bpf_tracing_link *tr_link =
2628                 container_of(link, struct bpf_tracing_link, link);
2629
2630         seq_printf(seq,
2631                    "attach_type:\t%d\n",
2632                    tr_link->attach_type);
2633 }
2634
2635 static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
2636                                            struct bpf_link_info *info)
2637 {
2638         struct bpf_tracing_link *tr_link =
2639                 container_of(link, struct bpf_tracing_link, link);
2640
2641         info->tracing.attach_type = tr_link->attach_type;
2642         bpf_trampoline_unpack_key(tr_link->trampoline->key,
2643                                   &info->tracing.target_obj_id,
2644                                   &info->tracing.target_btf_id);
2645
2646         return 0;
2647 }
2648
2649 static const struct bpf_link_ops bpf_tracing_link_lops = {
2650         .release = bpf_tracing_link_release,
2651         .dealloc = bpf_tracing_link_dealloc,
2652         .show_fdinfo = bpf_tracing_link_show_fdinfo,
2653         .fill_link_info = bpf_tracing_link_fill_link_info,
2654 };
2655
2656 static int bpf_tracing_prog_attach(struct bpf_prog *prog,
2657                                    int tgt_prog_fd,
2658                                    u32 btf_id)
2659 {
2660         struct bpf_link_primer link_primer;
2661         struct bpf_prog *tgt_prog = NULL;
2662         struct bpf_trampoline *tr = NULL;
2663         struct bpf_tracing_link *link;
2664         u64 key = 0;
2665         int err;
2666
2667         switch (prog->type) {
2668         case BPF_PROG_TYPE_TRACING:
2669                 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
2670                     prog->expected_attach_type != BPF_TRACE_FEXIT &&
2671                     prog->expected_attach_type != BPF_MODIFY_RETURN) {
2672                         err = -EINVAL;
2673                         goto out_put_prog;
2674                 }
2675                 break;
2676         case BPF_PROG_TYPE_EXT:
2677                 if (prog->expected_attach_type != 0) {
2678                         err = -EINVAL;
2679                         goto out_put_prog;
2680                 }
2681                 break;
2682         case BPF_PROG_TYPE_LSM:
2683                 if (prog->expected_attach_type != BPF_LSM_MAC) {
2684                         err = -EINVAL;
2685                         goto out_put_prog;
2686                 }
2687                 break;
2688         default:
2689                 err = -EINVAL;
2690                 goto out_put_prog;
2691         }
2692
2693         if (!!tgt_prog_fd != !!btf_id) {
2694                 err = -EINVAL;
2695                 goto out_put_prog;
2696         }
2697
2698         if (tgt_prog_fd) {
2699                 /* For now we only allow new targets for BPF_PROG_TYPE_EXT */
2700                 if (prog->type != BPF_PROG_TYPE_EXT) {
2701                         err = -EINVAL;
2702                         goto out_put_prog;
2703                 }
2704
2705                 tgt_prog = bpf_prog_get(tgt_prog_fd);
2706                 if (IS_ERR(tgt_prog)) {
2707                         err = PTR_ERR(tgt_prog);
2708                         tgt_prog = NULL;
2709                         goto out_put_prog;
2710                 }
2711
2712                 key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
2713         }
2714
2715         link = kzalloc(sizeof(*link), GFP_USER);
2716         if (!link) {
2717                 err = -ENOMEM;
2718                 goto out_put_prog;
2719         }
2720         bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING,
2721                       &bpf_tracing_link_lops, prog);
2722         link->attach_type = prog->expected_attach_type;
2723
2724         mutex_lock(&prog->aux->dst_mutex);
2725
2726         /* There are a few possible cases here:
2727          *
2728          * - if prog->aux->dst_trampoline is set, the program was just loaded
2729          *   and not yet attached to anything, so we can use the values stored
2730          *   in prog->aux
2731          *
2732          * - if prog->aux->dst_trampoline is NULL, the program has already been
2733          *   attached to a target and its initial target was cleared (below)
2734          *
2735          * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
2736          *   target_btf_id using the link_create API.
2737          *
2738          * - if tgt_prog == NULL when this function was called using the old
2739          *   raw_tracepoint_open API, and we need a target from prog->aux
2740          *
2741          * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program
2742          *   was detached and is going for re-attachment.
2743          */
2744         if (!prog->aux->dst_trampoline && !tgt_prog) {
2745                 /*
2746                  * Allow re-attach for TRACING and LSM programs. If it's
2747                  * currently linked, bpf_trampoline_link_prog will fail.
2748                  * EXT programs need to specify tgt_prog_fd, so they
2749                  * re-attach in separate code path.
2750                  */
2751                 if (prog->type != BPF_PROG_TYPE_TRACING &&
2752                     prog->type != BPF_PROG_TYPE_LSM) {
2753                         err = -EINVAL;
2754                         goto out_unlock;
2755                 }
2756                 btf_id = prog->aux->attach_btf_id;
2757                 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
2758         }
2759
2760         if (!prog->aux->dst_trampoline ||
2761             (key && key != prog->aux->dst_trampoline->key)) {
2762                 /* If there is no saved target, or the specified target is
2763                  * different from the destination specified at load time, we
2764                  * need a new trampoline and a check for compatibility
2765                  */
2766                 struct bpf_attach_target_info tgt_info = {};
2767
2768                 err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
2769                                               &tgt_info);
2770                 if (err)
2771                         goto out_unlock;
2772
2773                 tr = bpf_trampoline_get(key, &tgt_info);
2774                 if (!tr) {
2775                         err = -ENOMEM;
2776                         goto out_unlock;
2777                 }
2778         } else {
2779                 /* The caller didn't specify a target, or the target was the
2780                  * same as the destination supplied during program load. This
2781                  * means we can reuse the trampoline and reference from program
2782                  * load time, and there is no need to allocate a new one. This
2783                  * can only happen once for any program, as the saved values in
2784                  * prog->aux are cleared below.
2785                  */
2786                 tr = prog->aux->dst_trampoline;
2787                 tgt_prog = prog->aux->dst_prog;
2788         }
2789
2790         err = bpf_link_prime(&link->link, &link_primer);
2791         if (err)
2792                 goto out_unlock;
2793
2794         err = bpf_trampoline_link_prog(prog, tr);
2795         if (err) {
2796                 bpf_link_cleanup(&link_primer);
2797                 link = NULL;
2798                 goto out_unlock;
2799         }
2800
2801         link->tgt_prog = tgt_prog;
2802         link->trampoline = tr;
2803
2804         /* Always clear the trampoline and target prog from prog->aux to make
2805          * sure the original attach destination is not kept alive after a
2806          * program is (re-)attached to another target.
2807          */
2808         if (prog->aux->dst_prog &&
2809             (tgt_prog_fd || tr != prog->aux->dst_trampoline))
2810                 /* got extra prog ref from syscall, or attaching to different prog */
2811                 bpf_prog_put(prog->aux->dst_prog);
2812         if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
2813                 /* we allocated a new trampoline, so free the old one */
2814                 bpf_trampoline_put(prog->aux->dst_trampoline);
2815
2816         prog->aux->dst_prog = NULL;
2817         prog->aux->dst_trampoline = NULL;
2818         mutex_unlock(&prog->aux->dst_mutex);
2819
2820         return bpf_link_settle(&link_primer);
2821 out_unlock:
2822         if (tr && tr != prog->aux->dst_trampoline)
2823                 bpf_trampoline_put(tr);
2824         mutex_unlock(&prog->aux->dst_mutex);
2825         kfree(link);
2826 out_put_prog:
2827         if (tgt_prog_fd && tgt_prog)
2828                 bpf_prog_put(tgt_prog);
2829         return err;
2830 }
2831
2832 struct bpf_raw_tp_link {
2833         struct bpf_link link;
2834         struct bpf_raw_event_map *btp;
2835 };
2836
2837 static void bpf_raw_tp_link_release(struct bpf_link *link)
2838 {
2839         struct bpf_raw_tp_link *raw_tp =
2840                 container_of(link, struct bpf_raw_tp_link, link);
2841
2842         bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
2843         bpf_put_raw_tracepoint(raw_tp->btp);
2844 }
2845
2846 static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
2847 {
2848         struct bpf_raw_tp_link *raw_tp =
2849                 container_of(link, struct bpf_raw_tp_link, link);
2850
2851         kfree(raw_tp);
2852 }
2853
2854 static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
2855                                         struct seq_file *seq)
2856 {
2857         struct bpf_raw_tp_link *raw_tp_link =
2858                 container_of(link, struct bpf_raw_tp_link, link);
2859
2860         seq_printf(seq,
2861                    "tp_name:\t%s\n",
2862                    raw_tp_link->btp->tp->name);
2863 }
2864
2865 static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
2866                                           struct bpf_link_info *info)
2867 {
2868         struct bpf_raw_tp_link *raw_tp_link =
2869                 container_of(link, struct bpf_raw_tp_link, link);
2870         char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
2871         const char *tp_name = raw_tp_link->btp->tp->name;
2872         u32 ulen = info->raw_tracepoint.tp_name_len;
2873         size_t tp_len = strlen(tp_name);
2874
2875         if (!ulen ^ !ubuf)
2876                 return -EINVAL;
2877
2878         info->raw_tracepoint.tp_name_len = tp_len + 1;
2879
2880         if (!ubuf)
2881                 return 0;
2882
2883         if (ulen >= tp_len + 1) {
2884                 if (copy_to_user(ubuf, tp_name, tp_len + 1))
2885                         return -EFAULT;
2886         } else {
2887                 char zero = '\0';
2888
2889                 if (copy_to_user(ubuf, tp_name, ulen - 1))
2890                         return -EFAULT;
2891                 if (put_user(zero, ubuf + ulen - 1))
2892                         return -EFAULT;
2893                 return -ENOSPC;
2894         }
2895
2896         return 0;
2897 }
2898
2899 static const struct bpf_link_ops bpf_raw_tp_link_lops = {
2900         .release = bpf_raw_tp_link_release,
2901         .dealloc = bpf_raw_tp_link_dealloc,
2902         .show_fdinfo = bpf_raw_tp_link_show_fdinfo,
2903         .fill_link_info = bpf_raw_tp_link_fill_link_info,
2904 };
2905
2906 #ifdef CONFIG_PERF_EVENTS
2907 struct bpf_perf_link {
2908         struct bpf_link link;
2909         struct file *perf_file;
2910 };
2911
2912 static void bpf_perf_link_release(struct bpf_link *link)
2913 {
2914         struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
2915         struct perf_event *event = perf_link->perf_file->private_data;
2916
2917         perf_event_free_bpf_prog(event);
2918         fput(perf_link->perf_file);
2919 }
2920
2921 static void bpf_perf_link_dealloc(struct bpf_link *link)
2922 {
2923         struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
2924
2925         kfree(perf_link);
2926 }
2927
2928 static const struct bpf_link_ops bpf_perf_link_lops = {
2929         .release = bpf_perf_link_release,
2930         .dealloc = bpf_perf_link_dealloc,
2931 };
2932
2933 static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
2934 {
2935         struct bpf_link_primer link_primer;
2936         struct bpf_perf_link *link;
2937         struct perf_event *event;
2938         struct file *perf_file;
2939         int err;
2940
2941         if (attr->link_create.flags)
2942                 return -EINVAL;
2943
2944         perf_file = perf_event_get(attr->link_create.target_fd);
2945         if (IS_ERR(perf_file))
2946                 return PTR_ERR(perf_file);
2947
2948         link = kzalloc(sizeof(*link), GFP_USER);
2949         if (!link) {
2950                 err = -ENOMEM;
2951                 goto out_put_file;
2952         }
2953         bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog);
2954         link->perf_file = perf_file;
2955
2956         err = bpf_link_prime(&link->link, &link_primer);
2957         if (err) {
2958                 kfree(link);
2959                 goto out_put_file;
2960         }
2961
2962         event = perf_file->private_data;
2963         err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie);
2964         if (err) {
2965                 bpf_link_cleanup(&link_primer);
2966                 goto out_put_file;
2967         }
2968         /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */
2969         bpf_prog_inc(prog);
2970
2971         return bpf_link_settle(&link_primer);
2972
2973 out_put_file:
2974         fput(perf_file);
2975         return err;
2976 }
2977 #endif /* CONFIG_PERF_EVENTS */
2978
2979 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
2980
2981 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
2982 {
2983         struct bpf_link_primer link_primer;
2984         struct bpf_raw_tp_link *link;
2985         struct bpf_raw_event_map *btp;
2986         struct bpf_prog *prog;
2987         const char *tp_name;
2988         char buf[128];
2989         int err;
2990
2991         if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
2992                 return -EINVAL;
2993
2994         prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
2995         if (IS_ERR(prog))
2996                 return PTR_ERR(prog);
2997
2998         switch (prog->type) {
2999         case BPF_PROG_TYPE_TRACING:
3000         case BPF_PROG_TYPE_EXT:
3001         case BPF_PROG_TYPE_LSM:
3002                 if (attr->raw_tracepoint.name) {
3003                         /* The attach point for this category of programs
3004                          * should be specified via btf_id during program load.
3005                          */
3006                         err = -EINVAL;
3007                         goto out_put_prog;
3008                 }
3009                 if (prog->type == BPF_PROG_TYPE_TRACING &&
3010                     prog->expected_attach_type == BPF_TRACE_RAW_TP) {
3011                         tp_name = prog->aux->attach_func_name;
3012                         break;
3013                 }
3014                 err = bpf_tracing_prog_attach(prog, 0, 0);
3015                 if (err >= 0)
3016                         return err;
3017                 goto out_put_prog;
3018         case BPF_PROG_TYPE_RAW_TRACEPOINT:
3019         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
3020                 if (strncpy_from_user(buf,
3021                                       u64_to_user_ptr(attr->raw_tracepoint.name),
3022                                       sizeof(buf) - 1) < 0) {
3023                         err = -EFAULT;
3024                         goto out_put_prog;
3025                 }
3026                 buf[sizeof(buf) - 1] = 0;
3027                 tp_name = buf;
3028                 break;
3029         default:
3030                 err = -EINVAL;
3031                 goto out_put_prog;
3032         }
3033
3034         btp = bpf_get_raw_tracepoint(tp_name);
3035         if (!btp) {
3036                 err = -ENOENT;
3037                 goto out_put_prog;
3038         }
3039
3040         link = kzalloc(sizeof(*link), GFP_USER);
3041         if (!link) {
3042                 err = -ENOMEM;
3043                 goto out_put_btp;
3044         }
3045         bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
3046                       &bpf_raw_tp_link_lops, prog);
3047         link->btp = btp;
3048
3049         err = bpf_link_prime(&link->link, &link_primer);
3050         if (err) {
3051                 kfree(link);
3052                 goto out_put_btp;
3053         }
3054
3055         err = bpf_probe_register(link->btp, prog);
3056         if (err) {
3057                 bpf_link_cleanup(&link_primer);
3058                 goto out_put_btp;
3059         }
3060
3061         return bpf_link_settle(&link_primer);
3062
3063 out_put_btp:
3064         bpf_put_raw_tracepoint(btp);
3065 out_put_prog:
3066         bpf_prog_put(prog);
3067         return err;
3068 }
3069
3070 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
3071                                              enum bpf_attach_type attach_type)
3072 {
3073         switch (prog->type) {
3074         case BPF_PROG_TYPE_CGROUP_SOCK:
3075         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3076         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3077         case BPF_PROG_TYPE_SK_LOOKUP:
3078                 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
3079         case BPF_PROG_TYPE_CGROUP_SKB:
3080                 if (!capable(CAP_NET_ADMIN))
3081                         /* cg-skb progs can be loaded by unpriv user.
3082                          * check permissions at attach time.
3083                          */
3084                         return -EPERM;
3085                 return prog->enforce_expected_attach_type &&
3086                         prog->expected_attach_type != attach_type ?
3087                         -EINVAL : 0;
3088         default:
3089                 return 0;
3090         }
3091 }
3092
3093 static enum bpf_prog_type
3094 attach_type_to_prog_type(enum bpf_attach_type attach_type)
3095 {
3096         switch (attach_type) {
3097         case BPF_CGROUP_INET_INGRESS:
3098         case BPF_CGROUP_INET_EGRESS:
3099                 return BPF_PROG_TYPE_CGROUP_SKB;
3100         case BPF_CGROUP_INET_SOCK_CREATE:
3101         case BPF_CGROUP_INET_SOCK_RELEASE:
3102         case BPF_CGROUP_INET4_POST_BIND:
3103         case BPF_CGROUP_INET6_POST_BIND:
3104                 return BPF_PROG_TYPE_CGROUP_SOCK;
3105         case BPF_CGROUP_INET4_BIND:
3106         case BPF_CGROUP_INET6_BIND:
3107         case BPF_CGROUP_INET4_CONNECT:
3108         case BPF_CGROUP_INET6_CONNECT:
3109         case BPF_CGROUP_INET4_GETPEERNAME:
3110         case BPF_CGROUP_INET6_GETPEERNAME:
3111         case BPF_CGROUP_INET4_GETSOCKNAME:
3112         case BPF_CGROUP_INET6_GETSOCKNAME:
3113         case BPF_CGROUP_UDP4_SENDMSG:
3114         case BPF_CGROUP_UDP6_SENDMSG:
3115         case BPF_CGROUP_UDP4_RECVMSG:
3116         case BPF_CGROUP_UDP6_RECVMSG:
3117                 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
3118         case BPF_CGROUP_SOCK_OPS:
3119                 return BPF_PROG_TYPE_SOCK_OPS;
3120         case BPF_CGROUP_DEVICE:
3121                 return BPF_PROG_TYPE_CGROUP_DEVICE;
3122         case BPF_SK_MSG_VERDICT:
3123                 return BPF_PROG_TYPE_SK_MSG;
3124         case BPF_SK_SKB_STREAM_PARSER:
3125         case BPF_SK_SKB_STREAM_VERDICT:
3126         case BPF_SK_SKB_VERDICT:
3127                 return BPF_PROG_TYPE_SK_SKB;
3128         case BPF_LIRC_MODE2:
3129                 return BPF_PROG_TYPE_LIRC_MODE2;
3130         case BPF_FLOW_DISSECTOR:
3131                 return BPF_PROG_TYPE_FLOW_DISSECTOR;
3132         case BPF_CGROUP_SYSCTL:
3133                 return BPF_PROG_TYPE_CGROUP_SYSCTL;
3134         case BPF_CGROUP_GETSOCKOPT:
3135         case BPF_CGROUP_SETSOCKOPT:
3136                 return BPF_PROG_TYPE_CGROUP_SOCKOPT;
3137         case BPF_TRACE_ITER:
3138                 return BPF_PROG_TYPE_TRACING;
3139         case BPF_SK_LOOKUP:
3140                 return BPF_PROG_TYPE_SK_LOOKUP;
3141         case BPF_XDP:
3142                 return BPF_PROG_TYPE_XDP;
3143         default:
3144                 return BPF_PROG_TYPE_UNSPEC;
3145         }
3146 }
3147
3148 #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd
3149
3150 #define BPF_F_ATTACH_MASK \
3151         (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE)
3152
3153 static int bpf_prog_attach(const union bpf_attr *attr)
3154 {
3155         enum bpf_prog_type ptype;
3156         struct bpf_prog *prog;
3157         int ret;
3158
3159         if (CHECK_ATTR(BPF_PROG_ATTACH))
3160                 return -EINVAL;
3161
3162         if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
3163                 return -EINVAL;
3164
3165         ptype = attach_type_to_prog_type(attr->attach_type);
3166         if (ptype == BPF_PROG_TYPE_UNSPEC)
3167                 return -EINVAL;
3168
3169         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
3170         if (IS_ERR(prog))
3171                 return PTR_ERR(prog);
3172
3173         if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
3174                 bpf_prog_put(prog);
3175                 return -EINVAL;
3176         }
3177
3178         switch (ptype) {
3179         case BPF_PROG_TYPE_SK_SKB:
3180         case BPF_PROG_TYPE_SK_MSG:
3181                 ret = sock_map_get_from_fd(attr, prog);
3182                 break;
3183         case BPF_PROG_TYPE_LIRC_MODE2:
3184                 ret = lirc_prog_attach(attr, prog);
3185                 break;
3186         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3187                 ret = netns_bpf_prog_attach(attr, prog);
3188                 break;
3189         case BPF_PROG_TYPE_CGROUP_DEVICE:
3190         case BPF_PROG_TYPE_CGROUP_SKB:
3191         case BPF_PROG_TYPE_CGROUP_SOCK:
3192         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3193         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3194         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3195         case BPF_PROG_TYPE_SOCK_OPS:
3196                 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
3197                 break;
3198         default:
3199                 ret = -EINVAL;
3200         }
3201
3202         if (ret)
3203                 bpf_prog_put(prog);
3204         return ret;
3205 }
3206
3207 #define BPF_PROG_DETACH_LAST_FIELD attach_type
3208
3209 static int bpf_prog_detach(const union bpf_attr *attr)
3210 {
3211         enum bpf_prog_type ptype;
3212
3213         if (CHECK_ATTR(BPF_PROG_DETACH))
3214                 return -EINVAL;
3215
3216         ptype = attach_type_to_prog_type(attr->attach_type);
3217
3218         switch (ptype) {
3219         case BPF_PROG_TYPE_SK_MSG:
3220         case BPF_PROG_TYPE_SK_SKB:
3221                 return sock_map_prog_detach(attr, ptype);
3222         case BPF_PROG_TYPE_LIRC_MODE2:
3223                 return lirc_prog_detach(attr);
3224         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3225                 return netns_bpf_prog_detach(attr, ptype);
3226         case BPF_PROG_TYPE_CGROUP_DEVICE:
3227         case BPF_PROG_TYPE_CGROUP_SKB:
3228         case BPF_PROG_TYPE_CGROUP_SOCK:
3229         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3230         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3231         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3232         case BPF_PROG_TYPE_SOCK_OPS:
3233                 return cgroup_bpf_prog_detach(attr, ptype);
3234         default:
3235                 return -EINVAL;
3236         }
3237 }
3238
3239 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
3240
3241 static int bpf_prog_query(const union bpf_attr *attr,
3242                           union bpf_attr __user *uattr)
3243 {
3244         if (!capable(CAP_NET_ADMIN))
3245                 return -EPERM;
3246         if (CHECK_ATTR(BPF_PROG_QUERY))
3247                 return -EINVAL;
3248         if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
3249                 return -EINVAL;
3250
3251         switch (attr->query.attach_type) {
3252         case BPF_CGROUP_INET_INGRESS:
3253         case BPF_CGROUP_INET_EGRESS:
3254         case BPF_CGROUP_INET_SOCK_CREATE:
3255         case BPF_CGROUP_INET_SOCK_RELEASE:
3256         case BPF_CGROUP_INET4_BIND:
3257         case BPF_CGROUP_INET6_BIND:
3258         case BPF_CGROUP_INET4_POST_BIND:
3259         case BPF_CGROUP_INET6_POST_BIND:
3260         case BPF_CGROUP_INET4_CONNECT:
3261         case BPF_CGROUP_INET6_CONNECT:
3262         case BPF_CGROUP_INET4_GETPEERNAME:
3263         case BPF_CGROUP_INET6_GETPEERNAME:
3264         case BPF_CGROUP_INET4_GETSOCKNAME:
3265         case BPF_CGROUP_INET6_GETSOCKNAME:
3266         case BPF_CGROUP_UDP4_SENDMSG:
3267         case BPF_CGROUP_UDP6_SENDMSG:
3268         case BPF_CGROUP_UDP4_RECVMSG:
3269         case BPF_CGROUP_UDP6_RECVMSG:
3270         case BPF_CGROUP_SOCK_OPS:
3271         case BPF_CGROUP_DEVICE:
3272         case BPF_CGROUP_SYSCTL:
3273         case BPF_CGROUP_GETSOCKOPT:
3274         case BPF_CGROUP_SETSOCKOPT:
3275                 return cgroup_bpf_prog_query(attr, uattr);
3276         case BPF_LIRC_MODE2:
3277                 return lirc_prog_query(attr, uattr);
3278         case BPF_FLOW_DISSECTOR:
3279         case BPF_SK_LOOKUP:
3280                 return netns_bpf_prog_query(attr, uattr);
3281         default:
3282                 return -EINVAL;
3283         }
3284 }
3285
3286 #define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu
3287
3288 static int bpf_prog_test_run(const union bpf_attr *attr,
3289                              union bpf_attr __user *uattr)
3290 {
3291         struct bpf_prog *prog;
3292         int ret = -ENOTSUPP;
3293
3294         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
3295                 return -EINVAL;
3296
3297         if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
3298             (!attr->test.ctx_size_in && attr->test.ctx_in))
3299                 return -EINVAL;
3300
3301         if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
3302             (!attr->test.ctx_size_out && attr->test.ctx_out))
3303                 return -EINVAL;
3304
3305         prog = bpf_prog_get(attr->test.prog_fd);
3306         if (IS_ERR(prog))
3307                 return PTR_ERR(prog);
3308
3309         if (prog->aux->ops->test_run)
3310                 ret = prog->aux->ops->test_run(prog, attr, uattr);
3311
3312         bpf_prog_put(prog);
3313         return ret;
3314 }
3315
3316 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
3317
3318 static int bpf_obj_get_next_id(const union bpf_attr *attr,
3319                                union bpf_attr __user *uattr,
3320                                struct idr *idr,
3321                                spinlock_t *lock)
3322 {
3323         u32 next_id = attr->start_id;
3324         int err = 0;
3325
3326         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
3327                 return -EINVAL;
3328
3329         if (!capable(CAP_SYS_ADMIN))
3330                 return -EPERM;
3331
3332         next_id++;
3333         spin_lock_bh(lock);
3334         if (!idr_get_next(idr, &next_id))
3335                 err = -ENOENT;
3336         spin_unlock_bh(lock);
3337
3338         if (!err)
3339                 err = put_user(next_id, &uattr->next_id);
3340
3341         return err;
3342 }
3343
3344 struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
3345 {
3346         struct bpf_map *map;
3347
3348         spin_lock_bh(&map_idr_lock);
3349 again:
3350         map = idr_get_next(&map_idr, id);
3351         if (map) {
3352                 map = __bpf_map_inc_not_zero(map, false);
3353                 if (IS_ERR(map)) {
3354                         (*id)++;
3355                         goto again;
3356                 }
3357         }
3358         spin_unlock_bh(&map_idr_lock);
3359
3360         return map;
3361 }
3362
3363 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
3364 {
3365         struct bpf_prog *prog;
3366
3367         spin_lock_bh(&prog_idr_lock);
3368 again:
3369         prog = idr_get_next(&prog_idr, id);
3370         if (prog) {
3371                 prog = bpf_prog_inc_not_zero(prog);
3372                 if (IS_ERR(prog)) {
3373                         (*id)++;
3374                         goto again;
3375                 }
3376         }
3377         spin_unlock_bh(&prog_idr_lock);
3378
3379         return prog;
3380 }
3381
3382 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
3383
3384 struct bpf_prog *bpf_prog_by_id(u32 id)
3385 {
3386         struct bpf_prog *prog;
3387
3388         if (!id)
3389                 return ERR_PTR(-ENOENT);
3390
3391         spin_lock_bh(&prog_idr_lock);
3392         prog = idr_find(&prog_idr, id);
3393         if (prog)
3394                 prog = bpf_prog_inc_not_zero(prog);
3395         else
3396                 prog = ERR_PTR(-ENOENT);
3397         spin_unlock_bh(&prog_idr_lock);
3398         return prog;
3399 }
3400
3401 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
3402 {
3403         struct bpf_prog *prog;
3404         u32 id = attr->prog_id;
3405         int fd;
3406
3407         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
3408                 return -EINVAL;
3409
3410         if (!capable(CAP_SYS_ADMIN))
3411                 return -EPERM;
3412
3413         prog = bpf_prog_by_id(id);
3414         if (IS_ERR(prog))
3415                 return PTR_ERR(prog);
3416
3417         fd = bpf_prog_new_fd(prog);
3418         if (fd < 0)
3419                 bpf_prog_put(prog);
3420
3421         return fd;
3422 }
3423
3424 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
3425
3426 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
3427 {
3428         struct bpf_map *map;
3429         u32 id = attr->map_id;
3430         int f_flags;
3431         int fd;
3432
3433         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
3434             attr->open_flags & ~BPF_OBJ_FLAG_MASK)
3435                 return -EINVAL;
3436
3437         if (!capable(CAP_SYS_ADMIN))
3438                 return -EPERM;
3439
3440         f_flags = bpf_get_file_flag(attr->open_flags);
3441         if (f_flags < 0)
3442                 return f_flags;
3443
3444         spin_lock_bh(&map_idr_lock);
3445         map = idr_find(&map_idr, id);
3446         if (map)
3447                 map = __bpf_map_inc_not_zero(map, true);
3448         else
3449                 map = ERR_PTR(-ENOENT);
3450         spin_unlock_bh(&map_idr_lock);
3451
3452         if (IS_ERR(map))
3453                 return PTR_ERR(map);
3454
3455         fd = bpf_map_new_fd(map, f_flags);
3456         if (fd < 0)
3457                 bpf_map_put_with_uref(map);
3458
3459         return fd;
3460 }
3461
3462 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
3463                                               unsigned long addr, u32 *off,
3464                                               u32 *type)
3465 {
3466         const struct bpf_map *map;
3467         int i;
3468
3469         mutex_lock(&prog->aux->used_maps_mutex);
3470         for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
3471                 map = prog->aux->used_maps[i];
3472                 if (map == (void *)addr) {
3473                         *type = BPF_PSEUDO_MAP_FD;
3474                         goto out;
3475                 }
3476                 if (!map->ops->map_direct_value_meta)
3477                         continue;
3478                 if (!map->ops->map_direct_value_meta(map, addr, off)) {
3479                         *type = BPF_PSEUDO_MAP_VALUE;
3480                         goto out;
3481                 }
3482         }
3483         map = NULL;
3484
3485 out:
3486         mutex_unlock(&prog->aux->used_maps_mutex);
3487         return map;
3488 }
3489
3490 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
3491                                               const struct cred *f_cred)
3492 {
3493         const struct bpf_map *map;
3494         struct bpf_insn *insns;
3495         u32 off, type;
3496         u64 imm;
3497         u8 code;
3498         int i;
3499
3500         insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
3501                         GFP_USER);
3502         if (!insns)
3503                 return insns;
3504
3505         for (i = 0; i < prog->len; i++) {
3506                 code = insns[i].code;
3507
3508                 if (code == (BPF_JMP | BPF_TAIL_CALL)) {
3509                         insns[i].code = BPF_JMP | BPF_CALL;
3510                         insns[i].imm = BPF_FUNC_tail_call;
3511                         /* fall-through */
3512                 }
3513                 if (code == (BPF_JMP | BPF_CALL) ||
3514                     code == (BPF_JMP | BPF_CALL_ARGS)) {
3515                         if (code == (BPF_JMP | BPF_CALL_ARGS))
3516                                 insns[i].code = BPF_JMP | BPF_CALL;
3517                         if (!bpf_dump_raw_ok(f_cred))
3518                                 insns[i].imm = 0;
3519                         continue;
3520                 }
3521                 if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
3522                         insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
3523                         continue;
3524                 }
3525
3526                 if (code != (BPF_LD | BPF_IMM | BPF_DW))
3527                         continue;
3528
3529                 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
3530                 map = bpf_map_from_imm(prog, imm, &off, &type);
3531                 if (map) {
3532                         insns[i].src_reg = type;
3533                         insns[i].imm = map->id;
3534                         insns[i + 1].imm = off;
3535                         continue;
3536                 }
3537         }
3538
3539         return insns;
3540 }
3541
3542 static int set_info_rec_size(struct bpf_prog_info *info)
3543 {
3544         /*
3545          * Ensure info.*_rec_size is the same as kernel expected size
3546          *
3547          * or
3548          *
3549          * Only allow zero *_rec_size if both _rec_size and _cnt are
3550          * zero.  In this case, the kernel will set the expected
3551          * _rec_size back to the info.
3552          */
3553
3554         if ((info->nr_func_info || info->func_info_rec_size) &&
3555             info->func_info_rec_size != sizeof(struct bpf_func_info))
3556                 return -EINVAL;
3557
3558         if ((info->nr_line_info || info->line_info_rec_size) &&
3559             info->line_info_rec_size != sizeof(struct bpf_line_info))
3560                 return -EINVAL;
3561
3562         if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
3563             info->jited_line_info_rec_size != sizeof(__u64))
3564                 return -EINVAL;
3565
3566         info->func_info_rec_size = sizeof(struct bpf_func_info);
3567         info->line_info_rec_size = sizeof(struct bpf_line_info);
3568         info->jited_line_info_rec_size = sizeof(__u64);
3569
3570         return 0;
3571 }
3572
3573 static int bpf_prog_get_info_by_fd(struct file *file,
3574                                    struct bpf_prog *prog,
3575                                    const union bpf_attr *attr,
3576                                    union bpf_attr __user *uattr)
3577 {
3578         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3579         struct bpf_prog_info info;
3580         u32 info_len = attr->info.info_len;
3581         struct bpf_prog_stats stats;
3582         char __user *uinsns;
3583         u32 ulen;
3584         int err;
3585
3586         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3587         if (err)
3588                 return err;
3589         info_len = min_t(u32, sizeof(info), info_len);
3590
3591         memset(&info, 0, sizeof(info));
3592         if (copy_from_user(&info, uinfo, info_len))
3593                 return -EFAULT;
3594
3595         info.type = prog->type;
3596         info.id = prog->aux->id;
3597         info.load_time = prog->aux->load_time;
3598         info.created_by_uid = from_kuid_munged(current_user_ns(),
3599                                                prog->aux->user->uid);
3600         info.gpl_compatible = prog->gpl_compatible;
3601
3602         memcpy(info.tag, prog->tag, sizeof(prog->tag));
3603         memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
3604
3605         mutex_lock(&prog->aux->used_maps_mutex);
3606         ulen = info.nr_map_ids;
3607         info.nr_map_ids = prog->aux->used_map_cnt;
3608         ulen = min_t(u32, info.nr_map_ids, ulen);
3609         if (ulen) {
3610                 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
3611                 u32 i;
3612
3613                 for (i = 0; i < ulen; i++)
3614                         if (put_user(prog->aux->used_maps[i]->id,
3615                                      &user_map_ids[i])) {
3616                                 mutex_unlock(&prog->aux->used_maps_mutex);
3617                                 return -EFAULT;
3618                         }
3619         }
3620         mutex_unlock(&prog->aux->used_maps_mutex);
3621
3622         err = set_info_rec_size(&info);
3623         if (err)
3624                 return err;
3625
3626         bpf_prog_get_stats(prog, &stats);
3627         info.run_time_ns = stats.nsecs;
3628         info.run_cnt = stats.cnt;
3629         info.recursion_misses = stats.misses;
3630
3631         if (!bpf_capable()) {
3632                 info.jited_prog_len = 0;
3633                 info.xlated_prog_len = 0;
3634                 info.nr_jited_ksyms = 0;
3635                 info.nr_jited_func_lens = 0;
3636                 info.nr_func_info = 0;
3637                 info.nr_line_info = 0;
3638                 info.nr_jited_line_info = 0;
3639                 goto done;
3640         }
3641
3642         ulen = info.xlated_prog_len;
3643         info.xlated_prog_len = bpf_prog_insn_size(prog);
3644         if (info.xlated_prog_len && ulen) {
3645                 struct bpf_insn *insns_sanitized;
3646                 bool fault;
3647
3648                 if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) {
3649                         info.xlated_prog_insns = 0;
3650                         goto done;
3651                 }
3652                 insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
3653                 if (!insns_sanitized)
3654                         return -ENOMEM;
3655                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
3656                 ulen = min_t(u32, info.xlated_prog_len, ulen);
3657                 fault = copy_to_user(uinsns, insns_sanitized, ulen);
3658                 kfree(insns_sanitized);
3659                 if (fault)
3660                         return -EFAULT;
3661         }
3662
3663         if (bpf_prog_is_dev_bound(prog->aux)) {
3664                 err = bpf_prog_offload_info_fill(&info, prog);
3665                 if (err)
3666                         return err;
3667                 goto done;
3668         }
3669
3670         /* NOTE: the following code is supposed to be skipped for offload.
3671          * bpf_prog_offload_info_fill() is the place to fill similar fields
3672          * for offload.
3673          */
3674         ulen = info.jited_prog_len;
3675         if (prog->aux->func_cnt) {
3676                 u32 i;
3677
3678                 info.jited_prog_len = 0;
3679                 for (i = 0; i < prog->aux->func_cnt; i++)
3680                         info.jited_prog_len += prog->aux->func[i]->jited_len;
3681         } else {
3682                 info.jited_prog_len = prog->jited_len;
3683         }
3684
3685         if (info.jited_prog_len && ulen) {
3686                 if (bpf_dump_raw_ok(file->f_cred)) {
3687                         uinsns = u64_to_user_ptr(info.jited_prog_insns);
3688                         ulen = min_t(u32, info.jited_prog_len, ulen);
3689
3690                         /* for multi-function programs, copy the JITed
3691                          * instructions for all the functions
3692                          */
3693                         if (prog->aux->func_cnt) {
3694                                 u32 len, free, i;
3695                                 u8 *img;
3696
3697                                 free = ulen;
3698                                 for (i = 0; i < prog->aux->func_cnt; i++) {
3699                                         len = prog->aux->func[i]->jited_len;
3700                                         len = min_t(u32, len, free);
3701                                         img = (u8 *) prog->aux->func[i]->bpf_func;
3702                                         if (copy_to_user(uinsns, img, len))
3703                                                 return -EFAULT;
3704                                         uinsns += len;
3705                                         free -= len;
3706                                         if (!free)
3707                                                 break;
3708                                 }
3709                         } else {
3710                                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
3711                                         return -EFAULT;
3712                         }
3713                 } else {
3714                         info.jited_prog_insns = 0;
3715                 }
3716         }
3717
3718         ulen = info.nr_jited_ksyms;
3719         info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
3720         if (ulen) {
3721                 if (bpf_dump_raw_ok(file->f_cred)) {
3722                         unsigned long ksym_addr;
3723                         u64 __user *user_ksyms;
3724                         u32 i;
3725
3726                         /* copy the address of the kernel symbol
3727                          * corresponding to each function
3728                          */
3729                         ulen = min_t(u32, info.nr_jited_ksyms, ulen);
3730                         user_ksyms = u64_to_user_ptr(info.jited_ksyms);
3731                         if (prog->aux->func_cnt) {
3732                                 for (i = 0; i < ulen; i++) {
3733                                         ksym_addr = (unsigned long)
3734                                                 prog->aux->func[i]->bpf_func;
3735                                         if (put_user((u64) ksym_addr,
3736                                                      &user_ksyms[i]))
3737                                                 return -EFAULT;
3738                                 }
3739                         } else {
3740                                 ksym_addr = (unsigned long) prog->bpf_func;
3741                                 if (put_user((u64) ksym_addr, &user_ksyms[0]))
3742                                         return -EFAULT;
3743                         }
3744                 } else {
3745                         info.jited_ksyms = 0;
3746                 }
3747         }
3748
3749         ulen = info.nr_jited_func_lens;
3750         info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
3751         if (ulen) {
3752                 if (bpf_dump_raw_ok(file->f_cred)) {
3753                         u32 __user *user_lens;
3754                         u32 func_len, i;
3755
3756                         /* copy the JITed image lengths for each function */
3757                         ulen = min_t(u32, info.nr_jited_func_lens, ulen);
3758                         user_lens = u64_to_user_ptr(info.jited_func_lens);
3759                         if (prog->aux->func_cnt) {
3760                                 for (i = 0; i < ulen; i++) {
3761                                         func_len =
3762                                                 prog->aux->func[i]->jited_len;
3763                                         if (put_user(func_len, &user_lens[i]))
3764                                                 return -EFAULT;
3765                                 }
3766                         } else {
3767                                 func_len = prog->jited_len;
3768                                 if (put_user(func_len, &user_lens[0]))
3769                                         return -EFAULT;
3770                         }
3771                 } else {
3772                         info.jited_func_lens = 0;
3773                 }
3774         }
3775
3776         if (prog->aux->btf)
3777                 info.btf_id = btf_obj_id(prog->aux->btf);
3778
3779         ulen = info.nr_func_info;
3780         info.nr_func_info = prog->aux->func_info_cnt;
3781         if (info.nr_func_info && ulen) {
3782                 char __user *user_finfo;
3783
3784                 user_finfo = u64_to_user_ptr(info.func_info);
3785                 ulen = min_t(u32, info.nr_func_info, ulen);
3786                 if (copy_to_user(user_finfo, prog->aux->func_info,
3787                                  info.func_info_rec_size * ulen))
3788                         return -EFAULT;
3789         }
3790
3791         ulen = info.nr_line_info;
3792         info.nr_line_info = prog->aux->nr_linfo;
3793         if (info.nr_line_info && ulen) {
3794                 __u8 __user *user_linfo;
3795
3796                 user_linfo = u64_to_user_ptr(info.line_info);
3797                 ulen = min_t(u32, info.nr_line_info, ulen);
3798                 if (copy_to_user(user_linfo, prog->aux->linfo,
3799                                  info.line_info_rec_size * ulen))
3800                         return -EFAULT;
3801         }
3802
3803         ulen = info.nr_jited_line_info;
3804         if (prog->aux->jited_linfo)
3805                 info.nr_jited_line_info = prog->aux->nr_linfo;
3806         else
3807                 info.nr_jited_line_info = 0;
3808         if (info.nr_jited_line_info && ulen) {
3809                 if (bpf_dump_raw_ok(file->f_cred)) {
3810                         __u64 __user *user_linfo;
3811                         u32 i;
3812
3813                         user_linfo = u64_to_user_ptr(info.jited_line_info);
3814                         ulen = min_t(u32, info.nr_jited_line_info, ulen);
3815                         for (i = 0; i < ulen; i++) {
3816                                 if (put_user((__u64)(long)prog->aux->jited_linfo[i],
3817                                              &user_linfo[i]))
3818                                         return -EFAULT;
3819                         }
3820                 } else {
3821                         info.jited_line_info = 0;
3822                 }
3823         }
3824
3825         ulen = info.nr_prog_tags;
3826         info.nr_prog_tags = prog->aux->func_cnt ? : 1;
3827         if (ulen) {
3828                 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
3829                 u32 i;
3830
3831                 user_prog_tags = u64_to_user_ptr(info.prog_tags);
3832                 ulen = min_t(u32, info.nr_prog_tags, ulen);
3833                 if (prog->aux->func_cnt) {
3834                         for (i = 0; i < ulen; i++) {
3835                                 if (copy_to_user(user_prog_tags[i],
3836                                                  prog->aux->func[i]->tag,
3837                                                  BPF_TAG_SIZE))
3838                                         return -EFAULT;
3839                         }
3840                 } else {
3841                         if (copy_to_user(user_prog_tags[0],
3842                                          prog->tag, BPF_TAG_SIZE))
3843                                 return -EFAULT;
3844                 }
3845         }
3846
3847 done:
3848         if (copy_to_user(uinfo, &info, info_len) ||
3849             put_user(info_len, &uattr->info.info_len))
3850                 return -EFAULT;
3851
3852         return 0;
3853 }
3854
3855 static int bpf_map_get_info_by_fd(struct file *file,
3856                                   struct bpf_map *map,
3857                                   const union bpf_attr *attr,
3858                                   union bpf_attr __user *uattr)
3859 {
3860         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3861         struct bpf_map_info info;
3862         u32 info_len = attr->info.info_len;
3863         int err;
3864
3865         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3866         if (err)
3867                 return err;
3868         info_len = min_t(u32, sizeof(info), info_len);
3869
3870         memset(&info, 0, sizeof(info));
3871         info.type = map->map_type;
3872         info.id = map->id;
3873         info.key_size = map->key_size;
3874         info.value_size = map->value_size;
3875         info.max_entries = map->max_entries;
3876         info.map_flags = map->map_flags;
3877         memcpy(info.name, map->name, sizeof(map->name));
3878
3879         if (map->btf) {
3880                 info.btf_id = btf_obj_id(map->btf);
3881                 info.btf_key_type_id = map->btf_key_type_id;
3882                 info.btf_value_type_id = map->btf_value_type_id;
3883         }
3884         info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
3885
3886         if (bpf_map_is_dev_bound(map)) {
3887                 err = bpf_map_offload_info_fill(&info, map);
3888                 if (err)
3889                         return err;
3890         }
3891
3892         if (copy_to_user(uinfo, &info, info_len) ||
3893             put_user(info_len, &uattr->info.info_len))
3894                 return -EFAULT;
3895
3896         return 0;
3897 }
3898
3899 static int bpf_btf_get_info_by_fd(struct file *file,
3900                                   struct btf *btf,
3901                                   const union bpf_attr *attr,
3902                                   union bpf_attr __user *uattr)
3903 {
3904         struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3905         u32 info_len = attr->info.info_len;
3906         int err;
3907
3908         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
3909         if (err)
3910                 return err;
3911
3912         return btf_get_info_by_fd(btf, attr, uattr);
3913 }
3914
3915 static int bpf_link_get_info_by_fd(struct file *file,
3916                                   struct bpf_link *link,
3917                                   const union bpf_attr *attr,
3918                                   union bpf_attr __user *uattr)
3919 {
3920         struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3921         struct bpf_link_info info;
3922         u32 info_len = attr->info.info_len;
3923         int err;
3924
3925         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3926         if (err)
3927                 return err;
3928         info_len = min_t(u32, sizeof(info), info_len);
3929
3930         memset(&info, 0, sizeof(info));
3931         if (copy_from_user(&info, uinfo, info_len))
3932                 return -EFAULT;
3933
3934         info.type = link->type;
3935         info.id = link->id;
3936         info.prog_id = link->prog->aux->id;
3937
3938         if (link->ops->fill_link_info) {
3939                 err = link->ops->fill_link_info(link, &info);
3940                 if (err)
3941                         return err;
3942         }
3943
3944         if (copy_to_user(uinfo, &info, info_len) ||
3945             put_user(info_len, &uattr->info.info_len))
3946                 return -EFAULT;
3947
3948         return 0;
3949 }
3950
3951
3952 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
3953
3954 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
3955                                   union bpf_attr __user *uattr)
3956 {
3957         int ufd = attr->info.bpf_fd;
3958         struct fd f;
3959         int err;
3960
3961         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
3962                 return -EINVAL;
3963
3964         f = fdget(ufd);
3965         if (!f.file)
3966                 return -EBADFD;
3967
3968         if (f.file->f_op == &bpf_prog_fops)
3969                 err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr,
3970                                               uattr);
3971         else if (f.file->f_op == &bpf_map_fops)
3972                 err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr,
3973                                              uattr);
3974         else if (f.file->f_op == &btf_fops)
3975                 err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr);
3976         else if (f.file->f_op == &bpf_link_fops)
3977                 err = bpf_link_get_info_by_fd(f.file, f.file->private_data,
3978                                               attr, uattr);
3979         else
3980                 err = -EINVAL;
3981
3982         fdput(f);
3983         return err;
3984 }
3985
3986 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
3987
3988 static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr)
3989 {
3990         if (CHECK_ATTR(BPF_BTF_LOAD))
3991                 return -EINVAL;
3992
3993         if (!bpf_capable())
3994                 return -EPERM;
3995
3996         return btf_new_fd(attr, uattr);
3997 }
3998
3999 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
4000
4001 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
4002 {
4003         if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
4004                 return -EINVAL;
4005
4006         if (!capable(CAP_SYS_ADMIN))
4007                 return -EPERM;
4008
4009         return btf_get_fd_by_id(attr->btf_id);
4010 }
4011
4012 static int bpf_task_fd_query_copy(const union bpf_attr *attr,
4013                                     union bpf_attr __user *uattr,
4014                                     u32 prog_id, u32 fd_type,
4015                                     const char *buf, u64 probe_offset,
4016                                     u64 probe_addr)
4017 {
4018         char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
4019         u32 len = buf ? strlen(buf) : 0, input_len;
4020         int err = 0;
4021
4022         if (put_user(len, &uattr->task_fd_query.buf_len))
4023                 return -EFAULT;
4024         input_len = attr->task_fd_query.buf_len;
4025         if (input_len && ubuf) {
4026                 if (!len) {
4027                         /* nothing to copy, just make ubuf NULL terminated */
4028                         char zero = '\0';
4029
4030                         if (put_user(zero, ubuf))
4031                                 return -EFAULT;
4032                 } else if (input_len >= len + 1) {
4033                         /* ubuf can hold the string with NULL terminator */
4034                         if (copy_to_user(ubuf, buf, len + 1))
4035                                 return -EFAULT;
4036                 } else {
4037                         /* ubuf cannot hold the string with NULL terminator,
4038                          * do a partial copy with NULL terminator.
4039                          */
4040                         char zero = '\0';
4041
4042                         err = -ENOSPC;
4043                         if (copy_to_user(ubuf, buf, input_len - 1))
4044                                 return -EFAULT;
4045                         if (put_user(zero, ubuf + input_len - 1))
4046                                 return -EFAULT;
4047                 }
4048         }
4049
4050         if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
4051             put_user(fd_type, &uattr->task_fd_query.fd_type) ||
4052             put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
4053             put_user(probe_addr, &uattr->task_fd_query.probe_addr))
4054                 return -EFAULT;
4055
4056         return err;
4057 }
4058
4059 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
4060
4061 static int bpf_task_fd_query(const union bpf_attr *attr,
4062                              union bpf_attr __user *uattr)
4063 {
4064         pid_t pid = attr->task_fd_query.pid;
4065         u32 fd = attr->task_fd_query.fd;
4066         const struct perf_event *event;
4067         struct task_struct *task;
4068         struct file *file;
4069         int err;
4070
4071         if (CHECK_ATTR(BPF_TASK_FD_QUERY))
4072                 return -EINVAL;
4073
4074         if (!capable(CAP_SYS_ADMIN))
4075                 return -EPERM;
4076
4077         if (attr->task_fd_query.flags != 0)
4078                 return -EINVAL;
4079
4080         task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
4081         if (!task)
4082                 return -ENOENT;
4083
4084         err = 0;
4085         file = fget_task(task, fd);
4086         put_task_struct(task);
4087         if (!file)
4088                 return -EBADF;
4089
4090         if (file->f_op == &bpf_link_fops) {
4091                 struct bpf_link *link = file->private_data;
4092
4093                 if (link->ops == &bpf_raw_tp_link_lops) {
4094                         struct bpf_raw_tp_link *raw_tp =
4095                                 container_of(link, struct bpf_raw_tp_link, link);
4096                         struct bpf_raw_event_map *btp = raw_tp->btp;
4097
4098                         err = bpf_task_fd_query_copy(attr, uattr,
4099                                                      raw_tp->link.prog->aux->id,
4100                                                      BPF_FD_TYPE_RAW_TRACEPOINT,
4101                                                      btp->tp->name, 0, 0);
4102                         goto put_file;
4103                 }
4104                 goto out_not_supp;
4105         }
4106
4107         event = perf_get_event(file);
4108         if (!IS_ERR(event)) {
4109                 u64 probe_offset, probe_addr;
4110                 u32 prog_id, fd_type;
4111                 const char *buf;
4112
4113                 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
4114                                               &buf, &probe_offset,
4115                                               &probe_addr);
4116                 if (!err)
4117                         err = bpf_task_fd_query_copy(attr, uattr, prog_id,
4118                                                      fd_type, buf,
4119                                                      probe_offset,
4120                                                      probe_addr);
4121                 goto put_file;
4122         }
4123
4124 out_not_supp:
4125         err = -ENOTSUPP;
4126 put_file:
4127         fput(file);
4128         return err;
4129 }
4130
4131 #define BPF_MAP_BATCH_LAST_FIELD batch.flags
4132
4133 #define BPF_DO_BATCH(fn)                        \
4134         do {                                    \
4135                 if (!fn) {                      \
4136                         err = -ENOTSUPP;        \
4137                         goto err_put;           \
4138                 }                               \
4139                 err = fn(map, attr, uattr);     \
4140         } while (0)
4141
4142 static int bpf_map_do_batch(const union bpf_attr *attr,
4143                             union bpf_attr __user *uattr,
4144                             int cmd)
4145 {
4146         struct bpf_map *map;
4147         int err, ufd;
4148         struct fd f;
4149
4150         if (CHECK_ATTR(BPF_MAP_BATCH))
4151                 return -EINVAL;
4152
4153         ufd = attr->batch.map_fd;
4154         f = fdget(ufd);
4155         map = __bpf_map_get(f);
4156         if (IS_ERR(map))
4157                 return PTR_ERR(map);
4158
4159         if ((cmd == BPF_MAP_LOOKUP_BATCH ||
4160              cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) &&
4161             !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
4162                 err = -EPERM;
4163                 goto err_put;
4164         }
4165
4166         if (cmd != BPF_MAP_LOOKUP_BATCH &&
4167             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
4168                 err = -EPERM;
4169                 goto err_put;
4170         }
4171
4172         if (cmd == BPF_MAP_LOOKUP_BATCH)
4173                 BPF_DO_BATCH(map->ops->map_lookup_batch);
4174         else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
4175                 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch);
4176         else if (cmd == BPF_MAP_UPDATE_BATCH)
4177                 BPF_DO_BATCH(map->ops->map_update_batch);
4178         else
4179                 BPF_DO_BATCH(map->ops->map_delete_batch);
4180
4181 err_put:
4182         fdput(f);
4183         return err;
4184 }
4185
4186 static int tracing_bpf_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
4187                                    struct bpf_prog *prog)
4188 {
4189         if (attr->link_create.attach_type != prog->expected_attach_type)
4190                 return -EINVAL;
4191
4192         if (prog->expected_attach_type == BPF_TRACE_ITER)
4193                 return bpf_iter_link_attach(attr, uattr, prog);
4194         else if (prog->type == BPF_PROG_TYPE_EXT)
4195                 return bpf_tracing_prog_attach(prog,
4196                                                attr->link_create.target_fd,
4197                                                attr->link_create.target_btf_id);
4198         return -EINVAL;
4199 }
4200
4201 #define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len
4202 static int link_create(union bpf_attr *attr, bpfptr_t uattr)
4203 {
4204         enum bpf_prog_type ptype;
4205         struct bpf_prog *prog;
4206         int ret;
4207
4208         if (CHECK_ATTR(BPF_LINK_CREATE))
4209                 return -EINVAL;
4210
4211         prog = bpf_prog_get(attr->link_create.prog_fd);
4212         if (IS_ERR(prog))
4213                 return PTR_ERR(prog);
4214
4215         ret = bpf_prog_attach_check_attach_type(prog,
4216                                                 attr->link_create.attach_type);
4217         if (ret)
4218                 goto out;
4219
4220         switch (prog->type) {
4221         case BPF_PROG_TYPE_EXT:
4222                 ret = tracing_bpf_link_attach(attr, uattr, prog);
4223                 goto out;
4224         case BPF_PROG_TYPE_PERF_EVENT:
4225         case BPF_PROG_TYPE_KPROBE:
4226         case BPF_PROG_TYPE_TRACEPOINT:
4227                 if (attr->link_create.attach_type != BPF_PERF_EVENT) {
4228                         ret = -EINVAL;
4229                         goto out;
4230                 }
4231                 ptype = prog->type;
4232                 break;
4233         default:
4234                 ptype = attach_type_to_prog_type(attr->link_create.attach_type);
4235                 if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) {
4236                         ret = -EINVAL;
4237                         goto out;
4238                 }
4239                 break;
4240         }
4241
4242         switch (ptype) {
4243         case BPF_PROG_TYPE_CGROUP_SKB:
4244         case BPF_PROG_TYPE_CGROUP_SOCK:
4245         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4246         case BPF_PROG_TYPE_SOCK_OPS:
4247         case BPF_PROG_TYPE_CGROUP_DEVICE:
4248         case BPF_PROG_TYPE_CGROUP_SYSCTL:
4249         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4250                 ret = cgroup_bpf_link_attach(attr, prog);
4251                 break;
4252         case BPF_PROG_TYPE_TRACING:
4253                 ret = tracing_bpf_link_attach(attr, uattr, prog);
4254                 break;
4255         case BPF_PROG_TYPE_FLOW_DISSECTOR:
4256         case BPF_PROG_TYPE_SK_LOOKUP:
4257                 ret = netns_bpf_link_create(attr, prog);
4258                 break;
4259 #ifdef CONFIG_NET
4260         case BPF_PROG_TYPE_XDP:
4261                 ret = bpf_xdp_link_attach(attr, prog);
4262                 break;
4263 #endif
4264 #ifdef CONFIG_PERF_EVENTS
4265         case BPF_PROG_TYPE_PERF_EVENT:
4266         case BPF_PROG_TYPE_TRACEPOINT:
4267         case BPF_PROG_TYPE_KPROBE:
4268                 ret = bpf_perf_link_attach(attr, prog);
4269                 break;
4270 #endif
4271         default:
4272                 ret = -EINVAL;
4273         }
4274
4275 out:
4276         if (ret < 0)
4277                 bpf_prog_put(prog);
4278         return ret;
4279 }
4280
4281 #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
4282
4283 static int link_update(union bpf_attr *attr)
4284 {
4285         struct bpf_prog *old_prog = NULL, *new_prog;
4286         struct bpf_link *link;
4287         u32 flags;
4288         int ret;
4289
4290         if (CHECK_ATTR(BPF_LINK_UPDATE))
4291                 return -EINVAL;
4292
4293         flags = attr->link_update.flags;
4294         if (flags & ~BPF_F_REPLACE)
4295                 return -EINVAL;
4296
4297         link = bpf_link_get_from_fd(attr->link_update.link_fd);
4298         if (IS_ERR(link))
4299                 return PTR_ERR(link);
4300
4301         new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
4302         if (IS_ERR(new_prog)) {
4303                 ret = PTR_ERR(new_prog);
4304                 goto out_put_link;
4305         }
4306
4307         if (flags & BPF_F_REPLACE) {
4308                 old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
4309                 if (IS_ERR(old_prog)) {
4310                         ret = PTR_ERR(old_prog);
4311                         old_prog = NULL;
4312                         goto out_put_progs;
4313                 }
4314         } else if (attr->link_update.old_prog_fd) {
4315                 ret = -EINVAL;
4316                 goto out_put_progs;
4317         }
4318
4319         if (link->ops->update_prog)
4320                 ret = link->ops->update_prog(link, new_prog, old_prog);
4321         else
4322                 ret = -EINVAL;
4323
4324 out_put_progs:
4325         if (old_prog)
4326                 bpf_prog_put(old_prog);
4327         if (ret)
4328                 bpf_prog_put(new_prog);
4329 out_put_link:
4330         bpf_link_put(link);
4331         return ret;
4332 }
4333
4334 #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
4335
4336 static int link_detach(union bpf_attr *attr)
4337 {
4338         struct bpf_link *link;
4339         int ret;
4340
4341         if (CHECK_ATTR(BPF_LINK_DETACH))
4342                 return -EINVAL;
4343
4344         link = bpf_link_get_from_fd(attr->link_detach.link_fd);
4345         if (IS_ERR(link))
4346                 return PTR_ERR(link);
4347
4348         if (link->ops->detach)
4349                 ret = link->ops->detach(link);
4350         else
4351                 ret = -EOPNOTSUPP;
4352
4353         bpf_link_put(link);
4354         return ret;
4355 }
4356
4357 static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
4358 {
4359         return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
4360 }
4361
4362 struct bpf_link *bpf_link_by_id(u32 id)
4363 {
4364         struct bpf_link *link;
4365
4366         if (!id)
4367                 return ERR_PTR(-ENOENT);
4368
4369         spin_lock_bh(&link_idr_lock);
4370         /* before link is "settled", ID is 0, pretend it doesn't exist yet */
4371         link = idr_find(&link_idr, id);
4372         if (link) {
4373                 if (link->id)
4374                         link = bpf_link_inc_not_zero(link);
4375                 else
4376                         link = ERR_PTR(-EAGAIN);
4377         } else {
4378                 link = ERR_PTR(-ENOENT);
4379         }
4380         spin_unlock_bh(&link_idr_lock);
4381         return link;
4382 }
4383
4384 #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
4385
4386 static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
4387 {
4388         struct bpf_link *link;
4389         u32 id = attr->link_id;
4390         int fd;
4391
4392         if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
4393                 return -EINVAL;
4394
4395         if (!capable(CAP_SYS_ADMIN))
4396                 return -EPERM;
4397
4398         link = bpf_link_by_id(id);
4399         if (IS_ERR(link))
4400                 return PTR_ERR(link);
4401
4402         fd = bpf_link_new_fd(link);
4403         if (fd < 0)
4404                 bpf_link_put(link);
4405
4406         return fd;
4407 }
4408
4409 DEFINE_MUTEX(bpf_stats_enabled_mutex);
4410
4411 static int bpf_stats_release(struct inode *inode, struct file *file)
4412 {
4413         mutex_lock(&bpf_stats_enabled_mutex);
4414         static_key_slow_dec(&bpf_stats_enabled_key.key);
4415         mutex_unlock(&bpf_stats_enabled_mutex);
4416         return 0;
4417 }
4418
4419 static const struct file_operations bpf_stats_fops = {
4420         .release = bpf_stats_release,
4421 };
4422
4423 static int bpf_enable_runtime_stats(void)
4424 {
4425         int fd;
4426
4427         mutex_lock(&bpf_stats_enabled_mutex);
4428
4429         /* Set a very high limit to avoid overflow */
4430         if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
4431                 mutex_unlock(&bpf_stats_enabled_mutex);
4432                 return -EBUSY;
4433         }
4434
4435         fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
4436         if (fd >= 0)
4437                 static_key_slow_inc(&bpf_stats_enabled_key.key);
4438
4439         mutex_unlock(&bpf_stats_enabled_mutex);
4440         return fd;
4441 }
4442
4443 #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
4444
4445 static int bpf_enable_stats(union bpf_attr *attr)
4446 {
4447
4448         if (CHECK_ATTR(BPF_ENABLE_STATS))
4449                 return -EINVAL;
4450
4451         if (!capable(CAP_SYS_ADMIN))
4452                 return -EPERM;
4453
4454         switch (attr->enable_stats.type) {
4455         case BPF_STATS_RUN_TIME:
4456                 return bpf_enable_runtime_stats();
4457         default:
4458                 break;
4459         }
4460         return -EINVAL;
4461 }
4462
4463 #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
4464
4465 static int bpf_iter_create(union bpf_attr *attr)
4466 {
4467         struct bpf_link *link;
4468         int err;
4469
4470         if (CHECK_ATTR(BPF_ITER_CREATE))
4471                 return -EINVAL;
4472
4473         if (attr->iter_create.flags)
4474                 return -EINVAL;
4475
4476         link = bpf_link_get_from_fd(attr->iter_create.link_fd);
4477         if (IS_ERR(link))
4478                 return PTR_ERR(link);
4479
4480         err = bpf_iter_new_fd(link);
4481         bpf_link_put(link);
4482
4483         return err;
4484 }
4485
4486 #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
4487
4488 static int bpf_prog_bind_map(union bpf_attr *attr)
4489 {
4490         struct bpf_prog *prog;
4491         struct bpf_map *map;
4492         struct bpf_map **used_maps_old, **used_maps_new;
4493         int i, ret = 0;
4494
4495         if (CHECK_ATTR(BPF_PROG_BIND_MAP))
4496                 return -EINVAL;
4497
4498         if (attr->prog_bind_map.flags)
4499                 return -EINVAL;
4500
4501         prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
4502         if (IS_ERR(prog))
4503                 return PTR_ERR(prog);
4504
4505         map = bpf_map_get(attr->prog_bind_map.map_fd);
4506         if (IS_ERR(map)) {
4507                 ret = PTR_ERR(map);
4508                 goto out_prog_put;
4509         }
4510
4511         mutex_lock(&prog->aux->used_maps_mutex);
4512
4513         used_maps_old = prog->aux->used_maps;
4514
4515         for (i = 0; i < prog->aux->used_map_cnt; i++)
4516                 if (used_maps_old[i] == map) {
4517                         bpf_map_put(map);
4518                         goto out_unlock;
4519                 }
4520
4521         used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
4522                                       sizeof(used_maps_new[0]),
4523                                       GFP_KERNEL);
4524         if (!used_maps_new) {
4525                 ret = -ENOMEM;
4526                 goto out_unlock;
4527         }
4528
4529         memcpy(used_maps_new, used_maps_old,
4530                sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
4531         used_maps_new[prog->aux->used_map_cnt] = map;
4532
4533         prog->aux->used_map_cnt++;
4534         prog->aux->used_maps = used_maps_new;
4535
4536         kfree(used_maps_old);
4537
4538 out_unlock:
4539         mutex_unlock(&prog->aux->used_maps_mutex);
4540
4541         if (ret)
4542                 bpf_map_put(map);
4543 out_prog_put:
4544         bpf_prog_put(prog);
4545         return ret;
4546 }
4547
4548 static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size)
4549 {
4550         union bpf_attr attr;
4551         int err;
4552
4553         if (sysctl_unprivileged_bpf_disabled && !bpf_capable())
4554                 return -EPERM;
4555
4556         err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
4557         if (err)
4558                 return err;
4559         size = min_t(u32, size, sizeof(attr));
4560
4561         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
4562         memset(&attr, 0, sizeof(attr));
4563         if (copy_from_bpfptr(&attr, uattr, size) != 0)
4564                 return -EFAULT;
4565
4566         err = security_bpf(cmd, &attr, size);
4567         if (err < 0)
4568                 return err;
4569
4570         switch (cmd) {
4571         case BPF_MAP_CREATE:
4572                 err = map_create(&attr);
4573                 break;
4574         case BPF_MAP_LOOKUP_ELEM:
4575                 err = map_lookup_elem(&attr);
4576                 break;
4577         case BPF_MAP_UPDATE_ELEM:
4578                 err = map_update_elem(&attr, uattr);
4579                 break;
4580         case BPF_MAP_DELETE_ELEM:
4581                 err = map_delete_elem(&attr);
4582                 break;
4583         case BPF_MAP_GET_NEXT_KEY:
4584                 err = map_get_next_key(&attr);
4585                 break;
4586         case BPF_MAP_FREEZE:
4587                 err = map_freeze(&attr);
4588                 break;
4589         case BPF_PROG_LOAD:
4590                 err = bpf_prog_load(&attr, uattr);
4591                 break;
4592         case BPF_OBJ_PIN:
4593                 err = bpf_obj_pin(&attr);
4594                 break;
4595         case BPF_OBJ_GET:
4596                 err = bpf_obj_get(&attr);
4597                 break;
4598         case BPF_PROG_ATTACH:
4599                 err = bpf_prog_attach(&attr);
4600                 break;
4601         case BPF_PROG_DETACH:
4602                 err = bpf_prog_detach(&attr);
4603                 break;
4604         case BPF_PROG_QUERY:
4605                 err = bpf_prog_query(&attr, uattr.user);
4606                 break;
4607         case BPF_PROG_TEST_RUN:
4608                 err = bpf_prog_test_run(&attr, uattr.user);
4609                 break;
4610         case BPF_PROG_GET_NEXT_ID:
4611                 err = bpf_obj_get_next_id(&attr, uattr.user,
4612                                           &prog_idr, &prog_idr_lock);
4613                 break;
4614         case BPF_MAP_GET_NEXT_ID:
4615                 err = bpf_obj_get_next_id(&attr, uattr.user,
4616                                           &map_idr, &map_idr_lock);
4617                 break;
4618         case BPF_BTF_GET_NEXT_ID:
4619                 err = bpf_obj_get_next_id(&attr, uattr.user,
4620                                           &btf_idr, &btf_idr_lock);
4621                 break;
4622         case BPF_PROG_GET_FD_BY_ID:
4623                 err = bpf_prog_get_fd_by_id(&attr);
4624                 break;
4625         case BPF_MAP_GET_FD_BY_ID:
4626                 err = bpf_map_get_fd_by_id(&attr);
4627                 break;
4628         case BPF_OBJ_GET_INFO_BY_FD:
4629                 err = bpf_obj_get_info_by_fd(&attr, uattr.user);
4630                 break;
4631         case BPF_RAW_TRACEPOINT_OPEN:
4632                 err = bpf_raw_tracepoint_open(&attr);
4633                 break;
4634         case BPF_BTF_LOAD:
4635                 err = bpf_btf_load(&attr, uattr);
4636                 break;
4637         case BPF_BTF_GET_FD_BY_ID:
4638                 err = bpf_btf_get_fd_by_id(&attr);
4639                 break;
4640         case BPF_TASK_FD_QUERY:
4641                 err = bpf_task_fd_query(&attr, uattr.user);
4642                 break;
4643         case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
4644                 err = map_lookup_and_delete_elem(&attr);
4645                 break;
4646         case BPF_MAP_LOOKUP_BATCH:
4647                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH);
4648                 break;
4649         case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
4650                 err = bpf_map_do_batch(&attr, uattr.user,
4651                                        BPF_MAP_LOOKUP_AND_DELETE_BATCH);
4652                 break;
4653         case BPF_MAP_UPDATE_BATCH:
4654                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH);
4655                 break;
4656         case BPF_MAP_DELETE_BATCH:
4657                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH);
4658                 break;
4659         case BPF_LINK_CREATE:
4660                 err = link_create(&attr, uattr);
4661                 break;
4662         case BPF_LINK_UPDATE:
4663                 err = link_update(&attr);
4664                 break;
4665         case BPF_LINK_GET_FD_BY_ID:
4666                 err = bpf_link_get_fd_by_id(&attr);
4667                 break;
4668         case BPF_LINK_GET_NEXT_ID:
4669                 err = bpf_obj_get_next_id(&attr, uattr.user,
4670                                           &link_idr, &link_idr_lock);
4671                 break;
4672         case BPF_ENABLE_STATS:
4673                 err = bpf_enable_stats(&attr);
4674                 break;
4675         case BPF_ITER_CREATE:
4676                 err = bpf_iter_create(&attr);
4677                 break;
4678         case BPF_LINK_DETACH:
4679                 err = link_detach(&attr);
4680                 break;
4681         case BPF_PROG_BIND_MAP:
4682                 err = bpf_prog_bind_map(&attr);
4683                 break;
4684         default:
4685                 err = -EINVAL;
4686                 break;
4687         }
4688
4689         return err;
4690 }
4691
4692 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
4693 {
4694         return __sys_bpf(cmd, USER_BPFPTR(uattr), size);
4695 }
4696
4697 static bool syscall_prog_is_valid_access(int off, int size,
4698                                          enum bpf_access_type type,
4699                                          const struct bpf_prog *prog,
4700                                          struct bpf_insn_access_aux *info)
4701 {
4702         if (off < 0 || off >= U16_MAX)
4703                 return false;
4704         if (off % size != 0)
4705                 return false;
4706         return true;
4707 }
4708
4709 BPF_CALL_3(bpf_sys_bpf, int, cmd, void *, attr, u32, attr_size)
4710 {
4711         switch (cmd) {
4712         case BPF_MAP_CREATE:
4713         case BPF_MAP_UPDATE_ELEM:
4714         case BPF_MAP_FREEZE:
4715         case BPF_PROG_LOAD:
4716         case BPF_BTF_LOAD:
4717                 break;
4718         /* case BPF_PROG_TEST_RUN:
4719          * is not part of this list to prevent recursive test_run
4720          */
4721         default:
4722                 return -EINVAL;
4723         }
4724         return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size);
4725 }
4726
4727 static const struct bpf_func_proto bpf_sys_bpf_proto = {
4728         .func           = bpf_sys_bpf,
4729         .gpl_only       = false,
4730         .ret_type       = RET_INTEGER,
4731         .arg1_type      = ARG_ANYTHING,
4732         .arg2_type      = ARG_PTR_TO_MEM,
4733         .arg3_type      = ARG_CONST_SIZE,
4734 };
4735
4736 const struct bpf_func_proto * __weak
4737 tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
4738 {
4739         return bpf_base_func_proto(func_id);
4740 }
4741
4742 BPF_CALL_1(bpf_sys_close, u32, fd)
4743 {
4744         /* When bpf program calls this helper there should not be
4745          * an fdget() without matching completed fdput().
4746          * This helper is allowed in the following callchain only:
4747          * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close
4748          */
4749         return close_fd(fd);
4750 }
4751
4752 static const struct bpf_func_proto bpf_sys_close_proto = {
4753         .func           = bpf_sys_close,
4754         .gpl_only       = false,
4755         .ret_type       = RET_INTEGER,
4756         .arg1_type      = ARG_ANYTHING,
4757 };
4758
4759 static const struct bpf_func_proto *
4760 syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
4761 {
4762         switch (func_id) {
4763         case BPF_FUNC_sys_bpf:
4764                 return &bpf_sys_bpf_proto;
4765         case BPF_FUNC_btf_find_by_name_kind:
4766                 return &bpf_btf_find_by_name_kind_proto;
4767         case BPF_FUNC_sys_close:
4768                 return &bpf_sys_close_proto;
4769         default:
4770                 return tracing_prog_func_proto(func_id, prog);
4771         }
4772 }
4773
4774 const struct bpf_verifier_ops bpf_syscall_verifier_ops = {
4775         .get_func_proto  = syscall_prog_func_proto,
4776         .is_valid_access = syscall_prog_is_valid_access,
4777 };
4778
4779 const struct bpf_prog_ops bpf_syscall_prog_ops = {
4780         .test_run = bpf_prog_test_run_syscall,
4781 };