tools/testing/selftests/bpf/progs/map_kptr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <vmlinux.h>
   3 #include <bpf/bpf_tracing.h>
   4 #include <bpf/bpf_helpers.h>
   5 #include "../bpf_testmod/bpf_testmod_kfunc.h"
   6
   7 struct map_value {
   8         struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr;
   9         struct prog_test_ref_kfunc __kptr *ref_ptr;
  10 };
  11
  12 struct array_map {
  13         __uint(type, BPF_MAP_TYPE_ARRAY);
  14         __type(key, int);
  15         __type(value, struct map_value);
  16         __uint(max_entries, 1);
  17 } array_map SEC(".maps");
  18
  19 struct pcpu_array_map {
  20         __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
  21         __type(key, int);
  22         __type(value, struct map_value);
  23         __uint(max_entries, 1);
  24 } pcpu_array_map SEC(".maps");
  25
  26 struct hash_map {
  27         __uint(type, BPF_MAP_TYPE_HASH);
  28         __type(key, int);
  29         __type(value, struct map_value);
  30         __uint(max_entries, 1);
  31 } hash_map SEC(".maps");
  32
  33 struct pcpu_hash_map {
  34         __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
  35         __type(key, int);
  36         __type(value, struct map_value);
  37         __uint(max_entries, 1);
  38 } pcpu_hash_map SEC(".maps");
  39
  40 struct hash_malloc_map {
  41         __uint(type, BPF_MAP_TYPE_HASH);
  42         __type(key, int);
  43         __type(value, struct map_value);
  44         __uint(max_entries, 1);
  45         __uint(map_flags, BPF_F_NO_PREALLOC);
  46 } hash_malloc_map SEC(".maps");
  47
  48 struct pcpu_hash_malloc_map {
  49         __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
  50         __type(key, int);
  51         __type(value, struct map_value);
  52         __uint(max_entries, 1);
  53         __uint(map_flags, BPF_F_NO_PREALLOC);
  54 } pcpu_hash_malloc_map SEC(".maps");
  55
  56 struct lru_hash_map {
  57         __uint(type, BPF_MAP_TYPE_LRU_HASH);
  58         __type(key, int);
  59         __type(value, struct map_value);
  60         __uint(max_entries, 1);
  61 } lru_hash_map SEC(".maps");
  62
  63 struct lru_pcpu_hash_map {
  64         __uint(type, BPF_MAP_TYPE_LRU_PERCPU_HASH);
  65         __type(key, int);
  66         __type(value, struct map_value);
  67         __uint(max_entries, 1);
  68 } lru_pcpu_hash_map SEC(".maps");
  69
  70 struct cgrp_ls_map {
  71         __uint(type, BPF_MAP_TYPE_CGRP_STORAGE);
  72         __uint(map_flags, BPF_F_NO_PREALLOC);
  73         __type(key, int);
  74         __type(value, struct map_value);
  75 } cgrp_ls_map SEC(".maps");
  76
  77 struct task_ls_map {
  78         __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
  79         __uint(map_flags, BPF_F_NO_PREALLOC);
  80         __type(key, int);
  81         __type(value, struct map_value);
  82 } task_ls_map SEC(".maps");
  83
  84 struct inode_ls_map {
  85         __uint(type, BPF_MAP_TYPE_INODE_STORAGE);
  86         __uint(map_flags, BPF_F_NO_PREALLOC);
  87         __type(key, int);
  88         __type(value, struct map_value);
  89 } inode_ls_map SEC(".maps");
  90
  91 struct sk_ls_map {
  92         __uint(type, BPF_MAP_TYPE_SK_STORAGE);
  93         __uint(map_flags, BPF_F_NO_PREALLOC);
  94         __type(key, int);
  95         __type(value, struct map_value);
  96 } sk_ls_map SEC(".maps");
  97
  98 #define DEFINE_MAP_OF_MAP(map_type, inner_map_type, name)       \
  99         struct {                                                \
 100                 __uint(type, map_type);                         \
 101                 __uint(max_entries, 1);                         \
 102                 __uint(key_size, sizeof(int));                  \
 103                 __uint(value_size, sizeof(int));                \
 104                 __array(values, struct inner_map_type);         \
 105         } name SEC(".maps") = {                                 \
 106                 .values = { [0] = &inner_map_type },            \
 107         }
 108
 109 DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, array_map, array_of_array_maps);
 110 DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, hash_map, array_of_hash_maps);
 111 DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, hash_malloc_map, array_of_hash_malloc_maps);
 112 DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, lru_hash_map, array_of_lru_hash_maps);
 113 DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, array_map, hash_of_array_maps);
 114 DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, hash_map, hash_of_hash_maps);
 115 DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, hash_malloc_map, hash_of_hash_malloc_maps);
 116 DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, lru_hash_map, hash_of_lru_hash_maps);
 117
 118 #define WRITE_ONCE(x, val) ((*(volatile typeof(x) *) &(x)) = (val))
 119
 120 static void test_kptr_unref(struct map_value *v)
 121 {
 122         struct prog_test_ref_kfunc *p;
 123
 124         p = v->unref_ptr;
 125         /* store untrusted_ptr_or_null_ */
 126         WRITE_ONCE(v->unref_ptr, p);
 127         if (!p)
 128                 return;
 129         if (p->a + p->b > 100)
 130                 return;
 131         /* store untrusted_ptr_ */
 132         WRITE_ONCE(v->unref_ptr, p);
 133         /* store NULL */
 134         WRITE_ONCE(v->unref_ptr, NULL);
 135 }
 136
 137 static void test_kptr_ref(struct map_value *v)
 138 {
 139         struct prog_test_ref_kfunc *p;
 140
 141         p = v->ref_ptr;
 142         /* store ptr_or_null_ */
 143         WRITE_ONCE(v->unref_ptr, p);
 144         if (!p)
 145                 return;
 146         /*
 147          * p is rcu_ptr_prog_test_ref_kfunc,
 148          * because bpf prog is non-sleepable and runs in RCU CS.
 149          * p can be passed to kfunc that requires KF_RCU.
 150          */
 151         bpf_kfunc_call_test_ref(p);
 152         if (p->a + p->b > 100)
 153                 return;
 154         /* store NULL */
 155         p = bpf_kptr_xchg(&v->ref_ptr, NULL);
 156         if (!p)
 157                 return;
 158         /*
 159          * p is trusted_ptr_prog_test_ref_kfunc.
 160          * p can be passed to kfunc that requires KF_RCU.
 161          */
 162         bpf_kfunc_call_test_ref(p);
 163         if (p->a + p->b > 100) {
 164                 bpf_kfunc_call_test_release(p);
 165                 return;
 166         }
 167         /* store ptr_ */
 168         WRITE_ONCE(v->unref_ptr, p);
 169         bpf_kfunc_call_test_release(p);
 170
 171         p = bpf_kfunc_call_test_acquire(&(unsigned long){0});
 172         if (!p)
 173                 return;
 174         /* store ptr_ */
 175         p = bpf_kptr_xchg(&v->ref_ptr, p);
 176         if (!p)
 177                 return;
 178         if (p->a + p->b > 100) {
 179                 bpf_kfunc_call_test_release(p);
 180                 return;
 181         }
 182         bpf_kfunc_call_test_release(p);
 183 }
 184
 185 static void test_kptr(struct map_value *v)
 186 {
 187         test_kptr_unref(v);
 188         test_kptr_ref(v);
 189 }
 190
 191 SEC("tc")
 192 int test_map_kptr(struct __sk_buff *ctx)
 193 {
 194         struct map_value *v;
 195         int key = 0;
 196
 197 #define TEST(map)                                       \
 198         v = bpf_map_lookup_elem(&map, &key);            \
 199         if (!v)                                         \
 200                 return 0;                               \
 201         test_kptr(v)
 202
 203         TEST(array_map);
 204         TEST(hash_map);
 205         TEST(hash_malloc_map);
 206         TEST(lru_hash_map);
 207
 208 #undef TEST
 209         return 0;
 210 }
 211
 212 SEC("tp_btf/cgroup_mkdir")
 213 int BPF_PROG(test_cgrp_map_kptr, struct cgroup *cgrp, const char *path)
 214 {
 215         struct map_value *v;
 216
 217         v = bpf_cgrp_storage_get(&cgrp_ls_map, cgrp, NULL, BPF_LOCAL_STORAGE_GET_F_CREATE);
 218         if (v)
 219                 test_kptr(v);
 220         return 0;
 221 }
 222
 223 SEC("lsm/inode_unlink")
 224 int BPF_PROG(test_task_map_kptr, struct inode *inode, struct dentry *victim)
 225 {
 226         struct task_struct *task;
 227         struct map_value *v;
 228
 229         task = bpf_get_current_task_btf();
 230         if (!task)
 231                 return 0;
 232         v = bpf_task_storage_get(&task_ls_map, task, NULL, BPF_LOCAL_STORAGE_GET_F_CREATE);
 233         if (v)
 234                 test_kptr(v);
 235         return 0;
 236 }
 237
 238 SEC("lsm/inode_unlink")
 239 int BPF_PROG(test_inode_map_kptr, struct inode *inode, struct dentry *victim)
 240 {
 241         struct map_value *v;
 242
 243         v = bpf_inode_storage_get(&inode_ls_map, inode, NULL, BPF_LOCAL_STORAGE_GET_F_CREATE);
 244         if (v)
 245                 test_kptr(v);
 246         return 0;
 247 }
 248
 249 SEC("tc")
 250 int test_sk_map_kptr(struct __sk_buff *ctx)
 251 {
 252         struct map_value *v;
 253         struct bpf_sock *sk;
 254
 255         sk = ctx->sk;
 256         if (!sk)
 257                 return 0;
 258         v = bpf_sk_storage_get(&sk_ls_map, sk, NULL, BPF_LOCAL_STORAGE_GET_F_CREATE);
 259         if (v)
 260                 test_kptr(v);
 261         return 0;
 262 }
 263
 264 SEC("tc")
 265 int test_map_in_map_kptr(struct __sk_buff *ctx)
 266 {
 267         struct map_value *v;
 268         int key = 0;
 269         void *map;
 270
 271 #define TEST(map_in_map)                                \
 272         map = bpf_map_lookup_elem(&map_in_map, &key);   \
 273         if (!map)                                       \
 274                 return 0;                               \
 275         v = bpf_map_lookup_elem(map, &key);             \
 276         if (!v)                                         \
 277                 return 0;                               \
 278         test_kptr(v)
 279
 280         TEST(array_of_array_maps);
 281         TEST(array_of_hash_maps);
 282         TEST(array_of_hash_malloc_maps);
 283         TEST(array_of_lru_hash_maps);
 284         TEST(hash_of_array_maps);
 285         TEST(hash_of_hash_maps);
 286         TEST(hash_of_hash_malloc_maps);
 287         TEST(hash_of_lru_hash_maps);
 288
 289 #undef TEST
 290         return 0;
 291 }
 292
 293 int ref = 1;
 294
 295 static __always_inline
 296 int test_map_kptr_ref_pre(struct map_value *v)
 297 {
 298         struct prog_test_ref_kfunc *p, *p_st;
 299         unsigned long arg = 0;
 300         int ret;
 301
 302         p = bpf_kfunc_call_test_acquire(&arg);
 303         if (!p)
 304                 return 1;
 305         ref++;
 306
 307         p_st = p->next;
 308         if (p_st->cnt.refs.counter != ref) {
 309                 ret = 2;
 310                 goto end;
 311         }
 312
 313         p = bpf_kptr_xchg(&v->ref_ptr, p);
 314         if (p) {
 315                 ret = 3;
 316                 goto end;
 317         }
 318         if (p_st->cnt.refs.counter != ref)
 319                 return 4;
 320
 321         p = bpf_kptr_xchg(&v->ref_ptr, NULL);
 322         if (!p)
 323                 return 5;
 324         bpf_kfunc_call_test_release(p);
 325         ref--;
 326         if (p_st->cnt.refs.counter != ref)
 327                 return 6;
 328
 329         p = bpf_kfunc_call_test_acquire(&arg);
 330         if (!p)
 331                 return 7;
 332         ref++;
 333         p = bpf_kptr_xchg(&v->ref_ptr, p);
 334         if (p) {
 335                 ret = 8;
 336                 goto end;
 337         }
 338         if (p_st->cnt.refs.counter != ref)
 339                 return 9;
 340         /* Leave in map */
 341
 342         return 0;
 343 end:
 344         ref--;
 345         bpf_kfunc_call_test_release(p);
 346         return ret;
 347 }
 348
 349 static __always_inline
 350 int test_map_kptr_ref_post(struct map_value *v)
 351 {
 352         struct prog_test_ref_kfunc *p, *p_st;
 353
 354         p_st = v->ref_ptr;
 355         if (!p_st || p_st->cnt.refs.counter != ref)
 356                 return 1;
 357
 358         p = bpf_kptr_xchg(&v->ref_ptr, NULL);
 359         if (!p)
 360                 return 2;
 361         if (p_st->cnt.refs.counter != ref) {
 362                 bpf_kfunc_call_test_release(p);
 363                 return 3;
 364         }
 365
 366         p = bpf_kptr_xchg(&v->ref_ptr, p);
 367         if (p) {
 368                 bpf_kfunc_call_test_release(p);
 369                 return 4;
 370         }
 371         if (p_st->cnt.refs.counter != ref)
 372                 return 5;
 373
 374         return 0;
 375 }
 376
 377 #define TEST(map)                            \
 378         v = bpf_map_lookup_elem(&map, &key); \
 379         if (!v)                              \
 380                 return -1;                   \
 381         ret = test_map_kptr_ref_pre(v);      \
 382         if (ret)                             \
 383                 return ret;
 384
 385 #define TEST_PCPU(map)                                 \
 386         v = bpf_map_lookup_percpu_elem(&map, &key, 0); \
 387         if (!v)                                        \
 388                 return -1;                             \
 389         ret = test_map_kptr_ref_pre(v);                \
 390         if (ret)                                       \
 391                 return ret;
 392
 393 SEC("tc")
 394 int test_map_kptr_ref1(struct __sk_buff *ctx)
 395 {
 396         struct map_value *v, val = {};
 397         int key = 0, ret;
 398
 399         bpf_map_update_elem(&hash_map, &key, &val, 0);
 400         bpf_map_update_elem(&hash_malloc_map, &key, &val, 0);
 401         bpf_map_update_elem(&lru_hash_map, &key, &val, 0);
 402
 403         bpf_map_update_elem(&pcpu_hash_map, &key, &val, 0);
 404         bpf_map_update_elem(&pcpu_hash_malloc_map, &key, &val, 0);
 405         bpf_map_update_elem(&lru_pcpu_hash_map, &key, &val, 0);
 406
 407         TEST(array_map);
 408         TEST(hash_map);
 409         TEST(hash_malloc_map);
 410         TEST(lru_hash_map);
 411
 412         TEST_PCPU(pcpu_array_map);
 413         TEST_PCPU(pcpu_hash_map);
 414         TEST_PCPU(pcpu_hash_malloc_map);
 415         TEST_PCPU(lru_pcpu_hash_map);
 416
 417         return 0;
 418 }
 419
 420 #undef TEST
 421 #undef TEST_PCPU
 422
 423 #define TEST(map)                            \
 424         v = bpf_map_lookup_elem(&map, &key); \
 425         if (!v)                              \
 426                 return -1;                   \
 427         ret = test_map_kptr_ref_post(v);     \
 428         if (ret)                             \
 429                 return ret;
 430
 431 #define TEST_PCPU(map)                                 \
 432         v = bpf_map_lookup_percpu_elem(&map, &key, 0); \
 433         if (!v)                                        \
 434                 return -1;                             \
 435         ret = test_map_kptr_ref_post(v);               \
 436         if (ret)                                       \
 437                 return ret;
 438
 439 SEC("tc")
 440 int test_map_kptr_ref2(struct __sk_buff *ctx)
 441 {
 442         struct map_value *v;
 443         int key = 0, ret;
 444
 445         TEST(array_map);
 446         TEST(hash_map);
 447         TEST(hash_malloc_map);
 448         TEST(lru_hash_map);
 449
 450         TEST_PCPU(pcpu_array_map);
 451         TEST_PCPU(pcpu_hash_map);
 452         TEST_PCPU(pcpu_hash_malloc_map);
 453         TEST_PCPU(lru_pcpu_hash_map);
 454
 455         return 0;
 456 }
 457
 458 #undef TEST
 459 #undef TEST_PCPU
 460
 461 SEC("tc")
 462 int test_map_kptr_ref3(struct __sk_buff *ctx)
 463 {
 464         struct prog_test_ref_kfunc *p;
 465         unsigned long sp = 0;
 466
 467         p = bpf_kfunc_call_test_acquire(&sp);
 468         if (!p)
 469                 return 1;
 470         ref++;
 471         if (p->cnt.refs.counter != ref) {
 472                 bpf_kfunc_call_test_release(p);
 473                 return 2;
 474         }
 475         bpf_kfunc_call_test_release(p);
 476         ref--;
 477         return 0;
 478 }
 479
 480 SEC("syscall")
 481 int test_ls_map_kptr_ref1(void *ctx)
 482 {
 483         struct task_struct *current;
 484         struct map_value *v;
 485
 486         current = bpf_get_current_task_btf();
 487         if (!current)
 488                 return 100;
 489         v = bpf_task_storage_get(&task_ls_map, current, NULL, 0);
 490         if (v)
 491                 return 150;
 492         v = bpf_task_storage_get(&task_ls_map, current, NULL, BPF_LOCAL_STORAGE_GET_F_CREATE);
 493         if (!v)
 494                 return 200;
 495         return test_map_kptr_ref_pre(v);
 496 }
 497
 498 SEC("syscall")
 499 int test_ls_map_kptr_ref2(void *ctx)
 500 {
 501         struct task_struct *current;
 502         struct map_value *v;
 503
 504         current = bpf_get_current_task_btf();
 505         if (!current)
 506                 return 100;
 507         v = bpf_task_storage_get(&task_ls_map, current, NULL, 0);
 508         if (!v)
 509                 return 200;
 510         return test_map_kptr_ref_post(v);
 511 }
 512
 513 SEC("syscall")
 514 int test_ls_map_kptr_ref_del(void *ctx)
 515 {
 516         struct task_struct *current;
 517         struct map_value *v;
 518
 519         current = bpf_get_current_task_btf();
 520         if (!current)
 521                 return 100;
 522         v = bpf_task_storage_get(&task_ls_map, current, NULL, 0);
 523         if (!v)
 524                 return 200;
 525         if (!v->ref_ptr)
 526                 return 300;
 527         return bpf_task_storage_delete(&task_ls_map, current);
 528 }
 529
 530 char _license[] SEC("license") = "GPL";