tools/testing/selftests/bpf/prog_tests/tailcalls.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <test_progs.h>
   3 #include <network_helpers.h>
   4
   5 /* test_tailcall_1 checks basic functionality by patching multiple locations
   6  * in a single program for a single tail call slot with nop->jmp, jmp->nop
   7  * and jmp->jmp rewrites. Also checks for nop->nop.
   8  */
   9 static void test_tailcall_1(void)
  10 {
  11         int err, map_fd, prog_fd, main_fd, i, j;
  12         struct bpf_map *prog_array;
  13         struct bpf_program *prog;
  14         struct bpf_object *obj;
  15         __u32 retval, duration;
  16         char prog_name[32];
  17         char buff[128] = {};
  18
  19         err = bpf_prog_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
  20                             &prog_fd);
  21         if (CHECK_FAIL(err))
  22                 return;
  23
  24         prog = bpf_object__find_program_by_title(obj, "classifier");
  25         if (CHECK_FAIL(!prog))
  26                 goto out;
  27
  28         main_fd = bpf_program__fd(prog);
  29         if (CHECK_FAIL(main_fd < 0))
  30                 goto out;
  31
  32         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
  33         if (CHECK_FAIL(!prog_array))
  34                 goto out;
  35
  36         map_fd = bpf_map__fd(prog_array);
  37         if (CHECK_FAIL(map_fd < 0))
  38                 goto out;
  39
  40         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  41                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
  42
  43                 prog = bpf_object__find_program_by_title(obj, prog_name);
  44                 if (CHECK_FAIL(!prog))
  45                         goto out;
  46
  47                 prog_fd = bpf_program__fd(prog);
  48                 if (CHECK_FAIL(prog_fd < 0))
  49                         goto out;
  50
  51                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
  52                 if (CHECK_FAIL(err))
  53                         goto out;
  54         }
  55
  56         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  57                 err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  58                                         &duration, &retval, NULL);
  59                 CHECK(err || retval != i, "tailcall",
  60                       "err %d errno %d retval %d\n", err, errno, retval);
  61
  62                 err = bpf_map_delete_elem(map_fd, &i);
  63                 if (CHECK_FAIL(err))
  64                         goto out;
  65         }
  66
  67         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  68                                 &duration, &retval, NULL);
  69         CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
  70               err, errno, retval);
  71
  72         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  73                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
  74
  75                 prog = bpf_object__find_program_by_title(obj, prog_name);
  76                 if (CHECK_FAIL(!prog))
  77                         goto out;
  78
  79                 prog_fd = bpf_program__fd(prog);
  80                 if (CHECK_FAIL(prog_fd < 0))
  81                         goto out;
  82
  83                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
  84                 if (CHECK_FAIL(err))
  85                         goto out;
  86         }
  87
  88         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  89                                 &duration, &retval, NULL);
  90         CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
  91               err, errno, retval);
  92
  93         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  94                 j = bpf_map__def(prog_array)->max_entries - 1 - i;
  95                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", j);
  96
  97                 prog = bpf_object__find_program_by_title(obj, prog_name);
  98                 if (CHECK_FAIL(!prog))
  99                         goto out;
 100
 101                 prog_fd = bpf_program__fd(prog);
 102                 if (CHECK_FAIL(prog_fd < 0))
 103                         goto out;
 104
 105                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 106                 if (CHECK_FAIL(err))
 107                         goto out;
 108         }
 109
 110         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 111                 j = bpf_map__def(prog_array)->max_entries - 1 - i;
 112
 113                 err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 114                                         &duration, &retval, NULL);
 115                 CHECK(err || retval != j, "tailcall",
 116                       "err %d errno %d retval %d\n", err, errno, retval);
 117
 118                 err = bpf_map_delete_elem(map_fd, &i);
 119                 if (CHECK_FAIL(err))
 120                         goto out;
 121         }
 122
 123         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 124                                 &duration, &retval, NULL);
 125         CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
 126               err, errno, retval);
 127
 128         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 129                 err = bpf_map_delete_elem(map_fd, &i);
 130                 if (CHECK_FAIL(err >= 0 || errno != ENOENT))
 131                         goto out;
 132
 133                 err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 134                                         &duration, &retval, NULL);
 135                 CHECK(err || retval != 3, "tailcall",
 136                       "err %d errno %d retval %d\n", err, errno, retval);
 137         }
 138
 139 out:
 140         bpf_object__close(obj);
 141 }
 142
 143 /* test_tailcall_2 checks that patching multiple programs for a single
 144  * tail call slot works. It also jumps through several programs and tests
 145  * the tail call limit counter.
 146  */
 147 static void test_tailcall_2(void)
 148 {
 149         int err, map_fd, prog_fd, main_fd, i;
 150         struct bpf_map *prog_array;
 151         struct bpf_program *prog;
 152         struct bpf_object *obj;
 153         __u32 retval, duration;
 154         char prog_name[32];
 155         char buff[128] = {};
 156
 157         err = bpf_prog_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 158                             &prog_fd);
 159         if (CHECK_FAIL(err))
 160                 return;
 161
 162         prog = bpf_object__find_program_by_title(obj, "classifier");
 163         if (CHECK_FAIL(!prog))
 164                 goto out;
 165
 166         main_fd = bpf_program__fd(prog);
 167         if (CHECK_FAIL(main_fd < 0))
 168                 goto out;
 169
 170         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 171         if (CHECK_FAIL(!prog_array))
 172                 goto out;
 173
 174         map_fd = bpf_map__fd(prog_array);
 175         if (CHECK_FAIL(map_fd < 0))
 176                 goto out;
 177
 178         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 179                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 180
 181                 prog = bpf_object__find_program_by_title(obj, prog_name);
 182                 if (CHECK_FAIL(!prog))
 183                         goto out;
 184
 185                 prog_fd = bpf_program__fd(prog);
 186                 if (CHECK_FAIL(prog_fd < 0))
 187                         goto out;
 188
 189                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 190                 if (CHECK_FAIL(err))
 191                         goto out;
 192         }
 193
 194         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 195                                 &duration, &retval, NULL);
 196         CHECK(err || retval != 2, "tailcall", "err %d errno %d retval %d\n",
 197               err, errno, retval);
 198
 199         i = 2;
 200         err = bpf_map_delete_elem(map_fd, &i);
 201         if (CHECK_FAIL(err))
 202                 goto out;
 203
 204         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 205                                 &duration, &retval, NULL);
 206         CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 207               err, errno, retval);
 208
 209         i = 0;
 210         err = bpf_map_delete_elem(map_fd, &i);
 211         if (CHECK_FAIL(err))
 212                 goto out;
 213
 214         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 215                                 &duration, &retval, NULL);
 216         CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
 217               err, errno, retval);
 218 out:
 219         bpf_object__close(obj);
 220 }
 221
 222 /* test_tailcall_3 checks that the count value of the tail call limit
 223  * enforcement matches with expectations.
 224  */
 225 static void test_tailcall_3(void)
 226 {
 227         int err, map_fd, prog_fd, main_fd, data_fd, i, val;
 228         struct bpf_map *prog_array, *data_map;
 229         struct bpf_program *prog;
 230         struct bpf_object *obj;
 231         __u32 retval, duration;
 232         char buff[128] = {};
 233
 234         err = bpf_prog_load("tailcall3.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 235                             &prog_fd);
 236         if (CHECK_FAIL(err))
 237                 return;
 238
 239         prog = bpf_object__find_program_by_title(obj, "classifier");
 240         if (CHECK_FAIL(!prog))
 241                 goto out;
 242
 243         main_fd = bpf_program__fd(prog);
 244         if (CHECK_FAIL(main_fd < 0))
 245                 goto out;
 246
 247         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 248         if (CHECK_FAIL(!prog_array))
 249                 goto out;
 250
 251         map_fd = bpf_map__fd(prog_array);
 252         if (CHECK_FAIL(map_fd < 0))
 253                 goto out;
 254
 255         prog = bpf_object__find_program_by_title(obj, "classifier/0");
 256         if (CHECK_FAIL(!prog))
 257                 goto out;
 258
 259         prog_fd = bpf_program__fd(prog);
 260         if (CHECK_FAIL(prog_fd < 0))
 261                 goto out;
 262
 263         i = 0;
 264         err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 265         if (CHECK_FAIL(err))
 266                 goto out;
 267
 268         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 269                                 &duration, &retval, NULL);
 270         CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 271               err, errno, retval);
 272
 273         data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 274         if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 275                 return;
 276
 277         data_fd = bpf_map__fd(data_map);
 278         if (CHECK_FAIL(map_fd < 0))
 279                 return;
 280
 281         i = 0;
 282         err = bpf_map_lookup_elem(data_fd, &i, &val);
 283         CHECK(err || val != 33, "tailcall count", "err %d errno %d count %d\n",
 284               err, errno, val);
 285
 286         i = 0;
 287         err = bpf_map_delete_elem(map_fd, &i);
 288         if (CHECK_FAIL(err))
 289                 goto out;
 290
 291         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 292                                 &duration, &retval, NULL);
 293         CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 294               err, errno, retval);
 295 out:
 296         bpf_object__close(obj);
 297 }
 298
 299 /* test_tailcall_4 checks that the kernel properly selects indirect jump
 300  * for the case where the key is not known. Latter is passed via global
 301  * data to select different targets we can compare return value of.
 302  */
 303 static void test_tailcall_4(void)
 304 {
 305         int err, map_fd, prog_fd, main_fd, data_fd, i;
 306         struct bpf_map *prog_array, *data_map;
 307         struct bpf_program *prog;
 308         struct bpf_object *obj;
 309         __u32 retval, duration;
 310         static const int zero = 0;
 311         char buff[128] = {};
 312         char prog_name[32];
 313
 314         err = bpf_prog_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 315                             &prog_fd);
 316         if (CHECK_FAIL(err))
 317                 return;
 318
 319         prog = bpf_object__find_program_by_title(obj, "classifier");
 320         if (CHECK_FAIL(!prog))
 321                 goto out;
 322
 323         main_fd = bpf_program__fd(prog);
 324         if (CHECK_FAIL(main_fd < 0))
 325                 goto out;
 326
 327         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 328         if (CHECK_FAIL(!prog_array))
 329                 goto out;
 330
 331         map_fd = bpf_map__fd(prog_array);
 332         if (CHECK_FAIL(map_fd < 0))
 333                 goto out;
 334
 335         data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 336         if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 337                 return;
 338
 339         data_fd = bpf_map__fd(data_map);
 340         if (CHECK_FAIL(map_fd < 0))
 341                 return;
 342
 343         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 344                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 345
 346                 prog = bpf_object__find_program_by_title(obj, prog_name);
 347                 if (CHECK_FAIL(!prog))
 348                         goto out;
 349
 350                 prog_fd = bpf_program__fd(prog);
 351                 if (CHECK_FAIL(prog_fd < 0))
 352                         goto out;
 353
 354                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 355                 if (CHECK_FAIL(err))
 356                         goto out;
 357         }
 358
 359         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 360                 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
 361                 if (CHECK_FAIL(err))
 362                         goto out;
 363
 364                 err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 365                                         &duration, &retval, NULL);
 366                 CHECK(err || retval != i, "tailcall",
 367                       "err %d errno %d retval %d\n", err, errno, retval);
 368         }
 369
 370         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 371                 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
 372                 if (CHECK_FAIL(err))
 373                         goto out;
 374
 375                 err = bpf_map_delete_elem(map_fd, &i);
 376                 if (CHECK_FAIL(err))
 377                         goto out;
 378
 379                 err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 380                                         &duration, &retval, NULL);
 381                 CHECK(err || retval != 3, "tailcall",
 382                       "err %d errno %d retval %d\n", err, errno, retval);
 383         }
 384 out:
 385         bpf_object__close(obj);
 386 }
 387
 388 /* test_tailcall_5 probes similarly to test_tailcall_4 that the kernel generates
 389  * an indirect jump when the keys are const but different from different branches.
 390  */
 391 static void test_tailcall_5(void)
 392 {
 393         int err, map_fd, prog_fd, main_fd, data_fd, i, key[] = { 1111, 1234, 5678 };
 394         struct bpf_map *prog_array, *data_map;
 395         struct bpf_program *prog;
 396         struct bpf_object *obj;
 397         __u32 retval, duration;
 398         static const int zero = 0;
 399         char buff[128] = {};
 400         char prog_name[32];
 401
 402         err = bpf_prog_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 403                             &prog_fd);
 404         if (CHECK_FAIL(err))
 405                 return;
 406
 407         prog = bpf_object__find_program_by_title(obj, "classifier");
 408         if (CHECK_FAIL(!prog))
 409                 goto out;
 410
 411         main_fd = bpf_program__fd(prog);
 412         if (CHECK_FAIL(main_fd < 0))
 413                 goto out;
 414
 415         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 416         if (CHECK_FAIL(!prog_array))
 417                 goto out;
 418
 419         map_fd = bpf_map__fd(prog_array);
 420         if (CHECK_FAIL(map_fd < 0))
 421                 goto out;
 422
 423         data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 424         if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 425                 return;
 426
 427         data_fd = bpf_map__fd(data_map);
 428         if (CHECK_FAIL(map_fd < 0))
 429                 return;
 430
 431         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 432                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 433
 434                 prog = bpf_object__find_program_by_title(obj, prog_name);
 435                 if (CHECK_FAIL(!prog))
 436                         goto out;
 437
 438                 prog_fd = bpf_program__fd(prog);
 439                 if (CHECK_FAIL(prog_fd < 0))
 440                         goto out;
 441
 442                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 443                 if (CHECK_FAIL(err))
 444                         goto out;
 445         }
 446
 447         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 448                 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
 449                 if (CHECK_FAIL(err))
 450                         goto out;
 451
 452                 err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 453                                         &duration, &retval, NULL);
 454                 CHECK(err || retval != i, "tailcall",
 455                       "err %d errno %d retval %d\n", err, errno, retval);
 456         }
 457
 458         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 459                 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
 460                 if (CHECK_FAIL(err))
 461                         goto out;
 462
 463                 err = bpf_map_delete_elem(map_fd, &i);
 464                 if (CHECK_FAIL(err))
 465                         goto out;
 466
 467                 err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 468                                         &duration, &retval, NULL);
 469                 CHECK(err || retval != 3, "tailcall",
 470                       "err %d errno %d retval %d\n", err, errno, retval);
 471         }
 472 out:
 473         bpf_object__close(obj);
 474 }
 475
 476 /* test_tailcall_bpf2bpf_1 purpose is to make sure that tailcalls are working
 477  * correctly in correlation with BPF subprograms
 478  */
 479 static void test_tailcall_bpf2bpf_1(void)
 480 {
 481         int err, map_fd, prog_fd, main_fd, i;
 482         struct bpf_map *prog_array;
 483         struct bpf_program *prog;
 484         struct bpf_object *obj;
 485         __u32 retval, duration;
 486         char prog_name[32];
 487
 488         err = bpf_prog_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS,
 489                             &obj, &prog_fd);
 490         if (CHECK_FAIL(err))
 491                 return;
 492
 493         prog = bpf_object__find_program_by_title(obj, "classifier");
 494         if (CHECK_FAIL(!prog))
 495                 goto out;
 496
 497         main_fd = bpf_program__fd(prog);
 498         if (CHECK_FAIL(main_fd < 0))
 499                 goto out;
 500
 501         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 502         if (CHECK_FAIL(!prog_array))
 503                 goto out;
 504
 505         map_fd = bpf_map__fd(prog_array);
 506         if (CHECK_FAIL(map_fd < 0))
 507                 goto out;
 508
 509         /* nop -> jmp */
 510         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 511                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 512
 513                 prog = bpf_object__find_program_by_title(obj, prog_name);
 514                 if (CHECK_FAIL(!prog))
 515                         goto out;
 516
 517                 prog_fd = bpf_program__fd(prog);
 518                 if (CHECK_FAIL(prog_fd < 0))
 519                         goto out;
 520
 521                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 522                 if (CHECK_FAIL(err))
 523                         goto out;
 524         }
 525
 526         err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 527                                 0, &retval, &duration);
 528         CHECK(err || retval != 1, "tailcall",
 529               "err %d errno %d retval %d\n", err, errno, retval);
 530
 531         /* jmp -> nop, call subprog that will do tailcall */
 532         i = 1;
 533         err = bpf_map_delete_elem(map_fd, &i);
 534         if (CHECK_FAIL(err))
 535                 goto out;
 536
 537         err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 538                                 0, &retval, &duration);
 539         CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 540               err, errno, retval);
 541
 542         /* make sure that subprog can access ctx and entry prog that
 543          * called this subprog can properly return
 544          */
 545         i = 0;
 546         err = bpf_map_delete_elem(map_fd, &i);
 547         if (CHECK_FAIL(err))
 548                 goto out;
 549
 550         err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 551                                 0, &retval, &duration);
 552         CHECK(err || retval != sizeof(pkt_v4) * 2,
 553               "tailcall", "err %d errno %d retval %d\n",
 554               err, errno, retval);
 555 out:
 556         bpf_object__close(obj);
 557 }
 558
 559 /* test_tailcall_bpf2bpf_2 checks that the count value of the tail call limit
 560  * enforcement matches with expectations when tailcall is preceded with
 561  * bpf2bpf call.
 562  */
 563 static void test_tailcall_bpf2bpf_2(void)
 564 {
 565         int err, map_fd, prog_fd, main_fd, data_fd, i, val;
 566         struct bpf_map *prog_array, *data_map;
 567         struct bpf_program *prog;
 568         struct bpf_object *obj;
 569         __u32 retval, duration;
 570         char buff[128] = {};
 571
 572         err = bpf_prog_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS,
 573                             &obj, &prog_fd);
 574         if (CHECK_FAIL(err))
 575                 return;
 576
 577         prog = bpf_object__find_program_by_title(obj, "classifier");
 578         if (CHECK_FAIL(!prog))
 579                 goto out;
 580
 581         main_fd = bpf_program__fd(prog);
 582         if (CHECK_FAIL(main_fd < 0))
 583                 goto out;
 584
 585         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 586         if (CHECK_FAIL(!prog_array))
 587                 goto out;
 588
 589         map_fd = bpf_map__fd(prog_array);
 590         if (CHECK_FAIL(map_fd < 0))
 591                 goto out;
 592
 593         prog = bpf_object__find_program_by_title(obj, "classifier/0");
 594         if (CHECK_FAIL(!prog))
 595                 goto out;
 596
 597         prog_fd = bpf_program__fd(prog);
 598         if (CHECK_FAIL(prog_fd < 0))
 599                 goto out;
 600
 601         i = 0;
 602         err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 603         if (CHECK_FAIL(err))
 604                 goto out;
 605
 606         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 607                                 &duration, &retval, NULL);
 608         CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 609               err, errno, retval);
 610
 611         data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 612         if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 613                 return;
 614
 615         data_fd = bpf_map__fd(data_map);
 616         if (CHECK_FAIL(map_fd < 0))
 617                 return;
 618
 619         i = 0;
 620         err = bpf_map_lookup_elem(data_fd, &i, &val);
 621         CHECK(err || val != 33, "tailcall count", "err %d errno %d count %d\n",
 622               err, errno, val);
 623
 624         i = 0;
 625         err = bpf_map_delete_elem(map_fd, &i);
 626         if (CHECK_FAIL(err))
 627                 goto out;
 628
 629         err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 630                                 &duration, &retval, NULL);
 631         CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 632               err, errno, retval);
 633 out:
 634         bpf_object__close(obj);
 635 }
 636
 637 /* test_tailcall_bpf2bpf_3 checks that non-trivial amount of stack (up to
 638  * 256 bytes) can be used within bpf subprograms that have the tailcalls
 639  * in them
 640  */
 641 static void test_tailcall_bpf2bpf_3(void)
 642 {
 643         int err, map_fd, prog_fd, main_fd, i;
 644         struct bpf_map *prog_array;
 645         struct bpf_program *prog;
 646         struct bpf_object *obj;
 647         __u32 retval, duration;
 648         char prog_name[32];
 649
 650         err = bpf_prog_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS,
 651                             &obj, &prog_fd);
 652         if (CHECK_FAIL(err))
 653                 return;
 654
 655         prog = bpf_object__find_program_by_title(obj, "classifier");
 656         if (CHECK_FAIL(!prog))
 657                 goto out;
 658
 659         main_fd = bpf_program__fd(prog);
 660         if (CHECK_FAIL(main_fd < 0))
 661                 goto out;
 662
 663         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 664         if (CHECK_FAIL(!prog_array))
 665                 goto out;
 666
 667         map_fd = bpf_map__fd(prog_array);
 668         if (CHECK_FAIL(map_fd < 0))
 669                 goto out;
 670
 671         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 672                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 673
 674                 prog = bpf_object__find_program_by_title(obj, prog_name);
 675                 if (CHECK_FAIL(!prog))
 676                         goto out;
 677
 678                 prog_fd = bpf_program__fd(prog);
 679                 if (CHECK_FAIL(prog_fd < 0))
 680                         goto out;
 681
 682                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 683                 if (CHECK_FAIL(err))
 684                         goto out;
 685         }
 686
 687         err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 688                                 &duration, &retval, NULL);
 689         CHECK(err || retval != sizeof(pkt_v4) * 3,
 690               "tailcall", "err %d errno %d retval %d\n",
 691               err, errno, retval);
 692
 693         i = 1;
 694         err = bpf_map_delete_elem(map_fd, &i);
 695         if (CHECK_FAIL(err))
 696                 goto out;
 697
 698         err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 699                                 &duration, &retval, NULL);
 700         CHECK(err || retval != sizeof(pkt_v4),
 701               "tailcall", "err %d errno %d retval %d\n",
 702               err, errno, retval);
 703
 704         i = 0;
 705         err = bpf_map_delete_elem(map_fd, &i);
 706         if (CHECK_FAIL(err))
 707                 goto out;
 708
 709         err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 710                                 &duration, &retval, NULL);
 711         CHECK(err || retval != sizeof(pkt_v4) * 2,
 712               "tailcall", "err %d errno %d retval %d\n",
 713               err, errno, retval);
 714 out:
 715         bpf_object__close(obj);
 716 }
 717
 718 #include "tailcall_bpf2bpf4.skel.h"
 719
 720 /* test_tailcall_bpf2bpf_4 checks that tailcall counter is correctly preserved
 721  * across tailcalls combined with bpf2bpf calls. for making sure that tailcall
 722  * counter behaves correctly, bpf program will go through following flow:
 723  *
 724  * entry -> entry_subprog -> tailcall0 -> bpf_func0 -> subprog0 ->
 725  * -> tailcall1 -> bpf_func1 -> subprog1 -> tailcall2 -> bpf_func2 ->
 726  * subprog2 [here bump global counter] --------^
 727  *
 728  * We go through first two tailcalls and start counting from the subprog2 where
 729  * the loop begins. At the end of the test make sure that the global counter is
 730  * equal to 31, because tailcall counter includes the first two tailcalls
 731  * whereas global counter is incremented only on loop presented on flow above.
 732  *
 733  * The noise parameter is used to insert bpf_map_update calls into the logic
 734  * to force verifier to patch instructions. This allows us to ensure jump
 735  * logic remains correct with instruction movement.
 736  */
 737 static void test_tailcall_bpf2bpf_4(bool noise)
 738 {
 739         int err, map_fd, prog_fd, main_fd, data_fd, i;
 740         struct tailcall_bpf2bpf4__bss val;
 741         struct bpf_map *prog_array, *data_map;
 742         struct bpf_program *prog;
 743         struct bpf_object *obj;
 744         __u32 retval, duration;
 745         char prog_name[32];
 746
 747         err = bpf_prog_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS,
 748                             &obj, &prog_fd);
 749         if (CHECK_FAIL(err))
 750                 return;
 751
 752         prog = bpf_object__find_program_by_title(obj, "classifier");
 753         if (CHECK_FAIL(!prog))
 754                 goto out;
 755
 756         main_fd = bpf_program__fd(prog);
 757         if (CHECK_FAIL(main_fd < 0))
 758                 goto out;
 759
 760         prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 761         if (CHECK_FAIL(!prog_array))
 762                 goto out;
 763
 764         map_fd = bpf_map__fd(prog_array);
 765         if (CHECK_FAIL(map_fd < 0))
 766                 goto out;
 767
 768         for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 769                 snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 770
 771                 prog = bpf_object__find_program_by_title(obj, prog_name);
 772                 if (CHECK_FAIL(!prog))
 773                         goto out;
 774
 775                 prog_fd = bpf_program__fd(prog);
 776                 if (CHECK_FAIL(prog_fd < 0))
 777                         goto out;
 778
 779                 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 780                 if (CHECK_FAIL(err))
 781                         goto out;
 782         }
 783
 784         data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 785         if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 786                 return;
 787
 788         data_fd = bpf_map__fd(data_map);
 789         if (CHECK_FAIL(map_fd < 0))
 790                 return;
 791
 792         i = 0;
 793         val.noise = noise;
 794         val.count = 0;
 795         err = bpf_map_update_elem(data_fd, &i, &val, BPF_ANY);
 796         if (CHECK_FAIL(err))
 797                 goto out;
 798
 799         err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 800                                 &duration, &retval, NULL);
 801         CHECK(err || retval != sizeof(pkt_v4) * 3, "tailcall", "err %d errno %d retval %d\n",
 802               err, errno, retval);
 803
 804         i = 0;
 805         err = bpf_map_lookup_elem(data_fd, &i, &val);
 806         CHECK(err || val.count != 31, "tailcall count", "err %d errno %d count %d\n",
 807               err, errno, val.count);
 808
 809 out:
 810         bpf_object__close(obj);
 811 }
 812
 813 void test_tailcalls(void)
 814 {
 815         if (test__start_subtest("tailcall_1"))
 816                 test_tailcall_1();
 817         if (test__start_subtest("tailcall_2"))
 818                 test_tailcall_2();
 819         if (test__start_subtest("tailcall_3"))
 820                 test_tailcall_3();
 821         if (test__start_subtest("tailcall_4"))
 822                 test_tailcall_4();
 823         if (test__start_subtest("tailcall_5"))
 824                 test_tailcall_5();
 825         if (test__start_subtest("tailcall_bpf2bpf_1"))
 826                 test_tailcall_bpf2bpf_1();
 827         if (test__start_subtest("tailcall_bpf2bpf_2"))
 828                 test_tailcall_bpf2bpf_2();
 829         if (test__start_subtest("tailcall_bpf2bpf_3"))
 830                 test_tailcall_bpf2bpf_3();
 831         if (test__start_subtest("tailcall_bpf2bpf_4"))
 832                 test_tailcall_bpf2bpf_4(false);
 833         if (test__start_subtest("tailcall_bpf2bpf_5"))
 834                 test_tailcall_bpf2bpf_4(true);
 835 }