tools/testing/selftests/bpf/test_progs.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2017 Facebook
   3  */
   4 #define _GNU_SOURCE
   5 #include "test_progs.h"
   6 #include "testing_helpers.h"
   7 #include "cgroup_helpers.h"
   8 #include <argp.h>
   9 #include <pthread.h>
  10 #include <sched.h>
  11 #include <signal.h>
  12 #include <string.h>
  13 #include <execinfo.h> /* backtrace */
  14 #include <linux/membarrier.h>
  15 #include <sys/sysinfo.h> /* get_nprocs */
  16 #include <netinet/in.h>
  17 #include <sys/select.h>
  18 #include <sys/socket.h>
  19 #include <sys/un.h>
  20 #include <bpf/btf.h>
  21 #include "json_writer.h"
  22
  23 static bool verbose(void)
  24 {
  25         return env.verbosity > VERBOSE_NONE;
  26 }
  27
  28 static void stdio_hijack_init(char **log_buf, size_t *log_cnt)
  29 {
  30 #ifdef __GLIBC__
  31         if (verbose() && env.worker_id == -1) {
  32                 /* nothing to do, output to stdout by default */
  33                 return;
  34         }
  35
  36         fflush(stdout);
  37         fflush(stderr);
  38
  39         stdout = open_memstream(log_buf, log_cnt);
  40         if (!stdout) {
  41                 stdout = env.stdout;
  42                 perror("open_memstream");
  43                 return;
  44         }
  45
  46         if (env.subtest_state)
  47                 env.subtest_state->stdout = stdout;
  48         else
  49                 env.test_state->stdout = stdout;
  50
  51         stderr = stdout;
  52 #endif
  53 }
  54
  55 static void stdio_hijack(char **log_buf, size_t *log_cnt)
  56 {
  57 #ifdef __GLIBC__
  58         if (verbose() && env.worker_id == -1) {
  59                 /* nothing to do, output to stdout by default */
  60                 return;
  61         }
  62
  63         env.stdout = stdout;
  64         env.stderr = stderr;
  65
  66         stdio_hijack_init(log_buf, log_cnt);
  67 #endif
  68 }
  69
  70 static void stdio_restore_cleanup(void)
  71 {
  72 #ifdef __GLIBC__
  73         if (verbose() && env.worker_id == -1) {
  74                 /* nothing to do, output to stdout by default */
  75                 return;
  76         }
  77
  78         fflush(stdout);
  79
  80         if (env.subtest_state) {
  81                 fclose(env.subtest_state->stdout);
  82                 env.subtest_state->stdout = NULL;
  83                 stdout = env.test_state->stdout;
  84                 stderr = env.test_state->stdout;
  85         } else {
  86                 fclose(env.test_state->stdout);
  87                 env.test_state->stdout = NULL;
  88         }
  89 #endif
  90 }
  91
  92 static void stdio_restore(void)
  93 {
  94 #ifdef __GLIBC__
  95         if (verbose() && env.worker_id == -1) {
  96                 /* nothing to do, output to stdout by default */
  97                 return;
  98         }
  99
 100         if (stdout == env.stdout)
 101                 return;
 102
 103         stdio_restore_cleanup();
 104
 105         stdout = env.stdout;
 106         stderr = env.stderr;
 107 #endif
 108 }
 109
 110 /* Adapted from perf/util/string.c */
 111 static bool glob_match(const char *str, const char *pat)
 112 {
 113         while (*str && *pat && *pat != '*') {
 114                 if (*str != *pat)
 115                         return false;
 116                 str++;
 117                 pat++;
 118         }
 119         /* Check wild card */
 120         if (*pat == '*') {
 121                 while (*pat == '*')
 122                         pat++;
 123                 if (!*pat) /* Tail wild card matches all */
 124                         return true;
 125                 while (*str)
 126                         if (glob_match(str++, pat))
 127                                 return true;
 128         }
 129         return !*str && !*pat;
 130 }
 131
 132 #define EXIT_NO_TEST            2
 133 #define EXIT_ERR_SETUP_INFRA    3
 134
 135 /* defined in test_progs.h */
 136 struct test_env env = {};
 137
 138 struct prog_test_def {
 139         const char *test_name;
 140         int test_num;
 141         void (*run_test)(void);
 142         void (*run_serial_test)(void);
 143         bool should_run;
 144         bool need_cgroup_cleanup;
 145 };
 146
 147 /* Override C runtime library's usleep() implementation to ensure nanosleep()
 148  * is always called. Usleep is frequently used in selftests as a way to
 149  * trigger kprobe and tracepoints.
 150  */
 151 int usleep(useconds_t usec)
 152 {
 153         struct timespec ts = {
 154                 .tv_sec = usec / 1000000,
 155                 .tv_nsec = (usec % 1000000) * 1000,
 156         };
 157
 158         return syscall(__NR_nanosleep, &ts, NULL);
 159 }
 160
 161 static bool should_run(struct test_selector *sel, int num, const char *name)
 162 {
 163         int i;
 164
 165         for (i = 0; i < sel->blacklist.cnt; i++) {
 166                 if (glob_match(name, sel->blacklist.tests[i].name) &&
 167                     !sel->blacklist.tests[i].subtest_cnt)
 168                         return false;
 169         }
 170
 171         for (i = 0; i < sel->whitelist.cnt; i++) {
 172                 if (glob_match(name, sel->whitelist.tests[i].name))
 173                         return true;
 174         }
 175
 176         if (!sel->whitelist.cnt && !sel->num_set)
 177                 return true;
 178
 179         return num < sel->num_set_len && sel->num_set[num];
 180 }
 181
 182 static bool should_run_subtest(struct test_selector *sel,
 183                                struct test_selector *subtest_sel,
 184                                int subtest_num,
 185                                const char *test_name,
 186                                const char *subtest_name)
 187 {
 188         int i, j;
 189
 190         for (i = 0; i < sel->blacklist.cnt; i++) {
 191                 if (glob_match(test_name, sel->blacklist.tests[i].name)) {
 192                         if (!sel->blacklist.tests[i].subtest_cnt)
 193                                 return false;
 194
 195                         for (j = 0; j < sel->blacklist.tests[i].subtest_cnt; j++) {
 196                                 if (glob_match(subtest_name,
 197                                                sel->blacklist.tests[i].subtests[j]))
 198                                         return false;
 199                         }
 200                 }
 201         }
 202
 203         for (i = 0; i < sel->whitelist.cnt; i++) {
 204                 if (glob_match(test_name, sel->whitelist.tests[i].name)) {
 205                         if (!sel->whitelist.tests[i].subtest_cnt)
 206                                 return true;
 207
 208                         for (j = 0; j < sel->whitelist.tests[i].subtest_cnt; j++) {
 209                                 if (glob_match(subtest_name,
 210                                                sel->whitelist.tests[i].subtests[j]))
 211                                         return true;
 212                         }
 213                 }
 214         }
 215
 216         if (!sel->whitelist.cnt && !subtest_sel->num_set)
 217                 return true;
 218
 219         return subtest_num < subtest_sel->num_set_len && subtest_sel->num_set[subtest_num];
 220 }
 221
 222 static char *test_result(bool failed, bool skipped)
 223 {
 224         return failed ? "FAIL" : (skipped ? "SKIP" : "OK");
 225 }
 226
 227 #define TEST_NUM_WIDTH 7
 228
 229 static void print_test_result(const struct prog_test_def *test, const struct test_state *test_state)
 230 {
 231         int skipped_cnt = test_state->skip_cnt;
 232         int subtests_cnt = test_state->subtest_num;
 233
 234         fprintf(env.stdout, "#%-*d %s:", TEST_NUM_WIDTH, test->test_num, test->test_name);
 235         if (test_state->error_cnt)
 236                 fprintf(env.stdout, "FAIL");
 237         else if (!skipped_cnt)
 238                 fprintf(env.stdout, "OK");
 239         else if (skipped_cnt == subtests_cnt || !subtests_cnt)
 240                 fprintf(env.stdout, "SKIP");
 241         else
 242                 fprintf(env.stdout, "OK (SKIP: %d/%d)", skipped_cnt, subtests_cnt);
 243
 244         fprintf(env.stdout, "\n");
 245 }
 246
 247 static void print_test_log(char *log_buf, size_t log_cnt)
 248 {
 249         log_buf[log_cnt] = '\0';
 250         fprintf(env.stdout, "%s", log_buf);
 251         if (log_buf[log_cnt - 1] != '\n')
 252                 fprintf(env.stdout, "\n");
 253 }
 254
 255 static void print_subtest_name(int test_num, int subtest_num,
 256                                const char *test_name, char *subtest_name,
 257                                char *result)
 258 {
 259         char test_num_str[TEST_NUM_WIDTH + 1];
 260
 261         snprintf(test_num_str, sizeof(test_num_str), "%d/%d", test_num, subtest_num);
 262
 263         fprintf(env.stdout, "#%-*s %s/%s",
 264                 TEST_NUM_WIDTH, test_num_str,
 265                 test_name, subtest_name);
 266
 267         if (result)
 268                 fprintf(env.stdout, ":%s", result);
 269
 270         fprintf(env.stdout, "\n");
 271 }
 272
 273 static void jsonw_write_log_message(json_writer_t *w, char *log_buf, size_t log_cnt)
 274 {
 275         /* open_memstream (from stdio_hijack_init) ensures that log_bug is terminated by a
 276          * null byte. Yet in parallel mode, log_buf will be NULL if there is no message.
 277          */
 278         if (log_cnt) {
 279                 jsonw_string_field(w, "message", log_buf);
 280         } else {
 281                 jsonw_string_field(w, "message", "");
 282         }
 283 }
 284
 285 static void dump_test_log(const struct prog_test_def *test,
 286                           const struct test_state *test_state,
 287                           bool skip_ok_subtests,
 288                           bool par_exec_result,
 289                           json_writer_t *w)
 290 {
 291         bool test_failed = test_state->error_cnt > 0;
 292         bool force_log = test_state->force_log;
 293         bool print_test = verbose() || force_log || test_failed;
 294         int i;
 295         struct subtest_state *subtest_state;
 296         bool subtest_failed;
 297         bool subtest_filtered;
 298         bool print_subtest;
 299
 300         /* we do not print anything in the worker thread */
 301         if (env.worker_id != -1)
 302                 return;
 303
 304         /* there is nothing to print when verbose log is used and execution
 305          * is not in parallel mode
 306          */
 307         if (verbose() && !par_exec_result)
 308                 return;
 309
 310         if (test_state->log_cnt && print_test)
 311                 print_test_log(test_state->log_buf, test_state->log_cnt);
 312
 313         if (w && print_test) {
 314                 jsonw_start_object(w);
 315                 jsonw_string_field(w, "name", test->test_name);
 316                 jsonw_uint_field(w, "number", test->test_num);
 317                 jsonw_write_log_message(w, test_state->log_buf, test_state->log_cnt);
 318                 jsonw_bool_field(w, "failed", test_failed);
 319                 jsonw_name(w, "subtests");
 320                 jsonw_start_array(w);
 321         }
 322
 323         for (i = 0; i < test_state->subtest_num; i++) {
 324                 subtest_state = &test_state->subtest_states[i];
 325                 subtest_failed = subtest_state->error_cnt;
 326                 subtest_filtered = subtest_state->filtered;
 327                 print_subtest = verbose() || force_log || subtest_failed;
 328
 329                 if ((skip_ok_subtests && !subtest_failed) || subtest_filtered)
 330                         continue;
 331
 332                 if (subtest_state->log_cnt && print_subtest) {
 333                         print_test_log(subtest_state->log_buf,
 334                                        subtest_state->log_cnt);
 335                 }
 336
 337                 print_subtest_name(test->test_num, i + 1,
 338                                    test->test_name, subtest_state->name,
 339                                    test_result(subtest_state->error_cnt,
 340                                                subtest_state->skipped));
 341
 342                 if (w && print_subtest) {
 343                         jsonw_start_object(w);
 344                         jsonw_string_field(w, "name", subtest_state->name);
 345                         jsonw_uint_field(w, "number", i+1);
 346                         jsonw_write_log_message(w, subtest_state->log_buf, subtest_state->log_cnt);
 347                         jsonw_bool_field(w, "failed", subtest_failed);
 348                         jsonw_end_object(w);
 349                 }
 350         }
 351
 352         if (w && print_test) {
 353                 jsonw_end_array(w);
 354                 jsonw_end_object(w);
 355         }
 356
 357         print_test_result(test, test_state);
 358 }
 359
 360 static void stdio_restore(void);
 361
 362 /* A bunch of tests set custom affinity per-thread and/or per-process. Reset
 363  * it after each test/sub-test.
 364  */
 365 static void reset_affinity(void)
 366 {
 367         cpu_set_t cpuset;
 368         int i, err;
 369
 370         CPU_ZERO(&cpuset);
 371         for (i = 0; i < env.nr_cpus; i++)
 372                 CPU_SET(i, &cpuset);
 373
 374         err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
 375         if (err < 0) {
 376                 stdio_restore();
 377                 fprintf(stderr, "Failed to reset process affinity: %d!\n", err);
 378                 exit(EXIT_ERR_SETUP_INFRA);
 379         }
 380         err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
 381         if (err < 0) {
 382                 stdio_restore();
 383                 fprintf(stderr, "Failed to reset thread affinity: %d!\n", err);
 384                 exit(EXIT_ERR_SETUP_INFRA);
 385         }
 386 }
 387
 388 static void save_netns(void)
 389 {
 390         env.saved_netns_fd = open("/proc/self/ns/net", O_RDONLY);
 391         if (env.saved_netns_fd == -1) {
 392                 perror("open(/proc/self/ns/net)");
 393                 exit(EXIT_ERR_SETUP_INFRA);
 394         }
 395 }
 396
 397 static void restore_netns(void)
 398 {
 399         if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) {
 400                 stdio_restore();
 401                 perror("setns(CLONE_NEWNS)");
 402                 exit(EXIT_ERR_SETUP_INFRA);
 403         }
 404 }
 405
 406 void test__end_subtest(void)
 407 {
 408         struct prog_test_def *test = env.test;
 409         struct test_state *test_state = env.test_state;
 410         struct subtest_state *subtest_state = env.subtest_state;
 411
 412         if (subtest_state->error_cnt) {
 413                 test_state->error_cnt++;
 414         } else {
 415                 if (!subtest_state->skipped)
 416                         test_state->sub_succ_cnt++;
 417                 else
 418                         test_state->skip_cnt++;
 419         }
 420
 421         if (verbose() && !env.workers)
 422                 print_subtest_name(test->test_num, test_state->subtest_num,
 423                                    test->test_name, subtest_state->name,
 424                                    test_result(subtest_state->error_cnt,
 425                                                subtest_state->skipped));
 426
 427         stdio_restore_cleanup();
 428         env.subtest_state = NULL;
 429 }
 430
 431 bool test__start_subtest(const char *subtest_name)
 432 {
 433         struct prog_test_def *test = env.test;
 434         struct test_state *state = env.test_state;
 435         struct subtest_state *subtest_state;
 436         size_t sub_state_size = sizeof(*subtest_state);
 437
 438         if (env.subtest_state)
 439                 test__end_subtest();
 440
 441         state->subtest_num++;
 442         state->subtest_states =
 443                 realloc(state->subtest_states,
 444                         state->subtest_num * sub_state_size);
 445         if (!state->subtest_states) {
 446                 fprintf(stderr, "Not enough memory to allocate subtest result\n");
 447                 return false;
 448         }
 449
 450         subtest_state = &state->subtest_states[state->subtest_num - 1];
 451
 452         memset(subtest_state, 0, sub_state_size);
 453
 454         if (!subtest_name || !subtest_name[0]) {
 455                 fprintf(env.stderr,
 456                         "Subtest #%d didn't provide sub-test name!\n",
 457                         state->subtest_num);
 458                 return false;
 459         }
 460
 461         subtest_state->name = strdup(subtest_name);
 462         if (!subtest_state->name) {
 463                 fprintf(env.stderr,
 464                         "Subtest #%d: failed to copy subtest name!\n",
 465                         state->subtest_num);
 466                 return false;
 467         }
 468
 469         if (!should_run_subtest(&env.test_selector,
 470                                 &env.subtest_selector,
 471                                 state->subtest_num,
 472                                 test->test_name,
 473                                 subtest_name)) {
 474                 subtest_state->filtered = true;
 475                 return false;
 476         }
 477
 478         env.subtest_state = subtest_state;
 479         stdio_hijack_init(&subtest_state->log_buf, &subtest_state->log_cnt);
 480
 481         return true;
 482 }
 483
 484 void test__force_log(void)
 485 {
 486         env.test_state->force_log = true;
 487 }
 488
 489 void test__skip(void)
 490 {
 491         if (env.subtest_state)
 492                 env.subtest_state->skipped = true;
 493         else
 494                 env.test_state->skip_cnt++;
 495 }
 496
 497 void test__fail(void)
 498 {
 499         if (env.subtest_state)
 500                 env.subtest_state->error_cnt++;
 501         else
 502                 env.test_state->error_cnt++;
 503 }
 504
 505 int test__join_cgroup(const char *path)
 506 {
 507         int fd;
 508
 509         if (!env.test->need_cgroup_cleanup) {
 510                 if (setup_cgroup_environment()) {
 511                         fprintf(stderr,
 512                                 "#%d %s: Failed to setup cgroup environment\n",
 513                                 env.test->test_num, env.test->test_name);
 514                         return -1;
 515                 }
 516
 517                 env.test->need_cgroup_cleanup = true;
 518         }
 519
 520         fd = create_and_get_cgroup(path);
 521         if (fd < 0) {
 522                 fprintf(stderr,
 523                         "#%d %s: Failed to create cgroup '%s' (errno=%d)\n",
 524                         env.test->test_num, env.test->test_name, path, errno);
 525                 return fd;
 526         }
 527
 528         if (join_cgroup(path)) {
 529                 fprintf(stderr,
 530                         "#%d %s: Failed to join cgroup '%s' (errno=%d)\n",
 531                         env.test->test_num, env.test->test_name, path, errno);
 532                 return -1;
 533         }
 534
 535         return fd;
 536 }
 537
 538 int bpf_find_map(const char *test, struct bpf_object *obj, const char *name)
 539 {
 540         struct bpf_map *map;
 541
 542         map = bpf_object__find_map_by_name(obj, name);
 543         if (!map) {
 544                 fprintf(stdout, "%s:FAIL:map '%s' not found\n", test, name);
 545                 test__fail();
 546                 return -1;
 547         }
 548         return bpf_map__fd(map);
 549 }
 550
 551 static bool is_jit_enabled(void)
 552 {
 553         const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
 554         bool enabled = false;
 555         int sysctl_fd;
 556
 557         sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
 558         if (sysctl_fd != -1) {
 559                 char tmpc;
 560
 561                 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
 562                         enabled = (tmpc != '0');
 563                 close(sysctl_fd);
 564         }
 565
 566         return enabled;
 567 }
 568
 569 int compare_map_keys(int map1_fd, int map2_fd)
 570 {
 571         __u32 key, next_key;
 572         char val_buf[PERF_MAX_STACK_DEPTH *
 573                      sizeof(struct bpf_stack_build_id)];
 574         int err;
 575
 576         err = bpf_map_get_next_key(map1_fd, NULL, &key);
 577         if (err)
 578                 return err;
 579         err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
 580         if (err)
 581                 return err;
 582
 583         while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
 584                 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
 585                 if (err)
 586                         return err;
 587
 588                 key = next_key;
 589         }
 590         if (errno != ENOENT)
 591                 return -1;
 592
 593         return 0;
 594 }
 595
 596 int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
 597 {
 598         __u32 key, next_key, *cur_key_p, *next_key_p;
 599         char *val_buf1, *val_buf2;
 600         int i, err = 0;
 601
 602         val_buf1 = malloc(stack_trace_len);
 603         val_buf2 = malloc(stack_trace_len);
 604         cur_key_p = NULL;
 605         next_key_p = &key;
 606         while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
 607                 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
 608                 if (err)
 609                         goto out;
 610                 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
 611                 if (err)
 612                         goto out;
 613                 for (i = 0; i < stack_trace_len; i++) {
 614                         if (val_buf1[i] != val_buf2[i]) {
 615                                 err = -1;
 616                                 goto out;
 617                         }
 618                 }
 619                 key = *next_key_p;
 620                 cur_key_p = &key;
 621                 next_key_p = &next_key;
 622         }
 623         if (errno != ENOENT)
 624                 err = -1;
 625
 626 out:
 627         free(val_buf1);
 628         free(val_buf2);
 629         return err;
 630 }
 631
 632 static int finit_module(int fd, const char *param_values, int flags)
 633 {
 634         return syscall(__NR_finit_module, fd, param_values, flags);
 635 }
 636
 637 static int delete_module(const char *name, int flags)
 638 {
 639         return syscall(__NR_delete_module, name, flags);
 640 }
 641
 642 /*
 643  * Trigger synchronize_rcu() in kernel.
 644  */
 645 int kern_sync_rcu(void)
 646 {
 647         return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0);
 648 }
 649
 650 static void unload_bpf_testmod(void)
 651 {
 652         if (kern_sync_rcu())
 653                 fprintf(env.stderr, "Failed to trigger kernel-side RCU sync!\n");
 654         if (delete_module("bpf_testmod", 0)) {
 655                 if (errno == ENOENT) {
 656                         if (verbose())
 657                                 fprintf(stdout, "bpf_testmod.ko is already unloaded.\n");
 658                         return;
 659                 }
 660                 fprintf(env.stderr, "Failed to unload bpf_testmod.ko from kernel: %d\n", -errno);
 661                 return;
 662         }
 663         if (verbose())
 664                 fprintf(stdout, "Successfully unloaded bpf_testmod.ko.\n");
 665 }
 666
 667 static int load_bpf_testmod(void)
 668 {
 669         int fd;
 670
 671         /* ensure previous instance of the module is unloaded */
 672         unload_bpf_testmod();
 673
 674         if (verbose())
 675                 fprintf(stdout, "Loading bpf_testmod.ko...\n");
 676
 677         fd = open("bpf_testmod.ko", O_RDONLY);
 678         if (fd < 0) {
 679                 fprintf(env.stderr, "Can't find bpf_testmod.ko kernel module: %d\n", -errno);
 680                 return -ENOENT;
 681         }
 682         if (finit_module(fd, "", 0)) {
 683                 fprintf(env.stderr, "Failed to load bpf_testmod.ko into the kernel: %d\n", -errno);
 684                 close(fd);
 685                 return -EINVAL;
 686         }
 687         close(fd);
 688
 689         if (verbose())
 690                 fprintf(stdout, "Successfully loaded bpf_testmod.ko.\n");
 691         return 0;
 692 }
 693
 694 /* extern declarations for test funcs */
 695 #define DEFINE_TEST(name)                               \
 696         extern void test_##name(void) __weak;           \
 697         extern void serial_test_##name(void) __weak;
 698 #include <prog_tests/tests.h>
 699 #undef DEFINE_TEST
 700
 701 static struct prog_test_def prog_test_defs[] = {
 702 #define DEFINE_TEST(name) {                     \
 703         .test_name = #name,                     \
 704         .run_test = &test_##name,               \
 705         .run_serial_test = &serial_test_##name, \
 706 },
 707 #include <prog_tests/tests.h>
 708 #undef DEFINE_TEST
 709 };
 710
 711 static const int prog_test_cnt = ARRAY_SIZE(prog_test_defs);
 712
 713 static struct test_state test_states[ARRAY_SIZE(prog_test_defs)];
 714
 715 const char *argp_program_version = "test_progs 0.1";
 716 const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
 717 static const char argp_program_doc[] = "BPF selftests test runner";
 718
 719 enum ARG_KEYS {
 720         ARG_TEST_NUM = 'n',
 721         ARG_TEST_NAME = 't',
 722         ARG_TEST_NAME_BLACKLIST = 'b',
 723         ARG_VERIFIER_STATS = 's',
 724         ARG_VERBOSE = 'v',
 725         ARG_GET_TEST_CNT = 'c',
 726         ARG_LIST_TEST_NAMES = 'l',
 727         ARG_TEST_NAME_GLOB_ALLOWLIST = 'a',
 728         ARG_TEST_NAME_GLOB_DENYLIST = 'd',
 729         ARG_NUM_WORKERS = 'j',
 730         ARG_DEBUG = -1,
 731         ARG_JSON_SUMMARY = 'J'
 732 };
 733
 734 static const struct argp_option opts[] = {
 735         { "num", ARG_TEST_NUM, "NUM", 0,
 736           "Run test number NUM only " },
 737         { "name", ARG_TEST_NAME, "NAMES", 0,
 738           "Run tests with names containing any string from NAMES list" },
 739         { "name-blacklist", ARG_TEST_NAME_BLACKLIST, "NAMES", 0,
 740           "Don't run tests with names containing any string from NAMES list" },
 741         { "verifier-stats", ARG_VERIFIER_STATS, NULL, 0,
 742           "Output verifier statistics", },
 743         { "verbose", ARG_VERBOSE, "LEVEL", OPTION_ARG_OPTIONAL,
 744           "Verbose output (use -vv or -vvv for progressively verbose output)" },
 745         { "count", ARG_GET_TEST_CNT, NULL, 0,
 746           "Get number of selected top-level tests " },
 747         { "list", ARG_LIST_TEST_NAMES, NULL, 0,
 748           "List test names that would run (without running them) " },
 749         { "allow", ARG_TEST_NAME_GLOB_ALLOWLIST, "NAMES", 0,
 750           "Run tests with name matching the pattern (supports '*' wildcard)." },
 751         { "deny", ARG_TEST_NAME_GLOB_DENYLIST, "NAMES", 0,
 752           "Don't run tests with name matching the pattern (supports '*' wildcard)." },
 753         { "workers", ARG_NUM_WORKERS, "WORKERS", OPTION_ARG_OPTIONAL,
 754           "Number of workers to run in parallel, default to number of cpus." },
 755         { "debug", ARG_DEBUG, NULL, 0,
 756           "print extra debug information for test_progs." },
 757         { "json-summary", ARG_JSON_SUMMARY, "FILE", 0, "Write report in json format to this file."},
 758         {},
 759 };
 760
 761 static int libbpf_print_fn(enum libbpf_print_level level,
 762                            const char *format, va_list args)
 763 {
 764         if (env.verbosity < VERBOSE_VERY && level == LIBBPF_DEBUG)
 765                 return 0;
 766         vfprintf(stdout, format, args);
 767         return 0;
 768 }
 769
 770 static void free_test_filter_set(const struct test_filter_set *set)
 771 {
 772         int i, j;
 773
 774         if (!set)
 775                 return;
 776
 777         for (i = 0; i < set->cnt; i++) {
 778                 free((void *)set->tests[i].name);
 779                 for (j = 0; j < set->tests[i].subtest_cnt; j++)
 780                         free((void *)set->tests[i].subtests[j]);
 781
 782                 free((void *)set->tests[i].subtests);
 783         }
 784
 785         free((void *)set->tests);
 786 }
 787
 788 static void free_test_selector(struct test_selector *test_selector)
 789 {
 790         free_test_filter_set(&test_selector->blacklist);
 791         free_test_filter_set(&test_selector->whitelist);
 792         free(test_selector->num_set);
 793 }
 794
 795 extern int extra_prog_load_log_flags;
 796
 797 static error_t parse_arg(int key, char *arg, struct argp_state *state)
 798 {
 799         struct test_env *env = state->input;
 800
 801         switch (key) {
 802         case ARG_TEST_NUM: {
 803                 char *subtest_str = strchr(arg, '/');
 804
 805                 if (subtest_str) {
 806                         *subtest_str = '\0';
 807                         if (parse_num_list(subtest_str + 1,
 808                                            &env->subtest_selector.num_set,
 809                                            &env->subtest_selector.num_set_len)) {
 810                                 fprintf(stderr,
 811                                         "Failed to parse subtest numbers.\n");
 812                                 return -EINVAL;
 813                         }
 814                 }
 815                 if (parse_num_list(arg, &env->test_selector.num_set,
 816                                    &env->test_selector.num_set_len)) {
 817                         fprintf(stderr, "Failed to parse test numbers.\n");
 818                         return -EINVAL;
 819                 }
 820                 break;
 821         }
 822         case ARG_TEST_NAME_GLOB_ALLOWLIST:
 823         case ARG_TEST_NAME: {
 824                 if (parse_test_list(arg,
 825                                     &env->test_selector.whitelist,
 826                                     key == ARG_TEST_NAME_GLOB_ALLOWLIST))
 827                         return -ENOMEM;
 828                 break;
 829         }
 830         case ARG_TEST_NAME_GLOB_DENYLIST:
 831         case ARG_TEST_NAME_BLACKLIST: {
 832                 if (parse_test_list(arg,
 833                                     &env->test_selector.blacklist,
 834                                     key == ARG_TEST_NAME_GLOB_DENYLIST))
 835                         return -ENOMEM;
 836                 break;
 837         }
 838         case ARG_VERIFIER_STATS:
 839                 env->verifier_stats = true;
 840                 break;
 841         case ARG_VERBOSE:
 842                 env->verbosity = VERBOSE_NORMAL;
 843                 if (arg) {
 844                         if (strcmp(arg, "v") == 0) {
 845                                 env->verbosity = VERBOSE_VERY;
 846                                 extra_prog_load_log_flags = 1;
 847                         } else if (strcmp(arg, "vv") == 0) {
 848                                 env->verbosity = VERBOSE_SUPER;
 849                                 extra_prog_load_log_flags = 2;
 850                         } else {
 851                                 fprintf(stderr,
 852                                         "Unrecognized verbosity setting ('%s'), only -v and -vv are supported\n",
 853                                         arg);
 854                                 return -EINVAL;
 855                         }
 856                 }
 857
 858                 if (verbose()) {
 859                         if (setenv("SELFTESTS_VERBOSE", "1", 1) == -1) {
 860                                 fprintf(stderr,
 861                                         "Unable to setenv SELFTESTS_VERBOSE=1 (errno=%d)",
 862                                         errno);
 863                                 return -EINVAL;
 864                         }
 865                 }
 866
 867                 break;
 868         case ARG_GET_TEST_CNT:
 869                 env->get_test_cnt = true;
 870                 break;
 871         case ARG_LIST_TEST_NAMES:
 872                 env->list_test_names = true;
 873                 break;
 874         case ARG_NUM_WORKERS:
 875                 if (arg) {
 876                         env->workers = atoi(arg);
 877                         if (!env->workers) {
 878                                 fprintf(stderr, "Invalid number of worker: %s.", arg);
 879                                 return -EINVAL;
 880                         }
 881                 } else {
 882                         env->workers = get_nprocs();
 883                 }
 884                 break;
 885         case ARG_DEBUG:
 886                 env->debug = true;
 887                 break;
 888         case ARG_JSON_SUMMARY:
 889                 env->json = fopen(arg, "w");
 890                 if (env->json == NULL) {
 891                         perror("Failed to open json summary file");
 892                         return -errno;
 893                 }
 894                 break;
 895         case ARGP_KEY_ARG:
 896                 argp_usage(state);
 897                 break;
 898         case ARGP_KEY_END:
 899                 break;
 900         default:
 901                 return ARGP_ERR_UNKNOWN;
 902         }
 903         return 0;
 904 }
 905
 906 /*
 907  * Determine if test_progs is running as a "flavored" test runner and switch
 908  * into corresponding sub-directory to load correct BPF objects.
 909  *
 910  * This is done by looking at executable name. If it contains "-flavor"
 911  * suffix, then we are running as a flavored test runner.
 912  */
 913 int cd_flavor_subdir(const char *exec_name)
 914 {
 915         /* General form of argv[0] passed here is:
 916          * some/path/to/test_progs[-flavor], where -flavor part is optional.
 917          * First cut out "test_progs[-flavor]" part, then extract "flavor"
 918          * part, if it's there.
 919          */
 920         const char *flavor = strrchr(exec_name, '/');
 921
 922         if (!flavor)
 923                 flavor = exec_name;
 924         else
 925                 flavor++;
 926
 927         flavor = strrchr(flavor, '-');
 928         if (!flavor)
 929                 return 0;
 930         flavor++;
 931         if (verbose())
 932                 fprintf(stdout, "Switching to flavor '%s' subdirectory...\n", flavor);
 933
 934         return chdir(flavor);
 935 }
 936
 937 int trigger_module_test_read(int read_sz)
 938 {
 939         int fd, err;
 940
 941         fd = open(BPF_TESTMOD_TEST_FILE, O_RDONLY);
 942         err = -errno;
 943         if (!ASSERT_GE(fd, 0, "testmod_file_open"))
 944                 return err;
 945
 946         read(fd, NULL, read_sz);
 947         close(fd);
 948
 949         return 0;
 950 }
 951
 952 int trigger_module_test_write(int write_sz)
 953 {
 954         int fd, err;
 955         char *buf = malloc(write_sz);
 956
 957         if (!buf)
 958                 return -ENOMEM;
 959
 960         memset(buf, 'a', write_sz);
 961         buf[write_sz-1] = '\0';
 962
 963         fd = open(BPF_TESTMOD_TEST_FILE, O_WRONLY);
 964         err = -errno;
 965         if (!ASSERT_GE(fd, 0, "testmod_file_open")) {
 966                 free(buf);
 967                 return err;
 968         }
 969
 970         write(fd, buf, write_sz);
 971         close(fd);
 972         free(buf);
 973         return 0;
 974 }
 975
 976 int write_sysctl(const char *sysctl, const char *value)
 977 {
 978         int fd, err, len;
 979
 980         fd = open(sysctl, O_WRONLY);
 981         if (!ASSERT_NEQ(fd, -1, "open sysctl"))
 982                 return -1;
 983
 984         len = strlen(value);
 985         err = write(fd, value, len);
 986         close(fd);
 987         if (!ASSERT_EQ(err, len, "write sysctl"))
 988                 return -1;
 989
 990         return 0;
 991 }
 992
 993 int get_bpf_max_tramp_links_from(struct btf *btf)
 994 {
 995         const struct btf_enum *e;
 996         const struct btf_type *t;
 997         __u32 i, type_cnt;
 998         const char *name;
 999         __u16 j, vlen;
1000
1001         for (i = 1, type_cnt = btf__type_cnt(btf); i < type_cnt; i++) {
1002                 t = btf__type_by_id(btf, i);
1003                 if (!t || !btf_is_enum(t) || t->name_off)
1004                         continue;
1005                 e = btf_enum(t);
1006                 for (j = 0, vlen = btf_vlen(t); j < vlen; j++, e++) {
1007                         name = btf__str_by_offset(btf, e->name_off);
1008                         if (name && !strcmp(name, "BPF_MAX_TRAMP_LINKS"))
1009                                 return e->val;
1010                 }
1011         }
1012
1013         return -1;
1014 }
1015
1016 int get_bpf_max_tramp_links(void)
1017 {
1018         struct btf *vmlinux_btf;
1019         int ret;
1020
1021         vmlinux_btf = btf__load_vmlinux_btf();
1022         if (!ASSERT_OK_PTR(vmlinux_btf, "vmlinux btf"))
1023                 return -1;
1024         ret = get_bpf_max_tramp_links_from(vmlinux_btf);
1025         btf__free(vmlinux_btf);
1026
1027         return ret;
1028 }
1029
1030 #define MAX_BACKTRACE_SZ 128
1031 void crash_handler(int signum)
1032 {
1033         void *bt[MAX_BACKTRACE_SZ];
1034         size_t sz;
1035
1036         sz = backtrace(bt, ARRAY_SIZE(bt));
1037
1038         if (env.stdout)
1039                 stdio_restore();
1040         if (env.test) {
1041                 env.test_state->error_cnt++;
1042                 dump_test_log(env.test, env.test_state, true, false, NULL);
1043         }
1044         if (env.worker_id != -1)
1045                 fprintf(stderr, "[%d]: ", env.worker_id);
1046         fprintf(stderr, "Caught signal #%d!\nStack trace:\n", signum);
1047         backtrace_symbols_fd(bt, sz, STDERR_FILENO);
1048 }
1049
1050 static void sigint_handler(int signum)
1051 {
1052         int i;
1053
1054         for (i = 0; i < env.workers; i++)
1055                 if (env.worker_socks[i] > 0)
1056                         close(env.worker_socks[i]);
1057 }
1058
1059 static int current_test_idx;
1060 static pthread_mutex_t current_test_lock;
1061 static pthread_mutex_t stdout_output_lock;
1062
1063 static inline const char *str_msg(const struct msg *msg, char *buf)
1064 {
1065         switch (msg->type) {
1066         case MSG_DO_TEST:
1067                 sprintf(buf, "MSG_DO_TEST %d", msg->do_test.num);
1068                 break;
1069         case MSG_TEST_DONE:
1070                 sprintf(buf, "MSG_TEST_DONE %d (log: %d)",
1071                         msg->test_done.num,
1072                         msg->test_done.have_log);
1073                 break;
1074         case MSG_SUBTEST_DONE:
1075                 sprintf(buf, "MSG_SUBTEST_DONE %d (log: %d)",
1076                         msg->subtest_done.num,
1077                         msg->subtest_done.have_log);
1078                 break;
1079         case MSG_TEST_LOG:
1080                 sprintf(buf, "MSG_TEST_LOG (cnt: %zu, last: %d)",
1081                         strlen(msg->test_log.log_buf),
1082                         msg->test_log.is_last);
1083                 break;
1084         case MSG_EXIT:
1085                 sprintf(buf, "MSG_EXIT");
1086                 break;
1087         default:
1088                 sprintf(buf, "UNKNOWN");
1089                 break;
1090         }
1091
1092         return buf;
1093 }
1094
1095 static int send_message(int sock, const struct msg *msg)
1096 {
1097         char buf[256];
1098
1099         if (env.debug)
1100                 fprintf(stderr, "Sending msg: %s\n", str_msg(msg, buf));
1101         return send(sock, msg, sizeof(*msg), 0);
1102 }
1103
1104 static int recv_message(int sock, struct msg *msg)
1105 {
1106         int ret;
1107         char buf[256];
1108
1109         memset(msg, 0, sizeof(*msg));
1110         ret = recv(sock, msg, sizeof(*msg), 0);
1111         if (ret >= 0) {
1112                 if (env.debug)
1113                         fprintf(stderr, "Received msg: %s\n", str_msg(msg, buf));
1114         }
1115         return ret;
1116 }
1117
1118 static void run_one_test(int test_num)
1119 {
1120         struct prog_test_def *test = &prog_test_defs[test_num];
1121         struct test_state *state = &test_states[test_num];
1122
1123         env.test = test;
1124         env.test_state = state;
1125
1126         stdio_hijack(&state->log_buf, &state->log_cnt);
1127
1128         if (test->run_test)
1129                 test->run_test();
1130         else if (test->run_serial_test)
1131                 test->run_serial_test();
1132
1133         /* ensure last sub-test is finalized properly */
1134         if (env.subtest_state)
1135                 test__end_subtest();
1136
1137         state->tested = true;
1138
1139         if (verbose() && env.worker_id == -1)
1140                 print_test_result(test, state);
1141
1142         reset_affinity();
1143         restore_netns();
1144         if (test->need_cgroup_cleanup)
1145                 cleanup_cgroup_environment();
1146
1147         stdio_restore();
1148
1149         dump_test_log(test, state, false, false, NULL);
1150 }
1151
1152 struct dispatch_data {
1153         int worker_id;
1154         int sock_fd;
1155 };
1156
1157 static int read_prog_test_msg(int sock_fd, struct msg *msg, enum msg_type type)
1158 {
1159         if (recv_message(sock_fd, msg) < 0)
1160                 return 1;
1161
1162         if (msg->type != type) {
1163                 printf("%s: unexpected message type %d. expected %d\n", __func__, msg->type, type);
1164                 return 1;
1165         }
1166
1167         return 0;
1168 }
1169
1170 static int dispatch_thread_read_log(int sock_fd, char **log_buf, size_t *log_cnt)
1171 {
1172         FILE *log_fp = NULL;
1173         int result = 0;
1174
1175         log_fp = open_memstream(log_buf, log_cnt);
1176         if (!log_fp)
1177                 return 1;
1178
1179         while (true) {
1180                 struct msg msg;
1181
1182                 if (read_prog_test_msg(sock_fd, &msg, MSG_TEST_LOG)) {
1183                         result = 1;
1184                         goto out;
1185                 }
1186
1187                 fprintf(log_fp, "%s", msg.test_log.log_buf);
1188                 if (msg.test_log.is_last)
1189                         break;
1190         }
1191
1192 out:
1193         fclose(log_fp);
1194         log_fp = NULL;
1195         return result;
1196 }
1197
1198 static int dispatch_thread_send_subtests(int sock_fd, struct test_state *state)
1199 {
1200         struct msg msg;
1201         struct subtest_state *subtest_state;
1202         int subtest_num = state->subtest_num;
1203
1204         state->subtest_states = malloc(subtest_num * sizeof(*subtest_state));
1205
1206         for (int i = 0; i < subtest_num; i++) {
1207                 subtest_state = &state->subtest_states[i];
1208
1209                 memset(subtest_state, 0, sizeof(*subtest_state));
1210
1211                 if (read_prog_test_msg(sock_fd, &msg, MSG_SUBTEST_DONE))
1212                         return 1;
1213
1214                 subtest_state->name = strdup(msg.subtest_done.name);
1215                 subtest_state->error_cnt = msg.subtest_done.error_cnt;
1216                 subtest_state->skipped = msg.subtest_done.skipped;
1217                 subtest_state->filtered = msg.subtest_done.filtered;
1218
1219                 /* collect all logs */
1220                 if (msg.subtest_done.have_log)
1221                         if (dispatch_thread_read_log(sock_fd,
1222                                                      &subtest_state->log_buf,
1223                                                      &subtest_state->log_cnt))
1224                                 return 1;
1225         }
1226
1227         return 0;
1228 }
1229
1230 static void *dispatch_thread(void *ctx)
1231 {
1232         struct dispatch_data *data = ctx;
1233         int sock_fd;
1234
1235         sock_fd = data->sock_fd;
1236
1237         while (true) {
1238                 int test_to_run = -1;
1239                 struct prog_test_def *test;
1240                 struct test_state *state;
1241
1242                 /* grab a test */
1243                 {
1244                         pthread_mutex_lock(&current_test_lock);
1245
1246                         if (current_test_idx >= prog_test_cnt) {
1247                                 pthread_mutex_unlock(&current_test_lock);
1248                                 goto done;
1249                         }
1250
1251                         test = &prog_test_defs[current_test_idx];
1252                         test_to_run = current_test_idx;
1253                         current_test_idx++;
1254
1255                         pthread_mutex_unlock(&current_test_lock);
1256                 }
1257
1258                 if (!test->should_run || test->run_serial_test)
1259                         continue;
1260
1261                 /* run test through worker */
1262                 {
1263                         struct msg msg_do_test;
1264
1265                         memset(&msg_do_test, 0, sizeof(msg_do_test));
1266                         msg_do_test.type = MSG_DO_TEST;
1267                         msg_do_test.do_test.num = test_to_run;
1268                         if (send_message(sock_fd, &msg_do_test) < 0) {
1269                                 perror("Fail to send command");
1270                                 goto done;
1271                         }
1272                         env.worker_current_test[data->worker_id] = test_to_run;
1273                 }
1274
1275                 /* wait for test done */
1276                 do {
1277                         struct msg msg;
1278
1279                         if (read_prog_test_msg(sock_fd, &msg, MSG_TEST_DONE))
1280                                 goto error;
1281                         if (test_to_run != msg.test_done.num)
1282                                 goto error;
1283
1284                         state = &test_states[test_to_run];
1285                         state->tested = true;
1286                         state->error_cnt = msg.test_done.error_cnt;
1287                         state->skip_cnt = msg.test_done.skip_cnt;
1288                         state->sub_succ_cnt = msg.test_done.sub_succ_cnt;
1289                         state->subtest_num = msg.test_done.subtest_num;
1290
1291                         /* collect all logs */
1292                         if (msg.test_done.have_log) {
1293                                 if (dispatch_thread_read_log(sock_fd,
1294                                                              &state->log_buf,
1295                                                              &state->log_cnt))
1296                                         goto error;
1297                         }
1298
1299                         /* collect all subtests and subtest logs */
1300                         if (!state->subtest_num)
1301                                 break;
1302
1303                         if (dispatch_thread_send_subtests(sock_fd, state))
1304                                 goto error;
1305                 } while (false);
1306
1307                 pthread_mutex_lock(&stdout_output_lock);
1308                 dump_test_log(test, state, false, true, NULL);
1309                 pthread_mutex_unlock(&stdout_output_lock);
1310         } /* while (true) */
1311 error:
1312         if (env.debug)
1313                 fprintf(stderr, "[%d]: Protocol/IO error: %s.\n", data->worker_id, strerror(errno));
1314
1315 done:
1316         {
1317                 struct msg msg_exit;
1318
1319                 msg_exit.type = MSG_EXIT;
1320                 if (send_message(sock_fd, &msg_exit) < 0) {
1321                         if (env.debug)
1322                                 fprintf(stderr, "[%d]: send_message msg_exit: %s.\n",
1323                                         data->worker_id, strerror(errno));
1324                 }
1325         }
1326         return NULL;
1327 }
1328
1329 static void calculate_summary_and_print_errors(struct test_env *env)
1330 {
1331         int i;
1332         int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, skip_cnt = 0;
1333         json_writer_t *w = NULL;
1334
1335         for (i = 0; i < prog_test_cnt; i++) {
1336                 struct test_state *state = &test_states[i];
1337
1338                 if (!state->tested)
1339                         continue;
1340
1341                 sub_succ_cnt += state->sub_succ_cnt;
1342                 skip_cnt += state->skip_cnt;
1343
1344                 if (state->error_cnt)
1345                         fail_cnt++;
1346                 else
1347                         succ_cnt++;
1348         }
1349
1350         if (env->json) {
1351                 w = jsonw_new(env->json);
1352                 if (!w)
1353                         fprintf(env->stderr, "Failed to create new JSON stream.");
1354         }
1355
1356         if (w) {
1357                 jsonw_start_object(w);
1358                 jsonw_uint_field(w, "success", succ_cnt);
1359                 jsonw_uint_field(w, "success_subtest", sub_succ_cnt);
1360                 jsonw_uint_field(w, "skipped", skip_cnt);
1361                 jsonw_uint_field(w, "failed", fail_cnt);
1362                 jsonw_name(w, "results");
1363                 jsonw_start_array(w);
1364         }
1365
1366         /*
1367          * We only print error logs summary when there are failed tests and
1368          * verbose mode is not enabled. Otherwise, results may be incosistent.
1369          *
1370          */
1371         if (!verbose() && fail_cnt) {
1372                 printf("\nAll error logs:\n");
1373
1374                 /* print error logs again */
1375                 for (i = 0; i < prog_test_cnt; i++) {
1376                         struct prog_test_def *test = &prog_test_defs[i];
1377                         struct test_state *state = &test_states[i];
1378
1379                         if (!state->tested || !state->error_cnt)
1380                                 continue;
1381
1382                         dump_test_log(test, state, true, true, w);
1383                 }
1384         }
1385
1386         if (w) {
1387                 jsonw_end_array(w);
1388                 jsonw_end_object(w);
1389                 jsonw_destroy(&w);
1390         }
1391
1392         if (env->json)
1393                 fclose(env->json);
1394
1395         printf("Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n",
1396                succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt);
1397
1398         env->succ_cnt = succ_cnt;
1399         env->sub_succ_cnt = sub_succ_cnt;
1400         env->fail_cnt = fail_cnt;
1401         env->skip_cnt = skip_cnt;
1402 }
1403
1404 static void server_main(void)
1405 {
1406         pthread_t *dispatcher_threads;
1407         struct dispatch_data *data;
1408         struct sigaction sigact_int = {
1409                 .sa_handler = sigint_handler,
1410                 .sa_flags = SA_RESETHAND,
1411         };
1412         int i;
1413
1414         sigaction(SIGINT, &sigact_int, NULL);
1415
1416         dispatcher_threads = calloc(sizeof(pthread_t), env.workers);
1417         data = calloc(sizeof(struct dispatch_data), env.workers);
1418
1419         env.worker_current_test = calloc(sizeof(int), env.workers);
1420         for (i = 0; i < env.workers; i++) {
1421                 int rc;
1422
1423                 data[i].worker_id = i;
1424                 data[i].sock_fd = env.worker_socks[i];
1425                 rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]);
1426                 if (rc < 0) {
1427                         perror("Failed to launch dispatcher thread");
1428                         exit(EXIT_ERR_SETUP_INFRA);
1429                 }
1430         }
1431
1432         /* wait for all dispatcher to finish */
1433         for (i = 0; i < env.workers; i++) {
1434                 while (true) {
1435                         int ret = pthread_tryjoin_np(dispatcher_threads[i], NULL);
1436
1437                         if (!ret) {
1438                                 break;
1439                         } else if (ret == EBUSY) {
1440                                 if (env.debug)
1441                                         fprintf(stderr, "Still waiting for thread %d (test %d).\n",
1442                                                 i,  env.worker_current_test[i] + 1);
1443                                 usleep(1000 * 1000);
1444                                 continue;
1445                         } else {
1446                                 fprintf(stderr, "Unexpected error joining dispatcher thread: %d", ret);
1447                                 break;
1448                         }
1449                 }
1450         }
1451         free(dispatcher_threads);
1452         free(env.worker_current_test);
1453         free(data);
1454
1455         /* run serial tests */
1456         save_netns();
1457
1458         for (int i = 0; i < prog_test_cnt; i++) {
1459                 struct prog_test_def *test = &prog_test_defs[i];
1460
1461                 if (!test->should_run || !test->run_serial_test)
1462                         continue;
1463
1464                 run_one_test(i);
1465         }
1466
1467         /* generate summary */
1468         fflush(stderr);
1469         fflush(stdout);
1470
1471         calculate_summary_and_print_errors(&env);
1472
1473         /* reap all workers */
1474         for (i = 0; i < env.workers; i++) {
1475                 int wstatus, pid;
1476
1477                 pid = waitpid(env.worker_pids[i], &wstatus, 0);
1478                 if (pid != env.worker_pids[i])
1479                         perror("Unable to reap worker");
1480         }
1481 }
1482
1483 static void worker_main_send_log(int sock, char *log_buf, size_t log_cnt)
1484 {
1485         char *src;
1486         size_t slen;
1487
1488         src = log_buf;
1489         slen = log_cnt;
1490         while (slen) {
1491                 struct msg msg_log;
1492                 char *dest;
1493                 size_t len;
1494
1495                 memset(&msg_log, 0, sizeof(msg_log));
1496                 msg_log.type = MSG_TEST_LOG;
1497                 dest = msg_log.test_log.log_buf;
1498                 len = slen >= MAX_LOG_TRUNK_SIZE ? MAX_LOG_TRUNK_SIZE : slen;
1499                 memcpy(dest, src, len);
1500
1501                 src += len;
1502                 slen -= len;
1503                 if (!slen)
1504                         msg_log.test_log.is_last = true;
1505
1506                 assert(send_message(sock, &msg_log) >= 0);
1507         }
1508 }
1509
1510 static void free_subtest_state(struct subtest_state *state)
1511 {
1512         if (state->log_buf) {
1513                 free(state->log_buf);
1514                 state->log_buf = NULL;
1515                 state->log_cnt = 0;
1516         }
1517         free(state->name);
1518         state->name = NULL;
1519 }
1520
1521 static int worker_main_send_subtests(int sock, struct test_state *state)
1522 {
1523         int i, result = 0;
1524         struct msg msg;
1525         struct subtest_state *subtest_state;
1526
1527         memset(&msg, 0, sizeof(msg));
1528         msg.type = MSG_SUBTEST_DONE;
1529
1530         for (i = 0; i < state->subtest_num; i++) {
1531                 subtest_state = &state->subtest_states[i];
1532
1533                 msg.subtest_done.num = i;
1534
1535                 strncpy(msg.subtest_done.name, subtest_state->name, MAX_SUBTEST_NAME);
1536
1537                 msg.subtest_done.error_cnt = subtest_state->error_cnt;
1538                 msg.subtest_done.skipped = subtest_state->skipped;
1539                 msg.subtest_done.filtered = subtest_state->filtered;
1540                 msg.subtest_done.have_log = false;
1541
1542                 if (verbose() || state->force_log || subtest_state->error_cnt) {
1543                         if (subtest_state->log_cnt)
1544                                 msg.subtest_done.have_log = true;
1545                 }
1546
1547                 if (send_message(sock, &msg) < 0) {
1548                         perror("Fail to send message done");
1549                         result = 1;
1550                         goto out;
1551                 }
1552
1553                 /* send logs */
1554                 if (msg.subtest_done.have_log)
1555                         worker_main_send_log(sock, subtest_state->log_buf, subtest_state->log_cnt);
1556
1557                 free_subtest_state(subtest_state);
1558                 free(subtest_state->name);
1559         }
1560
1561 out:
1562         for (; i < state->subtest_num; i++)
1563                 free_subtest_state(&state->subtest_states[i]);
1564         free(state->subtest_states);
1565         return result;
1566 }
1567
1568 static int worker_main(int sock)
1569 {
1570         save_netns();
1571
1572         while (true) {
1573                 /* receive command */
1574                 struct msg msg;
1575
1576                 if (recv_message(sock, &msg) < 0)
1577                         goto out;
1578
1579                 switch (msg.type) {
1580                 case MSG_EXIT:
1581                         if (env.debug)
1582                                 fprintf(stderr, "[%d]: worker exit.\n",
1583                                         env.worker_id);
1584                         goto out;
1585                 case MSG_DO_TEST: {
1586                         int test_to_run = msg.do_test.num;
1587                         struct prog_test_def *test = &prog_test_defs[test_to_run];
1588                         struct test_state *state = &test_states[test_to_run];
1589                         struct msg msg;
1590
1591                         if (env.debug)
1592                                 fprintf(stderr, "[%d]: #%d:%s running.\n",
1593                                         env.worker_id,
1594                                         test_to_run + 1,
1595                                         test->test_name);
1596
1597                         run_one_test(test_to_run);
1598
1599                         memset(&msg, 0, sizeof(msg));
1600                         msg.type = MSG_TEST_DONE;
1601                         msg.test_done.num = test_to_run;
1602                         msg.test_done.error_cnt = state->error_cnt;
1603                         msg.test_done.skip_cnt = state->skip_cnt;
1604                         msg.test_done.sub_succ_cnt = state->sub_succ_cnt;
1605                         msg.test_done.subtest_num = state->subtest_num;
1606                         msg.test_done.have_log = false;
1607
1608                         if (verbose() || state->force_log || state->error_cnt) {
1609                                 if (state->log_cnt)
1610                                         msg.test_done.have_log = true;
1611                         }
1612                         if (send_message(sock, &msg) < 0) {
1613                                 perror("Fail to send message done");
1614                                 goto out;
1615                         }
1616
1617                         /* send logs */
1618                         if (msg.test_done.have_log)
1619                                 worker_main_send_log(sock, state->log_buf, state->log_cnt);
1620
1621                         if (state->log_buf) {
1622                                 free(state->log_buf);
1623                                 state->log_buf = NULL;
1624                                 state->log_cnt = 0;
1625                         }
1626
1627                         if (state->subtest_num)
1628                                 if (worker_main_send_subtests(sock, state))
1629                                         goto out;
1630
1631                         if (env.debug)
1632                                 fprintf(stderr, "[%d]: #%d:%s done.\n",
1633                                         env.worker_id,
1634                                         test_to_run + 1,
1635                                         test->test_name);
1636                         break;
1637                 } /* case MSG_DO_TEST */
1638                 default:
1639                         if (env.debug)
1640                                 fprintf(stderr, "[%d]: unknown message.\n",  env.worker_id);
1641                         return -1;
1642                 }
1643         }
1644 out:
1645         return 0;
1646 }
1647
1648 static void free_test_states(void)
1649 {
1650         int i, j;
1651
1652         for (i = 0; i < ARRAY_SIZE(prog_test_defs); i++) {
1653                 struct test_state *test_state = &test_states[i];
1654
1655                 for (j = 0; j < test_state->subtest_num; j++)
1656                         free_subtest_state(&test_state->subtest_states[j]);
1657
1658                 free(test_state->subtest_states);
1659                 free(test_state->log_buf);
1660                 test_state->subtest_states = NULL;
1661                 test_state->log_buf = NULL;
1662         }
1663 }
1664
1665 int main(int argc, char **argv)
1666 {
1667         static const struct argp argp = {
1668                 .options = opts,
1669                 .parser = parse_arg,
1670                 .doc = argp_program_doc,
1671         };
1672         struct sigaction sigact = {
1673                 .sa_handler = crash_handler,
1674                 .sa_flags = SA_RESETHAND,
1675                 };
1676         int err, i;
1677
1678         sigaction(SIGSEGV, &sigact, NULL);
1679
1680         err = argp_parse(&argp, argc, argv, 0, NULL, &env);
1681         if (err)
1682                 return err;
1683
1684         err = cd_flavor_subdir(argv[0]);
1685         if (err)
1686                 return err;
1687
1688         /* Use libbpf 1.0 API mode */
1689         libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1690         libbpf_set_print(libbpf_print_fn);
1691
1692         srand(time(NULL));
1693
1694         env.jit_enabled = is_jit_enabled();
1695         env.nr_cpus = libbpf_num_possible_cpus();
1696         if (env.nr_cpus < 0) {
1697                 fprintf(stderr, "Failed to get number of CPUs: %d!\n",
1698                         env.nr_cpus);
1699                 return -1;
1700         }
1701
1702         env.stdout = stdout;
1703         env.stderr = stderr;
1704
1705         env.has_testmod = true;
1706         if (!env.list_test_names && load_bpf_testmod()) {
1707                 fprintf(env.stderr, "WARNING! Selftests relying on bpf_testmod.ko will be skipped.\n");
1708                 env.has_testmod = false;
1709         }
1710
1711         /* initializing tests */
1712         for (i = 0; i < prog_test_cnt; i++) {
1713                 struct prog_test_def *test = &prog_test_defs[i];
1714
1715                 test->test_num = i + 1;
1716                 test->should_run = should_run(&env.test_selector,
1717                                               test->test_num, test->test_name);
1718
1719                 if ((test->run_test == NULL && test->run_serial_test == NULL) ||
1720                     (test->run_test != NULL && test->run_serial_test != NULL)) {
1721                         fprintf(stderr, "Test %d:%s must have either test_%s() or serial_test_%sl() defined.\n",
1722                                 test->test_num, test->test_name, test->test_name, test->test_name);
1723                         exit(EXIT_ERR_SETUP_INFRA);
1724                 }
1725         }
1726
1727         /* ignore workers if we are just listing */
1728         if (env.get_test_cnt || env.list_test_names)
1729                 env.workers = 0;
1730
1731         /* launch workers if requested */
1732         env.worker_id = -1; /* main process */
1733         if (env.workers) {
1734                 env.worker_pids = calloc(sizeof(__pid_t), env.workers);
1735                 env.worker_socks = calloc(sizeof(int), env.workers);
1736                 if (env.debug)
1737                         fprintf(stdout, "Launching %d workers.\n", env.workers);
1738                 for (i = 0; i < env.workers; i++) {
1739                         int sv[2];
1740                         pid_t pid;
1741
1742                         if (socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, sv) < 0) {
1743                                 perror("Fail to create worker socket");
1744                                 return -1;
1745                         }
1746                         pid = fork();
1747                         if (pid < 0) {
1748                                 perror("Failed to fork worker");
1749                                 return -1;
1750                         } else if (pid != 0) { /* main process */
1751                                 close(sv[1]);
1752                                 env.worker_pids[i] = pid;
1753                                 env.worker_socks[i] = sv[0];
1754                         } else { /* inside each worker process */
1755                                 close(sv[0]);
1756                                 env.worker_id = i;
1757                                 return worker_main(sv[1]);
1758                         }
1759                 }
1760
1761                 if (env.worker_id == -1) {
1762                         server_main();
1763                         goto out;
1764                 }
1765         }
1766
1767         /* The rest of the main process */
1768
1769         /* on single mode */
1770         save_netns();
1771
1772         for (i = 0; i < prog_test_cnt; i++) {
1773                 struct prog_test_def *test = &prog_test_defs[i];
1774
1775                 if (!test->should_run)
1776                         continue;
1777
1778                 if (env.get_test_cnt) {
1779                         env.succ_cnt++;
1780                         continue;
1781                 }
1782
1783                 if (env.list_test_names) {
1784                         fprintf(env.stdout, "%s\n", test->test_name);
1785                         env.succ_cnt++;
1786                         continue;
1787                 }
1788
1789                 run_one_test(i);
1790         }
1791
1792         if (env.get_test_cnt) {
1793                 printf("%d\n", env.succ_cnt);
1794                 goto out;
1795         }
1796
1797         if (env.list_test_names)
1798                 goto out;
1799
1800         calculate_summary_and_print_errors(&env);
1801
1802         close(env.saved_netns_fd);
1803 out:
1804         if (!env.list_test_names && env.has_testmod)
1805                 unload_bpf_testmod();
1806
1807         free_test_selector(&env.test_selector);
1808         free_test_selector(&env.subtest_selector);
1809         free_test_states();
1810
1811         if (env.succ_cnt + env.fail_cnt + env.skip_cnt == 0)
1812                 return EXIT_NO_TEST;
1813
1814         return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
1815 }