tools/testing/selftests/bpf/test_progs.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2017 Facebook
   3  */
   4 #define _GNU_SOURCE
   5 #include "test_progs.h"
   6 #include "testing_helpers.h"
   7 #include "cgroup_helpers.h"
   8 #include <argp.h>
   9 #include <pthread.h>
  10 #include <sched.h>
  11 #include <signal.h>
  12 #include <string.h>
  13 #include <execinfo.h> /* backtrace */
  14 #include <sys/sysinfo.h> /* get_nprocs */
  15 #include <netinet/in.h>
  16 #include <sys/select.h>
  17 #include <sys/socket.h>
  18 #include <sys/un.h>
  19 #include <bpf/btf.h>
  20 #include "json_writer.h"
  21
  22 static bool verbose(void)
  23 {
  24         return env.verbosity > VERBOSE_NONE;
  25 }
  26
  27 static void stdio_hijack_init(char **log_buf, size_t *log_cnt)
  28 {
  29 #ifdef __GLIBC__
  30         if (verbose() && env.worker_id == -1) {
  31                 /* nothing to do, output to stdout by default */
  32                 return;
  33         }
  34
  35         fflush(stdout);
  36         fflush(stderr);
  37
  38         stdout = open_memstream(log_buf, log_cnt);
  39         if (!stdout) {
  40                 stdout = env.stdout;
  41                 perror("open_memstream");
  42                 return;
  43         }
  44
  45         if (env.subtest_state)
  46                 env.subtest_state->stdout = stdout;
  47         else
  48                 env.test_state->stdout = stdout;
  49
  50         stderr = stdout;
  51 #endif
  52 }
  53
  54 static void stdio_hijack(char **log_buf, size_t *log_cnt)
  55 {
  56 #ifdef __GLIBC__
  57         if (verbose() && env.worker_id == -1) {
  58                 /* nothing to do, output to stdout by default */
  59                 return;
  60         }
  61
  62         env.stdout = stdout;
  63         env.stderr = stderr;
  64
  65         stdio_hijack_init(log_buf, log_cnt);
  66 #endif
  67 }
  68
  69 static void stdio_restore_cleanup(void)
  70 {
  71 #ifdef __GLIBC__
  72         if (verbose() && env.worker_id == -1) {
  73                 /* nothing to do, output to stdout by default */
  74                 return;
  75         }
  76
  77         fflush(stdout);
  78
  79         if (env.subtest_state) {
  80                 fclose(env.subtest_state->stdout);
  81                 env.subtest_state->stdout = NULL;
  82                 stdout = env.test_state->stdout;
  83                 stderr = env.test_state->stdout;
  84         } else {
  85                 fclose(env.test_state->stdout);
  86                 env.test_state->stdout = NULL;
  87         }
  88 #endif
  89 }
  90
  91 static void stdio_restore(void)
  92 {
  93 #ifdef __GLIBC__
  94         if (verbose() && env.worker_id == -1) {
  95                 /* nothing to do, output to stdout by default */
  96                 return;
  97         }
  98
  99         if (stdout == env.stdout)
 100                 return;
 101
 102         stdio_restore_cleanup();
 103
 104         stdout = env.stdout;
 105         stderr = env.stderr;
 106 #endif
 107 }
 108
 109 /* Adapted from perf/util/string.c */
 110 static bool glob_match(const char *str, const char *pat)
 111 {
 112         while (*str && *pat && *pat != '*') {
 113                 if (*str != *pat)
 114                         return false;
 115                 str++;
 116                 pat++;
 117         }
 118         /* Check wild card */
 119         if (*pat == '*') {
 120                 while (*pat == '*')
 121                         pat++;
 122                 if (!*pat) /* Tail wild card matches all */
 123                         return true;
 124                 while (*str)
 125                         if (glob_match(str++, pat))
 126                                 return true;
 127         }
 128         return !*str && !*pat;
 129 }
 130
 131 #define EXIT_NO_TEST            2
 132 #define EXIT_ERR_SETUP_INFRA    3
 133
 134 /* defined in test_progs.h */
 135 struct test_env env = {};
 136
 137 struct prog_test_def {
 138         const char *test_name;
 139         int test_num;
 140         void (*run_test)(void);
 141         void (*run_serial_test)(void);
 142         bool should_run;
 143         bool need_cgroup_cleanup;
 144 };
 145
 146 /* Override C runtime library's usleep() implementation to ensure nanosleep()
 147  * is always called. Usleep is frequently used in selftests as a way to
 148  * trigger kprobe and tracepoints.
 149  */
 150 int usleep(useconds_t usec)
 151 {
 152         struct timespec ts = {
 153                 .tv_sec = usec / 1000000,
 154                 .tv_nsec = (usec % 1000000) * 1000,
 155         };
 156
 157         return syscall(__NR_nanosleep, &ts, NULL);
 158 }
 159
 160 static bool should_run(struct test_selector *sel, int num, const char *name)
 161 {
 162         int i;
 163
 164         for (i = 0; i < sel->blacklist.cnt; i++) {
 165                 if (glob_match(name, sel->blacklist.tests[i].name) &&
 166                     !sel->blacklist.tests[i].subtest_cnt)
 167                         return false;
 168         }
 169
 170         for (i = 0; i < sel->whitelist.cnt; i++) {
 171                 if (glob_match(name, sel->whitelist.tests[i].name))
 172                         return true;
 173         }
 174
 175         if (!sel->whitelist.cnt && !sel->num_set)
 176                 return true;
 177
 178         return num < sel->num_set_len && sel->num_set[num];
 179 }
 180
 181 static bool should_run_subtest(struct test_selector *sel,
 182                                struct test_selector *subtest_sel,
 183                                int subtest_num,
 184                                const char *test_name,
 185                                const char *subtest_name)
 186 {
 187         int i, j;
 188
 189         for (i = 0; i < sel->blacklist.cnt; i++) {
 190                 if (glob_match(test_name, sel->blacklist.tests[i].name)) {
 191                         if (!sel->blacklist.tests[i].subtest_cnt)
 192                                 return false;
 193
 194                         for (j = 0; j < sel->blacklist.tests[i].subtest_cnt; j++) {
 195                                 if (glob_match(subtest_name,
 196                                                sel->blacklist.tests[i].subtests[j]))
 197                                         return false;
 198                         }
 199                 }
 200         }
 201
 202         for (i = 0; i < sel->whitelist.cnt; i++) {
 203                 if (glob_match(test_name, sel->whitelist.tests[i].name)) {
 204                         if (!sel->whitelist.tests[i].subtest_cnt)
 205                                 return true;
 206
 207                         for (j = 0; j < sel->whitelist.tests[i].subtest_cnt; j++) {
 208                                 if (glob_match(subtest_name,
 209                                                sel->whitelist.tests[i].subtests[j]))
 210                                         return true;
 211                         }
 212                 }
 213         }
 214
 215         if (!sel->whitelist.cnt && !subtest_sel->num_set)
 216                 return true;
 217
 218         return subtest_num < subtest_sel->num_set_len && subtest_sel->num_set[subtest_num];
 219 }
 220
 221 static char *test_result(bool failed, bool skipped)
 222 {
 223         return failed ? "FAIL" : (skipped ? "SKIP" : "OK");
 224 }
 225
 226 #define TEST_NUM_WIDTH 7
 227
 228 static void print_test_result(const struct prog_test_def *test, const struct test_state *test_state)
 229 {
 230         int skipped_cnt = test_state->skip_cnt;
 231         int subtests_cnt = test_state->subtest_num;
 232
 233         fprintf(env.stdout, "#%-*d %s:", TEST_NUM_WIDTH, test->test_num, test->test_name);
 234         if (test_state->error_cnt)
 235                 fprintf(env.stdout, "FAIL");
 236         else if (!skipped_cnt)
 237                 fprintf(env.stdout, "OK");
 238         else if (skipped_cnt == subtests_cnt || !subtests_cnt)
 239                 fprintf(env.stdout, "SKIP");
 240         else
 241                 fprintf(env.stdout, "OK (SKIP: %d/%d)", skipped_cnt, subtests_cnt);
 242
 243         fprintf(env.stdout, "\n");
 244 }
 245
 246 static void print_test_log(char *log_buf, size_t log_cnt)
 247 {
 248         log_buf[log_cnt] = '\0';
 249         fprintf(env.stdout, "%s", log_buf);
 250         if (log_buf[log_cnt - 1] != '\n')
 251                 fprintf(env.stdout, "\n");
 252 }
 253
 254 static void print_subtest_name(int test_num, int subtest_num,
 255                                const char *test_name, char *subtest_name,
 256                                char *result)
 257 {
 258         char test_num_str[32];
 259
 260         snprintf(test_num_str, sizeof(test_num_str), "%d/%d", test_num, subtest_num);
 261
 262         fprintf(env.stdout, "#%-*s %s/%s",
 263                 TEST_NUM_WIDTH, test_num_str,
 264                 test_name, subtest_name);
 265
 266         if (result)
 267                 fprintf(env.stdout, ":%s", result);
 268
 269         fprintf(env.stdout, "\n");
 270 }
 271
 272 static void jsonw_write_log_message(json_writer_t *w, char *log_buf, size_t log_cnt)
 273 {
 274         /* open_memstream (from stdio_hijack_init) ensures that log_bug is terminated by a
 275          * null byte. Yet in parallel mode, log_buf will be NULL if there is no message.
 276          */
 277         if (log_cnt) {
 278                 jsonw_string_field(w, "message", log_buf);
 279         } else {
 280                 jsonw_string_field(w, "message", "");
 281         }
 282 }
 283
 284 static void dump_test_log(const struct prog_test_def *test,
 285                           const struct test_state *test_state,
 286                           bool skip_ok_subtests,
 287                           bool par_exec_result,
 288                           json_writer_t *w)
 289 {
 290         bool test_failed = test_state->error_cnt > 0;
 291         bool force_log = test_state->force_log;
 292         bool print_test = verbose() || force_log || test_failed;
 293         int i;
 294         struct subtest_state *subtest_state;
 295         bool subtest_failed;
 296         bool subtest_filtered;
 297         bool print_subtest;
 298
 299         /* we do not print anything in the worker thread */
 300         if (env.worker_id != -1)
 301                 return;
 302
 303         /* there is nothing to print when verbose log is used and execution
 304          * is not in parallel mode
 305          */
 306         if (verbose() && !par_exec_result)
 307                 return;
 308
 309         if (test_state->log_cnt && print_test)
 310                 print_test_log(test_state->log_buf, test_state->log_cnt);
 311
 312         if (w && print_test) {
 313                 jsonw_start_object(w);
 314                 jsonw_string_field(w, "name", test->test_name);
 315                 jsonw_uint_field(w, "number", test->test_num);
 316                 jsonw_write_log_message(w, test_state->log_buf, test_state->log_cnt);
 317                 jsonw_bool_field(w, "failed", test_failed);
 318                 jsonw_name(w, "subtests");
 319                 jsonw_start_array(w);
 320         }
 321
 322         for (i = 0; i < test_state->subtest_num; i++) {
 323                 subtest_state = &test_state->subtest_states[i];
 324                 subtest_failed = subtest_state->error_cnt;
 325                 subtest_filtered = subtest_state->filtered;
 326                 print_subtest = verbose() || force_log || subtest_failed;
 327
 328                 if ((skip_ok_subtests && !subtest_failed) || subtest_filtered)
 329                         continue;
 330
 331                 if (subtest_state->log_cnt && print_subtest) {
 332                         print_test_log(subtest_state->log_buf,
 333                                        subtest_state->log_cnt);
 334                 }
 335
 336                 print_subtest_name(test->test_num, i + 1,
 337                                    test->test_name, subtest_state->name,
 338                                    test_result(subtest_state->error_cnt,
 339                                                subtest_state->skipped));
 340
 341                 if (w && print_subtest) {
 342                         jsonw_start_object(w);
 343                         jsonw_string_field(w, "name", subtest_state->name);
 344                         jsonw_uint_field(w, "number", i+1);
 345                         jsonw_write_log_message(w, subtest_state->log_buf, subtest_state->log_cnt);
 346                         jsonw_bool_field(w, "failed", subtest_failed);
 347                         jsonw_end_object(w);
 348                 }
 349         }
 350
 351         if (w && print_test) {
 352                 jsonw_end_array(w);
 353                 jsonw_end_object(w);
 354         }
 355
 356         print_test_result(test, test_state);
 357 }
 358
 359 static void stdio_restore(void);
 360
 361 /* A bunch of tests set custom affinity per-thread and/or per-process. Reset
 362  * it after each test/sub-test.
 363  */
 364 static void reset_affinity(void)
 365 {
 366         cpu_set_t cpuset;
 367         int i, err;
 368
 369         CPU_ZERO(&cpuset);
 370         for (i = 0; i < env.nr_cpus; i++)
 371                 CPU_SET(i, &cpuset);
 372
 373         err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
 374         if (err < 0) {
 375                 stdio_restore();
 376                 fprintf(stderr, "Failed to reset process affinity: %d!\n", err);
 377                 exit(EXIT_ERR_SETUP_INFRA);
 378         }
 379         err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
 380         if (err < 0) {
 381                 stdio_restore();
 382                 fprintf(stderr, "Failed to reset thread affinity: %d!\n", err);
 383                 exit(EXIT_ERR_SETUP_INFRA);
 384         }
 385 }
 386
 387 static void save_netns(void)
 388 {
 389         env.saved_netns_fd = open("/proc/self/ns/net", O_RDONLY);
 390         if (env.saved_netns_fd == -1) {
 391                 perror("open(/proc/self/ns/net)");
 392                 exit(EXIT_ERR_SETUP_INFRA);
 393         }
 394 }
 395
 396 static void restore_netns(void)
 397 {
 398         if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) {
 399                 stdio_restore();
 400                 perror("setns(CLONE_NEWNS)");
 401                 exit(EXIT_ERR_SETUP_INFRA);
 402         }
 403 }
 404
 405 void test__end_subtest(void)
 406 {
 407         struct prog_test_def *test = env.test;
 408         struct test_state *test_state = env.test_state;
 409         struct subtest_state *subtest_state = env.subtest_state;
 410
 411         if (subtest_state->error_cnt) {
 412                 test_state->error_cnt++;
 413         } else {
 414                 if (!subtest_state->skipped)
 415                         test_state->sub_succ_cnt++;
 416                 else
 417                         test_state->skip_cnt++;
 418         }
 419
 420         if (verbose() && !env.workers)
 421                 print_subtest_name(test->test_num, test_state->subtest_num,
 422                                    test->test_name, subtest_state->name,
 423                                    test_result(subtest_state->error_cnt,
 424                                                subtest_state->skipped));
 425
 426         stdio_restore_cleanup();
 427         env.subtest_state = NULL;
 428 }
 429
 430 bool test__start_subtest(const char *subtest_name)
 431 {
 432         struct prog_test_def *test = env.test;
 433         struct test_state *state = env.test_state;
 434         struct subtest_state *subtest_state;
 435         size_t sub_state_size = sizeof(*subtest_state);
 436
 437         if (env.subtest_state)
 438                 test__end_subtest();
 439
 440         state->subtest_num++;
 441         state->subtest_states =
 442                 realloc(state->subtest_states,
 443                         state->subtest_num * sub_state_size);
 444         if (!state->subtest_states) {
 445                 fprintf(stderr, "Not enough memory to allocate subtest result\n");
 446                 return false;
 447         }
 448
 449         subtest_state = &state->subtest_states[state->subtest_num - 1];
 450
 451         memset(subtest_state, 0, sub_state_size);
 452
 453         if (!subtest_name || !subtest_name[0]) {
 454                 fprintf(env.stderr,
 455                         "Subtest #%d didn't provide sub-test name!\n",
 456                         state->subtest_num);
 457                 return false;
 458         }
 459
 460         subtest_state->name = strdup(subtest_name);
 461         if (!subtest_state->name) {
 462                 fprintf(env.stderr,
 463                         "Subtest #%d: failed to copy subtest name!\n",
 464                         state->subtest_num);
 465                 return false;
 466         }
 467
 468         if (!should_run_subtest(&env.test_selector,
 469                                 &env.subtest_selector,
 470                                 state->subtest_num,
 471                                 test->test_name,
 472                                 subtest_name)) {
 473                 subtest_state->filtered = true;
 474                 return false;
 475         }
 476
 477         env.subtest_state = subtest_state;
 478         stdio_hijack_init(&subtest_state->log_buf, &subtest_state->log_cnt);
 479
 480         return true;
 481 }
 482
 483 void test__force_log(void)
 484 {
 485         env.test_state->force_log = true;
 486 }
 487
 488 void test__skip(void)
 489 {
 490         if (env.subtest_state)
 491                 env.subtest_state->skipped = true;
 492         else
 493                 env.test_state->skip_cnt++;
 494 }
 495
 496 void test__fail(void)
 497 {
 498         if (env.subtest_state)
 499                 env.subtest_state->error_cnt++;
 500         else
 501                 env.test_state->error_cnt++;
 502 }
 503
 504 int test__join_cgroup(const char *path)
 505 {
 506         int fd;
 507
 508         if (!env.test->need_cgroup_cleanup) {
 509                 if (setup_cgroup_environment()) {
 510                         fprintf(stderr,
 511                                 "#%d %s: Failed to setup cgroup environment\n",
 512                                 env.test->test_num, env.test->test_name);
 513                         return -1;
 514                 }
 515
 516                 env.test->need_cgroup_cleanup = true;
 517         }
 518
 519         fd = create_and_get_cgroup(path);
 520         if (fd < 0) {
 521                 fprintf(stderr,
 522                         "#%d %s: Failed to create cgroup '%s' (errno=%d)\n",
 523                         env.test->test_num, env.test->test_name, path, errno);
 524                 return fd;
 525         }
 526
 527         if (join_cgroup(path)) {
 528                 fprintf(stderr,
 529                         "#%d %s: Failed to join cgroup '%s' (errno=%d)\n",
 530                         env.test->test_num, env.test->test_name, path, errno);
 531                 return -1;
 532         }
 533
 534         return fd;
 535 }
 536
 537 int bpf_find_map(const char *test, struct bpf_object *obj, const char *name)
 538 {
 539         struct bpf_map *map;
 540
 541         map = bpf_object__find_map_by_name(obj, name);
 542         if (!map) {
 543                 fprintf(stdout, "%s:FAIL:map '%s' not found\n", test, name);
 544                 test__fail();
 545                 return -1;
 546         }
 547         return bpf_map__fd(map);
 548 }
 549
 550 int compare_map_keys(int map1_fd, int map2_fd)
 551 {
 552         __u32 key, next_key;
 553         char val_buf[PERF_MAX_STACK_DEPTH *
 554                      sizeof(struct bpf_stack_build_id)];
 555         int err;
 556
 557         err = bpf_map_get_next_key(map1_fd, NULL, &key);
 558         if (err)
 559                 return err;
 560         err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
 561         if (err)
 562                 return err;
 563
 564         while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
 565                 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
 566                 if (err)
 567                         return err;
 568
 569                 key = next_key;
 570         }
 571         if (errno != ENOENT)
 572                 return -1;
 573
 574         return 0;
 575 }
 576
 577 int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
 578 {
 579         __u32 key, next_key, *cur_key_p, *next_key_p;
 580         char *val_buf1, *val_buf2;
 581         int i, err = 0;
 582
 583         val_buf1 = malloc(stack_trace_len);
 584         val_buf2 = malloc(stack_trace_len);
 585         cur_key_p = NULL;
 586         next_key_p = &key;
 587         while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
 588                 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
 589                 if (err)
 590                         goto out;
 591                 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
 592                 if (err)
 593                         goto out;
 594                 for (i = 0; i < stack_trace_len; i++) {
 595                         if (val_buf1[i] != val_buf2[i]) {
 596                                 err = -1;
 597                                 goto out;
 598                         }
 599                 }
 600                 key = *next_key_p;
 601                 cur_key_p = &key;
 602                 next_key_p = &next_key;
 603         }
 604         if (errno != ENOENT)
 605                 err = -1;
 606
 607 out:
 608         free(val_buf1);
 609         free(val_buf2);
 610         return err;
 611 }
 612
 613 /* extern declarations for test funcs */
 614 #define DEFINE_TEST(name)                               \
 615         extern void test_##name(void) __weak;           \
 616         extern void serial_test_##name(void) __weak;
 617 #include <prog_tests/tests.h>
 618 #undef DEFINE_TEST
 619
 620 static struct prog_test_def prog_test_defs[] = {
 621 #define DEFINE_TEST(name) {                     \
 622         .test_name = #name,                     \
 623         .run_test = &test_##name,               \
 624         .run_serial_test = &serial_test_##name, \
 625 },
 626 #include <prog_tests/tests.h>
 627 #undef DEFINE_TEST
 628 };
 629
 630 static const int prog_test_cnt = ARRAY_SIZE(prog_test_defs);
 631
 632 static struct test_state test_states[ARRAY_SIZE(prog_test_defs)];
 633
 634 const char *argp_program_version = "test_progs 0.1";
 635 const char *argp_program_bug_address = "<[email protected]>";
 636 static const char argp_program_doc[] =
 637 "BPF selftests test runner\v"
 638 "Options accepting the NAMES parameter take either a comma-separated list\n"
 639 "of test names, or a filename prefixed with @. The file contains one name\n"
 640 "(or wildcard pattern) per line, and comments beginning with # are ignored.\n"
 641 "\n"
 642 "These options can be passed repeatedly to read multiple files.\n";
 643
 644 enum ARG_KEYS {
 645         ARG_TEST_NUM = 'n',
 646         ARG_TEST_NAME = 't',
 647         ARG_TEST_NAME_BLACKLIST = 'b',
 648         ARG_VERIFIER_STATS = 's',
 649         ARG_VERBOSE = 'v',
 650         ARG_GET_TEST_CNT = 'c',
 651         ARG_LIST_TEST_NAMES = 'l',
 652         ARG_TEST_NAME_GLOB_ALLOWLIST = 'a',
 653         ARG_TEST_NAME_GLOB_DENYLIST = 'd',
 654         ARG_NUM_WORKERS = 'j',
 655         ARG_DEBUG = -1,
 656         ARG_JSON_SUMMARY = 'J'
 657 };
 658
 659 static const struct argp_option opts[] = {
 660         { "num", ARG_TEST_NUM, "NUM", 0,
 661           "Run test number NUM only " },
 662         { "name", ARG_TEST_NAME, "NAMES", 0,
 663           "Run tests with names containing any string from NAMES list" },
 664         { "name-blacklist", ARG_TEST_NAME_BLACKLIST, "NAMES", 0,
 665           "Don't run tests with names containing any string from NAMES list" },
 666         { "verifier-stats", ARG_VERIFIER_STATS, NULL, 0,
 667           "Output verifier statistics", },
 668         { "verbose", ARG_VERBOSE, "LEVEL", OPTION_ARG_OPTIONAL,
 669           "Verbose output (use -vv or -vvv for progressively verbose output)" },
 670         { "count", ARG_GET_TEST_CNT, NULL, 0,
 671           "Get number of selected top-level tests " },
 672         { "list", ARG_LIST_TEST_NAMES, NULL, 0,
 673           "List test names that would run (without running them) " },
 674         { "allow", ARG_TEST_NAME_GLOB_ALLOWLIST, "NAMES", 0,
 675           "Run tests with name matching the pattern (supports '*' wildcard)." },
 676         { "deny", ARG_TEST_NAME_GLOB_DENYLIST, "NAMES", 0,
 677           "Don't run tests with name matching the pattern (supports '*' wildcard)." },
 678         { "workers", ARG_NUM_WORKERS, "WORKERS", OPTION_ARG_OPTIONAL,
 679           "Number of workers to run in parallel, default to number of cpus." },
 680         { "debug", ARG_DEBUG, NULL, 0,
 681           "print extra debug information for test_progs." },
 682         { "json-summary", ARG_JSON_SUMMARY, "FILE", 0, "Write report in json format to this file."},
 683         {},
 684 };
 685
 686 static FILE *libbpf_capture_stream;
 687
 688 static struct {
 689         char *buf;
 690         size_t buf_sz;
 691 } libbpf_output_capture;
 692
 693 /* Creates a global memstream capturing INFO and WARN level output
 694  * passed to libbpf_print_fn.
 695  * Returns 0 on success, negative value on failure.
 696  * On failure the description is printed using PRINT_FAIL and
 697  * current test case is marked as fail.
 698  */
 699 int start_libbpf_log_capture(void)
 700 {
 701         if (libbpf_capture_stream) {
 702                 PRINT_FAIL("%s: libbpf_capture_stream != NULL\n", __func__);
 703                 return -EINVAL;
 704         }
 705
 706         libbpf_capture_stream = open_memstream(&libbpf_output_capture.buf,
 707                                                &libbpf_output_capture.buf_sz);
 708         if (!libbpf_capture_stream) {
 709                 PRINT_FAIL("%s: open_memstream failed errno=%d\n", __func__, errno);
 710                 return -EINVAL;
 711         }
 712
 713         return 0;
 714 }
 715
 716 /* Destroys global memstream created by start_libbpf_log_capture().
 717  * Returns a pointer to captured data which has to be freed.
 718  * Returned buffer is null terminated.
 719  */
 720 char *stop_libbpf_log_capture(void)
 721 {
 722         char *buf;
 723
 724         if (!libbpf_capture_stream)
 725                 return NULL;
 726
 727         fputc(0, libbpf_capture_stream);
 728         fclose(libbpf_capture_stream);
 729         libbpf_capture_stream = NULL;
 730         /* get 'buf' after fclose(), see open_memstream() documentation */
 731         buf = libbpf_output_capture.buf;
 732         memset(&libbpf_output_capture, 0, sizeof(libbpf_output_capture));
 733         return buf;
 734 }
 735
 736 static int libbpf_print_fn(enum libbpf_print_level level,
 737                            const char *format, va_list args)
 738 {
 739         if (libbpf_capture_stream && level != LIBBPF_DEBUG) {
 740                 va_list args2;
 741
 742                 va_copy(args2, args);
 743                 vfprintf(libbpf_capture_stream, format, args2);
 744         }
 745
 746         if (env.verbosity < VERBOSE_VERY && level == LIBBPF_DEBUG)
 747                 return 0;
 748
 749         vfprintf(stdout, format, args);
 750         return 0;
 751 }
 752
 753 static void free_test_filter_set(const struct test_filter_set *set)
 754 {
 755         int i, j;
 756
 757         if (!set)
 758                 return;
 759
 760         for (i = 0; i < set->cnt; i++) {
 761                 free((void *)set->tests[i].name);
 762                 for (j = 0; j < set->tests[i].subtest_cnt; j++)
 763                         free((void *)set->tests[i].subtests[j]);
 764
 765                 free((void *)set->tests[i].subtests);
 766         }
 767
 768         free((void *)set->tests);
 769 }
 770
 771 static void free_test_selector(struct test_selector *test_selector)
 772 {
 773         free_test_filter_set(&test_selector->blacklist);
 774         free_test_filter_set(&test_selector->whitelist);
 775         free(test_selector->num_set);
 776 }
 777
 778 extern int extra_prog_load_log_flags;
 779
 780 static error_t parse_arg(int key, char *arg, struct argp_state *state)
 781 {
 782         struct test_env *env = state->input;
 783         int err = 0;
 784
 785         switch (key) {
 786         case ARG_TEST_NUM: {
 787                 char *subtest_str = strchr(arg, '/');
 788
 789                 if (subtest_str) {
 790                         *subtest_str = '\0';
 791                         if (parse_num_list(subtest_str + 1,
 792                                            &env->subtest_selector.num_set,
 793                                            &env->subtest_selector.num_set_len)) {
 794                                 fprintf(stderr,
 795                                         "Failed to parse subtest numbers.\n");
 796                                 return -EINVAL;
 797                         }
 798                 }
 799                 if (parse_num_list(arg, &env->test_selector.num_set,
 800                                    &env->test_selector.num_set_len)) {
 801                         fprintf(stderr, "Failed to parse test numbers.\n");
 802                         return -EINVAL;
 803                 }
 804                 break;
 805         }
 806         case ARG_TEST_NAME_GLOB_ALLOWLIST:
 807         case ARG_TEST_NAME: {
 808                 if (arg[0] == '@')
 809                         err = parse_test_list_file(arg + 1,
 810                                                    &env->test_selector.whitelist,
 811                                                    key == ARG_TEST_NAME_GLOB_ALLOWLIST);
 812                 else
 813                         err = parse_test_list(arg,
 814                                               &env->test_selector.whitelist,
 815                                               key == ARG_TEST_NAME_GLOB_ALLOWLIST);
 816
 817                 break;
 818         }
 819         case ARG_TEST_NAME_GLOB_DENYLIST:
 820         case ARG_TEST_NAME_BLACKLIST: {
 821                 if (arg[0] == '@')
 822                         err = parse_test_list_file(arg + 1,
 823                                                    &env->test_selector.blacklist,
 824                                                    key == ARG_TEST_NAME_GLOB_DENYLIST);
 825                 else
 826                         err = parse_test_list(arg,
 827                                               &env->test_selector.blacklist,
 828                                               key == ARG_TEST_NAME_GLOB_DENYLIST);
 829
 830                 break;
 831         }
 832         case ARG_VERIFIER_STATS:
 833                 env->verifier_stats = true;
 834                 break;
 835         case ARG_VERBOSE:
 836                 env->verbosity = VERBOSE_NORMAL;
 837                 if (arg) {
 838                         if (strcmp(arg, "v") == 0) {
 839                                 env->verbosity = VERBOSE_VERY;
 840                                 extra_prog_load_log_flags = 1;
 841                         } else if (strcmp(arg, "vv") == 0) {
 842                                 env->verbosity = VERBOSE_SUPER;
 843                                 extra_prog_load_log_flags = 2;
 844                         } else {
 845                                 fprintf(stderr,
 846                                         "Unrecognized verbosity setting ('%s'), only -v and -vv are supported\n",
 847                                         arg);
 848                                 return -EINVAL;
 849                         }
 850                 }
 851
 852                 if (verbose()) {
 853                         if (setenv("SELFTESTS_VERBOSE", "1", 1) == -1) {
 854                                 fprintf(stderr,
 855                                         "Unable to setenv SELFTESTS_VERBOSE=1 (errno=%d)",
 856                                         errno);
 857                                 return -EINVAL;
 858                         }
 859                 }
 860
 861                 break;
 862         case ARG_GET_TEST_CNT:
 863                 env->get_test_cnt = true;
 864                 break;
 865         case ARG_LIST_TEST_NAMES:
 866                 env->list_test_names = true;
 867                 break;
 868         case ARG_NUM_WORKERS:
 869                 if (arg) {
 870                         env->workers = atoi(arg);
 871                         if (!env->workers) {
 872                                 fprintf(stderr, "Invalid number of worker: %s.", arg);
 873                                 return -EINVAL;
 874                         }
 875                 } else {
 876                         env->workers = get_nprocs();
 877                 }
 878                 break;
 879         case ARG_DEBUG:
 880                 env->debug = true;
 881                 break;
 882         case ARG_JSON_SUMMARY:
 883                 env->json = fopen(arg, "w");
 884                 if (env->json == NULL) {
 885                         perror("Failed to open json summary file");
 886                         return -errno;
 887                 }
 888                 break;
 889         case ARGP_KEY_ARG:
 890                 argp_usage(state);
 891                 break;
 892         case ARGP_KEY_END:
 893                 break;
 894         default:
 895                 return ARGP_ERR_UNKNOWN;
 896         }
 897         return err;
 898 }
 899
 900 /*
 901  * Determine if test_progs is running as a "flavored" test runner and switch
 902  * into corresponding sub-directory to load correct BPF objects.
 903  *
 904  * This is done by looking at executable name. If it contains "-flavor"
 905  * suffix, then we are running as a flavored test runner.
 906  */
 907 int cd_flavor_subdir(const char *exec_name)
 908 {
 909         /* General form of argv[0] passed here is:
 910          * some/path/to/test_progs[-flavor], where -flavor part is optional.
 911          * First cut out "test_progs[-flavor]" part, then extract "flavor"
 912          * part, if it's there.
 913          */
 914         const char *flavor = strrchr(exec_name, '/');
 915
 916         if (!flavor)
 917                 flavor = exec_name;
 918         else
 919                 flavor++;
 920
 921         flavor = strrchr(flavor, '-');
 922         if (!flavor)
 923                 return 0;
 924         flavor++;
 925         if (verbose())
 926                 fprintf(stdout, "Switching to flavor '%s' subdirectory...\n", flavor);
 927
 928         return chdir(flavor);
 929 }
 930
 931 int trigger_module_test_read(int read_sz)
 932 {
 933         int fd, err;
 934
 935         fd = open(BPF_TESTMOD_TEST_FILE, O_RDONLY);
 936         err = -errno;
 937         if (!ASSERT_GE(fd, 0, "testmod_file_open"))
 938                 return err;
 939
 940         read(fd, NULL, read_sz);
 941         close(fd);
 942
 943         return 0;
 944 }
 945
 946 int trigger_module_test_write(int write_sz)
 947 {
 948         int fd, err;
 949         char *buf = malloc(write_sz);
 950
 951         if (!buf)
 952                 return -ENOMEM;
 953
 954         memset(buf, 'a', write_sz);
 955         buf[write_sz-1] = '\0';
 956
 957         fd = open(BPF_TESTMOD_TEST_FILE, O_WRONLY);
 958         err = -errno;
 959         if (!ASSERT_GE(fd, 0, "testmod_file_open")) {
 960                 free(buf);
 961                 return err;
 962         }
 963
 964         write(fd, buf, write_sz);
 965         close(fd);
 966         free(buf);
 967         return 0;
 968 }
 969
 970 int write_sysctl(const char *sysctl, const char *value)
 971 {
 972         int fd, err, len;
 973
 974         fd = open(sysctl, O_WRONLY);
 975         if (!ASSERT_NEQ(fd, -1, "open sysctl"))
 976                 return -1;
 977
 978         len = strlen(value);
 979         err = write(fd, value, len);
 980         close(fd);
 981         if (!ASSERT_EQ(err, len, "write sysctl"))
 982                 return -1;
 983
 984         return 0;
 985 }
 986
 987 int get_bpf_max_tramp_links_from(struct btf *btf)
 988 {
 989         const struct btf_enum *e;
 990         const struct btf_type *t;
 991         __u32 i, type_cnt;
 992         const char *name;
 993         __u16 j, vlen;
 994
 995         for (i = 1, type_cnt = btf__type_cnt(btf); i < type_cnt; i++) {
 996                 t = btf__type_by_id(btf, i);
 997                 if (!t || !btf_is_enum(t) || t->name_off)
 998                         continue;
 999                 e = btf_enum(t);
1000                 for (j = 0, vlen = btf_vlen(t); j < vlen; j++, e++) {
1001                         name = btf__str_by_offset(btf, e->name_off);
1002                         if (name && !strcmp(name, "BPF_MAX_TRAMP_LINKS"))
1003                                 return e->val;
1004                 }
1005         }
1006
1007         return -1;
1008 }
1009
1010 int get_bpf_max_tramp_links(void)
1011 {
1012         struct btf *vmlinux_btf;
1013         int ret;
1014
1015         vmlinux_btf = btf__load_vmlinux_btf();
1016         if (!ASSERT_OK_PTR(vmlinux_btf, "vmlinux btf"))
1017                 return -1;
1018         ret = get_bpf_max_tramp_links_from(vmlinux_btf);
1019         btf__free(vmlinux_btf);
1020
1021         return ret;
1022 }
1023
1024 #define MAX_BACKTRACE_SZ 128
1025 void crash_handler(int signum)
1026 {
1027         void *bt[MAX_BACKTRACE_SZ];
1028         size_t sz;
1029
1030         sz = backtrace(bt, ARRAY_SIZE(bt));
1031
1032         if (env.stdout)
1033                 stdio_restore();
1034         if (env.test) {
1035                 env.test_state->error_cnt++;
1036                 dump_test_log(env.test, env.test_state, true, false, NULL);
1037         }
1038         if (env.worker_id != -1)
1039                 fprintf(stderr, "[%d]: ", env.worker_id);
1040         fprintf(stderr, "Caught signal #%d!\nStack trace:\n", signum);
1041         backtrace_symbols_fd(bt, sz, STDERR_FILENO);
1042 }
1043
1044 static void sigint_handler(int signum)
1045 {
1046         int i;
1047
1048         for (i = 0; i < env.workers; i++)
1049                 if (env.worker_socks[i] > 0)
1050                         close(env.worker_socks[i]);
1051 }
1052
1053 static int current_test_idx;
1054 static pthread_mutex_t current_test_lock;
1055 static pthread_mutex_t stdout_output_lock;
1056
1057 static inline const char *str_msg(const struct msg *msg, char *buf)
1058 {
1059         switch (msg->type) {
1060         case MSG_DO_TEST:
1061                 sprintf(buf, "MSG_DO_TEST %d", msg->do_test.num);
1062                 break;
1063         case MSG_TEST_DONE:
1064                 sprintf(buf, "MSG_TEST_DONE %d (log: %d)",
1065                         msg->test_done.num,
1066                         msg->test_done.have_log);
1067                 break;
1068         case MSG_SUBTEST_DONE:
1069                 sprintf(buf, "MSG_SUBTEST_DONE %d (log: %d)",
1070                         msg->subtest_done.num,
1071                         msg->subtest_done.have_log);
1072                 break;
1073         case MSG_TEST_LOG:
1074                 sprintf(buf, "MSG_TEST_LOG (cnt: %zu, last: %d)",
1075                         strlen(msg->test_log.log_buf),
1076                         msg->test_log.is_last);
1077                 break;
1078         case MSG_EXIT:
1079                 sprintf(buf, "MSG_EXIT");
1080                 break;
1081         default:
1082                 sprintf(buf, "UNKNOWN");
1083                 break;
1084         }
1085
1086         return buf;
1087 }
1088
1089 static int send_message(int sock, const struct msg *msg)
1090 {
1091         char buf[256];
1092
1093         if (env.debug)
1094                 fprintf(stderr, "Sending msg: %s\n", str_msg(msg, buf));
1095         return send(sock, msg, sizeof(*msg), 0);
1096 }
1097
1098 static int recv_message(int sock, struct msg *msg)
1099 {
1100         int ret;
1101         char buf[256];
1102
1103         memset(msg, 0, sizeof(*msg));
1104         ret = recv(sock, msg, sizeof(*msg), 0);
1105         if (ret >= 0) {
1106                 if (env.debug)
1107                         fprintf(stderr, "Received msg: %s\n", str_msg(msg, buf));
1108         }
1109         return ret;
1110 }
1111
1112 static void run_one_test(int test_num)
1113 {
1114         struct prog_test_def *test = &prog_test_defs[test_num];
1115         struct test_state *state = &test_states[test_num];
1116
1117         env.test = test;
1118         env.test_state = state;
1119
1120         stdio_hijack(&state->log_buf, &state->log_cnt);
1121
1122         if (test->run_test)
1123                 test->run_test();
1124         else if (test->run_serial_test)
1125                 test->run_serial_test();
1126
1127         /* ensure last sub-test is finalized properly */
1128         if (env.subtest_state)
1129                 test__end_subtest();
1130
1131         state->tested = true;
1132
1133         if (verbose() && env.worker_id == -1)
1134                 print_test_result(test, state);
1135
1136         reset_affinity();
1137         restore_netns();
1138         if (test->need_cgroup_cleanup)
1139                 cleanup_cgroup_environment();
1140
1141         stdio_restore();
1142         free(stop_libbpf_log_capture());
1143
1144         dump_test_log(test, state, false, false, NULL);
1145 }
1146
1147 struct dispatch_data {
1148         int worker_id;
1149         int sock_fd;
1150 };
1151
1152 static int read_prog_test_msg(int sock_fd, struct msg *msg, enum msg_type type)
1153 {
1154         if (recv_message(sock_fd, msg) < 0)
1155                 return 1;
1156
1157         if (msg->type != type) {
1158                 printf("%s: unexpected message type %d. expected %d\n", __func__, msg->type, type);
1159                 return 1;
1160         }
1161
1162         return 0;
1163 }
1164
1165 static int dispatch_thread_read_log(int sock_fd, char **log_buf, size_t *log_cnt)
1166 {
1167         FILE *log_fp = NULL;
1168         int result = 0;
1169
1170         log_fp = open_memstream(log_buf, log_cnt);
1171         if (!log_fp)
1172                 return 1;
1173
1174         while (true) {
1175                 struct msg msg;
1176
1177                 if (read_prog_test_msg(sock_fd, &msg, MSG_TEST_LOG)) {
1178                         result = 1;
1179                         goto out;
1180                 }
1181
1182                 fprintf(log_fp, "%s", msg.test_log.log_buf);
1183                 if (msg.test_log.is_last)
1184                         break;
1185         }
1186
1187 out:
1188         fclose(log_fp);
1189         log_fp = NULL;
1190         return result;
1191 }
1192
1193 static int dispatch_thread_send_subtests(int sock_fd, struct test_state *state)
1194 {
1195         struct msg msg;
1196         struct subtest_state *subtest_state;
1197         int subtest_num = state->subtest_num;
1198
1199         state->subtest_states = malloc(subtest_num * sizeof(*subtest_state));
1200
1201         for (int i = 0; i < subtest_num; i++) {
1202                 subtest_state = &state->subtest_states[i];
1203
1204                 memset(subtest_state, 0, sizeof(*subtest_state));
1205
1206                 if (read_prog_test_msg(sock_fd, &msg, MSG_SUBTEST_DONE))
1207                         return 1;
1208
1209                 subtest_state->name = strdup(msg.subtest_done.name);
1210                 subtest_state->error_cnt = msg.subtest_done.error_cnt;
1211                 subtest_state->skipped = msg.subtest_done.skipped;
1212                 subtest_state->filtered = msg.subtest_done.filtered;
1213
1214                 /* collect all logs */
1215                 if (msg.subtest_done.have_log)
1216                         if (dispatch_thread_read_log(sock_fd,
1217                                                      &subtest_state->log_buf,
1218                                                      &subtest_state->log_cnt))
1219                                 return 1;
1220         }
1221
1222         return 0;
1223 }
1224
1225 static void *dispatch_thread(void *ctx)
1226 {
1227         struct dispatch_data *data = ctx;
1228         int sock_fd;
1229
1230         sock_fd = data->sock_fd;
1231
1232         while (true) {
1233                 int test_to_run = -1;
1234                 struct prog_test_def *test;
1235                 struct test_state *state;
1236
1237                 /* grab a test */
1238                 {
1239                         pthread_mutex_lock(&current_test_lock);
1240
1241                         if (current_test_idx >= prog_test_cnt) {
1242                                 pthread_mutex_unlock(&current_test_lock);
1243                                 goto done;
1244                         }
1245
1246                         test = &prog_test_defs[current_test_idx];
1247                         test_to_run = current_test_idx;
1248                         current_test_idx++;
1249
1250                         pthread_mutex_unlock(&current_test_lock);
1251                 }
1252
1253                 if (!test->should_run || test->run_serial_test)
1254                         continue;
1255
1256                 /* run test through worker */
1257                 {
1258                         struct msg msg_do_test;
1259
1260                         memset(&msg_do_test, 0, sizeof(msg_do_test));
1261                         msg_do_test.type = MSG_DO_TEST;
1262                         msg_do_test.do_test.num = test_to_run;
1263                         if (send_message(sock_fd, &msg_do_test) < 0) {
1264                                 perror("Fail to send command");
1265                                 goto done;
1266                         }
1267                         env.worker_current_test[data->worker_id] = test_to_run;
1268                 }
1269
1270                 /* wait for test done */
1271                 do {
1272                         struct msg msg;
1273
1274                         if (read_prog_test_msg(sock_fd, &msg, MSG_TEST_DONE))
1275                                 goto error;
1276                         if (test_to_run != msg.test_done.num)
1277                                 goto error;
1278
1279                         state = &test_states[test_to_run];
1280                         state->tested = true;
1281                         state->error_cnt = msg.test_done.error_cnt;
1282                         state->skip_cnt = msg.test_done.skip_cnt;
1283                         state->sub_succ_cnt = msg.test_done.sub_succ_cnt;
1284                         state->subtest_num = msg.test_done.subtest_num;
1285
1286                         /* collect all logs */
1287                         if (msg.test_done.have_log) {
1288                                 if (dispatch_thread_read_log(sock_fd,
1289                                                              &state->log_buf,
1290                                                              &state->log_cnt))
1291                                         goto error;
1292                         }
1293
1294                         /* collect all subtests and subtest logs */
1295                         if (!state->subtest_num)
1296                                 break;
1297
1298                         if (dispatch_thread_send_subtests(sock_fd, state))
1299                                 goto error;
1300                 } while (false);
1301
1302                 pthread_mutex_lock(&stdout_output_lock);
1303                 dump_test_log(test, state, false, true, NULL);
1304                 pthread_mutex_unlock(&stdout_output_lock);
1305         } /* while (true) */
1306 error:
1307         if (env.debug)
1308                 fprintf(stderr, "[%d]: Protocol/IO error: %s.\n", data->worker_id, strerror(errno));
1309
1310 done:
1311         {
1312                 struct msg msg_exit;
1313
1314                 msg_exit.type = MSG_EXIT;
1315                 if (send_message(sock_fd, &msg_exit) < 0) {
1316                         if (env.debug)
1317                                 fprintf(stderr, "[%d]: send_message msg_exit: %s.\n",
1318                                         data->worker_id, strerror(errno));
1319                 }
1320         }
1321         return NULL;
1322 }
1323
1324 static void calculate_summary_and_print_errors(struct test_env *env)
1325 {
1326         int i;
1327         int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, skip_cnt = 0;
1328         json_writer_t *w = NULL;
1329
1330         for (i = 0; i < prog_test_cnt; i++) {
1331                 struct test_state *state = &test_states[i];
1332
1333                 if (!state->tested)
1334                         continue;
1335
1336                 sub_succ_cnt += state->sub_succ_cnt;
1337                 skip_cnt += state->skip_cnt;
1338
1339                 if (state->error_cnt)
1340                         fail_cnt++;
1341                 else
1342                         succ_cnt++;
1343         }
1344
1345         if (env->json) {
1346                 w = jsonw_new(env->json);
1347                 if (!w)
1348                         fprintf(env->stderr, "Failed to create new JSON stream.");
1349         }
1350
1351         if (w) {
1352                 jsonw_start_object(w);
1353                 jsonw_uint_field(w, "success", succ_cnt);
1354                 jsonw_uint_field(w, "success_subtest", sub_succ_cnt);
1355                 jsonw_uint_field(w, "skipped", skip_cnt);
1356                 jsonw_uint_field(w, "failed", fail_cnt);
1357                 jsonw_name(w, "results");
1358                 jsonw_start_array(w);
1359         }
1360
1361         /*
1362          * We only print error logs summary when there are failed tests and
1363          * verbose mode is not enabled. Otherwise, results may be incosistent.
1364          *
1365          */
1366         if (!verbose() && fail_cnt) {
1367                 printf("\nAll error logs:\n");
1368
1369                 /* print error logs again */
1370                 for (i = 0; i < prog_test_cnt; i++) {
1371                         struct prog_test_def *test = &prog_test_defs[i];
1372                         struct test_state *state = &test_states[i];
1373
1374                         if (!state->tested || !state->error_cnt)
1375                                 continue;
1376
1377                         dump_test_log(test, state, true, true, w);
1378                 }
1379         }
1380
1381         if (w) {
1382                 jsonw_end_array(w);
1383                 jsonw_end_object(w);
1384                 jsonw_destroy(&w);
1385         }
1386
1387         if (env->json)
1388                 fclose(env->json);
1389
1390         printf("Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n",
1391                succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt);
1392
1393         env->succ_cnt = succ_cnt;
1394         env->sub_succ_cnt = sub_succ_cnt;
1395         env->fail_cnt = fail_cnt;
1396         env->skip_cnt = skip_cnt;
1397 }
1398
1399 static void server_main(void)
1400 {
1401         pthread_t *dispatcher_threads;
1402         struct dispatch_data *data;
1403         struct sigaction sigact_int = {
1404                 .sa_handler = sigint_handler,
1405                 .sa_flags = SA_RESETHAND,
1406         };
1407         int i;
1408
1409         sigaction(SIGINT, &sigact_int, NULL);
1410
1411         dispatcher_threads = calloc(sizeof(pthread_t), env.workers);
1412         data = calloc(sizeof(struct dispatch_data), env.workers);
1413
1414         env.worker_current_test = calloc(sizeof(int), env.workers);
1415         for (i = 0; i < env.workers; i++) {
1416                 int rc;
1417
1418                 data[i].worker_id = i;
1419                 data[i].sock_fd = env.worker_socks[i];
1420                 rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]);
1421                 if (rc < 0) {
1422                         perror("Failed to launch dispatcher thread");
1423                         exit(EXIT_ERR_SETUP_INFRA);
1424                 }
1425         }
1426
1427         /* wait for all dispatcher to finish */
1428         for (i = 0; i < env.workers; i++) {
1429                 while (true) {
1430                         int ret = pthread_tryjoin_np(dispatcher_threads[i], NULL);
1431
1432                         if (!ret) {
1433                                 break;
1434                         } else if (ret == EBUSY) {
1435                                 if (env.debug)
1436                                         fprintf(stderr, "Still waiting for thread %d (test %d).\n",
1437                                                 i,  env.worker_current_test[i] + 1);
1438                                 usleep(1000 * 1000);
1439                                 continue;
1440                         } else {
1441                                 fprintf(stderr, "Unexpected error joining dispatcher thread: %d", ret);
1442                                 break;
1443                         }
1444                 }
1445         }
1446         free(dispatcher_threads);
1447         free(env.worker_current_test);
1448         free(data);
1449
1450         /* run serial tests */
1451         save_netns();
1452
1453         for (int i = 0; i < prog_test_cnt; i++) {
1454                 struct prog_test_def *test = &prog_test_defs[i];
1455
1456                 if (!test->should_run || !test->run_serial_test)
1457                         continue;
1458
1459                 run_one_test(i);
1460         }
1461
1462         /* generate summary */
1463         fflush(stderr);
1464         fflush(stdout);
1465
1466         calculate_summary_and_print_errors(&env);
1467
1468         /* reap all workers */
1469         for (i = 0; i < env.workers; i++) {
1470                 int wstatus, pid;
1471
1472                 pid = waitpid(env.worker_pids[i], &wstatus, 0);
1473                 if (pid != env.worker_pids[i])
1474                         perror("Unable to reap worker");
1475         }
1476 }
1477
1478 static void worker_main_send_log(int sock, char *log_buf, size_t log_cnt)
1479 {
1480         char *src;
1481         size_t slen;
1482
1483         src = log_buf;
1484         slen = log_cnt;
1485         while (slen) {
1486                 struct msg msg_log;
1487                 char *dest;
1488                 size_t len;
1489
1490                 memset(&msg_log, 0, sizeof(msg_log));
1491                 msg_log.type = MSG_TEST_LOG;
1492                 dest = msg_log.test_log.log_buf;
1493                 len = slen >= MAX_LOG_TRUNK_SIZE ? MAX_LOG_TRUNK_SIZE : slen;
1494                 memcpy(dest, src, len);
1495
1496                 src += len;
1497                 slen -= len;
1498                 if (!slen)
1499                         msg_log.test_log.is_last = true;
1500
1501                 assert(send_message(sock, &msg_log) >= 0);
1502         }
1503 }
1504
1505 static void free_subtest_state(struct subtest_state *state)
1506 {
1507         if (state->log_buf) {
1508                 free(state->log_buf);
1509                 state->log_buf = NULL;
1510                 state->log_cnt = 0;
1511         }
1512         free(state->name);
1513         state->name = NULL;
1514 }
1515
1516 static int worker_main_send_subtests(int sock, struct test_state *state)
1517 {
1518         int i, result = 0;
1519         struct msg msg;
1520         struct subtest_state *subtest_state;
1521
1522         memset(&msg, 0, sizeof(msg));
1523         msg.type = MSG_SUBTEST_DONE;
1524
1525         for (i = 0; i < state->subtest_num; i++) {
1526                 subtest_state = &state->subtest_states[i];
1527
1528                 msg.subtest_done.num = i;
1529
1530                 strncpy(msg.subtest_done.name, subtest_state->name, MAX_SUBTEST_NAME);
1531
1532                 msg.subtest_done.error_cnt = subtest_state->error_cnt;
1533                 msg.subtest_done.skipped = subtest_state->skipped;
1534                 msg.subtest_done.filtered = subtest_state->filtered;
1535                 msg.subtest_done.have_log = false;
1536
1537                 if (verbose() || state->force_log || subtest_state->error_cnt) {
1538                         if (subtest_state->log_cnt)
1539                                 msg.subtest_done.have_log = true;
1540                 }
1541
1542                 if (send_message(sock, &msg) < 0) {
1543                         perror("Fail to send message done");
1544                         result = 1;
1545                         goto out;
1546                 }
1547
1548                 /* send logs */
1549                 if (msg.subtest_done.have_log)
1550                         worker_main_send_log(sock, subtest_state->log_buf, subtest_state->log_cnt);
1551
1552                 free_subtest_state(subtest_state);
1553                 free(subtest_state->name);
1554         }
1555
1556 out:
1557         for (; i < state->subtest_num; i++)
1558                 free_subtest_state(&state->subtest_states[i]);
1559         free(state->subtest_states);
1560         return result;
1561 }
1562
1563 static int worker_main(int sock)
1564 {
1565         save_netns();
1566
1567         while (true) {
1568                 /* receive command */
1569                 struct msg msg;
1570
1571                 if (recv_message(sock, &msg) < 0)
1572                         goto out;
1573
1574                 switch (msg.type) {
1575                 case MSG_EXIT:
1576                         if (env.debug)
1577                                 fprintf(stderr, "[%d]: worker exit.\n",
1578                                         env.worker_id);
1579                         goto out;
1580                 case MSG_DO_TEST: {
1581                         int test_to_run = msg.do_test.num;
1582                         struct prog_test_def *test = &prog_test_defs[test_to_run];
1583                         struct test_state *state = &test_states[test_to_run];
1584                         struct msg msg;
1585
1586                         if (env.debug)
1587                                 fprintf(stderr, "[%d]: #%d:%s running.\n",
1588                                         env.worker_id,
1589                                         test_to_run + 1,
1590                                         test->test_name);
1591
1592                         run_one_test(test_to_run);
1593
1594                         memset(&msg, 0, sizeof(msg));
1595                         msg.type = MSG_TEST_DONE;
1596                         msg.test_done.num = test_to_run;
1597                         msg.test_done.error_cnt = state->error_cnt;
1598                         msg.test_done.skip_cnt = state->skip_cnt;
1599                         msg.test_done.sub_succ_cnt = state->sub_succ_cnt;
1600                         msg.test_done.subtest_num = state->subtest_num;
1601                         msg.test_done.have_log = false;
1602
1603                         if (verbose() || state->force_log || state->error_cnt) {
1604                                 if (state->log_cnt)
1605                                         msg.test_done.have_log = true;
1606                         }
1607                         if (send_message(sock, &msg) < 0) {
1608                                 perror("Fail to send message done");
1609                                 goto out;
1610                         }
1611
1612                         /* send logs */
1613                         if (msg.test_done.have_log)
1614                                 worker_main_send_log(sock, state->log_buf, state->log_cnt);
1615
1616                         if (state->log_buf) {
1617                                 free(state->log_buf);
1618                                 state->log_buf = NULL;
1619                                 state->log_cnt = 0;
1620                         }
1621
1622                         if (state->subtest_num)
1623                                 if (worker_main_send_subtests(sock, state))
1624                                         goto out;
1625
1626                         if (env.debug)
1627                                 fprintf(stderr, "[%d]: #%d:%s done.\n",
1628                                         env.worker_id,
1629                                         test_to_run + 1,
1630                                         test->test_name);
1631                         break;
1632                 } /* case MSG_DO_TEST */
1633                 default:
1634                         if (env.debug)
1635                                 fprintf(stderr, "[%d]: unknown message.\n",  env.worker_id);
1636                         return -1;
1637                 }
1638         }
1639 out:
1640         return 0;
1641 }
1642
1643 static void free_test_states(void)
1644 {
1645         int i, j;
1646
1647         for (i = 0; i < ARRAY_SIZE(prog_test_defs); i++) {
1648                 struct test_state *test_state = &test_states[i];
1649
1650                 for (j = 0; j < test_state->subtest_num; j++)
1651                         free_subtest_state(&test_state->subtest_states[j]);
1652
1653                 free(test_state->subtest_states);
1654                 free(test_state->log_buf);
1655                 test_state->subtest_states = NULL;
1656                 test_state->log_buf = NULL;
1657         }
1658 }
1659
1660 int main(int argc, char **argv)
1661 {
1662         static const struct argp argp = {
1663                 .options = opts,
1664                 .parser = parse_arg,
1665                 .doc = argp_program_doc,
1666         };
1667         struct sigaction sigact = {
1668                 .sa_handler = crash_handler,
1669                 .sa_flags = SA_RESETHAND,
1670                 };
1671         int err, i;
1672
1673         sigaction(SIGSEGV, &sigact, NULL);
1674
1675         err = argp_parse(&argp, argc, argv, 0, NULL, &env);
1676         if (err)
1677                 return err;
1678
1679         err = cd_flavor_subdir(argv[0]);
1680         if (err)
1681                 return err;
1682
1683         /* Use libbpf 1.0 API mode */
1684         libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1685         libbpf_set_print(libbpf_print_fn);
1686
1687         srand(time(NULL));
1688
1689         env.jit_enabled = is_jit_enabled();
1690         env.nr_cpus = libbpf_num_possible_cpus();
1691         if (env.nr_cpus < 0) {
1692                 fprintf(stderr, "Failed to get number of CPUs: %d!\n",
1693                         env.nr_cpus);
1694                 return -1;
1695         }
1696
1697         env.stdout = stdout;
1698         env.stderr = stderr;
1699
1700         env.has_testmod = true;
1701         if (!env.list_test_names) {
1702                 /* ensure previous instance of the module is unloaded */
1703                 unload_bpf_testmod(verbose());
1704
1705                 if (load_bpf_testmod(verbose())) {
1706                         fprintf(env.stderr, "WARNING! Selftests relying on bpf_testmod.ko will be skipped.\n");
1707                         env.has_testmod = false;
1708                 }
1709         }
1710
1711         /* initializing tests */
1712         for (i = 0; i < prog_test_cnt; i++) {
1713                 struct prog_test_def *test = &prog_test_defs[i];
1714
1715                 test->test_num = i + 1;
1716                 test->should_run = should_run(&env.test_selector,
1717                                               test->test_num, test->test_name);
1718
1719                 if ((test->run_test == NULL && test->run_serial_test == NULL) ||
1720                     (test->run_test != NULL && test->run_serial_test != NULL)) {
1721                         fprintf(stderr, "Test %d:%s must have either test_%s() or serial_test_%sl() defined.\n",
1722                                 test->test_num, test->test_name, test->test_name, test->test_name);
1723                         exit(EXIT_ERR_SETUP_INFRA);
1724                 }
1725         }
1726
1727         /* ignore workers if we are just listing */
1728         if (env.get_test_cnt || env.list_test_names)
1729                 env.workers = 0;
1730
1731         /* launch workers if requested */
1732         env.worker_id = -1; /* main process */
1733         if (env.workers) {
1734                 env.worker_pids = calloc(sizeof(__pid_t), env.workers);
1735                 env.worker_socks = calloc(sizeof(int), env.workers);
1736                 if (env.debug)
1737                         fprintf(stdout, "Launching %d workers.\n", env.workers);
1738                 for (i = 0; i < env.workers; i++) {
1739                         int sv[2];
1740                         pid_t pid;
1741
1742                         if (socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, sv) < 0) {
1743                                 perror("Fail to create worker socket");
1744                                 return -1;
1745                         }
1746                         pid = fork();
1747                         if (pid < 0) {
1748                                 perror("Failed to fork worker");
1749                                 return -1;
1750                         } else if (pid != 0) { /* main process */
1751                                 close(sv[1]);
1752                                 env.worker_pids[i] = pid;
1753                                 env.worker_socks[i] = sv[0];
1754                         } else { /* inside each worker process */
1755                                 close(sv[0]);
1756                                 env.worker_id = i;
1757                                 return worker_main(sv[1]);
1758                         }
1759                 }
1760
1761                 if (env.worker_id == -1) {
1762                         server_main();
1763                         goto out;
1764                 }
1765         }
1766
1767         /* The rest of the main process */
1768
1769         /* on single mode */
1770         save_netns();
1771
1772         for (i = 0; i < prog_test_cnt; i++) {
1773                 struct prog_test_def *test = &prog_test_defs[i];
1774
1775                 if (!test->should_run)
1776                         continue;
1777
1778                 if (env.get_test_cnt) {
1779                         env.succ_cnt++;
1780                         continue;
1781                 }
1782
1783                 if (env.list_test_names) {
1784                         fprintf(env.stdout, "%s\n", test->test_name);
1785                         env.succ_cnt++;
1786                         continue;
1787                 }
1788
1789                 run_one_test(i);
1790         }
1791
1792         if (env.get_test_cnt) {
1793                 printf("%d\n", env.succ_cnt);
1794                 goto out;
1795         }
1796
1797         if (env.list_test_names)
1798                 goto out;
1799
1800         calculate_summary_and_print_errors(&env);
1801
1802         close(env.saved_netns_fd);
1803 out:
1804         if (!env.list_test_names && env.has_testmod)
1805                 unload_bpf_testmod(verbose());
1806
1807         free_test_selector(&env.test_selector);
1808         free_test_selector(&env.subtest_selector);
1809         free_test_states();
1810
1811         if (env.succ_cnt + env.fail_cnt + env.skip_cnt == 0)
1812                 return EXIT_NO_TEST;
1813
1814         return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
1815 }