tools/perf/tests/builtin-test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * builtin-test.c
   4  *
   5  * Builtin regression testing command: ever growing number of sanity tests
   6  */
   7 #include <fcntl.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10 #include <string.h>
  11 #include <stdlib.h>
  12 #include <sys/types.h>
  13 #include <dirent.h>
  14 #include <sys/wait.h>
  15 #include <sys/stat.h>
  16 #include "builtin.h"
  17 #include "hist.h"
  18 #include "intlist.h"
  19 #include "tests.h"
  20 #include "debug.h"
  21 #include "color.h"
  22 #include <subcmd/parse-options.h>
  23 #include "string2.h"
  24 #include "symbol.h"
  25 #include "util/rlimit.h"
  26 #include <linux/kernel.h>
  27 #include <linux/string.h>
  28 #include <subcmd/exec-cmd.h>
  29 #include <linux/zalloc.h>
  30
  31 #include "builtin-test-list.h"
  32
  33 static bool dont_fork;
  34
  35 struct test_suite *__weak arch_tests[] = {
  36         NULL,
  37 };
  38
  39 static struct test_suite *generic_tests[] = {
  40         &suite__vmlinux_matches_kallsyms,
  41         &suite__openat_syscall_event,
  42         &suite__openat_syscall_event_on_all_cpus,
  43         &suite__basic_mmap,
  44         &suite__mem,
  45         &suite__parse_events,
  46         &suite__expr,
  47         &suite__PERF_RECORD,
  48         &suite__pmu,
  49         &suite__pmu_events,
  50         &suite__dso_data,
  51         &suite__dso_data_cache,
  52         &suite__dso_data_reopen,
  53         &suite__perf_evsel__roundtrip_name_test,
  54         &suite__perf_evsel__tp_sched_test,
  55         &suite__syscall_openat_tp_fields,
  56         &suite__attr,
  57         &suite__hists_link,
  58         &suite__python_use,
  59         &suite__bp_signal,
  60         &suite__bp_signal_overflow,
  61         &suite__bp_accounting,
  62         &suite__wp,
  63         &suite__task_exit,
  64         &suite__sw_clock_freq,
  65         &suite__code_reading,
  66         &suite__sample_parsing,
  67         &suite__keep_tracking,
  68         &suite__parse_no_sample_id_all,
  69         &suite__hists_filter,
  70         &suite__mmap_thread_lookup,
  71         &suite__thread_maps_share,
  72         &suite__hists_output,
  73         &suite__hists_cumulate,
  74         &suite__switch_tracking,
  75         &suite__fdarray__filter,
  76         &suite__fdarray__add,
  77         &suite__kmod_path__parse,
  78         &suite__thread_map,
  79         &suite__llvm,
  80         &suite__session_topology,
  81         &suite__bpf,
  82         &suite__thread_map_synthesize,
  83         &suite__thread_map_remove,
  84         &suite__cpu_map_synthesize,
  85         &suite__synthesize_stat_config,
  86         &suite__synthesize_stat,
  87         &suite__synthesize_stat_round,
  88         &suite__event_update,
  89         &suite__event_times,
  90         &suite__backward_ring_buffer,
  91         &suite__cpu_map_print,
  92         &suite__cpu_map_merge,
  93         &suite__sdt_event,
  94         &suite__is_printable_array,
  95         &suite__bitmap_print,
  96         &suite__perf_hooks,
  97         &suite__clang,
  98         &suite__unit_number__scnprint,
  99         &suite__mem2node,
 100         &suite__time_utils,
 101         &suite__jit_write_elf,
 102         &suite__pfm,
 103         &suite__api_io,
 104         &suite__maps__merge_in,
 105         &suite__demangle_java,
 106         &suite__demangle_ocaml,
 107         &suite__parse_metric,
 108         &suite__pe_file_parsing,
 109         &suite__expand_cgroup_events,
 110         &suite__perf_time_to_tsc,
 111         &suite__dlfilter,
 112         &suite__sigtrap,
 113         NULL,
 114 };
 115
 116 static struct test_suite **tests[] = {
 117         generic_tests,
 118         arch_tests,
 119 };
 120
 121 static int num_subtests(const struct test_suite *t)
 122 {
 123         int num;
 124
 125         if (!t->test_cases)
 126                 return 0;
 127
 128         num = 0;
 129         while (t->test_cases[num].name)
 130                 num++;
 131
 132         return num;
 133 }
 134
 135 static bool has_subtests(const struct test_suite *t)
 136 {
 137         return num_subtests(t) > 1;
 138 }
 139
 140 static const char *skip_reason(const struct test_suite *t, int subtest)
 141 {
 142         if (!t->test_cases)
 143                 return NULL;
 144
 145         return t->test_cases[subtest >= 0 ? subtest : 0].skip_reason;
 146 }
 147
 148 static const char *test_description(const struct test_suite *t, int subtest)
 149 {
 150         if (t->test_cases && subtest >= 0)
 151                 return t->test_cases[subtest].desc;
 152
 153         return t->desc;
 154 }
 155
 156 static test_fnptr test_function(const struct test_suite *t, int subtest)
 157 {
 158         if (subtest <= 0)
 159                 return t->test_cases[0].run_case;
 160
 161         return t->test_cases[subtest].run_case;
 162 }
 163
 164 static bool perf_test__matches(const char *desc, int curr, int argc, const char *argv[])
 165 {
 166         int i;
 167
 168         if (argc == 0)
 169                 return true;
 170
 171         for (i = 0; i < argc; ++i) {
 172                 char *end;
 173                 long nr = strtoul(argv[i], &end, 10);
 174
 175                 if (*end == '\0') {
 176                         if (nr == curr + 1)
 177                                 return true;
 178                         continue;
 179                 }
 180
 181                 if (strcasestr(desc, argv[i]))
 182                         return true;
 183         }
 184
 185         return false;
 186 }
 187
 188 static int run_test(struct test_suite *test, int subtest)
 189 {
 190         int status, err = -1, child = dont_fork ? 0 : fork();
 191         char sbuf[STRERR_BUFSIZE];
 192
 193         if (child < 0) {
 194                 pr_err("failed to fork test: %s\n",
 195                         str_error_r(errno, sbuf, sizeof(sbuf)));
 196                 return -1;
 197         }
 198
 199         if (!child) {
 200                 if (!dont_fork) {
 201                         pr_debug("test child forked, pid %d\n", getpid());
 202
 203                         if (verbose <= 0) {
 204                                 int nullfd = open("/dev/null", O_WRONLY);
 205
 206                                 if (nullfd >= 0) {
 207                                         close(STDERR_FILENO);
 208                                         close(STDOUT_FILENO);
 209
 210                                         dup2(nullfd, STDOUT_FILENO);
 211                                         dup2(STDOUT_FILENO, STDERR_FILENO);
 212                                         close(nullfd);
 213                                 }
 214                         } else {
 215                                 signal(SIGSEGV, sighandler_dump_stack);
 216                                 signal(SIGFPE, sighandler_dump_stack);
 217                         }
 218                 }
 219
 220                 err = test_function(test, subtest)(test, subtest);
 221                 if (!dont_fork)
 222                         exit(err);
 223         }
 224
 225         if (!dont_fork) {
 226                 wait(&status);
 227
 228                 if (WIFEXITED(status)) {
 229                         err = (signed char)WEXITSTATUS(status);
 230                         pr_debug("test child finished with %d\n", err);
 231                 } else if (WIFSIGNALED(status)) {
 232                         err = -1;
 233                         pr_debug("test child interrupted\n");
 234                 }
 235         }
 236
 237         return err;
 238 }
 239
 240 #define for_each_test(j, k, t)                  \
 241         for (j = 0; j < ARRAY_SIZE(tests); j++) \
 242                 for (k = 0, t = tests[j][k]; tests[j][k]; k++, t = tests[j][k])
 243
 244 static int test_and_print(struct test_suite *t, int subtest)
 245 {
 246         int err;
 247
 248         pr_debug("\n--- start ---\n");
 249         err = run_test(t, subtest);
 250         pr_debug("---- end ----\n");
 251
 252         if (!has_subtests(t))
 253                 pr_debug("%s:", t->desc);
 254         else
 255                 pr_debug("%s subtest %d:", t->desc, subtest + 1);
 256
 257         switch (err) {
 258         case TEST_OK:
 259                 pr_info(" Ok\n");
 260                 break;
 261         case TEST_SKIP: {
 262                 const char *reason = skip_reason(t, subtest);
 263
 264                 if (reason)
 265                         color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason);
 266                 else
 267                         color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
 268         }
 269                 break;
 270         case TEST_FAIL:
 271         default:
 272                 color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
 273                 break;
 274         }
 275
 276         return err;
 277 }
 278
 279 struct shell_test {
 280         const char *dir;
 281         const char *file;
 282 };
 283
 284 static int shell_test__run(struct test_suite *test, int subdir __maybe_unused)
 285 {
 286         int err;
 287         char script[PATH_MAX];
 288         struct shell_test *st = test->priv;
 289
 290         path__join(script, sizeof(script) - 3, st->dir, st->file);
 291
 292         if (verbose)
 293                 strncat(script, " -v", sizeof(script) - strlen(script) - 1);
 294
 295         err = system(script);
 296         if (!err)
 297                 return TEST_OK;
 298
 299         return WEXITSTATUS(err) == 2 ? TEST_SKIP : TEST_FAIL;
 300 }
 301
 302 static int run_shell_tests(int argc, const char *argv[], int i, int width,
 303                                 struct intlist *skiplist)
 304 {
 305         struct shell_test st;
 306         const struct script_file *files, *file;
 307
 308         files = list_script_files();
 309         if (!files)
 310                 return 0;
 311         for (file = files; file->dir; file++) {
 312                 int curr = i++;
 313                 struct test_case test_cases[] = {
 314                         {
 315                                 .desc = file->desc,
 316                                 .run_case = shell_test__run,
 317                         },
 318                         { .name = NULL, }
 319                 };
 320                 struct test_suite test_suite = {
 321                         .desc = test_cases[0].desc,
 322                         .test_cases = test_cases,
 323                         .priv = &st,
 324                 };
 325                 st.dir = file->dir;
 326
 327                 if (test_suite.desc == NULL ||
 328                     !perf_test__matches(test_suite.desc, curr, argc, argv))
 329                         continue;
 330
 331                 st.file = file->file;
 332                 pr_info("%3d: %-*s:", i, width, test_suite.desc);
 333
 334                 if (intlist__find(skiplist, i)) {
 335                         color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
 336                         continue;
 337                 }
 338
 339                 test_and_print(&test_suite, 0);
 340         }
 341         return 0;
 342 }
 343
 344 static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
 345 {
 346         struct test_suite *t;
 347         unsigned int j, k;
 348         int i = 0;
 349         int width = list_script_max_width();
 350
 351         for_each_test(j, k, t) {
 352                 int len = strlen(test_description(t, -1));
 353
 354                 if (width < len)
 355                         width = len;
 356         }
 357
 358         for_each_test(j, k, t) {
 359                 int curr = i++;
 360                 int subi;
 361
 362                 if (!perf_test__matches(test_description(t, -1), curr, argc, argv)) {
 363                         bool skip = true;
 364                         int subn;
 365
 366                         subn = num_subtests(t);
 367
 368                         for (subi = 0; subi < subn; subi++) {
 369                                 if (perf_test__matches(test_description(t, subi),
 370                                                         curr, argc, argv))
 371                                         skip = false;
 372                         }
 373
 374                         if (skip)
 375                                 continue;
 376                 }
 377
 378                 pr_info("%3d: %-*s:", i, width, test_description(t, -1));
 379
 380                 if (intlist__find(skiplist, i)) {
 381                         color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
 382                         continue;
 383                 }
 384
 385                 if (!has_subtests(t)) {
 386                         test_and_print(t, -1);
 387                 } else {
 388                         int subn = num_subtests(t);
 389                         /*
 390                          * minus 2 to align with normal testcases.
 391                          * For subtest we print additional '.x' in number.
 392                          * for example:
 393                          *
 394                          * 35: Test LLVM searching and compiling                        :
 395                          * 35.1: Basic BPF llvm compiling test                          : Ok
 396                          */
 397                         int subw = width > 2 ? width - 2 : width;
 398
 399                         if (subn <= 0) {
 400                                 color_fprintf(stderr, PERF_COLOR_YELLOW,
 401                                               " Skip (not compiled in)\n");
 402                                 continue;
 403                         }
 404                         pr_info("\n");
 405
 406                         for (subi = 0; subi < subn; subi++) {
 407                                 int len = strlen(test_description(t, subi));
 408
 409                                 if (subw < len)
 410                                         subw = len;
 411                         }
 412
 413                         for (subi = 0; subi < subn; subi++) {
 414                                 if (!perf_test__matches(test_description(t, subi),
 415                                                         curr, argc, argv))
 416                                         continue;
 417
 418                                 pr_info("%3d.%1d: %-*s:", i, subi + 1, subw,
 419                                         test_description(t, subi));
 420                                 test_and_print(t, subi);
 421                         }
 422                 }
 423         }
 424
 425         return run_shell_tests(argc, argv, i, width, skiplist);
 426 }
 427
 428 static int perf_test__list_shell(int argc, const char **argv, int i)
 429 {
 430         const struct script_file *files, *file;
 431
 432         files = list_script_files();
 433         if (!files)
 434                 return 0;
 435         for (file = files; file->dir; file++) {
 436                 int curr = i++;
 437                 struct test_suite t = {
 438                         .desc = file->desc
 439                 };
 440
 441                 if (!perf_test__matches(t.desc, curr, argc, argv))
 442                         continue;
 443
 444                 pr_info("%3d: %s\n", i, t.desc);
 445         }
 446         return 0;
 447 }
 448
 449 static int perf_test__list(int argc, const char **argv)
 450 {
 451         unsigned int j, k;
 452         struct test_suite *t;
 453         int i = 0;
 454
 455         for_each_test(j, k, t) {
 456                 int curr = i++;
 457
 458                 if (!perf_test__matches(test_description(t, -1), curr, argc, argv))
 459                         continue;
 460
 461                 pr_info("%3d: %s\n", i, test_description(t, -1));
 462
 463                 if (has_subtests(t)) {
 464                         int subn = num_subtests(t);
 465                         int subi;
 466
 467                         for (subi = 0; subi < subn; subi++)
 468                                 pr_info("%3d:%1d: %s\n", i, subi + 1,
 469                                         test_description(t, subi));
 470                 }
 471         }
 472
 473         perf_test__list_shell(argc, argv, i);
 474
 475         return 0;
 476 }
 477
 478 int cmd_test(int argc, const char **argv)
 479 {
 480         const char *test_usage[] = {
 481         "perf test [<options>] [{list <test-name-fragment>|[<test-name-fragments>|<test-numbers>]}]",
 482         NULL,
 483         };
 484         const char *skip = NULL;
 485         const struct option test_options[] = {
 486         OPT_STRING('s', "skip", &skip, "tests", "tests to skip"),
 487         OPT_INCR('v', "verbose", &verbose,
 488                     "be more verbose (show symbol address, etc)"),
 489         OPT_BOOLEAN('F', "dont-fork", &dont_fork,
 490                     "Do not fork for testcase"),
 491         OPT_END()
 492         };
 493         const char * const test_subcommands[] = { "list", NULL };
 494         struct intlist *skiplist = NULL;
 495         int ret = hists__init();
 496
 497         if (ret < 0)
 498                 return ret;
 499
 500         /* Unbuffered output */
 501         setvbuf(stdout, NULL, _IONBF, 0);
 502
 503         argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0);
 504         if (argc >= 1 && !strcmp(argv[0], "list"))
 505                 return perf_test__list(argc - 1, argv + 1);
 506
 507         symbol_conf.priv_size = sizeof(int);
 508         symbol_conf.sort_by_name = true;
 509         symbol_conf.try_vmlinux_path = true;
 510
 511         if (symbol__init(NULL) < 0)
 512                 return -1;
 513
 514         if (skip != NULL)
 515                 skiplist = intlist__new(skip);
 516         /*
 517          * Tests that create BPF maps, for instance, need more than the 64K
 518          * default:
 519          */
 520         rlimit__bump_memlock();
 521
 522         return __cmd_test(argc, argv, skiplist);
 523 }