tools/testing/selftests/bpf/bench.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2020 Facebook */
   3 #define _GNU_SOURCE
   4 #include <argp.h>
   5 #include <linux/compiler.h>
   6 #include <sys/time.h>
   7 #include <sched.h>
   8 #include <fcntl.h>
   9 #include <pthread.h>
  10 #include <sys/sysinfo.h>
  11 #include <signal.h>
  12 #include "bench.h"
  13 #include "testing_helpers.h"
  14
  15 struct env env = {
  16         .warmup_sec = 1,
  17         .duration_sec = 5,
  18         .affinity = false,
  19         .quiet = false,
  20         .consumer_cnt = 1,
  21         .producer_cnt = 1,
  22 };
  23
  24 static int libbpf_print_fn(enum libbpf_print_level level,
  25                     const char *format, va_list args)
  26 {
  27         if (level == LIBBPF_DEBUG && !env.verbose)
  28                 return 0;
  29         return vfprintf(stderr, format, args);
  30 }
  31
  32 void setup_libbpf(void)
  33 {
  34         libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
  35         libbpf_set_print(libbpf_print_fn);
  36 }
  37
  38 void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns)
  39 {
  40         long total = res->false_hits  + res->hits + res->drops;
  41
  42         printf("Iter %3d (%7.3lfus): ",
  43                iter, (delta_ns - 1000000000) / 1000.0);
  44
  45         printf("%ld false hits of %ld total operations. Percentage = %2.2f %%\n",
  46                res->false_hits, total, ((float)res->false_hits / total) * 100);
  47 }
  48
  49 void false_hits_report_final(struct bench_res res[], int res_cnt)
  50 {
  51         long total_hits = 0, total_drops = 0, total_false_hits = 0, total_ops = 0;
  52         int i;
  53
  54         for (i = 0; i < res_cnt; i++) {
  55                 total_hits += res[i].hits;
  56                 total_false_hits += res[i].false_hits;
  57                 total_drops += res[i].drops;
  58         }
  59         total_ops = total_hits + total_false_hits + total_drops;
  60
  61         printf("Summary: %ld false hits of %ld total operations. ",
  62                total_false_hits, total_ops);
  63         printf("Percentage =  %2.2f %%\n",
  64                ((float)total_false_hits / total_ops) * 100);
  65 }
  66
  67 void hits_drops_report_progress(int iter, struct bench_res *res, long delta_ns)
  68 {
  69         double hits_per_sec, drops_per_sec;
  70         double hits_per_prod;
  71
  72         hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
  73         hits_per_prod = hits_per_sec / env.producer_cnt;
  74         drops_per_sec = res->drops / 1000000.0 / (delta_ns / 1000000000.0);
  75
  76         printf("Iter %3d (%7.3lfus): ",
  77                iter, (delta_ns - 1000000000) / 1000.0);
  78
  79         printf("hits %8.3lfM/s (%7.3lfM/prod), drops %8.3lfM/s, total operations %8.3lfM/s\n",
  80                hits_per_sec, hits_per_prod, drops_per_sec, hits_per_sec + drops_per_sec);
  81 }
  82
  83 void
  84 grace_period_latency_basic_stats(struct bench_res res[], int res_cnt, struct basic_stats *gp_stat)
  85 {
  86         int i;
  87
  88         memset(gp_stat, 0, sizeof(struct basic_stats));
  89
  90         for (i = 0; i < res_cnt; i++)
  91                 gp_stat->mean += res[i].gp_ns / 1000.0 / (double)res[i].gp_ct / (0.0 + res_cnt);
  92
  93 #define IT_MEAN_DIFF (res[i].gp_ns / 1000.0 / (double)res[i].gp_ct - gp_stat->mean)
  94         if (res_cnt > 1) {
  95                 for (i = 0; i < res_cnt; i++)
  96                         gp_stat->stddev += (IT_MEAN_DIFF * IT_MEAN_DIFF) / (res_cnt - 1.0);
  97         }
  98         gp_stat->stddev = sqrt(gp_stat->stddev);
  99 #undef IT_MEAN_DIFF
 100 }
 101
 102 void
 103 grace_period_ticks_basic_stats(struct bench_res res[], int res_cnt, struct basic_stats *gp_stat)
 104 {
 105         int i;
 106
 107         memset(gp_stat, 0, sizeof(struct basic_stats));
 108         for (i = 0; i < res_cnt; i++)
 109                 gp_stat->mean += res[i].stime / (double)res[i].gp_ct / (0.0 + res_cnt);
 110
 111 #define IT_MEAN_DIFF (res[i].stime / (double)res[i].gp_ct - gp_stat->mean)
 112         if (res_cnt > 1) {
 113                 for (i = 0; i < res_cnt; i++)
 114                         gp_stat->stddev += (IT_MEAN_DIFF * IT_MEAN_DIFF) / (res_cnt - 1.0);
 115         }
 116         gp_stat->stddev = sqrt(gp_stat->stddev);
 117 #undef IT_MEAN_DIFF
 118 }
 119
 120 void hits_drops_report_final(struct bench_res res[], int res_cnt)
 121 {
 122         int i;
 123         double hits_mean = 0.0, drops_mean = 0.0, total_ops_mean = 0.0;
 124         double hits_stddev = 0.0, drops_stddev = 0.0, total_ops_stddev = 0.0;
 125         double total_ops;
 126
 127         for (i = 0; i < res_cnt; i++) {
 128                 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
 129                 drops_mean += res[i].drops / 1000000.0 / (0.0 + res_cnt);
 130         }
 131         total_ops_mean = hits_mean + drops_mean;
 132
 133         if (res_cnt > 1)  {
 134                 for (i = 0; i < res_cnt; i++) {
 135                         hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
 136                                        (hits_mean - res[i].hits / 1000000.0) /
 137                                        (res_cnt - 1.0);
 138                         drops_stddev += (drops_mean - res[i].drops / 1000000.0) *
 139                                         (drops_mean - res[i].drops / 1000000.0) /
 140                                         (res_cnt - 1.0);
 141                         total_ops = res[i].hits + res[i].drops;
 142                         total_ops_stddev += (total_ops_mean - total_ops / 1000000.0) *
 143                                         (total_ops_mean - total_ops / 1000000.0) /
 144                                         (res_cnt - 1.0);
 145                 }
 146                 hits_stddev = sqrt(hits_stddev);
 147                 drops_stddev = sqrt(drops_stddev);
 148                 total_ops_stddev = sqrt(total_ops_stddev);
 149         }
 150         printf("Summary: hits %8.3lf \u00B1 %5.3lfM/s (%7.3lfM/prod), ",
 151                hits_mean, hits_stddev, hits_mean / env.producer_cnt);
 152         printf("drops %8.3lf \u00B1 %5.3lfM/s, ",
 153                drops_mean, drops_stddev);
 154         printf("total operations %8.3lf \u00B1 %5.3lfM/s\n",
 155                total_ops_mean, total_ops_stddev);
 156 }
 157
 158 void ops_report_progress(int iter, struct bench_res *res, long delta_ns)
 159 {
 160         double hits_per_sec, hits_per_prod;
 161
 162         hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
 163         hits_per_prod = hits_per_sec / env.producer_cnt;
 164
 165         printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
 166
 167         printf("hits %8.3lfM/s (%7.3lfM/prod)\n", hits_per_sec, hits_per_prod);
 168 }
 169
 170 void ops_report_final(struct bench_res res[], int res_cnt)
 171 {
 172         double hits_mean = 0.0, hits_stddev = 0.0;
 173         int i;
 174
 175         for (i = 0; i < res_cnt; i++)
 176                 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
 177
 178         if (res_cnt > 1)  {
 179                 for (i = 0; i < res_cnt; i++)
 180                         hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
 181                                        (hits_mean - res[i].hits / 1000000.0) /
 182                                        (res_cnt - 1.0);
 183
 184                 hits_stddev = sqrt(hits_stddev);
 185         }
 186         printf("Summary: throughput %8.3lf \u00B1 %5.3lf M ops/s (%7.3lfM ops/prod), ",
 187                hits_mean, hits_stddev, hits_mean / env.producer_cnt);
 188         printf("latency %8.3lf ns/op\n", 1000.0 / hits_mean * env.producer_cnt);
 189 }
 190
 191 void local_storage_report_progress(int iter, struct bench_res *res,
 192                                    long delta_ns)
 193 {
 194         double important_hits_per_sec, hits_per_sec;
 195         double delta_sec = delta_ns / 1000000000.0;
 196
 197         hits_per_sec = res->hits / 1000000.0 / delta_sec;
 198         important_hits_per_sec = res->important_hits / 1000000.0 / delta_sec;
 199
 200         printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
 201
 202         printf("hits %8.3lfM/s ", hits_per_sec);
 203         printf("important_hits %8.3lfM/s\n", important_hits_per_sec);
 204 }
 205
 206 void local_storage_report_final(struct bench_res res[], int res_cnt)
 207 {
 208         double important_hits_mean = 0.0, important_hits_stddev = 0.0;
 209         double hits_mean = 0.0, hits_stddev = 0.0;
 210         int i;
 211
 212         for (i = 0; i < res_cnt; i++) {
 213                 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
 214                 important_hits_mean += res[i].important_hits / 1000000.0 / (0.0 + res_cnt);
 215         }
 216
 217         if (res_cnt > 1)  {
 218                 for (i = 0; i < res_cnt; i++) {
 219                         hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
 220                                        (hits_mean - res[i].hits / 1000000.0) /
 221                                        (res_cnt - 1.0);
 222                         important_hits_stddev +=
 223                                        (important_hits_mean - res[i].important_hits / 1000000.0) *
 224                                        (important_hits_mean - res[i].important_hits / 1000000.0) /
 225                                        (res_cnt - 1.0);
 226                 }
 227
 228                 hits_stddev = sqrt(hits_stddev);
 229                 important_hits_stddev = sqrt(important_hits_stddev);
 230         }
 231         printf("Summary: hits throughput %8.3lf \u00B1 %5.3lf M ops/s, ",
 232                hits_mean, hits_stddev);
 233         printf("hits latency %8.3lf ns/op, ", 1000.0 / hits_mean);
 234         printf("important_hits throughput %8.3lf \u00B1 %5.3lf M ops/s\n",
 235                important_hits_mean, important_hits_stddev);
 236 }
 237
 238 const char *argp_program_version = "benchmark";
 239 const char *argp_program_bug_address = "<[email protected]>";
 240 const char argp_program_doc[] =
 241 "benchmark    Generic benchmarking framework.\n"
 242 "\n"
 243 "This tool runs benchmarks.\n"
 244 "\n"
 245 "USAGE: benchmark <bench-name>\n"
 246 "\n"
 247 "EXAMPLES:\n"
 248 "    # run 'count-local' benchmark with 1 producer and 1 consumer\n"
 249 "    benchmark count-local\n"
 250 "    # run 'count-local' with 16 producer and 8 consumer thread, pinned to CPUs\n"
 251 "    benchmark -p16 -c8 -a count-local\n";
 252
 253 enum {
 254         ARG_PROD_AFFINITY_SET = 1000,
 255         ARG_CONS_AFFINITY_SET = 1001,
 256 };
 257
 258 static const struct argp_option opts[] = {
 259         { "list", 'l', NULL, 0, "List available benchmarks"},
 260         { "duration", 'd', "SEC", 0, "Duration of benchmark, seconds"},
 261         { "warmup", 'w', "SEC", 0, "Warm-up period, seconds"},
 262         { "producers", 'p', "NUM", 0, "Number of producer threads"},
 263         { "consumers", 'c', "NUM", 0, "Number of consumer threads"},
 264         { "verbose", 'v', NULL, 0, "Verbose debug output"},
 265         { "affinity", 'a', NULL, 0, "Set consumer/producer thread affinity"},
 266         { "quiet", 'q', NULL, 0, "Be more quiet"},
 267         { "prod-affinity", ARG_PROD_AFFINITY_SET, "CPUSET", 0,
 268           "Set of CPUs for producer threads; implies --affinity"},
 269         { "cons-affinity", ARG_CONS_AFFINITY_SET, "CPUSET", 0,
 270           "Set of CPUs for consumer threads; implies --affinity"},
 271         {},
 272 };
 273
 274 extern struct argp bench_ringbufs_argp;
 275 extern struct argp bench_bloom_map_argp;
 276 extern struct argp bench_bpf_loop_argp;
 277 extern struct argp bench_local_storage_argp;
 278 extern struct argp bench_local_storage_rcu_tasks_trace_argp;
 279 extern struct argp bench_strncmp_argp;
 280 extern struct argp bench_hashmap_lookup_argp;
 281 extern struct argp bench_local_storage_create_argp;
 282
 283 static const struct argp_child bench_parsers[] = {
 284         { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 },
 285         { &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 },
 286         { &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 },
 287         { &bench_local_storage_argp, 0, "local_storage benchmark", 0 },
 288         { &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 },
 289         { &bench_local_storage_rcu_tasks_trace_argp, 0,
 290                 "local_storage RCU Tasks Trace slowdown benchmark", 0 },
 291         { &bench_hashmap_lookup_argp, 0, "Hashmap lookup benchmark", 0 },
 292         { &bench_local_storage_create_argp, 0, "local-storage-create benchmark", 0 },
 293         {},
 294 };
 295
 296 /* Make pos_args global, so that we can run argp_parse twice, if necessary */
 297 static int pos_args;
 298
 299 static error_t parse_arg(int key, char *arg, struct argp_state *state)
 300 {
 301         switch (key) {
 302         case 'v':
 303                 env.verbose = true;
 304                 break;
 305         case 'l':
 306                 env.list = true;
 307                 break;
 308         case 'd':
 309                 env.duration_sec = strtol(arg, NULL, 10);
 310                 if (env.duration_sec <= 0) {
 311                         fprintf(stderr, "Invalid duration: %s\n", arg);
 312                         argp_usage(state);
 313                 }
 314                 break;
 315         case 'w':
 316                 env.warmup_sec = strtol(arg, NULL, 10);
 317                 if (env.warmup_sec <= 0) {
 318                         fprintf(stderr, "Invalid warm-up duration: %s\n", arg);
 319                         argp_usage(state);
 320                 }
 321                 break;
 322         case 'p':
 323                 env.producer_cnt = strtol(arg, NULL, 10);
 324                 if (env.producer_cnt <= 0) {
 325                         fprintf(stderr, "Invalid producer count: %s\n", arg);
 326                         argp_usage(state);
 327                 }
 328                 break;
 329         case 'c':
 330                 env.consumer_cnt = strtol(arg, NULL, 10);
 331                 if (env.consumer_cnt <= 0) {
 332                         fprintf(stderr, "Invalid consumer count: %s\n", arg);
 333                         argp_usage(state);
 334                 }
 335                 break;
 336         case 'a':
 337                 env.affinity = true;
 338                 break;
 339         case 'q':
 340                 env.quiet = true;
 341                 break;
 342         case ARG_PROD_AFFINITY_SET:
 343                 env.affinity = true;
 344                 if (parse_num_list(arg, &env.prod_cpus.cpus,
 345                                    &env.prod_cpus.cpus_len)) {
 346                         fprintf(stderr, "Invalid format of CPU set for producers.");
 347                         argp_usage(state);
 348                 }
 349                 break;
 350         case ARG_CONS_AFFINITY_SET:
 351                 env.affinity = true;
 352                 if (parse_num_list(arg, &env.cons_cpus.cpus,
 353                                    &env.cons_cpus.cpus_len)) {
 354                         fprintf(stderr, "Invalid format of CPU set for consumers.");
 355                         argp_usage(state);
 356                 }
 357                 break;
 358         case ARGP_KEY_ARG:
 359                 if (pos_args++) {
 360                         fprintf(stderr,
 361                                 "Unrecognized positional argument: %s\n", arg);
 362                         argp_usage(state);
 363                 }
 364                 env.bench_name = strdup(arg);
 365                 break;
 366         default:
 367                 return ARGP_ERR_UNKNOWN;
 368         }
 369         return 0;
 370 }
 371
 372 static void parse_cmdline_args_init(int argc, char **argv)
 373 {
 374         static const struct argp argp = {
 375                 .options = opts,
 376                 .parser = parse_arg,
 377                 .doc = argp_program_doc,
 378                 .children = bench_parsers,
 379         };
 380         if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
 381                 exit(1);
 382 }
 383
 384 static void parse_cmdline_args_final(int argc, char **argv)
 385 {
 386         struct argp_child bench_parsers[2] = {};
 387         const struct argp argp = {
 388                 .options = opts,
 389                 .parser = parse_arg,
 390                 .doc = argp_program_doc,
 391                 .children = bench_parsers,
 392         };
 393
 394         /* Parse arguments the second time with the correct set of parsers */
 395         if (bench->argp) {
 396                 bench_parsers[0].argp = bench->argp;
 397                 bench_parsers[0].header = bench->name;
 398                 pos_args = 0;
 399                 if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
 400                         exit(1);
 401         }
 402 }
 403
 404 static void collect_measurements(long delta_ns);
 405
 406 static __u64 last_time_ns;
 407 static void sigalarm_handler(int signo)
 408 {
 409         long new_time_ns = get_time_ns();
 410         long delta_ns = new_time_ns - last_time_ns;
 411
 412         collect_measurements(delta_ns);
 413
 414         last_time_ns = new_time_ns;
 415 }
 416
 417 /* set up periodic 1-second timer */
 418 static void setup_timer()
 419 {
 420         static struct sigaction sigalarm_action = {
 421                 .sa_handler = sigalarm_handler,
 422         };
 423         struct itimerval timer_settings = {};
 424         int err;
 425
 426         last_time_ns = get_time_ns();
 427         err = sigaction(SIGALRM, &sigalarm_action, NULL);
 428         if (err < 0) {
 429                 fprintf(stderr, "failed to install SIGALRM handler: %d\n", -errno);
 430                 exit(1);
 431         }
 432         timer_settings.it_interval.tv_sec = 1;
 433         timer_settings.it_value.tv_sec = 1;
 434         err = setitimer(ITIMER_REAL, &timer_settings, NULL);
 435         if (err < 0) {
 436                 fprintf(stderr, "failed to arm interval timer: %d\n", -errno);
 437                 exit(1);
 438         }
 439 }
 440
 441 static void set_thread_affinity(pthread_t thread, int cpu)
 442 {
 443         cpu_set_t cpuset;
 444
 445         CPU_ZERO(&cpuset);
 446         CPU_SET(cpu, &cpuset);
 447         if (pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset)) {
 448                 fprintf(stderr, "setting affinity to CPU #%d failed: %d\n",
 449                         cpu, errno);
 450                 exit(1);
 451         }
 452 }
 453
 454 static int next_cpu(struct cpu_set *cpu_set)
 455 {
 456         if (cpu_set->cpus) {
 457                 int i;
 458
 459                 /* find next available CPU */
 460                 for (i = cpu_set->next_cpu; i < cpu_set->cpus_len; i++) {
 461                         if (cpu_set->cpus[i]) {
 462                                 cpu_set->next_cpu = i + 1;
 463                                 return i;
 464                         }
 465                 }
 466                 fprintf(stderr, "Not enough CPUs specified, need CPU #%d or higher.\n", i);
 467                 exit(1);
 468         }
 469
 470         return cpu_set->next_cpu++;
 471 }
 472
 473 static struct bench_state {
 474         int res_cnt;
 475         struct bench_res *results;
 476         pthread_t *consumers;
 477         pthread_t *producers;
 478 } state;
 479
 480 const struct bench *bench = NULL;
 481
 482 extern const struct bench bench_count_global;
 483 extern const struct bench bench_count_local;
 484 extern const struct bench bench_rename_base;
 485 extern const struct bench bench_rename_kprobe;
 486 extern const struct bench bench_rename_kretprobe;
 487 extern const struct bench bench_rename_rawtp;
 488 extern const struct bench bench_rename_fentry;
 489 extern const struct bench bench_rename_fexit;
 490 extern const struct bench bench_trig_base;
 491 extern const struct bench bench_trig_tp;
 492 extern const struct bench bench_trig_rawtp;
 493 extern const struct bench bench_trig_kprobe;
 494 extern const struct bench bench_trig_fentry;
 495 extern const struct bench bench_trig_fentry_sleep;
 496 extern const struct bench bench_trig_fmodret;
 497 extern const struct bench bench_trig_uprobe_base;
 498 extern const struct bench bench_trig_uprobe_with_nop;
 499 extern const struct bench bench_trig_uretprobe_with_nop;
 500 extern const struct bench bench_trig_uprobe_without_nop;
 501 extern const struct bench bench_trig_uretprobe_without_nop;
 502 extern const struct bench bench_rb_libbpf;
 503 extern const struct bench bench_rb_custom;
 504 extern const struct bench bench_pb_libbpf;
 505 extern const struct bench bench_pb_custom;
 506 extern const struct bench bench_bloom_lookup;
 507 extern const struct bench bench_bloom_update;
 508 extern const struct bench bench_bloom_false_positive;
 509 extern const struct bench bench_hashmap_without_bloom;
 510 extern const struct bench bench_hashmap_with_bloom;
 511 extern const struct bench bench_bpf_loop;
 512 extern const struct bench bench_strncmp_no_helper;
 513 extern const struct bench bench_strncmp_helper;
 514 extern const struct bench bench_bpf_hashmap_full_update;
 515 extern const struct bench bench_local_storage_cache_seq_get;
 516 extern const struct bench bench_local_storage_cache_interleaved_get;
 517 extern const struct bench bench_local_storage_cache_hashmap_control;
 518 extern const struct bench bench_local_storage_tasks_trace;
 519 extern const struct bench bench_bpf_hashmap_lookup;
 520 extern const struct bench bench_local_storage_create;
 521
 522 static const struct bench *benchs[] = {
 523         &bench_count_global,
 524         &bench_count_local,
 525         &bench_rename_base,
 526         &bench_rename_kprobe,
 527         &bench_rename_kretprobe,
 528         &bench_rename_rawtp,
 529         &bench_rename_fentry,
 530         &bench_rename_fexit,
 531         &bench_trig_base,
 532         &bench_trig_tp,
 533         &bench_trig_rawtp,
 534         &bench_trig_kprobe,
 535         &bench_trig_fentry,
 536         &bench_trig_fentry_sleep,
 537         &bench_trig_fmodret,
 538         &bench_trig_uprobe_base,
 539         &bench_trig_uprobe_with_nop,
 540         &bench_trig_uretprobe_with_nop,
 541         &bench_trig_uprobe_without_nop,
 542         &bench_trig_uretprobe_without_nop,
 543         &bench_rb_libbpf,
 544         &bench_rb_custom,
 545         &bench_pb_libbpf,
 546         &bench_pb_custom,
 547         &bench_bloom_lookup,
 548         &bench_bloom_update,
 549         &bench_bloom_false_positive,
 550         &bench_hashmap_without_bloom,
 551         &bench_hashmap_with_bloom,
 552         &bench_bpf_loop,
 553         &bench_strncmp_no_helper,
 554         &bench_strncmp_helper,
 555         &bench_bpf_hashmap_full_update,
 556         &bench_local_storage_cache_seq_get,
 557         &bench_local_storage_cache_interleaved_get,
 558         &bench_local_storage_cache_hashmap_control,
 559         &bench_local_storage_tasks_trace,
 560         &bench_bpf_hashmap_lookup,
 561         &bench_local_storage_create,
 562 };
 563
 564 static void find_benchmark(void)
 565 {
 566         int i;
 567
 568         if (!env.bench_name) {
 569                 fprintf(stderr, "benchmark name is not specified\n");
 570                 exit(1);
 571         }
 572         for (i = 0; i < ARRAY_SIZE(benchs); i++) {
 573                 if (strcmp(benchs[i]->name, env.bench_name) == 0) {
 574                         bench = benchs[i];
 575                         break;
 576                 }
 577         }
 578         if (!bench) {
 579                 fprintf(stderr, "benchmark '%s' not found\n", env.bench_name);
 580                 exit(1);
 581         }
 582 }
 583
 584 static void setup_benchmark(void)
 585 {
 586         int i, err;
 587
 588         if (!env.quiet)
 589                 printf("Setting up benchmark '%s'...\n", bench->name);
 590
 591         state.producers = calloc(env.producer_cnt, sizeof(*state.producers));
 592         state.consumers = calloc(env.consumer_cnt, sizeof(*state.consumers));
 593         state.results = calloc(env.duration_sec + env.warmup_sec + 2,
 594                                sizeof(*state.results));
 595         if (!state.producers || !state.consumers || !state.results)
 596                 exit(1);
 597
 598         if (bench->validate)
 599                 bench->validate();
 600         if (bench->setup)
 601                 bench->setup();
 602
 603         for (i = 0; i < env.consumer_cnt; i++) {
 604                 err = pthread_create(&state.consumers[i], NULL,
 605                                      bench->consumer_thread, (void *)(long)i);
 606                 if (err) {
 607                         fprintf(stderr, "failed to create consumer thread #%d: %d\n",
 608                                 i, -errno);
 609                         exit(1);
 610                 }
 611                 if (env.affinity)
 612                         set_thread_affinity(state.consumers[i],
 613                                             next_cpu(&env.cons_cpus));
 614         }
 615
 616         /* unless explicit producer CPU list is specified, continue after
 617          * last consumer CPU
 618          */
 619         if (!env.prod_cpus.cpus)
 620                 env.prod_cpus.next_cpu = env.cons_cpus.next_cpu;
 621
 622         for (i = 0; i < env.producer_cnt; i++) {
 623                 err = pthread_create(&state.producers[i], NULL,
 624                                      bench->producer_thread, (void *)(long)i);
 625                 if (err) {
 626                         fprintf(stderr, "failed to create producer thread #%d: %d\n",
 627                                 i, -errno);
 628                         exit(1);
 629                 }
 630                 if (env.affinity)
 631                         set_thread_affinity(state.producers[i],
 632                                             next_cpu(&env.prod_cpus));
 633         }
 634
 635         if (!env.quiet)
 636                 printf("Benchmark '%s' started.\n", bench->name);
 637 }
 638
 639 static pthread_mutex_t bench_done_mtx = PTHREAD_MUTEX_INITIALIZER;
 640 static pthread_cond_t bench_done = PTHREAD_COND_INITIALIZER;
 641
 642 static void collect_measurements(long delta_ns) {
 643         int iter = state.res_cnt++;
 644         struct bench_res *res = &state.results[iter];
 645
 646         bench->measure(res);
 647
 648         if (bench->report_progress)
 649                 bench->report_progress(iter, res, delta_ns);
 650
 651         if (iter == env.duration_sec + env.warmup_sec) {
 652                 pthread_mutex_lock(&bench_done_mtx);
 653                 pthread_cond_signal(&bench_done);
 654                 pthread_mutex_unlock(&bench_done_mtx);
 655         }
 656 }
 657
 658 int main(int argc, char **argv)
 659 {
 660         parse_cmdline_args_init(argc, argv);
 661
 662         if (env.list) {
 663                 int i;
 664
 665                 printf("Available benchmarks:\n");
 666                 for (i = 0; i < ARRAY_SIZE(benchs); i++) {
 667                         printf("- %s\n", benchs[i]->name);
 668                 }
 669                 return 0;
 670         }
 671
 672         find_benchmark();
 673         parse_cmdline_args_final(argc, argv);
 674
 675         setup_benchmark();
 676
 677         setup_timer();
 678
 679         pthread_mutex_lock(&bench_done_mtx);
 680         pthread_cond_wait(&bench_done, &bench_done_mtx);
 681         pthread_mutex_unlock(&bench_done_mtx);
 682
 683         if (bench->report_final)
 684                 /* skip first sample */
 685                 bench->report_final(state.results + env.warmup_sec,
 686                                     state.res_cnt - env.warmup_sec);
 687
 688         return 0;
 689 }