tools/perf/builtin-c2c.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This is rewrite of original c2c tool introduced in here:
   4  *   http://lwn.net/Articles/588866/
   5  *
   6  * The original tool was changed to fit in current perf state.
   7  *
   8  * Original authors:
   9  *   Don Zickus <[email protected]>
  10  *   Dick Fowles <[email protected]>
  11  *   Joe Mario <[email protected]>
  12  */
  13 #include <errno.h>
  14 #include <inttypes.h>
  15 #include <linux/compiler.h>
  16 #include <linux/err.h>
  17 #include <linux/kernel.h>
  18 #include <linux/stringify.h>
  19 #include <linux/zalloc.h>
  20 #include <asm/bug.h>
  21 #include <sys/param.h>
  22 #include "debug.h"
  23 #include "builtin.h"
  24 #include <perf/cpumap.h>
  25 #include <subcmd/pager.h>
  26 #include <subcmd/parse-options.h>
  27 #include "map_symbol.h"
  28 #include "mem-events.h"
  29 #include "session.h"
  30 #include "hist.h"
  31 #include "sort.h"
  32 #include "tool.h"
  33 #include "cacheline.h"
  34 #include "data.h"
  35 #include "event.h"
  36 #include "evlist.h"
  37 #include "evsel.h"
  38 #include "ui/browsers/hists.h"
  39 #include "thread.h"
  40 #include "mem2node.h"
  41 #include "mem-info.h"
  42 #include "symbol.h"
  43 #include "ui/ui.h"
  44 #include "ui/progress.h"
  45 #include "pmus.h"
  46 #include "string2.h"
  47 #include "util/util.h"
  48
  49 struct c2c_hists {
  50         struct hists            hists;
  51         struct perf_hpp_list    list;
  52         struct c2c_stats        stats;
  53 };
  54
  55 struct compute_stats {
  56         struct stats             lcl_hitm;
  57         struct stats             rmt_hitm;
  58         struct stats             lcl_peer;
  59         struct stats             rmt_peer;
  60         struct stats             load;
  61 };
  62
  63 struct c2c_hist_entry {
  64         struct c2c_hists        *hists;
  65         struct c2c_stats         stats;
  66         unsigned long           *cpuset;
  67         unsigned long           *nodeset;
  68         struct c2c_stats        *node_stats;
  69         unsigned int             cacheline_idx;
  70
  71         struct compute_stats     cstats;
  72
  73         unsigned long            paddr;
  74         unsigned long            paddr_cnt;
  75         bool                     paddr_zero;
  76         char                    *nodestr;
  77
  78         /*
  79          * must be at the end,
  80          * because of its callchain dynamic entry
  81          */
  82         struct hist_entry       he;
  83 };
  84
  85 static char const *coalesce_default = "iaddr";
  86
  87 struct perf_c2c {
  88         struct perf_tool        tool;
  89         struct c2c_hists        hists;
  90         struct mem2node         mem2node;
  91
  92         unsigned long           **nodes;
  93         int                      nodes_cnt;
  94         int                      cpus_cnt;
  95         int                     *cpu2node;
  96         int                      node_info;
  97
  98         bool                     show_src;
  99         bool                     show_all;
 100         bool                     use_stdio;
 101         bool                     stats_only;
 102         bool                     symbol_full;
 103         bool                     stitch_lbr;
 104
 105         /* Shared cache line stats */
 106         struct c2c_stats        shared_clines_stats;
 107         int                     shared_clines;
 108
 109         int                      display;
 110
 111         const char              *coalesce;
 112         char                    *cl_sort;
 113         char                    *cl_resort;
 114         char                    *cl_output;
 115 };
 116
 117 enum {
 118         DISPLAY_LCL_HITM,
 119         DISPLAY_RMT_HITM,
 120         DISPLAY_TOT_HITM,
 121         DISPLAY_SNP_PEER,
 122         DISPLAY_MAX,
 123 };
 124
 125 static const char *display_str[DISPLAY_MAX] = {
 126         [DISPLAY_LCL_HITM] = "Local HITMs",
 127         [DISPLAY_RMT_HITM] = "Remote HITMs",
 128         [DISPLAY_TOT_HITM] = "Total HITMs",
 129         [DISPLAY_SNP_PEER] = "Peer Snoop",
 130 };
 131
 132 static const struct option c2c_options[] = {
 133         OPT_INCR('v', "verbose", &verbose, "be more verbose (show counter open errors, etc)"),
 134         OPT_END()
 135 };
 136
 137 static struct perf_c2c c2c;
 138
 139 static void *c2c_he_zalloc(size_t size)
 140 {
 141         struct c2c_hist_entry *c2c_he;
 142
 143         c2c_he = zalloc(size + sizeof(*c2c_he));
 144         if (!c2c_he)
 145                 return NULL;
 146
 147         c2c_he->cpuset = bitmap_zalloc(c2c.cpus_cnt);
 148         if (!c2c_he->cpuset)
 149                 goto out_free;
 150
 151         c2c_he->nodeset = bitmap_zalloc(c2c.nodes_cnt);
 152         if (!c2c_he->nodeset)
 153                 goto out_free;
 154
 155         c2c_he->node_stats = zalloc(c2c.nodes_cnt * sizeof(*c2c_he->node_stats));
 156         if (!c2c_he->node_stats)
 157                 goto out_free;
 158
 159         init_stats(&c2c_he->cstats.lcl_hitm);
 160         init_stats(&c2c_he->cstats.rmt_hitm);
 161         init_stats(&c2c_he->cstats.lcl_peer);
 162         init_stats(&c2c_he->cstats.rmt_peer);
 163         init_stats(&c2c_he->cstats.load);
 164
 165         return &c2c_he->he;
 166
 167 out_free:
 168         zfree(&c2c_he->nodeset);
 169         zfree(&c2c_he->cpuset);
 170         free(c2c_he);
 171         return NULL;
 172 }
 173
 174 static void c2c_he_free(void *he)
 175 {
 176         struct c2c_hist_entry *c2c_he;
 177
 178         c2c_he = container_of(he, struct c2c_hist_entry, he);
 179         if (c2c_he->hists) {
 180                 hists__delete_entries(&c2c_he->hists->hists);
 181                 zfree(&c2c_he->hists);
 182         }
 183
 184         zfree(&c2c_he->cpuset);
 185         zfree(&c2c_he->nodeset);
 186         zfree(&c2c_he->nodestr);
 187         zfree(&c2c_he->node_stats);
 188         free(c2c_he);
 189 }
 190
 191 static struct hist_entry_ops c2c_entry_ops = {
 192         .new    = c2c_he_zalloc,
 193         .free   = c2c_he_free,
 194 };
 195
 196 static int c2c_hists__init(struct c2c_hists *hists,
 197                            const char *sort,
 198                            int nr_header_lines);
 199
 200 static struct c2c_hists*
 201 he__get_c2c_hists(struct hist_entry *he,
 202                   const char *sort,
 203                   int nr_header_lines)
 204 {
 205         struct c2c_hist_entry *c2c_he;
 206         struct c2c_hists *hists;
 207         int ret;
 208
 209         c2c_he = container_of(he, struct c2c_hist_entry, he);
 210         if (c2c_he->hists)
 211                 return c2c_he->hists;
 212
 213         hists = c2c_he->hists = zalloc(sizeof(*hists));
 214         if (!hists)
 215                 return NULL;
 216
 217         ret = c2c_hists__init(hists, sort, nr_header_lines);
 218         if (ret) {
 219                 free(hists);
 220                 return NULL;
 221         }
 222
 223         return hists;
 224 }
 225
 226 static void c2c_he__set_cpu(struct c2c_hist_entry *c2c_he,
 227                             struct perf_sample *sample)
 228 {
 229         if (WARN_ONCE(sample->cpu == (unsigned int) -1,
 230                       "WARNING: no sample cpu value"))
 231                 return;
 232
 233         __set_bit(sample->cpu, c2c_he->cpuset);
 234 }
 235
 236 static void c2c_he__set_node(struct c2c_hist_entry *c2c_he,
 237                              struct perf_sample *sample)
 238 {
 239         int node;
 240
 241         if (!sample->phys_addr) {
 242                 c2c_he->paddr_zero = true;
 243                 return;
 244         }
 245
 246         node = mem2node__node(&c2c.mem2node, sample->phys_addr);
 247         if (WARN_ONCE(node < 0, "WARNING: failed to find node\n"))
 248                 return;
 249
 250         __set_bit(node, c2c_he->nodeset);
 251
 252         if (c2c_he->paddr != sample->phys_addr) {
 253                 c2c_he->paddr_cnt++;
 254                 c2c_he->paddr = sample->phys_addr;
 255         }
 256 }
 257
 258 static void compute_stats(struct c2c_hist_entry *c2c_he,
 259                           struct c2c_stats *stats,
 260                           u64 weight)
 261 {
 262         struct compute_stats *cstats = &c2c_he->cstats;
 263
 264         if (stats->rmt_hitm)
 265                 update_stats(&cstats->rmt_hitm, weight);
 266         else if (stats->lcl_hitm)
 267                 update_stats(&cstats->lcl_hitm, weight);
 268         else if (stats->rmt_peer)
 269                 update_stats(&cstats->rmt_peer, weight);
 270         else if (stats->lcl_peer)
 271                 update_stats(&cstats->lcl_peer, weight);
 272         else if (stats->load)
 273                 update_stats(&cstats->load, weight);
 274 }
 275
 276 static int process_sample_event(struct perf_tool *tool __maybe_unused,
 277                                 union perf_event *event,
 278                                 struct perf_sample *sample,
 279                                 struct evsel *evsel,
 280                                 struct machine *machine)
 281 {
 282         struct c2c_hists *c2c_hists = &c2c.hists;
 283         struct c2c_hist_entry *c2c_he;
 284         struct c2c_stats stats = { .nr_entries = 0, };
 285         struct hist_entry *he;
 286         struct addr_location al;
 287         struct mem_info *mi, *mi_dup;
 288         struct callchain_cursor *cursor;
 289         int ret;
 290
 291         addr_location__init(&al);
 292         if (machine__resolve(machine, &al, sample) < 0) {
 293                 pr_debug("problem processing %d event, skipping it.\n",
 294                          event->header.type);
 295                 ret = -1;
 296                 goto out;
 297         }
 298
 299         if (c2c.stitch_lbr)
 300                 thread__set_lbr_stitch_enable(al.thread, true);
 301
 302         cursor = get_tls_callchain_cursor();
 303         ret = sample__resolve_callchain(sample, cursor, NULL,
 304                                         evsel, &al, sysctl_perf_event_max_stack);
 305         if (ret)
 306                 goto out;
 307
 308         mi = sample__resolve_mem(sample, &al);
 309         if (mi == NULL) {
 310                 ret = -ENOMEM;
 311                 goto out;
 312         }
 313
 314         /*
 315          * The mi object is released in hists__add_entry_ops,
 316          * if it gets sorted out into existing data, so we need
 317          * to take the copy now.
 318          */
 319         mi_dup = mem_info__get(mi);
 320
 321         c2c_decode_stats(&stats, mi);
 322
 323         he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops,
 324                                   &al, NULL, NULL, mi, NULL,
 325                                   sample, true);
 326         if (he == NULL)
 327                 goto free_mi;
 328
 329         c2c_he = container_of(he, struct c2c_hist_entry, he);
 330         c2c_add_stats(&c2c_he->stats, &stats);
 331         c2c_add_stats(&c2c_hists->stats, &stats);
 332
 333         c2c_he__set_cpu(c2c_he, sample);
 334         c2c_he__set_node(c2c_he, sample);
 335
 336         hists__inc_nr_samples(&c2c_hists->hists, he->filtered);
 337         ret = hist_entry__append_callchain(he, sample);
 338
 339         if (!ret) {
 340                 /*
 341                  * There's already been warning about missing
 342                  * sample's cpu value. Let's account all to
 343                  * node 0 in this case, without any further
 344                  * warning.
 345                  *
 346                  * Doing node stats only for single callchain data.
 347                  */
 348                 int cpu = sample->cpu == (unsigned int) -1 ? 0 : sample->cpu;
 349                 int node = c2c.cpu2node[cpu];
 350
 351                 mi = mi_dup;
 352
 353                 c2c_hists = he__get_c2c_hists(he, c2c.cl_sort, 2);
 354                 if (!c2c_hists)
 355                         goto free_mi;
 356
 357                 he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops,
 358                                           &al, NULL, NULL, mi, NULL,
 359                                           sample, true);
 360                 if (he == NULL)
 361                         goto free_mi;
 362
 363                 c2c_he = container_of(he, struct c2c_hist_entry, he);
 364                 c2c_add_stats(&c2c_he->stats, &stats);
 365                 c2c_add_stats(&c2c_hists->stats, &stats);
 366                 c2c_add_stats(&c2c_he->node_stats[node], &stats);
 367
 368                 compute_stats(c2c_he, &stats, sample->weight);
 369
 370                 c2c_he__set_cpu(c2c_he, sample);
 371                 c2c_he__set_node(c2c_he, sample);
 372
 373                 hists__inc_nr_samples(&c2c_hists->hists, he->filtered);
 374                 ret = hist_entry__append_callchain(he, sample);
 375         }
 376
 377 out:
 378         addr_location__exit(&al);
 379         return ret;
 380
 381 free_mi:
 382         mem_info__put(mi_dup);
 383         mem_info__put(mi);
 384         ret = -ENOMEM;
 385         goto out;
 386 }
 387
 388 static struct perf_c2c c2c = {
 389         .tool = {
 390                 .sample         = process_sample_event,
 391                 .mmap           = perf_event__process_mmap,
 392                 .mmap2          = perf_event__process_mmap2,
 393                 .comm           = perf_event__process_comm,
 394                 .exit           = perf_event__process_exit,
 395                 .fork           = perf_event__process_fork,
 396                 .lost           = perf_event__process_lost,
 397                 .attr           = perf_event__process_attr,
 398                 .auxtrace_info  = perf_event__process_auxtrace_info,
 399                 .auxtrace       = perf_event__process_auxtrace,
 400                 .auxtrace_error = perf_event__process_auxtrace_error,
 401                 .ordered_events = true,
 402                 .ordering_requires_timestamps = true,
 403         },
 404 };
 405
 406 static const char * const c2c_usage[] = {
 407         "perf c2c {record|report}",
 408         NULL
 409 };
 410
 411 static const char * const __usage_report[] = {
 412         "perf c2c report",
 413         NULL
 414 };
 415
 416 static const char * const *report_c2c_usage = __usage_report;
 417
 418 #define C2C_HEADER_MAX 2
 419
 420 struct c2c_header {
 421         struct {
 422                 const char *text;
 423                 int         span;
 424         } line[C2C_HEADER_MAX];
 425 };
 426
 427 struct c2c_dimension {
 428         struct c2c_header        header;
 429         const char              *name;
 430         int                      width;
 431         struct sort_entry       *se;
 432
 433         int64_t (*cmp)(struct perf_hpp_fmt *fmt,
 434                        struct hist_entry *, struct hist_entry *);
 435         int   (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 436                        struct hist_entry *he);
 437         int   (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 438                        struct hist_entry *he);
 439 };
 440
 441 struct c2c_fmt {
 442         struct perf_hpp_fmt      fmt;
 443         struct c2c_dimension    *dim;
 444 };
 445
 446 #define SYMBOL_WIDTH 30
 447
 448 static struct c2c_dimension dim_symbol;
 449 static struct c2c_dimension dim_srcline;
 450
 451 static int symbol_width(struct hists *hists, struct sort_entry *se)
 452 {
 453         int width = hists__col_len(hists, se->se_width_idx);
 454
 455         if (!c2c.symbol_full)
 456                 width = MIN(width, SYMBOL_WIDTH);
 457
 458         return width;
 459 }
 460
 461 static int c2c_width(struct perf_hpp_fmt *fmt,
 462                      struct perf_hpp *hpp __maybe_unused,
 463                      struct hists *hists)
 464 {
 465         struct c2c_fmt *c2c_fmt;
 466         struct c2c_dimension *dim;
 467
 468         c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
 469         dim = c2c_fmt->dim;
 470
 471         if (dim == &dim_symbol || dim == &dim_srcline)
 472                 return symbol_width(hists, dim->se);
 473
 474         return dim->se ? hists__col_len(hists, dim->se->se_width_idx) :
 475                          c2c_fmt->dim->width;
 476 }
 477
 478 static int c2c_header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 479                       struct hists *hists, int line, int *span)
 480 {
 481         struct perf_hpp_list *hpp_list = hists->hpp_list;
 482         struct c2c_fmt *c2c_fmt;
 483         struct c2c_dimension *dim;
 484         const char *text = NULL;
 485         int width = c2c_width(fmt, hpp, hists);
 486
 487         c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
 488         dim = c2c_fmt->dim;
 489
 490         if (dim->se) {
 491                 text = dim->header.line[line].text;
 492                 /* Use the last line from sort_entry if not defined. */
 493                 if (!text && (line == hpp_list->nr_header_lines - 1))
 494                         text = dim->se->se_header;
 495         } else {
 496                 text = dim->header.line[line].text;
 497
 498                 if (*span) {
 499                         (*span)--;
 500                         return 0;
 501                 } else {
 502                         *span = dim->header.line[line].span;
 503                 }
 504         }
 505
 506         if (text == NULL)
 507                 text = "";
 508
 509         return scnprintf(hpp->buf, hpp->size, "%*s", width, text);
 510 }
 511
 512 #define HEX_STR(__s, __v)                               \
 513 ({                                                      \
 514         scnprintf(__s, sizeof(__s), "0x%" PRIx64, __v); \
 515         __s;                                            \
 516 })
 517
 518 static int64_t
 519 dcacheline_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 520                struct hist_entry *left, struct hist_entry *right)
 521 {
 522         return sort__dcacheline_cmp(left, right);
 523 }
 524
 525 static int dcacheline_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 526                             struct hist_entry *he)
 527 {
 528         uint64_t addr = 0;
 529         int width = c2c_width(fmt, hpp, he->hists);
 530         char buf[20];
 531
 532         if (he->mem_info)
 533                 addr = cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl);
 534
 535         return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
 536 }
 537
 538 static int
 539 dcacheline_node_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 540                       struct hist_entry *he)
 541 {
 542         struct c2c_hist_entry *c2c_he;
 543         int width = c2c_width(fmt, hpp, he->hists);
 544
 545         c2c_he = container_of(he, struct c2c_hist_entry, he);
 546         if (WARN_ON_ONCE(!c2c_he->nodestr))
 547                 return 0;
 548
 549         return scnprintf(hpp->buf, hpp->size, "%*s", width, c2c_he->nodestr);
 550 }
 551
 552 static int
 553 dcacheline_node_count(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 554                       struct hist_entry *he)
 555 {
 556         struct c2c_hist_entry *c2c_he;
 557         int width = c2c_width(fmt, hpp, he->hists);
 558
 559         c2c_he = container_of(he, struct c2c_hist_entry, he);
 560         return scnprintf(hpp->buf, hpp->size, "%*lu", width, c2c_he->paddr_cnt);
 561 }
 562
 563 static int offset_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 564                         struct hist_entry *he)
 565 {
 566         uint64_t addr = 0;
 567         int width = c2c_width(fmt, hpp, he->hists);
 568         char buf[20];
 569
 570         if (he->mem_info)
 571                 addr = cl_offset(mem_info__daddr(he->mem_info)->al_addr, chk_double_cl);
 572
 573         return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
 574 }
 575
 576 static int64_t
 577 offset_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 578            struct hist_entry *left, struct hist_entry *right)
 579 {
 580         uint64_t l = 0, r = 0;
 581
 582         if (left->mem_info)
 583                 l = cl_offset(mem_info__daddr(left->mem_info)->addr, chk_double_cl);
 584
 585         if (right->mem_info)
 586                 r = cl_offset(mem_info__daddr(right->mem_info)->addr, chk_double_cl);
 587
 588         return (int64_t)(r - l);
 589 }
 590
 591 static int
 592 iaddr_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 593             struct hist_entry *he)
 594 {
 595         uint64_t addr = 0;
 596         int width = c2c_width(fmt, hpp, he->hists);
 597         char buf[20];
 598
 599         if (he->mem_info)
 600                 addr = mem_info__iaddr(he->mem_info)->addr;
 601
 602         return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
 603 }
 604
 605 static int64_t
 606 iaddr_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 607           struct hist_entry *left, struct hist_entry *right)
 608 {
 609         return sort__iaddr_cmp(left, right);
 610 }
 611
 612 static int
 613 tot_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 614                struct hist_entry *he)
 615 {
 616         struct c2c_hist_entry *c2c_he;
 617         int width = c2c_width(fmt, hpp, he->hists);
 618         unsigned int tot_hitm;
 619
 620         c2c_he = container_of(he, struct c2c_hist_entry, he);
 621         tot_hitm = c2c_he->stats.lcl_hitm + c2c_he->stats.rmt_hitm;
 622
 623         return scnprintf(hpp->buf, hpp->size, "%*u", width, tot_hitm);
 624 }
 625
 626 static int64_t
 627 tot_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 628              struct hist_entry *left, struct hist_entry *right)
 629 {
 630         struct c2c_hist_entry *c2c_left;
 631         struct c2c_hist_entry *c2c_right;
 632         uint64_t tot_hitm_left;
 633         uint64_t tot_hitm_right;
 634
 635         c2c_left  = container_of(left, struct c2c_hist_entry, he);
 636         c2c_right = container_of(right, struct c2c_hist_entry, he);
 637
 638         tot_hitm_left  = c2c_left->stats.lcl_hitm + c2c_left->stats.rmt_hitm;
 639         tot_hitm_right = c2c_right->stats.lcl_hitm + c2c_right->stats.rmt_hitm;
 640
 641         return tot_hitm_left - tot_hitm_right;
 642 }
 643
 644 #define STAT_FN_ENTRY(__f)                                      \
 645 static int                                                      \
 646 __f ## _entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,   \
 647               struct hist_entry *he)                            \
 648 {                                                               \
 649         struct c2c_hist_entry *c2c_he;                          \
 650         int width = c2c_width(fmt, hpp, he->hists);             \
 651                                                                 \
 652         c2c_he = container_of(he, struct c2c_hist_entry, he);   \
 653         return scnprintf(hpp->buf, hpp->size, "%*u", width,     \
 654                          c2c_he->stats.__f);                    \
 655 }
 656
 657 #define STAT_FN_CMP(__f)                                                \
 658 static int64_t                                                          \
 659 __f ## _cmp(struct perf_hpp_fmt *fmt __maybe_unused,                    \
 660             struct hist_entry *left, struct hist_entry *right)          \
 661 {                                                                       \
 662         struct c2c_hist_entry *c2c_left, *c2c_right;                    \
 663                                                                         \
 664         c2c_left  = container_of(left, struct c2c_hist_entry, he);      \
 665         c2c_right = container_of(right, struct c2c_hist_entry, he);     \
 666         return (uint64_t) c2c_left->stats.__f -                         \
 667                (uint64_t) c2c_right->stats.__f;                         \
 668 }
 669
 670 #define STAT_FN(__f)            \
 671         STAT_FN_ENTRY(__f)      \
 672         STAT_FN_CMP(__f)
 673
 674 STAT_FN(rmt_hitm)
 675 STAT_FN(lcl_hitm)
 676 STAT_FN(rmt_peer)
 677 STAT_FN(lcl_peer)
 678 STAT_FN(tot_peer)
 679 STAT_FN(store)
 680 STAT_FN(st_l1hit)
 681 STAT_FN(st_l1miss)
 682 STAT_FN(st_na)
 683 STAT_FN(ld_fbhit)
 684 STAT_FN(ld_l1hit)
 685 STAT_FN(ld_l2hit)
 686 STAT_FN(ld_llchit)
 687 STAT_FN(rmt_hit)
 688
 689 static uint64_t get_load_llc_misses(struct c2c_stats *stats)
 690 {
 691         return stats->lcl_dram +
 692                stats->rmt_dram +
 693                stats->rmt_hitm +
 694                stats->rmt_hit;
 695 }
 696
 697 static uint64_t get_load_cache_hits(struct c2c_stats *stats)
 698 {
 699         return stats->ld_fbhit +
 700                stats->ld_l1hit +
 701                stats->ld_l2hit +
 702                stats->ld_llchit +
 703                stats->lcl_hitm;
 704 }
 705
 706 static uint64_t get_stores(struct c2c_stats *stats)
 707 {
 708         return stats->st_l1hit +
 709                stats->st_l1miss +
 710                stats->st_na;
 711 }
 712
 713 static uint64_t total_records(struct c2c_stats *stats)
 714 {
 715         return get_load_llc_misses(stats) +
 716                get_load_cache_hits(stats) +
 717                get_stores(stats);
 718 }
 719
 720 static int
 721 tot_recs_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 722                 struct hist_entry *he)
 723 {
 724         struct c2c_hist_entry *c2c_he;
 725         int width = c2c_width(fmt, hpp, he->hists);
 726         uint64_t tot_recs;
 727
 728         c2c_he = container_of(he, struct c2c_hist_entry, he);
 729         tot_recs = total_records(&c2c_he->stats);
 730
 731         return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, tot_recs);
 732 }
 733
 734 static int64_t
 735 tot_recs_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 736              struct hist_entry *left, struct hist_entry *right)
 737 {
 738         struct c2c_hist_entry *c2c_left;
 739         struct c2c_hist_entry *c2c_right;
 740         uint64_t tot_recs_left;
 741         uint64_t tot_recs_right;
 742
 743         c2c_left  = container_of(left, struct c2c_hist_entry, he);
 744         c2c_right = container_of(right, struct c2c_hist_entry, he);
 745
 746         tot_recs_left  = total_records(&c2c_left->stats);
 747         tot_recs_right = total_records(&c2c_right->stats);
 748
 749         return tot_recs_left - tot_recs_right;
 750 }
 751
 752 static uint64_t total_loads(struct c2c_stats *stats)
 753 {
 754         return get_load_llc_misses(stats) +
 755                get_load_cache_hits(stats);
 756 }
 757
 758 static int
 759 tot_loads_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 760                 struct hist_entry *he)
 761 {
 762         struct c2c_hist_entry *c2c_he;
 763         int width = c2c_width(fmt, hpp, he->hists);
 764         uint64_t tot_recs;
 765
 766         c2c_he = container_of(he, struct c2c_hist_entry, he);
 767         tot_recs = total_loads(&c2c_he->stats);
 768
 769         return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, tot_recs);
 770 }
 771
 772 static int64_t
 773 tot_loads_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 774               struct hist_entry *left, struct hist_entry *right)
 775 {
 776         struct c2c_hist_entry *c2c_left;
 777         struct c2c_hist_entry *c2c_right;
 778         uint64_t tot_recs_left;
 779         uint64_t tot_recs_right;
 780
 781         c2c_left  = container_of(left, struct c2c_hist_entry, he);
 782         c2c_right = container_of(right, struct c2c_hist_entry, he);
 783
 784         tot_recs_left  = total_loads(&c2c_left->stats);
 785         tot_recs_right = total_loads(&c2c_right->stats);
 786
 787         return tot_recs_left - tot_recs_right;
 788 }
 789
 790 typedef double (get_percent_cb)(struct c2c_hist_entry *);
 791
 792 static int
 793 percent_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 794               struct hist_entry *he, get_percent_cb get_percent)
 795 {
 796         struct c2c_hist_entry *c2c_he;
 797         int width = c2c_width(fmt, hpp, he->hists);
 798         double per;
 799
 800         c2c_he = container_of(he, struct c2c_hist_entry, he);
 801         per = get_percent(c2c_he);
 802
 803 #ifdef HAVE_SLANG_SUPPORT
 804         if (use_browser)
 805                 return __hpp__slsmg_color_printf(hpp, "%*.2f%%", width - 1, per);
 806 #endif
 807         return hpp_color_scnprintf(hpp, "%*.2f%%", width - 1, per);
 808 }
 809
 810 static double percent_costly_snoop(struct c2c_hist_entry *c2c_he)
 811 {
 812         struct c2c_hists *hists;
 813         struct c2c_stats *stats;
 814         struct c2c_stats *total;
 815         int tot = 0, st = 0;
 816         double p;
 817
 818         hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);
 819         stats = &c2c_he->stats;
 820         total = &hists->stats;
 821
 822         switch (c2c.display) {
 823         case DISPLAY_RMT_HITM:
 824                 st  = stats->rmt_hitm;
 825                 tot = total->rmt_hitm;
 826                 break;
 827         case DISPLAY_LCL_HITM:
 828                 st  = stats->lcl_hitm;
 829                 tot = total->lcl_hitm;
 830                 break;
 831         case DISPLAY_TOT_HITM:
 832                 st  = stats->tot_hitm;
 833                 tot = total->tot_hitm;
 834                 break;
 835         case DISPLAY_SNP_PEER:
 836                 st  = stats->tot_peer;
 837                 tot = total->tot_peer;
 838                 break;
 839         default:
 840                 break;
 841         }
 842
 843         p = tot ? (double) st / tot : 0;
 844
 845         return 100 * p;
 846 }
 847
 848 #define PERC_STR(__s, __v)                              \
 849 ({                                                      \
 850         scnprintf(__s, sizeof(__s), "%.2F%%", __v);     \
 851         __s;                                            \
 852 })
 853
 854 static int
 855 percent_costly_snoop_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 856                            struct hist_entry *he)
 857 {
 858         struct c2c_hist_entry *c2c_he;
 859         int width = c2c_width(fmt, hpp, he->hists);
 860         char buf[10];
 861         double per;
 862
 863         c2c_he = container_of(he, struct c2c_hist_entry, he);
 864         per = percent_costly_snoop(c2c_he);
 865         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
 866 }
 867
 868 static int
 869 percent_costly_snoop_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 870                            struct hist_entry *he)
 871 {
 872         return percent_color(fmt, hpp, he, percent_costly_snoop);
 873 }
 874
 875 static int64_t
 876 percent_costly_snoop_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 877                          struct hist_entry *left, struct hist_entry *right)
 878 {
 879         struct c2c_hist_entry *c2c_left;
 880         struct c2c_hist_entry *c2c_right;
 881         double per_left;
 882         double per_right;
 883
 884         c2c_left  = container_of(left, struct c2c_hist_entry, he);
 885         c2c_right = container_of(right, struct c2c_hist_entry, he);
 886
 887         per_left  = percent_costly_snoop(c2c_left);
 888         per_right = percent_costly_snoop(c2c_right);
 889
 890         return per_left - per_right;
 891 }
 892
 893 static struct c2c_stats *he_stats(struct hist_entry *he)
 894 {
 895         struct c2c_hist_entry *c2c_he;
 896
 897         c2c_he = container_of(he, struct c2c_hist_entry, he);
 898         return &c2c_he->stats;
 899 }
 900
 901 static struct c2c_stats *total_stats(struct hist_entry *he)
 902 {
 903         struct c2c_hists *hists;
 904
 905         hists = container_of(he->hists, struct c2c_hists, hists);
 906         return &hists->stats;
 907 }
 908
 909 static double percent(u32 st, u32 tot)
 910 {
 911         return tot ? 100. * (double) st / (double) tot : 0;
 912 }
 913
 914 #define PERCENT(__h, __f) percent(he_stats(__h)->__f, total_stats(__h)->__f)
 915
 916 #define PERCENT_FN(__f)                                                         \
 917 static double percent_ ## __f(struct c2c_hist_entry *c2c_he)                    \
 918 {                                                                               \
 919         struct c2c_hists *hists;                                                \
 920                                                                                 \
 921         hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);        \
 922         return percent(c2c_he->stats.__f, hists->stats.__f);                    \
 923 }
 924
 925 PERCENT_FN(rmt_hitm)
 926 PERCENT_FN(lcl_hitm)
 927 PERCENT_FN(rmt_peer)
 928 PERCENT_FN(lcl_peer)
 929 PERCENT_FN(st_l1hit)
 930 PERCENT_FN(st_l1miss)
 931 PERCENT_FN(st_na)
 932
 933 static int
 934 percent_rmt_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 935                        struct hist_entry *he)
 936 {
 937         int width = c2c_width(fmt, hpp, he->hists);
 938         double per = PERCENT(he, rmt_hitm);
 939         char buf[10];
 940
 941         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
 942 }
 943
 944 static int
 945 percent_rmt_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 946                        struct hist_entry *he)
 947 {
 948         return percent_color(fmt, hpp, he, percent_rmt_hitm);
 949 }
 950
 951 static int64_t
 952 percent_rmt_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 953                      struct hist_entry *left, struct hist_entry *right)
 954 {
 955         double per_left;
 956         double per_right;
 957
 958         per_left  = PERCENT(left, rmt_hitm);
 959         per_right = PERCENT(right, rmt_hitm);
 960
 961         return per_left - per_right;
 962 }
 963
 964 static int
 965 percent_lcl_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 966                        struct hist_entry *he)
 967 {
 968         int width = c2c_width(fmt, hpp, he->hists);
 969         double per = PERCENT(he, lcl_hitm);
 970         char buf[10];
 971
 972         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
 973 }
 974
 975 static int
 976 percent_lcl_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 977                        struct hist_entry *he)
 978 {
 979         return percent_color(fmt, hpp, he, percent_lcl_hitm);
 980 }
 981
 982 static int64_t
 983 percent_lcl_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 984                      struct hist_entry *left, struct hist_entry *right)
 985 {
 986         double per_left;
 987         double per_right;
 988
 989         per_left  = PERCENT(left, lcl_hitm);
 990         per_right = PERCENT(right, lcl_hitm);
 991
 992         return per_left - per_right;
 993 }
 994
 995 static int
 996 percent_lcl_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 997                        struct hist_entry *he)
 998 {
 999         int width = c2c_width(fmt, hpp, he->hists);
1000         double per = PERCENT(he, lcl_peer);
1001         char buf[10];
1002
1003         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1004 }
1005
1006 static int
1007 percent_lcl_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1008                        struct hist_entry *he)
1009 {
1010         return percent_color(fmt, hpp, he, percent_lcl_peer);
1011 }
1012
1013 static int64_t
1014 percent_lcl_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1015                      struct hist_entry *left, struct hist_entry *right)
1016 {
1017         double per_left;
1018         double per_right;
1019
1020         per_left  = PERCENT(left, lcl_peer);
1021         per_right = PERCENT(right, lcl_peer);
1022
1023         return per_left - per_right;
1024 }
1025
1026 static int
1027 percent_rmt_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1028                        struct hist_entry *he)
1029 {
1030         int width = c2c_width(fmt, hpp, he->hists);
1031         double per = PERCENT(he, rmt_peer);
1032         char buf[10];
1033
1034         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1035 }
1036
1037 static int
1038 percent_rmt_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1039                        struct hist_entry *he)
1040 {
1041         return percent_color(fmt, hpp, he, percent_rmt_peer);
1042 }
1043
1044 static int64_t
1045 percent_rmt_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1046                      struct hist_entry *left, struct hist_entry *right)
1047 {
1048         double per_left;
1049         double per_right;
1050
1051         per_left  = PERCENT(left, rmt_peer);
1052         per_right = PERCENT(right, rmt_peer);
1053
1054         return per_left - per_right;
1055 }
1056
1057 static int
1058 percent_stores_l1hit_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1059                            struct hist_entry *he)
1060 {
1061         int width = c2c_width(fmt, hpp, he->hists);
1062         double per = PERCENT(he, st_l1hit);
1063         char buf[10];
1064
1065         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1066 }
1067
1068 static int
1069 percent_stores_l1hit_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1070                            struct hist_entry *he)
1071 {
1072         return percent_color(fmt, hpp, he, percent_st_l1hit);
1073 }
1074
1075 static int64_t
1076 percent_stores_l1hit_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1077                         struct hist_entry *left, struct hist_entry *right)
1078 {
1079         double per_left;
1080         double per_right;
1081
1082         per_left  = PERCENT(left, st_l1hit);
1083         per_right = PERCENT(right, st_l1hit);
1084
1085         return per_left - per_right;
1086 }
1087
1088 static int
1089 percent_stores_l1miss_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1090                            struct hist_entry *he)
1091 {
1092         int width = c2c_width(fmt, hpp, he->hists);
1093         double per = PERCENT(he, st_l1miss);
1094         char buf[10];
1095
1096         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1097 }
1098
1099 static int
1100 percent_stores_l1miss_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1101                             struct hist_entry *he)
1102 {
1103         return percent_color(fmt, hpp, he, percent_st_l1miss);
1104 }
1105
1106 static int64_t
1107 percent_stores_l1miss_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1108                           struct hist_entry *left, struct hist_entry *right)
1109 {
1110         double per_left;
1111         double per_right;
1112
1113         per_left  = PERCENT(left, st_l1miss);
1114         per_right = PERCENT(right, st_l1miss);
1115
1116         return per_left - per_right;
1117 }
1118
1119 static int
1120 percent_stores_na_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1121                         struct hist_entry *he)
1122 {
1123         int width = c2c_width(fmt, hpp, he->hists);
1124         double per = PERCENT(he, st_na);
1125         char buf[10];
1126
1127         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1128 }
1129
1130 static int
1131 percent_stores_na_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1132                         struct hist_entry *he)
1133 {
1134         return percent_color(fmt, hpp, he, percent_st_na);
1135 }
1136
1137 static int64_t
1138 percent_stores_na_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1139                       struct hist_entry *left, struct hist_entry *right)
1140 {
1141         double per_left;
1142         double per_right;
1143
1144         per_left  = PERCENT(left, st_na);
1145         per_right = PERCENT(right, st_na);
1146
1147         return per_left - per_right;
1148 }
1149
1150 STAT_FN(lcl_dram)
1151 STAT_FN(rmt_dram)
1152
1153 static int
1154 pid_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1155           struct hist_entry *he)
1156 {
1157         int width = c2c_width(fmt, hpp, he->hists);
1158
1159         return scnprintf(hpp->buf, hpp->size, "%*d", width, thread__pid(he->thread));
1160 }
1161
1162 static int64_t
1163 pid_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1164         struct hist_entry *left, struct hist_entry *right)
1165 {
1166         return thread__pid(left->thread) - thread__pid(right->thread);
1167 }
1168
1169 static int64_t
1170 empty_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1171           struct hist_entry *left __maybe_unused,
1172           struct hist_entry *right __maybe_unused)
1173 {
1174         return 0;
1175 }
1176
1177 static int display_metrics(struct perf_hpp *hpp, u32 val, u32 sum)
1178 {
1179         int ret;
1180
1181         if (sum != 0)
1182                 ret = scnprintf(hpp->buf, hpp->size, "%5.1f%% ",
1183                                 percent(val, sum));
1184         else
1185                 ret = scnprintf(hpp->buf, hpp->size, "%6s ", "n/a");
1186
1187         return ret;
1188 }
1189
1190 static int
1191 node_entry(struct perf_hpp_fmt *fmt __maybe_unused, struct perf_hpp *hpp,
1192            struct hist_entry *he)
1193 {
1194         struct c2c_hist_entry *c2c_he;
1195         bool first = true;
1196         int node;
1197         int ret = 0;
1198
1199         c2c_he = container_of(he, struct c2c_hist_entry, he);
1200
1201         for (node = 0; node < c2c.nodes_cnt; node++) {
1202                 DECLARE_BITMAP(set, c2c.cpus_cnt);
1203
1204                 bitmap_zero(set, c2c.cpus_cnt);
1205                 bitmap_and(set, c2c_he->cpuset, c2c.nodes[node], c2c.cpus_cnt);
1206
1207                 if (bitmap_empty(set, c2c.cpus_cnt)) {
1208                         if (c2c.node_info == 1) {
1209                                 ret = scnprintf(hpp->buf, hpp->size, "%21s", " ");
1210                                 advance_hpp(hpp, ret);
1211                         }
1212                         continue;
1213                 }
1214
1215                 if (!first) {
1216                         ret = scnprintf(hpp->buf, hpp->size, " ");
1217                         advance_hpp(hpp, ret);
1218                 }
1219
1220                 switch (c2c.node_info) {
1221                 case 0:
1222                         ret = scnprintf(hpp->buf, hpp->size, "%2d", node);
1223                         advance_hpp(hpp, ret);
1224                         break;
1225                 case 1:
1226                 {
1227                         int num = bitmap_weight(set, c2c.cpus_cnt);
1228                         struct c2c_stats *stats = &c2c_he->node_stats[node];
1229
1230                         ret = scnprintf(hpp->buf, hpp->size, "%2d{%2d ", node, num);
1231                         advance_hpp(hpp, ret);
1232
1233                         switch (c2c.display) {
1234                         case DISPLAY_RMT_HITM:
1235                                 ret = display_metrics(hpp, stats->rmt_hitm,
1236                                                       c2c_he->stats.rmt_hitm);
1237                                 break;
1238                         case DISPLAY_LCL_HITM:
1239                                 ret = display_metrics(hpp, stats->lcl_hitm,
1240                                                       c2c_he->stats.lcl_hitm);
1241                                 break;
1242                         case DISPLAY_TOT_HITM:
1243                                 ret = display_metrics(hpp, stats->tot_hitm,
1244                                                       c2c_he->stats.tot_hitm);
1245                                 break;
1246                         case DISPLAY_SNP_PEER:
1247                                 ret = display_metrics(hpp, stats->tot_peer,
1248                                                       c2c_he->stats.tot_peer);
1249                                 break;
1250                         default:
1251                                 break;
1252                         }
1253
1254                         advance_hpp(hpp, ret);
1255
1256                         if (c2c_he->stats.store > 0) {
1257                                 ret = scnprintf(hpp->buf, hpp->size, "%5.1f%%}",
1258                                                 percent(stats->store, c2c_he->stats.store));
1259                         } else {
1260                                 ret = scnprintf(hpp->buf, hpp->size, "%6s}", "n/a");
1261                         }
1262
1263                         advance_hpp(hpp, ret);
1264                         break;
1265                 }
1266                 case 2:
1267                         ret = scnprintf(hpp->buf, hpp->size, "%2d{", node);
1268                         advance_hpp(hpp, ret);
1269
1270                         ret = bitmap_scnprintf(set, c2c.cpus_cnt, hpp->buf, hpp->size);
1271                         advance_hpp(hpp, ret);
1272
1273                         ret = scnprintf(hpp->buf, hpp->size, "}");
1274                         advance_hpp(hpp, ret);
1275                         break;
1276                 default:
1277                         break;
1278                 }
1279
1280                 first = false;
1281         }
1282
1283         return 0;
1284 }
1285
1286 static int
1287 mean_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1288            struct hist_entry *he, double mean)
1289 {
1290         int width = c2c_width(fmt, hpp, he->hists);
1291         char buf[10];
1292
1293         scnprintf(buf, 10, "%6.0f", mean);
1294         return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1295 }
1296
1297 #define MEAN_ENTRY(__func, __val)                                               \
1298 static int                                                                      \
1299 __func(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he)   \
1300 {                                                                               \
1301         struct c2c_hist_entry *c2c_he;                                          \
1302         c2c_he = container_of(he, struct c2c_hist_entry, he);                   \
1303         return mean_entry(fmt, hpp, he, avg_stats(&c2c_he->cstats.__val));      \
1304 }
1305
1306 MEAN_ENTRY(mean_rmt_entry,  rmt_hitm);
1307 MEAN_ENTRY(mean_lcl_entry,  lcl_hitm);
1308 MEAN_ENTRY(mean_load_entry, load);
1309 MEAN_ENTRY(mean_rmt_peer_entry, rmt_peer);
1310 MEAN_ENTRY(mean_lcl_peer_entry, lcl_peer);
1311
1312 static int
1313 cpucnt_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1314              struct hist_entry *he)
1315 {
1316         struct c2c_hist_entry *c2c_he;
1317         int width = c2c_width(fmt, hpp, he->hists);
1318         char buf[10];
1319
1320         c2c_he = container_of(he, struct c2c_hist_entry, he);
1321
1322         scnprintf(buf, 10, "%d", bitmap_weight(c2c_he->cpuset, c2c.cpus_cnt));
1323         return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1324 }
1325
1326 static int
1327 cl_idx_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1328              struct hist_entry *he)
1329 {
1330         struct c2c_hist_entry *c2c_he;
1331         int width = c2c_width(fmt, hpp, he->hists);
1332         char buf[10];
1333
1334         c2c_he = container_of(he, struct c2c_hist_entry, he);
1335
1336         scnprintf(buf, 10, "%u", c2c_he->cacheline_idx);
1337         return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1338 }
1339
1340 static int
1341 cl_idx_empty_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1342                    struct hist_entry *he)
1343 {
1344         int width = c2c_width(fmt, hpp, he->hists);
1345
1346         return scnprintf(hpp->buf, hpp->size, "%*s", width, "");
1347 }
1348
1349 #define HEADER_LOW(__h)                 \
1350         {                               \
1351                 .line[1] = {            \
1352                         .text = __h,    \
1353                 },                      \
1354         }
1355
1356 #define HEADER_BOTH(__h0, __h1)         \
1357         {                               \
1358                 .line[0] = {            \
1359                         .text = __h0,   \
1360                 },                      \
1361                 .line[1] = {            \
1362                         .text = __h1,   \
1363                 },                      \
1364         }
1365
1366 #define HEADER_SPAN(__h0, __h1, __s)    \
1367         {                               \
1368                 .line[0] = {            \
1369                         .text = __h0,   \
1370                         .span = __s,    \
1371                 },                      \
1372                 .line[1] = {            \
1373                         .text = __h1,   \
1374                 },                      \
1375         }
1376
1377 #define HEADER_SPAN_LOW(__h)            \
1378         {                               \
1379                 .line[1] = {            \
1380                         .text = __h,    \
1381                 },                      \
1382         }
1383
1384 static struct c2c_dimension dim_dcacheline = {
1385         .header         = HEADER_SPAN("--- Cacheline ----", "Address", 2),
1386         .name           = "dcacheline",
1387         .cmp            = dcacheline_cmp,
1388         .entry          = dcacheline_entry,
1389         .width          = 18,
1390 };
1391
1392 static struct c2c_dimension dim_dcacheline_node = {
1393         .header         = HEADER_LOW("Node"),
1394         .name           = "dcacheline_node",
1395         .cmp            = empty_cmp,
1396         .entry          = dcacheline_node_entry,
1397         .width          = 4,
1398 };
1399
1400 static struct c2c_dimension dim_dcacheline_count = {
1401         .header         = HEADER_LOW("PA cnt"),
1402         .name           = "dcacheline_count",
1403         .cmp            = empty_cmp,
1404         .entry          = dcacheline_node_count,
1405         .width          = 6,
1406 };
1407
1408 static struct c2c_header header_offset_tui = HEADER_SPAN("-----", "Off", 2);
1409
1410 static struct c2c_dimension dim_offset = {
1411         .header         = HEADER_SPAN("--- Data address -", "Offset", 2),
1412         .name           = "offset",
1413         .cmp            = offset_cmp,
1414         .entry          = offset_entry,
1415         .width          = 18,
1416 };
1417
1418 static struct c2c_dimension dim_offset_node = {
1419         .header         = HEADER_LOW("Node"),
1420         .name           = "offset_node",
1421         .cmp            = empty_cmp,
1422         .entry          = dcacheline_node_entry,
1423         .width          = 4,
1424 };
1425
1426 static struct c2c_dimension dim_iaddr = {
1427         .header         = HEADER_LOW("Code address"),
1428         .name           = "iaddr",
1429         .cmp            = iaddr_cmp,
1430         .entry          = iaddr_entry,
1431         .width          = 18,
1432 };
1433
1434 static struct c2c_dimension dim_tot_hitm = {
1435         .header         = HEADER_SPAN("------- Load Hitm -------", "Total", 2),
1436         .name           = "tot_hitm",
1437         .cmp            = tot_hitm_cmp,
1438         .entry          = tot_hitm_entry,
1439         .width          = 7,
1440 };
1441
1442 static struct c2c_dimension dim_lcl_hitm = {
1443         .header         = HEADER_SPAN_LOW("LclHitm"),
1444         .name           = "lcl_hitm",
1445         .cmp            = lcl_hitm_cmp,
1446         .entry          = lcl_hitm_entry,
1447         .width          = 7,
1448 };
1449
1450 static struct c2c_dimension dim_rmt_hitm = {
1451         .header         = HEADER_SPAN_LOW("RmtHitm"),
1452         .name           = "rmt_hitm",
1453         .cmp            = rmt_hitm_cmp,
1454         .entry          = rmt_hitm_entry,
1455         .width          = 7,
1456 };
1457
1458 static struct c2c_dimension dim_tot_peer = {
1459         .header         = HEADER_SPAN("------- Load Peer -------", "Total", 2),
1460         .name           = "tot_peer",
1461         .cmp            = tot_peer_cmp,
1462         .entry          = tot_peer_entry,
1463         .width          = 7,
1464 };
1465
1466 static struct c2c_dimension dim_lcl_peer = {
1467         .header         = HEADER_SPAN_LOW("Local"),
1468         .name           = "lcl_peer",
1469         .cmp            = lcl_peer_cmp,
1470         .entry          = lcl_peer_entry,
1471         .width          = 7,
1472 };
1473
1474 static struct c2c_dimension dim_rmt_peer = {
1475         .header         = HEADER_SPAN_LOW("Remote"),
1476         .name           = "rmt_peer",
1477         .cmp            = rmt_peer_cmp,
1478         .entry          = rmt_peer_entry,
1479         .width          = 7,
1480 };
1481
1482 static struct c2c_dimension dim_cl_rmt_hitm = {
1483         .header         = HEADER_SPAN("----- HITM -----", "Rmt", 1),
1484         .name           = "cl_rmt_hitm",
1485         .cmp            = rmt_hitm_cmp,
1486         .entry          = rmt_hitm_entry,
1487         .width          = 7,
1488 };
1489
1490 static struct c2c_dimension dim_cl_lcl_hitm = {
1491         .header         = HEADER_SPAN_LOW("Lcl"),
1492         .name           = "cl_lcl_hitm",
1493         .cmp            = lcl_hitm_cmp,
1494         .entry          = lcl_hitm_entry,
1495         .width          = 7,
1496 };
1497
1498 static struct c2c_dimension dim_cl_rmt_peer = {
1499         .header         = HEADER_SPAN("----- Peer -----", "Rmt", 1),
1500         .name           = "cl_rmt_peer",
1501         .cmp            = rmt_peer_cmp,
1502         .entry          = rmt_peer_entry,
1503         .width          = 7,
1504 };
1505
1506 static struct c2c_dimension dim_cl_lcl_peer = {
1507         .header         = HEADER_SPAN_LOW("Lcl"),
1508         .name           = "cl_lcl_peer",
1509         .cmp            = lcl_peer_cmp,
1510         .entry          = lcl_peer_entry,
1511         .width          = 7,
1512 };
1513
1514 static struct c2c_dimension dim_tot_stores = {
1515         .header         = HEADER_BOTH("Total", "Stores"),
1516         .name           = "tot_stores",
1517         .cmp            = store_cmp,
1518         .entry          = store_entry,
1519         .width          = 7,
1520 };
1521
1522 static struct c2c_dimension dim_stores_l1hit = {
1523         .header         = HEADER_SPAN("--------- Stores --------", "L1Hit", 2),
1524         .name           = "stores_l1hit",
1525         .cmp            = st_l1hit_cmp,
1526         .entry          = st_l1hit_entry,
1527         .width          = 7,
1528 };
1529
1530 static struct c2c_dimension dim_stores_l1miss = {
1531         .header         = HEADER_SPAN_LOW("L1Miss"),
1532         .name           = "stores_l1miss",
1533         .cmp            = st_l1miss_cmp,
1534         .entry          = st_l1miss_entry,
1535         .width          = 7,
1536 };
1537
1538 static struct c2c_dimension dim_stores_na = {
1539         .header         = HEADER_SPAN_LOW("N/A"),
1540         .name           = "stores_na",
1541         .cmp            = st_na_cmp,
1542         .entry          = st_na_entry,
1543         .width          = 7,
1544 };
1545
1546 static struct c2c_dimension dim_cl_stores_l1hit = {
1547         .header         = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2),
1548         .name           = "cl_stores_l1hit",
1549         .cmp            = st_l1hit_cmp,
1550         .entry          = st_l1hit_entry,
1551         .width          = 7,
1552 };
1553
1554 static struct c2c_dimension dim_cl_stores_l1miss = {
1555         .header         = HEADER_SPAN_LOW("L1 Miss"),
1556         .name           = "cl_stores_l1miss",
1557         .cmp            = st_l1miss_cmp,
1558         .entry          = st_l1miss_entry,
1559         .width          = 7,
1560 };
1561
1562 static struct c2c_dimension dim_cl_stores_na = {
1563         .header         = HEADER_SPAN_LOW("N/A"),
1564         .name           = "cl_stores_na",
1565         .cmp            = st_na_cmp,
1566         .entry          = st_na_entry,
1567         .width          = 7,
1568 };
1569
1570 static struct c2c_dimension dim_ld_fbhit = {
1571         .header         = HEADER_SPAN("----- Core Load Hit -----", "FB", 2),
1572         .name           = "ld_fbhit",
1573         .cmp            = ld_fbhit_cmp,
1574         .entry          = ld_fbhit_entry,
1575         .width          = 7,
1576 };
1577
1578 static struct c2c_dimension dim_ld_l1hit = {
1579         .header         = HEADER_SPAN_LOW("L1"),
1580         .name           = "ld_l1hit",
1581         .cmp            = ld_l1hit_cmp,
1582         .entry          = ld_l1hit_entry,
1583         .width          = 7,
1584 };
1585
1586 static struct c2c_dimension dim_ld_l2hit = {
1587         .header         = HEADER_SPAN_LOW("L2"),
1588         .name           = "ld_l2hit",
1589         .cmp            = ld_l2hit_cmp,
1590         .entry          = ld_l2hit_entry,
1591         .width          = 7,
1592 };
1593
1594 static struct c2c_dimension dim_ld_llchit = {
1595         .header         = HEADER_SPAN("- LLC Load Hit --", "LclHit", 1),
1596         .name           = "ld_lclhit",
1597         .cmp            = ld_llchit_cmp,
1598         .entry          = ld_llchit_entry,
1599         .width          = 8,
1600 };
1601
1602 static struct c2c_dimension dim_ld_rmthit = {
1603         .header         = HEADER_SPAN("- RMT Load Hit --", "RmtHit", 1),
1604         .name           = "ld_rmthit",
1605         .cmp            = rmt_hit_cmp,
1606         .entry          = rmt_hit_entry,
1607         .width          = 8,
1608 };
1609
1610 static struct c2c_dimension dim_tot_recs = {
1611         .header         = HEADER_BOTH("Total", "records"),
1612         .name           = "tot_recs",
1613         .cmp            = tot_recs_cmp,
1614         .entry          = tot_recs_entry,
1615         .width          = 7,
1616 };
1617
1618 static struct c2c_dimension dim_tot_loads = {
1619         .header         = HEADER_BOTH("Total", "Loads"),
1620         .name           = "tot_loads",
1621         .cmp            = tot_loads_cmp,
1622         .entry          = tot_loads_entry,
1623         .width          = 7,
1624 };
1625
1626 static struct c2c_header percent_costly_snoop_header[] = {
1627         [DISPLAY_LCL_HITM] = HEADER_BOTH("Lcl", "Hitm"),
1628         [DISPLAY_RMT_HITM] = HEADER_BOTH("Rmt", "Hitm"),
1629         [DISPLAY_TOT_HITM] = HEADER_BOTH("Tot", "Hitm"),
1630         [DISPLAY_SNP_PEER] = HEADER_BOTH("Peer", "Snoop"),
1631 };
1632
1633 static struct c2c_dimension dim_percent_costly_snoop = {
1634         .name           = "percent_costly_snoop",
1635         .cmp            = percent_costly_snoop_cmp,
1636         .entry          = percent_costly_snoop_entry,
1637         .color          = percent_costly_snoop_color,
1638         .width          = 7,
1639 };
1640
1641 static struct c2c_dimension dim_percent_rmt_hitm = {
1642         .header         = HEADER_SPAN("----- HITM -----", "RmtHitm", 1),
1643         .name           = "percent_rmt_hitm",
1644         .cmp            = percent_rmt_hitm_cmp,
1645         .entry          = percent_rmt_hitm_entry,
1646         .color          = percent_rmt_hitm_color,
1647         .width          = 7,
1648 };
1649
1650 static struct c2c_dimension dim_percent_lcl_hitm = {
1651         .header         = HEADER_SPAN_LOW("LclHitm"),
1652         .name           = "percent_lcl_hitm",
1653         .cmp            = percent_lcl_hitm_cmp,
1654         .entry          = percent_lcl_hitm_entry,
1655         .color          = percent_lcl_hitm_color,
1656         .width          = 7,
1657 };
1658
1659 static struct c2c_dimension dim_percent_rmt_peer = {
1660         .header         = HEADER_SPAN("-- Peer Snoop --", "Rmt", 1),
1661         .name           = "percent_rmt_peer",
1662         .cmp            = percent_rmt_peer_cmp,
1663         .entry          = percent_rmt_peer_entry,
1664         .color          = percent_rmt_peer_color,
1665         .width          = 7,
1666 };
1667
1668 static struct c2c_dimension dim_percent_lcl_peer = {
1669         .header         = HEADER_SPAN_LOW("Lcl"),
1670         .name           = "percent_lcl_peer",
1671         .cmp            = percent_lcl_peer_cmp,
1672         .entry          = percent_lcl_peer_entry,
1673         .color          = percent_lcl_peer_color,
1674         .width          = 7,
1675 };
1676
1677 static struct c2c_dimension dim_percent_stores_l1hit = {
1678         .header         = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2),
1679         .name           = "percent_stores_l1hit",
1680         .cmp            = percent_stores_l1hit_cmp,
1681         .entry          = percent_stores_l1hit_entry,
1682         .color          = percent_stores_l1hit_color,
1683         .width          = 7,
1684 };
1685
1686 static struct c2c_dimension dim_percent_stores_l1miss = {
1687         .header         = HEADER_SPAN_LOW("L1 Miss"),
1688         .name           = "percent_stores_l1miss",
1689         .cmp            = percent_stores_l1miss_cmp,
1690         .entry          = percent_stores_l1miss_entry,
1691         .color          = percent_stores_l1miss_color,
1692         .width          = 7,
1693 };
1694
1695 static struct c2c_dimension dim_percent_stores_na = {
1696         .header         = HEADER_SPAN_LOW("N/A"),
1697         .name           = "percent_stores_na",
1698         .cmp            = percent_stores_na_cmp,
1699         .entry          = percent_stores_na_entry,
1700         .color          = percent_stores_na_color,
1701         .width          = 7,
1702 };
1703
1704 static struct c2c_dimension dim_dram_lcl = {
1705         .header         = HEADER_SPAN("--- Load Dram ----", "Lcl", 1),
1706         .name           = "dram_lcl",
1707         .cmp            = lcl_dram_cmp,
1708         .entry          = lcl_dram_entry,
1709         .width          = 8,
1710 };
1711
1712 static struct c2c_dimension dim_dram_rmt = {
1713         .header         = HEADER_SPAN_LOW("Rmt"),
1714         .name           = "dram_rmt",
1715         .cmp            = rmt_dram_cmp,
1716         .entry          = rmt_dram_entry,
1717         .width          = 8,
1718 };
1719
1720 static struct c2c_dimension dim_pid = {
1721         .header         = HEADER_LOW("Pid"),
1722         .name           = "pid",
1723         .cmp            = pid_cmp,
1724         .entry          = pid_entry,
1725         .width          = 7,
1726 };
1727
1728 static struct c2c_dimension dim_tid = {
1729         .header         = HEADER_LOW("Tid"),
1730         .name           = "tid",
1731         .se             = &sort_thread,
1732 };
1733
1734 static struct c2c_dimension dim_symbol = {
1735         .name           = "symbol",
1736         .se             = &sort_sym,
1737 };
1738
1739 static struct c2c_dimension dim_dso = {
1740         .header         = HEADER_BOTH("Shared", "Object"),
1741         .name           = "dso",
1742         .se             = &sort_dso,
1743 };
1744
1745 static struct c2c_dimension dim_node = {
1746         .name           = "node",
1747         .cmp            = empty_cmp,
1748         .entry          = node_entry,
1749         .width          = 4,
1750 };
1751
1752 static struct c2c_dimension dim_mean_rmt = {
1753         .header         = HEADER_SPAN("---------- cycles ----------", "rmt hitm", 2),
1754         .name           = "mean_rmt",
1755         .cmp            = empty_cmp,
1756         .entry          = mean_rmt_entry,
1757         .width          = 8,
1758 };
1759
1760 static struct c2c_dimension dim_mean_lcl = {
1761         .header         = HEADER_SPAN_LOW("lcl hitm"),
1762         .name           = "mean_lcl",
1763         .cmp            = empty_cmp,
1764         .entry          = mean_lcl_entry,
1765         .width          = 8,
1766 };
1767
1768 static struct c2c_dimension dim_mean_load = {
1769         .header         = HEADER_SPAN_LOW("load"),
1770         .name           = "mean_load",
1771         .cmp            = empty_cmp,
1772         .entry          = mean_load_entry,
1773         .width          = 8,
1774 };
1775
1776 static struct c2c_dimension dim_mean_rmt_peer = {
1777         .header         = HEADER_SPAN("---------- cycles ----------", "rmt peer", 2),
1778         .name           = "mean_rmt_peer",
1779         .cmp            = empty_cmp,
1780         .entry          = mean_rmt_peer_entry,
1781         .width          = 8,
1782 };
1783
1784 static struct c2c_dimension dim_mean_lcl_peer = {
1785         .header         = HEADER_SPAN_LOW("lcl peer"),
1786         .name           = "mean_lcl_peer",
1787         .cmp            = empty_cmp,
1788         .entry          = mean_lcl_peer_entry,
1789         .width          = 8,
1790 };
1791
1792 static struct c2c_dimension dim_cpucnt = {
1793         .header         = HEADER_BOTH("cpu", "cnt"),
1794         .name           = "cpucnt",
1795         .cmp            = empty_cmp,
1796         .entry          = cpucnt_entry,
1797         .width          = 8,
1798 };
1799
1800 static struct c2c_dimension dim_srcline = {
1801         .name           = "cl_srcline",
1802         .se             = &sort_srcline,
1803 };
1804
1805 static struct c2c_dimension dim_dcacheline_idx = {
1806         .header         = HEADER_LOW("Index"),
1807         .name           = "cl_idx",
1808         .cmp            = empty_cmp,
1809         .entry          = cl_idx_entry,
1810         .width          = 5,
1811 };
1812
1813 static struct c2c_dimension dim_dcacheline_num = {
1814         .header         = HEADER_LOW("Num"),
1815         .name           = "cl_num",
1816         .cmp            = empty_cmp,
1817         .entry          = cl_idx_entry,
1818         .width          = 5,
1819 };
1820
1821 static struct c2c_dimension dim_dcacheline_num_empty = {
1822         .header         = HEADER_LOW("Num"),
1823         .name           = "cl_num_empty",
1824         .cmp            = empty_cmp,
1825         .entry          = cl_idx_empty_entry,
1826         .width          = 5,
1827 };
1828
1829 static struct c2c_dimension *dimensions[] = {
1830         &dim_dcacheline,
1831         &dim_dcacheline_node,
1832         &dim_dcacheline_count,
1833         &dim_offset,
1834         &dim_offset_node,
1835         &dim_iaddr,
1836         &dim_tot_hitm,
1837         &dim_lcl_hitm,
1838         &dim_rmt_hitm,
1839         &dim_tot_peer,
1840         &dim_lcl_peer,
1841         &dim_rmt_peer,
1842         &dim_cl_lcl_hitm,
1843         &dim_cl_rmt_hitm,
1844         &dim_cl_lcl_peer,
1845         &dim_cl_rmt_peer,
1846         &dim_tot_stores,
1847         &dim_stores_l1hit,
1848         &dim_stores_l1miss,
1849         &dim_stores_na,
1850         &dim_cl_stores_l1hit,
1851         &dim_cl_stores_l1miss,
1852         &dim_cl_stores_na,
1853         &dim_ld_fbhit,
1854         &dim_ld_l1hit,
1855         &dim_ld_l2hit,
1856         &dim_ld_llchit,
1857         &dim_ld_rmthit,
1858         &dim_tot_recs,
1859         &dim_tot_loads,
1860         &dim_percent_costly_snoop,
1861         &dim_percent_rmt_hitm,
1862         &dim_percent_lcl_hitm,
1863         &dim_percent_rmt_peer,
1864         &dim_percent_lcl_peer,
1865         &dim_percent_stores_l1hit,
1866         &dim_percent_stores_l1miss,
1867         &dim_percent_stores_na,
1868         &dim_dram_lcl,
1869         &dim_dram_rmt,
1870         &dim_pid,
1871         &dim_tid,
1872         &dim_symbol,
1873         &dim_dso,
1874         &dim_node,
1875         &dim_mean_rmt,
1876         &dim_mean_lcl,
1877         &dim_mean_rmt_peer,
1878         &dim_mean_lcl_peer,
1879         &dim_mean_load,
1880         &dim_cpucnt,
1881         &dim_srcline,
1882         &dim_dcacheline_idx,
1883         &dim_dcacheline_num,
1884         &dim_dcacheline_num_empty,
1885         NULL,
1886 };
1887
1888 static void fmt_free(struct perf_hpp_fmt *fmt)
1889 {
1890         struct c2c_fmt *c2c_fmt;
1891
1892         c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1893         free(c2c_fmt);
1894 }
1895
1896 static bool fmt_equal(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b)
1897 {
1898         struct c2c_fmt *c2c_a = container_of(a, struct c2c_fmt, fmt);
1899         struct c2c_fmt *c2c_b = container_of(b, struct c2c_fmt, fmt);
1900
1901         return c2c_a->dim == c2c_b->dim;
1902 }
1903
1904 static struct c2c_dimension *get_dimension(const char *name)
1905 {
1906         unsigned int i;
1907
1908         for (i = 0; dimensions[i]; i++) {
1909                 struct c2c_dimension *dim = dimensions[i];
1910
1911                 if (!strcmp(dim->name, name))
1912                         return dim;
1913         }
1914
1915         return NULL;
1916 }
1917
1918 static int c2c_se_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1919                         struct hist_entry *he)
1920 {
1921         struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1922         struct c2c_dimension *dim = c2c_fmt->dim;
1923         size_t len = fmt->user_len;
1924
1925         if (!len) {
1926                 len = hists__col_len(he->hists, dim->se->se_width_idx);
1927
1928                 if (dim == &dim_symbol || dim == &dim_srcline)
1929                         len = symbol_width(he->hists, dim->se);
1930         }
1931
1932         return dim->se->se_snprintf(he, hpp->buf, hpp->size, len);
1933 }
1934
1935 static int64_t c2c_se_cmp(struct perf_hpp_fmt *fmt,
1936                           struct hist_entry *a, struct hist_entry *b)
1937 {
1938         struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1939         struct c2c_dimension *dim = c2c_fmt->dim;
1940
1941         return dim->se->se_cmp(a, b);
1942 }
1943
1944 static int64_t c2c_se_collapse(struct perf_hpp_fmt *fmt,
1945                                struct hist_entry *a, struct hist_entry *b)
1946 {
1947         struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1948         struct c2c_dimension *dim = c2c_fmt->dim;
1949         int64_t (*collapse_fn)(struct hist_entry *, struct hist_entry *);
1950
1951         collapse_fn = dim->se->se_collapse ?: dim->se->se_cmp;
1952         return collapse_fn(a, b);
1953 }
1954
1955 static struct c2c_fmt *get_format(const char *name)
1956 {
1957         struct c2c_dimension *dim = get_dimension(name);
1958         struct c2c_fmt *c2c_fmt;
1959         struct perf_hpp_fmt *fmt;
1960
1961         if (!dim)
1962                 return NULL;
1963
1964         c2c_fmt = zalloc(sizeof(*c2c_fmt));
1965         if (!c2c_fmt)
1966                 return NULL;
1967
1968         c2c_fmt->dim = dim;
1969
1970         fmt = &c2c_fmt->fmt;
1971         INIT_LIST_HEAD(&fmt->list);
1972         INIT_LIST_HEAD(&fmt->sort_list);
1973
1974         fmt->cmp        = dim->se ? c2c_se_cmp   : dim->cmp;
1975         fmt->sort       = dim->se ? c2c_se_cmp   : dim->cmp;
1976         fmt->color      = dim->se ? NULL         : dim->color;
1977         fmt->entry      = dim->se ? c2c_se_entry : dim->entry;
1978         fmt->header     = c2c_header;
1979         fmt->width      = c2c_width;
1980         fmt->collapse   = dim->se ? c2c_se_collapse : dim->cmp;
1981         fmt->equal      = fmt_equal;
1982         fmt->free       = fmt_free;
1983
1984         return c2c_fmt;
1985 }
1986
1987 static int c2c_hists__init_output(struct perf_hpp_list *hpp_list, char *name)
1988 {
1989         struct c2c_fmt *c2c_fmt = get_format(name);
1990
1991         if (!c2c_fmt) {
1992                 reset_dimensions();
1993                 return output_field_add(hpp_list, name);
1994         }
1995
1996         perf_hpp_list__column_register(hpp_list, &c2c_fmt->fmt);
1997         return 0;
1998 }
1999
2000 static int c2c_hists__init_sort(struct perf_hpp_list *hpp_list, char *name)
2001 {
2002         struct c2c_fmt *c2c_fmt = get_format(name);
2003         struct c2c_dimension *dim;
2004
2005         if (!c2c_fmt) {
2006                 reset_dimensions();
2007                 return sort_dimension__add(hpp_list, name, NULL, 0);
2008         }
2009
2010         dim = c2c_fmt->dim;
2011         if (dim == &dim_dso)
2012                 hpp_list->dso = 1;
2013
2014         perf_hpp_list__register_sort_field(hpp_list, &c2c_fmt->fmt);
2015         return 0;
2016 }
2017
2018 #define PARSE_LIST(_list, _fn)                                                  \
2019         do {                                                                    \
2020                 char *tmp, *tok;                                                \
2021                 ret = 0;                                                        \
2022                                                                                 \
2023                 if (!_list)                                                     \
2024                         break;                                                  \
2025                                                                                 \
2026                 for (tok = strtok_r((char *)_list, ", ", &tmp);                 \
2027                                 tok; tok = strtok_r(NULL, ", ", &tmp)) {        \
2028                         ret = _fn(hpp_list, tok);                               \
2029                         if (ret == -EINVAL) {                                   \
2030                                 pr_err("Invalid --fields key: `%s'", tok);      \
2031                                 break;                                          \
2032                         } else if (ret == -ESRCH) {                             \
2033                                 pr_err("Unknown --fields key: `%s'", tok);      \
2034                                 break;                                          \
2035                         }                                                       \
2036                 }                                                               \
2037         } while (0)
2038
2039 static int hpp_list__parse(struct perf_hpp_list *hpp_list,
2040                            const char *output_,
2041                            const char *sort_)
2042 {
2043         char *output = output_ ? strdup(output_) : NULL;
2044         char *sort   = sort_   ? strdup(sort_) : NULL;
2045         int ret;
2046
2047         PARSE_LIST(output, c2c_hists__init_output);
2048         PARSE_LIST(sort,   c2c_hists__init_sort);
2049
2050         /* copy sort keys to output fields */
2051         perf_hpp__setup_output_field(hpp_list);
2052
2053         /*
2054          * We don't need other sorting keys other than those
2055          * we already specified. It also really slows down
2056          * the processing a lot with big number of output
2057          * fields, so switching this off for c2c.
2058          */
2059
2060 #if 0
2061         /* and then copy output fields to sort keys */
2062         perf_hpp__append_sort_keys(&hists->list);
2063 #endif
2064
2065         free(output);
2066         free(sort);
2067         return ret;
2068 }
2069
2070 static int c2c_hists__init(struct c2c_hists *hists,
2071                            const char *sort,
2072                            int nr_header_lines)
2073 {
2074         __hists__init(&hists->hists, &hists->list);
2075
2076         /*
2077          * Initialize only with sort fields, we need to resort
2078          * later anyway, and that's where we add output fields
2079          * as well.
2080          */
2081         perf_hpp_list__init(&hists->list);
2082
2083         /* Overload number of header lines.*/
2084         hists->list.nr_header_lines = nr_header_lines;
2085
2086         return hpp_list__parse(&hists->list, NULL, sort);
2087 }
2088
2089 static int c2c_hists__reinit(struct c2c_hists *c2c_hists,
2090                              const char *output,
2091                              const char *sort)
2092 {
2093         perf_hpp__reset_output_field(&c2c_hists->list);
2094         return hpp_list__parse(&c2c_hists->list, output, sort);
2095 }
2096
2097 #define DISPLAY_LINE_LIMIT  0.001
2098
2099 static u8 filter_display(u32 val, u32 sum)
2100 {
2101         if (sum == 0 || ((double)val / sum) < DISPLAY_LINE_LIMIT)
2102                 return HIST_FILTER__C2C;
2103
2104         return 0;
2105 }
2106
2107 static bool he__display(struct hist_entry *he, struct c2c_stats *stats)
2108 {
2109         struct c2c_hist_entry *c2c_he;
2110
2111         if (c2c.show_all)
2112                 return true;
2113
2114         c2c_he = container_of(he, struct c2c_hist_entry, he);
2115
2116         switch (c2c.display) {
2117         case DISPLAY_LCL_HITM:
2118                 he->filtered = filter_display(c2c_he->stats.lcl_hitm,
2119                                               stats->lcl_hitm);
2120                 break;
2121         case DISPLAY_RMT_HITM:
2122                 he->filtered = filter_display(c2c_he->stats.rmt_hitm,
2123                                               stats->rmt_hitm);
2124                 break;
2125         case DISPLAY_TOT_HITM:
2126                 he->filtered = filter_display(c2c_he->stats.tot_hitm,
2127                                               stats->tot_hitm);
2128                 break;
2129         case DISPLAY_SNP_PEER:
2130                 he->filtered = filter_display(c2c_he->stats.tot_peer,
2131                                               stats->tot_peer);
2132                 break;
2133         default:
2134                 break;
2135         }
2136
2137         return he->filtered == 0;
2138 }
2139
2140 static inline bool is_valid_hist_entry(struct hist_entry *he)
2141 {
2142         struct c2c_hist_entry *c2c_he;
2143         bool has_record = false;
2144
2145         c2c_he = container_of(he, struct c2c_hist_entry, he);
2146
2147         /* It's a valid entry if contains stores */
2148         if (c2c_he->stats.store)
2149                 return true;
2150
2151         switch (c2c.display) {
2152         case DISPLAY_LCL_HITM:
2153                 has_record = !!c2c_he->stats.lcl_hitm;
2154                 break;
2155         case DISPLAY_RMT_HITM:
2156                 has_record = !!c2c_he->stats.rmt_hitm;
2157                 break;
2158         case DISPLAY_TOT_HITM:
2159                 has_record = !!c2c_he->stats.tot_hitm;
2160                 break;
2161         case DISPLAY_SNP_PEER:
2162                 has_record = !!c2c_he->stats.tot_peer;
2163         default:
2164                 break;
2165         }
2166
2167         return has_record;
2168 }
2169
2170 static void set_node_width(struct c2c_hist_entry *c2c_he, int len)
2171 {
2172         struct c2c_dimension *dim;
2173
2174         dim = &c2c.hists == c2c_he->hists ?
2175               &dim_dcacheline_node : &dim_offset_node;
2176
2177         if (len > dim->width)
2178                 dim->width = len;
2179 }
2180
2181 static int set_nodestr(struct c2c_hist_entry *c2c_he)
2182 {
2183         char buf[30];
2184         int len;
2185
2186         if (c2c_he->nodestr)
2187                 return 0;
2188
2189         if (!bitmap_empty(c2c_he->nodeset, c2c.nodes_cnt)) {
2190                 len = bitmap_scnprintf(c2c_he->nodeset, c2c.nodes_cnt,
2191                                       buf, sizeof(buf));
2192         } else {
2193                 len = scnprintf(buf, sizeof(buf), "N/A");
2194         }
2195
2196         set_node_width(c2c_he, len);
2197         c2c_he->nodestr = strdup(buf);
2198         return c2c_he->nodestr ? 0 : -ENOMEM;
2199 }
2200
2201 static void calc_width(struct c2c_hist_entry *c2c_he)
2202 {
2203         struct c2c_hists *c2c_hists;
2204
2205         c2c_hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);
2206         hists__calc_col_len(&c2c_hists->hists, &c2c_he->he);
2207         set_nodestr(c2c_he);
2208 }
2209
2210 static int filter_cb(struct hist_entry *he, void *arg __maybe_unused)
2211 {
2212         struct c2c_hist_entry *c2c_he;
2213
2214         c2c_he = container_of(he, struct c2c_hist_entry, he);
2215
2216         if (c2c.show_src && !he->srcline)
2217                 he->srcline = hist_entry__srcline(he);
2218
2219         calc_width(c2c_he);
2220
2221         if (!is_valid_hist_entry(he))
2222                 he->filtered = HIST_FILTER__C2C;
2223
2224         return 0;
2225 }
2226
2227 static int resort_cl_cb(struct hist_entry *he, void *arg __maybe_unused)
2228 {
2229         struct c2c_hist_entry *c2c_he;
2230         struct c2c_hists *c2c_hists;
2231         bool display = he__display(he, &c2c.shared_clines_stats);
2232
2233         c2c_he = container_of(he, struct c2c_hist_entry, he);
2234         c2c_hists = c2c_he->hists;
2235
2236         if (display && c2c_hists) {
2237                 static unsigned int idx;
2238
2239                 c2c_he->cacheline_idx = idx++;
2240                 calc_width(c2c_he);
2241
2242                 c2c_hists__reinit(c2c_hists, c2c.cl_output, c2c.cl_resort);
2243
2244                 hists__collapse_resort(&c2c_hists->hists, NULL);
2245                 hists__output_resort_cb(&c2c_hists->hists, NULL, filter_cb);
2246         }
2247
2248         return 0;
2249 }
2250
2251 static struct c2c_header header_node_0 = HEADER_LOW("Node");
2252 static struct c2c_header header_node_1_hitms_stores =
2253                 HEADER_LOW("Node{cpus %hitms %stores}");
2254 static struct c2c_header header_node_1_peers_stores =
2255                 HEADER_LOW("Node{cpus %peers %stores}");
2256 static struct c2c_header header_node_2 = HEADER_LOW("Node{cpu list}");
2257
2258 static void setup_nodes_header(void)
2259 {
2260         switch (c2c.node_info) {
2261         case 0:
2262                 dim_node.header = header_node_0;
2263                 break;
2264         case 1:
2265                 if (c2c.display == DISPLAY_SNP_PEER)
2266                         dim_node.header = header_node_1_peers_stores;
2267                 else
2268                         dim_node.header = header_node_1_hitms_stores;
2269                 break;
2270         case 2:
2271                 dim_node.header = header_node_2;
2272                 break;
2273         default:
2274                 break;
2275         }
2276
2277         return;
2278 }
2279
2280 static int setup_nodes(struct perf_session *session)
2281 {
2282         struct numa_node *n;
2283         unsigned long **nodes;
2284         int node, idx;
2285         struct perf_cpu cpu;
2286         int *cpu2node;
2287
2288         if (c2c.node_info > 2)
2289                 c2c.node_info = 2;
2290
2291         c2c.nodes_cnt = session->header.env.nr_numa_nodes;
2292         c2c.cpus_cnt  = session->header.env.nr_cpus_avail;
2293
2294         n = session->header.env.numa_nodes;
2295         if (!n)
2296                 return -EINVAL;
2297
2298         nodes = zalloc(sizeof(unsigned long *) * c2c.nodes_cnt);
2299         if (!nodes)
2300                 return -ENOMEM;
2301
2302         c2c.nodes = nodes;
2303
2304         cpu2node = zalloc(sizeof(int) * c2c.cpus_cnt);
2305         if (!cpu2node)
2306                 return -ENOMEM;
2307
2308         for (idx = 0; idx < c2c.cpus_cnt; idx++)
2309                 cpu2node[idx] = -1;
2310
2311         c2c.cpu2node = cpu2node;
2312
2313         for (node = 0; node < c2c.nodes_cnt; node++) {
2314                 struct perf_cpu_map *map = n[node].map;
2315                 unsigned long *set;
2316
2317                 set = bitmap_zalloc(c2c.cpus_cnt);
2318                 if (!set)
2319                         return -ENOMEM;
2320
2321                 nodes[node] = set;
2322
2323                 perf_cpu_map__for_each_cpu_skip_any(cpu, idx, map) {
2324                         __set_bit(cpu.cpu, set);
2325
2326                         if (WARN_ONCE(cpu2node[cpu.cpu] != -1, "node/cpu topology bug"))
2327                                 return -EINVAL;
2328
2329                         cpu2node[cpu.cpu] = node;
2330                 }
2331         }
2332
2333         setup_nodes_header();
2334         return 0;
2335 }
2336
2337 #define HAS_HITMS(__h) ((__h)->stats.lcl_hitm || (__h)->stats.rmt_hitm)
2338 #define HAS_PEER(__h) ((__h)->stats.lcl_peer || (__h)->stats.rmt_peer)
2339
2340 static int resort_shared_cl_cb(struct hist_entry *he, void *arg __maybe_unused)
2341 {
2342         struct c2c_hist_entry *c2c_he;
2343         c2c_he = container_of(he, struct c2c_hist_entry, he);
2344
2345         if (HAS_HITMS(c2c_he) || HAS_PEER(c2c_he)) {
2346                 c2c.shared_clines++;
2347                 c2c_add_stats(&c2c.shared_clines_stats, &c2c_he->stats);
2348         }
2349
2350         return 0;
2351 }
2352
2353 static int hists__iterate_cb(struct hists *hists, hists__resort_cb_t cb)
2354 {
2355         struct rb_node *next = rb_first_cached(&hists->entries);
2356         int ret = 0;
2357
2358         while (next) {
2359                 struct hist_entry *he;
2360
2361                 he = rb_entry(next, struct hist_entry, rb_node);
2362                 ret = cb(he, NULL);
2363                 if (ret)
2364                         break;
2365                 next = rb_next(&he->rb_node);
2366         }
2367
2368         return ret;
2369 }
2370
2371 static void print_c2c__display_stats(FILE *out)
2372 {
2373         int llc_misses;
2374         struct c2c_stats *stats = &c2c.hists.stats;
2375
2376         llc_misses = get_load_llc_misses(stats);
2377
2378         fprintf(out, "=================================================\n");
2379         fprintf(out, "            Trace Event Information              \n");
2380         fprintf(out, "=================================================\n");
2381         fprintf(out, "  Total records                     : %10d\n", stats->nr_entries);
2382         fprintf(out, "  Locked Load/Store Operations      : %10d\n", stats->locks);
2383         fprintf(out, "  Load Operations                   : %10d\n", stats->load);
2384         fprintf(out, "  Loads - uncacheable               : %10d\n", stats->ld_uncache);
2385         fprintf(out, "  Loads - IO                        : %10d\n", stats->ld_io);
2386         fprintf(out, "  Loads - Miss                      : %10d\n", stats->ld_miss);
2387         fprintf(out, "  Loads - no mapping                : %10d\n", stats->ld_noadrs);
2388         fprintf(out, "  Load Fill Buffer Hit              : %10d\n", stats->ld_fbhit);
2389         fprintf(out, "  Load L1D hit                      : %10d\n", stats->ld_l1hit);
2390         fprintf(out, "  Load L2D hit                      : %10d\n", stats->ld_l2hit);
2391         fprintf(out, "  Load LLC hit                      : %10d\n", stats->ld_llchit + stats->lcl_hitm);
2392         fprintf(out, "  Load Local HITM                   : %10d\n", stats->lcl_hitm);
2393         fprintf(out, "  Load Remote HITM                  : %10d\n", stats->rmt_hitm);
2394         fprintf(out, "  Load Remote HIT                   : %10d\n", stats->rmt_hit);
2395         fprintf(out, "  Load Local DRAM                   : %10d\n", stats->lcl_dram);
2396         fprintf(out, "  Load Remote DRAM                  : %10d\n", stats->rmt_dram);
2397         fprintf(out, "  Load MESI State Exclusive         : %10d\n", stats->ld_excl);
2398         fprintf(out, "  Load MESI State Shared            : %10d\n", stats->ld_shared);
2399         fprintf(out, "  Load LLC Misses                   : %10d\n", llc_misses);
2400         fprintf(out, "  Load access blocked by data       : %10d\n", stats->blk_data);
2401         fprintf(out, "  Load access blocked by address    : %10d\n", stats->blk_addr);
2402         fprintf(out, "  Load HIT Local Peer               : %10d\n", stats->lcl_peer);
2403         fprintf(out, "  Load HIT Remote Peer              : %10d\n", stats->rmt_peer);
2404         fprintf(out, "  LLC Misses to Local DRAM          : %10.1f%%\n", ((double)stats->lcl_dram/(double)llc_misses) * 100.);
2405         fprintf(out, "  LLC Misses to Remote DRAM         : %10.1f%%\n", ((double)stats->rmt_dram/(double)llc_misses) * 100.);
2406         fprintf(out, "  LLC Misses to Remote cache (HIT)  : %10.1f%%\n", ((double)stats->rmt_hit /(double)llc_misses) * 100.);
2407         fprintf(out, "  LLC Misses to Remote cache (HITM) : %10.1f%%\n", ((double)stats->rmt_hitm/(double)llc_misses) * 100.);
2408         fprintf(out, "  Store Operations                  : %10d\n", stats->store);
2409         fprintf(out, "  Store - uncacheable               : %10d\n", stats->st_uncache);
2410         fprintf(out, "  Store - no mapping                : %10d\n", stats->st_noadrs);
2411         fprintf(out, "  Store L1D Hit                     : %10d\n", stats->st_l1hit);
2412         fprintf(out, "  Store L1D Miss                    : %10d\n", stats->st_l1miss);
2413         fprintf(out, "  Store No available memory level   : %10d\n", stats->st_na);
2414         fprintf(out, "  No Page Map Rejects               : %10d\n", stats->nomap);
2415         fprintf(out, "  Unable to parse data source       : %10d\n", stats->noparse);
2416 }
2417
2418 static void print_shared_cacheline_info(FILE *out)
2419 {
2420         struct c2c_stats *stats = &c2c.shared_clines_stats;
2421         int hitm_cnt = stats->lcl_hitm + stats->rmt_hitm;
2422
2423         fprintf(out, "=================================================\n");
2424         fprintf(out, "    Global Shared Cache Line Event Information   \n");
2425         fprintf(out, "=================================================\n");
2426         fprintf(out, "  Total Shared Cache Lines          : %10d\n", c2c.shared_clines);
2427         fprintf(out, "  Load HITs on shared lines         : %10d\n", stats->load);
2428         fprintf(out, "  Fill Buffer Hits on shared lines  : %10d\n", stats->ld_fbhit);
2429         fprintf(out, "  L1D hits on shared lines          : %10d\n", stats->ld_l1hit);
2430         fprintf(out, "  L2D hits on shared lines          : %10d\n", stats->ld_l2hit);
2431         fprintf(out, "  LLC hits on shared lines          : %10d\n", stats->ld_llchit + stats->lcl_hitm);
2432         fprintf(out, "  Load hits on peer cache or nodes  : %10d\n", stats->lcl_peer + stats->rmt_peer);
2433         fprintf(out, "  Locked Access on shared lines     : %10d\n", stats->locks);
2434         fprintf(out, "  Blocked Access on shared lines    : %10d\n", stats->blk_data + stats->blk_addr);
2435         fprintf(out, "  Store HITs on shared lines        : %10d\n", stats->store);
2436         fprintf(out, "  Store L1D hits on shared lines    : %10d\n", stats->st_l1hit);
2437         fprintf(out, "  Store No available memory level   : %10d\n", stats->st_na);
2438         fprintf(out, "  Total Merged records              : %10d\n", hitm_cnt + stats->store);
2439 }
2440
2441 static void print_cacheline(struct c2c_hists *c2c_hists,
2442                             struct hist_entry *he_cl,
2443                             struct perf_hpp_list *hpp_list,
2444                             FILE *out)
2445 {
2446         char bf[1000];
2447         struct perf_hpp hpp = {
2448                 .buf            = bf,
2449                 .size           = 1000,
2450         };
2451         static bool once;
2452
2453         if (!once) {
2454                 hists__fprintf_headers(&c2c_hists->hists, out);
2455                 once = true;
2456         } else {
2457                 fprintf(out, "\n");
2458         }
2459
2460         fprintf(out, "  ----------------------------------------------------------------------\n");
2461         __hist_entry__snprintf(he_cl, &hpp, hpp_list);
2462         fprintf(out, "%s\n", bf);
2463         fprintf(out, "  ----------------------------------------------------------------------\n");
2464
2465         hists__fprintf(&c2c_hists->hists, false, 0, 0, 0, out, false);
2466 }
2467
2468 static void print_pareto(FILE *out)
2469 {
2470         struct perf_hpp_list hpp_list;
2471         struct rb_node *nd;
2472         int ret;
2473         const char *cl_output;
2474
2475         if (c2c.display != DISPLAY_SNP_PEER)
2476                 cl_output = "cl_num,"
2477                             "cl_rmt_hitm,"
2478                             "cl_lcl_hitm,"
2479                             "cl_stores_l1hit,"
2480                             "cl_stores_l1miss,"
2481                             "cl_stores_na,"
2482                             "dcacheline";
2483         else
2484                 cl_output = "cl_num,"
2485                             "cl_rmt_peer,"
2486                             "cl_lcl_peer,"
2487                             "cl_stores_l1hit,"
2488                             "cl_stores_l1miss,"
2489                             "cl_stores_na,"
2490                             "dcacheline";
2491
2492         perf_hpp_list__init(&hpp_list);
2493         ret = hpp_list__parse(&hpp_list, cl_output, NULL);
2494
2495         if (WARN_ONCE(ret, "failed to setup sort entries\n"))
2496                 return;
2497
2498         nd = rb_first_cached(&c2c.hists.hists.entries);
2499
2500         for (; nd; nd = rb_next(nd)) {
2501                 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
2502                 struct c2c_hist_entry *c2c_he;
2503
2504                 if (he->filtered)
2505                         continue;
2506
2507                 c2c_he = container_of(he, struct c2c_hist_entry, he);
2508                 print_cacheline(c2c_he->hists, he, &hpp_list, out);
2509         }
2510 }
2511
2512 static void print_c2c_info(FILE *out, struct perf_session *session)
2513 {
2514         struct evlist *evlist = session->evlist;
2515         struct evsel *evsel;
2516         bool first = true;
2517
2518         fprintf(out, "=================================================\n");
2519         fprintf(out, "                 c2c details                     \n");
2520         fprintf(out, "=================================================\n");
2521
2522         evlist__for_each_entry(evlist, evsel) {
2523                 fprintf(out, "%-36s: %s\n", first ? "  Events" : "", evsel__name(evsel));
2524                 first = false;
2525         }
2526         fprintf(out, "  Cachelines sort on                : %s\n",
2527                 display_str[c2c.display]);
2528         fprintf(out, "  Cacheline data grouping           : %s\n", c2c.cl_sort);
2529 }
2530
2531 static void perf_c2c__hists_fprintf(FILE *out, struct perf_session *session)
2532 {
2533         setup_pager();
2534
2535         print_c2c__display_stats(out);
2536         fprintf(out, "\n");
2537         print_shared_cacheline_info(out);
2538         fprintf(out, "\n");
2539         print_c2c_info(out, session);
2540
2541         if (c2c.stats_only)
2542                 return;
2543
2544         fprintf(out, "\n");
2545         fprintf(out, "=================================================\n");
2546         fprintf(out, "           Shared Data Cache Line Table          \n");
2547         fprintf(out, "=================================================\n");
2548         fprintf(out, "#\n");
2549
2550         hists__fprintf(&c2c.hists.hists, true, 0, 0, 0, stdout, true);
2551
2552         fprintf(out, "\n");
2553         fprintf(out, "=================================================\n");
2554         fprintf(out, "      Shared Cache Line Distribution Pareto      \n");
2555         fprintf(out, "=================================================\n");
2556         fprintf(out, "#\n");
2557
2558         print_pareto(out);
2559 }
2560
2561 #ifdef HAVE_SLANG_SUPPORT
2562 static void c2c_browser__update_nr_entries(struct hist_browser *hb)
2563 {
2564         u64 nr_entries = 0;
2565         struct rb_node *nd = rb_first_cached(&hb->hists->entries);
2566
2567         while (nd) {
2568                 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
2569
2570                 if (!he->filtered)
2571                         nr_entries++;
2572
2573                 nd = rb_next(nd);
2574         }
2575
2576         hb->nr_non_filtered_entries = nr_entries;
2577 }
2578
2579 struct c2c_cacheline_browser {
2580         struct hist_browser      hb;
2581         struct hist_entry       *he;
2582 };
2583
2584 static int
2585 perf_c2c_cacheline_browser__title(struct hist_browser *browser,
2586                                   char *bf, size_t size)
2587 {
2588         struct c2c_cacheline_browser *cl_browser;
2589         struct hist_entry *he;
2590         uint64_t addr = 0;
2591
2592         cl_browser = container_of(browser, struct c2c_cacheline_browser, hb);
2593         he = cl_browser->he;
2594
2595         if (he->mem_info)
2596                 addr = cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl);
2597
2598         scnprintf(bf, size, "Cacheline 0x%lx", addr);
2599         return 0;
2600 }
2601
2602 static struct c2c_cacheline_browser*
2603 c2c_cacheline_browser__new(struct hists *hists, struct hist_entry *he)
2604 {
2605         struct c2c_cacheline_browser *browser;
2606
2607         browser = zalloc(sizeof(*browser));
2608         if (browser) {
2609                 hist_browser__init(&browser->hb, hists);
2610                 browser->hb.c2c_filter  = true;
2611                 browser->hb.title       = perf_c2c_cacheline_browser__title;
2612                 browser->he             = he;
2613         }
2614
2615         return browser;
2616 }
2617
2618 static int perf_c2c__browse_cacheline(struct hist_entry *he)
2619 {
2620         struct c2c_hist_entry *c2c_he;
2621         struct c2c_hists *c2c_hists;
2622         struct c2c_cacheline_browser *cl_browser;
2623         struct hist_browser *browser;
2624         int key = -1;
2625         static const char help[] =
2626         " ENTER         Toggle callchains (if present) \n"
2627         " n             Toggle Node details info \n"
2628         " s             Toggle full length of symbol and source line columns \n"
2629         " q             Return back to cacheline list \n";
2630
2631         if (!he)
2632                 return 0;
2633
2634         /* Display compact version first. */
2635         c2c.symbol_full = false;
2636
2637         c2c_he = container_of(he, struct c2c_hist_entry, he);
2638         c2c_hists = c2c_he->hists;
2639
2640         cl_browser = c2c_cacheline_browser__new(&c2c_hists->hists, he);
2641         if (cl_browser == NULL)
2642                 return -1;
2643
2644         browser = &cl_browser->hb;
2645
2646         /* reset abort key so that it can get Ctrl-C as a key */
2647         SLang_reset_tty();
2648         SLang_init_tty(0, 0, 0);
2649
2650         c2c_browser__update_nr_entries(browser);
2651
2652         while (1) {
2653                 key = hist_browser__run(browser, "? - help", true, 0);
2654
2655                 switch (key) {
2656                 case 's':
2657                         c2c.symbol_full = !c2c.symbol_full;
2658                         break;
2659                 case 'n':
2660                         c2c.node_info = (c2c.node_info + 1) % 3;
2661                         setup_nodes_header();
2662                         break;
2663                 case 'q':
2664                         goto out;
2665                 case '?':
2666                         ui_browser__help_window(&browser->b, help);
2667                         break;
2668                 default:
2669                         break;
2670                 }
2671         }
2672
2673 out:
2674         free(cl_browser);
2675         return 0;
2676 }
2677
2678 static int perf_c2c_browser__title(struct hist_browser *browser,
2679                                    char *bf, size_t size)
2680 {
2681         scnprintf(bf, size,
2682                   "Shared Data Cache Line Table     "
2683                   "(%lu entries, sorted on %s)",
2684                   browser->nr_non_filtered_entries,
2685                   display_str[c2c.display]);
2686         return 0;
2687 }
2688
2689 static struct hist_browser*
2690 perf_c2c_browser__new(struct hists *hists)
2691 {
2692         struct hist_browser *browser = hist_browser__new(hists);
2693
2694         if (browser) {
2695                 browser->title = perf_c2c_browser__title;
2696                 browser->c2c_filter = true;
2697         }
2698
2699         return browser;
2700 }
2701
2702 static int perf_c2c__hists_browse(struct hists *hists)
2703 {
2704         struct hist_browser *browser;
2705         int key = -1;
2706         static const char help[] =
2707         " d             Display cacheline details \n"
2708         " ENTER         Toggle callchains (if present) \n"
2709         " q             Quit \n";
2710
2711         browser = perf_c2c_browser__new(hists);
2712         if (browser == NULL)
2713                 return -1;
2714
2715         /* reset abort key so that it can get Ctrl-C as a key */
2716         SLang_reset_tty();
2717         SLang_init_tty(0, 0, 0);
2718
2719         c2c_browser__update_nr_entries(browser);
2720
2721         while (1) {
2722                 key = hist_browser__run(browser, "? - help", true, 0);
2723
2724                 switch (key) {
2725                 case 'q':
2726                         goto out;
2727                 case 'd':
2728                         perf_c2c__browse_cacheline(browser->he_selection);
2729                         break;
2730                 case '?':
2731                         ui_browser__help_window(&browser->b, help);
2732                         break;
2733                 default:
2734                         break;
2735                 }
2736         }
2737
2738 out:
2739         hist_browser__delete(browser);
2740         return 0;
2741 }
2742
2743 static void perf_c2c_display(struct perf_session *session)
2744 {
2745         if (use_browser == 0)
2746                 perf_c2c__hists_fprintf(stdout, session);
2747         else
2748                 perf_c2c__hists_browse(&c2c.hists.hists);
2749 }
2750 #else
2751 static void perf_c2c_display(struct perf_session *session)
2752 {
2753         use_browser = 0;
2754         perf_c2c__hists_fprintf(stdout, session);
2755 }
2756 #endif /* HAVE_SLANG_SUPPORT */
2757
2758 static char *fill_line(const char *orig, int len)
2759 {
2760         int i, j, olen = strlen(orig);
2761         char *buf;
2762
2763         buf = zalloc(len + 1);
2764         if (!buf)
2765                 return NULL;
2766
2767         j = len / 2 - olen / 2;
2768
2769         for (i = 0; i < j - 1; i++)
2770                 buf[i] = '-';
2771
2772         buf[i++] = ' ';
2773
2774         strcpy(buf + i, orig);
2775
2776         i += olen;
2777
2778         buf[i++] = ' ';
2779
2780         for (; i < len; i++)
2781                 buf[i] = '-';
2782
2783         return buf;
2784 }
2785
2786 static int ui_quirks(void)
2787 {
2788         const char *nodestr = "Data address";
2789         char *buf;
2790
2791         if (!c2c.use_stdio) {
2792                 dim_offset.width  = 5;
2793                 dim_offset.header = header_offset_tui;
2794                 nodestr = chk_double_cl ? "Double-CL" : "CL";
2795         }
2796
2797         dim_percent_costly_snoop.header = percent_costly_snoop_header[c2c.display];
2798
2799         /* Fix the zero line for dcacheline column. */
2800         buf = fill_line(chk_double_cl ? "Double-Cacheline" : "Cacheline",
2801                                 dim_dcacheline.width +
2802                                 dim_dcacheline_node.width +
2803                                 dim_dcacheline_count.width + 4);
2804         if (!buf)
2805                 return -ENOMEM;
2806
2807         dim_dcacheline.header.line[0].text = buf;
2808
2809         /* Fix the zero line for offset column. */
2810         buf = fill_line(nodestr, dim_offset.width +
2811                                  dim_offset_node.width +
2812                                  dim_dcacheline_count.width + 4);
2813         if (!buf)
2814                 return -ENOMEM;
2815
2816         dim_offset.header.line[0].text = buf;
2817
2818         return 0;
2819 }
2820
2821 #define CALLCHAIN_DEFAULT_OPT  "graph,0.5,caller,function,percent"
2822
2823 const char callchain_help[] = "Display call graph (stack chain/backtrace):\n\n"
2824                                 CALLCHAIN_REPORT_HELP
2825                                 "\n\t\t\t\tDefault: " CALLCHAIN_DEFAULT_OPT;
2826
2827 static int
2828 parse_callchain_opt(const struct option *opt, const char *arg, int unset)
2829 {
2830         struct callchain_param *callchain = opt->value;
2831
2832         callchain->enabled = !unset;
2833         /*
2834          * --no-call-graph
2835          */
2836         if (unset) {
2837                 symbol_conf.use_callchain = false;
2838                 callchain->mode = CHAIN_NONE;
2839                 return 0;
2840         }
2841
2842         return parse_callchain_report_opt(arg);
2843 }
2844
2845 static int setup_callchain(struct evlist *evlist)
2846 {
2847         u64 sample_type = evlist__combined_sample_type(evlist);
2848         enum perf_call_graph_mode mode = CALLCHAIN_NONE;
2849
2850         if ((sample_type & PERF_SAMPLE_REGS_USER) &&
2851             (sample_type & PERF_SAMPLE_STACK_USER)) {
2852                 mode = CALLCHAIN_DWARF;
2853                 dwarf_callchain_users = true;
2854         } else if (sample_type & PERF_SAMPLE_BRANCH_STACK)
2855                 mode = CALLCHAIN_LBR;
2856         else if (sample_type & PERF_SAMPLE_CALLCHAIN)
2857                 mode = CALLCHAIN_FP;
2858
2859         if (!callchain_param.enabled &&
2860             callchain_param.mode != CHAIN_NONE &&
2861             mode != CALLCHAIN_NONE) {
2862                 symbol_conf.use_callchain = true;
2863                 if (callchain_register_param(&callchain_param) < 0) {
2864                         ui__error("Can't register callchain params.\n");
2865                         return -EINVAL;
2866                 }
2867         }
2868
2869         if (c2c.stitch_lbr && (mode != CALLCHAIN_LBR)) {
2870                 ui__warning("Can't find LBR callchain. Switch off --stitch-lbr.\n"
2871                             "Please apply --call-graph lbr when recording.\n");
2872                 c2c.stitch_lbr = false;
2873         }
2874
2875         callchain_param.record_mode = mode;
2876         callchain_param.min_percent = 0;
2877         return 0;
2878 }
2879
2880 static int setup_display(const char *str)
2881 {
2882         const char *display = str;
2883
2884         if (!strcmp(display, "tot"))
2885                 c2c.display = DISPLAY_TOT_HITM;
2886         else if (!strcmp(display, "rmt"))
2887                 c2c.display = DISPLAY_RMT_HITM;
2888         else if (!strcmp(display, "lcl"))
2889                 c2c.display = DISPLAY_LCL_HITM;
2890         else if (!strcmp(display, "peer"))
2891                 c2c.display = DISPLAY_SNP_PEER;
2892         else {
2893                 pr_err("failed: unknown display type: %s\n", str);
2894                 return -1;
2895         }
2896
2897         return 0;
2898 }
2899
2900 #define for_each_token(__tok, __buf, __sep, __tmp)              \
2901         for (__tok = strtok_r(__buf, __sep, &__tmp); __tok;     \
2902              __tok = strtok_r(NULL,  __sep, &__tmp))
2903
2904 static int build_cl_output(char *cl_sort, bool no_source)
2905 {
2906         char *tok, *tmp, *buf = strdup(cl_sort);
2907         bool add_pid   = false;
2908         bool add_tid   = false;
2909         bool add_iaddr = false;
2910         bool add_sym   = false;
2911         bool add_dso   = false;
2912         bool add_src   = false;
2913         int ret = 0;
2914
2915         if (!buf)
2916                 return -ENOMEM;
2917
2918         for_each_token(tok, buf, ",", tmp) {
2919                 if (!strcmp(tok, "tid")) {
2920                         add_tid = true;
2921                 } else if (!strcmp(tok, "pid")) {
2922                         add_pid = true;
2923                 } else if (!strcmp(tok, "iaddr")) {
2924                         add_iaddr = true;
2925                         add_sym   = true;
2926                         add_dso   = true;
2927                         add_src   = no_source ? false : true;
2928                 } else if (!strcmp(tok, "dso")) {
2929                         add_dso = true;
2930                 } else if (strcmp(tok, "offset")) {
2931                         pr_err("unrecognized sort token: %s\n", tok);
2932                         ret = -EINVAL;
2933                         goto err;
2934                 }
2935         }
2936
2937         if (asprintf(&c2c.cl_output,
2938                 "%s%s%s%s%s%s%s%s%s%s%s%s",
2939                 c2c.use_stdio ? "cl_num_empty," : "",
2940                 c2c.display == DISPLAY_SNP_PEER ? "percent_rmt_peer,"
2941                                                   "percent_lcl_peer," :
2942                                                   "percent_rmt_hitm,"
2943                                                   "percent_lcl_hitm,",
2944                 "percent_stores_l1hit,"
2945                 "percent_stores_l1miss,"
2946                 "percent_stores_na,"
2947                 "offset,offset_node,dcacheline_count,",
2948                 add_pid   ? "pid," : "",
2949                 add_tid   ? "tid," : "",
2950                 add_iaddr ? "iaddr," : "",
2951                 c2c.display == DISPLAY_SNP_PEER ? "mean_rmt_peer,"
2952                                                   "mean_lcl_peer," :
2953                                                   "mean_rmt,"
2954                                                   "mean_lcl,",
2955                 "mean_load,"
2956                 "tot_recs,"
2957                 "cpucnt,",
2958                 add_sym ? "symbol," : "",
2959                 add_dso ? "dso," : "",
2960                 add_src ? "cl_srcline," : "",
2961                 "node") < 0) {
2962                 ret = -ENOMEM;
2963                 goto err;
2964         }
2965
2966         c2c.show_src = add_src;
2967 err:
2968         free(buf);
2969         return ret;
2970 }
2971
2972 static int setup_coalesce(const char *coalesce, bool no_source)
2973 {
2974         const char *c = coalesce ?: coalesce_default;
2975         const char *sort_str = NULL;
2976
2977         if (asprintf(&c2c.cl_sort, "offset,%s", c) < 0)
2978                 return -ENOMEM;
2979
2980         if (build_cl_output(c2c.cl_sort, no_source))
2981                 return -1;
2982
2983         if (c2c.display == DISPLAY_TOT_HITM)
2984                 sort_str = "tot_hitm";
2985         else if (c2c.display == DISPLAY_RMT_HITM)
2986                 sort_str = "rmt_hitm,lcl_hitm";
2987         else if (c2c.display == DISPLAY_LCL_HITM)
2988                 sort_str = "lcl_hitm,rmt_hitm";
2989         else if (c2c.display == DISPLAY_SNP_PEER)
2990                 sort_str = "tot_peer";
2991
2992         if (asprintf(&c2c.cl_resort, "offset,%s", sort_str) < 0)
2993                 return -ENOMEM;
2994
2995         pr_debug("coalesce sort   fields: %s\n", c2c.cl_sort);
2996         pr_debug("coalesce resort fields: %s\n", c2c.cl_resort);
2997         pr_debug("coalesce output fields: %s\n", c2c.cl_output);
2998         return 0;
2999 }
3000
3001 static int perf_c2c__report(int argc, const char **argv)
3002 {
3003         struct itrace_synth_opts itrace_synth_opts = {
3004                 .set = true,
3005                 .mem = true,    /* Only enable memory event */
3006                 .default_no_sample = true,
3007         };
3008
3009         struct perf_session *session;
3010         struct ui_progress prog;
3011         struct perf_data data = {
3012                 .mode = PERF_DATA_MODE_READ,
3013         };
3014         char callchain_default_opt[] = CALLCHAIN_DEFAULT_OPT;
3015         const char *display = NULL;
3016         const char *coalesce = NULL;
3017         bool no_source = false;
3018         const struct option options[] = {
3019         OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
3020                    "file", "vmlinux pathname"),
3021         OPT_STRING('i', "input", &input_name, "file",
3022                    "the input file to process"),
3023         OPT_INCR('N', "node-info", &c2c.node_info,
3024                  "show extra node info in report (repeat for more info)"),
3025         OPT_BOOLEAN(0, "stdio", &c2c.use_stdio, "Use the stdio interface"),
3026         OPT_BOOLEAN(0, "stats", &c2c.stats_only,
3027                     "Display only statistic tables (implies --stdio)"),
3028         OPT_BOOLEAN(0, "full-symbols", &c2c.symbol_full,
3029                     "Display full length of symbols"),
3030         OPT_BOOLEAN(0, "no-source", &no_source,
3031                     "Do not display Source Line column"),
3032         OPT_BOOLEAN(0, "show-all", &c2c.show_all,
3033                     "Show all captured HITM lines."),
3034         OPT_CALLBACK_DEFAULT('g', "call-graph", &callchain_param,
3035                              "print_type,threshold[,print_limit],order,sort_key[,branch],value",
3036                              callchain_help, &parse_callchain_opt,
3037                              callchain_default_opt),
3038         OPT_STRING('d', "display", &display, "Switch HITM output type", "tot,lcl,rmt,peer"),
3039         OPT_STRING('c', "coalesce", &coalesce, "coalesce fields",
3040                    "coalesce fields: pid,tid,iaddr,dso"),
3041         OPT_BOOLEAN('f', "force", &symbol_conf.force, "don't complain, do it"),
3042         OPT_BOOLEAN(0, "stitch-lbr", &c2c.stitch_lbr,
3043                     "Enable LBR callgraph stitching approach"),
3044         OPT_BOOLEAN(0, "double-cl", &chk_double_cl, "Detect adjacent cacheline false sharing"),
3045         OPT_PARENT(c2c_options),
3046         OPT_END()
3047         };
3048         int err = 0;
3049         const char *output_str, *sort_str = NULL;
3050
3051         argc = parse_options(argc, argv, options, report_c2c_usage,
3052                              PARSE_OPT_STOP_AT_NON_OPTION);
3053         if (argc)
3054                 usage_with_options(report_c2c_usage, options);
3055
3056 #ifndef HAVE_SLANG_SUPPORT
3057         c2c.use_stdio = true;
3058 #endif
3059
3060         if (c2c.stats_only)
3061                 c2c.use_stdio = true;
3062
3063         err = symbol__validate_sym_arguments();
3064         if (err)
3065                 goto out;
3066
3067         if (!input_name || !strlen(input_name))
3068                 input_name = "perf.data";
3069
3070         data.path  = input_name;
3071         data.force = symbol_conf.force;
3072
3073         session = perf_session__new(&data, &c2c.tool);
3074         if (IS_ERR(session)) {
3075                 err = PTR_ERR(session);
3076                 pr_debug("Error creating perf session\n");
3077                 goto out;
3078         }
3079
3080         /*
3081          * Use the 'tot' as default display type if user doesn't specify it;
3082          * since Arm64 platform doesn't support HITMs flag, use 'peer' as the
3083          * default display type.
3084          */
3085         if (!display) {
3086                 if (!strcmp(perf_env__arch(&session->header.env), "arm64"))
3087                         display = "peer";
3088                 else
3089                         display = "tot";
3090         }
3091
3092         err = setup_display(display);
3093         if (err)
3094                 goto out_session;
3095
3096         err = setup_coalesce(coalesce, no_source);
3097         if (err) {
3098                 pr_debug("Failed to initialize hists\n");
3099                 goto out_session;
3100         }
3101
3102         err = c2c_hists__init(&c2c.hists, "dcacheline", 2);
3103         if (err) {
3104                 pr_debug("Failed to initialize hists\n");
3105                 goto out_session;
3106         }
3107
3108         session->itrace_synth_opts = &itrace_synth_opts;
3109
3110         err = setup_nodes(session);
3111         if (err) {
3112                 pr_err("Failed setup nodes\n");
3113                 goto out_session;
3114         }
3115
3116         err = mem2node__init(&c2c.mem2node, &session->header.env);
3117         if (err)
3118                 goto out_session;
3119
3120         err = setup_callchain(session->evlist);
3121         if (err)
3122                 goto out_mem2node;
3123
3124         if (symbol__init(&session->header.env) < 0)
3125                 goto out_mem2node;
3126
3127         /* No pipe support at the moment. */
3128         if (perf_data__is_pipe(session->data)) {
3129                 pr_debug("No pipe support at the moment.\n");
3130                 goto out_mem2node;
3131         }
3132
3133         if (c2c.use_stdio)
3134                 use_browser = 0;
3135         else
3136                 use_browser = 1;
3137
3138         setup_browser(false);
3139
3140         err = perf_session__process_events(session);
3141         if (err) {
3142                 pr_err("failed to process sample\n");
3143                 goto out_mem2node;
3144         }
3145
3146         if (c2c.display != DISPLAY_SNP_PEER)
3147                 output_str = "cl_idx,"
3148                              "dcacheline,"
3149                              "dcacheline_node,"
3150                              "dcacheline_count,"
3151                              "percent_costly_snoop,"
3152                              "tot_hitm,lcl_hitm,rmt_hitm,"
3153                              "tot_recs,"
3154                              "tot_loads,"
3155                              "tot_stores,"
3156                              "stores_l1hit,stores_l1miss,stores_na,"
3157                              "ld_fbhit,ld_l1hit,ld_l2hit,"
3158                              "ld_lclhit,lcl_hitm,"
3159                              "ld_rmthit,rmt_hitm,"
3160                              "dram_lcl,dram_rmt";
3161         else
3162                 output_str = "cl_idx,"
3163                              "dcacheline,"
3164                              "dcacheline_node,"
3165                              "dcacheline_count,"
3166                              "percent_costly_snoop,"
3167                              "tot_peer,lcl_peer,rmt_peer,"
3168                              "tot_recs,"
3169                              "tot_loads,"
3170                              "tot_stores,"
3171                              "stores_l1hit,stores_l1miss,stores_na,"
3172                              "ld_fbhit,ld_l1hit,ld_l2hit,"
3173                              "ld_lclhit,lcl_hitm,"
3174                              "ld_rmthit,rmt_hitm,"
3175                              "dram_lcl,dram_rmt";
3176
3177         if (c2c.display == DISPLAY_TOT_HITM)
3178                 sort_str = "tot_hitm";
3179         else if (c2c.display == DISPLAY_RMT_HITM)
3180                 sort_str = "rmt_hitm";
3181         else if (c2c.display == DISPLAY_LCL_HITM)
3182                 sort_str = "lcl_hitm";
3183         else if (c2c.display == DISPLAY_SNP_PEER)
3184                 sort_str = "tot_peer";
3185
3186         c2c_hists__reinit(&c2c.hists, output_str, sort_str);
3187
3188         ui_progress__init(&prog, c2c.hists.hists.nr_entries, "Sorting...");
3189
3190         hists__collapse_resort(&c2c.hists.hists, NULL);
3191         hists__output_resort_cb(&c2c.hists.hists, &prog, resort_shared_cl_cb);
3192         hists__iterate_cb(&c2c.hists.hists, resort_cl_cb);
3193
3194         ui_progress__finish();
3195
3196         if (ui_quirks()) {
3197                 pr_err("failed to setup UI\n");
3198                 goto out_mem2node;
3199         }
3200
3201         perf_c2c_display(session);
3202
3203 out_mem2node:
3204         mem2node__exit(&c2c.mem2node);
3205 out_session:
3206         perf_session__delete(session);
3207 out:
3208         return err;
3209 }
3210
3211 static int parse_record_events(const struct option *opt,
3212                                const char *str, int unset __maybe_unused)
3213 {
3214         bool *event_set = (bool *) opt->value;
3215         struct perf_pmu *pmu;
3216
3217         pmu = perf_mem_events_find_pmu();
3218         if (!pmu) {
3219                 pr_err("failed: there is no PMU that supports perf c2c\n");
3220                 exit(-1);
3221         }
3222
3223         if (!strcmp(str, "list")) {
3224                 perf_pmu__mem_events_list(pmu);
3225                 exit(0);
3226         }
3227         if (perf_pmu__mem_events_parse(pmu, str))
3228                 exit(-1);
3229
3230         *event_set = true;
3231         return 0;
3232 }
3233
3234
3235 static const char * const __usage_record[] = {
3236         "perf c2c record [<options>] [<command>]",
3237         "perf c2c record [<options>] -- <command> [<options>]",
3238         NULL
3239 };
3240
3241 static const char * const *record_mem_usage = __usage_record;
3242
3243 static int perf_c2c__record(int argc, const char **argv)
3244 {
3245         int rec_argc, i = 0, j;
3246         const char **rec_argv;
3247         int ret;
3248         bool all_user = false, all_kernel = false;
3249         bool event_set = false;
3250         struct perf_mem_event *e;
3251         struct perf_pmu *pmu;
3252         struct option options[] = {
3253         OPT_CALLBACK('e', "event", &event_set, "event",
3254                      "event selector. Use 'perf c2c record -e list' to list available events",
3255                      parse_record_events),
3256         OPT_BOOLEAN('u', "all-user", &all_user, "collect only user level data"),
3257         OPT_BOOLEAN('k', "all-kernel", &all_kernel, "collect only kernel level data"),
3258         OPT_UINTEGER('l', "ldlat", &perf_mem_events__loads_ldlat, "setup mem-loads latency"),
3259         OPT_PARENT(c2c_options),
3260         OPT_END()
3261         };
3262
3263         pmu = perf_mem_events_find_pmu();
3264         if (!pmu) {
3265                 pr_err("failed: no PMU supports the memory events\n");
3266                 return -1;
3267         }
3268
3269         if (perf_pmu__mem_events_init(pmu)) {
3270                 pr_err("failed: memory events not supported\n");
3271                 return -1;
3272         }
3273
3274         argc = parse_options(argc, argv, options, record_mem_usage,
3275                              PARSE_OPT_KEEP_UNKNOWN);
3276
3277         /* Max number of arguments multiplied by number of PMUs that can support them. */
3278         rec_argc = argc + 11 * (perf_pmu__mem_events_num_mem_pmus(pmu) + 1);
3279
3280         rec_argv = calloc(rec_argc + 1, sizeof(char *));
3281         if (!rec_argv)
3282                 return -1;
3283
3284         rec_argv[i++] = "record";
3285
3286         if (!event_set) {
3287                 e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD_STORE);
3288                 /*
3289                  * The load and store operations are required, use the event
3290                  * PERF_MEM_EVENTS__LOAD_STORE if it is supported.
3291                  */
3292                 if (e->tag) {
3293                         e->record = true;
3294                         rec_argv[i++] = "-W";
3295                 } else {
3296                         e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD);
3297                         e->record = true;
3298
3299                         e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__STORE);
3300                         e->record = true;
3301                 }
3302         }
3303
3304         e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD);
3305         if (e->record)
3306                 rec_argv[i++] = "-W";
3307
3308         rec_argv[i++] = "-d";
3309         rec_argv[i++] = "--phys-data";
3310         rec_argv[i++] = "--sample-cpu";
3311
3312         ret = perf_mem_events__record_args(rec_argv, &i);
3313         if (ret)
3314                 goto out;
3315
3316         if (all_user)
3317                 rec_argv[i++] = "--all-user";
3318
3319         if (all_kernel)
3320                 rec_argv[i++] = "--all-kernel";
3321
3322         for (j = 0; j < argc; j++, i++)
3323                 rec_argv[i] = argv[j];
3324
3325         if (verbose > 0) {
3326                 pr_debug("calling: ");
3327
3328                 j = 0;
3329
3330                 while (rec_argv[j]) {
3331                         pr_debug("%s ", rec_argv[j]);
3332                         j++;
3333                 }
3334                 pr_debug("\n");
3335         }
3336
3337         ret = cmd_record(i, rec_argv);
3338 out:
3339         free(rec_argv);
3340         return ret;
3341 }
3342
3343 int cmd_c2c(int argc, const char **argv)
3344 {
3345         argc = parse_options(argc, argv, c2c_options, c2c_usage,
3346                              PARSE_OPT_STOP_AT_NON_OPTION);
3347
3348         if (!argc)
3349                 usage_with_options(c2c_usage, c2c_options);
3350
3351         if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
3352                 return perf_c2c__record(argc, argv);
3353         } else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) {
3354                 return perf_c2c__report(argc, argv);
3355         } else {
3356                 usage_with_options(c2c_usage, c2c_options);
3357         }
3358
3359         return 0;
3360 }