tools/perf/builtin-c2c.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This is rewrite of original c2c tool introduced in here:
   4  *   http://lwn.net/Articles/588866/
   5  *
   6  * The original tool was changed to fit in current perf state.
   7  *
   8  * Original authors:
   9  *   Don Zickus <[email protected]>
  10  *   Dick Fowles <[email protected]>
  11  *   Joe Mario <[email protected]>
  12  */
  13 #include <errno.h>
  14 #include <inttypes.h>
  15 #include <linux/compiler.h>
  16 #include <linux/err.h>
  17 #include <linux/kernel.h>
  18 #include <linux/stringify.h>
  19 #include <linux/zalloc.h>
  20 #include <asm/bug.h>
  21 #include <sys/param.h>
  22 #include "debug.h"
  23 #include "builtin.h"
  24 #include <perf/cpumap.h>
  25 #include <subcmd/pager.h>
  26 #include <subcmd/parse-options.h>
  27 #include "map_symbol.h"
  28 #include "mem-events.h"
  29 #include "session.h"
  30 #include "hist.h"
  31 #include "sort.h"
  32 #include "tool.h"
  33 #include "cacheline.h"
  34 #include "data.h"
  35 #include "event.h"
  36 #include "evlist.h"
  37 #include "evsel.h"
  38 #include "ui/browsers/hists.h"
  39 #include "thread.h"
  40 #include "mem2node.h"
  41 #include "mem-info.h"
  42 #include "symbol.h"
  43 #include "ui/ui.h"
  44 #include "ui/progress.h"
  45 #include "pmus.h"
  46 #include "string2.h"
  47 #include "util/util.h"
  48
  49 struct c2c_hists {
  50         struct hists            hists;
  51         struct perf_hpp_list    list;
  52         struct c2c_stats        stats;
  53 };
  54
  55 struct compute_stats {
  56         struct stats             lcl_hitm;
  57         struct stats             rmt_hitm;
  58         struct stats             lcl_peer;
  59         struct stats             rmt_peer;
  60         struct stats             load;
  61 };
  62
  63 struct c2c_hist_entry {
  64         struct c2c_hists        *hists;
  65         struct c2c_stats         stats;
  66         unsigned long           *cpuset;
  67         unsigned long           *nodeset;
  68         struct c2c_stats        *node_stats;
  69         unsigned int             cacheline_idx;
  70
  71         struct compute_stats     cstats;
  72
  73         unsigned long            paddr;
  74         unsigned long            paddr_cnt;
  75         bool                     paddr_zero;
  76         char                    *nodestr;
  77
  78         /*
  79          * must be at the end,
  80          * because of its callchain dynamic entry
  81          */
  82         struct hist_entry       he;
  83 };
  84
  85 static char const *coalesce_default = "iaddr";
  86
  87 struct perf_c2c {
  88         struct perf_tool        tool;
  89         struct c2c_hists        hists;
  90         struct mem2node         mem2node;
  91
  92         unsigned long           **nodes;
  93         int                      nodes_cnt;
  94         int                      cpus_cnt;
  95         int                     *cpu2node;
  96         int                      node_info;
  97
  98         bool                     show_src;
  99         bool                     show_all;
 100         bool                     use_stdio;
 101         bool                     stats_only;
 102         bool                     symbol_full;
 103         bool                     stitch_lbr;
 104
 105         /* Shared cache line stats */
 106         struct c2c_stats        shared_clines_stats;
 107         int                     shared_clines;
 108
 109         int                      display;
 110
 111         const char              *coalesce;
 112         char                    *cl_sort;
 113         char                    *cl_resort;
 114         char                    *cl_output;
 115 };
 116
 117 enum {
 118         DISPLAY_LCL_HITM,
 119         DISPLAY_RMT_HITM,
 120         DISPLAY_TOT_HITM,
 121         DISPLAY_SNP_PEER,
 122         DISPLAY_MAX,
 123 };
 124
 125 static const char *display_str[DISPLAY_MAX] = {
 126         [DISPLAY_LCL_HITM] = "Local HITMs",
 127         [DISPLAY_RMT_HITM] = "Remote HITMs",
 128         [DISPLAY_TOT_HITM] = "Total HITMs",
 129         [DISPLAY_SNP_PEER] = "Peer Snoop",
 130 };
 131
 132 static const struct option c2c_options[] = {
 133         OPT_INCR('v', "verbose", &verbose, "be more verbose (show counter open errors, etc)"),
 134         OPT_END()
 135 };
 136
 137 static struct perf_c2c c2c;
 138
 139 static void *c2c_he_zalloc(size_t size)
 140 {
 141         struct c2c_hist_entry *c2c_he;
 142
 143         c2c_he = zalloc(size + sizeof(*c2c_he));
 144         if (!c2c_he)
 145                 return NULL;
 146
 147         c2c_he->cpuset = bitmap_zalloc(c2c.cpus_cnt);
 148         if (!c2c_he->cpuset)
 149                 goto out_free;
 150
 151         c2c_he->nodeset = bitmap_zalloc(c2c.nodes_cnt);
 152         if (!c2c_he->nodeset)
 153                 goto out_free;
 154
 155         c2c_he->node_stats = zalloc(c2c.nodes_cnt * sizeof(*c2c_he->node_stats));
 156         if (!c2c_he->node_stats)
 157                 goto out_free;
 158
 159         init_stats(&c2c_he->cstats.lcl_hitm);
 160         init_stats(&c2c_he->cstats.rmt_hitm);
 161         init_stats(&c2c_he->cstats.lcl_peer);
 162         init_stats(&c2c_he->cstats.rmt_peer);
 163         init_stats(&c2c_he->cstats.load);
 164
 165         return &c2c_he->he;
 166
 167 out_free:
 168         zfree(&c2c_he->nodeset);
 169         zfree(&c2c_he->cpuset);
 170         free(c2c_he);
 171         return NULL;
 172 }
 173
 174 static void c2c_he_free(void *he)
 175 {
 176         struct c2c_hist_entry *c2c_he;
 177
 178         c2c_he = container_of(he, struct c2c_hist_entry, he);
 179         if (c2c_he->hists) {
 180                 hists__delete_entries(&c2c_he->hists->hists);
 181                 zfree(&c2c_he->hists);
 182         }
 183
 184         zfree(&c2c_he->cpuset);
 185         zfree(&c2c_he->nodeset);
 186         zfree(&c2c_he->nodestr);
 187         zfree(&c2c_he->node_stats);
 188         free(c2c_he);
 189 }
 190
 191 static struct hist_entry_ops c2c_entry_ops = {
 192         .new    = c2c_he_zalloc,
 193         .free   = c2c_he_free,
 194 };
 195
 196 static int c2c_hists__init(struct c2c_hists *hists,
 197                            const char *sort,
 198                            int nr_header_lines);
 199
 200 static struct c2c_hists*
 201 he__get_c2c_hists(struct hist_entry *he,
 202                   const char *sort,
 203                   int nr_header_lines)
 204 {
 205         struct c2c_hist_entry *c2c_he;
 206         struct c2c_hists *hists;
 207         int ret;
 208
 209         c2c_he = container_of(he, struct c2c_hist_entry, he);
 210         if (c2c_he->hists)
 211                 return c2c_he->hists;
 212
 213         hists = c2c_he->hists = zalloc(sizeof(*hists));
 214         if (!hists)
 215                 return NULL;
 216
 217         ret = c2c_hists__init(hists, sort, nr_header_lines);
 218         if (ret) {
 219                 free(hists);
 220                 return NULL;
 221         }
 222
 223         return hists;
 224 }
 225
 226 static void c2c_he__set_cpu(struct c2c_hist_entry *c2c_he,
 227                             struct perf_sample *sample)
 228 {
 229         if (WARN_ONCE(sample->cpu == (unsigned int) -1,
 230                       "WARNING: no sample cpu value"))
 231                 return;
 232
 233         __set_bit(sample->cpu, c2c_he->cpuset);
 234 }
 235
 236 static void c2c_he__set_node(struct c2c_hist_entry *c2c_he,
 237                              struct perf_sample *sample)
 238 {
 239         int node;
 240
 241         if (!sample->phys_addr) {
 242                 c2c_he->paddr_zero = true;
 243                 return;
 244         }
 245
 246         node = mem2node__node(&c2c.mem2node, sample->phys_addr);
 247         if (WARN_ONCE(node < 0, "WARNING: failed to find node\n"))
 248                 return;
 249
 250         __set_bit(node, c2c_he->nodeset);
 251
 252         if (c2c_he->paddr != sample->phys_addr) {
 253                 c2c_he->paddr_cnt++;
 254                 c2c_he->paddr = sample->phys_addr;
 255         }
 256 }
 257
 258 static void compute_stats(struct c2c_hist_entry *c2c_he,
 259                           struct c2c_stats *stats,
 260                           u64 weight)
 261 {
 262         struct compute_stats *cstats = &c2c_he->cstats;
 263
 264         if (stats->rmt_hitm)
 265                 update_stats(&cstats->rmt_hitm, weight);
 266         else if (stats->lcl_hitm)
 267                 update_stats(&cstats->lcl_hitm, weight);
 268         else if (stats->rmt_peer)
 269                 update_stats(&cstats->rmt_peer, weight);
 270         else if (stats->lcl_peer)
 271                 update_stats(&cstats->lcl_peer, weight);
 272         else if (stats->load)
 273                 update_stats(&cstats->load, weight);
 274 }
 275
 276 static int process_sample_event(const struct perf_tool *tool __maybe_unused,
 277                                 union perf_event *event,
 278                                 struct perf_sample *sample,
 279                                 struct evsel *evsel,
 280                                 struct machine *machine)
 281 {
 282         struct c2c_hists *c2c_hists = &c2c.hists;
 283         struct c2c_hist_entry *c2c_he;
 284         struct c2c_stats stats = { .nr_entries = 0, };
 285         struct hist_entry *he;
 286         struct addr_location al;
 287         struct mem_info *mi, *mi_dup;
 288         struct callchain_cursor *cursor;
 289         int ret;
 290
 291         addr_location__init(&al);
 292         if (machine__resolve(machine, &al, sample) < 0) {
 293                 pr_debug("problem processing %d event, skipping it.\n",
 294                          event->header.type);
 295                 ret = -1;
 296                 goto out;
 297         }
 298
 299         if (c2c.stitch_lbr)
 300                 thread__set_lbr_stitch_enable(al.thread, true);
 301
 302         cursor = get_tls_callchain_cursor();
 303         ret = sample__resolve_callchain(sample, cursor, NULL,
 304                                         evsel, &al, sysctl_perf_event_max_stack);
 305         if (ret)
 306                 goto out;
 307
 308         mi = sample__resolve_mem(sample, &al);
 309         if (mi == NULL) {
 310                 ret = -ENOMEM;
 311                 goto out;
 312         }
 313
 314         /*
 315          * The mi object is released in hists__add_entry_ops,
 316          * if it gets sorted out into existing data, so we need
 317          * to take the copy now.
 318          */
 319         mi_dup = mem_info__get(mi);
 320
 321         c2c_decode_stats(&stats, mi);
 322
 323         he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops,
 324                                   &al, NULL, NULL, mi, NULL,
 325                                   sample, true);
 326         if (he == NULL)
 327                 goto free_mi;
 328
 329         c2c_he = container_of(he, struct c2c_hist_entry, he);
 330         c2c_add_stats(&c2c_he->stats, &stats);
 331         c2c_add_stats(&c2c_hists->stats, &stats);
 332
 333         c2c_he__set_cpu(c2c_he, sample);
 334         c2c_he__set_node(c2c_he, sample);
 335
 336         hists__inc_nr_samples(&c2c_hists->hists, he->filtered);
 337         ret = hist_entry__append_callchain(he, sample);
 338
 339         if (!ret) {
 340                 /*
 341                  * There's already been warning about missing
 342                  * sample's cpu value. Let's account all to
 343                  * node 0 in this case, without any further
 344                  * warning.
 345                  *
 346                  * Doing node stats only for single callchain data.
 347                  */
 348                 int cpu = sample->cpu == (unsigned int) -1 ? 0 : sample->cpu;
 349                 int node = c2c.cpu2node[cpu];
 350
 351                 mi = mi_dup;
 352
 353                 c2c_hists = he__get_c2c_hists(he, c2c.cl_sort, 2);
 354                 if (!c2c_hists)
 355                         goto free_mi;
 356
 357                 he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops,
 358                                           &al, NULL, NULL, mi, NULL,
 359                                           sample, true);
 360                 if (he == NULL)
 361                         goto free_mi;
 362
 363                 c2c_he = container_of(he, struct c2c_hist_entry, he);
 364                 c2c_add_stats(&c2c_he->stats, &stats);
 365                 c2c_add_stats(&c2c_hists->stats, &stats);
 366                 c2c_add_stats(&c2c_he->node_stats[node], &stats);
 367
 368                 compute_stats(c2c_he, &stats, sample->weight);
 369
 370                 c2c_he__set_cpu(c2c_he, sample);
 371                 c2c_he__set_node(c2c_he, sample);
 372
 373                 hists__inc_nr_samples(&c2c_hists->hists, he->filtered);
 374                 ret = hist_entry__append_callchain(he, sample);
 375         }
 376
 377 out:
 378         addr_location__exit(&al);
 379         return ret;
 380
 381 free_mi:
 382         mem_info__put(mi_dup);
 383         mem_info__put(mi);
 384         ret = -ENOMEM;
 385         goto out;
 386 }
 387
 388 static const char * const c2c_usage[] = {
 389         "perf c2c {record|report}",
 390         NULL
 391 };
 392
 393 static const char * const __usage_report[] = {
 394         "perf c2c report",
 395         NULL
 396 };
 397
 398 static const char * const *report_c2c_usage = __usage_report;
 399
 400 #define C2C_HEADER_MAX 2
 401
 402 struct c2c_header {
 403         struct {
 404                 const char *text;
 405                 int         span;
 406         } line[C2C_HEADER_MAX];
 407 };
 408
 409 struct c2c_dimension {
 410         struct c2c_header        header;
 411         const char              *name;
 412         int                      width;
 413         struct sort_entry       *se;
 414
 415         int64_t (*cmp)(struct perf_hpp_fmt *fmt,
 416                        struct hist_entry *, struct hist_entry *);
 417         int   (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 418                        struct hist_entry *he);
 419         int   (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 420                        struct hist_entry *he);
 421 };
 422
 423 struct c2c_fmt {
 424         struct perf_hpp_fmt      fmt;
 425         struct c2c_dimension    *dim;
 426 };
 427
 428 #define SYMBOL_WIDTH 30
 429
 430 static struct c2c_dimension dim_symbol;
 431 static struct c2c_dimension dim_srcline;
 432
 433 static int symbol_width(struct hists *hists, struct sort_entry *se)
 434 {
 435         int width = hists__col_len(hists, se->se_width_idx);
 436
 437         if (!c2c.symbol_full)
 438                 width = MIN(width, SYMBOL_WIDTH);
 439
 440         return width;
 441 }
 442
 443 static int c2c_width(struct perf_hpp_fmt *fmt,
 444                      struct perf_hpp *hpp __maybe_unused,
 445                      struct hists *hists)
 446 {
 447         struct c2c_fmt *c2c_fmt;
 448         struct c2c_dimension *dim;
 449
 450         c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
 451         dim = c2c_fmt->dim;
 452
 453         if (dim == &dim_symbol || dim == &dim_srcline)
 454                 return symbol_width(hists, dim->se);
 455
 456         return dim->se ? hists__col_len(hists, dim->se->se_width_idx) :
 457                          c2c_fmt->dim->width;
 458 }
 459
 460 static int c2c_header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 461                       struct hists *hists, int line, int *span)
 462 {
 463         struct perf_hpp_list *hpp_list = hists->hpp_list;
 464         struct c2c_fmt *c2c_fmt;
 465         struct c2c_dimension *dim;
 466         const char *text = NULL;
 467         int width = c2c_width(fmt, hpp, hists);
 468
 469         c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
 470         dim = c2c_fmt->dim;
 471
 472         if (dim->se) {
 473                 text = dim->header.line[line].text;
 474                 /* Use the last line from sort_entry if not defined. */
 475                 if (!text && (line == hpp_list->nr_header_lines - 1))
 476                         text = dim->se->se_header;
 477         } else {
 478                 text = dim->header.line[line].text;
 479
 480                 if (*span) {
 481                         (*span)--;
 482                         return 0;
 483                 } else {
 484                         *span = dim->header.line[line].span;
 485                 }
 486         }
 487
 488         if (text == NULL)
 489                 text = "";
 490
 491         return scnprintf(hpp->buf, hpp->size, "%*s", width, text);
 492 }
 493
 494 #define HEX_STR(__s, __v)                               \
 495 ({                                                      \
 496         scnprintf(__s, sizeof(__s), "0x%" PRIx64, __v); \
 497         __s;                                            \
 498 })
 499
 500 static int64_t
 501 dcacheline_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 502                struct hist_entry *left, struct hist_entry *right)
 503 {
 504         return sort__dcacheline_cmp(left, right);
 505 }
 506
 507 static int dcacheline_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 508                             struct hist_entry *he)
 509 {
 510         uint64_t addr = 0;
 511         int width = c2c_width(fmt, hpp, he->hists);
 512         char buf[20];
 513
 514         if (he->mem_info)
 515                 addr = cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl);
 516
 517         return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
 518 }
 519
 520 static int
 521 dcacheline_node_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 522                       struct hist_entry *he)
 523 {
 524         struct c2c_hist_entry *c2c_he;
 525         int width = c2c_width(fmt, hpp, he->hists);
 526
 527         c2c_he = container_of(he, struct c2c_hist_entry, he);
 528         if (WARN_ON_ONCE(!c2c_he->nodestr))
 529                 return 0;
 530
 531         return scnprintf(hpp->buf, hpp->size, "%*s", width, c2c_he->nodestr);
 532 }
 533
 534 static int
 535 dcacheline_node_count(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 536                       struct hist_entry *he)
 537 {
 538         struct c2c_hist_entry *c2c_he;
 539         int width = c2c_width(fmt, hpp, he->hists);
 540
 541         c2c_he = container_of(he, struct c2c_hist_entry, he);
 542         return scnprintf(hpp->buf, hpp->size, "%*lu", width, c2c_he->paddr_cnt);
 543 }
 544
 545 static int offset_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 546                         struct hist_entry *he)
 547 {
 548         uint64_t addr = 0;
 549         int width = c2c_width(fmt, hpp, he->hists);
 550         char buf[20];
 551
 552         if (he->mem_info)
 553                 addr = cl_offset(mem_info__daddr(he->mem_info)->al_addr, chk_double_cl);
 554
 555         return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
 556 }
 557
 558 static int64_t
 559 offset_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 560            struct hist_entry *left, struct hist_entry *right)
 561 {
 562         uint64_t l = 0, r = 0;
 563
 564         if (left->mem_info)
 565                 l = cl_offset(mem_info__daddr(left->mem_info)->addr, chk_double_cl);
 566
 567         if (right->mem_info)
 568                 r = cl_offset(mem_info__daddr(right->mem_info)->addr, chk_double_cl);
 569
 570         return (int64_t)(r - l);
 571 }
 572
 573 static int
 574 iaddr_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 575             struct hist_entry *he)
 576 {
 577         uint64_t addr = 0;
 578         int width = c2c_width(fmt, hpp, he->hists);
 579         char buf[20];
 580
 581         if (he->mem_info)
 582                 addr = mem_info__iaddr(he->mem_info)->addr;
 583
 584         return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
 585 }
 586
 587 static int64_t
 588 iaddr_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 589           struct hist_entry *left, struct hist_entry *right)
 590 {
 591         return sort__iaddr_cmp(left, right);
 592 }
 593
 594 static int
 595 tot_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 596                struct hist_entry *he)
 597 {
 598         struct c2c_hist_entry *c2c_he;
 599         int width = c2c_width(fmt, hpp, he->hists);
 600         unsigned int tot_hitm;
 601
 602         c2c_he = container_of(he, struct c2c_hist_entry, he);
 603         tot_hitm = c2c_he->stats.lcl_hitm + c2c_he->stats.rmt_hitm;
 604
 605         return scnprintf(hpp->buf, hpp->size, "%*u", width, tot_hitm);
 606 }
 607
 608 static int64_t
 609 tot_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 610              struct hist_entry *left, struct hist_entry *right)
 611 {
 612         struct c2c_hist_entry *c2c_left;
 613         struct c2c_hist_entry *c2c_right;
 614         uint64_t tot_hitm_left;
 615         uint64_t tot_hitm_right;
 616
 617         c2c_left  = container_of(left, struct c2c_hist_entry, he);
 618         c2c_right = container_of(right, struct c2c_hist_entry, he);
 619
 620         tot_hitm_left  = c2c_left->stats.lcl_hitm + c2c_left->stats.rmt_hitm;
 621         tot_hitm_right = c2c_right->stats.lcl_hitm + c2c_right->stats.rmt_hitm;
 622
 623         return tot_hitm_left - tot_hitm_right;
 624 }
 625
 626 #define STAT_FN_ENTRY(__f)                                      \
 627 static int                                                      \
 628 __f ## _entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,   \
 629               struct hist_entry *he)                            \
 630 {                                                               \
 631         struct c2c_hist_entry *c2c_he;                          \
 632         int width = c2c_width(fmt, hpp, he->hists);             \
 633                                                                 \
 634         c2c_he = container_of(he, struct c2c_hist_entry, he);   \
 635         return scnprintf(hpp->buf, hpp->size, "%*u", width,     \
 636                          c2c_he->stats.__f);                    \
 637 }
 638
 639 #define STAT_FN_CMP(__f)                                                \
 640 static int64_t                                                          \
 641 __f ## _cmp(struct perf_hpp_fmt *fmt __maybe_unused,                    \
 642             struct hist_entry *left, struct hist_entry *right)          \
 643 {                                                                       \
 644         struct c2c_hist_entry *c2c_left, *c2c_right;                    \
 645                                                                         \
 646         c2c_left  = container_of(left, struct c2c_hist_entry, he);      \
 647         c2c_right = container_of(right, struct c2c_hist_entry, he);     \
 648         return (uint64_t) c2c_left->stats.__f -                         \
 649                (uint64_t) c2c_right->stats.__f;                         \
 650 }
 651
 652 #define STAT_FN(__f)            \
 653         STAT_FN_ENTRY(__f)      \
 654         STAT_FN_CMP(__f)
 655
 656 STAT_FN(rmt_hitm)
 657 STAT_FN(lcl_hitm)
 658 STAT_FN(rmt_peer)
 659 STAT_FN(lcl_peer)
 660 STAT_FN(tot_peer)
 661 STAT_FN(store)
 662 STAT_FN(st_l1hit)
 663 STAT_FN(st_l1miss)
 664 STAT_FN(st_na)
 665 STAT_FN(ld_fbhit)
 666 STAT_FN(ld_l1hit)
 667 STAT_FN(ld_l2hit)
 668 STAT_FN(ld_llchit)
 669 STAT_FN(rmt_hit)
 670
 671 static uint64_t get_load_llc_misses(struct c2c_stats *stats)
 672 {
 673         return stats->lcl_dram +
 674                stats->rmt_dram +
 675                stats->rmt_hitm +
 676                stats->rmt_hit;
 677 }
 678
 679 static uint64_t get_load_cache_hits(struct c2c_stats *stats)
 680 {
 681         return stats->ld_fbhit +
 682                stats->ld_l1hit +
 683                stats->ld_l2hit +
 684                stats->ld_llchit +
 685                stats->lcl_hitm;
 686 }
 687
 688 static uint64_t get_stores(struct c2c_stats *stats)
 689 {
 690         return stats->st_l1hit +
 691                stats->st_l1miss +
 692                stats->st_na;
 693 }
 694
 695 static uint64_t total_records(struct c2c_stats *stats)
 696 {
 697         return get_load_llc_misses(stats) +
 698                get_load_cache_hits(stats) +
 699                get_stores(stats);
 700 }
 701
 702 static int
 703 tot_recs_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 704                 struct hist_entry *he)
 705 {
 706         struct c2c_hist_entry *c2c_he;
 707         int width = c2c_width(fmt, hpp, he->hists);
 708         uint64_t tot_recs;
 709
 710         c2c_he = container_of(he, struct c2c_hist_entry, he);
 711         tot_recs = total_records(&c2c_he->stats);
 712
 713         return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, tot_recs);
 714 }
 715
 716 static int64_t
 717 tot_recs_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 718              struct hist_entry *left, struct hist_entry *right)
 719 {
 720         struct c2c_hist_entry *c2c_left;
 721         struct c2c_hist_entry *c2c_right;
 722         uint64_t tot_recs_left;
 723         uint64_t tot_recs_right;
 724
 725         c2c_left  = container_of(left, struct c2c_hist_entry, he);
 726         c2c_right = container_of(right, struct c2c_hist_entry, he);
 727
 728         tot_recs_left  = total_records(&c2c_left->stats);
 729         tot_recs_right = total_records(&c2c_right->stats);
 730
 731         return tot_recs_left - tot_recs_right;
 732 }
 733
 734 static uint64_t total_loads(struct c2c_stats *stats)
 735 {
 736         return get_load_llc_misses(stats) +
 737                get_load_cache_hits(stats);
 738 }
 739
 740 static int
 741 tot_loads_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 742                 struct hist_entry *he)
 743 {
 744         struct c2c_hist_entry *c2c_he;
 745         int width = c2c_width(fmt, hpp, he->hists);
 746         uint64_t tot_recs;
 747
 748         c2c_he = container_of(he, struct c2c_hist_entry, he);
 749         tot_recs = total_loads(&c2c_he->stats);
 750
 751         return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, tot_recs);
 752 }
 753
 754 static int64_t
 755 tot_loads_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 756               struct hist_entry *left, struct hist_entry *right)
 757 {
 758         struct c2c_hist_entry *c2c_left;
 759         struct c2c_hist_entry *c2c_right;
 760         uint64_t tot_recs_left;
 761         uint64_t tot_recs_right;
 762
 763         c2c_left  = container_of(left, struct c2c_hist_entry, he);
 764         c2c_right = container_of(right, struct c2c_hist_entry, he);
 765
 766         tot_recs_left  = total_loads(&c2c_left->stats);
 767         tot_recs_right = total_loads(&c2c_right->stats);
 768
 769         return tot_recs_left - tot_recs_right;
 770 }
 771
 772 typedef double (get_percent_cb)(struct c2c_hist_entry *);
 773
 774 static int
 775 percent_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 776               struct hist_entry *he, get_percent_cb get_percent)
 777 {
 778         struct c2c_hist_entry *c2c_he;
 779         int width = c2c_width(fmt, hpp, he->hists);
 780         double per;
 781
 782         c2c_he = container_of(he, struct c2c_hist_entry, he);
 783         per = get_percent(c2c_he);
 784
 785 #ifdef HAVE_SLANG_SUPPORT
 786         if (use_browser)
 787                 return __hpp__slsmg_color_printf(hpp, "%*.2f%%", width - 1, per);
 788 #endif
 789         return hpp_color_scnprintf(hpp, "%*.2f%%", width - 1, per);
 790 }
 791
 792 static double percent_costly_snoop(struct c2c_hist_entry *c2c_he)
 793 {
 794         struct c2c_hists *hists;
 795         struct c2c_stats *stats;
 796         struct c2c_stats *total;
 797         int tot = 0, st = 0;
 798         double p;
 799
 800         hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);
 801         stats = &c2c_he->stats;
 802         total = &hists->stats;
 803
 804         switch (c2c.display) {
 805         case DISPLAY_RMT_HITM:
 806                 st  = stats->rmt_hitm;
 807                 tot = total->rmt_hitm;
 808                 break;
 809         case DISPLAY_LCL_HITM:
 810                 st  = stats->lcl_hitm;
 811                 tot = total->lcl_hitm;
 812                 break;
 813         case DISPLAY_TOT_HITM:
 814                 st  = stats->tot_hitm;
 815                 tot = total->tot_hitm;
 816                 break;
 817         case DISPLAY_SNP_PEER:
 818                 st  = stats->tot_peer;
 819                 tot = total->tot_peer;
 820                 break;
 821         default:
 822                 break;
 823         }
 824
 825         p = tot ? (double) st / tot : 0;
 826
 827         return 100 * p;
 828 }
 829
 830 #define PERC_STR(__s, __v)                              \
 831 ({                                                      \
 832         scnprintf(__s, sizeof(__s), "%.2F%%", __v);     \
 833         __s;                                            \
 834 })
 835
 836 static int
 837 percent_costly_snoop_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 838                            struct hist_entry *he)
 839 {
 840         struct c2c_hist_entry *c2c_he;
 841         int width = c2c_width(fmt, hpp, he->hists);
 842         char buf[10];
 843         double per;
 844
 845         c2c_he = container_of(he, struct c2c_hist_entry, he);
 846         per = percent_costly_snoop(c2c_he);
 847         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
 848 }
 849
 850 static int
 851 percent_costly_snoop_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 852                            struct hist_entry *he)
 853 {
 854         return percent_color(fmt, hpp, he, percent_costly_snoop);
 855 }
 856
 857 static int64_t
 858 percent_costly_snoop_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 859                          struct hist_entry *left, struct hist_entry *right)
 860 {
 861         struct c2c_hist_entry *c2c_left;
 862         struct c2c_hist_entry *c2c_right;
 863         double per_left;
 864         double per_right;
 865
 866         c2c_left  = container_of(left, struct c2c_hist_entry, he);
 867         c2c_right = container_of(right, struct c2c_hist_entry, he);
 868
 869         per_left  = percent_costly_snoop(c2c_left);
 870         per_right = percent_costly_snoop(c2c_right);
 871
 872         return per_left - per_right;
 873 }
 874
 875 static struct c2c_stats *he_stats(struct hist_entry *he)
 876 {
 877         struct c2c_hist_entry *c2c_he;
 878
 879         c2c_he = container_of(he, struct c2c_hist_entry, he);
 880         return &c2c_he->stats;
 881 }
 882
 883 static struct c2c_stats *total_stats(struct hist_entry *he)
 884 {
 885         struct c2c_hists *hists;
 886
 887         hists = container_of(he->hists, struct c2c_hists, hists);
 888         return &hists->stats;
 889 }
 890
 891 static double percent(u32 st, u32 tot)
 892 {
 893         return tot ? 100. * (double) st / (double) tot : 0;
 894 }
 895
 896 #define PERCENT(__h, __f) percent(he_stats(__h)->__f, total_stats(__h)->__f)
 897
 898 #define PERCENT_FN(__f)                                                         \
 899 static double percent_ ## __f(struct c2c_hist_entry *c2c_he)                    \
 900 {                                                                               \
 901         struct c2c_hists *hists;                                                \
 902                                                                                 \
 903         hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);        \
 904         return percent(c2c_he->stats.__f, hists->stats.__f);                    \
 905 }
 906
 907 PERCENT_FN(rmt_hitm)
 908 PERCENT_FN(lcl_hitm)
 909 PERCENT_FN(rmt_peer)
 910 PERCENT_FN(lcl_peer)
 911 PERCENT_FN(st_l1hit)
 912 PERCENT_FN(st_l1miss)
 913 PERCENT_FN(st_na)
 914
 915 static int
 916 percent_rmt_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 917                        struct hist_entry *he)
 918 {
 919         int width = c2c_width(fmt, hpp, he->hists);
 920         double per = PERCENT(he, rmt_hitm);
 921         char buf[10];
 922
 923         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
 924 }
 925
 926 static int
 927 percent_rmt_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 928                        struct hist_entry *he)
 929 {
 930         return percent_color(fmt, hpp, he, percent_rmt_hitm);
 931 }
 932
 933 static int64_t
 934 percent_rmt_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 935                      struct hist_entry *left, struct hist_entry *right)
 936 {
 937         double per_left;
 938         double per_right;
 939
 940         per_left  = PERCENT(left, rmt_hitm);
 941         per_right = PERCENT(right, rmt_hitm);
 942
 943         return per_left - per_right;
 944 }
 945
 946 static int
 947 percent_lcl_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 948                        struct hist_entry *he)
 949 {
 950         int width = c2c_width(fmt, hpp, he->hists);
 951         double per = PERCENT(he, lcl_hitm);
 952         char buf[10];
 953
 954         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
 955 }
 956
 957 static int
 958 percent_lcl_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 959                        struct hist_entry *he)
 960 {
 961         return percent_color(fmt, hpp, he, percent_lcl_hitm);
 962 }
 963
 964 static int64_t
 965 percent_lcl_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 966                      struct hist_entry *left, struct hist_entry *right)
 967 {
 968         double per_left;
 969         double per_right;
 970
 971         per_left  = PERCENT(left, lcl_hitm);
 972         per_right = PERCENT(right, lcl_hitm);
 973
 974         return per_left - per_right;
 975 }
 976
 977 static int
 978 percent_lcl_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 979                        struct hist_entry *he)
 980 {
 981         int width = c2c_width(fmt, hpp, he->hists);
 982         double per = PERCENT(he, lcl_peer);
 983         char buf[10];
 984
 985         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
 986 }
 987
 988 static int
 989 percent_lcl_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
 990                        struct hist_entry *he)
 991 {
 992         return percent_color(fmt, hpp, he, percent_lcl_peer);
 993 }
 994
 995 static int64_t
 996 percent_lcl_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
 997                      struct hist_entry *left, struct hist_entry *right)
 998 {
 999         double per_left;
1000         double per_right;
1001
1002         per_left  = PERCENT(left, lcl_peer);
1003         per_right = PERCENT(right, lcl_peer);
1004
1005         return per_left - per_right;
1006 }
1007
1008 static int
1009 percent_rmt_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1010                        struct hist_entry *he)
1011 {
1012         int width = c2c_width(fmt, hpp, he->hists);
1013         double per = PERCENT(he, rmt_peer);
1014         char buf[10];
1015
1016         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1017 }
1018
1019 static int
1020 percent_rmt_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1021                        struct hist_entry *he)
1022 {
1023         return percent_color(fmt, hpp, he, percent_rmt_peer);
1024 }
1025
1026 static int64_t
1027 percent_rmt_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1028                      struct hist_entry *left, struct hist_entry *right)
1029 {
1030         double per_left;
1031         double per_right;
1032
1033         per_left  = PERCENT(left, rmt_peer);
1034         per_right = PERCENT(right, rmt_peer);
1035
1036         return per_left - per_right;
1037 }
1038
1039 static int
1040 percent_stores_l1hit_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1041                            struct hist_entry *he)
1042 {
1043         int width = c2c_width(fmt, hpp, he->hists);
1044         double per = PERCENT(he, st_l1hit);
1045         char buf[10];
1046
1047         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1048 }
1049
1050 static int
1051 percent_stores_l1hit_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1052                            struct hist_entry *he)
1053 {
1054         return percent_color(fmt, hpp, he, percent_st_l1hit);
1055 }
1056
1057 static int64_t
1058 percent_stores_l1hit_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1059                         struct hist_entry *left, struct hist_entry *right)
1060 {
1061         double per_left;
1062         double per_right;
1063
1064         per_left  = PERCENT(left, st_l1hit);
1065         per_right = PERCENT(right, st_l1hit);
1066
1067         return per_left - per_right;
1068 }
1069
1070 static int
1071 percent_stores_l1miss_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1072                            struct hist_entry *he)
1073 {
1074         int width = c2c_width(fmt, hpp, he->hists);
1075         double per = PERCENT(he, st_l1miss);
1076         char buf[10];
1077
1078         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1079 }
1080
1081 static int
1082 percent_stores_l1miss_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1083                             struct hist_entry *he)
1084 {
1085         return percent_color(fmt, hpp, he, percent_st_l1miss);
1086 }
1087
1088 static int64_t
1089 percent_stores_l1miss_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1090                           struct hist_entry *left, struct hist_entry *right)
1091 {
1092         double per_left;
1093         double per_right;
1094
1095         per_left  = PERCENT(left, st_l1miss);
1096         per_right = PERCENT(right, st_l1miss);
1097
1098         return per_left - per_right;
1099 }
1100
1101 static int
1102 percent_stores_na_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1103                         struct hist_entry *he)
1104 {
1105         int width = c2c_width(fmt, hpp, he->hists);
1106         double per = PERCENT(he, st_na);
1107         char buf[10];
1108
1109         return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1110 }
1111
1112 static int
1113 percent_stores_na_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1114                         struct hist_entry *he)
1115 {
1116         return percent_color(fmt, hpp, he, percent_st_na);
1117 }
1118
1119 static int64_t
1120 percent_stores_na_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1121                       struct hist_entry *left, struct hist_entry *right)
1122 {
1123         double per_left;
1124         double per_right;
1125
1126         per_left  = PERCENT(left, st_na);
1127         per_right = PERCENT(right, st_na);
1128
1129         return per_left - per_right;
1130 }
1131
1132 STAT_FN(lcl_dram)
1133 STAT_FN(rmt_dram)
1134
1135 static int
1136 pid_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1137           struct hist_entry *he)
1138 {
1139         int width = c2c_width(fmt, hpp, he->hists);
1140
1141         return scnprintf(hpp->buf, hpp->size, "%*d", width, thread__pid(he->thread));
1142 }
1143
1144 static int64_t
1145 pid_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1146         struct hist_entry *left, struct hist_entry *right)
1147 {
1148         return thread__pid(left->thread) - thread__pid(right->thread);
1149 }
1150
1151 static int64_t
1152 empty_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1153           struct hist_entry *left __maybe_unused,
1154           struct hist_entry *right __maybe_unused)
1155 {
1156         return 0;
1157 }
1158
1159 static int display_metrics(struct perf_hpp *hpp, u32 val, u32 sum)
1160 {
1161         int ret;
1162
1163         if (sum != 0)
1164                 ret = scnprintf(hpp->buf, hpp->size, "%5.1f%% ",
1165                                 percent(val, sum));
1166         else
1167                 ret = scnprintf(hpp->buf, hpp->size, "%6s ", "n/a");
1168
1169         return ret;
1170 }
1171
1172 static int
1173 node_entry(struct perf_hpp_fmt *fmt __maybe_unused, struct perf_hpp *hpp,
1174            struct hist_entry *he)
1175 {
1176         struct c2c_hist_entry *c2c_he;
1177         bool first = true;
1178         int node;
1179         int ret = 0;
1180
1181         c2c_he = container_of(he, struct c2c_hist_entry, he);
1182
1183         for (node = 0; node < c2c.nodes_cnt; node++) {
1184                 DECLARE_BITMAP(set, c2c.cpus_cnt);
1185
1186                 bitmap_zero(set, c2c.cpus_cnt);
1187                 bitmap_and(set, c2c_he->cpuset, c2c.nodes[node], c2c.cpus_cnt);
1188
1189                 if (bitmap_empty(set, c2c.cpus_cnt)) {
1190                         if (c2c.node_info == 1) {
1191                                 ret = scnprintf(hpp->buf, hpp->size, "%21s", " ");
1192                                 advance_hpp(hpp, ret);
1193                         }
1194                         continue;
1195                 }
1196
1197                 if (!first) {
1198                         ret = scnprintf(hpp->buf, hpp->size, " ");
1199                         advance_hpp(hpp, ret);
1200                 }
1201
1202                 switch (c2c.node_info) {
1203                 case 0:
1204                         ret = scnprintf(hpp->buf, hpp->size, "%2d", node);
1205                         advance_hpp(hpp, ret);
1206                         break;
1207                 case 1:
1208                 {
1209                         int num = bitmap_weight(set, c2c.cpus_cnt);
1210                         struct c2c_stats *stats = &c2c_he->node_stats[node];
1211
1212                         ret = scnprintf(hpp->buf, hpp->size, "%2d{%2d ", node, num);
1213                         advance_hpp(hpp, ret);
1214
1215                         switch (c2c.display) {
1216                         case DISPLAY_RMT_HITM:
1217                                 ret = display_metrics(hpp, stats->rmt_hitm,
1218                                                       c2c_he->stats.rmt_hitm);
1219                                 break;
1220                         case DISPLAY_LCL_HITM:
1221                                 ret = display_metrics(hpp, stats->lcl_hitm,
1222                                                       c2c_he->stats.lcl_hitm);
1223                                 break;
1224                         case DISPLAY_TOT_HITM:
1225                                 ret = display_metrics(hpp, stats->tot_hitm,
1226                                                       c2c_he->stats.tot_hitm);
1227                                 break;
1228                         case DISPLAY_SNP_PEER:
1229                                 ret = display_metrics(hpp, stats->tot_peer,
1230                                                       c2c_he->stats.tot_peer);
1231                                 break;
1232                         default:
1233                                 break;
1234                         }
1235
1236                         advance_hpp(hpp, ret);
1237
1238                         if (c2c_he->stats.store > 0) {
1239                                 ret = scnprintf(hpp->buf, hpp->size, "%5.1f%%}",
1240                                                 percent(stats->store, c2c_he->stats.store));
1241                         } else {
1242                                 ret = scnprintf(hpp->buf, hpp->size, "%6s}", "n/a");
1243                         }
1244
1245                         advance_hpp(hpp, ret);
1246                         break;
1247                 }
1248                 case 2:
1249                         ret = scnprintf(hpp->buf, hpp->size, "%2d{", node);
1250                         advance_hpp(hpp, ret);
1251
1252                         ret = bitmap_scnprintf(set, c2c.cpus_cnt, hpp->buf, hpp->size);
1253                         advance_hpp(hpp, ret);
1254
1255                         ret = scnprintf(hpp->buf, hpp->size, "}");
1256                         advance_hpp(hpp, ret);
1257                         break;
1258                 default:
1259                         break;
1260                 }
1261
1262                 first = false;
1263         }
1264
1265         return 0;
1266 }
1267
1268 static int
1269 mean_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1270            struct hist_entry *he, double mean)
1271 {
1272         int width = c2c_width(fmt, hpp, he->hists);
1273         char buf[10];
1274
1275         scnprintf(buf, 10, "%6.0f", mean);
1276         return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1277 }
1278
1279 #define MEAN_ENTRY(__func, __val)                                               \
1280 static int                                                                      \
1281 __func(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he)   \
1282 {                                                                               \
1283         struct c2c_hist_entry *c2c_he;                                          \
1284         c2c_he = container_of(he, struct c2c_hist_entry, he);                   \
1285         return mean_entry(fmt, hpp, he, avg_stats(&c2c_he->cstats.__val));      \
1286 }
1287
1288 MEAN_ENTRY(mean_rmt_entry,  rmt_hitm);
1289 MEAN_ENTRY(mean_lcl_entry,  lcl_hitm);
1290 MEAN_ENTRY(mean_load_entry, load);
1291 MEAN_ENTRY(mean_rmt_peer_entry, rmt_peer);
1292 MEAN_ENTRY(mean_lcl_peer_entry, lcl_peer);
1293
1294 static int
1295 cpucnt_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1296              struct hist_entry *he)
1297 {
1298         struct c2c_hist_entry *c2c_he;
1299         int width = c2c_width(fmt, hpp, he->hists);
1300         char buf[10];
1301
1302         c2c_he = container_of(he, struct c2c_hist_entry, he);
1303
1304         scnprintf(buf, 10, "%d", bitmap_weight(c2c_he->cpuset, c2c.cpus_cnt));
1305         return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1306 }
1307
1308 static int
1309 cl_idx_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1310              struct hist_entry *he)
1311 {
1312         struct c2c_hist_entry *c2c_he;
1313         int width = c2c_width(fmt, hpp, he->hists);
1314         char buf[10];
1315
1316         c2c_he = container_of(he, struct c2c_hist_entry, he);
1317
1318         scnprintf(buf, 10, "%u", c2c_he->cacheline_idx);
1319         return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1320 }
1321
1322 static int
1323 cl_idx_empty_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1324                    struct hist_entry *he)
1325 {
1326         int width = c2c_width(fmt, hpp, he->hists);
1327
1328         return scnprintf(hpp->buf, hpp->size, "%*s", width, "");
1329 }
1330
1331 #define HEADER_LOW(__h)                 \
1332         {                               \
1333                 .line[1] = {            \
1334                         .text = __h,    \
1335                 },                      \
1336         }
1337
1338 #define HEADER_BOTH(__h0, __h1)         \
1339         {                               \
1340                 .line[0] = {            \
1341                         .text = __h0,   \
1342                 },                      \
1343                 .line[1] = {            \
1344                         .text = __h1,   \
1345                 },                      \
1346         }
1347
1348 #define HEADER_SPAN(__h0, __h1, __s)    \
1349         {                               \
1350                 .line[0] = {            \
1351                         .text = __h0,   \
1352                         .span = __s,    \
1353                 },                      \
1354                 .line[1] = {            \
1355                         .text = __h1,   \
1356                 },                      \
1357         }
1358
1359 #define HEADER_SPAN_LOW(__h)            \
1360         {                               \
1361                 .line[1] = {            \
1362                         .text = __h,    \
1363                 },                      \
1364         }
1365
1366 static struct c2c_dimension dim_dcacheline = {
1367         .header         = HEADER_SPAN("--- Cacheline ----", "Address", 2),
1368         .name           = "dcacheline",
1369         .cmp            = dcacheline_cmp,
1370         .entry          = dcacheline_entry,
1371         .width          = 18,
1372 };
1373
1374 static struct c2c_dimension dim_dcacheline_node = {
1375         .header         = HEADER_LOW("Node"),
1376         .name           = "dcacheline_node",
1377         .cmp            = empty_cmp,
1378         .entry          = dcacheline_node_entry,
1379         .width          = 4,
1380 };
1381
1382 static struct c2c_dimension dim_dcacheline_count = {
1383         .header         = HEADER_LOW("PA cnt"),
1384         .name           = "dcacheline_count",
1385         .cmp            = empty_cmp,
1386         .entry          = dcacheline_node_count,
1387         .width          = 6,
1388 };
1389
1390 static struct c2c_header header_offset_tui = HEADER_SPAN("-----", "Off", 2);
1391
1392 static struct c2c_dimension dim_offset = {
1393         .header         = HEADER_SPAN("--- Data address -", "Offset", 2),
1394         .name           = "offset",
1395         .cmp            = offset_cmp,
1396         .entry          = offset_entry,
1397         .width          = 18,
1398 };
1399
1400 static struct c2c_dimension dim_offset_node = {
1401         .header         = HEADER_LOW("Node"),
1402         .name           = "offset_node",
1403         .cmp            = empty_cmp,
1404         .entry          = dcacheline_node_entry,
1405         .width          = 4,
1406 };
1407
1408 static struct c2c_dimension dim_iaddr = {
1409         .header         = HEADER_LOW("Code address"),
1410         .name           = "iaddr",
1411         .cmp            = iaddr_cmp,
1412         .entry          = iaddr_entry,
1413         .width          = 18,
1414 };
1415
1416 static struct c2c_dimension dim_tot_hitm = {
1417         .header         = HEADER_SPAN("------- Load Hitm -------", "Total", 2),
1418         .name           = "tot_hitm",
1419         .cmp            = tot_hitm_cmp,
1420         .entry          = tot_hitm_entry,
1421         .width          = 7,
1422 };
1423
1424 static struct c2c_dimension dim_lcl_hitm = {
1425         .header         = HEADER_SPAN_LOW("LclHitm"),
1426         .name           = "lcl_hitm",
1427         .cmp            = lcl_hitm_cmp,
1428         .entry          = lcl_hitm_entry,
1429         .width          = 7,
1430 };
1431
1432 static struct c2c_dimension dim_rmt_hitm = {
1433         .header         = HEADER_SPAN_LOW("RmtHitm"),
1434         .name           = "rmt_hitm",
1435         .cmp            = rmt_hitm_cmp,
1436         .entry          = rmt_hitm_entry,
1437         .width          = 7,
1438 };
1439
1440 static struct c2c_dimension dim_tot_peer = {
1441         .header         = HEADER_SPAN("------- Load Peer -------", "Total", 2),
1442         .name           = "tot_peer",
1443         .cmp            = tot_peer_cmp,
1444         .entry          = tot_peer_entry,
1445         .width          = 7,
1446 };
1447
1448 static struct c2c_dimension dim_lcl_peer = {
1449         .header         = HEADER_SPAN_LOW("Local"),
1450         .name           = "lcl_peer",
1451         .cmp            = lcl_peer_cmp,
1452         .entry          = lcl_peer_entry,
1453         .width          = 7,
1454 };
1455
1456 static struct c2c_dimension dim_rmt_peer = {
1457         .header         = HEADER_SPAN_LOW("Remote"),
1458         .name           = "rmt_peer",
1459         .cmp            = rmt_peer_cmp,
1460         .entry          = rmt_peer_entry,
1461         .width          = 7,
1462 };
1463
1464 static struct c2c_dimension dim_cl_rmt_hitm = {
1465         .header         = HEADER_SPAN("----- HITM -----", "Rmt", 1),
1466         .name           = "cl_rmt_hitm",
1467         .cmp            = rmt_hitm_cmp,
1468         .entry          = rmt_hitm_entry,
1469         .width          = 7,
1470 };
1471
1472 static struct c2c_dimension dim_cl_lcl_hitm = {
1473         .header         = HEADER_SPAN_LOW("Lcl"),
1474         .name           = "cl_lcl_hitm",
1475         .cmp            = lcl_hitm_cmp,
1476         .entry          = lcl_hitm_entry,
1477         .width          = 7,
1478 };
1479
1480 static struct c2c_dimension dim_cl_rmt_peer = {
1481         .header         = HEADER_SPAN("----- Peer -----", "Rmt", 1),
1482         .name           = "cl_rmt_peer",
1483         .cmp            = rmt_peer_cmp,
1484         .entry          = rmt_peer_entry,
1485         .width          = 7,
1486 };
1487
1488 static struct c2c_dimension dim_cl_lcl_peer = {
1489         .header         = HEADER_SPAN_LOW("Lcl"),
1490         .name           = "cl_lcl_peer",
1491         .cmp            = lcl_peer_cmp,
1492         .entry          = lcl_peer_entry,
1493         .width          = 7,
1494 };
1495
1496 static struct c2c_dimension dim_tot_stores = {
1497         .header         = HEADER_BOTH("Total", "Stores"),
1498         .name           = "tot_stores",
1499         .cmp            = store_cmp,
1500         .entry          = store_entry,
1501         .width          = 7,
1502 };
1503
1504 static struct c2c_dimension dim_stores_l1hit = {
1505         .header         = HEADER_SPAN("--------- Stores --------", "L1Hit", 2),
1506         .name           = "stores_l1hit",
1507         .cmp            = st_l1hit_cmp,
1508         .entry          = st_l1hit_entry,
1509         .width          = 7,
1510 };
1511
1512 static struct c2c_dimension dim_stores_l1miss = {
1513         .header         = HEADER_SPAN_LOW("L1Miss"),
1514         .name           = "stores_l1miss",
1515         .cmp            = st_l1miss_cmp,
1516         .entry          = st_l1miss_entry,
1517         .width          = 7,
1518 };
1519
1520 static struct c2c_dimension dim_stores_na = {
1521         .header         = HEADER_SPAN_LOW("N/A"),
1522         .name           = "stores_na",
1523         .cmp            = st_na_cmp,
1524         .entry          = st_na_entry,
1525         .width          = 7,
1526 };
1527
1528 static struct c2c_dimension dim_cl_stores_l1hit = {
1529         .header         = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2),
1530         .name           = "cl_stores_l1hit",
1531         .cmp            = st_l1hit_cmp,
1532         .entry          = st_l1hit_entry,
1533         .width          = 7,
1534 };
1535
1536 static struct c2c_dimension dim_cl_stores_l1miss = {
1537         .header         = HEADER_SPAN_LOW("L1 Miss"),
1538         .name           = "cl_stores_l1miss",
1539         .cmp            = st_l1miss_cmp,
1540         .entry          = st_l1miss_entry,
1541         .width          = 7,
1542 };
1543
1544 static struct c2c_dimension dim_cl_stores_na = {
1545         .header         = HEADER_SPAN_LOW("N/A"),
1546         .name           = "cl_stores_na",
1547         .cmp            = st_na_cmp,
1548         .entry          = st_na_entry,
1549         .width          = 7,
1550 };
1551
1552 static struct c2c_dimension dim_ld_fbhit = {
1553         .header         = HEADER_SPAN("----- Core Load Hit -----", "FB", 2),
1554         .name           = "ld_fbhit",
1555         .cmp            = ld_fbhit_cmp,
1556         .entry          = ld_fbhit_entry,
1557         .width          = 7,
1558 };
1559
1560 static struct c2c_dimension dim_ld_l1hit = {
1561         .header         = HEADER_SPAN_LOW("L1"),
1562         .name           = "ld_l1hit",
1563         .cmp            = ld_l1hit_cmp,
1564         .entry          = ld_l1hit_entry,
1565         .width          = 7,
1566 };
1567
1568 static struct c2c_dimension dim_ld_l2hit = {
1569         .header         = HEADER_SPAN_LOW("L2"),
1570         .name           = "ld_l2hit",
1571         .cmp            = ld_l2hit_cmp,
1572         .entry          = ld_l2hit_entry,
1573         .width          = 7,
1574 };
1575
1576 static struct c2c_dimension dim_ld_llchit = {
1577         .header         = HEADER_SPAN("- LLC Load Hit --", "LclHit", 1),
1578         .name           = "ld_lclhit",
1579         .cmp            = ld_llchit_cmp,
1580         .entry          = ld_llchit_entry,
1581         .width          = 8,
1582 };
1583
1584 static struct c2c_dimension dim_ld_rmthit = {
1585         .header         = HEADER_SPAN("- RMT Load Hit --", "RmtHit", 1),
1586         .name           = "ld_rmthit",
1587         .cmp            = rmt_hit_cmp,
1588         .entry          = rmt_hit_entry,
1589         .width          = 8,
1590 };
1591
1592 static struct c2c_dimension dim_tot_recs = {
1593         .header         = HEADER_BOTH("Total", "records"),
1594         .name           = "tot_recs",
1595         .cmp            = tot_recs_cmp,
1596         .entry          = tot_recs_entry,
1597         .width          = 7,
1598 };
1599
1600 static struct c2c_dimension dim_tot_loads = {
1601         .header         = HEADER_BOTH("Total", "Loads"),
1602         .name           = "tot_loads",
1603         .cmp            = tot_loads_cmp,
1604         .entry          = tot_loads_entry,
1605         .width          = 7,
1606 };
1607
1608 static struct c2c_header percent_costly_snoop_header[] = {
1609         [DISPLAY_LCL_HITM] = HEADER_BOTH("Lcl", "Hitm"),
1610         [DISPLAY_RMT_HITM] = HEADER_BOTH("Rmt", "Hitm"),
1611         [DISPLAY_TOT_HITM] = HEADER_BOTH("Tot", "Hitm"),
1612         [DISPLAY_SNP_PEER] = HEADER_BOTH("Peer", "Snoop"),
1613 };
1614
1615 static struct c2c_dimension dim_percent_costly_snoop = {
1616         .name           = "percent_costly_snoop",
1617         .cmp            = percent_costly_snoop_cmp,
1618         .entry          = percent_costly_snoop_entry,
1619         .color          = percent_costly_snoop_color,
1620         .width          = 7,
1621 };
1622
1623 static struct c2c_dimension dim_percent_rmt_hitm = {
1624         .header         = HEADER_SPAN("----- HITM -----", "RmtHitm", 1),
1625         .name           = "percent_rmt_hitm",
1626         .cmp            = percent_rmt_hitm_cmp,
1627         .entry          = percent_rmt_hitm_entry,
1628         .color          = percent_rmt_hitm_color,
1629         .width          = 7,
1630 };
1631
1632 static struct c2c_dimension dim_percent_lcl_hitm = {
1633         .header         = HEADER_SPAN_LOW("LclHitm"),
1634         .name           = "percent_lcl_hitm",
1635         .cmp            = percent_lcl_hitm_cmp,
1636         .entry          = percent_lcl_hitm_entry,
1637         .color          = percent_lcl_hitm_color,
1638         .width          = 7,
1639 };
1640
1641 static struct c2c_dimension dim_percent_rmt_peer = {
1642         .header         = HEADER_SPAN("-- Peer Snoop --", "Rmt", 1),
1643         .name           = "percent_rmt_peer",
1644         .cmp            = percent_rmt_peer_cmp,
1645         .entry          = percent_rmt_peer_entry,
1646         .color          = percent_rmt_peer_color,
1647         .width          = 7,
1648 };
1649
1650 static struct c2c_dimension dim_percent_lcl_peer = {
1651         .header         = HEADER_SPAN_LOW("Lcl"),
1652         .name           = "percent_lcl_peer",
1653         .cmp            = percent_lcl_peer_cmp,
1654         .entry          = percent_lcl_peer_entry,
1655         .color          = percent_lcl_peer_color,
1656         .width          = 7,
1657 };
1658
1659 static struct c2c_dimension dim_percent_stores_l1hit = {
1660         .header         = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2),
1661         .name           = "percent_stores_l1hit",
1662         .cmp            = percent_stores_l1hit_cmp,
1663         .entry          = percent_stores_l1hit_entry,
1664         .color          = percent_stores_l1hit_color,
1665         .width          = 7,
1666 };
1667
1668 static struct c2c_dimension dim_percent_stores_l1miss = {
1669         .header         = HEADER_SPAN_LOW("L1 Miss"),
1670         .name           = "percent_stores_l1miss",
1671         .cmp            = percent_stores_l1miss_cmp,
1672         .entry          = percent_stores_l1miss_entry,
1673         .color          = percent_stores_l1miss_color,
1674         .width          = 7,
1675 };
1676
1677 static struct c2c_dimension dim_percent_stores_na = {
1678         .header         = HEADER_SPAN_LOW("N/A"),
1679         .name           = "percent_stores_na",
1680         .cmp            = percent_stores_na_cmp,
1681         .entry          = percent_stores_na_entry,
1682         .color          = percent_stores_na_color,
1683         .width          = 7,
1684 };
1685
1686 static struct c2c_dimension dim_dram_lcl = {
1687         .header         = HEADER_SPAN("--- Load Dram ----", "Lcl", 1),
1688         .name           = "dram_lcl",
1689         .cmp            = lcl_dram_cmp,
1690         .entry          = lcl_dram_entry,
1691         .width          = 8,
1692 };
1693
1694 static struct c2c_dimension dim_dram_rmt = {
1695         .header         = HEADER_SPAN_LOW("Rmt"),
1696         .name           = "dram_rmt",
1697         .cmp            = rmt_dram_cmp,
1698         .entry          = rmt_dram_entry,
1699         .width          = 8,
1700 };
1701
1702 static struct c2c_dimension dim_pid = {
1703         .header         = HEADER_LOW("Pid"),
1704         .name           = "pid",
1705         .cmp            = pid_cmp,
1706         .entry          = pid_entry,
1707         .width          = 7,
1708 };
1709
1710 static struct c2c_dimension dim_tid = {
1711         .header         = HEADER_LOW("Tid"),
1712         .name           = "tid",
1713         .se             = &sort_thread,
1714 };
1715
1716 static struct c2c_dimension dim_symbol = {
1717         .name           = "symbol",
1718         .se             = &sort_sym,
1719 };
1720
1721 static struct c2c_dimension dim_dso = {
1722         .header         = HEADER_BOTH("Shared", "Object"),
1723         .name           = "dso",
1724         .se             = &sort_dso,
1725 };
1726
1727 static struct c2c_dimension dim_node = {
1728         .name           = "node",
1729         .cmp            = empty_cmp,
1730         .entry          = node_entry,
1731         .width          = 4,
1732 };
1733
1734 static struct c2c_dimension dim_mean_rmt = {
1735         .header         = HEADER_SPAN("---------- cycles ----------", "rmt hitm", 2),
1736         .name           = "mean_rmt",
1737         .cmp            = empty_cmp,
1738         .entry          = mean_rmt_entry,
1739         .width          = 8,
1740 };
1741
1742 static struct c2c_dimension dim_mean_lcl = {
1743         .header         = HEADER_SPAN_LOW("lcl hitm"),
1744         .name           = "mean_lcl",
1745         .cmp            = empty_cmp,
1746         .entry          = mean_lcl_entry,
1747         .width          = 8,
1748 };
1749
1750 static struct c2c_dimension dim_mean_load = {
1751         .header         = HEADER_SPAN_LOW("load"),
1752         .name           = "mean_load",
1753         .cmp            = empty_cmp,
1754         .entry          = mean_load_entry,
1755         .width          = 8,
1756 };
1757
1758 static struct c2c_dimension dim_mean_rmt_peer = {
1759         .header         = HEADER_SPAN("---------- cycles ----------", "rmt peer", 2),
1760         .name           = "mean_rmt_peer",
1761         .cmp            = empty_cmp,
1762         .entry          = mean_rmt_peer_entry,
1763         .width          = 8,
1764 };
1765
1766 static struct c2c_dimension dim_mean_lcl_peer = {
1767         .header         = HEADER_SPAN_LOW("lcl peer"),
1768         .name           = "mean_lcl_peer",
1769         .cmp            = empty_cmp,
1770         .entry          = mean_lcl_peer_entry,
1771         .width          = 8,
1772 };
1773
1774 static struct c2c_dimension dim_cpucnt = {
1775         .header         = HEADER_BOTH("cpu", "cnt"),
1776         .name           = "cpucnt",
1777         .cmp            = empty_cmp,
1778         .entry          = cpucnt_entry,
1779         .width          = 8,
1780 };
1781
1782 static struct c2c_dimension dim_srcline = {
1783         .name           = "cl_srcline",
1784         .se             = &sort_srcline,
1785 };
1786
1787 static struct c2c_dimension dim_dcacheline_idx = {
1788         .header         = HEADER_LOW("Index"),
1789         .name           = "cl_idx",
1790         .cmp            = empty_cmp,
1791         .entry          = cl_idx_entry,
1792         .width          = 5,
1793 };
1794
1795 static struct c2c_dimension dim_dcacheline_num = {
1796         .header         = HEADER_LOW("Num"),
1797         .name           = "cl_num",
1798         .cmp            = empty_cmp,
1799         .entry          = cl_idx_entry,
1800         .width          = 5,
1801 };
1802
1803 static struct c2c_dimension dim_dcacheline_num_empty = {
1804         .header         = HEADER_LOW("Num"),
1805         .name           = "cl_num_empty",
1806         .cmp            = empty_cmp,
1807         .entry          = cl_idx_empty_entry,
1808         .width          = 5,
1809 };
1810
1811 static struct c2c_dimension *dimensions[] = {
1812         &dim_dcacheline,
1813         &dim_dcacheline_node,
1814         &dim_dcacheline_count,
1815         &dim_offset,
1816         &dim_offset_node,
1817         &dim_iaddr,
1818         &dim_tot_hitm,
1819         &dim_lcl_hitm,
1820         &dim_rmt_hitm,
1821         &dim_tot_peer,
1822         &dim_lcl_peer,
1823         &dim_rmt_peer,
1824         &dim_cl_lcl_hitm,
1825         &dim_cl_rmt_hitm,
1826         &dim_cl_lcl_peer,
1827         &dim_cl_rmt_peer,
1828         &dim_tot_stores,
1829         &dim_stores_l1hit,
1830         &dim_stores_l1miss,
1831         &dim_stores_na,
1832         &dim_cl_stores_l1hit,
1833         &dim_cl_stores_l1miss,
1834         &dim_cl_stores_na,
1835         &dim_ld_fbhit,
1836         &dim_ld_l1hit,
1837         &dim_ld_l2hit,
1838         &dim_ld_llchit,
1839         &dim_ld_rmthit,
1840         &dim_tot_recs,
1841         &dim_tot_loads,
1842         &dim_percent_costly_snoop,
1843         &dim_percent_rmt_hitm,
1844         &dim_percent_lcl_hitm,
1845         &dim_percent_rmt_peer,
1846         &dim_percent_lcl_peer,
1847         &dim_percent_stores_l1hit,
1848         &dim_percent_stores_l1miss,
1849         &dim_percent_stores_na,
1850         &dim_dram_lcl,
1851         &dim_dram_rmt,
1852         &dim_pid,
1853         &dim_tid,
1854         &dim_symbol,
1855         &dim_dso,
1856         &dim_node,
1857         &dim_mean_rmt,
1858         &dim_mean_lcl,
1859         &dim_mean_rmt_peer,
1860         &dim_mean_lcl_peer,
1861         &dim_mean_load,
1862         &dim_cpucnt,
1863         &dim_srcline,
1864         &dim_dcacheline_idx,
1865         &dim_dcacheline_num,
1866         &dim_dcacheline_num_empty,
1867         NULL,
1868 };
1869
1870 static void fmt_free(struct perf_hpp_fmt *fmt)
1871 {
1872         struct c2c_fmt *c2c_fmt;
1873
1874         c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1875         free(c2c_fmt);
1876 }
1877
1878 static bool fmt_equal(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b)
1879 {
1880         struct c2c_fmt *c2c_a = container_of(a, struct c2c_fmt, fmt);
1881         struct c2c_fmt *c2c_b = container_of(b, struct c2c_fmt, fmt);
1882
1883         return c2c_a->dim == c2c_b->dim;
1884 }
1885
1886 static struct c2c_dimension *get_dimension(const char *name)
1887 {
1888         unsigned int i;
1889
1890         for (i = 0; dimensions[i]; i++) {
1891                 struct c2c_dimension *dim = dimensions[i];
1892
1893                 if (!strcmp(dim->name, name))
1894                         return dim;
1895         }
1896
1897         return NULL;
1898 }
1899
1900 static int c2c_se_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1901                         struct hist_entry *he)
1902 {
1903         struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1904         struct c2c_dimension *dim = c2c_fmt->dim;
1905         size_t len = fmt->user_len;
1906
1907         if (!len) {
1908                 len = hists__col_len(he->hists, dim->se->se_width_idx);
1909
1910                 if (dim == &dim_symbol || dim == &dim_srcline)
1911                         len = symbol_width(he->hists, dim->se);
1912         }
1913
1914         return dim->se->se_snprintf(he, hpp->buf, hpp->size, len);
1915 }
1916
1917 static int64_t c2c_se_cmp(struct perf_hpp_fmt *fmt,
1918                           struct hist_entry *a, struct hist_entry *b)
1919 {
1920         struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1921         struct c2c_dimension *dim = c2c_fmt->dim;
1922
1923         return dim->se->se_cmp(a, b);
1924 }
1925
1926 static int64_t c2c_se_collapse(struct perf_hpp_fmt *fmt,
1927                                struct hist_entry *a, struct hist_entry *b)
1928 {
1929         struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1930         struct c2c_dimension *dim = c2c_fmt->dim;
1931         int64_t (*collapse_fn)(struct hist_entry *, struct hist_entry *);
1932
1933         collapse_fn = dim->se->se_collapse ?: dim->se->se_cmp;
1934         return collapse_fn(a, b);
1935 }
1936
1937 static struct c2c_fmt *get_format(const char *name)
1938 {
1939         struct c2c_dimension *dim = get_dimension(name);
1940         struct c2c_fmt *c2c_fmt;
1941         struct perf_hpp_fmt *fmt;
1942
1943         if (!dim)
1944                 return NULL;
1945
1946         c2c_fmt = zalloc(sizeof(*c2c_fmt));
1947         if (!c2c_fmt)
1948                 return NULL;
1949
1950         c2c_fmt->dim = dim;
1951
1952         fmt = &c2c_fmt->fmt;
1953         INIT_LIST_HEAD(&fmt->list);
1954         INIT_LIST_HEAD(&fmt->sort_list);
1955
1956         fmt->cmp        = dim->se ? c2c_se_cmp   : dim->cmp;
1957         fmt->sort       = dim->se ? c2c_se_cmp   : dim->cmp;
1958         fmt->color      = dim->se ? NULL         : dim->color;
1959         fmt->entry      = dim->se ? c2c_se_entry : dim->entry;
1960         fmt->header     = c2c_header;
1961         fmt->width      = c2c_width;
1962         fmt->collapse   = dim->se ? c2c_se_collapse : dim->cmp;
1963         fmt->equal      = fmt_equal;
1964         fmt->free       = fmt_free;
1965
1966         return c2c_fmt;
1967 }
1968
1969 static int c2c_hists__init_output(struct perf_hpp_list *hpp_list, char *name)
1970 {
1971         struct c2c_fmt *c2c_fmt = get_format(name);
1972
1973         if (!c2c_fmt) {
1974                 reset_dimensions();
1975                 return output_field_add(hpp_list, name);
1976         }
1977
1978         perf_hpp_list__column_register(hpp_list, &c2c_fmt->fmt);
1979         return 0;
1980 }
1981
1982 static int c2c_hists__init_sort(struct perf_hpp_list *hpp_list, char *name)
1983 {
1984         struct c2c_fmt *c2c_fmt = get_format(name);
1985         struct c2c_dimension *dim;
1986
1987         if (!c2c_fmt) {
1988                 reset_dimensions();
1989                 return sort_dimension__add(hpp_list, name, NULL, 0);
1990         }
1991
1992         dim = c2c_fmt->dim;
1993         if (dim == &dim_dso)
1994                 hpp_list->dso = 1;
1995
1996         perf_hpp_list__register_sort_field(hpp_list, &c2c_fmt->fmt);
1997         return 0;
1998 }
1999
2000 #define PARSE_LIST(_list, _fn)                                                  \
2001         do {                                                                    \
2002                 char *tmp, *tok;                                                \
2003                 ret = 0;                                                        \
2004                                                                                 \
2005                 if (!_list)                                                     \
2006                         break;                                                  \
2007                                                                                 \
2008                 for (tok = strtok_r((char *)_list, ", ", &tmp);                 \
2009                                 tok; tok = strtok_r(NULL, ", ", &tmp)) {        \
2010                         ret = _fn(hpp_list, tok);                               \
2011                         if (ret == -EINVAL) {                                   \
2012                                 pr_err("Invalid --fields key: `%s'", tok);      \
2013                                 break;                                          \
2014                         } else if (ret == -ESRCH) {                             \
2015                                 pr_err("Unknown --fields key: `%s'", tok);      \
2016                                 break;                                          \
2017                         }                                                       \
2018                 }                                                               \
2019         } while (0)
2020
2021 static int hpp_list__parse(struct perf_hpp_list *hpp_list,
2022                            const char *output_,
2023                            const char *sort_)
2024 {
2025         char *output = output_ ? strdup(output_) : NULL;
2026         char *sort   = sort_   ? strdup(sort_) : NULL;
2027         int ret;
2028
2029         PARSE_LIST(output, c2c_hists__init_output);
2030         PARSE_LIST(sort,   c2c_hists__init_sort);
2031
2032         /* copy sort keys to output fields */
2033         perf_hpp__setup_output_field(hpp_list);
2034
2035         /*
2036          * We don't need other sorting keys other than those
2037          * we already specified. It also really slows down
2038          * the processing a lot with big number of output
2039          * fields, so switching this off for c2c.
2040          */
2041
2042 #if 0
2043         /* and then copy output fields to sort keys */
2044         perf_hpp__append_sort_keys(&hists->list);
2045 #endif
2046
2047         free(output);
2048         free(sort);
2049         return ret;
2050 }
2051
2052 static int c2c_hists__init(struct c2c_hists *hists,
2053                            const char *sort,
2054                            int nr_header_lines)
2055 {
2056         __hists__init(&hists->hists, &hists->list);
2057
2058         /*
2059          * Initialize only with sort fields, we need to resort
2060          * later anyway, and that's where we add output fields
2061          * as well.
2062          */
2063         perf_hpp_list__init(&hists->list);
2064
2065         /* Overload number of header lines.*/
2066         hists->list.nr_header_lines = nr_header_lines;
2067
2068         return hpp_list__parse(&hists->list, NULL, sort);
2069 }
2070
2071 static int c2c_hists__reinit(struct c2c_hists *c2c_hists,
2072                              const char *output,
2073                              const char *sort)
2074 {
2075         perf_hpp__reset_output_field(&c2c_hists->list);
2076         return hpp_list__parse(&c2c_hists->list, output, sort);
2077 }
2078
2079 #define DISPLAY_LINE_LIMIT  0.001
2080
2081 static u8 filter_display(u32 val, u32 sum)
2082 {
2083         if (sum == 0 || ((double)val / sum) < DISPLAY_LINE_LIMIT)
2084                 return HIST_FILTER__C2C;
2085
2086         return 0;
2087 }
2088
2089 static bool he__display(struct hist_entry *he, struct c2c_stats *stats)
2090 {
2091         struct c2c_hist_entry *c2c_he;
2092
2093         if (c2c.show_all)
2094                 return true;
2095
2096         c2c_he = container_of(he, struct c2c_hist_entry, he);
2097
2098         switch (c2c.display) {
2099         case DISPLAY_LCL_HITM:
2100                 he->filtered = filter_display(c2c_he->stats.lcl_hitm,
2101                                               stats->lcl_hitm);
2102                 break;
2103         case DISPLAY_RMT_HITM:
2104                 he->filtered = filter_display(c2c_he->stats.rmt_hitm,
2105                                               stats->rmt_hitm);
2106                 break;
2107         case DISPLAY_TOT_HITM:
2108                 he->filtered = filter_display(c2c_he->stats.tot_hitm,
2109                                               stats->tot_hitm);
2110                 break;
2111         case DISPLAY_SNP_PEER:
2112                 he->filtered = filter_display(c2c_he->stats.tot_peer,
2113                                               stats->tot_peer);
2114                 break;
2115         default:
2116                 break;
2117         }
2118
2119         return he->filtered == 0;
2120 }
2121
2122 static inline bool is_valid_hist_entry(struct hist_entry *he)
2123 {
2124         struct c2c_hist_entry *c2c_he;
2125         bool has_record = false;
2126
2127         c2c_he = container_of(he, struct c2c_hist_entry, he);
2128
2129         /* It's a valid entry if contains stores */
2130         if (c2c_he->stats.store)
2131                 return true;
2132
2133         switch (c2c.display) {
2134         case DISPLAY_LCL_HITM:
2135                 has_record = !!c2c_he->stats.lcl_hitm;
2136                 break;
2137         case DISPLAY_RMT_HITM:
2138                 has_record = !!c2c_he->stats.rmt_hitm;
2139                 break;
2140         case DISPLAY_TOT_HITM:
2141                 has_record = !!c2c_he->stats.tot_hitm;
2142                 break;
2143         case DISPLAY_SNP_PEER:
2144                 has_record = !!c2c_he->stats.tot_peer;
2145         default:
2146                 break;
2147         }
2148
2149         return has_record;
2150 }
2151
2152 static void set_node_width(struct c2c_hist_entry *c2c_he, int len)
2153 {
2154         struct c2c_dimension *dim;
2155
2156         dim = &c2c.hists == c2c_he->hists ?
2157               &dim_dcacheline_node : &dim_offset_node;
2158
2159         if (len > dim->width)
2160                 dim->width = len;
2161 }
2162
2163 static int set_nodestr(struct c2c_hist_entry *c2c_he)
2164 {
2165         char buf[30];
2166         int len;
2167
2168         if (c2c_he->nodestr)
2169                 return 0;
2170
2171         if (!bitmap_empty(c2c_he->nodeset, c2c.nodes_cnt)) {
2172                 len = bitmap_scnprintf(c2c_he->nodeset, c2c.nodes_cnt,
2173                                       buf, sizeof(buf));
2174         } else {
2175                 len = scnprintf(buf, sizeof(buf), "N/A");
2176         }
2177
2178         set_node_width(c2c_he, len);
2179         c2c_he->nodestr = strdup(buf);
2180         return c2c_he->nodestr ? 0 : -ENOMEM;
2181 }
2182
2183 static void calc_width(struct c2c_hist_entry *c2c_he)
2184 {
2185         struct c2c_hists *c2c_hists;
2186
2187         c2c_hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);
2188         hists__calc_col_len(&c2c_hists->hists, &c2c_he->he);
2189         set_nodestr(c2c_he);
2190 }
2191
2192 static int filter_cb(struct hist_entry *he, void *arg __maybe_unused)
2193 {
2194         struct c2c_hist_entry *c2c_he;
2195
2196         c2c_he = container_of(he, struct c2c_hist_entry, he);
2197
2198         if (c2c.show_src && !he->srcline)
2199                 he->srcline = hist_entry__srcline(he);
2200
2201         calc_width(c2c_he);
2202
2203         if (!is_valid_hist_entry(he))
2204                 he->filtered = HIST_FILTER__C2C;
2205
2206         return 0;
2207 }
2208
2209 static int resort_cl_cb(struct hist_entry *he, void *arg __maybe_unused)
2210 {
2211         struct c2c_hist_entry *c2c_he;
2212         struct c2c_hists *c2c_hists;
2213         bool display = he__display(he, &c2c.shared_clines_stats);
2214
2215         c2c_he = container_of(he, struct c2c_hist_entry, he);
2216         c2c_hists = c2c_he->hists;
2217
2218         if (display && c2c_hists) {
2219                 static unsigned int idx;
2220
2221                 c2c_he->cacheline_idx = idx++;
2222                 calc_width(c2c_he);
2223
2224                 c2c_hists__reinit(c2c_hists, c2c.cl_output, c2c.cl_resort);
2225
2226                 hists__collapse_resort(&c2c_hists->hists, NULL);
2227                 hists__output_resort_cb(&c2c_hists->hists, NULL, filter_cb);
2228         }
2229
2230         return 0;
2231 }
2232
2233 static struct c2c_header header_node_0 = HEADER_LOW("Node");
2234 static struct c2c_header header_node_1_hitms_stores =
2235                 HEADER_LOW("Node{cpus %hitms %stores}");
2236 static struct c2c_header header_node_1_peers_stores =
2237                 HEADER_LOW("Node{cpus %peers %stores}");
2238 static struct c2c_header header_node_2 = HEADER_LOW("Node{cpu list}");
2239
2240 static void setup_nodes_header(void)
2241 {
2242         switch (c2c.node_info) {
2243         case 0:
2244                 dim_node.header = header_node_0;
2245                 break;
2246         case 1:
2247                 if (c2c.display == DISPLAY_SNP_PEER)
2248                         dim_node.header = header_node_1_peers_stores;
2249                 else
2250                         dim_node.header = header_node_1_hitms_stores;
2251                 break;
2252         case 2:
2253                 dim_node.header = header_node_2;
2254                 break;
2255         default:
2256                 break;
2257         }
2258
2259         return;
2260 }
2261
2262 static int setup_nodes(struct perf_session *session)
2263 {
2264         struct numa_node *n;
2265         unsigned long **nodes;
2266         int node, idx;
2267         struct perf_cpu cpu;
2268         int *cpu2node;
2269
2270         if (c2c.node_info > 2)
2271                 c2c.node_info = 2;
2272
2273         c2c.nodes_cnt = session->header.env.nr_numa_nodes;
2274         c2c.cpus_cnt  = session->header.env.nr_cpus_avail;
2275
2276         n = session->header.env.numa_nodes;
2277         if (!n)
2278                 return -EINVAL;
2279
2280         nodes = zalloc(sizeof(unsigned long *) * c2c.nodes_cnt);
2281         if (!nodes)
2282                 return -ENOMEM;
2283
2284         c2c.nodes = nodes;
2285
2286         cpu2node = zalloc(sizeof(int) * c2c.cpus_cnt);
2287         if (!cpu2node)
2288                 return -ENOMEM;
2289
2290         for (idx = 0; idx < c2c.cpus_cnt; idx++)
2291                 cpu2node[idx] = -1;
2292
2293         c2c.cpu2node = cpu2node;
2294
2295         for (node = 0; node < c2c.nodes_cnt; node++) {
2296                 struct perf_cpu_map *map = n[node].map;
2297                 unsigned long *set;
2298
2299                 set = bitmap_zalloc(c2c.cpus_cnt);
2300                 if (!set)
2301                         return -ENOMEM;
2302
2303                 nodes[node] = set;
2304
2305                 perf_cpu_map__for_each_cpu_skip_any(cpu, idx, map) {
2306                         __set_bit(cpu.cpu, set);
2307
2308                         if (WARN_ONCE(cpu2node[cpu.cpu] != -1, "node/cpu topology bug"))
2309                                 return -EINVAL;
2310
2311                         cpu2node[cpu.cpu] = node;
2312                 }
2313         }
2314
2315         setup_nodes_header();
2316         return 0;
2317 }
2318
2319 #define HAS_HITMS(__h) ((__h)->stats.lcl_hitm || (__h)->stats.rmt_hitm)
2320 #define HAS_PEER(__h) ((__h)->stats.lcl_peer || (__h)->stats.rmt_peer)
2321
2322 static int resort_shared_cl_cb(struct hist_entry *he, void *arg __maybe_unused)
2323 {
2324         struct c2c_hist_entry *c2c_he;
2325         c2c_he = container_of(he, struct c2c_hist_entry, he);
2326
2327         if (HAS_HITMS(c2c_he) || HAS_PEER(c2c_he)) {
2328                 c2c.shared_clines++;
2329                 c2c_add_stats(&c2c.shared_clines_stats, &c2c_he->stats);
2330         }
2331
2332         return 0;
2333 }
2334
2335 static int hists__iterate_cb(struct hists *hists, hists__resort_cb_t cb)
2336 {
2337         struct rb_node *next = rb_first_cached(&hists->entries);
2338         int ret = 0;
2339
2340         while (next) {
2341                 struct hist_entry *he;
2342
2343                 he = rb_entry(next, struct hist_entry, rb_node);
2344                 ret = cb(he, NULL);
2345                 if (ret)
2346                         break;
2347                 next = rb_next(&he->rb_node);
2348         }
2349
2350         return ret;
2351 }
2352
2353 static void print_c2c__display_stats(FILE *out)
2354 {
2355         int llc_misses;
2356         struct c2c_stats *stats = &c2c.hists.stats;
2357
2358         llc_misses = get_load_llc_misses(stats);
2359
2360         fprintf(out, "=================================================\n");
2361         fprintf(out, "            Trace Event Information              \n");
2362         fprintf(out, "=================================================\n");
2363         fprintf(out, "  Total records                     : %10d\n", stats->nr_entries);
2364         fprintf(out, "  Locked Load/Store Operations      : %10d\n", stats->locks);
2365         fprintf(out, "  Load Operations                   : %10d\n", stats->load);
2366         fprintf(out, "  Loads - uncacheable               : %10d\n", stats->ld_uncache);
2367         fprintf(out, "  Loads - IO                        : %10d\n", stats->ld_io);
2368         fprintf(out, "  Loads - Miss                      : %10d\n", stats->ld_miss);
2369         fprintf(out, "  Loads - no mapping                : %10d\n", stats->ld_noadrs);
2370         fprintf(out, "  Load Fill Buffer Hit              : %10d\n", stats->ld_fbhit);
2371         fprintf(out, "  Load L1D hit                      : %10d\n", stats->ld_l1hit);
2372         fprintf(out, "  Load L2D hit                      : %10d\n", stats->ld_l2hit);
2373         fprintf(out, "  Load LLC hit                      : %10d\n", stats->ld_llchit + stats->lcl_hitm);
2374         fprintf(out, "  Load Local HITM                   : %10d\n", stats->lcl_hitm);
2375         fprintf(out, "  Load Remote HITM                  : %10d\n", stats->rmt_hitm);
2376         fprintf(out, "  Load Remote HIT                   : %10d\n", stats->rmt_hit);
2377         fprintf(out, "  Load Local DRAM                   : %10d\n", stats->lcl_dram);
2378         fprintf(out, "  Load Remote DRAM                  : %10d\n", stats->rmt_dram);
2379         fprintf(out, "  Load MESI State Exclusive         : %10d\n", stats->ld_excl);
2380         fprintf(out, "  Load MESI State Shared            : %10d\n", stats->ld_shared);
2381         fprintf(out, "  Load LLC Misses                   : %10d\n", llc_misses);
2382         fprintf(out, "  Load access blocked by data       : %10d\n", stats->blk_data);
2383         fprintf(out, "  Load access blocked by address    : %10d\n", stats->blk_addr);
2384         fprintf(out, "  Load HIT Local Peer               : %10d\n", stats->lcl_peer);
2385         fprintf(out, "  Load HIT Remote Peer              : %10d\n", stats->rmt_peer);
2386         fprintf(out, "  LLC Misses to Local DRAM          : %10.1f%%\n", ((double)stats->lcl_dram/(double)llc_misses) * 100.);
2387         fprintf(out, "  LLC Misses to Remote DRAM         : %10.1f%%\n", ((double)stats->rmt_dram/(double)llc_misses) * 100.);
2388         fprintf(out, "  LLC Misses to Remote cache (HIT)  : %10.1f%%\n", ((double)stats->rmt_hit /(double)llc_misses) * 100.);
2389         fprintf(out, "  LLC Misses to Remote cache (HITM) : %10.1f%%\n", ((double)stats->rmt_hitm/(double)llc_misses) * 100.);
2390         fprintf(out, "  Store Operations                  : %10d\n", stats->store);
2391         fprintf(out, "  Store - uncacheable               : %10d\n", stats->st_uncache);
2392         fprintf(out, "  Store - no mapping                : %10d\n", stats->st_noadrs);
2393         fprintf(out, "  Store L1D Hit                     : %10d\n", stats->st_l1hit);
2394         fprintf(out, "  Store L1D Miss                    : %10d\n", stats->st_l1miss);
2395         fprintf(out, "  Store No available memory level   : %10d\n", stats->st_na);
2396         fprintf(out, "  No Page Map Rejects               : %10d\n", stats->nomap);
2397         fprintf(out, "  Unable to parse data source       : %10d\n", stats->noparse);
2398 }
2399
2400 static void print_shared_cacheline_info(FILE *out)
2401 {
2402         struct c2c_stats *stats = &c2c.shared_clines_stats;
2403         int hitm_cnt = stats->lcl_hitm + stats->rmt_hitm;
2404
2405         fprintf(out, "=================================================\n");
2406         fprintf(out, "    Global Shared Cache Line Event Information   \n");
2407         fprintf(out, "=================================================\n");
2408         fprintf(out, "  Total Shared Cache Lines          : %10d\n", c2c.shared_clines);
2409         fprintf(out, "  Load HITs on shared lines         : %10d\n", stats->load);
2410         fprintf(out, "  Fill Buffer Hits on shared lines  : %10d\n", stats->ld_fbhit);
2411         fprintf(out, "  L1D hits on shared lines          : %10d\n", stats->ld_l1hit);
2412         fprintf(out, "  L2D hits on shared lines          : %10d\n", stats->ld_l2hit);
2413         fprintf(out, "  LLC hits on shared lines          : %10d\n", stats->ld_llchit + stats->lcl_hitm);
2414         fprintf(out, "  Load hits on peer cache or nodes  : %10d\n", stats->lcl_peer + stats->rmt_peer);
2415         fprintf(out, "  Locked Access on shared lines     : %10d\n", stats->locks);
2416         fprintf(out, "  Blocked Access on shared lines    : %10d\n", stats->blk_data + stats->blk_addr);
2417         fprintf(out, "  Store HITs on shared lines        : %10d\n", stats->store);
2418         fprintf(out, "  Store L1D hits on shared lines    : %10d\n", stats->st_l1hit);
2419         fprintf(out, "  Store No available memory level   : %10d\n", stats->st_na);
2420         fprintf(out, "  Total Merged records              : %10d\n", hitm_cnt + stats->store);
2421 }
2422
2423 static void print_cacheline(struct c2c_hists *c2c_hists,
2424                             struct hist_entry *he_cl,
2425                             struct perf_hpp_list *hpp_list,
2426                             FILE *out)
2427 {
2428         char bf[1000];
2429         struct perf_hpp hpp = {
2430                 .buf            = bf,
2431                 .size           = 1000,
2432         };
2433         static bool once;
2434
2435         if (!once) {
2436                 hists__fprintf_headers(&c2c_hists->hists, out);
2437                 once = true;
2438         } else {
2439                 fprintf(out, "\n");
2440         }
2441
2442         fprintf(out, "  ----------------------------------------------------------------------\n");
2443         __hist_entry__snprintf(he_cl, &hpp, hpp_list);
2444         fprintf(out, "%s\n", bf);
2445         fprintf(out, "  ----------------------------------------------------------------------\n");
2446
2447         hists__fprintf(&c2c_hists->hists, false, 0, 0, 0, out, false);
2448 }
2449
2450 static void print_pareto(FILE *out)
2451 {
2452         struct perf_hpp_list hpp_list;
2453         struct rb_node *nd;
2454         int ret;
2455         const char *cl_output;
2456
2457         if (c2c.display != DISPLAY_SNP_PEER)
2458                 cl_output = "cl_num,"
2459                             "cl_rmt_hitm,"
2460                             "cl_lcl_hitm,"
2461                             "cl_stores_l1hit,"
2462                             "cl_stores_l1miss,"
2463                             "cl_stores_na,"
2464                             "dcacheline";
2465         else
2466                 cl_output = "cl_num,"
2467                             "cl_rmt_peer,"
2468                             "cl_lcl_peer,"
2469                             "cl_stores_l1hit,"
2470                             "cl_stores_l1miss,"
2471                             "cl_stores_na,"
2472                             "dcacheline";
2473
2474         perf_hpp_list__init(&hpp_list);
2475         ret = hpp_list__parse(&hpp_list, cl_output, NULL);
2476
2477         if (WARN_ONCE(ret, "failed to setup sort entries\n"))
2478                 return;
2479
2480         nd = rb_first_cached(&c2c.hists.hists.entries);
2481
2482         for (; nd; nd = rb_next(nd)) {
2483                 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
2484                 struct c2c_hist_entry *c2c_he;
2485
2486                 if (he->filtered)
2487                         continue;
2488
2489                 c2c_he = container_of(he, struct c2c_hist_entry, he);
2490                 print_cacheline(c2c_he->hists, he, &hpp_list, out);
2491         }
2492 }
2493
2494 static void print_c2c_info(FILE *out, struct perf_session *session)
2495 {
2496         struct evlist *evlist = session->evlist;
2497         struct evsel *evsel;
2498         bool first = true;
2499
2500         fprintf(out, "=================================================\n");
2501         fprintf(out, "                 c2c details                     \n");
2502         fprintf(out, "=================================================\n");
2503
2504         evlist__for_each_entry(evlist, evsel) {
2505                 fprintf(out, "%-36s: %s\n", first ? "  Events" : "", evsel__name(evsel));
2506                 first = false;
2507         }
2508         fprintf(out, "  Cachelines sort on                : %s\n",
2509                 display_str[c2c.display]);
2510         fprintf(out, "  Cacheline data grouping           : %s\n", c2c.cl_sort);
2511 }
2512
2513 static void perf_c2c__hists_fprintf(FILE *out, struct perf_session *session)
2514 {
2515         setup_pager();
2516
2517         print_c2c__display_stats(out);
2518         fprintf(out, "\n");
2519         print_shared_cacheline_info(out);
2520         fprintf(out, "\n");
2521         print_c2c_info(out, session);
2522
2523         if (c2c.stats_only)
2524                 return;
2525
2526         fprintf(out, "\n");
2527         fprintf(out, "=================================================\n");
2528         fprintf(out, "           Shared Data Cache Line Table          \n");
2529         fprintf(out, "=================================================\n");
2530         fprintf(out, "#\n");
2531
2532         hists__fprintf(&c2c.hists.hists, true, 0, 0, 0, stdout, true);
2533
2534         fprintf(out, "\n");
2535         fprintf(out, "=================================================\n");
2536         fprintf(out, "      Shared Cache Line Distribution Pareto      \n");
2537         fprintf(out, "=================================================\n");
2538         fprintf(out, "#\n");
2539
2540         print_pareto(out);
2541 }
2542
2543 #ifdef HAVE_SLANG_SUPPORT
2544 static void c2c_browser__update_nr_entries(struct hist_browser *hb)
2545 {
2546         u64 nr_entries = 0;
2547         struct rb_node *nd = rb_first_cached(&hb->hists->entries);
2548
2549         while (nd) {
2550                 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
2551
2552                 if (!he->filtered)
2553                         nr_entries++;
2554
2555                 nd = rb_next(nd);
2556         }
2557
2558         hb->nr_non_filtered_entries = nr_entries;
2559 }
2560
2561 struct c2c_cacheline_browser {
2562         struct hist_browser      hb;
2563         struct hist_entry       *he;
2564 };
2565
2566 static int
2567 perf_c2c_cacheline_browser__title(struct hist_browser *browser,
2568                                   char *bf, size_t size)
2569 {
2570         struct c2c_cacheline_browser *cl_browser;
2571         struct hist_entry *he;
2572         uint64_t addr = 0;
2573
2574         cl_browser = container_of(browser, struct c2c_cacheline_browser, hb);
2575         he = cl_browser->he;
2576
2577         if (he->mem_info)
2578                 addr = cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl);
2579
2580         scnprintf(bf, size, "Cacheline 0x%lx", addr);
2581         return 0;
2582 }
2583
2584 static struct c2c_cacheline_browser*
2585 c2c_cacheline_browser__new(struct hists *hists, struct hist_entry *he)
2586 {
2587         struct c2c_cacheline_browser *browser;
2588
2589         browser = zalloc(sizeof(*browser));
2590         if (browser) {
2591                 hist_browser__init(&browser->hb, hists);
2592                 browser->hb.c2c_filter  = true;
2593                 browser->hb.title       = perf_c2c_cacheline_browser__title;
2594                 browser->he             = he;
2595         }
2596
2597         return browser;
2598 }
2599
2600 static int perf_c2c__browse_cacheline(struct hist_entry *he)
2601 {
2602         struct c2c_hist_entry *c2c_he;
2603         struct c2c_hists *c2c_hists;
2604         struct c2c_cacheline_browser *cl_browser;
2605         struct hist_browser *browser;
2606         int key = -1;
2607         static const char help[] =
2608         " ENTER         Toggle callchains (if present) \n"
2609         " n             Toggle Node details info \n"
2610         " s             Toggle full length of symbol and source line columns \n"
2611         " q             Return back to cacheline list \n";
2612
2613         if (!he)
2614                 return 0;
2615
2616         /* Display compact version first. */
2617         c2c.symbol_full = false;
2618
2619         c2c_he = container_of(he, struct c2c_hist_entry, he);
2620         c2c_hists = c2c_he->hists;
2621
2622         cl_browser = c2c_cacheline_browser__new(&c2c_hists->hists, he);
2623         if (cl_browser == NULL)
2624                 return -1;
2625
2626         browser = &cl_browser->hb;
2627
2628         /* reset abort key so that it can get Ctrl-C as a key */
2629         SLang_reset_tty();
2630         SLang_init_tty(0, 0, 0);
2631
2632         c2c_browser__update_nr_entries(browser);
2633
2634         while (1) {
2635                 key = hist_browser__run(browser, "? - help", true, 0);
2636
2637                 switch (key) {
2638                 case 's':
2639                         c2c.symbol_full = !c2c.symbol_full;
2640                         break;
2641                 case 'n':
2642                         c2c.node_info = (c2c.node_info + 1) % 3;
2643                         setup_nodes_header();
2644                         break;
2645                 case 'q':
2646                         goto out;
2647                 case '?':
2648                         ui_browser__help_window(&browser->b, help);
2649                         break;
2650                 default:
2651                         break;
2652                 }
2653         }
2654
2655 out:
2656         free(cl_browser);
2657         return 0;
2658 }
2659
2660 static int perf_c2c_browser__title(struct hist_browser *browser,
2661                                    char *bf, size_t size)
2662 {
2663         scnprintf(bf, size,
2664                   "Shared Data Cache Line Table     "
2665                   "(%lu entries, sorted on %s)",
2666                   browser->nr_non_filtered_entries,
2667                   display_str[c2c.display]);
2668         return 0;
2669 }
2670
2671 static struct hist_browser*
2672 perf_c2c_browser__new(struct hists *hists)
2673 {
2674         struct hist_browser *browser = hist_browser__new(hists);
2675
2676         if (browser) {
2677                 browser->title = perf_c2c_browser__title;
2678                 browser->c2c_filter = true;
2679         }
2680
2681         return browser;
2682 }
2683
2684 static int perf_c2c__hists_browse(struct hists *hists)
2685 {
2686         struct hist_browser *browser;
2687         int key = -1;
2688         static const char help[] =
2689         " d             Display cacheline details \n"
2690         " ENTER         Toggle callchains (if present) \n"
2691         " q             Quit \n";
2692
2693         browser = perf_c2c_browser__new(hists);
2694         if (browser == NULL)
2695                 return -1;
2696
2697         /* reset abort key so that it can get Ctrl-C as a key */
2698         SLang_reset_tty();
2699         SLang_init_tty(0, 0, 0);
2700
2701         c2c_browser__update_nr_entries(browser);
2702
2703         while (1) {
2704                 key = hist_browser__run(browser, "? - help", true, 0);
2705
2706                 switch (key) {
2707                 case 'q':
2708                         goto out;
2709                 case 'd':
2710                         perf_c2c__browse_cacheline(browser->he_selection);
2711                         break;
2712                 case '?':
2713                         ui_browser__help_window(&browser->b, help);
2714                         break;
2715                 default:
2716                         break;
2717                 }
2718         }
2719
2720 out:
2721         hist_browser__delete(browser);
2722         return 0;
2723 }
2724
2725 static void perf_c2c_display(struct perf_session *session)
2726 {
2727         if (use_browser == 0)
2728                 perf_c2c__hists_fprintf(stdout, session);
2729         else
2730                 perf_c2c__hists_browse(&c2c.hists.hists);
2731 }
2732 #else
2733 static void perf_c2c_display(struct perf_session *session)
2734 {
2735         use_browser = 0;
2736         perf_c2c__hists_fprintf(stdout, session);
2737 }
2738 #endif /* HAVE_SLANG_SUPPORT */
2739
2740 static char *fill_line(const char *orig, int len)
2741 {
2742         int i, j, olen = strlen(orig);
2743         char *buf;
2744
2745         buf = zalloc(len + 1);
2746         if (!buf)
2747                 return NULL;
2748
2749         j = len / 2 - olen / 2;
2750
2751         for (i = 0; i < j - 1; i++)
2752                 buf[i] = '-';
2753
2754         buf[i++] = ' ';
2755
2756         strcpy(buf + i, orig);
2757
2758         i += olen;
2759
2760         buf[i++] = ' ';
2761
2762         for (; i < len; i++)
2763                 buf[i] = '-';
2764
2765         return buf;
2766 }
2767
2768 static int ui_quirks(void)
2769 {
2770         const char *nodestr = "Data address";
2771         char *buf;
2772
2773         if (!c2c.use_stdio) {
2774                 dim_offset.width  = 5;
2775                 dim_offset.header = header_offset_tui;
2776                 nodestr = chk_double_cl ? "Double-CL" : "CL";
2777         }
2778
2779         dim_percent_costly_snoop.header = percent_costly_snoop_header[c2c.display];
2780
2781         /* Fix the zero line for dcacheline column. */
2782         buf = fill_line(chk_double_cl ? "Double-Cacheline" : "Cacheline",
2783                                 dim_dcacheline.width +
2784                                 dim_dcacheline_node.width +
2785                                 dim_dcacheline_count.width + 4);
2786         if (!buf)
2787                 return -ENOMEM;
2788
2789         dim_dcacheline.header.line[0].text = buf;
2790
2791         /* Fix the zero line for offset column. */
2792         buf = fill_line(nodestr, dim_offset.width +
2793                                  dim_offset_node.width +
2794                                  dim_dcacheline_count.width + 4);
2795         if (!buf)
2796                 return -ENOMEM;
2797
2798         dim_offset.header.line[0].text = buf;
2799
2800         return 0;
2801 }
2802
2803 #define CALLCHAIN_DEFAULT_OPT  "graph,0.5,caller,function,percent"
2804
2805 const char callchain_help[] = "Display call graph (stack chain/backtrace):\n\n"
2806                                 CALLCHAIN_REPORT_HELP
2807                                 "\n\t\t\t\tDefault: " CALLCHAIN_DEFAULT_OPT;
2808
2809 static int
2810 parse_callchain_opt(const struct option *opt, const char *arg, int unset)
2811 {
2812         struct callchain_param *callchain = opt->value;
2813
2814         callchain->enabled = !unset;
2815         /*
2816          * --no-call-graph
2817          */
2818         if (unset) {
2819                 symbol_conf.use_callchain = false;
2820                 callchain->mode = CHAIN_NONE;
2821                 return 0;
2822         }
2823
2824         return parse_callchain_report_opt(arg);
2825 }
2826
2827 static int setup_callchain(struct evlist *evlist)
2828 {
2829         u64 sample_type = evlist__combined_sample_type(evlist);
2830         enum perf_call_graph_mode mode = CALLCHAIN_NONE;
2831
2832         if ((sample_type & PERF_SAMPLE_REGS_USER) &&
2833             (sample_type & PERF_SAMPLE_STACK_USER)) {
2834                 mode = CALLCHAIN_DWARF;
2835                 dwarf_callchain_users = true;
2836         } else if (sample_type & PERF_SAMPLE_BRANCH_STACK)
2837                 mode = CALLCHAIN_LBR;
2838         else if (sample_type & PERF_SAMPLE_CALLCHAIN)
2839                 mode = CALLCHAIN_FP;
2840
2841         if (!callchain_param.enabled &&
2842             callchain_param.mode != CHAIN_NONE &&
2843             mode != CALLCHAIN_NONE) {
2844                 symbol_conf.use_callchain = true;
2845                 if (callchain_register_param(&callchain_param) < 0) {
2846                         ui__error("Can't register callchain params.\n");
2847                         return -EINVAL;
2848                 }
2849         }
2850
2851         if (c2c.stitch_lbr && (mode != CALLCHAIN_LBR)) {
2852                 ui__warning("Can't find LBR callchain. Switch off --stitch-lbr.\n"
2853                             "Please apply --call-graph lbr when recording.\n");
2854                 c2c.stitch_lbr = false;
2855         }
2856
2857         callchain_param.record_mode = mode;
2858         callchain_param.min_percent = 0;
2859         return 0;
2860 }
2861
2862 static int setup_display(const char *str)
2863 {
2864         const char *display = str;
2865
2866         if (!strcmp(display, "tot"))
2867                 c2c.display = DISPLAY_TOT_HITM;
2868         else if (!strcmp(display, "rmt"))
2869                 c2c.display = DISPLAY_RMT_HITM;
2870         else if (!strcmp(display, "lcl"))
2871                 c2c.display = DISPLAY_LCL_HITM;
2872         else if (!strcmp(display, "peer"))
2873                 c2c.display = DISPLAY_SNP_PEER;
2874         else {
2875                 pr_err("failed: unknown display type: %s\n", str);
2876                 return -1;
2877         }
2878
2879         return 0;
2880 }
2881
2882 #define for_each_token(__tok, __buf, __sep, __tmp)              \
2883         for (__tok = strtok_r(__buf, __sep, &__tmp); __tok;     \
2884              __tok = strtok_r(NULL,  __sep, &__tmp))
2885
2886 static int build_cl_output(char *cl_sort, bool no_source)
2887 {
2888         char *tok, *tmp, *buf = strdup(cl_sort);
2889         bool add_pid   = false;
2890         bool add_tid   = false;
2891         bool add_iaddr = false;
2892         bool add_sym   = false;
2893         bool add_dso   = false;
2894         bool add_src   = false;
2895         int ret = 0;
2896
2897         if (!buf)
2898                 return -ENOMEM;
2899
2900         for_each_token(tok, buf, ",", tmp) {
2901                 if (!strcmp(tok, "tid")) {
2902                         add_tid = true;
2903                 } else if (!strcmp(tok, "pid")) {
2904                         add_pid = true;
2905                 } else if (!strcmp(tok, "iaddr")) {
2906                         add_iaddr = true;
2907                         add_sym   = true;
2908                         add_dso   = true;
2909                         add_src   = no_source ? false : true;
2910                 } else if (!strcmp(tok, "dso")) {
2911                         add_dso = true;
2912                 } else if (strcmp(tok, "offset")) {
2913                         pr_err("unrecognized sort token: %s\n", tok);
2914                         ret = -EINVAL;
2915                         goto err;
2916                 }
2917         }
2918
2919         if (asprintf(&c2c.cl_output,
2920                 "%s%s%s%s%s%s%s%s%s%s%s%s",
2921                 c2c.use_stdio ? "cl_num_empty," : "",
2922                 c2c.display == DISPLAY_SNP_PEER ? "percent_rmt_peer,"
2923                                                   "percent_lcl_peer," :
2924                                                   "percent_rmt_hitm,"
2925                                                   "percent_lcl_hitm,",
2926                 "percent_stores_l1hit,"
2927                 "percent_stores_l1miss,"
2928                 "percent_stores_na,"
2929                 "offset,offset_node,dcacheline_count,",
2930                 add_pid   ? "pid," : "",
2931                 add_tid   ? "tid," : "",
2932                 add_iaddr ? "iaddr," : "",
2933                 c2c.display == DISPLAY_SNP_PEER ? "mean_rmt_peer,"
2934                                                   "mean_lcl_peer," :
2935                                                   "mean_rmt,"
2936                                                   "mean_lcl,",
2937                 "mean_load,"
2938                 "tot_recs,"
2939                 "cpucnt,",
2940                 add_sym ? "symbol," : "",
2941                 add_dso ? "dso," : "",
2942                 add_src ? "cl_srcline," : "",
2943                 "node") < 0) {
2944                 ret = -ENOMEM;
2945                 goto err;
2946         }
2947
2948         c2c.show_src = add_src;
2949 err:
2950         free(buf);
2951         return ret;
2952 }
2953
2954 static int setup_coalesce(const char *coalesce, bool no_source)
2955 {
2956         const char *c = coalesce ?: coalesce_default;
2957         const char *sort_str = NULL;
2958
2959         if (asprintf(&c2c.cl_sort, "offset,%s", c) < 0)
2960                 return -ENOMEM;
2961
2962         if (build_cl_output(c2c.cl_sort, no_source))
2963                 return -1;
2964
2965         if (c2c.display == DISPLAY_TOT_HITM)
2966                 sort_str = "tot_hitm";
2967         else if (c2c.display == DISPLAY_RMT_HITM)
2968                 sort_str = "rmt_hitm,lcl_hitm";
2969         else if (c2c.display == DISPLAY_LCL_HITM)
2970                 sort_str = "lcl_hitm,rmt_hitm";
2971         else if (c2c.display == DISPLAY_SNP_PEER)
2972                 sort_str = "tot_peer";
2973
2974         if (asprintf(&c2c.cl_resort, "offset,%s", sort_str) < 0)
2975                 return -ENOMEM;
2976
2977         pr_debug("coalesce sort   fields: %s\n", c2c.cl_sort);
2978         pr_debug("coalesce resort fields: %s\n", c2c.cl_resort);
2979         pr_debug("coalesce output fields: %s\n", c2c.cl_output);
2980         return 0;
2981 }
2982
2983 static int perf_c2c__report(int argc, const char **argv)
2984 {
2985         struct itrace_synth_opts itrace_synth_opts = {
2986                 .set = true,
2987                 .mem = true,    /* Only enable memory event */
2988                 .default_no_sample = true,
2989         };
2990
2991         struct perf_session *session;
2992         struct ui_progress prog;
2993         struct perf_data data = {
2994                 .mode = PERF_DATA_MODE_READ,
2995         };
2996         char callchain_default_opt[] = CALLCHAIN_DEFAULT_OPT;
2997         const char *display = NULL;
2998         const char *coalesce = NULL;
2999         bool no_source = false;
3000         const struct option options[] = {
3001         OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
3002                    "file", "vmlinux pathname"),
3003         OPT_STRING('i', "input", &input_name, "file",
3004                    "the input file to process"),
3005         OPT_INCR('N', "node-info", &c2c.node_info,
3006                  "show extra node info in report (repeat for more info)"),
3007         OPT_BOOLEAN(0, "stdio", &c2c.use_stdio, "Use the stdio interface"),
3008         OPT_BOOLEAN(0, "stats", &c2c.stats_only,
3009                     "Display only statistic tables (implies --stdio)"),
3010         OPT_BOOLEAN(0, "full-symbols", &c2c.symbol_full,
3011                     "Display full length of symbols"),
3012         OPT_BOOLEAN(0, "no-source", &no_source,
3013                     "Do not display Source Line column"),
3014         OPT_BOOLEAN(0, "show-all", &c2c.show_all,
3015                     "Show all captured HITM lines."),
3016         OPT_CALLBACK_DEFAULT('g', "call-graph", &callchain_param,
3017                              "print_type,threshold[,print_limit],order,sort_key[,branch],value",
3018                              callchain_help, &parse_callchain_opt,
3019                              callchain_default_opt),
3020         OPT_STRING('d', "display", &display, "Switch HITM output type", "tot,lcl,rmt,peer"),
3021         OPT_STRING('c', "coalesce", &coalesce, "coalesce fields",
3022                    "coalesce fields: pid,tid,iaddr,dso"),
3023         OPT_BOOLEAN('f', "force", &symbol_conf.force, "don't complain, do it"),
3024         OPT_BOOLEAN(0, "stitch-lbr", &c2c.stitch_lbr,
3025                     "Enable LBR callgraph stitching approach"),
3026         OPT_BOOLEAN(0, "double-cl", &chk_double_cl, "Detect adjacent cacheline false sharing"),
3027         OPT_PARENT(c2c_options),
3028         OPT_END()
3029         };
3030         int err = 0;
3031         const char *output_str, *sort_str = NULL;
3032
3033         argc = parse_options(argc, argv, options, report_c2c_usage,
3034                              PARSE_OPT_STOP_AT_NON_OPTION);
3035         if (argc)
3036                 usage_with_options(report_c2c_usage, options);
3037
3038 #ifndef HAVE_SLANG_SUPPORT
3039         c2c.use_stdio = true;
3040 #endif
3041
3042         if (c2c.stats_only)
3043                 c2c.use_stdio = true;
3044
3045         err = symbol__validate_sym_arguments();
3046         if (err)
3047                 goto out;
3048
3049         if (!input_name || !strlen(input_name))
3050                 input_name = "perf.data";
3051
3052         data.path  = input_name;
3053         data.force = symbol_conf.force;
3054
3055         perf_tool__init(&c2c.tool, /*ordered_events=*/true);
3056         c2c.tool.sample         = process_sample_event;
3057         c2c.tool.mmap           = perf_event__process_mmap;
3058         c2c.tool.mmap2          = perf_event__process_mmap2;
3059         c2c.tool.comm           = perf_event__process_comm;
3060         c2c.tool.exit           = perf_event__process_exit;
3061         c2c.tool.fork           = perf_event__process_fork;
3062         c2c.tool.lost           = perf_event__process_lost;
3063         c2c.tool.attr           = perf_event__process_attr;
3064         c2c.tool.auxtrace_info  = perf_event__process_auxtrace_info;
3065         c2c.tool.auxtrace       = perf_event__process_auxtrace;
3066         c2c.tool.auxtrace_error = perf_event__process_auxtrace_error;
3067         c2c.tool.ordering_requires_timestamps = true;
3068         session = perf_session__new(&data, &c2c.tool);
3069         if (IS_ERR(session)) {
3070                 err = PTR_ERR(session);
3071                 pr_debug("Error creating perf session\n");
3072                 goto out;
3073         }
3074
3075         /*
3076          * Use the 'tot' as default display type if user doesn't specify it;
3077          * since Arm64 platform doesn't support HITMs flag, use 'peer' as the
3078          * default display type.
3079          */
3080         if (!display) {
3081                 if (!strcmp(perf_env__arch(&session->header.env), "arm64"))
3082                         display = "peer";
3083                 else
3084                         display = "tot";
3085         }
3086
3087         err = setup_display(display);
3088         if (err)
3089                 goto out_session;
3090
3091         err = setup_coalesce(coalesce, no_source);
3092         if (err) {
3093                 pr_debug("Failed to initialize hists\n");
3094                 goto out_session;
3095         }
3096
3097         err = c2c_hists__init(&c2c.hists, "dcacheline", 2);
3098         if (err) {
3099                 pr_debug("Failed to initialize hists\n");
3100                 goto out_session;
3101         }
3102
3103         session->itrace_synth_opts = &itrace_synth_opts;
3104
3105         err = setup_nodes(session);
3106         if (err) {
3107                 pr_err("Failed setup nodes\n");
3108                 goto out_session;
3109         }
3110
3111         err = mem2node__init(&c2c.mem2node, &session->header.env);
3112         if (err)
3113                 goto out_session;
3114
3115         err = setup_callchain(session->evlist);
3116         if (err)
3117                 goto out_mem2node;
3118
3119         if (symbol__init(&session->header.env) < 0)
3120                 goto out_mem2node;
3121
3122         /* No pipe support at the moment. */
3123         if (perf_data__is_pipe(session->data)) {
3124                 pr_debug("No pipe support at the moment.\n");
3125                 goto out_mem2node;
3126         }
3127
3128         if (c2c.use_stdio)
3129                 use_browser = 0;
3130         else
3131                 use_browser = 1;
3132
3133         setup_browser(false);
3134
3135         err = perf_session__process_events(session);
3136         if (err) {
3137                 pr_err("failed to process sample\n");
3138                 goto out_mem2node;
3139         }
3140
3141         if (c2c.display != DISPLAY_SNP_PEER)
3142                 output_str = "cl_idx,"
3143                              "dcacheline,"
3144                              "dcacheline_node,"
3145                              "dcacheline_count,"
3146                              "percent_costly_snoop,"
3147                              "tot_hitm,lcl_hitm,rmt_hitm,"
3148                              "tot_recs,"
3149                              "tot_loads,"
3150                              "tot_stores,"
3151                              "stores_l1hit,stores_l1miss,stores_na,"
3152                              "ld_fbhit,ld_l1hit,ld_l2hit,"
3153                              "ld_lclhit,lcl_hitm,"
3154                              "ld_rmthit,rmt_hitm,"
3155                              "dram_lcl,dram_rmt";
3156         else
3157                 output_str = "cl_idx,"
3158                              "dcacheline,"
3159                              "dcacheline_node,"
3160                              "dcacheline_count,"
3161                              "percent_costly_snoop,"
3162                              "tot_peer,lcl_peer,rmt_peer,"
3163                              "tot_recs,"
3164                              "tot_loads,"
3165                              "tot_stores,"
3166                              "stores_l1hit,stores_l1miss,stores_na,"
3167                              "ld_fbhit,ld_l1hit,ld_l2hit,"
3168                              "ld_lclhit,lcl_hitm,"
3169                              "ld_rmthit,rmt_hitm,"
3170                              "dram_lcl,dram_rmt";
3171
3172         if (c2c.display == DISPLAY_TOT_HITM)
3173                 sort_str = "tot_hitm";
3174         else if (c2c.display == DISPLAY_RMT_HITM)
3175                 sort_str = "rmt_hitm";
3176         else if (c2c.display == DISPLAY_LCL_HITM)
3177                 sort_str = "lcl_hitm";
3178         else if (c2c.display == DISPLAY_SNP_PEER)
3179                 sort_str = "tot_peer";
3180
3181         c2c_hists__reinit(&c2c.hists, output_str, sort_str);
3182
3183         ui_progress__init(&prog, c2c.hists.hists.nr_entries, "Sorting...");
3184
3185         hists__collapse_resort(&c2c.hists.hists, NULL);
3186         hists__output_resort_cb(&c2c.hists.hists, &prog, resort_shared_cl_cb);
3187         hists__iterate_cb(&c2c.hists.hists, resort_cl_cb);
3188
3189         ui_progress__finish();
3190
3191         if (ui_quirks()) {
3192                 pr_err("failed to setup UI\n");
3193                 goto out_mem2node;
3194         }
3195
3196         perf_c2c_display(session);
3197
3198 out_mem2node:
3199         mem2node__exit(&c2c.mem2node);
3200 out_session:
3201         perf_session__delete(session);
3202 out:
3203         return err;
3204 }
3205
3206 static int parse_record_events(const struct option *opt,
3207                                const char *str, int unset __maybe_unused)
3208 {
3209         bool *event_set = (bool *) opt->value;
3210         struct perf_pmu *pmu;
3211
3212         pmu = perf_mem_events_find_pmu();
3213         if (!pmu) {
3214                 pr_err("failed: there is no PMU that supports perf c2c\n");
3215                 exit(-1);
3216         }
3217
3218         if (!strcmp(str, "list")) {
3219                 perf_pmu__mem_events_list(pmu);
3220                 exit(0);
3221         }
3222         if (perf_pmu__mem_events_parse(pmu, str))
3223                 exit(-1);
3224
3225         *event_set = true;
3226         return 0;
3227 }
3228
3229
3230 static const char * const __usage_record[] = {
3231         "perf c2c record [<options>] [<command>]",
3232         "perf c2c record [<options>] -- <command> [<options>]",
3233         NULL
3234 };
3235
3236 static const char * const *record_mem_usage = __usage_record;
3237
3238 static int perf_c2c__record(int argc, const char **argv)
3239 {
3240         int rec_argc, i = 0, j;
3241         const char **rec_argv;
3242         int ret;
3243         bool all_user = false, all_kernel = false;
3244         bool event_set = false;
3245         struct perf_mem_event *e;
3246         struct perf_pmu *pmu;
3247         struct option options[] = {
3248         OPT_CALLBACK('e', "event", &event_set, "event",
3249                      "event selector. Use 'perf c2c record -e list' to list available events",
3250                      parse_record_events),
3251         OPT_BOOLEAN('u', "all-user", &all_user, "collect only user level data"),
3252         OPT_BOOLEAN('k', "all-kernel", &all_kernel, "collect only kernel level data"),
3253         OPT_UINTEGER('l', "ldlat", &perf_mem_events__loads_ldlat, "setup mem-loads latency"),
3254         OPT_PARENT(c2c_options),
3255         OPT_END()
3256         };
3257
3258         pmu = perf_mem_events_find_pmu();
3259         if (!pmu) {
3260                 pr_err("failed: no PMU supports the memory events\n");
3261                 return -1;
3262         }
3263
3264         if (perf_pmu__mem_events_init()) {
3265                 pr_err("failed: memory events not supported\n");
3266                 return -1;
3267         }
3268
3269         argc = parse_options(argc, argv, options, record_mem_usage,
3270                              PARSE_OPT_KEEP_UNKNOWN);
3271
3272         /* Max number of arguments multiplied by number of PMUs that can support them. */
3273         rec_argc = argc + 11 * (perf_pmu__mem_events_num_mem_pmus(pmu) + 1);
3274
3275         rec_argv = calloc(rec_argc + 1, sizeof(char *));
3276         if (!rec_argv)
3277                 return -1;
3278
3279         rec_argv[i++] = "record";
3280
3281         if (!event_set) {
3282                 e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD_STORE);
3283                 /*
3284                  * The load and store operations are required, use the event
3285                  * PERF_MEM_EVENTS__LOAD_STORE if it is supported.
3286                  */
3287                 if (e->tag) {
3288                         perf_mem_record[PERF_MEM_EVENTS__LOAD_STORE] = true;
3289                         rec_argv[i++] = "-W";
3290                 } else {
3291                         perf_mem_record[PERF_MEM_EVENTS__LOAD] = true;
3292                         perf_mem_record[PERF_MEM_EVENTS__STORE] = true;
3293                 }
3294         }
3295
3296         if (perf_mem_record[PERF_MEM_EVENTS__LOAD])
3297                 rec_argv[i++] = "-W";
3298
3299         rec_argv[i++] = "-d";
3300         rec_argv[i++] = "--phys-data";
3301         rec_argv[i++] = "--sample-cpu";
3302
3303         ret = perf_mem_events__record_args(rec_argv, &i);
3304         if (ret)
3305                 goto out;
3306
3307         if (all_user)
3308                 rec_argv[i++] = "--all-user";
3309
3310         if (all_kernel)
3311                 rec_argv[i++] = "--all-kernel";
3312
3313         for (j = 0; j < argc; j++, i++)
3314                 rec_argv[i] = argv[j];
3315
3316         if (verbose > 0) {
3317                 pr_debug("calling: ");
3318
3319                 j = 0;
3320
3321                 while (rec_argv[j]) {
3322                         pr_debug("%s ", rec_argv[j]);
3323                         j++;
3324                 }
3325                 pr_debug("\n");
3326         }
3327
3328         ret = cmd_record(i, rec_argv);
3329 out:
3330         free(rec_argv);
3331         return ret;
3332 }
3333
3334 int cmd_c2c(int argc, const char **argv)
3335 {
3336         argc = parse_options(argc, argv, c2c_options, c2c_usage,
3337                              PARSE_OPT_STOP_AT_NON_OPTION);
3338
3339         if (!argc)
3340                 usage_with_options(c2c_usage, c2c_options);
3341
3342         if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
3343                 return perf_c2c__record(argc, argv);
3344         } else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) {
3345                 return perf_c2c__report(argc, argv);
3346         } else {
3347                 usage_with_options(c2c_usage, c2c_options);
3348         }
3349
3350         return 0;
3351 }