tools/perf/util/machine.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <dirent.h>
   3 #include <errno.h>
   4 #include <inttypes.h>
   5 #include <regex.h>
   6 #include <stdlib.h>
   7 #include "callchain.h"
   8 #include "debug.h"
   9 #include "dso.h"
  10 #include "env.h"
  11 #include "event.h"
  12 #include "evsel.h"
  13 #include "hist.h"
  14 #include "machine.h"
  15 #include "map.h"
  16 #include "map_symbol.h"
  17 #include "branch.h"
  18 #include "mem-events.h"
  19 #include "path.h"
  20 #include "srcline.h"
  21 #include "symbol.h"
  22 #include "sort.h"
  23 #include "strlist.h"
  24 #include "target.h"
  25 #include "thread.h"
  26 #include "util.h"
  27 #include "vdso.h"
  28 #include <stdbool.h>
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include "unwind.h"
  33 #include "linux/hash.h"
  34 #include "asm/bug.h"
  35 #include "bpf-event.h"
  36 #include <internal/lib.h> // page_size
  37 #include "cgroup.h"
  38 #include "arm64-frame-pointer-unwind-support.h"
  39
  40 #include <linux/ctype.h>
  41 #include <symbol/kallsyms.h>
  42 #include <linux/mman.h>
  43 #include <linux/string.h>
  44 #include <linux/zalloc.h>
  45
  46 static void __machine__remove_thread(struct machine *machine, struct thread *th, bool lock);
  47 static int append_inlines(struct callchain_cursor *cursor, struct map_symbol *ms, u64 ip);
  48
  49 static struct dso *machine__kernel_dso(struct machine *machine)
  50 {
  51         return map__dso(machine->vmlinux_map);
  52 }
  53
  54 static void dsos__init(struct dsos *dsos)
  55 {
  56         INIT_LIST_HEAD(&dsos->head);
  57         dsos->root = RB_ROOT;
  58         init_rwsem(&dsos->lock);
  59 }
  60
  61 static void machine__threads_init(struct machine *machine)
  62 {
  63         int i;
  64
  65         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
  66                 struct threads *threads = &machine->threads[i];
  67                 threads->entries = RB_ROOT_CACHED;
  68                 init_rwsem(&threads->lock);
  69                 threads->nr = 0;
  70                 INIT_LIST_HEAD(&threads->dead);
  71                 threads->last_match = NULL;
  72         }
  73 }
  74
  75 static int machine__set_mmap_name(struct machine *machine)
  76 {
  77         if (machine__is_host(machine))
  78                 machine->mmap_name = strdup("[kernel.kallsyms]");
  79         else if (machine__is_default_guest(machine))
  80                 machine->mmap_name = strdup("[guest.kernel.kallsyms]");
  81         else if (asprintf(&machine->mmap_name, "[guest.kernel.kallsyms.%d]",
  82                           machine->pid) < 0)
  83                 machine->mmap_name = NULL;
  84
  85         return machine->mmap_name ? 0 : -ENOMEM;
  86 }
  87
  88 static void thread__set_guest_comm(struct thread *thread, pid_t pid)
  89 {
  90         char comm[64];
  91
  92         snprintf(comm, sizeof(comm), "[guest/%d]", pid);
  93         thread__set_comm(thread, comm, 0);
  94 }
  95
  96 int machine__init(struct machine *machine, const char *root_dir, pid_t pid)
  97 {
  98         int err = -ENOMEM;
  99
 100         memset(machine, 0, sizeof(*machine));
 101         machine->kmaps = maps__new(machine);
 102         if (machine->kmaps == NULL)
 103                 return -ENOMEM;
 104
 105         RB_CLEAR_NODE(&machine->rb_node);
 106         dsos__init(&machine->dsos);
 107
 108         machine__threads_init(machine);
 109
 110         machine->vdso_info = NULL;
 111         machine->env = NULL;
 112
 113         machine->pid = pid;
 114
 115         machine->id_hdr_size = 0;
 116         machine->kptr_restrict_warned = false;
 117         machine->comm_exec = false;
 118         machine->kernel_start = 0;
 119         machine->vmlinux_map = NULL;
 120
 121         machine->root_dir = strdup(root_dir);
 122         if (machine->root_dir == NULL)
 123                 goto out;
 124
 125         if (machine__set_mmap_name(machine))
 126                 goto out;
 127
 128         if (pid != HOST_KERNEL_ID) {
 129                 struct thread *thread = machine__findnew_thread(machine, -1,
 130                                                                 pid);
 131
 132                 if (thread == NULL)
 133                         goto out;
 134
 135                 thread__set_guest_comm(thread, pid);
 136                 thread__put(thread);
 137         }
 138
 139         machine->current_tid = NULL;
 140         err = 0;
 141
 142 out:
 143         if (err) {
 144                 zfree(&machine->kmaps);
 145                 zfree(&machine->root_dir);
 146                 zfree(&machine->mmap_name);
 147         }
 148         return 0;
 149 }
 150
 151 struct machine *machine__new_host(void)
 152 {
 153         struct machine *machine = malloc(sizeof(*machine));
 154
 155         if (machine != NULL) {
 156                 machine__init(machine, "", HOST_KERNEL_ID);
 157
 158                 if (machine__create_kernel_maps(machine) < 0)
 159                         goto out_delete;
 160         }
 161
 162         return machine;
 163 out_delete:
 164         free(machine);
 165         return NULL;
 166 }
 167
 168 struct machine *machine__new_kallsyms(void)
 169 {
 170         struct machine *machine = machine__new_host();
 171         /*
 172          * FIXME:
 173          * 1) We should switch to machine__load_kallsyms(), i.e. not explicitly
 174          *    ask for not using the kcore parsing code, once this one is fixed
 175          *    to create a map per module.
 176          */
 177         if (machine && machine__load_kallsyms(machine, "/proc/kallsyms") <= 0) {
 178                 machine__delete(machine);
 179                 machine = NULL;
 180         }
 181
 182         return machine;
 183 }
 184
 185 static void dsos__purge(struct dsos *dsos)
 186 {
 187         struct dso *pos, *n;
 188
 189         down_write(&dsos->lock);
 190
 191         list_for_each_entry_safe(pos, n, &dsos->head, node) {
 192                 RB_CLEAR_NODE(&pos->rb_node);
 193                 pos->root = NULL;
 194                 list_del_init(&pos->node);
 195                 dso__put(pos);
 196         }
 197
 198         up_write(&dsos->lock);
 199 }
 200
 201 static void dsos__exit(struct dsos *dsos)
 202 {
 203         dsos__purge(dsos);
 204         exit_rwsem(&dsos->lock);
 205 }
 206
 207 void machine__delete_threads(struct machine *machine)
 208 {
 209         struct rb_node *nd;
 210         int i;
 211
 212         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
 213                 struct threads *threads = &machine->threads[i];
 214                 down_write(&threads->lock);
 215                 nd = rb_first_cached(&threads->entries);
 216                 while (nd) {
 217                         struct thread *t = rb_entry(nd, struct thread, rb_node);
 218
 219                         nd = rb_next(nd);
 220                         __machine__remove_thread(machine, t, false);
 221                 }
 222                 up_write(&threads->lock);
 223         }
 224 }
 225
 226 void machine__exit(struct machine *machine)
 227 {
 228         int i;
 229
 230         if (machine == NULL)
 231                 return;
 232
 233         machine__destroy_kernel_maps(machine);
 234         maps__delete(machine->kmaps);
 235         dsos__exit(&machine->dsos);
 236         machine__exit_vdso(machine);
 237         zfree(&machine->root_dir);
 238         zfree(&machine->mmap_name);
 239         zfree(&machine->current_tid);
 240         zfree(&machine->kallsyms_filename);
 241
 242         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
 243                 struct threads *threads = &machine->threads[i];
 244                 struct thread *thread, *n;
 245                 /*
 246                  * Forget about the dead, at this point whatever threads were
 247                  * left in the dead lists better have a reference count taken
 248                  * by who is using them, and then, when they drop those references
 249                  * and it finally hits zero, thread__put() will check and see that
 250                  * its not in the dead threads list and will not try to remove it
 251                  * from there, just calling thread__delete() straight away.
 252                  */
 253                 list_for_each_entry_safe(thread, n, &threads->dead, node)
 254                         list_del_init(&thread->node);
 255
 256                 exit_rwsem(&threads->lock);
 257         }
 258 }
 259
 260 void machine__delete(struct machine *machine)
 261 {
 262         if (machine) {
 263                 machine__exit(machine);
 264                 free(machine);
 265         }
 266 }
 267
 268 void machines__init(struct machines *machines)
 269 {
 270         machine__init(&machines->host, "", HOST_KERNEL_ID);
 271         machines->guests = RB_ROOT_CACHED;
 272 }
 273
 274 void machines__exit(struct machines *machines)
 275 {
 276         machine__exit(&machines->host);
 277         /* XXX exit guest */
 278 }
 279
 280 struct machine *machines__add(struct machines *machines, pid_t pid,
 281                               const char *root_dir)
 282 {
 283         struct rb_node **p = &machines->guests.rb_root.rb_node;
 284         struct rb_node *parent = NULL;
 285         struct machine *pos, *machine = malloc(sizeof(*machine));
 286         bool leftmost = true;
 287
 288         if (machine == NULL)
 289                 return NULL;
 290
 291         if (machine__init(machine, root_dir, pid) != 0) {
 292                 free(machine);
 293                 return NULL;
 294         }
 295
 296         while (*p != NULL) {
 297                 parent = *p;
 298                 pos = rb_entry(parent, struct machine, rb_node);
 299                 if (pid < pos->pid)
 300                         p = &(*p)->rb_left;
 301                 else {
 302                         p = &(*p)->rb_right;
 303                         leftmost = false;
 304                 }
 305         }
 306
 307         rb_link_node(&machine->rb_node, parent, p);
 308         rb_insert_color_cached(&machine->rb_node, &machines->guests, leftmost);
 309
 310         machine->machines = machines;
 311
 312         return machine;
 313 }
 314
 315 void machines__set_comm_exec(struct machines *machines, bool comm_exec)
 316 {
 317         struct rb_node *nd;
 318
 319         machines->host.comm_exec = comm_exec;
 320
 321         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 322                 struct machine *machine = rb_entry(nd, struct machine, rb_node);
 323
 324                 machine->comm_exec = comm_exec;
 325         }
 326 }
 327
 328 struct machine *machines__find(struct machines *machines, pid_t pid)
 329 {
 330         struct rb_node **p = &machines->guests.rb_root.rb_node;
 331         struct rb_node *parent = NULL;
 332         struct machine *machine;
 333         struct machine *default_machine = NULL;
 334
 335         if (pid == HOST_KERNEL_ID)
 336                 return &machines->host;
 337
 338         while (*p != NULL) {
 339                 parent = *p;
 340                 machine = rb_entry(parent, struct machine, rb_node);
 341                 if (pid < machine->pid)
 342                         p = &(*p)->rb_left;
 343                 else if (pid > machine->pid)
 344                         p = &(*p)->rb_right;
 345                 else
 346                         return machine;
 347                 if (!machine->pid)
 348                         default_machine = machine;
 349         }
 350
 351         return default_machine;
 352 }
 353
 354 struct machine *machines__findnew(struct machines *machines, pid_t pid)
 355 {
 356         char path[PATH_MAX];
 357         const char *root_dir = "";
 358         struct machine *machine = machines__find(machines, pid);
 359
 360         if (machine && (machine->pid == pid))
 361                 goto out;
 362
 363         if ((pid != HOST_KERNEL_ID) &&
 364             (pid != DEFAULT_GUEST_KERNEL_ID) &&
 365             (symbol_conf.guestmount)) {
 366                 sprintf(path, "%s/%d", symbol_conf.guestmount, pid);
 367                 if (access(path, R_OK)) {
 368                         static struct strlist *seen;
 369
 370                         if (!seen)
 371                                 seen = strlist__new(NULL, NULL);
 372
 373                         if (!strlist__has_entry(seen, path)) {
 374                                 pr_err("Can't access file %s\n", path);
 375                                 strlist__add(seen, path);
 376                         }
 377                         machine = NULL;
 378                         goto out;
 379                 }
 380                 root_dir = path;
 381         }
 382
 383         machine = machines__add(machines, pid, root_dir);
 384 out:
 385         return machine;
 386 }
 387
 388 struct machine *machines__find_guest(struct machines *machines, pid_t pid)
 389 {
 390         struct machine *machine = machines__find(machines, pid);
 391
 392         if (!machine)
 393                 machine = machines__findnew(machines, DEFAULT_GUEST_KERNEL_ID);
 394         return machine;
 395 }
 396
 397 /*
 398  * A common case for KVM test programs is that the test program acts as the
 399  * hypervisor, creating, running and destroying the virtual machine, and
 400  * providing the guest object code from its own object code. In this case,
 401  * the VM is not running an OS, but only the functions loaded into it by the
 402  * hypervisor test program, and conveniently, loaded at the same virtual
 403  * addresses.
 404  *
 405  * Normally to resolve addresses, MMAP events are needed to map addresses
 406  * back to the object code and debug symbols for that object code.
 407  *
 408  * Currently, there is no way to get such mapping information from guests
 409  * but, in the scenario described above, the guest has the same mappings
 410  * as the hypervisor, so support for that scenario can be achieved.
 411  *
 412  * To support that, copy the host thread's maps to the guest thread's maps.
 413  * Note, we do not discover the guest until we encounter a guest event,
 414  * which works well because it is not until then that we know that the host
 415  * thread's maps have been set up.
 416  *
 417  * This function returns the guest thread. Apart from keeping the data
 418  * structures sane, using a thread belonging to the guest machine, instead
 419  * of the host thread, allows it to have its own comm (refer
 420  * thread__set_guest_comm()).
 421  */
 422 static struct thread *findnew_guest_code(struct machine *machine,
 423                                          struct machine *host_machine,
 424                                          pid_t pid)
 425 {
 426         struct thread *host_thread;
 427         struct thread *thread;
 428         int err;
 429
 430         if (!machine)
 431                 return NULL;
 432
 433         thread = machine__findnew_thread(machine, -1, pid);
 434         if (!thread)
 435                 return NULL;
 436
 437         /* Assume maps are set up if there are any */
 438         if (maps__nr_maps(thread->maps))
 439                 return thread;
 440
 441         host_thread = machine__find_thread(host_machine, -1, pid);
 442         if (!host_thread)
 443                 goto out_err;
 444
 445         thread__set_guest_comm(thread, pid);
 446
 447         /*
 448          * Guest code can be found in hypervisor process at the same address
 449          * so copy host maps.
 450          */
 451         err = maps__clone(thread, host_thread->maps);
 452         thread__put(host_thread);
 453         if (err)
 454                 goto out_err;
 455
 456         return thread;
 457
 458 out_err:
 459         thread__zput(thread);
 460         return NULL;
 461 }
 462
 463 struct thread *machines__findnew_guest_code(struct machines *machines, pid_t pid)
 464 {
 465         struct machine *host_machine = machines__find(machines, HOST_KERNEL_ID);
 466         struct machine *machine = machines__findnew(machines, pid);
 467
 468         return findnew_guest_code(machine, host_machine, pid);
 469 }
 470
 471 struct thread *machine__findnew_guest_code(struct machine *machine, pid_t pid)
 472 {
 473         struct machines *machines = machine->machines;
 474         struct machine *host_machine;
 475
 476         if (!machines)
 477                 return NULL;
 478
 479         host_machine = machines__find(machines, HOST_KERNEL_ID);
 480
 481         return findnew_guest_code(machine, host_machine, pid);
 482 }
 483
 484 void machines__process_guests(struct machines *machines,
 485                               machine__process_t process, void *data)
 486 {
 487         struct rb_node *nd;
 488
 489         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 490                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
 491                 process(pos, data);
 492         }
 493 }
 494
 495 void machines__set_id_hdr_size(struct machines *machines, u16 id_hdr_size)
 496 {
 497         struct rb_node *node;
 498         struct machine *machine;
 499
 500         machines->host.id_hdr_size = id_hdr_size;
 501
 502         for (node = rb_first_cached(&machines->guests); node;
 503              node = rb_next(node)) {
 504                 machine = rb_entry(node, struct machine, rb_node);
 505                 machine->id_hdr_size = id_hdr_size;
 506         }
 507
 508         return;
 509 }
 510
 511 static void machine__update_thread_pid(struct machine *machine,
 512                                        struct thread *th, pid_t pid)
 513 {
 514         struct thread *leader;
 515
 516         if (pid == th->pid_ || pid == -1 || th->pid_ != -1)
 517                 return;
 518
 519         th->pid_ = pid;
 520
 521         if (th->pid_ == th->tid)
 522                 return;
 523
 524         leader = __machine__findnew_thread(machine, th->pid_, th->pid_);
 525         if (!leader)
 526                 goto out_err;
 527
 528         if (!leader->maps)
 529                 leader->maps = maps__new(machine);
 530
 531         if (!leader->maps)
 532                 goto out_err;
 533
 534         if (th->maps == leader->maps)
 535                 return;
 536
 537         if (th->maps) {
 538                 /*
 539                  * Maps are created from MMAP events which provide the pid and
 540                  * tid.  Consequently there never should be any maps on a thread
 541                  * with an unknown pid.  Just print an error if there are.
 542                  */
 543                 if (!maps__empty(th->maps))
 544                         pr_err("Discarding thread maps for %d:%d\n",
 545                                th->pid_, th->tid);
 546                 maps__put(th->maps);
 547         }
 548
 549         th->maps = maps__get(leader->maps);
 550 out_put:
 551         thread__put(leader);
 552         return;
 553 out_err:
 554         pr_err("Failed to join map groups for %d:%d\n", th->pid_, th->tid);
 555         goto out_put;
 556 }
 557
 558 /*
 559  * Front-end cache - TID lookups come in blocks,
 560  * so most of the time we dont have to look up
 561  * the full rbtree:
 562  */
 563 static struct thread*
 564 __threads__get_last_match(struct threads *threads, struct machine *machine,
 565                           int pid, int tid)
 566 {
 567         struct thread *th;
 568
 569         th = threads->last_match;
 570         if (th != NULL) {
 571                 if (th->tid == tid) {
 572                         machine__update_thread_pid(machine, th, pid);
 573                         return thread__get(th);
 574                 }
 575
 576                 threads->last_match = NULL;
 577         }
 578
 579         return NULL;
 580 }
 581
 582 static struct thread*
 583 threads__get_last_match(struct threads *threads, struct machine *machine,
 584                         int pid, int tid)
 585 {
 586         struct thread *th = NULL;
 587
 588         if (perf_singlethreaded)
 589                 th = __threads__get_last_match(threads, machine, pid, tid);
 590
 591         return th;
 592 }
 593
 594 static void
 595 __threads__set_last_match(struct threads *threads, struct thread *th)
 596 {
 597         threads->last_match = th;
 598 }
 599
 600 static void
 601 threads__set_last_match(struct threads *threads, struct thread *th)
 602 {
 603         if (perf_singlethreaded)
 604                 __threads__set_last_match(threads, th);
 605 }
 606
 607 /*
 608  * Caller must eventually drop thread->refcnt returned with a successful
 609  * lookup/new thread inserted.
 610  */
 611 static struct thread *____machine__findnew_thread(struct machine *machine,
 612                                                   struct threads *threads,
 613                                                   pid_t pid, pid_t tid,
 614                                                   bool create)
 615 {
 616         struct rb_node **p = &threads->entries.rb_root.rb_node;
 617         struct rb_node *parent = NULL;
 618         struct thread *th;
 619         bool leftmost = true;
 620
 621         th = threads__get_last_match(threads, machine, pid, tid);
 622         if (th)
 623                 return th;
 624
 625         while (*p != NULL) {
 626                 parent = *p;
 627                 th = rb_entry(parent, struct thread, rb_node);
 628
 629                 if (th->tid == tid) {
 630                         threads__set_last_match(threads, th);
 631                         machine__update_thread_pid(machine, th, pid);
 632                         return thread__get(th);
 633                 }
 634
 635                 if (tid < th->tid)
 636                         p = &(*p)->rb_left;
 637                 else {
 638                         p = &(*p)->rb_right;
 639                         leftmost = false;
 640                 }
 641         }
 642
 643         if (!create)
 644                 return NULL;
 645
 646         th = thread__new(pid, tid);
 647         if (th != NULL) {
 648                 rb_link_node(&th->rb_node, parent, p);
 649                 rb_insert_color_cached(&th->rb_node, &threads->entries, leftmost);
 650
 651                 /*
 652                  * We have to initialize maps separately after rb tree is updated.
 653                  *
 654                  * The reason is that we call machine__findnew_thread
 655                  * within thread__init_maps to find the thread
 656                  * leader and that would screwed the rb tree.
 657                  */
 658                 if (thread__init_maps(th, machine)) {
 659                         rb_erase_cached(&th->rb_node, &threads->entries);
 660                         RB_CLEAR_NODE(&th->rb_node);
 661                         thread__put(th);
 662                         return NULL;
 663                 }
 664                 /*
 665                  * It is now in the rbtree, get a ref
 666                  */
 667                 thread__get(th);
 668                 threads__set_last_match(threads, th);
 669                 ++threads->nr;
 670         }
 671
 672         return th;
 673 }
 674
 675 struct thread *__machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid)
 676 {
 677         return ____machine__findnew_thread(machine, machine__threads(machine, tid), pid, tid, true);
 678 }
 679
 680 struct thread *machine__findnew_thread(struct machine *machine, pid_t pid,
 681                                        pid_t tid)
 682 {
 683         struct threads *threads = machine__threads(machine, tid);
 684         struct thread *th;
 685
 686         down_write(&threads->lock);
 687         th = __machine__findnew_thread(machine, pid, tid);
 688         up_write(&threads->lock);
 689         return th;
 690 }
 691
 692 struct thread *machine__find_thread(struct machine *machine, pid_t pid,
 693                                     pid_t tid)
 694 {
 695         struct threads *threads = machine__threads(machine, tid);
 696         struct thread *th;
 697
 698         down_read(&threads->lock);
 699         th =  ____machine__findnew_thread(machine, threads, pid, tid, false);
 700         up_read(&threads->lock);
 701         return th;
 702 }
 703
 704 /*
 705  * Threads are identified by pid and tid, and the idle task has pid == tid == 0.
 706  * So here a single thread is created for that, but actually there is a separate
 707  * idle task per cpu, so there should be one 'struct thread' per cpu, but there
 708  * is only 1. That causes problems for some tools, requiring workarounds. For
 709  * example get_idle_thread() in builtin-sched.c, or thread_stack__per_cpu().
 710  */
 711 struct thread *machine__idle_thread(struct machine *machine)
 712 {
 713         struct thread *thread = machine__findnew_thread(machine, 0, 0);
 714
 715         if (!thread || thread__set_comm(thread, "swapper", 0) ||
 716             thread__set_namespaces(thread, 0, NULL))
 717                 pr_err("problem inserting idle task for machine pid %d\n", machine->pid);
 718
 719         return thread;
 720 }
 721
 722 struct comm *machine__thread_exec_comm(struct machine *machine,
 723                                        struct thread *thread)
 724 {
 725         if (machine->comm_exec)
 726                 return thread__exec_comm(thread);
 727         else
 728                 return thread__comm(thread);
 729 }
 730
 731 int machine__process_comm_event(struct machine *machine, union perf_event *event,
 732                                 struct perf_sample *sample)
 733 {
 734         struct thread *thread = machine__findnew_thread(machine,
 735                                                         event->comm.pid,
 736                                                         event->comm.tid);
 737         bool exec = event->header.misc & PERF_RECORD_MISC_COMM_EXEC;
 738         int err = 0;
 739
 740         if (exec)
 741                 machine->comm_exec = true;
 742
 743         if (dump_trace)
 744                 perf_event__fprintf_comm(event, stdout);
 745
 746         if (thread == NULL ||
 747             __thread__set_comm(thread, event->comm.comm, sample->time, exec)) {
 748                 dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n");
 749                 err = -1;
 750         }
 751
 752         thread__put(thread);
 753
 754         return err;
 755 }
 756
 757 int machine__process_namespaces_event(struct machine *machine __maybe_unused,
 758                                       union perf_event *event,
 759                                       struct perf_sample *sample __maybe_unused)
 760 {
 761         struct thread *thread = machine__findnew_thread(machine,
 762                                                         event->namespaces.pid,
 763                                                         event->namespaces.tid);
 764         int err = 0;
 765
 766         WARN_ONCE(event->namespaces.nr_namespaces > NR_NAMESPACES,
 767                   "\nWARNING: kernel seems to support more namespaces than perf"
 768                   " tool.\nTry updating the perf tool..\n\n");
 769
 770         WARN_ONCE(event->namespaces.nr_namespaces < NR_NAMESPACES,
 771                   "\nWARNING: perf tool seems to support more namespaces than"
 772                   " the kernel.\nTry updating the kernel..\n\n");
 773
 774         if (dump_trace)
 775                 perf_event__fprintf_namespaces(event, stdout);
 776
 777         if (thread == NULL ||
 778             thread__set_namespaces(thread, sample->time, &event->namespaces)) {
 779                 dump_printf("problem processing PERF_RECORD_NAMESPACES, skipping event.\n");
 780                 err = -1;
 781         }
 782
 783         thread__put(thread);
 784
 785         return err;
 786 }
 787
 788 int machine__process_cgroup_event(struct machine *machine,
 789                                   union perf_event *event,
 790                                   struct perf_sample *sample __maybe_unused)
 791 {
 792         struct cgroup *cgrp;
 793
 794         if (dump_trace)
 795                 perf_event__fprintf_cgroup(event, stdout);
 796
 797         cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path);
 798         if (cgrp == NULL)
 799                 return -ENOMEM;
 800
 801         return 0;
 802 }
 803
 804 int machine__process_lost_event(struct machine *machine __maybe_unused,
 805                                 union perf_event *event, struct perf_sample *sample __maybe_unused)
 806 {
 807         dump_printf(": id:%" PRI_lu64 ": lost:%" PRI_lu64 "\n",
 808                     event->lost.id, event->lost.lost);
 809         return 0;
 810 }
 811
 812 int machine__process_lost_samples_event(struct machine *machine __maybe_unused,
 813                                         union perf_event *event, struct perf_sample *sample)
 814 {
 815         dump_printf(": id:%" PRIu64 ": lost samples :%" PRI_lu64 "\n",
 816                     sample->id, event->lost_samples.lost);
 817         return 0;
 818 }
 819
 820 static struct dso *machine__findnew_module_dso(struct machine *machine,
 821                                                struct kmod_path *m,
 822                                                const char *filename)
 823 {
 824         struct dso *dso;
 825
 826         down_write(&machine->dsos.lock);
 827
 828         dso = __dsos__find(&machine->dsos, m->name, true);
 829         if (!dso) {
 830                 dso = __dsos__addnew(&machine->dsos, m->name);
 831                 if (dso == NULL)
 832                         goto out_unlock;
 833
 834                 dso__set_module_info(dso, m, machine);
 835                 dso__set_long_name(dso, strdup(filename), true);
 836                 dso->kernel = DSO_SPACE__KERNEL;
 837         }
 838
 839         dso__get(dso);
 840 out_unlock:
 841         up_write(&machine->dsos.lock);
 842         return dso;
 843 }
 844
 845 int machine__process_aux_event(struct machine *machine __maybe_unused,
 846                                union perf_event *event)
 847 {
 848         if (dump_trace)
 849                 perf_event__fprintf_aux(event, stdout);
 850         return 0;
 851 }
 852
 853 int machine__process_itrace_start_event(struct machine *machine __maybe_unused,
 854                                         union perf_event *event)
 855 {
 856         if (dump_trace)
 857                 perf_event__fprintf_itrace_start(event, stdout);
 858         return 0;
 859 }
 860
 861 int machine__process_aux_output_hw_id_event(struct machine *machine __maybe_unused,
 862                                             union perf_event *event)
 863 {
 864         if (dump_trace)
 865                 perf_event__fprintf_aux_output_hw_id(event, stdout);
 866         return 0;
 867 }
 868
 869 int machine__process_switch_event(struct machine *machine __maybe_unused,
 870                                   union perf_event *event)
 871 {
 872         if (dump_trace)
 873                 perf_event__fprintf_switch(event, stdout);
 874         return 0;
 875 }
 876
 877 static int machine__process_ksymbol_register(struct machine *machine,
 878                                              union perf_event *event,
 879                                              struct perf_sample *sample __maybe_unused)
 880 {
 881         struct symbol *sym;
 882         struct dso *dso;
 883         struct map *map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr);
 884         bool put_map = false;
 885         int err = 0;
 886
 887         if (!map) {
 888                 dso = dso__new(event->ksymbol.name);
 889
 890                 if (!dso) {
 891                         err = -ENOMEM;
 892                         goto out;
 893                 }
 894                 dso->kernel = DSO_SPACE__KERNEL;
 895                 map = map__new2(0, dso);
 896                 dso__put(dso);
 897                 if (!map) {
 898                         err = -ENOMEM;
 899                         goto out;
 900                 }
 901                 /*
 902                  * The inserted map has a get on it, we need to put to release
 903                  * the reference count here, but do it after all accesses are
 904                  * done.
 905                  */
 906                 put_map = true;
 907                 if (event->ksymbol.ksym_type == PERF_RECORD_KSYMBOL_TYPE_OOL) {
 908                         dso->binary_type = DSO_BINARY_TYPE__OOL;
 909                         dso->data.file_size = event->ksymbol.len;
 910                         dso__set_loaded(dso);
 911                 }
 912
 913                 map__set_start(map, event->ksymbol.addr);
 914                 map__set_end(map, map__start(map) + event->ksymbol.len);
 915                 err = maps__insert(machine__kernel_maps(machine), map);
 916                 if (err) {
 917                         err = -ENOMEM;
 918                         goto out;
 919                 }
 920
 921                 dso__set_loaded(dso);
 922
 923                 if (is_bpf_image(event->ksymbol.name)) {
 924                         dso->binary_type = DSO_BINARY_TYPE__BPF_IMAGE;
 925                         dso__set_long_name(dso, "", false);
 926                 }
 927         } else {
 928                 dso = map__dso(map);
 929         }
 930
 931         sym = symbol__new(map__map_ip(map, map__start(map)),
 932                           event->ksymbol.len,
 933                           0, 0, event->ksymbol.name);
 934         if (!sym) {
 935                 err = -ENOMEM;
 936                 goto out;
 937         }
 938         dso__insert_symbol(dso, sym);
 939 out:
 940         if (put_map)
 941                 map__put(map);
 942         return err;
 943 }
 944
 945 static int machine__process_ksymbol_unregister(struct machine *machine,
 946                                                union perf_event *event,
 947                                                struct perf_sample *sample __maybe_unused)
 948 {
 949         struct symbol *sym;
 950         struct map *map;
 951
 952         map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr);
 953         if (!map)
 954                 return 0;
 955
 956         if (RC_CHK_ACCESS(map) != RC_CHK_ACCESS(machine->vmlinux_map))
 957                 maps__remove(machine__kernel_maps(machine), map);
 958         else {
 959                 struct dso *dso = map__dso(map);
 960
 961                 sym = dso__find_symbol(dso, map__map_ip(map, map__start(map)));
 962                 if (sym)
 963                         dso__delete_symbol(dso, sym);
 964         }
 965
 966         return 0;
 967 }
 968
 969 int machine__process_ksymbol(struct machine *machine __maybe_unused,
 970                              union perf_event *event,
 971                              struct perf_sample *sample)
 972 {
 973         if (dump_trace)
 974                 perf_event__fprintf_ksymbol(event, stdout);
 975
 976         if (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER)
 977                 return machine__process_ksymbol_unregister(machine, event,
 978                                                            sample);
 979         return machine__process_ksymbol_register(machine, event, sample);
 980 }
 981
 982 int machine__process_text_poke(struct machine *machine, union perf_event *event,
 983                                struct perf_sample *sample __maybe_unused)
 984 {
 985         struct map *map = maps__find(machine__kernel_maps(machine), event->text_poke.addr);
 986         u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
 987         struct dso *dso = map ? map__dso(map) : NULL;
 988
 989         if (dump_trace)
 990                 perf_event__fprintf_text_poke(event, machine, stdout);
 991
 992         if (!event->text_poke.new_len)
 993                 return 0;
 994
 995         if (cpumode != PERF_RECORD_MISC_KERNEL) {
 996                 pr_debug("%s: unsupported cpumode - ignoring\n", __func__);
 997                 return 0;
 998         }
 999
1000         if (dso) {
1001                 u8 *new_bytes = event->text_poke.bytes + event->text_poke.old_len;
1002                 int ret;
1003
1004                 /*
1005                  * Kernel maps might be changed when loading symbols so loading
1006                  * must be done prior to using kernel maps.
1007                  */
1008                 map__load(map);
1009                 ret = dso__data_write_cache_addr(dso, map, machine,
1010                                                  event->text_poke.addr,
1011                                                  new_bytes,
1012                                                  event->text_poke.new_len);
1013                 if (ret != event->text_poke.new_len)
1014                         pr_debug("Failed to write kernel text poke at %#" PRI_lx64 "\n",
1015                                  event->text_poke.addr);
1016         } else {
1017                 pr_debug("Failed to find kernel text poke address map for %#" PRI_lx64 "\n",
1018                          event->text_poke.addr);
1019         }
1020
1021         return 0;
1022 }
1023
1024 static struct map *machine__addnew_module_map(struct machine *machine, u64 start,
1025                                               const char *filename)
1026 {
1027         struct map *map = NULL;
1028         struct kmod_path m;
1029         struct dso *dso;
1030         int err;
1031
1032         if (kmod_path__parse_name(&m, filename))
1033                 return NULL;
1034
1035         dso = machine__findnew_module_dso(machine, &m, filename);
1036         if (dso == NULL)
1037                 goto out;
1038
1039         map = map__new2(start, dso);
1040         if (map == NULL)
1041                 goto out;
1042
1043         err = maps__insert(machine__kernel_maps(machine), map);
1044         /* If maps__insert failed, return NULL. */
1045         if (err) {
1046                 map__put(map);
1047                 map = NULL;
1048         }
1049 out:
1050         /* put the dso here, corresponding to  machine__findnew_module_dso */
1051         dso__put(dso);
1052         zfree(&m.name);
1053         return map;
1054 }
1055
1056 size_t machines__fprintf_dsos(struct machines *machines, FILE *fp)
1057 {
1058         struct rb_node *nd;
1059         size_t ret = __dsos__fprintf(&machines->host.dsos.head, fp);
1060
1061         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
1062                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
1063                 ret += __dsos__fprintf(&pos->dsos.head, fp);
1064         }
1065
1066         return ret;
1067 }
1068
1069 size_t machine__fprintf_dsos_buildid(struct machine *m, FILE *fp,
1070                                      bool (skip)(struct dso *dso, int parm), int parm)
1071 {
1072         return __dsos__fprintf_buildid(&m->dsos.head, fp, skip, parm);
1073 }
1074
1075 size_t machines__fprintf_dsos_buildid(struct machines *machines, FILE *fp,
1076                                      bool (skip)(struct dso *dso, int parm), int parm)
1077 {
1078         struct rb_node *nd;
1079         size_t ret = machine__fprintf_dsos_buildid(&machines->host, fp, skip, parm);
1080
1081         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
1082                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
1083                 ret += machine__fprintf_dsos_buildid(pos, fp, skip, parm);
1084         }
1085         return ret;
1086 }
1087
1088 size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp)
1089 {
1090         int i;
1091         size_t printed = 0;
1092         struct dso *kdso = machine__kernel_dso(machine);
1093
1094         if (kdso->has_build_id) {
1095                 char filename[PATH_MAX];
1096                 if (dso__build_id_filename(kdso, filename, sizeof(filename),
1097                                            false))
1098                         printed += fprintf(fp, "[0] %s\n", filename);
1099         }
1100
1101         for (i = 0; i < vmlinux_path__nr_entries; ++i)
1102                 printed += fprintf(fp, "[%d] %s\n",
1103                                    i + kdso->has_build_id, vmlinux_path[i]);
1104
1105         return printed;
1106 }
1107
1108 size_t machine__fprintf(struct machine *machine, FILE *fp)
1109 {
1110         struct rb_node *nd;
1111         size_t ret;
1112         int i;
1113
1114         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
1115                 struct threads *threads = &machine->threads[i];
1116
1117                 down_read(&threads->lock);
1118
1119                 ret = fprintf(fp, "Threads: %u\n", threads->nr);
1120
1121                 for (nd = rb_first_cached(&threads->entries); nd;
1122                      nd = rb_next(nd)) {
1123                         struct thread *pos = rb_entry(nd, struct thread, rb_node);
1124
1125                         ret += thread__fprintf(pos, fp);
1126                 }
1127
1128                 up_read(&threads->lock);
1129         }
1130         return ret;
1131 }
1132
1133 static struct dso *machine__get_kernel(struct machine *machine)
1134 {
1135         const char *vmlinux_name = machine->mmap_name;
1136         struct dso *kernel;
1137
1138         if (machine__is_host(machine)) {
1139                 if (symbol_conf.vmlinux_name)
1140                         vmlinux_name = symbol_conf.vmlinux_name;
1141
1142                 kernel = machine__findnew_kernel(machine, vmlinux_name,
1143                                                  "[kernel]", DSO_SPACE__KERNEL);
1144         } else {
1145                 if (symbol_conf.default_guest_vmlinux_name)
1146                         vmlinux_name = symbol_conf.default_guest_vmlinux_name;
1147
1148                 kernel = machine__findnew_kernel(machine, vmlinux_name,
1149                                                  "[guest.kernel]",
1150                                                  DSO_SPACE__KERNEL_GUEST);
1151         }
1152
1153         if (kernel != NULL && (!kernel->has_build_id))
1154                 dso__read_running_kernel_build_id(kernel, machine);
1155
1156         return kernel;
1157 }
1158
1159 void machine__get_kallsyms_filename(struct machine *machine, char *buf,
1160                                     size_t bufsz)
1161 {
1162         if (machine__is_default_guest(machine))
1163                 scnprintf(buf, bufsz, "%s", symbol_conf.default_guest_kallsyms);
1164         else
1165                 scnprintf(buf, bufsz, "%s/proc/kallsyms", machine->root_dir);
1166 }
1167
1168 const char *ref_reloc_sym_names[] = {"_text", "_stext", NULL};
1169
1170 /* Figure out the start address of kernel map from /proc/kallsyms.
1171  * Returns the name of the start symbol in *symbol_name. Pass in NULL as
1172  * symbol_name if it's not that important.
1173  */
1174 static int machine__get_running_kernel_start(struct machine *machine,
1175                                              const char **symbol_name,
1176                                              u64 *start, u64 *end)
1177 {
1178         char filename[PATH_MAX];
1179         int i, err = -1;
1180         const char *name;
1181         u64 addr = 0;
1182
1183         machine__get_kallsyms_filename(machine, filename, PATH_MAX);
1184
1185         if (symbol__restricted_filename(filename, "/proc/kallsyms"))
1186                 return 0;
1187
1188         for (i = 0; (name = ref_reloc_sym_names[i]) != NULL; i++) {
1189                 err = kallsyms__get_function_start(filename, name, &addr);
1190                 if (!err)
1191                         break;
1192         }
1193
1194         if (err)
1195                 return -1;
1196
1197         if (symbol_name)
1198                 *symbol_name = name;
1199
1200         *start = addr;
1201
1202         err = kallsyms__get_function_start(filename, "_etext", &addr);
1203         if (!err)
1204                 *end = addr;
1205
1206         return 0;
1207 }
1208
1209 int machine__create_extra_kernel_map(struct machine *machine,
1210                                      struct dso *kernel,
1211                                      struct extra_kernel_map *xm)
1212 {
1213         struct kmap *kmap;
1214         struct map *map;
1215         int err;
1216
1217         map = map__new2(xm->start, kernel);
1218         if (!map)
1219                 return -ENOMEM;
1220
1221         map__set_end(map, xm->end);
1222         map__set_pgoff(map, xm->pgoff);
1223
1224         kmap = map__kmap(map);
1225
1226         strlcpy(kmap->name, xm->name, KMAP_NAME_LEN);
1227
1228         err = maps__insert(machine__kernel_maps(machine), map);
1229
1230         if (!err) {
1231                 pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n",
1232                         kmap->name, map__start(map), map__end(map));
1233         }
1234
1235         map__put(map);
1236
1237         return err;
1238 }
1239
1240 static u64 find_entry_trampoline(struct dso *dso)
1241 {
1242         /* Duplicates are removed so lookup all aliases */
1243         const char *syms[] = {
1244                 "_entry_trampoline",
1245                 "__entry_trampoline_start",
1246                 "entry_SYSCALL_64_trampoline",
1247         };
1248         struct symbol *sym = dso__first_symbol(dso);
1249         unsigned int i;
1250
1251         for (; sym; sym = dso__next_symbol(sym)) {
1252                 if (sym->binding != STB_GLOBAL)
1253                         continue;
1254                 for (i = 0; i < ARRAY_SIZE(syms); i++) {
1255                         if (!strcmp(sym->name, syms[i]))
1256                                 return sym->start;
1257                 }
1258         }
1259
1260         return 0;
1261 }
1262
1263 /*
1264  * These values can be used for kernels that do not have symbols for the entry
1265  * trampolines in kallsyms.
1266  */
1267 #define X86_64_CPU_ENTRY_AREA_PER_CPU   0xfffffe0000000000ULL
1268 #define X86_64_CPU_ENTRY_AREA_SIZE      0x2c000
1269 #define X86_64_ENTRY_TRAMPOLINE         0x6000
1270
1271 /* Map x86_64 PTI entry trampolines */
1272 int machine__map_x86_64_entry_trampolines(struct machine *machine,
1273                                           struct dso *kernel)
1274 {
1275         struct maps *kmaps = machine__kernel_maps(machine);
1276         int nr_cpus_avail, cpu;
1277         bool found = false;
1278         struct map_rb_node *rb_node;
1279         u64 pgoff;
1280
1281         /*
1282          * In the vmlinux case, pgoff is a virtual address which must now be
1283          * mapped to a vmlinux offset.
1284          */
1285         maps__for_each_entry(kmaps, rb_node) {
1286                 struct map *dest_map, *map = rb_node->map;
1287                 struct kmap *kmap = __map__kmap(map);
1288
1289                 if (!kmap || !is_entry_trampoline(kmap->name))
1290                         continue;
1291
1292                 dest_map = maps__find(kmaps, map__pgoff(map));
1293                 if (dest_map != map)
1294                         map__set_pgoff(map, map__map_ip(dest_map, map__pgoff(map)));
1295                 found = true;
1296         }
1297         if (found || machine->trampolines_mapped)
1298                 return 0;
1299
1300         pgoff = find_entry_trampoline(kernel);
1301         if (!pgoff)
1302                 return 0;
1303
1304         nr_cpus_avail = machine__nr_cpus_avail(machine);
1305
1306         /* Add a 1 page map for each CPU's entry trampoline */
1307         for (cpu = 0; cpu < nr_cpus_avail; cpu++) {
1308                 u64 va = X86_64_CPU_ENTRY_AREA_PER_CPU +
1309                          cpu * X86_64_CPU_ENTRY_AREA_SIZE +
1310                          X86_64_ENTRY_TRAMPOLINE;
1311                 struct extra_kernel_map xm = {
1312                         .start = va,
1313                         .end   = va + page_size,
1314                         .pgoff = pgoff,
1315                 };
1316
1317                 strlcpy(xm.name, ENTRY_TRAMPOLINE_NAME, KMAP_NAME_LEN);
1318
1319                 if (machine__create_extra_kernel_map(machine, kernel, &xm) < 0)
1320                         return -1;
1321         }
1322
1323         machine->trampolines_mapped = nr_cpus_avail;
1324
1325         return 0;
1326 }
1327
1328 int __weak machine__create_extra_kernel_maps(struct machine *machine __maybe_unused,
1329                                              struct dso *kernel __maybe_unused)
1330 {
1331         return 0;
1332 }
1333
1334 static int
1335 __machine__create_kernel_maps(struct machine *machine, struct dso *kernel)
1336 {
1337         /* In case of renewal the kernel map, destroy previous one */
1338         machine__destroy_kernel_maps(machine);
1339
1340         map__put(machine->vmlinux_map);
1341         machine->vmlinux_map = map__new2(0, kernel);
1342         if (machine->vmlinux_map == NULL)
1343                 return -ENOMEM;
1344
1345         map__set_map_ip(machine->vmlinux_map, identity__map_ip);
1346         map__set_unmap_ip(machine->vmlinux_map, identity__map_ip);
1347         return maps__insert(machine__kernel_maps(machine), machine->vmlinux_map);
1348 }
1349
1350 void machine__destroy_kernel_maps(struct machine *machine)
1351 {
1352         struct kmap *kmap;
1353         struct map *map = machine__kernel_map(machine);
1354
1355         if (map == NULL)
1356                 return;
1357
1358         kmap = map__kmap(map);
1359         maps__remove(machine__kernel_maps(machine), map);
1360         if (kmap && kmap->ref_reloc_sym) {
1361                 zfree((char **)&kmap->ref_reloc_sym->name);
1362                 zfree(&kmap->ref_reloc_sym);
1363         }
1364
1365         map__zput(machine->vmlinux_map);
1366 }
1367
1368 int machines__create_guest_kernel_maps(struct machines *machines)
1369 {
1370         int ret = 0;
1371         struct dirent **namelist = NULL;
1372         int i, items = 0;
1373         char path[PATH_MAX];
1374         pid_t pid;
1375         char *endp;
1376
1377         if (symbol_conf.default_guest_vmlinux_name ||
1378             symbol_conf.default_guest_modules ||
1379             symbol_conf.default_guest_kallsyms) {
1380                 machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID);
1381         }
1382
1383         if (symbol_conf.guestmount) {
1384                 items = scandir(symbol_conf.guestmount, &namelist, NULL, NULL);
1385                 if (items <= 0)
1386                         return -ENOENT;
1387                 for (i = 0; i < items; i++) {
1388                         if (!isdigit(namelist[i]->d_name[0])) {
1389                                 /* Filter out . and .. */
1390                                 continue;
1391                         }
1392                         pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10);
1393                         if ((*endp != '\0') ||
1394                             (endp == namelist[i]->d_name) ||
1395                             (errno == ERANGE)) {
1396                                 pr_debug("invalid directory (%s). Skipping.\n",
1397                                          namelist[i]->d_name);
1398                                 continue;
1399                         }
1400                         sprintf(path, "%s/%s/proc/kallsyms",
1401                                 symbol_conf.guestmount,
1402                                 namelist[i]->d_name);
1403                         ret = access(path, R_OK);
1404                         if (ret) {
1405                                 pr_debug("Can't access file %s\n", path);
1406                                 goto failure;
1407                         }
1408                         machines__create_kernel_maps(machines, pid);
1409                 }
1410 failure:
1411                 free(namelist);
1412         }
1413
1414         return ret;
1415 }
1416
1417 void machines__destroy_kernel_maps(struct machines *machines)
1418 {
1419         struct rb_node *next = rb_first_cached(&machines->guests);
1420
1421         machine__destroy_kernel_maps(&machines->host);
1422
1423         while (next) {
1424                 struct machine *pos = rb_entry(next, struct machine, rb_node);
1425
1426                 next = rb_next(&pos->rb_node);
1427                 rb_erase_cached(&pos->rb_node, &machines->guests);
1428                 machine__delete(pos);
1429         }
1430 }
1431
1432 int machines__create_kernel_maps(struct machines *machines, pid_t pid)
1433 {
1434         struct machine *machine = machines__findnew(machines, pid);
1435
1436         if (machine == NULL)
1437                 return -1;
1438
1439         return machine__create_kernel_maps(machine);
1440 }
1441
1442 int machine__load_kallsyms(struct machine *machine, const char *filename)
1443 {
1444         struct map *map = machine__kernel_map(machine);
1445         struct dso *dso = map__dso(map);
1446         int ret = __dso__load_kallsyms(dso, filename, map, true);
1447
1448         if (ret > 0) {
1449                 dso__set_loaded(dso);
1450                 /*
1451                  * Since /proc/kallsyms will have multiple sessions for the
1452                  * kernel, with modules between them, fixup the end of all
1453                  * sections.
1454                  */
1455                 maps__fixup_end(machine__kernel_maps(machine));
1456         }
1457
1458         return ret;
1459 }
1460
1461 int machine__load_vmlinux_path(struct machine *machine)
1462 {
1463         struct map *map = machine__kernel_map(machine);
1464         struct dso *dso = map__dso(map);
1465         int ret = dso__load_vmlinux_path(dso, map);
1466
1467         if (ret > 0)
1468                 dso__set_loaded(dso);
1469
1470         return ret;
1471 }
1472
1473 static char *get_kernel_version(const char *root_dir)
1474 {
1475         char version[PATH_MAX];
1476         FILE *file;
1477         char *name, *tmp;
1478         const char *prefix = "Linux version ";
1479
1480         sprintf(version, "%s/proc/version", root_dir);
1481         file = fopen(version, "r");
1482         if (!file)
1483                 return NULL;
1484
1485         tmp = fgets(version, sizeof(version), file);
1486         fclose(file);
1487         if (!tmp)
1488                 return NULL;
1489
1490         name = strstr(version, prefix);
1491         if (!name)
1492                 return NULL;
1493         name += strlen(prefix);
1494         tmp = strchr(name, ' ');
1495         if (tmp)
1496                 *tmp = '\0';
1497
1498         return strdup(name);
1499 }
1500
1501 static bool is_kmod_dso(struct dso *dso)
1502 {
1503         return dso->symtab_type == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE ||
1504                dso->symtab_type == DSO_BINARY_TYPE__GUEST_KMODULE;
1505 }
1506
1507 static int maps__set_module_path(struct maps *maps, const char *path, struct kmod_path *m)
1508 {
1509         char *long_name;
1510         struct dso *dso;
1511         struct map *map = maps__find_by_name(maps, m->name);
1512
1513         if (map == NULL)
1514                 return 0;
1515
1516         long_name = strdup(path);
1517         if (long_name == NULL)
1518                 return -ENOMEM;
1519
1520         dso = map__dso(map);
1521         dso__set_long_name(dso, long_name, true);
1522         dso__kernel_module_get_build_id(dso, "");
1523
1524         /*
1525          * Full name could reveal us kmod compression, so
1526          * we need to update the symtab_type if needed.
1527          */
1528         if (m->comp && is_kmod_dso(dso)) {
1529                 dso->symtab_type++;
1530                 dso->comp = m->comp;
1531         }
1532
1533         return 0;
1534 }
1535
1536 static int maps__set_modules_path_dir(struct maps *maps, const char *dir_name, int depth)
1537 {
1538         struct dirent *dent;
1539         DIR *dir = opendir(dir_name);
1540         int ret = 0;
1541
1542         if (!dir) {
1543                 pr_debug("%s: cannot open %s dir\n", __func__, dir_name);
1544                 return -1;
1545         }
1546
1547         while ((dent = readdir(dir)) != NULL) {
1548                 char path[PATH_MAX];
1549                 struct stat st;
1550
1551                 /*sshfs might return bad dent->d_type, so we have to stat*/
1552                 path__join(path, sizeof(path), dir_name, dent->d_name);
1553                 if (stat(path, &st))
1554                         continue;
1555
1556                 if (S_ISDIR(st.st_mode)) {
1557                         if (!strcmp(dent->d_name, ".") ||
1558                             !strcmp(dent->d_name, ".."))
1559                                 continue;
1560
1561                         /* Do not follow top-level source and build symlinks */
1562                         if (depth == 0) {
1563                                 if (!strcmp(dent->d_name, "source") ||
1564                                     !strcmp(dent->d_name, "build"))
1565                                         continue;
1566                         }
1567
1568                         ret = maps__set_modules_path_dir(maps, path, depth + 1);
1569                         if (ret < 0)
1570                                 goto out;
1571                 } else {
1572                         struct kmod_path m;
1573
1574                         ret = kmod_path__parse_name(&m, dent->d_name);
1575                         if (ret)
1576                                 goto out;
1577
1578                         if (m.kmod)
1579                                 ret = maps__set_module_path(maps, path, &m);
1580
1581                         zfree(&m.name);
1582
1583                         if (ret)
1584                                 goto out;
1585                 }
1586         }
1587
1588 out:
1589         closedir(dir);
1590         return ret;
1591 }
1592
1593 static int machine__set_modules_path(struct machine *machine)
1594 {
1595         char *version;
1596         char modules_path[PATH_MAX];
1597
1598         version = get_kernel_version(machine->root_dir);
1599         if (!version)
1600                 return -1;
1601
1602         snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s",
1603                  machine->root_dir, version);
1604         free(version);
1605
1606         return maps__set_modules_path_dir(machine__kernel_maps(machine), modules_path, 0);
1607 }
1608 int __weak arch__fix_module_text_start(u64 *start __maybe_unused,
1609                                 u64 *size __maybe_unused,
1610                                 const char *name __maybe_unused)
1611 {
1612         return 0;
1613 }
1614
1615 static int machine__create_module(void *arg, const char *name, u64 start,
1616                                   u64 size)
1617 {
1618         struct machine *machine = arg;
1619         struct map *map;
1620
1621         if (arch__fix_module_text_start(&start, &size, name) < 0)
1622                 return -1;
1623
1624         map = machine__addnew_module_map(machine, start, name);
1625         if (map == NULL)
1626                 return -1;
1627         map__set_end(map, start + size);
1628
1629         dso__kernel_module_get_build_id(map__dso(map), machine->root_dir);
1630         map__put(map);
1631         return 0;
1632 }
1633
1634 static int machine__create_modules(struct machine *machine)
1635 {
1636         const char *modules;
1637         char path[PATH_MAX];
1638
1639         if (machine__is_default_guest(machine)) {
1640                 modules = symbol_conf.default_guest_modules;
1641         } else {
1642                 snprintf(path, PATH_MAX, "%s/proc/modules", machine->root_dir);
1643                 modules = path;
1644         }
1645
1646         if (symbol__restricted_filename(modules, "/proc/modules"))
1647                 return -1;
1648
1649         if (modules__parse(modules, machine, machine__create_module))
1650                 return -1;
1651
1652         if (!machine__set_modules_path(machine))
1653                 return 0;
1654
1655         pr_debug("Problems setting modules path maps, continuing anyway...\n");
1656
1657         return 0;
1658 }
1659
1660 static void machine__set_kernel_mmap(struct machine *machine,
1661                                      u64 start, u64 end)
1662 {
1663         map__set_start(machine->vmlinux_map, start);
1664         map__set_end(machine->vmlinux_map, end);
1665         /*
1666          * Be a bit paranoid here, some perf.data file came with
1667          * a zero sized synthesized MMAP event for the kernel.
1668          */
1669         if (start == 0 && end == 0)
1670                 map__set_end(machine->vmlinux_map, ~0ULL);
1671 }
1672
1673 static int machine__update_kernel_mmap(struct machine *machine,
1674                                      u64 start, u64 end)
1675 {
1676         struct map *orig, *updated;
1677         int err;
1678
1679         orig = machine->vmlinux_map;
1680         updated = map__get(orig);
1681
1682         machine->vmlinux_map = updated;
1683         machine__set_kernel_mmap(machine, start, end);
1684         maps__remove(machine__kernel_maps(machine), orig);
1685         err = maps__insert(machine__kernel_maps(machine), updated);
1686         map__put(orig);
1687
1688         return err;
1689 }
1690
1691 int machine__create_kernel_maps(struct machine *machine)
1692 {
1693         struct dso *kernel = machine__get_kernel(machine);
1694         const char *name = NULL;
1695         u64 start = 0, end = ~0ULL;
1696         int ret;
1697
1698         if (kernel == NULL)
1699                 return -1;
1700
1701         ret = __machine__create_kernel_maps(machine, kernel);
1702         if (ret < 0)
1703                 goto out_put;
1704
1705         if (symbol_conf.use_modules && machine__create_modules(machine) < 0) {
1706                 if (machine__is_host(machine))
1707                         pr_debug("Problems creating module maps, "
1708                                  "continuing anyway...\n");
1709                 else
1710                         pr_debug("Problems creating module maps for guest %d, "
1711                                  "continuing anyway...\n", machine->pid);
1712         }
1713
1714         if (!machine__get_running_kernel_start(machine, &name, &start, &end)) {
1715                 if (name &&
1716                     map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, name, start)) {
1717                         machine__destroy_kernel_maps(machine);
1718                         ret = -1;
1719                         goto out_put;
1720                 }
1721
1722                 /*
1723                  * we have a real start address now, so re-order the kmaps
1724                  * assume it's the last in the kmaps
1725                  */
1726                 ret = machine__update_kernel_mmap(machine, start, end);
1727                 if (ret < 0)
1728                         goto out_put;
1729         }
1730
1731         if (machine__create_extra_kernel_maps(machine, kernel))
1732                 pr_debug("Problems creating extra kernel maps, continuing anyway...\n");
1733
1734         if (end == ~0ULL) {
1735                 /* update end address of the kernel map using adjacent module address */
1736                 struct map_rb_node *rb_node = maps__find_node(machine__kernel_maps(machine),
1737                                                         machine__kernel_map(machine));
1738                 struct map_rb_node *next = map_rb_node__next(rb_node);
1739
1740                 if (next)
1741                         machine__set_kernel_mmap(machine, start, map__start(next->map));
1742         }
1743
1744 out_put:
1745         dso__put(kernel);
1746         return ret;
1747 }
1748
1749 static bool machine__uses_kcore(struct machine *machine)
1750 {
1751         struct dso *dso;
1752
1753         list_for_each_entry(dso, &machine->dsos.head, node) {
1754                 if (dso__is_kcore(dso))
1755                         return true;
1756         }
1757
1758         return false;
1759 }
1760
1761 static bool perf_event__is_extra_kernel_mmap(struct machine *machine,
1762                                              struct extra_kernel_map *xm)
1763 {
1764         return machine__is(machine, "x86_64") &&
1765                is_entry_trampoline(xm->name);
1766 }
1767
1768 static int machine__process_extra_kernel_map(struct machine *machine,
1769                                              struct extra_kernel_map *xm)
1770 {
1771         struct dso *kernel = machine__kernel_dso(machine);
1772
1773         if (kernel == NULL)
1774                 return -1;
1775
1776         return machine__create_extra_kernel_map(machine, kernel, xm);
1777 }
1778
1779 static int machine__process_kernel_mmap_event(struct machine *machine,
1780                                               struct extra_kernel_map *xm,
1781                                               struct build_id *bid)
1782 {
1783         struct map *map;
1784         enum dso_space_type dso_space;
1785         bool is_kernel_mmap;
1786         const char *mmap_name = machine->mmap_name;
1787
1788         /* If we have maps from kcore then we do not need or want any others */
1789         if (machine__uses_kcore(machine))
1790                 return 0;
1791
1792         if (machine__is_host(machine))
1793                 dso_space = DSO_SPACE__KERNEL;
1794         else
1795                 dso_space = DSO_SPACE__KERNEL_GUEST;
1796
1797         is_kernel_mmap = memcmp(xm->name, mmap_name, strlen(mmap_name) - 1) == 0;
1798         if (!is_kernel_mmap && !machine__is_host(machine)) {
1799                 /*
1800                  * If the event was recorded inside the guest and injected into
1801                  * the host perf.data file, then it will match a host mmap_name,
1802                  * so try that - see machine__set_mmap_name().
1803                  */
1804                 mmap_name = "[kernel.kallsyms]";
1805                 is_kernel_mmap = memcmp(xm->name, mmap_name, strlen(mmap_name) - 1) == 0;
1806         }
1807         if (xm->name[0] == '/' ||
1808             (!is_kernel_mmap && xm->name[0] == '[')) {
1809                 map = machine__addnew_module_map(machine, xm->start,
1810                                                  xm->name);
1811                 if (map == NULL)
1812                         goto out_problem;
1813
1814                 map__set_end(map, map__start(map) + xm->end - xm->start);
1815
1816                 if (build_id__is_defined(bid))
1817                         dso__set_build_id(map__dso(map), bid);
1818
1819         } else if (is_kernel_mmap) {
1820                 const char *symbol_name = xm->name + strlen(mmap_name);
1821                 /*
1822                  * Should be there already, from the build-id table in
1823                  * the header.
1824                  */
1825                 struct dso *kernel = NULL;
1826                 struct dso *dso;
1827
1828                 down_read(&machine->dsos.lock);
1829
1830                 list_for_each_entry(dso, &machine->dsos.head, node) {
1831
1832                         /*
1833                          * The cpumode passed to is_kernel_module is not the
1834                          * cpumode of *this* event. If we insist on passing
1835                          * correct cpumode to is_kernel_module, we should
1836                          * record the cpumode when we adding this dso to the
1837                          * linked list.
1838                          *
1839                          * However we don't really need passing correct
1840                          * cpumode.  We know the correct cpumode must be kernel
1841                          * mode (if not, we should not link it onto kernel_dsos
1842                          * list).
1843                          *
1844                          * Therefore, we pass PERF_RECORD_MISC_CPUMODE_UNKNOWN.
1845                          * is_kernel_module() treats it as a kernel cpumode.
1846                          */
1847
1848                         if (!dso->kernel ||
1849                             is_kernel_module(dso->long_name,
1850                                              PERF_RECORD_MISC_CPUMODE_UNKNOWN))
1851                                 continue;
1852
1853
1854                         kernel = dso;
1855                         break;
1856                 }
1857
1858                 up_read(&machine->dsos.lock);
1859
1860                 if (kernel == NULL)
1861                         kernel = machine__findnew_dso(machine, machine->mmap_name);
1862                 if (kernel == NULL)
1863                         goto out_problem;
1864
1865                 kernel->kernel = dso_space;
1866                 if (__machine__create_kernel_maps(machine, kernel) < 0) {
1867                         dso__put(kernel);
1868                         goto out_problem;
1869                 }
1870
1871                 if (strstr(kernel->long_name, "vmlinux"))
1872                         dso__set_short_name(kernel, "[kernel.vmlinux]", false);
1873
1874                 if (machine__update_kernel_mmap(machine, xm->start, xm->end) < 0) {
1875                         dso__put(kernel);
1876                         goto out_problem;
1877                 }
1878
1879                 if (build_id__is_defined(bid))
1880                         dso__set_build_id(kernel, bid);
1881
1882                 /*
1883                  * Avoid using a zero address (kptr_restrict) for the ref reloc
1884                  * symbol. Effectively having zero here means that at record
1885                  * time /proc/sys/kernel/kptr_restrict was non zero.
1886                  */
1887                 if (xm->pgoff != 0) {
1888                         map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map,
1889                                                         symbol_name,
1890                                                         xm->pgoff);
1891                 }
1892
1893                 if (machine__is_default_guest(machine)) {
1894                         /*
1895                          * preload dso of guest kernel and modules
1896                          */
1897                         dso__load(kernel, machine__kernel_map(machine));
1898                 }
1899         } else if (perf_event__is_extra_kernel_mmap(machine, xm)) {
1900                 return machine__process_extra_kernel_map(machine, xm);
1901         }
1902         return 0;
1903 out_problem:
1904         return -1;
1905 }
1906
1907 int machine__process_mmap2_event(struct machine *machine,
1908                                  union perf_event *event,
1909                                  struct perf_sample *sample)
1910 {
1911         struct thread *thread;
1912         struct map *map;
1913         struct dso_id dso_id = {
1914                 .maj = event->mmap2.maj,
1915                 .min = event->mmap2.min,
1916                 .ino = event->mmap2.ino,
1917                 .ino_generation = event->mmap2.ino_generation,
1918         };
1919         struct build_id __bid, *bid = NULL;
1920         int ret = 0;
1921
1922         if (dump_trace)
1923                 perf_event__fprintf_mmap2(event, stdout);
1924
1925         if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
1926                 bid = &__bid;
1927                 build_id__init(bid, event->mmap2.build_id, event->mmap2.build_id_size);
1928         }
1929
1930         if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1931             sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1932                 struct extra_kernel_map xm = {
1933                         .start = event->mmap2.start,
1934                         .end   = event->mmap2.start + event->mmap2.len,
1935                         .pgoff = event->mmap2.pgoff,
1936                 };
1937
1938                 strlcpy(xm.name, event->mmap2.filename, KMAP_NAME_LEN);
1939                 ret = machine__process_kernel_mmap_event(machine, &xm, bid);
1940                 if (ret < 0)
1941                         goto out_problem;
1942                 return 0;
1943         }
1944
1945         thread = machine__findnew_thread(machine, event->mmap2.pid,
1946                                         event->mmap2.tid);
1947         if (thread == NULL)
1948                 goto out_problem;
1949
1950         map = map__new(machine, event->mmap2.start,
1951                         event->mmap2.len, event->mmap2.pgoff,
1952                         &dso_id, event->mmap2.prot,
1953                         event->mmap2.flags, bid,
1954                         event->mmap2.filename, thread);
1955
1956         if (map == NULL)
1957                 goto out_problem_map;
1958
1959         ret = thread__insert_map(thread, map);
1960         if (ret)
1961                 goto out_problem_insert;
1962
1963         thread__put(thread);
1964         map__put(map);
1965         return 0;
1966
1967 out_problem_insert:
1968         map__put(map);
1969 out_problem_map:
1970         thread__put(thread);
1971 out_problem:
1972         dump_printf("problem processing PERF_RECORD_MMAP2, skipping event.\n");
1973         return 0;
1974 }
1975
1976 int machine__process_mmap_event(struct machine *machine, union perf_event *event,
1977                                 struct perf_sample *sample)
1978 {
1979         struct thread *thread;
1980         struct map *map;
1981         u32 prot = 0;
1982         int ret = 0;
1983
1984         if (dump_trace)
1985                 perf_event__fprintf_mmap(event, stdout);
1986
1987         if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1988             sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1989                 struct extra_kernel_map xm = {
1990                         .start = event->mmap.start,
1991                         .end   = event->mmap.start + event->mmap.len,
1992                         .pgoff = event->mmap.pgoff,
1993                 };
1994
1995                 strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN);
1996                 ret = machine__process_kernel_mmap_event(machine, &xm, NULL);
1997                 if (ret < 0)
1998                         goto out_problem;
1999                 return 0;
2000         }
2001
2002         thread = machine__findnew_thread(machine, event->mmap.pid,
2003                                          event->mmap.tid);
2004         if (thread == NULL)
2005                 goto out_problem;
2006
2007         if (!(event->header.misc & PERF_RECORD_MISC_MMAP_DATA))
2008                 prot = PROT_EXEC;
2009
2010         map = map__new(machine, event->mmap.start,
2011                         event->mmap.len, event->mmap.pgoff,
2012                         NULL, prot, 0, NULL, event->mmap.filename, thread);
2013
2014         if (map == NULL)
2015                 goto out_problem_map;
2016
2017         ret = thread__insert_map(thread, map);
2018         if (ret)
2019                 goto out_problem_insert;
2020
2021         thread__put(thread);
2022         map__put(map);
2023         return 0;
2024
2025 out_problem_insert:
2026         map__put(map);
2027 out_problem_map:
2028         thread__put(thread);
2029 out_problem:
2030         dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n");
2031         return 0;
2032 }
2033
2034 static void __machine__remove_thread(struct machine *machine, struct thread *th, bool lock)
2035 {
2036         struct threads *threads = machine__threads(machine, th->tid);
2037
2038         if (threads->last_match == th)
2039                 threads__set_last_match(threads, NULL);
2040
2041         if (lock)
2042                 down_write(&threads->lock);
2043
2044         BUG_ON(refcount_read(&th->refcnt) == 0);
2045
2046         rb_erase_cached(&th->rb_node, &threads->entries);
2047         RB_CLEAR_NODE(&th->rb_node);
2048         --threads->nr;
2049         /*
2050          * Move it first to the dead_threads list, then drop the reference,
2051          * if this is the last reference, then the thread__delete destructor
2052          * will be called and we will remove it from the dead_threads list.
2053          */
2054         list_add_tail(&th->node, &threads->dead);
2055
2056         /*
2057          * We need to do the put here because if this is the last refcount,
2058          * then we will be touching the threads->dead head when removing the
2059          * thread.
2060          */
2061         thread__put(th);
2062
2063         if (lock)
2064                 up_write(&threads->lock);
2065 }
2066
2067 void machine__remove_thread(struct machine *machine, struct thread *th)
2068 {
2069         return __machine__remove_thread(machine, th, true);
2070 }
2071
2072 int machine__process_fork_event(struct machine *machine, union perf_event *event,
2073                                 struct perf_sample *sample)
2074 {
2075         struct thread *thread = machine__find_thread(machine,
2076                                                      event->fork.pid,
2077                                                      event->fork.tid);
2078         struct thread *parent = machine__findnew_thread(machine,
2079                                                         event->fork.ppid,
2080                                                         event->fork.ptid);
2081         bool do_maps_clone = true;
2082         int err = 0;
2083
2084         if (dump_trace)
2085                 perf_event__fprintf_task(event, stdout);
2086
2087         /*
2088          * There may be an existing thread that is not actually the parent,
2089          * either because we are processing events out of order, or because the
2090          * (fork) event that would have removed the thread was lost. Assume the
2091          * latter case and continue on as best we can.
2092          */
2093         if (parent->pid_ != (pid_t)event->fork.ppid) {
2094                 dump_printf("removing erroneous parent thread %d/%d\n",
2095                             parent->pid_, parent->tid);
2096                 machine__remove_thread(machine, parent);
2097                 thread__put(parent);
2098                 parent = machine__findnew_thread(machine, event->fork.ppid,
2099                                                  event->fork.ptid);
2100         }
2101
2102         /* if a thread currently exists for the thread id remove it */
2103         if (thread != NULL) {
2104                 machine__remove_thread(machine, thread);
2105                 thread__put(thread);
2106         }
2107
2108         thread = machine__findnew_thread(machine, event->fork.pid,
2109                                          event->fork.tid);
2110         /*
2111          * When synthesizing FORK events, we are trying to create thread
2112          * objects for the already running tasks on the machine.
2113          *
2114          * Normally, for a kernel FORK event, we want to clone the parent's
2115          * maps because that is what the kernel just did.
2116          *
2117          * But when synthesizing, this should not be done.  If we do, we end up
2118          * with overlapping maps as we process the synthesized MMAP2 events that
2119          * get delivered shortly thereafter.
2120          *
2121          * Use the FORK event misc flags in an internal way to signal this
2122          * situation, so we can elide the map clone when appropriate.
2123          */
2124         if (event->fork.header.misc & PERF_RECORD_MISC_FORK_EXEC)
2125                 do_maps_clone = false;
2126
2127         if (thread == NULL || parent == NULL ||
2128             thread__fork(thread, parent, sample->time, do_maps_clone) < 0) {
2129                 dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
2130                 err = -1;
2131         }
2132         thread__put(thread);
2133         thread__put(parent);
2134
2135         return err;
2136 }
2137
2138 int machine__process_exit_event(struct machine *machine, union perf_event *event,
2139                                 struct perf_sample *sample __maybe_unused)
2140 {
2141         struct thread *thread = machine__find_thread(machine,
2142                                                      event->fork.pid,
2143                                                      event->fork.tid);
2144
2145         if (dump_trace)
2146                 perf_event__fprintf_task(event, stdout);
2147
2148         if (thread != NULL) {
2149                 thread__exited(thread);
2150                 thread__put(thread);
2151         }
2152
2153         return 0;
2154 }
2155
2156 int machine__process_event(struct machine *machine, union perf_event *event,
2157                            struct perf_sample *sample)
2158 {
2159         int ret;
2160
2161         switch (event->header.type) {
2162         case PERF_RECORD_COMM:
2163                 ret = machine__process_comm_event(machine, event, sample); break;
2164         case PERF_RECORD_MMAP:
2165                 ret = machine__process_mmap_event(machine, event, sample); break;
2166         case PERF_RECORD_NAMESPACES:
2167                 ret = machine__process_namespaces_event(machine, event, sample); break;
2168         case PERF_RECORD_CGROUP:
2169                 ret = machine__process_cgroup_event(machine, event, sample); break;
2170         case PERF_RECORD_MMAP2:
2171                 ret = machine__process_mmap2_event(machine, event, sample); break;
2172         case PERF_RECORD_FORK:
2173                 ret = machine__process_fork_event(machine, event, sample); break;
2174         case PERF_RECORD_EXIT:
2175                 ret = machine__process_exit_event(machine, event, sample); break;
2176         case PERF_RECORD_LOST:
2177                 ret = machine__process_lost_event(machine, event, sample); break;
2178         case PERF_RECORD_AUX:
2179                 ret = machine__process_aux_event(machine, event); break;
2180         case PERF_RECORD_ITRACE_START:
2181                 ret = machine__process_itrace_start_event(machine, event); break;
2182         case PERF_RECORD_LOST_SAMPLES:
2183                 ret = machine__process_lost_samples_event(machine, event, sample); break;
2184         case PERF_RECORD_SWITCH:
2185         case PERF_RECORD_SWITCH_CPU_WIDE:
2186                 ret = machine__process_switch_event(machine, event); break;
2187         case PERF_RECORD_KSYMBOL:
2188                 ret = machine__process_ksymbol(machine, event, sample); break;
2189         case PERF_RECORD_BPF_EVENT:
2190                 ret = machine__process_bpf(machine, event, sample); break;
2191         case PERF_RECORD_TEXT_POKE:
2192                 ret = machine__process_text_poke(machine, event, sample); break;
2193         case PERF_RECORD_AUX_OUTPUT_HW_ID:
2194                 ret = machine__process_aux_output_hw_id_event(machine, event); break;
2195         default:
2196                 ret = -1;
2197                 break;
2198         }
2199
2200         return ret;
2201 }
2202
2203 static bool symbol__match_regex(struct symbol *sym, regex_t *regex)
2204 {
2205         if (!regexec(regex, sym->name, 0, NULL, 0))
2206                 return true;
2207         return false;
2208 }
2209
2210 static void ip__resolve_ams(struct thread *thread,
2211                             struct addr_map_symbol *ams,
2212                             u64 ip)
2213 {
2214         struct addr_location al;
2215
2216         memset(&al, 0, sizeof(al));
2217         /*
2218          * We cannot use the header.misc hint to determine whether a
2219          * branch stack address is user, kernel, guest, hypervisor.
2220          * Branches may straddle the kernel/user/hypervisor boundaries.
2221          * Thus, we have to try consecutively until we find a match
2222          * or else, the symbol is unknown
2223          */
2224         thread__find_cpumode_addr_location(thread, ip, &al);
2225
2226         ams->addr = ip;
2227         ams->al_addr = al.addr;
2228         ams->al_level = al.level;
2229         ams->ms.maps = al.maps;
2230         ams->ms.sym = al.sym;
2231         ams->ms.map = al.map;
2232         ams->phys_addr = 0;
2233         ams->data_page_size = 0;
2234 }
2235
2236 static void ip__resolve_data(struct thread *thread,
2237                              u8 m, struct addr_map_symbol *ams,
2238                              u64 addr, u64 phys_addr, u64 daddr_page_size)
2239 {
2240         struct addr_location al;
2241
2242         memset(&al, 0, sizeof(al));
2243
2244         thread__find_symbol(thread, m, addr, &al);
2245
2246         ams->addr = addr;
2247         ams->al_addr = al.addr;
2248         ams->al_level = al.level;
2249         ams->ms.maps = al.maps;
2250         ams->ms.sym = al.sym;
2251         ams->ms.map = al.map;
2252         ams->phys_addr = phys_addr;
2253         ams->data_page_size = daddr_page_size;
2254 }
2255
2256 struct mem_info *sample__resolve_mem(struct perf_sample *sample,
2257                                      struct addr_location *al)
2258 {
2259         struct mem_info *mi = mem_info__new();
2260
2261         if (!mi)
2262                 return NULL;
2263
2264         ip__resolve_ams(al->thread, &mi->iaddr, sample->ip);
2265         ip__resolve_data(al->thread, al->cpumode, &mi->daddr,
2266                          sample->addr, sample->phys_addr,
2267                          sample->data_page_size);
2268         mi->data_src.val = sample->data_src;
2269
2270         return mi;
2271 }
2272
2273 static char *callchain_srcline(struct map_symbol *ms, u64 ip)
2274 {
2275         struct map *map = ms->map;
2276         char *srcline = NULL;
2277         struct dso *dso;
2278
2279         if (!map || callchain_param.key == CCKEY_FUNCTION)
2280                 return srcline;
2281
2282         dso = map__dso(map);
2283         srcline = srcline__tree_find(&dso->srclines, ip);
2284         if (!srcline) {
2285                 bool show_sym = false;
2286                 bool show_addr = callchain_param.key == CCKEY_ADDRESS;
2287
2288                 srcline = get_srcline(dso, map__rip_2objdump(map, ip),
2289                                       ms->sym, show_sym, show_addr, ip);
2290                 srcline__tree_insert(&dso->srclines, ip, srcline);
2291         }
2292
2293         return srcline;
2294 }
2295
2296 struct iterations {
2297         int nr_loop_iter;
2298         u64 cycles;
2299 };
2300
2301 static int add_callchain_ip(struct thread *thread,
2302                             struct callchain_cursor *cursor,
2303                             struct symbol **parent,
2304                             struct addr_location *root_al,
2305                             u8 *cpumode,
2306                             u64 ip,
2307                             bool branch,
2308                             struct branch_flags *flags,
2309                             struct iterations *iter,
2310                             u64 branch_from)
2311 {
2312         struct map_symbol ms;
2313         struct addr_location al;
2314         int nr_loop_iter = 0, err;
2315         u64 iter_cycles = 0;
2316         const char *srcline = NULL;
2317
2318         al.filtered = 0;
2319         al.sym = NULL;
2320         al.srcline = NULL;
2321         if (!cpumode) {
2322                 thread__find_cpumode_addr_location(thread, ip, &al);
2323         } else {
2324                 if (ip >= PERF_CONTEXT_MAX) {
2325                         switch (ip) {
2326                         case PERF_CONTEXT_HV:
2327                                 *cpumode = PERF_RECORD_MISC_HYPERVISOR;
2328                                 break;
2329                         case PERF_CONTEXT_KERNEL:
2330                                 *cpumode = PERF_RECORD_MISC_KERNEL;
2331                                 break;
2332                         case PERF_CONTEXT_USER:
2333                                 *cpumode = PERF_RECORD_MISC_USER;
2334                                 break;
2335                         default:
2336                                 pr_debug("invalid callchain context: "
2337                                          "%"PRId64"\n", (s64) ip);
2338                                 /*
2339                                  * It seems the callchain is corrupted.
2340                                  * Discard all.
2341                                  */
2342                                 callchain_cursor_reset(cursor);
2343                                 return 1;
2344                         }
2345                         return 0;
2346                 }
2347                 thread__find_symbol(thread, *cpumode, ip, &al);
2348         }
2349
2350         if (al.sym != NULL) {
2351                 if (perf_hpp_list.parent && !*parent &&
2352                     symbol__match_regex(al.sym, &parent_regex))
2353                         *parent = al.sym;
2354                 else if (have_ignore_callees && root_al &&
2355                   symbol__match_regex(al.sym, &ignore_callees_regex)) {
2356                         /* Treat this symbol as the root,
2357                            forgetting its callees. */
2358                         *root_al = al;
2359                         callchain_cursor_reset(cursor);
2360                 }
2361         }
2362
2363         if (symbol_conf.hide_unresolved && al.sym == NULL)
2364                 return 0;
2365
2366         if (iter) {
2367                 nr_loop_iter = iter->nr_loop_iter;
2368                 iter_cycles = iter->cycles;
2369         }
2370
2371         ms.maps = al.maps;
2372         ms.map = al.map;
2373         ms.sym = al.sym;
2374
2375         if (!branch && append_inlines(cursor, &ms, ip) == 0)
2376                 return 0;
2377
2378         srcline = callchain_srcline(&ms, al.addr);
2379         err = callchain_cursor_append(cursor, ip, &ms,
2380                                       branch, flags, nr_loop_iter,
2381                                       iter_cycles, branch_from, srcline);
2382         map__put(al.map);
2383         return err;
2384 }
2385
2386 struct branch_info *sample__resolve_bstack(struct perf_sample *sample,
2387                                            struct addr_location *al)
2388 {
2389         unsigned int i;
2390         const struct branch_stack *bs = sample->branch_stack;
2391         struct branch_entry *entries = perf_sample__branch_entries(sample);
2392         struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info));
2393
2394         if (!bi)
2395                 return NULL;
2396
2397         for (i = 0; i < bs->nr; i++) {
2398                 ip__resolve_ams(al->thread, &bi[i].to, entries[i].to);
2399                 ip__resolve_ams(al->thread, &bi[i].from, entries[i].from);
2400                 bi[i].flags = entries[i].flags;
2401         }
2402         return bi;
2403 }
2404
2405 static void save_iterations(struct iterations *iter,
2406                             struct branch_entry *be, int nr)
2407 {
2408         int i;
2409
2410         iter->nr_loop_iter++;
2411         iter->cycles = 0;
2412
2413         for (i = 0; i < nr; i++)
2414                 iter->cycles += be[i].flags.cycles;
2415 }
2416
2417 #define CHASHSZ 127
2418 #define CHASHBITS 7
2419 #define NO_ENTRY 0xff
2420
2421 #define PERF_MAX_BRANCH_DEPTH 127
2422
2423 /* Remove loops. */
2424 static int remove_loops(struct branch_entry *l, int nr,
2425                         struct iterations *iter)
2426 {
2427         int i, j, off;
2428         unsigned char chash[CHASHSZ];
2429
2430         memset(chash, NO_ENTRY, sizeof(chash));
2431
2432         BUG_ON(PERF_MAX_BRANCH_DEPTH > 255);
2433
2434         for (i = 0; i < nr; i++) {
2435                 int h = hash_64(l[i].from, CHASHBITS) % CHASHSZ;
2436
2437                 /* no collision handling for now */
2438                 if (chash[h] == NO_ENTRY) {
2439                         chash[h] = i;
2440                 } else if (l[chash[h]].from == l[i].from) {
2441                         bool is_loop = true;
2442                         /* check if it is a real loop */
2443                         off = 0;
2444                         for (j = chash[h]; j < i && i + off < nr; j++, off++)
2445                                 if (l[j].from != l[i + off].from) {
2446                                         is_loop = false;
2447                                         break;
2448                                 }
2449                         if (is_loop) {
2450                                 j = nr - (i + off);
2451                                 if (j > 0) {
2452                                         save_iterations(iter + i + off,
2453                                                 l + i, off);
2454
2455                                         memmove(iter + i, iter + i + off,
2456                                                 j * sizeof(*iter));
2457
2458                                         memmove(l + i, l + i + off,
2459                                                 j * sizeof(*l));
2460                                 }
2461
2462                                 nr -= off;
2463                         }
2464                 }
2465         }
2466         return nr;
2467 }
2468
2469 static int lbr_callchain_add_kernel_ip(struct thread *thread,
2470                                        struct callchain_cursor *cursor,
2471                                        struct perf_sample *sample,
2472                                        struct symbol **parent,
2473                                        struct addr_location *root_al,
2474                                        u64 branch_from,
2475                                        bool callee, int end)
2476 {
2477         struct ip_callchain *chain = sample->callchain;
2478         u8 cpumode = PERF_RECORD_MISC_USER;
2479         int err, i;
2480
2481         if (callee) {
2482                 for (i = 0; i < end + 1; i++) {
2483                         err = add_callchain_ip(thread, cursor, parent,
2484                                                root_al, &cpumode, chain->ips[i],
2485                                                false, NULL, NULL, branch_from);
2486                         if (err)
2487                                 return err;
2488                 }
2489                 return 0;
2490         }
2491
2492         for (i = end; i >= 0; i--) {
2493                 err = add_callchain_ip(thread, cursor, parent,
2494                                        root_al, &cpumode, chain->ips[i],
2495                                        false, NULL, NULL, branch_from);
2496                 if (err)
2497                         return err;
2498         }
2499
2500         return 0;
2501 }
2502
2503 static void save_lbr_cursor_node(struct thread *thread,
2504                                  struct callchain_cursor *cursor,
2505                                  int idx)
2506 {
2507         struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
2508
2509         if (!lbr_stitch)
2510                 return;
2511
2512         if (cursor->pos == cursor->nr) {
2513                 lbr_stitch->prev_lbr_cursor[idx].valid = false;
2514                 return;
2515         }
2516
2517         if (!cursor->curr)
2518                 cursor->curr = cursor->first;
2519         else
2520                 cursor->curr = cursor->curr->next;
2521         memcpy(&lbr_stitch->prev_lbr_cursor[idx], cursor->curr,
2522                sizeof(struct callchain_cursor_node));
2523
2524         lbr_stitch->prev_lbr_cursor[idx].valid = true;
2525         cursor->pos++;
2526 }
2527
2528 static int lbr_callchain_add_lbr_ip(struct thread *thread,
2529                                     struct callchain_cursor *cursor,
2530                                     struct perf_sample *sample,
2531                                     struct symbol **parent,
2532                                     struct addr_location *root_al,
2533                                     u64 *branch_from,
2534                                     bool callee)
2535 {
2536         struct branch_stack *lbr_stack = sample->branch_stack;
2537         struct branch_entry *entries = perf_sample__branch_entries(sample);
2538         u8 cpumode = PERF_RECORD_MISC_USER;
2539         int lbr_nr = lbr_stack->nr;
2540         struct branch_flags *flags;
2541         int err, i;
2542         u64 ip;
2543
2544         /*
2545          * The curr and pos are not used in writing session. They are cleared
2546          * in callchain_cursor_commit() when the writing session is closed.
2547          * Using curr and pos to track the current cursor node.
2548          */
2549         if (thread->lbr_stitch) {
2550                 cursor->curr = NULL;
2551                 cursor->pos = cursor->nr;
2552                 if (cursor->nr) {
2553                         cursor->curr = cursor->first;
2554                         for (i = 0; i < (int)(cursor->nr - 1); i++)
2555                                 cursor->curr = cursor->curr->next;
2556                 }
2557         }
2558
2559         if (callee) {
2560                 /* Add LBR ip from first entries.to */
2561                 ip = entries[0].to;
2562                 flags = &entries[0].flags;
2563                 *branch_from = entries[0].from;
2564                 err = add_callchain_ip(thread, cursor, parent,
2565                                        root_al, &cpumode, ip,
2566                                        true, flags, NULL,
2567                                        *branch_from);
2568                 if (err)
2569                         return err;
2570
2571                 /*
2572                  * The number of cursor node increases.
2573                  * Move the current cursor node.
2574                  * But does not need to save current cursor node for entry 0.
2575                  * It's impossible to stitch the whole LBRs of previous sample.
2576                  */
2577                 if (thread->lbr_stitch && (cursor->pos != cursor->nr)) {
2578                         if (!cursor->curr)
2579                                 cursor->curr = cursor->first;
2580                         else
2581                                 cursor->curr = cursor->curr->next;
2582                         cursor->pos++;
2583                 }
2584
2585                 /* Add LBR ip from entries.from one by one. */
2586                 for (i = 0; i < lbr_nr; i++) {
2587                         ip = entries[i].from;
2588                         flags = &entries[i].flags;
2589                         err = add_callchain_ip(thread, cursor, parent,
2590                                                root_al, &cpumode, ip,
2591                                                true, flags, NULL,
2592                                                *branch_from);
2593                         if (err)
2594                                 return err;
2595                         save_lbr_cursor_node(thread, cursor, i);
2596                 }
2597                 return 0;
2598         }
2599
2600         /* Add LBR ip from entries.from one by one. */
2601         for (i = lbr_nr - 1; i >= 0; i--) {
2602                 ip = entries[i].from;
2603                 flags = &entries[i].flags;
2604                 err = add_callchain_ip(thread, cursor, parent,
2605                                        root_al, &cpumode, ip,
2606                                        true, flags, NULL,
2607                                        *branch_from);
2608                 if (err)
2609                         return err;
2610                 save_lbr_cursor_node(thread, cursor, i);
2611         }
2612
2613         /* Add LBR ip from first entries.to */
2614         ip = entries[0].to;
2615         flags = &entries[0].flags;
2616         *branch_from = entries[0].from;
2617         err = add_callchain_ip(thread, cursor, parent,
2618                                root_al, &cpumode, ip,
2619                                true, flags, NULL,
2620                                *branch_from);
2621         if (err)
2622                 return err;
2623
2624         return 0;
2625 }
2626
2627 static int lbr_callchain_add_stitched_lbr_ip(struct thread *thread,
2628                                              struct callchain_cursor *cursor)
2629 {
2630         struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
2631         struct callchain_cursor_node *cnode;
2632         struct stitch_list *stitch_node;
2633         int err;
2634
2635         list_for_each_entry(stitch_node, &lbr_stitch->lists, node) {
2636                 cnode = &stitch_node->cursor;
2637
2638                 err = callchain_cursor_append(cursor, cnode->ip,
2639                                               &cnode->ms,
2640                                               cnode->branch,
2641                                               &cnode->branch_flags,
2642                                               cnode->nr_loop_iter,
2643                                               cnode->iter_cycles,
2644                                               cnode->branch_from,
2645                                               cnode->srcline);
2646                 if (err)
2647                         return err;
2648         }
2649         return 0;
2650 }
2651
2652 static struct stitch_list *get_stitch_node(struct thread *thread)
2653 {
2654         struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
2655         struct stitch_list *stitch_node;
2656
2657         if (!list_empty(&lbr_stitch->free_lists)) {
2658                 stitch_node = list_first_entry(&lbr_stitch->free_lists,
2659                                                struct stitch_list, node);
2660                 list_del(&stitch_node->node);
2661
2662                 return stitch_node;
2663         }
2664
2665         return malloc(sizeof(struct stitch_list));
2666 }
2667
2668 static bool has_stitched_lbr(struct thread *thread,
2669                              struct perf_sample *cur,
2670                              struct perf_sample *prev,
2671                              unsigned int max_lbr,
2672                              bool callee)
2673 {
2674         struct branch_stack *cur_stack = cur->branch_stack;
2675         struct branch_entry *cur_entries = perf_sample__branch_entries(cur);
2676         struct branch_stack *prev_stack = prev->branch_stack;
2677         struct branch_entry *prev_entries = perf_sample__branch_entries(prev);
2678         struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
2679         int i, j, nr_identical_branches = 0;
2680         struct stitch_list *stitch_node;
2681         u64 cur_base, distance;
2682
2683         if (!cur_stack || !prev_stack)
2684                 return false;
2685
2686         /* Find the physical index of the base-of-stack for current sample. */
2687         cur_base = max_lbr - cur_stack->nr + cur_stack->hw_idx + 1;
2688
2689         distance = (prev_stack->hw_idx > cur_base) ? (prev_stack->hw_idx - cur_base) :
2690                                                      (max_lbr + prev_stack->hw_idx - cur_base);
2691         /* Previous sample has shorter stack. Nothing can be stitched. */
2692         if (distance + 1 > prev_stack->nr)
2693                 return false;
2694
2695         /*
2696          * Check if there are identical LBRs between two samples.
2697          * Identical LBRs must have same from, to and flags values. Also,
2698          * they have to be saved in the same LBR registers (same physical
2699          * index).
2700          *
2701          * Starts from the base-of-stack of current sample.
2702          */
2703         for (i = distance, j = cur_stack->nr - 1; (i >= 0) && (j >= 0); i--, j--) {
2704                 if ((prev_entries[i].from != cur_entries[j].from) ||
2705                     (prev_entries[i].to != cur_entries[j].to) ||
2706                     (prev_entries[i].flags.value != cur_entries[j].flags.value))
2707                         break;
2708                 nr_identical_branches++;
2709         }
2710
2711         if (!nr_identical_branches)
2712                 return false;
2713
2714         /*
2715          * Save the LBRs between the base-of-stack of previous sample
2716          * and the base-of-stack of current sample into lbr_stitch->lists.
2717          * These LBRs will be stitched later.
2718          */
2719         for (i = prev_stack->nr - 1; i > (int)distance; i--) {
2720
2721                 if (!lbr_stitch->prev_lbr_cursor[i].valid)
2722                         continue;
2723
2724                 stitch_node = get_stitch_node(thread);
2725                 if (!stitch_node)
2726                         return false;
2727
2728                 memcpy(&stitch_node->cursor, &lbr_stitch->prev_lbr_cursor[i],
2729                        sizeof(struct callchain_cursor_node));
2730
2731                 if (callee)
2732                         list_add(&stitch_node->node, &lbr_stitch->lists);
2733                 else
2734                         list_add_tail(&stitch_node->node, &lbr_stitch->lists);
2735         }
2736
2737         return true;
2738 }
2739
2740 static bool alloc_lbr_stitch(struct thread *thread, unsigned int max_lbr)
2741 {
2742         if (thread->lbr_stitch)
2743                 return true;
2744
2745         thread->lbr_stitch = zalloc(sizeof(*thread->lbr_stitch));
2746         if (!thread->lbr_stitch)
2747                 goto err;
2748
2749         thread->lbr_stitch->prev_lbr_cursor = calloc(max_lbr + 1, sizeof(struct callchain_cursor_node));
2750         if (!thread->lbr_stitch->prev_lbr_cursor)
2751                 goto free_lbr_stitch;
2752
2753         INIT_LIST_HEAD(&thread->lbr_stitch->lists);
2754         INIT_LIST_HEAD(&thread->lbr_stitch->free_lists);
2755
2756         return true;
2757
2758 free_lbr_stitch:
2759         zfree(&thread->lbr_stitch);
2760 err:
2761         pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n");
2762         thread->lbr_stitch_enable = false;
2763         return false;
2764 }
2765
2766 /*
2767  * Resolve LBR callstack chain sample
2768  * Return:
2769  * 1 on success get LBR callchain information
2770  * 0 no available LBR callchain information, should try fp
2771  * negative error code on other errors.
2772  */
2773 static int resolve_lbr_callchain_sample(struct thread *thread,
2774                                         struct callchain_cursor *cursor,
2775                                         struct perf_sample *sample,
2776                                         struct symbol **parent,
2777                                         struct addr_location *root_al,
2778                                         int max_stack,
2779                                         unsigned int max_lbr)
2780 {
2781         bool callee = (callchain_param.order == ORDER_CALLEE);
2782         struct ip_callchain *chain = sample->callchain;
2783         int chain_nr = min(max_stack, (int)chain->nr), i;
2784         struct lbr_stitch *lbr_stitch;
2785         bool stitched_lbr = false;
2786         u64 branch_from = 0;
2787         int err;
2788
2789         for (i = 0; i < chain_nr; i++) {
2790                 if (chain->ips[i] == PERF_CONTEXT_USER)
2791                         break;
2792         }
2793
2794         /* LBR only affects the user callchain */
2795         if (i == chain_nr)
2796                 return 0;
2797
2798         if (thread->lbr_stitch_enable && !sample->no_hw_idx &&
2799             (max_lbr > 0) && alloc_lbr_stitch(thread, max_lbr)) {
2800                 lbr_stitch = thread->lbr_stitch;
2801
2802                 stitched_lbr = has_stitched_lbr(thread, sample,
2803                                                 &lbr_stitch->prev_sample,
2804                                                 max_lbr, callee);
2805
2806                 if (!stitched_lbr && !list_empty(&lbr_stitch->lists)) {
2807                         list_replace_init(&lbr_stitch->lists,
2808                                           &lbr_stitch->free_lists);
2809                 }
2810                 memcpy(&lbr_stitch->prev_sample, sample, sizeof(*sample));
2811         }
2812
2813         if (callee) {
2814                 /* Add kernel ip */
2815                 err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
2816                                                   parent, root_al, branch_from,
2817                                                   true, i);
2818                 if (err)
2819                         goto error;
2820
2821                 err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
2822                                                root_al, &branch_from, true);
2823                 if (err)
2824                         goto error;
2825
2826                 if (stitched_lbr) {
2827                         err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
2828                         if (err)
2829                                 goto error;
2830                 }
2831
2832         } else {
2833                 if (stitched_lbr) {
2834                         err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
2835                         if (err)
2836                                 goto error;
2837                 }
2838                 err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
2839                                                root_al, &branch_from, false);
2840                 if (err)
2841                         goto error;
2842
2843                 /* Add kernel ip */
2844                 err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
2845                                                   parent, root_al, branch_from,
2846                                                   false, i);
2847                 if (err)
2848                         goto error;
2849         }
2850         return 1;
2851
2852 error:
2853         return (err < 0) ? err : 0;
2854 }
2855
2856 static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread,
2857                              struct callchain_cursor *cursor,
2858                              struct symbol **parent,
2859                              struct addr_location *root_al,
2860                              u8 *cpumode, int ent)
2861 {
2862         int err = 0;
2863
2864         while (--ent >= 0) {
2865                 u64 ip = chain->ips[ent];
2866
2867                 if (ip >= PERF_CONTEXT_MAX) {
2868                         err = add_callchain_ip(thread, cursor, parent,
2869                                                root_al, cpumode, ip,
2870                                                false, NULL, NULL, 0);
2871                         break;
2872                 }
2873         }
2874         return err;
2875 }
2876
2877 static u64 get_leaf_frame_caller(struct perf_sample *sample,
2878                 struct thread *thread, int usr_idx)
2879 {
2880         if (machine__normalized_is(maps__machine(thread->maps), "arm64"))
2881                 return get_leaf_frame_caller_aarch64(sample, thread, usr_idx);
2882         else
2883                 return 0;
2884 }
2885
2886 static int thread__resolve_callchain_sample(struct thread *thread,
2887                                             struct callchain_cursor *cursor,
2888                                             struct evsel *evsel,
2889                                             struct perf_sample *sample,
2890                                             struct symbol **parent,
2891                                             struct addr_location *root_al,
2892                                             int max_stack)
2893 {
2894         struct branch_stack *branch = sample->branch_stack;
2895         struct branch_entry *entries = perf_sample__branch_entries(sample);
2896         struct ip_callchain *chain = sample->callchain;
2897         int chain_nr = 0;
2898         u8 cpumode = PERF_RECORD_MISC_USER;
2899         int i, j, err, nr_entries, usr_idx;
2900         int skip_idx = -1;
2901         int first_call = 0;
2902         u64 leaf_frame_caller;
2903
2904         if (chain)
2905                 chain_nr = chain->nr;
2906
2907         if (evsel__has_branch_callstack(evsel)) {
2908                 struct perf_env *env = evsel__env(evsel);
2909
2910                 err = resolve_lbr_callchain_sample(thread, cursor, sample, parent,
2911                                                    root_al, max_stack,
2912                                                    !env ? 0 : env->max_branches);
2913                 if (err)
2914                         return (err < 0) ? err : 0;
2915         }
2916
2917         /*
2918          * Based on DWARF debug information, some architectures skip
2919          * a callchain entry saved by the kernel.
2920          */
2921         skip_idx = arch_skip_callchain_idx(thread, chain);
2922
2923         /*
2924          * Add branches to call stack for easier browsing. This gives
2925          * more context for a sample than just the callers.
2926          *
2927          * This uses individual histograms of paths compared to the
2928          * aggregated histograms the normal LBR mode uses.
2929          *
2930          * Limitations for now:
2931          * - No extra filters
2932          * - No annotations (should annotate somehow)
2933          */
2934
2935         if (branch && callchain_param.branch_callstack) {
2936                 int nr = min(max_stack, (int)branch->nr);
2937                 struct branch_entry be[nr];
2938                 struct iterations iter[nr];
2939
2940                 if (branch->nr > PERF_MAX_BRANCH_DEPTH) {
2941                         pr_warning("corrupted branch chain. skipping...\n");
2942                         goto check_calls;
2943                 }
2944
2945                 for (i = 0; i < nr; i++) {
2946                         if (callchain_param.order == ORDER_CALLEE) {
2947                                 be[i] = entries[i];
2948
2949                                 if (chain == NULL)
2950                                         continue;
2951
2952                                 /*
2953                                  * Check for overlap into the callchain.
2954                                  * The return address is one off compared to
2955                                  * the branch entry. To adjust for this
2956                                  * assume the calling instruction is not longer
2957                                  * than 8 bytes.
2958                                  */
2959                                 if (i == skip_idx ||
2960                                     chain->ips[first_call] >= PERF_CONTEXT_MAX)
2961                                         first_call++;
2962                                 else if (be[i].from < chain->ips[first_call] &&
2963                                     be[i].from >= chain->ips[first_call] - 8)
2964                                         first_call++;
2965                         } else
2966                                 be[i] = entries[branch->nr - i - 1];
2967                 }
2968
2969                 memset(iter, 0, sizeof(struct iterations) * nr);
2970                 nr = remove_loops(be, nr, iter);
2971
2972                 for (i = 0; i < nr; i++) {
2973                         err = add_callchain_ip(thread, cursor, parent,
2974                                                root_al,
2975                                                NULL, be[i].to,
2976                                                true, &be[i].flags,
2977                                                NULL, be[i].from);
2978
2979                         if (!err)
2980                                 err = add_callchain_ip(thread, cursor, parent, root_al,
2981                                                        NULL, be[i].from,
2982                                                        true, &be[i].flags,
2983                                                        &iter[i], 0);
2984                         if (err == -EINVAL)
2985                                 break;
2986                         if (err)
2987                                 return err;
2988                 }
2989
2990                 if (chain_nr == 0)
2991                         return 0;
2992
2993                 chain_nr -= nr;
2994         }
2995
2996 check_calls:
2997         if (chain && callchain_param.order != ORDER_CALLEE) {
2998                 err = find_prev_cpumode(chain, thread, cursor, parent, root_al,
2999                                         &cpumode, chain->nr - first_call);
3000                 if (err)
3001                         return (err < 0) ? err : 0;
3002         }
3003         for (i = first_call, nr_entries = 0;
3004              i < chain_nr && nr_entries < max_stack; i++) {
3005                 u64 ip;
3006
3007                 if (callchain_param.order == ORDER_CALLEE)
3008                         j = i;
3009                 else
3010                         j = chain->nr - i - 1;
3011
3012 #ifdef HAVE_SKIP_CALLCHAIN_IDX
3013                 if (j == skip_idx)
3014                         continue;
3015 #endif
3016                 ip = chain->ips[j];
3017                 if (ip < PERF_CONTEXT_MAX)
3018                        ++nr_entries;
3019                 else if (callchain_param.order != ORDER_CALLEE) {
3020                         err = find_prev_cpumode(chain, thread, cursor, parent,
3021                                                 root_al, &cpumode, j);
3022                         if (err)
3023                                 return (err < 0) ? err : 0;
3024                         continue;
3025                 }
3026
3027                 /*
3028                  * PERF_CONTEXT_USER allows us to locate where the user stack ends.
3029                  * Depending on callchain_param.order and the position of PERF_CONTEXT_USER,
3030                  * the index will be different in order to add the missing frame
3031                  * at the right place.
3032                  */
3033
3034                 usr_idx = callchain_param.order == ORDER_CALLEE ? j-2 : j-1;
3035
3036                 if (usr_idx >= 0 && chain->ips[usr_idx] == PERF_CONTEXT_USER) {
3037
3038                         leaf_frame_caller = get_leaf_frame_caller(sample, thread, usr_idx);
3039
3040                         /*
3041                          * check if leaf_frame_Caller != ip to not add the same
3042                          * value twice.
3043                          */
3044
3045                         if (leaf_frame_caller && leaf_frame_caller != ip) {
3046
3047                                 err = add_callchain_ip(thread, cursor, parent,
3048                                                root_al, &cpumode, leaf_frame_caller,
3049                                                false, NULL, NULL, 0);
3050                                 if (err)
3051                                         return (err < 0) ? err : 0;
3052                         }
3053                 }
3054
3055                 err = add_callchain_ip(thread, cursor, parent,
3056                                        root_al, &cpumode, ip,
3057                                        false, NULL, NULL, 0);
3058
3059                 if (err)
3060                         return (err < 0) ? err : 0;
3061         }
3062
3063         return 0;
3064 }
3065
3066 static int append_inlines(struct callchain_cursor *cursor, struct map_symbol *ms, u64 ip)
3067 {
3068         struct symbol *sym = ms->sym;
3069         struct map *map = ms->map;
3070         struct inline_node *inline_node;
3071         struct inline_list *ilist;
3072         struct dso *dso;
3073         u64 addr;
3074         int ret = 1;
3075
3076         if (!symbol_conf.inline_name || !map || !sym)
3077                 return ret;
3078
3079         addr = map__dso_map_ip(map, ip);
3080         addr = map__rip_2objdump(map, addr);
3081         dso = map__dso(map);
3082
3083         inline_node = inlines__tree_find(&dso->inlined_nodes, addr);
3084         if (!inline_node) {
3085                 inline_node = dso__parse_addr_inlines(dso, addr, sym);
3086                 if (!inline_node)
3087                         return ret;
3088                 inlines__tree_insert(&dso->inlined_nodes, inline_node);
3089         }
3090
3091         list_for_each_entry(ilist, &inline_node->val, list) {
3092                 struct map_symbol ilist_ms = {
3093                         .maps = ms->maps,
3094                         .map = map,
3095                         .sym = ilist->symbol,
3096                 };
3097                 ret = callchain_cursor_append(cursor, ip, &ilist_ms, false,
3098                                               NULL, 0, 0, 0, ilist->srcline);
3099
3100                 if (ret != 0)
3101                         return ret;
3102         }
3103
3104         return ret;
3105 }
3106
3107 static int unwind_entry(struct unwind_entry *entry, void *arg)
3108 {
3109         struct callchain_cursor *cursor = arg;
3110         const char *srcline = NULL;
3111         u64 addr = entry->ip;
3112
3113         if (symbol_conf.hide_unresolved && entry->ms.sym == NULL)
3114                 return 0;
3115
3116         if (append_inlines(cursor, &entry->ms, entry->ip) == 0)
3117                 return 0;
3118
3119         /*
3120          * Convert entry->ip from a virtual address to an offset in
3121          * its corresponding binary.
3122          */
3123         if (entry->ms.map)
3124                 addr = map__dso_map_ip(entry->ms.map, entry->ip);
3125
3126         srcline = callchain_srcline(&entry->ms, addr);
3127         return callchain_cursor_append(cursor, entry->ip, &entry->ms,
3128                                        false, NULL, 0, 0, 0, srcline);
3129 }
3130
3131 static int thread__resolve_callchain_unwind(struct thread *thread,
3132                                             struct callchain_cursor *cursor,
3133                                             struct evsel *evsel,
3134                                             struct perf_sample *sample,
3135                                             int max_stack)
3136 {
3137         /* Can we do dwarf post unwind? */
3138         if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) &&
3139               (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER)))
3140                 return 0;
3141
3142         /* Bail out if nothing was captured. */
3143         if ((!sample->user_regs.regs) ||
3144             (!sample->user_stack.size))
3145                 return 0;
3146
3147         return unwind__get_entries(unwind_entry, cursor,
3148                                    thread, sample, max_stack, false);
3149 }
3150
3151 int thread__resolve_callchain(struct thread *thread,
3152                               struct callchain_cursor *cursor,
3153                               struct evsel *evsel,
3154                               struct perf_sample *sample,
3155                               struct symbol **parent,
3156                               struct addr_location *root_al,
3157                               int max_stack)
3158 {
3159         int ret = 0;
3160
3161         callchain_cursor_reset(cursor);
3162
3163         if (callchain_param.order == ORDER_CALLEE) {
3164                 ret = thread__resolve_callchain_sample(thread, cursor,
3165                                                        evsel, sample,
3166                                                        parent, root_al,
3167                                                        max_stack);
3168                 if (ret)
3169                         return ret;
3170                 ret = thread__resolve_callchain_unwind(thread, cursor,
3171                                                        evsel, sample,
3172                                                        max_stack);
3173         } else {
3174                 ret = thread__resolve_callchain_unwind(thread, cursor,
3175                                                        evsel, sample,
3176                                                        max_stack);
3177                 if (ret)
3178                         return ret;
3179                 ret = thread__resolve_callchain_sample(thread, cursor,
3180                                                        evsel, sample,
3181                                                        parent, root_al,
3182                                                        max_stack);
3183         }
3184
3185         return ret;
3186 }
3187
3188 int machine__for_each_thread(struct machine *machine,
3189                              int (*fn)(struct thread *thread, void *p),
3190                              void *priv)
3191 {
3192         struct threads *threads;
3193         struct rb_node *nd;
3194         struct thread *thread;
3195         int rc = 0;
3196         int i;
3197
3198         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
3199                 threads = &machine->threads[i];
3200                 for (nd = rb_first_cached(&threads->entries); nd;
3201                      nd = rb_next(nd)) {
3202                         thread = rb_entry(nd, struct thread, rb_node);
3203                         rc = fn(thread, priv);
3204                         if (rc != 0)
3205                                 return rc;
3206                 }
3207
3208                 list_for_each_entry(thread, &threads->dead, node) {
3209                         rc = fn(thread, priv);
3210                         if (rc != 0)
3211                                 return rc;
3212                 }
3213         }
3214         return rc;
3215 }
3216
3217 int machines__for_each_thread(struct machines *machines,
3218                               int (*fn)(struct thread *thread, void *p),
3219                               void *priv)
3220 {
3221         struct rb_node *nd;
3222         int rc = 0;
3223
3224         rc = machine__for_each_thread(&machines->host, fn, priv);
3225         if (rc != 0)
3226                 return rc;
3227
3228         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
3229                 struct machine *machine = rb_entry(nd, struct machine, rb_node);
3230
3231                 rc = machine__for_each_thread(machine, fn, priv);
3232                 if (rc != 0)
3233                         return rc;
3234         }
3235         return rc;
3236 }
3237
3238 pid_t machine__get_current_tid(struct machine *machine, int cpu)
3239 {
3240         if (cpu < 0 || (size_t)cpu >= machine->current_tid_sz)
3241                 return -1;
3242
3243         return machine->current_tid[cpu];
3244 }
3245
3246 int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid,
3247                              pid_t tid)
3248 {
3249         struct thread *thread;
3250         const pid_t init_val = -1;
3251
3252         if (cpu < 0)
3253                 return -EINVAL;
3254
3255         if (realloc_array_as_needed(machine->current_tid,
3256                                     machine->current_tid_sz,
3257                                     (unsigned int)cpu,
3258                                     &init_val))
3259                 return -ENOMEM;
3260
3261         machine->current_tid[cpu] = tid;
3262
3263         thread = machine__findnew_thread(machine, pid, tid);
3264         if (!thread)
3265                 return -ENOMEM;
3266
3267         thread->cpu = cpu;
3268         thread__put(thread);
3269
3270         return 0;
3271 }
3272
3273 /*
3274  * Compares the raw arch string. N.B. see instead perf_env__arch() or
3275  * machine__normalized_is() if a normalized arch is needed.
3276  */
3277 bool machine__is(struct machine *machine, const char *arch)
3278 {
3279         return machine && !strcmp(perf_env__raw_arch(machine->env), arch);
3280 }
3281
3282 bool machine__normalized_is(struct machine *machine, const char *arch)
3283 {
3284         return machine && !strcmp(perf_env__arch(machine->env), arch);
3285 }
3286
3287 int machine__nr_cpus_avail(struct machine *machine)
3288 {
3289         return machine ? perf_env__nr_cpus_avail(machine->env) : 0;
3290 }
3291
3292 int machine__get_kernel_start(struct machine *machine)
3293 {
3294         struct map *map = machine__kernel_map(machine);
3295         int err = 0;
3296
3297         /*
3298          * The only addresses above 2^63 are kernel addresses of a 64-bit
3299          * kernel.  Note that addresses are unsigned so that on a 32-bit system
3300          * all addresses including kernel addresses are less than 2^32.  In
3301          * that case (32-bit system), if the kernel mapping is unknown, all
3302          * addresses will be assumed to be in user space - see
3303          * machine__kernel_ip().
3304          */
3305         machine->kernel_start = 1ULL << 63;
3306         if (map) {
3307                 err = map__load(map);
3308                 /*
3309                  * On x86_64, PTI entry trampolines are less than the
3310                  * start of kernel text, but still above 2^63. So leave
3311                  * kernel_start = 1ULL << 63 for x86_64.
3312                  */
3313                 if (!err && !machine__is(machine, "x86_64"))
3314                         machine->kernel_start = map__start(map);
3315         }
3316         return err;
3317 }
3318
3319 u8 machine__addr_cpumode(struct machine *machine, u8 cpumode, u64 addr)
3320 {
3321         u8 addr_cpumode = cpumode;
3322         bool kernel_ip;
3323
3324         if (!machine->single_address_space)
3325                 goto out;
3326
3327         kernel_ip = machine__kernel_ip(machine, addr);
3328         switch (cpumode) {
3329         case PERF_RECORD_MISC_KERNEL:
3330         case PERF_RECORD_MISC_USER:
3331                 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_KERNEL :
3332                                            PERF_RECORD_MISC_USER;
3333                 break;
3334         case PERF_RECORD_MISC_GUEST_KERNEL:
3335         case PERF_RECORD_MISC_GUEST_USER:
3336                 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_GUEST_KERNEL :
3337                                            PERF_RECORD_MISC_GUEST_USER;
3338                 break;
3339         default:
3340                 break;
3341         }
3342 out:
3343         return addr_cpumode;
3344 }
3345
3346 struct dso *machine__findnew_dso_id(struct machine *machine, const char *filename, struct dso_id *id)
3347 {
3348         return dsos__findnew_id(&machine->dsos, filename, id);
3349 }
3350
3351 struct dso *machine__findnew_dso(struct machine *machine, const char *filename)
3352 {
3353         return machine__findnew_dso_id(machine, filename, NULL);
3354 }
3355
3356 char *machine__resolve_kernel_addr(void *vmachine, unsigned long long *addrp, char **modp)
3357 {
3358         struct machine *machine = vmachine;
3359         struct map *map;
3360         struct symbol *sym = machine__find_kernel_symbol(machine, *addrp, &map);
3361
3362         if (sym == NULL)
3363                 return NULL;
3364
3365         *modp = __map__is_kmodule(map) ? (char *)map__dso(map)->short_name : NULL;
3366         *addrp = map__unmap_ip(map, sym->start);
3367         return sym->name;
3368 }
3369
3370 int machine__for_each_dso(struct machine *machine, machine__dso_t fn, void *priv)
3371 {
3372         struct dso *pos;
3373         int err = 0;
3374
3375         list_for_each_entry(pos, &machine->dsos.head, node) {
3376                 if (fn(pos, machine, priv))
3377                         err = -1;
3378         }
3379         return err;
3380 }
3381
3382 int machine__for_each_kernel_map(struct machine *machine, machine__map_t fn, void *priv)
3383 {
3384         struct maps *maps = machine__kernel_maps(machine);
3385         struct map_rb_node *pos;
3386         int err = 0;
3387
3388         maps__for_each_entry(maps, pos) {
3389                 err = fn(pos->map, priv);
3390                 if (err != 0) {
3391                         break;
3392                 }
3393         }
3394         return err;
3395 }
3396
3397 bool machine__is_lock_function(struct machine *machine, u64 addr)
3398 {
3399         if (!machine->sched.text_start) {
3400                 struct map *kmap;
3401                 struct symbol *sym = machine__find_kernel_symbol_by_name(machine, "__sched_text_start", &kmap);
3402
3403                 if (!sym) {
3404                         /* to avoid retry */
3405                         machine->sched.text_start = 1;
3406                         return false;
3407                 }
3408
3409                 machine->sched.text_start = map__unmap_ip(kmap, sym->start);
3410
3411                 /* should not fail from here */
3412                 sym = machine__find_kernel_symbol_by_name(machine, "__sched_text_end", &kmap);
3413                 machine->sched.text_end = map__unmap_ip(kmap, sym->start);
3414
3415                 sym = machine__find_kernel_symbol_by_name(machine, "__lock_text_start", &kmap);
3416                 machine->lock.text_start = map__unmap_ip(kmap, sym->start);
3417
3418                 sym = machine__find_kernel_symbol_by_name(machine, "__lock_text_end", &kmap);
3419                 machine->lock.text_end = map__unmap_ip(kmap, sym->start);
3420         }
3421
3422         /* failed to get kernel symbols */
3423         if (machine->sched.text_start == 1)
3424                 return false;
3425
3426         /* mutex and rwsem functions are in sched text section */
3427         if (machine->sched.text_start <= addr && addr < machine->sched.text_end)
3428                 return true;
3429
3430         /* spinlock functions are in lock text section */
3431         if (machine->lock.text_start <= addr && addr < machine->lock.text_end)
3432                 return true;
3433
3434         return false;
3435 }